Fork choice and metadata framework for Engine (#8401)

* Add light client TODO item

* Move existing total-difficulty-based fork choice check to Engine

* Abstract total difficulty and block provider as Machine::BlockMetadata and Machine::BlockProvider

* Decouple "generate_metadata" logic to Engine

* Use fixed BlockMetadata and BlockProvider type for null and instantseal

In this way they can use total difficulty fork choice check

* Extend blockdetails with metadatas and finalized info

* Extra data update: mark_finalized and update_metadatas

* Check finalized block in Blockchain

* Fix a test constructor in verification mod

* Add total difficulty trait

* Fix type import

* Db migration to V13 with metadata column

* Address grumbles

* metadatas -> metadata

* Use generic type for update_metadata to avoid passing HashMap all around

* Remove metadata in blockdetails

* [WIP] Implement a generic metadata architecture

* [WIP] Metadata insertion logic in BlockChain

* typo: Value -> Self::Value

* [WIP] Temporarily remove Engine::is_new_best interface

So that we don't have too many type errors.

* [WIP] Fix more type errors

* [WIP] ExtendedHeader::PartialEq

* [WIP] Change metadata type Option<Vec<u8>> to Vec<u8>

* [WIP] Remove Metadata Error

* [WIP] Clean up error conversion

* [WIP] finalized -> is_finalized

* [WIP] Mark all fields in ExtrasInsert as pub

* [WIP] Remove unused import

* [WIP] Keep only local metadata info

* Mark metadata as optional

* [WIP] Revert metadata db change in BlockChain

* [WIP] Put finalization in unclosed state

* Use metadata interface in BlockDetail

* [WIP] Fix current build failures

* [WIP] Remove unused blockmetadata struct

* Remove DB migration info

* [WIP] Typo

* Use ExtendedHeader to implement fork choice check

* Implement is_new_best using Ancestry iterator

* Use expect instead of panic

* [WIP] Add ancestry Engine support via on_new_block

* Fix tests

* Emission of ancestry actions

* use_short_version should take account of metadata

* Engine::is_new_best -> Engine::fork_choice

* Use proper expect format as defined in #1026

* panic -> expect

* ancestry_header -> ancestry_with_metadata

* Boxed iterator -> &mut iterator

* Fix tests

* is_new_best -> primitive_fork_choice

* Document how fork_choice works

* Engine::fork_choice -> Engine::primitive_fork_choice

* comment: clarify types of finalization where Engine::primitive_fork_choice works

* Expose FinalizationInfo to Engine

* Fix tests due to merging

* Remove TotalDifficulty trait

* Do not pass FinalizationInfo to Engine

If there's finalized blocks in from route, choose the old branch without calling `Engine::fork_choice`.

* Fix compile

* Fix unused import

* Remove is_to_route_finalized

When no block reorg passes a finalized block, this variable is always false.

* Address format grumbles

* Fix docs: mark_finalized returns None if block hash is not found

`blockchain` mod does not yet have an Error type, so we still temporarily use None here.

* Fix inaccurate tree_route None expect description

machine/src/lib.rs

@@ -40,13 +40,35 @@ pub trait Header {
 	fn number(&self) -> u64;
 }
 
-/// a header with an associated score (difficulty in PoW terms)
+/// A header with an associated score (difficulty in PoW terms)
 pub trait ScoredHeader: Header {
+	type Value;
+
 	/// Get the score of this header.
-	fn score(&self) -> &U256;
+	fn score(&self) -> &Self::Value;
 
 	/// Set the score of this header.
-	fn set_score(&mut self, score: U256);
+	fn set_score(&mut self, score: Self::Value);
+}
+
+/// A header with associated total score.
+pub trait TotalScoredHeader: Header {
+	type Value;
+
+	/// Get the total score of this header.
+	fn total_score(&self) -> Self::Value;
+}
+
+/// A header with finalized information.
+pub trait FinalizableHeader: Header {
+	/// Get whether this header is considered finalized, so that it will never be replaced in reorganization.
+	fn is_finalized(&self) -> bool;
+}
+
+/// A header with metadata information.
+pub trait WithMetadataHeader: Header {
+	/// Get the current header metadata.
+	fn metadata(&self) -> Option<&[u8]>;
 }
 
 /// A "live" block is one which is in the process of the transition.
@@ -73,16 +95,36 @@ pub trait Transactions: LiveBlock {
 	fn transactions(&self) -> &[Self::Transaction];
 }
 
+/// Trait for blocks which have finalized information.
+pub trait Finalizable: LiveBlock {
+	/// Get whether the block is finalized.
+	fn is_finalized(&self) -> bool;
+	/// Mark the block as finalized.
+	fn mark_finalized(&mut self);
+}
+
+/// A state machine with block metadata.
+pub trait WithMetadata: LiveBlock {
+	/// Get the current live block metadata.
+	fn metadata(&self) -> Option<&[u8]>;
+	/// Set the current live block metadata.
+	fn set_metadata(&mut self, value: Option<Vec<u8>>);
+}
+
 /// Generalization of types surrounding blockchain-suitable state machines.
 pub trait Machine: for<'a> LocalizedMachine<'a> {
 	/// The block header type.
 	type Header: Header;
 	/// The live block type.
 	type LiveBlock: LiveBlock<Header=Self::Header>;
+	/// Block header with metadata information.
+	type ExtendedHeader: Header;
 	/// A handle to a blockchain client for this machine.
 	type EngineClient: ?Sized;
 	/// A description of needed auxiliary data.
 	type AuxiliaryRequest;
+	/// Actions taken on ancestry blocks when commiting a new block.
+	type AncestryAction;
 
 	/// Errors which can occur when querying or interacting with the machine.
 	type Error;