Remote transaction execution (#4684)

* return errors on database corruption

* fix tests, json tests

* fix remainder of build

* buffer flow -> request credits

* proving state backend

* generate transaction proofs from provider

* network messages for transaction proof

* transaction proof test

* test for transaction proof message

* fix call bug

* request transaction proofs from on_demand

* most of proved_execution rpc

* proved execution future

ethcore/src/client/client.rs

@@ -24,7 +24,7 @@ use time::precise_time_ns;
 
 // util
 use util::{Bytes, PerfTimer, Itertools, Mutex, RwLock, MutexGuard, Hashable};
-use util::{journaldb, TrieFactory, Trie};
+use util::{journaldb, DBValue, TrieFactory, Trie};
 use util::{U256, H256, Address, H2048, Uint, FixedHash};
 use util::trie::TrieSpec;
 use util::kvdb::*;
@@ -34,7 +34,7 @@ use io::*;
 use views::BlockView;
 use error::{ImportError, ExecutionError, CallError, BlockError, ImportResult, Error as EthcoreError};
 use header::BlockNumber;
-use state::{State, CleanupMode};
+use state::{self, State, CleanupMode};
 use spec::Spec;
 use basic_types::Seal;
 use engines::Engine;
@@ -308,18 +308,24 @@ impl Client {
 	}
 
 	/// The env info as of the best block.
-	fn latest_env_info(&self) -> EnvInfo {
-		let header = self.best_block_header();
+	pub fn latest_env_info(&self) -> EnvInfo {
+		self.env_info(BlockId::Latest).expect("Best block header always stored; qed")
+	}
 
-		EnvInfo {
-			number: header.number(),
-			author: header.author(),
-			timestamp: header.timestamp(),
-			difficulty: header.difficulty(),
-			last_hashes: self.build_last_hashes(header.hash()),
-			gas_used: U256::default(),
-			gas_limit: header.gas_limit(),
-		}
+	/// The env info as of a given block.
+	/// returns `None` if the block unknown.
+	pub fn env_info(&self, id: BlockId) -> Option<EnvInfo> {
+		self.block_header(id).map(|header| {
+			EnvInfo {
+				number: header.number(),
+				author: header.author(),
+				timestamp: header.timestamp(),
+				difficulty: header.difficulty(),
+				last_hashes: self.build_last_hashes(header.parent_hash()),
+				gas_used: U256::default(),
+				gas_limit: header.gas_limit(),
+			}
+		})
 	}
 
 	fn build_last_hashes(&self, parent_hash: H256) -> Arc<LastHashes> {
@@ -874,17 +880,9 @@ impl snapshot::DatabaseRestore for Client {
 
 impl BlockChainClient for Client {
 	fn call(&self, t: &SignedTransaction, block: BlockId, analytics: CallAnalytics) -> Result<Executed, CallError> {
-		let header = self.block_header(block).ok_or(CallError::StatePruned)?;
-		let last_hashes = self.build_last_hashes(header.parent_hash());
-		let env_info = EnvInfo {
-			number: header.number(),
-			author: header.author(),
-			timestamp: header.timestamp(),
-			difficulty: header.difficulty(),
-			last_hashes: last_hashes,
-			gas_used: U256::zero(),
-			gas_limit: U256::max_value(),
-		};
+		let mut env_info = self.env_info(block).ok_or(CallError::StatePruned)?;
+		env_info.gas_limit = U256::max_value();
+
 		// that's just a copy of the state.
 		let mut state = self.state_at(block).ok_or(CallError::StatePruned)?;
 		let original_state = if analytics.state_diffing { Some(state.clone()) } else { None };
@@ -910,17 +908,13 @@ impl BlockChainClient for Client {
 
 	fn estimate_gas(&self, t: &SignedTransaction, block: BlockId) -> Result<U256, CallError> {
 		const UPPER_CEILING: u64 = 1_000_000_000_000u64;
-		let header = self.block_header(block).ok_or(CallError::StatePruned)?;
-		let last_hashes = self.build_last_hashes(header.parent_hash());
-		let env_info = EnvInfo {
-			number: header.number(),
-			author: header.author(),
-			timestamp: header.timestamp(),
-			difficulty: header.difficulty(),
-			last_hashes: last_hashes,
-			gas_used: U256::zero(),
-			gas_limit: UPPER_CEILING.into(),
+		let (mut upper, env_info)  = {
+			let mut env_info = self.env_info(block).ok_or(CallError::StatePruned)?;
+			let initial_upper = env_info.gas_limit;
+			env_info.gas_limit = UPPER_CEILING.into();
+			(initial_upper, env_info)
 		};
+
 		// that's just a copy of the state.
 		let original_state = self.state_at(block).ok_or(CallError::StatePruned)?;
 		let sender = t.sender();
@@ -946,7 +940,6 @@ impl BlockChainClient for Client {
 				.unwrap_or(false))
 		};
 
-		let mut upper = header.gas_limit();
 		if !cond(upper)? {
 			// impossible at block gas limit - try `UPPER_CEILING` instead.
 			// TODO: consider raising limit by powers of two.
@@ -989,7 +982,7 @@ impl BlockChainClient for Client {
 
 	fn replay(&self, id: TransactionId, analytics: CallAnalytics) -> Result<Executed, CallError> {
 		let address = self.transaction_address(id).ok_or(CallError::TransactionNotFound)?;
-		let header = self.block_header(BlockId::Hash(address.block_hash)).ok_or(CallError::StatePruned)?;
+		let mut env_info = self.env_info(BlockId::Hash(address.block_hash)).ok_or(CallError::StatePruned)?;
 		let body = self.block_body(BlockId::Hash(address.block_hash)).ok_or(CallError::StatePruned)?;
 		let mut state = self.state_at_beginning(BlockId::Hash(address.block_hash)).ok_or(CallError::StatePruned)?;
 		let mut txs = body.transactions();
@@ -999,16 +992,6 @@ impl BlockChainClient for Client {
 		}
 
 		let options = TransactOptions { tracing: analytics.transaction_tracing, vm_tracing: analytics.vm_tracing, check_nonce: false };
-		let last_hashes = self.build_last_hashes(header.hash());
-		let mut env_info = EnvInfo {
-			number: header.number(),
-			author: header.author(),
-			timestamp: header.timestamp(),
-			difficulty: header.difficulty(),
-			last_hashes: last_hashes,
-			gas_used: U256::default(),
-			gas_limit: header.gas_limit(),
-		};
 		const PROOF: &'static str = "Transactions fetched from blockchain; blockchain transactions are valid; qed";
 		let rest = txs.split_off(address.index);
 		for t in txs {
@@ -1620,6 +1603,25 @@ impl ::client::ProvingBlockChainClient for Client {
 			.and_then(|x| x)
 			.unwrap_or_else(Vec::new)
 	}
+
+	fn prove_transaction(&self, transaction: SignedTransaction, id: BlockId) -> Option<Vec<DBValue>> {
+		let (state, env_info) = match (self.state_at(id), self.env_info(id)) {
+			(Some(s), Some(e)) => (s, e),
+			_ => return None,
+		};
+		let mut jdb = self.state_db.lock().journal_db().boxed_clone();
+		let backend = state::backend::Proving::new(jdb.as_hashdb_mut());
+
+		let mut state = state.replace_backend(backend);
+		let options = TransactOptions { tracing: false, vm_tracing: false, check_nonce: false };
+		let res = Executive::new(&mut state, &env_info, &*self.engine, &self.factories.vm).transact(&transaction, options);
+
+		match res {
+			Err(ExecutionError::Internal(_)) => return None,
+			_ => return Some(state.drop().1.extract_proof()),
+		}
+	}
+
 }
 
 impl Drop for Client {

ethcore/src/client/test_client.rs

@@ -765,6 +765,10 @@ impl ProvingBlockChainClient for TestBlockChainClient {
 	fn code_by_hash(&self, _: H256, _: BlockId) -> Bytes {
 		Vec::new()
 	}
+
+	fn prove_transaction(&self, _: SignedTransaction, _: BlockId) -> Option<Vec<DBValue>> {
+		None
+	}
 }
 
 impl EngineClient for TestBlockChainClient {

ethcore/src/client/traits.rs

@@ -16,6 +16,7 @@
 
 use std::collections::BTreeMap;
 use util::{U256, Address, H256, H2048, Bytes, Itertools};
+use util::hashdb::DBValue;
 use blockchain::TreeRoute;
 use verification::queue::QueueInfo as BlockQueueInfo;
 use block::{OpenBlock, SealedBlock};
@@ -321,4 +322,7 @@ pub trait ProvingBlockChainClient: BlockChainClient {
 
 	/// Get code by address hash.
 	fn code_by_hash(&self, account_key: H256, id: BlockId) -> Bytes;
+
+	/// Prove execution of a transaction at the given block.
+	fn prove_transaction(&self, transaction: SignedTransaction, id: BlockId) -> Option<Vec<DBValue>>;
 }

ethcore/src/env_info.rs

@@ -14,6 +14,8 @@
 // You should have received a copy of the GNU General Public License
 // along with Parity.  If not, see <http://www.gnu.org/licenses/>.
 
+//! Environment information for transaction execution.
+
 use std::cmp;
 use std::sync::Arc;
 use util::{U256, Address, H256, Hashable};
@@ -25,7 +27,7 @@ use ethjson;
 pub type LastHashes = Vec<H256>;
 
 /// Information concerning the execution environment for a message-call/contract-creation.
-#[derive(Debug)]
+#[derive(Debug, Clone)]
 pub struct EnvInfo {
 	/// The block number.
 	pub number: BlockNumber,

ethcore/src/lib.rs

@@ -79,7 +79,6 @@
 //!   cargo build --release
 //!   ```
 
-
 extern crate ethcore_io as io;
 extern crate rustc_serialize;
 extern crate crypto;
@@ -141,12 +140,12 @@ pub mod action_params;
 pub mod db;
 pub mod verification;
 pub mod state;
+pub mod env_info;
 #[macro_use] pub mod evm;
 
 mod cache_manager;
 mod blooms;
 mod basic_types;
-mod env_info;
 mod pod_account;
 mod state_db;
 mod account_db;

ethcore/src/state/backend.rs

@@ -21,10 +21,12 @@
 //! should become general over time to the point where not even a
 //! merkle trie is strictly necessary.
 
+use std::collections::{HashSet, HashMap};
 use std::sync::Arc;
 
 use state::Account;
-use util::{Address, AsHashDB, HashDB, H256};
+use util::{Address, MemoryDB, Mutex, H256};
+use util::hashdb::{AsHashDB, HashDB, DBValue};
 
 /// State backend. See module docs for more details.
 pub trait Backend: Send {
@@ -64,21 +66,48 @@ pub trait Backend: Send {
 	fn is_known_null(&self, address: &Address) -> bool;
 }
 
-/// A raw backend which simply wraps a hashdb and does no caching.
-#[derive(Debug, Clone, PartialEq, Eq)]
-pub struct NoCache<T>(T);
+/// A raw backend used to check proofs of execution.
+///
+/// This doesn't delete anything since execution proofs won't have mangled keys
+/// and we want to avoid collisions.
+// TODO: when account lookup moved into backends, this won't rely as tenuously on intended
+// usage.
+#[derive(Clone, PartialEq)]
+pub struct ProofCheck(MemoryDB);
 
-impl<T> NoCache<T> {
-	/// Create a new `NoCache` backend.
-	pub fn new(inner: T) -> Self { NoCache(inner) }
-
-	/// Consume the backend, yielding the inner database.
-	pub fn into_inner(self) -> T { self.0 }
+impl ProofCheck {
+	/// Create a new `ProofCheck` backend from the given state items.
+	pub fn new(proof: &[DBValue]) -> Self {
+		let mut db = MemoryDB::new();
+		for item in proof { db.insert(item); }
+		ProofCheck(db)
+	}
 }
 
-impl<T: AsHashDB + Send> Backend for NoCache<T> {
-	fn as_hashdb(&self) -> &HashDB { self.0.as_hashdb() }
-	fn as_hashdb_mut(&mut self) -> &mut HashDB { self.0.as_hashdb_mut() }
+impl HashDB for ProofCheck {
+	fn keys(&self) -> HashMap<H256, i32> { self.0.keys() }
+	fn get(&self, key: &H256) -> Option<DBValue> {
+		self.0.get(key)
+	}
+
+	fn contains(&self, key: &H256) -> bool {
+		self.0.contains(key)
+	}
+
+	fn insert(&mut self, value: &[u8]) -> H256 {
+		self.0.insert(value)
+	}
+
+	fn emplace(&mut self, key: H256, value: DBValue) {
+		self.0.emplace(key, value)
+	}
+
+	fn remove(&mut self, _key: &H256) { }
+}
+
+impl Backend for ProofCheck {
+	fn as_hashdb(&self) -> &HashDB { self }
+	fn as_hashdb_mut(&mut self) -> &mut HashDB { self }
 	fn add_to_account_cache(&mut self, _addr: Address, _data: Option<Account>, _modified: bool) {}
 	fn cache_code(&self, _hash: H256, _code: Arc<Vec<u8>>) {}
 	fn get_cached_account(&self, _addr: &Address) -> Option<Option<Account>> { None }
@@ -91,3 +120,104 @@ impl<T: AsHashDB + Send> Backend for NoCache<T> {
 	fn note_non_null_account(&self, _address: &Address) {}
 	fn is_known_null(&self, _address: &Address) -> bool { false }
 }
+
+/// Proving state backend.
+/// This keeps track of all state values loaded during usage of this backend.
+/// The proof-of-execution can be extracted with `extract_proof`.
+///
+/// This doesn't cache anything or rely on the canonical state caches.
+pub struct Proving<H: AsHashDB> {
+	base: H, // state we're proving values from.
+	changed: MemoryDB, // changed state via insertions.
+	proof: Mutex<HashSet<DBValue>>,
+}
+
+impl<H: AsHashDB + Send + Sync> HashDB for Proving<H> {
+	fn keys(&self) -> HashMap<H256, i32> {
+		let mut keys = self.base.as_hashdb().keys();
+		keys.extend(self.changed.keys());
+		keys
+	}
+
+	fn get(&self, key: &H256) -> Option<DBValue> {
+		match self.base.as_hashdb().get(key) {
+			Some(val) => {
+				self.proof.lock().insert(val.clone());
+				Some(val)
+			}
+			None => self.changed.get(key)
+		}
+	}
+
+	fn contains(&self, key: &H256) -> bool {
+		self.get(key).is_some()
+	}
+
+	fn insert(&mut self, value: &[u8]) -> H256 {
+		self.changed.insert(value)
+	}
+
+	fn emplace(&mut self, key: H256, value: DBValue) {
+		self.changed.emplace(key, value)
+	}
+
+	fn remove(&mut self, key: &H256) {
+		// only remove from `changed`
+		if self.changed.contains(key) {
+			self.changed.remove(key)
+		}
+	}
+}
+
+impl<H: AsHashDB + Send + Sync> Backend for Proving<H> {
+	fn as_hashdb(&self) -> &HashDB {
+		self
+	}
+
+	fn as_hashdb_mut(&mut self) -> &mut HashDB {
+		self
+	}
+
+	fn add_to_account_cache(&mut self, _: Address, _: Option<Account>, _: bool) { }
+
+	fn cache_code(&self, _: H256, _: Arc<Vec<u8>>) { }
+
+	fn get_cached_account(&self, _: &Address) -> Option<Option<Account>> { None }
+
+	fn get_cached<F, U>(&self, _: &Address, _: F) -> Option<U>
+		where F: FnOnce(Option<&mut Account>) -> U
+	{
+		None
+	}
+
+	fn get_cached_code(&self, _: &H256) -> Option<Arc<Vec<u8>>> { None }
+	fn note_non_null_account(&self, _: &Address) { }
+	fn is_known_null(&self, _: &Address) -> bool { false }
+}
+
+impl<H: AsHashDB> Proving<H> {
+	/// Create a new `Proving` over a base database.
+	/// This will store all values ever fetched from that base.
+	pub fn new(base: H) -> Self {
+		Proving {
+			base: base,
+			changed: MemoryDB::new(),
+			proof: Mutex::new(HashSet::new()),
+		}
+	}
+
+	/// Consume the backend, extracting the gathered proof.
+	pub fn extract_proof(self) -> Vec<DBValue> {
+		self.proof.into_inner().into_iter().collect()
+	}
+}
+
+impl<H: AsHashDB + Clone> Clone for Proving<H> {
+	fn clone(&self) -> Self {
+		Proving {
+			base: self.base.clone(),
+			changed: self.changed.clone(),
+			proof: Mutex::new(self.proof.lock().clone()),
+		}
+	}
+}

ethcore/src/state/mod.rs

@@ -31,6 +31,7 @@ use factory::Factories;
 use trace::FlatTrace;
 use pod_account::*;
 use pod_state::{self, PodState};
+use types::executed::{Executed, ExecutionError};
 use types::state_diff::StateDiff;
 use transaction::SignedTransaction;
 use state_db::StateDB;
@@ -60,6 +61,17 @@ pub struct ApplyOutcome {
 /// Result type for the execution ("application") of a transaction.
 pub type ApplyResult = Result<ApplyOutcome, Error>;
 
+/// Return type of proof validity check.
+#[derive(Debug, Clone)]
+pub enum ProvedExecution {
+	/// Proof wasn't enough to complete execution.
+	BadProof,
+	/// The transaction failed, but not due to a bad proof.
+	Failed(ExecutionError),
+	/// The transaction successfully completd with the given proof.
+	Complete(Executed),
+}
+
 #[derive(Eq, PartialEq, Clone, Copy, Debug)]
 /// Account modification state. Used to check if the account was
 /// Modified in between commits and overall.
@@ -150,6 +162,39 @@ impl AccountEntry {
 	}
 }
 
+/// Check the given proof of execution.
+/// `Err(ExecutionError::Internal)` indicates failure, everything else indicates
+/// a successful proof (as the transaction itself may be poorly chosen).
+pub fn check_proof(
+	proof: &[::util::DBValue],
+	root: H256,
+	transaction: &SignedTransaction,
+	engine: &Engine,
+	env_info: &EnvInfo,
+) -> ProvedExecution {
+	let backend = self::backend::ProofCheck::new(proof);
+	let mut factories = Factories::default();
+	factories.accountdb = ::account_db::Factory::Plain;
+
+	let res = State::from_existing(
+		backend,
+		root,
+		engine.account_start_nonce(),
+		factories
+	);
+
+	let mut state = match res {
+		Ok(state) => state,
+		Err(_) => return ProvedExecution::BadProof,
+	};
+
+	match state.execute(env_info, engine, transaction, false) {
+		Ok(executed) => ProvedExecution::Complete(executed),
+		Err(ExecutionError::Internal(_)) => ProvedExecution::BadProof,
+		Err(e) => ProvedExecution::Failed(e),
+	}
+}
+
 /// Representation of the entire state of all accounts in the system.
 ///
 /// `State` can work together with `StateDB` to share account cache.
@@ -264,6 +309,19 @@ impl<B: Backend> State<B> {
 		Ok(state)
 	}
 
+	/// Swap the current backend for another.
+	// TODO: [rob] find a less hacky way to avoid duplication of `Client::state_at`.
+	pub fn replace_backend<T: Backend>(self, backend: T) -> State<T> {
+		State {
+			db: backend,
+			root: self.root,
+			cache: self.cache,
+			checkpoints: self.checkpoints,
+			account_start_nonce: self.account_start_nonce,
+			factories: self.factories,
+		}
+	}
+
 	/// Create a recoverable checkpoint of this state.
 	pub fn checkpoint(&mut self) {
 		self.checkpoints.get_mut().push(HashMap::new());
@@ -535,16 +593,12 @@ impl<B: Backend> State<B> {
 		Ok(())
 	}
 
-	/// Execute a given transaction.
+	/// Execute a given transaction, producing a receipt and an optional trace.
 	/// This will change the state accordingly.
 	pub fn apply(&mut self, env_info: &EnvInfo, engine: &Engine, t: &SignedTransaction, tracing: bool) -> ApplyResult {
 //		let old = self.to_pod();
 
-		let options = TransactOptions { tracing: tracing, vm_tracing: false, check_nonce: true };
-		let vm_factory = self.factories.vm.clone();
-		let e = Executive::new(self, env_info, engine, &vm_factory).transact(t, options)?;
-
-		// TODO uncomment once to_pod() works correctly.
+		let e = self.execute(env_info, engine, t, tracing)?;
 //		trace!("Applied transaction. Diff:\n{}\n", state_diff::diff_pod(&old, &self.to_pod()));
 		let state_root = if env_info.number < engine.params().eip98_transition {
 			self.commit()?;
@@ -557,6 +611,15 @@ impl<B: Backend> State<B> {
 		Ok(ApplyOutcome{receipt: receipt, trace: e.trace})
 	}
 
+	// Execute a given transaction.
+	fn execute(&mut self, env_info: &EnvInfo, engine: &Engine, t: &SignedTransaction, tracing: bool) -> Result<Executed, ExecutionError> {
+		let options = TransactOptions { tracing: tracing, vm_tracing: false, check_nonce: true };
+		let vm_factory = self.factories.vm.clone();
+
+		Executive::new(self, env_info, engine, &vm_factory).transact(t, options)
+	}
+
+
 	/// Commit accounts to SecTrieDBMut. This is similar to cpp-ethereum's dev::eth::commit.
 	/// `accounts` is mutable because we may need to commit the code or storage and record that.
 	#[cfg_attr(feature="dev", allow(match_ref_pats))]

ethcore/src/tests/client.rs

@@ -16,7 +16,8 @@
 
 use io::IoChannel;
 use client::{BlockChainClient, MiningBlockChainClient, Client, ClientConfig, BlockId};
-use state::CleanupMode;
+use state::{self, State, CleanupMode};
+use executive::Executive;
 use ethereum;
 use block::IsBlock;
 use tests::helpers::*;
@@ -341,3 +342,43 @@ fn does_not_propagate_delayed_transactions() {
 	assert_eq!(2, client.ready_transactions().len());
 	assert_eq!(2, client.miner().pending_transactions().len());
 }
+
+#[test]
+fn transaction_proof() {
+	use ::client::ProvingBlockChainClient;
+
+	let client_result = generate_dummy_client(0);
+	let client = client_result.reference();
+	let address = Address::random();
+	let test_spec = Spec::new_test();
+	for _ in 0..20 {
+		let mut b = client.prepare_open_block(Address::default(), (3141562.into(), 31415620.into()), vec![]);
+		b.block_mut().fields_mut().state.add_balance(&address, &5.into(), CleanupMode::NoEmpty).unwrap();
+		b.block_mut().fields_mut().state.commit().unwrap();
+		let b = b.close_and_lock().seal(&*test_spec.engine, vec![]).unwrap();
+		client.import_sealed_block(b).unwrap(); // account change is in the journal overlay
+	}
+
+	let transaction = Transaction {
+		nonce: 0.into(),
+		gas_price: 0.into(),
+		gas: 21000.into(),
+		action: Action::Call(Address::default()),
+		value: 5.into(),
+		data: Vec::new(),
+	}.fake_sign(address);
+
+	let proof = client.prove_transaction(transaction.clone(), BlockId::Latest).unwrap();
+	let backend = state::backend::ProofCheck::new(&proof);
+
+	let mut factories = ::factory::Factories::default();
+	factories.accountdb = ::account_db::Factory::Plain; // raw state values, no mangled keys.
+	let root = client.best_block_header().state_root();
+
+	let mut state = State::from_existing(backend, root, 0.into(), factories.clone()).unwrap();
+	Executive::new(&mut state, &client.latest_env_info(), &*test_spec.engine, &factories.vm)
+		.transact(&transaction, Default::default()).unwrap();
+
+	assert_eq!(state.balance(&Address::default()).unwrap(), 5.into());
+	assert_eq!(state.balance(&address).unwrap(), 95.into());
+}