take_snapshot and block chunking skeleton

ethcore/src/client/client.rs

@@ -753,12 +753,22 @@ impl<V> BlockChainClient for Client<V> where V: Verifier {
 	fn all_transactions(&self) -> Vec<SignedTransaction> {
 		self.miner.all_transactions()
 	}
+
+	fn take_snapshot(&self) {
+		let best_header = HeaderView::(&self.best_block_header());
+		let hash = best_header.hash();
+		let state_root = best_header.state_root();
+
+		// lock the state db to keep it consistent with the best block.
+		// clone the arc so we can loan out self to the block chunker.
+		let state_db = self.state_db.clone().lock().unwrap();
+	}
 }
 
 impl<V> MiningBlockChainClient for Client<V> where V: Verifier {
 	fn prepare_sealing(&self, author: Address, gas_floor_target: U256, extra_data: Bytes, transactions: Vec<SignedTransaction>)
 		-> (Option<ClosedBlock>, HashSet<H256>) {
-		let engine = self.engine.deref().deref();
+		let engine = &**self.engine;
 		let h = self.chain.best_block_hash();
 		let mut invalid_transactions = HashSet::new();
 

ethcore/src/client/mod.rs

@@ -194,6 +194,9 @@ pub trait BlockChainClient : Sync + Send {
 
 	/// list all transactions
 	fn all_transactions(&self) -> Vec<SignedTransaction>;
+
+	/// Generate a PV64 snapshot for the current best block.
+	fn take_snapshot(&self);
 }
 
 /// Extended client interface used for mining

ethcore/src/lib.rs

@@ -111,6 +111,7 @@ pub mod pod_state;
 pub mod engine;
 pub mod migrations;
 pub mod miner;
+pub mod pv64;
 
 mod blooms;
 mod db;

ethcore/src/pv64/mod.rs

@@ -0,0 +1,114 @@
+// Copyright 2015, 2016 Ethcore (UK) Ltd.
+// This file is part of Parity.
+
+// Parity is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Parity is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Parity.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Pv64 snapshot creation helpers.
+
+// Try to have chunks be around 16MB
+const PREFERRED_CHUNK_SIZE: usize = 16 * 1024 * 1024;
+
+// But tolerate ones within a quarter of a megabyte of that size.
+const SIZE_TOLERANCE: usize = 250 * 1024;
+
+use std::collections::VecDeque;
+use std::fs::File;
+use std::io::Write;
+use std::path::{Path, PathBuf};
+
+use client::BlockChainClient;
+use ids::BlockID;
+use views::{BlockView, HeaderView};
+
+use util::Hashable;
+use util::hash::H256;
+use util::rlp::{Stream, RlpStream};
+
+/// Used to build block chunks.
+pub struct BlockChunker<'a> {
+	client: &'a BlockChainClient,
+	// block, receipt rlp pairs.
+	rlps: VecDeque<(Bytes, Bytes)>,
+	genesis_hash: H256,
+	current_hash: H256,
+}
+
+impl<'a> BlockChunker<'a> {
+	/// Create a new BlockChunker given a client and the genesis hash.
+	pub fn new(client: &'a BlockChainClient, genesis_hash: H256) -> Self {
+		// Todo [rob]: find a way to reuse rlp allocations
+		BlockChunker {
+			client: client,
+			rlps: VecDeque<(Bytes, Bytes)>
+			genesis_hash: genesis_hash,
+			current_hash: HeaderView::new(&client.best_block_header()).hash(),
+		}
+	}
+
+	// Try to fill the buffers, moving backwards from current block hash.
+	// This will return true if it created a block chunk, false otherwise.
+	fn fill_buffers(&mut self) -> bool {
+		let mut loaded_size = 0;
+
+		while loaded_size < PREFERRED_CHUNK_SIZE && self.current_hash != self.genesis_hash {
+
+			// skip compression for now
+			let block = self.client.block(BlockID::Hash(self.current_hash)).unwrap();
+			let receipts = self.client.block_receipts(&self.current_hash).unwrap();
+
+			let new_loaded_size = loaded_size + (block.len() + receipts.len());
+			if new_loaded_size > PREFERRED_CHUNK_SIZE + SIZE_TOLERANCE {
+				return true;
+			} else {
+				loaded_size = new_loaded_size;
+			}
+
+			self.current_hash = BlockView::new(&block).header_view().parent_hash();
+
+			self.rlps.push_front((block, receipts));
+		}
+
+		loaded_size == 0
+	}
+
+	// write out the data in the buffers to a chunk on disk
+	fn write_chunk(&mut self, path: &Path) -> H256 {
+		// Todo: compress raw data, put parent hash and block number into chunk.
+		let mut rlp_stream = RlpStream::new_list(self.rlps.len());
+		for (block, receipts) in self.rlps.drain(..) {
+			rlp_stream.begin_list(2).append(&block).append(&receipts);
+		}
+
+		let raw_data = rlp_stream.out();
+		let hash_str = raw_data.sha3().to_string();
+
+		let file_path = PathBuf::new(path).push(&hash_str[2..]);
+		let mut file = File::create(file_path).unwrap();
+		file.write_all(&raw_data);
+	}
+
+	/// Create and write out all block chunks to disk, returning a vector of all
+	/// the hashes of block chunks created.
+	///
+	/// The path parameter is the directory to store the block chunks in.
+	/// This function assumes the directory exists already.
+	pub fn chunk_all(self, best_block_hash: H256, path: &Path) -> Vec<H256> {
+		let mut hash = best_block_hash;
+		let mut chunk_hashes = Vec::new();
+
+		while self.fill_buffers() {
+			chunk_hashes.push(self.write_chunk(path));
+		}
+	}
+}

ethcore/src/views.rs

@@ -145,7 +145,7 @@ impl<'a> BlockView<'a> {
 		self.rlp.val_at(0)
 	}
 
-	/// Create new header view obto block head rlp.
+	/// Create new header view onto block head rlp.
 	pub fn header_view(&self) -> HeaderView<'a> {
 		HeaderView::new_from_rlp(self.rlp.at(0))
 	}