Merge branch 'master' into tx_queue_integration

Conflicts:
	ethcore/src/client.rs
	sync/src/tests/helpers.rs

ethcore/src/blockchain/blockchain.rs

@@ -448,7 +448,8 @@ impl BlockChain {
 		let mut write_details = self.block_details.write().unwrap();
 		for (hash, details) in update.block_details.into_iter() {
 			batch.put_extras(&hash, &details);
-			write_details.insert(hash, details);
+			write_details.insert(hash.clone(), details);
+			self.note_used(CacheID::Extras(ExtrasIndex::BlockDetails, hash));
 		}
 
 		let mut write_receipts = self.block_receipts.write().unwrap();
@@ -759,6 +760,14 @@ impl BlockChain {
 
 				// TODO: handle block_hashes properly.
 				block_hashes.clear();
+
+				blocks.shrink_to_fit();
+				block_details.shrink_to_fit();
+ 				block_hashes.shrink_to_fit();
+ 				transaction_addresses.shrink_to_fit();
+ 				block_logs.shrink_to_fit();
+ 				blocks_blooms.shrink_to_fit();
+ 				block_receipts.shrink_to_fit();
 			}
 			if self.cache_size().total() < self.max_cache_size { break; }
 		}

ethcore/src/client.rs

@@ -16,6 +16,7 @@
 
 //! Blockchain database client.
 
+use std::marker::PhantomData;
 use util::*;
 use util::panics::*;
 use blockchain::{BlockChain, BlockProvider};
@@ -35,6 +36,7 @@ use transaction::LocalizedTransaction;
 use extras::TransactionAddress;
 use filter::Filter;
 use log_entry::LocalizedLogEntry;
+use util::keys::store::SecretStore;
 pub use block_queue::{BlockQueueConfig, BlockQueueInfo};
 pub use blockchain::{TreeRoute, BlockChainConfig, CacheSize as BlockChainCacheSize};
 
@@ -76,12 +78,24 @@ pub enum BlockStatus {
 }
 
 /// Client configuration. Includes configs for all sub-systems.
-#[derive(Debug, Default)]
+#[derive(Debug)]
 pub struct ClientConfig {
 	/// Block queue configuration.
 	pub queue: BlockQueueConfig,
 	/// Blockchain configuration.
 	pub blockchain: BlockChainConfig,
+	/// Prefer journal rather than archive.
+	pub prefer_journal: bool,
+}
+
+impl Default for ClientConfig {
+	fn default() -> ClientConfig {
+		ClientConfig {
+			queue: Default::default(),
+			blockchain: Default::default(),
+			prefer_journal: false,
+		}
+	}
 }
 
 /// Information about the blockchain gathered together.
@@ -126,6 +140,9 @@ pub trait BlockChainClient : Sync + Send {
 	/// Get address nonce.
 	fn nonce(&self, address: &Address) -> U256;
 
+	/// Get block hash.
+	fn block_hash(&self, id: BlockId) -> Option<H256>;
+
 	/// Get address code.
 	fn code(&self, address: &Address) -> Option<Bytes>;
 
@@ -188,7 +205,7 @@ impl ClientReport {
 
 /// Blockchain database client backed by a persistent database. Owns and manages a blockchain and a block queue.
 /// Call `import_block()` to import a block asynchronously; `flush_queue()` flushes the queue.
-pub struct Client {
+pub struct Client<V = CanonVerifier> where V: Verifier {
 	chain: Arc<RwLock<BlockChain>>,
 	engine: Arc<Box<Engine>>,
 	state_db: Mutex<JournalDB>,
@@ -201,18 +218,27 @@ pub struct Client {
 	sealing_block: Mutex<Option<ClosedBlock>>,
 	author: RwLock<Address>,
 	extra_data: RwLock<Bytes>,
+	verifier: PhantomData<V>,
+	secret_store: Arc<RwLock<SecretStore>>,
 }
 
 const HISTORY: u64 = 1000;
 const CLIENT_DB_VER_STR: &'static str = "4.0";
 
-impl Client {
+impl Client<CanonVerifier> {
 	/// Create a new client with given spec and DB path.
 	pub fn new(config: ClientConfig, spec: Spec, path: &Path, message_channel: IoChannel<NetSyncMessage> ) -> Result<Arc<Client>, Error> {
+		Client::<CanonVerifier>::new_with_verifier(config, spec, path, message_channel)
+	}
+}
+
+impl<V> Client<V> where V: Verifier {
+	///  Create a new client with given spec and DB path and custom verifier.
+	pub fn new_with_verifier(config: ClientConfig, spec: Spec, path: &Path, message_channel: IoChannel<NetSyncMessage> ) -> Result<Arc<Client>, Error> {
 		let mut dir = path.to_path_buf();
 		dir.push(H64::from(spec.genesis_header().hash()).hex());
 		//TODO: sec/fat: pruned/full versioning
-		dir.push(format!("v{}-sec-pruned", CLIENT_DB_VER_STR));
+		dir.push(format!("v{}-sec-{}", CLIENT_DB_VER_STR, if config.prefer_journal { "pruned" } else { "archive" }));
 		let path = dir.as_path();
 		let gb = spec.genesis_block();
 		let chain = Arc::new(RwLock::new(BlockChain::new(config.blockchain, &gb, path)));
@@ -220,7 +246,7 @@ impl Client {
 		state_path.push("state");
 
 		let engine = Arc::new(try!(spec.to_engine()));
-		let mut state_db = JournalDB::new(state_path.to_str().unwrap());
+		let mut state_db = JournalDB::from_prefs(state_path.to_str().unwrap(), config.prefer_journal);
 		if state_db.is_empty() && engine.spec().ensure_db_good(&mut state_db) {
 			state_db.commit(0, &engine.spec().genesis_header().hash(), None).expect("Error commiting genesis state to state DB");
 		}
@@ -229,6 +255,9 @@ impl Client {
 		let panic_handler = PanicHandler::new_in_arc();
 		panic_handler.forward_from(&block_queue);
 
+		let secret_store = Arc::new(RwLock::new(SecretStore::new()));
+		secret_store.write().unwrap().try_import_existing();
+
 		Ok(Arc::new(Client {
 			chain: chain,
 			engine: engine,
@@ -240,6 +269,8 @@ impl Client {
 			sealing_block: Mutex::new(None),
 			author: RwLock::new(Address::new()),
 			extra_data: RwLock::new(Vec::new()),
+			verifier: PhantomData,
+			secret_store: secret_store,
 		}))
 	}
 
@@ -264,6 +295,11 @@ impl Client {
 		last_hashes
 	}
 
+	/// Secret store (key manager)
+	pub fn secret_store(&self) -> &Arc<RwLock<SecretStore>> {
+		&self.secret_store
+	}
+
 	fn check_and_close_block(&self, block: &PreverifiedBlock) -> Result<ClosedBlock, ()> {
 		let engine = self.engine.deref().deref();
 		let header = &block.header;
@@ -302,7 +338,7 @@ impl Client {
 
 		// Final Verification
 		let closed_block = enact_result.unwrap();
-		if let Err(e) = verify_block_final(&header, closed_block.block().header()) {
+		if let Err(e) = V::verify_block_final(&header, closed_block.block().header()) {
 			warn!(target: "client", "Stage 4 block verification failed for #{} ({})\nError: {:?}", header.number(), header.hash(), e);
 			return Err(());
 		}
@@ -505,7 +541,7 @@ impl Client {
 
 // TODO: need MinerService MinerIoHandler
 
-impl BlockChainClient for Client {
+impl<V> BlockChainClient for Client<V> where V: Verifier {
 	fn block_header(&self, id: BlockId) -> Option<Bytes> {
 		let chain = self.chain.read().unwrap();
 		Self::block_hash(&chain, id).and_then(|hash| chain.block(&hash).map(|bytes| BlockView::new(&bytes).rlp().at(0).as_raw().to_vec()))
@@ -549,6 +585,11 @@ impl BlockChainClient for Client {
 		self.state().nonce(address)
 	}
 
+	fn block_hash(&self, id: BlockId) -> Option<H256> {
+		let chain = self.chain.read().unwrap();
+		Self::block_hash(&chain, id)
+	}
+
 	fn code(&self, address: &Address) -> Option<Bytes> {
 		self.state().code(address)
 	}

ethcore/src/filter.rs

@@ -42,6 +42,22 @@ pub struct Filter {
 	pub topics: [Option<Vec<H256>>; 4],
 }
 
+impl Clone for Filter {
+	fn clone(&self) -> Self {
+		let mut topics = [None, None, None, None];
+		for i in 0..4 {
+			topics[i] = self.topics[i].clone();
+		}
+
+		Filter {
+			from_block: self.from_block.clone(),
+			to_block: self.to_block.clone(),
+			address: self.address.clone(),
+			topics: topics
+		}
+	}
+}
+
 impl Filter {
 	/// Returns combinations of each address and topic.	
 	pub fn bloom_possibilities(&self) -> Vec<H2048> {

ethcore/src/tests/client.rs

@@ -108,6 +108,25 @@ fn can_collect_garbage() {
 	assert!(client.blockchain_cache_info().blocks < 100 * 1024);
 }
 
+#[test]
+fn can_handle_long_fork() {
+	let client_result = generate_dummy_client(1200);
+	let client = client_result.reference();
+	for _ in 0..10 {
+		client.import_verified_blocks(&IoChannel::disconnected());
+	}
+	assert_eq!(1200, client.chain_info().best_block_number);
+
+	push_blocks_to_client(client, 45, 1201, 800);
+	push_blocks_to_client(client, 49, 1201, 800);
+	push_blocks_to_client(client, 53, 1201, 600);
+
+	for _ in 0..20 {
+		client.import_verified_blocks(&IoChannel::disconnected());
+	}
+	assert_eq!(2000, client.chain_info().best_block_number);
+}
+
 #[test]
 fn can_mine() {
 	let dummy_blocks = get_good_dummy_block_seq(2);
@@ -122,7 +141,7 @@ fn can_mine() {
 				b.hash()
 			}
 			None => { panic!(); }
-		}	
+		}
 	};
 	assert!(client.submit_seal(pow_hash, vec![]).is_ok());
-}
+}

ethcore/src/tests/helpers.rs

@@ -156,10 +156,9 @@ pub fn generate_dummy_client(block_number: u32) -> GuardedTempResult<Arc<Client>
 		rolling_block_number = rolling_block_number + 1;
 		rolling_timestamp = rolling_timestamp + 10;
 
-		if let Err(_) = client.import_block(create_test_block(&header)) {
-			panic!("error importing block which is valid by definition");
+		if let Err(e) = client.import_block(create_test_block(&header)) {
+			panic!("error importing block which is valid by definition: {:?}", e);
 		}
-
 	}
 	client.flush_queue();
 	client.import_verified_blocks(&IoChannel::disconnected());
@@ -170,6 +169,34 @@ pub fn generate_dummy_client(block_number: u32) -> GuardedTempResult<Arc<Client>
 	}
 }
 
+pub fn push_blocks_to_client(client: &Arc<Client>, timestamp_salt: u64, starting_number: usize, block_number: usize) {
+	let test_spec = get_test_spec();
+	let test_engine = test_spec.to_engine().unwrap();
+	let state_root = test_engine.spec().genesis_header().state_root;
+	let mut rolling_hash = client.chain_info().best_block_hash;
+	let mut rolling_block_number = starting_number as u64;
+	let mut rolling_timestamp = timestamp_salt + starting_number as u64 * 10;
+
+	for _ in 0..block_number {
+		let mut header = Header::new();
+
+		header.gas_limit = decode(test_engine.spec().engine_params.get("minGasLimit").unwrap());
+		header.difficulty = decode(test_engine.spec().engine_params.get("minimumDifficulty").unwrap());
+		header.timestamp = rolling_timestamp;
+		header.number = rolling_block_number;
+		header.parent_hash = rolling_hash;
+		header.state_root = state_root.clone();
+
+		rolling_hash = header.hash();
+		rolling_block_number = rolling_block_number + 1;
+		rolling_timestamp = rolling_timestamp + 10;
+
+		if let Err(e) = client.import_block(create_test_block(&header)) {
+			panic!("error importing block which is valid by definition: {:?}", e);
+		}
+	}
+}
+
 pub fn get_test_client_with_blocks(blocks: Vec<Bytes>) -> GuardedTempResult<Arc<Client>> {
 	let dir = RandomTempPath::new();
 	let client = Client::new(ClientConfig::default(), get_test_spec(), dir.as_path(), IoChannel::disconnected()).unwrap();
@@ -253,18 +280,29 @@ pub fn get_temp_state_in(path: &Path) -> State {
 pub fn get_good_dummy_block_seq(count: usize) -> Vec<Bytes> {
 	let test_spec = get_test_spec();
 	let test_engine = test_spec.to_engine().unwrap();
-	let mut parent = test_engine.spec().genesis_header().hash();
+  	get_good_dummy_block_fork_seq(1, count, &test_engine.spec().genesis_header().hash())
+}
+
+pub fn get_good_dummy_block_fork_seq(start_number: usize, count: usize, parent_hash: &H256) -> Vec<Bytes> {
+	let test_spec = get_test_spec();
+	let test_engine = test_spec.to_engine().unwrap();
+	let mut rolling_timestamp = start_number as u64 * 10;
+	let mut parent = *parent_hash;
 	let mut r = Vec::new();
-	for i in 1 .. count + 1 {
+	for i in start_number .. start_number + count + 1 {
 		let mut block_header = Header::new();
 		block_header.gas_limit = decode(test_engine.spec().engine_params.get("minGasLimit").unwrap());
-		block_header.difficulty = decode(test_engine.spec().engine_params.get("minimumDifficulty").unwrap());
-		block_header.timestamp = i as u64;
+		block_header.difficulty = U256::from(i).mul(U256([0, 1, 0, 0]));
+		block_header.timestamp = rolling_timestamp;
 		block_header.number = i as u64;
 		block_header.parent_hash = parent;
 		block_header.state_root = test_engine.spec().genesis_header().state_root;
+
 		parent = block_header.hash();
+		rolling_timestamp = rolling_timestamp + 10;
+
 		r.push(create_test_block(&block_header));
+
 	}
 	r
 }

ethcore/src/verification/canon_verifier.rs

@@ -0,0 +1,28 @@
+// Copyright 2015, 2016 Ethcore (UK) Ltd.
+// This file is part of Parity.
+
+// Parity is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Parity is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Parity.  If not, see <http://www.gnu.org/licenses/>.
+
+use error::Error;
+use header::Header;
+use super::Verifier;
+use super::verification;
+
+pub struct CanonVerifier;
+
+impl Verifier for CanonVerifier {
+	fn verify_block_final(expected: &Header, got: &Header) -> Result<(), Error> {
+		verification::verify_block_final(expected, got)
+	}
+}

ethcore/src/verification/mod.rs

@@ -0,0 +1,25 @@
+// Copyright 2015, 2016 Ethcore (UK) Ltd.
+// This file is part of Parity.
+
+// Parity is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Parity is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Parity.  If not, see <http://www.gnu.org/licenses/>.
+
+pub mod verification;
+pub mod verifier;
+mod canon_verifier;
+mod noop_verifier;
+
+pub use self::verification::*;
+pub use self::verifier::Verifier;
+pub use self::canon_verifier::CanonVerifier;
+pub use self::noop_verifier::NoopVerifier;

ethcore/src/verification/noop_verifier.rs

@@ -0,0 +1,27 @@
+// Copyright 2015, 2016 Ethcore (UK) Ltd.
+// This file is part of Parity.
+
+// Parity is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Parity is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Parity.  If not, see <http://www.gnu.org/licenses/>.
+
+use error::Error;
+use header::Header;
+use super::Verifier;
+
+pub struct NoopVerifier;
+
+impl Verifier for NoopVerifier {
+	fn verify_block_final(_expected: &Header, _got: &Header) -> Result<(), Error> {
+		Ok(())
+	}
+}

ethcore/src/verification/verification.rs

@@ -61,7 +61,7 @@ pub fn verify_block_unordered(header: Header, bytes: Bytes, engine: &Engine) ->
 	for u in Rlp::new(&bytes).at(2).iter().map(|rlp| rlp.as_val::<Header>()) {
 		try!(engine.verify_block_unordered(&u, None));
 	}
-	// Verify transactions. 
+	// Verify transactions.
 	let mut transactions = Vec::new();
 	{
 		let v = BlockView::new(&bytes);
@@ -141,7 +141,7 @@ pub fn verify_block_family<BC>(header: &Header, bytes: &[u8], engine: &Engine, b
 			let uncle_parent = try!(bc.block_header(&uncle.parent_hash).ok_or_else(|| Error::from(BlockError::UnknownUncleParent(uncle.parent_hash.clone()))));
 			for _ in 0..depth {
 				match bc.block_details(&expected_uncle_parent) {
-					Some(details) => { 
+					Some(details) => {
 						expected_uncle_parent = details.parent;
 					},
 					None => break
@@ -468,7 +468,7 @@ mod tests {
 		header.number = 9;
 		check_fail(family_test(&create_test_block_with_data(&header, &good_transactions, &good_uncles), engine.deref(), &bc),
 			InvalidNumber(Mismatch { expected: parent.number + 1, found: header.number }));
-		
+
 		header = good.clone();
 		let mut bad_uncles = good_uncles.clone();
 		bad_uncles.push(good_uncle1.clone());

ethcore/src/verification/verifier.rs

@@ -0,0 +1,23 @@
+// Copyright 2015, 2016 Ethcore (UK) Ltd.
+// This file is part of Parity.
+
+// Parity is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Parity is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Parity.  If not, see <http://www.gnu.org/licenses/>.
+
+use error::Error;
+use header::Header;
+
+/// Should be used to verify blocks.
+pub trait Verifier: Send + Sync {
+	fn verify_block_final(expected: &Header, got: &Header) -> Result<(), Error>;
+}