Merge branch 'master' into ethminer_crate

Conflicts:
	ethcore/src/client/client.rs
	parity/main.rs
	sync/src/chain.rs
	sync/src/lib.rs

ethcore/src/block.rs

@@ -171,7 +171,7 @@ pub struct SealedBlock {
 
 impl<'x> OpenBlock<'x> {
 	/// Create a new OpenBlock ready for transaction pushing.
-	pub fn new(engine: &'x Engine, db: JournalDB, parent: &Header, last_hashes: LastHashes, author: Address, extra_data: Bytes) -> Self {
+	pub fn new(engine: &'x Engine, db: Box<JournalDB>, parent: &Header, last_hashes: LastHashes, author: Address, extra_data: Bytes) -> Self {
 		let mut r = OpenBlock {
 			block: ExecutedBlock::new(State::from_existing(db, parent.state_root().clone(), engine.account_start_nonce())),
 			engine: engine,
@@ -317,7 +317,7 @@ impl ClosedBlock {
 	}
 
 	/// Drop this object and return the underlieing database.
-	pub fn drain(self) -> JournalDB { self.block.state.drop().1 }
+	pub fn drain(self) -> Box<JournalDB> { self.block.state.drop().1 }
 }
 
 impl SealedBlock {
@@ -331,7 +331,7 @@ impl SealedBlock {
 	}
 
 	/// Drop this object and return the underlieing database.
-	pub fn drain(self) -> JournalDB { self.block.state.drop().1 }
+	pub fn drain(self) -> Box<JournalDB> { self.block.state.drop().1 }
 }
 
 impl IsBlock for SealedBlock {
@@ -339,10 +339,10 @@ impl IsBlock for SealedBlock {
 }
 
 /// Enact the block given by block header, transactions and uncles
-pub fn enact(header: &Header, transactions: &[SignedTransaction], uncles: &[Header], engine: &Engine, db: JournalDB, parent: &Header, last_hashes: LastHashes) -> Result<ClosedBlock, Error> {
+pub fn enact(header: &Header, transactions: &[SignedTransaction], uncles: &[Header], engine: &Engine, db: Box<JournalDB>, parent: &Header, last_hashes: LastHashes) -> Result<ClosedBlock, Error> {
 	{
 		if ::log::max_log_level() >= ::log::LogLevel::Trace {
-			let s = State::from_existing(db.clone(), parent.state_root().clone(), engine.account_start_nonce());
+			let s = State::from_existing(db.spawn(), parent.state_root().clone(), engine.account_start_nonce());
 			trace!("enact(): root={}, author={}, author_balance={}\n", s.root(), header.author(), s.balance(&header.author()));
 		}
 	}
@@ -357,20 +357,20 @@ pub fn enact(header: &Header, transactions: &[SignedTransaction], uncles: &[Head
 }
 
 /// Enact the block given by `block_bytes` using `engine` on the database `db` with given `parent` block header
-pub fn enact_bytes(block_bytes: &[u8], engine: &Engine, db: JournalDB, parent: &Header, last_hashes: LastHashes) -> Result<ClosedBlock, Error> {
+pub fn enact_bytes(block_bytes: &[u8], engine: &Engine, db: Box<JournalDB>, parent: &Header, last_hashes: LastHashes) -> Result<ClosedBlock, Error> {
 	let block = BlockView::new(block_bytes);
 	let header = block.header();
 	enact(&header, &block.transactions(), &block.uncles(), engine, db, parent, last_hashes)
 }
 
 /// Enact the block given by `block_bytes` using `engine` on the database `db` with given `parent` block header
-pub fn enact_verified(block: &PreverifiedBlock, engine: &Engine, db: JournalDB, parent: &Header, last_hashes: LastHashes) -> Result<ClosedBlock, Error> {
+pub fn enact_verified(block: &PreverifiedBlock, engine: &Engine, db: Box<JournalDB>, parent: &Header, last_hashes: LastHashes) -> Result<ClosedBlock, Error> {
 	let view = BlockView::new(&block.bytes);
 	enact(&block.header, &block.transactions, &view.uncles(), engine, db, parent, last_hashes)
 }
 
 /// Enact the block given by `block_bytes` using `engine` on the database `db` with given `parent` block header. Seal the block aferwards
-pub fn enact_and_seal(block_bytes: &[u8], engine: &Engine, db: JournalDB, parent: &Header, last_hashes: LastHashes) -> Result<SealedBlock, Error> {
+pub fn enact_and_seal(block_bytes: &[u8], engine: &Engine, db: Box<JournalDB>, parent: &Header, last_hashes: LastHashes) -> Result<SealedBlock, Error> {
 	let header = BlockView::new(block_bytes).header_view();
 	Ok(try!(try!(enact_bytes(block_bytes, engine, db, parent, last_hashes)).seal(engine, header.seal())))
 }
@@ -389,7 +389,7 @@ mod tests {
 		let genesis_header = engine.spec().genesis_header();
 		let mut db_result = get_temp_journal_db();
 		let mut db = db_result.take();
-		engine.spec().ensure_db_good(&mut db);
+		engine.spec().ensure_db_good(db.as_hashdb_mut());
 		let last_hashes = vec![genesis_header.hash()];
 		let b = OpenBlock::new(engine.deref(), db, &genesis_header, last_hashes, Address::zero(), vec![]);
 		let b = b.close();
@@ -404,14 +404,14 @@ mod tests {
 
 		let mut db_result = get_temp_journal_db();
 		let mut db = db_result.take();
-		engine.spec().ensure_db_good(&mut db);
+		engine.spec().ensure_db_good(db.as_hashdb_mut());
 		let b = OpenBlock::new(engine.deref(), db, &genesis_header, vec![genesis_header.hash()], Address::zero(), vec![]).close().seal(engine.deref(), vec![]).unwrap();
 		let orig_bytes = b.rlp_bytes();
 		let orig_db = b.drain();
 
 		let mut db_result = get_temp_journal_db();
 		let mut db = db_result.take();
-		engine.spec().ensure_db_good(&mut db);
+		engine.spec().ensure_db_good(db.as_hashdb_mut());
 		let e = enact_and_seal(&orig_bytes, engine.deref(), db, &genesis_header, vec![genesis_header.hash()]).unwrap();
 
 		assert_eq!(e.rlp_bytes(), orig_bytes);

ethcore/src/block_queue.rs

@@ -412,6 +412,7 @@ impl BlockQueue {
 		}
 	}
 
+	/// Optimise memory footprint of the heap fields.
 	pub fn collect_garbage(&self) {
 		{
 			self.verification.unverified.lock().unwrap().shrink_to_fit();

ethcore/src/client/client.rs

@@ -100,7 +100,7 @@ impl ClientReport {
 pub struct Client<V = CanonVerifier> where V: Verifier {
 	chain: Arc<BlockChain>,
 	engine: Arc<Box<Engine>>,
-	state_db: Mutex<JournalDB>,
+	state_db: Mutex<Box<JournalDB>>,
 	block_queue: BlockQueue,
 	report: RwLock<ClientReport>,
 	import_lock: Mutex<()>,
@@ -124,7 +124,8 @@ impl<V> Client<V> where V: Verifier {
 		let mut dir = path.to_path_buf();
 		dir.push(H64::from(spec.genesis_header().hash()).hex());
 		//TODO: sec/fat: pruned/full versioning
-		dir.push(format!("v{}-sec-{}", CLIENT_DB_VER_STR, if config.prefer_journal { "pruned" } else { "archive" }));
+		// version here is a bit useless now, since it's controlled only be the pruning algo.
+		dir.push(format!("v{}-sec-{}", CLIENT_DB_VER_STR, config.pruning));
 		let path = dir.as_path();
 		let gb = spec.genesis_block();
 		let chain = Arc::new(BlockChain::new(config.blockchain, &gb, path));
@@ -132,8 +133,10 @@ impl<V> Client<V> where V: Verifier {
 		state_path.push("state");
 
 		let engine = Arc::new(try!(spec.to_engine()));
-		let mut state_db = JournalDB::from_prefs(state_path.to_str().unwrap(), config.prefer_journal);
+		let state_path_str = state_path.to_str().unwrap();
-		if state_db.is_empty() && engine.spec().ensure_db_good(&mut state_db) {
+		let mut state_db = journaldb::new(state_path_str, config.pruning);
+
+		if state_db.is_empty() && engine.spec().ensure_db_good(state_db.as_hashdb_mut()) {
 			state_db.commit(0, &engine.spec().genesis_header().hash(), None).expect("Error commiting genesis state to state DB");
 		}
 
@@ -201,7 +204,7 @@ impl<V> Client<V> where V: Verifier {
 		// Enact Verified Block
 		let parent = chain_has_parent.unwrap();
 		let last_hashes = self.build_last_hashes(header.parent_hash.clone());
-		let db = self.state_db.lock().unwrap().clone();
+		let db = self.state_db.lock().unwrap().spawn();
 
 		let enact_result = enact_verified(&block, engine, db, &parent, last_hashes);
 		if let Err(e) = enact_result {
@@ -302,7 +305,7 @@ impl<V> Client<V> where V: Verifier {
 
 	/// Get a copy of the best block's state.
 	pub fn state(&self) -> State {
-		State::from_existing(self.state_db.lock().unwrap().clone(), HeaderView::new(&self.best_block_header()).state_root(), self.engine.account_start_nonce())
+		State::from_existing(self.state_db.lock().unwrap().spawn(), HeaderView::new(&self.best_block_header()).state_root(), self.engine.account_start_nonce())
 	}
 
 	/// Get info on the cache.
@@ -363,7 +366,7 @@ impl<V> BlockChainClient for Client<V> where V: Verifier {
 
 		let mut b = OpenBlock::new(
 			engine,
-			self.state_db.lock().unwrap().clone(),
+			self.state_db.lock().unwrap().spawn(),
 			match self.chain.block_header(&h) { Some(ref x) => x, None => {return None} },
 			self.build_last_hashes(h.clone()),
 			author,

ethcore/src/client/config.rs

@@ -16,6 +16,7 @@
 
 pub use block_queue::BlockQueueConfig;
 pub use blockchain::BlockChainConfig;
+use util::journaldb;
 
 /// Client configuration. Includes configs for all sub-systems.
 #[derive(Debug, Default)]
@@ -24,8 +25,8 @@ pub struct ClientConfig {
 	pub queue: BlockQueueConfig,
 	/// Blockchain configuration.
 	pub blockchain: BlockChainConfig,
-	/// Prefer journal rather than archive.
+	/// The JournalDB ("pruning") algorithm to use.
-	pub prefer_journal: bool,
+	pub pruning: journaldb::Algorithm,
 	/// The name of the client instance.
 	pub name: String,
 }

ethcore/src/ethereum/ethash.rs

@@ -298,7 +298,7 @@ mod tests {
 		let genesis_header = engine.spec().genesis_header();
 		let mut db_result = get_temp_journal_db();
 		let mut db = db_result.take();
-		engine.spec().ensure_db_good(&mut db);
+		engine.spec().ensure_db_good(db.as_hashdb_mut());
 		let last_hashes = vec![genesis_header.hash()];
 		let b = OpenBlock::new(engine.deref(), db, &genesis_header, last_hashes, Address::zero(), vec![]);
 		let b = b.close();
@@ -311,7 +311,7 @@ mod tests {
 		let genesis_header = engine.spec().genesis_header();
 		let mut db_result = get_temp_journal_db();
 		let mut db = db_result.take();
-		engine.spec().ensure_db_good(&mut db);
+		engine.spec().ensure_db_good(db.as_hashdb_mut());
 		let last_hashes = vec![genesis_header.hash()];
 		let mut b = OpenBlock::new(engine.deref(), db, &genesis_header, last_hashes, Address::zero(), vec![]);
 		let mut uncle = Header::new();

ethcore/src/ethereum/mod.rs

@@ -61,7 +61,7 @@ mod tests {
 		let genesis_header = engine.spec().genesis_header();
 		let mut db_result = get_temp_journal_db();
 		let mut db = db_result.take();
-		engine.spec().ensure_db_good(&mut db);
+		engine.spec().ensure_db_good(db.as_hashdb_mut());
 		let s = State::from_existing(db, genesis_header.state_root.clone(), engine.account_start_nonce());
 		assert_eq!(s.balance(&address_from_hex("0000000000000000000000000000000000000001")), U256::from(1u64));
 		assert_eq!(s.balance(&address_from_hex("0000000000000000000000000000000000000002")), U256::from(1u64));

ethcore/src/state.rs

@@ -31,7 +31,7 @@ pub type ApplyResult = Result<Receipt, Error>;
 
 /// Representation of the entire state of all accounts in the system.
 pub struct State {
-	db: JournalDB,
+	db: Box<JournalDB>,
 	root: H256,
 	cache: RefCell<HashMap<Address, Option<Account>>>,
 	snapshots: RefCell<Vec<HashMap<Address, Option<Option<Account>>>>>,
@@ -41,11 +41,11 @@ pub struct State {
 impl State {
 	/// Creates new state with empty state root
 	#[cfg(test)]
-	pub fn new(mut db: JournalDB, account_start_nonce: U256) -> State {
+	pub fn new(mut db: Box<JournalDB>, account_start_nonce: U256) -> State {
 		let mut root = H256::new();
 		{
 			// init trie and reset root too null
-			let _ = SecTrieDBMut::new(&mut db, &mut root);
+			let _ = SecTrieDBMut::new(db.as_hashdb_mut(), &mut root);
 		}
 
 		State {
@@ -58,10 +58,10 @@ impl State {
 	}
 
 	/// Creates new state with existing state root
-	pub fn from_existing(db: JournalDB, root: H256, account_start_nonce: U256) -> State {
+	pub fn from_existing(db: Box<JournalDB>, root: H256, account_start_nonce: U256) -> State {
 		{
 			// trie should panic! if root does not exist
-			let _ = SecTrieDB::new(&db, &root);
+			let _ = SecTrieDB::new(db.as_hashdb(), &root);
 		}
 
 		State {
@@ -126,7 +126,7 @@ impl State {
 	}
 
 	/// Destroy the current object and return root and database.
-	pub fn drop(self) -> (H256, JournalDB) {
+	pub fn drop(self) -> (H256, Box<JournalDB>) {
 		(self.root, self.db)
 	}
 
@@ -148,7 +148,7 @@ impl State {
 
 	/// Determine whether an account exists.
 	pub fn exists(&self, a: &Address) -> bool {
-		self.cache.borrow().get(&a).unwrap_or(&None).is_some() || SecTrieDB::new(&self.db, &self.root).contains(&a)
+		self.cache.borrow().get(&a).unwrap_or(&None).is_some() || SecTrieDB::new(self.db.as_hashdb(), &self.root).contains(&a)
 	}
 
 	/// Get the balance of account `a`.
@@ -163,7 +163,7 @@ impl State {
 
 	/// Mutate storage of account `address` so that it is `value` for `key`.
 	pub fn storage_at(&self, address: &Address, key: &H256) -> H256 {
-		self.get(address, false).as_ref().map_or(H256::new(), |a|a.storage_at(&AccountDB::new(&self.db, address), key))
+		self.get(address, false).as_ref().map_or(H256::new(), |a|a.storage_at(&AccountDB::new(self.db.as_hashdb(), address), key))
 	}
 
 	/// Mutate storage of account `a` so that it is `value` for `key`.
@@ -253,7 +253,7 @@ impl State {
 	/// Commits our cached account changes into the trie.
 	pub fn commit(&mut self) {
 		assert!(self.snapshots.borrow().is_empty());
-		Self::commit_into(&mut self.db, &mut self.root, self.cache.borrow_mut().deref_mut());
+		Self::commit_into(self.db.as_hashdb_mut(), &mut self.root, self.cache.borrow_mut().deref_mut());
 	}
 
 	#[cfg(test)]
@@ -285,11 +285,11 @@ impl State {
 	fn get<'a>(&'a self, a: &Address, require_code: bool) -> &'a Option<Account> {
 		let have_key = self.cache.borrow().contains_key(a);
 		if !have_key {
-			self.insert_cache(a, SecTrieDB::new(&self.db, &self.root).get(&a).map(Account::from_rlp))
+			self.insert_cache(a, SecTrieDB::new(self.db.as_hashdb(), &self.root).get(&a).map(Account::from_rlp))
 		}
 		if require_code {
 			if let Some(ref mut account) = self.cache.borrow_mut().get_mut(a).unwrap().as_mut() {
-				account.cache_code(&AccountDB::new(&self.db, a));
+				account.cache_code(&AccountDB::new(self.db.as_hashdb(), a));
 			}
 		}
 		unsafe { ::std::mem::transmute(self.cache.borrow().get(a).unwrap()) }
@@ -305,7 +305,7 @@ impl State {
 	fn require_or_from<'a, F: FnOnce() -> Account, G: FnOnce(&mut Account)>(&self, a: &Address, require_code: bool, default: F, not_default: G) -> &'a mut Account {
 		let have_key = self.cache.borrow().contains_key(a);
 		if !have_key {
-			self.insert_cache(a, SecTrieDB::new(&self.db, &self.root).get(&a).map(Account::from_rlp))
+			self.insert_cache(a, SecTrieDB::new(self.db.as_hashdb(), &self.root).get(&a).map(Account::from_rlp))
 		} else {
 			self.note_cache(a);
 		}
@@ -318,7 +318,7 @@ impl State {
 
 		unsafe { ::std::mem::transmute(self.cache.borrow_mut().get_mut(a).unwrap().as_mut().map(|account| {
 			if require_code {
-				account.cache_code(&AccountDB::new(&self.db, a));
+				account.cache_code(&AccountDB::new(self.db.as_hashdb(), a));
 			}
 			account
 		}).unwrap()) }

ethcore/src/tests/helpers.rs

@@ -250,9 +250,9 @@ pub fn generate_dummy_empty_blockchain() -> GuardedTempResult<BlockChain> {
 	}
 }
 
-pub fn get_temp_journal_db() -> GuardedTempResult<JournalDB> {
+pub fn get_temp_journal_db() -> GuardedTempResult<Box<JournalDB>> {
 	let temp = RandomTempPath::new();
-	let journal_db = JournalDB::new(temp.as_str());
+	let journal_db = journaldb::new(temp.as_str(), journaldb::Algorithm::EarlyMerge);
 	GuardedTempResult {
 		_temp: temp,
 		result: Some(journal_db)
@@ -268,8 +268,8 @@ pub fn get_temp_state() -> GuardedTempResult<State> {
 	}
 }
 
-pub fn get_temp_journal_db_in(path: &Path) -> JournalDB {
+pub fn get_temp_journal_db_in(path: &Path) -> Box<JournalDB> {
-	JournalDB::new(path.to_str().unwrap())
+	journaldb::new(path.to_str().unwrap(), journaldb::Algorithm::EarlyMerge)
 }
 
 pub fn get_temp_state_in(path: &Path) -> State {

parity/main.rs

@@ -80,6 +80,11 @@ Usage:
 Protocol Options:
   --chain CHAIN            Specify the blockchain type. CHAIN may be either a JSON chain specification file
                            or olympic, frontier, homestead, mainnet, morden, or testnet [default: homestead].
+  --testnet                Equivalent to --chain testnet (geth-compatible).
+  --networkid INDEX        Override the network identifier from the chain we are on.
+  --pruning METHOD         Configure pruning of the state/storage trie. METHOD may be one of: archive,
+                           light (experimental) [default: archive].
+  -d --datadir PATH        Specify the database & configuration directory path [default: $HOME/.parity]
   --db-path PATH           Specify the database & configuration directory path [default: $HOME/.parity]
   --pruning                Client should prune the state/storage trie.
   --keys-path PATH         Specify the path for JSON key files to be found [default: $HOME/.web3/keys]
@@ -101,7 +106,12 @@ API and Console Options:
   --jsonrpc-port PORT      Specify the port portion of the JSONRPC API server [default: 8545].
   --jsonrpc-cors URL       Specify CORS header for JSON-RPC API responses [default: null].
   --jsonrpc-apis APIS      Specify the APIs available through the JSONRPC interface. APIS is a comma-delimited
-                           list of API name. Possible name are web3, eth and net. [default: web3,eth,net].
+                           list of API name. Possible names are web3, eth and net. [default: web3,eth,net].
+  --rpc                    Equivalent to --jsonrpc (geth-compatible).
+  --rpcaddr HOST           Equivalent to --jsonrpc-addr HOST (geth-compatible).
+  --rpcport PORT           Equivalent to --jsonrpc-port PORT (geth-compatible).
+  --rpcapi APIS            Equivalent to --jsonrpc-apis APIS (geth-compatible).
+  --rpccorsdomain URL      Equivalent to --jsonrpc-cors URL (geth-compatible).
 
 Sealing/Mining Options:
   --gasprice GAS           Minimal gas price a transaction must have to be accepted for mining [default: 20000000000].
@@ -150,7 +160,7 @@ struct Args {
 	flag_cache: Option<usize>,
 	flag_keys_path: String,
 	flag_bootnodes: Option<String>,
-	flag_pruning: bool,
+	flag_pruning: String,
 	flag_no_bootstrap: bool,
 	flag_port: u16,
 	flag_peers: usize,
@@ -371,7 +381,14 @@ impl Configuration {
 				client_config.blockchain.max_cache_size = self.args.flag_cache_max_size;
 			}
 		}
-		client_config.prefer_journal = self.args.flag_pruning;
+		client_config.pruning = match self.args.flag_pruning.as_str() {
+			"" => journaldb::Algorithm::Archive,
+			"archive" => journaldb::Algorithm::Archive,
+			"pruned" => journaldb::Algorithm::EarlyMerge,
+//			"fast" => journaldb::Algorithm::OverlayRecent,	// TODO: @arkpar uncomment this once option 2 is merged.
+//			"slow" => journaldb::Algorithm::RefCounted,		// TODO: @gavofyork uncomment this once ref-count algo is merged.
+			_ => { die!("Invalid pruning method given."); }
+		};
 		client_config.name = self.args.flag_identity.clone();
 		client_config.queue.max_mem_use = self.args.flag_queue_max_size;
 		client_config

util/src/hashdb.rs

@@ -20,7 +20,7 @@ use bytes::*;
 use std::collections::HashMap;
 
 /// Trait modelling datastore keyed by a 32-byte Keccak hash.
-pub trait HashDB {
+pub trait HashDB : AsHashDB {
 	/// Get the keys in the database together with number of underlying references.
 	fn keys(&self) -> HashMap<H256, i32>;
 
@@ -111,3 +111,16 @@ pub trait HashDB {
 	/// ```
 	fn remove(&mut self, key: &H256) { self.kill(key) }
 }
+
+/// Upcast trait.
+pub trait AsHashDB {
+	/// Perform upcast to HashDB for anything that derives from HashDB.
+	fn as_hashdb(&self) -> &HashDB;
+	/// Perform mutable upcast to HashDB for anything that derives from HashDB.
+	fn as_hashdb_mut(&mut self) -> &mut HashDB;
+}
+
+impl<T: HashDB> AsHashDB for T {
+	fn as_hashdb(&self) -> &HashDB { self }
+	fn as_hashdb_mut(&mut self) -> &mut HashDB { self }
+}

util/src/journaldb/archivedb.rs

@@ -0,0 +1,388 @@
+// Copyright 2015, 2016 Ethcore (UK) Ltd.
+// This file is part of Parity.
+
+// Parity is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Parity is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Parity.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Disk-backed HashDB implementation.
+
+use common::*;
+use rlp::*;
+use hashdb::*;
+use memorydb::*;
+use super::traits::JournalDB;
+use kvdb::{Database, DBTransaction, DatabaseConfig};
+#[cfg(test)]
+use std::env;
+
+/// Implementation of the HashDB trait for a disk-backed database with a memory overlay
+/// and latent-removal semantics.
+///
+/// Like OverlayDB, there is a memory overlay; `commit()` must be called in order to
+/// write operations out to disk. Unlike OverlayDB, `remove()` operations do not take effect
+/// immediately. Rather some age (based on a linear but arbitrary metric) must pass before
+/// the removals actually take effect.
+pub struct ArchiveDB {
+	overlay: MemoryDB,
+	backing: Arc<Database>,
+}
+
+// all keys must be at least 12 bytes
+const LATEST_ERA_KEY : [u8; 12] = [ b'l', b'a', b's', b't', 0, 0, 0, 0, 0, 0, 0, 0 ];
+const VERSION_KEY : [u8; 12] = [ b'j', b'v', b'e', b'r', 0, 0, 0, 0, 0, 0, 0, 0 ];
+const DB_VERSION : u32 = 259;
+
+impl ArchiveDB {
+	/// Create a new instance from file
+	pub fn new(path: &str) -> ArchiveDB {
+		let opts = DatabaseConfig {
+			prefix_size: Some(12) //use 12 bytes as prefix, this must match account_db prefix
+		};
+		let backing = Database::open(&opts, path).unwrap_or_else(|e| {
+			panic!("Error opening state db: {}", e);
+		});
+		if !backing.is_empty() {
+			match backing.get(&VERSION_KEY).map(|d| d.map(|v| decode::<u32>(&v))) {
+				Ok(Some(DB_VERSION)) => {},
+				v => panic!("Incompatible DB version, expected {}, got {:?}", DB_VERSION, v)
+			}
+		} else {
+			backing.put(&VERSION_KEY, &encode(&DB_VERSION)).expect("Error writing version to database");
+		}
+
+		ArchiveDB {
+			overlay: MemoryDB::new(),
+			backing: Arc::new(backing),
+		}
+	}
+
+	/// Create a new instance with an anonymous temporary database.
+	#[cfg(test)]
+	fn new_temp() -> ArchiveDB {
+		let mut dir = env::temp_dir();
+		dir.push(H32::random().hex());
+		Self::new(dir.to_str().unwrap())
+	}
+
+	fn payload(&self, key: &H256) -> Option<Bytes> {
+		self.backing.get(&key.bytes()).expect("Low-level database error. Some issue with your hard disk?").map(|v| v.to_vec())
+	}
+}
+
+impl HashDB for ArchiveDB {
+	fn keys(&self) -> HashMap<H256, i32> {
+		let mut ret: HashMap<H256, i32> = HashMap::new();
+		for (key, _) in self.backing.iter() {
+			let h = H256::from_slice(key.deref());
+			ret.insert(h, 1);
+		}
+
+		for (key, refs) in self.overlay.keys().into_iter() {
+			let refs = *ret.get(&key).unwrap_or(&0) + refs;
+			ret.insert(key, refs);
+		}
+		ret
+	}
+
+	fn lookup(&self, key: &H256) -> Option<&[u8]> {
+		let k = self.overlay.raw(key);
+		match k {
+			Some(&(ref d, rc)) if rc > 0 => Some(d),
+			_ => {
+				if let Some(x) = self.payload(key) {
+					Some(&self.overlay.denote(key, x).0)
+				}
+				else {
+					None
+				}
+			}
+		}
+	}
+
+	fn exists(&self, key: &H256) -> bool {
+		self.lookup(key).is_some()
+	}
+
+	fn insert(&mut self, value: &[u8]) -> H256 {
+		self.overlay.insert(value)
+	}
+	fn emplace(&mut self, key: H256, value: Bytes) {
+		self.overlay.emplace(key, value);
+	}
+	fn kill(&mut self, key: &H256) {
+		self.overlay.kill(key);
+	}
+}
+
+impl JournalDB for ArchiveDB {
+	fn spawn(&self) -> Box<JournalDB> {
+		Box::new(ArchiveDB {
+			overlay: MemoryDB::new(),
+			backing: self.backing.clone(),
+		})
+	}
+
+	fn mem_used(&self) -> usize {
+		self.overlay.mem_used()
+ 	}
+
+	fn is_empty(&self) -> bool {
+		self.backing.get(&LATEST_ERA_KEY).expect("Low level database error").is_none()
+	}
+
+	fn commit(&mut self, _: u64, _: &H256, _: Option<(u64, H256)>) -> Result<u32, UtilError> {
+		let batch = DBTransaction::new();
+		let mut inserts = 0usize;
+		let mut deletes = 0usize;
+		for i in self.overlay.drain().into_iter() {
+			let (key, (value, rc)) = i;
+			if rc > 0 {
+				assert!(rc == 1);
+				batch.put(&key.bytes(), &value).expect("Low-level database error. Some issue with your hard disk?");
+				inserts += 1;
+			}
+			if rc < 0 {
+				assert!(rc == -1);
+				deletes += 1;
+			}
+		}
+		try!(self.backing.write(batch));
+		Ok((inserts + deletes) as u32)
+	}
+}
+
+#[cfg(test)]
+mod tests {
+	use common::*;
+	use super::*;
+	use hashdb::*;
+	use journaldb::traits::JournalDB;
+
+	#[test]
+	fn insert_same_in_fork() {
+		// history is 1
+		let mut jdb = ArchiveDB::new_temp();
+
+		let x = jdb.insert(b"X");
+		jdb.commit(1, &b"1".sha3(), None).unwrap();
+		jdb.commit(2, &b"2".sha3(), None).unwrap();
+		jdb.commit(3, &b"1002a".sha3(), Some((1, b"1".sha3()))).unwrap();
+		jdb.commit(4, &b"1003a".sha3(), Some((2, b"2".sha3()))).unwrap();
+
+		jdb.remove(&x);
+		jdb.commit(3, &b"1002b".sha3(), Some((1, b"1".sha3()))).unwrap();
+		let x = jdb.insert(b"X");
+		jdb.commit(4, &b"1003b".sha3(), Some((2, b"2".sha3()))).unwrap();
+
+		jdb.commit(5, &b"1004a".sha3(), Some((3, b"1002a".sha3()))).unwrap();
+		jdb.commit(6, &b"1005a".sha3(), Some((4, b"1003a".sha3()))).unwrap();
+
+		assert!(jdb.exists(&x));
+	}
+
+	#[test]
+	fn long_history() {
+		// history is 3
+		let mut jdb = ArchiveDB::new_temp();
+		let h = jdb.insert(b"foo");
+		jdb.commit(0, &b"0".sha3(), None).unwrap();
+		assert!(jdb.exists(&h));
+		jdb.remove(&h);
+		jdb.commit(1, &b"1".sha3(), None).unwrap();
+		assert!(jdb.exists(&h));
+		jdb.commit(2, &b"2".sha3(), None).unwrap();
+		assert!(jdb.exists(&h));
+		jdb.commit(3, &b"3".sha3(), Some((0, b"0".sha3()))).unwrap();
+		assert!(jdb.exists(&h));
+		jdb.commit(4, &b"4".sha3(), Some((1, b"1".sha3()))).unwrap();
+	}
+
+	#[test]
+	fn complex() {
+		// history is 1
+		let mut jdb = ArchiveDB::new_temp();
+
+		let foo = jdb.insert(b"foo");
+		let bar = jdb.insert(b"bar");
+		jdb.commit(0, &b"0".sha3(), None).unwrap();
+		assert!(jdb.exists(&foo));
+		assert!(jdb.exists(&bar));
+
+		jdb.remove(&foo);
+		jdb.remove(&bar);
+		let baz = jdb.insert(b"baz");
+		jdb.commit(1, &b"1".sha3(), Some((0, b"0".sha3()))).unwrap();
+		assert!(jdb.exists(&foo));
+		assert!(jdb.exists(&bar));
+		assert!(jdb.exists(&baz));
+
+		let foo = jdb.insert(b"foo");
+		jdb.remove(&baz);
+		jdb.commit(2, &b"2".sha3(), Some((1, b"1".sha3()))).unwrap();
+		assert!(jdb.exists(&foo));
+		assert!(jdb.exists(&baz));
+
+		jdb.remove(&foo);
+		jdb.commit(3, &b"3".sha3(), Some((2, b"2".sha3()))).unwrap();
+		assert!(jdb.exists(&foo));
+
+		jdb.commit(4, &b"4".sha3(), Some((3, b"3".sha3()))).unwrap();
+	}
+
+	#[test]
+	fn fork() {
+		// history is 1
+		let mut jdb = ArchiveDB::new_temp();
+
+		let foo = jdb.insert(b"foo");
+		let bar = jdb.insert(b"bar");
+		jdb.commit(0, &b"0".sha3(), None).unwrap();
+		assert!(jdb.exists(&foo));
+		assert!(jdb.exists(&bar));
+
+		jdb.remove(&foo);
+		let baz = jdb.insert(b"baz");
+		jdb.commit(1, &b"1a".sha3(), Some((0, b"0".sha3()))).unwrap();
+
+		jdb.remove(&bar);
+		jdb.commit(1, &b"1b".sha3(), Some((0, b"0".sha3()))).unwrap();
+
+		assert!(jdb.exists(&foo));
+		assert!(jdb.exists(&bar));
+		assert!(jdb.exists(&baz));
+
+		jdb.commit(2, &b"2b".sha3(), Some((1, b"1b".sha3()))).unwrap();
+		assert!(jdb.exists(&foo));
+	}
+
+	#[test]
+	fn overwrite() {
+		// history is 1
+		let mut jdb = ArchiveDB::new_temp();
+
+		let foo = jdb.insert(b"foo");
+		jdb.commit(0, &b"0".sha3(), None).unwrap();
+		assert!(jdb.exists(&foo));
+
+		jdb.remove(&foo);
+		jdb.commit(1, &b"1".sha3(), Some((0, b"0".sha3()))).unwrap();
+		jdb.insert(b"foo");
+		assert!(jdb.exists(&foo));
+		jdb.commit(2, &b"2".sha3(), Some((1, b"1".sha3()))).unwrap();
+		assert!(jdb.exists(&foo));
+		jdb.commit(3, &b"2".sha3(), Some((0, b"2".sha3()))).unwrap();
+		assert!(jdb.exists(&foo));
+	}
+
+	#[test]
+	fn fork_same_key() {
+		// history is 1
+		let mut jdb = ArchiveDB::new_temp();
+		jdb.commit(0, &b"0".sha3(), None).unwrap();
+
+		let foo = jdb.insert(b"foo");
+		jdb.commit(1, &b"1a".sha3(), Some((0, b"0".sha3()))).unwrap();
+
+		jdb.insert(b"foo");
+		jdb.commit(1, &b"1b".sha3(), Some((0, b"0".sha3()))).unwrap();
+		assert!(jdb.exists(&foo));
+
+		jdb.commit(2, &b"2a".sha3(), Some((1, b"1a".sha3()))).unwrap();
+		assert!(jdb.exists(&foo));
+	}
+
+
+	#[test]
+	fn reopen() {
+		let mut dir = ::std::env::temp_dir();
+		dir.push(H32::random().hex());
+		let bar = H256::random();
+
+		let foo = {
+			let mut jdb = ArchiveDB::new(dir.to_str().unwrap());
+			// history is 1
+			let foo = jdb.insert(b"foo");
+			jdb.emplace(bar.clone(), b"bar".to_vec());
+			jdb.commit(0, &b"0".sha3(), None).unwrap();
+			foo
+		};
+
+		{
+			let mut jdb = ArchiveDB::new(dir.to_str().unwrap());
+			jdb.remove(&foo);
+			jdb.commit(1, &b"1".sha3(), Some((0, b"0".sha3()))).unwrap();
+		}
+
+		{
+			let mut jdb = ArchiveDB::new(dir.to_str().unwrap());
+			assert!(jdb.exists(&foo));
+			assert!(jdb.exists(&bar));
+			jdb.commit(2, &b"2".sha3(), Some((1, b"1".sha3()))).unwrap();
+		}
+	}
+
+	#[test]
+	fn reopen_remove() {
+		let mut dir = ::std::env::temp_dir();
+		dir.push(H32::random().hex());
+
+		let foo = {
+			let mut jdb = ArchiveDB::new(dir.to_str().unwrap());
+			// history is 1
+			let foo = jdb.insert(b"foo");
+			jdb.commit(0, &b"0".sha3(), None).unwrap();
+			jdb.commit(1, &b"1".sha3(), Some((0, b"0".sha3()))).unwrap();
+
+			// foo is ancient history.
+
+			jdb.insert(b"foo");
+			jdb.commit(2, &b"2".sha3(), Some((1, b"1".sha3()))).unwrap();
+			foo
+		};
+
+		{
+			let mut jdb = ArchiveDB::new(dir.to_str().unwrap());
+			jdb.remove(&foo);
+			jdb.commit(3, &b"3".sha3(), Some((2, b"2".sha3()))).unwrap();
+			assert!(jdb.exists(&foo));
+			jdb.remove(&foo);
+			jdb.commit(4, &b"4".sha3(), Some((3, b"3".sha3()))).unwrap();
+			jdb.commit(5, &b"5".sha3(), Some((4, b"4".sha3()))).unwrap();
+		}
+	}
+	#[test]
+	fn reopen_fork() {
+		let mut dir = ::std::env::temp_dir();
+		dir.push(H32::random().hex());
+		let (foo, bar, baz) = {
+			let mut jdb = ArchiveDB::new(dir.to_str().unwrap());
+			// history is 1
+			let foo = jdb.insert(b"foo");
+			let bar = jdb.insert(b"bar");
+			jdb.commit(0, &b"0".sha3(), None).unwrap();
+			jdb.remove(&foo);
+			let baz = jdb.insert(b"baz");
+			jdb.commit(1, &b"1a".sha3(), Some((0, b"0".sha3()))).unwrap();
+
+			jdb.remove(&bar);
+			jdb.commit(1, &b"1b".sha3(), Some((0, b"0".sha3()))).unwrap();
+			(foo, bar, baz)
+		};
+
+		{
+			let mut jdb = ArchiveDB::new(dir.to_str().unwrap());
+			jdb.commit(2, &b"2b".sha3(), Some((1, b"1b".sha3()))).unwrap();
+			assert!(jdb.exists(&foo));
+		}
+	}
+}

util/src/journaldb/mod.rs

@@ -0,0 +1,78 @@
+// Copyright 2015, 2016 Ethcore (UK) Ltd.
+// This file is part of Parity.
+
+// Parity is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Parity is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Parity.  If not, see <http://www.gnu.org/licenses/>.
+
+//! JournalDB interface and implementation.
+
+use common::*;
+
+/// Export the journaldb module.
+pub mod traits;
+mod archivedb;
+mod optiononedb;
+
+/// Export the JournalDB trait.
+pub use self::traits::JournalDB;
+
+/// A journal database algorithm.
+#[derive(Debug)]
+pub enum Algorithm {
+	/// Keep all keys forever.
+	Archive,
+
+	/// Ancient and recent history maintained separately; recent history lasts for particular
+	/// number of blocks.
+	///
+	/// Inserts go into backing database, journal retains knowledge of whether backing DB key is
+	/// ancient or recent. Non-canon inserts get explicitly reverted and removed from backing DB.
+	EarlyMerge,
+
+	/// Ancient and recent history maintained separately; recent history lasts for particular
+	/// number of blocks.
+	///
+	/// Inserts go into memory overlay, which is tried for key fetches. Memory overlay gets
+	/// flushed in backing only at end of recent history.
+	OverlayRecent,
+
+	/// Ancient and recent history maintained separately; recent history lasts for particular
+	/// number of blocks.
+	///
+	/// References are counted in disk-backed DB.
+	RefCounted,
+}
+
+impl Default for Algorithm {
+	fn default() -> Algorithm { Algorithm::Archive }
+}
+
+impl fmt::Display for Algorithm {
+	fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+		write!(f, "{}", match self {
+			&Algorithm::Archive => "archive",
+			&Algorithm::EarlyMerge => "earlymerge",
+			&Algorithm::OverlayRecent => "overlayrecent",
+			&Algorithm::RefCounted => "refcounted",
+		})
+	}
+}
+
+/// Create a new JournalDB trait object.
+pub fn new(path: &str, algorithm: Algorithm) -> Box<JournalDB> {
+	match algorithm {
+		Algorithm::Archive => Box::new(archivedb::ArchiveDB::new(path)),
+		Algorithm::EarlyMerge => Box::new(optiononedb::OptionOneDB::new(path)),
+		_ => unimplemented!(),
+	}
+}

util/src/journaldb/optiononedb.rs

@@ -20,6 +20,7 @@ use common::*;
 use rlp::*;
 use hashdb::*;
 use memorydb::*;
+use super::traits::JournalDB;
 use kvdb::{Database, DBTransaction, DatabaseConfig};
 #[cfg(test)]
 use std::env;
@@ -31,63 +32,38 @@ use std::env;
 /// write operations out to disk. Unlike OverlayDB, `remove()` operations do not take effect
 /// immediately. Rather some age (based on a linear but arbitrary metric) must pass before
 /// the removals actually take effect.
-pub struct JournalDB {
+pub struct OptionOneDB {
 	overlay: MemoryDB,
 	backing: Arc<Database>,
 	counters: Option<Arc<RwLock<HashMap<H256, i32>>>>,
 }
 
-impl Clone for JournalDB {
-	fn clone(&self) -> JournalDB {
-		JournalDB {
-			overlay: MemoryDB::new(),
-			backing: self.backing.clone(),
-			counters: self.counters.clone(),
-		}
-	}
-}
-
 // all keys must be at least 12 bytes
 const LATEST_ERA_KEY : [u8; 12] = [ b'l', b'a', b's', b't', 0, 0, 0, 0, 0, 0, 0, 0 ];
 const VERSION_KEY : [u8; 12] = [ b'j', b'v', b'e', b'r', 0, 0, 0, 0, 0, 0, 0, 0 ];
-
 const DB_VERSION : u32 = 3;
-const DB_VERSION_NO_JOURNAL : u32 = 3 + 256;
-
 const PADDING : [u8; 10] = [ 0u8; 10 ];
 
-impl JournalDB {
+impl OptionOneDB {
 	/// Create a new instance from file
-	pub fn new(path: &str) -> JournalDB {
+	pub fn new(path: &str) -> OptionOneDB {
-		Self::from_prefs(path, true)
-	}
-
-	/// Create a new instance from file
-	pub fn from_prefs(path: &str, prefer_journal: bool) -> JournalDB {
 		let opts = DatabaseConfig {
 			prefix_size: Some(12) //use 12 bytes as prefix, this must match account_db prefix
 		};
 		let backing = Database::open(&opts, path).unwrap_or_else(|e| {
 			panic!("Error opening state db: {}", e);
 		});
-		let with_journal;
 		if !backing.is_empty() {
 			match backing.get(&VERSION_KEY).map(|d| d.map(|v| decode::<u32>(&v))) {
-				Ok(Some(DB_VERSION)) => { with_journal = true; },
+				Ok(Some(DB_VERSION)) => {},
-				Ok(Some(DB_VERSION_NO_JOURNAL)) => { with_journal = false; },
 				v => panic!("Incompatible DB version, expected {}, got {:?}", DB_VERSION, v)
 			}
 		} else {
-			backing.put(&VERSION_KEY, &encode(&(if prefer_journal { DB_VERSION } else { DB_VERSION_NO_JOURNAL }))).expect("Error writing version to database");
+			backing.put(&VERSION_KEY, &encode(&DB_VERSION)).expect("Error writing version to database");
-			with_journal = prefer_journal;
 		}
 
-		let counters = if with_journal {
+		let counters = Some(Arc::new(RwLock::new(OptionOneDB::read_counters(&backing))));
-			Some(Arc::new(RwLock::new(JournalDB::read_counters(&backing))))
+		OptionOneDB {
-		} else {
-			None
-		};
-		JournalDB {
 			overlay: MemoryDB::new(),
 			backing: Arc::new(backing),
 			counters: counters,
@@ -96,55 +72,12 @@ impl JournalDB {
 
 	/// Create a new instance with an anonymous temporary database.
 	#[cfg(test)]
-	pub fn new_temp() -> JournalDB {
+	fn new_temp() -> OptionOneDB {
 		let mut dir = env::temp_dir();
 		dir.push(H32::random().hex());
 		Self::new(dir.to_str().unwrap())
 	}
 
-	/// Check if this database has any commits
-	pub fn is_empty(&self) -> bool {
-		self.backing.get(&LATEST_ERA_KEY).expect("Low level database error").is_none()
-	}
-
-	/// Commit all recent insert operations.
-	pub fn commit(&mut self, now: u64, id: &H256, end: Option<(u64, H256)>) -> Result<u32, UtilError> {
-		let have_counters = self.counters.is_some();
-		if have_counters {
-			self.commit_with_counters(now, id, end)
-		} else {
-			self.commit_without_counters()
-		}
-	}
-
-	/// Drain the overlay and place it into a batch for the DB.
-	fn batch_overlay_insertions(overlay: &mut MemoryDB, batch: &DBTransaction) -> usize {
-		let mut inserts = 0usize;
-		let mut deletes = 0usize;
-		for i in overlay.drain().into_iter() {
-			let (key, (value, rc)) = i;
-			if rc > 0 {
-				assert!(rc == 1);
-				batch.put(&key.bytes(), &value).expect("Low-level database error. Some issue with your hard disk?");
-				inserts += 1;
-			}
-			if rc < 0 {
-				assert!(rc == -1);
-				deletes += 1;
-			}
-		}
-		trace!("commit: Inserted {}, Deleted {} nodes", inserts, deletes);
-		inserts + deletes
-	}
-
-	/// Just commit the overlay into the backing DB.
-	fn commit_without_counters(&mut self) -> Result<u32, UtilError> {
-		let batch = DBTransaction::new();
-		let ret = Self::batch_overlay_insertions(&mut self.overlay, &batch);
-		try!(self.backing.write(batch));
-		Ok(ret as u32)
-	}
-
 	fn morph_key(key: &H256, index: u8) -> Bytes {
 		let mut ret = key.bytes().to_owned();
 		ret.push(index);
@@ -226,9 +159,106 @@ impl JournalDB {
 		}
 	}
 
-	/// Commit all recent insert operations and historical removals from the old era
+	fn payload(&self, key: &H256) -> Option<Bytes> {
-	/// to the backing database.
+		self.backing.get(&key.bytes()).expect("Low-level database error. Some issue with your hard disk?").map(|v| v.to_vec())
-	fn commit_with_counters(&mut self, now: u64, id: &H256, end: Option<(u64, H256)>) -> Result<u32, UtilError> {
+	}
+
+	fn read_counters(db: &Database) -> HashMap<H256, i32> {
+		let mut counters = HashMap::new();
+		if let Some(val) = db.get(&LATEST_ERA_KEY).expect("Low-level database error.") {
+			let mut era = decode::<u64>(&val);
+			loop {
+				let mut index = 0usize;
+				while let Some(rlp_data) = db.get({
+					let mut r = RlpStream::new_list(3);
+					r.append(&era);
+					r.append(&index);
+					r.append(&&PADDING[..]);
+					&r.drain()
+				}).expect("Low-level database error.") {
+					trace!("read_counters: era={}, index={}", era, index);
+					let rlp = Rlp::new(&rlp_data);
+					let inserts: Vec<H256> = rlp.val_at(1);
+					Self::replay_keys(&inserts, db, &mut counters);
+					index += 1;
+				};
+				if index == 0 || era == 0 {
+					break;
+				}
+				era -= 1;
+			}
+		}
+		trace!("Recovered {} counters", counters.len());
+		counters
+	}
+}
+
+impl HashDB for OptionOneDB {
+	fn keys(&self) -> HashMap<H256, i32> {
+		let mut ret: HashMap<H256, i32> = HashMap::new();
+		for (key, _) in self.backing.iter() {
+			let h = H256::from_slice(key.deref());
+			ret.insert(h, 1);
+		}
+
+		for (key, refs) in self.overlay.keys().into_iter() {
+			let refs = *ret.get(&key).unwrap_or(&0) + refs;
+			ret.insert(key, refs);
+		}
+		ret
+	}
+
+	fn lookup(&self, key: &H256) -> Option<&[u8]> {
+		let k = self.overlay.raw(key);
+		match k {
+			Some(&(ref d, rc)) if rc > 0 => Some(d),
+			_ => {
+				if let Some(x) = self.payload(key) {
+					Some(&self.overlay.denote(key, x).0)
+				}
+				else {
+					None
+				}
+			}
+		}
+	}
+
+	fn exists(&self, key: &H256) -> bool {
+		self.lookup(key).is_some()
+	}
+
+	fn insert(&mut self, value: &[u8]) -> H256 {
+		self.overlay.insert(value)
+	}
+	fn emplace(&mut self, key: H256, value: Bytes) {
+		self.overlay.emplace(key, value);
+	}
+	fn kill(&mut self, key: &H256) {
+		self.overlay.kill(key);
+	}
+}
+
+impl JournalDB for OptionOneDB {
+	fn spawn(&self) -> Box<JournalDB> {
+		Box::new(OptionOneDB {
+			overlay: MemoryDB::new(),
+			backing: self.backing.clone(),
+			counters: self.counters.clone(),
+		})
+	}
+
+	fn mem_used(&self) -> usize {
+		self.overlay.mem_used() + match self.counters {
+			Some(ref c) => c.read().unwrap().heap_size_of_children(),
+			None => 0
+		}
+ 	}
+
+	fn is_empty(&self) -> bool {
+		self.backing.get(&LATEST_ERA_KEY).expect("Low level database error").is_none()
+	}
+
+	fn commit(&mut self, now: u64, id: &H256, end: Option<(u64, H256)>) -> Result<u32, UtilError> {
 		// journal format:
 		// [era, 0] => [ id, [insert_0, ...], [remove_0, ...] ]
 		// [era, 1] => [ id, [insert_0, ...], [remove_0, ...] ]
@@ -339,99 +369,13 @@ impl JournalDB {
 				try!(batch.delete(&last));
 				index += 1;
 			}
-			trace!("JournalDB: delete journal for time #{}.{}, (canon was {})", end_era, index, canon_id);
+			trace!("OptionOneDB: delete journal for time #{}.{}, (canon was {})", end_era, index, canon_id);
 		}
 
 		try!(self.backing.write(batch));
-//		trace!("JournalDB::commit() deleted {} nodes", deletes);
+//		trace!("OptionOneDB::commit() deleted {} nodes", deletes);
 		Ok(0)
 	}
-
-	fn payload(&self, key: &H256) -> Option<Bytes> {
-		self.backing.get(&key.bytes()).expect("Low-level database error. Some issue with your hard disk?").map(|v| v.to_vec())
-	}
-
-	fn read_counters(db: &Database) -> HashMap<H256, i32> {
-		let mut counters = HashMap::new();
-		if let Some(val) = db.get(&LATEST_ERA_KEY).expect("Low-level database error.") {
-			let mut era = decode::<u64>(&val);
-			loop {
-				let mut index = 0usize;
-				while let Some(rlp_data) = db.get({
-					let mut r = RlpStream::new_list(3);
-					r.append(&era);
-					r.append(&index);
-					r.append(&&PADDING[..]);
-					&r.drain()
-				}).expect("Low-level database error.") {
-					trace!("read_counters: era={}, index={}", era, index);
-					let rlp = Rlp::new(&rlp_data);
-					let inserts: Vec<H256> = rlp.val_at(1);
-					Self::replay_keys(&inserts, db, &mut counters);
-					index += 1;
-				};
-				if index == 0 || era == 0 {
-					break;
-				}
-				era -= 1;
-			}
-		}
-		trace!("Recovered {} counters", counters.len());
-		counters
-	}
-
-	/// Returns heap memory size used
-	pub fn mem_used(&self) -> usize {
-		self.overlay.mem_used() + match self.counters {
-			Some(ref c) => c.read().unwrap().heap_size_of_children(),
-			None => 0
-		}
- 	}
- }
-
-impl HashDB for JournalDB {
-	fn keys(&self) -> HashMap<H256, i32> {
-		let mut ret: HashMap<H256, i32> = HashMap::new();
-		for (key, _) in self.backing.iter() {
-			let h = H256::from_slice(key.deref());
-			ret.insert(h, 1);
-		}
-
-		for (key, refs) in self.overlay.keys().into_iter() {
-			let refs = *ret.get(&key).unwrap_or(&0) + refs;
-			ret.insert(key, refs);
-		}
-		ret
-	}
-
-	fn lookup(&self, key: &H256) -> Option<&[u8]> {
-		let k = self.overlay.raw(key);
-		match k {
-			Some(&(ref d, rc)) if rc > 0 => Some(d),
-			_ => {
-				if let Some(x) = self.payload(key) {
-					Some(&self.overlay.denote(key, x).0)
-				}
-				else {
-					None
-				}
-			}
-		}
-	}
-
-	fn exists(&self, key: &H256) -> bool {
-		self.lookup(key).is_some()
-	}
-
-	fn insert(&mut self, value: &[u8]) -> H256 {
-		self.overlay.insert(value)
-	}
-	fn emplace(&mut self, key: H256, value: Bytes) {
-		self.overlay.emplace(key, value);
-	}
-	fn kill(&mut self, key: &H256) {
-		self.overlay.kill(key);
-	}
 }
 
 #[cfg(test)]
@@ -439,11 +383,12 @@ mod tests {
 	use common::*;
 	use super::*;
 	use hashdb::*;
+	use journaldb::traits::JournalDB;
 
 	#[test]
 	fn insert_same_in_fork() {
 		// history is 1
-		let mut jdb = JournalDB::new_temp();
+		let mut jdb = OptionOneDB::new_temp();
 
 		let x = jdb.insert(b"X");
 		jdb.commit(1, &b"1".sha3(), None).unwrap();
@@ -465,7 +410,7 @@ mod tests {
 	#[test]
 	fn long_history() {
 		// history is 3
-		let mut jdb = JournalDB::new_temp();
+		let mut jdb = OptionOneDB::new_temp();
 		let h = jdb.insert(b"foo");
 		jdb.commit(0, &b"0".sha3(), None).unwrap();
 		assert!(jdb.exists(&h));
@@ -483,7 +428,7 @@ mod tests {
 	#[test]
 	fn complex() {
 		// history is 1
-		let mut jdb = JournalDB::new_temp();
+		let mut jdb = OptionOneDB::new_temp();
 
 		let foo = jdb.insert(b"foo");
 		let bar = jdb.insert(b"bar");
@@ -521,7 +466,7 @@ mod tests {
 	#[test]
 	fn fork() {
 		// history is 1
-		let mut jdb = JournalDB::new_temp();
+		let mut jdb = OptionOneDB::new_temp();
 
 		let foo = jdb.insert(b"foo");
 		let bar = jdb.insert(b"bar");
@@ -549,7 +494,7 @@ mod tests {
 	#[test]
 	fn overwrite() {
 		// history is 1
-		let mut jdb = JournalDB::new_temp();
+		let mut jdb = OptionOneDB::new_temp();
 
 		let foo = jdb.insert(b"foo");
 		jdb.commit(0, &b"0".sha3(), None).unwrap();
@@ -568,7 +513,7 @@ mod tests {
 	#[test]
 	fn fork_same_key() {
 		// history is 1
-		let mut jdb = JournalDB::new_temp();
+		let mut jdb = OptionOneDB::new_temp();
 		jdb.commit(0, &b"0".sha3(), None).unwrap();
 
 		let foo = jdb.insert(b"foo");
@@ -590,7 +535,7 @@ mod tests {
 		let bar = H256::random();
 
 		let foo = {
-			let mut jdb = JournalDB::new(dir.to_str().unwrap());
+			let mut jdb = OptionOneDB::new(dir.to_str().unwrap());
 			// history is 1
 			let foo = jdb.insert(b"foo");
 			jdb.emplace(bar.clone(), b"bar".to_vec());
@@ -599,13 +544,13 @@ mod tests {
 		};
 
 		{
-			let mut jdb = JournalDB::new(dir.to_str().unwrap());
+			let mut jdb = OptionOneDB::new(dir.to_str().unwrap());
 			jdb.remove(&foo);
 			jdb.commit(1, &b"1".sha3(), Some((0, b"0".sha3()))).unwrap();
 		}
 
 		{
-			let mut jdb = JournalDB::new(dir.to_str().unwrap());
+			let mut jdb = OptionOneDB::new(dir.to_str().unwrap());
 			assert!(jdb.exists(&foo));
 			assert!(jdb.exists(&bar));
 			jdb.commit(2, &b"2".sha3(), Some((1, b"1".sha3()))).unwrap();
@@ -619,7 +564,7 @@ mod tests {
 		dir.push(H32::random().hex());
 
 		let foo = {
-			let mut jdb = JournalDB::new(dir.to_str().unwrap());
+			let mut jdb = OptionOneDB::new(dir.to_str().unwrap());
 			// history is 1
 			let foo = jdb.insert(b"foo");
 			jdb.commit(0, &b"0".sha3(), None).unwrap();
@@ -633,7 +578,7 @@ mod tests {
 		};
 
 		{
-			let mut jdb = JournalDB::new(dir.to_str().unwrap());
+			let mut jdb = OptionOneDB::new(dir.to_str().unwrap());
 			jdb.remove(&foo);
 			jdb.commit(3, &b"3".sha3(), Some((2, b"2".sha3()))).unwrap();
 			assert!(jdb.exists(&foo));
@@ -648,7 +593,7 @@ mod tests {
 		let mut dir = ::std::env::temp_dir();
 		dir.push(H32::random().hex());
 		let (foo, bar, baz) = {
-			let mut jdb = JournalDB::new(dir.to_str().unwrap());
+			let mut jdb = OptionOneDB::new(dir.to_str().unwrap());
 			// history is 1
 			let foo = jdb.insert(b"foo");
 			let bar = jdb.insert(b"bar");
@@ -663,7 +608,7 @@ mod tests {
 		};
 
 		{
-			let mut jdb = JournalDB::new(dir.to_str().unwrap());
+			let mut jdb = OptionOneDB::new(dir.to_str().unwrap());
 			jdb.commit(2, &b"2b".sha3(), Some((1, b"1b".sha3()))).unwrap();
 			assert!(jdb.exists(&foo));
 			assert!(!jdb.exists(&baz));

util/src/journaldb/traits.rs

@@ -0,0 +1,37 @@
+// Copyright 2015, 2016 Ethcore (UK) Ltd.
+// This file is part of Parity.
+
+// Parity is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Parity is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Parity.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Disk-backed HashDB implementation.
+
+use common::*;
+use hashdb::*;
+
+/// A HashDB which can manage a short-term journal potentially containing many forks of mutually
+/// exclusive actions.
+pub trait JournalDB : HashDB + Send + Sync {
+	/// Return a copy of ourself, in a box.
+	fn spawn(&self) -> Box<JournalDB>;
+
+	/// Returns heap memory size used
+	fn mem_used(&self) -> usize;
+
+	/// Check if this database has any commits
+	fn is_empty(&self) -> bool;
+
+	/// Commit all recent insert operations and canonical historical commits' removals from the
+	/// old era to the backing database, reverting any non-canonical historical commit's inserts.
+	fn commit(&mut self, now: u64, id: &H256, end: Option<(u64, H256)>) -> Result<u32, UtilError>;
+}

util/src/lib.rs

@@ -154,7 +154,7 @@ pub use rlp::*;
 pub use hashdb::*;
 pub use memorydb::*;
 pub use overlaydb::*;
-pub use journaldb::*;
+pub use journaldb::JournalDB;
 pub use math::*;
 pub use crypto::*;
 pub use triehash::*;

util/src/overlaydb.rs

@@ -36,6 +36,7 @@ use kvdb::{Database};
 ///
 /// `lookup()` and `contains()` maintain normal behaviour - all `insert()` and `remove()` 
 /// queries have an immediate effect in terms of these functions.
+//#[derive(Clone)]
 pub struct OverlayDB {
 	overlay: MemoryDB,
 	backing: Arc<Database>,