Merge pull request #927 from ethcore/pruning_auto

Pruning auto

ethcore/src/client/client.rs

@@ -17,6 +17,7 @@
 //! Blockchain database client.
 
 use std::marker::PhantomData;
+use std::path::PathBuf;
 use util::*;
 use util::panics::*;
 use views::BlockView;
@@ -126,22 +127,29 @@ impl Client<CanonVerifier> {
 	}
 }
 
+pub fn get_db_path(path: &Path, pruning: journaldb::Algorithm, genesis_hash: H256) -> PathBuf {
+	let mut dir = path.to_path_buf();
+	dir.push(H64::from(genesis_hash).hex());
+	//TODO: sec/fat: pruned/full versioning
+	// version here is a bit useless now, since it's controlled only be the pruning algo.
+	dir.push(format!("v{}-sec-{}", CLIENT_DB_VER_STR, pruning));
+	dir
+}
+
+pub fn append_path(path: &Path, item: &str) -> String {
+	let mut p = path.to_path_buf();
+	p.push(item);
+	p.to_str().unwrap().to_owned()
+}
+
 impl<V> Client<V> where V: Verifier {
 	///  Create a new client with given spec and DB path and custom verifier.
 	pub fn new_with_verifier(config: ClientConfig, spec: Spec, path: &Path, message_channel: IoChannel<NetSyncMessage> ) -> Result<Arc<Client<V>>, Error> {
-		let mut dir = path.to_path_buf();
+		let path = get_db_path(path, config.pruning, spec.genesis_header().hash());
-		dir.push(H64::from(spec.genesis_header().hash()).hex());
-		//TODO: sec/fat: pruned/full versioning
-		// version here is a bit useless now, since it's controlled only be the pruning algo.
-		dir.push(format!("v{}-sec-{}", CLIENT_DB_VER_STR, config.pruning));
-		let path = dir.as_path();
 		let gb = spec.genesis_block();
-		let chain = Arc::new(BlockChain::new(config.blockchain, &gb, path));
+		let chain = Arc::new(BlockChain::new(config.blockchain, &gb, &path));
-		let mut state_path = path.to_path_buf();
-		state_path.push("state");
 
-		let state_path_str = state_path.to_str().unwrap();
+		let mut state_db = journaldb::new(&append_path(&path, "state"), config.pruning);
-		let mut state_db = journaldb::new(state_path_str, config.pruning);
 
 		if state_db.is_empty() && spec.ensure_db_good(state_db.as_hashdb_mut()) {
 			state_db.commit(0, &spec.genesis_header().hash(), None).expect("Error commiting genesis state to state DB");

parity/main.rs

@@ -128,11 +128,11 @@ Networking Options:
 
 API and Console Options:
   -j --jsonrpc             Enable the JSON-RPC API server.
+  --jsonrpc-port PORT      Specify the port portion of the JSONRPC API server
+                           [default: 8545].
   --jsonrpc-interface IP   Specify the hostname portion of the JSONRPC API
                            server, IP should be an interface's IP address, or
                            all (all interfaces) or local [default: local].
-  --jsonrpc-port PORT      Specify the port portion of the JSONRPC API server
-                           [default: 8545].
   --jsonrpc-cors URL       Specify CORS header for JSON-RPC API responses
                            [default: null].
   --jsonrpc-apis APIS      Specify the APIs available through the JSONRPC
@@ -171,8 +171,14 @@ Sealing/Mining Options:
 
 Footprint Options:
   --pruning METHOD         Configure pruning of the state/storage trie. METHOD
-                           may be one of: archive, basic (experimental), fast
+                           may be one of auto, archive, basic, fast, light:
-                           (experimental) [default: archive].
+                           archive - keep all state trie data. No pruning.
+                           basic - reference count in disk DB. Slow but light.
+                           fast - maintain journal overlay. Fast but 50MB used.
+                           light - early merges with partial tracking. Fast
+                           and light. Experimental!
+                           auto - use the method most recently synced or
+                           default to archive if none synced [default: auto].
   --cache-pref-size BYTES  Specify the prefered size of the blockchain cache in
                            bytes [default: 16384].
   --cache-max-size BYTES   Specify the maximum size of the blockchain cache in
@@ -538,7 +544,25 @@ impl Configuration {
 		ret
 	}
 
-	fn client_config(&self) -> ClientConfig {
+	fn find_best_db(&self, spec: &Spec) -> Option<journaldb::Algorithm> {
+		let mut ret = None;
+		let mut latest_era = None;
+		let jdb_types = [journaldb::Algorithm::Archive, journaldb::Algorithm::EarlyMerge, journaldb::Algorithm::OverlayRecent, journaldb::Algorithm::RefCounted];
+		for i in jdb_types.into_iter() {
+			let db = journaldb::new(&append_path(&get_db_path(&Path::new(&self.path()), *i, spec.genesis_header().hash()), "state"), *i);
+			match (latest_era, db.latest_era()) {
+				(Some(best), Some(this)) if best >= this => {}
+				(_, None) => {}
+				(_, Some(this)) => {
+					latest_era = Some(this);
+					ret = Some(*i);
+				}
+			}
+		}
+		ret
+	}
+
+	fn client_config(&self, spec: &Spec) -> ClientConfig {
 		let mut client_config = ClientConfig::default();
 		match self.args.flag_cache {
 			Some(mb) => {
@@ -555,8 +579,10 @@ impl Configuration {
 			"light" => journaldb::Algorithm::EarlyMerge,
 			"fast" => journaldb::Algorithm::OverlayRecent,
 			"basic" => journaldb::Algorithm::RefCounted,
+			"auto" => self.find_best_db(spec).unwrap_or(journaldb::Algorithm::OverlayRecent),
 			_ => { die!("Invalid pruning method given."); }
 		};
+		info!("Using state DB of {}", client_config.pruning);
 		client_config.name = self.args.flag_identity.clone();
 		client_config.queue.max_mem_use = self.args.flag_queue_max_size;
 		client_config
@@ -651,13 +677,14 @@ impl Configuration {
 		let spec = self.spec();
 		let net_settings = self.net_settings(&spec);
 		let sync_config = self.sync_config(&spec);
+		let client_config = self.client_config(&spec);
 
 		// Secret Store
 		let account_service = Arc::new(self.account_service());
 
 		// Build client
 		let mut service = ClientService::start(
-			self.client_config(), spec, net_settings, &Path::new(&self.path())
+			client_config, spec, net_settings, &Path::new(&self.path())
 		).unwrap_or_else(|e| die_with_error(e));
 
 		panic_handler.forward_from(&service);

util/src/journaldb/archivedb.rs

@@ -41,7 +41,7 @@ pub struct ArchiveDB {
 // all keys must be at least 12 bytes
 const LATEST_ERA_KEY : [u8; 12] = [ b'l', b'a', b's', b't', 0, 0, 0, 0, 0, 0, 0, 0 ];
 const VERSION_KEY : [u8; 12] = [ b'j', b'v', b'e', b'r', 0, 0, 0, 0, 0, 0, 0, 0 ];
-const DB_VERSION : u32 = 259;
+const DB_VERSION : u32 = 0x103;
 
 impl ArchiveDB {
 	/// Create a new instance from file
@@ -55,7 +55,7 @@ impl ArchiveDB {
 		if !backing.is_empty() {
 			match backing.get(&VERSION_KEY).map(|d| d.map(|v| decode::<u32>(&v))) {
 				Ok(Some(DB_VERSION)) => {},
-				v => panic!("Incompatible DB version, expected {}, got {:?}", DB_VERSION, v)
+				v => panic!("Incompatible DB version, expected {}, got {:?}; to resolve, remove {} and restart.", DB_VERSION, v, path)
 			}
 		} else {
 			backing.put(&VERSION_KEY, &encode(&DB_VERSION)).expect("Error writing version to database");
@@ -168,6 +168,8 @@ impl JournalDB for ArchiveDB {
 		Ok((inserts + deletes) as u32)
 	}
 
+	fn latest_era(&self) -> Option<u64> { self.latest_era }
+
 	fn state(&self, id: &H256) -> Option<Bytes> {
 		self.backing.get_by_prefix(&id.bytes()[0..12]).and_then(|b| Some(b.to_vec()))
 	}

util/src/journaldb/earlymergedb.rs

@@ -70,7 +70,7 @@ pub struct EarlyMergeDB {
 // all keys must be at least 12 bytes
 const LATEST_ERA_KEY : [u8; 12] = [ b'l', b'a', b's', b't', 0, 0, 0, 0, 0, 0, 0, 0 ];
 const VERSION_KEY : [u8; 12] = [ b'j', b'v', b'e', b'r', 0, 0, 0, 0, 0, 0, 0, 0 ];
-const DB_VERSION : u32 = 3;
+const DB_VERSION : u32 = 0x003;
 const PADDING : [u8; 10] = [ 0u8; 10 ];
 
 impl EarlyMergeDB {
@@ -85,7 +85,7 @@ impl EarlyMergeDB {
 		if !backing.is_empty() {
 			match backing.get(&VERSION_KEY).map(|d| d.map(|v| decode::<u32>(&v))) {
 				Ok(Some(DB_VERSION)) => {},
-				v => panic!("Incompatible DB version, expected {}, got {:?}", DB_VERSION, v)
+				v => panic!("Incompatible DB version, expected {}, got {:?}; to resolve, remove {} and restart.", DB_VERSION, v, path)
 			}
 		} else {
 			backing.put(&VERSION_KEY, &encode(&DB_VERSION)).expect("Error writing version to database");
@@ -333,6 +333,8 @@ impl JournalDB for EarlyMergeDB {
 		self.backing.get(&LATEST_ERA_KEY).expect("Low level database error").is_none()
 	}
 
+	fn latest_era(&self) -> Option<u64> { self.latest_era }
+
 	fn mem_used(&self) -> usize {
 		self.overlay.mem_used() + match self.refs {
 			Some(ref c) => c.read().unwrap().heap_size_of_children(),
@@ -340,7 +342,6 @@ impl JournalDB for EarlyMergeDB {
 		}
  	}
 
-
 	#[cfg_attr(feature="dev", allow(cyclomatic_complexity))]
 	fn commit(&mut self, now: u64, id: &H256, end: Option<(u64, H256)>) -> Result<u32, UtilError> {
 		// journal format:

util/src/journaldb/mod.rs

@@ -29,7 +29,7 @@ mod refcounteddb;
 pub use self::traits::JournalDB;
 
 /// A journal database algorithm.
-#[derive(Debug)]
+#[derive(Debug, Clone, Copy)]
 pub enum Algorithm {
 	/// Keep all keys forever.
 	Archive,

util/src/journaldb/overlayrecentdb.rs

@@ -95,7 +95,7 @@ impl Clone for OverlayRecentDB {
 // all keys must be at least 12 bytes
 const LATEST_ERA_KEY : [u8; 12] = [ b'l', b'a', b's', b't', 0, 0, 0, 0, 0, 0, 0, 0 ];
 const VERSION_KEY : [u8; 12] = [ b'j', b'v', b'e', b'r', 0, 0, 0, 0, 0, 0, 0, 0 ];
-const DB_VERSION : u32 = 0x200 + 3;
+const DB_VERSION : u32 = 0x203;
 const PADDING : [u8; 10] = [ 0u8; 10 ];
 
 impl OverlayRecentDB {
@@ -115,7 +115,7 @@ impl OverlayRecentDB {
 		if !backing.is_empty() {
 			match backing.get(&VERSION_KEY).map(|d| d.map(|v| decode::<u32>(&v))) {
 				Ok(Some(DB_VERSION)) => {}
-				v => panic!("Incompatible DB version, expected {}, got {:?}", DB_VERSION, v)
+				v => panic!("Incompatible DB version, expected {}, got {:?}; to resolve, remove {} and restart.", DB_VERSION, v, path)
 			}
 		} else {
 			backing.put(&VERSION_KEY, &encode(&DB_VERSION)).expect("Error writing version to database");
@@ -213,6 +213,8 @@ impl JournalDB for OverlayRecentDB {
 		self.backing.get(&LATEST_ERA_KEY).expect("Low level database error").is_none()
 	}
 
+	fn latest_era(&self) -> Option<u64> { self.journal_overlay.read().unwrap().latest_era }
+
 	fn commit(&mut self, now: u64, id: &H256, end: Option<(u64, H256)>) -> Result<u32, UtilError> {
 		// record new commit's details.
 		trace!("commit: #{} ({}), end era: {:?}", now, id, end);

util/src/journaldb/refcounteddb.rs

@@ -42,7 +42,7 @@ pub struct RefCountedDB {
 
 const LATEST_ERA_KEY : [u8; 12] = [ b'l', b'a', b's', b't', 0, 0, 0, 0, 0, 0, 0, 0 ];
 const VERSION_KEY : [u8; 12] = [ b'j', b'v', b'e', b'r', 0, 0, 0, 0, 0, 0, 0, 0 ];
-const DB_VERSION : u32 = 512;
+const DB_VERSION : u32 = 0x200;
 const PADDING : [u8; 10] = [ 0u8; 10 ];
 
 impl RefCountedDB {
@@ -57,7 +57,7 @@ impl RefCountedDB {
 		if !backing.is_empty() {
 			match backing.get(&VERSION_KEY).map(|d| d.map(|v| decode::<u32>(&v))) {
 				Ok(Some(DB_VERSION)) => {},
-				v => panic!("Incompatible DB version, expected {}, got {:?}", DB_VERSION, v)
+				v => panic!("Incompatible DB version, expected {}, got {:?}; to resolve, remove {} and restart.", DB_VERSION, v, path)
 			}
 		} else {
 			backing.put(&VERSION_KEY, &encode(&DB_VERSION)).expect("Error writing version to database");
@@ -112,6 +112,8 @@ impl JournalDB for RefCountedDB {
 		self.latest_era.is_none()
 	}
 
+	fn latest_era(&self) -> Option<u64> { self.latest_era }
+
 	fn commit(&mut self, now: u64, id: &H256, end: Option<(u64, H256)>) -> Result<u32, UtilError> {
 		// journal format:
 		// [era, 0] => [ id, [insert_0, ...], [remove_0, ...] ]
@@ -220,6 +222,25 @@ mod tests {
 		assert!(!jdb.exists(&h));
 	}
 
+	#[test]
+	fn latest_era_should_work() {
+		// history is 3
+		let mut jdb = RefCountedDB::new_temp();
+		assert_eq!(jdb.latest_era(), None);
+		let h = jdb.insert(b"foo");
+		jdb.commit(0, &b"0".sha3(), None).unwrap();
+		assert_eq!(jdb.latest_era(), Some(0));
+		jdb.remove(&h);
+		jdb.commit(1, &b"1".sha3(), None).unwrap();
+		assert_eq!(jdb.latest_era(), Some(1));
+		jdb.commit(2, &b"2".sha3(), None).unwrap();
+		assert_eq!(jdb.latest_era(), Some(2));
+		jdb.commit(3, &b"3".sha3(), Some((0, b"0".sha3()))).unwrap();
+		assert_eq!(jdb.latest_era(), Some(3));
+		jdb.commit(4, &b"4".sha3(), Some((1, b"1".sha3()))).unwrap();
+		assert_eq!(jdb.latest_era(), Some(4));
+	}
+
 	#[test]
 	fn complex() {
 		// history is 1

util/src/journaldb/traits.rs

@@ -31,6 +31,9 @@ pub trait JournalDB : HashDB + Send + Sync {
 	/// Check if this database has any commits
 	fn is_empty(&self) -> bool;
 
+	/// Get the latest era in the DB. None if there isn't yet any data in there.
+	fn latest_era(&self) -> Option<u64>;
+
 	/// Commit all recent insert operations and canonical historical commits' removals from the
 	/// old era to the backing database, reverting any non-canonical historical commit's inserts.
 	fn commit(&mut self, now: u64, id: &H256, end: Option<(u64, H256)>) -> Result<u32, UtilError>;