From fa95419c2747bbd71835530be9b1f10ddd44912b Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Mon, 11 Apr 2016 18:42:50 -0700 Subject: [PATCH] --pruning=auto option. --- ethcore/src/client/client.rs | 30 +++++++++++++++++---------- parity/main.rs | 17 ++++++++------- util/src/journaldb/archivedb.rs | 6 +++--- util/src/journaldb/earlymergedb.rs | 7 +++---- util/src/journaldb/mod.rs | 2 +- util/src/journaldb/overlayrecentdb.rs | 6 +++--- util/src/journaldb/refcounteddb.rs | 6 +++--- util/src/journaldb/traits.rs | 2 +- 8 files changed, 43 insertions(+), 33 deletions(-) diff --git a/ethcore/src/client/client.rs b/ethcore/src/client/client.rs index 781bea7fa..d399b1cf1 100644 --- a/ethcore/src/client/client.rs +++ b/ethcore/src/client/client.rs @@ -17,6 +17,7 @@ //! Blockchain database client. use std::marker::PhantomData; +use std::path::PathBuf; use util::*; use util::panics::*; use views::BlockView; @@ -126,22 +127,29 @@ impl Client { } } +pub fn get_db_path(path: &Path, pruning: journaldb::Algorithm, genesis_hash: H256) -> PathBuf { + let mut dir = path.to_path_buf(); + dir.push(H64::from(genesis_hash).hex()); + //TODO: sec/fat: pruned/full versioning + // version here is a bit useless now, since it's controlled only be the pruning algo. + dir.push(format!("v{}-sec-{}", CLIENT_DB_VER_STR, pruning)); + dir +} + +pub fn append_path(path: &Path, item: &str) -> String { + let mut p = path.to_path_buf(); + p.push(item); + p.to_str().unwrap().to_owned() +} + impl Client where V: Verifier { /// Create a new client with given spec and DB path and custom verifier. pub fn new_with_verifier(config: ClientConfig, spec: Spec, path: &Path, message_channel: IoChannel ) -> Result>, Error> { - let mut dir = path.to_path_buf(); - dir.push(H64::from(spec.genesis_header().hash()).hex()); - //TODO: sec/fat: pruned/full versioning - // version here is a bit useless now, since it's controlled only be the pruning algo. - dir.push(format!("v{}-sec-{}", CLIENT_DB_VER_STR, config.pruning)); - let path = dir.as_path(); + let path = get_db_path(path, config.pruning, spec.genesis_header().hash()); let gb = spec.genesis_block(); - let chain = Arc::new(BlockChain::new(config.blockchain, &gb, path)); - let mut state_path = path.to_path_buf(); - state_path.push("state"); + let chain = Arc::new(BlockChain::new(config.blockchain, &gb, &path)); - let state_path_str = state_path.to_str().unwrap(); - let mut state_db = journaldb::new(state_path_str, config.pruning); + let mut state_db = journaldb::new(&append_path(&path, "state"), config.pruning); if state_db.is_empty() && spec.ensure_db_good(state_db.as_hashdb_mut()) { state_db.commit(0, &spec.genesis_header().hash(), None).expect("Error commiting genesis state to state DB"); diff --git a/parity/main.rs b/parity/main.rs index af454a518..a4d9655bd 100644 --- a/parity/main.rs +++ b/parity/main.rs @@ -542,24 +542,25 @@ impl Configuration { ret } - fn find_best_db(path: &str) -> Option { + fn find_best_db(&self, spec: &Spec) -> Option { let mut ret = None; let mut latest_era = None; - for i in [journaldb::Algorithm::Archive, journaldb::Algorithm::EarlyMerge, journaldb::Algorithm::OverlayRecent, journaldb::Algorithm::RefCounted].into_iter() { - let db = journaldb::new(path, i); + let jdb_types = [journaldb::Algorithm::Archive, journaldb::Algorithm::EarlyMerge, journaldb::Algorithm::OverlayRecent, journaldb::Algorithm::RefCounted]; + for i in jdb_types.into_iter() { + let db = journaldb::new(&append_path(&get_db_path(&Path::new(&self.path()), *i, spec.genesis_header().hash()), "state"), *i); match (latest_era, db.latest_era()) { (Some(best), Some(this)) if best >= this => {} (_, None) => {} (_, Some(this)) => { latest_era = Some(this); - ret = Some(i); + ret = Some(*i); } } } ret } - fn client_config(&self) -> ClientConfig { + fn client_config(&self, spec: &Spec) -> ClientConfig { let mut client_config = ClientConfig::default(); match self.args.flag_cache { Some(mb) => { @@ -576,9 +577,10 @@ impl Configuration { "light" => journaldb::Algorithm::EarlyMerge, "fast" => journaldb::Algorithm::OverlayRecent, "basic" => journaldb::Algorithm::RefCounted, - "auto" => Self::find_best_db().unwrap_or(journaldb::Algorithm::OverlayRecent), + "auto" => self.find_best_db(spec).unwrap_or(journaldb::Algorithm::OverlayRecent), _ => { die!("Invalid pruning method given."); } }; + info!("Using state DB of {}", client_config.pruning); client_config.name = self.args.flag_identity.clone(); client_config.queue.max_mem_use = self.args.flag_queue_max_size; client_config @@ -673,13 +675,14 @@ impl Configuration { let spec = self.spec(); let net_settings = self.net_settings(&spec); let sync_config = self.sync_config(&spec); + let client_config = self.client_config(&spec); // Secret Store let account_service = Arc::new(self.account_service()); // Build client let mut service = ClientService::start( - self.client_config(), spec, net_settings, &Path::new(&self.path()) + client_config, spec, net_settings, &Path::new(&self.path()) ).unwrap_or_else(|e| die_with_error(e)); panic_handler.forward_from(&service); diff --git a/util/src/journaldb/archivedb.rs b/util/src/journaldb/archivedb.rs index 8f112b18b..4fa8ce6f3 100644 --- a/util/src/journaldb/archivedb.rs +++ b/util/src/journaldb/archivedb.rs @@ -41,7 +41,7 @@ pub struct ArchiveDB { // all keys must be at least 12 bytes const LATEST_ERA_KEY : [u8; 12] = [ b'l', b'a', b's', b't', 0, 0, 0, 0, 0, 0, 0, 0 ]; const VERSION_KEY : [u8; 12] = [ b'j', b'v', b'e', b'r', 0, 0, 0, 0, 0, 0, 0, 0 ]; -const DB_VERSION : u32 = 259; +const DB_VERSION : u32 = 0x103; impl ArchiveDB { /// Create a new instance from file @@ -55,7 +55,7 @@ impl ArchiveDB { if !backing.is_empty() { match backing.get(&VERSION_KEY).map(|d| d.map(|v| decode::(&v))) { Ok(Some(DB_VERSION)) => {}, - v => panic!("Incompatible DB version, expected {}, got {:?}", DB_VERSION, v) + v => panic!("Incompatible DB version, expected {}, got {:?}; to resolve, remove {} and restart.", DB_VERSION, v, path) } } else { backing.put(&VERSION_KEY, &encode(&DB_VERSION)).expect("Error writing version to database"); @@ -168,7 +168,7 @@ impl JournalDB for ArchiveDB { Ok((inserts + deletes) as u32) } - fn latest_era() -> Option { self.latest_era } + fn latest_era(&self) -> Option { self.latest_era } fn state(&self, id: &H256) -> Option { self.backing.get_by_prefix(&id.bytes()[0..12]).and_then(|b| Some(b.to_vec())) diff --git a/util/src/journaldb/earlymergedb.rs b/util/src/journaldb/earlymergedb.rs index 41bf78efa..62fe0023a 100644 --- a/util/src/journaldb/earlymergedb.rs +++ b/util/src/journaldb/earlymergedb.rs @@ -70,7 +70,7 @@ pub struct EarlyMergeDB { // all keys must be at least 12 bytes const LATEST_ERA_KEY : [u8; 12] = [ b'l', b'a', b's', b't', 0, 0, 0, 0, 0, 0, 0, 0 ]; const VERSION_KEY : [u8; 12] = [ b'j', b'v', b'e', b'r', 0, 0, 0, 0, 0, 0, 0, 0 ]; -const DB_VERSION : u32 = 3; +const DB_VERSION : u32 = 0x003; const PADDING : [u8; 10] = [ 0u8; 10 ]; impl EarlyMergeDB { @@ -85,7 +85,7 @@ impl EarlyMergeDB { if !backing.is_empty() { match backing.get(&VERSION_KEY).map(|d| d.map(|v| decode::(&v))) { Ok(Some(DB_VERSION)) => {}, - v => panic!("Incompatible DB version, expected {}, got {:?}", DB_VERSION, v) + v => panic!("Incompatible DB version, expected {}, got {:?}; to resolve, remove {} and restart.", DB_VERSION, v, path) } } else { backing.put(&VERSION_KEY, &encode(&DB_VERSION)).expect("Error writing version to database"); @@ -333,7 +333,7 @@ impl JournalDB for EarlyMergeDB { self.backing.get(&LATEST_ERA_KEY).expect("Low level database error").is_none() } - fn latest_era() -> Option { self.latest_era } + fn latest_era(&self) -> Option { self.latest_era } fn mem_used(&self) -> usize { self.overlay.mem_used() + match self.refs { @@ -342,7 +342,6 @@ impl JournalDB for EarlyMergeDB { } } - #[cfg_attr(feature="dev", allow(cyclomatic_complexity))] fn commit(&mut self, now: u64, id: &H256, end: Option<(u64, H256)>) -> Result { // journal format: diff --git a/util/src/journaldb/mod.rs b/util/src/journaldb/mod.rs index f65aebde1..6e389b3c0 100644 --- a/util/src/journaldb/mod.rs +++ b/util/src/journaldb/mod.rs @@ -29,7 +29,7 @@ mod refcounteddb; pub use self::traits::JournalDB; /// A journal database algorithm. -#[derive(Debug)] +#[derive(Debug, Clone, Copy)] pub enum Algorithm { /// Keep all keys forever. Archive, diff --git a/util/src/journaldb/overlayrecentdb.rs b/util/src/journaldb/overlayrecentdb.rs index e3adb3983..9c68b9255 100644 --- a/util/src/journaldb/overlayrecentdb.rs +++ b/util/src/journaldb/overlayrecentdb.rs @@ -95,7 +95,7 @@ impl Clone for OverlayRecentDB { // all keys must be at least 12 bytes const LATEST_ERA_KEY : [u8; 12] = [ b'l', b'a', b's', b't', 0, 0, 0, 0, 0, 0, 0, 0 ]; const VERSION_KEY : [u8; 12] = [ b'j', b'v', b'e', b'r', 0, 0, 0, 0, 0, 0, 0, 0 ]; -const DB_VERSION : u32 = 0x200 + 3; +const DB_VERSION : u32 = 0x203; const PADDING : [u8; 10] = [ 0u8; 10 ]; impl OverlayRecentDB { @@ -115,7 +115,7 @@ impl OverlayRecentDB { if !backing.is_empty() { match backing.get(&VERSION_KEY).map(|d| d.map(|v| decode::(&v))) { Ok(Some(DB_VERSION)) => {} - v => panic!("Incompatible DB version, expected {}, got {:?}", DB_VERSION, v) + v => panic!("Incompatible DB version, expected {}, got {:?}; to resolve, remove {} and restart.", DB_VERSION, v, path) } } else { backing.put(&VERSION_KEY, &encode(&DB_VERSION)).expect("Error writing version to database"); @@ -213,7 +213,7 @@ impl JournalDB for OverlayRecentDB { self.backing.get(&LATEST_ERA_KEY).expect("Low level database error").is_none() } - fn latest_era() -> Option { self.latest_era } + fn latest_era(&self) -> Option { self.journal_overlay.read().unwrap().latest_era } fn commit(&mut self, now: u64, id: &H256, end: Option<(u64, H256)>) -> Result { // record new commit's details. diff --git a/util/src/journaldb/refcounteddb.rs b/util/src/journaldb/refcounteddb.rs index ef74ce1cf..0df4d76b1 100644 --- a/util/src/journaldb/refcounteddb.rs +++ b/util/src/journaldb/refcounteddb.rs @@ -42,7 +42,7 @@ pub struct RefCountedDB { const LATEST_ERA_KEY : [u8; 12] = [ b'l', b'a', b's', b't', 0, 0, 0, 0, 0, 0, 0, 0 ]; const VERSION_KEY : [u8; 12] = [ b'j', b'v', b'e', b'r', 0, 0, 0, 0, 0, 0, 0, 0 ]; -const DB_VERSION : u32 = 512; +const DB_VERSION : u32 = 0x200; const PADDING : [u8; 10] = [ 0u8; 10 ]; impl RefCountedDB { @@ -57,7 +57,7 @@ impl RefCountedDB { if !backing.is_empty() { match backing.get(&VERSION_KEY).map(|d| d.map(|v| decode::(&v))) { Ok(Some(DB_VERSION)) => {}, - v => panic!("Incompatible DB version, expected {}, got {:?}", DB_VERSION, v) + v => panic!("Incompatible DB version, expected {}, got {:?}; to resolve, remove {} and restart.", DB_VERSION, v, path) } } else { backing.put(&VERSION_KEY, &encode(&DB_VERSION)).expect("Error writing version to database"); @@ -112,7 +112,7 @@ impl JournalDB for RefCountedDB { self.latest_era.is_none() } - fn latest_era() -> Option { self.latest_era } + fn latest_era(&self) -> Option { self.latest_era } fn commit(&mut self, now: u64, id: &H256, end: Option<(u64, H256)>) -> Result { // journal format: diff --git a/util/src/journaldb/traits.rs b/util/src/journaldb/traits.rs index 87efcdaf8..10212f976 100644 --- a/util/src/journaldb/traits.rs +++ b/util/src/journaldb/traits.rs @@ -32,7 +32,7 @@ pub trait JournalDB : HashDB + Send + Sync { fn is_empty(&self) -> bool; /// Get the latest era in the DB. None if there isn't yet any data in there. - fn latest_era() -> Option { self.latest_era } + fn latest_era(&self) -> Option; /// Commit all recent insert operations and canonical historical commits' removals from the /// old era to the backing database, reverting any non-canonical historical commit's inserts.