From 6e97496b27f771620d69b159053863f92f4e9b82 Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Mon, 11 Apr 2016 15:51:14 -0700 Subject: [PATCH 1/2] Support for --pruning=auto. --- parity/main.rs | 32 +++++++++++++++++++++++---- util/src/journaldb/archivedb.rs | 2 ++ util/src/journaldb/earlymergedb.rs | 2 ++ util/src/journaldb/overlayrecentdb.rs | 2 ++ util/src/journaldb/refcounteddb.rs | 21 ++++++++++++++++++ util/src/journaldb/traits.rs | 3 +++ 6 files changed, 58 insertions(+), 4 deletions(-) diff --git a/parity/main.rs b/parity/main.rs index e2e4c9c37..af454a518 100644 --- a/parity/main.rs +++ b/parity/main.rs @@ -126,11 +126,11 @@ Networking Options: API and Console Options: -j --jsonrpc Enable the JSON-RPC API server. + --jsonrpc-port PORT Specify the port portion of the JSONRPC API server + [default: 8545]. --jsonrpc-interface IP Specify the hostname portion of the JSONRPC API server, IP should be an interface's IP address, or all (all interfaces) or local [default: local]. - --jsonrpc-port PORT Specify the port portion of the JSONRPC API server - [default: 8545]. --jsonrpc-cors URL Specify CORS header for JSON-RPC API responses [default: null]. --jsonrpc-apis APIS Specify the APIs available through the JSONRPC @@ -169,8 +169,14 @@ Sealing/Mining Options: Footprint Options: --pruning METHOD Configure pruning of the state/storage trie. METHOD - may be one of: archive, basic (experimental), fast - (experimental) [default: archive]. + may be one of auto, archive, basic, fast, light: + archive - keep all state trie data. No pruning. + basic - reference count in disk DB. Slow but light. + fast - maintain journal overlay. Fast but 50MB used. + light - early merges with partial tracking. Fast + and light. Experimental! + auto - use the method most recently synced or + default to archive if none synced [default: auto]. --cache-pref-size BYTES Specify the prefered size of the blockchain cache in bytes [default: 16384]. --cache-max-size BYTES Specify the maximum size of the blockchain cache in @@ -536,6 +542,23 @@ impl Configuration { ret } + fn find_best_db(path: &str) -> Option { + let mut ret = None; + let mut latest_era = None; + for i in [journaldb::Algorithm::Archive, journaldb::Algorithm::EarlyMerge, journaldb::Algorithm::OverlayRecent, journaldb::Algorithm::RefCounted].into_iter() { + let db = journaldb::new(path, i); + match (latest_era, db.latest_era()) { + (Some(best), Some(this)) if best >= this => {} + (_, None) => {} + (_, Some(this)) => { + latest_era = Some(this); + ret = Some(i); + } + } + } + ret + } + fn client_config(&self) -> ClientConfig { let mut client_config = ClientConfig::default(); match self.args.flag_cache { @@ -553,6 +576,7 @@ impl Configuration { "light" => journaldb::Algorithm::EarlyMerge, "fast" => journaldb::Algorithm::OverlayRecent, "basic" => journaldb::Algorithm::RefCounted, + "auto" => Self::find_best_db().unwrap_or(journaldb::Algorithm::OverlayRecent), _ => { die!("Invalid pruning method given."); } }; client_config.name = self.args.flag_identity.clone(); diff --git a/util/src/journaldb/archivedb.rs b/util/src/journaldb/archivedb.rs index 380e8e423..8f112b18b 100644 --- a/util/src/journaldb/archivedb.rs +++ b/util/src/journaldb/archivedb.rs @@ -168,6 +168,8 @@ impl JournalDB for ArchiveDB { Ok((inserts + deletes) as u32) } + fn latest_era() -> Option { self.latest_era } + fn state(&self, id: &H256) -> Option { self.backing.get_by_prefix(&id.bytes()[0..12]).and_then(|b| Some(b.to_vec())) } diff --git a/util/src/journaldb/earlymergedb.rs b/util/src/journaldb/earlymergedb.rs index eada4bbaa..41bf78efa 100644 --- a/util/src/journaldb/earlymergedb.rs +++ b/util/src/journaldb/earlymergedb.rs @@ -333,6 +333,8 @@ impl JournalDB for EarlyMergeDB { self.backing.get(&LATEST_ERA_KEY).expect("Low level database error").is_none() } + fn latest_era() -> Option { self.latest_era } + fn mem_used(&self) -> usize { self.overlay.mem_used() + match self.refs { Some(ref c) => c.read().unwrap().heap_size_of_children(), diff --git a/util/src/journaldb/overlayrecentdb.rs b/util/src/journaldb/overlayrecentdb.rs index 0b9ad4fda..e3adb3983 100644 --- a/util/src/journaldb/overlayrecentdb.rs +++ b/util/src/journaldb/overlayrecentdb.rs @@ -213,6 +213,8 @@ impl JournalDB for OverlayRecentDB { self.backing.get(&LATEST_ERA_KEY).expect("Low level database error").is_none() } + fn latest_era() -> Option { self.latest_era } + fn commit(&mut self, now: u64, id: &H256, end: Option<(u64, H256)>) -> Result { // record new commit's details. trace!("commit: #{} ({}), end era: {:?}", now, id, end); diff --git a/util/src/journaldb/refcounteddb.rs b/util/src/journaldb/refcounteddb.rs index e69eccab7..ef74ce1cf 100644 --- a/util/src/journaldb/refcounteddb.rs +++ b/util/src/journaldb/refcounteddb.rs @@ -112,6 +112,8 @@ impl JournalDB for RefCountedDB { self.latest_era.is_none() } + fn latest_era() -> Option { self.latest_era } + fn commit(&mut self, now: u64, id: &H256, end: Option<(u64, H256)>) -> Result { // journal format: // [era, 0] => [ id, [insert_0, ...], [remove_0, ...] ] @@ -220,6 +222,25 @@ mod tests { assert!(!jdb.exists(&h)); } + #[test] + fn latest_era_should_work() { + // history is 3 + let mut jdb = RefCountedDB::new_temp(); + assert_eq!(jdb.latest_era(), None); + let h = jdb.insert(b"foo"); + jdb.commit(0, &b"0".sha3(), None).unwrap(); + assert_eq!(jdb.latest_era(), Some(0)); + jdb.remove(&h); + jdb.commit(1, &b"1".sha3(), None).unwrap(); + assert_eq!(jdb.latest_era(), Some(1)); + jdb.commit(2, &b"2".sha3(), None).unwrap(); + assert_eq!(jdb.latest_era(), Some(2)); + jdb.commit(3, &b"3".sha3(), Some((0, b"0".sha3()))).unwrap(); + assert_eq!(jdb.latest_era(), Some(3)); + jdb.commit(4, &b"4".sha3(), Some((1, b"1".sha3()))).unwrap(); + assert_eq!(jdb.latest_era(), Some(4)); + } + #[test] fn complex() { // history is 1 diff --git a/util/src/journaldb/traits.rs b/util/src/journaldb/traits.rs index b1ba27957..87efcdaf8 100644 --- a/util/src/journaldb/traits.rs +++ b/util/src/journaldb/traits.rs @@ -31,6 +31,9 @@ pub trait JournalDB : HashDB + Send + Sync { /// Check if this database has any commits fn is_empty(&self) -> bool; + /// Get the latest era in the DB. None if there isn't yet any data in there. + fn latest_era() -> Option { self.latest_era } + /// Commit all recent insert operations and canonical historical commits' removals from the /// old era to the backing database, reverting any non-canonical historical commit's inserts. fn commit(&mut self, now: u64, id: &H256, end: Option<(u64, H256)>) -> Result; From fa95419c2747bbd71835530be9b1f10ddd44912b Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Mon, 11 Apr 2016 18:42:50 -0700 Subject: [PATCH 2/2] --pruning=auto option. --- ethcore/src/client/client.rs | 30 +++++++++++++++++---------- parity/main.rs | 17 ++++++++------- util/src/journaldb/archivedb.rs | 6 +++--- util/src/journaldb/earlymergedb.rs | 7 +++---- util/src/journaldb/mod.rs | 2 +- util/src/journaldb/overlayrecentdb.rs | 6 +++--- util/src/journaldb/refcounteddb.rs | 6 +++--- util/src/journaldb/traits.rs | 2 +- 8 files changed, 43 insertions(+), 33 deletions(-) diff --git a/ethcore/src/client/client.rs b/ethcore/src/client/client.rs index 781bea7fa..d399b1cf1 100644 --- a/ethcore/src/client/client.rs +++ b/ethcore/src/client/client.rs @@ -17,6 +17,7 @@ //! Blockchain database client. use std::marker::PhantomData; +use std::path::PathBuf; use util::*; use util::panics::*; use views::BlockView; @@ -126,22 +127,29 @@ impl Client { } } +pub fn get_db_path(path: &Path, pruning: journaldb::Algorithm, genesis_hash: H256) -> PathBuf { + let mut dir = path.to_path_buf(); + dir.push(H64::from(genesis_hash).hex()); + //TODO: sec/fat: pruned/full versioning + // version here is a bit useless now, since it's controlled only be the pruning algo. + dir.push(format!("v{}-sec-{}", CLIENT_DB_VER_STR, pruning)); + dir +} + +pub fn append_path(path: &Path, item: &str) -> String { + let mut p = path.to_path_buf(); + p.push(item); + p.to_str().unwrap().to_owned() +} + impl Client where V: Verifier { /// Create a new client with given spec and DB path and custom verifier. pub fn new_with_verifier(config: ClientConfig, spec: Spec, path: &Path, message_channel: IoChannel ) -> Result>, Error> { - let mut dir = path.to_path_buf(); - dir.push(H64::from(spec.genesis_header().hash()).hex()); - //TODO: sec/fat: pruned/full versioning - // version here is a bit useless now, since it's controlled only be the pruning algo. - dir.push(format!("v{}-sec-{}", CLIENT_DB_VER_STR, config.pruning)); - let path = dir.as_path(); + let path = get_db_path(path, config.pruning, spec.genesis_header().hash()); let gb = spec.genesis_block(); - let chain = Arc::new(BlockChain::new(config.blockchain, &gb, path)); - let mut state_path = path.to_path_buf(); - state_path.push("state"); + let chain = Arc::new(BlockChain::new(config.blockchain, &gb, &path)); - let state_path_str = state_path.to_str().unwrap(); - let mut state_db = journaldb::new(state_path_str, config.pruning); + let mut state_db = journaldb::new(&append_path(&path, "state"), config.pruning); if state_db.is_empty() && spec.ensure_db_good(state_db.as_hashdb_mut()) { state_db.commit(0, &spec.genesis_header().hash(), None).expect("Error commiting genesis state to state DB"); diff --git a/parity/main.rs b/parity/main.rs index af454a518..a4d9655bd 100644 --- a/parity/main.rs +++ b/parity/main.rs @@ -542,24 +542,25 @@ impl Configuration { ret } - fn find_best_db(path: &str) -> Option { + fn find_best_db(&self, spec: &Spec) -> Option { let mut ret = None; let mut latest_era = None; - for i in [journaldb::Algorithm::Archive, journaldb::Algorithm::EarlyMerge, journaldb::Algorithm::OverlayRecent, journaldb::Algorithm::RefCounted].into_iter() { - let db = journaldb::new(path, i); + let jdb_types = [journaldb::Algorithm::Archive, journaldb::Algorithm::EarlyMerge, journaldb::Algorithm::OverlayRecent, journaldb::Algorithm::RefCounted]; + for i in jdb_types.into_iter() { + let db = journaldb::new(&append_path(&get_db_path(&Path::new(&self.path()), *i, spec.genesis_header().hash()), "state"), *i); match (latest_era, db.latest_era()) { (Some(best), Some(this)) if best >= this => {} (_, None) => {} (_, Some(this)) => { latest_era = Some(this); - ret = Some(i); + ret = Some(*i); } } } ret } - fn client_config(&self) -> ClientConfig { + fn client_config(&self, spec: &Spec) -> ClientConfig { let mut client_config = ClientConfig::default(); match self.args.flag_cache { Some(mb) => { @@ -576,9 +577,10 @@ impl Configuration { "light" => journaldb::Algorithm::EarlyMerge, "fast" => journaldb::Algorithm::OverlayRecent, "basic" => journaldb::Algorithm::RefCounted, - "auto" => Self::find_best_db().unwrap_or(journaldb::Algorithm::OverlayRecent), + "auto" => self.find_best_db(spec).unwrap_or(journaldb::Algorithm::OverlayRecent), _ => { die!("Invalid pruning method given."); } }; + info!("Using state DB of {}", client_config.pruning); client_config.name = self.args.flag_identity.clone(); client_config.queue.max_mem_use = self.args.flag_queue_max_size; client_config @@ -673,13 +675,14 @@ impl Configuration { let spec = self.spec(); let net_settings = self.net_settings(&spec); let sync_config = self.sync_config(&spec); + let client_config = self.client_config(&spec); // Secret Store let account_service = Arc::new(self.account_service()); // Build client let mut service = ClientService::start( - self.client_config(), spec, net_settings, &Path::new(&self.path()) + client_config, spec, net_settings, &Path::new(&self.path()) ).unwrap_or_else(|e| die_with_error(e)); panic_handler.forward_from(&service); diff --git a/util/src/journaldb/archivedb.rs b/util/src/journaldb/archivedb.rs index 8f112b18b..4fa8ce6f3 100644 --- a/util/src/journaldb/archivedb.rs +++ b/util/src/journaldb/archivedb.rs @@ -41,7 +41,7 @@ pub struct ArchiveDB { // all keys must be at least 12 bytes const LATEST_ERA_KEY : [u8; 12] = [ b'l', b'a', b's', b't', 0, 0, 0, 0, 0, 0, 0, 0 ]; const VERSION_KEY : [u8; 12] = [ b'j', b'v', b'e', b'r', 0, 0, 0, 0, 0, 0, 0, 0 ]; -const DB_VERSION : u32 = 259; +const DB_VERSION : u32 = 0x103; impl ArchiveDB { /// Create a new instance from file @@ -55,7 +55,7 @@ impl ArchiveDB { if !backing.is_empty() { match backing.get(&VERSION_KEY).map(|d| d.map(|v| decode::(&v))) { Ok(Some(DB_VERSION)) => {}, - v => panic!("Incompatible DB version, expected {}, got {:?}", DB_VERSION, v) + v => panic!("Incompatible DB version, expected {}, got {:?}; to resolve, remove {} and restart.", DB_VERSION, v, path) } } else { backing.put(&VERSION_KEY, &encode(&DB_VERSION)).expect("Error writing version to database"); @@ -168,7 +168,7 @@ impl JournalDB for ArchiveDB { Ok((inserts + deletes) as u32) } - fn latest_era() -> Option { self.latest_era } + fn latest_era(&self) -> Option { self.latest_era } fn state(&self, id: &H256) -> Option { self.backing.get_by_prefix(&id.bytes()[0..12]).and_then(|b| Some(b.to_vec())) diff --git a/util/src/journaldb/earlymergedb.rs b/util/src/journaldb/earlymergedb.rs index 41bf78efa..62fe0023a 100644 --- a/util/src/journaldb/earlymergedb.rs +++ b/util/src/journaldb/earlymergedb.rs @@ -70,7 +70,7 @@ pub struct EarlyMergeDB { // all keys must be at least 12 bytes const LATEST_ERA_KEY : [u8; 12] = [ b'l', b'a', b's', b't', 0, 0, 0, 0, 0, 0, 0, 0 ]; const VERSION_KEY : [u8; 12] = [ b'j', b'v', b'e', b'r', 0, 0, 0, 0, 0, 0, 0, 0 ]; -const DB_VERSION : u32 = 3; +const DB_VERSION : u32 = 0x003; const PADDING : [u8; 10] = [ 0u8; 10 ]; impl EarlyMergeDB { @@ -85,7 +85,7 @@ impl EarlyMergeDB { if !backing.is_empty() { match backing.get(&VERSION_KEY).map(|d| d.map(|v| decode::(&v))) { Ok(Some(DB_VERSION)) => {}, - v => panic!("Incompatible DB version, expected {}, got {:?}", DB_VERSION, v) + v => panic!("Incompatible DB version, expected {}, got {:?}; to resolve, remove {} and restart.", DB_VERSION, v, path) } } else { backing.put(&VERSION_KEY, &encode(&DB_VERSION)).expect("Error writing version to database"); @@ -333,7 +333,7 @@ impl JournalDB for EarlyMergeDB { self.backing.get(&LATEST_ERA_KEY).expect("Low level database error").is_none() } - fn latest_era() -> Option { self.latest_era } + fn latest_era(&self) -> Option { self.latest_era } fn mem_used(&self) -> usize { self.overlay.mem_used() + match self.refs { @@ -342,7 +342,6 @@ impl JournalDB for EarlyMergeDB { } } - #[cfg_attr(feature="dev", allow(cyclomatic_complexity))] fn commit(&mut self, now: u64, id: &H256, end: Option<(u64, H256)>) -> Result { // journal format: diff --git a/util/src/journaldb/mod.rs b/util/src/journaldb/mod.rs index f65aebde1..6e389b3c0 100644 --- a/util/src/journaldb/mod.rs +++ b/util/src/journaldb/mod.rs @@ -29,7 +29,7 @@ mod refcounteddb; pub use self::traits::JournalDB; /// A journal database algorithm. -#[derive(Debug)] +#[derive(Debug, Clone, Copy)] pub enum Algorithm { /// Keep all keys forever. Archive, diff --git a/util/src/journaldb/overlayrecentdb.rs b/util/src/journaldb/overlayrecentdb.rs index e3adb3983..9c68b9255 100644 --- a/util/src/journaldb/overlayrecentdb.rs +++ b/util/src/journaldb/overlayrecentdb.rs @@ -95,7 +95,7 @@ impl Clone for OverlayRecentDB { // all keys must be at least 12 bytes const LATEST_ERA_KEY : [u8; 12] = [ b'l', b'a', b's', b't', 0, 0, 0, 0, 0, 0, 0, 0 ]; const VERSION_KEY : [u8; 12] = [ b'j', b'v', b'e', b'r', 0, 0, 0, 0, 0, 0, 0, 0 ]; -const DB_VERSION : u32 = 0x200 + 3; +const DB_VERSION : u32 = 0x203; const PADDING : [u8; 10] = [ 0u8; 10 ]; impl OverlayRecentDB { @@ -115,7 +115,7 @@ impl OverlayRecentDB { if !backing.is_empty() { match backing.get(&VERSION_KEY).map(|d| d.map(|v| decode::(&v))) { Ok(Some(DB_VERSION)) => {} - v => panic!("Incompatible DB version, expected {}, got {:?}", DB_VERSION, v) + v => panic!("Incompatible DB version, expected {}, got {:?}; to resolve, remove {} and restart.", DB_VERSION, v, path) } } else { backing.put(&VERSION_KEY, &encode(&DB_VERSION)).expect("Error writing version to database"); @@ -213,7 +213,7 @@ impl JournalDB for OverlayRecentDB { self.backing.get(&LATEST_ERA_KEY).expect("Low level database error").is_none() } - fn latest_era() -> Option { self.latest_era } + fn latest_era(&self) -> Option { self.journal_overlay.read().unwrap().latest_era } fn commit(&mut self, now: u64, id: &H256, end: Option<(u64, H256)>) -> Result { // record new commit's details. diff --git a/util/src/journaldb/refcounteddb.rs b/util/src/journaldb/refcounteddb.rs index ef74ce1cf..0df4d76b1 100644 --- a/util/src/journaldb/refcounteddb.rs +++ b/util/src/journaldb/refcounteddb.rs @@ -42,7 +42,7 @@ pub struct RefCountedDB { const LATEST_ERA_KEY : [u8; 12] = [ b'l', b'a', b's', b't', 0, 0, 0, 0, 0, 0, 0, 0 ]; const VERSION_KEY : [u8; 12] = [ b'j', b'v', b'e', b'r', 0, 0, 0, 0, 0, 0, 0, 0 ]; -const DB_VERSION : u32 = 512; +const DB_VERSION : u32 = 0x200; const PADDING : [u8; 10] = [ 0u8; 10 ]; impl RefCountedDB { @@ -57,7 +57,7 @@ impl RefCountedDB { if !backing.is_empty() { match backing.get(&VERSION_KEY).map(|d| d.map(|v| decode::(&v))) { Ok(Some(DB_VERSION)) => {}, - v => panic!("Incompatible DB version, expected {}, got {:?}", DB_VERSION, v) + v => panic!("Incompatible DB version, expected {}, got {:?}; to resolve, remove {} and restart.", DB_VERSION, v, path) } } else { backing.put(&VERSION_KEY, &encode(&DB_VERSION)).expect("Error writing version to database"); @@ -112,7 +112,7 @@ impl JournalDB for RefCountedDB { self.latest_era.is_none() } - fn latest_era() -> Option { self.latest_era } + fn latest_era(&self) -> Option { self.latest_era } fn commit(&mut self, now: u64, id: &H256, end: Option<(u64, H256)>) -> Result { // journal format: diff --git a/util/src/journaldb/traits.rs b/util/src/journaldb/traits.rs index 87efcdaf8..10212f976 100644 --- a/util/src/journaldb/traits.rs +++ b/util/src/journaldb/traits.rs @@ -32,7 +32,7 @@ pub trait JournalDB : HashDB + Send + Sync { fn is_empty(&self) -> bool; /// Get the latest era in the DB. None if there isn't yet any data in there. - fn latest_era() -> Option { self.latest_era } + fn latest_era(&self) -> Option; /// Commit all recent insert operations and canonical historical commits' removals from the /// old era to the backing database, reverting any non-canonical historical commit's inserts.