grassrootseconomics/openethereum
5
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Merge pull request #927 from ethcore/pruning_auto

Browse Source 
Pruning auto
This commit is contained in:
Nikolay Volf 2016-04-12 16:00:39 +03:00
commit bdb35cfaed
8 changed files with 91 additions and 27 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

30
ethcore/src/client/client.rs
View File

@ -17,6 +17,7 @@
//! Blockchain database client.
use std::marker::PhantomData;
use std::path::PathBuf;
use util::*;
use util::panics::*;
use views::BlockView;
@ -126,22 +127,29 @@ impl Client<CanonVerifier> {
}
}
pub fn get_db_path(path: &Path, pruning: journaldb::Algorithm, genesis_hash: H256) -> PathBuf {
let mut dir = path.to_path_buf();
dir.push(H64::from(genesis_hash).hex());
//TODO: sec/fat: pruned/full versioning
// version here is a bit useless now, since it's controlled only be the pruning algo.
dir.push(format!("v{}-sec-{}", CLIENT_DB_VER_STR, pruning));
dir
}
pub fn append_path(path: &Path, item: &str) -> String {
let mut p = path.to_path_buf();
p.push(item);
p.to_str().unwrap().to_owned()
}
impl<V> Client<V> where V: Verifier {
/// Create a new client with given spec and DB path and custom verifier.
pub fn new_with_verifier(config: ClientConfig, spec: Spec, path: &Path, message_channel: IoChannel<NetSyncMessage> ) -> Result<Arc<Client<V>>, Error> {
let mut dir = path.to_path_buf();
dir.push(H64::from(spec.genesis_header().hash()).hex());
//TODO: sec/fat: pruned/full versioning
// version here is a bit useless now, since it's controlled only be the pruning algo.
dir.push(format!("v{}-sec-{}", CLIENT_DB_VER_STR, config.pruning));
let path = dir.as_path();
let path = get_db_path(path, config.pruning, spec.genesis_header().hash());
let gb = spec.genesis_block();
let chain = Arc::new(BlockChain::new(config.blockchain, &gb, path));
let mut state_path = path.to_path_buf();
state_path.push("state");
let chain = Arc::new(BlockChain::new(config.blockchain, &gb, &path));
let state_path_str = state_path.to_str().unwrap();
let mut state_db = journaldb::new(state_path_str, config.pruning);
let mut state_db = journaldb::new(&append_path(&path, "state"), config.pruning);
if state_db.is_empty() && spec.ensure_db_good(state_db.as_hashdb_mut()) {
state_db.commit(0, &spec.genesis_header().hash(), None).expect("Error commiting genesis state to state DB");

39
parity/main.rs
View File

@ -128,11 +128,11 @@ Networking Options:
API and Console Options:
-j --jsonrpc Enable the JSON-RPC API server.
--jsonrpc-port PORT Specify the port portion of the JSONRPC API server
[default: 8545].
--jsonrpc-interface IP Specify the hostname portion of the JSONRPC API
server, IP should be an interface's IP address, or
all (all interfaces) or local [default: local].
--jsonrpc-port PORT Specify the port portion of the JSONRPC API server
[default: 8545].
--jsonrpc-cors URL Specify CORS header for JSON-RPC API responses
[default: null].
--jsonrpc-apis APIS Specify the APIs available through the JSONRPC
@ -171,8 +171,14 @@ Sealing/Mining Options:
Footprint Options:
--pruning METHOD Configure pruning of the state/storage trie. METHOD
may be one of: archive, basic (experimental), fast
(experimental) [default: archive].
may be one of auto, archive, basic, fast, light:
archive - keep all state trie data. No pruning.
basic - reference count in disk DB. Slow but light.
fast - maintain journal overlay. Fast but 50MB used.
light - early merges with partial tracking. Fast
and light. Experimental!
auto - use the method most recently synced or
default to archive if none synced [default: auto].
--cache-pref-size BYTES Specify the prefered size of the blockchain cache in
bytes [default: 16384].
--cache-max-size BYTES Specify the maximum size of the blockchain cache in
@ -538,7 +544,25 @@ impl Configuration {
ret
}
fn client_config(&self) -> ClientConfig {
fn find_best_db(&self, spec: &Spec) -> Option<journaldb::Algorithm> {
let mut ret = None;
let mut latest_era = None;
let jdb_types = [journaldb::Algorithm::Archive, journaldb::Algorithm::EarlyMerge, journaldb::Algorithm::OverlayRecent, journaldb::Algorithm::RefCounted];
for i in jdb_types.into_iter() {
let db = journaldb::new(&append_path(&get_db_path(&Path::new(&self.path()), *i, spec.genesis_header().hash()), "state"), *i);
match (latest_era, db.latest_era()) {
(Some(best), Some(this)) if best >= this => {}
(_, None) => {}
(_, Some(this)) => {
latest_era = Some(this);
ret = Some(*i);
}
}
}
ret
}
fn client_config(&self, spec: &Spec) -> ClientConfig {
let mut client_config = ClientConfig::default();
match self.args.flag_cache {
Some(mb) => {
@ -555,8 +579,10 @@ impl Configuration {
"light" => journaldb::Algorithm::EarlyMerge,
"fast" => journaldb::Algorithm::OverlayRecent,
"basic" => journaldb::Algorithm::RefCounted,
"auto" => self.find_best_db(spec).unwrap_or(journaldb::Algorithm::OverlayRecent),
_ => { die!("Invalid pruning method given."); }
};
info!("Using state DB of {}", client_config.pruning);
client_config.name = self.args.flag_identity.clone();
client_config.queue.max_mem_use = self.args.flag_queue_max_size;
client_config
@ -651,13 +677,14 @@ impl Configuration {
let spec = self.spec();
let net_settings = self.net_settings(&spec);
let sync_config = self.sync_config(&spec);
let client_config = self.client_config(&spec);
// Secret Store
let account_service = Arc::new(self.account_service());
// Build client
let mut service = ClientService::start(
self.client_config(), spec, net_settings, &Path::new(&self.path())
client_config, spec, net_settings, &Path::new(&self.path())
).unwrap_or_else(|e| die_with_error(e));
panic_handler.forward_from(&service);

6
util/src/journaldb/archivedb.rs
View File

@ -41,7 +41,7 @@ pub struct ArchiveDB {
// all keys must be at least 12 bytes
const LATEST_ERA_KEY : [u8; 12] = [ b'l', b'a', b's', b't', 0, 0, 0, 0, 0, 0, 0, 0 ];
const VERSION_KEY : [u8; 12] = [ b'j', b'v', b'e', b'r', 0, 0, 0, 0, 0, 0, 0, 0 ];
const DB_VERSION : u32 = 259;
const DB_VERSION : u32 = 0x103;
impl ArchiveDB {
/// Create a new instance from file
@ -55,7 +55,7 @@ impl ArchiveDB {
if !backing.is_empty() {
match backing.get(&VERSION_KEY).map(|d| d.map(|v| decode::<u32>(&v))) {
Ok(Some(DB_VERSION)) => {},
v => panic!("Incompatible DB version, expected {}, got {:?}", DB_VERSION, v)
v => panic!("Incompatible DB version, expected {}, got {:?}; to resolve, remove {} and restart.", DB_VERSION, v, path)
}
} else {
backing.put(&VERSION_KEY, &encode(&DB_VERSION)).expect("Error writing version to database");
@ -168,6 +168,8 @@ impl JournalDB for ArchiveDB {
Ok((inserts + deletes) as u32)
}
fn latest_era(&self) -> Option<u64> { self.latest_era }
fn state(&self, id: &H256) -> Option<Bytes> {
self.backing.get_by_prefix(&id.bytes()[0..12]).and_then(|b| Some(b.to_vec()))
}

7
util/src/journaldb/earlymergedb.rs
View File

@ -70,7 +70,7 @@ pub struct EarlyMergeDB {
// all keys must be at least 12 bytes
const LATEST_ERA_KEY : [u8; 12] = [ b'l', b'a', b's', b't', 0, 0, 0, 0, 0, 0, 0, 0 ];
const VERSION_KEY : [u8; 12] = [ b'j', b'v', b'e', b'r', 0, 0, 0, 0, 0, 0, 0, 0 ];
const DB_VERSION : u32 = 3;
const DB_VERSION : u32 = 0x003;
const PADDING : [u8; 10] = [ 0u8; 10 ];
impl EarlyMergeDB {
@ -85,7 +85,7 @@ impl EarlyMergeDB {
if !backing.is_empty() {
match backing.get(&VERSION_KEY).map(|d| d.map(|v| decode::<u32>(&v))) {
Ok(Some(DB_VERSION)) => {},
v => panic!("Incompatible DB version, expected {}, got {:?}", DB_VERSION, v)
v => panic!("Incompatible DB version, expected {}, got {:?}; to resolve, remove {} and restart.", DB_VERSION, v, path)
}
} else {
backing.put(&VERSION_KEY, &encode(&DB_VERSION)).expect("Error writing version to database");
@ -333,6 +333,8 @@ impl JournalDB for EarlyMergeDB {
self.backing.get(&LATEST_ERA_KEY).expect("Low level database error").is_none()
}
fn latest_era(&self) -> Option<u64> { self.latest_era }
fn mem_used(&self) -> usize {
self.overlay.mem_used() + match self.refs {
Some(ref c) => c.read().unwrap().heap_size_of_children(),
@ -340,7 +342,6 @@ impl JournalDB for EarlyMergeDB {
}
}
#[cfg_attr(feature="dev", allow(cyclomatic_complexity))]
fn commit(&mut self, now: u64, id: &H256, end: Option<(u64, H256)>) -> Result<u32, UtilError> {
// journal format:

2
util/src/journaldb/mod.rs
View File

@ -29,7 +29,7 @@ mod refcounteddb;
pub use self::traits::JournalDB;
/// A journal database algorithm.
#[derive(Debug)]
#[derive(Debug, Clone, Copy)]
pub enum Algorithm {
/// Keep all keys forever.
Archive,

6
util/src/journaldb/overlayrecentdb.rs
View File

@ -95,7 +95,7 @@ impl Clone for OverlayRecentDB {
// all keys must be at least 12 bytes
const LATEST_ERA_KEY : [u8; 12] = [ b'l', b'a', b's', b't', 0, 0, 0, 0, 0, 0, 0, 0 ];
const VERSION_KEY : [u8; 12] = [ b'j', b'v', b'e', b'r', 0, 0, 0, 0, 0, 0, 0, 0 ];
const DB_VERSION : u32 = 0x200 + 3;
const DB_VERSION : u32 = 0x203;
const PADDING : [u8; 10] = [ 0u8; 10 ];
impl OverlayRecentDB {
@ -115,7 +115,7 @@ impl OverlayRecentDB {
if !backing.is_empty() {
match backing.get(&VERSION_KEY).map(|d| d.map(|v| decode::<u32>(&v))) {
Ok(Some(DB_VERSION)) => {}
v => panic!("Incompatible DB version, expected {}, got {:?}", DB_VERSION, v)
v => panic!("Incompatible DB version, expected {}, got {:?}; to resolve, remove {} and restart.", DB_VERSION, v, path)
}
} else {
backing.put(&VERSION_KEY, &encode(&DB_VERSION)).expect("Error writing version to database");
@ -213,6 +213,8 @@ impl JournalDB for OverlayRecentDB {
self.backing.get(&LATEST_ERA_KEY).expect("Low level database error").is_none()
}
fn latest_era(&self) -> Option<u64> { self.journal_overlay.read().unwrap().latest_era }
fn commit(&mut self, now: u64, id: &H256, end: Option<(u64, H256)>) -> Result<u32, UtilError> {
// record new commit's details.
trace!("commit: #{} ({}), end era: {:?}", now, id, end);

25
util/src/journaldb/refcounteddb.rs
View File

@ -42,7 +42,7 @@ pub struct RefCountedDB {
const LATEST_ERA_KEY : [u8; 12] = [ b'l', b'a', b's', b't', 0, 0, 0, 0, 0, 0, 0, 0 ];
const VERSION_KEY : [u8; 12] = [ b'j', b'v', b'e', b'r', 0, 0, 0, 0, 0, 0, 0, 0 ];
const DB_VERSION : u32 = 512;
const DB_VERSION : u32 = 0x200;
const PADDING : [u8; 10] = [ 0u8; 10 ];
impl RefCountedDB {
@ -57,7 +57,7 @@ impl RefCountedDB {
if !backing.is_empty() {
match backing.get(&VERSION_KEY).map(|d| d.map(|v| decode::<u32>(&v))) {
Ok(Some(DB_VERSION)) => {},
v => panic!("Incompatible DB version, expected {}, got {:?}", DB_VERSION, v)
v => panic!("Incompatible DB version, expected {}, got {:?}; to resolve, remove {} and restart.", DB_VERSION, v, path)
}
} else {
backing.put(&VERSION_KEY, &encode(&DB_VERSION)).expect("Error writing version to database");
@ -112,6 +112,8 @@ impl JournalDB for RefCountedDB {
self.latest_era.is_none()
}
fn latest_era(&self) -> Option<u64> { self.latest_era }
fn commit(&mut self, now: u64, id: &H256, end: Option<(u64, H256)>) -> Result<u32, UtilError> {
// journal format:
// [era, 0] => [ id, [insert_0, ...], [remove_0, ...] ]
@ -220,6 +222,25 @@ mod tests {
assert!(!jdb.exists(&h));
}
#[test]
fn latest_era_should_work() {
// history is 3
let mut jdb = RefCountedDB::new_temp();
assert_eq!(jdb.latest_era(), None);
let h = jdb.insert(b"foo");
jdb.commit(0, &b"0".sha3(), None).unwrap();
assert_eq!(jdb.latest_era(), Some(0));
jdb.remove(&h);
jdb.commit(1, &b"1".sha3(), None).unwrap();
assert_eq!(jdb.latest_era(), Some(1));
jdb.commit(2, &b"2".sha3(), None).unwrap();
assert_eq!(jdb.latest_era(), Some(2));
jdb.commit(3, &b"3".sha3(), Some((0, b"0".sha3()))).unwrap();
assert_eq!(jdb.latest_era(), Some(3));
jdb.commit(4, &b"4".sha3(), Some((1, b"1".sha3()))).unwrap();
assert_eq!(jdb.latest_era(), Some(4));
}
#[test]
fn complex() {
// history is 1

3
util/src/journaldb/traits.rs
View File

@ -31,6 +31,9 @@ pub trait JournalDB : HashDB + Send + Sync {
/// Check if this database has any commits
fn is_empty(&self) -> bool;
/// Get the latest era in the DB. None if there isn't yet any data in there.
fn latest_era(&self) -> Option<u64>;
/// Commit all recent insert operations and canonical historical commits' removals from the
/// old era to the backing database, reverting any non-canonical historical commit's inserts.
fn commit(&mut self, now: u64, id: &H256, end: Option<(u64, H256)>) -> Result<u32, UtilError>;