grassrootseconomics/openethereum
8
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Polishing Actually enable fat db pr (#1974) (#2048)

Browse Source 
* Actually enable fat db, and do RPCs for it.

* Implement HashDB traits for AccountDB.

* user defaults

* finished user defaults

* user defaults are network-dependent

* added tests for newly added functions, logger is initialized first

* dir cleanup in progress

* user_file is placed next to snapshots

* fixing requested change
This commit is contained in:
Marek Kotewicz
2016-10-03 11:13:10 +02:00
committed by Gav Wood
parent 0dcdaa7a2a
commit 06fe768ac2
20 changed files with 188 additions and 34 deletions
Download Patch File Download Diff File Expand all files Collapse all files

26
parity/blockchain.rs
View File

@@ -30,8 +30,8 @@ use ethcore::client::{Mode, DatabaseCompactionProfile, VMType, BlockImportError,
use ethcore::error::ImportError;
use ethcore::miner::Miner;
use cache::CacheConfig;
use params::{SpecType, Pruning, Switch, tracing_switch_to_bool};
use informant::{Informant, MillisecondDuration};
use params::{SpecType, Pruning, Switch, tracing_switch_to_bool, fatdb_switch_to_bool};
use io_handler::ImportIoHandler;
use helpers::{to_client_config, execute_upgrades};
use dir::Directories;
@@ -81,6 +81,7 @@ pub struct ImportBlockchain {
pub wal: bool,
pub mode: Mode,
pub tracing: Switch,
pub fat_db: Switch,
pub vm_type: VMType,
}
@@ -96,6 +97,7 @@ pub struct ExportBlockchain {
pub compaction: DatabaseCompactionProfile,
pub wal: bool,
pub mode: Mode,
pub fat_db: Switch,
pub tracing: Switch,
pub from_block: BlockID,
pub to_block: BlockID,
@@ -135,14 +137,17 @@ fn execute_import(cmd: ImportBlockchain) -> Result<String, String> {
// load user defaults
let mut user_defaults = try!(UserDefaults::load(&user_defaults_path));
// check if tracing is on
let tracing = try!(tracing_switch_to_bool(cmd.tracing, &user_defaults));
fdlimit::raise_fd_limit();
// select pruning algorithm
let algorithm = cmd.pruning.to_algorithm(&user_defaults);
// check if tracing is on
let tracing = try!(tracing_switch_to_bool(cmd.tracing, &user_defaults));
// check if fatdb is on
let fat_db = try!(fatdb_switch_to_bool(cmd.fat_db, &user_defaults, algorithm));
// prepare client and snapshot paths.
let client_path = db_dirs.client_path(algorithm);
let snapshot_path = db_dirs.snapshot_path();
@@ -151,7 +156,7 @@ fn execute_import(cmd: ImportBlockchain) -> Result<String, String> {
try!(execute_upgrades(&db_dirs, algorithm, cmd.compaction.compaction_profile()));
// prepare client config
let client_config = to_client_config(&cmd.cache_config, cmd.mode, tracing, cmd.compaction, cmd.wal, cmd.vm_type, "".into(), algorithm);
let client_config = to_client_config(&cmd.cache_config, cmd.mode, tracing, fat_db, cmd.compaction, cmd.wal, cmd.vm_type, "".into(), algorithm);
// build client
let service = try!(ClientService::start(
@@ -283,14 +288,17 @@ fn execute_export(cmd: ExportBlockchain) -> Result<String, String> {
// load user defaults
let user_defaults = try!(UserDefaults::load(&user_defaults_path));
// check if tracing is on
let tracing = try!(tracing_switch_to_bool(cmd.tracing, &user_defaults));
fdlimit::raise_fd_limit();
// select pruning algorithm
let algorithm = cmd.pruning.to_algorithm(&user_defaults);
// check if tracing is on
let tracing = try!(tracing_switch_to_bool(cmd.tracing, &user_defaults));
// check if fatdb is on
let fat_db = try!(fatdb_switch_to_bool(cmd.fat_db, &user_defaults, algorithm));
// prepare client and snapshot paths.
let client_path = db_dirs.client_path(algorithm);
let snapshot_path = db_dirs.snapshot_path();
@@ -299,7 +307,7 @@ fn execute_export(cmd: ExportBlockchain) -> Result<String, String> {
try!(execute_upgrades(&db_dirs, algorithm, cmd.compaction.compaction_profile()));
// prepare client config
let client_config = to_client_config(&cmd.cache_config, cmd.mode, tracing, cmd.compaction, cmd.wal, VMType::default(), "".into(), algorithm);
let client_config = to_client_config(&cmd.cache_config, cmd.mode, tracing, fat_db, cmd.compaction, cmd.wal, VMType::default(), "".into(), algorithm);
let service = try!(ClientService::start(
client_config,

2
parity/cli/config.full.toml
View File

@@ -82,7 +82,7 @@ cache_size_queue = 50
cache_size = 128 # Overrides above caches with total size
fast_and_loose = false
db_compaction = "ssd"
fat_db = false
fat_db = "auto"
[snapshots]
disable_periodic = false

2
parity/cli/config.toml
View File

@@ -49,7 +49,7 @@ cache_size_db = 128
cache_size_blocks = 16
cache_size_queue = 100
db_compaction = "ssd"
fat_db = true
fat_db = "off"
[snapshots]
disable_periodic = true

8
parity/cli/mod.rs
View File

@@ -217,7 +217,7 @@ usage! {
or |c: &Config| otry!(c.footprint).fast_and_loose.clone(),
flag_db_compaction: String = "ssd",
or |c: &Config| otry!(c.footprint).db_compaction.clone(),
flag_fat_db: bool = false,
flag_fat_db: String = "auto",
or |c: &Config| otry!(c.footprint).fat_db.clone(),
// -- Import/Export Options
@@ -362,7 +362,7 @@ struct Footprint {
cache_size_blocks: Option<u32>,
cache_size_queue: Option<u32>,
db_compaction: Option<String>,
fat_db: Option<bool>,
fat_db: Option<String>,
}
#[derive(Default, Debug, PartialEq, RustcDecodable)]
@@ -535,7 +535,7 @@ mod tests {
flag_cache_size: Some(128),
flag_fast_and_loose: false,
flag_db_compaction: "ssd".into(),
flag_fat_db: false,
flag_fat_db: "auto".into(),
// -- Import/Export Options
flag_from: "1".into(),
@@ -687,7 +687,7 @@ mod tests {
cache_size_blocks: Some(16),
cache_size_queue: Some(100),
db_compaction: Some("ssd".into()),
fat_db: Some(true),
fat_db: Some("off".into()),
}),
snapshots: Some(Snapshots {
disable_periodic: Some(true),

5
parity/cli/usage.txt
View File

@@ -217,7 +217,10 @@ Footprint Options:
--db-compaction TYPE Database compaction type. TYPE may be one of:
ssd - suitable for SSDs and fast HDDs;
hdd - suitable for slow HDDs (default: {flag_db_compaction}).
--fat-db Fat database. (default: {flag_fat_db})
--fat-db BOOL Build appropriate information to allow enumeration
of all accounts and storage keys. Doubles the size
of the state database. BOOL may be one of on, off
or auto. (default: {flag_fat_db})
Import/Export Options:
--from BLOCK Export from block BLOCK, which may be an index or

10
parity/configuration.rs
View File

@@ -84,6 +84,7 @@ impl Configuration {
let cache_config = self.cache_config();
let spec = try!(self.chain().parse());
let tracing = try!(self.args.flag_tracing.parse());
let fat_db = try!(self.args.flag_fat_db.parse());
let compaction = try!(self.args.flag_db_compaction.parse());
let wal = !self.args.flag_fast_and_loose;
let enable_network = self.enable_network(&mode);
@@ -140,6 +141,7 @@ impl Configuration {
wal: wal,
mode: mode,
tracing: tracing,
fat_db: fat_db,
vm_type: vm_type,
};
Cmd::Blockchain(BlockchainCmd::Import(import_cmd))
@@ -156,6 +158,7 @@ impl Configuration {
wal: wal,
mode: mode,
tracing: tracing,
fat_db: fat_db,
from_block: try!(to_block_id(&self.args.flag_from)),
to_block: try!(to_block_id(&self.args.flag_to)),
};
@@ -169,6 +172,7 @@ impl Configuration {
logger_config: logger_config,
mode: mode,
tracing: tracing,
fat_db: fat_db,
compaction: compaction,
file_path: self.args.arg_file.clone(),
wal: wal,
@@ -185,6 +189,7 @@ impl Configuration {
logger_config: logger_config,
mode: mode,
tracing: tracing,
fat_db: fat_db,
compaction: compaction,
file_path: self.args.arg_file.clone(),
wal: wal,
@@ -216,6 +221,7 @@ impl Configuration {
miner_extras: try!(self.miner_extras()),
mode: mode,
tracing: tracing,
fat_db: fat_db,
compaction: compaction,
wal: wal,
vm_type: vm_type,
@@ -717,6 +723,7 @@ mod tests {
wal: true,
mode: Default::default(),
tracing: Default::default(),
fat_db: Default::default(),
vm_type: VMType::Interpreter,
})));
}
@@ -737,6 +744,7 @@ mod tests {
wal: true,
mode: Default::default(),
tracing: Default::default(),
fat_db: Default::default(),
from_block: BlockID::Number(1),
to_block: BlockID::Latest,
})));
@@ -758,6 +766,7 @@ mod tests {
wal: true,
mode: Default::default(),
tracing: Default::default(),
fat_db: Default::default(),
from_block: BlockID::Number(1),
to_block: BlockID::Latest,
})));
@@ -804,6 +813,7 @@ mod tests {
ui: false,
name: "".into(),
custom_bootnodes: false,
fat_db: Default::default(),
no_periodic_snapshot: false,
}));
}

2
parity/helpers.rs
View File

@@ -191,6 +191,7 @@ pub fn to_client_config(
cache_config: &CacheConfig,
mode: Mode,
tracing: bool,
fat_db: bool,
compaction: DatabaseCompactionProfile,
wal: bool,
vm_type: VMType,
@@ -217,6 +218,7 @@ pub fn to_client_config(
client_config.mode = mode;
client_config.tracing.enabled = tracing;
client_config.fat_db = fat_db;
client_config.pruning = pruning;
client_config.db_compaction = compaction;
client_config.db_wal = wal;

13
parity/params.rs
View File

@@ -252,6 +252,19 @@ pub fn tracing_switch_to_bool(switch: Switch, user_defaults: &UserDefaults) -> R
}
}
pub fn fatdb_switch_to_bool(switch: Switch, user_defaults: &UserDefaults, algorithm: Algorithm) -> Result<bool, String> {
if algorithm != Algorithm::Archive {
return Err("Fat DB is not supported with the chosen pruning option. Please rerun with `--pruning=archive`".into());
}
match (user_defaults.is_first_launch, switch, user_defaults.fat_db) {
(false, Switch::On, false) => Err("FatDB resync required".into()),
(_, Switch::On, _) => Ok(true),
(_, Switch::Off, _) => Ok(false),
(_, Switch::Auto, def) => Ok(def),
}
}
#[cfg(test)]
mod tests {
use util::journaldb::Algorithm;

26
parity/run.rs
View File

@@ -35,7 +35,10 @@ use rpc::{HttpServer, IpcServer, HttpConfiguration, IpcConfiguration};
use signer::SignerServer;
use dapps::WebappServer;
use io_handler::ClientIoHandler;
use params::{SpecType, Pruning, AccountsConfig, GasPricerConfig, MinerExtras, Switch, tracing_switch_to_bool};
use params::{
SpecType, Pruning, AccountsConfig, GasPricerConfig, MinerExtras, Switch,
tracing_switch_to_bool, fatdb_switch_to_bool,
};
use helpers::{to_client_config, execute_upgrades, passwords_from_files};
use dir::Directories;
use cache::CacheConfig;
@@ -72,6 +75,7 @@ pub struct RunCmd {
pub miner_extras: MinerExtras,
pub mode: Mode,
pub tracing: Switch,
pub fat_db: Switch,
pub compaction: DatabaseCompactionProfile,
pub wal: bool,
pub vm_type: VMType,
@@ -115,11 +119,14 @@ pub fn execute(cmd: RunCmd) -> Result<(), String> {
// load user defaults
let mut user_defaults = try!(UserDefaults::load(&user_defaults_path));
// select pruning algorithm
let algorithm = cmd.pruning.to_algorithm(&user_defaults);
// check if tracing is on
let tracing = try!(tracing_switch_to_bool(cmd.tracing, &user_defaults));
// select pruning algorithm
let algorithm = cmd.pruning.to_algorithm(&user_defaults);
// check if fatdb is on
let fat_db = try!(fatdb_switch_to_bool(cmd.fat_db, &user_defaults, algorithm));
// prepare client and snapshot paths.
let client_path = db_dirs.client_path(algorithm);
@@ -135,7 +142,17 @@ pub fn execute(cmd: RunCmd) -> Result<(), String> {
// display info about used pruning algorithm
info!("Starting {}", Colour::White.bold().paint(version()));
info!("Using state DB journalling strategy {}", Colour::White.bold().paint(algorithm.as_str()));
info!("State DB configuation: {}{}{}",
Colour::White.bold().paint(algorithm.as_str()),
match fat_db {
true => Colour::White.bold().paint(" +Fat").to_string(),
false => "".to_owned(),
},
match tracing {
true => Colour::White.bold().paint(" +Trace").to_string(),
false => "".to_owned(),
}
);
// display warning about using experimental journaldb alorithm
if !algorithm.is_stable() {
@@ -171,6 +188,7 @@ pub fn execute(cmd: RunCmd) -> Result<(), String> {
&cmd.cache_config,
cmd.mode,
tracing,
fat_db,
cmd.compaction,
cmd.wal,
cmd.vm_type,

14
parity/snapshot.rs
View File

@@ -30,7 +30,7 @@ use ethcore::miner::Miner;
use ethcore::ids::BlockID;
use cache::CacheConfig;
use params::{SpecType, Pruning, Switch, tracing_switch_to_bool};
use params::{SpecType, Pruning, Switch, tracing_switch_to_bool, fatdb_switch_to_bool};
use helpers::{to_client_config, execute_upgrades};
use dir::Directories;
use user_defaults::UserDefaults;
@@ -57,6 +57,7 @@ pub struct SnapshotCommand {
pub logger_config: LogConfig,
pub mode: Mode,
pub tracing: Switch,
pub fat_db: Switch,
pub compaction: DatabaseCompactionProfile,
pub file_path: Option<String>,
pub wal: bool,
@@ -139,9 +140,6 @@ impl SnapshotCommand {
// load user defaults
let user_defaults = try!(UserDefaults::load(&user_defaults_path));
// check if tracing is on
let tracing = try!(tracing_switch_to_bool(self.tracing, &user_defaults));
// Setup logging
let _logger = setup_log(&self.logger_config);
@@ -150,6 +148,12 @@ impl SnapshotCommand {
// select pruning algorithm
let algorithm = self.pruning.to_algorithm(&user_defaults);
// check if tracing is on
let tracing = try!(tracing_switch_to_bool(self.tracing, &user_defaults));
// check if fatdb is on
let fat_db = try!(fatdb_switch_to_bool(self.fat_db, &user_defaults, algorithm));
// prepare client and snapshot paths.
let client_path = db_dirs.client_path(algorithm);
let snapshot_path = db_dirs.snapshot_path();
@@ -158,7 +162,7 @@ impl SnapshotCommand {
try!(execute_upgrades(&db_dirs, algorithm, self.compaction.compaction_profile()));
// prepare client config
let client_config = to_client_config(&self.cache_config, self.mode, tracing, self.compaction, self.wal, VMType::default(), "".into(), algorithm);
let client_config = to_client_config(&self.cache_config, self.mode, tracing, fat_db, self.compaction, self.wal, VMType::default(), "".into(), algorithm);
let service = try!(ClientService::start(
client_config,

6
parity/user_defaults.rs
View File

@@ -30,6 +30,7 @@ pub struct UserDefaults {
pub is_first_launch: bool,
pub pruning: Algorithm,
pub tracing: bool,
pub fat_db: bool,
}
impl Serialize for UserDefaults {
@@ -38,6 +39,7 @@ impl Serialize for UserDefaults {
let mut map: BTreeMap<String, Value> = BTreeMap::new();
map.insert("pruning".into(), Value::String(self.pruning.as_str().into()));
map.insert("tracing".into(), Value::Bool(self.tracing));
map.insert("fat_db".into(), Value::Bool(self.fat_db));
map.serialize(serializer)
}
}
@@ -62,11 +64,14 @@ impl Visitor for UserDefaultsVisitor {
let pruning = try!(pruning.parse().map_err(|_| Error::custom("invalid pruning method")));
let tracing: Value = try!(map.remove("tracing".into()).ok_or_else(|| Error::custom("missing tracing")));
let tracing = try!(tracing.as_bool().ok_or_else(|| Error::custom("invalid tracing value")));
let fat_db: Value = map.remove("fat_db".into()).unwrap_or_else(|| Value::Bool(false));
let fat_db = try!(fat_db.as_bool().ok_or_else(|| Error::custom("invalid fat_db value")));
let user_defaults = UserDefaults {
is_first_launch: false,
pruning: pruning,
tracing: tracing,
fat_db: fat_db,
};
Ok(user_defaults)
@@ -79,6 +84,7 @@ impl Default for UserDefaults {
is_first_launch: true,
pruning: Algorithm::default(),
tracing: false,
fat_db: false,
}
}
}