openethereum/util/journaldb/src/lib.rs

212 lines
6.2 KiB
Rust
Raw Normal View History

// Copyright 2015-2018 Parity Technologies (UK) Ltd.
2016-03-11 13:50:39 +01:00
// This file is part of Parity.
// Parity is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// Parity is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
// You should have received a copy of the GNU General Public License
// along with Parity. If not, see <http://www.gnu.org/licenses/>.
2016-04-06 10:07:24 +02:00
//! `JournalDB` interface and implementation.
2016-03-11 13:50:39 +01:00
2017-10-17 10:40:45 +02:00
extern crate heapsize;
#[macro_use]
extern crate log;
extern crate ethereum_types;
extern crate ethcore_bytes as bytes;
extern crate hashdb;
extern crate keccak_hasher;
extern crate kvdb;
2017-10-17 10:40:45 +02:00
extern crate memorydb;
extern crate parking_lot;
extern crate plain_hasher;
2017-10-17 10:40:45 +02:00
extern crate rlp;
#[cfg(test)]
extern crate ethcore_logger;
#[cfg(test)]
2017-11-10 19:04:55 +01:00
extern crate keccak_hash as keccak;
2017-10-17 10:40:45 +02:00
#[cfg(test)]
extern crate kvdb_memorydb;
use std::{fmt, str, io};
2017-07-29 21:56:42 +02:00
use std::sync::Arc;
2016-03-11 13:50:39 +01:00
/// Export the journaldb module.
mod traits;
2016-03-11 13:50:39 +01:00
mod archivedb;
mod earlymergedb;
mod overlayrecentdb;
2016-03-13 18:09:44 +01:00
mod refcounteddb;
mod util;
mod as_hash_db_impls;
2016-03-11 13:50:39 +01:00
pub mod overlaydb;
2016-04-06 10:07:24 +02:00
/// Export the `JournalDB` trait.
2016-03-11 13:50:39 +01:00
pub use self::traits::JournalDB;
`Client` refactoring (#7038) * Improves `BestBlock` comment * Improves `TraceDB` comment * Improves `journaldb::Algorithm` comment. Probably the whole enum should be renamed to `Strategy` or something alike. * Comments some of the `Client`'s fields * Deglobs client imports * Fixes comments * Extracts `import_lock` to `Importer` struct * Extracts `verifier` to `Importer` struct * Extracts `block_queue` to `Importer` struct * Extracts `miner` to `Importer` struct * Extracts `ancient_verifier` to `Importer` struct * Extracts `rng` to `Importer` struct * Extracts `import_old_block` to `Importer` struct * Adds `Nonce` trait * Adds `Balance` trait * Adds `ChainInfo` trait * Fixes imports for tests using `chain_info` method * Adds `BlockInfo` trait * Adds more `ChainInfo` imports * Adds `BlockInfo` imports * Adds `ReopenBlock` trait * Adds `PrepareOpenBlock` trait * Fixes import in tests * Adds `CallContract` trait * Fixes imports in tests using `call_contract` method * Adds `TransactionInfo` trait * Adds `RegistryInfo` trait * Fixes imports in tests using `registry_address` method * Adds `ScheduleInfo` trait * Adds `ImportSealedBlock` trait * Fixes imports in test using `import_sealed_block` method * Adds `BroadcastProposalBlock` trait * Migrates `Miner` to static dispatch * Fixes tests * Moves `calculate_enacted_retracted` to `Importer` * Moves import-related methods to `Importer` * Removes redundant `import_old_block` wrapper * Extracts `import_block*` into separate trait * Fixes tests * Handles `Pending` in `LightFetch` * Handles `Pending` in filters * Handles `Pending` in `ParityClient` * Handles `Pending` in `EthClient` * Removes `BlockId::Pending`, partly refactors dependent code * Adds `StateInfo` trait * Exports `StateOrBlock` and `BlockChain` types from `client` module * Refactors `balance` RPC using generic API * Refactors `storage_at` RPC using generic API * Makes `MinerService::pending_state`'s return type dynamic * Adds `StateOrBlock` and `BlockChain` types * Adds impl of `client::BlockChain` for `Client` * Exports `StateInfo` trait from `client` module * Missing `self` use To be fixed up to "Adds impl of `client::BlockChain` for `Client`" * Adds `number_to_id` and refactors dependent RPC methods * Refactors `code_at` using generic API * Adds `StateClient` trait * Refactors RPC to use `StateClient` trait * Reverts `client::BlockChain` trait stuff, refactors methods to accept `StateOrBlock` * Refactors TestClient * Adds helper function `block_number_to_id` * Uses `block_number_to_id` instead of local function * Handles `Pending` in `list_accounts` and `list_storage_keys` * Attempt to use associated types for state instead of trait objects * Simplifies `state_at_beginning` * Extracts `call` and `call_many` into separate trait * Refactors `build_last_hashes` to accept reference * Exports `Call` type from the module * Refactors `call` and `call_many` to accept state and header * Exports `state_at` in `StateClient` * Exports `pending_block_header` from `MinerService` * Refactors RPC `call` method using new API * Adds missing parentheses * Refactors `parity::call` to use new call API * Update .gitlab-ci.yml fix gitlab lint * Fixes error handling * Refactors `traces::call` and `call_many` to use new call API * Refactors `call_contract` * Refactors `block_header` * Refactors internal RPC method `block` * Moves `estimate_gas` to `Call` trait, refactors parameters * Refactors `estimate_gas` in RPC * Refactors `uncle` * Refactors RPC `transaction` * Covers missing branches * Makes it all compile, fixes compiler grumbles * Adds casts in `blockchain` module * Fixes `PendingBlock` tests, work on `MinerService` * Adds test stubs for StateClient and EngineInfo * Makes `state_db` public * Adds missing impls for `TestBlockChainClient` * Adds trait documentation * Adds missing docs to the `state_db` module * Fixes trivial compilation errors * Moves `code_hash` method to a `BlockInfo` trait * Refactors `Verifier` to be generic over client * Refactors `TransactionFilter` to be generic over client * Refactors `Miner` and `Client` to reflect changes in verifier and txfilter API * Moves `ServiceTransactionChecker` back to `ethcore` * Fixes trait bounds in `Miner` API * Fixes `Client` * Fixes lifetime bound in `FullFamilyParams` * Adds comments to `FullFamilyParams` * Fixes imports in `ethcore` * Fixes BlockNumber handling in `code_at` and `replay_block_transactions` * fix compile issues * First step to redundant trait merge * Fixes compilation error in RPC tests * Adds mock `State` as a stub for `TestClient` * Handles `StateOrBlock::State` in `TestBlockChainClient::balance` * Fixes `transaction_count` RPC * Fixes `transaction_count` * Moves `service_transaction.json` to the `contracts` subfolder * Fixes compilation errors in tests * Refactors client to use `AccountData` * Refactors client to use `BlockChain` * Refactors miner to use aggregate traits * Adds `SealedBlockImporter` trait * Refactors miner to use `SealedBlockImporter` trait * Removes unused imports * Simplifies `RegistryInfo::registry_address` * Fixes indentation * Removes commented out trait bound
2018-03-03 18:42:13 +01:00
/// Journal database operating strategy.
cli overhaul (#1600) * cli commands * cleanup parity/signer * cleanup parity/signer * remove redundant import of signer crate from main.rs * cli cleanup in progress * cli cleanup in progress * moved few commonly used functions to separate methods with tests * cleaning up blockchain import in progress * cleaning up blockchain import in progress2 * cleaning up blockchain import in progress3 * tests for database compaction profile parsing * cleaning up blockchain import in progress4 * cleaning up blockchain import in progress5 * blockchain import * export blockchain in progress * cleanup execute_export * Configuration::to_duration cleaned up * removed unused code, tests for to_duration * cleanup Configuration::mode function * parsing some of the cli params in params.rs * rpc and signer are no longer optional * move importing extern crates to main.rs file * swipe dies from rpc module * swipe dies from dapps * finding deprecated * several tests and fixes for parity * parity cleanup in progress * cleanup price parsing * parity cleanup in progress * swiped all dies * parity cleanup in progress * replace usages of from_str with parse() in parity/params.rs * removed few more from_str * split parity/params.rs into params and helpers * removed wildcard import from configuration.rs * cleanup directories/path creation * cleaning up run cmd * moved LoggerConfig * defaults for cli params * fixed indention in raise_fd_limit * tests for rpc_apis * tests for default ipc and rpc settings * ipc socket * cleanup in progress * account service * cleanup miner config * BlockChain commands use Directiores structure now * client_config * network settings and dapps configuration * removing warnings * default logger config * fixed client_path * overhaul * fixing export && import * default export DataFormat * import and export also upgrade db * fixed export && import * polishing pr * polishing pr * fixed custom bootnodes * fixed daemonize on windows * fixed setting up enable network * finished pr * fixed compiling on windows * Fixed warning; windows build * Better cache management * Fixed tests on windows * Fixed test * Restored pruning method names * --cache alias * Fixed more tests * Ensure default options actually listed as valid [ci:skip]
2016-07-25 16:09:47 +02:00
#[derive(Debug, PartialEq, Clone, Copy)]
pub enum Algorithm {
/// Keep all keys forever.
Archive,
2016-03-11 13:50:39 +01:00
/// Ancient and recent history maintained separately; recent history lasts for particular
/// number of blocks.
///
/// Inserts go into backing database, journal retains knowledge of whether backing DB key is
/// ancient or recent. Non-canon inserts get explicitly reverted and removed from backing DB.
EarlyMerge,
/// Ancient and recent history maintained separately; recent history lasts for particular
/// number of blocks.
///
/// Inserts go into memory overlay, which is tried for key fetches. Memory overlay gets
/// flushed in backing only at end of recent history.
OverlayRecent,
/// Ancient and recent history maintained separately; recent history lasts for particular
/// number of blocks.
///
/// References are counted in disk-backed DB.
RefCounted,
}
2017-07-29 17:12:07 +02:00
impl str::FromStr for Algorithm {
cli overhaul (#1600) * cli commands * cleanup parity/signer * cleanup parity/signer * remove redundant import of signer crate from main.rs * cli cleanup in progress * cli cleanup in progress * moved few commonly used functions to separate methods with tests * cleaning up blockchain import in progress * cleaning up blockchain import in progress2 * cleaning up blockchain import in progress3 * tests for database compaction profile parsing * cleaning up blockchain import in progress4 * cleaning up blockchain import in progress5 * blockchain import * export blockchain in progress * cleanup execute_export * Configuration::to_duration cleaned up * removed unused code, tests for to_duration * cleanup Configuration::mode function * parsing some of the cli params in params.rs * rpc and signer are no longer optional * move importing extern crates to main.rs file * swipe dies from rpc module * swipe dies from dapps * finding deprecated * several tests and fixes for parity * parity cleanup in progress * cleanup price parsing * parity cleanup in progress * swiped all dies * parity cleanup in progress * replace usages of from_str with parse() in parity/params.rs * removed few more from_str * split parity/params.rs into params and helpers * removed wildcard import from configuration.rs * cleanup directories/path creation * cleaning up run cmd * moved LoggerConfig * defaults for cli params * fixed indention in raise_fd_limit * tests for rpc_apis * tests for default ipc and rpc settings * ipc socket * cleanup in progress * account service * cleanup miner config * BlockChain commands use Directiores structure now * client_config * network settings and dapps configuration * removing warnings * default logger config * fixed client_path * overhaul * fixing export && import * default export DataFormat * import and export also upgrade db * fixed export && import * polishing pr * polishing pr * fixed custom bootnodes * fixed daemonize on windows * fixed setting up enable network * finished pr * fixed compiling on windows * Fixed warning; windows build * Better cache management * Fixed tests on windows * Fixed test * Restored pruning method names * --cache alias * Fixed more tests * Ensure default options actually listed as valid [ci:skip]
2016-07-25 16:09:47 +02:00
type Err = String;
fn from_str(s: &str) -> Result<Self, Self::Err> {
match s {
"archive" => Ok(Algorithm::Archive),
"light" => Ok(Algorithm::EarlyMerge),
"fast" => Ok(Algorithm::OverlayRecent),
"basic" => Ok(Algorithm::RefCounted),
e => Err(format!("Invalid algorithm: {}", e)),
}
}
}
impl Algorithm {
/// Returns static str describing journal database algorithm.
pub fn as_str(&self) -> &'static str {
match *self {
Algorithm::Archive => "archive",
Algorithm::EarlyMerge => "light",
Algorithm::OverlayRecent => "fast",
Algorithm::RefCounted => "basic",
}
}
2016-07-25 17:45:55 +02:00
/// Returns static str describing journal database algorithm.
pub fn as_internal_name_str(&self) -> &'static str {
match *self {
Algorithm::Archive => "archive",
Algorithm::EarlyMerge => "earlymerge",
Algorithm::OverlayRecent => "overlayrecent",
Algorithm::RefCounted => "refcounted",
}
}
cli overhaul (#1600) * cli commands * cleanup parity/signer * cleanup parity/signer * remove redundant import of signer crate from main.rs * cli cleanup in progress * cli cleanup in progress * moved few commonly used functions to separate methods with tests * cleaning up blockchain import in progress * cleaning up blockchain import in progress2 * cleaning up blockchain import in progress3 * tests for database compaction profile parsing * cleaning up blockchain import in progress4 * cleaning up blockchain import in progress5 * blockchain import * export blockchain in progress * cleanup execute_export * Configuration::to_duration cleaned up * removed unused code, tests for to_duration * cleanup Configuration::mode function * parsing some of the cli params in params.rs * rpc and signer are no longer optional * move importing extern crates to main.rs file * swipe dies from rpc module * swipe dies from dapps * finding deprecated * several tests and fixes for parity * parity cleanup in progress * cleanup price parsing * parity cleanup in progress * swiped all dies * parity cleanup in progress * replace usages of from_str with parse() in parity/params.rs * removed few more from_str * split parity/params.rs into params and helpers * removed wildcard import from configuration.rs * cleanup directories/path creation * cleaning up run cmd * moved LoggerConfig * defaults for cli params * fixed indention in raise_fd_limit * tests for rpc_apis * tests for default ipc and rpc settings * ipc socket * cleanup in progress * account service * cleanup miner config * BlockChain commands use Directiores structure now * client_config * network settings and dapps configuration * removing warnings * default logger config * fixed client_path * overhaul * fixing export && import * default export DataFormat * import and export also upgrade db * fixed export && import * polishing pr * polishing pr * fixed custom bootnodes * fixed daemonize on windows * fixed setting up enable network * finished pr * fixed compiling on windows * Fixed warning; windows build * Better cache management * Fixed tests on windows * Fixed test * Restored pruning method names * --cache alias * Fixed more tests * Ensure default options actually listed as valid [ci:skip]
2016-07-25 16:09:47 +02:00
/// Returns true if pruning strategy is stable
pub fn is_stable(&self) -> bool {
match *self {
Algorithm::Archive | Algorithm::OverlayRecent => true,
_ => false,
}
}
/// Returns all algorithm types.
pub fn all_types() -> Vec<Algorithm> {
vec![Algorithm::Archive, Algorithm::EarlyMerge, Algorithm::OverlayRecent, Algorithm::RefCounted]
}
}
impl fmt::Display for Algorithm {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
cli overhaul (#1600) * cli commands * cleanup parity/signer * cleanup parity/signer * remove redundant import of signer crate from main.rs * cli cleanup in progress * cli cleanup in progress * moved few commonly used functions to separate methods with tests * cleaning up blockchain import in progress * cleaning up blockchain import in progress2 * cleaning up blockchain import in progress3 * tests for database compaction profile parsing * cleaning up blockchain import in progress4 * cleaning up blockchain import in progress5 * blockchain import * export blockchain in progress * cleanup execute_export * Configuration::to_duration cleaned up * removed unused code, tests for to_duration * cleanup Configuration::mode function * parsing some of the cli params in params.rs * rpc and signer are no longer optional * move importing extern crates to main.rs file * swipe dies from rpc module * swipe dies from dapps * finding deprecated * several tests and fixes for parity * parity cleanup in progress * cleanup price parsing * parity cleanup in progress * swiped all dies * parity cleanup in progress * replace usages of from_str with parse() in parity/params.rs * removed few more from_str * split parity/params.rs into params and helpers * removed wildcard import from configuration.rs * cleanup directories/path creation * cleaning up run cmd * moved LoggerConfig * defaults for cli params * fixed indention in raise_fd_limit * tests for rpc_apis * tests for default ipc and rpc settings * ipc socket * cleanup in progress * account service * cleanup miner config * BlockChain commands use Directiores structure now * client_config * network settings and dapps configuration * removing warnings * default logger config * fixed client_path * overhaul * fixing export && import * default export DataFormat * import and export also upgrade db * fixed export && import * polishing pr * polishing pr * fixed custom bootnodes * fixed daemonize on windows * fixed setting up enable network * finished pr * fixed compiling on windows * Fixed warning; windows build * Better cache management * Fixed tests on windows * Fixed test * Restored pruning method names * --cache alias * Fixed more tests * Ensure default options actually listed as valid [ci:skip]
2016-07-25 16:09:47 +02:00
write!(f, "{}", self.as_str())
}
}
/// Create a new `JournalDB` trait object over a generic key-value database.
pub fn new(backing: Arc<::kvdb::KeyValueDB>, algorithm: Algorithm, col: Option<u32>) -> Box<JournalDB> {
match algorithm {
Algorithm::Archive => Box::new(archivedb::ArchiveDB::new(backing, col)),
Algorithm::EarlyMerge => Box::new(earlymergedb::EarlyMergeDB::new(backing, col)),
Algorithm::OverlayRecent => Box::new(overlayrecentdb::OverlayRecentDB::new(backing, col)),
Algorithm::RefCounted => Box::new(refcounteddb::RefCountedDB::new(backing, col)),
}
}
// all keys must be at least 12 bytes
2017-04-19 14:58:19 +02:00
const DB_PREFIX_LEN : usize = ::kvdb::PREFIX_LEN;
const LATEST_ERA_KEY : [u8; ::kvdb::PREFIX_LEN] = [ b'l', b'a', b's', b't', 0, 0, 0, 0, 0, 0, 0, 0 ];
cli overhaul (#1600) * cli commands * cleanup parity/signer * cleanup parity/signer * remove redundant import of signer crate from main.rs * cli cleanup in progress * cli cleanup in progress * moved few commonly used functions to separate methods with tests * cleaning up blockchain import in progress * cleaning up blockchain import in progress2 * cleaning up blockchain import in progress3 * tests for database compaction profile parsing * cleaning up blockchain import in progress4 * cleaning up blockchain import in progress5 * blockchain import * export blockchain in progress * cleanup execute_export * Configuration::to_duration cleaned up * removed unused code, tests for to_duration * cleanup Configuration::mode function * parsing some of the cli params in params.rs * rpc and signer are no longer optional * move importing extern crates to main.rs file * swipe dies from rpc module * swipe dies from dapps * finding deprecated * several tests and fixes for parity * parity cleanup in progress * cleanup price parsing * parity cleanup in progress * swiped all dies * parity cleanup in progress * replace usages of from_str with parse() in parity/params.rs * removed few more from_str * split parity/params.rs into params and helpers * removed wildcard import from configuration.rs * cleanup directories/path creation * cleaning up run cmd * moved LoggerConfig * defaults for cli params * fixed indention in raise_fd_limit * tests for rpc_apis * tests for default ipc and rpc settings * ipc socket * cleanup in progress * account service * cleanup miner config * BlockChain commands use Directiores structure now * client_config * network settings and dapps configuration * removing warnings * default logger config * fixed client_path * overhaul * fixing export && import * default export DataFormat * import and export also upgrade db * fixed export && import * polishing pr * polishing pr * fixed custom bootnodes * fixed daemonize on windows * fixed setting up enable network * finished pr * fixed compiling on windows * Fixed warning; windows build * Better cache management * Fixed tests on windows * Fixed test * Restored pruning method names * --cache alias * Fixed more tests * Ensure default options actually listed as valid [ci:skip]
2016-07-25 16:09:47 +02:00
fn error_key_already_exists(hash: &ethereum_types::H256) -> io::Error {
io::Error::new(io::ErrorKind::AlreadyExists, hash.to_string())
}
fn error_negatively_reference_hash(hash: &ethereum_types::H256) -> io::Error {
io::Error::new(io::ErrorKind::Other, format!("Entry {} removed from database more times than it was added.", hash))
}
cli overhaul (#1600) * cli commands * cleanup parity/signer * cleanup parity/signer * remove redundant import of signer crate from main.rs * cli cleanup in progress * cli cleanup in progress * moved few commonly used functions to separate methods with tests * cleaning up blockchain import in progress * cleaning up blockchain import in progress2 * cleaning up blockchain import in progress3 * tests for database compaction profile parsing * cleaning up blockchain import in progress4 * cleaning up blockchain import in progress5 * blockchain import * export blockchain in progress * cleanup execute_export * Configuration::to_duration cleaned up * removed unused code, tests for to_duration * cleanup Configuration::mode function * parsing some of the cli params in params.rs * rpc and signer are no longer optional * move importing extern crates to main.rs file * swipe dies from rpc module * swipe dies from dapps * finding deprecated * several tests and fixes for parity * parity cleanup in progress * cleanup price parsing * parity cleanup in progress * swiped all dies * parity cleanup in progress * replace usages of from_str with parse() in parity/params.rs * removed few more from_str * split parity/params.rs into params and helpers * removed wildcard import from configuration.rs * cleanup directories/path creation * cleaning up run cmd * moved LoggerConfig * defaults for cli params * fixed indention in raise_fd_limit * tests for rpc_apis * tests for default ipc and rpc settings * ipc socket * cleanup in progress * account service * cleanup miner config * BlockChain commands use Directiores structure now * client_config * network settings and dapps configuration * removing warnings * default logger config * fixed client_path * overhaul * fixing export && import * default export DataFormat * import and export also upgrade db * fixed export && import * polishing pr * polishing pr * fixed custom bootnodes * fixed daemonize on windows * fixed setting up enable network * finished pr * fixed compiling on windows * Fixed warning; windows build * Better cache management * Fixed tests on windows * Fixed test * Restored pruning method names * --cache alias * Fixed more tests * Ensure default options actually listed as valid [ci:skip]
2016-07-25 16:09:47 +02:00
#[cfg(test)]
mod tests {
use super::Algorithm;
#[test]
fn test_journal_algorithm_parsing() {
assert_eq!(Algorithm::Archive, "archive".parse().unwrap());
assert_eq!(Algorithm::EarlyMerge, "light".parse().unwrap());
assert_eq!(Algorithm::OverlayRecent, "fast".parse().unwrap());
assert_eq!(Algorithm::RefCounted, "basic".parse().unwrap());
}
#[test]
fn test_journal_algorithm_printing() {
assert_eq!(Algorithm::Archive.to_string(), "archive".to_owned());
assert_eq!(Algorithm::EarlyMerge.to_string(), "light".to_owned());
assert_eq!(Algorithm::OverlayRecent.to_string(), "fast".to_owned());
assert_eq!(Algorithm::RefCounted.to_string(), "basic".to_owned());
}
#[test]
fn test_journal_algorithm_is_stable() {
assert!(Algorithm::Archive.is_stable());
assert!(Algorithm::OverlayRecent.is_stable());
assert!(!Algorithm::EarlyMerge.is_stable());
assert!(!Algorithm::RefCounted.is_stable());
}
#[test]
fn test_journal_algorithm_all_types() {
// compiling should fail if some cases are not covered
let mut archive = 0;
let mut earlymerge = 0;
let mut overlayrecent = 0;
let mut refcounted = 0;
for a in &Algorithm::all_types() {
match *a {
Algorithm::Archive => archive += 1,
Algorithm::EarlyMerge => earlymerge += 1,
Algorithm::OverlayRecent => overlayrecent += 1,
Algorithm::RefCounted => refcounted += 1,
}
}
assert_eq!(archive, 1);
assert_eq!(earlymerge, 1);
assert_eq!(overlayrecent, 1);
assert_eq!(refcounted, 1);
}
}