From 84a48142defadd8b48654299c5d4d532f992261d Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Thu, 10 Mar 2016 19:50:04 +0100 Subject: [PATCH 01/17] Add more geth options. --- parity/main.rs | 39 +++++++++++++++++++++++---------------- 1 file changed, 23 insertions(+), 16 deletions(-) diff --git a/parity/main.rs b/parity/main.rs index 68d45bc04..cced0ed0f 100644 --- a/parity/main.rs +++ b/parity/main.rs @@ -70,9 +70,9 @@ Parity. Ethereum Client. Copyright 2015, 2016 Ethcore (UK) Limited Usage: - parity daemon [options] [ --no-bootstrap | ... ] + parity daemon [options] parity account (new | list) - parity [options] [ --no-bootstrap | ... ] + parity [options] Protocol Options: --chain CHAIN Specify the blockchain type. CHAIN may be either a JSON chain specification file @@ -85,13 +85,13 @@ Protocol Options: --identity NAME Specify your node's name. Networking Options: - --no-bootstrap Don't bother trying to connect to any nodes initially. - --listen-address URL Specify the IP/port on which to listen for peers [default: 0.0.0.0:30304]. - --public-address URL Specify the IP/port on which peers may connect. - --address URL Equivalent to --listen-address URL --public-address URL. - --peers NUM Try to maintain that many peers [default: 25]. + --no-bootstrap Don't bother trying to connect to standard bootnodes. + --bootnodes NODES Specify additional comma-separated bootnodes. --no-discovery Disable new peer discovery. - --no-upnp Disable trying to figure out the correct public adderss over UPnP. + --peers NUM Try to maintain that many peers [default: 25]. + --port PORT Override the port for the node to listen on, supercedes --address. + --nat METHOD Specify method to use for determining public address. Must be one of: any, none, + upnp, extip:(IP) [default: upnp]. --node-key KEY Specify node secret key, either as 64-character hex string or input to SHA3 operation. API and Console Options: @@ -101,16 +101,11 @@ API and Console Options: --jsonrpc-cors URL Specify CORS header for JSON-RPC API responses [default: null]. --jsonrpc-apis APIS Specify the APIs available through the JSONRPC interface. APIS is a comma-delimited list of API name. Possible name are web3, eth and net. [default: web3,eth,net]. - --rpc Equivalent to --jsonrpc (geth-compatible). - --rpcaddr HOST Equivalent to --jsonrpc-addr HOST (geth-compatible). - --rpcport PORT Equivalent to --jsonrpc-port PORT (geth-compatible). - --rpcapi APIS Equivalent to --jsonrpc-apis APIS (geth-compatible). - --rpccorsdomain URL Equivalent to --jsonrpc-cors URL (geth-compatible). Sealing/Mining Options: --author ADDRESS Specify the block author (aka "coinbase") address for sending block rewards from sealed blocks [default: 0037a6b811ffeb6e072da21179d11b1406371c63]. - --extradata STRING Specify a custom extra-data for authored blocks, no more than 32 characters. + --extra-data STRING Specify a custom extra-data for authored blocks, no more than 32 characters. Memory Footprint Options: --cache-pref-size BYTES Specify the prefered size of the blockchain cache in bytes [default: 16384]. @@ -119,6 +114,18 @@ Memory Footprint Options: --cache MEGABYTES Set total amount of cache to use for the entire system, mutually exclusive with other cache options (geth-compatible). +Geth-Compatibility Options + --rpc Equivalent to --jsonrpc. + --rpcaddr HOST Equivalent to --jsonrpc-addr HOST. + --rpcport PORT Equivalent to --jsonrpc-port PORT. + --rpcapi APIS Equivalent to --jsonrpc-apis APIS. + --rpccorsdomain URL Equivalent to --jsonrpc-cors URL. + --maxpeers COUNT Equivalent to --peers COUNT. + --nodekey KEY Equivalent to --node-key KEY. + --nodiscover Equivalent to --no-discovery. + --etherbase ADDRESS Equivalent to --author ADDRESS. + --extradata STRING Equivalent to --extra-data STRING. + Miscellaneous Options: -l --logging LOGGING Specify the logging level. -v --version Show information about version. @@ -145,7 +152,7 @@ struct Args { flag_listen_address: String, flag_public_address: Option, flag_address: Option, - flag_peers: usize, + flag_maxpeers: usize, flag_no_discovery: bool, flag_no_upnp: bool, flag_node_key: Option, @@ -323,7 +330,7 @@ impl Configuration { ret.public_address = public; ret.use_secret = self.args.flag_node_key.as_ref().map(|s| Secret::from_str(&s).unwrap_or_else(|_| s.sha3())); ret.discovery_enabled = !self.args.flag_no_discovery; - ret.ideal_peers = self.args.flag_peers as u32; + ret.ideal_peers = self.args.flag_maxpeers as u32; let mut net_path = PathBuf::from(&self.path()); net_path.push("network"); ret.config_path = Some(net_path.to_str().unwrap().to_owned()); From 29916edb91092d95a5a08c10075bea2bb7098a3f Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Thu, 10 Mar 2016 21:36:45 +0100 Subject: [PATCH 02/17] More geth compatibility. --- parity/main.rs | 101 +++++++++++++++++++++++++------------------------ 1 file changed, 52 insertions(+), 49 deletions(-) diff --git a/parity/main.rs b/parity/main.rs index cced0ed0f..402017f3c 100644 --- a/parity/main.rs +++ b/parity/main.rs @@ -37,7 +37,7 @@ extern crate rpassword; #[cfg(feature = "rpc")] extern crate ethcore_rpc as rpc; -use std::net::{SocketAddr}; +use std::net::{SocketAddr, IpAddr}; use std::env; use std::process::exit; use std::path::PathBuf; @@ -77,21 +77,19 @@ Usage: Protocol Options: --chain CHAIN Specify the blockchain type. CHAIN may be either a JSON chain specification file or olympic, frontier, homestead, mainnet, morden, or testnet [default: homestead]. - --testnet Equivalent to --chain testnet (geth-compatible). - --networkid INDEX Override the network identifier from the chain we are on. - --archive Client should not prune the state/storage trie. - -d --datadir PATH Specify the database & configuration directory path [default: $HOME/.parity] + --db-path PATH Specify the database & configuration directory path [default: $HOME/.parity] --keys-path PATH Specify the path for JSON key files to be found [default: $HOME/.web3/keys] --identity NAME Specify your node's name. + --archive Client should not prune the state/storage trie. Networking Options: - --no-bootstrap Don't bother trying to connect to standard bootnodes. - --bootnodes NODES Specify additional comma-separated bootnodes. - --no-discovery Disable new peer discovery. + --port PORT Override the port on which the node should listen [default: 30303]. --peers NUM Try to maintain that many peers [default: 25]. - --port PORT Override the port for the node to listen on, supercedes --address. --nat METHOD Specify method to use for determining public address. Must be one of: any, none, - upnp, extip:(IP) [default: upnp]. + upnp, extip:(IP) [default: any]. + --bootnodes NODES Specify additional comma-separated bootnodes. + --no-bootstrap Don't bother trying to connect to standard bootnodes. + --no-discovery Disable new peer discovery. --node-key KEY Specify node secret key, either as 64-character hex string or input to SHA3 operation. API and Console Options: @@ -115,6 +113,9 @@ Memory Footprint Options: other cache options (geth-compatible). Geth-Compatibility Options + --datadir PATH Equivalent to --db-path PATH. + --testnet Equivalent to --chain testnet. + --networkid INDEX Override the network identifier from the chain we are on. --rpc Equivalent to --jsonrpc. --rpcaddr HOST Equivalent to --jsonrpc-addr HOST. --rpcport PORT Equivalent to --jsonrpc-port PORT. @@ -139,22 +140,18 @@ struct Args { cmd_new: bool, cmd_list: bool, arg_pid_file: String, - arg_enode: Vec, flag_chain: String, - flag_testnet: bool, - flag_datadir: String, - flag_networkid: Option, + flag_db_path: String, flag_identity: String, flag_cache: Option, flag_keys_path: String, flag_archive: bool, + flag_bootnodes: Option, flag_no_bootstrap: bool, - flag_listen_address: String, - flag_public_address: Option, - flag_address: Option, - flag_maxpeers: usize, + flag_port: u16, + flag_peers: usize, flag_no_discovery: bool, - flag_no_upnp: bool, + flag_nat: String, flag_node_key: Option, flag_cache_pref_size: usize, flag_cache_max_size: usize, @@ -164,15 +161,24 @@ struct Args { flag_jsonrpc_port: u16, flag_jsonrpc_cors: String, flag_jsonrpc_apis: String, + flag_logging: Option, + flag_version: bool, + // geth-compatibility... + flag_nodekey: Option, + flag_nodiscover: bool, + flag_maxpeers: Option, + flag_author: String, + flag_extra_data: Option, + flag_datadir: Option, + flag_extradata: Option, + flag_etherbase: Option, flag_rpc: bool, flag_rpcaddr: Option, flag_rpcport: Option, flag_rpccorsdomain: Option, flag_rpcapi: Option, - flag_logging: Option, - flag_version: bool, - flag_author: String, - flag_extra_data: Option, + flag_testnet: bool, + flag_networkid: Option, } fn setup_log(init: &Option) { @@ -252,15 +258,17 @@ impl Configuration { } fn path(&self) -> String { - self.args.flag_datadir.replace("$HOME", env::home_dir().unwrap().to_str().unwrap()) + let d = self.args.flag_datadir.as_ref().unwrap_or(&self.args.flag_db_path); + d.replace("$HOME", env::home_dir().unwrap().to_str().unwrap()) } fn author(&self) -> Address { - Address::from_str(&self.args.flag_author).unwrap_or_else(|_| die!("{}: Invalid address for --author. Must be 40 hex characters, without the 0x at the beginning.", self.args.flag_author)) + let d = self.args.flag_etherbase.as_ref().unwrap_or(&self.args.flag_author); + Address::from_str(d).unwrap_or_else(|_| die!("{}: Invalid address for --author. Must be 40 hex characters, without the 0x at the beginning.", self.args.flag_author)) } fn extra_data(&self) -> Bytes { - match self.args.flag_extra_data { + match self.args.flag_extradata.as_ref().or(self.args.flag_extra_data.as_ref()) { Some(ref x) if x.len() <= 32 => x.as_bytes().to_owned(), None => version_data(), Some(ref x) => { die!("{}: Extra data must be at most 32 characters.", x); } @@ -292,45 +300,40 @@ impl Configuration { } fn init_nodes(&self, spec: &Spec) -> Vec { - if self.args.flag_no_bootstrap { Vec::new() } else { - match self.args.arg_enode.len() { - 0 => spec.nodes().clone(), - _ => self.args.arg_enode.iter().map(|s| Self::normalize_enode(s).unwrap_or_else(||die!("{}: Invalid node address format given for a boot node.", s))).collect(), - } + let mut r = if self.args.flag_no_bootstrap { Vec::new() } else { spec.nodes().clone() }; + if let Some(ref x) = self.args.flag_bootnodes { + r.extend(x.split(",").map(|s| Self::normalize_enode(s).unwrap_or_else(||die!("{}: Invalid node address format given for a boot node.", s)))); } + r } #[cfg_attr(all(nightly, feature="dev"), allow(useless_format))] fn net_addresses(&self) -> (Option, Option) { - let mut listen_address = None; - let mut public_address = None; + let listen_address = Some(SocketAddr::new(IpAddr::from_str("127.0.0.1").unwrap(), self.args.flag_port)); + + let host = if self.args.flag_nat.starts_with("extip:") { + &self.args.flag_nat[6..] + } else { + "127.0.0.1" + }; + let public_address = Some(SocketAddr::new( + IpAddr::from_str(&host).unwrap_or_else(|_| die!("{}: Invalid host given with --net extip:", host)), + self.args.flag_port + )); - if let Some(ref a) = self.args.flag_address { - public_address = Some(SocketAddr::from_str(a.as_ref()).unwrap_or_else(|_| die!("{}: Invalid listen/public address given with --address", a))); - listen_address = public_address; - } - if listen_address.is_none() { - listen_address = Some(SocketAddr::from_str(self.args.flag_listen_address.as_ref()).unwrap_or_else(|_| die!("{}: Invalid listen/public address given with --listen-address", self.args.flag_listen_address))); - } - if let Some(ref a) = self.args.flag_public_address { - if public_address.is_some() { - die!("Conflicting flags provided: --address and --public-address"); - } - public_address = Some(SocketAddr::from_str(a.as_ref()).unwrap_or_else(|_| die!("{}: Invalid listen/public address given with --public-address", a))); - } (listen_address, public_address) } fn net_settings(&self, spec: &Spec) -> NetworkConfiguration { let mut ret = NetworkConfiguration::new(); - ret.nat_enabled = !self.args.flag_no_upnp; + ret.nat_enabled = self.args.flag_nat == "any" || self.args.flag_nat == "upnp"; ret.boot_nodes = self.init_nodes(spec); let (listen, public) = self.net_addresses(); ret.listen_address = listen; ret.public_address = public; ret.use_secret = self.args.flag_node_key.as_ref().map(|s| Secret::from_str(&s).unwrap_or_else(|_| s.sha3())); - ret.discovery_enabled = !self.args.flag_no_discovery; - ret.ideal_peers = self.args.flag_maxpeers as u32; + ret.discovery_enabled = !self.args.flag_no_discovery && !self.args.flag_nodiscover; + ret.ideal_peers = self.args.flag_maxpeers.unwrap_or(self.args.flag_peers) as u32; let mut net_path = PathBuf::from(&self.path()); net_path.push("network"); ret.config_path = Some(net_path.to_str().unwrap().to_owned()); From d9c462a3d3bd5a15b5c8dc6c8424b85b4179349c Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Fri, 11 Mar 2016 10:05:27 +0100 Subject: [PATCH 03/17] Use proper listen address. Tidyups. --- parity/main.rs | 18 +++++++----------- 1 file changed, 7 insertions(+), 11 deletions(-) diff --git a/parity/main.rs b/parity/main.rs index d6c3516d0..732e94f02 100644 --- a/parity/main.rs +++ b/parity/main.rs @@ -302,25 +302,21 @@ impl Configuration { fn init_nodes(&self, spec: &Spec) -> Vec { let mut r = if self.args.flag_no_bootstrap { Vec::new() } else { spec.nodes().clone() }; if let Some(ref x) = self.args.flag_bootnodes { - r.extend(x.split(",").map(|s| Self::normalize_enode(s).unwrap_or_else(||die!("{}: Invalid node address format given for a boot node.", s)))); + r.extend(x.split(",").map(|s| Self::normalize_enode(s).unwrap_or_else(|| die!("{}: Invalid node address format given for a boot node.", s)))); } r } #[cfg_attr(all(nightly, feature="dev"), allow(useless_format))] fn net_addresses(&self) -> (Option, Option) { - let listen_address = Some(SocketAddr::new(IpAddr::from_str("127.0.0.1").unwrap(), self.args.flag_port)); - - let host = if self.args.flag_nat.starts_with("extip:") { - &self.args.flag_nat[6..] + let listen_address = Some(SocketAddr::new(IpAddr::from_str("0.0.0.0").unwrap(), self.args.flag_port)); + let public_address = if self.args.flag_nat.starts_with("extip:") { + let host = &self.args.flag_nat[6..]; + let host = IpAddr::from_str(host).unwrap_or_else(|_| die!("Invalid host given with `--nat extip:{}`", host)); + Some(SocketAddr::new(host, self.args.flag_port)) } else { - "127.0.0.1" + listen_address.clone() }; - let public_address = Some(SocketAddr::new( - IpAddr::from_str(&host).unwrap_or_else(|_| die!("{}: Invalid host given with --net extip:", host)), - self.args.flag_port - )); - (listen_address, public_address) } From 190630cc6bd3ed4d076f6b6490a431ee4b5180ec Mon Sep 17 00:00:00 2001 From: debris Date: Fri, 11 Mar 2016 12:31:45 +0100 Subject: [PATCH 04/17] separated transaction_request to its own submodule, added basic tests for it --- rpc/src/v1/impls/eth.rs | 3 +- rpc/src/v1/types/mod.rs.in | 3 +- rpc/src/v1/types/transaction.rs | 32 +------- rpc/src/v1/types/transaction_request.rs | 101 ++++++++++++++++++++++++ 4 files changed, 106 insertions(+), 33 deletions(-) create mode 100644 rpc/src/v1/types/transaction_request.rs diff --git a/rpc/src/v1/impls/eth.rs b/rpc/src/v1/impls/eth.rs index 102d0da61..38e363624 100644 --- a/rpc/src/v1/impls/eth.rs +++ b/rpc/src/v1/impls/eth.rs @@ -27,6 +27,7 @@ use ethcore::block::{IsBlock}; use ethcore::views::*; use ethcore::ethereum::Ethash; use ethcore::ethereum::denominations::shannon; +use ethcore::transaction::Transaction as EthTransaction; use v1::traits::{Eth, EthFilter}; use v1::types::{Block, BlockTransactions, BlockNumber, Bytes, SyncStatus, SyncInfo, Transaction, TransactionRequest, OptionalValue, Index, Filter, Log}; use v1::helpers::{PollFilter, PollManager}; @@ -274,7 +275,7 @@ impl Eth for EthClient where C: BlockChainClient + 'static, S: match accounts.account_secret(&transaction_request.from) { Ok(secret) => { let sync = take_weak!(self.sync); - let (transaction, _) = transaction_request.to_eth(); + let transaction: EthTransaction = transaction_request.into(); let signed_transaction = transaction.sign(&secret); let hash = signed_transaction.hash(); sync.insert_transaction(signed_transaction); diff --git a/rpc/src/v1/types/mod.rs.in b/rpc/src/v1/types/mod.rs.in index 2b2390ecb..ebc3bc0ff 100644 --- a/rpc/src/v1/types/mod.rs.in +++ b/rpc/src/v1/types/mod.rs.in @@ -23,6 +23,7 @@ mod log; mod optionals; mod sync; mod transaction; +mod transaction_request; pub use self::block::{Block, BlockTransactions}; pub use self::block_number::BlockNumber; @@ -33,5 +34,5 @@ pub use self::log::Log; pub use self::optionals::OptionalValue; pub use self::sync::{SyncStatus, SyncInfo}; pub use self::transaction::Transaction; -pub use self::transaction::TransactionRequest; +pub use self::transaction_request::TransactionRequest; diff --git a/rpc/src/v1/types/transaction.rs b/rpc/src/v1/types/transaction.rs index 17b42cfcf..0518a58ea 100644 --- a/rpc/src/v1/types/transaction.rs +++ b/rpc/src/v1/types/transaction.rs @@ -17,8 +17,7 @@ use util::numbers::*; use ethcore::transaction::{LocalizedTransaction, Action}; use v1::types::{Bytes, OptionalValue}; -use serde::{Deserializer, Error}; -use ethcore; +use serde::Error; #[derive(Debug, Default, Serialize)] pub struct Transaction { @@ -39,35 +38,6 @@ pub struct Transaction { pub input: Bytes } -#[derive(Debug, Default, Serialize, Deserialize)] -pub struct TransactionRequest { - pub from: Address, - pub to: Option
, - #[serde(rename="gasPrice")] - pub gas_price: Option, - pub gas: Option, - pub value: Option, - pub data: Bytes, - pub nonce: Option, -} - -impl TransactionRequest { - /// maps transaction request to the transaction that can be signed and inserted - pub fn to_eth(self) -> (ethcore::transaction::Transaction, Address) { - (ethcore::transaction::Transaction { - nonce: self.nonce.unwrap_or(U256::zero()), - action: match self.to { - None => ethcore::transaction::Action::Create, - Some(addr) => ethcore::transaction::Action::Call(addr) - }, - gas: self.gas.unwrap_or(U256::zero()), - gas_price: self.gas_price.unwrap_or(U256::zero()), - value: self.value.unwrap_or(U256::zero()), - data: self.data.to_vec() - }, self.from) - } -} - impl From for Transaction { fn from(t: LocalizedTransaction) -> Transaction { Transaction { diff --git a/rpc/src/v1/types/transaction_request.rs b/rpc/src/v1/types/transaction_request.rs new file mode 100644 index 000000000..a61b11c25 --- /dev/null +++ b/rpc/src/v1/types/transaction_request.rs @@ -0,0 +1,101 @@ +// Copyright 2015, 2016 Ethcore (UK) Ltd. +// This file is part of Parity. + +// Parity is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Parity is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Parity. If not, see . + +use util::hash::Address; +use util::numbers::{Uint, U256}; +use ethcore::transaction::{Action, Transaction}; +use v1::types::Bytes; + +#[derive(Debug, Default, Deserialize)] +pub struct TransactionRequest { + pub from: Address, + pub to: Option
, + #[serde(rename="gasPrice")] + pub gas_price: Option, + pub gas: Option, + pub value: Option, + pub data: Bytes, + pub nonce: Option, +} + +impl Into for TransactionRequest { + fn into(self) -> Transaction { + Transaction { + nonce: self.nonce.unwrap_or(U256::zero()), + action: match self.to { + None => Action::Create, + Some(addr) => Action::Call(addr) + }, + gas: self.gas.unwrap_or(U256::zero()), + gas_price: self.gas_price.unwrap_or(U256::zero()), + value: self.value.unwrap_or(U256::zero()), + data: self.data.to_vec() + } + } +} + +#[cfg(test)] +mod tests { + use util::numbers::{Uint, U256}; + use util::hash::Address; + use ethcore::transaction::{Transaction, Action}; + use v1::types::Bytes; + use super::*; + + #[test] + fn transaction_request_into_transaction() { + let tr = TransactionRequest { + from: Address::default(), + to: Some(Address::from(10)), + gas_price: Some(U256::from(20)), + gas: Some(U256::from(10_000)), + value: Some(U256::from(1)), + data: Bytes::new(vec![10, 20]), + nonce: Some(U256::from(12)), + }; + + assert_eq!(Transaction { + nonce: U256::from(12), + action: Action::Call(Address::from(10)), + gas: U256::from(10_000), + gas_price: U256::from(20), + value: U256::from(1), + data: vec![10, 20], + }, tr.into()); + } + + #[test] + fn empty_transaction_request_into_transaction() { + let tr = TransactionRequest { + from: Address::default(), + to: None, + gas_price: None, + gas: None, + value: None, + data: Bytes::new(vec![]), + nonce: None, + }; + + assert_eq!(Transaction { + nonce: U256::zero(), + action: Action::Create, + gas: U256::zero(), + gas_price: U256::zero(), + value: U256::zero(), + data: vec![], + }, tr.into()); + } +} From 756f96413045e63a3dccf801f992c46ed22f7dfb Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Fri, 11 Mar 2016 12:54:48 +0100 Subject: [PATCH 05/17] JournalDB -> Box, and it's a trait. --- ethcore/src/block.rs | 16 ++--- ethcore/src/client.rs | 12 ++-- ethcore/src/state.rs | 26 +++---- ethcore/src/tests/helpers.rs | 8 +-- util/src/hashdb.rs | 15 ++++- util/src/journaldb.rs | 127 +++++++++++++++++++---------------- util/src/overlaydb.rs | 1 + 7 files changed, 116 insertions(+), 89 deletions(-) diff --git a/ethcore/src/block.rs b/ethcore/src/block.rs index b3894db94..ea9e91781 100644 --- a/ethcore/src/block.rs +++ b/ethcore/src/block.rs @@ -171,7 +171,7 @@ pub struct SealedBlock { impl<'x> OpenBlock<'x> { /// Create a new OpenBlock ready for transaction pushing. - pub fn new(engine: &'x Engine, db: JournalDB, parent: &Header, last_hashes: LastHashes, author: Address, extra_data: Bytes) -> Self { + pub fn new(engine: &'x Engine, db: Box, parent: &Header, last_hashes: LastHashes, author: Address, extra_data: Bytes) -> Self { let mut r = OpenBlock { block: ExecutedBlock::new(State::from_existing(db, parent.state_root().clone(), engine.account_start_nonce())), engine: engine, @@ -317,7 +317,7 @@ impl ClosedBlock { } /// Drop this object and return the underlieing database. - pub fn drain(self) -> JournalDB { self.block.state.drop().1 } + pub fn drain(self) -> Box { self.block.state.drop().1 } } impl SealedBlock { @@ -331,7 +331,7 @@ impl SealedBlock { } /// Drop this object and return the underlieing database. - pub fn drain(self) -> JournalDB { self.block.state.drop().1 } + pub fn drain(self) -> Box { self.block.state.drop().1 } } impl IsBlock for SealedBlock { @@ -339,10 +339,10 @@ impl IsBlock for SealedBlock { } /// Enact the block given by block header, transactions and uncles -pub fn enact(header: &Header, transactions: &[SignedTransaction], uncles: &[Header], engine: &Engine, db: JournalDB, parent: &Header, last_hashes: LastHashes) -> Result { +pub fn enact(header: &Header, transactions: &[SignedTransaction], uncles: &[Header], engine: &Engine, db: Box, parent: &Header, last_hashes: LastHashes) -> Result { { if ::log::max_log_level() >= ::log::LogLevel::Trace { - let s = State::from_existing(db.clone(), parent.state_root().clone(), engine.account_start_nonce()); + let s = State::from_existing(db.spawn(), parent.state_root().clone(), engine.account_start_nonce()); trace!("enact(): root={}, author={}, author_balance={}\n", s.root(), header.author(), s.balance(&header.author())); } } @@ -357,20 +357,20 @@ pub fn enact(header: &Header, transactions: &[SignedTransaction], uncles: &[Head } /// Enact the block given by `block_bytes` using `engine` on the database `db` with given `parent` block header -pub fn enact_bytes(block_bytes: &[u8], engine: &Engine, db: JournalDB, parent: &Header, last_hashes: LastHashes) -> Result { +pub fn enact_bytes(block_bytes: &[u8], engine: &Engine, db: Box, parent: &Header, last_hashes: LastHashes) -> Result { let block = BlockView::new(block_bytes); let header = block.header(); enact(&header, &block.transactions(), &block.uncles(), engine, db, parent, last_hashes) } /// Enact the block given by `block_bytes` using `engine` on the database `db` with given `parent` block header -pub fn enact_verified(block: &PreverifiedBlock, engine: &Engine, db: JournalDB, parent: &Header, last_hashes: LastHashes) -> Result { +pub fn enact_verified(block: &PreverifiedBlock, engine: &Engine, db: Box, parent: &Header, last_hashes: LastHashes) -> Result { let view = BlockView::new(&block.bytes); enact(&block.header, &block.transactions, &view.uncles(), engine, db, parent, last_hashes) } /// Enact the block given by `block_bytes` using `engine` on the database `db` with given `parent` block header. Seal the block aferwards -pub fn enact_and_seal(block_bytes: &[u8], engine: &Engine, db: JournalDB, parent: &Header, last_hashes: LastHashes) -> Result { +pub fn enact_and_seal(block_bytes: &[u8], engine: &Engine, db: Box, parent: &Header, last_hashes: LastHashes) -> Result { let header = BlockView::new(block_bytes).header_view(); Ok(try!(try!(enact_bytes(block_bytes, engine, db, parent, last_hashes)).seal(engine, header.seal()))) } diff --git a/ethcore/src/client.rs b/ethcore/src/client.rs index b342cef15..7ccf094d2 100644 --- a/ethcore/src/client.rs +++ b/ethcore/src/client.rs @@ -214,7 +214,7 @@ impl ClientReport { pub struct Client where V: Verifier { chain: Arc>, engine: Arc>, - state_db: Mutex, + state_db: Mutex>, block_queue: RwLock, report: RwLock, import_lock: Mutex<()>, @@ -253,8 +253,8 @@ impl Client where V: Verifier { state_path.push("state"); let engine = Arc::new(try!(spec.to_engine())); - let mut state_db = JournalDB::from_prefs(state_path.to_str().unwrap(), config.prefer_journal); - if state_db.is_empty() && engine.spec().ensure_db_good(&mut state_db) { + let mut state_db = Box::new(OptionOneDB::from_prefs(state_path.to_str().unwrap(), config.prefer_journal)); + if state_db.is_empty() && engine.spec().ensure_db_good(state_db.deref_mut()) { state_db.commit(0, &engine.spec().genesis_header().hash(), None).expect("Error commiting genesis state to state DB"); } @@ -336,7 +336,7 @@ impl Client where V: Verifier { // Enact Verified Block let parent = chain_has_parent.unwrap(); let last_hashes = self.build_last_hashes(header.parent_hash.clone()); - let db = self.state_db.lock().unwrap().clone(); + let db = self.state_db.lock().unwrap().spawn(); let enact_result = enact_verified(&block, engine, db, &parent, last_hashes); if let Err(e) = enact_result { @@ -438,7 +438,7 @@ impl Client where V: Verifier { /// Get a copy of the best block's state. pub fn state(&self) -> State { - State::from_existing(self.state_db.lock().unwrap().clone(), HeaderView::new(&self.best_block_header()).state_root(), self.engine.account_start_nonce()) + State::from_existing(self.state_db.lock().unwrap().spawn(), HeaderView::new(&self.best_block_header()).state_root(), self.engine.account_start_nonce()) } /// Get info on the cache. @@ -507,7 +507,7 @@ impl Client where V: Verifier { let h = self.chain.read().unwrap().best_block_hash(); let mut b = OpenBlock::new( self.engine.deref().deref(), - self.state_db.lock().unwrap().clone(), + self.state_db.lock().unwrap().spawn(), match self.chain.read().unwrap().block_header(&h) { Some(ref x) => x, None => {return;} }, self.build_last_hashes(h.clone()), self.author(), diff --git a/ethcore/src/state.rs b/ethcore/src/state.rs index 7c1064abf..9a3d1791d 100644 --- a/ethcore/src/state.rs +++ b/ethcore/src/state.rs @@ -31,7 +31,7 @@ pub type ApplyResult = Result; /// Representation of the entire state of all accounts in the system. pub struct State { - db: JournalDB, + db: Box, root: H256, cache: RefCell>>, snapshots: RefCell>>>>, @@ -41,11 +41,11 @@ pub struct State { impl State { /// Creates new state with empty state root #[cfg(test)] - pub fn new(mut db: JournalDB, account_start_nonce: U256) -> State { + pub fn new(mut db: Box, account_start_nonce: U256) -> State { let mut root = H256::new(); { // init trie and reset root too null - let _ = SecTrieDBMut::new(&mut db, &mut root); + let _ = SecTrieDBMut::new(db.deref_mut(), &mut root); } State { @@ -58,10 +58,10 @@ impl State { } /// Creates new state with existing state root - pub fn from_existing(db: JournalDB, root: H256, account_start_nonce: U256) -> State { + pub fn from_existing(db: Box, root: H256, account_start_nonce: U256) -> State { { // trie should panic! if root does not exist - let _ = SecTrieDB::new(&db, &root); + let _ = SecTrieDB::new(db.as_hashdb(), &root); } State { @@ -126,7 +126,7 @@ impl State { } /// Destroy the current object and return root and database. - pub fn drop(self) -> (H256, JournalDB) { + pub fn drop(self) -> (H256, Box) { (self.root, self.db) } @@ -148,7 +148,7 @@ impl State { /// Determine whether an account exists. pub fn exists(&self, a: &Address) -> bool { - self.cache.borrow().get(&a).unwrap_or(&None).is_some() || SecTrieDB::new(&self.db, &self.root).contains(&a) + self.cache.borrow().get(&a).unwrap_or(&None).is_some() || SecTrieDB::new(self.db.as_hashdb(), &self.root).contains(&a) } /// Get the balance of account `a`. @@ -163,7 +163,7 @@ impl State { /// Mutate storage of account `address` so that it is `value` for `key`. pub fn storage_at(&self, address: &Address, key: &H256) -> H256 { - self.get(address, false).as_ref().map_or(H256::new(), |a|a.storage_at(&AccountDB::new(&self.db, address), key)) + self.get(address, false).as_ref().map_or(H256::new(), |a|a.storage_at(&AccountDB::new(self.db.as_hashdb(), address), key)) } /// Mutate storage of account `a` so that it is `value` for `key`. @@ -253,7 +253,7 @@ impl State { /// Commits our cached account changes into the trie. pub fn commit(&mut self) { assert!(self.snapshots.borrow().is_empty()); - Self::commit_into(&mut self.db, &mut self.root, self.cache.borrow_mut().deref_mut()); + Self::commit_into(self.db.as_hashdb_mut(), &mut self.root, self.cache.borrow_mut().deref_mut()); } #[cfg(test)] @@ -285,11 +285,11 @@ impl State { fn get<'a>(&'a self, a: &Address, require_code: bool) -> &'a Option { let have_key = self.cache.borrow().contains_key(a); if !have_key { - self.insert_cache(a, SecTrieDB::new(&self.db, &self.root).get(&a).map(Account::from_rlp)) + self.insert_cache(a, SecTrieDB::new(self.db.as_hashdb(), &self.root).get(&a).map(Account::from_rlp)) } if require_code { if let Some(ref mut account) = self.cache.borrow_mut().get_mut(a).unwrap().as_mut() { - account.cache_code(&AccountDB::new(&self.db, a)); + account.cache_code(&AccountDB::new(self.db.as_hashdb(), a)); } } unsafe { ::std::mem::transmute(self.cache.borrow().get(a).unwrap()) } @@ -305,7 +305,7 @@ impl State { fn require_or_from<'a, F: FnOnce() -> Account, G: FnOnce(&mut Account)>(&self, a: &Address, require_code: bool, default: F, not_default: G) -> &'a mut Account { let have_key = self.cache.borrow().contains_key(a); if !have_key { - self.insert_cache(a, SecTrieDB::new(&self.db, &self.root).get(&a).map(Account::from_rlp)) + self.insert_cache(a, SecTrieDB::new(self.db.as_hashdb(), &self.root).get(&a).map(Account::from_rlp)) } else { self.note_cache(a); } @@ -318,7 +318,7 @@ impl State { unsafe { ::std::mem::transmute(self.cache.borrow_mut().get_mut(a).unwrap().as_mut().map(|account| { if require_code { - account.cache_code(&AccountDB::new(&self.db, a)); + account.cache_code(&AccountDB::new(self.db.as_hashdb(), a)); } account }).unwrap()) } diff --git a/ethcore/src/tests/helpers.rs b/ethcore/src/tests/helpers.rs index bb9a44614..7b99f68d5 100644 --- a/ethcore/src/tests/helpers.rs +++ b/ethcore/src/tests/helpers.rs @@ -250,9 +250,9 @@ pub fn generate_dummy_empty_blockchain() -> GuardedTempResult { } } -pub fn get_temp_journal_db() -> GuardedTempResult { +pub fn get_temp_journal_db() -> GuardedTempResult> { let temp = RandomTempPath::new(); - let journal_db = JournalDB::new(temp.as_str()); + let journal_db = Box::new(OptionOneDB::new(temp.as_str())); GuardedTempResult { _temp: temp, result: Some(journal_db) @@ -268,8 +268,8 @@ pub fn get_temp_state() -> GuardedTempResult { } } -pub fn get_temp_journal_db_in(path: &Path) -> JournalDB { - JournalDB::new(path.to_str().unwrap()) +pub fn get_temp_journal_db_in(path: &Path) -> Box { + Box::new(OptionOneDB::new(path.to_str().unwrap())) } pub fn get_temp_state_in(path: &Path) -> State { diff --git a/util/src/hashdb.rs b/util/src/hashdb.rs index 4d8cbaba1..e622c4b99 100644 --- a/util/src/hashdb.rs +++ b/util/src/hashdb.rs @@ -20,7 +20,7 @@ use bytes::*; use std::collections::HashMap; /// Trait modelling datastore keyed by a 32-byte Keccak hash. -pub trait HashDB { +pub trait HashDB : AsHashDB { /// Get the keys in the database together with number of underlying references. fn keys(&self) -> HashMap; @@ -111,3 +111,16 @@ pub trait HashDB { /// ``` fn remove(&mut self, key: &H256) { self.kill(key) } } + +/// Upcast trait. +pub trait AsHashDB { + /// Perform upcast to HashDB for anything that derives from HashDB. + fn as_hashdb(&self) -> &HashDB; + /// Perform mutable upcast to HashDB for anything that derives from HashDB. + fn as_hashdb_mut(&mut self) -> &mut HashDB; +} + +impl AsHashDB for T { + fn as_hashdb(&self) -> &HashDB { self } + fn as_hashdb_mut(&mut self) -> &mut HashDB { self } +} diff --git a/util/src/journaldb.rs b/util/src/journaldb.rs index 35ad83fa0..23fd08011 100644 --- a/util/src/journaldb.rs +++ b/util/src/journaldb.rs @@ -24,6 +24,22 @@ use kvdb::{Database, DBTransaction, DatabaseConfig}; #[cfg(test)] use std::env; +/// A HashDB which can manage a short-term journal potentially containing many forks of mutually +/// exclusive actions. +pub trait JournalDB : HashDB { + /// Return a copy of ourself, in a box. + fn spawn(&self) -> Box; + + /// Returns heap memory size used + fn mem_used(&self) -> usize; + + /// Check if this database has any commits + fn is_empty(&self) -> bool; + + /// Commit all recent insert operations. + fn commit(&mut self, now: u64, id: &H256, end: Option<(u64, H256)>) -> Result; +} + /// Implementation of the HashDB trait for a disk-backed database with a memory overlay /// and latent-removal semantics. /// @@ -31,22 +47,12 @@ use std::env; /// write operations out to disk. Unlike OverlayDB, `remove()` operations do not take effect /// immediately. Rather some age (based on a linear but arbitrary metric) must pass before /// the removals actually take effect. -pub struct JournalDB { +pub struct OptionOneDB { overlay: MemoryDB, backing: Arc, counters: Option>>>, } -impl Clone for JournalDB { - fn clone(&self) -> JournalDB { - JournalDB { - overlay: MemoryDB::new(), - backing: self.backing.clone(), - counters: self.counters.clone(), - } - } -} - // all keys must be at least 12 bytes const LATEST_ERA_KEY : [u8; 12] = [ b'l', b'a', b's', b't', 0, 0, 0, 0, 0, 0, 0, 0 ]; const VERSION_KEY : [u8; 12] = [ b'j', b'v', b'e', b'r', 0, 0, 0, 0, 0, 0, 0, 0 ]; @@ -56,14 +62,14 @@ const DB_VERSION_NO_JOURNAL : u32 = 3 + 256; const PADDING : [u8; 10] = [ 0u8; 10 ]; -impl JournalDB { +impl OptionOneDB { /// Create a new instance from file - pub fn new(path: &str) -> JournalDB { + pub fn new(path: &str) -> OptionOneDB { Self::from_prefs(path, true) } /// Create a new instance from file - pub fn from_prefs(path: &str, prefer_journal: bool) -> JournalDB { + pub fn from_prefs(path: &str, prefer_journal: bool) -> OptionOneDB { let opts = DatabaseConfig { prefix_size: Some(12) //use 12 bytes as prefix, this must match account_db prefix }; @@ -83,11 +89,11 @@ impl JournalDB { } let counters = if with_journal { - Some(Arc::new(RwLock::new(JournalDB::read_counters(&backing)))) + Some(Arc::new(RwLock::new(OptionOneDB::read_counters(&backing)))) } else { None }; - JournalDB { + OptionOneDB { overlay: MemoryDB::new(), backing: Arc::new(backing), counters: counters, @@ -96,27 +102,12 @@ impl JournalDB { /// Create a new instance with an anonymous temporary database. #[cfg(test)] - pub fn new_temp() -> JournalDB { + fn new_temp() -> OptionOneDB { let mut dir = env::temp_dir(); dir.push(H32::random().hex()); Self::new(dir.to_str().unwrap()) } - /// Check if this database has any commits - pub fn is_empty(&self) -> bool { - self.backing.get(&LATEST_ERA_KEY).expect("Low level database error").is_none() - } - - /// Commit all recent insert operations. - pub fn commit(&mut self, now: u64, id: &H256, end: Option<(u64, H256)>) -> Result { - let have_counters = self.counters.is_some(); - if have_counters { - self.commit_with_counters(now, id, end) - } else { - self.commit_without_counters() - } - } - /// Drain the overlay and place it into a batch for the DB. fn batch_overlay_insertions(overlay: &mut MemoryDB, batch: &DBTransaction) -> usize { let mut inserts = 0usize; @@ -339,11 +330,11 @@ impl JournalDB { try!(batch.delete(&last)); index += 1; } - trace!("JournalDB: delete journal for time #{}.{}, (canon was {})", end_era, index, canon_id); + trace!("OptionOneDB: delete journal for time #{}.{}, (canon was {})", end_era, index, canon_id); } try!(self.backing.write(batch)); -// trace!("JournalDB::commit() deleted {} nodes", deletes); +// trace!("OptionOneDB::commit() deleted {} nodes", deletes); Ok(0) } @@ -379,17 +370,9 @@ impl JournalDB { trace!("Recovered {} counters", counters.len()); counters } +} - /// Returns heap memory size used - pub fn mem_used(&self) -> usize { - self.overlay.mem_used() + match self.counters { - Some(ref c) => c.read().unwrap().heap_size_of_children(), - None => 0 - } - } - } - -impl HashDB for JournalDB { +impl HashDB for OptionOneDB { fn keys(&self) -> HashMap { let mut ret: HashMap = HashMap::new(); for (key, _) in self.backing.iter() { @@ -434,6 +417,36 @@ impl HashDB for JournalDB { } } +impl JournalDB for OptionOneDB { + fn spawn(&self) -> Box { + Box::new(OptionOneDB { + overlay: MemoryDB::new(), + backing: self.backing.clone(), + counters: self.counters.clone(), + }) + } + + fn mem_used(&self) -> usize { + self.overlay.mem_used() + match self.counters { + Some(ref c) => c.read().unwrap().heap_size_of_children(), + None => 0 + } + } + + fn is_empty(&self) -> bool { + self.backing.get(&LATEST_ERA_KEY).expect("Low level database error").is_none() + } + + fn commit(&mut self, now: u64, id: &H256, end: Option<(u64, H256)>) -> Result { + let have_counters = self.counters.is_some(); + if have_counters { + self.commit_with_counters(now, id, end) + } else { + self.commit_without_counters() + } + } +} + #[cfg(test)] mod tests { use common::*; @@ -443,7 +456,7 @@ mod tests { #[test] fn insert_same_in_fork() { // history is 1 - let mut jdb = JournalDB::new_temp(); + let mut jdb = OptionOneDB::new_temp(); let x = jdb.insert(b"X"); jdb.commit(1, &b"1".sha3(), None).unwrap(); @@ -465,7 +478,7 @@ mod tests { #[test] fn long_history() { // history is 3 - let mut jdb = JournalDB::new_temp(); + let mut jdb = OptionOneDB::new_temp(); let h = jdb.insert(b"foo"); jdb.commit(0, &b"0".sha3(), None).unwrap(); assert!(jdb.exists(&h)); @@ -483,7 +496,7 @@ mod tests { #[test] fn complex() { // history is 1 - let mut jdb = JournalDB::new_temp(); + let mut jdb = OptionOneDB::new_temp(); let foo = jdb.insert(b"foo"); let bar = jdb.insert(b"bar"); @@ -521,7 +534,7 @@ mod tests { #[test] fn fork() { // history is 1 - let mut jdb = JournalDB::new_temp(); + let mut jdb = OptionOneDB::new_temp(); let foo = jdb.insert(b"foo"); let bar = jdb.insert(b"bar"); @@ -549,7 +562,7 @@ mod tests { #[test] fn overwrite() { // history is 1 - let mut jdb = JournalDB::new_temp(); + let mut jdb = OptionOneDB::new_temp(); let foo = jdb.insert(b"foo"); jdb.commit(0, &b"0".sha3(), None).unwrap(); @@ -568,7 +581,7 @@ mod tests { #[test] fn fork_same_key() { // history is 1 - let mut jdb = JournalDB::new_temp(); + let mut jdb = OptionOneDB::new_temp(); jdb.commit(0, &b"0".sha3(), None).unwrap(); let foo = jdb.insert(b"foo"); @@ -590,7 +603,7 @@ mod tests { let bar = H256::random(); let foo = { - let mut jdb = JournalDB::new(dir.to_str().unwrap()); + let mut jdb = OptionOneDB::new(dir.to_str().unwrap()); // history is 1 let foo = jdb.insert(b"foo"); jdb.emplace(bar.clone(), b"bar".to_vec()); @@ -599,13 +612,13 @@ mod tests { }; { - let mut jdb = JournalDB::new(dir.to_str().unwrap()); + let mut jdb = OptionOneDB::new(dir.to_str().unwrap()); jdb.remove(&foo); jdb.commit(1, &b"1".sha3(), Some((0, b"0".sha3()))).unwrap(); } { - let mut jdb = JournalDB::new(dir.to_str().unwrap()); + let mut jdb = OptionOneDB::new(dir.to_str().unwrap()); assert!(jdb.exists(&foo)); assert!(jdb.exists(&bar)); jdb.commit(2, &b"2".sha3(), Some((1, b"1".sha3()))).unwrap(); @@ -619,7 +632,7 @@ mod tests { dir.push(H32::random().hex()); let foo = { - let mut jdb = JournalDB::new(dir.to_str().unwrap()); + let mut jdb = OptionOneDB::new(dir.to_str().unwrap()); // history is 1 let foo = jdb.insert(b"foo"); jdb.commit(0, &b"0".sha3(), None).unwrap(); @@ -633,7 +646,7 @@ mod tests { }; { - let mut jdb = JournalDB::new(dir.to_str().unwrap()); + let mut jdb = OptionOneDB::new(dir.to_str().unwrap()); jdb.remove(&foo); jdb.commit(3, &b"3".sha3(), Some((2, b"2".sha3()))).unwrap(); assert!(jdb.exists(&foo)); @@ -648,7 +661,7 @@ mod tests { let mut dir = ::std::env::temp_dir(); dir.push(H32::random().hex()); let (foo, bar, baz) = { - let mut jdb = JournalDB::new(dir.to_str().unwrap()); + let mut jdb = OptionOneDB::new(dir.to_str().unwrap()); // history is 1 let foo = jdb.insert(b"foo"); let bar = jdb.insert(b"bar"); @@ -663,7 +676,7 @@ mod tests { }; { - let mut jdb = JournalDB::new(dir.to_str().unwrap()); + let mut jdb = OptionOneDB::new(dir.to_str().unwrap()); jdb.commit(2, &b"2b".sha3(), Some((1, b"1b".sha3()))).unwrap(); assert!(jdb.exists(&foo)); assert!(!jdb.exists(&baz)); diff --git a/util/src/overlaydb.rs b/util/src/overlaydb.rs index 3c80f4148..f14677d05 100644 --- a/util/src/overlaydb.rs +++ b/util/src/overlaydb.rs @@ -36,6 +36,7 @@ use kvdb::{Database}; /// /// `lookup()` and `contains()` maintain normal behaviour - all `insert()` and `remove()` /// queries have an immediate effect in terms of these functions. +#[derive(Clone)] pub struct OverlayDB { overlay: MemoryDB, backing: Arc, From 99c5794929e0c14b524f949fcc16a5ecbda88fee Mon Sep 17 00:00:00 2001 From: Nikolay Volf Date: Fri, 11 Mar 2016 16:00:30 +0400 Subject: [PATCH 06/17] fix warning for transaction_queue.add usage --- sync/src/chain.rs | 4 ++-- sync/src/lib.rs | 3 ++- sync/src/transaction_queue.rs | 4 ++-- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/sync/src/chain.rs b/sync/src/chain.rs index 866838bec..d65685dfe 100644 --- a/sync/src/chain.rs +++ b/sync/src/chain.rs @@ -1306,11 +1306,11 @@ impl ChainSync { } /// Add transaction to the transaction queue - pub fn insert_transaction(&self, transaction: ethcore::transaction::SignedTransaction, fetch_nonce: &T) + pub fn insert_transaction(&self, transaction: ethcore::transaction::SignedTransaction, fetch_nonce: &T) -> Result<(), Error> where T: Fn(&Address) -> U256 { let mut queue = self.transaction_queue.lock().unwrap(); - queue.add(transaction, fetch_nonce); + queue.add(transaction, fetch_nonce) } } diff --git a/sync/src/lib.rs b/sync/src/lib.rs index 3b79e5614..a416c8829 100644 --- a/sync/src/lib.rs +++ b/sync/src/lib.rs @@ -146,7 +146,8 @@ impl SyncProvider for EthSync { let nonce_fn = |a: &Address| self.chain.state().nonce(a) + U256::one(); let sync = self.sync.write().unwrap(); - sync.insert_transaction(transaction, &nonce_fn); + sync.insert_transaction(transaction, &nonce_fn).unwrap_or_else( + |e| warn!(target: "sync", "Error inserting transaction to queue: {:?}", e)); } } diff --git a/sync/src/transaction_queue.rs b/sync/src/transaction_queue.rs index 243939a4c..618eb6a0b 100644 --- a/sync/src/transaction_queue.rs +++ b/sync/src/transaction_queue.rs @@ -684,8 +684,8 @@ mod test { let mut txq = TransactionQueue::new(); let (tx, tx2) = new_txs(U256::from(1)); - txq.add(tx.clone(), &prev_nonce); - txq.add(tx2.clone(), &prev_nonce); + txq.add(tx.clone(), &prev_nonce).unwrap(); + txq.add(tx2.clone(), &prev_nonce).unwrap(); assert_eq!(txq.status().future, 2); // when From d71c5d4c17f26484fb15511d155e02f7a27eab3b Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Fri, 11 Mar 2016 13:19:10 +0100 Subject: [PATCH 07/17] Place Sync/Send in trait. --- ethcore/src/block.rs | 14 +++++++------- ethcore/src/client/client.rs | 2 +- ethcore/src/state.rs | 8 ++++---- ethcore/src/tests/helpers.rs | 4 ++-- util/src/journaldb.rs | 6 +++--- 5 files changed, 17 insertions(+), 17 deletions(-) diff --git a/ethcore/src/block.rs b/ethcore/src/block.rs index ea9e91781..ab5086273 100644 --- a/ethcore/src/block.rs +++ b/ethcore/src/block.rs @@ -171,7 +171,7 @@ pub struct SealedBlock { impl<'x> OpenBlock<'x> { /// Create a new OpenBlock ready for transaction pushing. - pub fn new(engine: &'x Engine, db: Box, parent: &Header, last_hashes: LastHashes, author: Address, extra_data: Bytes) -> Self { + pub fn new(engine: &'x Engine, db: Box>, parent: &Header, last_hashes: LastHashes, author: Address, extra_data: Bytes) -> Self { let mut r = OpenBlock { block: ExecutedBlock::new(State::from_existing(db, parent.state_root().clone(), engine.account_start_nonce())), engine: engine, @@ -317,7 +317,7 @@ impl ClosedBlock { } /// Drop this object and return the underlieing database. - pub fn drain(self) -> Box { self.block.state.drop().1 } + pub fn drain(self) -> Box> { self.block.state.drop().1 } } impl SealedBlock { @@ -331,7 +331,7 @@ impl SealedBlock { } /// Drop this object and return the underlieing database. - pub fn drain(self) -> Box { self.block.state.drop().1 } + pub fn drain(self) -> Box> { self.block.state.drop().1 } } impl IsBlock for SealedBlock { @@ -339,7 +339,7 @@ impl IsBlock for SealedBlock { } /// Enact the block given by block header, transactions and uncles -pub fn enact(header: &Header, transactions: &[SignedTransaction], uncles: &[Header], engine: &Engine, db: Box, parent: &Header, last_hashes: LastHashes) -> Result { +pub fn enact(header: &Header, transactions: &[SignedTransaction], uncles: &[Header], engine: &Engine, db: Box>, parent: &Header, last_hashes: LastHashes) -> Result { { if ::log::max_log_level() >= ::log::LogLevel::Trace { let s = State::from_existing(db.spawn(), parent.state_root().clone(), engine.account_start_nonce()); @@ -357,20 +357,20 @@ pub fn enact(header: &Header, transactions: &[SignedTransaction], uncles: &[Head } /// Enact the block given by `block_bytes` using `engine` on the database `db` with given `parent` block header -pub fn enact_bytes(block_bytes: &[u8], engine: &Engine, db: Box, parent: &Header, last_hashes: LastHashes) -> Result { +pub fn enact_bytes(block_bytes: &[u8], engine: &Engine, db: Box>, parent: &Header, last_hashes: LastHashes) -> Result { let block = BlockView::new(block_bytes); let header = block.header(); enact(&header, &block.transactions(), &block.uncles(), engine, db, parent, last_hashes) } /// Enact the block given by `block_bytes` using `engine` on the database `db` with given `parent` block header -pub fn enact_verified(block: &PreverifiedBlock, engine: &Engine, db: Box, parent: &Header, last_hashes: LastHashes) -> Result { +pub fn enact_verified(block: &PreverifiedBlock, engine: &Engine, db: Box>, parent: &Header, last_hashes: LastHashes) -> Result { let view = BlockView::new(&block.bytes); enact(&block.header, &block.transactions, &view.uncles(), engine, db, parent, last_hashes) } /// Enact the block given by `block_bytes` using `engine` on the database `db` with given `parent` block header. Seal the block aferwards -pub fn enact_and_seal(block_bytes: &[u8], engine: &Engine, db: Box, parent: &Header, last_hashes: LastHashes) -> Result { +pub fn enact_and_seal(block_bytes: &[u8], engine: &Engine, db: Box>, parent: &Header, last_hashes: LastHashes) -> Result { let header = BlockView::new(block_bytes).header_view(); Ok(try!(try!(enact_bytes(block_bytes, engine, db, parent, last_hashes)).seal(engine, header.seal()))) } diff --git a/ethcore/src/client/client.rs b/ethcore/src/client/client.rs index e2fe7bb90..ea0919bcc 100644 --- a/ethcore/src/client/client.rs +++ b/ethcore/src/client/client.rs @@ -101,7 +101,7 @@ impl ClientReport { pub struct Client where V: Verifier { chain: Arc, engine: Arc>, - state_db: Mutex>, + state_db: Mutex>>, block_queue: BlockQueue, report: RwLock, import_lock: Mutex<()>, diff --git a/ethcore/src/state.rs b/ethcore/src/state.rs index 9a3d1791d..222b88643 100644 --- a/ethcore/src/state.rs +++ b/ethcore/src/state.rs @@ -31,7 +31,7 @@ pub type ApplyResult = Result; /// Representation of the entire state of all accounts in the system. pub struct State { - db: Box, + db: Box>, root: H256, cache: RefCell>>, snapshots: RefCell>>>>, @@ -41,7 +41,7 @@ pub struct State { impl State { /// Creates new state with empty state root #[cfg(test)] - pub fn new(mut db: Box, account_start_nonce: U256) -> State { + pub fn new(mut db: Box>, account_start_nonce: U256) -> State { let mut root = H256::new(); { // init trie and reset root too null @@ -58,7 +58,7 @@ impl State { } /// Creates new state with existing state root - pub fn from_existing(db: Box, root: H256, account_start_nonce: U256) -> State { + pub fn from_existing(db: Box>, root: H256, account_start_nonce: U256) -> State { { // trie should panic! if root does not exist let _ = SecTrieDB::new(db.as_hashdb(), &root); @@ -126,7 +126,7 @@ impl State { } /// Destroy the current object and return root and database. - pub fn drop(self) -> (H256, Box) { + pub fn drop(self) -> (H256, Box>) { (self.root, self.db) } diff --git a/ethcore/src/tests/helpers.rs b/ethcore/src/tests/helpers.rs index 7b99f68d5..85e311e82 100644 --- a/ethcore/src/tests/helpers.rs +++ b/ethcore/src/tests/helpers.rs @@ -250,7 +250,7 @@ pub fn generate_dummy_empty_blockchain() -> GuardedTempResult { } } -pub fn get_temp_journal_db() -> GuardedTempResult> { +pub fn get_temp_journal_db() -> GuardedTempResult>> { let temp = RandomTempPath::new(); let journal_db = Box::new(OptionOneDB::new(temp.as_str())); GuardedTempResult { @@ -268,7 +268,7 @@ pub fn get_temp_state() -> GuardedTempResult { } } -pub fn get_temp_journal_db_in(path: &Path) -> Box { +pub fn get_temp_journal_db_in(path: &Path) -> Box> { Box::new(OptionOneDB::new(path.to_str().unwrap())) } diff --git a/util/src/journaldb.rs b/util/src/journaldb.rs index 23fd08011..c6243ace0 100644 --- a/util/src/journaldb.rs +++ b/util/src/journaldb.rs @@ -26,9 +26,9 @@ use std::env; /// A HashDB which can manage a short-term journal potentially containing many forks of mutually /// exclusive actions. -pub trait JournalDB : HashDB { +pub trait JournalDB : HashDB + Sync + Send { /// Return a copy of ourself, in a box. - fn spawn(&self) -> Box; + fn spawn(&self) -> Box>; /// Returns heap memory size used fn mem_used(&self) -> usize; @@ -418,7 +418,7 @@ impl HashDB for OptionOneDB { } impl JournalDB for OptionOneDB { - fn spawn(&self) -> Box { + fn spawn(&self) -> Box> { Box::new(OptionOneDB { overlay: MemoryDB::new(), backing: self.backing.clone(), From 2a856a13f04271fbe78576858aedca3b2c4b659c Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Fri, 11 Mar 2016 13:21:53 +0100 Subject: [PATCH 08/17] Obvious typo fix. --- ethcore/src/block.rs | 14 +++++++------- ethcore/src/client/client.rs | 2 +- ethcore/src/state.rs | 8 ++++---- ethcore/src/tests/helpers.rs | 4 ++-- util/src/journaldb.rs | 4 ++-- 5 files changed, 16 insertions(+), 16 deletions(-) diff --git a/ethcore/src/block.rs b/ethcore/src/block.rs index ab5086273..f3a4feaf0 100644 --- a/ethcore/src/block.rs +++ b/ethcore/src/block.rs @@ -171,7 +171,7 @@ pub struct SealedBlock { impl<'x> OpenBlock<'x> { /// Create a new OpenBlock ready for transaction pushing. - pub fn new(engine: &'x Engine, db: Box>, parent: &Header, last_hashes: LastHashes, author: Address, extra_data: Bytes) -> Self { + pub fn new(engine: &'x Engine, db: Box, parent: &Header, last_hashes: LastHashes, author: Address, extra_data: Bytes) -> Self { let mut r = OpenBlock { block: ExecutedBlock::new(State::from_existing(db, parent.state_root().clone(), engine.account_start_nonce())), engine: engine, @@ -317,7 +317,7 @@ impl ClosedBlock { } /// Drop this object and return the underlieing database. - pub fn drain(self) -> Box> { self.block.state.drop().1 } + pub fn drain(self) -> Box { self.block.state.drop().1 } } impl SealedBlock { @@ -331,7 +331,7 @@ impl SealedBlock { } /// Drop this object and return the underlieing database. - pub fn drain(self) -> Box> { self.block.state.drop().1 } + pub fn drain(self) -> Box { self.block.state.drop().1 } } impl IsBlock for SealedBlock { @@ -339,7 +339,7 @@ impl IsBlock for SealedBlock { } /// Enact the block given by block header, transactions and uncles -pub fn enact(header: &Header, transactions: &[SignedTransaction], uncles: &[Header], engine: &Engine, db: Box>, parent: &Header, last_hashes: LastHashes) -> Result { +pub fn enact(header: &Header, transactions: &[SignedTransaction], uncles: &[Header], engine: &Engine, db: Box, parent: &Header, last_hashes: LastHashes) -> Result { { if ::log::max_log_level() >= ::log::LogLevel::Trace { let s = State::from_existing(db.spawn(), parent.state_root().clone(), engine.account_start_nonce()); @@ -357,20 +357,20 @@ pub fn enact(header: &Header, transactions: &[SignedTransaction], uncles: &[Head } /// Enact the block given by `block_bytes` using `engine` on the database `db` with given `parent` block header -pub fn enact_bytes(block_bytes: &[u8], engine: &Engine, db: Box>, parent: &Header, last_hashes: LastHashes) -> Result { +pub fn enact_bytes(block_bytes: &[u8], engine: &Engine, db: Box, parent: &Header, last_hashes: LastHashes) -> Result { let block = BlockView::new(block_bytes); let header = block.header(); enact(&header, &block.transactions(), &block.uncles(), engine, db, parent, last_hashes) } /// Enact the block given by `block_bytes` using `engine` on the database `db` with given `parent` block header -pub fn enact_verified(block: &PreverifiedBlock, engine: &Engine, db: Box>, parent: &Header, last_hashes: LastHashes) -> Result { +pub fn enact_verified(block: &PreverifiedBlock, engine: &Engine, db: Box, parent: &Header, last_hashes: LastHashes) -> Result { let view = BlockView::new(&block.bytes); enact(&block.header, &block.transactions, &view.uncles(), engine, db, parent, last_hashes) } /// Enact the block given by `block_bytes` using `engine` on the database `db` with given `parent` block header. Seal the block aferwards -pub fn enact_and_seal(block_bytes: &[u8], engine: &Engine, db: Box>, parent: &Header, last_hashes: LastHashes) -> Result { +pub fn enact_and_seal(block_bytes: &[u8], engine: &Engine, db: Box, parent: &Header, last_hashes: LastHashes) -> Result { let header = BlockView::new(block_bytes).header_view(); Ok(try!(try!(enact_bytes(block_bytes, engine, db, parent, last_hashes)).seal(engine, header.seal()))) } diff --git a/ethcore/src/client/client.rs b/ethcore/src/client/client.rs index ea0919bcc..3c8a28380 100644 --- a/ethcore/src/client/client.rs +++ b/ethcore/src/client/client.rs @@ -101,7 +101,7 @@ impl ClientReport { pub struct Client where V: Verifier { chain: Arc, engine: Arc>, - state_db: Mutex>>, + state_db: Mutex>, block_queue: BlockQueue, report: RwLock, import_lock: Mutex<()>, diff --git a/ethcore/src/state.rs b/ethcore/src/state.rs index 222b88643..519debcc1 100644 --- a/ethcore/src/state.rs +++ b/ethcore/src/state.rs @@ -31,7 +31,7 @@ pub type ApplyResult = Result; /// Representation of the entire state of all accounts in the system. pub struct State { - db: Box>, + db: Box, root: H256, cache: RefCell>>, snapshots: RefCell>>>>, @@ -41,7 +41,7 @@ pub struct State { impl State { /// Creates new state with empty state root #[cfg(test)] - pub fn new(mut db: Box>, account_start_nonce: U256) -> State { + pub fn new(mut db: Box, account_start_nonce: U256) -> State { let mut root = H256::new(); { // init trie and reset root too null @@ -58,7 +58,7 @@ impl State { } /// Creates new state with existing state root - pub fn from_existing(db: Box>, root: H256, account_start_nonce: U256) -> State { + pub fn from_existing(db: Box, root: H256, account_start_nonce: U256) -> State { { // trie should panic! if root does not exist let _ = SecTrieDB::new(db.as_hashdb(), &root); @@ -126,7 +126,7 @@ impl State { } /// Destroy the current object and return root and database. - pub fn drop(self) -> (H256, Box>) { + pub fn drop(self) -> (H256, Box) { (self.root, self.db) } diff --git a/ethcore/src/tests/helpers.rs b/ethcore/src/tests/helpers.rs index 85e311e82..0bb6b5015 100644 --- a/ethcore/src/tests/helpers.rs +++ b/ethcore/src/tests/helpers.rs @@ -250,7 +250,7 @@ pub fn generate_dummy_empty_blockchain() -> GuardedTempResult { } } -pub fn get_temp_journal_db() -> GuardedTempResult>> { +pub fn get_temp_journal_db() -> GuardedTempResult> { let temp = RandomTempPath::new(); let journal_db = Box::new(OptionOneDB::new(temp.as_str())); GuardedTempResult { @@ -268,7 +268,7 @@ pub fn get_temp_state() -> GuardedTempResult { } } -pub fn get_temp_journal_db_in(path: &Path) -> Box> { +pub fn get_temp_journal_db_in(path: &Path) -> Box { Box::new(OptionOneDB::new(path.to_str().unwrap())) } diff --git a/util/src/journaldb.rs b/util/src/journaldb.rs index c6243ace0..e7a670a08 100644 --- a/util/src/journaldb.rs +++ b/util/src/journaldb.rs @@ -28,7 +28,7 @@ use std::env; /// exclusive actions. pub trait JournalDB : HashDB + Sync + Send { /// Return a copy of ourself, in a box. - fn spawn(&self) -> Box>; + fn spawn(&self) -> Box; /// Returns heap memory size used fn mem_used(&self) -> usize; @@ -418,7 +418,7 @@ impl HashDB for OptionOneDB { } impl JournalDB for OptionOneDB { - fn spawn(&self) -> Box> { + fn spawn(&self) -> Box { Box::new(OptionOneDB { overlay: MemoryDB::new(), backing: self.backing.clone(), From 4771fdf0fb448d72e3b386dee78b308cd40f1c45 Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Fri, 11 Mar 2016 13:50:39 +0100 Subject: [PATCH 09/17] Rearrange journaldb infrastructure. --- ethcore/src/block_queue.rs | 1 + ethcore/src/client/client.rs | 10 +- util/src/journaldb/archivedb.rs | 387 ++++++++++++++++++ util/src/journaldb/mod.rs | 33 ++ .../optiononedb.rs} | 279 +++++-------- util/src/journaldb/traits.rs | 37 ++ 6 files changed, 571 insertions(+), 176 deletions(-) create mode 100644 util/src/journaldb/archivedb.rs create mode 100644 util/src/journaldb/mod.rs rename util/src/{journaldb.rs => journaldb/optiononedb.rs} (88%) create mode 100644 util/src/journaldb/traits.rs diff --git a/ethcore/src/block_queue.rs b/ethcore/src/block_queue.rs index 4e335f705..8b7143b0b 100644 --- a/ethcore/src/block_queue.rs +++ b/ethcore/src/block_queue.rs @@ -412,6 +412,7 @@ impl BlockQueue { } } + /// Optimise memory footprint of the heap fields. pub fn collect_garbage(&self) { { self.verification.unverified.lock().unwrap().shrink_to_fit(); diff --git a/ethcore/src/client/client.rs b/ethcore/src/client/client.rs index 3c8a28380..70a3eb92a 100644 --- a/ethcore/src/client/client.rs +++ b/ethcore/src/client/client.rs @@ -139,8 +139,14 @@ impl Client where V: Verifier { state_path.push("state"); let engine = Arc::new(try!(spec.to_engine())); - let mut state_db = Box::new(OptionOneDB::from_prefs(state_path.to_str().unwrap(), config.prefer_journal)); - if state_db.is_empty() && engine.spec().ensure_db_good(state_db.deref_mut()) { + let state_path_str = state_path.to_str().unwrap(); + let mut state_db = if config.prefer_journal { + new_optiononedb(state_path_str) + } else { + new_archivedb(state_path_str) + }; + + if state_db.is_empty() && engine.spec().ensure_db_good(state_db.as_hashdb_mut()) { state_db.commit(0, &engine.spec().genesis_header().hash(), None).expect("Error commiting genesis state to state DB"); } diff --git a/util/src/journaldb/archivedb.rs b/util/src/journaldb/archivedb.rs new file mode 100644 index 000000000..28cc4130a --- /dev/null +++ b/util/src/journaldb/archivedb.rs @@ -0,0 +1,387 @@ +// Copyright 2015, 2016 Ethcore (UK) Ltd. +// This file is part of Parity. + +// Parity is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Parity is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Parity. If not, see . + +//! Disk-backed HashDB implementation. + +use common::*; +use rlp::*; +use hashdb::*; +use memorydb::*; +use super::traits::JournalDB; +use kvdb::{Database, DBTransaction, DatabaseConfig}; +#[cfg(test)] +use std::env; + +/// Implementation of the HashDB trait for a disk-backed database with a memory overlay +/// and latent-removal semantics. +/// +/// Like OverlayDB, there is a memory overlay; `commit()` must be called in order to +/// write operations out to disk. Unlike OverlayDB, `remove()` operations do not take effect +/// immediately. Rather some age (based on a linear but arbitrary metric) must pass before +/// the removals actually take effect. +pub struct ArchiveDB { + overlay: MemoryDB, + backing: Arc, +} + +// all keys must be at least 12 bytes +const LATEST_ERA_KEY : [u8; 12] = [ b'l', b'a', b's', b't', 0, 0, 0, 0, 0, 0, 0, 0 ]; +const VERSION_KEY : [u8; 12] = [ b'j', b'v', b'e', b'r', 0, 0, 0, 0, 0, 0, 0, 0 ]; +const DB_VERSION : u32 = 259; + +impl ArchiveDB { + /// Create a new instance from file + pub fn new(path: &str) -> ArchiveDB { + let opts = DatabaseConfig { + prefix_size: Some(12) //use 12 bytes as prefix, this must match account_db prefix + }; + let backing = Database::open(&opts, path).unwrap_or_else(|e| { + panic!("Error opening state db: {}", e); + }); + if !backing.is_empty() { + match backing.get(&VERSION_KEY).map(|d| d.map(|v| decode::(&v))) { + Ok(Some(DB_VERSION)) => {}, + v => panic!("Incompatible DB version, expected {}, got {:?}", DB_VERSION, v) + } + } else { + backing.put(&VERSION_KEY, &encode(&DB_VERSION)).expect("Error writing version to database"); + } + + ArchiveDB { + overlay: MemoryDB::new(), + backing: Arc::new(backing), + } + } + + /// Create a new instance with an anonymous temporary database. + #[cfg(test)] + fn new_temp() -> ArchiveDB { + let mut dir = env::temp_dir(); + dir.push(H32::random().hex()); + Self::new(dir.to_str().unwrap()) + } + + fn payload(&self, key: &H256) -> Option { + self.backing.get(&key.bytes()).expect("Low-level database error. Some issue with your hard disk?").map(|v| v.to_vec()) + } +} + +impl HashDB for ArchiveDB { + fn keys(&self) -> HashMap { + let mut ret: HashMap = HashMap::new(); + for (key, _) in self.backing.iter() { + let h = H256::from_slice(key.deref()); + ret.insert(h, 1); + } + + for (key, refs) in self.overlay.keys().into_iter() { + let refs = *ret.get(&key).unwrap_or(&0) + refs; + ret.insert(key, refs); + } + ret + } + + fn lookup(&self, key: &H256) -> Option<&[u8]> { + let k = self.overlay.raw(key); + match k { + Some(&(ref d, rc)) if rc > 0 => Some(d), + _ => { + if let Some(x) = self.payload(key) { + Some(&self.overlay.denote(key, x).0) + } + else { + None + } + } + } + } + + fn exists(&self, key: &H256) -> bool { + self.lookup(key).is_some() + } + + fn insert(&mut self, value: &[u8]) -> H256 { + self.overlay.insert(value) + } + fn emplace(&mut self, key: H256, value: Bytes) { + self.overlay.emplace(key, value); + } + fn kill(&mut self, key: &H256) { + self.overlay.kill(key); + } +} + +impl JournalDB for ArchiveDB { + fn spawn(&self) -> Box { + Box::new(ArchiveDB { + overlay: MemoryDB::new(), + backing: self.backing.clone(), + }) + } + + fn mem_used(&self) -> usize { + self.overlay.mem_used() + } + + fn is_empty(&self) -> bool { + self.backing.get(&LATEST_ERA_KEY).expect("Low level database error").is_none() + } + + fn commit(&mut self, _: u64, _: &H256, _: Option<(u64, H256)>) -> Result { + let batch = DBTransaction::new(); + let mut inserts = 0usize; + let mut deletes = 0usize; + for i in self.overlay.drain().into_iter() { + let (key, (value, rc)) = i; + if rc > 0 { + assert!(rc == 1); + batch.put(&key.bytes(), &value).expect("Low-level database error. Some issue with your hard disk?"); + inserts += 1; + } + if rc < 0 { + assert!(rc == -1); + deletes += 1; + } + } + try!(self.backing.write(batch)); + Ok((inserts + deletes) as u32) + } +} + +#[cfg(test)] +mod tests { + use common::*; + use super::*; + use hashdb::*; + + #[test] + fn insert_same_in_fork() { + // history is 1 + let mut jdb = ArchiveDB::new_temp(); + + let x = jdb.insert(b"X"); + jdb.commit(1, &b"1".sha3(), None).unwrap(); + jdb.commit(2, &b"2".sha3(), None).unwrap(); + jdb.commit(3, &b"1002a".sha3(), Some((1, b"1".sha3()))).unwrap(); + jdb.commit(4, &b"1003a".sha3(), Some((2, b"2".sha3()))).unwrap(); + + jdb.remove(&x); + jdb.commit(3, &b"1002b".sha3(), Some((1, b"1".sha3()))).unwrap(); + let x = jdb.insert(b"X"); + jdb.commit(4, &b"1003b".sha3(), Some((2, b"2".sha3()))).unwrap(); + + jdb.commit(5, &b"1004a".sha3(), Some((3, b"1002a".sha3()))).unwrap(); + jdb.commit(6, &b"1005a".sha3(), Some((4, b"1003a".sha3()))).unwrap(); + + assert!(jdb.exists(&x)); + } + + #[test] + fn long_history() { + // history is 3 + let mut jdb = ArchiveDB::new_temp(); + let h = jdb.insert(b"foo"); + jdb.commit(0, &b"0".sha3(), None).unwrap(); + assert!(jdb.exists(&h)); + jdb.remove(&h); + jdb.commit(1, &b"1".sha3(), None).unwrap(); + assert!(jdb.exists(&h)); + jdb.commit(2, &b"2".sha3(), None).unwrap(); + assert!(jdb.exists(&h)); + jdb.commit(3, &b"3".sha3(), Some((0, b"0".sha3()))).unwrap(); + assert!(jdb.exists(&h)); + jdb.commit(4, &b"4".sha3(), Some((1, b"1".sha3()))).unwrap(); + } + + #[test] + fn complex() { + // history is 1 + let mut jdb = ArchiveDB::new_temp(); + + let foo = jdb.insert(b"foo"); + let bar = jdb.insert(b"bar"); + jdb.commit(0, &b"0".sha3(), None).unwrap(); + assert!(jdb.exists(&foo)); + assert!(jdb.exists(&bar)); + + jdb.remove(&foo); + jdb.remove(&bar); + let baz = jdb.insert(b"baz"); + jdb.commit(1, &b"1".sha3(), Some((0, b"0".sha3()))).unwrap(); + assert!(jdb.exists(&foo)); + assert!(jdb.exists(&bar)); + assert!(jdb.exists(&baz)); + + let foo = jdb.insert(b"foo"); + jdb.remove(&baz); + jdb.commit(2, &b"2".sha3(), Some((1, b"1".sha3()))).unwrap(); + assert!(jdb.exists(&foo)); + assert!(jdb.exists(&baz)); + + jdb.remove(&foo); + jdb.commit(3, &b"3".sha3(), Some((2, b"2".sha3()))).unwrap(); + assert!(jdb.exists(&foo)); + + jdb.commit(4, &b"4".sha3(), Some((3, b"3".sha3()))).unwrap(); + } + + #[test] + fn fork() { + // history is 1 + let mut jdb = ArchiveDB::new_temp(); + + let foo = jdb.insert(b"foo"); + let bar = jdb.insert(b"bar"); + jdb.commit(0, &b"0".sha3(), None).unwrap(); + assert!(jdb.exists(&foo)); + assert!(jdb.exists(&bar)); + + jdb.remove(&foo); + let baz = jdb.insert(b"baz"); + jdb.commit(1, &b"1a".sha3(), Some((0, b"0".sha3()))).unwrap(); + + jdb.remove(&bar); + jdb.commit(1, &b"1b".sha3(), Some((0, b"0".sha3()))).unwrap(); + + assert!(jdb.exists(&foo)); + assert!(jdb.exists(&bar)); + assert!(jdb.exists(&baz)); + + jdb.commit(2, &b"2b".sha3(), Some((1, b"1b".sha3()))).unwrap(); + assert!(jdb.exists(&foo)); + } + + #[test] + fn overwrite() { + // history is 1 + let mut jdb = ArchiveDB::new_temp(); + + let foo = jdb.insert(b"foo"); + jdb.commit(0, &b"0".sha3(), None).unwrap(); + assert!(jdb.exists(&foo)); + + jdb.remove(&foo); + jdb.commit(1, &b"1".sha3(), Some((0, b"0".sha3()))).unwrap(); + jdb.insert(b"foo"); + assert!(jdb.exists(&foo)); + jdb.commit(2, &b"2".sha3(), Some((1, b"1".sha3()))).unwrap(); + assert!(jdb.exists(&foo)); + jdb.commit(3, &b"2".sha3(), Some((0, b"2".sha3()))).unwrap(); + assert!(jdb.exists(&foo)); + } + + #[test] + fn fork_same_key() { + // history is 1 + let mut jdb = ArchiveDB::new_temp(); + jdb.commit(0, &b"0".sha3(), None).unwrap(); + + let foo = jdb.insert(b"foo"); + jdb.commit(1, &b"1a".sha3(), Some((0, b"0".sha3()))).unwrap(); + + jdb.insert(b"foo"); + jdb.commit(1, &b"1b".sha3(), Some((0, b"0".sha3()))).unwrap(); + assert!(jdb.exists(&foo)); + + jdb.commit(2, &b"2a".sha3(), Some((1, b"1a".sha3()))).unwrap(); + assert!(jdb.exists(&foo)); + } + + + #[test] + fn reopen() { + let mut dir = ::std::env::temp_dir(); + dir.push(H32::random().hex()); + let bar = H256::random(); + + let foo = { + let mut jdb = ArchiveDB::new(dir.to_str().unwrap()); + // history is 1 + let foo = jdb.insert(b"foo"); + jdb.emplace(bar.clone(), b"bar".to_vec()); + jdb.commit(0, &b"0".sha3(), None).unwrap(); + foo + }; + + { + let mut jdb = ArchiveDB::new(dir.to_str().unwrap()); + jdb.remove(&foo); + jdb.commit(1, &b"1".sha3(), Some((0, b"0".sha3()))).unwrap(); + } + + { + let mut jdb = ArchiveDB::new(dir.to_str().unwrap()); + assert!(jdb.exists(&foo)); + assert!(jdb.exists(&bar)); + jdb.commit(2, &b"2".sha3(), Some((1, b"1".sha3()))).unwrap(); + } + } + + #[test] + fn reopen_remove() { + let mut dir = ::std::env::temp_dir(); + dir.push(H32::random().hex()); + + let foo = { + let mut jdb = ArchiveDB::new(dir.to_str().unwrap()); + // history is 1 + let foo = jdb.insert(b"foo"); + jdb.commit(0, &b"0".sha3(), None).unwrap(); + jdb.commit(1, &b"1".sha3(), Some((0, b"0".sha3()))).unwrap(); + + // foo is ancient history. + + jdb.insert(b"foo"); + jdb.commit(2, &b"2".sha3(), Some((1, b"1".sha3()))).unwrap(); + foo + }; + + { + let mut jdb = ArchiveDB::new(dir.to_str().unwrap()); + jdb.remove(&foo); + jdb.commit(3, &b"3".sha3(), Some((2, b"2".sha3()))).unwrap(); + assert!(jdb.exists(&foo)); + jdb.remove(&foo); + jdb.commit(4, &b"4".sha3(), Some((3, b"3".sha3()))).unwrap(); + jdb.commit(5, &b"5".sha3(), Some((4, b"4".sha3()))).unwrap(); + } + } + #[test] + fn reopen_fork() { + let mut dir = ::std::env::temp_dir(); + dir.push(H32::random().hex()); + let (foo, bar, baz) = { + let mut jdb = ArchiveDB::new(dir.to_str().unwrap()); + // history is 1 + let foo = jdb.insert(b"foo"); + let bar = jdb.insert(b"bar"); + jdb.commit(0, &b"0".sha3(), None).unwrap(); + jdb.remove(&foo); + let baz = jdb.insert(b"baz"); + jdb.commit(1, &b"1a".sha3(), Some((0, b"0".sha3()))).unwrap(); + + jdb.remove(&bar); + jdb.commit(1, &b"1b".sha3(), Some((0, b"0".sha3()))).unwrap(); + (foo, bar, baz) + }; + + { + let mut jdb = ArchiveDB::new(dir.to_str().unwrap()); + jdb.commit(2, &b"2b".sha3(), Some((1, b"1b".sha3()))).unwrap(); + assert!(jdb.exists(&foo)); + } + } +} diff --git a/util/src/journaldb/mod.rs b/util/src/journaldb/mod.rs new file mode 100644 index 000000000..fdb825d51 --- /dev/null +++ b/util/src/journaldb/mod.rs @@ -0,0 +1,33 @@ +// Copyright 2015, 2016 Ethcore (UK) Ltd. +// This file is part of Parity. + +// Parity is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Parity is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Parity. If not, see . + +//! JournalDB interface and implementation. + +use common::*; + +/// Export the journaldb module. +pub mod traits; +mod archivedb; +mod optiononedb; + +/// Export the JournalDB trait. +pub use self::traits::JournalDB; + +/// Create a new JournalDB trait object which is an ArchiveDB. +pub fn new_archivedb(path: &str) -> Box { Box::new(archivedb::ArchiveDB::new(path)) } + +/// Create a new JournalDB trait object which is an OptionOneDB. +pub fn new_optiononedb(path: &str) -> Box { Box::new(optiononedb::OptionOneDB::new(path)) } diff --git a/util/src/journaldb.rs b/util/src/journaldb/optiononedb.rs similarity index 88% rename from util/src/journaldb.rs rename to util/src/journaldb/optiononedb.rs index e7a670a08..58bb88277 100644 --- a/util/src/journaldb.rs +++ b/util/src/journaldb/optiononedb.rs @@ -20,26 +20,11 @@ use common::*; use rlp::*; use hashdb::*; use memorydb::*; +use super::traits::JournalDB; use kvdb::{Database, DBTransaction, DatabaseConfig}; #[cfg(test)] use std::env; -/// A HashDB which can manage a short-term journal potentially containing many forks of mutually -/// exclusive actions. -pub trait JournalDB : HashDB + Sync + Send { - /// Return a copy of ourself, in a box. - fn spawn(&self) -> Box; - - /// Returns heap memory size used - fn mem_used(&self) -> usize; - - /// Check if this database has any commits - fn is_empty(&self) -> bool; - - /// Commit all recent insert operations. - fn commit(&mut self, now: u64, id: &H256, end: Option<(u64, H256)>) -> Result; -} - /// Implementation of the HashDB trait for a disk-backed database with a memory overlay /// and latent-removal semantics. /// @@ -56,43 +41,28 @@ pub struct OptionOneDB { // all keys must be at least 12 bytes const LATEST_ERA_KEY : [u8; 12] = [ b'l', b'a', b's', b't', 0, 0, 0, 0, 0, 0, 0, 0 ]; const VERSION_KEY : [u8; 12] = [ b'j', b'v', b'e', b'r', 0, 0, 0, 0, 0, 0, 0, 0 ]; - const DB_VERSION : u32 = 3; -const DB_VERSION_NO_JOURNAL : u32 = 3 + 256; - const PADDING : [u8; 10] = [ 0u8; 10 ]; impl OptionOneDB { /// Create a new instance from file pub fn new(path: &str) -> OptionOneDB { - Self::from_prefs(path, true) - } - - /// Create a new instance from file - pub fn from_prefs(path: &str, prefer_journal: bool) -> OptionOneDB { let opts = DatabaseConfig { prefix_size: Some(12) //use 12 bytes as prefix, this must match account_db prefix }; let backing = Database::open(&opts, path).unwrap_or_else(|e| { panic!("Error opening state db: {}", e); }); - let with_journal; if !backing.is_empty() { match backing.get(&VERSION_KEY).map(|d| d.map(|v| decode::(&v))) { - Ok(Some(DB_VERSION)) => { with_journal = true; }, - Ok(Some(DB_VERSION_NO_JOURNAL)) => { with_journal = false; }, + Ok(Some(DB_VERSION)) => {}, v => panic!("Incompatible DB version, expected {}, got {:?}", DB_VERSION, v) } } else { - backing.put(&VERSION_KEY, &encode(&(if prefer_journal { DB_VERSION } else { DB_VERSION_NO_JOURNAL }))).expect("Error writing version to database"); - with_journal = prefer_journal; + backing.put(&VERSION_KEY, &encode(&DB_VERSION)).expect("Error writing version to database"); } - - let counters = if with_journal { - Some(Arc::new(RwLock::new(OptionOneDB::read_counters(&backing)))) - } else { - None - }; + + let counters = Some(Arc::new(RwLock::new(OptionOneDB::read_counters(&backing)))); OptionOneDB { overlay: MemoryDB::new(), backing: Arc::new(backing), @@ -108,34 +78,6 @@ impl OptionOneDB { Self::new(dir.to_str().unwrap()) } - /// Drain the overlay and place it into a batch for the DB. - fn batch_overlay_insertions(overlay: &mut MemoryDB, batch: &DBTransaction) -> usize { - let mut inserts = 0usize; - let mut deletes = 0usize; - for i in overlay.drain().into_iter() { - let (key, (value, rc)) = i; - if rc > 0 { - assert!(rc == 1); - batch.put(&key.bytes(), &value).expect("Low-level database error. Some issue with your hard disk?"); - inserts += 1; - } - if rc < 0 { - assert!(rc == -1); - deletes += 1; - } - } - trace!("commit: Inserted {}, Deleted {} nodes", inserts, deletes); - inserts + deletes - } - - /// Just commit the overlay into the backing DB. - fn commit_without_counters(&mut self) -> Result { - let batch = DBTransaction::new(); - let ret = Self::batch_overlay_insertions(&mut self.overlay, &batch); - try!(self.backing.write(batch)); - Ok(ret as u32) - } - fn morph_key(key: &H256, index: u8) -> Bytes { let mut ret = key.bytes().to_owned(); ret.push(index); @@ -217,9 +159,106 @@ impl OptionOneDB { } } - /// Commit all recent insert operations and historical removals from the old era - /// to the backing database. - fn commit_with_counters(&mut self, now: u64, id: &H256, end: Option<(u64, H256)>) -> Result { + fn payload(&self, key: &H256) -> Option { + self.backing.get(&key.bytes()).expect("Low-level database error. Some issue with your hard disk?").map(|v| v.to_vec()) + } + + fn read_counters(db: &Database) -> HashMap { + let mut counters = HashMap::new(); + if let Some(val) = db.get(&LATEST_ERA_KEY).expect("Low-level database error.") { + let mut era = decode::(&val); + loop { + let mut index = 0usize; + while let Some(rlp_data) = db.get({ + let mut r = RlpStream::new_list(3); + r.append(&era); + r.append(&index); + r.append(&&PADDING[..]); + &r.drain() + }).expect("Low-level database error.") { + trace!("read_counters: era={}, index={}", era, index); + let rlp = Rlp::new(&rlp_data); + let inserts: Vec = rlp.val_at(1); + Self::replay_keys(&inserts, db, &mut counters); + index += 1; + }; + if index == 0 || era == 0 { + break; + } + era -= 1; + } + } + trace!("Recovered {} counters", counters.len()); + counters + } +} + +impl HashDB for OptionOneDB { + fn keys(&self) -> HashMap { + let mut ret: HashMap = HashMap::new(); + for (key, _) in self.backing.iter() { + let h = H256::from_slice(key.deref()); + ret.insert(h, 1); + } + + for (key, refs) in self.overlay.keys().into_iter() { + let refs = *ret.get(&key).unwrap_or(&0) + refs; + ret.insert(key, refs); + } + ret + } + + fn lookup(&self, key: &H256) -> Option<&[u8]> { + let k = self.overlay.raw(key); + match k { + Some(&(ref d, rc)) if rc > 0 => Some(d), + _ => { + if let Some(x) = self.payload(key) { + Some(&self.overlay.denote(key, x).0) + } + else { + None + } + } + } + } + + fn exists(&self, key: &H256) -> bool { + self.lookup(key).is_some() + } + + fn insert(&mut self, value: &[u8]) -> H256 { + self.overlay.insert(value) + } + fn emplace(&mut self, key: H256, value: Bytes) { + self.overlay.emplace(key, value); + } + fn kill(&mut self, key: &H256) { + self.overlay.kill(key); + } +} + +impl JournalDB for OptionOneDB { + fn spawn(&self) -> Box { + Box::new(OptionOneDB { + overlay: MemoryDB::new(), + backing: self.backing.clone(), + counters: self.counters.clone(), + }) + } + + fn mem_used(&self) -> usize { + self.overlay.mem_used() + match self.counters { + Some(ref c) => c.read().unwrap().heap_size_of_children(), + None => 0 + } + } + + fn is_empty(&self) -> bool { + self.backing.get(&LATEST_ERA_KEY).expect("Low level database error").is_none() + } + + fn commit(&mut self, now: u64, id: &H256, end: Option<(u64, H256)>) -> Result { // journal format: // [era, 0] => [ id, [insert_0, ...], [remove_0, ...] ] // [era, 1] => [ id, [insert_0, ...], [remove_0, ...] ] @@ -337,114 +376,6 @@ impl OptionOneDB { // trace!("OptionOneDB::commit() deleted {} nodes", deletes); Ok(0) } - - fn payload(&self, key: &H256) -> Option { - self.backing.get(&key.bytes()).expect("Low-level database error. Some issue with your hard disk?").map(|v| v.to_vec()) - } - - fn read_counters(db: &Database) -> HashMap { - let mut counters = HashMap::new(); - if let Some(val) = db.get(&LATEST_ERA_KEY).expect("Low-level database error.") { - let mut era = decode::(&val); - loop { - let mut index = 0usize; - while let Some(rlp_data) = db.get({ - let mut r = RlpStream::new_list(3); - r.append(&era); - r.append(&index); - r.append(&&PADDING[..]); - &r.drain() - }).expect("Low-level database error.") { - trace!("read_counters: era={}, index={}", era, index); - let rlp = Rlp::new(&rlp_data); - let inserts: Vec = rlp.val_at(1); - Self::replay_keys(&inserts, db, &mut counters); - index += 1; - }; - if index == 0 || era == 0 { - break; - } - era -= 1; - } - } - trace!("Recovered {} counters", counters.len()); - counters - } -} - -impl HashDB for OptionOneDB { - fn keys(&self) -> HashMap { - let mut ret: HashMap = HashMap::new(); - for (key, _) in self.backing.iter() { - let h = H256::from_slice(key.deref()); - ret.insert(h, 1); - } - - for (key, refs) in self.overlay.keys().into_iter() { - let refs = *ret.get(&key).unwrap_or(&0) + refs; - ret.insert(key, refs); - } - ret - } - - fn lookup(&self, key: &H256) -> Option<&[u8]> { - let k = self.overlay.raw(key); - match k { - Some(&(ref d, rc)) if rc > 0 => Some(d), - _ => { - if let Some(x) = self.payload(key) { - Some(&self.overlay.denote(key, x).0) - } - else { - None - } - } - } - } - - fn exists(&self, key: &H256) -> bool { - self.lookup(key).is_some() - } - - fn insert(&mut self, value: &[u8]) -> H256 { - self.overlay.insert(value) - } - fn emplace(&mut self, key: H256, value: Bytes) { - self.overlay.emplace(key, value); - } - fn kill(&mut self, key: &H256) { - self.overlay.kill(key); - } -} - -impl JournalDB for OptionOneDB { - fn spawn(&self) -> Box { - Box::new(OptionOneDB { - overlay: MemoryDB::new(), - backing: self.backing.clone(), - counters: self.counters.clone(), - }) - } - - fn mem_used(&self) -> usize { - self.overlay.mem_used() + match self.counters { - Some(ref c) => c.read().unwrap().heap_size_of_children(), - None => 0 - } - } - - fn is_empty(&self) -> bool { - self.backing.get(&LATEST_ERA_KEY).expect("Low level database error").is_none() - } - - fn commit(&mut self, now: u64, id: &H256, end: Option<(u64, H256)>) -> Result { - let have_counters = self.counters.is_some(); - if have_counters { - self.commit_with_counters(now, id, end) - } else { - self.commit_without_counters() - } - } } #[cfg(test)] diff --git a/util/src/journaldb/traits.rs b/util/src/journaldb/traits.rs new file mode 100644 index 000000000..25e132339 --- /dev/null +++ b/util/src/journaldb/traits.rs @@ -0,0 +1,37 @@ +// Copyright 2015, 2016 Ethcore (UK) Ltd. +// This file is part of Parity. + +// Parity is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Parity is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Parity. If not, see . + +//! Disk-backed HashDB implementation. + +use common::*; +use hashdb::*; + +/// A HashDB which can manage a short-term journal potentially containing many forks of mutually +/// exclusive actions. +pub trait JournalDB : HashDB + Send + Sync { + /// Return a copy of ourself, in a box. + fn spawn(&self) -> Box; + + /// Returns heap memory size used + fn mem_used(&self) -> usize; + + /// Check if this database has any commits + fn is_empty(&self) -> bool; + + /// Commit all recent insert operations and canonical historical commits' removals from the + /// old era to the backing database, reverting any non-canonical historical commit's inserts. + fn commit(&mut self, now: u64, id: &H256, end: Option<(u64, H256)>) -> Result; +} From ecd33a60931be9616245984149a46aba12457f36 Mon Sep 17 00:00:00 2001 From: debris Date: Fri, 11 Mar 2016 13:54:52 +0100 Subject: [PATCH 10/17] fixed U256 and transaction request deserialization, added tests for transaction request --- rpc/src/v1/types/bytes.rs | 2 +- rpc/src/v1/types/transaction_request.rs | 64 ++++++++++++++++++++----- util/bigint/src/uint.rs | 8 +--- util/src/hash.rs | 2 +- 4 files changed, 55 insertions(+), 21 deletions(-) diff --git a/rpc/src/v1/types/bytes.rs b/rpc/src/v1/types/bytes.rs index 466fbebde..0b14c30e8 100644 --- a/rpc/src/v1/types/bytes.rs +++ b/rpc/src/v1/types/bytes.rs @@ -20,7 +20,7 @@ use serde::de::Visitor; use util::common::FromHex; /// Wrapper structure around vector of bytes. -#[derive(Debug)] +#[derive(Debug, PartialEq)] pub struct Bytes(Vec); impl Bytes { diff --git a/rpc/src/v1/types/transaction_request.rs b/rpc/src/v1/types/transaction_request.rs index a61b11c25..d40402ab5 100644 --- a/rpc/src/v1/types/transaction_request.rs +++ b/rpc/src/v1/types/transaction_request.rs @@ -19,7 +19,7 @@ use util::numbers::{Uint, U256}; use ethcore::transaction::{Action, Transaction}; use v1::types::Bytes; -#[derive(Debug, Default, Deserialize)] +#[derive(Debug, Default, PartialEq, Deserialize)] pub struct TransactionRequest { pub from: Address, pub to: Option
, @@ -27,28 +27,26 @@ pub struct TransactionRequest { pub gas_price: Option, pub gas: Option, pub value: Option, - pub data: Bytes, + pub data: Option, pub nonce: Option, } impl Into for TransactionRequest { fn into(self) -> Transaction { Transaction { - nonce: self.nonce.unwrap_or(U256::zero()), - action: match self.to { - None => Action::Create, - Some(addr) => Action::Call(addr) - }, - gas: self.gas.unwrap_or(U256::zero()), - gas_price: self.gas_price.unwrap_or(U256::zero()), - value: self.value.unwrap_or(U256::zero()), - data: self.data.to_vec() + nonce: self.nonce.unwrap_or_else(U256::zero), + action: self.to.map_or(Action::Create, Action::Call), + gas: self.gas.unwrap_or_else(U256::zero), + gas_price: self.gas_price.unwrap_or_else(U256::zero), + value: self.value.unwrap_or_else(U256::zero), + data: self.data.map_or_else(Vec::new, |d| d.to_vec()), } } } #[cfg(test)] mod tests { + use serde_json; use util::numbers::{Uint, U256}; use util::hash::Address; use ethcore::transaction::{Transaction, Action}; @@ -63,7 +61,7 @@ mod tests { gas_price: Some(U256::from(20)), gas: Some(U256::from(10_000)), value: Some(U256::from(1)), - data: Bytes::new(vec![10, 20]), + data: Some(Bytes::new(vec![10, 20])), nonce: Some(U256::from(12)), }; @@ -85,7 +83,7 @@ mod tests { gas_price: None, gas: None, value: None, - data: Bytes::new(vec![]), + data: None, nonce: None, }; @@ -98,4 +96,44 @@ mod tests { data: vec![], }, tr.into()); } + + #[test] + fn transaction_request_deserialize() { + let s = r#"{ + "from":"0x0000000000000000000000000000000000000001", + "to":"0x0000000000000000000000000000000000000002", + "gasPrice":"0x1", + "gas":"0x2", + "value":"0x3", + "data":"0x123456", + "nonce":"0x4" + }"#; + let deserialized: TransactionRequest = serde_json::from_str(s).unwrap(); + + assert_eq!(deserialized, TransactionRequest { + from: Address::from(1), + to: Some(Address::from(2)), + gas_price: Some(U256::from(1)), + gas: Some(U256::from(2)), + value: Some(U256::from(3)), + data: Some(Bytes::new(vec![0x12, 0x34, 0x56])), + nonce: Some(U256::from(4)), + }); + } + + #[test] + fn transaction_request_deserialize_empty() { + let s = r#"{"from":"0x0000000000000000000000000000000000000001"}"#; + let deserialized: TransactionRequest = serde_json::from_str(s).unwrap(); + + assert_eq!(deserialized, TransactionRequest { + from: Address::from(1), + to: None, + gas_price: None, + gas: None, + value: None, + data: None, + nonce: None, + }); + } } diff --git a/util/bigint/src/uint.rs b/util/bigint/src/uint.rs index 959df0944..47888fd88 100644 --- a/util/bigint/src/uint.rs +++ b/util/bigint/src/uint.rs @@ -39,7 +39,6 @@ use std::fmt; use std::cmp; -use std::mem; use std::str::{FromStr}; use std::convert::From; use std::hash::{Hash, Hasher}; @@ -788,14 +787,11 @@ macro_rules! construct_uint { fn visit_str(&mut self, value: &str) -> Result where E: serde::Error { // 0x + len - if value.len() != 2 + $n_words / 8 { + if value.len() > 2 + $n_words * 16 { return Err(serde::Error::custom("Invalid length.")); } - match $name::from_str(&value[2..]) { - Ok(val) => Ok(val), - Err(_) => { return Err(serde::Error::custom("Invalid length.")); } - } + $name::from_str(&value[2..]).map_err(|_| serde::Error::custom("Invalid hex value.")) } fn visit_string(&mut self, value: String) -> Result where E: serde::Error { diff --git a/util/src/hash.rs b/util/src/hash.rs index 4eb96b53e..3dc15116d 100644 --- a/util/src/hash.rs +++ b/util/src/hash.rs @@ -257,7 +257,7 @@ macro_rules! impl_hash { return Err(serde::Error::custom("Invalid length.")); } - value[2..].from_hex().map(|ref v| $from::from_slice(v)).map_err(|_| serde::Error::custom("Invalid valid hex.")) + value[2..].from_hex().map(|ref v| $from::from_slice(v)).map_err(|_| serde::Error::custom("Invalid hex value.")) } fn visit_string(&mut self, value: String) -> Result where E: serde::Error { From 51cfd4b0ea03a7a22ae0c0034a4ac0c8093a05c6 Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Fri, 11 Mar 2016 13:58:11 +0100 Subject: [PATCH 11/17] Remove unneeded clone. --- util/src/overlaydb.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/util/src/overlaydb.rs b/util/src/overlaydb.rs index f14677d05..7c9b6b04b 100644 --- a/util/src/overlaydb.rs +++ b/util/src/overlaydb.rs @@ -36,7 +36,7 @@ use kvdb::{Database}; /// /// `lookup()` and `contains()` maintain normal behaviour - all `insert()` and `remove()` /// queries have an immediate effect in terms of these functions. -#[derive(Clone)] +//#[derive(Clone)] pub struct OverlayDB { overlay: MemoryDB, backing: Arc, From 38d470f3bcc584a3f0f29e3670ebf10e2a2ddad9 Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Fri, 11 Mar 2016 14:45:19 +0100 Subject: [PATCH 12/17] Reorganise command line options into more general engine. --- ethcore/src/client/client.rs | 9 ++---- ethcore/src/client/config.rs | 5 ++-- parity/main.rs | 17 ++++++++---- util/src/journaldb/mod.rs | 53 +++++++++++++++++++++++++++++++++--- util/src/lib.rs | 2 +- 5 files changed, 68 insertions(+), 18 deletions(-) diff --git a/ethcore/src/client/client.rs b/ethcore/src/client/client.rs index 70a3eb92a..1148a0140 100644 --- a/ethcore/src/client/client.rs +++ b/ethcore/src/client/client.rs @@ -131,7 +131,8 @@ impl Client where V: Verifier { let mut dir = path.to_path_buf(); dir.push(H64::from(spec.genesis_header().hash()).hex()); //TODO: sec/fat: pruned/full versioning - dir.push(format!("v{}-sec-{}", CLIENT_DB_VER_STR, if config.prefer_journal { "pruned" } else { "archive" })); + // version here is a bit useless now, since it's controlled only be the pruning algo. + dir.push(format!("v{}-sec-{}", CLIENT_DB_VER_STR, config.pruning)); let path = dir.as_path(); let gb = spec.genesis_block(); let chain = Arc::new(BlockChain::new(config.blockchain, &gb, path)); @@ -140,11 +141,7 @@ impl Client where V: Verifier { let engine = Arc::new(try!(spec.to_engine())); let state_path_str = state_path.to_str().unwrap(); - let mut state_db = if config.prefer_journal { - new_optiononedb(state_path_str) - } else { - new_archivedb(state_path_str) - }; + let mut state_db = journaldb::new(state_path_str, config.pruning); if state_db.is_empty() && engine.spec().ensure_db_good(state_db.as_hashdb_mut()) { state_db.commit(0, &engine.spec().genesis_header().hash(), None).expect("Error commiting genesis state to state DB"); diff --git a/ethcore/src/client/config.rs b/ethcore/src/client/config.rs index 484c8d0c6..89e95ea06 100644 --- a/ethcore/src/client/config.rs +++ b/ethcore/src/client/config.rs @@ -16,6 +16,7 @@ pub use block_queue::BlockQueueConfig; pub use blockchain::BlockChainConfig; +use util::journaldb; /// Client configuration. Includes configs for all sub-systems. #[derive(Debug, Default)] @@ -24,8 +25,8 @@ pub struct ClientConfig { pub queue: BlockQueueConfig, /// Blockchain configuration. pub blockchain: BlockChainConfig, - /// Prefer journal rather than archive. - pub prefer_journal: bool, + /// The JournalDB ("pruning") algorithm to use. + pub pruning: journaldb::Algorithm, /// The name of the client instance. pub name: String, } diff --git a/parity/main.rs b/parity/main.rs index b6ed5cba3..bf1e24203 100644 --- a/parity/main.rs +++ b/parity/main.rs @@ -80,7 +80,8 @@ Protocol Options: or olympic, frontier, homestead, mainnet, morden, or testnet [default: homestead]. --testnet Equivalent to --chain testnet (geth-compatible). --networkid INDEX Override the network identifier from the chain we are on. - --pruning Client should prune the state/storage trie. + --pruning METHOD Configure pruning of the state/storage trie. METHOD may be one of: archive, + light (experimental) [default: archive]. -d --datadir PATH Specify the database & configuration directory path [default: $HOME/.parity] --keys-path PATH Specify the path for JSON key files to be found [default: $HOME/.web3/keys] --identity NAME Specify your node's name. @@ -101,7 +102,7 @@ API and Console Options: --jsonrpc-port PORT Specify the port portion of the JSONRPC API server [default: 8545]. --jsonrpc-cors URL Specify CORS header for JSON-RPC API responses [default: null]. --jsonrpc-apis APIS Specify the APIs available through the JSONRPC interface. APIS is a comma-delimited - list of API name. Possible name are web3, eth and net. [default: web3,eth,net]. + list of API name. Possible names are web3, eth and net. [default: web3,eth,net]. --rpc Equivalent to --jsonrpc (geth-compatible). --rpcaddr HOST Equivalent to --jsonrpc-addr HOST (geth-compatible). --rpcport PORT Equivalent to --jsonrpc-port PORT (geth-compatible). @@ -141,7 +142,7 @@ struct Args { flag_identity: String, flag_cache: Option, flag_keys_path: String, - flag_pruning: bool, + flag_pruning: String, flag_no_bootstrap: bool, flag_listen_address: String, flag_public_address: Option, @@ -403,7 +404,13 @@ impl Configuration { client_config.blockchain.max_cache_size = self.args.flag_cache_max_size; } } - client_config.prefer_journal = self.args.flag_pruning; + client_config.pruning = match self.args.flag_pruning.as_str() { + "archive" => journaldb::Algorithm::Archive, + "pruned" => journaldb::Algorithm::EarlyMerge, +// "fast" => journaldb::Algorithm::OverlayRecent, // TODO: @arkpar uncomment this once option 2 is merged. +// "slow" => journaldb::Algorithm::RefCounted, // TODO: @gavofyork uncomment this once ref-count algo is merged. + _ => { die!("{}: Invalid pruning method given.", self.args.flag_pruning); } + }; client_config.name = self.args.flag_identity.clone(); client_config.queue.max_mem_use = self.args.flag_queue_max_size; let mut service = ClientService::start(client_config, spec, net_settings, &Path::new(&self.path())).unwrap(); @@ -424,7 +431,7 @@ impl Configuration { self.args.flag_rpcaddr.as_ref().unwrap_or(&self.args.flag_jsonrpc_addr), self.args.flag_rpcport.unwrap_or(self.args.flag_jsonrpc_port) ); - SocketAddr::from_str(&url).unwrap_or_else(|_|die!("{}: Invalid JSONRPC listen host/port given.", url)); + SocketAddr::from_str(&url).unwrap_or_else(|_| die!("{}: Invalid JSONRPC listen host/port given.", url)); let cors = self.args.flag_rpccorsdomain.as_ref().unwrap_or(&self.args.flag_jsonrpc_cors); // TODO: use this as the API list. let apis = self.args.flag_rpcapi.as_ref().unwrap_or(&self.args.flag_jsonrpc_apis); diff --git a/util/src/journaldb/mod.rs b/util/src/journaldb/mod.rs index fdb825d51..0cee7dd8d 100644 --- a/util/src/journaldb/mod.rs +++ b/util/src/journaldb/mod.rs @@ -26,8 +26,53 @@ mod optiononedb; /// Export the JournalDB trait. pub use self::traits::JournalDB; -/// Create a new JournalDB trait object which is an ArchiveDB. -pub fn new_archivedb(path: &str) -> Box { Box::new(archivedb::ArchiveDB::new(path)) } +/// A journal database algorithm. +#[derive(Debug)] +pub enum Algorithm { + /// Keep all keys forever. + Archive, -/// Create a new JournalDB trait object which is an OptionOneDB. -pub fn new_optiononedb(path: &str) -> Box { Box::new(optiononedb::OptionOneDB::new(path)) } + /// Ancient and recent history maintained separately; recent history lasts for particular + /// number of blocks. + /// + /// Inserts go into backing database, journal retains knowledge of whether backing DB key is + /// ancient or recent. Non-canon inserts get explicitly reverted and removed from backing DB. + EarlyMerge, + + /// Ancient and recent history maintained separately; recent history lasts for particular + /// number of blocks. + /// + /// Inserts go into memory overlay, which is tried for key fetches. Memory overlay gets + /// flushed in backing only at end of recent history. + OverlayRecent, + + /// Ancient and recent history maintained separately; recent history lasts for particular + /// number of blocks. + /// + /// References are counted in disk-backed DB. + RefCounted, +} + +impl Default for Algorithm { + fn default() -> Algorithm { Algorithm::Archive } +} + +impl fmt::Display for Algorithm { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + write!(f, "{}", match self { + &Algorithm::Archive => "archive", + &Algorithm::EarlyMerge => "earlymerge", + &Algorithm::OverlayRecent => "overlayrecent", + &Algorithm::RefCounted => "refcounted", + }) + } +} + +/// Create a new JournalDB trait object. +pub fn new(path: &str, algorithm: Algorithm) -> Box { + match algorithm { + Algorithm::Archive => Box::new(archivedb::ArchiveDB::new(path)), + Algorithm::EarlyMerge => Box::new(optiononedb::OptionOneDB::new(path)), + _ => unimplemented!(), + } +} diff --git a/util/src/lib.rs b/util/src/lib.rs index 59d66a325..de9934f36 100644 --- a/util/src/lib.rs +++ b/util/src/lib.rs @@ -154,7 +154,7 @@ pub use rlp::*; pub use hashdb::*; pub use memorydb::*; pub use overlaydb::*; -pub use journaldb::*; +pub use journaldb::JournalDB; pub use math::*; pub use crypto::*; pub use triehash::*; From 8ae103087ddc87b3f61f1c0b21dcbb5ce5919fd5 Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Fri, 11 Mar 2016 15:07:43 +0100 Subject: [PATCH 13/17] Fixups for new API. --- ethcore/src/tests/helpers.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ethcore/src/tests/helpers.rs b/ethcore/src/tests/helpers.rs index 56ea6b1d3..dc3068560 100644 --- a/ethcore/src/tests/helpers.rs +++ b/ethcore/src/tests/helpers.rs @@ -252,7 +252,7 @@ pub fn generate_dummy_empty_blockchain() -> GuardedTempResult { pub fn get_temp_journal_db() -> GuardedTempResult> { let temp = RandomTempPath::new(); - let journal_db: Box = Box::new(OptionOneDB::new(temp.as_str())); + let journal_db = journaldb::new(temp.as_str(), journaldb::Algorithm::EarlyMerge); GuardedTempResult { _temp: temp, result: Some(journal_db) @@ -269,7 +269,7 @@ pub fn get_temp_state() -> GuardedTempResult { } pub fn get_temp_journal_db_in(path: &Path) -> Box { - Box::new(OptionOneDB::new(path.to_str().unwrap())) + journaldb::new(path.to_str().unwrap(), journaldb::Algorithm::EarlyMerge) } pub fn get_temp_state_in(path: &Path) -> State { From 5499f4530c0034451e9642fc6bba2eb67cdb321e Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Fri, 11 Mar 2016 15:01:15 +0100 Subject: [PATCH 14/17] Fix tests. --- ethcore/src/block.rs | 6 +++--- ethcore/src/ethereum/ethash.rs | 4 ++-- ethcore/src/ethereum/mod.rs | 2 +- ethcore/src/state.rs | 2 +- ethcore/src/tests/helpers.rs | 2 +- 5 files changed, 8 insertions(+), 8 deletions(-) diff --git a/ethcore/src/block.rs b/ethcore/src/block.rs index f3a4feaf0..4f23cf0a0 100644 --- a/ethcore/src/block.rs +++ b/ethcore/src/block.rs @@ -389,7 +389,7 @@ mod tests { let genesis_header = engine.spec().genesis_header(); let mut db_result = get_temp_journal_db(); let mut db = db_result.take(); - engine.spec().ensure_db_good(&mut db); + engine.spec().ensure_db_good(db.as_hashdb_mut()); let last_hashes = vec![genesis_header.hash()]; let b = OpenBlock::new(engine.deref(), db, &genesis_header, last_hashes, Address::zero(), vec![]); let b = b.close(); @@ -404,14 +404,14 @@ mod tests { let mut db_result = get_temp_journal_db(); let mut db = db_result.take(); - engine.spec().ensure_db_good(&mut db); + engine.spec().ensure_db_good(db.as_hashdb_mut()); let b = OpenBlock::new(engine.deref(), db, &genesis_header, vec![genesis_header.hash()], Address::zero(), vec![]).close().seal(engine.deref(), vec![]).unwrap(); let orig_bytes = b.rlp_bytes(); let orig_db = b.drain(); let mut db_result = get_temp_journal_db(); let mut db = db_result.take(); - engine.spec().ensure_db_good(&mut db); + engine.spec().ensure_db_good(db.as_hashdb_mut()); let e = enact_and_seal(&orig_bytes, engine.deref(), db, &genesis_header, vec![genesis_header.hash()]).unwrap(); assert_eq!(e.rlp_bytes(), orig_bytes); diff --git a/ethcore/src/ethereum/ethash.rs b/ethcore/src/ethereum/ethash.rs index b0c0e4a9f..a97002a2a 100644 --- a/ethcore/src/ethereum/ethash.rs +++ b/ethcore/src/ethereum/ethash.rs @@ -298,7 +298,7 @@ mod tests { let genesis_header = engine.spec().genesis_header(); let mut db_result = get_temp_journal_db(); let mut db = db_result.take(); - engine.spec().ensure_db_good(&mut db); + engine.spec().ensure_db_good(db.as_hashdb_mut()); let last_hashes = vec![genesis_header.hash()]; let b = OpenBlock::new(engine.deref(), db, &genesis_header, last_hashes, Address::zero(), vec![]); let b = b.close(); @@ -311,7 +311,7 @@ mod tests { let genesis_header = engine.spec().genesis_header(); let mut db_result = get_temp_journal_db(); let mut db = db_result.take(); - engine.spec().ensure_db_good(&mut db); + engine.spec().ensure_db_good(db.as_hashdb_mut()); let last_hashes = vec![genesis_header.hash()]; let mut b = OpenBlock::new(engine.deref(), db, &genesis_header, last_hashes, Address::zero(), vec![]); let mut uncle = Header::new(); diff --git a/ethcore/src/ethereum/mod.rs b/ethcore/src/ethereum/mod.rs index 0d1dcd8d5..8c2ae6b37 100644 --- a/ethcore/src/ethereum/mod.rs +++ b/ethcore/src/ethereum/mod.rs @@ -61,7 +61,7 @@ mod tests { let genesis_header = engine.spec().genesis_header(); let mut db_result = get_temp_journal_db(); let mut db = db_result.take(); - engine.spec().ensure_db_good(&mut db); + engine.spec().ensure_db_good(db.as_hashdb_mut()); let s = State::from_existing(db, genesis_header.state_root.clone(), engine.account_start_nonce()); assert_eq!(s.balance(&address_from_hex("0000000000000000000000000000000000000001")), U256::from(1u64)); assert_eq!(s.balance(&address_from_hex("0000000000000000000000000000000000000002")), U256::from(1u64)); diff --git a/ethcore/src/state.rs b/ethcore/src/state.rs index 519debcc1..60149d3eb 100644 --- a/ethcore/src/state.rs +++ b/ethcore/src/state.rs @@ -45,7 +45,7 @@ impl State { let mut root = H256::new(); { // init trie and reset root too null - let _ = SecTrieDBMut::new(db.deref_mut(), &mut root); + let _ = SecTrieDBMut::new(db.as_hashdb_mut(), &mut root); } State { diff --git a/ethcore/src/tests/helpers.rs b/ethcore/src/tests/helpers.rs index 0bb6b5015..56ea6b1d3 100644 --- a/ethcore/src/tests/helpers.rs +++ b/ethcore/src/tests/helpers.rs @@ -252,7 +252,7 @@ pub fn generate_dummy_empty_blockchain() -> GuardedTempResult { pub fn get_temp_journal_db() -> GuardedTempResult> { let temp = RandomTempPath::new(); - let journal_db = Box::new(OptionOneDB::new(temp.as_str())); + let journal_db: Box = Box::new(OptionOneDB::new(temp.as_str())); GuardedTempResult { _temp: temp, result: Some(journal_db) From 04af38bb0de889f33145bd36c22e6e67f82c4e25 Mon Sep 17 00:00:00 2001 From: Nikolay Volf Date: Fri, 11 Mar 2016 18:54:28 +0400 Subject: [PATCH 15/17] fix test compilation --- util/src/journaldb/archivedb.rs | 1 + util/src/journaldb/optiononedb.rs | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/util/src/journaldb/archivedb.rs b/util/src/journaldb/archivedb.rs index 28cc4130a..e7da1b737 100644 --- a/util/src/journaldb/archivedb.rs +++ b/util/src/journaldb/archivedb.rs @@ -166,6 +166,7 @@ mod tests { use common::*; use super::*; use hashdb::*; + use journaldb::traits::JournalDB; #[test] fn insert_same_in_fork() { diff --git a/util/src/journaldb/optiononedb.rs b/util/src/journaldb/optiononedb.rs index 58bb88277..dfa7c8ec1 100644 --- a/util/src/journaldb/optiononedb.rs +++ b/util/src/journaldb/optiononedb.rs @@ -61,7 +61,7 @@ impl OptionOneDB { } else { backing.put(&VERSION_KEY, &encode(&DB_VERSION)).expect("Error writing version to database"); } - + let counters = Some(Arc::new(RwLock::new(OptionOneDB::read_counters(&backing)))); OptionOneDB { overlay: MemoryDB::new(), @@ -383,6 +383,7 @@ mod tests { use common::*; use super::*; use hashdb::*; + use journaldb::traits::JournalDB; #[test] fn insert_same_in_fork() { From 179569f9f810b9b9d6352653eb892afb87b2df2a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tomasz=20Drwi=C4=99ga?= Date: Fri, 11 Mar 2016 16:01:18 +0100 Subject: [PATCH 16/17] Adding std::mem back --- util/bigint/src/uint.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/util/bigint/src/uint.rs b/util/bigint/src/uint.rs index 3997d2e66..c18ed839c 100644 --- a/util/bigint/src/uint.rs +++ b/util/bigint/src/uint.rs @@ -36,6 +36,9 @@ //! The functions here are designed to be fast. //! + +#[cfg(all(asm_available, target_arch="x86_64"))] +use std::mem; use std::fmt; use std::cmp; From 89dbc2ac25e8ce6eec482143869a160932e50e82 Mon Sep 17 00:00:00 2001 From: Nikolay Volf Date: Fri, 11 Mar 2016 18:08:29 +0300 Subject: [PATCH 17/17] [ci skip] update readme to exclude beta spec (stable is ok) --- README.md | 3 --- 1 file changed, 3 deletions(-) diff --git a/README.md b/README.md index 4fd2a53cc..47a27e30e 100644 --- a/README.md +++ b/README.md @@ -34,9 +34,6 @@ Then, download and build Parity: git clone https://github.com/ethcore/parity cd parity -# parity should be built with rust beta -multirust override beta - # build in release mode cargo build --release ```