Backports for beta 2.2.2 (#9976)

* version: bump beta to 2.2.2

* Add experimental RPCs flag (#9928)

* WiP

* Enable experimental RPCs.

* Keep existing blocks when restoring a Snapshot (#8643)

* Rename db_restore => client

* First step: make it compile!

* Second step: working implementation!

* Refactoring

* Fix tests

* PR Grumbles

* PR Grumbles WIP

* Migrate ancient blocks interating backward

* Early return in block migration if snapshot is aborted

* Remove RwLock getter (PR Grumble I)

* Remove dependency on `Client`: only used Traits

* Add test for recovering aborted snapshot recovery

* Add test for migrating old blocks

* Fix build

* PR Grumble I

* PR Grumble II

* PR Grumble III

* PR Grumble IV

* PR Grumble V

* PR Grumble VI

* Fix one test

* Fix test

* PR Grumble

* PR Grumbles

* PR Grumbles II

* Fix tests

* Release RwLock earlier

* Revert Cargo.lock

* Update _update ancient block_ logic: set local in `commit`

* Update typo in ethcore/src/snapshot/service.rs

Co-Authored-By: ngotchac <ngotchac@gmail.com>

* Adjust requests costs for light client (#9925)

* PIP Table Cost relative to average peers instead of max peers

* Add tracing in PIP new_cost_table

* Update stat peer_count

* Use number of leeching peers for Light serve costs

* Fix test::light_params_load_share_depends_on_max_peers (wrong type)

* Remove (now) useless test

* Remove `load_share` from LightParams.Config
Prevent div. by 0

* Add LEECHER_COUNT_FACTOR

* PR Grumble: u64 to u32 for f64 casting

* Prevent u32 overflow for avg_peer_count

* Add tests for LightSync::Statistics

* Fix empty steps (#9939)

* Don't send empty step twice or empty step then block.

* Perform basic validation of locally sealed blocks.

* Don't include empty step twice.

* prevent silent errors in daemon mode, closes #9367 (#9946)

* Fix a deadlock (#9952)

* Update informant:
  - decimal in Mgas/s
  - print every 5s (not randomly between 5s and 10s)

* Fix dead-lock in `blockchain.rs`

* Update locks ordering

* Fix light client informant while syncing (#9932)

* Add `is_idle` to LightSync to check importing status

* Use SyncStateWrapper to make sure is_idle gets updates

* Update is_major_import to use verified queue size as well

* Add comment for `is_idle`

* Add Debug to `SyncStateWrapper`

* `fn get` -> `fn into_inner`

*  ci: rearrange pipeline by logic (#9970)

* ci: rearrange pipeline by logic

* ci: rename docs script

* fix docker build (#9971)

* Deny unknown fields for chainspec (#9972)

* Add deny_unknown_fields to chainspec

* Add tests and fix existing one

* Remove serde_ignored dependency for chainspec

* Fix rpc test eth chain spec

* Fix starting_nonce_test spec

* Improve block and transaction propagation (#9954)

* Refactor sync to add priority tasks.

* Send priority tasks notifications.

* Propagate blocks, optimize transactions.

* Implement transaction propagation. Use sync_channel.

* Tone down info.

* Prevent deadlock by not waiting forever for sync lock.

* Fix lock order.

* Don't use sync_channel to prevent deadlocks.

* Fix tests.

* Fix unstable peers and slowness in sync (#9967)

* Don't sync all peers after each response

* Update formating

* Fix tests: add `continue_sync` to `Sync_step`

* Update ethcore/sync/src/chain/mod.rs

Co-Authored-By: ngotchac <ngotchac@gmail.com>

* fix rpc middlewares

* fix Cargo.lock

* json: resolve merge in spec

* rpc: fix starting_nonce_test

* ci: allow nightl job to fail
This commit is contained in:
Afri Schoedon
2018-11-29 10:57:49 +01:00
committed by GitHub
parent 5c56fc5023
commit 78ceec6c6e
67 changed files with 1854 additions and 653 deletions

View File

@@ -229,6 +229,7 @@ pub struct BlockChain {
cache_man: Mutex<CacheManager<CacheId>>,
pending_best_ancient_block: RwLock<Option<Option<BestAncientBlock>>>,
pending_best_block: RwLock<Option<BestBlock>>,
pending_block_hashes: RwLock<HashMap<BlockNumber, H256>>,
pending_block_details: RwLock<HashMap<H256, BlockDetails>>,
@@ -538,6 +539,7 @@ impl BlockChain {
block_receipts: RwLock::new(HashMap::new()),
db: db.clone(),
cache_man: Mutex::new(cache_man),
pending_best_ancient_block: RwLock::new(None),
pending_best_block: RwLock::new(None),
pending_block_hashes: RwLock::new(HashMap::new()),
pending_block_details: RwLock::new(HashMap::new()),
@@ -808,18 +810,7 @@ impl BlockChain {
}, is_best);
if is_ancient {
let mut best_ancient_block = self.best_ancient_block.write();
let ancient_number = best_ancient_block.as_ref().map_or(0, |b| b.number);
if self.block_hash(block_number + 1).is_some() {
batch.delete(db::COL_EXTRA, b"ancient");
*best_ancient_block = None;
} else if block_number > ancient_number {
batch.put(db::COL_EXTRA, b"ancient", &hash);
*best_ancient_block = Some(BestAncientBlock {
hash: hash,
number: block_number,
});
}
self.set_best_ancient_block(block_number, &hash, batch);
}
false
@@ -860,6 +851,84 @@ impl BlockChain {
}
}
/// Update the best ancient block to the given hash, after checking that
/// it's directly linked to the currently known best ancient block
pub fn update_best_ancient_block(&self, hash: &H256) {
// Get the block view of the next ancient block (it must
// be in DB at this point)
let block_view = match self.block(hash) {
Some(v) => v,
None => return,
};
// So that `best_ancient_block` gets unlocked before calling
// `set_best_ancient_block`
{
// Get the target hash ; if there are no ancient block,
// it means that the chain is already fully linked
// Release the `best_ancient_block` RwLock
let target_hash = {
let best_ancient_block = self.best_ancient_block.read();
let cur_ancient_block = match *best_ancient_block {
Some(ref b) => b,
None => return,
};
// Ensure that the new best ancient block is after the current one
if block_view.number() <= cur_ancient_block.number {
return;
}
cur_ancient_block.hash.clone()
};
let mut block_hash = *hash;
let mut is_linked = false;
loop {
if block_hash == target_hash {
is_linked = true;
break;
}
match self.block_details(&block_hash) {
Some(block_details) => {
block_hash = block_details.parent;
},
None => break,
}
}
if !is_linked {
trace!(target: "blockchain", "The given block {:x} is not linked to the known ancient block {:x}", hash, target_hash);
return;
}
}
let mut batch = self.db.key_value().transaction();
self.set_best_ancient_block(block_view.number(), hash, &mut batch);
self.db.key_value().write(batch).expect("Low level database error.");
}
/// Set the best ancient block with the given value: private method
/// `best_ancient_block` must not be locked, otherwise a DeadLock would occur
fn set_best_ancient_block(&self, block_number: BlockNumber, block_hash: &H256, batch: &mut DBTransaction) {
let mut pending_best_ancient_block = self.pending_best_ancient_block.write();
let ancient_number = self.best_ancient_block.read().as_ref().map_or(0, |b| b.number);
if self.block_hash(block_number + 1).is_some() {
trace!(target: "blockchain", "The two ends of the chain have met.");
batch.delete(db::COL_EXTRA, b"ancient");
*pending_best_ancient_block = Some(None);
} else if block_number > ancient_number {
trace!(target: "blockchain", "Updating the best ancient block to {}.", block_number);
batch.put(db::COL_EXTRA, b"ancient", &block_hash);
*pending_best_ancient_block = Some(Some(BestAncientBlock {
hash: *block_hash,
number: block_number,
}));
}
}
/// Insert an epoch transition. Provide an epoch number being transitioned to
/// and epoch transition object.
///
@@ -1112,15 +1181,21 @@ impl BlockChain {
/// Apply pending insertion updates
pub fn commit(&self) {
let mut pending_best_ancient_block = self.pending_best_ancient_block.write();
let mut pending_best_block = self.pending_best_block.write();
let mut pending_write_hashes = self.pending_block_hashes.write();
let mut pending_block_details = self.pending_block_details.write();
let mut pending_write_txs = self.pending_transaction_addresses.write();
let mut best_block = self.best_block.write();
let mut best_ancient_block = self.best_ancient_block.write();
let mut write_block_details = self.block_details.write();
let mut write_hashes = self.block_hashes.write();
let mut write_txs = self.transaction_addresses.write();
// update best ancient block
if let Some(block_option) = pending_best_ancient_block.take() {
*best_ancient_block = block_option;
}
// update best block
if let Some(block) = pending_best_block.take() {
*best_block = block;

View File

@@ -15,7 +15,7 @@
// along with Parity. If not, see <http://www.gnu.org/licenses/>.
use bytes::Bytes;
use ethereum_types::H256;
use ethereum_types::{H256, U256};
use transaction::UnverifiedTransaction;
use blockchain::ImportRoute;
use std::time::Duration;
@@ -141,7 +141,15 @@ pub trait ChainNotify : Send + Sync {
}
/// fires when chain broadcasts a message
fn broadcast(&self, _message_type: ChainMessageType) {}
fn broadcast(&self, _message_type: ChainMessageType) {
// does nothing by default
}
/// fires when new block is about to be imported
/// implementations should be light
fn block_pre_import(&self, _bytes: &Bytes, _hash: &H256, _difficulty: &U256) {
// does nothing by default
}
/// fires when new transactions are received from a peer
fn transactions_received(&self,

View File

@@ -32,7 +32,7 @@ use kvdb::{DBValue, KeyValueDB, DBTransaction};
// other
use ethereum_types::{H256, Address, U256};
use block::{IsBlock, LockedBlock, Drain, ClosedBlock, OpenBlock, enact_verified, SealedBlock};
use blockchain::{BlockChain, BlockChainDB, BlockProvider, TreeRoute, ImportRoute, TransactionAddress, ExtrasInsert};
use blockchain::{BlockReceipts, BlockChain, BlockChainDB, BlockProvider, TreeRoute, ImportRoute, TransactionAddress, ExtrasInsert};
use client::ancient_import::AncientVerifier;
use client::{
Nonce, Balance, ChainInfo, BlockInfo, CallContract, TransactionInfo,
@@ -66,7 +66,7 @@ use ethcore_miner::pool::VerifiedTransaction;
use parking_lot::{Mutex, RwLock};
use rand::OsRng;
use receipt::{Receipt, LocalizedReceipt};
use snapshot::{self, io as snapshot_io};
use snapshot::{self, io as snapshot_io, SnapshotClient};
use spec::Spec;
use state_db::StateDB;
use state::{self, State};
@@ -881,7 +881,7 @@ impl Client {
/// Flush the block import queue.
pub fn flush_queue(&self) {
self.importer.block_queue.flush();
while !self.importer.block_queue.queue_info().is_empty() {
while !self.importer.block_queue.is_empty() {
self.import_verified_blocks();
}
}
@@ -1005,6 +1005,16 @@ impl Client {
self.importer.miner.clone()
}
#[cfg(test)]
pub fn state_db(&self) -> ::parking_lot::RwLockReadGuard<StateDB> {
self.state_db.read()
}
#[cfg(test)]
pub fn chain(&self) -> Arc<BlockChain> {
self.chain.read().clone()
}
/// Replace io channel. Useful for testing.
pub fn set_io_channel(&self, io_channel: IoChannel<ClientIoMessage>) {
*self.io_channel.write() = io_channel;
@@ -1413,8 +1423,21 @@ impl ImportBlock for Client {
bail!(EthcoreErrorKind::Block(BlockError::UnknownParent(unverified.parent_hash())));
}
let raw = if self.importer.block_queue.is_empty() {
Some((
unverified.bytes.clone(),
unverified.header.hash(),
*unverified.header.difficulty(),
))
} else { None };
match self.importer.block_queue.import(unverified) {
Ok(res) => Ok(res),
Ok(hash) => {
if let Some((raw, hash, difficulty)) = raw {
self.notify(move |n| n.block_pre_import(&raw, &hash, &difficulty));
}
Ok(hash)
},
// we only care about block errors (not import errors)
Err((block, EthcoreError(EthcoreErrorKind::Block(err), _))) => {
self.importer.bad_blocks.report(block.bytes, format!("{:?}", err));
@@ -1817,7 +1840,7 @@ impl BlockChainClient for Client {
Some(receipt)
}
fn block_receipts(&self, id: BlockId) -> Option<Vec<LocalizedReceipt>> {
fn localized_block_receipts(&self, id: BlockId) -> Option<Vec<LocalizedReceipt>> {
let hash = self.block_hash(id)?;
let chain = self.chain.read();
@@ -1860,14 +1883,18 @@ impl BlockChainClient for Client {
self.state_db.read().journal_db().state(hash)
}
fn encoded_block_receipts(&self, hash: &H256) -> Option<Bytes> {
self.chain.read().block_receipts(hash).map(|receipts| ::rlp::encode(&receipts))
fn block_receipts(&self, hash: &H256) -> Option<BlockReceipts> {
self.chain.read().block_receipts(hash)
}
fn queue_info(&self) -> BlockQueueInfo {
self.importer.block_queue.queue_info()
}
fn is_queue_empty(&self) -> bool {
self.importer.block_queue.is_empty()
}
fn clear_queue(&self) {
self.importer.block_queue.clear();
}
@@ -2277,26 +2304,37 @@ impl ScheduleInfo for Client {
impl ImportSealedBlock for Client {
fn import_sealed_block(&self, block: SealedBlock) -> EthcoreResult<H256> {
let h = block.header().hash();
let start = Instant::now();
let raw = block.rlp_bytes();
let header = block.header().clone();
let hash = header.hash();
self.notify(|n| n.block_pre_import(&raw, &hash, header.difficulty()));
let route = {
// Do a super duper basic verification to detect potential bugs
if let Err(e) = self.engine.verify_block_basic(&header) {
self.importer.bad_blocks.report(
block.rlp_bytes(),
format!("Detected an issue with locally sealed block: {}", e),
);
return Err(e.into());
}
// scope for self.import_lock
let _import_lock = self.importer.import_lock.lock();
trace_time!("import_sealed_block");
let number = block.header().number();
let block_data = block.rlp_bytes();
let header = block.header().clone();
let route = self.importer.commit_block(block, &header, encoded::Block::new(block_data), self);
trace!(target: "client", "Imported sealed block #{} ({})", number, h);
trace!(target: "client", "Imported sealed block #{} ({})", header.number(), hash);
self.state_db.write().sync_cache(&route.enacted, &route.retracted, false);
route
};
let route = ChainRoute::from([route].as_ref());
self.importer.miner.chain_new_blocks(
self,
&[h.clone()],
&[hash],
&[],
route.enacted(),
route.retracted(),
@@ -2304,16 +2342,16 @@ impl ImportSealedBlock for Client {
);
self.notify(|notify| {
notify.new_blocks(
vec![h.clone()],
vec![hash],
vec![],
route.clone(),
vec![h.clone()],
vec![hash],
vec![],
start.elapsed(),
);
});
self.db.read().key_value().flush().expect("DB flush failed.");
Ok(h)
Ok(hash)
}
}
@@ -2406,6 +2444,8 @@ impl ProvingBlockChainClient for Client {
}
}
impl SnapshotClient for Client {}
impl Drop for Client {
fn drop(&mut self) {
self.engine.stop();
@@ -2504,7 +2544,7 @@ mod tests {
use test_helpers::{generate_dummy_client_with_data};
let client = generate_dummy_client_with_data(2, 2, &[1.into(), 1.into()]);
let receipts = client.block_receipts(BlockId::Latest).unwrap();
let receipts = client.localized_block_receipts(BlockId::Latest).unwrap();
assert_eq!(receipts.len(), 2);
assert_eq!(receipts[0].transaction_index, 0);

View File

@@ -686,7 +686,7 @@ impl BlockChainClient for TestBlockChainClient {
self.receipts.read().get(&id).cloned()
}
fn block_receipts(&self, _id: BlockId) -> Option<Vec<LocalizedReceipt>> {
fn localized_block_receipts(&self, _id: BlockId) -> Option<Vec<LocalizedReceipt>> {
Some(self.receipts.read().values().cloned().collect())
}
@@ -789,16 +789,14 @@ impl BlockChainClient for TestBlockChainClient {
None
}
fn encoded_block_receipts(&self, hash: &H256) -> Option<Bytes> {
fn block_receipts(&self, hash: &H256) -> Option<BlockReceipts> {
// starts with 'f' ?
if *hash > H256::from("f000000000000000000000000000000000000000000000000000000000000000") {
let receipt = BlockReceipts::new(vec![Receipt::new(
TransactionOutcome::StateRoot(H256::zero()),
U256::zero(),
vec![])]);
let mut rlp = RlpStream::new();
rlp.append(&receipt);
return Some(rlp.out());
return Some(receipt);
}
None
}

View File

@@ -20,7 +20,7 @@ use std::sync::Arc;
use itertools::Itertools;
use block::{OpenBlock, SealedBlock, ClosedBlock};
use blockchain::TreeRoute;
use blockchain::{BlockReceipts, TreeRoute};
use client::Mode;
use encoded;
use vm::LastHashes;
@@ -282,7 +282,7 @@ pub trait BlockChainClient : Sync + Send + AccountData + BlockChain + CallContra
fn transaction_receipt(&self, id: TransactionId) -> Option<LocalizedReceipt>;
/// Get localized receipts for all transaction in given block.
fn block_receipts(&self, id: BlockId) -> Option<Vec<LocalizedReceipt>>;
fn localized_block_receipts(&self, id: BlockId) -> Option<Vec<LocalizedReceipt>>;
/// Get a tree route between `from` and `to`.
/// See `BlockChain::tree_route`.
@@ -294,12 +294,17 @@ pub trait BlockChainClient : Sync + Send + AccountData + BlockChain + CallContra
/// Get latest state node
fn state_data(&self, hash: &H256) -> Option<Bytes>;
/// Get raw block receipts data by block header hash.
fn encoded_block_receipts(&self, hash: &H256) -> Option<Bytes>;
/// Get block receipts data by block header hash.
fn block_receipts(&self, hash: &H256) -> Option<BlockReceipts>;
/// Get block queue information.
fn queue_info(&self) -> BlockQueueInfo;
/// Returns true if block queue is empty.
fn is_queue_empty(&self) -> bool {
self.queue_info().is_empty()
}
/// Clear block queue and abort all import activity.
fn clear_queue(&self);

View File

@@ -16,8 +16,8 @@
//! A blockchain engine that supports a non-instant BFT proof-of-authority.
use std::collections::{BTreeMap, HashSet};
use std::fmt;
use std::collections::{BTreeMap, BTreeSet, HashSet};
use std::{cmp, fmt};
use std::iter::FromIterator;
use std::ops::Deref;
use std::sync::atomic::{AtomicUsize, AtomicBool, Ordering as AtomicOrdering};
@@ -123,10 +123,10 @@ struct Step {
}
impl Step {
fn load(&self) -> usize { self.inner.load(AtomicOrdering::SeqCst) }
fn load(&self) -> u64 { self.inner.load(AtomicOrdering::SeqCst) as u64 }
fn duration_remaining(&self) -> Duration {
let now = unix_now();
let expected_seconds = (self.load() as u64)
let expected_seconds = self.load()
.checked_add(1)
.and_then(|ctr| ctr.checked_mul(self.duration as u64))
.map(Duration::from_secs);
@@ -162,8 +162,8 @@ impl Step {
}
}
fn check_future(&self, given: usize) -> Result<(), Option<OutOfBounds<u64>>> {
const REJECTED_STEP_DRIFT: usize = 4;
fn check_future(&self, given: u64) -> Result<(), Option<OutOfBounds<u64>>> {
const REJECTED_STEP_DRIFT: u64 = 4;
// Verify if the step is correct.
if given <= self.load() {
@@ -182,8 +182,8 @@ impl Step {
let d = self.duration as u64;
Err(Some(OutOfBounds {
min: None,
max: Some(d * current as u64),
found: d * given as u64,
max: Some(d * current),
found: d * given,
}))
} else {
Ok(())
@@ -192,8 +192,8 @@ impl Step {
}
// Chain scoring: total weight is sqrt(U256::max_value())*height - step
fn calculate_score(parent_step: U256, current_step: U256, current_empty_steps: U256) -> U256 {
U256::from(U128::max_value()) + parent_step - current_step + current_empty_steps
fn calculate_score(parent_step: u64, current_step: u64, current_empty_steps: usize) -> U256 {
U256::from(U128::max_value()) + U256::from(parent_step) - U256::from(current_step) + U256::from(current_empty_steps)
}
struct EpochManager {
@@ -284,13 +284,26 @@ impl EpochManager {
/// A message broadcast by authorities when it's their turn to seal a block but there are no
/// transactions. Other authorities accumulate these messages and later include them in the seal as
/// proof.
#[derive(Clone, Debug)]
#[derive(Clone, Debug, PartialEq, Eq)]
struct EmptyStep {
signature: H520,
step: usize,
step: u64,
parent_hash: H256,
}
impl PartialOrd for EmptyStep {
fn partial_cmp(&self, other: &Self) -> Option<cmp::Ordering> {
Some(self.cmp(other))
}
}
impl Ord for EmptyStep {
fn cmp(&self, other: &Self) -> cmp::Ordering {
self.step.cmp(&other.step)
.then_with(|| self.parent_hash.cmp(&other.parent_hash))
.then_with(|| self.signature.cmp(&other.signature))
}
}
impl EmptyStep {
fn from_sealed(sealed_empty_step: SealedEmptyStep, parent_hash: &H256) -> EmptyStep {
let signature = sealed_empty_step.signature;
@@ -353,7 +366,7 @@ pub fn empty_step_full_rlp(signature: &H520, empty_step_rlp: &[u8]) -> Vec<u8> {
s.out()
}
pub fn empty_step_rlp(step: usize, parent_hash: &H256) -> Vec<u8> {
pub fn empty_step_rlp(step: u64, parent_hash: &H256) -> Vec<u8> {
let mut s = RlpStream::new_list(2);
s.append(&step).append(parent_hash);
s.out()
@@ -365,7 +378,7 @@ pub fn empty_step_rlp(step: usize, parent_hash: &H256) -> Vec<u8> {
/// empty message is included.
struct SealedEmptyStep {
signature: H520,
step: usize,
step: u64,
}
impl Encodable for SealedEmptyStep {
@@ -399,7 +412,7 @@ pub struct AuthorityRound {
validators: Box<ValidatorSet>,
validate_score_transition: u64,
validate_step_transition: u64,
empty_steps: Mutex<Vec<EmptyStep>>,
empty_steps: Mutex<BTreeSet<EmptyStep>>,
epoch_manager: Mutex<EpochManager>,
immediate_transitions: bool,
block_reward: U256,
@@ -494,7 +507,7 @@ fn header_expected_seal_fields(header: &Header, empty_steps_transition: u64) ->
}
}
fn header_step(header: &Header, empty_steps_transition: u64) -> Result<usize, ::rlp::DecoderError> {
fn header_step(header: &Header, empty_steps_transition: u64) -> Result<u64, ::rlp::DecoderError> {
let expected_seal_fields = header_expected_seal_fields(header, empty_steps_transition);
Rlp::new(&header.seal().get(0).expect(
&format!("was either checked with verify_block_basic or is genesis; has {} fields; qed (Make sure the spec file has a correct genesis seal)", expected_seal_fields))).as_val()
@@ -533,17 +546,17 @@ fn header_empty_steps_signers(header: &Header, empty_steps_transition: u64) -> R
}
}
fn step_proposer(validators: &ValidatorSet, bh: &H256, step: usize) -> Address {
let proposer = validators.get(bh, step);
fn step_proposer(validators: &ValidatorSet, bh: &H256, step: u64) -> Address {
let proposer = validators.get(bh, step as usize);
trace!(target: "engine", "Fetched proposer for step {}: {}", step, proposer);
proposer
}
fn is_step_proposer(validators: &ValidatorSet, bh: &H256, step: usize, address: &Address) -> bool {
fn is_step_proposer(validators: &ValidatorSet, bh: &H256, step: u64, address: &Address) -> bool {
step_proposer(validators, bh, step) == *address
}
fn verify_timestamp(step: &Step, header_step: usize) -> Result<(), BlockError> {
fn verify_timestamp(step: &Step, header_step: u64) -> Result<(), BlockError> {
match step.check_future(header_step) {
Err(None) => {
trace!(target: "engine", "verify_timestamp: block from the future");
@@ -564,7 +577,7 @@ fn verify_external(header: &Header, validators: &ValidatorSet, empty_steps_trans
let header_step = header_step(header, empty_steps_transition)?;
let proposer_signature = header_signature(header, empty_steps_transition)?;
let correct_proposer = validators.get(header.parent_hash(), header_step);
let correct_proposer = validators.get(header.parent_hash(), header_step as usize);
let is_invalid_proposer = *header.author() != correct_proposer || {
let empty_steps_rlp = if header.number() >= empty_steps_transition {
Some(header_empty_steps_raw(header))
@@ -634,13 +647,13 @@ impl AuthorityRound {
panic!("authority_round: step duration can't be zero")
}
let should_timeout = our_params.start_step.is_none();
let initial_step = our_params.start_step.unwrap_or_else(|| (unix_now().as_secs() / (our_params.step_duration as u64))) as usize;
let initial_step = our_params.start_step.unwrap_or_else(|| (unix_now().as_secs() / (our_params.step_duration as u64)));
let engine = Arc::new(
AuthorityRound {
transition_service: IoService::<()>::start()?,
step: Arc::new(PermissionedStep {
inner: Step {
inner: AtomicUsize::new(initial_step),
inner: AtomicUsize::new(initial_step as usize),
calibrate: our_params.start_step.is_none(),
duration: our_params.step_duration,
},
@@ -651,7 +664,7 @@ impl AuthorityRound {
validators: our_params.validators,
validate_score_transition: our_params.validate_score_transition,
validate_step_transition: our_params.validate_step_transition,
empty_steps: Mutex::new(Vec::new()),
empty_steps: Default::default(),
epoch_manager: Mutex::new(EpochManager::blank()),
immediate_transitions: our_params.immediate_transitions,
block_reward: our_params.block_reward,
@@ -699,22 +712,41 @@ impl AuthorityRound {
})
}
fn empty_steps(&self, from_step: U256, to_step: U256, parent_hash: H256) -> Vec<EmptyStep> {
self.empty_steps.lock().iter().filter(|e| {
U256::from(e.step) > from_step &&
U256::from(e.step) < to_step &&
e.parent_hash == parent_hash
}).cloned().collect()
fn empty_steps(&self, from_step: u64, to_step: u64, parent_hash: H256) -> Vec<EmptyStep> {
let from = EmptyStep {
step: from_step + 1,
parent_hash,
signature: Default::default(),
};
let to = EmptyStep {
step: to_step,
parent_hash: Default::default(),
signature: Default::default(),
};
if from >= to {
return vec![];
}
self.empty_steps.lock()
.range(from..to)
.filter(|e| e.parent_hash == parent_hash)
.cloned()
.collect()
}
fn clear_empty_steps(&self, step: U256) {
fn clear_empty_steps(&self, step: u64) {
// clear old `empty_steps` messages
self.empty_steps.lock().retain(|e| U256::from(e.step) > step);
let mut empty_steps = self.empty_steps.lock();
*empty_steps = empty_steps.split_off(&EmptyStep {
step: step + 1,
parent_hash: Default::default(),
signature: Default::default(),
});
}
fn handle_empty_step_message(&self, empty_step: EmptyStep) {
let mut empty_steps = self.empty_steps.lock();
empty_steps.push(empty_step);
self.empty_steps.lock().insert(empty_step);
}
fn generate_empty_step(&self, parent_hash: &H256) {
@@ -744,7 +776,7 @@ impl AuthorityRound {
}
}
fn report_skipped(&self, header: &Header, current_step: usize, parent_step: usize, validators: &ValidatorSet, set_number: u64) {
fn report_skipped(&self, header: &Header, current_step: u64, parent_step: u64, validators: &ValidatorSet, set_number: u64) {
// we're building on top of the genesis block so don't report any skipped steps
if header.number() == 1 {
return;
@@ -937,12 +969,12 @@ impl Engine<EthereumMachine> for AuthorityRound {
let current_step = self.step.inner.load();
let current_empty_steps_len = if header.number() >= self.empty_steps_transition {
self.empty_steps(parent_step.into(), current_step.into(), parent.hash()).len()
self.empty_steps(parent_step, current_step, parent.hash()).len()
} else {
0
};
let score = calculate_score(parent_step.into(), current_step.into(), current_empty_steps_len.into());
let score = calculate_score(parent_step, current_step, current_empty_steps_len);
header.set_difficulty(score);
}
@@ -986,8 +1018,8 @@ impl Engine<EthereumMachine> for AuthorityRound {
}
let header = block.header();
let parent_step: U256 = header_step(parent, self.empty_steps_transition)
.expect("Header has been verified; qed").into();
let parent_step = header_step(parent, self.empty_steps_transition)
.expect("Header has been verified; qed");
let step = self.step.inner.load();
@@ -1022,7 +1054,7 @@ impl Engine<EthereumMachine> for AuthorityRound {
if is_step_proposer(&*validators, header.parent_hash(), step, header.author()) {
// this is guarded against by `can_propose` unless the block was signed
// on the same step (implies same key) and on a different node.
if parent_step == step.into() {
if parent_step == step {
warn!("Attempted to seal block on the same step as parent. Is this authority sealing with more than one node?");
return Seal::None;
}
@@ -1034,7 +1066,10 @@ impl Engine<EthereumMachine> for AuthorityRound {
block.transactions().is_empty() &&
empty_steps.len() < self.maximum_empty_steps {
self.generate_empty_step(header.parent_hash());
if self.step.can_propose.compare_and_swap(true, false, AtomicOrdering::SeqCst) {
self.generate_empty_step(header.parent_hash());
}
return Seal::None;
}
@@ -1058,7 +1093,7 @@ impl Engine<EthereumMachine> for AuthorityRound {
// report any skipped primaries between the parent block and
// the block we're sealing, unless we have empty steps enabled
if header.number() < self.empty_steps_transition {
self.report_skipped(header, step, u64::from(parent_step) as usize, &*validators, set_number);
self.report_skipped(header, step, parent_step, &*validators, set_number);
}
let mut fields = vec![
@@ -1593,12 +1628,12 @@ mod tests {
// Two validators.
// Spec starts with step 2.
header.set_difficulty(calculate_score(U256::from(0), U256::from(2), U256::zero()));
header.set_difficulty(calculate_score(0, 2, 0));
let signature = tap.sign(addr, Some("0".into()), header.bare_hash()).unwrap();
header.set_seal(vec![encode(&2usize), encode(&(&*signature as &[u8]))]);
assert!(engine.verify_block_family(&header, &parent_header).is_ok());
assert!(engine.verify_block_external(&header).is_err());
header.set_difficulty(calculate_score(U256::from(0), U256::from(1), U256::zero()));
header.set_difficulty(calculate_score(0, 1, 0));
let signature = tap.sign(addr, Some("0".into()), header.bare_hash()).unwrap();
header.set_seal(vec![encode(&1usize), encode(&(&*signature as &[u8]))]);
assert!(engine.verify_block_family(&header, &parent_header).is_ok());
@@ -1622,7 +1657,7 @@ mod tests {
// Two validators.
// Spec starts with step 2.
header.set_difficulty(calculate_score(U256::from(0), U256::from(1), U256::zero()));
header.set_difficulty(calculate_score(0, 1, 0));
let signature = tap.sign(addr, Some("0".into()), header.bare_hash()).unwrap();
header.set_seal(vec![encode(&1usize), encode(&(&*signature as &[u8]))]);
assert!(engine.verify_block_family(&header, &parent_header).is_ok());
@@ -1650,10 +1685,10 @@ mod tests {
// Two validators.
// Spec starts with step 2.
header.set_seal(vec![encode(&5usize), encode(&(&*signature as &[u8]))]);
header.set_difficulty(calculate_score(U256::from(4), U256::from(5), U256::zero()));
header.set_difficulty(calculate_score(4, 5, 0));
assert!(engine.verify_block_family(&header, &parent_header).is_ok());
header.set_seal(vec![encode(&3usize), encode(&(&*signature as &[u8]))]);
header.set_difficulty(calculate_score(U256::from(4), U256::from(3), U256::zero()));
header.set_difficulty(calculate_score(4, 3, 0));
assert!(engine.verify_block_family(&header, &parent_header).is_err());
}
@@ -1687,7 +1722,7 @@ mod tests {
parent_header.set_seal(vec![encode(&1usize)]);
parent_header.set_gas_limit("222222".parse::<U256>().unwrap());
let mut header: Header = Header::default();
header.set_difficulty(calculate_score(U256::from(1), U256::from(3), U256::zero()));
header.set_difficulty(calculate_score(1, 3, 0));
header.set_gas_limit("222222".parse::<U256>().unwrap());
header.set_seal(vec![encode(&3usize)]);
@@ -1801,14 +1836,14 @@ mod tests {
(spec, tap, accounts)
}
fn empty_step(engine: &EthEngine, step: usize, parent_hash: &H256) -> EmptyStep {
fn empty_step(engine: &EthEngine, step: u64, parent_hash: &H256) -> EmptyStep {
let empty_step_rlp = super::empty_step_rlp(step, parent_hash);
let signature = engine.sign(keccak(&empty_step_rlp)).unwrap().into();
let parent_hash = parent_hash.clone();
EmptyStep { step, signature, parent_hash }
}
fn sealed_empty_step(engine: &EthEngine, step: usize, parent_hash: &H256) -> SealedEmptyStep {
fn sealed_empty_step(engine: &EthEngine, step: u64, parent_hash: &H256) -> SealedEmptyStep {
let empty_step_rlp = super::empty_step_rlp(step, parent_hash);
let signature = engine.sign(keccak(&empty_step_rlp)).unwrap().into();
SealedEmptyStep { signature, step }
@@ -1844,6 +1879,11 @@ mod tests {
// we've received the message
assert!(notify.messages.read().contains(&empty_step_rlp));
let len = notify.messages.read().len();
// make sure that we don't generate empty step for the second time
assert_eq!(engine.generate_seal(b1.block(), &genesis_header), Seal::None);
assert_eq!(len, notify.messages.read().len());
}
#[test]
@@ -2058,7 +2098,7 @@ mod tests {
let empty_step3 = sealed_empty_step(engine, 3, &parent_header.hash());
let empty_steps = vec![empty_step2, empty_step3];
header.set_difficulty(calculate_score(U256::from(0), U256::from(4), U256::from(2)));
header.set_difficulty(calculate_score(0, 4, 2));
let signature = tap.sign(addr1, Some("1".into()), header.bare_hash()).unwrap();
header.set_seal(vec![
encode(&4usize),
@@ -2173,4 +2213,52 @@ mod tests {
BTreeMap::default(),
);
}
#[test]
fn test_empty_steps() {
let last_benign = Arc::new(AtomicUsize::new(0));
let params = AuthorityRoundParams {
step_duration: 4,
start_step: Some(1),
validators: Box::new(TestSet::new(Default::default(), last_benign.clone())),
validate_score_transition: 0,
validate_step_transition: 0,
immediate_transitions: true,
maximum_uncle_count_transition: 0,
maximum_uncle_count: 0,
empty_steps_transition: 0,
maximum_empty_steps: 10,
block_reward: Default::default(),
block_reward_contract_transition: 0,
block_reward_contract: Default::default(),
};
let mut c_params = ::spec::CommonParams::default();
c_params.gas_limit_bound_divisor = 5.into();
let machine = ::machine::EthereumMachine::regular(c_params, Default::default());
let engine = AuthorityRound::new(params, machine).unwrap();
let parent_hash: H256 = 1.into();
let signature = H520::default();
let step = |step: u64| EmptyStep {
step,
parent_hash,
signature,
};
engine.handle_empty_step_message(step(1));
engine.handle_empty_step_message(step(3));
engine.handle_empty_step_message(step(2));
engine.handle_empty_step_message(step(1));
assert_eq!(engine.empty_steps(0, 4, parent_hash), vec![step(1), step(2), step(3)]);
assert_eq!(engine.empty_steps(2, 3, parent_hash), vec![]);
assert_eq!(engine.empty_steps(2, 4, parent_hash), vec![step(3)]);
engine.clear_empty_steps(2);
assert_eq!(engine.empty_steps(0, 3, parent_hash), vec![]);
assert_eq!(engine.empty_steps(0, 4, parent_hash), vec![step(3)]);
}
}

View File

@@ -139,6 +139,9 @@ extern crate trace_time;
#[cfg_attr(test, macro_use)]
extern crate evm;
#[cfg(test)]
extern crate env_logger;
pub extern crate ethstore;
#[macro_use]

View File

@@ -576,7 +576,7 @@ impl Miner {
trace!(target: "miner", "requires_reseal: sealing enabled");
// Disable sealing if there were no requests for SEALING_TIMEOUT_IN_BLOCKS
let had_requests = sealing.last_request.map(|last_request|
let had_requests = sealing.last_request.map(|last_request|
best_block.saturating_sub(last_request) <= SEALING_TIMEOUT_IN_BLOCKS
).unwrap_or(false);

View File

@@ -65,6 +65,8 @@ pub enum Error {
BadEpochProof(u64),
/// Wrong chunk format.
WrongChunkFormat(String),
/// Unlinked ancient block chain
UnlinkedAncientBlockChain,
}
impl fmt::Display for Error {
@@ -91,6 +93,7 @@ impl fmt::Display for Error {
Error::SnapshotsUnsupported => write!(f, "Snapshots unsupported by consensus engine."),
Error::BadEpochProof(i) => write!(f, "Bad epoch proof for transition to epoch {}", i),
Error::WrongChunkFormat(ref msg) => write!(f, "Wrong chunk format: {}", msg),
Error::UnlinkedAncientBlockChain => write!(f, "Unlinked ancient blocks chain"),
}
}
}

View File

@@ -56,7 +56,7 @@ use rand::{Rng, OsRng};
pub use self::error::Error;
pub use self::consensus::*;
pub use self::service::{Service, DatabaseRestore};
pub use self::service::{SnapshotClient, Service, DatabaseRestore};
pub use self::traits::SnapshotService;
pub use self::watcher::Watcher;
pub use types::snapshot_manifest::ManifestData;

View File

@@ -22,12 +22,13 @@ use std::fs::{self, File};
use std::path::PathBuf;
use std::sync::Arc;
use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
use std::cmp;
use super::{ManifestData, StateRebuilder, Rebuilder, RestorationStatus, SnapshotService, MAX_CHUNK_SIZE};
use super::io::{SnapshotReader, LooseReader, SnapshotWriter, LooseWriter};
use blockchain::{BlockChain, BlockChainDB, BlockChainDBHandler};
use client::{Client, ChainInfo, ClientIoMessage};
use client::{BlockInfo, BlockChainClient, Client, ChainInfo, ClientIoMessage};
use engines::EthEngine;
use error::{Error, ErrorKind as SnapshotErrorKind};
use snapshot::{Error as SnapshotError};
@@ -40,6 +41,7 @@ use ethereum_types::H256;
use parking_lot::{Mutex, RwLock, RwLockReadGuard};
use bytes::Bytes;
use journaldb::Algorithm;
use kvdb::DBTransaction;
use snappy;
/// Helper for removing directories in case of error.
@@ -203,6 +205,9 @@ impl Restoration {
/// Type alias for client io channel.
pub type Channel = IoChannel<ClientIoMessage>;
/// Trait alias for the Client Service used
pub trait SnapshotClient: BlockChainClient + BlockInfo + DatabaseRestore {}
/// Snapshot service parameters.
pub struct ServiceParams {
/// The consensus engine this is built on.
@@ -219,7 +224,7 @@ pub struct ServiceParams {
/// Usually "<chain hash>/snapshot"
pub snapshot_root: PathBuf,
/// A handle for database restoration.
pub db_restore: Arc<DatabaseRestore>,
pub client: Arc<SnapshotClient>,
}
/// `SnapshotService` implementation.
@@ -236,7 +241,7 @@ pub struct Service {
genesis_block: Bytes,
state_chunks: AtomicUsize,
block_chunks: AtomicUsize,
db_restore: Arc<DatabaseRestore>,
client: Arc<SnapshotClient>,
progress: super::Progress,
taking_snapshot: AtomicBool,
restoring_snapshot: AtomicBool,
@@ -257,7 +262,7 @@ impl Service {
genesis_block: params.genesis_block,
state_chunks: AtomicUsize::new(0),
block_chunks: AtomicUsize::new(0),
db_restore: params.db_restore,
client: params.client,
progress: Default::default(),
taking_snapshot: AtomicBool::new(false),
restoring_snapshot: AtomicBool::new(false),
@@ -334,12 +339,110 @@ impl Service {
// replace one the client's database with our own.
fn replace_client_db(&self) -> Result<(), Error> {
let our_db = self.restoration_db();
let migrated_blocks = self.migrate_blocks()?;
trace!(target: "snapshot", "Migrated {} ancient blocks", migrated_blocks);
self.db_restore.restore_db(&*our_db.to_string_lossy())?;
let rest_db = self.restoration_db();
self.client.restore_db(&*rest_db.to_string_lossy())?;
Ok(())
}
// Migrate the blocks in the current DB into the new chain
fn migrate_blocks(&self) -> Result<usize, Error> {
// Count the number of migrated blocks
let mut count = 0;
let rest_db = self.restoration_db();
let cur_chain_info = self.client.chain_info();
let next_db = self.restoration_db_handler.open(&rest_db)?;
let next_chain = BlockChain::new(Default::default(), &[], next_db.clone());
let next_chain_info = next_chain.chain_info();
// The old database looks like this:
// [genesis, best_ancient_block] ... [first_block, best_block]
// If we are fully synced neither `best_ancient_block` nor `first_block` is set, and we can assume that the whole range from [genesis, best_block] is imported.
// The new database only contains the tip of the chain ([first_block, best_block]),
// so the useful set of blocks is defined as:
// [0 ... min(new.first_block, best_ancient_block or best_block)]
let find_range = || -> Option<(H256, H256)> {
let next_available_from = next_chain_info.first_block_number?;
let cur_available_to = cur_chain_info.ancient_block_number.unwrap_or(cur_chain_info.best_block_number);
let highest_block_num = cmp::min(next_available_from.saturating_sub(1), cur_available_to);
if highest_block_num == 0 {
return None;
}
trace!(target: "snapshot", "Trying to import ancient blocks until {}", highest_block_num);
// Here we start from the highest block number and go backward to 0,
// thus starting at `highest_block_num` and targetting `0`.
let target_hash = self.client.block_hash(BlockId::Number(0))?;
let start_hash = self.client.block_hash(BlockId::Number(highest_block_num))?;
Some((start_hash, target_hash))
};
let (start_hash, target_hash) = match find_range() {
Some(x) => x,
None => return Ok(0),
};
let mut batch = DBTransaction::new();
let mut parent_hash = start_hash;
while parent_hash != target_hash {
// Early return if restoration is aborted
if !self.restoring_snapshot.load(Ordering::SeqCst) {
return Ok(count);
}
let block = self.client.block(BlockId::Hash(parent_hash)).ok_or(::snapshot::error::Error::UnlinkedAncientBlockChain)?;
parent_hash = block.parent_hash();
let block_number = block.number();
let block_receipts = self.client.block_receipts(&block.hash());
let parent_total_difficulty = self.client.block_total_difficulty(BlockId::Hash(parent_hash));
match (block_receipts, parent_total_difficulty) {
(Some(block_receipts), Some(parent_total_difficulty)) => {
let block_receipts = block_receipts.receipts;
next_chain.insert_unordered_block(&mut batch, block, block_receipts, Some(parent_total_difficulty), false, true);
count += 1;
},
_ => break,
}
// Writting changes to DB and logging every now and then
if block_number % 1_000 == 0 {
next_db.key_value().write_buffered(batch);
next_chain.commit();
next_db.key_value().flush().expect("DB flush failed.");
batch = DBTransaction::new();
}
if block_number % 10_000 == 0 {
trace!(target: "snapshot", "Block restoration at #{}", block_number);
}
}
// Final commit to the DB
next_db.key_value().write_buffered(batch);
next_chain.commit();
next_db.key_value().flush().expect("DB flush failed.");
// We couldn't reach the targeted hash
if parent_hash != target_hash {
return Err(::snapshot::error::Error::UnlinkedAncientBlockChain.into());
}
// Update best ancient block in the Next Chain
next_chain.update_best_ancient_block(&start_hash);
Ok(count)
}
/// Get a reference to the snapshot reader.
pub fn reader(&self) -> RwLockReadGuard<Option<LooseReader>> {
self.reader.read()
@@ -480,12 +583,16 @@ impl Service {
// Import previous chunks, continue if it fails
self.import_prev_chunks(&mut res, manifest).ok();
*self.status.lock() = RestorationStatus::Ongoing {
state_chunks: state_chunks as u32,
block_chunks: block_chunks as u32,
state_chunks_done: self.state_chunks.load(Ordering::SeqCst) as u32,
block_chunks_done: self.block_chunks.load(Ordering::SeqCst) as u32,
};
// It could be that the restoration failed or completed in the meanwhile
let mut restoration_status = self.status.lock();
if let RestorationStatus::Initializing { .. } = *restoration_status {
*restoration_status = RestorationStatus::Ongoing {
state_chunks: state_chunks as u32,
block_chunks: block_chunks as u32,
state_chunks_done: self.state_chunks.load(Ordering::SeqCst) as u32,
block_chunks_done: self.block_chunks.load(Ordering::SeqCst) as u32,
};
}
Ok(())
}
@@ -752,26 +859,19 @@ impl Drop for Service {
#[cfg(test)]
mod tests {
use std::sync::Arc;
use client::ClientIoMessage;
use io::{IoService};
use spec::Spec;
use journaldb::Algorithm;
use error::Error;
use snapshot::{ManifestData, RestorationStatus, SnapshotService};
use super::*;
use tempdir::TempDir;
use test_helpers::restoration_db_handler;
struct NoopDBRestore;
impl DatabaseRestore for NoopDBRestore {
fn restore_db(&self, _new_db: &str) -> Result<(), Error> {
Ok(())
}
}
use test_helpers::{generate_dummy_client_with_spec_and_data, restoration_db_handler};
#[test]
fn sends_async_messages() {
let gas_prices = vec![1.into(), 2.into(), 3.into(), 999.into()];
let client = generate_dummy_client_with_spec_and_data(Spec::new_null, 400, 5, &gas_prices);
let service = IoService::<ClientIoMessage>::start().unwrap();
let spec = Spec::new_test();
@@ -785,7 +885,7 @@ mod tests {
pruning: Algorithm::Archive,
channel: service.channel(),
snapshot_root: dir,
db_restore: Arc::new(NoopDBRestore),
client: client,
};
let service = Service::new(snapshot_params).unwrap();

View File

@@ -16,26 +16,23 @@
//! Tests for the snapshot service.
use std::fs;
use std::sync::Arc;
use tempdir::TempDir;
use client::{Client, BlockInfo};
use blockchain::BlockProvider;
use client::{Client, ClientConfig, ImportBlock, BlockInfo};
use ids::BlockId;
use snapshot::io::{PackedReader, PackedWriter, SnapshotReader, SnapshotWriter};
use snapshot::service::{Service, ServiceParams};
use snapshot::{self, ManifestData, SnapshotService};
use snapshot::{chunk_state, chunk_secondary, ManifestData, Progress, SnapshotService, RestorationStatus};
use spec::Spec;
use test_helpers::{generate_dummy_client_with_spec_and_data, restoration_db_handler};
use test_helpers::{new_db, new_temp_db, generate_dummy_client_with_spec_and_data, restoration_db_handler};
use parking_lot::Mutex;
use io::IoChannel;
use kvdb_rocksdb::DatabaseConfig;
struct NoopDBRestore;
impl snapshot::DatabaseRestore for NoopDBRestore {
fn restore_db(&self, _new_db: &str) -> Result<(), ::error::Error> {
Ok(())
}
}
use verification::queue::kind::blocks::Unverified;
#[test]
fn restored_is_equivalent() {
@@ -46,7 +43,6 @@ fn restored_is_equivalent() {
const TX_PER: usize = 5;
let gas_prices = vec![1.into(), 2.into(), 3.into(), 999.into()];
let client = generate_dummy_client_with_spec_and_data(Spec::new_null, NUM_BLOCKS, TX_PER, &gas_prices);
let tempdir = TempDir::new("").unwrap();
@@ -73,7 +69,7 @@ fn restored_is_equivalent() {
pruning: ::journaldb::Algorithm::Archive,
channel: IoChannel::disconnected(),
snapshot_root: path,
db_restore: client2.clone(),
client: client2.clone(),
};
let service = Service::new(service_params).unwrap();
@@ -94,7 +90,7 @@ fn restored_is_equivalent() {
service.feed_block_chunk(hash, &chunk);
}
assert_eq!(service.status(), ::snapshot::RestorationStatus::Inactive);
assert_eq!(service.status(), RestorationStatus::Inactive);
for x in 0..NUM_BLOCKS {
let block1 = client.block(BlockId::Number(x as u64)).unwrap();
@@ -106,6 +102,9 @@ fn restored_is_equivalent() {
#[test]
fn guards_delete_folders() {
let gas_prices = vec![1.into(), 2.into(), 3.into(), 999.into()];
let client = generate_dummy_client_with_spec_and_data(Spec::new_null, 400, 5, &gas_prices);
let spec = Spec::new_null();
let tempdir = TempDir::new("").unwrap();
let service_params = ServiceParams {
@@ -115,7 +114,7 @@ fn guards_delete_folders() {
pruning: ::journaldb::Algorithm::Archive,
channel: IoChannel::disconnected(),
snapshot_root: tempdir.path().to_owned(),
db_restore: Arc::new(NoopDBRestore),
client: client,
};
let service = Service::new(service_params).unwrap();
@@ -146,3 +145,201 @@ fn guards_delete_folders() {
assert!(!path.join("db").exists());
assert!(path.join("temp").exists());
}
#[test]
fn keep_ancient_blocks() {
::env_logger::init().ok();
// Test variables
const NUM_BLOCKS: u64 = 500;
const NUM_SNAPSHOT_BLOCKS: u64 = 300;
const SNAPSHOT_MODE: ::snapshot::PowSnapshot = ::snapshot::PowSnapshot { blocks: NUM_SNAPSHOT_BLOCKS, max_restore_blocks: NUM_SNAPSHOT_BLOCKS };
// Temporary folders
let tempdir = TempDir::new("").unwrap();
let snapshot_path = tempdir.path().join("SNAP");
// Generate blocks
let gas_prices = vec![1.into(), 2.into(), 3.into(), 999.into()];
let spec_f = Spec::new_null;
let spec = spec_f();
let client = generate_dummy_client_with_spec_and_data(spec_f, NUM_BLOCKS as u32, 5, &gas_prices);
let bc = client.chain();
// Create the Snapshot
let best_hash = bc.best_block_hash();
let writer = Mutex::new(PackedWriter::new(&snapshot_path).unwrap());
let block_hashes = chunk_secondary(
Box::new(SNAPSHOT_MODE),
&bc,
best_hash,
&writer,
&Progress::default()
).unwrap();
let state_db = client.state_db().journal_db().boxed_clone();
let start_header = bc.block_header_data(&best_hash).unwrap();
let state_root = start_header.state_root();
let state_hashes = chunk_state(
state_db.as_hashdb(),
&state_root,
&writer,
&Progress::default(),
None
).unwrap();
let manifest = ::snapshot::ManifestData {
version: 2,
state_hashes: state_hashes,
state_root: state_root,
block_hashes: block_hashes,
block_number: NUM_BLOCKS,
block_hash: best_hash,
};
writer.into_inner().finish(manifest.clone()).unwrap();
// Initialize the Client
let db_config = DatabaseConfig::with_columns(::db::NUM_COLUMNS);
let client_db = new_temp_db(&tempdir.path());
let client2 = Client::new(
ClientConfig::default(),
&spec,
client_db,
Arc::new(::miner::Miner::new_for_tests(&spec, None)),
IoChannel::disconnected(),
).unwrap();
// Add some ancient blocks
for block_number in 1..50 {
let block_hash = bc.block_hash(block_number).unwrap();
let block = bc.block(&block_hash).unwrap();
client2.import_block(Unverified::from_rlp(block.into_inner()).unwrap()).unwrap();
}
client2.import_verified_blocks();
client2.flush_queue();
// Restore the Snapshot
let reader = PackedReader::new(&snapshot_path).unwrap().unwrap();
let service_params = ServiceParams {
engine: spec.engine.clone(),
genesis_block: spec.genesis_block(),
restoration_db_handler: restoration_db_handler(db_config),
pruning: ::journaldb::Algorithm::Archive,
channel: IoChannel::disconnected(),
snapshot_root: tempdir.path().to_owned(),
client: client2.clone(),
};
let service = Service::new(service_params).unwrap();
service.init_restore(manifest.clone(), false).unwrap();
for hash in &manifest.block_hashes {
let chunk = reader.chunk(*hash).unwrap();
service.feed_block_chunk(*hash, &chunk);
}
for hash in &manifest.state_hashes {
let chunk = reader.chunk(*hash).unwrap();
service.feed_state_chunk(*hash, &chunk);
}
match service.status() {
RestorationStatus::Inactive => (),
RestorationStatus::Failed => panic!("Snapshot Restoration has failed."),
RestorationStatus::Ongoing { .. } => panic!("Snapshot Restoration should be done."),
_ => panic!("Invalid Snapshot Service status."),
}
// Check that the latest block number is the right one
assert_eq!(client2.block(BlockId::Latest).unwrap().number(), NUM_BLOCKS as u64);
// Check that we have blocks in [NUM_BLOCKS - NUM_SNAPSHOT_BLOCKS + 1 ; NUM_BLOCKS]
// but none before
assert!(client2.block(BlockId::Number(NUM_BLOCKS - NUM_SNAPSHOT_BLOCKS + 1)).is_some());
assert!(client2.block(BlockId::Number(100)).is_none());
// Check that the first 50 blocks have been migrated
for block_number in 1..49 {
assert!(client2.block(BlockId::Number(block_number)).is_some());
}
}
#[test]
fn recover_aborted_recovery() {
::env_logger::init().ok();
const NUM_BLOCKS: u32 = 400;
let gas_prices = vec![1.into(), 2.into(), 3.into(), 999.into()];
let client = generate_dummy_client_with_spec_and_data(Spec::new_null, NUM_BLOCKS, 5, &gas_prices);
let spec = Spec::new_null();
let tempdir = TempDir::new("").unwrap();
let db_config = DatabaseConfig::with_columns(::db::NUM_COLUMNS);
let client_db = new_db();
let client2 = Client::new(
Default::default(),
&spec,
client_db,
Arc::new(::miner::Miner::new_for_tests(&spec, None)),
IoChannel::disconnected(),
).unwrap();
let service_params = ServiceParams {
engine: spec.engine.clone(),
genesis_block: spec.genesis_block(),
restoration_db_handler: restoration_db_handler(db_config),
pruning: ::journaldb::Algorithm::Archive,
channel: IoChannel::disconnected(),
snapshot_root: tempdir.path().to_owned(),
client: client2.clone(),
};
let service = Service::new(service_params).unwrap();
service.take_snapshot(&client, NUM_BLOCKS as u64).unwrap();
let manifest = service.manifest().unwrap();
service.init_restore(manifest.clone(), true).unwrap();
// Restore only the state chunks
for hash in &manifest.state_hashes {
let chunk = service.chunk(*hash).unwrap();
service.feed_state_chunk(*hash, &chunk);
}
match service.status() {
RestorationStatus::Ongoing { block_chunks_done, state_chunks_done, .. } => {
assert_eq!(state_chunks_done, manifest.state_hashes.len() as u32);
assert_eq!(block_chunks_done, 0);
},
e => panic!("Snapshot restoration must be ongoing ; {:?}", e),
}
// Abort the restore...
service.abort_restore();
// And try again!
service.init_restore(manifest.clone(), true).unwrap();
match service.status() {
RestorationStatus::Ongoing { block_chunks_done, state_chunks_done, .. } => {
assert_eq!(state_chunks_done, manifest.state_hashes.len() as u32);
assert_eq!(block_chunks_done, 0);
},
e => panic!("Snapshot restoration must be ongoing ; {:?}", e),
}
// Remove the snapshot directory, and restart the restoration
// It shouldn't have restored any previous blocks
fs::remove_dir_all(tempdir.path()).unwrap();
// And try again!
service.init_restore(manifest.clone(), true).unwrap();
match service.status() {
RestorationStatus::Ongoing { block_chunks_done, state_chunks_done, .. } => {
assert_eq!(block_chunks_done, 0);
assert_eq!(state_chunks_done, 0);
},
_ => panic!("Snapshot restoration must be ongoing"),
}
}

View File

@@ -41,7 +41,7 @@ use transaction::{Action, Transaction, SignedTransaction};
use views::BlockView;
use blooms_db;
use kvdb::KeyValueDB;
use kvdb_rocksdb;
use kvdb_rocksdb::{self, Database, DatabaseConfig};
use tempdir::TempDir;
use verification::queue::kind::blocks::Unverified;
use encoded;
@@ -263,30 +263,30 @@ pub fn get_test_client_with_blocks(blocks: Vec<Bytes>) -> Arc<Client> {
client
}
struct TestBlockChainDB {
_blooms_dir: TempDir,
_trace_blooms_dir: TempDir,
blooms: blooms_db::Database,
trace_blooms: blooms_db::Database,
key_value: Arc<KeyValueDB>,
}
impl BlockChainDB for TestBlockChainDB {
fn key_value(&self) -> &Arc<KeyValueDB> {
&self.key_value
}
fn blooms(&self) -> &blooms_db::Database {
&self.blooms
}
fn trace_blooms(&self) -> &blooms_db::Database {
&self.trace_blooms
}
}
/// Creates new test instance of `BlockChainDB`
pub fn new_db() -> Arc<BlockChainDB> {
struct TestBlockChainDB {
_blooms_dir: TempDir,
_trace_blooms_dir: TempDir,
blooms: blooms_db::Database,
trace_blooms: blooms_db::Database,
key_value: Arc<KeyValueDB>,
}
impl BlockChainDB for TestBlockChainDB {
fn key_value(&self) -> &Arc<KeyValueDB> {
&self.key_value
}
fn blooms(&self) -> &blooms_db::Database {
&self.blooms
}
fn trace_blooms(&self) -> &blooms_db::Database {
&self.trace_blooms
}
}
let blooms_dir = TempDir::new("").unwrap();
let trace_blooms_dir = TempDir::new("").unwrap();
@@ -301,6 +301,26 @@ pub fn new_db() -> Arc<BlockChainDB> {
Arc::new(db)
}
/// Creates a new temporary `BlockChainDB` on FS
pub fn new_temp_db(tempdir: &Path) -> Arc<BlockChainDB> {
let blooms_dir = TempDir::new("").unwrap();
let trace_blooms_dir = TempDir::new("").unwrap();
let key_value_dir = tempdir.join("key_value");
let db_config = DatabaseConfig::with_columns(::db::NUM_COLUMNS);
let key_value_db = Database::open(&db_config, key_value_dir.to_str().unwrap()).unwrap();
let db = TestBlockChainDB {
blooms: blooms_db::Database::open(blooms_dir.path()).unwrap(),
trace_blooms: blooms_db::Database::open(trace_blooms_dir.path()).unwrap(),
_blooms_dir: blooms_dir,
_trace_blooms_dir: trace_blooms_dir,
key_value: Arc::new(key_value_db)
};
Arc::new(db)
}
/// Creates new instance of KeyValueDBHandler
pub fn restoration_db_handler(config: kvdb_rocksdb::DatabaseConfig) -> Box<BlockChainDBHandler> {
struct RestorationDBHandler {

View File

@@ -583,6 +583,13 @@ impl<K: Kind> VerificationQueue<K> {
result
}
/// Returns true if there is nothing currently in the queue.
/// TODO [ToDr] Optimize to avoid locking
pub fn is_empty(&self) -> bool {
let v = &self.verification;
v.unverified.lock().is_empty() && v.verifying.lock().is_empty() && v.verified.lock().is_empty()
}
/// Get queue status.
pub fn queue_info(&self) -> QueueInfo {
use std::mem::size_of;