From 8599a11a0b2e1b50ef16f6b3878d35d34a6cf506 Mon Sep 17 00:00:00 2001 From: Robert Habermeier Date: Mon, 31 Oct 2016 17:32:53 +0100 Subject: [PATCH] Snapshot sync and block gap info in `eth_syncing` (#2948) * provide snapshot sync info in eth_syncing * specify block gap in eth_syncing * Extend eth_syncing with warp, format the output properly * adjust serialization tests for sync info * whitespace --- ethcore/src/client/test_client.rs | 14 ++++-- js/src/api/format/output.js | 22 +++++++++ js/src/api/format/output.spec.js | 18 ++++++- js/src/api/rpc/eth/eth.js | 5 +- js/src/jsonrpc/interfaces/eth.js | 14 +++++- parity/rpc_apis.rs | 3 ++ parity/run.rs | 1 + rpc/src/v1/impls/eth.rs | 47 +++++++++++++++--- rpc/src/v1/tests/eth.rs | 10 +++- rpc/src/v1/tests/helpers/mod.rs | 2 + rpc/src/v1/tests/helpers/snapshot_service.rs | 51 ++++++++++++++++++++ rpc/src/v1/tests/mocked/eth.rs | 32 ++++++++++-- rpc/src/v1/types/sync.rs | 24 +++++++-- 13 files changed, 220 insertions(+), 23 deletions(-) create mode 100644 rpc/src/v1/tests/helpers/snapshot_service.rs diff --git a/ethcore/src/client/test_client.rs b/ethcore/src/client/test_client.rs index 29a48f02f..a12d94ae4 100644 --- a/ethcore/src/client/test_client.rs +++ b/ethcore/src/client/test_client.rs @@ -84,6 +84,10 @@ pub struct TestBlockChainClient { pub vm_factory: EvmFactory, /// Timestamp assigned to latest sealed block pub latest_block_timestamp: RwLock, + /// Ancient block info. + pub ancient_block: RwLock>, + /// First block info. + pub first_block: RwLock>, } #[derive(Clone)] @@ -133,6 +137,8 @@ impl TestBlockChainClient { spec: spec, vm_factory: EvmFactory::new(VMType::Interpreter, 1024 * 1024), latest_block_timestamp: RwLock::new(10_000_000), + ancient_block: RwLock::new(None), + first_block: RwLock::new(None), }; client.add_blocks(1, EachBlockWith::Nothing); // add genesis block client.genesis_hash = client.last_hash.read().clone(); @@ -594,10 +600,10 @@ impl BlockChainClient for TestBlockChainClient { genesis_hash: self.genesis_hash.clone(), best_block_hash: self.last_hash.read().clone(), best_block_number: self.blocks.read().len() as BlockNumber - 1, - first_block_hash: None, - first_block_number: None, - ancient_block_hash: None, - ancient_block_number: None, + first_block_hash: self.first_block.read().as_ref().map(|x| x.0), + first_block_number: self.first_block.read().as_ref().map(|x| x.1), + ancient_block_hash: self.ancient_block.read().as_ref().map(|x| x.0), + ancient_block_number: self.ancient_block.read().as_ref().map(|x| x.1) } } diff --git a/js/src/api/format/output.js b/js/src/api/format/output.js index 425c8d5a8..6f785ed9b 100644 --- a/js/src/api/format/output.js +++ b/js/src/api/format/output.js @@ -153,6 +153,28 @@ export function outSignerRequest (request) { return request; } +export function outSyncing (syncing) { + if (syncing && syncing !== 'false') { + Object.keys(syncing).forEach((key) => { + switch (key) { + case 'currentBlock': + case 'highestBlock': + case 'startingBlock': + case 'warpChunksAmount': + case 'warpChunksProcessed': + syncing[key] = outNumber(syncing[key]); + break; + + case 'blockGap': + syncing[key] = syncing[key].map(outNumber); + break; + } + }); + } + + return syncing; +} + export function outTransaction (tx) { if (tx) { Object.keys(tx).forEach((key) => { diff --git a/js/src/api/format/output.spec.js b/js/src/api/format/output.spec.js index f21fbbdac..aac433d70 100644 --- a/js/src/api/format/output.spec.js +++ b/js/src/api/format/output.spec.js @@ -16,7 +16,7 @@ import BigNumber from 'bignumber.js'; -import { outBlock, outAccountInfo, outAddress, outDate, outHistogram, outNumber, outPeers, outReceipt, outTransaction, outTrace } from './output'; +import { outBlock, outAccountInfo, outAddress, outDate, outHistogram, outNumber, outPeers, outReceipt, outSyncing, outTransaction, outTrace } from './output'; import { isAddress, isBigNumber, isInstanceOf } from '../../../test/types'; describe('api/format/output', () => { @@ -203,6 +203,22 @@ describe('api/format/output', () => { }); }); + describe('outSyncing', () => { + ['currentBlock', 'highestBlock', 'startingBlock', 'warpChunksAmount', 'warpChunksProcessed'].forEach((input) => { + it(`formats ${input} numbers as a number`, () => { + expect(outSyncing({ [input]: '0x123' })).to.deep.equal({ + [input]: new BigNumber('0x123') + }); + }); + }); + + it('formats blockGap properly', () => { + expect(outSyncing({ blockGap: [0x123, 0x456] })).to.deep.equal({ + blockGap: [new BigNumber(0x123), new BigNumber(0x456)] + }); + }); + }); + describe('outTransaction', () => { ['from', 'to'].forEach((input) => { it(`formats ${input} address as address`, () => { diff --git a/js/src/api/rpc/eth/eth.js b/js/src/api/rpc/eth/eth.js index 3d99f82b8..703f3ed11 100644 --- a/js/src/api/rpc/eth/eth.js +++ b/js/src/api/rpc/eth/eth.js @@ -15,7 +15,7 @@ // along with Parity. If not, see . import { inAddress, inBlockNumber, inData, inFilter, inHash, inHex, inNumber16, inOptions } from '../../format/input'; -import { outAddress, outBlock, outLog, outNumber, outReceipt, outTransaction } from '../../format/output'; +import { outAddress, outBlock, outLog, outNumber, outReceipt, outSyncing, outTransaction } from '../../format/output'; export default class Eth { constructor (transport) { @@ -314,7 +314,8 @@ export default class Eth { syncing () { return this._transport - .execute('eth_syncing'); + .execute('eth_syncing') + .then(outSyncing); } uninstallFilter (filterId) { diff --git a/js/src/jsonrpc/interfaces/eth.js b/js/src/jsonrpc/interfaces/eth.js index 33bf9d66d..f1c8fb86f 100644 --- a/js/src/jsonrpc/interfaces/eth.js +++ b/js/src/jsonrpc/interfaces/eth.js @@ -1003,7 +1003,7 @@ export default { details: { startingBlock: { type: Quantity, - desc: 'The block at which the import started (will only be reset, after the sync reached his head)' + desc: 'The block at which the import started (will only be reset, after the sync reached this head)' }, currentBlock: { type: Quantity, @@ -1012,6 +1012,18 @@ export default { highestBlock: { type: Quantity, desc: 'The estimated highest block' + }, + blockGap: { + type: Array, + desc: 'Array of "first", "last", such that [first, last) are all missing from the chain' + }, + warpChunksAmount: { + type: Quantity, + desc: 'Total amount of snapshot chunks' + }, + warpChunksProcessed: { + type: Quantity, + desc: 'Total amount of snapshot chunks processed' } } } diff --git a/parity/rpc_apis.rs b/parity/rpc_apis.rs index 491f58a1b..1aae27dd2 100644 --- a/parity/rpc_apis.rs +++ b/parity/rpc_apis.rs @@ -23,6 +23,7 @@ use util::RotatingLogger; use ethcore::miner::{Miner, ExternalMiner}; use ethcore::client::Client; use ethcore::account_provider::AccountProvider; +use ethcore::snapshot::SnapshotService; use ethsync::{ManageNetwork, SyncProvider}; use ethcore_rpc::{Extendable, NetworkSettings}; pub use ethcore_rpc::SignerService; @@ -97,6 +98,7 @@ impl FromStr for ApiSet { pub struct Dependencies { pub signer_service: Arc, pub client: Arc, + pub snapshot: Arc, pub sync: Arc, pub net: Arc, pub secret_store: Arc, @@ -161,6 +163,7 @@ pub fn setup_rpc(server: T, deps: Arc, apis: ApiSet Api::Eth => { let client = EthClient::new( &deps.client, + &deps.snapshot, &deps.sync, &deps.secret_store, &deps.miner, diff --git a/parity/run.rs b/parity/run.rs index 7e7e7456a..8a3ae06ec 100644 --- a/parity/run.rs +++ b/parity/run.rs @@ -251,6 +251,7 @@ pub fn execute(cmd: RunCmd) -> Result<(), String> { signer_service: Arc::new(rpc_apis::SignerService::new(move || { signer::generate_new_token(signer_path.clone()).map_err(|e| format!("{:?}", e)) }, cmd.signer_port)), + snapshot: snapshot_service.clone(), client: client.clone(), sync: sync_provider.clone(), net: manage_network.clone(), diff --git a/rpc/src/v1/impls/eth.rs b/rpc/src/v1/impls/eth.rs index e414fe765..18f2cc37c 100644 --- a/rpc/src/v1/impls/eth.rs +++ b/rpc/src/v1/impls/eth.rs @@ -40,6 +40,7 @@ use ethcore::ethereum::Ethash; use ethcore::transaction::{Transaction as EthTransaction, SignedTransaction, Action}; use ethcore::log_entry::LogEntry; use ethcore::filter::Filter as EthcoreFilter; +use ethcore::snapshot::SnapshotService; use self::ethash::SeedHashCompute; use v1::traits::Eth; use v1::types::{ @@ -70,13 +71,15 @@ impl Default for EthClientOptions { } /// Eth rpc implementation. -pub struct EthClient where +pub struct EthClient where C: MiningBlockChainClient, + SN: SnapshotService, S: SyncProvider, M: MinerService, EM: ExternalMinerService { client: Weak, + snapshot: Weak, sync: Weak, accounts: Weak, miner: Weak, @@ -85,17 +88,26 @@ pub struct EthClient where options: EthClientOptions, } -impl EthClient where +impl EthClient where C: MiningBlockChainClient, + SN: SnapshotService, S: SyncProvider, M: MinerService, EM: ExternalMinerService { /// Creates new EthClient. - pub fn new(client: &Arc, sync: &Arc, accounts: &Arc, miner: &Arc, em: &Arc, options: EthClientOptions) - -> EthClient { + pub fn new( + client: &Arc, + snapshot: &Arc, + sync: &Arc, + accounts: &Arc, + miner: &Arc, + em: &Arc, + options: EthClientOptions + ) -> Self { EthClient { client: Arc::downgrade(client), + snapshot: Arc::downgrade(snapshot), sync: Arc::downgrade(sync), miner: Arc::downgrade(miner), accounts: Arc::downgrade(accounts), @@ -220,8 +232,9 @@ pub fn pending_logs(miner: &M, best_block: EthBlockNumber, filter: &EthcoreFi const MAX_QUEUE_SIZE_TO_MINE_ON: usize = 4; // because uncles go back 6. -impl EthClient where +impl EthClient where C: MiningBlockChainClient + 'static, + SN: SnapshotService + 'static, S: SyncProvider + 'static, M: MinerService + 'static, EM: ExternalMinerService + 'static { @@ -239,8 +252,9 @@ static SOLC: &'static str = "solc.exe"; #[cfg(not(windows))] static SOLC: &'static str = "solc"; -impl Eth for EthClient where +impl Eth for EthClient where C: MiningBlockChainClient + 'static, + SN: SnapshotService + 'static, S: SyncProvider + 'static, M: MinerService + 'static, EM: ExternalMinerService + 'static { @@ -253,16 +267,33 @@ impl Eth for EthClient where } fn syncing(&self) -> Result { + use ethcore::snapshot::RestorationStatus; + try!(self.active()); let status = take_weak!(self.sync).status(); let client = take_weak!(self.client); - if is_major_importing(Some(status.state), client.queue_info()) { - let current_block = U256::from(client.chain_info().best_block_number); + let snapshot_status = take_weak!(self.snapshot).status(); + + let (warping, warp_chunks_amount, warp_chunks_processed) = match snapshot_status { + RestorationStatus::Ongoing { state_chunks, block_chunks, state_chunks_done, block_chunks_done } => + (true, Some(block_chunks + state_chunks), Some(block_chunks_done + state_chunks_done)), + _ => (false, None, None), + }; + + + if warping || is_major_importing(Some(status.state), client.queue_info()) { + let chain_info = client.chain_info(); + let current_block = U256::from(chain_info.best_block_number); let highest_block = U256::from(status.highest_block_number.unwrap_or(status.start_block_number)); + let gap = chain_info.ancient_block_number.map(|x| U256::from(x + 1)) + .and_then(|first| chain_info.first_block_number.map(|last| (first, U256::from(last)))); let info = SyncInfo { starting_block: status.start_block_number.into(), current_block: current_block.into(), highest_block: highest_block.into(), + warp_chunks_amount: warp_chunks_amount.map(|x| U256::from(x as u64)).map(Into::into), + warp_chunks_processed: warp_chunks_processed.map(|x| U256::from(x as u64)).map(Into::into), + block_gap: gap.map(|(x, y)| (x.into(), y.into())), }; Ok(SyncStatus::Info(info)) } else { diff --git a/rpc/src/v1/tests/eth.rs b/rpc/src/v1/tests/eth.rs index d556d11ef..c4341622e 100644 --- a/rpc/src/v1/tests/eth.rs +++ b/rpc/src/v1/tests/eth.rs @@ -37,7 +37,7 @@ use v1::impls::{EthClient, EthSigningUnsafeClient}; use v1::types::U256 as NU256; use v1::traits::eth::Eth; use v1::traits::eth_signing::EthSigning; -use v1::tests::helpers::{TestSyncProvider, Config}; +use v1::tests::helpers::{TestSnapshotService, TestSyncProvider, Config}; fn account_provider() -> Arc { Arc::new(AccountProvider::transient_provider()) @@ -73,6 +73,10 @@ fn miner_service(spec: &Spec, accounts: Arc) -> Arc { ) } +fn snapshot_service() -> Arc { + Arc::new(TestSnapshotService::new()) +} + fn make_spec(chain: &BlockChain) -> Spec { let genesis = Genesis::from(chain.genesis()); let mut spec = ethereum::new_frontier_test(); @@ -86,6 +90,7 @@ fn make_spec(chain: &BlockChain) -> Spec { struct EthTester { client: Arc, _miner: Arc, + _snapshot: Arc, accounts: Arc, handler: IoHandler, } @@ -112,6 +117,7 @@ impl EthTester { let dir = RandomTempPath::new(); let account_provider = account_provider(); let miner_service = miner_service(&spec, account_provider.clone()); + let snapshot_service = snapshot_service(); let db_config = ::util::kvdb::DatabaseConfig::with_columns(::ethcore::db::NUM_COLUMNS); let client = Client::new( @@ -127,6 +133,7 @@ impl EthTester { let eth_client = EthClient::new( &client, + &snapshot_service, &sync_provider, &account_provider, &miner_service, @@ -145,6 +152,7 @@ impl EthTester { EthTester { _miner: miner_service, + _snapshot: snapshot_service, client: client, accounts: account_provider, handler: handler, diff --git a/rpc/src/v1/tests/helpers/mod.rs b/rpc/src/v1/tests/helpers/mod.rs index 234bae1be..321a9717e 100644 --- a/rpc/src/v1/tests/helpers/mod.rs +++ b/rpc/src/v1/tests/helpers/mod.rs @@ -19,7 +19,9 @@ mod sync_provider; mod miner_service; mod fetch; +mod snapshot_service; pub use self::sync_provider::{Config, TestSyncProvider}; pub use self::miner_service::TestMinerService; pub use self::fetch::TestFetch; +pub use self::snapshot_service::TestSnapshotService; \ No newline at end of file diff --git a/rpc/src/v1/tests/helpers/snapshot_service.rs b/rpc/src/v1/tests/helpers/snapshot_service.rs new file mode 100644 index 000000000..b78aa374d --- /dev/null +++ b/rpc/src/v1/tests/helpers/snapshot_service.rs @@ -0,0 +1,51 @@ +// Copyright 2015, 2016 Ethcore (UK) Ltd. +// This file is part of Parity. + +// Parity is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Parity is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Parity. If not, see . + +use ethcore::snapshot::{ManifestData, RestorationStatus, SnapshotService}; + +use util::{Bytes, Mutex}; +use util::hash::H256; + +/// Mocked snapshot service (used for sync info extensions). +pub struct TestSnapshotService { + status: Mutex, +} + +impl TestSnapshotService { + /// Create a test snapshot service. Only the `status` function matters -- it'll + /// return `Inactive` by default. + pub fn new() -> Self { + TestSnapshotService { + status: Mutex::new(RestorationStatus::Inactive), + } + } + + /// Set the restoration status. + pub fn set_status(&self, status: RestorationStatus) { + *self.status.lock() = status; + } +} + +impl SnapshotService for TestSnapshotService { + fn manifest(&self) -> Option { None } + fn chunk(&self, _hash: H256) -> Option { None } + fn status(&self) -> RestorationStatus { self.status.lock().clone() } + fn begin_restore(&self, _manifest: ManifestData) { } + fn abort_restore(&self) { } + fn restore_state_chunk(&self, _hash: H256, _chunk: Bytes) { } + fn restore_block_chunk(&self, _hash: H256, _chunk: Bytes) { } + fn provide_canon_hashes(&self, _hashes: &[(u64, H256)]) { } +} \ No newline at end of file diff --git a/rpc/src/v1/tests/mocked/eth.rs b/rpc/src/v1/tests/mocked/eth.rs index 5cf951530..24aa5e480 100644 --- a/rpc/src/v1/tests/mocked/eth.rs +++ b/rpc/src/v1/tests/mocked/eth.rs @@ -28,7 +28,7 @@ use ethcore::transaction::{Transaction, Action}; use ethcore::miner::{ExternalMiner, MinerService}; use ethsync::SyncState; use v1::{Eth, EthClient, EthClientOptions, EthFilter, EthFilterClient, EthSigning, EthSigningUnsafeClient}; -use v1::tests::helpers::{TestSyncProvider, Config, TestMinerService}; +use v1::tests::helpers::{TestSyncProvider, Config, TestMinerService, TestSnapshotService}; use rustc_serialize::hex::ToHex; use time::get_time; @@ -52,11 +52,16 @@ fn miner_service() -> Arc { Arc::new(TestMinerService::default()) } +fn snapshot_service() -> Arc { + Arc::new(TestSnapshotService::new()) +} + struct EthTester { pub client: Arc, pub sync: Arc, pub accounts_provider: Arc, pub miner: Arc, + pub snapshot: Arc, hashrates: Arc>>, pub io: IoHandler, } @@ -73,9 +78,10 @@ impl EthTester { let sync = sync_provider(); let ap = accounts_provider(); let miner = miner_service(); + let snapshot = snapshot_service(); let hashrates = Arc::new(Mutex::new(HashMap::new())); let external_miner = Arc::new(ExternalMiner::new(hashrates.clone())); - let eth = EthClient::new(&client, &sync, &ap, &miner, &external_miner, options).to_delegate(); + let eth = EthClient::new(&client, &snapshot, &sync, &ap, &miner, &external_miner, options).to_delegate(); let filter = EthFilterClient::new(&client, &miner).to_delegate(); let sign = EthSigningUnsafeClient::new(&client, &ap, &miner).to_delegate(); let io = IoHandler::new(); @@ -88,6 +94,7 @@ impl EthTester { sync: sync, accounts_provider: ap, miner: miner, + snapshot: snapshot, io: io, hashrates: hashrates, } @@ -109,6 +116,8 @@ fn rpc_eth_protocol_version() { #[test] fn rpc_eth_syncing() { + use ethcore::snapshot::RestorationStatus; + let request = r#"{"jsonrpc": "2.0", "method": "eth_syncing", "params": [], "id": 1}"#; let tester = EthTester::default(); @@ -125,10 +134,27 @@ fn rpc_eth_syncing() { // "sync" to 1000 blocks. // causes TestBlockChainClient to return 1000 for its best block number. tester.add_blocks(1000, EachBlockWith::Nothing); + *tester.client.ancient_block.write() = Some((H256::new(), 5)); + *tester.client.first_block.write() = Some((H256::from(U256::from(1234)), 3333)); - let true_res = r#"{"jsonrpc":"2.0","result":{"currentBlock":"0x3e8","highestBlock":"0x9c4","startingBlock":"0x0"},"id":1}"#; + let true_res = r#"{"jsonrpc":"2.0","result":{"blockGap":["0x6","0xd05"],"currentBlock":"0x3e8","highestBlock":"0x9c4","startingBlock":"0x0","warpChunksAmount":null,"warpChunksProcessed":null},"id":1}"#; assert_eq!(tester.io.handle_request_sync(request), Some(true_res.to_owned())); + *tester.client.ancient_block.write() = None; + *tester.client.first_block.write() = None; + + let snap_res = r#"{"jsonrpc":"2.0","result":{"blockGap":null,"currentBlock":"0x3e8","highestBlock":"0x9c4","startingBlock":"0x0","warpChunksAmount":"0x32","warpChunksProcessed":"0x18"},"id":1}"#; + tester.snapshot.set_status(RestorationStatus::Ongoing { + state_chunks: 40, + block_chunks: 10, + state_chunks_done: 18, + block_chunks_done: 6, + }); + + assert_eq!(tester.io.handle_request_sync(request), Some(snap_res.to_owned())); + + tester.snapshot.set_status(RestorationStatus::Inactive); + // finish "syncing" tester.add_blocks(1500, EachBlockWith::Nothing); diff --git a/rpc/src/v1/types/sync.rs b/rpc/src/v1/types/sync.rs index 9a7f733c1..a0f61e799 100644 --- a/rpc/src/v1/types/sync.rs +++ b/rpc/src/v1/types/sync.rs @@ -30,6 +30,15 @@ pub struct SyncInfo { /// Highest block seen so far #[serde(rename="highestBlock")] pub highest_block: U256, + /// Warp sync snapshot chunks total. + #[serde(rename="warpChunksAmount")] + pub warp_chunks_amount: Option, + /// Warp sync snpashot chunks processed. + #[serde(rename="warpChunksProcessed")] + pub warp_chunks_processed: Option, + /// Describes the gap in the blockchain, if there is one: (first, last) + #[serde(rename="blockGap")] + pub block_gap: Option<(U256, U256)>, } /// Peers info @@ -53,7 +62,7 @@ pub struct PeerInfo { /// Node client ID pub name: String, /// Capabilities - pub caps: Vec, + pub caps: Vec, /// Network information pub network: PeerNetworkInfo, /// Protocols information @@ -138,7 +147,7 @@ mod tests { fn test_serialize_sync_info() { let t = SyncInfo::default(); let serialized = serde_json::to_string(&t).unwrap(); - assert_eq!(serialized, r#"{"startingBlock":"0x0","currentBlock":"0x0","highestBlock":"0x0"}"#); + assert_eq!(serialized, r#"{"startingBlock":"0x0","currentBlock":"0x0","highestBlock":"0x0","warpChunksAmount":null,"warpChunksProcessed":null,"blockGap":null}"#); } #[test] @@ -156,6 +165,15 @@ mod tests { let t = SyncStatus::Info(SyncInfo::default()); let serialized = serde_json::to_string(&t).unwrap(); - assert_eq!(serialized, r#"{"startingBlock":"0x0","currentBlock":"0x0","highestBlock":"0x0"}"#); + assert_eq!(serialized, r#"{"startingBlock":"0x0","currentBlock":"0x0","highestBlock":"0x0","warpChunksAmount":null,"warpChunksProcessed":null,"blockGap":null}"#); + } + + #[test] + fn test_serialize_block_gap() { + let mut t = SyncInfo::default(); + t.block_gap = Some((1.into(), 5.into())); + + let serialized = serde_json::to_string(&t).unwrap(); + assert_eq!(serialized, r#"{"startingBlock":"0x0","currentBlock":"0x0","highestBlock":"0x0","warpChunksAmount":null,"warpChunksProcessed":null,"blockGap":["0x1","0x5"]}"#) } }