[beta] Backports (#8011)

* Hardware-wallet/usb-subscribe-refactor (#7860)

* Hardware-wallet fix

* More fine-grained initilization of callbacks by vendorID, productID and usb class
* Each device manufacturer gets a seperate handle thread each
* Replaced "dummy for loop" with a delay to wait for the device to boot-up properly
* Haven't been very carefully with checking dependencies cycles etc
* Inline comments explaining where shortcuts have been taken
* Need to test this on Windows machine and with Ledger (both models)

Signed-off-by: niklasad1 <niklasadolfsson1@gmail.com>

* Validate product_id of detected ledger devices

* closed_device => unlocked_device

* address comments

* add target in debug

* Address feedback

* Remove thread joining in HardwareWalletManager
* Remove thread handlers in HardwareWalletManager because this makes them unused

* fixed broken logs (#7934)

* fixed broken logs

* bring back old lock order

* removed bloom groups from blockchain

* revert unrelated changes

* simplify blockchain_block_blooms

* Bump WS (#7952)

* Calculate proper keccak256/sha3 using parity. (#7953)

* Increase max download limit to 128MB (#7965)

* fetch: increase max download limit to 64MB

* parity: increase download size limit for updater service

* Detect too large packets in snapshot sync. (#7977)

* fix traces, removed bloomchain crate, closes #7228, closes #7167 (#7979)

* Remvoe generator.rs

* Make block generator easier to use (#7888)

* Make block generator easier to use

* applied review suggestions

* rename BlockMetadata -> BlockOptions

* removed redundant uses of blockchain generator and genereator.next().unwrap() calls

Cargo.lock

@@ -181,11 +181,6 @@ dependencies = [
  "keccak-hash 0.1.0",
 ]
 
-[[package]]
-name = "bloomchain"
-version = "0.1.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-
 [[package]]
 name = "bn"
 version = "0.4.4"
@@ -476,7 +471,6 @@ version = "1.9.0"
 dependencies = [
  "ansi_term 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "bloomable 0.1.0",
- "bloomchain 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "bn 0.4.4 (git+https://github.com/paritytech/bn)",
  "byteorder 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "common-types 0.1.0",
@@ -1333,7 +1327,7 @@ dependencies = [
  "log 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)",
  "parking_lot 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)",
  "slab 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "ws 0.7.1 (git+https://github.com/tomusdrw/ws-rs)",
+ "ws 0.7.5 (git+https://github.com/tomusdrw/ws-rs)",
 ]
 
 [[package]]
@@ -3529,8 +3523,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 
 [[package]]
 name = "ws"
-version = "0.7.1"
+version = "0.7.5"
-source = "git+https://github.com/tomusdrw/ws-rs#f8306a798b7541d64624299a83a2c934f173beed"
+source = "git+https://github.com/tomusdrw/ws-rs#368ce39e2aa8700d568ca29dbacaecdf1bf749d1"
 dependencies = [
  "byteorder 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "bytes 0.4.5 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -3608,7 +3602,6 @@ dependencies = [
 "checksum bitflags 0.8.2 (registry+https://github.com/rust-lang/crates.io-index)" = "1370e9fc2a6ae53aea8b7a5110edbd08836ed87c88736dfabccade1c2b44bff4"
 "checksum bitflags 0.9.1 (registry+https://github.com/rust-lang/crates.io-index)" = "4efd02e230a02e18f92fc2735f44597385ed02ad8f831e7c1c1156ee5e1ab3a5"
 "checksum bitflags 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)" = "b3c30d3802dfb7281680d6285f2ccdaa8c2d8fee41f93805dba5c4cf50dc23cf"
-"checksum bloomchain 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "3f421095d2a76fc24cd3fb3f912b90df06be7689912b1bdb423caefae59c258d"
 "checksum bn 0.4.4 (git+https://github.com/paritytech/bn)" = "<none>"
 "checksum byteorder 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ff81738b726f5d099632ceaffe7fb65b90212e8dce59d518729e7e8634032d3d"
 "checksum bytes 0.4.5 (registry+https://github.com/rust-lang/crates.io-index)" = "d828f97b58cc5de3e40c421d0cf2132d6b2da4ee0e11b8632fa838f0f9333ad6"
@@ -3860,7 +3853,7 @@ dependencies = [
 "checksum wasmi 0.0.0 (git+https://github.com/pepyakin/wasmi)" = "<none>"
 "checksum winapi 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)" = "167dc9d6949a9b857f3451275e911c3f44255842c1f7a76f33c55103a909087a"
 "checksum winapi-build 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "2d315eee3b34aca4797b2da6b13ed88266e6d612562a0c46390af8299fc699bc"
-"checksum ws 0.7.1 (git+https://github.com/tomusdrw/ws-rs)" = "<none>"
+"checksum ws 0.7.5 (git+https://github.com/tomusdrw/ws-rs)" = "<none>"
 "checksum ws2_32-sys 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "d59cefebd0c892fa2dd6de581e937301d8552cb44489cdff035c6187cb63fa5e"
 "checksum xdg 2.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "a66b7c2281ebde13cf4391d70d4c7e5946c3c25e72a7b859ca8f677dcd0b0c61"
 "checksum xml-rs 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)" = "7ec6c39eaa68382c8e31e35239402c0a9489d4141a8ceb0c716099a0b515b562"

ethcore/Cargo.toml

@@ -8,7 +8,6 @@ authors = ["Parity Technologies <admin@parity.io>"]
 
 [dependencies]
 ansi_term = "0.9"
-bloomchain = "0.1"
 bn = { git = "https://github.com/paritytech/bn" }
 byteorder = "1.0"
 common-types = { path = "types" }

ethcore/src/blockchain/cache.rs

@@ -23,8 +23,6 @@ pub struct CacheSize {
 	pub block_details: usize,
 	/// Transaction addresses cache size.
 	pub transaction_addresses: usize,
-	/// Blooms cache size.
-	pub blocks_blooms: usize,
 	/// Block receipts size.
 	pub block_receipts: usize,
 }
@@ -32,6 +30,6 @@ pub struct CacheSize {
 impl CacheSize {
 	/// Total amount used by the cache.
 	pub fn total(&self) -> usize {
-		self.blocks + self.block_details + self.transaction_addresses + self.blocks_blooms + self.block_receipts
+		self.blocks + self.block_details + self.transaction_addresses + self.block_receipts
 	}
 }

ethcore/src/blockchain/extras.rs

@@ -18,8 +18,6 @@
 
 use std::ops;
 use std::io::Write;
-use bloomchain;
-use blooms::{GroupPosition, BloomGroup};
 use db::Key;
 use engines::epoch::{Transition as EpochTransition};
 use header::BlockNumber;
@@ -39,8 +37,6 @@ pub enum ExtrasIndex {
 	BlockHash = 1,
 	/// Transaction address index
 	TransactionAddress = 2,
-	/// Block blooms index
-	BlocksBlooms = 3,
 	/// Block receipts index
 	BlockReceipts = 4,
 	/// Epoch transition data index.
@@ -88,46 +84,6 @@ impl Key<BlockDetails> for H256 {
 	}
 }
 
-pub struct LogGroupKey([u8; 6]);
-
-impl ops::Deref for LogGroupKey {
-	type Target = [u8];
-
-	fn deref(&self) -> &Self::Target {
-		&self.0
-	}
-}
-
-#[derive(Debug, PartialEq, Eq, Hash, Clone)]
-pub struct LogGroupPosition(GroupPosition);
-
-impl From<bloomchain::group::GroupPosition> for LogGroupPosition {
-	fn from(position: bloomchain::group::GroupPosition) -> Self {
-		LogGroupPosition(From::from(position))
-	}
-}
-
-impl HeapSizeOf for LogGroupPosition {
-	fn heap_size_of_children(&self) -> usize {
-		self.0.heap_size_of_children()
-	}
-}
-
-impl Key<BloomGroup> for LogGroupPosition {
-	type Target = LogGroupKey;
-
-	fn key(&self) -> Self::Target {
-		let mut result = [0u8; 6];
-		result[0] = ExtrasIndex::BlocksBlooms as u8;
-		result[1] = self.0.level;
-		result[2] = (self.0.index >> 24) as u8;
-		result[3] = (self.0.index >> 16) as u8;
-		result[4] = (self.0.index >> 8) as u8;
-		result[5] = self.0.index as u8;
-		LogGroupKey(result)
-	}
-}
-
 impl Key<TransactionAddress> for H256 {
 	type Target = H264;
 

ethcore/src/blockchain/generator.rs

@@ -0,0 +1,218 @@
+// Copyright 2015-2017 Parity Technologies (UK) Ltd.
+// This file is part of Parity.
+
+// Parity is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Parity is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Parity.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Blockchain generator for tests.
+
+use std::collections::VecDeque;
+use bigint::prelude::{U256, H256, H2048 as Bloom};
+
+use bytes::Bytes;
+use header::Header;
+use rlp::encode;
+use transaction::SignedTransaction;
+use views::BlockView;
+
+/// Helper structure, used for encoding blocks.
+#[derive(Default, Clone, RlpEncodable)]
+pub struct Block {
+	pub header: Header,
+	pub transactions: Vec<SignedTransaction>,
+	pub uncles: Vec<Header>
+}
+
+impl Block {
+	#[inline]
+	pub fn header(&self) -> Header {
+		self.header.clone()
+	}
+
+	#[inline]
+	pub fn hash(&self) -> H256 {
+		BlockView::new(&self.encoded()).header_view().hash()
+	}
+
+	#[inline]
+	pub fn number(&self) -> u64 {
+		self.header.number()
+	}
+
+	#[inline]
+	pub fn encoded(&self) -> Bytes {
+		encode(self).into_vec()
+	}
+}
+
+#[derive(Debug)]
+pub struct BlockOptions {
+	pub difficulty: U256,
+	pub bloom: Bloom,
+	pub transactions: Vec<SignedTransaction>,
+}
+
+impl Default for BlockOptions {
+	fn default() -> Self {
+		BlockOptions {
+			difficulty: 10.into(),
+			bloom: Bloom::default(),
+			transactions: Vec::new(),
+		}
+	}
+}
+
+#[derive(Clone)]
+pub struct BlockBuilder {
+	blocks: VecDeque<Block>,
+}
+
+impl BlockBuilder {
+	pub fn genesis() -> Self {
+		let mut blocks = VecDeque::with_capacity(1);
+		blocks.push_back(Block::default());
+
+		BlockBuilder {
+			blocks,
+		}
+	}
+
+	#[inline]
+	pub fn add_block(&self) -> Self {
+		self.add_block_with(|| BlockOptions::default())
+	}
+
+	#[inline]
+	pub fn add_blocks(&self, count: usize) -> Self {
+		self.add_blocks_with(count, || BlockOptions::default())
+	}
+
+	#[inline]
+	pub fn add_block_with<T>(&self, get_metadata: T) -> Self where T: Fn() -> BlockOptions {
+		self.add_blocks_with(1, get_metadata)
+	}
+
+	#[inline]
+	pub fn add_block_with_difficulty<T>(&self, difficulty: T) -> Self where T: Into<U256> {
+		let difficulty = difficulty.into();
+		self.add_blocks_with(1, move || BlockOptions {
+			difficulty,
+			..Default::default()
+		})
+	}
+
+	#[inline]
+	pub fn add_block_with_transactions<T>(&self, transactions: T) -> Self
+		where T: IntoIterator<Item = SignedTransaction> {
+		let transactions = transactions.into_iter().collect::<Vec<_>>();
+		self.add_blocks_with(1, || BlockOptions {
+			transactions: transactions.clone(),
+			..Default::default()
+		})
+	}
+
+	#[inline]
+	pub fn add_block_with_bloom(&self, bloom: Bloom) -> Self {
+		self.add_blocks_with(1, move || BlockOptions {
+			bloom,
+			..Default::default()
+		})
+	}
+
+	pub fn add_blocks_with<T>(&self, count: usize, get_metadata: T) -> Self where T: Fn() -> BlockOptions {
+		assert!(count > 0, "There must be at least 1 block");
+		let mut parent_hash = self.last().hash();
+		let mut parent_number = self.last().number();
+		let mut blocks = VecDeque::with_capacity(count);
+		for _ in 0..count {
+			let mut block = Block::default();
+			let metadata = get_metadata();
+			let block_number = parent_number + 1;
+			block.header.set_parent_hash(parent_hash);
+			block.header.set_number(block_number);
+			block.header.set_log_bloom(metadata.bloom);
+			block.header.set_difficulty(metadata.difficulty);
+			block.transactions = metadata.transactions;
+
+			parent_hash = block.hash();
+			parent_number = block_number;
+
+			blocks.push_back(block);
+		}
+
+		BlockBuilder {
+			blocks,
+		}
+	}
+
+	#[inline]
+	pub fn last(&self) -> &Block {
+		self.blocks.back().expect("There is always at least 1 block")
+	}
+}
+
+#[derive(Clone)]
+pub struct BlockGenerator {
+	builders: VecDeque<BlockBuilder>,
+}
+
+impl BlockGenerator {
+	pub fn new<T>(builders: T) -> Self where T: IntoIterator<Item = BlockBuilder> {
+		BlockGenerator {
+			builders: builders.into_iter().collect(),
+		}
+	}
+}
+
+impl Iterator for BlockGenerator {
+	type Item = Block;
+
+	fn next(&mut self) -> Option<Self::Item> {
+		loop {
+			match self.builders.front_mut() {
+				Some(ref mut builder) => {
+					if let Some(block) = builder.blocks.pop_front() {
+						return Some(block);
+					}
+				},
+				None => return None,
+			}
+			self.builders.pop_front();
+		}
+
+	}
+}
+
+#[cfg(test)]
+mod tests {
+	use super::{BlockBuilder, BlockOptions, BlockGenerator};
+
+	#[test]
+	fn test_block_builder() {
+		let genesis = BlockBuilder::genesis();
+		let block_1 = genesis.add_block();
+		let block_1001 = block_1.add_blocks(1000);
+		let block_1002 = block_1001.add_block_with(|| BlockOptions::default());
+		let generator = BlockGenerator::new(vec![genesis, block_1, block_1001, block_1002]);
+		assert_eq!(generator.count(), 1003);
+	}
+
+	#[test]
+	fn test_block_builder_fork() {
+		let genesis = BlockBuilder::genesis();
+		let block_10a = genesis.add_blocks(10);
+		let block_11b = genesis.add_blocks(11);
+		assert_eq!(block_10a.last().number(), 10);
+		assert_eq!(block_11b.last().number(), 11);
+	}
+}

ethcore/src/blockchain/generator/block.rs

@@ -1,72 +0,0 @@
-// Copyright 2015-2017 Parity Technologies (UK) Ltd.
-// This file is part of Parity.
-
-// Parity is free software: you can redistribute it and/or modify
-// it under the terms of the GNU General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-
-// Parity is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU General Public License for more details.
-
-// You should have received a copy of the GNU General Public License
-// along with Parity.  If not, see <http://www.gnu.org/licenses/>.
-
-use rlp::*;
-use bigint::hash::{H256, H2048};
-use bytes::Bytes;
-use header::Header;
-use transaction::SignedTransaction;
-
-use super::fork::Forkable;
-use super::bloom::WithBloom;
-use super::complete::CompleteBlock;
-use super::transaction::WithTransaction;
-
-/// Helper structure, used for encoding blocks.
-#[derive(Default)]
-pub struct Block {
-	pub header: Header,
-	pub transactions: Vec<SignedTransaction>,
-	pub uncles: Vec<Header>
-}
-
-impl Encodable for Block {
-	fn rlp_append(&self, s: &mut RlpStream) {
-		s.begin_list(3);
-		s.append(&self.header);
-		s.append_list(&self.transactions);
-		s.append_list(&self.uncles);
-	}
-}
-
-impl Forkable for Block {
-	fn fork(mut self, fork_number: usize) -> Self where Self: Sized {
-		let difficulty = self.header.difficulty().clone() - fork_number.into();
-		self.header.set_difficulty(difficulty);
-		self
-	}
-}
-
-impl WithBloom for Block {
-	fn with_bloom(mut self, bloom: H2048) -> Self where Self: Sized {
-		self.header.set_log_bloom(bloom);
-		self
-	}
-}
-
-impl WithTransaction for Block {
-	fn with_transaction(mut self, transaction: SignedTransaction) -> Self where Self: Sized {
-		self.transactions.push(transaction);
-		self
-	}
-}
-
-impl CompleteBlock for Block {
-	fn complete(mut self, parent_hash: H256) -> Bytes {
-		self.header.set_parent_hash(parent_hash);
-		encode(&self).into_vec()
-	}
-}

ethcore/src/blockchain/generator/bloom.rs

@@ -1,35 +0,0 @@
-// Copyright 2015-2017 Parity Technologies (UK) Ltd.
-// This file is part of Parity.
-
-// Parity is free software: you can redistribute it and/or modify
-// it under the terms of the GNU General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-
-// Parity is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU General Public License for more details.
-
-// You should have received a copy of the GNU General Public License
-// along with Parity.  If not, see <http://www.gnu.org/licenses/>.
-
-use bigint::hash::H2048;
-
-pub trait WithBloom {
-	fn with_bloom(self, bloom: H2048) -> Self where Self: Sized;
-}
-
-pub struct Bloom<'a, I> where I: 'a {
-	pub iter: &'a mut I,
-	pub bloom: H2048,
-}
-
-impl<'a, I> Iterator for Bloom<'a, I> where I: Iterator, <I as Iterator>::Item: WithBloom {
-	type Item = <I as Iterator>::Item;
-
-	#[inline]
-	fn next(&mut self) -> Option<Self::Item> {
-		self.iter.next().map(|item| item.with_bloom(self.bloom.clone()))
-	}
-}

ethcore/src/blockchain/generator/complete.rs

@@ -1,52 +0,0 @@
-// Copyright 2015-2017 Parity Technologies (UK) Ltd.
-// This file is part of Parity.
-
-// Parity is free software: you can redistribute it and/or modify
-// it under the terms of the GNU General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-
-// Parity is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU General Public License for more details.
-
-// You should have received a copy of the GNU General Public License
-// along with Parity.  If not, see <http://www.gnu.org/licenses/>.
-
-use bigint::hash::H256;
-use bytes::Bytes;
-use views::BlockView;
-
-#[derive(Default, Clone)]
-pub struct BlockFinalizer {
-	parent_hash: H256
-}
-
-impl BlockFinalizer {
-	pub fn fork(&self) -> Self {
-		self.clone()
-	}
-}
-
-pub trait CompleteBlock {
-	fn complete(self, parent_hash: H256) -> Bytes;
-}
-
-pub struct Complete<'a, I> where I: 'a {
-	pub iter: &'a mut I,
-	pub finalizer: &'a mut BlockFinalizer,
-}
-
-impl<'a, I> Iterator for Complete<'a, I> where I: Iterator, <I as Iterator>::Item: CompleteBlock {
-	type Item = Bytes;
-
-	#[inline]
-	fn next(&mut self) -> Option<Self::Item> {
-		self.iter.next().map(|item| {
-			let rlp = item.complete(self.finalizer.parent_hash.clone());
-			self.finalizer.parent_hash = BlockView::new(&rlp).header_view().hash();
-			rlp
-		})
-	}
-}

ethcore/src/blockchain/generator/fork.rs

@@ -1,42 +0,0 @@
-// Copyright 2015-2017 Parity Technologies (UK) Ltd.
-// This file is part of Parity.
-
-// Parity is free software: you can redistribute it and/or modify
-// it under the terms of the GNU General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-
-// Parity is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU General Public License for more details.
-
-// You should have received a copy of the GNU General Public License
-// along with Parity.  If not, see <http://www.gnu.org/licenses/>.
-
-pub trait Forkable {
-	fn fork(self, fork_number: usize) -> Self where Self: Sized;
-}
-
-pub struct Fork<I> {
-	pub iter: I,
-	pub fork_number: usize,
-}
-
-impl<I> Clone for Fork<I> where I: Iterator + Clone {
-	fn clone(&self) -> Self {
-		Fork {
-			iter: self.iter.clone(),
-			fork_number: self.fork_number
-		}
-	}
-}
-
-impl<I> Iterator for Fork<I> where I: Iterator, <I as Iterator>::Item: Forkable {
-	type Item = <I as Iterator>::Item;
-
-	#[inline]
-	fn next(&mut self) -> Option<Self::Item> {
-		self.iter.next().map(|item| item.fork(self.fork_number))
-	}
-}

ethcore/src/blockchain/generator/generator.rs

@@ -1,179 +0,0 @@
-// Copyright 2015-2017 Parity Technologies (UK) Ltd.
-// This file is part of Parity.
-
-// Parity is free software: you can redistribute it and/or modify
-// it under the terms of the GNU General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-
-// Parity is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU General Public License for more details.
-
-// You should have received a copy of the GNU General Public License
-// along with Parity.  If not, see <http://www.gnu.org/licenses/>.
-
-use bigint::prelude::U256;
-use bigint::hash::H2048;
-use bytes::Bytes;
-use header::BlockNumber;
-use transaction::SignedTransaction;
-use super::fork::Fork;
-use super::bloom::Bloom;
-use super::complete::{BlockFinalizer, CompleteBlock, Complete};
-use super::block::Block;
-use super::transaction::Transaction;
-
-/// Chain iterator interface.
-pub trait ChainIterator: Iterator + Sized {
-	/// Should be called to create a fork of current iterator.
-	/// Blocks generated by fork will have lower difficulty than current chain.
-	fn fork(&self, fork_number: usize) -> Fork<Self> where Self: Clone;
-	/// Should be called to make every consecutive block have given bloom.
-	fn with_bloom(&mut self, bloom: H2048) -> Bloom<Self>;
-	/// Should be called to make every consecutive block have given transaction.
-	fn with_transaction(&mut self, transaction: SignedTransaction) -> Transaction<Self>;
-	/// Should be called to complete block. Without complete, block may have incorrect hash.
-	fn complete<'a>(&'a mut self, finalizer: &'a mut BlockFinalizer) -> Complete<'a, Self>;
-	/// Completes and generates block.
-	fn generate<'a>(&'a mut self, finalizer: &'a mut BlockFinalizer) -> Option<Bytes> where Self::Item: CompleteBlock;
-}
-
-impl<I> ChainIterator for I where I: Iterator + Sized {
-	fn fork(&self, fork_number: usize) -> Fork<Self> where I: Clone {
-		Fork {
-			iter: self.clone(),
-			fork_number: fork_number
-		}
-	}
-
-	fn with_bloom(&mut self, bloom: H2048) -> Bloom<Self> {
-		Bloom {
-			iter: self,
-			bloom: bloom
-		}
-	}
-
-	fn with_transaction(&mut self, transaction: SignedTransaction) -> Transaction<Self> {
-		Transaction {
-			iter: self,
-			transaction: transaction,
-		}
-	}
-
-	fn complete<'a>(&'a mut self, finalizer: &'a mut BlockFinalizer) -> Complete<'a, Self> {
-		Complete {
-			iter: self,
-			finalizer: finalizer
-		}
-	}
-
-	fn generate<'a>(&'a mut self, finalizer: &'a mut BlockFinalizer) -> Option<Bytes> where <I as Iterator>::Item: CompleteBlock {
-		self.complete(finalizer).next()
-	}
-}
-
-/// Blockchain generator.
-#[derive(Clone)]
-pub struct ChainGenerator {
-	/// Next block number.
-	number: BlockNumber,
-	/// Next block difficulty.
-	difficulty: U256,
-}
-
-impl ChainGenerator {
-	fn prepare_block(&self) -> Block {
-		let mut block = Block::default();
-		block.header.set_number(self.number);
-		block.header.set_difficulty(self.difficulty);
-		block
-	}
-}
-
-impl Default for ChainGenerator {
-	fn default() -> Self {
-		ChainGenerator {
-			number: 0,
-			difficulty: 1000.into(),
-		}
-	}
-}
-
-impl Iterator for ChainGenerator {
-	type Item = Block;
-
-	fn next(&mut self) -> Option<Self::Item> {
-		let block = self.prepare_block();
-		self.number += 1;
-		Some(block)
-	}
-}
-
-mod tests {
-	use bigint::hash::{H256, H2048};
-	use views::BlockView;
-	use blockchain::generator::{ChainIterator, ChainGenerator, BlockFinalizer};
-
-	#[test]
-	fn canon_chain_generator() {
-		let mut canon_chain = ChainGenerator::default();
-		let mut finalizer = BlockFinalizer::default();
-
-		let genesis_rlp = canon_chain.generate(&mut finalizer).unwrap();
-		let genesis = BlockView::new(&genesis_rlp);
-
-		assert_eq!(genesis.header_view().parent_hash(), H256::default());
-		assert_eq!(genesis.header_view().number(), 0);
-
-		let b1_rlp = canon_chain.generate(&mut finalizer).unwrap();
-		let b1 = BlockView::new(&b1_rlp);
-
-		assert_eq!(b1.header_view().parent_hash(), genesis.header_view().hash());
-		assert_eq!(b1.header_view().number(), 1);
-
-		let mut fork_chain = canon_chain.fork(1);
-
-		let b2_rlp_fork = fork_chain.generate(&mut finalizer.fork()).unwrap();
-		let b2_fork = BlockView::new(&b2_rlp_fork);
-
-		assert_eq!(b2_fork.header_view().parent_hash(), b1.header_view().hash());
-		assert_eq!(b2_fork.header_view().number(), 2);
-
-		let b2_rlp = canon_chain.generate(&mut finalizer).unwrap();
-		let b2 = BlockView::new(&b2_rlp);
-
-		assert_eq!(b2.header_view().parent_hash(), b1.header_view().hash());
-		assert_eq!(b2.header_view().number(), 2);
-		assert!(b2.header_view().difficulty() > b2_fork.header_view().difficulty());
-	}
-
-	#[test]
-	fn with_bloom_generator() {
-		let bloom = H2048([0x1; 256]);
-		let mut gen = ChainGenerator::default();
-		let mut finalizer = BlockFinalizer::default();
-
-		let block0_rlp = gen.with_bloom(bloom).generate(&mut finalizer).unwrap();
-		let block1_rlp = gen.generate(&mut finalizer).unwrap();
-		let block0 = BlockView::new(&block0_rlp);
-		let block1 = BlockView::new(&block1_rlp);
-
-		assert_eq!(block0.header_view().number(), 0);
-		assert_eq!(block0.header_view().parent_hash(), H256::default());
-
-		assert_eq!(block1.header_view().number(), 1);
-		assert_eq!(block1.header_view().parent_hash(), block0.header_view().hash());
-
-	}
-
-	#[test]
-	fn generate_1000_blocks() {
-		let generator = ChainGenerator::default();
-		let mut finalizer = BlockFinalizer::default();
-		let blocks: Vec<_> = generator.take(1000).complete(&mut finalizer).collect();
-		assert_eq!(blocks.len(), 1000);
-	}
-}
-

ethcore/src/blockchain/generator/mod.rs

@@ -1,27 +0,0 @@
-// Copyright 2015-2017 Parity Technologies (UK) Ltd.
-// This file is part of Parity.
-
-// Parity is free software: you can redistribute it and/or modify
-// it under the terms of the GNU General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-
-// Parity is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU General Public License for more details.
-
-// You should have received a copy of the GNU General Public License
-// along with Parity.  If not, see <http://www.gnu.org/licenses/>.
-
-//! Blockchain generator for tests.
-
-mod bloom;
-mod block;
-mod complete;
-mod fork;
-pub mod generator;
-mod transaction;
-
-pub use self::complete::BlockFinalizer;
-pub use self::generator::{ChainIterator, ChainGenerator};

ethcore/src/blockchain/generator/transaction.rs

@@ -1,35 +0,0 @@
-// Copyright 2015-2017 Parity Technologies (UK) Ltd.
-// This file is part of Parity.
-
-// Parity is free software: you can redistribute it and/or modify
-// it under the terms of the GNU General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-
-// Parity is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU General Public License for more details.
-
-// You should have received a copy of the GNU General Public License
-// along with Parity.  If not, see <http://www.gnu.org/licenses/>.
-
-use transaction::SignedTransaction;
-
-pub trait WithTransaction {
-	fn with_transaction(self, transaction: SignedTransaction) -> Self where Self: Sized;
-}
-
-pub struct Transaction<'a, I> where I: 'a {
-	pub iter: &'a mut I,
-	pub transaction: SignedTransaction,
-}
-
-impl <'a, I> Iterator for Transaction<'a, I> where I: Iterator, <I as Iterator>::Item: WithTransaction {
-	type Item = <I as Iterator>::Item;
-
-	#[inline]
-	fn next(&mut self) -> Option<Self::Item> {
-		self.iter.next().map(|item| item.with_transaction(self.transaction.clone()))
-	}
-}

ethcore/src/blockchain/update.rs

@@ -2,8 +2,7 @@ use std::collections::HashMap;
 use bigint::hash::H256;
 use header::BlockNumber;
 use blockchain::block_info::BlockInfo;
-use blooms::BloomGroup;
+use blockchain::extras::{BlockDetails, BlockReceipts, TransactionAddress};
-use super::extras::{BlockDetails, BlockReceipts, TransactionAddress, LogGroupPosition};
 
 /// Block extras update info.
 pub struct ExtrasUpdate<'a> {
@@ -19,8 +18,6 @@ pub struct ExtrasUpdate<'a> {
 	pub block_details: HashMap<H256, BlockDetails>,
 	/// Modified block receipts.
 	pub block_receipts: HashMap<H256, BlockReceipts>,
-	/// Modified blocks blooms.
-	pub blocks_blooms: HashMap<LogGroupPosition, BloomGroup>,
 	/// Modified transaction addresses (None signifies removed transactions).
 	pub transactions_addresses: HashMap<H256, Option<TransactionAddress>>,
 }

ethcore/src/blooms/bloom_group.rs

@@ -1,74 +0,0 @@
-// Copyright 2015-2017 Parity Technologies (UK) Ltd.
-// This file is part of Parity.
-
-// Parity is free software: you can redistribute it and/or modify
-// it under the terms of the GNU General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-
-// Parity is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU General Public License for more details.
-
-// You should have received a copy of the GNU General Public License
-// along with Parity.  If not, see <http://www.gnu.org/licenses/>.
-
-use bloomchain::group as bc;
-use rlp::*;
-use heapsize::HeapSizeOf;
-use super::Bloom;
-
-/// Represents group of X consecutive blooms.
-#[derive(Debug, Clone)]
-pub struct BloomGroup {
-	blooms: Vec<Bloom>,
-}
-
-impl From<bc::BloomGroup> for BloomGroup {
-	fn from(group: bc::BloomGroup) -> Self {
-		let blooms = group.blooms
-			.into_iter()
-			.map(From::from)
-			.collect();
-
-		BloomGroup {
-			blooms: blooms
-		}
-	}
-}
-
-impl Into<bc::BloomGroup> for BloomGroup {
-	fn into(self) -> bc::BloomGroup {
-		let blooms = self.blooms
-			.into_iter()
-			.map(Into::into)
-			.collect();
-
-		bc::BloomGroup {
-			blooms: blooms
-		}
-	}
-}
-
-impl Decodable for BloomGroup {
-	fn decode(rlp: &UntrustedRlp) -> Result<Self, DecoderError> {
-		let blooms = rlp.as_list()?;
-		let group = BloomGroup {
-			blooms: blooms
-		};
-		Ok(group)
-	}
-}
-
-impl Encodable for BloomGroup {
-	fn rlp_append(&self, s: &mut RlpStream) {
-		s.append_list(&self.blooms);
-	}
-}
-
-impl HeapSizeOf for BloomGroup {
-	fn heap_size_of_children(&self) -> usize {
-		self.blooms.heap_size_of_children()
-	}
-}

ethcore/src/blooms/group_position.rs

@@ -1,42 +0,0 @@
-// Copyright 2015-2017 Parity Technologies (UK) Ltd.
-// This file is part of Parity.
-
-// Parity is free software: you can redistribute it and/or modify
-// it under the terms of the GNU General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-
-// Parity is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU General Public License for more details.
-
-// You should have received a copy of the GNU General Public License
-// along with Parity.  If not, see <http://www.gnu.org/licenses/>.
-
-use bloomchain::group as bc;
-use heapsize::HeapSizeOf;
-
-/// Represents `BloomGroup` position in database.
-#[derive(PartialEq, Eq, Hash, Clone, Debug)]
-pub struct GroupPosition {
-	/// Bloom level.
-	pub level: u8,
-	/// Group index.
-	pub index: u32,
-}
-
-impl From<bc::GroupPosition> for GroupPosition {
-	fn from(p: bc::GroupPosition) -> Self {
-		GroupPosition {
-			level: p.level as u8,
-			index: p.index as u32,
-		}
-	}
-}
-
-impl HeapSizeOf for GroupPosition {
-	fn heap_size_of_children(&self) -> usize {
-		0
-	}
-}

ethcore/src/client/client.rs

@@ -1673,17 +1673,8 @@ impl BlockChainClient for Client {
 		};
 
 		let chain = self.chain.read();
-		let blocks = filter.bloom_possibilities().iter()
+		let blocks = chain.blocks_with_blooms(&filter.bloom_possibilities(), from, to);
-			.map(move |bloom| {
+		chain.logs(blocks, |entry| filter.matches(entry), filter.limit)
-				chain.blocks_with_bloom(bloom, from, to)
-			})
-			.flat_map(|m| m)
-			// remove duplicate elements
-			.collect::<HashSet<u64>>()
-			.into_iter()
-			.collect::<Vec<u64>>();
-
-		self.chain.read().logs(blocks, |entry| filter.matches(entry), filter.limit)
 	}
 
 	fn filter_traces(&self, filter: TraceFilter) -> Option<Vec<LocalizedTrace>> {

ethcore/src/lib.rs

@@ -54,7 +54,6 @@
 //!   cargo build --release
 //!   ```
 
-extern crate bloomchain;
 extern crate bn;
 extern crate byteorder;
 extern crate crossbeam;
@@ -155,7 +154,6 @@ pub mod verification;
 pub mod views;
 
 mod cache_manager;
-mod blooms;
 mod basic_types;
 mod pod_account;
 mod state_db;

ethcore/src/snapshot/error.rs

@@ -57,6 +57,8 @@ pub enum Error {
 	VersionNotSupported(u64),
 	/// Max chunk size is to small to fit basic account data.
 	ChunkTooSmall,
+	/// Oversized chunk
+	ChunkTooLarge,
 	/// Snapshots not supported by the consensus engine.
 	SnapshotsUnsupported,
 	/// Bad epoch transition.
@@ -85,6 +87,7 @@ impl fmt::Display for Error {
 			Error::Trie(ref err) => err.fmt(f),
 			Error::VersionNotSupported(ref ver) => write!(f, "Snapshot version {} is not supprted.", ver),
 			Error::ChunkTooSmall => write!(f, "Chunk size is too small."),
+			Error::ChunkTooLarge => write!(f, "Chunk size is too large."),
 			Error::SnapshotsUnsupported => write!(f, "Snapshots unsupported by consensus engine."),
 			Error::BadEpochProof(i) => write!(f, "Bad epoch proof for transition to epoch {}", i),
 			Error::WrongChunkFormat(ref msg) => write!(f, "Wrong chunk format: {}", msg),

ethcore/src/snapshot/mod.rs

@@ -77,6 +77,11 @@ mod traits;
 // Try to have chunks be around 4MB (before compression)
 const PREFERRED_CHUNK_SIZE: usize = 4 * 1024 * 1024;
 
+// Maximal chunk size (decompressed)
+// Snappy::decompressed_len estimation may sometimes yield results greater
+// than PREFERRED_CHUNK_SIZE so allow some threshold here.
+const MAX_CHUNK_SIZE: usize = PREFERRED_CHUNK_SIZE / 4 * 5;
+
 // Minimum supported state chunk version.
 const MIN_SUPPORTED_STATE_CHUNK_VERSION: u64 = 1;
 // current state chunk version.

ethcore/src/snapshot/service.rs

@@ -23,7 +23,7 @@ use std::path::PathBuf;
 use std::sync::Arc;
 use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
 
-use super::{ManifestData, StateRebuilder, Rebuilder, RestorationStatus, SnapshotService};
+use super::{ManifestData, StateRebuilder, Rebuilder, RestorationStatus, SnapshotService, MAX_CHUNK_SIZE};
 use super::io::{SnapshotReader, LooseReader, SnapshotWriter, LooseWriter};
 
 use blockchain::BlockChain;
@@ -130,6 +130,11 @@ impl Restoration {
 	// feeds a state chunk, aborts early if `flag` becomes false.
 	fn feed_state(&mut self, hash: H256, chunk: &[u8], flag: &AtomicBool) -> Result<(), Error> {
 		if self.state_chunks_left.contains(&hash) {
+			let expected_len = snappy::decompressed_len(chunk)?;
+			if expected_len > MAX_CHUNK_SIZE {
+				trace!(target: "snapshot", "Discarding large chunk: {} vs {}", expected_len, MAX_CHUNK_SIZE);
+				return Err(::snapshot::Error::ChunkTooLarge.into());
+			}
 			let len = snappy::decompress_into(chunk, &mut self.snappy_buffer)?;
 
 			self.state.feed(&self.snappy_buffer[..len], flag)?;
@@ -147,6 +152,11 @@ impl Restoration {
 	// feeds a block chunk
 	fn feed_blocks(&mut self, hash: H256, chunk: &[u8], engine: &EthEngine, flag: &AtomicBool) -> Result<(), Error> {
 		if self.block_chunks_left.contains(&hash) {
+			let expected_len = snappy::decompressed_len(chunk)?;
+			if expected_len > MAX_CHUNK_SIZE {
+				trace!(target: "snapshot", "Discarding large chunk: {} vs {}", expected_len, MAX_CHUNK_SIZE);
+				return Err(::snapshot::Error::ChunkTooLarge.into());
+			}
 			let len = snappy::decompress_into(chunk, &mut self.snappy_buffer)?;
 
 			self.secondary.feed(&self.snappy_buffer[..len], engine, flag)?;

ethcore/src/snapshot/tests/proof_of_work.rs

@@ -19,7 +19,7 @@
 use devtools::RandomTempPath;
 use error::Error;
 
-use blockchain::generator::{ChainGenerator, ChainIterator, BlockFinalizer};
+use blockchain::generator::{BlockGenerator, BlockBuilder};
 use blockchain::BlockChain;
 use snapshot::{chunk_secondary, Error as SnapshotError, Progress, SnapshotComponents};
 use snapshot::io::{PackedReader, PackedWriter, SnapshotReader, SnapshotWriter};
@@ -35,9 +35,10 @@ use std::sync::atomic::AtomicBool;
 const SNAPSHOT_MODE: ::snapshot::PowSnapshot = ::snapshot::PowSnapshot { blocks: 30000, max_restore_blocks: 30000 };
 
 fn chunk_and_restore(amount: u64) {
-	let mut canon_chain = ChainGenerator::default();
+	let genesis = BlockBuilder::genesis();
-	let mut finalizer = BlockFinalizer::default();
+	let rest = genesis.add_blocks(amount as usize);
-	let genesis = canon_chain.generate(&mut finalizer).unwrap();
+	let generator = BlockGenerator::new(vec![rest]);
+	let genesis = genesis.last();
 
 	let engine = ::spec::Spec::new_test().engine;
 	let new_path = RandomTempPath::create_dir();
@@ -45,13 +46,12 @@ fn chunk_and_restore(amount: u64) {
 	snapshot_path.push("SNAP");
 
 	let old_db = Arc::new(kvdb_memorydb::create(::db::NUM_COLUMNS.unwrap_or(0)));
-	let bc = BlockChain::new(Default::default(), &genesis, old_db.clone());
+	let bc = BlockChain::new(Default::default(), &genesis.encoded(), old_db.clone());
 
 	// build the blockchain.
 	let mut batch = DBTransaction::new();
-	for _ in 0..amount {
+	for block in generator {
-		let block = canon_chain.generate(&mut finalizer).unwrap();
+		bc.insert_block(&mut batch, &block.encoded(), vec![]);
-		bc.insert_block(&mut batch, &block, vec![]);
 		bc.commit();
 	}
 
@@ -82,7 +82,7 @@ fn chunk_and_restore(amount: u64) {
 
 	// restore it.
 	let new_db = Arc::new(kvdb_memorydb::create(::db::NUM_COLUMNS.unwrap_or(0)));
-	let new_chain = BlockChain::new(Default::default(), &genesis, new_db.clone());
+	let new_chain = BlockChain::new(Default::default(), &genesis.encoded(), new_db.clone());
 	let mut rebuilder = SNAPSHOT_MODE.rebuilder(new_chain, new_db.clone(), &manifest).unwrap();
 
 	let reader = PackedReader::new(&snapshot_path).unwrap().unwrap();
@@ -97,15 +97,19 @@ fn chunk_and_restore(amount: u64) {
 	drop(rebuilder);
 
 	// and test it.
-	let new_chain = BlockChain::new(Default::default(), &genesis, new_db);
+	let new_chain = BlockChain::new(Default::default(), &genesis.encoded(), new_db);
 	assert_eq!(new_chain.best_block_hash(), best_hash);
 }
 
 #[test]
-fn chunk_and_restore_500() { chunk_and_restore(500) }
+fn chunk_and_restore_500() {
+	chunk_and_restore(500)
+}
 
 #[test]
-fn chunk_and_restore_40k() { chunk_and_restore(40000) }
+fn chunk_and_restore_4k() {
+	chunk_and_restore(4000)
+}
 
 #[test]
 fn checks_flag() {
@@ -120,17 +124,12 @@ fn checks_flag() {
 
 	stream.append_empty_data().append_empty_data();
 
-	let genesis = {
+	let genesis = BlockBuilder::genesis();
-		let mut canon_chain = ChainGenerator::default();
-		let mut finalizer = BlockFinalizer::default();
-		canon_chain.generate(&mut finalizer).unwrap()
-	};
-
 	let chunk = stream.out();
 
 	let db = Arc::new(kvdb_memorydb::create(::db::NUM_COLUMNS.unwrap_or(0)));
 	let engine = ::spec::Spec::new_test().engine;
-	let chain = BlockChain::new(Default::default(), &genesis, db.clone());
+	let chain = BlockChain::new(Default::default(), &genesis.last().encoded(), db.clone());
 
 	let manifest = ::snapshot::ManifestData {
 		version: 2,

ethcore/src/trace/bloom.rs

@@ -1,77 +0,0 @@
-use bloomchain::Bloom;
-use bloomchain::group::{BloomGroup, GroupPosition};
-use basic_types::LogBloom;
-
-/// Helper structure representing bloom of the trace.
-#[derive(Clone, RlpEncodableWrapper, RlpDecodableWrapper)]
-pub struct BlockTracesBloom(LogBloom);
-
-impl From<LogBloom> for BlockTracesBloom {
-	fn from(bloom: LogBloom) -> BlockTracesBloom {
-		BlockTracesBloom(bloom)
-	}
-}
-
-impl From<Bloom> for BlockTracesBloom {
-	fn from(bloom: Bloom) -> BlockTracesBloom {
-		let bytes: [u8; 256] = bloom.into();
-		BlockTracesBloom(LogBloom::from(bytes))
-	}
-}
-
-impl Into<Bloom> for BlockTracesBloom {
-	fn into(self) -> Bloom {
-		let log = self.0;
-		Bloom::from(log.0)
-	}
-}
-
-/// Represents group of X consecutive blooms.
-#[derive(Clone, RlpEncodableWrapper, RlpDecodableWrapper)]
-pub struct BlockTracesBloomGroup {
-	blooms: Vec<BlockTracesBloom>,
-}
-
-impl From<BloomGroup> for BlockTracesBloomGroup {
-	fn from(group: BloomGroup) -> Self {
-		let blooms = group.blooms
-			.into_iter()
-			.map(From::from)
-			.collect();
-
-		BlockTracesBloomGroup {
-			blooms: blooms
-		}
-	}
-}
-
-impl Into<BloomGroup> for BlockTracesBloomGroup {
-	fn into(self) -> BloomGroup {
-		let blooms = self.blooms
-			.into_iter()
-			.map(Into::into)
-			.collect();
-
-		BloomGroup {
-			blooms: blooms
-		}
-	}
-}
-
-/// Represents `BloomGroup` position in database.
-#[derive(PartialEq, Eq, Hash, Clone, Debug)]
-pub struct TraceGroupPosition {
-	/// Bloom level.
-	pub level: u8,
-	/// Group index.
-	pub index: u32,
-}
-
-impl From<GroupPosition> for TraceGroupPosition {
-	fn from(p: GroupPosition) -> Self {
-		TraceGroupPosition {
-			level: p.level as u8,
-			index: p.index as u32,
-		}
-	}
-}

ethcore/src/trace/config.rs

@@ -15,7 +15,6 @@
 // along with Parity.  If not, see <http://www.gnu.org/licenses/>.
 
 //! Traces config.
-use bloomchain::Config as BloomConfig;
 
 /// Traces config.
 #[derive(Debug, PartialEq, Clone)]
@@ -23,8 +22,6 @@ pub struct Config {
 	/// Indicates if tracing should be enabled or not.
 	/// If it's None, it will be automatically configured.
 	pub enabled: bool,
-	/// Traces blooms configuration.
-	pub blooms: BloomConfig,
 	/// Preferef cache-size.
 	pub pref_cache_size: usize,
 	/// Max cache-size.
@@ -35,10 +32,6 @@ impl Default for Config {
 	fn default() -> Self {
 		Config {
 			enabled: false,
-			blooms: BloomConfig {
-				levels: 3,
-				elements_per_index: 16,
-			},
 			pref_cache_size: 15 * 1024 * 1024,
 			max_cache_size: 20 * 1024 * 1024,
 		}

ethcore/src/trace/db.rs

@@ -15,19 +15,15 @@
 // along with Parity.  If not, see <http://www.gnu.org/licenses/>.
 
 //! Trace database.
-use std::ops::Deref;
 use std::collections::{HashMap, VecDeque};
 use std::sync::Arc;
-use bloomchain::{Number, Config as BloomConfig};
-use bloomchain::group::{BloomGroupDatabase, BloomGroupChain, GroupPosition, BloomGroup};
 use heapsize::HeapSizeOf;
-use bigint::hash::{H256, H264};
+use bigint::hash::{H256, H264, H2048 as Bloom};
 use kvdb::{KeyValueDB, DBTransaction};
 use parking_lot::RwLock;
 use header::BlockNumber;
 use trace::{LocalizedTrace, Config, Filter, Database as TraceDatabase, ImportRequest, DatabaseExtras};
 use db::{self, Key, Writable, Readable, CacheUpdatePolicy};
-use blooms;
 use super::flat::{FlatTrace, FlatBlockTraces, FlatTransactionTraces};
 use cache_manager::CacheManager;
 
@@ -37,8 +33,8 @@ const TRACE_DB_VER: &'static [u8] = b"1.0";
 enum TraceDBIndex {
 	/// Block traces index.
 	BlockTraces = 0,
-	/// Trace bloom group index.
+	/// Blooms index.
-	BloomGroups = 1,
+	Blooms = 2,
 }
 
 impl Key<FlatBlockTraces> for H256 {
@@ -52,80 +48,37 @@ impl Key<FlatBlockTraces> for H256 {
 	}
 }
 
-/// Wrapper around `blooms::GroupPosition` so it could be
+impl Key<Bloom> for H256 {
-/// uniquely identified in the database.
+	type Target = H264;
-#[derive(Debug, PartialEq, Eq, Hash, Clone)]
-struct TraceGroupPosition(blooms::GroupPosition);
 
-impl From<GroupPosition> for TraceGroupPosition {
+	fn key(&self) -> H264 {
-	fn from(position: GroupPosition) -> Self {
+		let mut result = H264::default();
-		TraceGroupPosition(From::from(position))
+		result[0] = TraceDBIndex::Blooms as u8;
-	}
+		result[1..33].copy_from_slice(self);
-}
+		result
-
-impl HeapSizeOf for TraceGroupPosition {
-	fn heap_size_of_children(&self) -> usize {
-		0
-	}
-}
-
-/// Helper data structure created cause [u8; 6] does not implement Deref to &[u8].
-pub struct TraceGroupKey([u8; 6]);
-
-impl Deref for TraceGroupKey {
-	type Target = [u8];
-
-	fn deref(&self) -> &Self::Target {
-		&self.0
-	}
-}
-
-impl Key<blooms::BloomGroup> for TraceGroupPosition {
-	type Target = TraceGroupKey;
-
-	fn key(&self) -> Self::Target {
-		let mut result = [0u8; 6];
-		result[0] = TraceDBIndex::BloomGroups as u8;
-		result[1] = self.0.level;
-		result[2] = self.0.index as u8;
-		result[3] = (self.0.index >> 8) as u8;
-		result[4] = (self.0.index >> 16) as u8;
-		result[5] = (self.0.index >> 24) as u8;
-		TraceGroupKey(result)
 	}
 }
 
 #[derive(Debug, Hash, Eq, PartialEq)]
 enum CacheId {
 	Trace(H256),
-	Bloom(TraceGroupPosition),
+	Bloom(H256),
 }
 
 /// Trace database.
 pub struct TraceDB<T> where T: DatabaseExtras {
 	// cache
 	traces: RwLock<HashMap<H256, FlatBlockTraces>>,
-	blooms: RwLock<HashMap<TraceGroupPosition, blooms::BloomGroup>>,
+	blooms: RwLock<HashMap<H256, Bloom>>,
 	cache_manager: RwLock<CacheManager<CacheId>>,
 	// db
 	tracesdb: Arc<KeyValueDB>,
-	// config,
-	bloom_config: BloomConfig,
 	// tracing enabled
 	enabled: bool,
 	// extras
 	extras: Arc<T>,
 }
 
-impl<T> BloomGroupDatabase for TraceDB<T> where T: DatabaseExtras {
-	fn blooms_at(&self, position: &GroupPosition) -> Option<BloomGroup> {
-		let position = TraceGroupPosition::from(position.clone());
-		let result = self.tracesdb.read_with_cache(db::COL_TRACE, &self.blooms, &position).map(Into::into);
-		self.note_used(CacheId::Bloom(position));
-		result
-	}
-}
-
 impl<T> TraceDB<T> where T: DatabaseExtras {
 	/// Creates new instance of `TraceDB`.
 	pub fn new(config: Config, tracesdb: Arc<KeyValueDB>, extras: Arc<T>) -> Self {
@@ -137,13 +90,12 @@ impl<T> TraceDB<T> where T: DatabaseExtras {
 		tracesdb.write(batch).expect("failed to update version");
 
 		TraceDB {
-			traces: RwLock::new(HashMap::new()),
-			blooms: RwLock::new(HashMap::new()),
 			cache_manager: RwLock::new(CacheManager::new(config.pref_cache_size, config.max_cache_size, 10 * 1024)),
-			tracesdb: tracesdb,
+			tracesdb,
-			bloom_config: config.blooms,
 			enabled: config.enabled,
-			extras: extras,
+			extras,
+			traces: RwLock::default(),
+			blooms: RwLock::default(),
 		}
 	}
 
@@ -188,6 +140,12 @@ impl<T> TraceDB<T> where T: DatabaseExtras {
 		result
 	}
 
+	fn bloom(&self, block_hash: &H256) -> Option<Bloom> {
+		let result = self.tracesdb.read_with_cache(db::COL_TRACE, &self.blooms, block_hash);
+		self.note_used(CacheId::Bloom(block_hash.clone()));
+		result
+	}
+
 	/// Returns vector of transaction traces for given block.
 	fn transactions_traces(&self, block_hash: &H256) -> Option<Vec<FlatTransactionTraces>> {
 		self.traces(block_hash).map(Into::into)
@@ -264,49 +222,16 @@ impl<T> TraceDatabase for TraceDB<T> where T: DatabaseExtras {
 			return;
 		}
 
-		// now let's rebuild the blooms
-		if !request.enacted.is_empty() {
-			let range_start = request.block_number as Number + 1 - request.enacted.len();
-			let range_end = range_start + request.retracted;
-			let replaced_range = range_start..range_end;
-			let enacted_blooms = request.enacted
-				.iter()
-				// all traces are expected to be found here. That's why `expect` has been used
-				// instead of `filter_map`. If some traces haven't been found, it meens that
-				// traces database is corrupted or incomplete.
-				.map(|block_hash| if block_hash == &request.block_hash {
-					request.traces.bloom()
-				} else {
-					self.traces(block_hash).expect("Traces database is incomplete.").bloom()
-				})
-				.map(blooms::Bloom::from)
-				.map(Into::into)
-				.collect();
-
-			let chain = BloomGroupChain::new(self.bloom_config, self);
-			let trace_blooms = chain.replace(&replaced_range, enacted_blooms);
-			let blooms_to_insert = trace_blooms.into_iter()
-				.map(|p| (From::from(p.0), From::from(p.1)))
-				.collect::<HashMap<TraceGroupPosition, blooms::BloomGroup>>();
-
-			let blooms_keys: Vec<_> = blooms_to_insert.keys().cloned().collect();
-			let mut blooms = self.blooms.write();
-			batch.extend_with_cache(db::COL_TRACE, &mut *blooms, blooms_to_insert, CacheUpdatePolicy::Remove);
-			// note_used must be called after locking blooms to avoid cache/traces deadlock on garbage collection
-			for key in blooms_keys {
-				self.note_used(CacheId::Bloom(key));
-			}
-		}
-
 		// insert new block traces into the cache and the database
-		{
+		let mut traces = self.traces.write();
-			let mut traces = self.traces.write();
+		let mut blooms = self.blooms.write();
-			// it's important to use overwrite here,
+		// it's important to use overwrite here,
-			// cause this value might be queried by hash later
+		// cause this value might be queried by hash later
-			batch.write_with_cache(db::COL_TRACE, &mut *traces, request.block_hash, request.traces, CacheUpdatePolicy::Overwrite);
+		batch.write_with_cache(db::COL_TRACE, &mut *blooms, request.block_hash, request.traces.bloom(), CacheUpdatePolicy::Overwrite);
-			// note_used must be called after locking traces to avoid cache/traces deadlock on garbage collection
+		batch.write_with_cache(db::COL_TRACE, &mut *traces, request.block_hash, request.traces, CacheUpdatePolicy::Overwrite);
-			self.note_used(CacheId::Trace(request.block_hash.clone()));
+		// note_used must be called after locking traces to avoid cache/traces deadlock on garbage collection
-		}
+		self.note_used(CacheId::Trace(request.block_hash));
+		self.note_used(CacheId::Bloom(request.block_hash));
 	}
 
 	fn trace(&self, block_number: BlockNumber, tx_position: usize, trace_position: Vec<usize>) -> Option<LocalizedTrace> {
@@ -393,15 +318,17 @@ impl<T> TraceDatabase for TraceDB<T> where T: DatabaseExtras {
 	}
 
 	fn filter(&self, filter: &Filter) -> Vec<LocalizedTrace> {
-		let chain = BloomGroupChain::new(self.bloom_config, self);
+		let possibilities = filter.bloom_possibilities();
-		let numbers = chain.filter(filter);
+		// + 1, cause filters are inclusive
-		numbers.into_iter()
+		(filter.range.start..filter.range.end + 1).into_iter()
-			.flat_map(|n| {
+			.map(|n| n as BlockNumber)
-				let number = n as BlockNumber;
+			.filter_map(|n| self.extras.block_hash(n).map(|hash| (n, hash)))
-				let hash = self.extras.block_hash(number)
+			.filter(|&(_,ref hash)| {
-					.expect("Expected to find block hash. Extras db is probably corrupted");
+				let bloom = self.bloom(hash).expect("hash exists; qed");
-				let traces = self.traces(&hash)
+				possibilities.iter().any(|p| bloom.contains(p))
-					.expect("Expected to find a trace. Db is probably corrupted.");
+			})
+			.flat_map(|(number, hash)| {
+				let traces = self.traces(&hash).expect("hash exists; qed");
 				self.matching_block_traces(filter, traces, hash, number)
 			})
 			.collect()

ethcore/src/trace/mod.rs

@@ -16,7 +16,6 @@
 
 //! Tracing
 
-mod bloom;
 mod config;
 mod db;
 mod executive_tracer;

ethcore/src/trace/types/filter.rs

@@ -17,10 +17,10 @@
 //! Trace filters type definitions
 
 use std::ops::Range;
-use bloomchain::{Filter as BloomFilter, Bloom, Number};
 use hash::keccak;
 use util::Address;
 use bloomable::Bloomable;
+use bigint::prelude::H2048 as Bloom;
 use basic_types::LogBloom;
 use trace::flat::FlatTrace;
 use super::trace::{Action, Res};
@@ -87,22 +87,9 @@ pub struct Filter {
 	pub to_address: AddressesFilter,
 }
 
-impl BloomFilter for Filter {
-	fn bloom_possibilities(&self) -> Vec<Bloom> {
-		self.bloom_possibilities()
-			.into_iter()
-			.map(|b| Bloom::from(b.0))
-			.collect()
-	}
-
-	fn range(&self) -> Range<Number> {
-		self.range.clone()
-	}
-}
-
 impl Filter {
 	/// Returns combinations of each address.
-	fn bloom_possibilities(&self) -> Vec<LogBloom> {
+	pub fn bloom_possibilities(&self) -> Vec<Bloom> {
 		self.to_address.with_blooms(self.from_address.blooms())
 	}
 

ethcore/src/verification/verification.rs

@@ -461,7 +461,7 @@ mod tests {
 			unimplemented!()
 		}
 
-		fn blocks_with_bloom(&self, _bloom: &H2048, _from_block: BlockNumber, _to_block: BlockNumber) -> Vec<BlockNumber> {
+		fn blocks_with_blooms(&self, _blooms: &[H2048], _from_block: BlockNumber, _to_block: BlockNumber) -> Vec<BlockNumber> {
 			unimplemented!()
 		}
 

hw/src/ledger.rs

@@ -22,16 +22,21 @@ use super::{WalletInfo, KeyPath};
 use bigint::hash::H256;
 use ethkey::{Address, Signature};
 use hidapi;
+use libusb;
 use parking_lot::{Mutex, RwLock};
 
 use std::cmp::min;
 use std::fmt;
 use std::str::FromStr;
-use std::sync::Arc;
+use std::sync::{Arc, Weak};
 use std::time::Duration;
+use std::thread;
+
+/// Ledger vendor ID
+pub const LEDGER_VID: u16 = 0x2c97;
+/// Legder product IDs: [Nano S and Blue]
+pub const LEDGER_PIDS: [u16; 2] = [0x0000, 0x0001];
 
-const LEDGER_VID: u16 = 0x2c97;
-const LEDGER_PIDS: [u16; 2] = [0x0000, 0x0001]; // Nano S and Blue
 const ETH_DERIVATION_PATH_BE: [u8; 17] = [4, 0x80, 0, 0, 44, 0x80, 0, 0, 60, 0x80, 0, 0, 0, 0, 0, 0, 0]; // 44'/60'/0'/0
 const ETC_DERIVATION_PATH_BE: [u8; 21] = [5, 0x80, 0, 0, 44, 0x80, 0, 0, 60, 0x80, 0x02, 0x73, 0xd0, 0x80, 0, 0, 0, 0, 0, 0, 0]; // 44'/60'/160720'/0'/0
 
@@ -54,10 +59,14 @@ pub enum Error {
 	Protocol(&'static str),
 	/// Hidapi error.
 	Usb(hidapi::HidError),
+	/// Libusb error
+	LibUsb(libusb::Error),
 	/// Device with request key is not available.
 	KeyNotFound,
 	/// Signing has been cancelled by user.
 	UserCancel,
+	/// Invalid Device
+	InvalidDevice,
 }
 
 impl fmt::Display for Error {
@@ -65,8 +74,10 @@ impl fmt::Display for Error {
 		match *self {
 			Error::Protocol(ref s) => write!(f, "Ledger protocol error: {}", s),
 			Error::Usb(ref e) => write!(f, "USB communication error: {}", e),
+			Error::LibUsb(ref e) => write!(f, "LibUSB communication error: {}", e),
 			Error::KeyNotFound => write!(f, "Key not found"),
 			Error::UserCancel => write!(f, "Operation has been cancelled"),
+			Error::InvalidDevice => write!(f, "Unsupported product was entered"),
 		}
 	}
 }
@@ -77,6 +88,12 @@ impl From<hidapi::HidError> for Error {
 	}
 }
 
+impl From<libusb::Error> for Error {
+	fn from(err: libusb::Error) -> Error {
+		Error::LibUsb(err)
+	}
+}
+
 /// Ledger device manager.
 pub struct Manager {
 	usb: Arc<Mutex<hidapi::HidApi>>,
@@ -234,16 +251,7 @@ impl Manager {
 	fn open_path<R, F>(&self, f: F) -> Result<R, Error>
 		where F: Fn() -> Result<R, &'static str>
 	{
-		let mut err = Error::KeyNotFound;
+		f().map_err(Into::into)
-		// Try to open device a few times.
-		for _ in 0..10 {
-			match f() {
-				Ok(handle) => return Ok(handle),
-				Err(e) => err = From::from(e),
-			}
-			::std::thread::sleep(Duration::from_millis(200));
-		}
-		Err(err)
 	}
 
 	fn send_apdu(handle: &hidapi::HidDevice, command: u8, p1: u8, p2: u8, data: &[u8]) -> Result<Vec<u8>, Error> {
@@ -333,6 +341,54 @@ impl Manager {
 		message.truncate(new_len);
 		Ok(message)
 	}
+
+	fn is_valid_ledger(device: &libusb::Device) -> Result<(), Error> {
+		let desc = device.device_descriptor()?;
+		let vendor_id = desc.vendor_id();
+		let product_id = desc.product_id();
+
+		if vendor_id == LEDGER_VID && LEDGER_PIDS.contains(&product_id) {
+			Ok(())
+		} else {
+			Err(Error::InvalidDevice)
+		}
+	}
+
+}
+
+/// Ledger event handler
+/// A seperate thread is handling incoming events
+pub struct EventHandler {
+	ledger: Weak<Manager>,
+}
+
+impl EventHandler {
+	/// Ledger event handler constructor
+	pub fn new(ledger: Weak<Manager>) -> Self {
+		Self { ledger: ledger }
+	}
+}
+
+impl libusb::Hotplug for EventHandler {
+	fn device_arrived(&mut self, device: libusb::Device) {
+		if let (Some(ledger), Ok(_)) = (self.ledger.upgrade(), Manager::is_valid_ledger(&device)) {
+			debug!(target: "hw", "Ledger arrived");
+			// Wait for the device to boot up
+			thread::sleep(Duration::from_millis(1000));
+			if let Err(e) = ledger.update_devices() {
+				debug!(target: "hw", "Ledger connect error: {:?}", e);
+			}
+		}
+	}
+
+	fn device_left(&mut self, device: libusb::Device) {
+		if let (Some(ledger), Ok(_)) = (self.ledger.upgrade(), Manager::is_valid_ledger(&device)) {
+			debug!(target: "hw", "Ledger left");
+			if let Err(e) = ledger.update_devices() {
+				debug!(target: "hw", "Ledger disconnect error: {:?}", e);
+			}
+		}
+	}
 }
 
 #[test]

hw/src/lib.rs

@@ -33,13 +33,15 @@ use ethkey::{Address, Signature};
 
 use parking_lot::Mutex;
 use std::fmt;
-use std::sync::{Arc, Weak};
+use std::sync::Arc;
 use std::sync::atomic;
 use std::sync::atomic::AtomicBool;
 use std::thread;
 use std::time::Duration;
 use bigint::prelude::uint::U256;
 
+const USB_DEVICE_CLASS_DEVICE: u8 = 0;
+
 /// Hardware wallet error.
 #[derive(Debug)]
 pub enum Error {
@@ -128,84 +130,78 @@ impl From<libusb::Error> for Error {
 
 /// Hardware wallet management interface.
 pub struct HardwareWalletManager {
-	update_thread: Option<thread::JoinHandle<()>>,
 	exiting: Arc<AtomicBool>,
 	ledger: Arc<ledger::Manager>,
 	trezor: Arc<trezor::Manager>,
 }
 
-struct EventHandler {
-	ledger: Weak<ledger::Manager>,
-	trezor: Weak<trezor::Manager>,
-}
-
-impl libusb::Hotplug for EventHandler {
-	fn device_arrived(&mut self, _device: libusb::Device) {
-		debug!("USB Device arrived");
-		if let (Some(l), Some(t)) = (self.ledger.upgrade(), self.trezor.upgrade()) {
-			for _ in 0..10 {
-				let l_devices = l.update_devices().unwrap_or_else(|e| {
-					debug!("Error enumerating Ledger devices: {}", e);
-					0
-				});
-				let t_devices = t.update_devices().unwrap_or_else(|e| {
-					debug!("Error enumerating Trezor devices: {}", e);
-					0
-				});
-				if l_devices + t_devices > 0 {
-					break;
-				}
-				thread::sleep(Duration::from_millis(200));
-			}
-		}
-	}
-
-	fn device_left(&mut self, _device: libusb::Device) {
-		debug!("USB Device lost");
-		if let (Some(l), Some(t)) = (self.ledger.upgrade(), self.trezor.upgrade()) {
-			l.update_devices().unwrap_or_else(|e| {debug!("Error enumerating Ledger devices: {}", e); 0});
-			t.update_devices().unwrap_or_else(|e| {debug!("Error enumerating Trezor devices: {}", e); 0});
-		}
-	}
-}
 
 impl HardwareWalletManager {
+	/// Hardware wallet constructor
 	pub fn new() -> Result<HardwareWalletManager, Error> {
-		let usb_context = Arc::new(libusb::Context::new()?);
+		let usb_context_trezor = Arc::new(libusb::Context::new()?);
+		let usb_context_ledger = Arc::new(libusb::Context::new()?);
 		let hidapi = Arc::new(Mutex::new(hidapi::HidApi::new().map_err(|e| Error::Hid(e.to_string().clone()))?));
 		let ledger = Arc::new(ledger::Manager::new(hidapi.clone()));
 		let trezor = Arc::new(trezor::Manager::new(hidapi.clone()));
-		usb_context.register_callback(
+
-			None, None, None,
+		// Subscribe to TREZOR V1
-			Box::new(EventHandler {
+		// Note, this support only TREZOR V1 becasue TREZOR V2 has another vendorID for some reason
-				ledger: Arc::downgrade(&ledger),
+		// Also, we now only support one product as the second argument specifies
-				trezor: Arc::downgrade(&trezor),
+		usb_context_trezor.register_callback(
-			}),
+			Some(trezor::TREZOR_VID), Some(trezor::TREZOR_PIDS[0]), Some(USB_DEVICE_CLASS_DEVICE),
-		)?;
+			Box::new(trezor::EventHandler::new(Arc::downgrade(&trezor))))?;
+
+		// Subscribe to all Ledger Devices
+		// This means that we need to check that the given productID is supported
+		// None => LIBUSB_HOTPLUG_MATCH_ANY, in other words that all are subscribed to
+		// More info can be found: http://libusb.sourceforge.net/api-1.0/group__hotplug.html#gae6c5f1add6cc754005549c7259dc35ea
+		usb_context_ledger.register_callback(
+			Some(ledger::LEDGER_VID), None, Some(USB_DEVICE_CLASS_DEVICE),
+			Box::new(ledger::EventHandler::new(Arc::downgrade(&ledger))))?;
+
 		let exiting = Arc::new(AtomicBool::new(false));
-		let thread_exiting = exiting.clone();
+		let thread_exiting_ledger = exiting.clone();
+		let thread_exiting_trezor = exiting.clone();
 		let l = ledger.clone();
 		let t = trezor.clone();
-		let thread = thread::Builder::new()
+
-			.name("hw_wallet".to_string())
+		// Ledger event thread
+		thread::Builder::new()
+			.name("hw_wallet_ledger".to_string())
 			.spawn(move || {
 				if let Err(e) = l.update_devices() {
-					debug!("Error updating ledger devices: {}", e);
+					debug!(target: "hw", "Ledger couldn't connect at startup, error: {}", e);
-				}
+					//debug!("Ledger could not connect at startup, error: {}", e);
-				if let Err(e) = t.update_devices() {
-					debug!("Error updating trezor devices: {}", e);
 				}
 				loop {
-					usb_context.handle_events(Some(Duration::from_millis(500)))
+					usb_context_ledger.handle_events(Some(Duration::from_millis(500)))
-					           .unwrap_or_else(|e| debug!("Error processing USB events: {}", e));
+					           .unwrap_or_else(|e| debug!(target: "hw", "Ledger event handler error: {}", e));
-					if thread_exiting.load(atomic::Ordering::Acquire) {
+					if thread_exiting_ledger.load(atomic::Ordering::Acquire) {
 						break;
 					}
 				}
 			})
 			.ok();
+
+		// Trezor event thread
+		thread::Builder::new()
+			.name("hw_wallet_trezor".to_string())
+			.spawn(move || {
+				if let Err(e) = t.update_devices() {
+					debug!(target: "hw", "Trezor couldn't connect at startup, error: {}", e);
+				}
+				loop {
+					usb_context_trezor.handle_events(Some(Duration::from_millis(500)))
+					           .unwrap_or_else(|e| debug!(target: "hw", "Trezor event handler error: {}", e));
+					if thread_exiting_trezor.load(atomic::Ordering::Acquire) {
+						break;
+					}
+				}
+			})
+			.ok();
+
 		Ok(HardwareWalletManager {
-			update_thread: thread,
 			exiting: exiting,
 			ledger: ledger,
 			trezor: trezor,
@@ -259,10 +255,10 @@ impl HardwareWalletManager {
 
 impl Drop for HardwareWalletManager {
 	fn drop(&mut self) {
+		// Indicate to the USB Hotplug handlers that they
+		// shall terminate but don't wait for them to terminate.
+		// If they don't terminate for some reason USB Hotplug events will be handled
+		// even if the HardwareWalletManger has been dropped
 		self.exiting.store(true, atomic::Ordering::Release);
-		if let Some(thread) = self.update_thread.take() {
-			thread.thread().unpark();
-			thread.join().ok();
-		}
 	}
 }

hw/src/trezor.rs

@@ -24,23 +24,26 @@ use super::{WalletInfo, TransactionInfo, KeyPath};
 use bigint::hash::H256;
 use ethkey::{Address, Signature};
 use hidapi;
+use libusb;
 use parking_lot::{Mutex, RwLock};
 use protobuf;
 use protobuf::{Message, ProtobufEnum};
 use std::cmp::{min, max};
 use std::fmt;
-use std::sync::Arc;
+use std::sync::{Arc, Weak};
 use std::time::Duration;
 use bigint::prelude::uint::U256;
 
 use trezor_sys::messages::{EthereumAddress, PinMatrixAck, MessageType, EthereumTxRequest, EthereumSignTx, EthereumGetAddress, EthereumTxAck, ButtonAck};
 
-const TREZOR_VID: u16 = 0x534c;
+/// Trezor v1 vendor ID
-const TREZOR_PIDS: [u16; 1] = [0x0001]; // Trezor v1, keeping this as an array to leave room for Trezor v2 which is in progress
+pub const TREZOR_VID: u16 = 0x534c;
+/// Trezor product IDs
+pub const TREZOR_PIDS: [u16; 1] = [0x0001];
+
 const ETH_DERIVATION_PATH: [u32; 5] = [0x8000002C, 0x8000003C, 0x80000000, 0, 0]; // m/44'/60'/0'/0/0
 const ETC_DERIVATION_PATH: [u32; 5] = [0x8000002C, 0x8000003D, 0x80000000, 0, 0]; // m/44'/61'/0'/0/0
 
-
 /// Hardware wallet error.
 #[derive(Debug)]
 pub enum Error {
@@ -55,7 +58,7 @@ pub enum Error {
 	/// The Message Type given in the trezor RPC call is not something we recognize
 	BadMessageType,
 	/// Trying to read from a closed device at the given path
-	ClosedDevice(String),
+	LockedDevice(String),
 }
 
 impl fmt::Display for Error {
@@ -66,7 +69,7 @@ impl fmt::Display for Error {
 			Error::KeyNotFound => write!(f, "Key not found"),
 			Error::UserCancel => write!(f, "Operation has been cancelled"),
 			Error::BadMessageType => write!(f, "Bad Message Type in RPC call"),
-			Error::ClosedDevice(ref s) => write!(f, "Device is closed, needs PIN to perform operations: {}", s),
+			Error::LockedDevice(ref s) => write!(f, "Device is locked, needs PIN to perform operations: {}", s),
 		}
 	}
 }
@@ -83,11 +86,11 @@ impl From<protobuf::ProtobufError> for Error {
 	}
 }
 
-/// Ledger device manager.
+/// Ledger device manager
 pub struct Manager {
 	usb: Arc<Mutex<hidapi::HidApi>>,
 	devices: RwLock<Vec<Device>>,
-	closed_devices: RwLock<Vec<String>>,
+	locked_devices: RwLock<Vec<String>>,
 	key_path: RwLock<KeyPath>,
 }
 
@@ -109,7 +112,7 @@ impl Manager {
 		Manager {
 			usb: hidapi,
 			devices: RwLock::new(Vec::new()),
-			closed_devices: RwLock::new(Vec::new()),
+			locked_devices: RwLock::new(Vec::new()),
 			key_path: RwLock::new(KeyPath::Ethereum),
 		}
 	}
@@ -120,7 +123,7 @@ impl Manager {
 		usb.refresh_devices();
 		let devices = usb.devices();
 		let mut new_devices = Vec::new();
-		let mut closed_devices = Vec::new();
+		let mut locked_devices = Vec::new();
 		let mut error = None;
 		for usb_device in devices {
 			let is_trezor = usb_device.vendor_id == TREZOR_VID;
@@ -139,7 +142,7 @@ impl Manager {
 			}
 			match self.read_device_info(&usb, &usb_device) {
 				Ok(device) => new_devices.push(device),
-				Err(Error::ClosedDevice(path)) => closed_devices.push(path.to_string()),
+				Err(Error::LockedDevice(path)) => locked_devices.push(path.to_string()),
 				Err(e) => {
 					warn!("Error reading device: {:?}", e);
 					error = Some(e);
@@ -147,9 +150,9 @@ impl Manager {
 			}
 		}
 		let count = new_devices.len();
-		trace!("Got devices: {:?}, closed: {:?}", new_devices, closed_devices);
+		trace!("Got devices: {:?}, closed: {:?}", new_devices, locked_devices);
 		*self.devices.write() = new_devices;
-		*self.closed_devices.write() = closed_devices;
+		*self.locked_devices.write() = locked_devices;
 		match error {
 			Some(e) => Err(e),
 			None => Ok(count),
@@ -173,7 +176,7 @@ impl Manager {
 					},
 				})
 			}
-			Ok(None) => Err(Error::ClosedDevice(dev_info.path.clone())),
+			Ok(None) => Err(Error::LockedDevice(dev_info.path.clone())),
 			Err(e) => Err(e),
 		}
 	}
@@ -189,7 +192,7 @@ impl Manager {
 	}
 
 	pub fn list_locked_devices(&self) -> Vec<String> {
-		(*self.closed_devices.read()).clone()
+		(*self.locked_devices.read()).clone()
 	}
 
 	/// Get wallet info.
@@ -200,16 +203,7 @@ impl Manager {
 	fn open_path<R, F>(&self, f: F) -> Result<R, Error>
 		where F: Fn() -> Result<R, &'static str>
 	{
-		let mut err = Error::KeyNotFound;
+		f().map_err(Into::into)
-		// Try to open device a few times.
-		for _ in 0..10 {
-			match f() {
-				Ok(handle) => return Ok(handle),
-				Err(e) => err = From::from(e),
-			}
-			::std::thread::sleep(Duration::from_millis(200));
-		}
-		Err(err)
 	}
 
 	pub fn pin_matrix_ack(&self, device_path: &str, pin: &str) -> Result<bool, Error> {
@@ -406,6 +400,42 @@ impl Manager {
 	}
 }
 
+/// Trezor event handler
+/// A separate thread is handeling incoming events
+pub struct EventHandler {
+	trezor: Weak<Manager>,
+}
+
+impl EventHandler {
+	// Trezor event handler constructor
+	pub fn new(trezor: Weak<Manager>) -> Self {
+		Self { trezor: trezor }
+	}
+}
+
+impl libusb::Hotplug for EventHandler {
+	fn device_arrived(&mut self, _device: libusb::Device) {
+		debug!(target: "hw", "Trezor V1 arrived");
+		if let Some(trezor) = self.trezor.upgrade() {
+			// Wait for the device to boot up
+			::std::thread::sleep(Duration::from_millis(1000));
+			if let Err(e) = trezor.update_devices() {
+				debug!(target: "hw", "Trezor V1 connect error: {:?}", e);
+			}
+
+		}
+	}
+
+	fn device_left(&mut self, _device: libusb::Device) {
+		debug!(target: "hw", "Trezor V1 left");
+		if let Some(trezor) = self.trezor.upgrade() {
+			if let Err(e) = trezor.update_devices() {
+				debug!(target: "hw", "Trezor V1 disconnect error: {:?}", e);
+			}
+		}
+	}
+}
+
 #[test]
 #[ignore]
 /// This test can't be run without an actual trezor device connected

parity/run.rs

@@ -676,11 +676,15 @@ pub fn execute_impl(cmd: RunCmd, can_restart: bool, logger: Arc<RotatingLogger>)
 	let event_loop = EventLoop::spawn();
 
 	// the updater service
+	let mut updater_fetch = fetch.clone();
+	// parity binaries should be smaller than 128MB
+	updater_fetch.set_limit(Some(128 * 1024 * 1024));
+
 	let updater = Updater::new(
 		Arc::downgrade(&(service.client() as Arc<BlockChainClient>)),
 		Arc::downgrade(&sync_provider),
 		update_policy,
-		fetch.clone(),
+		updater_fetch,
 		event_loop.remote(),
 	);
 	service.add_notify(updater.clone());

scripts/gitlab-build.sh

@@ -22,13 +22,10 @@ echo "Parity version:     " $VER
 echo "Branch:             " $CI_BUILD_REF_NAME
 echo "--------------------"
 
-echo "Rhash version:"
 # NOTE for md5 and sha256 we want to display filename as well
 # hence we use --* instead of -p *
 MD5_BIN="rhash --md5"
 SHA256_BIN="rhash --sha256"
-# NOTE For SHA3 we need only hash (hence -p)
-SHA3_BIN="rhash -p %{sha3-256}"
 
 set_env () {
   echo "Set ENVIROMENT"
@@ -70,14 +67,12 @@ strip_binaries () {
 calculate_checksums () {
   echo "Checksum calculation:"
   rhash --version
+
   rm -rf *.md5
   rm -rf *.sha256
 
-  export SHA3="$($SHA3_BIN target/$PLATFORM/release/parity$S3WIN)"
+  BIN="target/$PLATFORM/release/parity$S3WIN"
-  # NOTE rhash 1.3.1 doesnt support keccak, workaround
+  export SHA3="$($BIN tools hash $BIN)"
-  if [ "$SHA3" == "%{sha3-256}" ]; then
-    export SHA3="$(target/$PLATFORM/release/parity$S3WIN tools hash target/$PLATFORM/release/parity$S3WIN)"
-  fi
 
   echo "Parity file SHA3: $SHA3"
   $MD5_BIN target/$PLATFORM/release/parity$S3WIN > parity$S3WIN.md5

util/fetch/src/client.rs

@@ -127,6 +127,11 @@ impl Client {
 		})
 	}
 
+	/// Sets a limit on the maximum download size.
+	pub fn set_limit(&mut self, limit: Option<usize>) {
+		self.limit = limit
+	}
+
 	fn client(&self) -> Result<Arc<reqwest::Client>, Error> {
 		{
 			let (ref time, ref client) = *self.client.read();
@@ -150,8 +155,8 @@ impl Fetch for Client {
 	type Result = CpuFuture<Response, Error>;
 
 	fn new() -> Result<Self, Error> {
-		// Max 50MB will be downloaded.
+		// Max 64MB will be downloaded.
-		Self::with_limit(Some(50*1024*1024))
+		Self::with_limit(Some(64 * 1024 * 1024))
 	}
 
 	fn process<F, I, E>(&self, f: F) -> BoxFuture<I, E> where