Use global state cache when mining (#2529)

Update appveyor rustc [beta]

.gitlab-ci.yml

@@ -289,5 +289,6 @@ test-linux:
     - ./test.sh --verbose
   tags:
     - rust-test
+    - javascript
   dependencies:
     - linux-stable

Cargo.lock

@@ -1,6 +1,6 @@
 [root]
 name = "parity"
-version = "1.3.4"
+version = "1.3.5"
 dependencies = [
  "ansi_term 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)",
  "clippy 0.0.80 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -20,7 +20,7 @@ dependencies = [
  "ethcore-logger 1.3.0",
  "ethcore-rpc 1.3.0",
  "ethcore-signer 1.3.0",
- "ethcore-util 1.3.4",
+ "ethcore-util 1.3.5",
  "ethsync 1.3.0",
  "fdlimit 0.1.0",
  "hyper 0.8.1 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -270,7 +270,7 @@ dependencies = [
  "ethcore-ipc 1.3.0",
  "ethcore-ipc-codegen 1.3.0",
  "ethcore-ipc-nano 1.3.0",
- "ethcore-util 1.3.4",
+ "ethcore-util 1.3.5",
  "ethjson 0.1.0",
  "ethstore 0.1.0",
  "evmjit 1.3.0",
@@ -294,7 +294,7 @@ version = "1.3.0"
 dependencies = [
  "clippy 0.0.80 (registry+https://github.com/rust-lang/crates.io-index)",
  "ethcore-rpc 1.3.0",
- "ethcore-util 1.3.4",
+ "ethcore-util 1.3.5",
  "hyper 0.9.4 (git+https://github.com/ethcore/hyper)",
  "jsonrpc-core 2.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
  "jsonrpc-http-server 6.1.0 (git+https://github.com/ethcore/jsonrpc-http-server.git)",
@@ -336,7 +336,7 @@ name = "ethcore-ipc"
 version = "1.3.0"
 dependencies = [
  "ethcore-devtools 1.3.0",
- "ethcore-util 1.3.4",
+ "ethcore-util 1.3.5",
  "nanomsg 0.5.1 (git+https://github.com/ethcore/nanomsg.rs.git)",
  "semver 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
@@ -381,7 +381,7 @@ dependencies = [
  "ethcore-ipc 1.3.0",
  "ethcore-ipc-codegen 1.3.0",
  "ethcore-ipc-nano 1.3.0",
- "ethcore-util 1.3.4",
+ "ethcore-util 1.3.5",
  "log 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)",
  "nanomsg 0.5.1 (git+https://github.com/ethcore/nanomsg.rs.git)",
  "semver 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -393,7 +393,7 @@ name = "ethcore-logger"
 version = "1.3.0"
 dependencies = [
  "env_logger 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)",
- "ethcore-util 1.3.4",
+ "ethcore-util 1.3.5",
  "isatty 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
  "lazy_static 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)",
  "log 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -408,7 +408,7 @@ dependencies = [
  "ansi_term 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)",
  "ethcore-devtools 1.3.0",
  "ethcore-io 1.3.0",
- "ethcore-util 1.3.4",
+ "ethcore-util 1.3.5",
  "igd 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)",
  "libc 0.2.12 (registry+https://github.com/rust-lang/crates.io-index)",
  "log 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -432,7 +432,7 @@ dependencies = [
  "ethcore-devtools 1.3.0",
  "ethcore-io 1.3.0",
  "ethcore-ipc 1.3.0",
- "ethcore-util 1.3.4",
+ "ethcore-util 1.3.5",
  "ethjson 0.1.0",
  "ethsync 1.3.0",
  "json-ipc-server 0.2.4 (git+https://github.com/ethcore/json-ipc-server.git?branch=beta)",
@@ -455,7 +455,7 @@ dependencies = [
  "env_logger 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)",
  "ethcore-io 1.3.0",
  "ethcore-rpc 1.3.0",
- "ethcore-util 1.3.4",
+ "ethcore-util 1.3.5",
  "jsonrpc-core 2.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
  "log 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)",
  "parity-dapps-signer 1.4.0 (git+https://github.com/ethcore/parity-ui.git)",
@@ -466,7 +466,7 @@ dependencies = [
 
 [[package]]
 name = "ethcore-util"
-version = "1.3.4"
+version = "1.3.5"
 dependencies = [
  "ansi_term 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)",
  "arrayvec 0.3.16 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -499,7 +499,7 @@ dependencies = [
 name = "ethjson"
 version = "0.1.0"
 dependencies = [
- "ethcore-util 1.3.4",
+ "ethcore-util 1.3.5",
  "rustc-serialize 0.3.19 (registry+https://github.com/rust-lang/crates.io-index)",
  "serde 0.7.9 (registry+https://github.com/rust-lang/crates.io-index)",
  "serde_codegen 0.7.9 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -547,7 +547,7 @@ dependencies = [
  "ethcore-ipc-codegen 1.3.0",
  "ethcore-ipc-nano 1.3.0",
  "ethcore-network 1.3.0",
- "ethcore-util 1.3.4",
+ "ethcore-util 1.3.5",
  "heapsize 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)",
  "log 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)",
  "parking_lot 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)",

Cargo.toml

@@ -1,7 +1,7 @@
 [package]
 description = "Ethcore client."
 name = "parity"
-version = "1.3.4"
+version = "1.3.5"
 license = "GPL-3.0"
 authors = ["Ethcore <admin@ethcore.io>"]
 build = "build.rs"

appveyor.yml

@@ -6,6 +6,7 @@ environment:
   certpass:
     secure: 0BgXJqxq9Ei34/hZ7121FQ==
   keyfile: C:\users\appveyor\Certificates.p12
+  RUSTFLAGS: -Zorbit=off
 
 branches:
   only:
@@ -18,10 +19,10 @@ branches:
 install:
   - git submodule update --init --recursive
   - ps: Install-Product node 6
-  - ps: Start-FileDownload "https://static.rust-lang.org/dist/rust-1.10.0-x86_64-pc-windows-msvc.exe"
+  - ps: Start-FileDownload "https://static.rust-lang.org/dist/rust-1.12.0-x86_64-pc-windows-msvc.exe"
   - ps: Start-FileDownload "https://github.com/ethcore/win-build/raw/master/SimpleFC.dll" -FileName nsis\SimpleFC.dll
   - ps: Start-FileDownload "https://github.com/ethcore/win-build/raw/master/vc_redist.x64.exe" -FileName nsis\vc_redist.x64.exe
-  - rust-1.10.0-x86_64-pc-windows-msvc.exe /VERYSILENT /NORESTART /DIR="C:\Program Files (x86)\Rust"
+  - rust-1.12.0-x86_64-pc-windows-msvc.exe /VERYSILENT /NORESTART /DIR="C:\Program Files (x86)\Rust"
   - SET PATH=%PATH%;C:\Program Files (x86)\Rust\bin;C:\Program Files (x86)\NSIS;C:\Program Files (x86)\Microsoft SDKs\Windows\v7.1A\Bin
   - rustc -V
   - cargo -V

ethcore/src/client/client.rs

@@ -1052,7 +1052,7 @@ impl MiningBlockChainClient for Client {
 			&self.vm_factory,
 			self.trie_factory.clone(),
 			false,	// TODO: this will need to be parameterised once we want to do immediate mining insertion.
-			self.state_db.lock().boxed_clone(),
+			self.state_db.lock().boxed_clone_canon(&h),
 			&self.chain.block_header(&h).expect("h is best block hash: so its header must exist: qed"),
 			self.build_last_hashes(h.clone()),
 			author,

ethcore/src/miner/miner.rs

@@ -211,7 +211,7 @@ impl Miner {
 	pub fn new(options: MinerOptions, gas_pricer: GasPricer, spec: &Spec, accounts: Option<Arc<AccountProvider>>) -> Arc<Miner> {
 		let work_poster = if !options.new_work_notify.is_empty() { Some(WorkPoster::new(&options.new_work_notify)) } else { None };
 		let txq = Arc::new(Mutex::new(TransactionQueue::with_limits(
-			options.tx_queue_strategy, options.tx_queue_size, options.tx_gas_limit
+			options.tx_queue_strategy, options.tx_queue_size, !U256::zero(), options.tx_gas_limit
 		)));
 		Arc::new(Miner {
 			transaction_queue: txq,
@@ -401,6 +401,8 @@ impl Miner {
 		let gas_limit = HeaderView::new(&chain.best_block_header()).gas_limit();
 		let mut queue = self.transaction_queue.lock();
 		queue.set_gas_limit(gas_limit);
+		// Set total qx queue gas limit to be 2x the block gas limit.
+		queue.set_total_gas_limit(gas_limit << 1);
 	}
 
 	/// Returns true if we had to prepare new pending block

ethcore/src/miner/transaction_queue.rs

@@ -279,6 +279,7 @@ struct TransactionSet {
 	by_priority: BTreeSet<TransactionOrder>,
 	by_address: Table<Address, U256, TransactionOrder>,
 	limit: usize,
+	gas_limit: U256,
 }
 
 impl TransactionSet {
@@ -299,14 +300,18 @@ impl TransactionSet {
 	/// It drops transactions from this set but also removes associated `VerifiedTransaction`.
 	/// Returns addresses and lowest nonces of transactions removed because of limit.
 	fn enforce_limit(&mut self, by_hash: &mut HashMap<H256, VerifiedTransaction>) -> Option<HashMap<Address, U256>> {
-		let len = self.by_priority.len();
-		if len <= self.limit {
-			return None;
-		}
+		let mut count = 0;
+		let mut gas: U256 = 0.into();
 		let to_drop : Vec<(Address, U256)> = {
 			self.by_priority
 				.iter()
-				.skip(self.limit)
+				.skip_while(|order| {
+					count = count + 1;
+					let r = gas.overflowing_add(order.gas);
+					if r.1 { return false }
+					gas = r.0;
+					count <= self.limit && gas <= self.gas_limit
+				})
 				.map(|order| by_hash.get(&order.hash)
 					.expect("All transactions in `self.by_priority` and `self.by_address` are kept in sync with `by_hash`."))
 				.map(|tx| (tx.sender(), tx.nonce()))
@@ -423,21 +428,23 @@ impl Default for TransactionQueue {
 impl TransactionQueue {
 	/// Creates new instance of this Queue
 	pub fn new(strategy: PrioritizationStrategy) -> Self {
-		Self::with_limits(strategy, 1024, !U256::zero())
+		Self::with_limits(strategy, 1024, !U256::zero(), !U256::zero())
 	}
 
 	/// Create new instance of this Queue with specified limits
-	pub fn with_limits(strategy: PrioritizationStrategy, limit: usize, tx_gas_limit: U256) -> Self {
+	pub fn with_limits(strategy: PrioritizationStrategy, limit: usize, gas_limit: U256, tx_gas_limit: U256) -> Self {
 		let current = TransactionSet {
 			by_priority: BTreeSet::new(),
 			by_address: Table::new(),
 			limit: limit,
+			gas_limit: gas_limit,
 		};
 
 		let future = TransactionSet {
 			by_priority: BTreeSet::new(),
 			by_address: Table::new(),
 			limit: limit,
+			gas_limit: gas_limit,
 		};
 
 		TransactionQueue {
@@ -488,6 +495,13 @@ impl TransactionQueue {
 		};
 	}
 
+	/// Sets new total gas limit.
+	pub fn set_total_gas_limit(&mut self, gas_limit: U256) {
+		self.future.gas_limit = gas_limit;
+		self.current.gas_limit = gas_limit;
+		self.future.enforce_limit(&mut self.by_hash);
+	}
+
 	/// Set the new limit for the amount of gas any individual transaction may have.
 	/// Any transaction already imported to the queue is not affected.
 	pub fn set_tx_gas_limit(&mut self, limit: U256) {
@@ -797,6 +811,16 @@ impl TransactionQueue {
 		let nonce = tx.nonce();
 		let hash = tx.hash();
 
+		{
+			// Rough size sanity check
+			let gas = &tx.transaction.gas;
+			if U256::from(tx.transaction.data.len()) > *gas {
+				// Droping transaction
+				trace!(target: "txqueue", "Dropping oversized transaction: {:?} (gas: {} < size {})", hash, gas, tx.transaction.data.len());
+				return Err(TransactionError::LimitReached);
+			}
+		}
+
 		// The transaction might be old, let's check that.
 		// This has to be the first test, otherwise calculating
 		// nonce height would result in overflow.
@@ -1049,7 +1073,8 @@ mod test {
 		let mut set = TransactionSet {
 			by_priority: BTreeSet::new(),
 			by_address: Table::new(),
-			limit: 1
+			limit: 1,
+			gas_limit: !U256::zero(),
 		};
 		let (tx1, tx2) = new_txs(U256::from(1));
 		let tx1 = VerifiedTransaction::new(tx1, TransactionOrigin::External).unwrap();
@@ -1088,7 +1113,8 @@ mod test {
 		let mut set = TransactionSet {
 			by_priority: BTreeSet::new(),
 			by_address: Table::new(),
-			limit: 1
+			limit: 1,
+			gas_limit: !U256::zero(),
 		};
 		// Create two transactions with same nonce
 		// (same hash)
@@ -1668,7 +1694,7 @@ mod test {
 	#[test]
 	fn should_drop_old_transactions_when_hitting_the_limit() {
 		// given
-		let mut txq = TransactionQueue::with_limits(PrioritizationStrategy::GasPriceOnly, 1, !U256::zero());
+		let mut txq = TransactionQueue::with_limits(PrioritizationStrategy::GasPriceOnly, 1, !U256::zero(), !U256::zero());
 		let (tx, tx2) = new_txs(U256::one());
 		let sender = tx.sender().unwrap();
 		let nonce = tx.nonce;
@@ -1690,7 +1716,7 @@ mod test {
 	#[test]
 	fn should_return_correct_nonces_when_dropped_because_of_limit() {
 		// given
-		let mut txq = TransactionQueue::with_limits(PrioritizationStrategy::GasPriceOnly, 2, !U256::zero());
+		let mut txq = TransactionQueue::with_limits(PrioritizationStrategy::GasPriceOnly, 2, !U256::zero(), !U256::zero());
 		let tx = new_tx();
 		let (tx1, tx2) = new_txs(U256::one());
 		let sender = tx1.sender().unwrap();
@@ -1711,7 +1737,7 @@ mod test {
 
 	#[test]
 	fn should_limit_future_transactions() {
-		let mut txq = TransactionQueue::with_limits(PrioritizationStrategy::GasPriceOnly, 1, !U256::zero());
+		let mut txq = TransactionQueue::with_limits(PrioritizationStrategy::GasPriceOnly, 1, !U256::zero(), !U256::zero());
 		txq.current.set_limit(10);
 		let (tx1, tx2) = new_txs_with_gas_price_diff(U256::from(4), U256::from(1));
 		let (tx3, tx4) = new_txs_with_gas_price_diff(U256::from(4), U256::from(2));
@@ -1728,6 +1754,16 @@ mod test {
 		assert_eq!(txq.status().future, 1);
 	}
 
+	#[test]
+	fn should_limit_by_gas() {
+		let mut txq = TransactionQueue::with_limits(PrioritizationStrategy::GasPriceOnly, 100, default_gas_val() * U256::from(2), !U256::zero());
+		let (tx1, _) = new_txs_with_gas_price_diff(U256::from(4), U256::from(1));
+		let (tx3, _) = new_txs_with_gas_price_diff(U256::from(4), U256::from(2));
+		txq.add(tx1.clone(), &default_nonce, TransactionOrigin::External).unwrap();
+		txq.add(tx3.clone(), &default_nonce, TransactionOrigin::External).unwrap();
+		assert_eq!(txq.status().pending, 2);
+	}
+
 	#[test]
 	fn should_drop_transactions_with_old_nonces() {
 		let mut txq = TransactionQueue::default();
@@ -1971,7 +2007,7 @@ mod test {
 	#[test]
 	fn should_keep_right_order_in_future() {
 		// given
-		let mut txq = TransactionQueue::with_limits(PrioritizationStrategy::GasPriceOnly, 1, !U256::zero());
+		let mut txq = TransactionQueue::with_limits(PrioritizationStrategy::GasPriceOnly, 1, !U256::zero(), !U256::zero());
 		let (tx1, tx2) = new_txs(U256::from(1));
 		let prev_nonce = |a: &Address| AccountDetails { nonce: default_nonce(a).nonce - U256::one(), balance:
 			default_nonce(a).balance };

nsis/installer.nsi

@@ -4,7 +4,7 @@
 !define DESCRIPTION "Fast, light, robust Ethereum implementation"
 !define VERSIONMAJOR 1
 !define VERSIONMINOR 3
-!define VERSIONBUILD 4
+!define VERSIONBUILD 5
 
 !addplugindir .\
 

sync/src/chain.rs

@@ -121,6 +121,7 @@ const MAX_ROUND_PARENTS: usize = 32;
 const MAX_NEW_HASHES: usize = 64;
 const MAX_TX_TO_IMPORT: usize = 512;
 const MAX_NEW_BLOCK_AGE: BlockNumber = 20;
+const MAX_TRANSACTION_SIZE: usize = 300*1024;
 
 const STATUS_PACKET: u8 = 0x00;
 const NEW_BLOCK_HASHES_PACKET: u8 = 0x01;
@@ -1079,7 +1080,12 @@ impl ChainSync {
 		item_count = min(item_count, MAX_TX_TO_IMPORT);
 		let mut transactions = Vec::with_capacity(item_count);
 		for i in 0 .. item_count {
-			let tx = try!(r.at(i)).as_raw().to_vec();
+			let rlp = try!(r.at(i));
+			if rlp.as_raw().len() > MAX_TRANSACTION_SIZE {
+				debug!("Skipped oversized transaction of {} bytes", rlp.as_raw().len());
+				continue;
+			}
+			let tx = rlp.as_raw().to_vec();
 			transactions.push(tx);
 		}
 		io.chain().queue_transactions(transactions);

util/Cargo.toml

@@ -3,7 +3,7 @@ description = "Ethcore utility library"
 homepage = "http://ethcore.io"
 license = "GPL-3.0"
 name = "ethcore-util"
-version = "1.3.4"
+version = "1.3.5"
 authors = ["Ethcore <admin@ethcore.io>"]
 build = "build.rs"