[beta] Backports (#8450)

* Use forked app_dirs crate for reverted Windows dir behavior  (#8438)

* Remove unused appdirs dependency in CLI

* Use forked app_dirs crate for reverted Windows dir behavior

* remove Tendermint extra_info due to seal inconsistencies (#8367)

* handle queue import errors a bit more gracefully (#8385)

* Improve VM executor stack size estimation rules (#8439)

* Improve VM executor stack size estimation rules

* typo: docs add "(Debug build)" comment

* Fix an off by one typo and set minimal stack size

This avoids the case if `depth_threshold == max_depth`. Usually setting stack size to zero will just rebound it to
platform minimal stack size, but we set it here just in case.

* Use saturating_sub to avoid potential overflow

Cargo.lock

@@ -14,7 +14,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 [[package]]
 name = "app_dirs"
 version = "1.2.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
+source = "git+https://github.com/paritytech/app-dirs-rs#0b37f9481ce29e9d5174ad185bca695b206368eb"
 dependencies = [
  "ole32-sys 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "shell32-sys 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -319,7 +319,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 name = "dir"
 version = "0.1.0"
 dependencies = [
- "app_dirs 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)",
+ "app_dirs 1.2.1 (git+https://github.com/paritytech/app-dirs-rs)",
  "ethereum-types 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)",
  "journaldb 0.1.0",
 ]
@@ -1865,7 +1865,6 @@ name = "parity"
 version = "1.10.1"
 dependencies = [
  "ansi_term 0.10.2 (registry+https://github.com/rust-lang/crates.io-index)",
- "app_dirs 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)",
  "clap 2.29.1 (registry+https://github.com/rust-lang/crates.io-index)",
  "ctrlc 1.1.1 (git+https://github.com/paritytech/rust-ctrlc.git)",
  "daemonize 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -3556,7 +3555,7 @@ dependencies = [
 [metadata]
 "checksum aho-corasick 0.6.4 (registry+https://github.com/rust-lang/crates.io-index)" = "d6531d44de723825aa81398a6415283229725a00fa30713812ab9323faa82fc4"
 "checksum ansi_term 0.10.2 (registry+https://github.com/rust-lang/crates.io-index)" = "6b3568b48b7cefa6b8ce125f9bb4989e52fbcc29ebea88df04cc7c5f12f70455"
-"checksum app_dirs 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "e73a24bad9bd6a94d6395382a6c69fe071708ae4409f763c5475e14ee896313d"
+"checksum app_dirs 1.2.1 (git+https://github.com/paritytech/app-dirs-rs)" = "<none>"
 "checksum arrayvec 0.4.7 (registry+https://github.com/rust-lang/crates.io-index)" = "a1e964f9e24d588183fcb43503abda40d288c8657dfc27311516ce2f05675aef"
 "checksum aster 0.41.0 (registry+https://github.com/rust-lang/crates.io-index)" = "4ccfdf7355d9db158df68f976ed030ab0f6578af811f5a7bb6dcf221ec24e0e0"
 "checksum atty 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "d912da0db7fa85514874458ca3651fe2cddace8d0b0505571dbdcd41ab490159"

Cargo.toml

@@ -27,7 +27,6 @@ toml = "0.4"
 serde = "1.0"
 serde_json = "1.0"
 serde_derive = "1.0"
-app_dirs = "1.1.1"
 futures = "0.1"
 futures-cpupool = "0.1"
 fdlimit = "0.1"

ethcore/src/engines/tendermint/mod.rs

@@ -27,7 +27,7 @@ mod params;
 
 use std::sync::{Weak, Arc};
 use std::sync::atomic::{AtomicUsize, Ordering as AtomicOrdering};
-use std::collections::{HashSet, BTreeMap};
+use std::collections::HashSet;
 use hash::keccak;
 use ethereum_types::{H256, H520, U128, U256, Address};
 use parking_lot::RwLock;
@@ -449,17 +449,6 @@ impl Engine<EthereumMachine> for Tendermint {
 
 	fn maximum_uncle_age(&self) -> usize { 0 }
 
-	/// Additional engine-specific information for the user/developer concerning `header`.
-	fn extra_info(&self, header: &Header) -> BTreeMap<String, String> {
-		let message = ConsensusMessage::new_proposal(header).expect("Invalid header.");
-		map![
-			"signature".into() => message.signature.to_string(),
-			"height".into() => message.vote_step.height.to_string(),
-			"view".into() => message.vote_step.view.to_string(),
-			"block_hash".into() => message.block_hash.as_ref().map(ToString::to_string).unwrap_or("".into())
-		]
-	}
-
 	fn populate_from_parent(&self, header: &mut Header, parent: &Header) {
 		// Chain scoring: total weight is sqrt(U256::max_value())*height - view
 		let new_difficulty = U256::from(U128::max_value())

ethcore/src/executive.rs

@@ -34,10 +34,21 @@ use transaction::{Action, SignedTransaction};
 use crossbeam;
 pub use executed::{Executed, ExecutionResult};
 
-/// Roughly estimate what stack size each level of evm depth will use
+#[cfg(debug_assertions)]
-/// TODO [todr] We probably need some more sophisticated calculations here (limit on my machine 132)
+/// Roughly estimate what stack size each level of evm depth will use. (Debug build)
-/// Maybe something like here: `https://github.com/ethereum/libethereum/blob/4db169b8504f2b87f7d5a481819cfb959fc65f6c/libethereum/ExtVM.cpp`
+const STACK_SIZE_PER_DEPTH: usize = 128 * 1024;
-const STACK_SIZE_PER_DEPTH: usize = 24*1024;
+
+#[cfg(not(debug_assertions))]
+/// Roughly estimate what stack size each level of evm depth will use.
+const STACK_SIZE_PER_DEPTH: usize = 24 * 1024;
+
+#[cfg(debug_assertions)]
+/// Entry stack overhead prior to execution. (Debug build)
+const STACK_SIZE_ENTRY_OVERHEAD: usize = 100 * 1024;
+
+#[cfg(not(debug_assertions))]
+/// Entry stack overhead prior to execution.
+const STACK_SIZE_ENTRY_OVERHEAD: usize = 20 * 1024;
 
 /// Returns new address created from address, nonce, and code hash
 pub fn contract_address(address_scheme: CreateContractAddress, sender: &Address, nonce: &U256, code: &[u8]) -> (Address, Option<H256>) {
@@ -332,12 +343,12 @@ impl<'a, B: 'a + StateBackend> Executive<'a, B> {
 		tracer: &mut T,
 		vm_tracer: &mut V
 	) -> vm::Result<FinalizationResult> where T: Tracer, V: VMTracer {
-
+		let local_stack_size = ::io::LOCAL_STACK_SIZE.with(|sz| sz.get());
-		let depth_threshold = ::io::LOCAL_STACK_SIZE.with(|sz| sz.get() / STACK_SIZE_PER_DEPTH);
+		let depth_threshold = local_stack_size.saturating_sub(STACK_SIZE_ENTRY_OVERHEAD) / STACK_SIZE_PER_DEPTH;
 		let static_call = params.call_type == CallType::StaticCall;
 
 		// Ordinary execution - keep VM in same thread
-		if (self.depth + 1) % depth_threshold != 0 {
+		if self.depth != depth_threshold {
 			let vm_factory = self.state.vm_factory();
 			let mut ext = self.as_externalities(OriginInfo::from(&params), unconfirmed_substate, output_policy, tracer, vm_tracer, static_call);
 			trace!(target: "executive", "ext.schedule.have_delegate_call: {}", ext.schedule().have_delegate_call);
@@ -345,17 +356,15 @@ impl<'a, B: 'a + StateBackend> Executive<'a, B> {
 			return vm.exec(params, &mut ext).finalize(ext);
 		}
 
-		// Start in new thread to reset stack
+		// Start in new thread with stack size needed up to max depth
-		// TODO [todr] No thread builder yet, so we need to reset once for a while
-		// https://github.com/aturon/crossbeam/issues/16
 		crossbeam::scope(|scope| {
 			let vm_factory = self.state.vm_factory();
 			let mut ext = self.as_externalities(OriginInfo::from(&params), unconfirmed_substate, output_policy, tracer, vm_tracer, static_call);
 
-			scope.spawn(move || {
+			scope.builder().stack_size(::std::cmp::max(schedule.max_depth.saturating_sub(depth_threshold) * STACK_SIZE_PER_DEPTH, local_stack_size)).spawn(move || {
 				let mut vm = vm_factory.create(&params, &schedule);
 				vm.exec(params, &mut ext).finalize(ext)
-			})
+			}).expect("Sub-thread creation cannot fail; the host might run out of resources; qed")
 		}).join()
 	}
 

parity/main.rs

@@ -19,7 +19,6 @@
 #![warn(missing_docs)]
 
 extern crate ansi_term;
-extern crate app_dirs;
 extern crate ctrlc;
 extern crate docopt;
 #[macro_use]

sync/src/light_sync/mod.rs

@@ -427,6 +427,8 @@ impl<L: AsLightClient> LightSync<L> {
 
 	// handles request dispatch, block import, state machine transitions, and timeouts.
 	fn maintain_sync(&self, ctx: &BasicContext) {
+		use ethcore::error::{BlockImportError, ImportError};
+
 		const DRAIN_AMOUNT: usize = 128;
 
 		let client = self.client.as_light_client();
@@ -453,7 +455,15 @@ impl<L: AsLightClient> LightSync<L> {
 				trace!(target: "sync", "Drained {} headers to import", sink.len());
 
 				for header in sink.drain(..) {
-					if let Err(e) = client.queue_header(header) {
+					match client.queue_header(header) {
+						Ok(_) => {}
+						Err(BlockImportError::Import(ImportError::AlreadyInChain)) => {
+							trace!(target: "sync", "Block already in chain. Continuing.");
+						},
+						Err(BlockImportError::Import(ImportError::AlreadyQueued)) => {
+							trace!(target: "sync", "Block already queued. Continuing.");
+						},
+						Err(e) => {
 							debug!(target: "sync", "Found bad header ({:?}). Reset to search state.", e);
 
 							self.begin_search(&mut state);
@@ -462,6 +472,7 @@ impl<L: AsLightClient> LightSync<L> {
 					}
 				}
 			}
+		}
 
 		// handle state transitions.
 		{

util/dir/Cargo.toml

@@ -6,4 +6,4 @@ authors = ["Parity Technologies <admin@parity.io>"]
 [dependencies]
 ethereum-types = "0.2"
 journaldb = { path = "../journaldb" }
-app_dirs = "1.1.1"
+app_dirs = { git = "https://github.com/paritytech/app-dirs-rs" }