[stable] Backports (#8008)

* Detect too large packets in snapshot sync. (#7977)

* Increase max download limit to 128MB (#7965)

* fetch: increase max download limit to 64MB

* parity: increase download size limit for updater service

* Calculate proper keccak256/sha3 using parity. (#7953)

* Bump WS (#7952)

Cargo.lock

@@ -1505,7 +1505,7 @@ dependencies = [
  "log 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)",
  "parking_lot 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)",
  "slab 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "ws 0.7.1 (git+https://github.com/tomusdrw/ws-rs)",
+ "ws 0.7.5 (git+https://github.com/tomusdrw/ws-rs)",
 ]
 
 [[package]]
@@ -3673,8 +3673,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 
 [[package]]
 name = "ws"
-version = "0.7.1"
+version = "0.7.5"
-source = "git+https://github.com/tomusdrw/ws-rs#f8306a798b7541d64624299a83a2c934f173beed"
+source = "git+https://github.com/tomusdrw/ws-rs#368ce39e2aa8700d568ca29dbacaecdf1bf749d1"
 dependencies = [
  "byteorder 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "bytes 0.4.5 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -4013,7 +4013,7 @@ dependencies = [
 "checksum wasm-utils 0.1.0 (git+https://github.com/paritytech/wasm-utils)" = "<none>"
 "checksum winapi 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)" = "167dc9d6949a9b857f3451275e911c3f44255842c1f7a76f33c55103a909087a"
 "checksum winapi-build 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "2d315eee3b34aca4797b2da6b13ed88266e6d612562a0c46390af8299fc699bc"
-"checksum ws 0.7.1 (git+https://github.com/tomusdrw/ws-rs)" = "<none>"
+"checksum ws 0.7.5 (git+https://github.com/tomusdrw/ws-rs)" = "<none>"
 "checksum ws2_32-sys 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "d59cefebd0c892fa2dd6de581e937301d8552cb44489cdff035c6187cb63fa5e"
 "checksum xdg 2.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "a66b7c2281ebde13cf4391d70d4c7e5946c3c25e72a7b859ca8f677dcd0b0c61"
 "checksum xml-rs 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)" = "7ec6c39eaa68382c8e31e35239402c0a9489d4141a8ceb0c716099a0b515b562"

ethcore/src/snapshot/error.rs

@@ -57,6 +57,8 @@ pub enum Error {
 	VersionNotSupported(u64),
 	/// Max chunk size is to small to fit basic account data.
 	ChunkTooSmall,
+	/// Oversized chunk
+	ChunkTooLarge,
 	/// Snapshots not supported by the consensus engine.
 	SnapshotsUnsupported,
 	/// Bad epoch transition.
@@ -85,6 +87,7 @@ impl fmt::Display for Error {
 			Error::Trie(ref err) => err.fmt(f),
 			Error::VersionNotSupported(ref ver) => write!(f, "Snapshot version {} is not supprted.", ver),
 			Error::ChunkTooSmall => write!(f, "Chunk size is too small."),
+			Error::ChunkTooLarge => write!(f, "Chunk size is too large."),
 			Error::SnapshotsUnsupported => write!(f, "Snapshots unsupported by consensus engine."),
 			Error::BadEpochProof(i) => write!(f, "Bad epoch proof for transition to epoch {}", i),
 			Error::WrongChunkFormat(ref msg) => write!(f, "Wrong chunk format: {}", msg),

ethcore/src/snapshot/mod.rs

@@ -86,6 +86,11 @@ mod traits {
 // Try to have chunks be around 4MB (before compression)
 const PREFERRED_CHUNK_SIZE: usize = 4 * 1024 * 1024;
 
+// Maximal chunk size (decompressed)
+// Snappy::decompressed_len estimation may sometimes yield results greater
+// than PREFERRED_CHUNK_SIZE so allow some threshold here.
+const MAX_CHUNK_SIZE: usize = PREFERRED_CHUNK_SIZE / 4 * 5;
+
 // Minimum supported state chunk version.
 const MIN_SUPPORTED_STATE_CHUNK_VERSION: u64 = 1;
 // current state chunk version.

ethcore/src/snapshot/service.rs

@@ -23,7 +23,7 @@ use std::path::PathBuf;
 use std::sync::Arc;
 use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
 
-use super::{ManifestData, StateRebuilder, Rebuilder, RestorationStatus, SnapshotService};
+use super::{ManifestData, StateRebuilder, Rebuilder, RestorationStatus, SnapshotService, MAX_CHUNK_SIZE};
 use super::io::{SnapshotReader, LooseReader, SnapshotWriter, LooseWriter};
 
 use blockchain::BlockChain;
@@ -130,6 +130,11 @@ impl Restoration {
 	// feeds a state chunk, aborts early if `flag` becomes false.
 	fn feed_state(&mut self, hash: H256, chunk: &[u8], flag: &AtomicBool) -> Result<(), Error> {
 		if self.state_chunks_left.contains(&hash) {
+			let expected_len = snappy::decompressed_len(chunk)?;
+			if expected_len > MAX_CHUNK_SIZE {
+				trace!(target: "snapshot", "Discarding large chunk: {} vs {}", expected_len, MAX_CHUNK_SIZE);
+				return Err(::snapshot::Error::ChunkTooLarge.into());
+			}
 			let len = snappy::decompress_into(chunk, &mut self.snappy_buffer)?;
 
 			self.state.feed(&self.snappy_buffer[..len], flag)?;
@@ -147,6 +152,11 @@ impl Restoration {
 	// feeds a block chunk
 	fn feed_blocks(&mut self, hash: H256, chunk: &[u8], engine: &EthEngine, flag: &AtomicBool) -> Result<(), Error> {
 		if self.block_chunks_left.contains(&hash) {
+			let expected_len = snappy::decompressed_len(chunk)?;
+			if expected_len > MAX_CHUNK_SIZE {
+				trace!(target: "snapshot", "Discarding large chunk: {} vs {}", expected_len, MAX_CHUNK_SIZE);
+				return Err(::snapshot::Error::ChunkTooLarge.into());
+			}
 			let len = snappy::decompress_into(chunk, &mut self.snappy_buffer)?;
 
 			self.secondary.feed(&self.snappy_buffer[..len], engine, flag)?;

parity/run.rs

@@ -680,11 +680,15 @@ pub fn execute_impl(cmd: RunCmd, can_restart: bool, logger: Arc<RotatingLogger>)
 	let event_loop = EventLoop::spawn();
 
 	// the updater service
+	let mut updater_fetch = fetch.clone();
+	// parity binaries should be smaller than 128MB
+	updater_fetch.set_limit(Some(128 * 1024 * 1024));
+
 	let updater = Updater::new(
 		Arc::downgrade(&(service.client() as Arc<BlockChainClient>)),
 		Arc::downgrade(&sync_provider),
 		update_policy,
-		fetch.clone(),
+		updater_fetch,
 		event_loop.remote(),
 	);
 	service.add_notify(updater.clone());

scripts/gitlab-build.sh

@@ -22,13 +22,10 @@ echo "Parity version:     " $VER
 echo "Branch:             " $CI_BUILD_REF_NAME
 echo "--------------------"
 
-echo "Rhash version:"
 # NOTE for md5 and sha256 we want to display filename as well
 # hence we use --* instead of -p *
 MD5_BIN="rhash --md5"
 SHA256_BIN="rhash --sha256"
-# NOTE For SHA3 we need only hash (hence -p)
-SHA3_BIN="rhash -p %{sha3-256}"
 
 set_env () {
   echo "Set ENVIROMENT"
@@ -70,14 +67,12 @@ strip_binaries () {
 calculate_checksums () {
   echo "Checksum calculation:"
   rhash --version
+
   rm -rf *.md5
   rm -rf *.sha256
 
-  export SHA3="$($SHA3_BIN target/$PLATFORM/release/parity$S3WIN)"
+  BIN="target/$PLATFORM/release/parity$S3WIN"
-  # NOTE rhash 1.3.1 doesnt support keccak, workaround
+  export SHA3="$($BIN tools hash $BIN)"
-  if [ "$SHA3" == "%{sha3-256}" ]; then
-    export SHA3="$(target/$PLATFORM/release/parity$S3WIN tools hash target/$PLATFORM/release/parity$S3WIN)"
-  fi
 
   echo "Parity file SHA3: $SHA3"
   $MD5_BIN target/$PLATFORM/release/parity$S3WIN > parity$S3WIN.md5

util/fetch/src/client.rs

@@ -127,6 +127,11 @@ impl Client {
 		})
 	}
 
+	/// Sets a limit on the maximum download size.
+	pub fn set_limit(&mut self, limit: Option<usize>) {
+		self.limit = limit
+	}
+
 	fn client(&self) -> Result<Arc<reqwest::Client>, Error> {
 		{
 			let (ref time, ref client) = *self.client.read();
@@ -150,8 +155,8 @@ impl Fetch for Client {
 	type Result = CpuFuture<Response, Error>;
 
 	fn new() -> Result<Self, Error> {
-		// Max 50MB will be downloaded.
+		// Max 64MB will be downloaded.
-		Self::with_limit(Some(50*1024*1024))
+		Self::with_limit(Some(64 * 1024 * 1024))
 	}
 
 	fn process<F, I, E>(&self, f: F) -> BoxFuture<I, E> where