diff --git a/Cargo.lock b/Cargo.lock index 8b1165995..ebcb7f693 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1505,7 +1505,7 @@ dependencies = [ "log 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)", "parking_lot 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", "slab 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)", - "ws 0.7.1 (git+https://github.com/tomusdrw/ws-rs)", + "ws 0.7.5 (git+https://github.com/tomusdrw/ws-rs)", ] [[package]] @@ -3673,8 +3673,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "ws" -version = "0.7.1" -source = "git+https://github.com/tomusdrw/ws-rs#f8306a798b7541d64624299a83a2c934f173beed" +version = "0.7.5" +source = "git+https://github.com/tomusdrw/ws-rs#368ce39e2aa8700d568ca29dbacaecdf1bf749d1" dependencies = [ "byteorder 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)", "bytes 0.4.5 (registry+https://github.com/rust-lang/crates.io-index)", @@ -4013,7 +4013,7 @@ dependencies = [ "checksum wasm-utils 0.1.0 (git+https://github.com/paritytech/wasm-utils)" = "" "checksum winapi 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)" = "167dc9d6949a9b857f3451275e911c3f44255842c1f7a76f33c55103a909087a" "checksum winapi-build 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "2d315eee3b34aca4797b2da6b13ed88266e6d612562a0c46390af8299fc699bc" -"checksum ws 0.7.1 (git+https://github.com/tomusdrw/ws-rs)" = "" +"checksum ws 0.7.5 (git+https://github.com/tomusdrw/ws-rs)" = "" "checksum ws2_32-sys 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "d59cefebd0c892fa2dd6de581e937301d8552cb44489cdff035c6187cb63fa5e" "checksum xdg 2.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "a66b7c2281ebde13cf4391d70d4c7e5946c3c25e72a7b859ca8f677dcd0b0c61" "checksum xml-rs 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)" = "7ec6c39eaa68382c8e31e35239402c0a9489d4141a8ceb0c716099a0b515b562" diff --git a/ethcore/src/snapshot/error.rs b/ethcore/src/snapshot/error.rs index bd0cb227b..33faab667 100644 --- a/ethcore/src/snapshot/error.rs +++ b/ethcore/src/snapshot/error.rs @@ -57,6 +57,8 @@ pub enum Error { VersionNotSupported(u64), /// Max chunk size is to small to fit basic account data. ChunkTooSmall, + /// Oversized chunk + ChunkTooLarge, /// Snapshots not supported by the consensus engine. SnapshotsUnsupported, /// Bad epoch transition. @@ -85,6 +87,7 @@ impl fmt::Display for Error { Error::Trie(ref err) => err.fmt(f), Error::VersionNotSupported(ref ver) => write!(f, "Snapshot version {} is not supprted.", ver), Error::ChunkTooSmall => write!(f, "Chunk size is too small."), + Error::ChunkTooLarge => write!(f, "Chunk size is too large."), Error::SnapshotsUnsupported => write!(f, "Snapshots unsupported by consensus engine."), Error::BadEpochProof(i) => write!(f, "Bad epoch proof for transition to epoch {}", i), Error::WrongChunkFormat(ref msg) => write!(f, "Wrong chunk format: {}", msg), diff --git a/ethcore/src/snapshot/mod.rs b/ethcore/src/snapshot/mod.rs index e33c47e2c..c6622373c 100644 --- a/ethcore/src/snapshot/mod.rs +++ b/ethcore/src/snapshot/mod.rs @@ -86,6 +86,11 @@ mod traits { // Try to have chunks be around 4MB (before compression) const PREFERRED_CHUNK_SIZE: usize = 4 * 1024 * 1024; +// Maximal chunk size (decompressed) +// Snappy::decompressed_len estimation may sometimes yield results greater +// than PREFERRED_CHUNK_SIZE so allow some threshold here. +const MAX_CHUNK_SIZE: usize = PREFERRED_CHUNK_SIZE / 4 * 5; + // Minimum supported state chunk version. const MIN_SUPPORTED_STATE_CHUNK_VERSION: u64 = 1; // current state chunk version. diff --git a/ethcore/src/snapshot/service.rs b/ethcore/src/snapshot/service.rs index ae6a34cfa..f87ceffc5 100644 --- a/ethcore/src/snapshot/service.rs +++ b/ethcore/src/snapshot/service.rs @@ -23,7 +23,7 @@ use std::path::PathBuf; use std::sync::Arc; use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; -use super::{ManifestData, StateRebuilder, Rebuilder, RestorationStatus, SnapshotService}; +use super::{ManifestData, StateRebuilder, Rebuilder, RestorationStatus, SnapshotService, MAX_CHUNK_SIZE}; use super::io::{SnapshotReader, LooseReader, SnapshotWriter, LooseWriter}; use blockchain::BlockChain; @@ -130,6 +130,11 @@ impl Restoration { // feeds a state chunk, aborts early if `flag` becomes false. fn feed_state(&mut self, hash: H256, chunk: &[u8], flag: &AtomicBool) -> Result<(), Error> { if self.state_chunks_left.contains(&hash) { + let expected_len = snappy::decompressed_len(chunk)?; + if expected_len > MAX_CHUNK_SIZE { + trace!(target: "snapshot", "Discarding large chunk: {} vs {}", expected_len, MAX_CHUNK_SIZE); + return Err(::snapshot::Error::ChunkTooLarge.into()); + } let len = snappy::decompress_into(chunk, &mut self.snappy_buffer)?; self.state.feed(&self.snappy_buffer[..len], flag)?; @@ -147,6 +152,11 @@ impl Restoration { // feeds a block chunk fn feed_blocks(&mut self, hash: H256, chunk: &[u8], engine: &EthEngine, flag: &AtomicBool) -> Result<(), Error> { if self.block_chunks_left.contains(&hash) { + let expected_len = snappy::decompressed_len(chunk)?; + if expected_len > MAX_CHUNK_SIZE { + trace!(target: "snapshot", "Discarding large chunk: {} vs {}", expected_len, MAX_CHUNK_SIZE); + return Err(::snapshot::Error::ChunkTooLarge.into()); + } let len = snappy::decompress_into(chunk, &mut self.snappy_buffer)?; self.secondary.feed(&self.snappy_buffer[..len], engine, flag)?; diff --git a/parity/run.rs b/parity/run.rs index 2bf1aad9e..928848eb1 100644 --- a/parity/run.rs +++ b/parity/run.rs @@ -680,11 +680,15 @@ pub fn execute_impl(cmd: RunCmd, can_restart: bool, logger: Arc) let event_loop = EventLoop::spawn(); // the updater service + let mut updater_fetch = fetch.clone(); + // parity binaries should be smaller than 128MB + updater_fetch.set_limit(Some(128 * 1024 * 1024)); + let updater = Updater::new( Arc::downgrade(&(service.client() as Arc)), Arc::downgrade(&sync_provider), update_policy, - fetch.clone(), + updater_fetch, event_loop.remote(), ); service.add_notify(updater.clone()); diff --git a/scripts/gitlab-build.sh b/scripts/gitlab-build.sh index 8d3ea9c1d..b2fdad4c5 100755 --- a/scripts/gitlab-build.sh +++ b/scripts/gitlab-build.sh @@ -22,13 +22,10 @@ echo "Parity version: " $VER echo "Branch: " $CI_BUILD_REF_NAME echo "--------------------" -echo "Rhash version:" # NOTE for md5 and sha256 we want to display filename as well # hence we use --* instead of -p * MD5_BIN="rhash --md5" SHA256_BIN="rhash --sha256" -# NOTE For SHA3 we need only hash (hence -p) -SHA3_BIN="rhash -p %{sha3-256}" set_env () { echo "Set ENVIROMENT" @@ -70,14 +67,12 @@ strip_binaries () { calculate_checksums () { echo "Checksum calculation:" rhash --version + rm -rf *.md5 rm -rf *.sha256 - export SHA3="$($SHA3_BIN target/$PLATFORM/release/parity$S3WIN)" - # NOTE rhash 1.3.1 doesnt support keccak, workaround - if [ "$SHA3" == "%{sha3-256}" ]; then - export SHA3="$(target/$PLATFORM/release/parity$S3WIN tools hash target/$PLATFORM/release/parity$S3WIN)" - fi + BIN="target/$PLATFORM/release/parity$S3WIN" + export SHA3="$($BIN tools hash $BIN)" echo "Parity file SHA3: $SHA3" $MD5_BIN target/$PLATFORM/release/parity$S3WIN > parity$S3WIN.md5 diff --git a/util/fetch/src/client.rs b/util/fetch/src/client.rs index a24cf991b..07c6a1624 100644 --- a/util/fetch/src/client.rs +++ b/util/fetch/src/client.rs @@ -127,6 +127,11 @@ impl Client { }) } + /// Sets a limit on the maximum download size. + pub fn set_limit(&mut self, limit: Option) { + self.limit = limit + } + fn client(&self) -> Result, Error> { { let (ref time, ref client) = *self.client.read(); @@ -150,8 +155,8 @@ impl Fetch for Client { type Result = CpuFuture; fn new() -> Result { - // Max 50MB will be downloaded. - Self::with_limit(Some(50*1024*1024)) + // Max 64MB will be downloaded. + Self::with_limit(Some(64 * 1024 * 1024)) } fn process(&self, f: F) -> BoxFuture where