grassrootseconomics/openethereum
5
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

[stable] Backports (#8008)

Browse Source 
* Detect too large packets in snapshot sync. (#7977)

* Increase max download limit to 128MB (#7965)

* fetch: increase max download limit to 64MB

* parity: increase download size limit for updater service

* Calculate proper keccak256/sha3 using parity. (#7953)

* Bump WS (#7952)
This commit is contained in:
Tomasz Drwięga 2018-02-27 20:07:23 +01:00 committed by Rando
parent a51e4d5d13
commit 2738c5cd74
7 changed files with 38 additions and 16 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
Cargo.lock generated
View File

@ -1505,7 +1505,7 @@ dependencies = [
"log 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)",
"parking_lot 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)",
"slab 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)",
"ws 0.7.1 (git+https://github.com/tomusdrw/ws-rs)",
"ws 0.7.5 (git+https://github.com/tomusdrw/ws-rs)",
]
[[package]]
@ -3673,8 +3673,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
[[package]]
name = "ws"
version = "0.7.1"
source = "git+https://github.com/tomusdrw/ws-rs#f8306a798b7541d64624299a83a2c934f173beed"
version = "0.7.5"
source = "git+https://github.com/tomusdrw/ws-rs#368ce39e2aa8700d568ca29dbacaecdf1bf749d1"
dependencies = [
"byteorder 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
"bytes 0.4.5 (registry+https://github.com/rust-lang/crates.io-index)",
@ -4013,7 +4013,7 @@ dependencies = [
"checksum wasm-utils 0.1.0 (git+https://github.com/paritytech/wasm-utils)" = "<none>"
"checksum winapi 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)" = "167dc9d6949a9b857f3451275e911c3f44255842c1f7a76f33c55103a909087a"
"checksum winapi-build 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "2d315eee3b34aca4797b2da6b13ed88266e6d612562a0c46390af8299fc699bc"
"checksum ws 0.7.1 (git+https://github.com/tomusdrw/ws-rs)" = "<none>"
"checksum ws 0.7.5 (git+https://github.com/tomusdrw/ws-rs)" = "<none>"
"checksum ws2_32-sys 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "d59cefebd0c892fa2dd6de581e937301d8552cb44489cdff035c6187cb63fa5e"
"checksum xdg 2.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "a66b7c2281ebde13cf4391d70d4c7e5946c3c25e72a7b859ca8f677dcd0b0c61"
"checksum xml-rs 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)" = "7ec6c39eaa68382c8e31e35239402c0a9489d4141a8ceb0c716099a0b515b562"

3
ethcore/src/snapshot/error.rs
View File

@ -57,6 +57,8 @@ pub enum Error {
VersionNotSupported(u64),
/// Max chunk size is to small to fit basic account data.
ChunkTooSmall,
/// Oversized chunk
ChunkTooLarge,
/// Snapshots not supported by the consensus engine.
SnapshotsUnsupported,
/// Bad epoch transition.
@ -85,6 +87,7 @@ impl fmt::Display for Error {
Error::Trie(ref err) => err.fmt(f),
Error::VersionNotSupported(ref ver) => write!(f, "Snapshot version {} is not supprted.", ver),
Error::ChunkTooSmall => write!(f, "Chunk size is too small."),
Error::ChunkTooLarge => write!(f, "Chunk size is too large."),
Error::SnapshotsUnsupported => write!(f, "Snapshots unsupported by consensus engine."),
Error::BadEpochProof(i) => write!(f, "Bad epoch proof for transition to epoch {}", i),
Error::WrongChunkFormat(ref msg) => write!(f, "Wrong chunk format: {}", msg),

5
ethcore/src/snapshot/mod.rs
View File

@ -86,6 +86,11 @@ mod traits {
// Try to have chunks be around 4MB (before compression)
const PREFERRED_CHUNK_SIZE: usize = 4 * 1024 * 1024;
// Maximal chunk size (decompressed)
// Snappy::decompressed_len estimation may sometimes yield results greater
// than PREFERRED_CHUNK_SIZE so allow some threshold here.
const MAX_CHUNK_SIZE: usize = PREFERRED_CHUNK_SIZE / 4 * 5;
// Minimum supported state chunk version.
const MIN_SUPPORTED_STATE_CHUNK_VERSION: u64 = 1;
// current state chunk version.

12
ethcore/src/snapshot/service.rs
View File

@ -23,7 +23,7 @@ use std::path::PathBuf;
use std::sync::Arc;
use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
use super::{ManifestData, StateRebuilder, Rebuilder, RestorationStatus, SnapshotService};
use super::{ManifestData, StateRebuilder, Rebuilder, RestorationStatus, SnapshotService, MAX_CHUNK_SIZE};
use super::io::{SnapshotReader, LooseReader, SnapshotWriter, LooseWriter};
use blockchain::BlockChain;
@ -130,6 +130,11 @@ impl Restoration {
// feeds a state chunk, aborts early if `flag` becomes false.
fn feed_state(&mut self, hash: H256, chunk: &[u8], flag: &AtomicBool) -> Result<(), Error> {
if self.state_chunks_left.contains(&hash) {
let expected_len = snappy::decompressed_len(chunk)?;
if expected_len > MAX_CHUNK_SIZE {
trace!(target: "snapshot", "Discarding large chunk: {} vs {}", expected_len, MAX_CHUNK_SIZE);
return Err(::snapshot::Error::ChunkTooLarge.into());
}
let len = snappy::decompress_into(chunk, &mut self.snappy_buffer)?;
self.state.feed(&self.snappy_buffer[..len], flag)?;
@ -147,6 +152,11 @@ impl Restoration {
// feeds a block chunk
fn feed_blocks(&mut self, hash: H256, chunk: &[u8], engine: &EthEngine, flag: &AtomicBool) -> Result<(), Error> {
if self.block_chunks_left.contains(&hash) {
let expected_len = snappy::decompressed_len(chunk)?;
if expected_len > MAX_CHUNK_SIZE {
trace!(target: "snapshot", "Discarding large chunk: {} vs {}", expected_len, MAX_CHUNK_SIZE);
return Err(::snapshot::Error::ChunkTooLarge.into());
}
let len = snappy::decompress_into(chunk, &mut self.snappy_buffer)?;
self.secondary.feed(&self.snappy_buffer[..len], engine, flag)?;

6
parity/run.rs
View File

@ -680,11 +680,15 @@ pub fn execute_impl(cmd: RunCmd, can_restart: bool, logger: Arc<RotatingLogger>)
let event_loop = EventLoop::spawn();
// the updater service
let mut updater_fetch = fetch.clone();
// parity binaries should be smaller than 128MB
updater_fetch.set_limit(Some(128 * 1024 * 1024));
let updater = Updater::new(
Arc::downgrade(&(service.client() as Arc<BlockChainClient>)),
Arc::downgrade(&sync_provider),
update_policy,
fetch.clone(),
updater_fetch,
event_loop.remote(),
);
service.add_notify(updater.clone());

11
scripts/gitlab-build.sh
View File

@ -22,13 +22,10 @@ echo "Parity version: " $VER
echo "Branch: " $CI_BUILD_REF_NAME
echo "--------------------"
echo "Rhash version:"
# NOTE for md5 and sha256 we want to display filename as well
# hence we use --* instead of -p *
MD5_BIN="rhash --md5"
SHA256_BIN="rhash --sha256"
# NOTE For SHA3 we need only hash (hence -p)
SHA3_BIN="rhash -p %{sha3-256}"
set_env () {
echo "Set ENVIROMENT"
@ -70,14 +67,12 @@ strip_binaries () {
calculate_checksums () {
echo "Checksum calculation:"
rhash --version
rm -rf *.md5
rm -rf *.sha256
export SHA3="$($SHA3_BIN target/$PLATFORM/release/parity$S3WIN)"
# NOTE rhash 1.3.1 doesnt support keccak, workaround
if [ "$SHA3" == "%{sha3-256}" ]; then
export SHA3="$(target/$PLATFORM/release/parity$S3WIN tools hash target/$PLATFORM/release/parity$S3WIN)"
fi
BIN="target/$PLATFORM/release/parity$S3WIN"
export SHA3="$($BIN tools hash $BIN)"
echo "Parity file SHA3: $SHA3"
$MD5_BIN target/$PLATFORM/release/parity$S3WIN > parity$S3WIN.md5

9
util/fetch/src/client.rs
View File

@ -127,6 +127,11 @@ impl Client {
})
}
/// Sets a limit on the maximum download size.
pub fn set_limit(&mut self, limit: Option<usize>) {
self.limit = limit
}
fn client(&self) -> Result<Arc<reqwest::Client>, Error> {
{
let (ref time, ref client) = *self.client.read();
@ -150,8 +155,8 @@ impl Fetch for Client {
type Result = CpuFuture<Response, Error>;
fn new() -> Result<Self, Error> {
// Max 50MB will be downloaded.
Self::with_limit(Some(50*1024*1024))
// Max 64MB will be downloaded.
Self::with_limit(Some(64 * 1024 * 1024))
}
fn process<F, I, E>(&self, f: F) -> BoxFuture<I, E> where