2.4.2 beta backports (#10488)

* version: bump beta

* Сaching through docker volume (#10477)

* _old codebase_ before docker update

* before docker update, testing runnr

* docker update, testing the caching

* distributed job cargo homes

* distributed job cargo homes 2

* distributed job cargo homes 3

* dockerfile with gitlab checkout, audit uses template

* dockerfile gets repo in volume

* change builds_dir

* trying docker cache for repo

* repo cached automatically

* after script is not concatenated

* check sccache non-cacheable reasons nature

* watch cache

* log sccache

* log sccache 2

* debug log sccache

* fix debug log sccache

* fix debug log sccache 2

* debug log cache 3

* debug log cache 3

* trace log all sccache

* test wo cargo cache

* test w removed cargo cache

* report non-cacheable reasons, cargo cache is back and empty

* report non-cacheable reasons, cargo cache is back and empty 2

* report non-cacheable reasons, cargo cache is back and empty 3

* wrap into after_script

* restore CI tags

`qa` -> `linux-docker`

* return to main runners, this will fail until config on runners And Dockerfile won't be updated

* typo fix CI lint

* return to docker tag

* fix win&mac build (#10486)

add CARGO_HOME:                      "${CI_PROJECT_DIR}/.cargo"

* fix(extract `timestamp_checked_add` as lib) (#10383)

* fix(extract `timestamp_checked_add` as lib)

* fix(whisper types): remove unused `EmptyTopics`

* fix(time-lib): feature-flag to use time-lib or std

This commit adds conditional compilation checks that falls back to `our time-lib` when
`time_checked_add` is not available in the standard library

Note, `time_checked_add` covers both `checked_add` and `checked_sub`

* fix(grumble): use cfg_attr to define rustc feature
This commit is contained in:
Sočik 2019-03-20 09:17:30 +01:00 committed by GitHub
parent 72c8b79035
commit e1c1ecf0a5
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
14 changed files with 165 additions and 73 deletions

View File

@ -9,10 +9,9 @@ variables:
GIT_STRATEGY: fetch
GIT_SUBMODULE_STRATEGY: recursive
CI_SERVER_NAME: "GitLab CI"
CARGO_HOME: "${CI_PROJECT_DIR}/.cargo"
CARGO_TARGET: x86_64-unknown-linux-gnu
.no_git: &no_git #disable git strategy
.no_git: &no_git # disable git strategy
variables:
GIT_STRATEGY: none
GIT_SUBMODULE_STRATEGY: none
@ -32,12 +31,17 @@ variables:
paths:
- artifacts/
.docker-cache-status: &docker-cache-status
dependencies: []
.docker-cache-status: &docker-cache-status
variables:
CARGO_HOME: "/cargo/${CI_JOB_NAME}"
before_script:
- SCCACHE_ERROR_LOG=/builds/parity/parity-ethereum/sccache_error.log RUST_LOG=sccache::server=debug sccache --start-server
- sccache -s
after_script:
- sccache -s
- echo "All crate-types:"
- grep 'parse_arguments.*--crate-type' sccache_error.log | sed -re 's/.*"--crate-type", "([^"]+)".*/\1/' | sort | uniq -c
- echo "Non-cacheable reasons:"
- grep CannotCache sccache_error.log | sed -re 's/.*CannotCache\((.+)\).*/\1/' | sort | uniq -c
tags:
- linux-docker
@ -47,43 +51,60 @@ cargo-check 0 3:
<<: *docker-cache-status
script:
- time cargo check --target $CARGO_TARGET --locked --no-default-features
- sccache -s
cargo-check 1 3:
stage: test
<<: *docker-cache-status
script:
- time cargo check --target $CARGO_TARGET --locked --manifest-path util/io/Cargo.toml --no-default-features
- sccache -s
cargo-check 2 3:
stage: test
<<: *docker-cache-status
script:
- time cargo check --target $CARGO_TARGET --locked --manifest-path util/io/Cargo.toml --features "mio"
- sccache -s
cargo-audit:
stage: test
<<: *docker-cache-status
script:
- cargo audit
tags:
- linux-docker
- sccache -s
validate-chainspecs:
stage: test
<<: *docker-cache-status
script:
- ./scripts/gitlab/validate-chainspecs.sh
- sccache -s
test-cpp:
stage: build
<<: *docker-cache-status
script:
- ./scripts/gitlab/test-cpp.sh
- sccache -s
test-linux:
stage: build
<<: *docker-cache-status
script:
- ./scripts/gitlab/test-linux.sh
- sccache -s
build-android:
stage: build
image: parity/rust-android:gitlab-ci
variables:
CARGO_TARGET: armv7-linux-androideabi
script:
- scripts/gitlab/build-linux.sh
tags:
- linux-docker
<<: *collect_artifacts
build-linux: &build-linux
stage: build
@ -91,6 +112,7 @@ build-linux: &build-linux
<<: *docker-cache-status
script:
- scripts/gitlab/build-linux.sh
- sccache -s
<<: *collect_artifacts
build-linux-i386:
@ -116,6 +138,7 @@ build-darwin:
only: *releaseable_branches
variables:
CARGO_TARGET: x86_64-apple-darwin
CARGO_HOME: "${CI_PROJECT_DIR}/.cargo"
CC: gcc
CXX: g++
script:
@ -129,6 +152,7 @@ build-windows:
only: *releaseable_branches
variables:
CARGO_TARGET: x86_64-pc-windows-msvc
CARGO_HOME: "${CI_PROJECT_DIR}/.cargo"
script:
- sh scripts/gitlab/build-windows.sh
tags:
@ -138,7 +162,7 @@ build-windows:
publish-docker:
stage: publish
only: *releaseable_branches
cache: {}
cache: {}
dependencies:
- build-linux
tags:
@ -152,7 +176,7 @@ publish-snap: &publish-snap
image: snapcore/snapcraft
variables:
BUILD_ARCH: amd64
cache: {}
cache: {}
dependencies:
- build-linux
tags:
@ -228,19 +252,9 @@ publish-docs:
- tags
except:
- nightly
cache: {}
cache: {}
script:
- scripts/gitlab/publish-docs.sh
tags:
- linux-docker
build-android:
stage: build
image: parity/rust-android:gitlab-ci
variables:
CARGO_TARGET: armv7-linux-androideabi
script:
- scripts/gitlab/build-linux.sh
tags:
- linux-docker
<<: *collect_artifacts

18
Cargo.lock generated
View File

@ -753,6 +753,7 @@ dependencies = [
"serde_derive 1.0.80 (registry+https://github.com/rust-lang/crates.io-index)",
"stats 0.1.0",
"tempdir 0.3.7 (registry+https://github.com/rust-lang/crates.io-index)",
"time-utils 0.1.0",
"trace-time 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
"trie-db 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)",
"trie-standardmap 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
@ -2441,7 +2442,7 @@ dependencies = [
"futures 0.1.25 (registry+https://github.com/rust-lang/crates.io-index)",
"jni 0.10.2 (registry+https://github.com/rust-lang/crates.io-index)",
"panic_hook 0.1.0",
"parity-ethereum 2.4.1",
"parity-ethereum 2.4.2",
"tokio 0.1.11 (registry+https://github.com/rust-lang/crates.io-index)",
"tokio-current-thread 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)",
]
@ -2471,7 +2472,7 @@ dependencies = [
[[package]]
name = "parity-ethereum"
version = "2.4.1"
version = "2.4.2"
dependencies = [
"ansi_term 0.10.2 (registry+https://github.com/rust-lang/crates.io-index)",
"atty 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)",
@ -2524,7 +2525,7 @@ dependencies = [
"parity-rpc 1.12.0",
"parity-runtime 0.1.0",
"parity-updater 1.12.0",
"parity-version 2.4.1",
"parity-version 2.4.2",
"parity-whisper 0.1.0",
"parking_lot 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)",
"pretty_assertions 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)",
@ -2675,7 +2676,7 @@ dependencies = [
"parity-crypto 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)",
"parity-runtime 0.1.0",
"parity-updater 1.12.0",
"parity-version 2.4.1",
"parity-version 2.4.2",
"parking_lot 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)",
"pretty_assertions 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)",
"rand 0.4.3 (registry+https://github.com/rust-lang/crates.io-index)",
@ -2773,7 +2774,7 @@ dependencies = [
"parity-bytes 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
"parity-hash-fetch 1.12.0",
"parity-path 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
"parity-version 2.4.1",
"parity-version 2.4.2",
"parking_lot 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)",
"rand 0.4.3 (registry+https://github.com/rust-lang/crates.io-index)",
"semver 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)",
@ -2783,7 +2784,7 @@ dependencies = [
[[package]]
name = "parity-version"
version = "2.4.1"
version = "2.4.2"
dependencies = [
"parity-bytes 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
"rlp 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)",
@ -2826,6 +2827,7 @@ dependencies = [
"serde_json 1.0.32 (registry+https://github.com/rust-lang/crates.io-index)",
"slab 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)",
"smallvec 0.6.5 (registry+https://github.com/rust-lang/crates.io-index)",
"time-utils 0.1.0",
"tiny-keccak 1.4.2 (registry+https://github.com/rust-lang/crates.io-index)",
]
@ -3781,6 +3783,10 @@ dependencies = [
"winapi 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "time-utils"
version = "0.1.0"
[[package]]
name = "timer"
version = "0.2.0"

View File

@ -2,7 +2,7 @@
description = "Parity Ethereum client"
name = "parity-ethereum"
# NOTE Make sure to update util/version/Cargo.toml as well
version = "2.4.1"
version = "2.4.2"
license = "GPL-3.0"
authors = ["Parity Technologies <admin@parity.io>"]
@ -138,7 +138,8 @@ members = [
"util/triehash-ethereum",
"util/keccak-hasher",
"util/patricia-trie-ethereum",
"util/fastmap"
"util/fastmap",
"util/time-utils"
]
[patch.crates-io]

View File

@ -64,6 +64,7 @@ serde = "1.0"
serde_derive = "1.0"
stats = { path = "../util/stats" }
tempdir = {version="0.3", optional = true}
time-utils = { path = "../util/time-utils" }
trace-time = "0.1"
triehash-ethereum = { version = "0.2", path = "../util/triehash-ethereum" }
unexpected = { path = "../util/unexpected" }

View File

@ -47,6 +47,9 @@ use types::header::{Header, ExtendedHeader};
use types::ancestry_action::AncestryAction;
use unexpected::{Mismatch, OutOfBounds};
#[cfg(not(time_checked_add))]
use time_utils::CheckedSystemTime;
mod finality;
/// `AuthorityRound` params.
@ -570,8 +573,15 @@ fn verify_timestamp(step: &Step, header_step: u64) -> Result<(), BlockError> {
// NOTE This error might be returned only in early stage of verification (Stage 1).
// Returning it further won't recover the sync process.
trace!(target: "engine", "verify_timestamp: block too early");
let oob = oob.map(|n| SystemTime::now() + Duration::from_secs(n));
Err(BlockError::TemporarilyInvalid(oob).into())
let now = SystemTime::now();
let found = now.checked_add(Duration::from_secs(oob.found)).ok_or(BlockError::TimestampOverflow)?;
let max = oob.max.and_then(|m| now.checked_add(Duration::from_secs(m)));
let min = oob.min.and_then(|m| now.checked_add(Duration::from_secs(m)));
let new_oob = OutOfBounds { min, max, found };
Err(BlockError::TemporarilyInvalid(new_oob).into())
},
Ok(_) => Ok(()),
}
@ -607,6 +617,7 @@ fn combine_proofs(signal_number: BlockNumber, set_proof: &[u8], finality_proof:
stream.out()
}
fn destructure_proofs(combined: &[u8]) -> Result<(BlockNumber, &[u8], &[u8]), Error> {
let rlp = Rlp::new(combined);
Ok((
@ -622,7 +633,7 @@ trait AsMillis {
impl AsMillis for Duration {
fn as_millis(&self) -> u64 {
self.as_secs()*1_000 + (self.subsec_nanos()/1_000_000) as u64
self.as_secs() * 1_000 + (self.subsec_nanos() / 1_000_000) as u64
}
}

View File

@ -15,6 +15,7 @@
// along with Parity Ethereum. If not, see <http://www.gnu.org/licenses/>.
#![warn(missing_docs, unused_extern_crates)]
#![cfg_attr(feature = "time_checked_add", feature(time_checked_add))]
//! Ethcore library
//!
@ -149,6 +150,9 @@ extern crate fetch;
#[cfg(all(test, feature = "price-info"))]
extern crate parity_runtime;
#[cfg(not(time_checked_add))]
extern crate time_utils;
pub mod block;
pub mod builtin;
pub mod client;

View File

@ -40,24 +40,8 @@ use types::{BlockNumber, header::Header};
use types::transaction::SignedTransaction;
use verification::queue::kind::blocks::Unverified;
/// Returns `Ok<SystemTime>` when the result less or equal to `i32::max_value` to prevent `SystemTime` to panic because
/// it is platform specific, may be i32 or i64.
///
/// `Err<BlockError::TimestampOver` otherwise.
///
// FIXME: @niklasad1 - remove this when and use `SystemTime::checked_add`
// when https://github.com/rust-lang/rust/issues/55940 is stabilized.
fn timestamp_checked_add(sys: SystemTime, d2: Duration) -> Result<SystemTime, BlockError> {
let d1 = sys.duration_since(UNIX_EPOCH).map_err(|_| BlockError::TimestampOverflow)?;
let total_time = d1.checked_add(d2).ok_or(BlockError::TimestampOverflow)?;
if total_time.as_secs() <= i32::max_value() as u64 {
Ok(sys + d2)
} else {
Err(BlockError::TimestampOverflow)
}
}
#[cfg(not(time_checked_add))]
use time_utils::CheckedSystemTime;
/// Preprocessed block data gathered in `verify_block_unordered` call
pub struct PreverifiedBlock {
@ -323,9 +307,11 @@ pub fn verify_header_params(header: &Header, engine: &EthEngine, is_full: bool,
if is_full {
const ACCEPTABLE_DRIFT: Duration = Duration::from_secs(15);
// this will resist overflow until `year 2037`
let max_time = SystemTime::now() + ACCEPTABLE_DRIFT;
let invalid_threshold = max_time + ACCEPTABLE_DRIFT * 9;
let timestamp = timestamp_checked_add(UNIX_EPOCH, Duration::from_secs(header.timestamp()))?;
let timestamp = UNIX_EPOCH.checked_add(Duration::from_secs(header.timestamp()))
.ok_or(BlockError::TimestampOverflow)?;
if timestamp > invalid_threshold {
return Err(From::from(BlockError::InvalidTimestamp(OutOfBounds { max: Some(max_time), min: None, found: timestamp })))
@ -347,8 +333,11 @@ fn verify_parent(header: &Header, parent: &Header, engine: &EthEngine) -> Result
let gas_limit_divisor = engine.params().gas_limit_bound_divisor;
if !engine.is_timestamp_valid(header.timestamp(), parent.timestamp()) {
let min = timestamp_checked_add(SystemTime::now(), Duration::from_secs(parent.timestamp().saturating_add(1)))?;
let found = timestamp_checked_add(SystemTime::now(), Duration::from_secs(header.timestamp()))?;
let now = SystemTime::now();
let min = now.checked_add(Duration::from_secs(parent.timestamp().saturating_add(1)))
.ok_or(BlockError::TimestampOverflow)?;
let found = now.checked_add(Duration::from_secs(header.timestamp()))
.ok_or(BlockError::TimestampOverflow)?;
return Err(From::from(BlockError::InvalidTimestamp(OutOfBounds { max: None, min: Some(min), found })))
}
if header.number() != parent.number() + 1 {
@ -835,11 +824,4 @@ mod tests {
check_fail(unordered_test(&create_test_block_with_data(&header, &bad_transactions, &[]), &engine), TooManyTransactions(keypair.address()));
unordered_test(&create_test_block_with_data(&header, &good_transactions, &[]), &engine).unwrap();
}
#[test]
fn checked_add_systime_dur() {
assert!(timestamp_checked_add(UNIX_EPOCH, Duration::new(i32::max_value() as u64 + 1, 0)).is_err());
assert!(timestamp_checked_add(UNIX_EPOCH, Duration::new(i32::max_value() as u64, 0)).is_ok());
assert!(timestamp_checked_add(UNIX_EPOCH, Duration::new(i32::max_value() as u64 - 1, 1_000_000_000)).is_ok());
}
}

View File

@ -0,0 +1,9 @@
[package]
name = "time-utils"
version = "0.1.0"
authors = ["Parity Technologies <admin@parity.io>"]
description = "Time utilities for checked arithmetic"
license = "GPL3"
edition = "2018"
[dependencies]

View File

@ -0,0 +1,50 @@
use std::time::{Duration, SystemTime, UNIX_EPOCH};
/// Temporary trait for `checked operations` on SystemTime until these are available in standard library
pub trait CheckedSystemTime {
/// Returns `Some<SystemTime>` when the result less or equal to `i32::max_value` to prevent `SystemTime` to panic because
/// it is platform specific, possible representations are i32, i64, u64 and Duration. `None` otherwise
fn checked_add(self, _d: Duration) -> Option<SystemTime>;
/// Returns `Some<SystemTime>` when the result is successful and `None` when it is not
fn checked_sub(self, _d: Duration) -> Option<SystemTime>;
}
impl CheckedSystemTime for SystemTime {
fn checked_add(self, dur: Duration) -> Option<SystemTime> {
let this_dur = self.duration_since(UNIX_EPOCH).ok()?;
let total_time = this_dur.checked_add(dur)?;
if total_time.as_secs() <= i32::max_value() as u64 {
Some(self + dur)
} else {
None
}
}
fn checked_sub(self, dur: Duration) -> Option<SystemTime> {
let this_dur = self.duration_since(UNIX_EPOCH).ok()?;
let total_time = this_dur.checked_sub(dur)?;
if total_time.as_secs() <= i32::max_value() as u64 {
Some(self - dur)
} else {
None
}
}
}
#[cfg(test)]
mod tests {
#[test]
fn it_works() {
use super::CheckedSystemTime;
use std::time::{Duration, SystemTime, UNIX_EPOCH};
assert!(CheckedSystemTime::checked_add(UNIX_EPOCH, Duration::new(i32::max_value() as u64 + 1, 0)).is_none());
assert!(CheckedSystemTime::checked_add(UNIX_EPOCH, Duration::new(i32::max_value() as u64, 0)).is_some());
assert!(CheckedSystemTime::checked_add(UNIX_EPOCH, Duration::new(i32::max_value() as u64 - 1, 1_000_000_000)).is_some());
assert!(CheckedSystemTime::checked_sub(UNIX_EPOCH, Duration::from_secs(120)).is_none());
assert!(CheckedSystemTime::checked_sub(SystemTime::now(), Duration::from_secs(1000)).is_some());
}
}

View File

@ -3,7 +3,7 @@
[package]
name = "parity-version"
# NOTE: this value is used for Parity Ethereum version string (via env CARGO_PKG_VERSION)
version = "2.4.1"
version = "2.4.2"
authors = ["Parity Technologies <admin@parity.io>"]
build = "build.rs"

View File

@ -24,6 +24,7 @@ serde_json = "1.0"
slab = "0.3"
smallvec = "0.6"
tiny-keccak = "1.4"
time-utils = { path = "../util/time-utils" }
jsonrpc-core = "10.0.1"
jsonrpc-derive = "10.0.2"

View File

@ -17,6 +17,8 @@
//! Whisper P2P messaging system as a DevP2P subprotocol, with RPC and Rust
//! interface.
#![cfg_attr(feature = "time_checked_add", feature(time_checked_add))]
extern crate byteorder;
extern crate parity_crypto as crypto;
extern crate ethcore_network as network;
@ -46,6 +48,9 @@ extern crate log;
#[macro_use]
extern crate serde_derive;
#[cfg(not(time_checked_add))]
extern crate time_utils;
#[cfg(test)]
extern crate serde_json;

View File

@ -24,6 +24,9 @@ use rlp::{self, DecoderError, RlpStream, Rlp};
use smallvec::SmallVec;
use tiny_keccak::{keccak256, Keccak};
#[cfg(not(time_checked_add))]
use time_utils::CheckedSystemTime;
/// Work-factor proved. Takes 3 parameters: size of message, time to live,
/// and hash.
///
@ -117,6 +120,7 @@ pub enum Error {
EmptyTopics,
LivesTooLong,
IssuedInFuture,
TimestampOverflow,
ZeroTTL,
}
@ -133,6 +137,7 @@ impl fmt::Display for Error {
Error::LivesTooLong => write!(f, "Message claims to be issued before the unix epoch."),
Error::IssuedInFuture => write!(f, "Message issued in future."),
Error::ZeroTTL => write!(f, "Message live for zero time."),
Error::TimestampOverflow => write!(f, "Timestamp overflow"),
Error::EmptyTopics => write!(f, "Message has no topics."),
}
}
@ -226,10 +231,6 @@ impl rlp::Decodable for Envelope {
}
}
/// Error indicating no topics.
#[derive(Debug, Copy, Clone)]
pub struct EmptyTopics;
/// Message creation parameters.
/// Pass this to `Message::create` to make a message.
pub struct CreateParams {
@ -255,11 +256,11 @@ pub struct Message {
impl Message {
/// Create a message from creation parameters.
/// Panics if TTL is 0.
pub fn create(params: CreateParams) -> Result<Self, EmptyTopics> {
pub fn create(params: CreateParams) -> Result<Self, Error> {
use byteorder::{BigEndian, ByteOrder};
use rand::{Rng, SeedableRng, XorShiftRng};
if params.topics.is_empty() { return Err(EmptyTopics) }
if params.topics.is_empty() { return Err(Error::EmptyTopics) }
let mut rng = {
let mut thread_rng = ::rand::thread_rng();
@ -270,7 +271,8 @@ impl Message {
assert!(params.ttl > 0);
let expiry = {
let after_mining = SystemTime::now() + Duration::from_millis(params.work);
let after_mining = SystemTime::now().checked_sub(Duration::from_millis(params.work))
.ok_or(Error::TimestampOverflow)?;
let since_epoch = after_mining.duration_since(time::UNIX_EPOCH)
.expect("time after now is after unix epoch; qed");
@ -357,7 +359,10 @@ impl Message {
(envelope.expiry - envelope.ttl).saturating_sub(LEEWAY_SECONDS)
);
if time::UNIX_EPOCH + issue_time_adjusted > now {
let issue_time_adjusted = time::UNIX_EPOCH.checked_add(issue_time_adjusted)
.ok_or(Error::TimestampOverflow)?;
if issue_time_adjusted > now {
return Err(Error::IssuedInFuture);
}
@ -400,8 +405,8 @@ impl Message {
}
/// Get the expiry time.
pub fn expiry(&self) -> SystemTime {
time::UNIX_EPOCH + Duration::from_secs(self.envelope.expiry)
pub fn expiry(&self) -> Option<SystemTime> {
time::UNIX_EPOCH.checked_add(Duration::from_secs(self.envelope.expiry))
}
/// Get the topics.

View File

@ -223,7 +223,10 @@ impl Messages {
}
}
let expiry = message.expiry();
let expiry = match message.expiry() {
Some(time) => time,
_ => return false,
};
self.cumulative_size += message.encoded_size();
@ -232,8 +235,8 @@ impl Messages {
let sorted_entry = SortedEntry {
slab_id: id,
work_proved: work_proved,
expiry: expiry,
work_proved,
expiry,
};
match self.sorted.binary_search(&sorted_entry) {