diff --git a/Cargo.lock b/Cargo.lock index 291ba89d3..79e018c7e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5,7 +5,7 @@ name = "account-db" version = "0.1.0" dependencies = [ "ethereum-types 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)", - "hash-db 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)", + "hash-db 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)", "keccak-hash 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", "keccak-hasher 0.1.1", "kvdb 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", @@ -20,7 +20,7 @@ dependencies = [ "common-types 0.1.0", "derive_more 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)", "ethereum-types 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)", - "hash-db 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)", + "hash-db 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)", "journaldb 0.2.0", "keccak-hash 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", "keccak-hasher 0.1.1", @@ -37,7 +37,7 @@ dependencies = [ "rlp_compress 0.1.0", "serde 1.0.99 (registry+https://github.com/rust-lang/crates.io-index)", "trace 0.1.0", - "trie-db 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)", + "trie-db 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)", "trie-vm-factories 0.1.0", ] @@ -82,6 +82,14 @@ dependencies = [ "stream-cipher 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "ahash" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "const-random 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "aho-corasick" version = "0.6.8" @@ -598,6 +606,7 @@ dependencies = [ "parity-crypto 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", "parity-snappy 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", "parity-util-mem 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", + "parking_lot 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", "patricia-trie-ethereum 0.1.0", "rlp 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", "rlp_derive 0.1.0", @@ -606,6 +615,24 @@ dependencies = [ "vm 0.1.0", ] +[[package]] +name = "const-random" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "const-random-macro 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)", + "proc-macro-hack 0.5.9 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "const-random-macro" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "proc-macro-hack 0.5.9 (registry+https://github.com/rust-lang/crates.io-index)", + "rand 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "criterion" version = "0.3.0" @@ -645,7 +672,7 @@ version = "0.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "crossbeam-epoch 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)", - "crossbeam-utils 0.6.5 (registry+https://github.com/rust-lang/crates.io-index)", + "crossbeam-utils 0.6.6 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -654,7 +681,7 @@ version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "crossbeam-epoch 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)", - "crossbeam-utils 0.6.5 (registry+https://github.com/rust-lang/crates.io-index)", + "crossbeam-utils 0.6.6 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -664,7 +691,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "arrayvec 0.4.11 (registry+https://github.com/rust-lang/crates.io-index)", "cfg-if 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)", - "crossbeam-utils 0.6.5 (registry+https://github.com/rust-lang/crates.io-index)", + "crossbeam-utils 0.6.6 (registry+https://github.com/rust-lang/crates.io-index)", "lazy_static 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)", "memoffset 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)", "scopeguard 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", @@ -675,7 +702,7 @@ name = "crossbeam-queue" version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "crossbeam-utils 0.6.5 (registry+https://github.com/rust-lang/crates.io-index)", + "crossbeam-utils 0.6.6 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -685,7 +712,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "crossbeam-utils" -version = "0.6.5" +version = "0.6.6" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "cfg-if 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)", @@ -1043,7 +1070,7 @@ dependencies = [ "executive-state 0.1.0", "fetch 0.1.0", "futures 0.1.29 (registry+https://github.com/rust-lang/crates.io-index)", - "hash-db 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)", + "hash-db 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)", "itertools 0.5.10 (registry+https://github.com/rust-lang/crates.io-index)", "journaldb 0.2.0", "keccak-hash 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", @@ -1067,6 +1094,7 @@ dependencies = [ "registrar 0.0.1", "rlp 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", "rustc-hex 2.0.1 (registry+https://github.com/rust-lang/crates.io-index)", + "scopeguard 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", "serde 1.0.99 (registry+https://github.com/rust-lang/crates.io-index)", "serde_derive 1.0.89 (registry+https://github.com/rust-lang/crates.io-index)", "serde_json 1.0.40 (registry+https://github.com/rust-lang/crates.io-index)", @@ -1077,7 +1105,7 @@ dependencies = [ "tempdir 0.3.7 (registry+https://github.com/rust-lang/crates.io-index)", "trace 0.1.0", "trace-time 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", - "trie-db 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)", + "trie-db 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)", "trie-standardmap 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)", "trie-vm-factories 0.1.0", "triehash-ethereum 0.2.0", @@ -1216,7 +1244,7 @@ dependencies = [ "failsafe 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)", "fastmap 0.1.0", "futures 0.1.29 (registry+https://github.com/rust-lang/crates.io-index)", - "hash-db 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)", + "hash-db 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)", "journaldb 0.2.0", "keccak-hash 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", "keccak-hasher 0.1.1", @@ -1239,7 +1267,7 @@ dependencies = [ "spec 0.1.0", "stats 0.1.0", "tempdir 0.3.7 (registry+https://github.com/rust-lang/crates.io-index)", - "trie-db 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)", + "trie-db 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)", "triehash-ethereum 0.2.0", "verification 0.1.0", "vm 0.1.0", @@ -1368,7 +1396,7 @@ dependencies = [ "ethjson 0.1.0", "fetch 0.1.0", "futures 0.1.29 (registry+https://github.com/rust-lang/crates.io-index)", - "hash-db 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)", + "hash-db 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)", "journaldb 0.2.0", "keccak-hash 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", "keccak-hasher 0.1.1", @@ -1393,7 +1421,7 @@ dependencies = [ "tiny-keccak 1.5.0 (registry+https://github.com/rust-lang/crates.io-index)", "trace 0.1.0", "transaction-pool 2.0.1 (registry+https://github.com/rust-lang/crates.io-index)", - "trie-db 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)", + "trie-db 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)", "url 2.1.0 (registry+https://github.com/rust-lang/crates.io-index)", "vm 0.1.0", ] @@ -1668,7 +1696,7 @@ dependencies = [ "ethcore 1.12.0", "ethereum-types 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)", "evm 0.1.0", - "hash-db 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)", + "hash-db 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)", "keccak-hash 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", "keccak-hasher 0.1.1", "kvdb 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", @@ -1681,7 +1709,7 @@ dependencies = [ "rustc-hex 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", "spec 0.1.0", "trace 0.1.0", - "trie-db 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)", + "trie-db 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)", "trie-vm-factories 0.1.0", "vm 0.1.0", ] @@ -1909,7 +1937,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "hash-db" -version = "0.15.0" +version = "0.15.2" source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] @@ -1920,6 +1948,15 @@ dependencies = [ "crunchy 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "hashbrown" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "ahash 0.2.16 (registry+https://github.com/rust-lang/crates.io-index)", + "autocfg 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "hashmap_core" version = "0.1.10" @@ -2224,7 +2261,7 @@ dependencies = [ "env_logger 0.5.13 (registry+https://github.com/rust-lang/crates.io-index)", "ethereum-types 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)", "fastmap 0.1.0", - "hash-db 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)", + "hash-db 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)", "keccak-hash 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", "keccak-hasher 0.1.1", "kvdb 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", @@ -2352,7 +2389,7 @@ name = "keccak-hasher" version = "0.1.1" dependencies = [ "ethereum-types 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)", - "hash-db 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)", + "hash-db 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)", "plain_hasher 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", "tiny-keccak 1.5.0 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -2362,7 +2399,7 @@ name = "keccak-hasher" version = "0.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "hash-db 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)", + "hash-db 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)", "hash256-std-hasher 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)", "tiny-keccak 1.5.0 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -2525,7 +2562,7 @@ dependencies = [ "account-state 0.1.0", "client-traits 0.1.0", "common-types 0.1.0", - "crossbeam-utils 0.6.5 (registry+https://github.com/rust-lang/crates.io-index)", + "crossbeam-utils 0.6.6 (registry+https://github.com/rust-lang/crates.io-index)", "ethabi 9.0.1 (registry+https://github.com/rust-lang/crates.io-index)", "ethabi-contract 9.0.0 (registry+https://github.com/rust-lang/crates.io-index)", "ethabi-derive 9.0.1 (registry+https://github.com/rust-lang/crates.io-index)", @@ -2610,7 +2647,7 @@ name = "memory-db" version = "0.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "hash-db 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)", + "hash-db 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)", "hashmap_core 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)", "parity-util-mem 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -3393,14 +3430,14 @@ version = "0.1.0" dependencies = [ "elastic-array 0.10.2 (registry+https://github.com/rust-lang/crates.io-index)", "ethereum-types 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)", - "hash-db 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)", + "hash-db 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)", "journaldb 0.2.0", "keccak-hash 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", "keccak-hasher 0.1.1", "memory-db 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)", "parity-bytes 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", "rlp 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", - "trie-db 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)", + "trie-db 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -3486,7 +3523,7 @@ dependencies = [ "common-types 0.1.0", "ethereum-types 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)", "ethjson 0.1.0", - "hash-db 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)", + "hash-db 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)", "itertools 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)", "keccak-hash 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", "keccak-hasher 0.1.1", @@ -3498,7 +3535,7 @@ dependencies = [ "rlp 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", "rustc-hex 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", "serde 1.0.99 (registry+https://github.com/rust-lang/crates.io-index)", - "trie-db 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)", + "trie-db 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)", "triehash-ethereum 0.2.0", ] @@ -3854,7 +3891,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "crossbeam-deque 0.6.3 (registry+https://github.com/rust-lang/crates.io-index)", "crossbeam-queue 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", - "crossbeam-utils 0.6.5 (registry+https://github.com/rust-lang/crates.io-index)", + "crossbeam-utils 0.6.6 (registry+https://github.com/rust-lang/crates.io-index)", "lazy_static 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)", "num_cpus 1.10.1 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -4216,7 +4253,7 @@ dependencies = [ "account-state 0.1.0", "client-traits 0.1.0", "common-types 0.1.0", - "crossbeam-utils 0.6.5 (registry+https://github.com/rust-lang/crates.io-index)", + "crossbeam-utils 0.6.6 (registry+https://github.com/rust-lang/crates.io-index)", "engine 0.1.0", "env_logger 0.5.13 (registry+https://github.com/rust-lang/crates.io-index)", "ethabi 9.0.1 (registry+https://github.com/rust-lang/crates.io-index)", @@ -4230,7 +4267,7 @@ dependencies = [ "ethcore-io 1.12.0", "ethereum-types 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)", "ethkey 0.4.0", - "hash-db 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)", + "hash-db 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)", "itertools 0.5.10 (registry+https://github.com/rust-lang/crates.io-index)", "journaldb 0.2.0", "keccak-hash 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", @@ -4248,11 +4285,12 @@ dependencies = [ "rand_xorshift 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", "rlp 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", "rlp_derive 0.1.0", + "scopeguard 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", "snapshot-tests 0.1.0", "spec 0.1.0", "state-db 0.1.0", "tempdir 0.3.7 (registry+https://github.com/rust-lang/crates.io-index)", - "trie-db 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)", + "trie-db 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)", "trie-standardmap 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)", "triehash-ethereum 0.2.0", ] @@ -4276,7 +4314,7 @@ dependencies = [ "ethcore-db 0.1.0", "ethcore-io 1.12.0", "ethereum-types 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)", - "hash-db 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)", + "hash-db 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)", "journaldb 0.2.0", "keccak-hash 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", "keccak-hasher 0.1.1", @@ -4295,7 +4333,7 @@ dependencies = [ "snapshot 0.1.0", "spec 0.1.0", "tempdir 0.3.7 (registry+https://github.com/rust-lang/crates.io-index)", - "trie-db 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)", + "trie-db 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)", "trie-standardmap 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)", "triehash-ethereum 0.2.0", ] @@ -4330,7 +4368,7 @@ dependencies = [ "ethjson 0.1.0", "evm 0.1.0", "executive-state 0.1.0", - "hash-db 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)", + "hash-db 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)", "instant-seal 0.1.0", "journaldb 0.2.0", "keccak-hash 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", @@ -4368,7 +4406,7 @@ dependencies = [ "ethcore-bloom-journal 0.1.0", "ethcore-db 0.1.0", "ethereum-types 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)", - "hash-db 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)", + "hash-db 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)", "journaldb 0.2.0", "keccak-hash 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", "keccak-hasher 0.1.1", @@ -4633,7 +4671,7 @@ name = "tokio-executor" version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "crossbeam-utils 0.6.5 (registry+https://github.com/rust-lang/crates.io-index)", + "crossbeam-utils 0.6.6 (registry+https://github.com/rust-lang/crates.io-index)", "futures 0.1.29 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -4734,7 +4772,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "crossbeam-deque 0.7.1 (registry+https://github.com/rust-lang/crates.io-index)", "crossbeam-queue 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", - "crossbeam-utils 0.6.5 (registry+https://github.com/rust-lang/crates.io-index)", + "crossbeam-utils 0.6.6 (registry+https://github.com/rust-lang/crates.io-index)", "futures 0.1.29 (registry+https://github.com/rust-lang/crates.io-index)", "lazy_static 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", @@ -4757,7 +4795,7 @@ name = "tokio-timer" version = "0.2.11" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "crossbeam-utils 0.6.5 (registry+https://github.com/rust-lang/crates.io-index)", + "crossbeam-utils 0.6.6 (registry+https://github.com/rust-lang/crates.io-index)", "futures 0.1.29 (registry+https://github.com/rust-lang/crates.io-index)", "slab 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)", "tokio-executor 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)", @@ -4862,12 +4900,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "trie-db" -version = "0.15.0" +version = "0.15.2" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "elastic-array 0.10.2 (registry+https://github.com/rust-lang/crates.io-index)", - "hash-db 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)", - "hashmap_core 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)", + "hash-db 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)", + "hashbrown 0.6.1 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", "rand 0.6.1 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -4877,7 +4915,7 @@ name = "trie-standardmap" version = "0.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "hash-db 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)", + "hash-db 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)", "keccak-hasher 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -4889,7 +4927,7 @@ dependencies = [ "evm 0.1.0", "keccak-hasher 0.1.1", "patricia-trie-ethereum 0.1.0", - "trie-db 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)", + "trie-db 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)", "vm 0.1.0", "wasm 0.1.0", ] @@ -4899,7 +4937,7 @@ name = "triehash" version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "hash-db 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)", + "hash-db 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)", "rlp 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -5344,6 +5382,7 @@ dependencies = [ "checksum aes-ctr 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "d2e5b0458ea3beae0d1d8c0f3946564f8e10f90646cf78c06b4351052058d1ee" "checksum aes-soft 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "cfd7e7ae3f9a1fb5c03b389fc6bb9a51400d0c13053f0dca698c832bfd893a0d" "checksum aesni 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)" = "2f70a6b5f971e473091ab7cfb5ffac6cde81666c4556751d8d5620ead8abf100" +"checksum ahash 0.2.16 (registry+https://github.com/rust-lang/crates.io-index)" = "b35dfc96a657c1842b4eb73180b65e37152d4b94d0eb5cb51708aee7826950b4" "checksum aho-corasick 0.6.8 (registry+https://github.com/rust-lang/crates.io-index)" = "68f56c7353e5a9547cbd76ed90f7bb5ffc3ba09d4ea9bd1d8c06c8b1142eeb5a" "checksum aho-corasick 0.7.6 (registry+https://github.com/rust-lang/crates.io-index)" = "58fb5e95d83b38284460a5fda7d6470aa0b8844d283a0b614b8535e880800d2d" "checksum ansi_term 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ee49baf6cb617b853aa8d93bf420db2383fab46d314482ca2803b40d5fde979b" @@ -5390,6 +5429,8 @@ dependencies = [ "checksum cloudabi 0.0.3 (registry+https://github.com/rust-lang/crates.io-index)" = "ddfc5b9aa5d4507acaf872de71051dfd0e309860e88966e1051e462a077aac4f" "checksum cmake 0.1.35 (registry+https://github.com/rust-lang/crates.io-index)" = "6ec65ee4f9c9d16f335091d23693457ed4928657ba4982289d7fafee03bc614a" "checksum combine 3.6.1 (registry+https://github.com/rust-lang/crates.io-index)" = "fc1d011beeed29187b8db2ac3925c8dd4d3e87db463dc9d2d2833985388fc5bc" +"checksum const-random 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)" = "7b641a8c9867e341f3295564203b1c250eb8ce6cb6126e007941f78c4d2ed7fe" +"checksum const-random-macro 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)" = "c750ec12b83377637110d5a57f5ae08e895b06c4b16e2bdbf1a94ef717428c59" "checksum criterion 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "938703e165481c8d612ea3479ac8342e5615185db37765162e762ec3523e2fc6" "checksum criterion-plot 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "eccdc6ce8bbe352ca89025bee672aa6d24f4eb8c53e3a8b5d1bc58011da072a2" "checksum crossbeam-deque 0.6.3 (registry+https://github.com/rust-lang/crates.io-index)" = "05e44b8cf3e1a625844d1750e1f7820da46044ff6d28f4d43e455ba3e5bb2c13" @@ -5397,7 +5438,7 @@ dependencies = [ "checksum crossbeam-epoch 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)" = "fedcd6772e37f3da2a9af9bf12ebe046c0dfe657992377b4df982a2b54cd37a9" "checksum crossbeam-queue 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "7c979cd6cfe72335896575c6b5688da489e420d36a27a0b9eb0c73db574b4a4b" "checksum crossbeam-utils 0.5.0 (registry+https://github.com/rust-lang/crates.io-index)" = "677d453a17e8bd2b913fa38e8b9cf04bcdbb5be790aa294f2389661d72036015" -"checksum crossbeam-utils 0.6.5 (registry+https://github.com/rust-lang/crates.io-index)" = "f8306fcef4a7b563b76b7dd949ca48f52bc1141aa067d2ea09565f3e2652aa5c" +"checksum crossbeam-utils 0.6.6 (registry+https://github.com/rust-lang/crates.io-index)" = "04973fa96e96579258a5091af6003abde64af786b860f18622b82e026cca60e6" "checksum crunchy 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)" = "a2f4a431c5c9f662e1200b7c7f02c34e91361150e382089a8f2dec3ba680cbda" "checksum crunchy 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "7a81dae078cea95a014a339291cec439d2f232ebe854a9d672b796c6afafa9b7" "checksum crypto-mac 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "4434400df11d95d556bac068ddfedd482915eb18fe8bea89bc80b6e4b1c179e5" @@ -5447,8 +5488,9 @@ dependencies = [ "checksum globset 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)" = "4743617a7464bbda3c8aec8558ff2f9429047e025771037df561d383337ff865" "checksum h2 0.1.26 (registry+https://github.com/rust-lang/crates.io-index)" = "a5b34c246847f938a410a03c5458c7fee2274436675e76d8b903c08efc29c462" "checksum hamming 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)" = "65043da274378d68241eb9a8f8f8aa54e349136f7b8e12f63e3ef44043cc30e1" -"checksum hash-db 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)" = "32c87fec93c4a2d264483ef843ac1930ae7c7999d97d73721305a5188b4c23a4" +"checksum hash-db 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)" = "d23bd4e7b5eda0d0f3a307e8b381fdc8ba9000f26fbe912250c0a4cc3956364a" "checksum hash256-std-hasher 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)" = "16293646125e09e5bc216d9f73fa81ab31c4f97007d56c036bbf15a58e970540" +"checksum hashbrown 0.6.1 (registry+https://github.com/rust-lang/crates.io-index)" = "6587d09be37fb98a11cb08b9000a3f592451c1b1b613ca69d949160e313a430a" "checksum hashmap_core 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)" = "8e04cb7a5051270ef3fa79f8c7604d581ecfa73d520e74f554e45541c4b5881a" "checksum heapsize 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)" = "1679e6ea370dee694f91f1dc469bf94cf8f52051d147aec3e1f9497c6fc22461" "checksum heck 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ea04fa3ead4e05e51a7c806fc07271fdbde4e246a6c6d1efd52e72230b771b82" @@ -5698,7 +5740,7 @@ dependencies = [ "checksum trace-time 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "dbe82f2f0bf1991e163e757baf044282823155dd326e70f44ce2186c3c320cc9" "checksum transaction-pool 2.0.1 (registry+https://github.com/rust-lang/crates.io-index)" = "454adc482e32785c3beab9415dd0f3c689f29cc2d16717eb62f6a784d53544b4" "checksum transient-hashmap 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)" = "aeb4b191d033a35edfce392a38cdcf9790b6cebcb30fa690c312c29da4dc433e" -"checksum trie-db 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)" = "9b65d609ae631d808c6c1cc23a622733d5a0b66a7d67e9f5cd5171562a1f4cb5" +"checksum trie-db 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)" = "d0b62d27e8aa1c07414549ac872480ac82380bab39e730242ab08d82d7cc098a" "checksum trie-standardmap 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)" = "64fda153c00484d640bc91334624be22ead0e5baca917d9fd53ff29bdebcf9b2" "checksum triehash 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)" = "61d0a66fa2412c7eb7816640e8ea14cf6bd63b6c824e72315b6ca76d33851134" "checksum try-lock 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "e604eb7b43c06650e854be16a2a03155743d3752dd1c943f6829e26b7a36e382" diff --git a/ethcore/Cargo.toml b/ethcore/Cargo.toml index f036010c5..ffbcdeaf6 100644 --- a/ethcore/Cargo.toml +++ b/ethcore/Cargo.toml @@ -51,6 +51,7 @@ rayon = "1.1" registrar = { path = "../util/registrar" } rlp = "0.4.0" rustc-hex = "2" +scopeguard = "1.0.0" serde = "1.0" serde_derive = "1.0" snapshot = { path = "snapshot" } diff --git a/ethcore/service/src/service.rs b/ethcore/service/src/service.rs index 06f1efe76..597ee6d3d 100644 --- a/ethcore/service/src/service.rs +++ b/ethcore/service/src/service.rs @@ -270,14 +270,12 @@ where ClientIoMessage::TakeSnapshot(num) => { let client = self.client.clone(); let snapshot = self.snapshot.clone(); - let res = thread::Builder::new().name("Periodic Snapshot".into()).spawn(move || { if let Err(e) = snapshot.take_snapshot(&*client, num) { match e { EthcoreError::Snapshot(SnapshotError::SnapshotAborted) => info!("Snapshot aborted"), _ => warn!("Failed to take snapshot at block #{}: {}", num, e), } - } }); diff --git a/ethcore/snapshot/Cargo.toml b/ethcore/snapshot/Cargo.toml index 882c9c5c9..00b9ab6ea 100644 --- a/ethcore/snapshot/Cargo.toml +++ b/ethcore/snapshot/Cargo.toml @@ -33,6 +33,7 @@ rand_xorshift = "0.2" parking_lot = "0.9" rlp = "0.4.2" rlp_derive = { path = "../../util/rlp-derive" } +scopeguard = "1.0.0" snappy = { package = "parity-snappy", version ="0.1.0" } state-db = { path = "../state-db" } trie-db = "0.15.0" diff --git a/ethcore/snapshot/snapshot-tests/src/account.rs b/ethcore/snapshot/snapshot-tests/src/account.rs index 09707c311..fd1e5941b 100644 --- a/ethcore/snapshot/snapshot-tests/src/account.rs +++ b/ethcore/snapshot/snapshot-tests/src/account.rs @@ -48,7 +48,7 @@ fn encoding_basic() { let thin_rlp = ::rlp::encode(&account); assert_eq!(::rlp::decode::(&thin_rlp).unwrap(), account); - let p = Progress::default(); + let p = Progress::new(); let fat_rlps = to_fat_rlps(&keccak(&addr), &account, &AccountDB::from_hash(db.as_hash_db(), keccak(addr)), &mut Default::default(), usize::max_value(), usize::max_value(), &p).unwrap(); let fat_rlp = Rlp::new(&fat_rlps[0]).at(1).unwrap(); assert_eq!(from_fat_rlp(&mut AccountDBMut::from_hash(db.as_hash_db_mut(), keccak(addr)), fat_rlp, H256::zero()).unwrap().0, account); @@ -69,7 +69,7 @@ fn encoding_version() { let thin_rlp = ::rlp::encode(&account); assert_eq!(::rlp::decode::(&thin_rlp).unwrap(), account); - let p = Progress::default(); + let p = Progress::new(); let fat_rlps = to_fat_rlps(&keccak(&addr), &account, &AccountDB::from_hash(db.as_hash_db(), keccak(addr)), &mut Default::default(), usize::max_value(), usize::max_value(), &p).unwrap(); let fat_rlp = Rlp::new(&fat_rlps[0]).at(1).unwrap(); assert_eq!(from_fat_rlp(&mut AccountDBMut::from_hash(db.as_hash_db_mut(), keccak(addr)), fat_rlp, H256::zero()).unwrap().0, account); @@ -96,7 +96,7 @@ fn encoding_storage() { let thin_rlp = ::rlp::encode(&account); assert_eq!(::rlp::decode::(&thin_rlp).unwrap(), account); - let p = Progress::default(); + let p = Progress::new(); let fat_rlp = to_fat_rlps(&keccak(&addr), &account, &AccountDB::from_hash(db.as_hash_db(), keccak(addr)), &mut Default::default(), usize::max_value(), usize::max_value(), &p).unwrap(); let fat_rlp = Rlp::new(&fat_rlp[0]).at(1).unwrap(); @@ -124,7 +124,7 @@ fn encoding_storage_split() { let thin_rlp = ::rlp::encode(&account); assert_eq!(::rlp::decode::(&thin_rlp).unwrap(), account); - let p = Progress::default(); + let p = Progress::new(); let fat_rlps = to_fat_rlps(&keccak(addr), &account, &AccountDB::from_hash(db.as_hash_db(), keccak(addr)), &mut Default::default(), 500, 1000, &p).unwrap(); let mut root = KECCAK_NULL_RLP; let mut restored_account = None; @@ -170,8 +170,8 @@ fn encoding_code() { }; let mut used_code = HashSet::new(); - let p1 = Progress::default(); - let p2 = Progress::default(); + let p1 = Progress::new(); + let p2 = Progress::new(); let fat_rlp1 = to_fat_rlps(&keccak(&addr1), &account1, &AccountDB::from_hash(db.as_hash_db(), keccak(addr1)), &mut used_code, usize::max_value(), usize::max_value(), &p1).unwrap(); let fat_rlp2 = to_fat_rlps(&keccak(&addr2), &account2, &AccountDB::from_hash(db.as_hash_db(), keccak(addr2)), &mut used_code, usize::max_value(), usize::max_value(), &p2).unwrap(); assert_eq!(used_code.len(), 1); diff --git a/ethcore/snapshot/snapshot-tests/src/helpers.rs b/ethcore/snapshot/snapshot-tests/src/helpers.rs index afc497906..0c419dd71 100644 --- a/ethcore/snapshot/snapshot-tests/src/helpers.rs +++ b/ethcore/snapshot/snapshot-tests/src/helpers.rs @@ -27,7 +27,8 @@ use client_traits::ChainInfo; use common_types::{ ids::BlockId, basic_account::BasicAccount, - errors::EthcoreError + errors::EthcoreError, + snapshot::Progress, }; use engine::Engine; use ethcore::client::Client; @@ -145,7 +146,7 @@ pub fn snap(client: &Client) -> (Box, TempDir) { let tempdir = TempDir::new("").unwrap(); let path = tempdir.path().join("file"); let writer = PackedWriter::new(&path).unwrap(); - let progress = Default::default(); + let progress = Progress::new(); let hash = client.chain_info().best_block_hash; client.take_snapshot(writer, BlockId::Hash(hash), &progress).unwrap(); diff --git a/ethcore/snapshot/snapshot-tests/src/proof_of_work.rs b/ethcore/snapshot/snapshot-tests/src/proof_of_work.rs index 81dcfe2cd..9d01432ff 100644 --- a/ethcore/snapshot/snapshot-tests/src/proof_of_work.rs +++ b/ethcore/snapshot/snapshot-tests/src/proof_of_work.rs @@ -74,7 +74,7 @@ fn chunk_and_restore(amount: u64) { &bc, best_hash, &writer, - &Progress::default() + &Progress::new() ).unwrap(); let manifest = ManifestData { diff --git a/ethcore/snapshot/snapshot-tests/src/service.rs b/ethcore/snapshot/snapshot-tests/src/service.rs index b2e7a9825..ebedd5472 100644 --- a/ethcore/snapshot/snapshot-tests/src/service.rs +++ b/ethcore/snapshot/snapshot-tests/src/service.rs @@ -278,7 +278,7 @@ fn keep_ancient_blocks() { &bc, best_hash, &writer, - &Progress::default() + &Progress::new() ).unwrap(); let state_db = client.state_db().journal_db().boxed_clone(); let start_header = bc.block_header_data(&best_hash).unwrap(); @@ -287,7 +287,7 @@ fn keep_ancient_blocks() { state_db.as_hash_db(), &state_root, &writer, - &Progress::default(), + &Progress::new(), None, 0 ).unwrap(); diff --git a/ethcore/snapshot/snapshot-tests/src/state.rs b/ethcore/snapshot/snapshot-tests/src/state.rs index 4cccd0d82..ef9057964 100644 --- a/ethcore/snapshot/snapshot-tests/src/state.rs +++ b/ethcore/snapshot/snapshot-tests/src/state.rs @@ -62,7 +62,7 @@ fn snap_and_restore() { let mut state_hashes = Vec::new(); for part in 0..SNAPSHOT_SUBPARTS { - let mut hashes = chunk_state(&old_db, &state_root, &writer, &Progress::default(), Some(part), 0).unwrap(); + let mut hashes = chunk_state(&old_db, &state_root, &writer, &Progress::new(), Some(part), 0).unwrap(); state_hashes.append(&mut hashes); } @@ -133,7 +133,7 @@ fn get_code_from_prev_chunk() { let mut make_chunk = |acc, hash| { let mut db = journaldb::new_memory_db(); AccountDBMut::from_hash(&mut db, hash).insert(EMPTY_PREFIX, &code[..]); - let p = Progress::default(); + let p = Progress::new(); let fat_rlp = to_fat_rlps(&hash, &acc, &AccountDB::from_hash(&db, hash), &mut used_code, usize::max_value(), usize::max_value(), &p).unwrap(); let mut stream = RlpStream::new_list(1); stream.append_raw(&fat_rlp[0], 1); @@ -178,7 +178,7 @@ fn checks_flag() { let state_root = producer.state_root(); let writer = Mutex::new(PackedWriter::new(&snap_file).unwrap()); - let state_hashes = chunk_state(&old_db, &state_root, &writer, &Progress::default(), None, 0).unwrap(); + let state_hashes = chunk_state(&old_db, &state_root, &writer, &Progress::new(), None, 0).unwrap(); writer.into_inner().finish(ManifestData { version: 2, diff --git a/ethcore/snapshot/snapshot-tests/src/watcher.rs b/ethcore/snapshot/snapshot-tests/src/watcher.rs index d7cf35ced..95e932a2a 100644 --- a/ethcore/snapshot/snapshot-tests/src/watcher.rs +++ b/ethcore/snapshot/snapshot-tests/src/watcher.rs @@ -41,8 +41,8 @@ impl Oracle for TestOracle { struct TestBroadcast(Option); impl Broadcast for TestBroadcast { - fn take_at(&self, num: Option) { - if num != self.0 { + fn request_snapshot_at(&self, num: u64) { + if Some(num) != self.0 { panic!("Watcher broadcast wrong number. Expected {:?}, found {:?}", self.0, num); } } diff --git a/ethcore/snapshot/src/consensus/work.rs b/ethcore/snapshot/src/consensus/work.rs index fede7e9d6..819e95ca0 100644 --- a/ethcore/snapshot/src/consensus/work.rs +++ b/ethcore/snapshot/src/consensus/work.rs @@ -178,7 +178,7 @@ impl<'a> PowWorker<'a> { let parent_hash = last_header.parent_hash(); let parent_total_difficulty = last_details.total_difficulty - last_header.difficulty(); - trace!(target: "snapshot", "parent last written block: {}", parent_hash); + trace!(target: "snapshot", "parent last written block: #{}/{}", parent_number, parent_hash); let num_entries = self.rlps.len(); let mut rlp_stream = RlpStream::new_list(3 + num_entries); diff --git a/ethcore/snapshot/src/lib.rs b/ethcore/snapshot/src/lib.rs index e6797a661..406aeeddb 100644 --- a/ethcore/snapshot/src/lib.rs +++ b/ethcore/snapshot/src/lib.rs @@ -129,35 +129,34 @@ pub fn take_snapshot( let state_root = start_header.state_root(); let block_number = start_header.number(); - info!("Taking snapshot starting at block {}", block_number); - + info!("Taking snapshot starting at block #{}/{:?}", block_number, block_hash); let version = chunker.current_version(); let writer = Mutex::new(writer); let (state_hashes, block_hashes) = thread::scope(|scope| -> Result<(Vec, Vec), Error> { let writer = &writer; - let block_guard = scope.spawn(move |_| { + let tb = scope.builder().name("Snapshot Worker - Blocks".to_string()); + let block_guard = tb.spawn(move |_| { chunk_secondary(chunker, chain, block_hash, writer, p) - }); + })?; // The number of threads must be between 1 and SNAPSHOT_SUBPARTS assert!(processing_threads >= 1, "Cannot use less than 1 threads for creating snapshots"); - let num_threads: usize = cmp::min(processing_threads, SNAPSHOT_SUBPARTS); + let num_threads = cmp::min(processing_threads, SNAPSHOT_SUBPARTS); info!(target: "snapshot", "Using {} threads for Snapshot creation.", num_threads); - let mut state_guards = Vec::with_capacity(num_threads as usize); + let mut state_guards = Vec::with_capacity(num_threads); for thread_idx in 0..num_threads { - let state_guard = scope.spawn(move |_| -> Result, Error> { + let tb = scope.builder().name(format!("Snapshot Worker #{} - State", thread_idx).to_string()); + let state_guard = tb.spawn(move |_| -> Result, Error> { let mut chunk_hashes = Vec::new(); - for part in (thread_idx..SNAPSHOT_SUBPARTS).step_by(num_threads) { - debug!(target: "snapshot", "Chunking part {} in thread {}", part, thread_idx); + debug!(target: "snapshot", "Chunking part {} of the state at {} in thread {}", part, block_number, thread_idx); let mut hashes = chunk_state(state_db, &state_root, writer, p, Some(part), thread_idx)?; chunk_hashes.append(&mut hashes); } - Ok(chunk_hashes) - }); + })?; state_guards.push(state_guard); } @@ -169,7 +168,8 @@ pub fn take_snapshot( state_hashes.extend(part_state_hashes); } - debug!(target: "snapshot", "Took a snapshot of {} accounts", p.accounts.load(Ordering::SeqCst)); + info!("Took a snapshot at #{} of {} accounts", block_number, p.accounts()); + Ok((state_hashes, block_hashes)) }).expect("Sub-thread never panics; qed")?; @@ -218,7 +218,7 @@ pub fn chunk_secondary<'a>( trace!(target: "snapshot", "wrote secondary chunk. hash: {:x}, size: {}, uncompressed size: {}", hash, size, raw_data.len()); - progress.size.fetch_add(size as u64, Ordering::SeqCst); + progress.update(0, size); chunk_hashes.push(hash); Ok(()) }; @@ -275,8 +275,7 @@ impl<'a> StateChunker<'a> { self.writer.lock().write_state_chunk(hash, compressed)?; trace!(target: "snapshot", "Thread {} wrote state chunk. size: {}, uncompressed size: {}", self.thread_idx, compressed_size, raw_data.len()); - self.progress.accounts.fetch_add(num_entries, Ordering::SeqCst); - self.progress.size.fetch_add(compressed_size as u64, Ordering::SeqCst); + self.progress.update(num_entries, compressed_size); self.hashes.push(hash); self.cur_size = 0; @@ -327,7 +326,7 @@ pub fn chunk_state<'a>( if let Some(part) = part { assert!(part < 16, "Wrong chunk state part number (must be <16) in snapshot creation."); - let part_offset = MAX_SNAPSHOT_SUBPARTS / SNAPSHOT_SUBPARTS; + let part_offset = MAX_SNAPSHOT_SUBPARTS / SNAPSHOT_SUBPARTS; // 16 let mut seek_from = vec![0; 32]; seek_from[0] = (part * part_offset) as u8; account_iter.seek(&seek_from)?; @@ -349,7 +348,15 @@ pub fn chunk_state<'a>( let account = ::rlp::decode(&*account_data)?; let account_db = AccountDB::from_hash(db, account_key_hash); - let fat_rlps = account::to_fat_rlps(&account_key_hash, &account, &account_db, &mut used_code, PREFERRED_CHUNK_SIZE - chunker.chunk_size(), PREFERRED_CHUNK_SIZE, progress)?; + let fat_rlps = account::to_fat_rlps( + &account_key_hash, + &account, + &account_db, + &mut used_code, + PREFERRED_CHUNK_SIZE - chunker.chunk_size(), + PREFERRED_CHUNK_SIZE, + progress + )?; for (i, fat_rlp) in fat_rlps.into_iter().enumerate() { if i > 0 { chunker.write_chunk()?; diff --git a/ethcore/snapshot/src/service.rs b/ethcore/snapshot/src/service.rs index 141a32360..da84482db 100644 --- a/ethcore/snapshot/src/service.rs +++ b/ethcore/snapshot/src/service.rs @@ -39,7 +39,7 @@ use ethcore_io::IoChannel; use journaldb::Algorithm; use keccak_hash::keccak; use kvdb::DBTransaction; -use log::{error, info, trace, warn}; +use log::{debug, error, info, trace, warn}; use parking_lot::{Mutex, RwLock, RwLockReadGuard}; use snappy; use trie_db::TrieError; @@ -212,7 +212,7 @@ impl Restoration { } self.guard.disarm(); - trace!(target: "snapshot", "restoration finalised correctly"); + trace!(target: "snapshot", "Restoration finalised correctly"); Ok(()) } @@ -280,7 +280,7 @@ impl Service where C: SnapshotClient + ChainInfo { state_chunks: AtomicUsize::new(0), block_chunks: AtomicUsize::new(0), client: params.client, - progress: Default::default(), + progress: Progress::new(), taking_snapshot: AtomicBool::new(false), restoring_snapshot: AtomicBool::new(false), }; @@ -413,7 +413,10 @@ impl Service where C: SnapshotClient + ChainInfo { Some(x) => x, None => return Ok(0), }; - + info!(target: "snapshot", "Migrating blocks from old db to new. Start: #{}/{:?}, Target: #{}/{:?}", + self.client.block_number(BlockId::Hash(start_hash)).unwrap_or_default(), start_hash, + self.client.block_number(BlockId::Hash(target_hash)).unwrap_or_default(), target_hash, + ); let mut batch = DBTransaction::new(); let mut parent_hash = start_hash; while parent_hash != target_hash { @@ -455,10 +458,10 @@ impl Service where C: SnapshotClient + ChainInfo { next_chain.commit(); next_db.key_value().flush().expect("DB flush failed."); batch = DBTransaction::new(); - } - if block_number % 10_000 == 0 { - info!(target: "snapshot", "Block restoration at #{}", block_number); + if block_number % 10_000 == 0 { + info!(target: "snapshot", "Block restoration at #{}", block_number); + } } } @@ -483,11 +486,13 @@ impl Service where C: SnapshotClient + ChainInfo { if self.progress.done() || !self.taking_snapshot.load(Ordering::SeqCst) { return } let p = &self.progress; - info!("Snapshot: {} accounts {} blocks {} bytes", p.accounts(), p.blocks(), p.size()); + info!("Snapshot: {} accounts, {} blocks, {} bytes", p.accounts(), p.blocks(), p.bytes()); + let rate = p.rate(); + debug!(target: "snapshot", "Current progress rate: {:.0} acc/s, {:.0} bytes/s (compressed)", rate.0, rate.1); } /// Take a snapshot at the block with the given number. - /// calling this while a restoration is in progress or vice versa + /// Calling this while a restoration is in progress or vice versa /// will lead to a race condition where the first one to finish will /// have their produced snapshot overwritten. pub fn take_snapshot(&self, client: &C, num: u64) -> Result<(), Error> { @@ -497,45 +502,40 @@ impl Service where C: SnapshotClient + ChainInfo { } info!("Taking snapshot at #{}", num); - self.progress.reset(); + { + scopeguard::defer! {{ + self.taking_snapshot.store(false, Ordering::SeqCst); + }} + let start_time = std::time::Instant::now(); + self.progress.reset(); - let temp_dir = self.temp_snapshot_dir(); - let snapshot_dir = self.snapshot_dir(); + let temp_dir = self.temp_snapshot_dir(); + let snapshot_dir = self.snapshot_dir(); - let _ = fs::remove_dir_all(&temp_dir); + let _ = fs::remove_dir_all(&temp_dir); // expected to fail - let writer = LooseWriter::new(temp_dir.clone())?; + let writer = LooseWriter::new(temp_dir.clone())?; - let guard = Guard::new(temp_dir.clone()); - let res = client.take_snapshot(writer, BlockId::Number(num), &self.progress); - self.taking_snapshot.store(false, Ordering::SeqCst); - if let Err(e) = res { - if client.chain_info().best_block_number >= num + client.pruning_history() { - // The state we were snapshotting was pruned before we could finish. - info!("Periodic snapshot failed: block state pruned. Run with a longer `--pruning-history` or with `--no-periodic-snapshot`"); - return Err(e); - } else { - return Err(e); + let guard = Guard::new(temp_dir.clone()); + let _ = client.take_snapshot(writer, BlockId::Number(num), &self.progress)?; + info!("Finished taking snapshot at #{}, in {:.0?}", num, start_time.elapsed()); + + // destroy the old snapshot reader. + let mut reader = self.reader.write(); + *reader = None; + + if snapshot_dir.exists() { + trace!(target: "snapshot", "Removing previous snapshot at {:?}", &snapshot_dir); + fs::remove_dir_all(&snapshot_dir)?; } + + fs::rename(temp_dir, &snapshot_dir)?; + trace!(target: "snapshot", "Moved new snapshot into place at {:?}", &snapshot_dir); + *reader = Some(LooseReader::new(snapshot_dir)?); + + guard.disarm(); + Ok(()) } - - info!("Finished taking snapshot at #{}", num); - - let mut reader = self.reader.write(); - - // destroy the old snapshot reader. - *reader = None; - - if snapshot_dir.exists() { - fs::remove_dir_all(&snapshot_dir)?; - } - - fs::rename(temp_dir, &snapshot_dir)?; - - *reader = Some(LooseReader::new(snapshot_dir)?); - - guard.disarm(); - Ok(()) } /// Initialize the restoration synchronously. @@ -654,13 +654,19 @@ impl Service where C: SnapshotClient + ChainInfo { Ok(()) } - /// Import a previous chunk at the given path. Returns whether the block was imported or not - fn import_prev_chunk(&self, restoration: &mut Option, manifest: &ManifestData, file: io::Result) -> Result { + /// Import a previous chunk at the given path. Returns whether the chunk was imported or not + fn import_prev_chunk( + &self, + restoration: &mut Option, + manifest: &ManifestData, + file: io::Result + ) -> Result { let file = file?; let path = file.path(); let mut file = File::open(path.clone())?; - let mut buffer = Vec::new(); + let filesize = file.metadata()?.len(); + let mut buffer = Vec::with_capacity(filesize as usize + 1); // +1 for EOF file.read_to_end(&mut buffer)?; let hash = keccak(&buffer); @@ -670,6 +676,7 @@ impl Service where C: SnapshotClient + ChainInfo { } else if manifest.state_hashes.contains(&hash) { true } else { + warn!(target: "snapshot", "Hash of the content of {:?} not present in the manifest block/state hashes.", path); return Ok(false); }; @@ -680,11 +687,10 @@ impl Service where C: SnapshotClient + ChainInfo { Ok(true) } - // finalize the restoration. this accepts an already-locked - // restoration as an argument -- so acquiring it again _will_ - // lead to deadlock. + // Finalize the restoration. This accepts an already-locked restoration as an argument -- so + // acquiring it again _will_ lead to deadlock. fn finalize_restoration(&self, rest: &mut Option) -> Result<(), Error> { - trace!(target: "snapshot", "finalizing restoration"); + trace!(target: "snapshot", "Finalizing restoration"); *self.status.lock() = RestorationStatus::Finalizing; let recover = rest.as_ref().map_or(false, |rest| rest.writer.is_some()); @@ -695,7 +701,7 @@ impl Service where C: SnapshotClient + ChainInfo { .unwrap_or(Ok(()))?; let migrated_blocks = self.migrate_blocks()?; - info!(target: "snapshot", "Migrated {} ancient blocks", migrated_blocks); + info!(target: "snapshot", "Migrated {} ancient blocks from the old DB", migrated_blocks); // replace the Client's database with the new one (restart the Client). self.client.restore_db(&*self.restoration_db().to_string_lossy())?; @@ -707,11 +713,11 @@ impl Service where C: SnapshotClient + ChainInfo { let snapshot_dir = self.snapshot_dir(); if snapshot_dir.exists() { - trace!(target: "snapshot", "removing old snapshot dir at {}", snapshot_dir.to_string_lossy()); + trace!(target: "snapshot", "Removing old snapshot dir at {}", snapshot_dir.to_string_lossy()); fs::remove_dir_all(&snapshot_dir)?; } - trace!(target: "snapshot", "copying restored snapshot files over"); + trace!(target: "snapshot", "Copying restored snapshot files over"); fs::rename(self.temp_recovery_dir(), &snapshot_dir)?; *reader = Some(LooseReader::new(snapshot_dir)?); diff --git a/ethcore/snapshot/src/traits.rs b/ethcore/snapshot/src/traits.rs index b72d6b7af..8f5e12a11 100644 --- a/ethcore/snapshot/src/traits.rs +++ b/ethcore/snapshot/src/traits.rs @@ -136,7 +136,7 @@ pub trait SnapshotComponents: Send { /// Snapshot related functionality pub trait SnapshotClient: BlockChainClient + BlockInfo + DatabaseRestore + BlockChainReset { /// Take a snapshot at the given block. - /// If the ID given is "latest", this will default to 1000 blocks behind. + /// If the BlockId is 'Latest', this will default to 1000 blocks behind. fn take_snapshot( &self, writer: W, @@ -148,7 +148,7 @@ pub trait SnapshotClient: BlockChainClient + BlockInfo + DatabaseRestore + Block /// Helper trait for broadcasting a block to take a snapshot at. pub trait Broadcast: Send + Sync { /// Start a snapshot from the given block number. - fn take_at(&self, num: Option); + fn request_snapshot_at(&self, num: u64); } diff --git a/ethcore/snapshot/src/watcher.rs b/ethcore/snapshot/src/watcher.rs index d6d9bcef8..34ed5189b 100644 --- a/ethcore/snapshot/src/watcher.rs +++ b/ethcore/snapshot/src/watcher.rs @@ -49,16 +49,11 @@ impl Oracle for StandardOracle } impl Broadcast for Mutex>> { - fn take_at(&self, num: Option) { - let num = match num { - Some(n) => n, - None => return, - }; - - trace!(target: "snapshot_watcher", "Snapshot requested at block #{}", num); - + fn request_snapshot_at(&self, num: u64) { if let Err(e) = self.lock().send(ClientIoMessage::TakeSnapshot(num)) { - warn!("Snapshot watcher disconnected from IoService: {}", e); + warn!(target: "snapshot_watcher", "Snapshot watcher disconnected from IoService: {}", e); + } else { + trace!(target: "snapshot_watcher", "Snapshot requested at block #{}", num); } } } @@ -68,7 +63,10 @@ impl Broadcast for Mutex>> { pub struct Watcher { oracle: Box, broadcast: Box, + // How often we attempt to take a snapshot: only snapshot on blocknumbers that are multiples of + // `period`. Always set to `SNAPSHOT_PERIOD`, i.e. 5000. period: u64, + // Start snapshots `history` blocks from the tip. Always set to `SNAPSHOT_HISTORY`, i.e. 100. history: u64, } @@ -106,18 +104,22 @@ impl ChainNotify for Watcher { fn new_blocks(&self, new_blocks: NewBlocks) { if self.oracle.is_major_importing() || new_blocks.has_more_blocks_to_import { return } - trace!(target: "snapshot_watcher", "{} imported", new_blocks.imported.len()); - + // Decide if it's time for a snapshot: the highest of the imported blocks is a multiple of 5000? let highest = new_blocks.imported.into_iter() + // Convert block hashes to block numbers for all newly imported blocks .filter_map(|h| self.oracle.to_number(h)) - .filter(|&num| num >= self.period + self.history) - .map(|num| num - self.history) - .filter(|num| num % self.period == 0) + // Subtract `history` (i.e. `SNAPSHOT_HISTORY`, i.e. 100) from the block numbers to stay + // clear of reorgs. + .map(|num| num.saturating_sub(self.history) ) + // …filter out blocks that do not fall on the a multiple of `period`. This regulates the + // frequency of snapshots and ensures more snapshots are produced from similar points in + // the chain. + .filter(|num| num % self.period == 0 ) + // Pick newest of the candidates: this is where we want to snapshot from. .fold(0, ::std::cmp::max); - match highest { - 0 => self.broadcast.take_at(None), - _ => self.broadcast.take_at(Some(highest)), + if highest > 0 { + self.broadcast.request_snapshot_at(highest); } } } diff --git a/ethcore/src/client/client.rs b/ethcore/src/client/client.rs index 4527dba81..c1c450486 100644 --- a/ethcore/src/client/client.rs +++ b/ethcore/src/client/client.rs @@ -20,7 +20,7 @@ use std::convert::TryFrom; use std::io::{BufRead, BufReader}; use std::str::from_utf8; use std::sync::{Arc, Weak}; -use std::sync::atomic::{AtomicBool, AtomicI64, Ordering as AtomicOrdering}; +use std::sync::atomic::{AtomicBool, AtomicI64, Ordering as AtomicOrdering, Ordering, AtomicU64}; use std::time::{Duration, Instant}; use ansi_term::Colour; @@ -204,6 +204,9 @@ pub struct Client { /// Database pruning strategy to use for StateDB pruning: journaldb::Algorithm, + /// Don't prune the state we're currently snapshotting + snapshotting_at: AtomicU64, + /// Client uses this to store blocks, traces, etc. db: RwLock>, @@ -780,6 +783,7 @@ impl Client { tracedb, engine, pruning: config.pruning.clone(), + snapshotting_at: AtomicU64::new(0), db: RwLock::new(db.clone()), state_db: RwLock::new(state_db), report: RwLock::new(Default::default()), @@ -964,13 +968,15 @@ impl Client { return Ok(()) } - let number = match state_db.journal_db().latest_era() { + let latest_era = match state_db.journal_db().latest_era() { Some(n) => n, None => return Ok(()), }; - // prune all ancient eras until we're below the memory target, - // but have at least the minimum number of states. + // Prune all ancient eras until we're below the memory target (default: 32Mb), + // but have at least the minimum number of states, i.e. `history`. + // If a snapshot is under way, no pruning happens and memory consumption is allowed to + // increase above the memory target until the snapshot has finished. loop { let needs_pruning = state_db.journal_db().journal_size() >= self.config.history_mem; @@ -979,17 +985,27 @@ impl Client { } match state_db.journal_db().earliest_era() { - Some(era) if era + self.history <= number => { - trace!(target: "client", "Pruning state for ancient era {}", era); - match chain.block_hash(era) { + Some(earliest_era) if earliest_era + self.history <= latest_era => { + let freeze_at = self.snapshotting_at.load(Ordering::SeqCst); + if freeze_at > 0 && freeze_at == earliest_era { + // Note: journal_db().mem_used() can be used for a more accurate memory + // consumption measurement but it can be expensive so sticking with the + // faster `journal_size()` instead. + trace!(target: "pruning", "Pruning is paused at era {} (snapshot under way); earliest era={}, latest era={}, journal_size={} – Not pruning.", + freeze_at, earliest_era, latest_era, state_db.journal_db().journal_size()); + break; + } + trace!(target: "pruning", "Pruning state for ancient era #{}; latest era={}, journal_size={}", + earliest_era, latest_era, state_db.journal_db().journal_size()); + match chain.block_hash(earliest_era) { Some(ancient_hash) => { let mut batch = DBTransaction::new(); - state_db.mark_canonical(&mut batch, era, &ancient_hash)?; + state_db.mark_canonical(&mut batch, earliest_era, &ancient_hash)?; self.db.read().key_value().write_buffered(batch); state_db.journal_db().flush(); } None => - debug!(target: "client", "Missing expected hash for block {}", era), + debug!(target: "pruning", "Missing expected hash for block {}", earliest_era), } } _ => break, // means that every era is kept, no pruning necessary. @@ -2533,48 +2549,61 @@ impl SnapshotClient for Client { return Err(EthcoreError::Snapshot(SnapshotError::SnapshotsUnsupported)); } let db = self.state_db.read().journal_db().boxed_clone(); - let best_block_number = self.chain_info().best_block_number; - let block_number = self.block_number(at).ok_or_else(|| SnapshotError::InvalidStartingBlock(at))?; - if db.is_prunable() && self.pruning_info().earliest_state > block_number { + let block_number = self.block_number(at).ok_or_else(|| SnapshotError::InvalidStartingBlock(at))?; + let earliest_era = db.earliest_era().unwrap_or(0); + if db.is_prunable() && earliest_era > block_number { return Err(SnapshotError::OldBlockPrunedDB.into()); } - let history = cmp::min(self.history, 1000); - let start_hash = match at { + let (actual_block_nr, block_hash) = match at { BlockId::Latest => { - let start_num = match db.earliest_era() { - Some(era) => cmp::max(era, best_block_number.saturating_sub(history)), - None => best_block_number.saturating_sub(history), - }; + // Start `self.history` blocks from the best block, but no further back than 1000 + // blocks (or earliest era, whichever is greatest). + let history = cmp::min(self.history, 1000); + let best_block_number = self.chain_info().best_block_number; + let start_num = cmp::max(earliest_era, best_block_number.saturating_sub(history)); match self.block_hash(BlockId::Number(start_num)) { - Some(h) => h, - None => return Err(SnapshotError::InvalidStartingBlock(at).into()), + Some(hash) => (start_num, hash), + None => { + error!(target: "snapshot", "Can't take snapshot at {:?}: missing hash for the starting block #{}", at, start_num); + return Err(SnapshotError::InvalidStartingBlock(at).into()) + }, } } _ => match self.block_hash(at) { - Some(hash) => hash, + Some(hash) => (block_number, hash), None => return Err(SnapshotError::InvalidStartingBlock(at).into()), }, }; let processing_threads = self.config.snapshot.processing_threads; - let chunker = snapshot::chunker(self.engine.snapshot_mode()).ok_or_else(|| SnapshotError::SnapshotsUnsupported)?; - snapshot::take_snapshot( - chunker, - &self.chain.read(), - start_hash, - db.as_hash_db(), - writer, - p, - processing_threads, - )?; - Ok(()) + trace!(target: "snapshot", "Snapshot requested at block {:?}. Using block #{}/{:?}. Earliest block: #{}, earliest state era #{}. Using {} threads.", + at, actual_block_nr, block_hash, self.pruning_info().earliest_chain, earliest_era, processing_threads, + ); + // Stop pruning from happening while the snapshot is under way. + self.snapshotting_at.store(actual_block_nr, Ordering::SeqCst); + { + scopeguard::defer! {{ + trace!(target: "snapshot", "Re-enabling pruning."); + self.snapshotting_at.store(0, Ordering::SeqCst) + }}; + let chunker = snapshot::chunker(self.engine.snapshot_mode()).ok_or_else(|| SnapshotError::SnapshotsUnsupported)?; + // Spawn threads and take snapshot + snapshot::take_snapshot( + chunker, + &self.chain.read(), + block_hash, + db.as_hash_db(), + writer, + p, + processing_threads, + )?; + Ok(()) + } } - - } impl ImportExportBlocks for Client { diff --git a/ethcore/src/client/config.rs b/ethcore/src/client/config.rs index 5a1c2b87e..23104c48b 100644 --- a/ethcore/src/client/config.rs +++ b/ethcore/src/client/config.rs @@ -90,7 +90,7 @@ pub struct ClientConfig { pub history: u64, /// Ideal memory usage for state pruning history. pub history_mem: usize, - /// Check seal valididity on block import + /// Check seal validity on block import pub check_seal: bool, /// Maximal number of transactions queued for verification in a separate thread. pub transaction_verification_queue_size: usize, diff --git a/ethcore/types/Cargo.toml b/ethcore/types/Cargo.toml index 509b9d13d..dd1dff2fb 100644 --- a/ethcore/types/Cargo.toml +++ b/ethcore/types/Cargo.toml @@ -15,6 +15,7 @@ parity-bytes = "0.1" parity-crypto = { version = "0.4.2", features = ["publickey"] } parity-util-mem = "0.2.0" parity-snappy = "0.1" +parking_lot = "0.9.0" patricia-trie-ethereum = { path = "../../util/patricia-trie-ethereum" } rlp = "0.4.0" rlp_derive = { path = "../../util/rlp-derive" } diff --git a/ethcore/types/src/lib.rs b/ethcore/types/src/lib.rs index 58ca86cae..ab17b8837 100644 --- a/ethcore/types/src/lib.rs +++ b/ethcore/types/src/lib.rs @@ -40,6 +40,7 @@ extern crate parity_crypto; #[macro_use] extern crate derive_more; extern crate keccak_hash as hash; +extern crate parking_lot; extern crate parity_bytes as bytes; extern crate patricia_trie_ethereum as ethtrie; extern crate rlp; diff --git a/ethcore/types/src/snapshot.rs b/ethcore/types/src/snapshot.rs index ed305d689..a3be631be 100644 --- a/ethcore/types/src/snapshot.rs +++ b/ethcore/types/src/snapshot.rs @@ -16,11 +16,13 @@ //! Snapshot type definitions -use std::sync::atomic::{AtomicBool, AtomicUsize, AtomicU64, Ordering}; +use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; +use std::time::Instant; -use ethereum_types::H256; -use rlp::{Rlp, RlpStream, DecoderError}; use bytes::Bytes; +use ethereum_types::H256; +use parking_lot::RwLock; +use rlp::{Rlp, RlpStream, DecoderError}; /// Modes of snapshotting pub enum Snapshotting { @@ -39,31 +41,53 @@ pub enum Snapshotting { } /// A progress indicator for snapshots. -#[derive(Debug, Default)] +#[derive(Debug)] pub struct Progress { /// Number of accounts processed so far - pub accounts: AtomicUsize, + accounts: AtomicUsize, + // Number of accounts processed at last tick. + prev_accounts: AtomicUsize, /// Number of blocks processed so far pub blocks: AtomicUsize, /// Size in bytes of a all compressed chunks processed so far - pub size: AtomicU64, + bytes: AtomicUsize, + // Number of bytes processed at last tick. + prev_bytes: AtomicUsize, /// Signals that the snapshotting process is completed pub done: AtomicBool, /// Signal snapshotting process to abort pub abort: AtomicBool, + + last_tick: RwLock, } impl Progress { + /// Create a new progress tracker. + pub fn new() -> Progress { + Progress { + accounts: AtomicUsize::new(0), + prev_accounts: AtomicUsize::new(0), + blocks: AtomicUsize::new(0), + bytes: AtomicUsize::new(0), + prev_bytes: AtomicUsize::new(0), + abort: AtomicBool::new(false), + done: AtomicBool::new(false), + last_tick: RwLock::new(Instant::now()), + } + } + /// Reset the progress. pub fn reset(&self) { self.accounts.store(0, Ordering::Release); self.blocks.store(0, Ordering::Release); - self.size.store(0, Ordering::Release); + self.bytes.store(0, Ordering::Release); self.abort.store(false, Ordering::Release); // atomic fence here to ensure the others are written first? // logs might very rarely get polluted if not. self.done.store(false, Ordering::Release); + + *self.last_tick.write() = Instant::now(); } /// Get the number of accounts snapshotted thus far. @@ -73,10 +97,37 @@ impl Progress { pub fn blocks(&self) -> usize { self.blocks.load(Ordering::Acquire) } /// Get the written size of the snapshot in bytes. - pub fn size(&self) -> u64 { self.size.load(Ordering::Acquire) } + pub fn bytes(&self) -> usize { self.bytes.load(Ordering::Acquire) } /// Whether the snapshot is complete. pub fn done(&self) -> bool { self.done.load(Ordering::Acquire) } + + /// Return the progress rate over the last tick (i.e. since last update). + pub fn rate(&self) -> (f64, f64) { + let last_tick = *self.last_tick.read(); + let dt = last_tick.elapsed().as_secs_f64(); + if dt < 1.0 { + return (0f64, 0f64); + } + let delta_acc = self.accounts.load(Ordering::Relaxed) + .saturating_sub(self.prev_accounts.load(Ordering::Relaxed)); + let delta_bytes = self.bytes.load(Ordering::Relaxed) + .saturating_sub(self.prev_bytes.load(Ordering::Relaxed)); + (delta_acc as f64 / dt, delta_bytes as f64 / dt) + } + + /// Update state progress counters and set the last tick. + pub fn update(&self, accounts_delta: usize, bytes_delta: usize) { + *self.last_tick.write() = Instant::now(); + self.prev_accounts.store( + self.accounts.fetch_add(accounts_delta, Ordering::SeqCst), + Ordering::SeqCst + ); + self.prev_bytes.store( + self.bytes.fetch_add(bytes_delta, Ordering::SeqCst), + Ordering::SeqCst + ); + } } /// Manifest data. diff --git a/parity/run.rs b/parity/run.rs index 891424eeb..987819431 100644 --- a/parity/run.rs +++ b/parity/run.rs @@ -68,10 +68,10 @@ use signer; use db; use registrar::RegistrarClient; -// how often to take periodic snapshots. +// How often we attempt to take a snapshot: only snapshot on blocknumbers that are multiples of this. const SNAPSHOT_PERIOD: u64 = 5000; -// how many blocks to wait before starting a periodic snapshot. +// Start snapshots `history` blocks from the tip. Should be smaller than `SNAPSHOT_HISTORY`. const SNAPSHOT_HISTORY: u64 = 100; // Number of minutes before a given gas price corpus should expire. diff --git a/parity/snapshot_cmd.rs b/parity/snapshot_cmd.rs index bd41400a3..8a9157ad6 100644 --- a/parity/snapshot_cmd.rs +++ b/parity/snapshot_cmd.rs @@ -257,18 +257,18 @@ impl SnapshotCommand { let writer = PackedWriter::new(&file_path) .map_err(|e| format!("Failed to open snapshot writer: {}", e))?; - let progress = Arc::new(Progress::default()); + let progress = Arc::new(Progress::new()); let p = progress.clone(); let informant_handle = ::std::thread::spawn(move || { ::std::thread::sleep(Duration::from_secs(5)); let mut last_size = 0; while !p.done() { - let cur_size = p.size(); + let cur_size = p.bytes(); if cur_size != last_size { last_size = cur_size; let bytes = ::informant::format_bytes(cur_size as usize); - info!("Snapshot: {} accounts {} blocks {}", p.accounts(), p.blocks(), bytes); + info!("Snapshot: {} accounts (state), {} blocks, {} bytes", p.accounts(), p.blocks(), bytes); } ::std::thread::sleep(Duration::from_secs(5)); diff --git a/util/journaldb/src/lib.rs b/util/journaldb/src/lib.rs index 0024b3638..739c25807 100644 --- a/util/journaldb/src/lib.rs +++ b/util/journaldb/src/lib.rs @@ -84,8 +84,8 @@ pub trait JournalDB: HashDB { /// Get backing database. fn backing(&self) -> &Arc; - /// Clear internal strucutres. This should called after changes have been written - /// to the backing strage + /// Clear internal structure. This should be called after changes have been written + /// to the backing storage. fn flush(&self) {} /// Consolidate all the insertions and deletions in the given memory overlay. diff --git a/util/journaldb/src/overlayrecentdb.rs b/util/journaldb/src/overlayrecentdb.rs index 0c0f92d5e..6ea285cd1 100644 --- a/util/journaldb/src/overlayrecentdb.rs +++ b/util/journaldb/src/overlayrecentdb.rs @@ -228,10 +228,10 @@ impl OverlayRecentDB { JournalOverlay { backing_overlay: overlay, pending_overlay: HashMap::default(), - journal: journal, - latest_era: latest_era, - earliest_era: earliest_era, - cumulative_size: cumulative_size, + journal, + latest_era, + earliest_era, + cumulative_size, } } @@ -264,7 +264,6 @@ impl JournalDB for OverlayRecentDB { fn journal_size(&self) -> usize { self.journal_overlay.read().cumulative_size - } fn is_empty(&self) -> bool { @@ -351,7 +350,7 @@ impl JournalDB for OverlayRecentDB { let mut ops = 0; // apply old commits' details - if let Some(ref mut records) = journal_overlay.journal.get_mut(&end_era) { + if let Some(records) = journal_overlay.journal.get_mut(&end_era) { let mut canon_insertions: Vec<(H256, DBValue)> = Vec::new(); let mut canon_deletions: Vec = Vec::new(); let mut overlay_deletions: Vec = Vec::new();