Pause pruning while snapshotting (#11178)

* WIP. Typos and logging.

* Format todos

* Pause pruning while a snapshot is under way
Logs, docs and todos

* Allocate memory for the full chunk

* Name snapshotting threads

* Ensure `taking_snapshot` is set to false whenever and however `take_snapshot`returns
Rename `take_at` to `request_snapshot_at`
Cleanup

* Let "in_progress" deletion fail
Fix tests

* Just use an atomic

* Review grumbles

* Finish the sentence

* Resolve a few todos and clarify comments.

* Calculate progress rate since last update

* Lockfile

* Fix tests

* typo

* Reinstate default snapshotting frequency
Cut down on the logging noise

* address grumble

* Log memory use with `journal_size()` and explain why.
This commit is contained in:
David 2019-10-24 16:46:32 +02:00 committed by GitHub
parent acf7c48d7e
commit ffeaee778c
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
24 changed files with 348 additions and 209 deletions

134
Cargo.lock generated
View File

@ -5,7 +5,7 @@ name = "account-db"
version = "0.1.0" version = "0.1.0"
dependencies = [ dependencies = [
"ethereum-types 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)", "ethereum-types 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)",
"hash-db 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)", "hash-db 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)",
"keccak-hash 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", "keccak-hash 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
"keccak-hasher 0.1.1", "keccak-hasher 0.1.1",
"kvdb 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", "kvdb 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
@ -20,7 +20,7 @@ dependencies = [
"common-types 0.1.0", "common-types 0.1.0",
"derive_more 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)", "derive_more 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)",
"ethereum-types 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)", "ethereum-types 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)",
"hash-db 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)", "hash-db 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)",
"journaldb 0.2.0", "journaldb 0.2.0",
"keccak-hash 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", "keccak-hash 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
"keccak-hasher 0.1.1", "keccak-hasher 0.1.1",
@ -37,7 +37,7 @@ dependencies = [
"rlp_compress 0.1.0", "rlp_compress 0.1.0",
"serde 1.0.99 (registry+https://github.com/rust-lang/crates.io-index)", "serde 1.0.99 (registry+https://github.com/rust-lang/crates.io-index)",
"trace 0.1.0", "trace 0.1.0",
"trie-db 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)", "trie-db 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)",
"trie-vm-factories 0.1.0", "trie-vm-factories 0.1.0",
] ]
@ -82,6 +82,14 @@ dependencies = [
"stream-cipher 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)", "stream-cipher 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)",
] ]
[[package]]
name = "ahash"
version = "0.2.16"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"const-random 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]] [[package]]
name = "aho-corasick" name = "aho-corasick"
version = "0.6.8" version = "0.6.8"
@ -598,6 +606,7 @@ dependencies = [
"parity-crypto 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", "parity-crypto 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)",
"parity-snappy 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", "parity-snappy 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
"parity-util-mem 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", "parity-util-mem 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)",
"parking_lot 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)",
"patricia-trie-ethereum 0.1.0", "patricia-trie-ethereum 0.1.0",
"rlp 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", "rlp 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)",
"rlp_derive 0.1.0", "rlp_derive 0.1.0",
@ -606,6 +615,24 @@ dependencies = [
"vm 0.1.0", "vm 0.1.0",
] ]
[[package]]
name = "const-random"
version = "0.1.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"const-random-macro 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)",
"proc-macro-hack 0.5.9 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "const-random-macro"
version = "0.1.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"proc-macro-hack 0.5.9 (registry+https://github.com/rust-lang/crates.io-index)",
"rand 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]] [[package]]
name = "criterion" name = "criterion"
version = "0.3.0" version = "0.3.0"
@ -645,7 +672,7 @@ version = "0.6.3"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [ dependencies = [
"crossbeam-epoch 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)", "crossbeam-epoch 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)",
"crossbeam-utils 0.6.5 (registry+https://github.com/rust-lang/crates.io-index)", "crossbeam-utils 0.6.6 (registry+https://github.com/rust-lang/crates.io-index)",
] ]
[[package]] [[package]]
@ -654,7 +681,7 @@ version = "0.7.1"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [ dependencies = [
"crossbeam-epoch 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)", "crossbeam-epoch 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)",
"crossbeam-utils 0.6.5 (registry+https://github.com/rust-lang/crates.io-index)", "crossbeam-utils 0.6.6 (registry+https://github.com/rust-lang/crates.io-index)",
] ]
[[package]] [[package]]
@ -664,7 +691,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [ dependencies = [
"arrayvec 0.4.11 (registry+https://github.com/rust-lang/crates.io-index)", "arrayvec 0.4.11 (registry+https://github.com/rust-lang/crates.io-index)",
"cfg-if 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)", "cfg-if 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)",
"crossbeam-utils 0.6.5 (registry+https://github.com/rust-lang/crates.io-index)", "crossbeam-utils 0.6.6 (registry+https://github.com/rust-lang/crates.io-index)",
"lazy_static 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)", "lazy_static 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)",
"memoffset 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)", "memoffset 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)",
"scopeguard 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", "scopeguard 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)",
@ -675,7 +702,7 @@ name = "crossbeam-queue"
version = "0.1.2" version = "0.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [ dependencies = [
"crossbeam-utils 0.6.5 (registry+https://github.com/rust-lang/crates.io-index)", "crossbeam-utils 0.6.6 (registry+https://github.com/rust-lang/crates.io-index)",
] ]
[[package]] [[package]]
@ -685,7 +712,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
[[package]] [[package]]
name = "crossbeam-utils" name = "crossbeam-utils"
version = "0.6.5" version = "0.6.6"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [ dependencies = [
"cfg-if 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)", "cfg-if 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)",
@ -1043,7 +1070,7 @@ dependencies = [
"executive-state 0.1.0", "executive-state 0.1.0",
"fetch 0.1.0", "fetch 0.1.0",
"futures 0.1.29 (registry+https://github.com/rust-lang/crates.io-index)", "futures 0.1.29 (registry+https://github.com/rust-lang/crates.io-index)",
"hash-db 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)", "hash-db 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)",
"itertools 0.5.10 (registry+https://github.com/rust-lang/crates.io-index)", "itertools 0.5.10 (registry+https://github.com/rust-lang/crates.io-index)",
"journaldb 0.2.0", "journaldb 0.2.0",
"keccak-hash 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", "keccak-hash 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
@ -1067,6 +1094,7 @@ dependencies = [
"registrar 0.0.1", "registrar 0.0.1",
"rlp 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", "rlp 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)",
"rustc-hex 2.0.1 (registry+https://github.com/rust-lang/crates.io-index)", "rustc-hex 2.0.1 (registry+https://github.com/rust-lang/crates.io-index)",
"scopeguard 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)",
"serde 1.0.99 (registry+https://github.com/rust-lang/crates.io-index)", "serde 1.0.99 (registry+https://github.com/rust-lang/crates.io-index)",
"serde_derive 1.0.89 (registry+https://github.com/rust-lang/crates.io-index)", "serde_derive 1.0.89 (registry+https://github.com/rust-lang/crates.io-index)",
"serde_json 1.0.40 (registry+https://github.com/rust-lang/crates.io-index)", "serde_json 1.0.40 (registry+https://github.com/rust-lang/crates.io-index)",
@ -1077,7 +1105,7 @@ dependencies = [
"tempdir 0.3.7 (registry+https://github.com/rust-lang/crates.io-index)", "tempdir 0.3.7 (registry+https://github.com/rust-lang/crates.io-index)",
"trace 0.1.0", "trace 0.1.0",
"trace-time 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", "trace-time 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
"trie-db 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)", "trie-db 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)",
"trie-standardmap 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)", "trie-standardmap 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)",
"trie-vm-factories 0.1.0", "trie-vm-factories 0.1.0",
"triehash-ethereum 0.2.0", "triehash-ethereum 0.2.0",
@ -1216,7 +1244,7 @@ dependencies = [
"failsafe 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)", "failsafe 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)",
"fastmap 0.1.0", "fastmap 0.1.0",
"futures 0.1.29 (registry+https://github.com/rust-lang/crates.io-index)", "futures 0.1.29 (registry+https://github.com/rust-lang/crates.io-index)",
"hash-db 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)", "hash-db 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)",
"journaldb 0.2.0", "journaldb 0.2.0",
"keccak-hash 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", "keccak-hash 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
"keccak-hasher 0.1.1", "keccak-hasher 0.1.1",
@ -1239,7 +1267,7 @@ dependencies = [
"spec 0.1.0", "spec 0.1.0",
"stats 0.1.0", "stats 0.1.0",
"tempdir 0.3.7 (registry+https://github.com/rust-lang/crates.io-index)", "tempdir 0.3.7 (registry+https://github.com/rust-lang/crates.io-index)",
"trie-db 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)", "trie-db 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)",
"triehash-ethereum 0.2.0", "triehash-ethereum 0.2.0",
"verification 0.1.0", "verification 0.1.0",
"vm 0.1.0", "vm 0.1.0",
@ -1368,7 +1396,7 @@ dependencies = [
"ethjson 0.1.0", "ethjson 0.1.0",
"fetch 0.1.0", "fetch 0.1.0",
"futures 0.1.29 (registry+https://github.com/rust-lang/crates.io-index)", "futures 0.1.29 (registry+https://github.com/rust-lang/crates.io-index)",
"hash-db 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)", "hash-db 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)",
"journaldb 0.2.0", "journaldb 0.2.0",
"keccak-hash 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", "keccak-hash 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
"keccak-hasher 0.1.1", "keccak-hasher 0.1.1",
@ -1393,7 +1421,7 @@ dependencies = [
"tiny-keccak 1.5.0 (registry+https://github.com/rust-lang/crates.io-index)", "tiny-keccak 1.5.0 (registry+https://github.com/rust-lang/crates.io-index)",
"trace 0.1.0", "trace 0.1.0",
"transaction-pool 2.0.1 (registry+https://github.com/rust-lang/crates.io-index)", "transaction-pool 2.0.1 (registry+https://github.com/rust-lang/crates.io-index)",
"trie-db 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)", "trie-db 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)",
"url 2.1.0 (registry+https://github.com/rust-lang/crates.io-index)", "url 2.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
"vm 0.1.0", "vm 0.1.0",
] ]
@ -1668,7 +1696,7 @@ dependencies = [
"ethcore 1.12.0", "ethcore 1.12.0",
"ethereum-types 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)", "ethereum-types 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)",
"evm 0.1.0", "evm 0.1.0",
"hash-db 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)", "hash-db 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)",
"keccak-hash 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", "keccak-hash 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
"keccak-hasher 0.1.1", "keccak-hasher 0.1.1",
"kvdb 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", "kvdb 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
@ -1681,7 +1709,7 @@ dependencies = [
"rustc-hex 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", "rustc-hex 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)",
"spec 0.1.0", "spec 0.1.0",
"trace 0.1.0", "trace 0.1.0",
"trie-db 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)", "trie-db 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)",
"trie-vm-factories 0.1.0", "trie-vm-factories 0.1.0",
"vm 0.1.0", "vm 0.1.0",
] ]
@ -1909,7 +1937,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
[[package]] [[package]]
name = "hash-db" name = "hash-db"
version = "0.15.0" version = "0.15.2"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
[[package]] [[package]]
@ -1920,6 +1948,15 @@ dependencies = [
"crunchy 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", "crunchy 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)",
] ]
[[package]]
name = "hashbrown"
version = "0.6.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"ahash 0.2.16 (registry+https://github.com/rust-lang/crates.io-index)",
"autocfg 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]] [[package]]
name = "hashmap_core" name = "hashmap_core"
version = "0.1.10" version = "0.1.10"
@ -2224,7 +2261,7 @@ dependencies = [
"env_logger 0.5.13 (registry+https://github.com/rust-lang/crates.io-index)", "env_logger 0.5.13 (registry+https://github.com/rust-lang/crates.io-index)",
"ethereum-types 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)", "ethereum-types 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)",
"fastmap 0.1.0", "fastmap 0.1.0",
"hash-db 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)", "hash-db 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)",
"keccak-hash 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", "keccak-hash 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
"keccak-hasher 0.1.1", "keccak-hasher 0.1.1",
"kvdb 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", "kvdb 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
@ -2352,7 +2389,7 @@ name = "keccak-hasher"
version = "0.1.1" version = "0.1.1"
dependencies = [ dependencies = [
"ethereum-types 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)", "ethereum-types 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)",
"hash-db 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)", "hash-db 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)",
"plain_hasher 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", "plain_hasher 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)",
"tiny-keccak 1.5.0 (registry+https://github.com/rust-lang/crates.io-index)", "tiny-keccak 1.5.0 (registry+https://github.com/rust-lang/crates.io-index)",
] ]
@ -2362,7 +2399,7 @@ name = "keccak-hasher"
version = "0.15.0" version = "0.15.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [ dependencies = [
"hash-db 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)", "hash-db 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)",
"hash256-std-hasher 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)", "hash256-std-hasher 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)",
"tiny-keccak 1.5.0 (registry+https://github.com/rust-lang/crates.io-index)", "tiny-keccak 1.5.0 (registry+https://github.com/rust-lang/crates.io-index)",
] ]
@ -2525,7 +2562,7 @@ dependencies = [
"account-state 0.1.0", "account-state 0.1.0",
"client-traits 0.1.0", "client-traits 0.1.0",
"common-types 0.1.0", "common-types 0.1.0",
"crossbeam-utils 0.6.5 (registry+https://github.com/rust-lang/crates.io-index)", "crossbeam-utils 0.6.6 (registry+https://github.com/rust-lang/crates.io-index)",
"ethabi 9.0.1 (registry+https://github.com/rust-lang/crates.io-index)", "ethabi 9.0.1 (registry+https://github.com/rust-lang/crates.io-index)",
"ethabi-contract 9.0.0 (registry+https://github.com/rust-lang/crates.io-index)", "ethabi-contract 9.0.0 (registry+https://github.com/rust-lang/crates.io-index)",
"ethabi-derive 9.0.1 (registry+https://github.com/rust-lang/crates.io-index)", "ethabi-derive 9.0.1 (registry+https://github.com/rust-lang/crates.io-index)",
@ -2610,7 +2647,7 @@ name = "memory-db"
version = "0.15.0" version = "0.15.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [ dependencies = [
"hash-db 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)", "hash-db 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)",
"hashmap_core 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)", "hashmap_core 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)",
"parity-util-mem 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", "parity-util-mem 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)",
] ]
@ -3393,14 +3430,14 @@ version = "0.1.0"
dependencies = [ dependencies = [
"elastic-array 0.10.2 (registry+https://github.com/rust-lang/crates.io-index)", "elastic-array 0.10.2 (registry+https://github.com/rust-lang/crates.io-index)",
"ethereum-types 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)", "ethereum-types 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)",
"hash-db 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)", "hash-db 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)",
"journaldb 0.2.0", "journaldb 0.2.0",
"keccak-hash 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", "keccak-hash 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
"keccak-hasher 0.1.1", "keccak-hasher 0.1.1",
"memory-db 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)", "memory-db 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)",
"parity-bytes 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", "parity-bytes 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
"rlp 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", "rlp 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)",
"trie-db 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)", "trie-db 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)",
] ]
[[package]] [[package]]
@ -3486,7 +3523,7 @@ dependencies = [
"common-types 0.1.0", "common-types 0.1.0",
"ethereum-types 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)", "ethereum-types 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)",
"ethjson 0.1.0", "ethjson 0.1.0",
"hash-db 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)", "hash-db 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)",
"itertools 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)", "itertools 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)",
"keccak-hash 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", "keccak-hash 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
"keccak-hasher 0.1.1", "keccak-hasher 0.1.1",
@ -3498,7 +3535,7 @@ dependencies = [
"rlp 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", "rlp 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)",
"rustc-hex 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", "rustc-hex 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)",
"serde 1.0.99 (registry+https://github.com/rust-lang/crates.io-index)", "serde 1.0.99 (registry+https://github.com/rust-lang/crates.io-index)",
"trie-db 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)", "trie-db 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)",
"triehash-ethereum 0.2.0", "triehash-ethereum 0.2.0",
] ]
@ -3854,7 +3891,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [ dependencies = [
"crossbeam-deque 0.6.3 (registry+https://github.com/rust-lang/crates.io-index)", "crossbeam-deque 0.6.3 (registry+https://github.com/rust-lang/crates.io-index)",
"crossbeam-queue 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", "crossbeam-queue 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)",
"crossbeam-utils 0.6.5 (registry+https://github.com/rust-lang/crates.io-index)", "crossbeam-utils 0.6.6 (registry+https://github.com/rust-lang/crates.io-index)",
"lazy_static 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)", "lazy_static 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)",
"num_cpus 1.10.1 (registry+https://github.com/rust-lang/crates.io-index)", "num_cpus 1.10.1 (registry+https://github.com/rust-lang/crates.io-index)",
] ]
@ -4216,7 +4253,7 @@ dependencies = [
"account-state 0.1.0", "account-state 0.1.0",
"client-traits 0.1.0", "client-traits 0.1.0",
"common-types 0.1.0", "common-types 0.1.0",
"crossbeam-utils 0.6.5 (registry+https://github.com/rust-lang/crates.io-index)", "crossbeam-utils 0.6.6 (registry+https://github.com/rust-lang/crates.io-index)",
"engine 0.1.0", "engine 0.1.0",
"env_logger 0.5.13 (registry+https://github.com/rust-lang/crates.io-index)", "env_logger 0.5.13 (registry+https://github.com/rust-lang/crates.io-index)",
"ethabi 9.0.1 (registry+https://github.com/rust-lang/crates.io-index)", "ethabi 9.0.1 (registry+https://github.com/rust-lang/crates.io-index)",
@ -4230,7 +4267,7 @@ dependencies = [
"ethcore-io 1.12.0", "ethcore-io 1.12.0",
"ethereum-types 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)", "ethereum-types 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)",
"ethkey 0.4.0", "ethkey 0.4.0",
"hash-db 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)", "hash-db 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)",
"itertools 0.5.10 (registry+https://github.com/rust-lang/crates.io-index)", "itertools 0.5.10 (registry+https://github.com/rust-lang/crates.io-index)",
"journaldb 0.2.0", "journaldb 0.2.0",
"keccak-hash 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", "keccak-hash 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
@ -4248,11 +4285,12 @@ dependencies = [
"rand_xorshift 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", "rand_xorshift 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)",
"rlp 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", "rlp 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)",
"rlp_derive 0.1.0", "rlp_derive 0.1.0",
"scopeguard 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)",
"snapshot-tests 0.1.0", "snapshot-tests 0.1.0",
"spec 0.1.0", "spec 0.1.0",
"state-db 0.1.0", "state-db 0.1.0",
"tempdir 0.3.7 (registry+https://github.com/rust-lang/crates.io-index)", "tempdir 0.3.7 (registry+https://github.com/rust-lang/crates.io-index)",
"trie-db 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)", "trie-db 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)",
"trie-standardmap 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)", "trie-standardmap 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)",
"triehash-ethereum 0.2.0", "triehash-ethereum 0.2.0",
] ]
@ -4276,7 +4314,7 @@ dependencies = [
"ethcore-db 0.1.0", "ethcore-db 0.1.0",
"ethcore-io 1.12.0", "ethcore-io 1.12.0",
"ethereum-types 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)", "ethereum-types 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)",
"hash-db 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)", "hash-db 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)",
"journaldb 0.2.0", "journaldb 0.2.0",
"keccak-hash 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", "keccak-hash 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
"keccak-hasher 0.1.1", "keccak-hasher 0.1.1",
@ -4295,7 +4333,7 @@ dependencies = [
"snapshot 0.1.0", "snapshot 0.1.0",
"spec 0.1.0", "spec 0.1.0",
"tempdir 0.3.7 (registry+https://github.com/rust-lang/crates.io-index)", "tempdir 0.3.7 (registry+https://github.com/rust-lang/crates.io-index)",
"trie-db 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)", "trie-db 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)",
"trie-standardmap 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)", "trie-standardmap 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)",
"triehash-ethereum 0.2.0", "triehash-ethereum 0.2.0",
] ]
@ -4330,7 +4368,7 @@ dependencies = [
"ethjson 0.1.0", "ethjson 0.1.0",
"evm 0.1.0", "evm 0.1.0",
"executive-state 0.1.0", "executive-state 0.1.0",
"hash-db 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)", "hash-db 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)",
"instant-seal 0.1.0", "instant-seal 0.1.0",
"journaldb 0.2.0", "journaldb 0.2.0",
"keccak-hash 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", "keccak-hash 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
@ -4368,7 +4406,7 @@ dependencies = [
"ethcore-bloom-journal 0.1.0", "ethcore-bloom-journal 0.1.0",
"ethcore-db 0.1.0", "ethcore-db 0.1.0",
"ethereum-types 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)", "ethereum-types 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)",
"hash-db 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)", "hash-db 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)",
"journaldb 0.2.0", "journaldb 0.2.0",
"keccak-hash 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", "keccak-hash 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
"keccak-hasher 0.1.1", "keccak-hasher 0.1.1",
@ -4633,7 +4671,7 @@ name = "tokio-executor"
version = "0.1.8" version = "0.1.8"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [ dependencies = [
"crossbeam-utils 0.6.5 (registry+https://github.com/rust-lang/crates.io-index)", "crossbeam-utils 0.6.6 (registry+https://github.com/rust-lang/crates.io-index)",
"futures 0.1.29 (registry+https://github.com/rust-lang/crates.io-index)", "futures 0.1.29 (registry+https://github.com/rust-lang/crates.io-index)",
] ]
@ -4734,7 +4772,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [ dependencies = [
"crossbeam-deque 0.7.1 (registry+https://github.com/rust-lang/crates.io-index)", "crossbeam-deque 0.7.1 (registry+https://github.com/rust-lang/crates.io-index)",
"crossbeam-queue 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", "crossbeam-queue 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)",
"crossbeam-utils 0.6.5 (registry+https://github.com/rust-lang/crates.io-index)", "crossbeam-utils 0.6.6 (registry+https://github.com/rust-lang/crates.io-index)",
"futures 0.1.29 (registry+https://github.com/rust-lang/crates.io-index)", "futures 0.1.29 (registry+https://github.com/rust-lang/crates.io-index)",
"lazy_static 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)", "lazy_static 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)",
"log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)",
@ -4757,7 +4795,7 @@ name = "tokio-timer"
version = "0.2.11" version = "0.2.11"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [ dependencies = [
"crossbeam-utils 0.6.5 (registry+https://github.com/rust-lang/crates.io-index)", "crossbeam-utils 0.6.6 (registry+https://github.com/rust-lang/crates.io-index)",
"futures 0.1.29 (registry+https://github.com/rust-lang/crates.io-index)", "futures 0.1.29 (registry+https://github.com/rust-lang/crates.io-index)",
"slab 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)", "slab 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)",
"tokio-executor 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)", "tokio-executor 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)",
@ -4862,12 +4900,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
[[package]] [[package]]
name = "trie-db" name = "trie-db"
version = "0.15.0" version = "0.15.2"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [ dependencies = [
"elastic-array 0.10.2 (registry+https://github.com/rust-lang/crates.io-index)", "elastic-array 0.10.2 (registry+https://github.com/rust-lang/crates.io-index)",
"hash-db 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)", "hash-db 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)",
"hashmap_core 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)", "hashbrown 0.6.1 (registry+https://github.com/rust-lang/crates.io-index)",
"log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)",
"rand 0.6.1 (registry+https://github.com/rust-lang/crates.io-index)", "rand 0.6.1 (registry+https://github.com/rust-lang/crates.io-index)",
] ]
@ -4877,7 +4915,7 @@ name = "trie-standardmap"
version = "0.15.0" version = "0.15.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [ dependencies = [
"hash-db 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)", "hash-db 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)",
"keccak-hasher 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)", "keccak-hasher 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)",
] ]
@ -4889,7 +4927,7 @@ dependencies = [
"evm 0.1.0", "evm 0.1.0",
"keccak-hasher 0.1.1", "keccak-hasher 0.1.1",
"patricia-trie-ethereum 0.1.0", "patricia-trie-ethereum 0.1.0",
"trie-db 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)", "trie-db 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)",
"vm 0.1.0", "vm 0.1.0",
"wasm 0.1.0", "wasm 0.1.0",
] ]
@ -4899,7 +4937,7 @@ name = "triehash"
version = "0.8.0" version = "0.8.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [ dependencies = [
"hash-db 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)", "hash-db 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)",
"rlp 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", "rlp 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)",
] ]
@ -5344,6 +5382,7 @@ dependencies = [
"checksum aes-ctr 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "d2e5b0458ea3beae0d1d8c0f3946564f8e10f90646cf78c06b4351052058d1ee" "checksum aes-ctr 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "d2e5b0458ea3beae0d1d8c0f3946564f8e10f90646cf78c06b4351052058d1ee"
"checksum aes-soft 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "cfd7e7ae3f9a1fb5c03b389fc6bb9a51400d0c13053f0dca698c832bfd893a0d" "checksum aes-soft 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "cfd7e7ae3f9a1fb5c03b389fc6bb9a51400d0c13053f0dca698c832bfd893a0d"
"checksum aesni 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)" = "2f70a6b5f971e473091ab7cfb5ffac6cde81666c4556751d8d5620ead8abf100" "checksum aesni 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)" = "2f70a6b5f971e473091ab7cfb5ffac6cde81666c4556751d8d5620ead8abf100"
"checksum ahash 0.2.16 (registry+https://github.com/rust-lang/crates.io-index)" = "b35dfc96a657c1842b4eb73180b65e37152d4b94d0eb5cb51708aee7826950b4"
"checksum aho-corasick 0.6.8 (registry+https://github.com/rust-lang/crates.io-index)" = "68f56c7353e5a9547cbd76ed90f7bb5ffc3ba09d4ea9bd1d8c06c8b1142eeb5a" "checksum aho-corasick 0.6.8 (registry+https://github.com/rust-lang/crates.io-index)" = "68f56c7353e5a9547cbd76ed90f7bb5ffc3ba09d4ea9bd1d8c06c8b1142eeb5a"
"checksum aho-corasick 0.7.6 (registry+https://github.com/rust-lang/crates.io-index)" = "58fb5e95d83b38284460a5fda7d6470aa0b8844d283a0b614b8535e880800d2d" "checksum aho-corasick 0.7.6 (registry+https://github.com/rust-lang/crates.io-index)" = "58fb5e95d83b38284460a5fda7d6470aa0b8844d283a0b614b8535e880800d2d"
"checksum ansi_term 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ee49baf6cb617b853aa8d93bf420db2383fab46d314482ca2803b40d5fde979b" "checksum ansi_term 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ee49baf6cb617b853aa8d93bf420db2383fab46d314482ca2803b40d5fde979b"
@ -5390,6 +5429,8 @@ dependencies = [
"checksum cloudabi 0.0.3 (registry+https://github.com/rust-lang/crates.io-index)" = "ddfc5b9aa5d4507acaf872de71051dfd0e309860e88966e1051e462a077aac4f" "checksum cloudabi 0.0.3 (registry+https://github.com/rust-lang/crates.io-index)" = "ddfc5b9aa5d4507acaf872de71051dfd0e309860e88966e1051e462a077aac4f"
"checksum cmake 0.1.35 (registry+https://github.com/rust-lang/crates.io-index)" = "6ec65ee4f9c9d16f335091d23693457ed4928657ba4982289d7fafee03bc614a" "checksum cmake 0.1.35 (registry+https://github.com/rust-lang/crates.io-index)" = "6ec65ee4f9c9d16f335091d23693457ed4928657ba4982289d7fafee03bc614a"
"checksum combine 3.6.1 (registry+https://github.com/rust-lang/crates.io-index)" = "fc1d011beeed29187b8db2ac3925c8dd4d3e87db463dc9d2d2833985388fc5bc" "checksum combine 3.6.1 (registry+https://github.com/rust-lang/crates.io-index)" = "fc1d011beeed29187b8db2ac3925c8dd4d3e87db463dc9d2d2833985388fc5bc"
"checksum const-random 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)" = "7b641a8c9867e341f3295564203b1c250eb8ce6cb6126e007941f78c4d2ed7fe"
"checksum const-random-macro 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)" = "c750ec12b83377637110d5a57f5ae08e895b06c4b16e2bdbf1a94ef717428c59"
"checksum criterion 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "938703e165481c8d612ea3479ac8342e5615185db37765162e762ec3523e2fc6" "checksum criterion 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "938703e165481c8d612ea3479ac8342e5615185db37765162e762ec3523e2fc6"
"checksum criterion-plot 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "eccdc6ce8bbe352ca89025bee672aa6d24f4eb8c53e3a8b5d1bc58011da072a2" "checksum criterion-plot 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "eccdc6ce8bbe352ca89025bee672aa6d24f4eb8c53e3a8b5d1bc58011da072a2"
"checksum crossbeam-deque 0.6.3 (registry+https://github.com/rust-lang/crates.io-index)" = "05e44b8cf3e1a625844d1750e1f7820da46044ff6d28f4d43e455ba3e5bb2c13" "checksum crossbeam-deque 0.6.3 (registry+https://github.com/rust-lang/crates.io-index)" = "05e44b8cf3e1a625844d1750e1f7820da46044ff6d28f4d43e455ba3e5bb2c13"
@ -5397,7 +5438,7 @@ dependencies = [
"checksum crossbeam-epoch 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)" = "fedcd6772e37f3da2a9af9bf12ebe046c0dfe657992377b4df982a2b54cd37a9" "checksum crossbeam-epoch 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)" = "fedcd6772e37f3da2a9af9bf12ebe046c0dfe657992377b4df982a2b54cd37a9"
"checksum crossbeam-queue 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "7c979cd6cfe72335896575c6b5688da489e420d36a27a0b9eb0c73db574b4a4b" "checksum crossbeam-queue 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "7c979cd6cfe72335896575c6b5688da489e420d36a27a0b9eb0c73db574b4a4b"
"checksum crossbeam-utils 0.5.0 (registry+https://github.com/rust-lang/crates.io-index)" = "677d453a17e8bd2b913fa38e8b9cf04bcdbb5be790aa294f2389661d72036015" "checksum crossbeam-utils 0.5.0 (registry+https://github.com/rust-lang/crates.io-index)" = "677d453a17e8bd2b913fa38e8b9cf04bcdbb5be790aa294f2389661d72036015"
"checksum crossbeam-utils 0.6.5 (registry+https://github.com/rust-lang/crates.io-index)" = "f8306fcef4a7b563b76b7dd949ca48f52bc1141aa067d2ea09565f3e2652aa5c" "checksum crossbeam-utils 0.6.6 (registry+https://github.com/rust-lang/crates.io-index)" = "04973fa96e96579258a5091af6003abde64af786b860f18622b82e026cca60e6"
"checksum crunchy 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)" = "a2f4a431c5c9f662e1200b7c7f02c34e91361150e382089a8f2dec3ba680cbda" "checksum crunchy 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)" = "a2f4a431c5c9f662e1200b7c7f02c34e91361150e382089a8f2dec3ba680cbda"
"checksum crunchy 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "7a81dae078cea95a014a339291cec439d2f232ebe854a9d672b796c6afafa9b7" "checksum crunchy 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "7a81dae078cea95a014a339291cec439d2f232ebe854a9d672b796c6afafa9b7"
"checksum crypto-mac 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "4434400df11d95d556bac068ddfedd482915eb18fe8bea89bc80b6e4b1c179e5" "checksum crypto-mac 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "4434400df11d95d556bac068ddfedd482915eb18fe8bea89bc80b6e4b1c179e5"
@ -5447,8 +5488,9 @@ dependencies = [
"checksum globset 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)" = "4743617a7464bbda3c8aec8558ff2f9429047e025771037df561d383337ff865" "checksum globset 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)" = "4743617a7464bbda3c8aec8558ff2f9429047e025771037df561d383337ff865"
"checksum h2 0.1.26 (registry+https://github.com/rust-lang/crates.io-index)" = "a5b34c246847f938a410a03c5458c7fee2274436675e76d8b903c08efc29c462" "checksum h2 0.1.26 (registry+https://github.com/rust-lang/crates.io-index)" = "a5b34c246847f938a410a03c5458c7fee2274436675e76d8b903c08efc29c462"
"checksum hamming 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)" = "65043da274378d68241eb9a8f8f8aa54e349136f7b8e12f63e3ef44043cc30e1" "checksum hamming 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)" = "65043da274378d68241eb9a8f8f8aa54e349136f7b8e12f63e3ef44043cc30e1"
"checksum hash-db 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)" = "32c87fec93c4a2d264483ef843ac1930ae7c7999d97d73721305a5188b4c23a4" "checksum hash-db 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)" = "d23bd4e7b5eda0d0f3a307e8b381fdc8ba9000f26fbe912250c0a4cc3956364a"
"checksum hash256-std-hasher 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)" = "16293646125e09e5bc216d9f73fa81ab31c4f97007d56c036bbf15a58e970540" "checksum hash256-std-hasher 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)" = "16293646125e09e5bc216d9f73fa81ab31c4f97007d56c036bbf15a58e970540"
"checksum hashbrown 0.6.1 (registry+https://github.com/rust-lang/crates.io-index)" = "6587d09be37fb98a11cb08b9000a3f592451c1b1b613ca69d949160e313a430a"
"checksum hashmap_core 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)" = "8e04cb7a5051270ef3fa79f8c7604d581ecfa73d520e74f554e45541c4b5881a" "checksum hashmap_core 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)" = "8e04cb7a5051270ef3fa79f8c7604d581ecfa73d520e74f554e45541c4b5881a"
"checksum heapsize 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)" = "1679e6ea370dee694f91f1dc469bf94cf8f52051d147aec3e1f9497c6fc22461" "checksum heapsize 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)" = "1679e6ea370dee694f91f1dc469bf94cf8f52051d147aec3e1f9497c6fc22461"
"checksum heck 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ea04fa3ead4e05e51a7c806fc07271fdbde4e246a6c6d1efd52e72230b771b82" "checksum heck 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ea04fa3ead4e05e51a7c806fc07271fdbde4e246a6c6d1efd52e72230b771b82"
@ -5698,7 +5740,7 @@ dependencies = [
"checksum trace-time 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "dbe82f2f0bf1991e163e757baf044282823155dd326e70f44ce2186c3c320cc9" "checksum trace-time 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "dbe82f2f0bf1991e163e757baf044282823155dd326e70f44ce2186c3c320cc9"
"checksum transaction-pool 2.0.1 (registry+https://github.com/rust-lang/crates.io-index)" = "454adc482e32785c3beab9415dd0f3c689f29cc2d16717eb62f6a784d53544b4" "checksum transaction-pool 2.0.1 (registry+https://github.com/rust-lang/crates.io-index)" = "454adc482e32785c3beab9415dd0f3c689f29cc2d16717eb62f6a784d53544b4"
"checksum transient-hashmap 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)" = "aeb4b191d033a35edfce392a38cdcf9790b6cebcb30fa690c312c29da4dc433e" "checksum transient-hashmap 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)" = "aeb4b191d033a35edfce392a38cdcf9790b6cebcb30fa690c312c29da4dc433e"
"checksum trie-db 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)" = "9b65d609ae631d808c6c1cc23a622733d5a0b66a7d67e9f5cd5171562a1f4cb5" "checksum trie-db 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)" = "d0b62d27e8aa1c07414549ac872480ac82380bab39e730242ab08d82d7cc098a"
"checksum trie-standardmap 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)" = "64fda153c00484d640bc91334624be22ead0e5baca917d9fd53ff29bdebcf9b2" "checksum trie-standardmap 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)" = "64fda153c00484d640bc91334624be22ead0e5baca917d9fd53ff29bdebcf9b2"
"checksum triehash 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)" = "61d0a66fa2412c7eb7816640e8ea14cf6bd63b6c824e72315b6ca76d33851134" "checksum triehash 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)" = "61d0a66fa2412c7eb7816640e8ea14cf6bd63b6c824e72315b6ca76d33851134"
"checksum try-lock 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "e604eb7b43c06650e854be16a2a03155743d3752dd1c943f6829e26b7a36e382" "checksum try-lock 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "e604eb7b43c06650e854be16a2a03155743d3752dd1c943f6829e26b7a36e382"

View File

@ -51,6 +51,7 @@ rayon = "1.1"
registrar = { path = "../util/registrar" } registrar = { path = "../util/registrar" }
rlp = "0.4.0" rlp = "0.4.0"
rustc-hex = "2" rustc-hex = "2"
scopeguard = "1.0.0"
serde = "1.0" serde = "1.0"
serde_derive = "1.0" serde_derive = "1.0"
snapshot = { path = "snapshot" } snapshot = { path = "snapshot" }

View File

@ -270,14 +270,12 @@ where
ClientIoMessage::TakeSnapshot(num) => { ClientIoMessage::TakeSnapshot(num) => {
let client = self.client.clone(); let client = self.client.clone();
let snapshot = self.snapshot.clone(); let snapshot = self.snapshot.clone();
let res = thread::Builder::new().name("Periodic Snapshot".into()).spawn(move || { let res = thread::Builder::new().name("Periodic Snapshot".into()).spawn(move || {
if let Err(e) = snapshot.take_snapshot(&*client, num) { if let Err(e) = snapshot.take_snapshot(&*client, num) {
match e { match e {
EthcoreError::Snapshot(SnapshotError::SnapshotAborted) => info!("Snapshot aborted"), EthcoreError::Snapshot(SnapshotError::SnapshotAborted) => info!("Snapshot aborted"),
_ => warn!("Failed to take snapshot at block #{}: {}", num, e), _ => warn!("Failed to take snapshot at block #{}: {}", num, e),
} }
} }
}); });

View File

@ -33,6 +33,7 @@ rand_xorshift = "0.2"
parking_lot = "0.9" parking_lot = "0.9"
rlp = "0.4.2" rlp = "0.4.2"
rlp_derive = { path = "../../util/rlp-derive" } rlp_derive = { path = "../../util/rlp-derive" }
scopeguard = "1.0.0"
snappy = { package = "parity-snappy", version ="0.1.0" } snappy = { package = "parity-snappy", version ="0.1.0" }
state-db = { path = "../state-db" } state-db = { path = "../state-db" }
trie-db = "0.15.0" trie-db = "0.15.0"

View File

@ -48,7 +48,7 @@ fn encoding_basic() {
let thin_rlp = ::rlp::encode(&account); let thin_rlp = ::rlp::encode(&account);
assert_eq!(::rlp::decode::<BasicAccount>(&thin_rlp).unwrap(), account); assert_eq!(::rlp::decode::<BasicAccount>(&thin_rlp).unwrap(), account);
let p = Progress::default(); let p = Progress::new();
let fat_rlps = to_fat_rlps(&keccak(&addr), &account, &AccountDB::from_hash(db.as_hash_db(), keccak(addr)), &mut Default::default(), usize::max_value(), usize::max_value(), &p).unwrap(); let fat_rlps = to_fat_rlps(&keccak(&addr), &account, &AccountDB::from_hash(db.as_hash_db(), keccak(addr)), &mut Default::default(), usize::max_value(), usize::max_value(), &p).unwrap();
let fat_rlp = Rlp::new(&fat_rlps[0]).at(1).unwrap(); let fat_rlp = Rlp::new(&fat_rlps[0]).at(1).unwrap();
assert_eq!(from_fat_rlp(&mut AccountDBMut::from_hash(db.as_hash_db_mut(), keccak(addr)), fat_rlp, H256::zero()).unwrap().0, account); assert_eq!(from_fat_rlp(&mut AccountDBMut::from_hash(db.as_hash_db_mut(), keccak(addr)), fat_rlp, H256::zero()).unwrap().0, account);
@ -69,7 +69,7 @@ fn encoding_version() {
let thin_rlp = ::rlp::encode(&account); let thin_rlp = ::rlp::encode(&account);
assert_eq!(::rlp::decode::<BasicAccount>(&thin_rlp).unwrap(), account); assert_eq!(::rlp::decode::<BasicAccount>(&thin_rlp).unwrap(), account);
let p = Progress::default(); let p = Progress::new();
let fat_rlps = to_fat_rlps(&keccak(&addr), &account, &AccountDB::from_hash(db.as_hash_db(), keccak(addr)), &mut Default::default(), usize::max_value(), usize::max_value(), &p).unwrap(); let fat_rlps = to_fat_rlps(&keccak(&addr), &account, &AccountDB::from_hash(db.as_hash_db(), keccak(addr)), &mut Default::default(), usize::max_value(), usize::max_value(), &p).unwrap();
let fat_rlp = Rlp::new(&fat_rlps[0]).at(1).unwrap(); let fat_rlp = Rlp::new(&fat_rlps[0]).at(1).unwrap();
assert_eq!(from_fat_rlp(&mut AccountDBMut::from_hash(db.as_hash_db_mut(), keccak(addr)), fat_rlp, H256::zero()).unwrap().0, account); assert_eq!(from_fat_rlp(&mut AccountDBMut::from_hash(db.as_hash_db_mut(), keccak(addr)), fat_rlp, H256::zero()).unwrap().0, account);
@ -96,7 +96,7 @@ fn encoding_storage() {
let thin_rlp = ::rlp::encode(&account); let thin_rlp = ::rlp::encode(&account);
assert_eq!(::rlp::decode::<BasicAccount>(&thin_rlp).unwrap(), account); assert_eq!(::rlp::decode::<BasicAccount>(&thin_rlp).unwrap(), account);
let p = Progress::default(); let p = Progress::new();
let fat_rlp = to_fat_rlps(&keccak(&addr), &account, &AccountDB::from_hash(db.as_hash_db(), keccak(addr)), &mut Default::default(), usize::max_value(), usize::max_value(), &p).unwrap(); let fat_rlp = to_fat_rlps(&keccak(&addr), &account, &AccountDB::from_hash(db.as_hash_db(), keccak(addr)), &mut Default::default(), usize::max_value(), usize::max_value(), &p).unwrap();
let fat_rlp = Rlp::new(&fat_rlp[0]).at(1).unwrap(); let fat_rlp = Rlp::new(&fat_rlp[0]).at(1).unwrap();
@ -124,7 +124,7 @@ fn encoding_storage_split() {
let thin_rlp = ::rlp::encode(&account); let thin_rlp = ::rlp::encode(&account);
assert_eq!(::rlp::decode::<BasicAccount>(&thin_rlp).unwrap(), account); assert_eq!(::rlp::decode::<BasicAccount>(&thin_rlp).unwrap(), account);
let p = Progress::default(); let p = Progress::new();
let fat_rlps = to_fat_rlps(&keccak(addr), &account, &AccountDB::from_hash(db.as_hash_db(), keccak(addr)), &mut Default::default(), 500, 1000, &p).unwrap(); let fat_rlps = to_fat_rlps(&keccak(addr), &account, &AccountDB::from_hash(db.as_hash_db(), keccak(addr)), &mut Default::default(), 500, 1000, &p).unwrap();
let mut root = KECCAK_NULL_RLP; let mut root = KECCAK_NULL_RLP;
let mut restored_account = None; let mut restored_account = None;
@ -170,8 +170,8 @@ fn encoding_code() {
}; };
let mut used_code = HashSet::new(); let mut used_code = HashSet::new();
let p1 = Progress::default(); let p1 = Progress::new();
let p2 = Progress::default(); let p2 = Progress::new();
let fat_rlp1 = to_fat_rlps(&keccak(&addr1), &account1, &AccountDB::from_hash(db.as_hash_db(), keccak(addr1)), &mut used_code, usize::max_value(), usize::max_value(), &p1).unwrap(); let fat_rlp1 = to_fat_rlps(&keccak(&addr1), &account1, &AccountDB::from_hash(db.as_hash_db(), keccak(addr1)), &mut used_code, usize::max_value(), usize::max_value(), &p1).unwrap();
let fat_rlp2 = to_fat_rlps(&keccak(&addr2), &account2, &AccountDB::from_hash(db.as_hash_db(), keccak(addr2)), &mut used_code, usize::max_value(), usize::max_value(), &p2).unwrap(); let fat_rlp2 = to_fat_rlps(&keccak(&addr2), &account2, &AccountDB::from_hash(db.as_hash_db(), keccak(addr2)), &mut used_code, usize::max_value(), usize::max_value(), &p2).unwrap();
assert_eq!(used_code.len(), 1); assert_eq!(used_code.len(), 1);

View File

@ -27,7 +27,8 @@ use client_traits::ChainInfo;
use common_types::{ use common_types::{
ids::BlockId, ids::BlockId,
basic_account::BasicAccount, basic_account::BasicAccount,
errors::EthcoreError errors::EthcoreError,
snapshot::Progress,
}; };
use engine::Engine; use engine::Engine;
use ethcore::client::Client; use ethcore::client::Client;
@ -145,7 +146,7 @@ pub fn snap(client: &Client) -> (Box<dyn SnapshotReader>, TempDir) {
let tempdir = TempDir::new("").unwrap(); let tempdir = TempDir::new("").unwrap();
let path = tempdir.path().join("file"); let path = tempdir.path().join("file");
let writer = PackedWriter::new(&path).unwrap(); let writer = PackedWriter::new(&path).unwrap();
let progress = Default::default(); let progress = Progress::new();
let hash = client.chain_info().best_block_hash; let hash = client.chain_info().best_block_hash;
client.take_snapshot(writer, BlockId::Hash(hash), &progress).unwrap(); client.take_snapshot(writer, BlockId::Hash(hash), &progress).unwrap();

View File

@ -74,7 +74,7 @@ fn chunk_and_restore(amount: u64) {
&bc, &bc,
best_hash, best_hash,
&writer, &writer,
&Progress::default() &Progress::new()
).unwrap(); ).unwrap();
let manifest = ManifestData { let manifest = ManifestData {

View File

@ -278,7 +278,7 @@ fn keep_ancient_blocks() {
&bc, &bc,
best_hash, best_hash,
&writer, &writer,
&Progress::default() &Progress::new()
).unwrap(); ).unwrap();
let state_db = client.state_db().journal_db().boxed_clone(); let state_db = client.state_db().journal_db().boxed_clone();
let start_header = bc.block_header_data(&best_hash).unwrap(); let start_header = bc.block_header_data(&best_hash).unwrap();
@ -287,7 +287,7 @@ fn keep_ancient_blocks() {
state_db.as_hash_db(), state_db.as_hash_db(),
&state_root, &state_root,
&writer, &writer,
&Progress::default(), &Progress::new(),
None, None,
0 0
).unwrap(); ).unwrap();

View File

@ -62,7 +62,7 @@ fn snap_and_restore() {
let mut state_hashes = Vec::new(); let mut state_hashes = Vec::new();
for part in 0..SNAPSHOT_SUBPARTS { for part in 0..SNAPSHOT_SUBPARTS {
let mut hashes = chunk_state(&old_db, &state_root, &writer, &Progress::default(), Some(part), 0).unwrap(); let mut hashes = chunk_state(&old_db, &state_root, &writer, &Progress::new(), Some(part), 0).unwrap();
state_hashes.append(&mut hashes); state_hashes.append(&mut hashes);
} }
@ -133,7 +133,7 @@ fn get_code_from_prev_chunk() {
let mut make_chunk = |acc, hash| { let mut make_chunk = |acc, hash| {
let mut db = journaldb::new_memory_db(); let mut db = journaldb::new_memory_db();
AccountDBMut::from_hash(&mut db, hash).insert(EMPTY_PREFIX, &code[..]); AccountDBMut::from_hash(&mut db, hash).insert(EMPTY_PREFIX, &code[..]);
let p = Progress::default(); let p = Progress::new();
let fat_rlp = to_fat_rlps(&hash, &acc, &AccountDB::from_hash(&db, hash), &mut used_code, usize::max_value(), usize::max_value(), &p).unwrap(); let fat_rlp = to_fat_rlps(&hash, &acc, &AccountDB::from_hash(&db, hash), &mut used_code, usize::max_value(), usize::max_value(), &p).unwrap();
let mut stream = RlpStream::new_list(1); let mut stream = RlpStream::new_list(1);
stream.append_raw(&fat_rlp[0], 1); stream.append_raw(&fat_rlp[0], 1);
@ -178,7 +178,7 @@ fn checks_flag() {
let state_root = producer.state_root(); let state_root = producer.state_root();
let writer = Mutex::new(PackedWriter::new(&snap_file).unwrap()); let writer = Mutex::new(PackedWriter::new(&snap_file).unwrap());
let state_hashes = chunk_state(&old_db, &state_root, &writer, &Progress::default(), None, 0).unwrap(); let state_hashes = chunk_state(&old_db, &state_root, &writer, &Progress::new(), None, 0).unwrap();
writer.into_inner().finish(ManifestData { writer.into_inner().finish(ManifestData {
version: 2, version: 2,

View File

@ -41,8 +41,8 @@ impl Oracle for TestOracle {
struct TestBroadcast(Option<u64>); struct TestBroadcast(Option<u64>);
impl Broadcast for TestBroadcast { impl Broadcast for TestBroadcast {
fn take_at(&self, num: Option<u64>) { fn request_snapshot_at(&self, num: u64) {
if num != self.0 { if Some(num) != self.0 {
panic!("Watcher broadcast wrong number. Expected {:?}, found {:?}", self.0, num); panic!("Watcher broadcast wrong number. Expected {:?}, found {:?}", self.0, num);
} }
} }

View File

@ -178,7 +178,7 @@ impl<'a> PowWorker<'a> {
let parent_hash = last_header.parent_hash(); let parent_hash = last_header.parent_hash();
let parent_total_difficulty = last_details.total_difficulty - last_header.difficulty(); let parent_total_difficulty = last_details.total_difficulty - last_header.difficulty();
trace!(target: "snapshot", "parent last written block: {}", parent_hash); trace!(target: "snapshot", "parent last written block: #{}/{}", parent_number, parent_hash);
let num_entries = self.rlps.len(); let num_entries = self.rlps.len();
let mut rlp_stream = RlpStream::new_list(3 + num_entries); let mut rlp_stream = RlpStream::new_list(3 + num_entries);

View File

@ -129,35 +129,34 @@ pub fn take_snapshot<W: SnapshotWriter + Send>(
let state_root = start_header.state_root(); let state_root = start_header.state_root();
let block_number = start_header.number(); let block_number = start_header.number();
info!("Taking snapshot starting at block {}", block_number); info!("Taking snapshot starting at block #{}/{:?}", block_number, block_hash);
let version = chunker.current_version(); let version = chunker.current_version();
let writer = Mutex::new(writer); let writer = Mutex::new(writer);
let (state_hashes, block_hashes) = thread::scope(|scope| -> Result<(Vec<H256>, Vec<H256>), Error> { let (state_hashes, block_hashes) = thread::scope(|scope| -> Result<(Vec<H256>, Vec<H256>), Error> {
let writer = &writer; let writer = &writer;
let block_guard = scope.spawn(move |_| { let tb = scope.builder().name("Snapshot Worker - Blocks".to_string());
let block_guard = tb.spawn(move |_| {
chunk_secondary(chunker, chain, block_hash, writer, p) chunk_secondary(chunker, chain, block_hash, writer, p)
}); })?;
// The number of threads must be between 1 and SNAPSHOT_SUBPARTS // The number of threads must be between 1 and SNAPSHOT_SUBPARTS
assert!(processing_threads >= 1, "Cannot use less than 1 threads for creating snapshots"); assert!(processing_threads >= 1, "Cannot use less than 1 threads for creating snapshots");
let num_threads: usize = cmp::min(processing_threads, SNAPSHOT_SUBPARTS); let num_threads = cmp::min(processing_threads, SNAPSHOT_SUBPARTS);
info!(target: "snapshot", "Using {} threads for Snapshot creation.", num_threads); info!(target: "snapshot", "Using {} threads for Snapshot creation.", num_threads);
let mut state_guards = Vec::with_capacity(num_threads as usize); let mut state_guards = Vec::with_capacity(num_threads);
for thread_idx in 0..num_threads { for thread_idx in 0..num_threads {
let state_guard = scope.spawn(move |_| -> Result<Vec<H256>, Error> { let tb = scope.builder().name(format!("Snapshot Worker #{} - State", thread_idx).to_string());
let state_guard = tb.spawn(move |_| -> Result<Vec<H256>, Error> {
let mut chunk_hashes = Vec::new(); let mut chunk_hashes = Vec::new();
for part in (thread_idx..SNAPSHOT_SUBPARTS).step_by(num_threads) { for part in (thread_idx..SNAPSHOT_SUBPARTS).step_by(num_threads) {
debug!(target: "snapshot", "Chunking part {} in thread {}", part, thread_idx); debug!(target: "snapshot", "Chunking part {} of the state at {} in thread {}", part, block_number, thread_idx);
let mut hashes = chunk_state(state_db, &state_root, writer, p, Some(part), thread_idx)?; let mut hashes = chunk_state(state_db, &state_root, writer, p, Some(part), thread_idx)?;
chunk_hashes.append(&mut hashes); chunk_hashes.append(&mut hashes);
} }
Ok(chunk_hashes) Ok(chunk_hashes)
}); })?;
state_guards.push(state_guard); state_guards.push(state_guard);
} }
@ -169,7 +168,8 @@ pub fn take_snapshot<W: SnapshotWriter + Send>(
state_hashes.extend(part_state_hashes); state_hashes.extend(part_state_hashes);
} }
debug!(target: "snapshot", "Took a snapshot of {} accounts", p.accounts.load(Ordering::SeqCst)); info!("Took a snapshot at #{} of {} accounts", block_number, p.accounts());
Ok((state_hashes, block_hashes)) Ok((state_hashes, block_hashes))
}).expect("Sub-thread never panics; qed")?; }).expect("Sub-thread never panics; qed")?;
@ -218,7 +218,7 @@ pub fn chunk_secondary<'a>(
trace!(target: "snapshot", "wrote secondary chunk. hash: {:x}, size: {}, uncompressed size: {}", trace!(target: "snapshot", "wrote secondary chunk. hash: {:x}, size: {}, uncompressed size: {}",
hash, size, raw_data.len()); hash, size, raw_data.len());
progress.size.fetch_add(size as u64, Ordering::SeqCst); progress.update(0, size);
chunk_hashes.push(hash); chunk_hashes.push(hash);
Ok(()) Ok(())
}; };
@ -275,8 +275,7 @@ impl<'a> StateChunker<'a> {
self.writer.lock().write_state_chunk(hash, compressed)?; self.writer.lock().write_state_chunk(hash, compressed)?;
trace!(target: "snapshot", "Thread {} wrote state chunk. size: {}, uncompressed size: {}", self.thread_idx, compressed_size, raw_data.len()); trace!(target: "snapshot", "Thread {} wrote state chunk. size: {}, uncompressed size: {}", self.thread_idx, compressed_size, raw_data.len());
self.progress.accounts.fetch_add(num_entries, Ordering::SeqCst); self.progress.update(num_entries, compressed_size);
self.progress.size.fetch_add(compressed_size as u64, Ordering::SeqCst);
self.hashes.push(hash); self.hashes.push(hash);
self.cur_size = 0; self.cur_size = 0;
@ -327,7 +326,7 @@ pub fn chunk_state<'a>(
if let Some(part) = part { if let Some(part) = part {
assert!(part < 16, "Wrong chunk state part number (must be <16) in snapshot creation."); assert!(part < 16, "Wrong chunk state part number (must be <16) in snapshot creation.");
let part_offset = MAX_SNAPSHOT_SUBPARTS / SNAPSHOT_SUBPARTS; let part_offset = MAX_SNAPSHOT_SUBPARTS / SNAPSHOT_SUBPARTS; // 16
let mut seek_from = vec![0; 32]; let mut seek_from = vec![0; 32];
seek_from[0] = (part * part_offset) as u8; seek_from[0] = (part * part_offset) as u8;
account_iter.seek(&seek_from)?; account_iter.seek(&seek_from)?;
@ -349,7 +348,15 @@ pub fn chunk_state<'a>(
let account = ::rlp::decode(&*account_data)?; let account = ::rlp::decode(&*account_data)?;
let account_db = AccountDB::from_hash(db, account_key_hash); let account_db = AccountDB::from_hash(db, account_key_hash);
let fat_rlps = account::to_fat_rlps(&account_key_hash, &account, &account_db, &mut used_code, PREFERRED_CHUNK_SIZE - chunker.chunk_size(), PREFERRED_CHUNK_SIZE, progress)?; let fat_rlps = account::to_fat_rlps(
&account_key_hash,
&account,
&account_db,
&mut used_code,
PREFERRED_CHUNK_SIZE - chunker.chunk_size(),
PREFERRED_CHUNK_SIZE,
progress
)?;
for (i, fat_rlp) in fat_rlps.into_iter().enumerate() { for (i, fat_rlp) in fat_rlps.into_iter().enumerate() {
if i > 0 { if i > 0 {
chunker.write_chunk()?; chunker.write_chunk()?;

View File

@ -39,7 +39,7 @@ use ethcore_io::IoChannel;
use journaldb::Algorithm; use journaldb::Algorithm;
use keccak_hash::keccak; use keccak_hash::keccak;
use kvdb::DBTransaction; use kvdb::DBTransaction;
use log::{error, info, trace, warn}; use log::{debug, error, info, trace, warn};
use parking_lot::{Mutex, RwLock, RwLockReadGuard}; use parking_lot::{Mutex, RwLock, RwLockReadGuard};
use snappy; use snappy;
use trie_db::TrieError; use trie_db::TrieError;
@ -212,7 +212,7 @@ impl Restoration {
} }
self.guard.disarm(); self.guard.disarm();
trace!(target: "snapshot", "restoration finalised correctly"); trace!(target: "snapshot", "Restoration finalised correctly");
Ok(()) Ok(())
} }
@ -280,7 +280,7 @@ impl<C> Service<C> where C: SnapshotClient + ChainInfo {
state_chunks: AtomicUsize::new(0), state_chunks: AtomicUsize::new(0),
block_chunks: AtomicUsize::new(0), block_chunks: AtomicUsize::new(0),
client: params.client, client: params.client,
progress: Default::default(), progress: Progress::new(),
taking_snapshot: AtomicBool::new(false), taking_snapshot: AtomicBool::new(false),
restoring_snapshot: AtomicBool::new(false), restoring_snapshot: AtomicBool::new(false),
}; };
@ -413,7 +413,10 @@ impl<C> Service<C> where C: SnapshotClient + ChainInfo {
Some(x) => x, Some(x) => x,
None => return Ok(0), None => return Ok(0),
}; };
info!(target: "snapshot", "Migrating blocks from old db to new. Start: #{}/{:?}, Target: #{}/{:?}",
self.client.block_number(BlockId::Hash(start_hash)).unwrap_or_default(), start_hash,
self.client.block_number(BlockId::Hash(target_hash)).unwrap_or_default(), target_hash,
);
let mut batch = DBTransaction::new(); let mut batch = DBTransaction::new();
let mut parent_hash = start_hash; let mut parent_hash = start_hash;
while parent_hash != target_hash { while parent_hash != target_hash {
@ -455,12 +458,12 @@ impl<C> Service<C> where C: SnapshotClient + ChainInfo {
next_chain.commit(); next_chain.commit();
next_db.key_value().flush().expect("DB flush failed."); next_db.key_value().flush().expect("DB flush failed.");
batch = DBTransaction::new(); batch = DBTransaction::new();
}
if block_number % 10_000 == 0 { if block_number % 10_000 == 0 {
info!(target: "snapshot", "Block restoration at #{}", block_number); info!(target: "snapshot", "Block restoration at #{}", block_number);
} }
} }
}
// Final commit to the DB // Final commit to the DB
next_db.key_value().write_buffered(batch); next_db.key_value().write_buffered(batch);
@ -483,11 +486,13 @@ impl<C> Service<C> where C: SnapshotClient + ChainInfo {
if self.progress.done() || !self.taking_snapshot.load(Ordering::SeqCst) { return } if self.progress.done() || !self.taking_snapshot.load(Ordering::SeqCst) { return }
let p = &self.progress; let p = &self.progress;
info!("Snapshot: {} accounts {} blocks {} bytes", p.accounts(), p.blocks(), p.size()); info!("Snapshot: {} accounts, {} blocks, {} bytes", p.accounts(), p.blocks(), p.bytes());
let rate = p.rate();
debug!(target: "snapshot", "Current progress rate: {:.0} acc/s, {:.0} bytes/s (compressed)", rate.0, rate.1);
} }
/// Take a snapshot at the block with the given number. /// Take a snapshot at the block with the given number.
/// calling this while a restoration is in progress or vice versa /// Calling this while a restoration is in progress or vice versa
/// will lead to a race condition where the first one to finish will /// will lead to a race condition where the first one to finish will
/// have their produced snapshot overwritten. /// have their produced snapshot overwritten.
pub fn take_snapshot(&self, client: &C, num: u64) -> Result<(), Error> { pub fn take_snapshot(&self, client: &C, num: u64) -> Result<(), Error> {
@ -497,46 +502,41 @@ impl<C> Service<C> where C: SnapshotClient + ChainInfo {
} }
info!("Taking snapshot at #{}", num); info!("Taking snapshot at #{}", num);
{
scopeguard::defer! {{
self.taking_snapshot.store(false, Ordering::SeqCst);
}}
let start_time = std::time::Instant::now();
self.progress.reset(); self.progress.reset();
let temp_dir = self.temp_snapshot_dir(); let temp_dir = self.temp_snapshot_dir();
let snapshot_dir = self.snapshot_dir(); let snapshot_dir = self.snapshot_dir();
let _ = fs::remove_dir_all(&temp_dir); let _ = fs::remove_dir_all(&temp_dir); // expected to fail
let writer = LooseWriter::new(temp_dir.clone())?; let writer = LooseWriter::new(temp_dir.clone())?;
let guard = Guard::new(temp_dir.clone()); let guard = Guard::new(temp_dir.clone());
let res = client.take_snapshot(writer, BlockId::Number(num), &self.progress); let _ = client.take_snapshot(writer, BlockId::Number(num), &self.progress)?;
self.taking_snapshot.store(false, Ordering::SeqCst); info!("Finished taking snapshot at #{}, in {:.0?}", num, start_time.elapsed());
if let Err(e) = res {
if client.chain_info().best_block_number >= num + client.pruning_history() {
// The state we were snapshotting was pruned before we could finish.
info!("Periodic snapshot failed: block state pruned. Run with a longer `--pruning-history` or with `--no-periodic-snapshot`");
return Err(e);
} else {
return Err(e);
}
}
info!("Finished taking snapshot at #{}", num);
let mut reader = self.reader.write();
// destroy the old snapshot reader. // destroy the old snapshot reader.
let mut reader = self.reader.write();
*reader = None; *reader = None;
if snapshot_dir.exists() { if snapshot_dir.exists() {
trace!(target: "snapshot", "Removing previous snapshot at {:?}", &snapshot_dir);
fs::remove_dir_all(&snapshot_dir)?; fs::remove_dir_all(&snapshot_dir)?;
} }
fs::rename(temp_dir, &snapshot_dir)?; fs::rename(temp_dir, &snapshot_dir)?;
trace!(target: "snapshot", "Moved new snapshot into place at {:?}", &snapshot_dir);
*reader = Some(LooseReader::new(snapshot_dir)?); *reader = Some(LooseReader::new(snapshot_dir)?);
guard.disarm(); guard.disarm();
Ok(()) Ok(())
} }
}
/// Initialize the restoration synchronously. /// Initialize the restoration synchronously.
/// The recover flag indicates whether to recover the restored snapshot. /// The recover flag indicates whether to recover the restored snapshot.
@ -654,13 +654,19 @@ impl<C> Service<C> where C: SnapshotClient + ChainInfo {
Ok(()) Ok(())
} }
/// Import a previous chunk at the given path. Returns whether the block was imported or not /// Import a previous chunk at the given path. Returns whether the chunk was imported or not
fn import_prev_chunk(&self, restoration: &mut Option<Restoration>, manifest: &ManifestData, file: io::Result<fs::DirEntry>) -> Result<bool, Error> { fn import_prev_chunk(
&self,
restoration: &mut Option<Restoration>,
manifest: &ManifestData,
file: io::Result<fs::DirEntry>
) -> Result<bool, Error> {
let file = file?; let file = file?;
let path = file.path(); let path = file.path();
let mut file = File::open(path.clone())?; let mut file = File::open(path.clone())?;
let mut buffer = Vec::new(); let filesize = file.metadata()?.len();
let mut buffer = Vec::with_capacity(filesize as usize + 1); // +1 for EOF
file.read_to_end(&mut buffer)?; file.read_to_end(&mut buffer)?;
let hash = keccak(&buffer); let hash = keccak(&buffer);
@ -670,6 +676,7 @@ impl<C> Service<C> where C: SnapshotClient + ChainInfo {
} else if manifest.state_hashes.contains(&hash) { } else if manifest.state_hashes.contains(&hash) {
true true
} else { } else {
warn!(target: "snapshot", "Hash of the content of {:?} not present in the manifest block/state hashes.", path);
return Ok(false); return Ok(false);
}; };
@ -680,11 +687,10 @@ impl<C> Service<C> where C: SnapshotClient + ChainInfo {
Ok(true) Ok(true)
} }
// finalize the restoration. this accepts an already-locked // Finalize the restoration. This accepts an already-locked restoration as an argument -- so
// restoration as an argument -- so acquiring it again _will_ // acquiring it again _will_ lead to deadlock.
// lead to deadlock.
fn finalize_restoration(&self, rest: &mut Option<Restoration>) -> Result<(), Error> { fn finalize_restoration(&self, rest: &mut Option<Restoration>) -> Result<(), Error> {
trace!(target: "snapshot", "finalizing restoration"); trace!(target: "snapshot", "Finalizing restoration");
*self.status.lock() = RestorationStatus::Finalizing; *self.status.lock() = RestorationStatus::Finalizing;
let recover = rest.as_ref().map_or(false, |rest| rest.writer.is_some()); let recover = rest.as_ref().map_or(false, |rest| rest.writer.is_some());
@ -695,7 +701,7 @@ impl<C> Service<C> where C: SnapshotClient + ChainInfo {
.unwrap_or(Ok(()))?; .unwrap_or(Ok(()))?;
let migrated_blocks = self.migrate_blocks()?; let migrated_blocks = self.migrate_blocks()?;
info!(target: "snapshot", "Migrated {} ancient blocks", migrated_blocks); info!(target: "snapshot", "Migrated {} ancient blocks from the old DB", migrated_blocks);
// replace the Client's database with the new one (restart the Client). // replace the Client's database with the new one (restart the Client).
self.client.restore_db(&*self.restoration_db().to_string_lossy())?; self.client.restore_db(&*self.restoration_db().to_string_lossy())?;
@ -707,11 +713,11 @@ impl<C> Service<C> where C: SnapshotClient + ChainInfo {
let snapshot_dir = self.snapshot_dir(); let snapshot_dir = self.snapshot_dir();
if snapshot_dir.exists() { if snapshot_dir.exists() {
trace!(target: "snapshot", "removing old snapshot dir at {}", snapshot_dir.to_string_lossy()); trace!(target: "snapshot", "Removing old snapshot dir at {}", snapshot_dir.to_string_lossy());
fs::remove_dir_all(&snapshot_dir)?; fs::remove_dir_all(&snapshot_dir)?;
} }
trace!(target: "snapshot", "copying restored snapshot files over"); trace!(target: "snapshot", "Copying restored snapshot files over");
fs::rename(self.temp_recovery_dir(), &snapshot_dir)?; fs::rename(self.temp_recovery_dir(), &snapshot_dir)?;
*reader = Some(LooseReader::new(snapshot_dir)?); *reader = Some(LooseReader::new(snapshot_dir)?);

View File

@ -136,7 +136,7 @@ pub trait SnapshotComponents: Send {
/// Snapshot related functionality /// Snapshot related functionality
pub trait SnapshotClient: BlockChainClient + BlockInfo + DatabaseRestore + BlockChainReset { pub trait SnapshotClient: BlockChainClient + BlockInfo + DatabaseRestore + BlockChainReset {
/// Take a snapshot at the given block. /// Take a snapshot at the given block.
/// If the ID given is "latest", this will default to 1000 blocks behind. /// If the BlockId is 'Latest', this will default to 1000 blocks behind.
fn take_snapshot<W: SnapshotWriter + Send>( fn take_snapshot<W: SnapshotWriter + Send>(
&self, &self,
writer: W, writer: W,
@ -148,7 +148,7 @@ pub trait SnapshotClient: BlockChainClient + BlockInfo + DatabaseRestore + Block
/// Helper trait for broadcasting a block to take a snapshot at. /// Helper trait for broadcasting a block to take a snapshot at.
pub trait Broadcast: Send + Sync { pub trait Broadcast: Send + Sync {
/// Start a snapshot from the given block number. /// Start a snapshot from the given block number.
fn take_at(&self, num: Option<u64>); fn request_snapshot_at(&self, num: u64);
} }

View File

@ -49,16 +49,11 @@ impl<F> Oracle for StandardOracle<F>
} }
impl<C: 'static> Broadcast for Mutex<IoChannel<ClientIoMessage<C>>> { impl<C: 'static> Broadcast for Mutex<IoChannel<ClientIoMessage<C>>> {
fn take_at(&self, num: Option<u64>) { fn request_snapshot_at(&self, num: u64) {
let num = match num {
Some(n) => n,
None => return,
};
trace!(target: "snapshot_watcher", "Snapshot requested at block #{}", num);
if let Err(e) = self.lock().send(ClientIoMessage::TakeSnapshot(num)) { if let Err(e) = self.lock().send(ClientIoMessage::TakeSnapshot(num)) {
warn!("Snapshot watcher disconnected from IoService: {}", e); warn!(target: "snapshot_watcher", "Snapshot watcher disconnected from IoService: {}", e);
} else {
trace!(target: "snapshot_watcher", "Snapshot requested at block #{}", num);
} }
} }
} }
@ -68,7 +63,10 @@ impl<C: 'static> Broadcast for Mutex<IoChannel<ClientIoMessage<C>>> {
pub struct Watcher { pub struct Watcher {
oracle: Box<dyn Oracle>, oracle: Box<dyn Oracle>,
broadcast: Box<dyn Broadcast>, broadcast: Box<dyn Broadcast>,
// How often we attempt to take a snapshot: only snapshot on blocknumbers that are multiples of
// `period`. Always set to `SNAPSHOT_PERIOD`, i.e. 5000.
period: u64, period: u64,
// Start snapshots `history` blocks from the tip. Always set to `SNAPSHOT_HISTORY`, i.e. 100.
history: u64, history: u64,
} }
@ -106,18 +104,22 @@ impl ChainNotify for Watcher {
fn new_blocks(&self, new_blocks: NewBlocks) { fn new_blocks(&self, new_blocks: NewBlocks) {
if self.oracle.is_major_importing() || new_blocks.has_more_blocks_to_import { return } if self.oracle.is_major_importing() || new_blocks.has_more_blocks_to_import { return }
trace!(target: "snapshot_watcher", "{} imported", new_blocks.imported.len()); // Decide if it's time for a snapshot: the highest of the imported blocks is a multiple of 5000?
let highest = new_blocks.imported.into_iter() let highest = new_blocks.imported.into_iter()
// Convert block hashes to block numbers for all newly imported blocks
.filter_map(|h| self.oracle.to_number(h)) .filter_map(|h| self.oracle.to_number(h))
.filter(|&num| num >= self.period + self.history) // Subtract `history` (i.e. `SNAPSHOT_HISTORY`, i.e. 100) from the block numbers to stay
.map(|num| num - self.history) // clear of reorgs.
.filter(|num| num % self.period == 0) .map(|num| num.saturating_sub(self.history) )
// …filter out blocks that do not fall on the a multiple of `period`. This regulates the
// frequency of snapshots and ensures more snapshots are produced from similar points in
// the chain.
.filter(|num| num % self.period == 0 )
// Pick newest of the candidates: this is where we want to snapshot from.
.fold(0, ::std::cmp::max); .fold(0, ::std::cmp::max);
match highest { if highest > 0 {
0 => self.broadcast.take_at(None), self.broadcast.request_snapshot_at(highest);
_ => self.broadcast.take_at(Some(highest)),
} }
} }
} }

View File

@ -20,7 +20,7 @@ use std::convert::TryFrom;
use std::io::{BufRead, BufReader}; use std::io::{BufRead, BufReader};
use std::str::from_utf8; use std::str::from_utf8;
use std::sync::{Arc, Weak}; use std::sync::{Arc, Weak};
use std::sync::atomic::{AtomicBool, AtomicI64, Ordering as AtomicOrdering}; use std::sync::atomic::{AtomicBool, AtomicI64, Ordering as AtomicOrdering, Ordering, AtomicU64};
use std::time::{Duration, Instant}; use std::time::{Duration, Instant};
use ansi_term::Colour; use ansi_term::Colour;
@ -204,6 +204,9 @@ pub struct Client {
/// Database pruning strategy to use for StateDB /// Database pruning strategy to use for StateDB
pruning: journaldb::Algorithm, pruning: journaldb::Algorithm,
/// Don't prune the state we're currently snapshotting
snapshotting_at: AtomicU64,
/// Client uses this to store blocks, traces, etc. /// Client uses this to store blocks, traces, etc.
db: RwLock<Arc<dyn BlockChainDB>>, db: RwLock<Arc<dyn BlockChainDB>>,
@ -780,6 +783,7 @@ impl Client {
tracedb, tracedb,
engine, engine,
pruning: config.pruning.clone(), pruning: config.pruning.clone(),
snapshotting_at: AtomicU64::new(0),
db: RwLock::new(db.clone()), db: RwLock::new(db.clone()),
state_db: RwLock::new(state_db), state_db: RwLock::new(state_db),
report: RwLock::new(Default::default()), report: RwLock::new(Default::default()),
@ -964,13 +968,15 @@ impl Client {
return Ok(()) return Ok(())
} }
let number = match state_db.journal_db().latest_era() { let latest_era = match state_db.journal_db().latest_era() {
Some(n) => n, Some(n) => n,
None => return Ok(()), None => return Ok(()),
}; };
// prune all ancient eras until we're below the memory target, // Prune all ancient eras until we're below the memory target (default: 32Mb),
// but have at least the minimum number of states. // but have at least the minimum number of states, i.e. `history`.
// If a snapshot is under way, no pruning happens and memory consumption is allowed to
// increase above the memory target until the snapshot has finished.
loop { loop {
let needs_pruning = state_db.journal_db().journal_size() >= self.config.history_mem; let needs_pruning = state_db.journal_db().journal_size() >= self.config.history_mem;
@ -979,17 +985,27 @@ impl Client {
} }
match state_db.journal_db().earliest_era() { match state_db.journal_db().earliest_era() {
Some(era) if era + self.history <= number => { Some(earliest_era) if earliest_era + self.history <= latest_era => {
trace!(target: "client", "Pruning state for ancient era {}", era); let freeze_at = self.snapshotting_at.load(Ordering::SeqCst);
match chain.block_hash(era) { if freeze_at > 0 && freeze_at == earliest_era {
// Note: journal_db().mem_used() can be used for a more accurate memory
// consumption measurement but it can be expensive so sticking with the
// faster `journal_size()` instead.
trace!(target: "pruning", "Pruning is paused at era {} (snapshot under way); earliest era={}, latest era={}, journal_size={} Not pruning.",
freeze_at, earliest_era, latest_era, state_db.journal_db().journal_size());
break;
}
trace!(target: "pruning", "Pruning state for ancient era #{}; latest era={}, journal_size={}",
earliest_era, latest_era, state_db.journal_db().journal_size());
match chain.block_hash(earliest_era) {
Some(ancient_hash) => { Some(ancient_hash) => {
let mut batch = DBTransaction::new(); let mut batch = DBTransaction::new();
state_db.mark_canonical(&mut batch, era, &ancient_hash)?; state_db.mark_canonical(&mut batch, earliest_era, &ancient_hash)?;
self.db.read().key_value().write_buffered(batch); self.db.read().key_value().write_buffered(batch);
state_db.journal_db().flush(); state_db.journal_db().flush();
} }
None => None =>
debug!(target: "client", "Missing expected hash for block {}", era), debug!(target: "pruning", "Missing expected hash for block {}", earliest_era),
} }
} }
_ => break, // means that every era is kept, no pruning necessary. _ => break, // means that every era is kept, no pruning necessary.
@ -2533,39 +2549,53 @@ impl SnapshotClient for Client {
return Err(EthcoreError::Snapshot(SnapshotError::SnapshotsUnsupported)); return Err(EthcoreError::Snapshot(SnapshotError::SnapshotsUnsupported));
} }
let db = self.state_db.read().journal_db().boxed_clone(); let db = self.state_db.read().journal_db().boxed_clone();
let best_block_number = self.chain_info().best_block_number;
let block_number = self.block_number(at).ok_or_else(|| SnapshotError::InvalidStartingBlock(at))?;
if db.is_prunable() && self.pruning_info().earliest_state > block_number { let block_number = self.block_number(at).ok_or_else(|| SnapshotError::InvalidStartingBlock(at))?;
let earliest_era = db.earliest_era().unwrap_or(0);
if db.is_prunable() && earliest_era > block_number {
return Err(SnapshotError::OldBlockPrunedDB.into()); return Err(SnapshotError::OldBlockPrunedDB.into());
} }
let history = cmp::min(self.history, 1000);
let start_hash = match at { let (actual_block_nr, block_hash) = match at {
BlockId::Latest => { BlockId::Latest => {
let start_num = match db.earliest_era() { // Start `self.history` blocks from the best block, but no further back than 1000
Some(era) => cmp::max(era, best_block_number.saturating_sub(history)), // blocks (or earliest era, whichever is greatest).
None => best_block_number.saturating_sub(history), let history = cmp::min(self.history, 1000);
}; let best_block_number = self.chain_info().best_block_number;
let start_num = cmp::max(earliest_era, best_block_number.saturating_sub(history));
match self.block_hash(BlockId::Number(start_num)) { match self.block_hash(BlockId::Number(start_num)) {
Some(h) => h, Some(hash) => (start_num, hash),
None => return Err(SnapshotError::InvalidStartingBlock(at).into()), None => {
error!(target: "snapshot", "Can't take snapshot at {:?}: missing hash for the starting block #{}", at, start_num);
return Err(SnapshotError::InvalidStartingBlock(at).into())
},
} }
} }
_ => match self.block_hash(at) { _ => match self.block_hash(at) {
Some(hash) => hash, Some(hash) => (block_number, hash),
None => return Err(SnapshotError::InvalidStartingBlock(at).into()), None => return Err(SnapshotError::InvalidStartingBlock(at).into()),
}, },
}; };
let processing_threads = self.config.snapshot.processing_threads; let processing_threads = self.config.snapshot.processing_threads;
trace!(target: "snapshot", "Snapshot requested at block {:?}. Using block #{}/{:?}. Earliest block: #{}, earliest state era #{}. Using {} threads.",
at, actual_block_nr, block_hash, self.pruning_info().earliest_chain, earliest_era, processing_threads,
);
// Stop pruning from happening while the snapshot is under way.
self.snapshotting_at.store(actual_block_nr, Ordering::SeqCst);
{
scopeguard::defer! {{
trace!(target: "snapshot", "Re-enabling pruning.");
self.snapshotting_at.store(0, Ordering::SeqCst)
}};
let chunker = snapshot::chunker(self.engine.snapshot_mode()).ok_or_else(|| SnapshotError::SnapshotsUnsupported)?; let chunker = snapshot::chunker(self.engine.snapshot_mode()).ok_or_else(|| SnapshotError::SnapshotsUnsupported)?;
// Spawn threads and take snapshot
snapshot::take_snapshot( snapshot::take_snapshot(
chunker, chunker,
&self.chain.read(), &self.chain.read(),
start_hash, block_hash,
db.as_hash_db(), db.as_hash_db(),
writer, writer,
p, p,
@ -2573,8 +2603,7 @@ impl SnapshotClient for Client {
)?; )?;
Ok(()) Ok(())
} }
}
} }
impl ImportExportBlocks for Client { impl ImportExportBlocks for Client {

View File

@ -90,7 +90,7 @@ pub struct ClientConfig {
pub history: u64, pub history: u64,
/// Ideal memory usage for state pruning history. /// Ideal memory usage for state pruning history.
pub history_mem: usize, pub history_mem: usize,
/// Check seal valididity on block import /// Check seal validity on block import
pub check_seal: bool, pub check_seal: bool,
/// Maximal number of transactions queued for verification in a separate thread. /// Maximal number of transactions queued for verification in a separate thread.
pub transaction_verification_queue_size: usize, pub transaction_verification_queue_size: usize,

View File

@ -15,6 +15,7 @@ parity-bytes = "0.1"
parity-crypto = { version = "0.4.2", features = ["publickey"] } parity-crypto = { version = "0.4.2", features = ["publickey"] }
parity-util-mem = "0.2.0" parity-util-mem = "0.2.0"
parity-snappy = "0.1" parity-snappy = "0.1"
parking_lot = "0.9.0"
patricia-trie-ethereum = { path = "../../util/patricia-trie-ethereum" } patricia-trie-ethereum = { path = "../../util/patricia-trie-ethereum" }
rlp = "0.4.0" rlp = "0.4.0"
rlp_derive = { path = "../../util/rlp-derive" } rlp_derive = { path = "../../util/rlp-derive" }

View File

@ -40,6 +40,7 @@ extern crate parity_crypto;
#[macro_use] #[macro_use]
extern crate derive_more; extern crate derive_more;
extern crate keccak_hash as hash; extern crate keccak_hash as hash;
extern crate parking_lot;
extern crate parity_bytes as bytes; extern crate parity_bytes as bytes;
extern crate patricia_trie_ethereum as ethtrie; extern crate patricia_trie_ethereum as ethtrie;
extern crate rlp; extern crate rlp;

View File

@ -16,11 +16,13 @@
//! Snapshot type definitions //! Snapshot type definitions
use std::sync::atomic::{AtomicBool, AtomicUsize, AtomicU64, Ordering}; use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
use std::time::Instant;
use ethereum_types::H256;
use rlp::{Rlp, RlpStream, DecoderError};
use bytes::Bytes; use bytes::Bytes;
use ethereum_types::H256;
use parking_lot::RwLock;
use rlp::{Rlp, RlpStream, DecoderError};
/// Modes of snapshotting /// Modes of snapshotting
pub enum Snapshotting { pub enum Snapshotting {
@ -39,31 +41,53 @@ pub enum Snapshotting {
} }
/// A progress indicator for snapshots. /// A progress indicator for snapshots.
#[derive(Debug, Default)] #[derive(Debug)]
pub struct Progress { pub struct Progress {
/// Number of accounts processed so far /// Number of accounts processed so far
pub accounts: AtomicUsize, accounts: AtomicUsize,
// Number of accounts processed at last tick.
prev_accounts: AtomicUsize,
/// Number of blocks processed so far /// Number of blocks processed so far
pub blocks: AtomicUsize, pub blocks: AtomicUsize,
/// Size in bytes of a all compressed chunks processed so far /// Size in bytes of a all compressed chunks processed so far
pub size: AtomicU64, bytes: AtomicUsize,
// Number of bytes processed at last tick.
prev_bytes: AtomicUsize,
/// Signals that the snapshotting process is completed /// Signals that the snapshotting process is completed
pub done: AtomicBool, pub done: AtomicBool,
/// Signal snapshotting process to abort /// Signal snapshotting process to abort
pub abort: AtomicBool, pub abort: AtomicBool,
last_tick: RwLock<Instant>,
} }
impl Progress { impl Progress {
/// Create a new progress tracker.
pub fn new() -> Progress {
Progress {
accounts: AtomicUsize::new(0),
prev_accounts: AtomicUsize::new(0),
blocks: AtomicUsize::new(0),
bytes: AtomicUsize::new(0),
prev_bytes: AtomicUsize::new(0),
abort: AtomicBool::new(false),
done: AtomicBool::new(false),
last_tick: RwLock::new(Instant::now()),
}
}
/// Reset the progress. /// Reset the progress.
pub fn reset(&self) { pub fn reset(&self) {
self.accounts.store(0, Ordering::Release); self.accounts.store(0, Ordering::Release);
self.blocks.store(0, Ordering::Release); self.blocks.store(0, Ordering::Release);
self.size.store(0, Ordering::Release); self.bytes.store(0, Ordering::Release);
self.abort.store(false, Ordering::Release); self.abort.store(false, Ordering::Release);
// atomic fence here to ensure the others are written first? // atomic fence here to ensure the others are written first?
// logs might very rarely get polluted if not. // logs might very rarely get polluted if not.
self.done.store(false, Ordering::Release); self.done.store(false, Ordering::Release);
*self.last_tick.write() = Instant::now();
} }
/// Get the number of accounts snapshotted thus far. /// Get the number of accounts snapshotted thus far.
@ -73,10 +97,37 @@ impl Progress {
pub fn blocks(&self) -> usize { self.blocks.load(Ordering::Acquire) } pub fn blocks(&self) -> usize { self.blocks.load(Ordering::Acquire) }
/// Get the written size of the snapshot in bytes. /// Get the written size of the snapshot in bytes.
pub fn size(&self) -> u64 { self.size.load(Ordering::Acquire) } pub fn bytes(&self) -> usize { self.bytes.load(Ordering::Acquire) }
/// Whether the snapshot is complete. /// Whether the snapshot is complete.
pub fn done(&self) -> bool { self.done.load(Ordering::Acquire) } pub fn done(&self) -> bool { self.done.load(Ordering::Acquire) }
/// Return the progress rate over the last tick (i.e. since last update).
pub fn rate(&self) -> (f64, f64) {
let last_tick = *self.last_tick.read();
let dt = last_tick.elapsed().as_secs_f64();
if dt < 1.0 {
return (0f64, 0f64);
}
let delta_acc = self.accounts.load(Ordering::Relaxed)
.saturating_sub(self.prev_accounts.load(Ordering::Relaxed));
let delta_bytes = self.bytes.load(Ordering::Relaxed)
.saturating_sub(self.prev_bytes.load(Ordering::Relaxed));
(delta_acc as f64 / dt, delta_bytes as f64 / dt)
}
/// Update state progress counters and set the last tick.
pub fn update(&self, accounts_delta: usize, bytes_delta: usize) {
*self.last_tick.write() = Instant::now();
self.prev_accounts.store(
self.accounts.fetch_add(accounts_delta, Ordering::SeqCst),
Ordering::SeqCst
);
self.prev_bytes.store(
self.bytes.fetch_add(bytes_delta, Ordering::SeqCst),
Ordering::SeqCst
);
}
} }
/// Manifest data. /// Manifest data.

View File

@ -68,10 +68,10 @@ use signer;
use db; use db;
use registrar::RegistrarClient; use registrar::RegistrarClient;
// how often to take periodic snapshots. // How often we attempt to take a snapshot: only snapshot on blocknumbers that are multiples of this.
const SNAPSHOT_PERIOD: u64 = 5000; const SNAPSHOT_PERIOD: u64 = 5000;
// how many blocks to wait before starting a periodic snapshot. // Start snapshots `history` blocks from the tip. Should be smaller than `SNAPSHOT_HISTORY`.
const SNAPSHOT_HISTORY: u64 = 100; const SNAPSHOT_HISTORY: u64 = 100;
// Number of minutes before a given gas price corpus should expire. // Number of minutes before a given gas price corpus should expire.

View File

@ -257,18 +257,18 @@ impl SnapshotCommand {
let writer = PackedWriter::new(&file_path) let writer = PackedWriter::new(&file_path)
.map_err(|e| format!("Failed to open snapshot writer: {}", e))?; .map_err(|e| format!("Failed to open snapshot writer: {}", e))?;
let progress = Arc::new(Progress::default()); let progress = Arc::new(Progress::new());
let p = progress.clone(); let p = progress.clone();
let informant_handle = ::std::thread::spawn(move || { let informant_handle = ::std::thread::spawn(move || {
::std::thread::sleep(Duration::from_secs(5)); ::std::thread::sleep(Duration::from_secs(5));
let mut last_size = 0; let mut last_size = 0;
while !p.done() { while !p.done() {
let cur_size = p.size(); let cur_size = p.bytes();
if cur_size != last_size { if cur_size != last_size {
last_size = cur_size; last_size = cur_size;
let bytes = ::informant::format_bytes(cur_size as usize); let bytes = ::informant::format_bytes(cur_size as usize);
info!("Snapshot: {} accounts {} blocks {}", p.accounts(), p.blocks(), bytes); info!("Snapshot: {} accounts (state), {} blocks, {} bytes", p.accounts(), p.blocks(), bytes);
} }
::std::thread::sleep(Duration::from_secs(5)); ::std::thread::sleep(Duration::from_secs(5));

View File

@ -84,8 +84,8 @@ pub trait JournalDB: HashDB<KeccakHasher, DBValue> {
/// Get backing database. /// Get backing database.
fn backing(&self) -> &Arc<dyn kvdb::KeyValueDB>; fn backing(&self) -> &Arc<dyn kvdb::KeyValueDB>;
/// Clear internal strucutres. This should called after changes have been written /// Clear internal structure. This should be called after changes have been written
/// to the backing strage /// to the backing storage.
fn flush(&self) {} fn flush(&self) {}
/// Consolidate all the insertions and deletions in the given memory overlay. /// Consolidate all the insertions and deletions in the given memory overlay.

View File

@ -228,10 +228,10 @@ impl OverlayRecentDB {
JournalOverlay { JournalOverlay {
backing_overlay: overlay, backing_overlay: overlay,
pending_overlay: HashMap::default(), pending_overlay: HashMap::default(),
journal: journal, journal,
latest_era: latest_era, latest_era,
earliest_era: earliest_era, earliest_era,
cumulative_size: cumulative_size, cumulative_size,
} }
} }
@ -264,7 +264,6 @@ impl JournalDB for OverlayRecentDB {
fn journal_size(&self) -> usize { fn journal_size(&self) -> usize {
self.journal_overlay.read().cumulative_size self.journal_overlay.read().cumulative_size
} }
fn is_empty(&self) -> bool { fn is_empty(&self) -> bool {
@ -351,7 +350,7 @@ impl JournalDB for OverlayRecentDB {
let mut ops = 0; let mut ops = 0;
// apply old commits' details // apply old commits' details
if let Some(ref mut records) = journal_overlay.journal.get_mut(&end_era) { if let Some(records) = journal_overlay.journal.get_mut(&end_era) {
let mut canon_insertions: Vec<(H256, DBValue)> = Vec::new(); let mut canon_insertions: Vec<(H256, DBValue)> = Vec::new();
let mut canon_deletions: Vec<H256> = Vec::new(); let mut canon_deletions: Vec<H256> = Vec::new();
let mut overlay_deletions: Vec<H256> = Vec::new(); let mut overlay_deletions: Vec<H256> = Vec::new();