Pause pruning while snapshotting (#11178)
* WIP. Typos and logging. * Format todos * Pause pruning while a snapshot is under way Logs, docs and todos * Allocate memory for the full chunk * Name snapshotting threads * Ensure `taking_snapshot` is set to false whenever and however `take_snapshot`returns Rename `take_at` to `request_snapshot_at` Cleanup * Let "in_progress" deletion fail Fix tests * Just use an atomic * Review grumbles * Finish the sentence * Resolve a few todos and clarify comments. * Calculate progress rate since last update * Lockfile * Fix tests * typo * Reinstate default snapshotting frequency Cut down on the logging noise * address grumble * Log memory use with `journal_size()` and explain why.
This commit is contained in:
David
GitHub
parent
acf7c48d7e
commit
ffeaee778c
134
Cargo.lock
generated
134
Cargo.lock
generated
@ -5,7 +5,7 @@ name = "account-db"
|
||||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"ethereum-types 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"hash-db 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"hash-db 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"keccak-hash 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"keccak-hasher 0.1.1",
|
||||
"kvdb 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
@ -20,7 +20,7 @@ dependencies = [
|
||||
"common-types 0.1.0",
|
||||
"derive_more 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"ethereum-types 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"hash-db 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"hash-db 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"journaldb 0.2.0",
|
||||
"keccak-hash 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"keccak-hasher 0.1.1",
|
||||
@ -37,7 +37,7 @@ dependencies = [
|
||||
"rlp_compress 0.1.0",
|
||||
"serde 1.0.99 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"trace 0.1.0",
|
||||
"trie-db 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"trie-db 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"trie-vm-factories 0.1.0",
|
||||
]
|
||||
|
||||
@ -82,6 +82,14 @@ dependencies = [
|
||||
"stream-cipher 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "ahash"
|
||||
version = "0.2.16"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
dependencies = [
|
||||
"const-random 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "aho-corasick"
|
||||
version = "0.6.8"
|
||||
@ -598,6 +606,7 @@ dependencies = [
|
||||
"parity-crypto 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"parity-snappy 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"parity-util-mem 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"parking_lot 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"patricia-trie-ethereum 0.1.0",
|
||||
"rlp 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"rlp_derive 0.1.0",
|
||||
@ -606,6 +615,24 @@ dependencies = [
|
||||
"vm 0.1.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "const-random"
|
||||
version = "0.1.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
dependencies = [
|
||||
"const-random-macro 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"proc-macro-hack 0.5.9 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "const-random-macro"
|
||||
version = "0.1.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
dependencies = [
|
||||
"proc-macro-hack 0.5.9 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"rand 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "criterion"
|
||||
version = "0.3.0"
|
||||
@ -645,7 +672,7 @@ version = "0.6.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
dependencies = [
|
||||
"crossbeam-epoch 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"crossbeam-utils 0.6.5 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"crossbeam-utils 0.6.6 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@ -654,7 +681,7 @@ version = "0.7.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
dependencies = [
|
||||
"crossbeam-epoch 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"crossbeam-utils 0.6.5 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"crossbeam-utils 0.6.6 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@ -664,7 +691,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
dependencies = [
|
||||
"arrayvec 0.4.11 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"cfg-if 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"crossbeam-utils 0.6.5 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"crossbeam-utils 0.6.6 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"lazy_static 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"memoffset 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"scopeguard 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
@ -675,7 +702,7 @@ name = "crossbeam-queue"
|
||||
version = "0.1.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
dependencies = [
|
||||
"crossbeam-utils 0.6.5 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"crossbeam-utils 0.6.6 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@ -685,7 +712,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
|
||||
[[package]]
|
||||
name = "crossbeam-utils"
|
||||
version = "0.6.5"
|
||||
version = "0.6.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
dependencies = [
|
||||
"cfg-if 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
@ -1043,7 +1070,7 @@ dependencies = [
|
||||
"executive-state 0.1.0",
|
||||
"fetch 0.1.0",
|
||||
"futures 0.1.29 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"hash-db 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"hash-db 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"itertools 0.5.10 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"journaldb 0.2.0",
|
||||
"keccak-hash 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
@ -1067,6 +1094,7 @@ dependencies = [
|
||||
"registrar 0.0.1",
|
||||
"rlp 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"rustc-hex 2.0.1 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"scopeguard 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"serde 1.0.99 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"serde_derive 1.0.89 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"serde_json 1.0.40 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
@ -1077,7 +1105,7 @@ dependencies = [
|
||||
"tempdir 0.3.7 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"trace 0.1.0",
|
||||
"trace-time 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"trie-db 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"trie-db 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"trie-standardmap 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"trie-vm-factories 0.1.0",
|
||||
"triehash-ethereum 0.2.0",
|
||||
@ -1216,7 +1244,7 @@ dependencies = [
|
||||
"failsafe 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"fastmap 0.1.0",
|
||||
"futures 0.1.29 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"hash-db 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"hash-db 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"journaldb 0.2.0",
|
||||
"keccak-hash 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"keccak-hasher 0.1.1",
|
||||
@ -1239,7 +1267,7 @@ dependencies = [
|
||||
"spec 0.1.0",
|
||||
"stats 0.1.0",
|
||||
"tempdir 0.3.7 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"trie-db 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"trie-db 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"triehash-ethereum 0.2.0",
|
||||
"verification 0.1.0",
|
||||
"vm 0.1.0",
|
||||
@ -1368,7 +1396,7 @@ dependencies = [
|
||||
"ethjson 0.1.0",
|
||||
"fetch 0.1.0",
|
||||
"futures 0.1.29 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"hash-db 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"hash-db 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"journaldb 0.2.0",
|
||||
"keccak-hash 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"keccak-hasher 0.1.1",
|
||||
@ -1393,7 +1421,7 @@ dependencies = [
|
||||
"tiny-keccak 1.5.0 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"trace 0.1.0",
|
||||
"transaction-pool 2.0.1 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"trie-db 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"trie-db 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"url 2.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"vm 0.1.0",
|
||||
]
|
||||
@ -1668,7 +1696,7 @@ dependencies = [
|
||||
"ethcore 1.12.0",
|
||||
"ethereum-types 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"evm 0.1.0",
|
||||
"hash-db 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"hash-db 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"keccak-hash 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"keccak-hasher 0.1.1",
|
||||
"kvdb 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
@ -1681,7 +1709,7 @@ dependencies = [
|
||||
"rustc-hex 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"spec 0.1.0",
|
||||
"trace 0.1.0",
|
||||
"trie-db 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"trie-db 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"trie-vm-factories 0.1.0",
|
||||
"vm 0.1.0",
|
||||
]
|
||||
@ -1909,7 +1937,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
|
||||
[[package]]
|
||||
name = "hash-db"
|
||||
version = "0.15.0"
|
||||
version = "0.15.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
|
||||
[[package]]
|
||||
@ -1920,6 +1948,15 @@ dependencies = [
|
||||
"crunchy 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "hashbrown"
|
||||
version = "0.6.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
dependencies = [
|
||||
"ahash 0.2.16 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"autocfg 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "hashmap_core"
|
||||
version = "0.1.10"
|
||||
@ -2224,7 +2261,7 @@ dependencies = [
|
||||
"env_logger 0.5.13 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"ethereum-types 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"fastmap 0.1.0",
|
||||
"hash-db 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"hash-db 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"keccak-hash 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"keccak-hasher 0.1.1",
|
||||
"kvdb 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
@ -2352,7 +2389,7 @@ name = "keccak-hasher"
|
||||
version = "0.1.1"
|
||||
dependencies = [
|
||||
"ethereum-types 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"hash-db 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"hash-db 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"plain_hasher 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"tiny-keccak 1.5.0 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
]
|
||||
@ -2362,7 +2399,7 @@ name = "keccak-hasher"
|
||||
version = "0.15.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
dependencies = [
|
||||
"hash-db 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"hash-db 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"hash256-std-hasher 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"tiny-keccak 1.5.0 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
]
|
||||
@ -2525,7 +2562,7 @@ dependencies = [
|
||||
"account-state 0.1.0",
|
||||
"client-traits 0.1.0",
|
||||
"common-types 0.1.0",
|
||||
"crossbeam-utils 0.6.5 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"crossbeam-utils 0.6.6 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"ethabi 9.0.1 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"ethabi-contract 9.0.0 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"ethabi-derive 9.0.1 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
@ -2610,7 +2647,7 @@ name = "memory-db"
|
||||
version = "0.15.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
dependencies = [
|
||||
"hash-db 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"hash-db 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"hashmap_core 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"parity-util-mem 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
]
|
||||
@ -3393,14 +3430,14 @@ version = "0.1.0"
|
||||
dependencies = [
|
||||
"elastic-array 0.10.2 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"ethereum-types 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"hash-db 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"hash-db 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"journaldb 0.2.0",
|
||||
"keccak-hash 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"keccak-hasher 0.1.1",
|
||||
"memory-db 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"parity-bytes 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"rlp 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"trie-db 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"trie-db 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@ -3486,7 +3523,7 @@ dependencies = [
|
||||
"common-types 0.1.0",
|
||||
"ethereum-types 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"ethjson 0.1.0",
|
||||
"hash-db 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"hash-db 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"itertools 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"keccak-hash 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"keccak-hasher 0.1.1",
|
||||
@ -3498,7 +3535,7 @@ dependencies = [
|
||||
"rlp 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"rustc-hex 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"serde 1.0.99 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"trie-db 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"trie-db 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"triehash-ethereum 0.2.0",
|
||||
]
|
||||
|
||||
@ -3854,7 +3891,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
dependencies = [
|
||||
"crossbeam-deque 0.6.3 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"crossbeam-queue 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"crossbeam-utils 0.6.5 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"crossbeam-utils 0.6.6 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"lazy_static 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"num_cpus 1.10.1 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
]
|
||||
@ -4216,7 +4253,7 @@ dependencies = [
|
||||
"account-state 0.1.0",
|
||||
"client-traits 0.1.0",
|
||||
"common-types 0.1.0",
|
||||
"crossbeam-utils 0.6.5 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"crossbeam-utils 0.6.6 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"engine 0.1.0",
|
||||
"env_logger 0.5.13 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"ethabi 9.0.1 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
@ -4230,7 +4267,7 @@ dependencies = [
|
||||
"ethcore-io 1.12.0",
|
||||
"ethereum-types 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"ethkey 0.4.0",
|
||||
"hash-db 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"hash-db 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"itertools 0.5.10 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"journaldb 0.2.0",
|
||||
"keccak-hash 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
@ -4248,11 +4285,12 @@ dependencies = [
|
||||
"rand_xorshift 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"rlp 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"rlp_derive 0.1.0",
|
||||
"scopeguard 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"snapshot-tests 0.1.0",
|
||||
"spec 0.1.0",
|
||||
"state-db 0.1.0",
|
||||
"tempdir 0.3.7 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"trie-db 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"trie-db 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"trie-standardmap 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"triehash-ethereum 0.2.0",
|
||||
]
|
||||
@ -4276,7 +4314,7 @@ dependencies = [
|
||||
"ethcore-db 0.1.0",
|
||||
"ethcore-io 1.12.0",
|
||||
"ethereum-types 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"hash-db 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"hash-db 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"journaldb 0.2.0",
|
||||
"keccak-hash 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"keccak-hasher 0.1.1",
|
||||
@ -4295,7 +4333,7 @@ dependencies = [
|
||||
"snapshot 0.1.0",
|
||||
"spec 0.1.0",
|
||||
"tempdir 0.3.7 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"trie-db 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"trie-db 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"trie-standardmap 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"triehash-ethereum 0.2.0",
|
||||
]
|
||||
@ -4330,7 +4368,7 @@ dependencies = [
|
||||
"ethjson 0.1.0",
|
||||
"evm 0.1.0",
|
||||
"executive-state 0.1.0",
|
||||
"hash-db 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"hash-db 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"instant-seal 0.1.0",
|
||||
"journaldb 0.2.0",
|
||||
"keccak-hash 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
@ -4368,7 +4406,7 @@ dependencies = [
|
||||
"ethcore-bloom-journal 0.1.0",
|
||||
"ethcore-db 0.1.0",
|
||||
"ethereum-types 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"hash-db 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"hash-db 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"journaldb 0.2.0",
|
||||
"keccak-hash 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"keccak-hasher 0.1.1",
|
||||
@ -4633,7 +4671,7 @@ name = "tokio-executor"
|
||||
version = "0.1.8"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
dependencies = [
|
||||
"crossbeam-utils 0.6.5 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"crossbeam-utils 0.6.6 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"futures 0.1.29 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
]
|
||||
|
||||
@ -4734,7 +4772,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
dependencies = [
|
||||
"crossbeam-deque 0.7.1 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"crossbeam-queue 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"crossbeam-utils 0.6.5 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"crossbeam-utils 0.6.6 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"futures 0.1.29 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"lazy_static 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
@ -4757,7 +4795,7 @@ name = "tokio-timer"
|
||||
version = "0.2.11"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
dependencies = [
|
||||
"crossbeam-utils 0.6.5 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"crossbeam-utils 0.6.6 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"futures 0.1.29 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"slab 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"tokio-executor 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
@ -4862,12 +4900,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
|
||||
[[package]]
|
||||
name = "trie-db"
|
||||
version = "0.15.0"
|
||||
version = "0.15.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
dependencies = [
|
||||
"elastic-array 0.10.2 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"hash-db 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"hashmap_core 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"hash-db 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"hashbrown 0.6.1 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"rand 0.6.1 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
]
|
||||
@ -4877,7 +4915,7 @@ name = "trie-standardmap"
|
||||
version = "0.15.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
dependencies = [
|
||||
"hash-db 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"hash-db 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"keccak-hasher 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
]
|
||||
|
||||
@ -4889,7 +4927,7 @@ dependencies = [
|
||||
"evm 0.1.0",
|
||||
"keccak-hasher 0.1.1",
|
||||
"patricia-trie-ethereum 0.1.0",
|
||||
"trie-db 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"trie-db 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"vm 0.1.0",
|
||||
"wasm 0.1.0",
|
||||
]
|
||||
@ -4899,7 +4937,7 @@ name = "triehash"
|
||||
version = "0.8.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
dependencies = [
|
||||
"hash-db 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"hash-db 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"rlp 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
]
|
||||
|
||||
@ -5344,6 +5382,7 @@ dependencies = [
|
||||
"checksum aes-ctr 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "d2e5b0458ea3beae0d1d8c0f3946564f8e10f90646cf78c06b4351052058d1ee"
|
||||
"checksum aes-soft 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "cfd7e7ae3f9a1fb5c03b389fc6bb9a51400d0c13053f0dca698c832bfd893a0d"
|
||||
"checksum aesni 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)" = "2f70a6b5f971e473091ab7cfb5ffac6cde81666c4556751d8d5620ead8abf100"
|
||||
"checksum ahash 0.2.16 (registry+https://github.com/rust-lang/crates.io-index)" = "b35dfc96a657c1842b4eb73180b65e37152d4b94d0eb5cb51708aee7826950b4"
|
||||
"checksum aho-corasick 0.6.8 (registry+https://github.com/rust-lang/crates.io-index)" = "68f56c7353e5a9547cbd76ed90f7bb5ffc3ba09d4ea9bd1d8c06c8b1142eeb5a"
|
||||
"checksum aho-corasick 0.7.6 (registry+https://github.com/rust-lang/crates.io-index)" = "58fb5e95d83b38284460a5fda7d6470aa0b8844d283a0b614b8535e880800d2d"
|
||||
"checksum ansi_term 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ee49baf6cb617b853aa8d93bf420db2383fab46d314482ca2803b40d5fde979b"
|
||||
@ -5390,6 +5429,8 @@ dependencies = [
|
||||
"checksum cloudabi 0.0.3 (registry+https://github.com/rust-lang/crates.io-index)" = "ddfc5b9aa5d4507acaf872de71051dfd0e309860e88966e1051e462a077aac4f"
|
||||
"checksum cmake 0.1.35 (registry+https://github.com/rust-lang/crates.io-index)" = "6ec65ee4f9c9d16f335091d23693457ed4928657ba4982289d7fafee03bc614a"
|
||||
"checksum combine 3.6.1 (registry+https://github.com/rust-lang/crates.io-index)" = "fc1d011beeed29187b8db2ac3925c8dd4d3e87db463dc9d2d2833985388fc5bc"
|
||||
"checksum const-random 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)" = "7b641a8c9867e341f3295564203b1c250eb8ce6cb6126e007941f78c4d2ed7fe"
|
||||
"checksum const-random-macro 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)" = "c750ec12b83377637110d5a57f5ae08e895b06c4b16e2bdbf1a94ef717428c59"
|
||||
"checksum criterion 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "938703e165481c8d612ea3479ac8342e5615185db37765162e762ec3523e2fc6"
|
||||
"checksum criterion-plot 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "eccdc6ce8bbe352ca89025bee672aa6d24f4eb8c53e3a8b5d1bc58011da072a2"
|
||||
"checksum crossbeam-deque 0.6.3 (registry+https://github.com/rust-lang/crates.io-index)" = "05e44b8cf3e1a625844d1750e1f7820da46044ff6d28f4d43e455ba3e5bb2c13"
|
||||
@ -5397,7 +5438,7 @@ dependencies = [
|
||||
"checksum crossbeam-epoch 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)" = "fedcd6772e37f3da2a9af9bf12ebe046c0dfe657992377b4df982a2b54cd37a9"
|
||||
"checksum crossbeam-queue 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "7c979cd6cfe72335896575c6b5688da489e420d36a27a0b9eb0c73db574b4a4b"
|
||||
"checksum crossbeam-utils 0.5.0 (registry+https://github.com/rust-lang/crates.io-index)" = "677d453a17e8bd2b913fa38e8b9cf04bcdbb5be790aa294f2389661d72036015"
|
||||
"checksum crossbeam-utils 0.6.5 (registry+https://github.com/rust-lang/crates.io-index)" = "f8306fcef4a7b563b76b7dd949ca48f52bc1141aa067d2ea09565f3e2652aa5c"
|
||||
"checksum crossbeam-utils 0.6.6 (registry+https://github.com/rust-lang/crates.io-index)" = "04973fa96e96579258a5091af6003abde64af786b860f18622b82e026cca60e6"
|
||||
"checksum crunchy 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)" = "a2f4a431c5c9f662e1200b7c7f02c34e91361150e382089a8f2dec3ba680cbda"
|
||||
"checksum crunchy 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "7a81dae078cea95a014a339291cec439d2f232ebe854a9d672b796c6afafa9b7"
|
||||
"checksum crypto-mac 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "4434400df11d95d556bac068ddfedd482915eb18fe8bea89bc80b6e4b1c179e5"
|
||||
@ -5447,8 +5488,9 @@ dependencies = [
|
||||
"checksum globset 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)" = "4743617a7464bbda3c8aec8558ff2f9429047e025771037df561d383337ff865"
|
||||
"checksum h2 0.1.26 (registry+https://github.com/rust-lang/crates.io-index)" = "a5b34c246847f938a410a03c5458c7fee2274436675e76d8b903c08efc29c462"
|
||||
"checksum hamming 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)" = "65043da274378d68241eb9a8f8f8aa54e349136f7b8e12f63e3ef44043cc30e1"
|
||||
"checksum hash-db 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)" = "32c87fec93c4a2d264483ef843ac1930ae7c7999d97d73721305a5188b4c23a4"
|
||||
"checksum hash-db 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)" = "d23bd4e7b5eda0d0f3a307e8b381fdc8ba9000f26fbe912250c0a4cc3956364a"
|
||||
"checksum hash256-std-hasher 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)" = "16293646125e09e5bc216d9f73fa81ab31c4f97007d56c036bbf15a58e970540"
|
||||
"checksum hashbrown 0.6.1 (registry+https://github.com/rust-lang/crates.io-index)" = "6587d09be37fb98a11cb08b9000a3f592451c1b1b613ca69d949160e313a430a"
|
||||
"checksum hashmap_core 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)" = "8e04cb7a5051270ef3fa79f8c7604d581ecfa73d520e74f554e45541c4b5881a"
|
||||
"checksum heapsize 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)" = "1679e6ea370dee694f91f1dc469bf94cf8f52051d147aec3e1f9497c6fc22461"
|
||||
"checksum heck 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ea04fa3ead4e05e51a7c806fc07271fdbde4e246a6c6d1efd52e72230b771b82"
|
||||
@ -5698,7 +5740,7 @@ dependencies = [
|
||||
"checksum trace-time 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "dbe82f2f0bf1991e163e757baf044282823155dd326e70f44ce2186c3c320cc9"
|
||||
"checksum transaction-pool 2.0.1 (registry+https://github.com/rust-lang/crates.io-index)" = "454adc482e32785c3beab9415dd0f3c689f29cc2d16717eb62f6a784d53544b4"
|
||||
"checksum transient-hashmap 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)" = "aeb4b191d033a35edfce392a38cdcf9790b6cebcb30fa690c312c29da4dc433e"
|
||||
"checksum trie-db 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)" = "9b65d609ae631d808c6c1cc23a622733d5a0b66a7d67e9f5cd5171562a1f4cb5"
|
||||
"checksum trie-db 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)" = "d0b62d27e8aa1c07414549ac872480ac82380bab39e730242ab08d82d7cc098a"
|
||||
"checksum trie-standardmap 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)" = "64fda153c00484d640bc91334624be22ead0e5baca917d9fd53ff29bdebcf9b2"
|
||||
"checksum triehash 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)" = "61d0a66fa2412c7eb7816640e8ea14cf6bd63b6c824e72315b6ca76d33851134"
|
||||
"checksum try-lock 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "e604eb7b43c06650e854be16a2a03155743d3752dd1c943f6829e26b7a36e382"
|
||||
|
||||
@ -51,6 +51,7 @@ rayon = "1.1"
|
||||
registrar = { path = "../util/registrar" }
|
||||
rlp = "0.4.0"
|
||||
rustc-hex = "2"
|
||||
scopeguard = "1.0.0"
|
||||
serde = "1.0"
|
||||
serde_derive = "1.0"
|
||||
snapshot = { path = "snapshot" }
|
||||
|
||||
@ -270,14 +270,12 @@ where
|
||||
ClientIoMessage::TakeSnapshot(num) => {
|
||||
let client = self.client.clone();
|
||||
let snapshot = self.snapshot.clone();
|
||||
|
||||
let res = thread::Builder::new().name("Periodic Snapshot".into()).spawn(move || {
|
||||
if let Err(e) = snapshot.take_snapshot(&*client, num) {
|
||||
match e {
|
||||
EthcoreError::Snapshot(SnapshotError::SnapshotAborted) => info!("Snapshot aborted"),
|
||||
_ => warn!("Failed to take snapshot at block #{}: {}", num, e),
|
||||
}
|
||||
|
||||
}
|
||||
});
|
||||
|
||||
|
||||
@ -33,6 +33,7 @@ rand_xorshift = "0.2"
|
||||
parking_lot = "0.9"
|
||||
rlp = "0.4.2"
|
||||
rlp_derive = { path = "../../util/rlp-derive" }
|
||||
scopeguard = "1.0.0"
|
||||
snappy = { package = "parity-snappy", version ="0.1.0" }
|
||||
state-db = { path = "../state-db" }
|
||||
trie-db = "0.15.0"
|
||||
|
||||
@ -48,7 +48,7 @@ fn encoding_basic() {
|
||||
|
||||
let thin_rlp = ::rlp::encode(&account);
|
||||
assert_eq!(::rlp::decode::<BasicAccount>(&thin_rlp).unwrap(), account);
|
||||
let p = Progress::default();
|
||||
let p = Progress::new();
|
||||
let fat_rlps = to_fat_rlps(&keccak(&addr), &account, &AccountDB::from_hash(db.as_hash_db(), keccak(addr)), &mut Default::default(), usize::max_value(), usize::max_value(), &p).unwrap();
|
||||
let fat_rlp = Rlp::new(&fat_rlps[0]).at(1).unwrap();
|
||||
assert_eq!(from_fat_rlp(&mut AccountDBMut::from_hash(db.as_hash_db_mut(), keccak(addr)), fat_rlp, H256::zero()).unwrap().0, account);
|
||||
@ -69,7 +69,7 @@ fn encoding_version() {
|
||||
|
||||
let thin_rlp = ::rlp::encode(&account);
|
||||
assert_eq!(::rlp::decode::<BasicAccount>(&thin_rlp).unwrap(), account);
|
||||
let p = Progress::default();
|
||||
let p = Progress::new();
|
||||
let fat_rlps = to_fat_rlps(&keccak(&addr), &account, &AccountDB::from_hash(db.as_hash_db(), keccak(addr)), &mut Default::default(), usize::max_value(), usize::max_value(), &p).unwrap();
|
||||
let fat_rlp = Rlp::new(&fat_rlps[0]).at(1).unwrap();
|
||||
assert_eq!(from_fat_rlp(&mut AccountDBMut::from_hash(db.as_hash_db_mut(), keccak(addr)), fat_rlp, H256::zero()).unwrap().0, account);
|
||||
@ -96,7 +96,7 @@ fn encoding_storage() {
|
||||
let thin_rlp = ::rlp::encode(&account);
|
||||
assert_eq!(::rlp::decode::<BasicAccount>(&thin_rlp).unwrap(), account);
|
||||
|
||||
let p = Progress::default();
|
||||
let p = Progress::new();
|
||||
|
||||
let fat_rlp = to_fat_rlps(&keccak(&addr), &account, &AccountDB::from_hash(db.as_hash_db(), keccak(addr)), &mut Default::default(), usize::max_value(), usize::max_value(), &p).unwrap();
|
||||
let fat_rlp = Rlp::new(&fat_rlp[0]).at(1).unwrap();
|
||||
@ -124,7 +124,7 @@ fn encoding_storage_split() {
|
||||
let thin_rlp = ::rlp::encode(&account);
|
||||
assert_eq!(::rlp::decode::<BasicAccount>(&thin_rlp).unwrap(), account);
|
||||
|
||||
let p = Progress::default();
|
||||
let p = Progress::new();
|
||||
let fat_rlps = to_fat_rlps(&keccak(addr), &account, &AccountDB::from_hash(db.as_hash_db(), keccak(addr)), &mut Default::default(), 500, 1000, &p).unwrap();
|
||||
let mut root = KECCAK_NULL_RLP;
|
||||
let mut restored_account = None;
|
||||
@ -170,8 +170,8 @@ fn encoding_code() {
|
||||
};
|
||||
|
||||
let mut used_code = HashSet::new();
|
||||
let p1 = Progress::default();
|
||||
let p2 = Progress::default();
|
||||
let p1 = Progress::new();
|
||||
let p2 = Progress::new();
|
||||
let fat_rlp1 = to_fat_rlps(&keccak(&addr1), &account1, &AccountDB::from_hash(db.as_hash_db(), keccak(addr1)), &mut used_code, usize::max_value(), usize::max_value(), &p1).unwrap();
|
||||
let fat_rlp2 = to_fat_rlps(&keccak(&addr2), &account2, &AccountDB::from_hash(db.as_hash_db(), keccak(addr2)), &mut used_code, usize::max_value(), usize::max_value(), &p2).unwrap();
|
||||
assert_eq!(used_code.len(), 1);
|
||||
|
||||
@ -27,7 +27,8 @@ use client_traits::ChainInfo;
|
||||
use common_types::{
|
||||
ids::BlockId,
|
||||
basic_account::BasicAccount,
|
||||
errors::EthcoreError
|
||||
errors::EthcoreError,
|
||||
snapshot::Progress,
|
||||
};
|
||||
use engine::Engine;
|
||||
use ethcore::client::Client;
|
||||
@ -145,7 +146,7 @@ pub fn snap(client: &Client) -> (Box<dyn SnapshotReader>, TempDir) {
|
||||
let tempdir = TempDir::new("").unwrap();
|
||||
let path = tempdir.path().join("file");
|
||||
let writer = PackedWriter::new(&path).unwrap();
|
||||
let progress = Default::default();
|
||||
let progress = Progress::new();
|
||||
|
||||
let hash = client.chain_info().best_block_hash;
|
||||
client.take_snapshot(writer, BlockId::Hash(hash), &progress).unwrap();
|
||||
|
||||
@ -74,7 +74,7 @@ fn chunk_and_restore(amount: u64) {
|
||||
&bc,
|
||||
best_hash,
|
||||
&writer,
|
||||
&Progress::default()
|
||||
&Progress::new()
|
||||
).unwrap();
|
||||
|
||||
let manifest = ManifestData {
|
||||
|
||||
@ -278,7 +278,7 @@ fn keep_ancient_blocks() {
|
||||
&bc,
|
||||
best_hash,
|
||||
&writer,
|
||||
&Progress::default()
|
||||
&Progress::new()
|
||||
).unwrap();
|
||||
let state_db = client.state_db().journal_db().boxed_clone();
|
||||
let start_header = bc.block_header_data(&best_hash).unwrap();
|
||||
@ -287,7 +287,7 @@ fn keep_ancient_blocks() {
|
||||
state_db.as_hash_db(),
|
||||
&state_root,
|
||||
&writer,
|
||||
&Progress::default(),
|
||||
&Progress::new(),
|
||||
None,
|
||||
0
|
||||
).unwrap();
|
||||
|
||||
@ -62,7 +62,7 @@ fn snap_and_restore() {
|
||||
|
||||
let mut state_hashes = Vec::new();
|
||||
for part in 0..SNAPSHOT_SUBPARTS {
|
||||
let mut hashes = chunk_state(&old_db, &state_root, &writer, &Progress::default(), Some(part), 0).unwrap();
|
||||
let mut hashes = chunk_state(&old_db, &state_root, &writer, &Progress::new(), Some(part), 0).unwrap();
|
||||
state_hashes.append(&mut hashes);
|
||||
}
|
||||
|
||||
@ -133,7 +133,7 @@ fn get_code_from_prev_chunk() {
|
||||
let mut make_chunk = |acc, hash| {
|
||||
let mut db = journaldb::new_memory_db();
|
||||
AccountDBMut::from_hash(&mut db, hash).insert(EMPTY_PREFIX, &code[..]);
|
||||
let p = Progress::default();
|
||||
let p = Progress::new();
|
||||
let fat_rlp = to_fat_rlps(&hash, &acc, &AccountDB::from_hash(&db, hash), &mut used_code, usize::max_value(), usize::max_value(), &p).unwrap();
|
||||
let mut stream = RlpStream::new_list(1);
|
||||
stream.append_raw(&fat_rlp[0], 1);
|
||||
@ -178,7 +178,7 @@ fn checks_flag() {
|
||||
let state_root = producer.state_root();
|
||||
let writer = Mutex::new(PackedWriter::new(&snap_file).unwrap());
|
||||
|
||||
let state_hashes = chunk_state(&old_db, &state_root, &writer, &Progress::default(), None, 0).unwrap();
|
||||
let state_hashes = chunk_state(&old_db, &state_root, &writer, &Progress::new(), None, 0).unwrap();
|
||||
|
||||
writer.into_inner().finish(ManifestData {
|
||||
version: 2,
|
||||
|
||||
@ -41,8 +41,8 @@ impl Oracle for TestOracle {
|
||||
|
||||
struct TestBroadcast(Option<u64>);
|
||||
impl Broadcast for TestBroadcast {
|
||||
fn take_at(&self, num: Option<u64>) {
|
||||
if num != self.0 {
|
||||
fn request_snapshot_at(&self, num: u64) {
|
||||
if Some(num) != self.0 {
|
||||
panic!("Watcher broadcast wrong number. Expected {:?}, found {:?}", self.0, num);
|
||||
}
|
||||
}
|
||||
|
||||
@ -178,7 +178,7 @@ impl<'a> PowWorker<'a> {
|
||||
let parent_hash = last_header.parent_hash();
|
||||
let parent_total_difficulty = last_details.total_difficulty - last_header.difficulty();
|
||||
|
||||
trace!(target: "snapshot", "parent last written block: {}", parent_hash);
|
||||
trace!(target: "snapshot", "parent last written block: #{}/{}", parent_number, parent_hash);
|
||||
|
||||
let num_entries = self.rlps.len();
|
||||
let mut rlp_stream = RlpStream::new_list(3 + num_entries);
|
||||
|
||||
@ -129,35 +129,34 @@ pub fn take_snapshot<W: SnapshotWriter + Send>(
|
||||
let state_root = start_header.state_root();
|
||||
let block_number = start_header.number();
|
||||
|
||||
info!("Taking snapshot starting at block {}", block_number);
|
||||
|
||||
info!("Taking snapshot starting at block #{}/{:?}", block_number, block_hash);
|
||||
let version = chunker.current_version();
|
||||
let writer = Mutex::new(writer);
|
||||
let (state_hashes, block_hashes) = thread::scope(|scope| -> Result<(Vec<H256>, Vec<H256>), Error> {
|
||||
let writer = &writer;
|
||||
let block_guard = scope.spawn(move |_| {
|
||||
let tb = scope.builder().name("Snapshot Worker - Blocks".to_string());
|
||||
let block_guard = tb.spawn(move |_| {
|
||||
chunk_secondary(chunker, chain, block_hash, writer, p)
|
||||
});
|
||||
})?;
|
||||
|
||||
// The number of threads must be between 1 and SNAPSHOT_SUBPARTS
|
||||
assert!(processing_threads >= 1, "Cannot use less than 1 threads for creating snapshots");
|
||||
let num_threads: usize = cmp::min(processing_threads, SNAPSHOT_SUBPARTS);
|
||||
let num_threads = cmp::min(processing_threads, SNAPSHOT_SUBPARTS);
|
||||
info!(target: "snapshot", "Using {} threads for Snapshot creation.", num_threads);
|
||||
|
||||
let mut state_guards = Vec::with_capacity(num_threads as usize);
|
||||
let mut state_guards = Vec::with_capacity(num_threads);
|
||||
|
||||
for thread_idx in 0..num_threads {
|
||||
let state_guard = scope.spawn(move |_| -> Result<Vec<H256>, Error> {
|
||||
let tb = scope.builder().name(format!("Snapshot Worker #{} - State", thread_idx).to_string());
|
||||
let state_guard = tb.spawn(move |_| -> Result<Vec<H256>, Error> {
|
||||
let mut chunk_hashes = Vec::new();
|
||||
|
||||
for part in (thread_idx..SNAPSHOT_SUBPARTS).step_by(num_threads) {
|
||||
debug!(target: "snapshot", "Chunking part {} in thread {}", part, thread_idx);
|
||||
debug!(target: "snapshot", "Chunking part {} of the state at {} in thread {}", part, block_number, thread_idx);
|
||||
let mut hashes = chunk_state(state_db, &state_root, writer, p, Some(part), thread_idx)?;
|
||||
chunk_hashes.append(&mut hashes);
|
||||
}
|
||||
|
||||
Ok(chunk_hashes)
|
||||
});
|
||||
})?;
|
||||
state_guards.push(state_guard);
|
||||
}
|
||||
|
||||
@ -169,7 +168,8 @@ pub fn take_snapshot<W: SnapshotWriter + Send>(
|
||||
state_hashes.extend(part_state_hashes);
|
||||
}
|
||||
|
||||
debug!(target: "snapshot", "Took a snapshot of {} accounts", p.accounts.load(Ordering::SeqCst));
|
||||
info!("Took a snapshot at #{} of {} accounts", block_number, p.accounts());
|
||||
|
||||
Ok((state_hashes, block_hashes))
|
||||
}).expect("Sub-thread never panics; qed")?;
|
||||
|
||||
@ -218,7 +218,7 @@ pub fn chunk_secondary<'a>(
|
||||
trace!(target: "snapshot", "wrote secondary chunk. hash: {:x}, size: {}, uncompressed size: {}",
|
||||
hash, size, raw_data.len());
|
||||
|
||||
progress.size.fetch_add(size as u64, Ordering::SeqCst);
|
||||
progress.update(0, size);
|
||||
chunk_hashes.push(hash);
|
||||
Ok(())
|
||||
};
|
||||
@ -275,8 +275,7 @@ impl<'a> StateChunker<'a> {
|
||||
self.writer.lock().write_state_chunk(hash, compressed)?;
|
||||
trace!(target: "snapshot", "Thread {} wrote state chunk. size: {}, uncompressed size: {}", self.thread_idx, compressed_size, raw_data.len());
|
||||
|
||||
self.progress.accounts.fetch_add(num_entries, Ordering::SeqCst);
|
||||
self.progress.size.fetch_add(compressed_size as u64, Ordering::SeqCst);
|
||||
self.progress.update(num_entries, compressed_size);
|
||||
|
||||
self.hashes.push(hash);
|
||||
self.cur_size = 0;
|
||||
@ -327,7 +326,7 @@ pub fn chunk_state<'a>(
|
||||
if let Some(part) = part {
|
||||
assert!(part < 16, "Wrong chunk state part number (must be <16) in snapshot creation.");
|
||||
|
||||
let part_offset = MAX_SNAPSHOT_SUBPARTS / SNAPSHOT_SUBPARTS;
|
||||
let part_offset = MAX_SNAPSHOT_SUBPARTS / SNAPSHOT_SUBPARTS; // 16
|
||||
let mut seek_from = vec![0; 32];
|
||||
seek_from[0] = (part * part_offset) as u8;
|
||||
account_iter.seek(&seek_from)?;
|
||||
@ -349,7 +348,15 @@ pub fn chunk_state<'a>(
|
||||
let account = ::rlp::decode(&*account_data)?;
|
||||
let account_db = AccountDB::from_hash(db, account_key_hash);
|
||||
|
||||
let fat_rlps = account::to_fat_rlps(&account_key_hash, &account, &account_db, &mut used_code, PREFERRED_CHUNK_SIZE - chunker.chunk_size(), PREFERRED_CHUNK_SIZE, progress)?;
|
||||
let fat_rlps = account::to_fat_rlps(
|
||||
&account_key_hash,
|
||||
&account,
|
||||
&account_db,
|
||||
&mut used_code,
|
||||
PREFERRED_CHUNK_SIZE - chunker.chunk_size(),
|
||||
PREFERRED_CHUNK_SIZE,
|
||||
progress
|
||||
)?;
|
||||
for (i, fat_rlp) in fat_rlps.into_iter().enumerate() {
|
||||
if i > 0 {
|
||||
chunker.write_chunk()?;
|
||||
|
||||
@ -39,7 +39,7 @@ use ethcore_io::IoChannel;
|
||||
use journaldb::Algorithm;
|
||||
use keccak_hash::keccak;
|
||||
use kvdb::DBTransaction;
|
||||
use log::{error, info, trace, warn};
|
||||
use log::{debug, error, info, trace, warn};
|
||||
use parking_lot::{Mutex, RwLock, RwLockReadGuard};
|
||||
use snappy;
|
||||
use trie_db::TrieError;
|
||||
@ -212,7 +212,7 @@ impl Restoration {
|
||||
}
|
||||
|
||||
self.guard.disarm();
|
||||
trace!(target: "snapshot", "restoration finalised correctly");
|
||||
trace!(target: "snapshot", "Restoration finalised correctly");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@ -280,7 +280,7 @@ impl<C> Service<C> where C: SnapshotClient + ChainInfo {
|
||||
state_chunks: AtomicUsize::new(0),
|
||||
block_chunks: AtomicUsize::new(0),
|
||||
client: params.client,
|
||||
progress: Default::default(),
|
||||
progress: Progress::new(),
|
||||
taking_snapshot: AtomicBool::new(false),
|
||||
restoring_snapshot: AtomicBool::new(false),
|
||||
};
|
||||
@ -413,7 +413,10 @@ impl<C> Service<C> where C: SnapshotClient + ChainInfo {
|
||||
Some(x) => x,
|
||||
None => return Ok(0),
|
||||
};
|
||||
|
||||
info!(target: "snapshot", "Migrating blocks from old db to new. Start: #{}/{:?}, Target: #{}/{:?}",
|
||||
self.client.block_number(BlockId::Hash(start_hash)).unwrap_or_default(), start_hash,
|
||||
self.client.block_number(BlockId::Hash(target_hash)).unwrap_or_default(), target_hash,
|
||||
);
|
||||
let mut batch = DBTransaction::new();
|
||||
let mut parent_hash = start_hash;
|
||||
while parent_hash != target_hash {
|
||||
@ -455,10 +458,10 @@ impl<C> Service<C> where C: SnapshotClient + ChainInfo {
|
||||
next_chain.commit();
|
||||
next_db.key_value().flush().expect("DB flush failed.");
|
||||
batch = DBTransaction::new();
|
||||
}
|
||||
|
||||
if block_number % 10_000 == 0 {
|
||||
info!(target: "snapshot", "Block restoration at #{}", block_number);
|
||||
if block_number % 10_000 == 0 {
|
||||
info!(target: "snapshot", "Block restoration at #{}", block_number);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@ -483,11 +486,13 @@ impl<C> Service<C> where C: SnapshotClient + ChainInfo {
|
||||
if self.progress.done() || !self.taking_snapshot.load(Ordering::SeqCst) { return }
|
||||
|
||||
let p = &self.progress;
|
||||
info!("Snapshot: {} accounts {} blocks {} bytes", p.accounts(), p.blocks(), p.size());
|
||||
info!("Snapshot: {} accounts, {} blocks, {} bytes", p.accounts(), p.blocks(), p.bytes());
|
||||
let rate = p.rate();
|
||||
debug!(target: "snapshot", "Current progress rate: {:.0} acc/s, {:.0} bytes/s (compressed)", rate.0, rate.1);
|
||||
}
|
||||
|
||||
/// Take a snapshot at the block with the given number.
|
||||
/// calling this while a restoration is in progress or vice versa
|
||||
/// Calling this while a restoration is in progress or vice versa
|
||||
/// will lead to a race condition where the first one to finish will
|
||||
/// have their produced snapshot overwritten.
|
||||
pub fn take_snapshot(&self, client: &C, num: u64) -> Result<(), Error> {
|
||||
@ -497,45 +502,40 @@ impl<C> Service<C> where C: SnapshotClient + ChainInfo {
|
||||
}
|
||||
|
||||
info!("Taking snapshot at #{}", num);
|
||||
self.progress.reset();
|
||||
{
|
||||
scopeguard::defer! {{
|
||||
self.taking_snapshot.store(false, Ordering::SeqCst);
|
||||
}}
|
||||
let start_time = std::time::Instant::now();
|
||||
self.progress.reset();
|
||||
|
||||
let temp_dir = self.temp_snapshot_dir();
|
||||
let snapshot_dir = self.snapshot_dir();
|
||||
let temp_dir = self.temp_snapshot_dir();
|
||||
let snapshot_dir = self.snapshot_dir();
|
||||
|
||||
let _ = fs::remove_dir_all(&temp_dir);
|
||||
let _ = fs::remove_dir_all(&temp_dir); // expected to fail
|
||||
|
||||
let writer = LooseWriter::new(temp_dir.clone())?;
|
||||
let writer = LooseWriter::new(temp_dir.clone())?;
|
||||
|
||||
let guard = Guard::new(temp_dir.clone());
|
||||
let res = client.take_snapshot(writer, BlockId::Number(num), &self.progress);
|
||||
self.taking_snapshot.store(false, Ordering::SeqCst);
|
||||
if let Err(e) = res {
|
||||
if client.chain_info().best_block_number >= num + client.pruning_history() {
|
||||
// The state we were snapshotting was pruned before we could finish.
|
||||
info!("Periodic snapshot failed: block state pruned. Run with a longer `--pruning-history` or with `--no-periodic-snapshot`");
|
||||
return Err(e);
|
||||
} else {
|
||||
return Err(e);
|
||||
let guard = Guard::new(temp_dir.clone());
|
||||
let _ = client.take_snapshot(writer, BlockId::Number(num), &self.progress)?;
|
||||
info!("Finished taking snapshot at #{}, in {:.0?}", num, start_time.elapsed());
|
||||
|
||||
// destroy the old snapshot reader.
|
||||
let mut reader = self.reader.write();
|
||||
*reader = None;
|
||||
|
||||
if snapshot_dir.exists() {
|
||||
trace!(target: "snapshot", "Removing previous snapshot at {:?}", &snapshot_dir);
|
||||
fs::remove_dir_all(&snapshot_dir)?;
|
||||
}
|
||||
|
||||
fs::rename(temp_dir, &snapshot_dir)?;
|
||||
trace!(target: "snapshot", "Moved new snapshot into place at {:?}", &snapshot_dir);
|
||||
*reader = Some(LooseReader::new(snapshot_dir)?);
|
||||
|
||||
guard.disarm();
|
||||
Ok(())
|
||||
}
|
||||
|
||||
info!("Finished taking snapshot at #{}", num);
|
||||
|
||||
let mut reader = self.reader.write();
|
||||
|
||||
// destroy the old snapshot reader.
|
||||
*reader = None;
|
||||
|
||||
if snapshot_dir.exists() {
|
||||
fs::remove_dir_all(&snapshot_dir)?;
|
||||
}
|
||||
|
||||
fs::rename(temp_dir, &snapshot_dir)?;
|
||||
|
||||
*reader = Some(LooseReader::new(snapshot_dir)?);
|
||||
|
||||
guard.disarm();
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Initialize the restoration synchronously.
|
||||
@ -654,13 +654,19 @@ impl<C> Service<C> where C: SnapshotClient + ChainInfo {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Import a previous chunk at the given path. Returns whether the block was imported or not
|
||||
fn import_prev_chunk(&self, restoration: &mut Option<Restoration>, manifest: &ManifestData, file: io::Result<fs::DirEntry>) -> Result<bool, Error> {
|
||||
/// Import a previous chunk at the given path. Returns whether the chunk was imported or not
|
||||
fn import_prev_chunk(
|
||||
&self,
|
||||
restoration: &mut Option<Restoration>,
|
||||
manifest: &ManifestData,
|
||||
file: io::Result<fs::DirEntry>
|
||||
) -> Result<bool, Error> {
|
||||
let file = file?;
|
||||
let path = file.path();
|
||||
|
||||
let mut file = File::open(path.clone())?;
|
||||
let mut buffer = Vec::new();
|
||||
let filesize = file.metadata()?.len();
|
||||
let mut buffer = Vec::with_capacity(filesize as usize + 1); // +1 for EOF
|
||||
file.read_to_end(&mut buffer)?;
|
||||
|
||||
let hash = keccak(&buffer);
|
||||
@ -670,6 +676,7 @@ impl<C> Service<C> where C: SnapshotClient + ChainInfo {
|
||||
} else if manifest.state_hashes.contains(&hash) {
|
||||
true
|
||||
} else {
|
||||
warn!(target: "snapshot", "Hash of the content of {:?} not present in the manifest block/state hashes.", path);
|
||||
return Ok(false);
|
||||
};
|
||||
|
||||
@ -680,11 +687,10 @@ impl<C> Service<C> where C: SnapshotClient + ChainInfo {
|
||||
Ok(true)
|
||||
}
|
||||
|
||||
// finalize the restoration. this accepts an already-locked
|
||||
// restoration as an argument -- so acquiring it again _will_
|
||||
// lead to deadlock.
|
||||
// Finalize the restoration. This accepts an already-locked restoration as an argument -- so
|
||||
// acquiring it again _will_ lead to deadlock.
|
||||
fn finalize_restoration(&self, rest: &mut Option<Restoration>) -> Result<(), Error> {
|
||||
trace!(target: "snapshot", "finalizing restoration");
|
||||
trace!(target: "snapshot", "Finalizing restoration");
|
||||
*self.status.lock() = RestorationStatus::Finalizing;
|
||||
|
||||
let recover = rest.as_ref().map_or(false, |rest| rest.writer.is_some());
|
||||
@ -695,7 +701,7 @@ impl<C> Service<C> where C: SnapshotClient + ChainInfo {
|
||||
.unwrap_or(Ok(()))?;
|
||||
|
||||
let migrated_blocks = self.migrate_blocks()?;
|
||||
info!(target: "snapshot", "Migrated {} ancient blocks", migrated_blocks);
|
||||
info!(target: "snapshot", "Migrated {} ancient blocks from the old DB", migrated_blocks);
|
||||
|
||||
// replace the Client's database with the new one (restart the Client).
|
||||
self.client.restore_db(&*self.restoration_db().to_string_lossy())?;
|
||||
@ -707,11 +713,11 @@ impl<C> Service<C> where C: SnapshotClient + ChainInfo {
|
||||
let snapshot_dir = self.snapshot_dir();
|
||||
|
||||
if snapshot_dir.exists() {
|
||||
trace!(target: "snapshot", "removing old snapshot dir at {}", snapshot_dir.to_string_lossy());
|
||||
trace!(target: "snapshot", "Removing old snapshot dir at {}", snapshot_dir.to_string_lossy());
|
||||
fs::remove_dir_all(&snapshot_dir)?;
|
||||
}
|
||||
|
||||
trace!(target: "snapshot", "copying restored snapshot files over");
|
||||
trace!(target: "snapshot", "Copying restored snapshot files over");
|
||||
fs::rename(self.temp_recovery_dir(), &snapshot_dir)?;
|
||||
|
||||
*reader = Some(LooseReader::new(snapshot_dir)?);
|
||||
|
||||
@ -136,7 +136,7 @@ pub trait SnapshotComponents: Send {
|
||||
/// Snapshot related functionality
|
||||
pub trait SnapshotClient: BlockChainClient + BlockInfo + DatabaseRestore + BlockChainReset {
|
||||
/// Take a snapshot at the given block.
|
||||
/// If the ID given is "latest", this will default to 1000 blocks behind.
|
||||
/// If the BlockId is 'Latest', this will default to 1000 blocks behind.
|
||||
fn take_snapshot<W: SnapshotWriter + Send>(
|
||||
&self,
|
||||
writer: W,
|
||||
@ -148,7 +148,7 @@ pub trait SnapshotClient: BlockChainClient + BlockInfo + DatabaseRestore + Block
|
||||
/// Helper trait for broadcasting a block to take a snapshot at.
|
||||
pub trait Broadcast: Send + Sync {
|
||||
/// Start a snapshot from the given block number.
|
||||
fn take_at(&self, num: Option<u64>);
|
||||
fn request_snapshot_at(&self, num: u64);
|
||||
}
|
||||
|
||||
|
||||
|
||||
@ -49,16 +49,11 @@ impl<F> Oracle for StandardOracle<F>
|
||||
}
|
||||
|
||||
impl<C: 'static> Broadcast for Mutex<IoChannel<ClientIoMessage<C>>> {
|
||||
fn take_at(&self, num: Option<u64>) {
|
||||
let num = match num {
|
||||
Some(n) => n,
|
||||
None => return,
|
||||
};
|
||||
|
||||
trace!(target: "snapshot_watcher", "Snapshot requested at block #{}", num);
|
||||
|
||||
fn request_snapshot_at(&self, num: u64) {
|
||||
if let Err(e) = self.lock().send(ClientIoMessage::TakeSnapshot(num)) {
|
||||
warn!("Snapshot watcher disconnected from IoService: {}", e);
|
||||
warn!(target: "snapshot_watcher", "Snapshot watcher disconnected from IoService: {}", e);
|
||||
} else {
|
||||
trace!(target: "snapshot_watcher", "Snapshot requested at block #{}", num);
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -68,7 +63,10 @@ impl<C: 'static> Broadcast for Mutex<IoChannel<ClientIoMessage<C>>> {
|
||||
pub struct Watcher {
|
||||
oracle: Box<dyn Oracle>,
|
||||
broadcast: Box<dyn Broadcast>,
|
||||
// How often we attempt to take a snapshot: only snapshot on blocknumbers that are multiples of
|
||||
// `period`. Always set to `SNAPSHOT_PERIOD`, i.e. 5000.
|
||||
period: u64,
|
||||
// Start snapshots `history` blocks from the tip. Always set to `SNAPSHOT_HISTORY`, i.e. 100.
|
||||
history: u64,
|
||||
}
|
||||
|
||||
@ -106,18 +104,22 @@ impl ChainNotify for Watcher {
|
||||
fn new_blocks(&self, new_blocks: NewBlocks) {
|
||||
if self.oracle.is_major_importing() || new_blocks.has_more_blocks_to_import { return }
|
||||
|
||||
trace!(target: "snapshot_watcher", "{} imported", new_blocks.imported.len());
|
||||
|
||||
// Decide if it's time for a snapshot: the highest of the imported blocks is a multiple of 5000?
|
||||
let highest = new_blocks.imported.into_iter()
|
||||
// Convert block hashes to block numbers for all newly imported blocks
|
||||
.filter_map(|h| self.oracle.to_number(h))
|
||||
.filter(|&num| num >= self.period + self.history)
|
||||
.map(|num| num - self.history)
|
||||
.filter(|num| num % self.period == 0)
|
||||
// Subtract `history` (i.e. `SNAPSHOT_HISTORY`, i.e. 100) from the block numbers to stay
|
||||
// clear of reorgs.
|
||||
.map(|num| num.saturating_sub(self.history) )
|
||||
// …filter out blocks that do not fall on the a multiple of `period`. This regulates the
|
||||
// frequency of snapshots and ensures more snapshots are produced from similar points in
|
||||
// the chain.
|
||||
.filter(|num| num % self.period == 0 )
|
||||
// Pick newest of the candidates: this is where we want to snapshot from.
|
||||
.fold(0, ::std::cmp::max);
|
||||
|
||||
match highest {
|
||||
0 => self.broadcast.take_at(None),
|
||||
_ => self.broadcast.take_at(Some(highest)),
|
||||
if highest > 0 {
|
||||
self.broadcast.request_snapshot_at(highest);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@ -20,7 +20,7 @@ use std::convert::TryFrom;
|
||||
use std::io::{BufRead, BufReader};
|
||||
use std::str::from_utf8;
|
||||
use std::sync::{Arc, Weak};
|
||||
use std::sync::atomic::{AtomicBool, AtomicI64, Ordering as AtomicOrdering};
|
||||
use std::sync::atomic::{AtomicBool, AtomicI64, Ordering as AtomicOrdering, Ordering, AtomicU64};
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
use ansi_term::Colour;
|
||||
@ -204,6 +204,9 @@ pub struct Client {
|
||||
/// Database pruning strategy to use for StateDB
|
||||
pruning: journaldb::Algorithm,
|
||||
|
||||
/// Don't prune the state we're currently snapshotting
|
||||
snapshotting_at: AtomicU64,
|
||||
|
||||
/// Client uses this to store blocks, traces, etc.
|
||||
db: RwLock<Arc<dyn BlockChainDB>>,
|
||||
|
||||
@ -780,6 +783,7 @@ impl Client {
|
||||
tracedb,
|
||||
engine,
|
||||
pruning: config.pruning.clone(),
|
||||
snapshotting_at: AtomicU64::new(0),
|
||||
db: RwLock::new(db.clone()),
|
||||
state_db: RwLock::new(state_db),
|
||||
report: RwLock::new(Default::default()),
|
||||
@ -964,13 +968,15 @@ impl Client {
|
||||
return Ok(())
|
||||
}
|
||||
|
||||
let number = match state_db.journal_db().latest_era() {
|
||||
let latest_era = match state_db.journal_db().latest_era() {
|
||||
Some(n) => n,
|
||||
None => return Ok(()),
|
||||
};
|
||||
|
||||
// prune all ancient eras until we're below the memory target,
|
||||
// but have at least the minimum number of states.
|
||||
// Prune all ancient eras until we're below the memory target (default: 32Mb),
|
||||
// but have at least the minimum number of states, i.e. `history`.
|
||||
// If a snapshot is under way, no pruning happens and memory consumption is allowed to
|
||||
// increase above the memory target until the snapshot has finished.
|
||||
loop {
|
||||
let needs_pruning = state_db.journal_db().journal_size() >= self.config.history_mem;
|
||||
|
||||
@ -979,17 +985,27 @@ impl Client {
|
||||
}
|
||||
|
||||
match state_db.journal_db().earliest_era() {
|
||||
Some(era) if era + self.history <= number => {
|
||||
trace!(target: "client", "Pruning state for ancient era {}", era);
|
||||
match chain.block_hash(era) {
|
||||
Some(earliest_era) if earliest_era + self.history <= latest_era => {
|
||||
let freeze_at = self.snapshotting_at.load(Ordering::SeqCst);
|
||||
if freeze_at > 0 && freeze_at == earliest_era {
|
||||
// Note: journal_db().mem_used() can be used for a more accurate memory
|
||||
// consumption measurement but it can be expensive so sticking with the
|
||||
// faster `journal_size()` instead.
|
||||
trace!(target: "pruning", "Pruning is paused at era {} (snapshot under way); earliest era={}, latest era={}, journal_size={} – Not pruning.",
|
||||
freeze_at, earliest_era, latest_era, state_db.journal_db().journal_size());
|
||||
break;
|
||||
}
|
||||
trace!(target: "pruning", "Pruning state for ancient era #{}; latest era={}, journal_size={}",
|
||||
earliest_era, latest_era, state_db.journal_db().journal_size());
|
||||
match chain.block_hash(earliest_era) {
|
||||
Some(ancient_hash) => {
|
||||
let mut batch = DBTransaction::new();
|
||||
state_db.mark_canonical(&mut batch, era, &ancient_hash)?;
|
||||
state_db.mark_canonical(&mut batch, earliest_era, &ancient_hash)?;
|
||||
self.db.read().key_value().write_buffered(batch);
|
||||
state_db.journal_db().flush();
|
||||
}
|
||||
None =>
|
||||
debug!(target: "client", "Missing expected hash for block {}", era),
|
||||
debug!(target: "pruning", "Missing expected hash for block {}", earliest_era),
|
||||
}
|
||||
}
|
||||
_ => break, // means that every era is kept, no pruning necessary.
|
||||
@ -2533,48 +2549,61 @@ impl SnapshotClient for Client {
|
||||
return Err(EthcoreError::Snapshot(SnapshotError::SnapshotsUnsupported));
|
||||
}
|
||||
let db = self.state_db.read().journal_db().boxed_clone();
|
||||
let best_block_number = self.chain_info().best_block_number;
|
||||
let block_number = self.block_number(at).ok_or_else(|| SnapshotError::InvalidStartingBlock(at))?;
|
||||
|
||||
if db.is_prunable() && self.pruning_info().earliest_state > block_number {
|
||||
let block_number = self.block_number(at).ok_or_else(|| SnapshotError::InvalidStartingBlock(at))?;
|
||||
let earliest_era = db.earliest_era().unwrap_or(0);
|
||||
if db.is_prunable() && earliest_era > block_number {
|
||||
return Err(SnapshotError::OldBlockPrunedDB.into());
|
||||
}
|
||||
|
||||
let history = cmp::min(self.history, 1000);
|
||||
|
||||
let start_hash = match at {
|
||||
let (actual_block_nr, block_hash) = match at {
|
||||
BlockId::Latest => {
|
||||
let start_num = match db.earliest_era() {
|
||||
Some(era) => cmp::max(era, best_block_number.saturating_sub(history)),
|
||||
None => best_block_number.saturating_sub(history),
|
||||
};
|
||||
// Start `self.history` blocks from the best block, but no further back than 1000
|
||||
// blocks (or earliest era, whichever is greatest).
|
||||
let history = cmp::min(self.history, 1000);
|
||||
let best_block_number = self.chain_info().best_block_number;
|
||||
let start_num = cmp::max(earliest_era, best_block_number.saturating_sub(history));
|
||||
|
||||
match self.block_hash(BlockId::Number(start_num)) {
|
||||
Some(h) => h,
|
||||
None => return Err(SnapshotError::InvalidStartingBlock(at).into()),
|
||||
Some(hash) => (start_num, hash),
|
||||
None => {
|
||||
error!(target: "snapshot", "Can't take snapshot at {:?}: missing hash for the starting block #{}", at, start_num);
|
||||
return Err(SnapshotError::InvalidStartingBlock(at).into())
|
||||
},
|
||||
}
|
||||
}
|
||||
_ => match self.block_hash(at) {
|
||||
Some(hash) => hash,
|
||||
Some(hash) => (block_number, hash),
|
||||
None => return Err(SnapshotError::InvalidStartingBlock(at).into()),
|
||||
},
|
||||
};
|
||||
|
||||
let processing_threads = self.config.snapshot.processing_threads;
|
||||
let chunker = snapshot::chunker(self.engine.snapshot_mode()).ok_or_else(|| SnapshotError::SnapshotsUnsupported)?;
|
||||
snapshot::take_snapshot(
|
||||
chunker,
|
||||
&self.chain.read(),
|
||||
start_hash,
|
||||
db.as_hash_db(),
|
||||
writer,
|
||||
p,
|
||||
processing_threads,
|
||||
)?;
|
||||
Ok(())
|
||||
trace!(target: "snapshot", "Snapshot requested at block {:?}. Using block #{}/{:?}. Earliest block: #{}, earliest state era #{}. Using {} threads.",
|
||||
at, actual_block_nr, block_hash, self.pruning_info().earliest_chain, earliest_era, processing_threads,
|
||||
);
|
||||
// Stop pruning from happening while the snapshot is under way.
|
||||
self.snapshotting_at.store(actual_block_nr, Ordering::SeqCst);
|
||||
{
|
||||
scopeguard::defer! {{
|
||||
trace!(target: "snapshot", "Re-enabling pruning.");
|
||||
self.snapshotting_at.store(0, Ordering::SeqCst)
|
||||
}};
|
||||
let chunker = snapshot::chunker(self.engine.snapshot_mode()).ok_or_else(|| SnapshotError::SnapshotsUnsupported)?;
|
||||
// Spawn threads and take snapshot
|
||||
snapshot::take_snapshot(
|
||||
chunker,
|
||||
&self.chain.read(),
|
||||
block_hash,
|
||||
db.as_hash_db(),
|
||||
writer,
|
||||
p,
|
||||
processing_threads,
|
||||
)?;
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
||||
impl ImportExportBlocks for Client {
|
||||
|
||||
@ -90,7 +90,7 @@ pub struct ClientConfig {
|
||||
pub history: u64,
|
||||
/// Ideal memory usage for state pruning history.
|
||||
pub history_mem: usize,
|
||||
/// Check seal valididity on block import
|
||||
/// Check seal validity on block import
|
||||
pub check_seal: bool,
|
||||
/// Maximal number of transactions queued for verification in a separate thread.
|
||||
pub transaction_verification_queue_size: usize,
|
||||
|
||||
@ -15,6 +15,7 @@ parity-bytes = "0.1"
|
||||
parity-crypto = { version = "0.4.2", features = ["publickey"] }
|
||||
parity-util-mem = "0.2.0"
|
||||
parity-snappy = "0.1"
|
||||
parking_lot = "0.9.0"
|
||||
patricia-trie-ethereum = { path = "../../util/patricia-trie-ethereum" }
|
||||
rlp = "0.4.0"
|
||||
rlp_derive = { path = "../../util/rlp-derive" }
|
||||
|
||||
@ -40,6 +40,7 @@ extern crate parity_crypto;
|
||||
#[macro_use]
|
||||
extern crate derive_more;
|
||||
extern crate keccak_hash as hash;
|
||||
extern crate parking_lot;
|
||||
extern crate parity_bytes as bytes;
|
||||
extern crate patricia_trie_ethereum as ethtrie;
|
||||
extern crate rlp;
|
||||
|
||||
@ -16,11 +16,13 @@
|
||||
|
||||
//! Snapshot type definitions
|
||||
|
||||
use std::sync::atomic::{AtomicBool, AtomicUsize, AtomicU64, Ordering};
|
||||
use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
|
||||
use std::time::Instant;
|
||||
|
||||
use ethereum_types::H256;
|
||||
use rlp::{Rlp, RlpStream, DecoderError};
|
||||
use bytes::Bytes;
|
||||
use ethereum_types::H256;
|
||||
use parking_lot::RwLock;
|
||||
use rlp::{Rlp, RlpStream, DecoderError};
|
||||
|
||||
/// Modes of snapshotting
|
||||
pub enum Snapshotting {
|
||||
@ -39,31 +41,53 @@ pub enum Snapshotting {
|
||||
}
|
||||
|
||||
/// A progress indicator for snapshots.
|
||||
#[derive(Debug, Default)]
|
||||
#[derive(Debug)]
|
||||
pub struct Progress {
|
||||
/// Number of accounts processed so far
|
||||
pub accounts: AtomicUsize,
|
||||
accounts: AtomicUsize,
|
||||
// Number of accounts processed at last tick.
|
||||
prev_accounts: AtomicUsize,
|
||||
/// Number of blocks processed so far
|
||||
pub blocks: AtomicUsize,
|
||||
/// Size in bytes of a all compressed chunks processed so far
|
||||
pub size: AtomicU64,
|
||||
bytes: AtomicUsize,
|
||||
// Number of bytes processed at last tick.
|
||||
prev_bytes: AtomicUsize,
|
||||
/// Signals that the snapshotting process is completed
|
||||
pub done: AtomicBool,
|
||||
/// Signal snapshotting process to abort
|
||||
pub abort: AtomicBool,
|
||||
|
||||
last_tick: RwLock<Instant>,
|
||||
}
|
||||
|
||||
impl Progress {
|
||||
/// Create a new progress tracker.
|
||||
pub fn new() -> Progress {
|
||||
Progress {
|
||||
accounts: AtomicUsize::new(0),
|
||||
prev_accounts: AtomicUsize::new(0),
|
||||
blocks: AtomicUsize::new(0),
|
||||
bytes: AtomicUsize::new(0),
|
||||
prev_bytes: AtomicUsize::new(0),
|
||||
abort: AtomicBool::new(false),
|
||||
done: AtomicBool::new(false),
|
||||
last_tick: RwLock::new(Instant::now()),
|
||||
}
|
||||
}
|
||||
|
||||
/// Reset the progress.
|
||||
pub fn reset(&self) {
|
||||
self.accounts.store(0, Ordering::Release);
|
||||
self.blocks.store(0, Ordering::Release);
|
||||
self.size.store(0, Ordering::Release);
|
||||
self.bytes.store(0, Ordering::Release);
|
||||
self.abort.store(false, Ordering::Release);
|
||||
|
||||
// atomic fence here to ensure the others are written first?
|
||||
// logs might very rarely get polluted if not.
|
||||
self.done.store(false, Ordering::Release);
|
||||
|
||||
*self.last_tick.write() = Instant::now();
|
||||
}
|
||||
|
||||
/// Get the number of accounts snapshotted thus far.
|
||||
@ -73,10 +97,37 @@ impl Progress {
|
||||
pub fn blocks(&self) -> usize { self.blocks.load(Ordering::Acquire) }
|
||||
|
||||
/// Get the written size of the snapshot in bytes.
|
||||
pub fn size(&self) -> u64 { self.size.load(Ordering::Acquire) }
|
||||
pub fn bytes(&self) -> usize { self.bytes.load(Ordering::Acquire) }
|
||||
|
||||
/// Whether the snapshot is complete.
|
||||
pub fn done(&self) -> bool { self.done.load(Ordering::Acquire) }
|
||||
|
||||
/// Return the progress rate over the last tick (i.e. since last update).
|
||||
pub fn rate(&self) -> (f64, f64) {
|
||||
let last_tick = *self.last_tick.read();
|
||||
let dt = last_tick.elapsed().as_secs_f64();
|
||||
if dt < 1.0 {
|
||||
return (0f64, 0f64);
|
||||
}
|
||||
let delta_acc = self.accounts.load(Ordering::Relaxed)
|
||||
.saturating_sub(self.prev_accounts.load(Ordering::Relaxed));
|
||||
let delta_bytes = self.bytes.load(Ordering::Relaxed)
|
||||
.saturating_sub(self.prev_bytes.load(Ordering::Relaxed));
|
||||
(delta_acc as f64 / dt, delta_bytes as f64 / dt)
|
||||
}
|
||||
|
||||
/// Update state progress counters and set the last tick.
|
||||
pub fn update(&self, accounts_delta: usize, bytes_delta: usize) {
|
||||
*self.last_tick.write() = Instant::now();
|
||||
self.prev_accounts.store(
|
||||
self.accounts.fetch_add(accounts_delta, Ordering::SeqCst),
|
||||
Ordering::SeqCst
|
||||
);
|
||||
self.prev_bytes.store(
|
||||
self.bytes.fetch_add(bytes_delta, Ordering::SeqCst),
|
||||
Ordering::SeqCst
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/// Manifest data.
|
||||
|
||||
@ -68,10 +68,10 @@ use signer;
|
||||
use db;
|
||||
use registrar::RegistrarClient;
|
||||
|
||||
// how often to take periodic snapshots.
|
||||
// How often we attempt to take a snapshot: only snapshot on blocknumbers that are multiples of this.
|
||||
const SNAPSHOT_PERIOD: u64 = 5000;
|
||||
|
||||
// how many blocks to wait before starting a periodic snapshot.
|
||||
// Start snapshots `history` blocks from the tip. Should be smaller than `SNAPSHOT_HISTORY`.
|
||||
const SNAPSHOT_HISTORY: u64 = 100;
|
||||
|
||||
// Number of minutes before a given gas price corpus should expire.
|
||||
|
||||
@ -257,18 +257,18 @@ impl SnapshotCommand {
|
||||
let writer = PackedWriter::new(&file_path)
|
||||
.map_err(|e| format!("Failed to open snapshot writer: {}", e))?;
|
||||
|
||||
let progress = Arc::new(Progress::default());
|
||||
let progress = Arc::new(Progress::new());
|
||||
let p = progress.clone();
|
||||
let informant_handle = ::std::thread::spawn(move || {
|
||||
::std::thread::sleep(Duration::from_secs(5));
|
||||
|
||||
let mut last_size = 0;
|
||||
while !p.done() {
|
||||
let cur_size = p.size();
|
||||
let cur_size = p.bytes();
|
||||
if cur_size != last_size {
|
||||
last_size = cur_size;
|
||||
let bytes = ::informant::format_bytes(cur_size as usize);
|
||||
info!("Snapshot: {} accounts {} blocks {}", p.accounts(), p.blocks(), bytes);
|
||||
info!("Snapshot: {} accounts (state), {} blocks, {} bytes", p.accounts(), p.blocks(), bytes);
|
||||
}
|
||||
|
||||
::std::thread::sleep(Duration::from_secs(5));
|
||||
|
||||
@ -84,8 +84,8 @@ pub trait JournalDB: HashDB<KeccakHasher, DBValue> {
|
||||
/// Get backing database.
|
||||
fn backing(&self) -> &Arc<dyn kvdb::KeyValueDB>;
|
||||
|
||||
/// Clear internal strucutres. This should called after changes have been written
|
||||
/// to the backing strage
|
||||
/// Clear internal structure. This should be called after changes have been written
|
||||
/// to the backing storage.
|
||||
fn flush(&self) {}
|
||||
|
||||
/// Consolidate all the insertions and deletions in the given memory overlay.
|
||||
|
||||
@ -228,10 +228,10 @@ impl OverlayRecentDB {
|
||||
JournalOverlay {
|
||||
backing_overlay: overlay,
|
||||
pending_overlay: HashMap::default(),
|
||||
journal: journal,
|
||||
latest_era: latest_era,
|
||||
earliest_era: earliest_era,
|
||||
cumulative_size: cumulative_size,
|
||||
journal,
|
||||
latest_era,
|
||||
earliest_era,
|
||||
cumulative_size,
|
||||
}
|
||||
}
|
||||
|
||||
@ -264,7 +264,6 @@ impl JournalDB for OverlayRecentDB {
|
||||
|
||||
fn journal_size(&self) -> usize {
|
||||
self.journal_overlay.read().cumulative_size
|
||||
|
||||
}
|
||||
|
||||
fn is_empty(&self) -> bool {
|
||||
@ -351,7 +350,7 @@ impl JournalDB for OverlayRecentDB {
|
||||
|
||||
let mut ops = 0;
|
||||
// apply old commits' details
|
||||
if let Some(ref mut records) = journal_overlay.journal.get_mut(&end_era) {
|
||||
if let Some(records) = journal_overlay.journal.get_mut(&end_era) {
|
||||
let mut canon_insertions: Vec<(H256, DBValue)> = Vec::new();
|
||||
let mut canon_deletions: Vec<H256> = Vec::new();
|
||||
let mut overlay_deletions: Vec<H256> = Vec::new();
|
||||
|
||||
Loading…
Reference in New Issue
Block a user