Pause pruning while snapshotting (#11178)

* WIP. Typos and logging.

* Format todos

* Pause pruning while a snapshot is under way
Logs, docs and todos

* Allocate memory for the full chunk

* Name snapshotting threads

* Ensure `taking_snapshot` is set to false whenever and however `take_snapshot`returns
Rename `take_at` to `request_snapshot_at`
Cleanup

* Let "in_progress" deletion fail
Fix tests

* Just use an atomic

* Review grumbles

* Finish the sentence

* Resolve a few todos and clarify comments.

* Calculate progress rate since last update

* Lockfile

* Fix tests

* typo

* Reinstate default snapshotting frequency
Cut down on the logging noise

* address grumble

* Log memory use with `journal_size()` and explain why.

Cargo.lock

@@ -5,7 +5,7 @@ name = "account-db"
 version = "0.1.0"
 dependencies = [
  "ethereum-types 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "hash-db 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "hash-db 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)",
  "keccak-hash 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "keccak-hasher 0.1.1",
  "kvdb 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -20,7 +20,7 @@ dependencies = [
  "common-types 0.1.0",
  "derive_more 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "ethereum-types 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "hash-db 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "hash-db 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)",
  "journaldb 0.2.0",
  "keccak-hash 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "keccak-hasher 0.1.1",
@@ -37,7 +37,7 @@ dependencies = [
  "rlp_compress 0.1.0",
  "serde 1.0.99 (registry+https://github.com/rust-lang/crates.io-index)",
  "trace 0.1.0",
- "trie-db 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "trie-db 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)",
  "trie-vm-factories 0.1.0",
 ]
 
@@ -82,6 +82,14 @@ dependencies = [
  "stream-cipher 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
+[[package]]
+name = "ahash"
+version = "0.2.16"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "const-random 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
 [[package]]
 name = "aho-corasick"
 version = "0.6.8"
@@ -598,6 +606,7 @@ dependencies = [
  "parity-crypto 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)",
  "parity-snappy 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "parity-util-mem 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "parking_lot 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "patricia-trie-ethereum 0.1.0",
  "rlp 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)",
  "rlp_derive 0.1.0",
@@ -606,6 +615,24 @@ dependencies = [
  "vm 0.1.0",
 ]
 
+[[package]]
+name = "const-random"
+version = "0.1.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "const-random-macro 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)",
+ "proc-macro-hack 0.5.9 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "const-random-macro"
+version = "0.1.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "proc-macro-hack 0.5.9 (registry+https://github.com/rust-lang/crates.io-index)",
+ "rand 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
 [[package]]
 name = "criterion"
 version = "0.3.0"
@@ -645,7 +672,7 @@ version = "0.6.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 dependencies = [
  "crossbeam-epoch 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)",
- "crossbeam-utils 0.6.5 (registry+https://github.com/rust-lang/crates.io-index)",
+ "crossbeam-utils 0.6.6 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
 [[package]]
@@ -654,7 +681,7 @@ version = "0.7.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 dependencies = [
  "crossbeam-epoch 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)",
- "crossbeam-utils 0.6.5 (registry+https://github.com/rust-lang/crates.io-index)",
+ "crossbeam-utils 0.6.6 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
 [[package]]
@@ -664,7 +691,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 dependencies = [
  "arrayvec 0.4.11 (registry+https://github.com/rust-lang/crates.io-index)",
  "cfg-if 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)",
- "crossbeam-utils 0.6.5 (registry+https://github.com/rust-lang/crates.io-index)",
+ "crossbeam-utils 0.6.6 (registry+https://github.com/rust-lang/crates.io-index)",
  "lazy_static 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "memoffset 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)",
  "scopeguard 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -675,7 +702,7 @@ name = "crossbeam-queue"
 version = "0.1.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 dependencies = [
- "crossbeam-utils 0.6.5 (registry+https://github.com/rust-lang/crates.io-index)",
+ "crossbeam-utils 0.6.6 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
 [[package]]
@@ -685,7 +712,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 
 [[package]]
 name = "crossbeam-utils"
-version = "0.6.5"
+version = "0.6.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 dependencies = [
  "cfg-if 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -1043,7 +1070,7 @@ dependencies = [
  "executive-state 0.1.0",
  "fetch 0.1.0",
  "futures 0.1.29 (registry+https://github.com/rust-lang/crates.io-index)",
- "hash-db 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "hash-db 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)",
  "itertools 0.5.10 (registry+https://github.com/rust-lang/crates.io-index)",
  "journaldb 0.2.0",
  "keccak-hash 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -1067,6 +1094,7 @@ dependencies = [
  "registrar 0.0.1",
  "rlp 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)",
  "rustc-hex 2.0.1 (registry+https://github.com/rust-lang/crates.io-index)",
+ "scopeguard 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "serde 1.0.99 (registry+https://github.com/rust-lang/crates.io-index)",
  "serde_derive 1.0.89 (registry+https://github.com/rust-lang/crates.io-index)",
  "serde_json 1.0.40 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -1077,7 +1105,7 @@ dependencies = [
  "tempdir 0.3.7 (registry+https://github.com/rust-lang/crates.io-index)",
  "trace 0.1.0",
  "trace-time 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
- "trie-db 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "trie-db 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)",
  "trie-standardmap 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "trie-vm-factories 0.1.0",
  "triehash-ethereum 0.2.0",
@@ -1216,7 +1244,7 @@ dependencies = [
  "failsafe 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "fastmap 0.1.0",
  "futures 0.1.29 (registry+https://github.com/rust-lang/crates.io-index)",
- "hash-db 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "hash-db 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)",
  "journaldb 0.2.0",
  "keccak-hash 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "keccak-hasher 0.1.1",
@@ -1239,7 +1267,7 @@ dependencies = [
  "spec 0.1.0",
  "stats 0.1.0",
  "tempdir 0.3.7 (registry+https://github.com/rust-lang/crates.io-index)",
- "trie-db 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "trie-db 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)",
  "triehash-ethereum 0.2.0",
  "verification 0.1.0",
  "vm 0.1.0",
@@ -1368,7 +1396,7 @@ dependencies = [
  "ethjson 0.1.0",
  "fetch 0.1.0",
  "futures 0.1.29 (registry+https://github.com/rust-lang/crates.io-index)",
- "hash-db 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "hash-db 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)",
  "journaldb 0.2.0",
  "keccak-hash 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "keccak-hasher 0.1.1",
@@ -1393,7 +1421,7 @@ dependencies = [
  "tiny-keccak 1.5.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "trace 0.1.0",
  "transaction-pool 2.0.1 (registry+https://github.com/rust-lang/crates.io-index)",
- "trie-db 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "trie-db 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)",
  "url 2.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "vm 0.1.0",
 ]
@@ -1668,7 +1696,7 @@ dependencies = [
  "ethcore 1.12.0",
  "ethereum-types 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "evm 0.1.0",
- "hash-db 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "hash-db 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)",
  "keccak-hash 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "keccak-hasher 0.1.1",
  "kvdb 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -1681,7 +1709,7 @@ dependencies = [
  "rustc-hex 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "spec 0.1.0",
  "trace 0.1.0",
- "trie-db 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "trie-db 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)",
  "trie-vm-factories 0.1.0",
  "vm 0.1.0",
 ]
@@ -1909,7 +1937,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 
 [[package]]
 name = "hash-db"
-version = "0.15.0"
+version = "0.15.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 
 [[package]]
@@ -1920,6 +1948,15 @@ dependencies = [
  "crunchy 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
+[[package]]
+name = "hashbrown"
+version = "0.6.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "ahash 0.2.16 (registry+https://github.com/rust-lang/crates.io-index)",
+ "autocfg 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
 [[package]]
 name = "hashmap_core"
 version = "0.1.10"
@@ -2224,7 +2261,7 @@ dependencies = [
  "env_logger 0.5.13 (registry+https://github.com/rust-lang/crates.io-index)",
  "ethereum-types 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "fastmap 0.1.0",
- "hash-db 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "hash-db 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)",
  "keccak-hash 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "keccak-hasher 0.1.1",
  "kvdb 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -2352,7 +2389,7 @@ name = "keccak-hasher"
 version = "0.1.1"
 dependencies = [
  "ethereum-types 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "hash-db 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "hash-db 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)",
  "plain_hasher 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "tiny-keccak 1.5.0 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
@@ -2362,7 +2399,7 @@ name = "keccak-hasher"
 version = "0.15.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 dependencies = [
- "hash-db 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "hash-db 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)",
  "hash256-std-hasher 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "tiny-keccak 1.5.0 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
@@ -2525,7 +2562,7 @@ dependencies = [
  "account-state 0.1.0",
  "client-traits 0.1.0",
  "common-types 0.1.0",
- "crossbeam-utils 0.6.5 (registry+https://github.com/rust-lang/crates.io-index)",
+ "crossbeam-utils 0.6.6 (registry+https://github.com/rust-lang/crates.io-index)",
  "ethabi 9.0.1 (registry+https://github.com/rust-lang/crates.io-index)",
  "ethabi-contract 9.0.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "ethabi-derive 9.0.1 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -2610,7 +2647,7 @@ name = "memory-db"
 version = "0.15.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 dependencies = [
- "hash-db 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "hash-db 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)",
  "hashmap_core 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)",
  "parity-util-mem 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
@@ -3393,14 +3430,14 @@ version = "0.1.0"
 dependencies = [
  "elastic-array 0.10.2 (registry+https://github.com/rust-lang/crates.io-index)",
  "ethereum-types 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "hash-db 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "hash-db 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)",
  "journaldb 0.2.0",
  "keccak-hash 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "keccak-hasher 0.1.1",
  "memory-db 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "parity-bytes 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "rlp 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)",
- "trie-db 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "trie-db 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
 [[package]]
@@ -3486,7 +3523,7 @@ dependencies = [
  "common-types 0.1.0",
  "ethereum-types 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "ethjson 0.1.0",
- "hash-db 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "hash-db 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)",
  "itertools 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "keccak-hash 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "keccak-hasher 0.1.1",
@@ -3498,7 +3535,7 @@ dependencies = [
  "rlp 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)",
  "rustc-hex 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "serde 1.0.99 (registry+https://github.com/rust-lang/crates.io-index)",
- "trie-db 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "trie-db 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)",
  "triehash-ethereum 0.2.0",
 ]
 
@@ -3854,7 +3891,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 dependencies = [
  "crossbeam-deque 0.6.3 (registry+https://github.com/rust-lang/crates.io-index)",
  "crossbeam-queue 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)",
- "crossbeam-utils 0.6.5 (registry+https://github.com/rust-lang/crates.io-index)",
+ "crossbeam-utils 0.6.6 (registry+https://github.com/rust-lang/crates.io-index)",
  "lazy_static 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "num_cpus 1.10.1 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
@@ -4216,7 +4253,7 @@ dependencies = [
  "account-state 0.1.0",
  "client-traits 0.1.0",
  "common-types 0.1.0",
- "crossbeam-utils 0.6.5 (registry+https://github.com/rust-lang/crates.io-index)",
+ "crossbeam-utils 0.6.6 (registry+https://github.com/rust-lang/crates.io-index)",
  "engine 0.1.0",
  "env_logger 0.5.13 (registry+https://github.com/rust-lang/crates.io-index)",
  "ethabi 9.0.1 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -4230,7 +4267,7 @@ dependencies = [
  "ethcore-io 1.12.0",
  "ethereum-types 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "ethkey 0.4.0",
- "hash-db 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "hash-db 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)",
  "itertools 0.5.10 (registry+https://github.com/rust-lang/crates.io-index)",
  "journaldb 0.2.0",
  "keccak-hash 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -4248,11 +4285,12 @@ dependencies = [
  "rand_xorshift 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "rlp 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)",
  "rlp_derive 0.1.0",
+ "scopeguard 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "snapshot-tests 0.1.0",
  "spec 0.1.0",
  "state-db 0.1.0",
  "tempdir 0.3.7 (registry+https://github.com/rust-lang/crates.io-index)",
- "trie-db 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "trie-db 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)",
  "trie-standardmap 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "triehash-ethereum 0.2.0",
 ]
@@ -4276,7 +4314,7 @@ dependencies = [
  "ethcore-db 0.1.0",
  "ethcore-io 1.12.0",
  "ethereum-types 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "hash-db 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "hash-db 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)",
  "journaldb 0.2.0",
  "keccak-hash 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "keccak-hasher 0.1.1",
@@ -4295,7 +4333,7 @@ dependencies = [
  "snapshot 0.1.0",
  "spec 0.1.0",
  "tempdir 0.3.7 (registry+https://github.com/rust-lang/crates.io-index)",
- "trie-db 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "trie-db 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)",
  "trie-standardmap 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "triehash-ethereum 0.2.0",
 ]
@@ -4330,7 +4368,7 @@ dependencies = [
  "ethjson 0.1.0",
  "evm 0.1.0",
  "executive-state 0.1.0",
- "hash-db 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "hash-db 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)",
  "instant-seal 0.1.0",
  "journaldb 0.2.0",
  "keccak-hash 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -4368,7 +4406,7 @@ dependencies = [
  "ethcore-bloom-journal 0.1.0",
  "ethcore-db 0.1.0",
  "ethereum-types 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "hash-db 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "hash-db 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)",
  "journaldb 0.2.0",
  "keccak-hash 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "keccak-hasher 0.1.1",
@@ -4633,7 +4671,7 @@ name = "tokio-executor"
 version = "0.1.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 dependencies = [
- "crossbeam-utils 0.6.5 (registry+https://github.com/rust-lang/crates.io-index)",
+ "crossbeam-utils 0.6.6 (registry+https://github.com/rust-lang/crates.io-index)",
  "futures 0.1.29 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
@@ -4734,7 +4772,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 dependencies = [
  "crossbeam-deque 0.7.1 (registry+https://github.com/rust-lang/crates.io-index)",
  "crossbeam-queue 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)",
- "crossbeam-utils 0.6.5 (registry+https://github.com/rust-lang/crates.io-index)",
+ "crossbeam-utils 0.6.6 (registry+https://github.com/rust-lang/crates.io-index)",
  "futures 0.1.29 (registry+https://github.com/rust-lang/crates.io-index)",
  "lazy_static 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -4757,7 +4795,7 @@ name = "tokio-timer"
 version = "0.2.11"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 dependencies = [
- "crossbeam-utils 0.6.5 (registry+https://github.com/rust-lang/crates.io-index)",
+ "crossbeam-utils 0.6.6 (registry+https://github.com/rust-lang/crates.io-index)",
  "futures 0.1.29 (registry+https://github.com/rust-lang/crates.io-index)",
  "slab 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)",
  "tokio-executor 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -4862,12 +4900,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 
 [[package]]
 name = "trie-db"
-version = "0.15.0"
+version = "0.15.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 dependencies = [
  "elastic-array 0.10.2 (registry+https://github.com/rust-lang/crates.io-index)",
- "hash-db 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "hash-db 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)",
- "hashmap_core 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)",
+ "hashbrown 0.6.1 (registry+https://github.com/rust-lang/crates.io-index)",
  "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)",
  "rand 0.6.1 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
@@ -4877,7 +4915,7 @@ name = "trie-standardmap"
 version = "0.15.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 dependencies = [
- "hash-db 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "hash-db 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)",
  "keccak-hasher 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
@@ -4889,7 +4927,7 @@ dependencies = [
  "evm 0.1.0",
  "keccak-hasher 0.1.1",
  "patricia-trie-ethereum 0.1.0",
- "trie-db 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "trie-db 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)",
  "vm 0.1.0",
  "wasm 0.1.0",
 ]
@@ -4899,7 +4937,7 @@ name = "triehash"
 version = "0.8.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 dependencies = [
- "hash-db 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "hash-db 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)",
  "rlp 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
@@ -5344,6 +5382,7 @@ dependencies = [
 "checksum aes-ctr 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "d2e5b0458ea3beae0d1d8c0f3946564f8e10f90646cf78c06b4351052058d1ee"
 "checksum aes-soft 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "cfd7e7ae3f9a1fb5c03b389fc6bb9a51400d0c13053f0dca698c832bfd893a0d"
 "checksum aesni 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)" = "2f70a6b5f971e473091ab7cfb5ffac6cde81666c4556751d8d5620ead8abf100"
+"checksum ahash 0.2.16 (registry+https://github.com/rust-lang/crates.io-index)" = "b35dfc96a657c1842b4eb73180b65e37152d4b94d0eb5cb51708aee7826950b4"
 "checksum aho-corasick 0.6.8 (registry+https://github.com/rust-lang/crates.io-index)" = "68f56c7353e5a9547cbd76ed90f7bb5ffc3ba09d4ea9bd1d8c06c8b1142eeb5a"
 "checksum aho-corasick 0.7.6 (registry+https://github.com/rust-lang/crates.io-index)" = "58fb5e95d83b38284460a5fda7d6470aa0b8844d283a0b614b8535e880800d2d"
 "checksum ansi_term 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ee49baf6cb617b853aa8d93bf420db2383fab46d314482ca2803b40d5fde979b"
@@ -5390,6 +5429,8 @@ dependencies = [
 "checksum cloudabi 0.0.3 (registry+https://github.com/rust-lang/crates.io-index)" = "ddfc5b9aa5d4507acaf872de71051dfd0e309860e88966e1051e462a077aac4f"
 "checksum cmake 0.1.35 (registry+https://github.com/rust-lang/crates.io-index)" = "6ec65ee4f9c9d16f335091d23693457ed4928657ba4982289d7fafee03bc614a"
 "checksum combine 3.6.1 (registry+https://github.com/rust-lang/crates.io-index)" = "fc1d011beeed29187b8db2ac3925c8dd4d3e87db463dc9d2d2833985388fc5bc"
+"checksum const-random 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)" = "7b641a8c9867e341f3295564203b1c250eb8ce6cb6126e007941f78c4d2ed7fe"
+"checksum const-random-macro 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)" = "c750ec12b83377637110d5a57f5ae08e895b06c4b16e2bdbf1a94ef717428c59"
 "checksum criterion 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "938703e165481c8d612ea3479ac8342e5615185db37765162e762ec3523e2fc6"
 "checksum criterion-plot 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "eccdc6ce8bbe352ca89025bee672aa6d24f4eb8c53e3a8b5d1bc58011da072a2"
 "checksum crossbeam-deque 0.6.3 (registry+https://github.com/rust-lang/crates.io-index)" = "05e44b8cf3e1a625844d1750e1f7820da46044ff6d28f4d43e455ba3e5bb2c13"
@@ -5397,7 +5438,7 @@ dependencies = [
 "checksum crossbeam-epoch 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)" = "fedcd6772e37f3da2a9af9bf12ebe046c0dfe657992377b4df982a2b54cd37a9"
 "checksum crossbeam-queue 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "7c979cd6cfe72335896575c6b5688da489e420d36a27a0b9eb0c73db574b4a4b"
 "checksum crossbeam-utils 0.5.0 (registry+https://github.com/rust-lang/crates.io-index)" = "677d453a17e8bd2b913fa38e8b9cf04bcdbb5be790aa294f2389661d72036015"
-"checksum crossbeam-utils 0.6.5 (registry+https://github.com/rust-lang/crates.io-index)" = "f8306fcef4a7b563b76b7dd949ca48f52bc1141aa067d2ea09565f3e2652aa5c"
+"checksum crossbeam-utils 0.6.6 (registry+https://github.com/rust-lang/crates.io-index)" = "04973fa96e96579258a5091af6003abde64af786b860f18622b82e026cca60e6"
 "checksum crunchy 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)" = "a2f4a431c5c9f662e1200b7c7f02c34e91361150e382089a8f2dec3ba680cbda"
 "checksum crunchy 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "7a81dae078cea95a014a339291cec439d2f232ebe854a9d672b796c6afafa9b7"
 "checksum crypto-mac 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "4434400df11d95d556bac068ddfedd482915eb18fe8bea89bc80b6e4b1c179e5"
@@ -5447,8 +5488,9 @@ dependencies = [
 "checksum globset 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)" = "4743617a7464bbda3c8aec8558ff2f9429047e025771037df561d383337ff865"
 "checksum h2 0.1.26 (registry+https://github.com/rust-lang/crates.io-index)" = "a5b34c246847f938a410a03c5458c7fee2274436675e76d8b903c08efc29c462"
 "checksum hamming 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)" = "65043da274378d68241eb9a8f8f8aa54e349136f7b8e12f63e3ef44043cc30e1"
-"checksum hash-db 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)" = "32c87fec93c4a2d264483ef843ac1930ae7c7999d97d73721305a5188b4c23a4"
+"checksum hash-db 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)" = "d23bd4e7b5eda0d0f3a307e8b381fdc8ba9000f26fbe912250c0a4cc3956364a"
 "checksum hash256-std-hasher 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)" = "16293646125e09e5bc216d9f73fa81ab31c4f97007d56c036bbf15a58e970540"
+"checksum hashbrown 0.6.1 (registry+https://github.com/rust-lang/crates.io-index)" = "6587d09be37fb98a11cb08b9000a3f592451c1b1b613ca69d949160e313a430a"
 "checksum hashmap_core 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)" = "8e04cb7a5051270ef3fa79f8c7604d581ecfa73d520e74f554e45541c4b5881a"
 "checksum heapsize 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)" = "1679e6ea370dee694f91f1dc469bf94cf8f52051d147aec3e1f9497c6fc22461"
 "checksum heck 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ea04fa3ead4e05e51a7c806fc07271fdbde4e246a6c6d1efd52e72230b771b82"
@@ -5698,7 +5740,7 @@ dependencies = [
 "checksum trace-time 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "dbe82f2f0bf1991e163e757baf044282823155dd326e70f44ce2186c3c320cc9"
 "checksum transaction-pool 2.0.1 (registry+https://github.com/rust-lang/crates.io-index)" = "454adc482e32785c3beab9415dd0f3c689f29cc2d16717eb62f6a784d53544b4"
 "checksum transient-hashmap 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)" = "aeb4b191d033a35edfce392a38cdcf9790b6cebcb30fa690c312c29da4dc433e"
-"checksum trie-db 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)" = "9b65d609ae631d808c6c1cc23a622733d5a0b66a7d67e9f5cd5171562a1f4cb5"
+"checksum trie-db 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)" = "d0b62d27e8aa1c07414549ac872480ac82380bab39e730242ab08d82d7cc098a"
 "checksum trie-standardmap 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)" = "64fda153c00484d640bc91334624be22ead0e5baca917d9fd53ff29bdebcf9b2"
 "checksum triehash 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)" = "61d0a66fa2412c7eb7816640e8ea14cf6bd63b6c824e72315b6ca76d33851134"
 "checksum try-lock 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "e604eb7b43c06650e854be16a2a03155743d3752dd1c943f6829e26b7a36e382"

ethcore/Cargo.toml

@@ -51,6 +51,7 @@ rayon = "1.1"
 registrar = { path = "../util/registrar" }
 rlp = "0.4.0"
 rustc-hex = "2"
+scopeguard = "1.0.0"
 serde = "1.0"
 serde_derive = "1.0"
 snapshot = { path = "snapshot" }

ethcore/service/src/service.rs

@@ -270,14 +270,12 @@ where
 			ClientIoMessage::TakeSnapshot(num) => {
 				let client = self.client.clone();
 				let snapshot = self.snapshot.clone();
-
 				let res = thread::Builder::new().name("Periodic Snapshot".into()).spawn(move || {
 					if let Err(e) = snapshot.take_snapshot(&*client, num) {
 						match e {
 							EthcoreError::Snapshot(SnapshotError::SnapshotAborted) => info!("Snapshot aborted"),
 							_ => warn!("Failed to take snapshot at block #{}: {}", num, e),
 						}
-
 					}
 				});
 

ethcore/snapshot/Cargo.toml

@@ -33,6 +33,7 @@ rand_xorshift = "0.2"
 parking_lot = "0.9"
 rlp = "0.4.2"
 rlp_derive = { path = "../../util/rlp-derive" }
+scopeguard = "1.0.0"
 snappy = { package = "parity-snappy", version ="0.1.0" }
 state-db = { path = "../state-db" }
 trie-db = "0.15.0"

ethcore/snapshot/snapshot-tests/src/account.rs

@@ -48,7 +48,7 @@ fn encoding_basic() {
 
 	let thin_rlp = ::rlp::encode(&account);
 	assert_eq!(::rlp::decode::<BasicAccount>(&thin_rlp).unwrap(), account);
-	let p = Progress::default();
+	let p = Progress::new();
 	let fat_rlps = to_fat_rlps(&keccak(&addr), &account, &AccountDB::from_hash(db.as_hash_db(), keccak(addr)), &mut Default::default(), usize::max_value(), usize::max_value(), &p).unwrap();
 	let fat_rlp = Rlp::new(&fat_rlps[0]).at(1).unwrap();
 	assert_eq!(from_fat_rlp(&mut AccountDBMut::from_hash(db.as_hash_db_mut(), keccak(addr)), fat_rlp, H256::zero()).unwrap().0, account);
@@ -69,7 +69,7 @@ fn encoding_version() {
 
 	let thin_rlp = ::rlp::encode(&account);
 	assert_eq!(::rlp::decode::<BasicAccount>(&thin_rlp).unwrap(), account);
-	let p = Progress::default();
+	let p = Progress::new();
 	let fat_rlps = to_fat_rlps(&keccak(&addr), &account, &AccountDB::from_hash(db.as_hash_db(), keccak(addr)), &mut Default::default(), usize::max_value(), usize::max_value(), &p).unwrap();
 	let fat_rlp = Rlp::new(&fat_rlps[0]).at(1).unwrap();
 	assert_eq!(from_fat_rlp(&mut AccountDBMut::from_hash(db.as_hash_db_mut(), keccak(addr)), fat_rlp, H256::zero()).unwrap().0, account);
@@ -96,7 +96,7 @@ fn encoding_storage() {
 	let thin_rlp = ::rlp::encode(&account);
 	assert_eq!(::rlp::decode::<BasicAccount>(&thin_rlp).unwrap(), account);
 
-	let p = Progress::default();
+	let p = Progress::new();
 
 	let fat_rlp = to_fat_rlps(&keccak(&addr), &account, &AccountDB::from_hash(db.as_hash_db(), keccak(addr)), &mut Default::default(), usize::max_value(), usize::max_value(), &p).unwrap();
 	let fat_rlp = Rlp::new(&fat_rlp[0]).at(1).unwrap();
@@ -124,7 +124,7 @@ fn encoding_storage_split() {
 	let thin_rlp = ::rlp::encode(&account);
 	assert_eq!(::rlp::decode::<BasicAccount>(&thin_rlp).unwrap(), account);
 
-	let p = Progress::default();
+	let p = Progress::new();
 	let fat_rlps = to_fat_rlps(&keccak(addr), &account, &AccountDB::from_hash(db.as_hash_db(), keccak(addr)), &mut Default::default(), 500, 1000, &p).unwrap();
 	let mut root = KECCAK_NULL_RLP;
 	let mut restored_account = None;
@@ -170,8 +170,8 @@ fn encoding_code() {
 	};
 
 	let mut used_code = HashSet::new();
-	let p1 = Progress::default();
+	let p1 = Progress::new();
-	let p2 = Progress::default();
+	let p2 = Progress::new();
 	let fat_rlp1 = to_fat_rlps(&keccak(&addr1), &account1, &AccountDB::from_hash(db.as_hash_db(), keccak(addr1)), &mut used_code, usize::max_value(), usize::max_value(), &p1).unwrap();
 	let fat_rlp2 = to_fat_rlps(&keccak(&addr2), &account2, &AccountDB::from_hash(db.as_hash_db(), keccak(addr2)), &mut used_code, usize::max_value(), usize::max_value(), &p2).unwrap();
 	assert_eq!(used_code.len(), 1);

ethcore/snapshot/snapshot-tests/src/helpers.rs

@@ -27,7 +27,8 @@ use client_traits::ChainInfo;
 use common_types::{
 	ids::BlockId,
 	basic_account::BasicAccount,
-	errors::EthcoreError
+	errors::EthcoreError,
+	snapshot::Progress,
 };
 use engine::Engine;
 use ethcore::client::Client;
@@ -145,7 +146,7 @@ pub fn snap(client: &Client) -> (Box<dyn SnapshotReader>, TempDir) {
 	let tempdir = TempDir::new("").unwrap();
 	let path = tempdir.path().join("file");
 	let writer = PackedWriter::new(&path).unwrap();
-	let progress = Default::default();
+	let progress = Progress::new();
 
 	let hash = client.chain_info().best_block_hash;
 	client.take_snapshot(writer, BlockId::Hash(hash), &progress).unwrap();

ethcore/snapshot/snapshot-tests/src/proof_of_work.rs

@@ -74,7 +74,7 @@ fn chunk_and_restore(amount: u64) {
 		&bc,
 		best_hash,
 		&writer,
-		&Progress::default()
+		&Progress::new()
 	).unwrap();
 
 	let manifest = ManifestData {

ethcore/snapshot/snapshot-tests/src/service.rs

@@ -278,7 +278,7 @@ fn keep_ancient_blocks() {
 		&bc,
 		best_hash,
 		&writer,
-		&Progress::default()
+		&Progress::new()
 	).unwrap();
 	let state_db = client.state_db().journal_db().boxed_clone();
 	let start_header = bc.block_header_data(&best_hash).unwrap();
@@ -287,7 +287,7 @@ fn keep_ancient_blocks() {
 		state_db.as_hash_db(),
 		&state_root,
 		&writer,
-		&Progress::default(),
+		&Progress::new(),
 		None,
 		0
 	).unwrap();

ethcore/snapshot/snapshot-tests/src/state.rs

@@ -62,7 +62,7 @@ fn snap_and_restore() {
 
 	let mut state_hashes = Vec::new();
 	for part in 0..SNAPSHOT_SUBPARTS {
-		let mut hashes = chunk_state(&old_db, &state_root, &writer, &Progress::default(), Some(part), 0).unwrap();
+		let mut hashes = chunk_state(&old_db, &state_root, &writer, &Progress::new(), Some(part), 0).unwrap();
 		state_hashes.append(&mut hashes);
 	}
 
@@ -133,7 +133,7 @@ fn get_code_from_prev_chunk() {
 	let mut make_chunk = |acc, hash| {
 		let mut db = journaldb::new_memory_db();
 		AccountDBMut::from_hash(&mut db, hash).insert(EMPTY_PREFIX, &code[..]);
-		let p = Progress::default();
+		let p = Progress::new();
 		let fat_rlp = to_fat_rlps(&hash, &acc, &AccountDB::from_hash(&db, hash), &mut used_code, usize::max_value(), usize::max_value(), &p).unwrap();
 		let mut stream = RlpStream::new_list(1);
 		stream.append_raw(&fat_rlp[0], 1);
@@ -178,7 +178,7 @@ fn checks_flag() {
 	let state_root = producer.state_root();
 	let writer = Mutex::new(PackedWriter::new(&snap_file).unwrap());
 
-	let state_hashes = chunk_state(&old_db, &state_root, &writer, &Progress::default(), None, 0).unwrap();
+	let state_hashes = chunk_state(&old_db, &state_root, &writer, &Progress::new(), None, 0).unwrap();
 
 	writer.into_inner().finish(ManifestData {
 		version: 2,

ethcore/snapshot/snapshot-tests/src/watcher.rs

@@ -41,8 +41,8 @@ impl Oracle for TestOracle {
 
 struct TestBroadcast(Option<u64>);
 impl Broadcast for TestBroadcast {
-	fn take_at(&self, num: Option<u64>) {
+	fn request_snapshot_at(&self, num: u64) {
-		if num != self.0 {
+		if Some(num) != self.0 {
 			panic!("Watcher broadcast wrong number. Expected {:?}, found {:?}", self.0, num);
 		}
 	}

ethcore/snapshot/src/consensus/work.rs

@@ -178,7 +178,7 @@ impl<'a> PowWorker<'a> {
 		let parent_hash = last_header.parent_hash();
 		let parent_total_difficulty = last_details.total_difficulty - last_header.difficulty();
 
-		trace!(target: "snapshot", "parent last written block: {}", parent_hash);
+		trace!(target: "snapshot", "parent last written block: #{}/{}", parent_number, parent_hash);
 
 		let num_entries = self.rlps.len();
 		let mut rlp_stream = RlpStream::new_list(3 + num_entries);

ethcore/snapshot/src/lib.rs

@@ -129,35 +129,34 @@ pub fn take_snapshot<W: SnapshotWriter + Send>(
 	let state_root = start_header.state_root();
 	let block_number = start_header.number();
 
-	info!("Taking snapshot starting at block {}", block_number);
+	info!("Taking snapshot starting at block #{}/{:?}", block_number, block_hash);
-
 	let version = chunker.current_version();
 	let writer = Mutex::new(writer);
 	let (state_hashes, block_hashes) = thread::scope(|scope| -> Result<(Vec<H256>, Vec<H256>), Error> {
 		let writer = &writer;
-		let block_guard = scope.spawn(move |_| {
+		let tb = scope.builder().name("Snapshot Worker - Blocks".to_string());
+		let block_guard = tb.spawn(move |_| {
 			chunk_secondary(chunker, chain, block_hash, writer, p)
-		});
+		})?;
 
 		// The number of threads must be between 1 and SNAPSHOT_SUBPARTS
 		assert!(processing_threads >= 1, "Cannot use less than 1 threads for creating snapshots");
-		let num_threads: usize = cmp::min(processing_threads, SNAPSHOT_SUBPARTS);
+		let num_threads = cmp::min(processing_threads, SNAPSHOT_SUBPARTS);
 		info!(target: "snapshot", "Using {} threads for Snapshot creation.", num_threads);
 
-		let mut state_guards = Vec::with_capacity(num_threads as usize);
+		let mut state_guards = Vec::with_capacity(num_threads);
 
 		for thread_idx in 0..num_threads {
-			let state_guard = scope.spawn(move |_| -> Result<Vec<H256>, Error> {
+			let tb = scope.builder().name(format!("Snapshot Worker #{} - State", thread_idx).to_string());
+			let state_guard = tb.spawn(move |_| -> Result<Vec<H256>, Error> {
 				let mut chunk_hashes = Vec::new();
-
 				for part in (thread_idx..SNAPSHOT_SUBPARTS).step_by(num_threads) {
-					debug!(target: "snapshot", "Chunking part {} in thread {}", part, thread_idx);
+					debug!(target: "snapshot", "Chunking part {} of the state at {} in thread {}", part, block_number, thread_idx);
 					let mut hashes = chunk_state(state_db, &state_root, writer, p, Some(part), thread_idx)?;
 					chunk_hashes.append(&mut hashes);
 				}
-
 				Ok(chunk_hashes)
-			});
+			})?;
 			state_guards.push(state_guard);
 		}
 
@@ -169,7 +168,8 @@ pub fn take_snapshot<W: SnapshotWriter + Send>(
 			state_hashes.extend(part_state_hashes);
 		}
 
-		debug!(target: "snapshot", "Took a snapshot of {} accounts", p.accounts.load(Ordering::SeqCst));
+		info!("Took a snapshot at #{} of {} accounts", block_number, p.accounts());
+
 		Ok((state_hashes, block_hashes))
 	}).expect("Sub-thread never panics; qed")?;
 
@@ -218,7 +218,7 @@ pub fn chunk_secondary<'a>(
 			trace!(target: "snapshot", "wrote secondary chunk. hash: {:x}, size: {}, uncompressed size: {}",
 				hash, size, raw_data.len());
 
-			progress.size.fetch_add(size as u64, Ordering::SeqCst);
+			progress.update(0, size);
 			chunk_hashes.push(hash);
 			Ok(())
 		};
@@ -275,8 +275,7 @@ impl<'a> StateChunker<'a> {
 		self.writer.lock().write_state_chunk(hash, compressed)?;
 		trace!(target: "snapshot", "Thread {} wrote state chunk. size: {}, uncompressed size: {}", self.thread_idx, compressed_size, raw_data.len());
 
-		self.progress.accounts.fetch_add(num_entries, Ordering::SeqCst);
+		self.progress.update(num_entries, compressed_size);
-		self.progress.size.fetch_add(compressed_size as u64, Ordering::SeqCst);
 
 		self.hashes.push(hash);
 		self.cur_size = 0;
@@ -327,7 +326,7 @@ pub fn chunk_state<'a>(
 	if let Some(part) = part {
 		assert!(part < 16, "Wrong chunk state part number (must be <16) in snapshot creation.");
 
-		let part_offset = MAX_SNAPSHOT_SUBPARTS / SNAPSHOT_SUBPARTS;
+		let part_offset = MAX_SNAPSHOT_SUBPARTS / SNAPSHOT_SUBPARTS; // 16
 		let mut seek_from = vec![0; 32];
 		seek_from[0] = (part * part_offset) as u8;
 		account_iter.seek(&seek_from)?;
@@ -349,7 +348,15 @@ pub fn chunk_state<'a>(
 		let account = ::rlp::decode(&*account_data)?;
 		let account_db = AccountDB::from_hash(db, account_key_hash);
 
-		let fat_rlps = account::to_fat_rlps(&account_key_hash, &account, &account_db, &mut used_code, PREFERRED_CHUNK_SIZE - chunker.chunk_size(), PREFERRED_CHUNK_SIZE, progress)?;
+		let fat_rlps = account::to_fat_rlps(
+			&account_key_hash,
+			&account,
+			&account_db,
+			&mut used_code,
+			PREFERRED_CHUNK_SIZE - chunker.chunk_size(),
+			PREFERRED_CHUNK_SIZE,
+			progress
+		)?;
 		for (i, fat_rlp) in fat_rlps.into_iter().enumerate() {
 			if i > 0 {
 				chunker.write_chunk()?;

ethcore/snapshot/src/service.rs

@@ -39,7 +39,7 @@ use ethcore_io::IoChannel;
 use journaldb::Algorithm;
 use keccak_hash::keccak;
 use kvdb::DBTransaction;
-use log::{error, info, trace, warn};
+use log::{debug, error, info, trace, warn};
 use parking_lot::{Mutex, RwLock, RwLockReadGuard};
 use snappy;
 use trie_db::TrieError;
@@ -212,7 +212,7 @@ impl Restoration {
 		}
 
 		self.guard.disarm();
-		trace!(target: "snapshot", "restoration finalised correctly");
+		trace!(target: "snapshot", "Restoration finalised correctly");
 		Ok(())
 	}
 
@@ -280,7 +280,7 @@ impl<C> Service<C> where C: SnapshotClient + ChainInfo {
 			state_chunks: AtomicUsize::new(0),
 			block_chunks: AtomicUsize::new(0),
 			client: params.client,
-			progress: Default::default(),
+			progress: Progress::new(),
 			taking_snapshot: AtomicBool::new(false),
 			restoring_snapshot: AtomicBool::new(false),
 		};
@@ -413,7 +413,10 @@ impl<C> Service<C> where C: SnapshotClient + ChainInfo {
 			Some(x) => x,
 			None => return Ok(0),
 		};
-
+		info!(target: "snapshot", "Migrating blocks from old db to new. Start: #{}/{:?}, Target: #{}/{:?}",
+			self.client.block_number(BlockId::Hash(start_hash)).unwrap_or_default(), start_hash,
+			self.client.block_number(BlockId::Hash(target_hash)).unwrap_or_default(), target_hash,
+		);
 		let mut batch = DBTransaction::new();
 		let mut parent_hash = start_hash;
 		while parent_hash != target_hash {
@@ -455,12 +458,12 @@ impl<C> Service<C> where C: SnapshotClient + ChainInfo {
 				next_chain.commit();
 				next_db.key_value().flush().expect("DB flush failed.");
 				batch = DBTransaction::new();
-			}
 
 				if block_number % 10_000 == 0 {
 					info!(target: "snapshot", "Block restoration at #{}", block_number);
 				}
 			}
+		}
 
 		// Final commit to the DB
 		next_db.key_value().write_buffered(batch);
@@ -483,11 +486,13 @@ impl<C> Service<C> where C: SnapshotClient + ChainInfo {
 		if self.progress.done() || !self.taking_snapshot.load(Ordering::SeqCst) { return }
 
 		let p = &self.progress;
-		info!("Snapshot: {} accounts {} blocks {} bytes", p.accounts(), p.blocks(), p.size());
+		info!("Snapshot: {} accounts, {} blocks, {} bytes", p.accounts(), p.blocks(), p.bytes());
+		let rate = p.rate();
+		debug!(target: "snapshot", "Current progress rate: {:.0} acc/s, {:.0} bytes/s (compressed)", rate.0, rate.1);
 	}
 
 	/// Take a snapshot at the block with the given number.
-	/// calling this while a restoration is in progress or vice versa
+	/// Calling this while a restoration is in progress or vice versa
 	/// will lead to a race condition where the first one to finish will
 	/// have their produced snapshot overwritten.
 	pub fn take_snapshot(&self, client: &C, num: u64) -> Result<(), Error> {
@@ -497,46 +502,41 @@ impl<C> Service<C> where C: SnapshotClient + ChainInfo {
 		}
 
 		info!("Taking snapshot at #{}", num);
+		{
+			scopeguard::defer! {{
+				self.taking_snapshot.store(false, Ordering::SeqCst);
+			}}
+			let start_time = std::time::Instant::now();
 			self.progress.reset();
 
 			let temp_dir = self.temp_snapshot_dir();
 			let snapshot_dir = self.snapshot_dir();
 
-		let _ = fs::remove_dir_all(&temp_dir);
+			let _ = fs::remove_dir_all(&temp_dir); // expected to fail
 
 			let writer = LooseWriter::new(temp_dir.clone())?;
 
 			let guard = Guard::new(temp_dir.clone());
-		let res = client.take_snapshot(writer, BlockId::Number(num), &self.progress);
+			let _ = client.take_snapshot(writer, BlockId::Number(num), &self.progress)?;
-		self.taking_snapshot.store(false, Ordering::SeqCst);
+			info!("Finished taking snapshot at #{}, in {:.0?}", num, start_time.elapsed());
-		if let Err(e) = res {
-			if client.chain_info().best_block_number >= num + client.pruning_history() {
-				// The state we were snapshotting was pruned before we could finish.
-				info!("Periodic snapshot failed: block state pruned. Run with a longer `--pruning-history` or with `--no-periodic-snapshot`");
-				return Err(e);
-			} else {
-				return Err(e);
-			}
-		}
-
-		info!("Finished taking snapshot at #{}", num);
-
-		let mut reader = self.reader.write();
 
 			// destroy the old snapshot reader.
+			let mut reader = self.reader.write();
 			*reader = None;
 
 			if snapshot_dir.exists() {
+				trace!(target: "snapshot", "Removing previous snapshot at {:?}", &snapshot_dir);
 				fs::remove_dir_all(&snapshot_dir)?;
 			}
 
 			fs::rename(temp_dir, &snapshot_dir)?;
-
+			trace!(target: "snapshot", "Moved new snapshot into place at {:?}", &snapshot_dir);
 			*reader = Some(LooseReader::new(snapshot_dir)?);
 
 			guard.disarm();
 			Ok(())
 		}
+	}
 
 	/// Initialize the restoration synchronously.
 	/// The recover flag indicates whether to recover the restored snapshot.
@@ -654,13 +654,19 @@ impl<C> Service<C> where C: SnapshotClient + ChainInfo {
 		Ok(())
 	}
 
-	/// Import a previous chunk at the given path. Returns whether the block was imported or not
+	/// Import a previous chunk at the given path. Returns whether the chunk was imported or not
-	fn import_prev_chunk(&self, restoration: &mut Option<Restoration>, manifest: &ManifestData, file: io::Result<fs::DirEntry>) -> Result<bool, Error> {
+	fn import_prev_chunk(
+		&self,
+		restoration: &mut Option<Restoration>,
+		manifest: &ManifestData,
+		file: io::Result<fs::DirEntry>
+	) -> Result<bool, Error> {
 		let file = file?;
 		let path = file.path();
 
 		let mut file = File::open(path.clone())?;
-		let mut buffer = Vec::new();
+		let filesize = file.metadata()?.len();
+		let mut buffer = Vec::with_capacity(filesize as usize + 1); // +1 for EOF
 		file.read_to_end(&mut buffer)?;
 
 		let hash = keccak(&buffer);
@@ -670,6 +676,7 @@ impl<C> Service<C> where C: SnapshotClient + ChainInfo {
 		} else if manifest.state_hashes.contains(&hash) {
 			true
 		} else {
+			warn!(target: "snapshot", "Hash of the content of {:?} not present in the manifest block/state hashes.", path);
 			return Ok(false);
 		};
 
@@ -680,11 +687,10 @@ impl<C> Service<C> where C: SnapshotClient + ChainInfo {
 		Ok(true)
 	}
 
-	// finalize the restoration. this accepts an already-locked
+	// Finalize the restoration. This accepts an already-locked restoration as an argument -- so
-	// restoration as an argument -- so acquiring it again _will_
+	// acquiring it again _will_ lead to deadlock.
-	// lead to deadlock.
 	fn finalize_restoration(&self, rest: &mut Option<Restoration>) -> Result<(), Error> {
-		trace!(target: "snapshot", "finalizing restoration");
+		trace!(target: "snapshot", "Finalizing restoration");
 		*self.status.lock() = RestorationStatus::Finalizing;
 
 		let recover = rest.as_ref().map_or(false, |rest| rest.writer.is_some());
@@ -695,7 +701,7 @@ impl<C> Service<C> where C: SnapshotClient + ChainInfo {
 			.unwrap_or(Ok(()))?;
 
 		let migrated_blocks = self.migrate_blocks()?;
-		info!(target: "snapshot", "Migrated {} ancient blocks", migrated_blocks);
+		info!(target: "snapshot", "Migrated {} ancient blocks from the old DB", migrated_blocks);
 
 		// replace the Client's database with the new one (restart the Client).
 		self.client.restore_db(&*self.restoration_db().to_string_lossy())?;
@@ -707,11 +713,11 @@ impl<C> Service<C> where C: SnapshotClient + ChainInfo {
 			let snapshot_dir = self.snapshot_dir();
 
 			if snapshot_dir.exists() {
-				trace!(target: "snapshot", "removing old snapshot dir at {}", snapshot_dir.to_string_lossy());
+				trace!(target: "snapshot", "Removing old snapshot dir at {}", snapshot_dir.to_string_lossy());
 				fs::remove_dir_all(&snapshot_dir)?;
 			}
 
-			trace!(target: "snapshot", "copying restored snapshot files over");
+			trace!(target: "snapshot", "Copying restored snapshot files over");
 			fs::rename(self.temp_recovery_dir(), &snapshot_dir)?;
 
 			*reader = Some(LooseReader::new(snapshot_dir)?);

ethcore/snapshot/src/traits.rs

@@ -136,7 +136,7 @@ pub trait SnapshotComponents: Send {
 /// Snapshot related functionality
 pub trait SnapshotClient: BlockChainClient + BlockInfo + DatabaseRestore + BlockChainReset {
 	/// Take a snapshot at the given block.
-	/// If the ID given is "latest", this will default to 1000 blocks behind.
+	/// If the BlockId is 'Latest', this will default to 1000 blocks behind.
 	fn take_snapshot<W: SnapshotWriter + Send>(
 		&self,
 		writer: W,
@@ -148,7 +148,7 @@ pub trait SnapshotClient: BlockChainClient + BlockInfo + DatabaseRestore + Block
 /// Helper trait for broadcasting a block to take a snapshot at.
 pub trait Broadcast: Send + Sync {
 	/// Start a snapshot from the given block number.
-	fn take_at(&self, num: Option<u64>);
+	fn request_snapshot_at(&self, num: u64);
 }
 
 

ethcore/snapshot/src/watcher.rs

@@ -49,16 +49,11 @@ impl<F> Oracle for StandardOracle<F>
 }
 
 impl<C: 'static> Broadcast for Mutex<IoChannel<ClientIoMessage<C>>> {
-	fn take_at(&self, num: Option<u64>) {
+	fn request_snapshot_at(&self, num: u64) {
-		let num = match num {
-			Some(n) => n,
-			None => return,
-		};
-
-		trace!(target: "snapshot_watcher", "Snapshot requested at block #{}", num);
-
 		if let Err(e) = self.lock().send(ClientIoMessage::TakeSnapshot(num)) {
-			warn!("Snapshot watcher disconnected from IoService: {}", e);
+			warn!(target: "snapshot_watcher", "Snapshot watcher disconnected from IoService: {}", e);
+		} else {
+			trace!(target: "snapshot_watcher", "Snapshot requested at block #{}", num);
 		}
 	}
 }
@@ -68,7 +63,10 @@ impl<C: 'static> Broadcast for Mutex<IoChannel<ClientIoMessage<C>>> {
 pub struct Watcher {
 	oracle: Box<dyn Oracle>,
 	broadcast: Box<dyn Broadcast>,
+	// How often we attempt to take a snapshot: only snapshot on blocknumbers that are multiples of
+	// `period`. Always set to `SNAPSHOT_PERIOD`, i.e. 5000.
 	period: u64,
+	// Start snapshots `history` blocks from the tip. Always set to `SNAPSHOT_HISTORY`, i.e. 100.
 	history: u64,
 }
 
@@ -106,18 +104,22 @@ impl ChainNotify for Watcher {
 	fn new_blocks(&self, new_blocks: NewBlocks) {
 		if self.oracle.is_major_importing() || new_blocks.has_more_blocks_to_import { return }
 
-		trace!(target: "snapshot_watcher", "{} imported", new_blocks.imported.len());
+		// Decide if it's time for a snapshot: the highest of the imported blocks is a multiple of 5000?
-
 		let highest = new_blocks.imported.into_iter()
+			// Convert block hashes to block numbers for all newly imported blocks
 			.filter_map(|h| self.oracle.to_number(h))
-			.filter(|&num| num >= self.period + self.history)
+			// Subtract `history` (i.e. `SNAPSHOT_HISTORY`, i.e. 100) from the block numbers to stay
-			.map(|num| num - self.history)
+			// clear of reorgs.
-			.filter(|num| num % self.period == 0)
+			.map(|num| num.saturating_sub(self.history) )
+			// …filter out blocks that do not fall on the a multiple of `period`. This regulates the
+			// frequency of snapshots and ensures more snapshots are produced from similar points in
+			// the chain.
+			.filter(|num| num % self.period == 0 )
+			// Pick newest of the candidates: this is where we want to snapshot from.
 			.fold(0, ::std::cmp::max);
 
-		match highest {
+		if highest > 0 {
-			0 => self.broadcast.take_at(None),
+			self.broadcast.request_snapshot_at(highest);
-			_ => self.broadcast.take_at(Some(highest)),
 		}
 	}
 }

ethcore/src/client/client.rs

@@ -20,7 +20,7 @@ use std::convert::TryFrom;
 use std::io::{BufRead, BufReader};
 use std::str::from_utf8;
 use std::sync::{Arc, Weak};
-use std::sync::atomic::{AtomicBool, AtomicI64, Ordering as AtomicOrdering};
+use std::sync::atomic::{AtomicBool, AtomicI64, Ordering as AtomicOrdering, Ordering, AtomicU64};
 use std::time::{Duration, Instant};
 
 use ansi_term::Colour;
@@ -204,6 +204,9 @@ pub struct Client {
 	/// Database pruning strategy to use for StateDB
 	pruning: journaldb::Algorithm,
 
+	/// Don't prune the state we're currently snapshotting
+	snapshotting_at: AtomicU64,
+
 	/// Client uses this to store blocks, traces, etc.
 	db: RwLock<Arc<dyn BlockChainDB>>,
 
@@ -780,6 +783,7 @@ impl Client {
 			tracedb,
 			engine,
 			pruning: config.pruning.clone(),
+			snapshotting_at: AtomicU64::new(0),
 			db: RwLock::new(db.clone()),
 			state_db: RwLock::new(state_db),
 			report: RwLock::new(Default::default()),
@@ -964,13 +968,15 @@ impl Client {
 			return Ok(())
 		}
 
-		let number = match state_db.journal_db().latest_era() {
+		let latest_era = match state_db.journal_db().latest_era() {
 			Some(n) => n,
 			None => return Ok(()),
 		};
 
-		// prune all ancient eras until we're below the memory target,
+		// Prune all ancient eras until we're below the memory target (default: 32Mb),
-		// but have at least the minimum number of states.
+		// but have at least the minimum number of states, i.e. `history`.
+		// If a snapshot is under way, no pruning happens and memory consumption is allowed to
+		// increase above the memory target until the snapshot has finished.
 		loop {
 			let needs_pruning = state_db.journal_db().journal_size() >= self.config.history_mem;
 
@@ -979,17 +985,27 @@ impl Client {
 			}
 
 			match state_db.journal_db().earliest_era() {
-				Some(era) if era + self.history <= number => {
+				Some(earliest_era) if earliest_era + self.history <= latest_era => {
-					trace!(target: "client", "Pruning state for ancient era {}", era);
+					let freeze_at = self.snapshotting_at.load(Ordering::SeqCst);
-					match chain.block_hash(era) {
+					if freeze_at > 0 && freeze_at == earliest_era {
+						// Note: journal_db().mem_used() can be used for a more accurate memory
+						// consumption measurement but it can be expensive so sticking with the
+						// faster `journal_size()` instead.
+						trace!(target: "pruning", "Pruning is paused at era {} (snapshot under way); earliest era={}, latest era={}, journal_size={} – Not pruning.",
+						       freeze_at, earliest_era, latest_era, state_db.journal_db().journal_size());
+						break;
+					}
+					trace!(target: "pruning", "Pruning state for ancient era #{}; latest era={}, journal_size={}",
+					       earliest_era, latest_era, state_db.journal_db().journal_size());
+					match chain.block_hash(earliest_era) {
 						Some(ancient_hash) => {
 							let mut batch = DBTransaction::new();
-							state_db.mark_canonical(&mut batch, era, &ancient_hash)?;
+							state_db.mark_canonical(&mut batch, earliest_era, &ancient_hash)?;
 							self.db.read().key_value().write_buffered(batch);
 							state_db.journal_db().flush();
 						}
 						None =>
-							debug!(target: "client", "Missing expected hash for block {}", era),
+							debug!(target: "pruning", "Missing expected hash for block {}", earliest_era),
 					}
 				}
 				_ => break, // means that every era is kept, no pruning necessary.
@@ -2533,39 +2549,53 @@ impl SnapshotClient for Client {
 			return Err(EthcoreError::Snapshot(SnapshotError::SnapshotsUnsupported));
 		}
 		let db = self.state_db.read().journal_db().boxed_clone();
-		let best_block_number = self.chain_info().best_block_number;
-		let block_number = self.block_number(at).ok_or_else(|| SnapshotError::InvalidStartingBlock(at))?;
 
-		if db.is_prunable() && self.pruning_info().earliest_state > block_number {
+		let block_number = self.block_number(at).ok_or_else(|| SnapshotError::InvalidStartingBlock(at))?;
+		let earliest_era = db.earliest_era().unwrap_or(0);
+		if db.is_prunable() && earliest_era > block_number {
 			return Err(SnapshotError::OldBlockPrunedDB.into());
 		}
 
-		let history = cmp::min(self.history, 1000);
 
-		let start_hash = match at {
+		let (actual_block_nr, block_hash) = match at {
 			BlockId::Latest => {
-				let start_num = match db.earliest_era() {
+				// Start `self.history` blocks from the best block, but no further back than 1000
-					Some(era) => cmp::max(era, best_block_number.saturating_sub(history)),
+				// blocks (or earliest era, whichever is greatest).
-					None => best_block_number.saturating_sub(history),
+				let history = cmp::min(self.history, 1000);
-				};
+				let best_block_number = self.chain_info().best_block_number;
+				let start_num = cmp::max(earliest_era, best_block_number.saturating_sub(history));
 
 				match self.block_hash(BlockId::Number(start_num)) {
-					Some(h) => h,
+					Some(hash) => (start_num, hash),
-					None => return Err(SnapshotError::InvalidStartingBlock(at).into()),
+					None => {
+						error!(target: "snapshot", "Can't take snapshot at {:?}: missing hash for the starting block #{}", at, start_num);
+						return Err(SnapshotError::InvalidStartingBlock(at).into())
+					},
 				}
 			}
 			_ => match self.block_hash(at) {
-				Some(hash) => hash,
+				Some(hash) => (block_number, hash),
 				None => return Err(SnapshotError::InvalidStartingBlock(at).into()),
 			},
 		};
 
 		let processing_threads = self.config.snapshot.processing_threads;
+		trace!(target: "snapshot", "Snapshot requested at block {:?}. Using block #{}/{:?}. Earliest block: #{}, earliest state era #{}. Using {} threads.",
+			at, actual_block_nr, block_hash, self.pruning_info().earliest_chain, earliest_era, processing_threads,
+		);
+		// Stop pruning from happening while the snapshot is under way.
+		self.snapshotting_at.store(actual_block_nr, Ordering::SeqCst);
+		{
+			scopeguard::defer! {{
+				trace!(target: "snapshot", "Re-enabling pruning.");
+				self.snapshotting_at.store(0, Ordering::SeqCst)
+			}};
 			let chunker = snapshot::chunker(self.engine.snapshot_mode()).ok_or_else(|| SnapshotError::SnapshotsUnsupported)?;
+			// Spawn threads and take snapshot
 			snapshot::take_snapshot(
 				chunker,
 				&self.chain.read(),
-			start_hash,
+				block_hash,
 				db.as_hash_db(),
 				writer,
 				p,
@@ -2573,8 +2603,7 @@ impl SnapshotClient for Client {
 			)?;
 			Ok(())
 		}
-
+	}
-
 }
 
 impl ImportExportBlocks for Client {

ethcore/src/client/config.rs

@@ -90,7 +90,7 @@ pub struct ClientConfig {
 	pub history: u64,
 	/// Ideal memory usage for state pruning history.
 	pub history_mem: usize,
-	/// Check seal valididity on block import
+	/// Check seal validity on block import
 	pub check_seal: bool,
 	/// Maximal number of transactions queued for verification in a separate thread.
 	pub transaction_verification_queue_size: usize,

ethcore/types/Cargo.toml

@@ -15,6 +15,7 @@ parity-bytes = "0.1"
 parity-crypto = { version = "0.4.2", features = ["publickey"] }
 parity-util-mem = "0.2.0"
 parity-snappy = "0.1"
+parking_lot = "0.9.0"
 patricia-trie-ethereum = { path = "../../util/patricia-trie-ethereum" }
 rlp = "0.4.0"
 rlp_derive = { path = "../../util/rlp-derive" }

ethcore/types/src/lib.rs

@@ -40,6 +40,7 @@ extern crate parity_crypto;
 #[macro_use]
 extern crate derive_more;
 extern crate keccak_hash as hash;
+extern crate parking_lot;
 extern crate parity_bytes as bytes;
 extern crate patricia_trie_ethereum as ethtrie;
 extern crate rlp;

ethcore/types/src/snapshot.rs

@@ -16,11 +16,13 @@
 
 //! Snapshot type definitions
 
-use std::sync::atomic::{AtomicBool, AtomicUsize, AtomicU64, Ordering};
+use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
+use std::time::Instant;
 
-use ethereum_types::H256;
-use rlp::{Rlp, RlpStream, DecoderError};
 use bytes::Bytes;
+use ethereum_types::H256;
+use parking_lot::RwLock;
+use rlp::{Rlp, RlpStream, DecoderError};
 
 /// Modes of snapshotting
 pub enum Snapshotting {
@@ -39,31 +41,53 @@ pub enum Snapshotting {
 }
 
 /// A progress indicator for snapshots.
-#[derive(Debug, Default)]
+#[derive(Debug)]
 pub struct Progress {
 	/// Number of accounts processed so far
-	pub accounts: AtomicUsize,
+	accounts: AtomicUsize,
+	// Number of accounts processed at last tick.
+	prev_accounts: AtomicUsize,
 	/// Number of blocks processed so far
 	pub blocks: AtomicUsize,
 	/// Size in bytes of a all compressed chunks processed so far
-	pub size: AtomicU64,
+	bytes: AtomicUsize,
+	// Number of bytes processed at last tick.
+	prev_bytes: AtomicUsize,
 	/// Signals that the snapshotting process is completed
 	pub done: AtomicBool,
 	/// Signal snapshotting process to abort
 	pub abort: AtomicBool,
+
+	last_tick: RwLock<Instant>,
 }
 
 impl Progress {
+	/// Create a new progress tracker.
+	pub fn new() -> Progress {
+		Progress {
+			accounts: AtomicUsize::new(0),
+			prev_accounts: AtomicUsize::new(0),
+			blocks: AtomicUsize::new(0),
+			bytes: AtomicUsize::new(0),
+			prev_bytes: AtomicUsize::new(0),
+			abort: AtomicBool::new(false),
+			done: AtomicBool::new(false),
+			last_tick: RwLock::new(Instant::now()),
+		}
+	}
+
 	/// Reset the progress.
 	pub fn reset(&self) {
 		self.accounts.store(0, Ordering::Release);
 		self.blocks.store(0, Ordering::Release);
-		self.size.store(0, Ordering::Release);
+		self.bytes.store(0, Ordering::Release);
 		self.abort.store(false, Ordering::Release);
 
 		// atomic fence here to ensure the others are written first?
 		// logs might very rarely get polluted if not.
 		self.done.store(false, Ordering::Release);
+
+		*self.last_tick.write() = Instant::now();
 	}
 
 	/// Get the number of accounts snapshotted thus far.
@@ -73,10 +97,37 @@ impl Progress {
 	pub fn blocks(&self) -> usize { self.blocks.load(Ordering::Acquire) }
 
 	/// Get the written size of the snapshot in bytes.
-	pub fn size(&self) -> u64 { self.size.load(Ordering::Acquire) }
+	pub fn bytes(&self) -> usize { self.bytes.load(Ordering::Acquire) }
 
 	/// Whether the snapshot is complete.
 	pub fn done(&self) -> bool  { self.done.load(Ordering::Acquire) }
+
+	/// Return the progress rate over the last tick (i.e. since last update).
+	pub fn rate(&self) -> (f64, f64) {
+		let last_tick = *self.last_tick.read();
+		let dt = last_tick.elapsed().as_secs_f64();
+		if dt < 1.0 {
+			return (0f64, 0f64);
+		}
+		let delta_acc = self.accounts.load(Ordering::Relaxed)
+			.saturating_sub(self.prev_accounts.load(Ordering::Relaxed));
+		let delta_bytes = self.bytes.load(Ordering::Relaxed)
+			.saturating_sub(self.prev_bytes.load(Ordering::Relaxed));
+		(delta_acc as f64 / dt, delta_bytes as f64 / dt)
+	}
+
+	/// Update state progress counters and set the last tick.
+	pub fn update(&self, accounts_delta: usize, bytes_delta: usize) {
+		*self.last_tick.write() = Instant::now();
+		self.prev_accounts.store(
+			self.accounts.fetch_add(accounts_delta, Ordering::SeqCst),
+			Ordering::SeqCst
+		);
+		self.prev_bytes.store(
+			self.bytes.fetch_add(bytes_delta, Ordering::SeqCst),
+			Ordering::SeqCst
+		);
+	}
 }
 
 /// Manifest data.

parity/run.rs

@@ -68,10 +68,10 @@ use signer;
 use db;
 use registrar::RegistrarClient;
 
-// how often to take periodic snapshots.
+// How often we attempt to take a snapshot: only snapshot on blocknumbers that are multiples of this.
 const SNAPSHOT_PERIOD: u64 = 5000;
 
-// how many blocks to wait before starting a periodic snapshot.
+// Start snapshots `history` blocks from the tip. Should be smaller than `SNAPSHOT_HISTORY`.
 const SNAPSHOT_HISTORY: u64 = 100;
 
 // Number of minutes before a given gas price corpus should expire.

parity/snapshot_cmd.rs

@@ -257,18 +257,18 @@ impl SnapshotCommand {
 		let writer = PackedWriter::new(&file_path)
 			.map_err(|e| format!("Failed to open snapshot writer: {}", e))?;
 
-		let progress = Arc::new(Progress::default());
+		let progress = Arc::new(Progress::new());
 		let p = progress.clone();
 		let informant_handle = ::std::thread::spawn(move || {
 			::std::thread::sleep(Duration::from_secs(5));
 
 			let mut last_size = 0;
 			while !p.done() {
-				let cur_size = p.size();
+				let cur_size = p.bytes();
 				if cur_size != last_size {
 					last_size = cur_size;
 					let bytes = ::informant::format_bytes(cur_size as usize);
-					info!("Snapshot: {} accounts {} blocks {}", p.accounts(), p.blocks(), bytes);
+					info!("Snapshot: {} accounts (state), {} blocks, {} bytes", p.accounts(), p.blocks(), bytes);
 				}
 
 				::std::thread::sleep(Duration::from_secs(5));

util/journaldb/src/lib.rs

@@ -84,8 +84,8 @@ pub trait JournalDB: HashDB<KeccakHasher, DBValue> {
 	/// Get backing database.
 	fn backing(&self) -> &Arc<dyn kvdb::KeyValueDB>;
 
-	/// Clear internal strucutres. This should called after changes have been written
+	/// Clear internal structure. This should be called after changes have been written
-	/// to the backing strage
+	/// to the backing storage.
 	fn flush(&self) {}
 
 	/// Consolidate all the insertions and deletions in the given memory overlay.

util/journaldb/src/overlayrecentdb.rs

@@ -228,10 +228,10 @@ impl OverlayRecentDB {
 		JournalOverlay {
 			backing_overlay: overlay,
 			pending_overlay: HashMap::default(),
-			journal: journal,
+			journal,
-			latest_era: latest_era,
+			latest_era,
-			earliest_era: earliest_era,
+			earliest_era,
-			cumulative_size: cumulative_size,
+			cumulative_size,
 		}
 	}
 
@@ -264,7 +264,6 @@ impl JournalDB for OverlayRecentDB {
 
 	fn journal_size(&self) -> usize {
 		self.journal_overlay.read().cumulative_size
-
 	}
 
 	fn is_empty(&self) -> bool {
@@ -351,7 +350,7 @@ impl JournalDB for OverlayRecentDB {
 
 		let mut ops = 0;
 		// apply old commits' details
-		if let Some(ref mut records) = journal_overlay.journal.get_mut(&end_era) {
+		if let Some(records) = journal_overlay.journal.get_mut(&end_era) {
 			let mut canon_insertions: Vec<(H256, DBValue)> = Vec::new();
 			let mut canon_deletions: Vec<H256> = Vec::new();
 			let mut overlay_deletions: Vec<H256> = Vec::new();