Merge branch 'master' into journaldb_commit

.gitlab-ci.yml

@@ -19,9 +19,11 @@ linux-stable:
   script:
     - cargo build --release --verbose
     - strip target/release/parity
+    - md5sum target/release/parity >> parity.md5
     - aws configure set aws_access_key_id $s3_key 
     - aws configure set aws_secret_access_key $s3_secret
     - aws s3api put-object --bucket builds-parity --key $CI_BUILD_REF_NAME/x86_64-unknown-linux-gnu/parity --body target/release/parity
+    - aws s3api put-object --bucket builds-parity --key $CI_BUILD_REF_NAME/x86_64-unknown-linux-gnu/parity.md5 --body parity.md5
   tags:
     - rust
     - rust-stable
@@ -40,9 +42,11 @@ linux-stable-14.04:
   script:
     - cargo build --release --verbose
     - strip target/release/parity
+    - md5sum target/release/parity >> parity.md5
     - aws configure set aws_access_key_id $s3_key 
     - aws configure set aws_secret_access_key $s3_secret
     - aws s3api put-object --bucket builds-parity --key $CI_BUILD_REF_NAME/x86_64-unknown-ubuntu_14_04-gnu/parity --body target/release/parity
+    - aws s3api put-object --bucket builds-parity --key $CI_BUILD_REF_NAME/x86_64-unknown-ubuntu_14_04-gnu/parity.md5 --body parity.md5
   tags:
     - rust
     - rust-14.04
@@ -101,9 +105,11 @@ linux-centos:
     - export CC="gcc"
     - cargo build --release --verbose
     - strip target/release/parity
+    - md5sum target/release/parity >> parity.md5
     - aws configure set aws_access_key_id $s3_key 
     - aws configure set aws_secret_access_key $s3_secret
     - aws s3api put-object --bucket builds-parity --key $CI_BUILD_REF_NAME/x86_64-unknown-centos-gnu/parity --body target/release/parity
+    - aws s3api put-object --bucket builds-parity --key $CI_BUILD_REF_NAME/x86_64-unknown-centos-gnu/parity.md5 --body parity.md5
   tags:
     - rust
     - rust-centos
@@ -115,7 +121,6 @@ linux-armv7:
   stage: build
   image: ethcore/rust-armv7:latest
   only:
-    - master
     - beta
     - tags
     - stable
@@ -127,9 +132,11 @@ linux-armv7:
     - cat .cargo/config
     - cargo build --target armv7-unknown-linux-gnueabihf --release --verbose
     - arm-linux-gnueabihf-strip target/armv7-unknown-linux-gnueabihf/release/parity
+    - md5sum target/armv7-unknown-linux-gnueabihf/release/parity >> parity.md5
     - aws configure set aws_access_key_id $s3_key 
     - aws configure set aws_secret_access_key $s3_secret
     - aws s3api put-object --bucket builds-parity --key $CI_BUILD_REF_NAME/armv7-unknown-linux-gnueabihf/parity --body target/armv7-unknown-linux-gnueabihf/release/parity
+    - aws s3api put-object --bucket builds-parity --key $CI_BUILD_REF_NAME/armv7-unknown-linux-gnueabihf/parity.md5 --body parity.md5
   tags:
     - rust
     - rust-arm
@@ -142,7 +149,6 @@ linux-arm:
   stage: build
   image: ethcore/rust-arm:latest
   only:
-    - master
     - beta
     - tags
     - stable
@@ -154,9 +160,11 @@ linux-arm:
     - cat .cargo/config
     - cargo build --target arm-unknown-linux-gnueabihf --release --verbose
     - arm-linux-gnueabihf-strip target/arm-unknown-linux-gnueabihf/release/parity
+    - md5sum target/arm-unknown-linux-gnueabihf/release/parity >> parity.md5
     - aws configure set aws_access_key_id $s3_key 
     - aws configure set aws_secret_access_key $s3_secret
     - aws s3api put-object --bucket builds-parity --key $CI_BUILD_REF_NAME/arm-unknown-linux-gnueabihf/parity --body target/arm-unknown-linux-gnueabihf/release/parity
+    - aws s3api put-object --bucket builds-parity --key $CI_BUILD_REF_NAME/arm-unknown-linux-gnueabihf/parity.md5 --body parity.md5
   tags:
     - rust
     - rust-arm
@@ -169,7 +177,6 @@ linux-armv6:
   stage: build
   image: ethcore/rust-armv6:latest
   only:
-    - master
     - beta
     - tags
     - stable
@@ -181,9 +188,11 @@ linux-armv6:
     - cat .cargo/config
     - cargo build --target arm-unknown-linux-gnueabi --release --verbose
     - arm-linux-gnueabi-strip target/arm-unknown-linux-gnueabi/release/parity
+    - md5sum target/arm-unknown-linux-gnueabi/release/parity >> parity.md5 
     - aws configure set aws_access_key_id $s3_key 
     - aws configure set aws_secret_access_key $s3_secret
     - aws s3api put-object --bucket builds-parity --key $CI_BUILD_REF_NAME/arm-unknown-linux-gnueabi/parity --body target/arm-unknown-linux-gnueabi/release/parity
+    - aws s3api put-object --bucket builds-parity --key $CI_BUILD_REF_NAME/arm-unknown-linux-gnueabi/parity.md5 --body parity.md5
   tags:
     - rust
     - rust-arm
@@ -196,7 +205,6 @@ linux-aarch64:
   stage: build
   image: ethcore/rust-aarch64:latest
   only:
-    - master
     - beta
     - tags
     - stable
@@ -208,9 +216,11 @@ linux-aarch64:
     - cat .cargo/config
     - cargo build --target aarch64-unknown-linux-gnu --release --verbose
     - aarch64-linux-gnu-strip target/aarch64-unknown-linux-gnu/release/parity
+    - md5sum target/aarch64-unknown-linux-gnu/release/parity >> parity.md5
     - aws configure set aws_access_key_id $s3_key 
     - aws configure set aws_secret_access_key $s3_secret
     - aws s3api put-object --bucket builds-parity --key $CI_BUILD_REF_NAME/aarch64-unknown-linux-gnu/parity --body target/aarch64-unknown-linux-gnu/release/parity
+    - aws s3api put-object --bucket builds-parity --key $CI_BUILD_REF_NAME/aarch64-unknown-linux-gnu/parity.md5 --body parity.md5
   tags:
     - rust
     - rust-arm
@@ -228,9 +238,11 @@ darwin:
     - stable
   script:
     - cargo build --release --verbose
+    - md5sum target/release/parity >> parity.md5
     - aws configure set aws_access_key_id $s3_key 
     - aws configure set aws_secret_access_key $s3_secret
     - aws s3api put-object --bucket builds-parity --key $CI_BUILD_REF_NAME/x86_64-apple-darwin/parity --body target/release/parity
+    - aws s3api put-object --bucket builds-parity --key $CI_BUILD_REF_NAME/x86_64-apple-darwin/parity.md5 --body parity.md5
   tags:
     - osx
   artifacts:
@@ -248,24 +260,49 @@ windows:
     - set INCLUDE=C:\Program Files (x86)\Microsoft SDKs\Windows\v7.1A\Include;C:\vs2015\VC\include;C:\Program Files (x86)\Windows Kits\10\Include\10.0.10240.0\ucrt
     - set LIB=C:\vs2015\VC\lib;C:\Program Files (x86)\Windows Kits\10\Lib\10.0.10240.0\ucrt\x64
     - set RUST_BACKTRACE=1
+    - set RUSTFLAGS=%RUSTFLAGS% -Zorbit=off -D warnings
     - rustup default stable-x86_64-pc-windows-msvc
     - cargo build --release --verbose
+    - curl -sL --url "https://github.com/ethcore/win-build/raw/master/SimpleFC.dll" -o nsis\SimpleFC.dll
+    - curl -sL --url "https://github.com/ethcore/win-build/raw/master/vc_redist.x64.exe" -o nsis\vc_redist.x64.exe
+    - signtool sign /f %keyfile% /p %certpass% target\release\parity.exe
+    - cd nsis
+    - makensis.exe installer.nsi
+    - copy installer.exe InstallParity.exe
+    - signtool sign /f %keyfile% /p %certpass% InstallParity.exe
+    - md5sums InstallParity.exe > InstallParity.exe.md5
+    - zip win-installer.zip InstallParity.exe InstallParity.exe.md5
+    - md5sums win-installer.zip > win-installer.zip.md5
+    - cd ..\target\release\
+    - md5sums parity.exe parity.pdb > parity.md5
+    - md5sums parity.exe > parity.exe.md5
+    - zip parity.zip parity.exe parity.pdb parity.md5
+    - md5sums parity.zip > parity.zip.md5
+    - cd ..\..
     - aws configure set aws_access_key_id %s3_key%
     - aws configure set aws_secret_access_key %s3_secret%
-    - aws s3api put-object --bucket builds-parity --key %CI_BUILD_REF_NAME%/x86_64-pc-windows-msvc/parity --body target/release/parity.exe
+    - aws s3api put-object --bucket builds-parity --key %CI_BUILD_REF_NAME%/x86_64-pc-windows-msvc/parity.exe --body target\release\parity.exe
-    - aws s3api put-object --bucket builds-parity --key %CI_BUILD_REF_NAME%/x86_64-pc-windows-msvc/parity --body target/release/parity.pdb
+    - aws s3api put-object --bucket builds-parity --key %CI_BUILD_REF_NAME%/x86_64-pc-windows-msvc/parity.exe.md5 --body target\release\parity.exe.md5
+    - aws s3api put-object --bucket builds-parity --key %CI_BUILD_REF_NAME%/x86_64-pc-windows-msvc/parity.zip --body target\release\parity.zip
+    - aws s3api put-object --bucket builds-parity --key %CI_BUILD_REF_NAME%/x86_64-pc-windows-msvc/parity.zip.md5 --body target\release\parity.zip.md5
+    - aws s3api put-object --bucket builds-parity --key %CI_BUILD_REF_NAME%/x86_64-pc-windows-msvc/InstallParity.exe --body nsis\InstallParity.exe
+    - aws s3api put-object --bucket builds-parity --key %CI_BUILD_REF_NAME%/x86_64-pc-windows-msvc/InstallParity.exe.md5 --body nsis\InstallParity.exe.md5
+    - aws s3api put-object --bucket builds-parity --key %CI_BUILD_REF_NAME%/x86_64-pc-windows-msvc/win-installer.zip --body nsis\win-installer.zip
+    - aws s3api put-object --bucket builds-parity --key %CI_BUILD_REF_NAME%/x86_64-pc-windows-msvc/win-installer.zip.md5 --body nsis\win-installer.zip.md5
   tags:
    - rust-windows
   artifacts:
     paths:
     - target/release/parity.exe
     - target/release/parity.pdb
+    - nsis/InstallParity.exe
     name: "x86_64-pc-windows-msvc_parity"
 test-linux:
   stage: test
   before_script:
     - git submodule update --init --recursive
   script:
+    - export RUST_BACKTRACE=1
     - ./test.sh --verbose
   tags:
     - rust-test

.travis.yml

@@ -31,6 +31,7 @@ env:
   - RUN_COVERAGE="false"
   - RUN_DOCS="false"
   - TEST_OPTIONS=""
+  - RUSTFLAGS="-D warnings"
   # GH_TOKEN for documentation
   - secure: bumJASbZSU8bxJ0EyPUJmu16AiV9EXOpyOj86Jlq/Ty9CfwGqsSXt96uDyE+OUJf34RUFQMsw0nk37/zC4lcn6kqk2wpuH3N/o85Zo/cVZY/NusBWLQqtT5VbYWsV+u2Ua4Tmmsw8yVYQhYwU2ZOejNpflL+Cs9XGgORp1L+/gMRMC2y5Se6ZhwnKPQlRJ8LGsG1dzjQULxzADIt3/zuspNBS8a2urJwlHfGMkvHDoUWCviP/GXoSqw3TZR7FmKyxE19I8n9+iSvm9+oZZquvcgfUxMHn8Gq/b44UbPvjtFOg2yam4xdWXF/RyWCHdc/R9EHorSABeCbefIsm+zcUF3/YQxwpSxM4IZEeH2rTiC7dcrsKw3XsO16xFQz5YI5Bay+CT/wTdMmJd7DdYz7Dyf+pOvcM9WOf/zorxYWSBOMYy0uzbusU2iyIghQ82s7E/Ahg+WARtPgkuTLSB5aL1oCTBKHqQscMr7lo5Ti6RpWLxEdTQMBznc+bMr+6dEtkEcG9zqc6cE9XX+ox3wTU6+HVMfQ1ltCntJ4UKcw3A6INEbw9wgocQa812CIASQ2fE+SCAbz6JxBjIAlFUnD1lUB7S8PdMPwn9plfQgKQ2A5YZqg6FnBdf0rQXIJYxQWKHXj/rBHSUCT0tHACDlzTA+EwWggvkP5AGIxRxm8jhw=
   - KCOV_CMD="./kcov-master/tmp/usr/local/bin/kcov"

Cargo.lock

@@ -37,6 +37,8 @@ dependencies = [
  "rustc-serialize 0.3.19 (registry+https://github.com/rust-lang/crates.io-index)",
  "rustc_version 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)",
  "semver 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)",
+ "serde 0.8.4 (registry+https://github.com/rust-lang/crates.io-index)",
+ "serde_json 0.8.1 (registry+https://github.com/rust-lang/crates.io-index)",
  "time 0.1.35 (registry+https://github.com/rust-lang/crates.io-index)",
  "toml 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "winapi 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -118,6 +120,11 @@ name = "bloomchain"
 version = "0.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 
+[[package]]
+name = "byteorder"
+version = "0.5.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+
 [[package]]
 name = "bytes"
 version = "0.3.0"
@@ -233,7 +240,7 @@ version = "0.5.4"
 source = "git+https://github.com/ethcore/rust-secp256k1#a9a0b1be1f39560ca86e8fc8e55e205a753ff25c"
 dependencies = [
  "arrayvec 0.3.16 (registry+https://github.com/rust-lang/crates.io-index)",
- "gcc 0.3.28 (registry+https://github.com/rust-lang/crates.io-index)",
+ "gcc 0.3.35 (registry+https://github.com/rust-lang/crates.io-index)",
  "libc 0.2.15 (registry+https://github.com/rust-lang/crates.io-index)",
  "rand 0.3.14 (registry+https://github.com/rust-lang/crates.io-index)",
  "rustc-serialize 0.3.19 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -268,10 +275,12 @@ version = "1.4.0"
 dependencies = [
  "bit-set 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "bloomchain 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "byteorder 0.5.3 (registry+https://github.com/rust-lang/crates.io-index)",
  "clippy 0.0.90 (registry+https://github.com/rust-lang/crates.io-index)",
  "crossbeam 0.2.9 (registry+https://github.com/rust-lang/crates.io-index)",
  "env_logger 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)",
  "ethash 1.4.0",
+ "ethcore-bloom-journal 0.1.0",
  "ethcore-devtools 1.4.0",
  "ethcore-io 1.4.0",
  "ethcore-ipc 1.4.0",
@@ -281,10 +290,12 @@ dependencies = [
  "ethjson 0.1.0",
  "ethkey 0.2.0",
  "ethstore 0.1.0",
+ "evmjit 1.4.0",
  "heapsize 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)",
  "hyper 0.9.4 (git+https://github.com/ethcore/hyper)",
  "lazy_static 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)",
  "log 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)",
+ "lru-cache 0.0.7 (git+https://github.com/contain-rs/lru-cache)",
  "num_cpus 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)",
  "rand 0.3.14 (registry+https://github.com/rust-lang/crates.io-index)",
  "rayon 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -297,7 +308,7 @@ dependencies = [
 
 [[package]]
 name = "ethcore-bigint"
-version = "0.1.0"
+version = "0.1.1"
 dependencies = [
  "heapsize 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)",
  "rand 0.3.14 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -305,6 +316,10 @@ dependencies = [
  "rustc_version 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
+[[package]]
+name = "ethcore-bloom-journal"
+version = "0.1.0"
+
 [[package]]
 name = "ethcore-dapps"
 version = "1.4.0"
@@ -314,7 +329,7 @@ dependencies = [
  "ethcore-devtools 1.4.0",
  "ethcore-rpc 1.4.0",
  "ethcore-util 1.4.0",
- "https-fetch 0.1.0",
+ "fetch 0.1.0",
  "hyper 0.9.4 (git+https://github.com/ethcore/hyper)",
  "jsonrpc-core 3.0.2 (registry+https://github.com/rust-lang/crates.io-index)",
  "jsonrpc-http-server 6.1.0 (git+https://github.com/ethcore/jsonrpc-http-server.git)",
@@ -384,6 +399,7 @@ dependencies = [
  "log 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)",
  "nanomsg 0.5.1 (git+https://github.com/ethcore/nanomsg.rs.git)",
  "semver 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)",
+ "time 0.1.35 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
 [[package]]
@@ -463,6 +479,7 @@ dependencies = [
  "ethkey 0.2.0",
  "ethstore 0.1.0",
  "ethsync 1.4.0",
+ "fetch 0.1.0",
  "json-ipc-server 0.2.4 (git+https://github.com/ethcore/json-ipc-server.git)",
  "jsonrpc-core 3.0.2 (registry+https://github.com/rust-lang/crates.io-index)",
  "jsonrpc-http-server 6.1.0 (git+https://github.com/ethcore/jsonrpc-http-server.git)",
@@ -522,7 +539,8 @@ dependencies = [
  "elastic-array 0.5.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "env_logger 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)",
  "eth-secp256k1 0.5.4 (git+https://github.com/ethcore/rust-secp256k1)",
- "ethcore-bigint 0.1.0",
+ "ethcore-bigint 0.1.1",
+ "ethcore-bloom-journal 0.1.0",
  "ethcore-devtools 1.4.0",
  "heapsize 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)",
  "itertools 0.4.13 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -550,7 +568,7 @@ name = "ethcrypto"
 version = "0.1.0"
 dependencies = [
  "eth-secp256k1 0.5.4 (git+https://github.com/ethcore/rust-secp256k1)",
- "ethcore-bigint 0.1.0",
+ "ethcore-bigint 0.1.1",
  "ethkey 0.2.0",
  "rust-crypto 0.2.36 (registry+https://github.com/rust-lang/crates.io-index)",
  "tiny-keccak 1.0.5 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -573,7 +591,7 @@ version = "0.2.0"
 dependencies = [
  "docopt 0.6.80 (registry+https://github.com/rust-lang/crates.io-index)",
  "eth-secp256k1 0.5.4 (git+https://github.com/ethcore/rust-secp256k1)",
- "ethcore-bigint 0.1.0",
+ "ethcore-bigint 0.1.1",
  "lazy_static 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)",
  "rand 0.3.14 (registry+https://github.com/rust-lang/crates.io-index)",
  "rustc-serialize 0.3.19 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -622,6 +640,13 @@ dependencies = [
  "time 0.1.35 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
+[[package]]
+name = "evmjit"
+version = "1.4.0"
+dependencies = [
+ "tiny-keccak 1.0.5 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
 [[package]]
 name = "fdlimit"
 version = "0.1.0"
@@ -629,6 +654,16 @@ dependencies = [
  "libc 0.2.15 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
+[[package]]
+name = "fetch"
+version = "0.1.0"
+dependencies = [
+ "https-fetch 0.1.0",
+ "hyper 0.9.4 (git+https://github.com/ethcore/hyper)",
+ "log 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)",
+ "rand 0.3.14 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
 [[package]]
 name = "flate2"
 version = "0.2.14"
@@ -640,8 +675,11 @@ dependencies = [
 
 [[package]]
 name = "gcc"
-version = "0.3.28"
+version = "0.3.35"
 source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "rayon 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)",
+]
 
 [[package]]
 name = "glob"
@@ -680,7 +718,7 @@ version = "0.1.0"
 dependencies = [
  "log 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)",
  "mio 0.5.1 (git+https://github.com/ethcore/mio?branch=v0.5.x)",
- "rustls 0.1.1 (git+https://github.com/ctz/rustls)",
+ "rustls 0.1.2 (git+https://github.com/ctz/rustls)",
 ]
 
 [[package]]
@@ -842,6 +880,11 @@ name = "libc"
 version = "0.2.15"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 
+[[package]]
+name = "linked-hash-map"
+version = "0.2.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+
 [[package]]
 name = "linked-hash-map"
 version = "0.3.0"
@@ -852,6 +895,14 @@ name = "log"
 version = "0.3.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 
+[[package]]
+name = "lru-cache"
+version = "0.0.7"
+source = "git+https://github.com/contain-rs/lru-cache#13255e33c45ceb69a4b143f235a4322df5fb580e"
+dependencies = [
+ "linked-hash-map 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
 [[package]]
 name = "matches"
 version = "0.1.2"
@@ -889,7 +940,7 @@ name = "miniz-sys"
 version = "0.1.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 dependencies = [
- "gcc 0.3.28 (registry+https://github.com/rust-lang/crates.io-index)",
+ "gcc 0.3.35 (registry+https://github.com/rust-lang/crates.io-index)",
  "libc 0.2.15 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
@@ -975,7 +1026,7 @@ name = "nanomsg-sys"
 version = "0.5.0"
 source = "git+https://github.com/ethcore/nanomsg.rs.git#c40fe442c9afaea5b38009a3d992ca044dcceb00"
 dependencies = [
- "gcc 0.3.28 (registry+https://github.com/rust-lang/crates.io-index)",
+ "gcc 0.3.35 (registry+https://github.com/rust-lang/crates.io-index)",
  "libc 0.2.15 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
@@ -1335,7 +1386,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 
 [[package]]
 name = "ring"
-version = "0.3.1"
+version = "0.4.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 dependencies = [
  "lazy_static 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -1347,7 +1398,7 @@ name = "rlp"
 version = "0.1.0"
 dependencies = [
  "elastic-array 0.5.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "ethcore-bigint 0.1.0",
+ "ethcore-bigint 0.1.1",
  "lazy_static 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)",
  "rustc-serialize 0.3.19 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
@@ -1355,7 +1406,7 @@ dependencies = [
 [[package]]
 name = "rocksdb"
 version = "0.4.5"
-source = "git+https://github.com/ethcore/rust-rocksdb#485dd747a2c9a9f910fc8ac696fc9edf5fa22aa3"
+source = "git+https://github.com/ethcore/rust-rocksdb#ffc7c82380fe8569f85ae6743f7f620af2d4a679"
 dependencies = [
  "libc 0.2.15 (registry+https://github.com/rust-lang/crates.io-index)",
  "rocksdb-sys 0.3.0 (git+https://github.com/ethcore/rust-rocksdb)",
@@ -1364,9 +1415,9 @@ dependencies = [
 [[package]]
 name = "rocksdb-sys"
 version = "0.3.0"
-source = "git+https://github.com/ethcore/rust-rocksdb#485dd747a2c9a9f910fc8ac696fc9edf5fa22aa3"
+source = "git+https://github.com/ethcore/rust-rocksdb#ffc7c82380fe8569f85ae6743f7f620af2d4a679"
 dependencies = [
- "gcc 0.3.28 (registry+https://github.com/rust-lang/crates.io-index)",
+ "gcc 0.3.35 (registry+https://github.com/rust-lang/crates.io-index)",
  "libc 0.2.15 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
@@ -1397,7 +1448,7 @@ name = "rust-crypto"
 version = "0.2.36"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 dependencies = [
- "gcc 0.3.28 (registry+https://github.com/rust-lang/crates.io-index)",
+ "gcc 0.3.35 (registry+https://github.com/rust-lang/crates.io-index)",
  "libc 0.2.15 (registry+https://github.com/rust-lang/crates.io-index)",
  "rand 0.3.14 (registry+https://github.com/rust-lang/crates.io-index)",
  "rustc-serialize 0.3.19 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -1419,15 +1470,15 @@ dependencies = [
 
 [[package]]
 name = "rustls"
-version = "0.1.1"
+version = "0.1.2"
-source = "git+https://github.com/ctz/rustls#a9c5a79f49337e22ac05bb1ea114240bdbe0fdd2"
+source = "git+https://github.com/ctz/rustls#3d2db624997004b7b18ba4463d6081f37598b2f5"
 dependencies = [
  "base64 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)",
  "log 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)",
- "ring 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)",
+ "ring 0.4.3 (registry+https://github.com/rust-lang/crates.io-index)",
  "time 0.1.35 (registry+https://github.com/rust-lang/crates.io-index)",
  "untrusted 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)",
- "webpki 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)",
+ "webpki 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
 [[package]]
@@ -1490,7 +1541,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 name = "sha3"
 version = "0.1.0"
 dependencies = [
- "gcc 0.3.28 (registry+https://github.com/rust-lang/crates.io-index)",
+ "gcc 0.3.35 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
 [[package]]
@@ -1778,10 +1829,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 
 [[package]]
 name = "webpki"
-version = "0.2.2"
+version = "0.3.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 dependencies = [
- "ring 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)",
+ "ring 0.4.3 (registry+https://github.com/rust-lang/crates.io-index)",
  "rustc-serialize 0.3.19 (registry+https://github.com/rust-lang/crates.io-index)",
  "time 0.1.35 (registry+https://github.com/rust-lang/crates.io-index)",
  "untrusted 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -1861,6 +1912,7 @@ dependencies = [
 "checksum bitflags 0.5.0 (registry+https://github.com/rust-lang/crates.io-index)" = "4f67931368edf3a9a51d29886d245f1c3db2f1ef0dcc9e35ff70341b78c10d23"
 "checksum blastfig 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "09640e0509d97d5cdff03a9f5daf087a8e04c735c3b113a75139634a19cfc7b2"
 "checksum bloomchain 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "3f421095d2a76fc24cd3fb3f912b90df06be7689912b1bdb423caefae59c258d"
+"checksum byteorder 0.5.3 (registry+https://github.com/rust-lang/crates.io-index)" = "0fc10e8cc6b2580fda3f36eb6dc5316657f812a3df879a44a66fc9f0fdbc4855"
 "checksum bytes 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "c129aff112dcc562970abb69e2508b40850dd24c274761bb50fb8a0067ba6c27"
 "checksum bytes 0.4.0-dev (git+https://github.com/carllerche/bytes)" = "<none>"
 "checksum cfg-if 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "de1e760d7b6535af4241fca8bd8adf68e2e7edacc6b29f5d399050c5e48cf88c"
@@ -1878,7 +1930,7 @@ dependencies = [
 "checksum eth-secp256k1 0.5.4 (git+https://github.com/ethcore/rust-secp256k1)" = "<none>"
 "checksum ethabi 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "f7b0c53453517f620847be51943db329276ae52f2e210cfc659e81182864be2f"
 "checksum flate2 0.2.14 (registry+https://github.com/rust-lang/crates.io-index)" = "3eeb481e957304178d2e782f2da1257f1434dfecbae883bafb61ada2a9fea3bb"
-"checksum gcc 0.3.28 (registry+https://github.com/rust-lang/crates.io-index)" = "3da3a2cbaeb01363c8e3704fd9fd0eb2ceb17c6f27abd4c1ef040fb57d20dc79"
+"checksum gcc 0.3.35 (registry+https://github.com/rust-lang/crates.io-index)" = "91ecd03771effb0c968fd6950b37e89476a578aaf1c70297d8e92b6516ec3312"
 "checksum glob 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)" = "8be18de09a56b60ed0edf84bc9df007e30040691af7acd1c41874faac5895bfb"
 "checksum hamming 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)" = "65043da274378d68241eb9a8f8f8aa54e349136f7b8e12f63e3ef44043cc30e1"
 "checksum heapsize 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)" = "abb306abb8d398e053cfb1b3e7b72c2f580be048b85745c52652954f8ad1439c"
@@ -1893,14 +1945,16 @@ dependencies = [
 "checksum itoa 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "ae3088ea4baeceb0284ee9eea42f591226e6beaecf65373e41b38d95a1b8e7a1"
 "checksum json-ipc-server 0.2.4 (git+https://github.com/ethcore/json-ipc-server.git)" = "<none>"
 "checksum json-tcp-server 0.1.0 (git+https://github.com/ethcore/json-tcp-server)" = "<none>"
-"checksum jsonrpc-core 3.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "e913b3c809aab9378889da8b990b4a46b98bd4794c8117946a1cf63c5f87bcde"
+"checksum jsonrpc-core 3.0.2 (registry+https://github.com/rust-lang/crates.io-index)" = "3c5094610b07f28f3edaf3947b732dadb31dbba4941d4d0c1c7a8350208f4414"
 "checksum jsonrpc-http-server 6.1.0 (git+https://github.com/ethcore/jsonrpc-http-server.git)" = "<none>"
 "checksum kernel32-sys 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "7507624b29483431c0ba2d82aece8ca6cdba9382bff4ddd0f7490560c056098d"
 "checksum language-tags 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "a91d884b6667cd606bb5a69aa0c99ba811a115fc68915e7056ec08a46e93199a"
 "checksum lazy_static 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "49247ec2a285bb3dcb23cbd9c35193c025e7251bfce77c1d5da97e6362dffe7f"
 "checksum libc 0.2.15 (registry+https://github.com/rust-lang/crates.io-index)" = "23e3757828fa702a20072c37ff47938e9dd331b92fac6e223d26d4b7a55f7ee2"
+"checksum linked-hash-map 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "bda158e0dabeb97ee8a401f4d17e479d6b891a14de0bba79d5cc2d4d325b5e48"
 "checksum linked-hash-map 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "6d262045c5b87c0861b3f004610afd0e2c851e2908d08b6c870cbb9d5f494ecd"
 "checksum log 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)" = "ab83497bf8bf4ed2a74259c1c802351fcd67a65baa86394b6ba73c36f4838054"
+"checksum lru-cache 0.0.7 (git+https://github.com/contain-rs/lru-cache)" = "<none>"
 "checksum matches 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "15305656809ce5a4805b1ff2946892810992197ce1270ff79baded852187942e"
 "checksum memchr 0.1.11 (registry+https://github.com/rust-lang/crates.io-index)" = "d8b629fb514376c675b98c1421e80b151d3817ac42d7c667717d282761418d20"
 "checksum mime 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "a74cc2587bf97c49f3f5bab62860d6abf3902ca73b66b51d9b049fbdcd727bd2"
@@ -1954,7 +2008,7 @@ dependencies = [
 "checksum rayon 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)" = "655df67c314c30fa3055a365eae276eb88aa4f3413a352a1ab32c1320eda41ea"
 "checksum regex 0.1.68 (registry+https://github.com/rust-lang/crates.io-index)" = "b4329b8928a284580a1c63ec9d846b12f6d3472317243ff7077aff11f23f2b29"
 "checksum regex-syntax 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "841591b1e05609a643e3b4d0045fce04f701daba7151ddcd3ad47b080693d5a9"
-"checksum ring 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "d059a6a96d3be79042e3f70eb97945912839265f9d8ab45b921abaf266c70dbb"
+"checksum ring 0.4.3 (registry+https://github.com/rust-lang/crates.io-index)" = "0d2f6547bf9640f1d3cc4e771f82374ec8fd237c17eeb3ff5cd5ccbe22377a09"
 "checksum rocksdb 0.4.5 (git+https://github.com/ethcore/rust-rocksdb)" = "<none>"
 "checksum rocksdb-sys 0.3.0 (git+https://github.com/ethcore/rust-rocksdb)" = "<none>"
 "checksum rotor 0.6.3 (git+https://github.com/ethcore/rotor)" = "<none>"
@@ -1962,7 +2016,7 @@ dependencies = [
 "checksum rust-crypto 0.2.36 (registry+https://github.com/rust-lang/crates.io-index)" = "f76d05d3993fd5f4af9434e8e436db163a12a9d40e1a58a726f27a01dfd12a2a"
 "checksum rustc-serialize 0.3.19 (registry+https://github.com/rust-lang/crates.io-index)" = "6159e4e6e559c81bd706afe9c8fd68f547d3e851ce12e76b1de7914bab61691b"
 "checksum rustc_version 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)" = "c5f5376ea5e30ce23c03eb77cbe4962b988deead10910c372b226388b594c084"
-"checksum rustls 0.1.1 (git+https://github.com/ctz/rustls)" = "<none>"
+"checksum rustls 0.1.2 (git+https://github.com/ctz/rustls)" = "<none>"
 "checksum semver 0.1.20 (registry+https://github.com/rust-lang/crates.io-index)" = "d4f410fedcf71af0345d7607d246e7ad15faaadd49d240ee3b24e5dc21a820ac"
 "checksum semver 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "2d5b7638a1f03815d94e88cb3b3c08e87f0db4d683ef499d1836aaf70a45623f"
 "checksum serde 0.8.4 (registry+https://github.com/rust-lang/crates.io-index)" = "b1dfda9ebb31d29fa8b94d7eb3031a86a8dcec065f0fe268a30f98867bf45775"
@@ -2007,7 +2061,7 @@ dependencies = [
 "checksum vecio 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "0795a11576d29ae80525a3fda315bf7b534f8feb9d34101e5fe63fb95bb2fd24"
 "checksum vergen 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "56b639f935488eb40f06d17c3e3bcc3054f6f75d264e187b1107c8d1cba8d31c"
 "checksum void 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)" = "6a02e4885ed3bc0f2de90ea6dd45ebcbb66dacffe03547fadbb0eeae2770887d"
-"checksum webpki 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "5dc10a815fabbb0c3145c1153240528f3a8703a47e26e8dbb4a5d4f6386200ad"
+"checksum webpki 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "813503a5985585e0812d430cd1328ee322f47f66629c8ed4ecab939cf9e92f91"
 "checksum winapi 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)" = "4dfaaa8fbdaa618fa6914b59b2769d690dd7521920a18d84b42d254678dd5fd4"
 "checksum winapi-build 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "2d315eee3b34aca4797b2da6b13ed88266e6d612562a0c46390af8299fc699bc"
 "checksum ws 0.5.2 (git+https://github.com/ethcore/ws-rs.git?branch=mio-upstream-stable)" = "<none>"

Cargo.toml

@@ -44,6 +44,8 @@ json-ipc-server = { git = "https://github.com/ethcore/json-ipc-server.git" }
 ethcore-dapps = { path = "dapps", optional = true }
 clippy = { version = "0.0.90", optional = true}
 ethcore-stratum = { path = "stratum" }
+serde = "0.8.0"
+serde_json = "0.8.0"
 
 [target.'cfg(windows)'.dependencies]
 winapi = "0.2"
@@ -60,19 +62,22 @@ default = ["ui", "use-precompiled-js", "ipc"]
 ui = ["dapps", "ethcore-signer/ui"]
 use-precompiled-js = ["ethcore-dapps/use-precompiled-js", "ethcore-signer/use-precompiled-js"]
 dapps = ["ethcore-dapps"]
-ipc = ["ethcore/ipc"]
+ipc = ["ethcore/ipc", "ethsync/ipc"]
+jit = ["ethcore/jit"]
 dev = ["clippy", "ethcore/dev", "ethcore-util/dev", "ethsync/dev", "ethcore-rpc/dev", "ethcore-dapps/dev", "ethcore-signer/dev"]
 json-tests = ["ethcore/json-tests"]
 stratum = ["ipc"]
 ethkey-cli = ["ethcore/ethkey-cli"]
 ethstore-cli = ["ethcore/ethstore-cli"]
 evm-debug = ["ethcore/evm-debug"]
+evm-debug-tests = ["ethcore/evm-debug-tests"]
+slow-blocks = ["ethcore/slow-blocks"]
 
 [[bin]]
 path = "parity/main.rs"
 name = "parity"
 
 [profile.release]
-debug = true
+debug = false
 lto = false
 

README.md

@@ -5,6 +5,7 @@
 
 [Internal Documentation][doc-url]
 
+
 Be sure to check out [our wiki][wiki-url] for more information.
 
 [travis-image]: https://travis-ci.org/ethcore/parity.svg?branch=master
@@ -14,12 +15,15 @@ Be sure to check out [our wiki][wiki-url] for more information.
 [gitter-image]: https://badges.gitter.im/Join%20Chat.svg
 [gitter-url]: https://gitter.im/ethcore/parity?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge
 [license-image]: https://img.shields.io/badge/license-GPL%20v3-green.svg
-[license-url]: http://www.gnu.org/licenses/gpl-3.0.en.html
+[license-url]: https://www.gnu.org/licenses/gpl-3.0.en.html
-[doc-url]: http://ethcore.github.io/parity/ethcore/index.html
+[doc-url]: https://ethcore.github.io/parity/ethcore/index.html
 [wiki-url]: https://github.com/ethcore/parity/wiki
 
+**Requires Rust version 1.12.0 to build**
+
 ----
 
+
 ## About Parity
 
 Parity's goal is to be the fastest, lightest, and most secure Ethereum client. We are developing Parity using the sophisticated and
@@ -96,9 +100,9 @@ and Parity will begin syncing the Ethereum blockchain.
 ### Using systemd service file
 To start Parity as a regular user using systemd init:
 
-1. Copy ```parity/scripts/parity.service``` to your
+1. Copy `parity/scripts/parity.service` to your
-systemd user directory (usually ```~/.config/systemd/user```).
+systemd user directory (usually `~/.config/systemd/user`).
-2. To pass any argument to Parity, write a ```~/.parity/parity.conf``` file this way:
+2. To pass any argument to Parity, write a `~/.parity/parity.conf` file this way:
-```ARGS="ARG1 ARG2 ARG3"```.
+`ARGS="ARG1 ARG2 ARG3"`.
 
-	Example: ```ARGS="ui --geth --identity MyMachine"```.
+	Example: `ARGS="ui --geth --identity MyMachine"`.

appveyor.yml

@@ -6,6 +6,7 @@ environment:
   certpass:
     secure: 0BgXJqxq9Ei34/hZ7121FQ==
   keyfile: C:\users\appveyor\Certificates.p12
+  RUSTFLAGS: -Zorbit=off -D warnings
 
 branches:
   only:
@@ -18,10 +19,10 @@ branches:
 install:
   - git submodule update --init --recursive
   - ps: Install-Product node 6
-  - ps: Start-FileDownload "https://static.rust-lang.org/dist/rust-1.10.0-x86_64-pc-windows-msvc.exe"
+  - ps: Start-FileDownload "https://static.rust-lang.org/dist/rust-1.12.0-x86_64-pc-windows-msvc.exe"
   - ps: Start-FileDownload "https://github.com/ethcore/win-build/raw/master/SimpleFC.dll" -FileName nsis\SimpleFC.dll
   - ps: Start-FileDownload "https://github.com/ethcore/win-build/raw/master/vc_redist.x64.exe" -FileName nsis\vc_redist.x64.exe
-  - rust-1.10.0-x86_64-pc-windows-msvc.exe /VERYSILENT /NORESTART /DIR="C:\Program Files (x86)\Rust"
+  - rust-1.12.0-x86_64-pc-windows-msvc.exe /VERYSILENT /NORESTART /DIR="C:\Program Files (x86)\Rust"
   - SET PATH=%PATH%;C:\Program Files (x86)\Rust\bin;C:\Program Files (x86)\NSIS;C:\Program Files (x86)\Microsoft SDKs\Windows\v7.1A\Bin
   - rustc -V
   - cargo -V

dapps/Cargo.toml

@@ -26,7 +26,7 @@ linked-hash-map = "0.3"
 ethcore-devtools = { path = "../devtools" }
 ethcore-rpc = { path = "../rpc" }
 ethcore-util = { path = "../util" }
-https-fetch = { path = "../util/https-fetch" }
+fetch = { path = "../util/fetch" }
 parity-dapps = { git = "https://github.com/ethcore/parity-ui.git", version = "1.4" }
 # List of apps
 parity-dapps-status = { git = "https://github.com/ethcore/parity-ui.git", version = "1.4" }

dapps/src/api/api.rs

@@ -92,11 +92,14 @@ impl server::Handler<net::HttpStream> for RestApiRouter {
 		}
 
 		let url = url.expect("Check for None early-exists above; qed");
-		let path = self.path.take().expect("on_request called only once, and path is always defined in new; qed");
+		let mut path = self.path.take().expect("on_request called only once, and path is always defined in new; qed");
 		let control = self.control.take().expect("on_request called only once, and control is always defined in new; qed");
 
 		let endpoint = url.path.get(1).map(|v| v.as_str());
 		let hash = url.path.get(2).map(|v| v.as_str());
+		// at this point path.app_id contains 'api', adjust it to the hash properly, otherwise
+		// we will try and retrieve 'api' as the hash when doing the /api/content route
+		if let Some(hash) = hash.clone() { path.app_id = hash.to_owned() }
 
 		let handler = endpoint.and_then(|v| match v {
 			"apps" => Some(as_json(&self.api.list_apps())),

dapps/src/apps/fetcher.rs

@@ -122,7 +122,7 @@ impl<R: URLHint> ContentFetcher<R> {
 				},
 				// We need to start fetching app
 				None => {
-					trace!(target: "dapps", "Content unavailable. Fetching...");
+					trace!(target: "dapps", "Content unavailable. Fetching... {:?}", content_id);
 					let content_hex = content_id.from_hex().expect("to_handler is called only when `contains` returns true.");
 					let content = self.resolver.resolve(content_hex);
 
@@ -415,4 +415,3 @@ mod tests {
 		assert_eq!(fetcher.contains("test3"), false);
 	}
 }
-

dapps/src/handlers/client/mod.rs

@@ -1,113 +0,0 @@
-// Copyright 2015, 2016 Ethcore (UK) Ltd.
-// This file is part of Parity.
-
-// Parity is free software: you can redistribute it and/or modify
-// it under the terms of the GNU General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-
-// Parity is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU General Public License for more details.
-
-// You should have received a copy of the GNU General Public License
-// along with Parity.  If not, see <http://www.gnu.org/licenses/>.
-
-//! Hyper Client Handlers
-
-pub mod fetch_file;
-
-use std::env;
-use std::sync::{mpsc, Arc};
-use std::sync::atomic::AtomicBool;
-use std::path::PathBuf;
-
-use hyper;
-use https_fetch as https;
-
-use random_filename;
-use self::fetch_file::{Fetch, Error as HttpFetchError};
-
-pub type FetchResult = Result<PathBuf, FetchError>;
-
-#[derive(Debug)]
-pub enum FetchError {
-	InvalidUrl,
-	Http(HttpFetchError),
-	Https(https::FetchError),
-	Other(String),
-}
-
-impl From<HttpFetchError> for FetchError {
-	fn from(e: HttpFetchError) -> Self {
-		FetchError::Http(e)
-	}
-}
-
-pub struct Client {
-	http_client: hyper::Client<Fetch>,
-	https_client: https::Client,
-}
-
-impl Client {
-	pub fn new() -> Self {
-		Client {
-			http_client: hyper::Client::new().expect("Unable to initialize http client."),
-			https_client: https::Client::new().expect("Unable to initialize https client."),
-		}
-	}
-
-	pub fn close(self) {
-		self.http_client.close();
-		self.https_client.close();
-	}
-
-	pub fn request(&mut self, url: &str, abort: Arc<AtomicBool>, on_done: Box<Fn() + Send>) -> Result<mpsc::Receiver<FetchResult>, FetchError> {
-		let is_https = url.starts_with("https://");
-		let url = try!(url.parse().map_err(|_| FetchError::InvalidUrl));
-		trace!(target: "dapps", "Fetching from: {:?}", url);
-		if is_https {
-			let url = try!(Self::convert_url(url));
-
-			let (tx, rx) = mpsc::channel();
-			let temp_path = Self::temp_path();
-			let res = self.https_client.fetch_to_file(url, temp_path.clone(), abort, move |result| {
-				let res = tx.send(
-					result.map(|_| temp_path).map_err(FetchError::Https)
-				);
-				if let Err(_) = res {
-					warn!("Fetch finished, but no one was listening");
-				}
-				on_done();
-			});
-
-			match res {
-				Ok(_) => Ok(rx),
-				Err(e) => Err(FetchError::Other(format!("{:?}", e))),
-			}
-		} else {
-			let (tx, rx) = mpsc::channel();
-			let res = self.http_client.request(url, Fetch::new(tx, abort, on_done));
-
-			match res {
-				Ok(_) => Ok(rx),
-				Err(e) => Err(FetchError::Other(format!("{:?}", e))),
-			}
-		}
-	}
-
-	fn convert_url(url: hyper::Url) -> Result<https::Url, FetchError> {
-		let host = format!("{}", try!(url.host().ok_or(FetchError::InvalidUrl)));
-		let port = try!(url.port_or_known_default().ok_or(FetchError::InvalidUrl));
-		https::Url::new(&host, port, url.path()).map_err(|_| FetchError::InvalidUrl)
-	}
-
-	fn temp_path() -> PathBuf {
-		let mut dir = env::temp_dir();
-		dir.push(random_filename());
-		dir
-	}
-}
-
-

dapps/src/handlers/fetch.rs

@@ -22,13 +22,13 @@ use std::sync::{mpsc, Arc};
 use std::sync::atomic::{AtomicBool, Ordering};
 use std::time::{Instant, Duration};
 use util::Mutex;
+use fetch::{Client, Fetch, FetchResult};
 
 use hyper::{server, Decoder, Encoder, Next, Method, Control};
 use hyper::net::HttpStream;
 use hyper::status::StatusCode;
 
 use handlers::{ContentHandler, Redirection};
-use handlers::client::{Client, FetchResult};
 use apps::redirection_address;
 use page::LocalPageEndpoint;
 
@@ -159,7 +159,7 @@ impl<H: ContentValidator> ContentFetcherHandler<H> {
 		handler: H) -> (Self, Arc<FetchControl>) {
 
 		let fetch_control = Arc::new(FetchControl::default());
-		let client = Client::new();
+		let client = Client::default();
 		let handler = ContentFetcherHandler {
 			fetch_control: fetch_control.clone(),
 			control: Some(control),

dapps/src/handlers/mod.rs

@@ -21,7 +21,6 @@ mod echo;
 mod content;
 mod redirect;
 mod fetch;
-pub mod client;
 
 pub use self::auth::AuthRequiredHandler;
 pub use self::echo::EchoHandler;

dapps/src/lib.rs

@@ -58,10 +58,10 @@ extern crate jsonrpc_http_server;
 extern crate mime_guess;
 extern crate rustc_serialize;
 extern crate parity_dapps;
-extern crate https_fetch;
 extern crate ethcore_rpc;
 extern crate ethcore_util as util;
 extern crate linked_hash_map;
+extern crate fetch;
 #[cfg(test)]
 extern crate ethcore_devtools as devtools;
 

db/src/database.rs

@@ -157,7 +157,7 @@ impl Drop for Database {
 	}
 }
 
-#[derive(Ipc)]
+#[ipc]
 impl DatabaseService for Database {
 	fn open(&self, config: DatabaseConfig, path: String) -> Result<(), Error> {
 		let mut db = self.db.write();

ethash/src/compute.rs

@@ -202,6 +202,9 @@ impl SeedHashCompute {
 	}
 }
 
+pub fn slow_get_seedhash(block_number: u64) -> H256 {
+	SeedHashCompute::resume_compute_seedhash([0u8; 32], 0, block_number / ETHASH_EPOCH_LENGTH)
+}
 
 #[inline]
 fn fnv_hash(x: u32, y: u32) -> u32 {

ethash/src/lib.rs

@@ -26,7 +26,7 @@ mod compute;
 
 use std::mem;
 use compute::Light;
-pub use compute::{ETHASH_EPOCH_LENGTH, H256, ProofOfWork, SeedHashCompute, quick_get_difficulty};
+pub use compute::{ETHASH_EPOCH_LENGTH, H256, ProofOfWork, SeedHashCompute, quick_get_difficulty, slow_get_seedhash};
 
 use std::sync::Arc;
 use parking_lot::Mutex;

ethcore/Cargo.toml

@@ -37,6 +37,9 @@ ethkey = { path = "../ethkey" }
 ethcore-ipc-nano = { path = "../ipc/nano" }
 rlp = { path = "../util/rlp" }
 rand = "0.3"
+lru-cache = { git = "https://github.com/contain-rs/lru-cache" }
+ethcore-bloom-journal = { path = "../util/bloom" }
+byteorder = "0.5"
 
 [dependencies.hyper]
 git = "https://github.com/ethcore/hyper"
@@ -44,7 +47,9 @@ default-features = false
 
 [features]
 jit = ["evmjit"]
-evm-debug = []
+evm-debug = ["slow-blocks"]
+evm-debug-tests = ["evm-debug"]
+slow-blocks = [] # Use SLOW_TX_DURATION="50" (compile time!) to track transactions over 50ms
 json-tests = []
 test-heavy = []
 dev = ["clippy"]

ethcore/build.rs

@@ -18,7 +18,7 @@ extern crate ethcore_ipc_codegen;
 
 fn main() {
 	ethcore_ipc_codegen::derive_binary("src/types/mod.rs.in").unwrap();
-	ethcore_ipc_codegen::derive_ipc("src/client/traits.rs").unwrap();
+	ethcore_ipc_codegen::derive_ipc_cond("src/client/traits.rs", cfg!(feature="ipc")).unwrap();
-	ethcore_ipc_codegen::derive_ipc("src/snapshot/snapshot_service_trait.rs").unwrap();
+	ethcore_ipc_codegen::derive_ipc_cond("src/snapshot/snapshot_service_trait.rs", cfg!(feature="ipc")).unwrap();
-	ethcore_ipc_codegen::derive_ipc("src/client/chain_notify.rs").unwrap();
+	ethcore_ipc_codegen::derive_ipc_cond("src/client/chain_notify.rs", cfg!(feature="ipc")).unwrap();
 }

ethcore/res/ethereum/expanse.json

@@ -0,0 +1,69 @@
+{
+	"name": "Expanse",
+	"forkName": "expanse",
+	"engine": {
+		"Ethash": {
+			"params": {
+				"gasLimitBoundDivisor": "0x0400",
+				"minimumDifficulty": "0x020000",
+				"difficultyBoundDivisor": "0x0800",
+				"difficultyIncrementDivisor": "60",
+				"durationLimit": "0x3C",
+				"blockReward": "0x6f05b59d3b200000",
+				"registrar" : "0x6c221ca53705f3497ec90ca7b84c59ae7382fc21",
+				"frontierCompatibilityModeLimit": "0x30d40",
+				"difficultyHardforkTransition": "0x59d9",
+				"difficultyHardforkBoundDivisor": "0x0200",
+				"bombDefuseTransition": "0x30d40"
+			}
+		}
+	},
+	"params": {
+		"accountStartNonce": "0x00",
+		"maximumExtraDataSize": "0x20",
+		"minGasLimit": "0x1388",
+		"networkID": "0x1",
+		"subprotocolName": "exp"
+	},
+	"genesis": {
+		"seal": {
+			"ethereum": {
+				"nonce": "0x214652414e4b4f21",
+				"mixHash": "0x0000000000000000000000000000000000000000000000000000000000000000"
+			}
+		},
+		"difficulty": "0x40000000",
+		"author": "0x93decab0cd745598860f782ac1e8f046cb99e898",
+		"timestamp": "0x00",
+		"parentHash": "0x0000000000000000000000000000000000000000000000000000000000000000",
+		"extraData": "0x4672616e6b6f497346726565646f6d",
+		"gasLimit": "0x1388"
+	},
+	"nodes": [
+		"enode://7f335a047654f3e70d6f91312a7cf89c39704011f1a584e2698250db3d63817e74b88e26b7854111e16b2c9d0c7173c05419aeee2d0321850227b126d8b1be3f@46.101.156.249:42786",
+		"enode://df872f81e25f72356152b44cab662caf1f2e57c3a156ecd20e9ac9246272af68a2031b4239a0bc831f2c6ab34733a041464d46b3ea36dce88d6c11714446e06b@178.62.208.109:42786",
+		"enode://96d3919b903e7f5ad59ac2f73c43be9172d9d27e2771355db03fd194732b795829a31fe2ea6de109d0804786c39a807e155f065b4b94c6fce167becd0ac02383@45.55.22.34:42786",
+		"enode://5f6c625bf287e3c08aad568de42d868781e961cbda805c8397cfb7be97e229419bef9a5a25a75f97632787106bba8a7caf9060fab3887ad2cfbeb182ab0f433f@46.101.182.53:42786",
+		"enode://d33a8d4c2c38a08971ed975b750f21d54c927c0bf7415931e214465a8d01651ecffe4401e1db913f398383381413c78105656d665d83f385244ab302d6138414@128.199.183.48:42786",
+		"enode://df872f81e25f72356152b44cab662caf1f2e57c3a156ecd20e9ac9246272af68a2031b4239a0bc831f2c6ab34733a041464d46b3ea36dce88d6c11714446e06b@178.62.208.109:42786",
+		"enode://f6f0d6b9b7d02ec9e8e4a16e38675f3621ea5e69860c739a65c1597ca28aefb3cec7a6d84e471ac927d42a1b64c1cbdefad75e7ce8872d57548ddcece20afdd1@159.203.64.95:42786"
+	],
+	"accounts": {
+		"0000000000000000000000000000000000000001": { "builtin": { "name": "ecrecover", "pricing": { "linear": { "base": 3000, "word": 0 } } } },
+		"0000000000000000000000000000000000000002": { "builtin": { "name": "sha256", "pricing": { "linear": { "base": 60, "word": 12 } } } },
+		"0000000000000000000000000000000000000003": { "builtin": { "name": "ripemd160", "pricing": { "linear": { "base": 600, "word": 120 } } } },
+		"0000000000000000000000000000000000000004": { "builtin": { "name": "identity", "pricing": { "linear": { "base": 15, "word": 3 } } } },
+		"bb94f0ceb32257275b2a7a9c094c13e469b4563e": {
+			"balance": "10000000000000000000000000"
+		},
+		"15656715068ab0dbdf0ab00748a8a19e40f28192": {
+			"balance": "1000000000000000000000000"
+		},
+		"c075fa11f85bda3aaba67106226aaf086ac16f4e": {
+			"balance": "100000000000000000000000"
+		},
+		"93decab0cd745598860f782ac1e8f046cb99e898": {
+			"balance": "10000000000000000000000"
+		}
+	}
+}

ethcore/res/ethereum/frontier.json

@@ -157,6 +157,9 @@
 		"stateRoot": "0xd7f8974fb5ac78d9ac099b9ad5018bedc2ce0a72dad1827a1709da30580f0544"
 	},
 	"nodes": [
+		"enode://cd6611461840543d5b9c56fbf088736154c699c43973b3a1a32390cf27106f87e58a818a606ccb05f3866de95a4fe860786fea71bf891ea95f234480d3022aa3@136.243.154.245:30303",
+		"enode://bcc7240543fe2cf86f5e9093d05753dd83343f8fda7bf0e833f65985c73afccf8f981301e13ef49c4804491eab043647374df1c4adf85766af88a624ecc3330e@136.243.154.244:30303",
+		"enode://ed4227681ca8c70beb2277b9e870353a9693f12e7c548c35df6bca6a956934d6f659999c2decb31f75ce217822eefca149ace914f1cbe461ed5a2ebaf9501455@88.212.206.70:30303",
 		"enode://a979fb575495b8d6db44f750317d0f4622bf4c2aa3365d6af7c284339968eef29b69ad0dce72a4d8db5ebb4968de0e3bec910127f134779fbcb0cb6d3331163c@52.16.188.185:30303",
 		"enode://de471bccee3d042261d52e9bff31458daecc406142b401d4cd848f677479f73104b9fdeb090af9583d3391b7f10cb2ba9e26865dd5fca4fcdc0fb1e3b723c786@54.94.239.50:30303",
 		"enode://1118980bf48b0a3640bdba04e0fe78b1add18e1cd99bf22d53daac1fd9972ad650df52176e7c7d89d1114cfef2bc23a2959aa54998a46afcf7d91809f0855082@52.74.57.123:30303",

ethcore/res/ethereum/morden.json

@@ -8,7 +8,7 @@
 				"difficultyBoundDivisor": "0x0800",
 				"durationLimit": "0x0d",
 				"blockReward": "0x4563918244F40000",
-				"registrar": "0x8e4e9b13d4b45cb0befc93c3061b1408f67316b2",
+				"registrar": "0x52dff57a8a1532e6afb3dc07e2af58bb9eb05b3d",
 				"frontierCompatibilityModeLimit": "0x789b0"
 			}
 		}

ethcore/res/ethereum/tests

@@ -1 +1 @@
-Subproject commit ac5475d676536cb945f98e9ff98384c01abd0599
+Subproject commit 8f07dbc8294a32db5ebe8098925fcefc2eab3e71

ethcore/src/account_db.rs

@@ -121,6 +121,10 @@ impl<'db> HashDB for AccountDB<'db>{
 	fn remove(&mut self, _key: &H256) {
 		unimplemented!()
 	}
+
+	fn get_aux(&self, hash: &[u8]) -> Option<Vec<u8>> {
+		self.db.get_aux(hash)
+	}
 }
 
 /// DB backend wrapper for Account trie
@@ -193,6 +197,18 @@ impl<'db> HashDB for AccountDBMut<'db>{
 		let key = combine_key(&self.address_hash, key);
 		self.db.remove(&key)
 	}
+
+	fn insert_aux(&mut self, hash: Vec<u8>, value: Vec<u8>) {
+		self.db.insert_aux(hash, value);
+	}
+
+	fn get_aux(&self, hash: &[u8]) -> Option<Vec<u8>> {
+		self.db.get_aux(hash)
+	}
+
+	fn remove_aux(&mut self, hash: &[u8]) {
+		self.db.remove_aux(hash);
+	}
 }
 
 struct Wrapping<'db>(&'db HashDB);

ethcore/src/action_params.rs

@@ -43,6 +43,8 @@ impl ActionValue {
 pub struct ActionParams {
 	/// Address of currently executed code.
 	pub code_address: Address,
+	/// Hash of currently executed code.
+	pub code_hash: H256,
 	/// Receive address. Usually equal to code_address,
 	/// except when called using CALLCODE.
 	pub address: Address,
@@ -57,7 +59,7 @@ pub struct ActionParams {
 	/// Transaction value.
 	pub value: ActionValue,
 	/// Code being executed.
-	pub code: Option<Bytes>,
+	pub code: Option<Arc<Bytes>>,
 	/// Input data.
 	pub data: Option<Bytes>,
 	/// Type of call
@@ -70,6 +72,7 @@ impl Default for ActionParams {
 	fn default() -> ActionParams {
 		ActionParams {
 			code_address: Address::new(),
+			code_hash: SHA3_EMPTY,
 			address: Address::new(),
 			sender: Address::new(),
 			origin: Address::new(),
@@ -88,10 +91,11 @@ impl From<ethjson::vm::Transaction> for ActionParams {
 		let address: Address = t.address.into();
 		ActionParams {
 			code_address: Address::new(),
+			code_hash: (&*t.code).sha3(),
 			address: address,
 			sender: t.sender.into(),
 			origin: t.origin.into(),
-			code: Some(t.code.into()),
+			code: Some(Arc::new(t.code.into())),
 			data: Some(t.data.into()),
 			gas: t.gas.into(),
 			gas_price: t.gas_price.into(),

ethcore/src/block.rs

@@ -16,13 +16,26 @@
 
 //! Blockchain block.
 
-use common::*;
+use std::sync::Arc;
+use std::collections::HashSet;
+
+use rlp::{UntrustedRlp, RlpStream, Encodable, Decodable, Decoder, DecoderError, View, Stream};
+use util::{Bytes, Address, Uint, FixedHash, Hashable, U256, H256, ordered_trie_root, SHA3_NULL_RLP};
+use util::error::{Mismatch, OutOfBounds};
+
+use basic_types::{LogBloom, Seal};
+use env_info::{EnvInfo, LastHashes};
 use engines::Engine;
-use state::*;
+use error::{Error, BlockError, TransactionError};
-use verification::PreverifiedBlock;
-use trace::FlatTrace;
 use factory::Factories;
-use rlp::*;
+use header::Header;
+use receipt::Receipt;
+use state::State;
+use state_db::StateDB;
+use trace::FlatTrace;
+use transaction::SignedTransaction;
+use verification::PreverifiedBlock;
+use views::BlockView;
 
 /// A block, encoded as it is on the block chain.
 #[derive(Default, Debug, Clone, PartialEq)]
@@ -69,7 +82,7 @@ impl Decodable for Block {
 	}
 }
 
-/// Internal type for a block's common elements.
+/// An internal type for a block's common elements.
 #[derive(Clone)]
 pub struct ExecutedBlock {
 	base: Block,
@@ -179,7 +192,7 @@ pub trait IsBlock {
 /// Trait for a object that has a state database.
 pub trait Drain {
 	/// Drop this object and return the underlieing database.
-	fn drain(self) -> Box<JournalDB>;
+	fn drain(self) -> StateDB;
 }
 
 impl IsBlock for ExecutedBlock {
@@ -205,6 +218,7 @@ pub struct ClosedBlock {
 	block: ExecutedBlock,
 	uncle_bytes: Bytes,
 	last_hashes: Arc<LastHashes>,
+	unclosed_state: State,
 }
 
 /// Just like `ClosedBlock` except that we can't reopen it and it's faster.
@@ -231,7 +245,7 @@ impl<'x> OpenBlock<'x> {
 		engine: &'x Engine,
 		factories: Factories,
 		tracing: bool,
-		db: Box<JournalDB>,
+		db: StateDB,
 		parent: &Header,
 		last_hashes: Arc<LastHashes>,
 		author: Address,
@@ -346,8 +360,7 @@ impl<'x> OpenBlock<'x> {
 	pub fn close(self) -> ClosedBlock {
 		let mut s = self;
 
-		// take a snapshot so the engine's changes can be rolled back.
+		let unclosed_state = s.block.state.clone();
-		s.block.state.snapshot();
 
 		s.engine.on_close_block(&mut s.block);
 		s.block.base.header.set_transactions_root(ordered_trie_root(s.block.base.transactions.iter().map(|e| e.rlp_bytes().to_vec())));
@@ -362,6 +375,7 @@ impl<'x> OpenBlock<'x> {
 			block: s.block,
 			uncle_bytes: uncle_bytes,
 			last_hashes: s.last_hashes,
+			unclosed_state: unclosed_state,
 		}
 	}
 
@@ -369,9 +383,6 @@ impl<'x> OpenBlock<'x> {
 	pub fn close_and_lock(self) -> LockedBlock {
 		let mut s = self;
 
-		// take a snapshot so the engine's changes can be rolled back.
-		s.block.state.snapshot();
-
 		s.engine.on_close_block(&mut s.block);
 		if s.block.base.header.transactions_root().is_zero() || s.block.base.header.transactions_root() == &SHA3_NULL_RLP {
 			s.block.base.header.set_transactions_root(ordered_trie_root(s.block.base.transactions.iter().map(|e| e.rlp_bytes().to_vec())));
@@ -388,11 +399,10 @@ impl<'x> OpenBlock<'x> {
 		s.block.base.header.set_log_bloom(s.block.receipts.iter().fold(LogBloom::zero(), |mut b, r| {b = &b | &r.log_bloom; b})); //TODO: use |= operator
 		s.block.base.header.set_gas_used(s.block.receipts.last().map_or(U256::zero(), |r| r.gas_used));
 
-		ClosedBlock {
+		LockedBlock {
 			block: s.block,
 			uncle_bytes: uncle_bytes,
-			last_hashes: s.last_hashes,
+		}
-		}.lock()
 	}
 }
 
@@ -413,17 +423,7 @@ impl ClosedBlock {
 	pub fn hash(&self) -> H256 { self.header().rlp_sha3(Seal::Without) }
 
 	/// Turn this into a `LockedBlock`, unable to be reopened again.
-	pub fn lock(mut self) -> LockedBlock {
+	pub fn lock(self) -> LockedBlock {
-		// finalize the changes made by the engine.
-		self.block.state.clear_snapshot();
-		if let Err(e) = self.block.state.commit() {
-			warn!("Error committing closed block's state: {:?}", e);
-		}
-
-		// set the state root here, after commit recalculates with the block
-		// rewards.
-		self.block.base.header.set_state_root(self.block.state.root().clone());
-
 		LockedBlock {
 			block: self.block,
 			uncle_bytes: self.uncle_bytes,
@@ -431,12 +431,12 @@ impl ClosedBlock {
 	}
 
 	/// Given an engine reference, reopen the `ClosedBlock` into an `OpenBlock`.
-	pub fn reopen(mut self, engine: &Engine) -> OpenBlock {
+	pub fn reopen(self, engine: &Engine) -> OpenBlock {
 		// revert rewards (i.e. set state back at last transaction's state).
-		self.block.state.revert_snapshot();
+		let mut block = self.block;
-
+		block.state = self.unclosed_state;
 		OpenBlock {
-			block: self.block,
+			block: block,
 			engine: engine,
 			last_hashes: self.last_hashes,
 		}
@@ -462,11 +462,11 @@ impl LockedBlock {
 	/// Provide a valid seal in order to turn this into a `SealedBlock`.
 	/// This does check the validity of `seal` with the engine.
 	/// Returns the `ClosedBlock` back again if the seal is no good.
-	pub fn try_seal(self, engine: &Engine, seal: Vec<Bytes>) -> Result<SealedBlock, LockedBlock> {
+	pub fn try_seal(self, engine: &Engine, seal: Vec<Bytes>) -> Result<SealedBlock, (Error, LockedBlock)> {
 		let mut s = self;
 		s.block.base.header.set_seal(seal);
 		match engine.verify_block_seal(&s.block.base.header) {
-			Err(_) => Err(s),
+			Err(e) => Err((e, s)),
 			_ => Ok(SealedBlock { block: s.block, uncle_bytes: s.uncle_bytes }),
 		}
 	}
@@ -474,7 +474,9 @@ impl LockedBlock {
 
 impl Drain for LockedBlock {
 	/// Drop this object and return the underlieing database.
-	fn drain(self) -> Box<JournalDB> { self.block.state.drop().1 }
+	fn drain(self) -> StateDB {
+		self.block.state.drop().1
+	}
 }
 
 impl SealedBlock {
@@ -490,7 +492,9 @@ impl SealedBlock {
 
 impl Drain for SealedBlock {
 	/// Drop this object and return the underlieing database.
-	fn drain(self) -> Box<JournalDB> { self.block.state.drop().1 }
+	fn drain(self) -> StateDB {
+		self.block.state.drop().1
+	}
 }
 
 impl IsBlock for SealedBlock {
@@ -505,7 +509,7 @@ pub fn enact(
 	uncles: &[Header],
 	engine: &Engine,
 	tracing: bool,
-	db: Box<JournalDB>,
+	db: StateDB,
 	parent: &Header,
 	last_hashes: Arc<LastHashes>,
 	factories: Factories,
@@ -526,18 +530,71 @@ pub fn enact(
 	b.set_uncles_hash(header.uncles_hash().clone());
 	b.set_transactions_root(header.transactions_root().clone());
 	b.set_receipts_root(header.receipts_root().clone());
-	for t in transactions { try!(b.push_transaction(t.clone(), None)); }
+
-	for u in uncles { try!(b.push_uncle(u.clone())); }
+	try!(push_transactions(&mut b, transactions));
+	for u in uncles {
+		try!(b.push_uncle(u.clone()));
+	}
 	Ok(b.close_and_lock())
 }
 
+#[inline(always)]
+#[cfg(not(feature = "slow-blocks"))]
+fn push_transactions(block: &mut OpenBlock, transactions: &[SignedTransaction]) -> Result<(), Error> {
+	for t in transactions {
+		try!(block.push_transaction(t.clone(), None));
+	}
+	Ok(())
+}
+
+#[cfg(feature = "slow-blocks")]
+fn push_transactions(block: &mut OpenBlock, transactions: &[SignedTransaction]) -> Result<(), Error> {
+	use std::time;
+
+	let slow_tx = option_env!("SLOW_TX_DURATION").and_then(|v| v.parse().ok()).unwrap_or(100);
+	for t in transactions {
+		let hash = t.hash();
+		let start = time::Instant::now();
+		try!(block.push_transaction(t.clone(), None));
+		let took = start.elapsed();
+		if took > time::Duration::from_millis(slow_tx) {
+			warn!("Heavy transaction in block {:?}: {:?}", block.header().number(), hash);
+		}
+	}
+	Ok(())
+}
+
 /// Enact the block given by `block_bytes` using `engine` on the database `db` with given `parent` block header
 #[cfg_attr(feature="dev", allow(too_many_arguments))]
-pub fn enact_bytes(
+pub fn enact_verified(
+	block: &PreverifiedBlock,
+	engine: &Engine,
+	tracing: bool,
+	db: StateDB,
+	parent: &Header,
+	last_hashes: Arc<LastHashes>,
+	factories: Factories,
+) -> Result<LockedBlock, Error> {
+	let view = BlockView::new(&block.bytes);
+	enact(&block.header, &block.transactions, &view.uncles(), engine, tracing, db, parent, last_hashes, factories)
+}
+
+#[cfg(test)]
+mod tests {
+	use tests::helpers::*;
+	use super::*;
+	use common::*;
+	use engines::Engine;
+	use factory::Factories;
+	use state_db::StateDB;
+
+	/// Enact the block given by `block_bytes` using `engine` on the database `db` with given `parent` block header
+	#[cfg_attr(feature="dev", allow(too_many_arguments))]
+	fn enact_bytes(
 		block_bytes: &[u8],
 		engine: &Engine,
 		tracing: bool,
-	db: Box<JournalDB>,
+		db: StateDB,
 		parent: &Header,
 		last_hashes: Arc<LastHashes>,
 		factories: Factories,
@@ -547,28 +604,13 @@ pub fn enact_bytes(
 		enact(&header, &block.transactions(), &block.uncles(), engine, tracing, db, parent, last_hashes, factories)
 	}
 
-/// Enact the block given by `block_bytes` using `engine` on the database `db` with given `parent` block header
-#[cfg_attr(feature="dev", allow(too_many_arguments))]
-pub fn enact_verified(
-	block: &PreverifiedBlock,
-	engine: &Engine,
-	tracing: bool,
-	db: Box<JournalDB>,
-	parent: &Header,
-	last_hashes: Arc<LastHashes>,
-	factories: Factories,
-) -> Result<LockedBlock, Error> {
-	let view = BlockView::new(&block.bytes);
-	enact(&block.header, &block.transactions, &view.uncles(), engine, tracing, db, parent, last_hashes, factories)
-}
-
 	/// Enact the block given by `block_bytes` using `engine` on the database `db` with given `parent` block header. Seal the block aferwards
 	#[cfg_attr(feature="dev", allow(too_many_arguments))]
-pub fn enact_and_seal(
+	fn enact_and_seal(
 		block_bytes: &[u8],
 		engine: &Engine,
 		tracing: bool,
-	db: Box<JournalDB>,
+		db: StateDB,
 		parent: &Header,
 		last_hashes: Arc<LastHashes>,
 		factories: Factories,
@@ -577,20 +619,14 @@ pub fn enact_and_seal(
 		Ok(try!(try!(enact_bytes(block_bytes, engine, tracing, db, parent, last_hashes, factories)).seal(engine, header.seal())))
 	}
 
-#[cfg(test)]
-mod tests {
-	use tests::helpers::*;
-	use super::*;
-	use common::*;
-
 	#[test]
 	fn open_block() {
 		use spec::*;
 		let spec = Spec::new_test();
 		let genesis_header = spec.genesis_header();
-		let mut db_result = get_temp_journal_db();
+		let mut db_result = get_temp_state_db();
 		let mut db = db_result.take();
-		spec.ensure_db_good(db.as_hashdb_mut()).unwrap();
+		spec.ensure_db_good(&mut db).unwrap();
 		let last_hashes = Arc::new(vec![genesis_header.hash()]);
 		let b = OpenBlock::new(&*spec.engine, Default::default(), false, db, &genesis_header, last_hashes, Address::zero(), (3141562.into(), 31415620.into()), vec![]).unwrap();
 		let b = b.close_and_lock();
@@ -604,25 +640,25 @@ mod tests {
 		let engine = &*spec.engine;
 		let genesis_header = spec.genesis_header();
 
-		let mut db_result = get_temp_journal_db();
+		let mut db_result = get_temp_state_db();
 		let mut db = db_result.take();
-		spec.ensure_db_good(db.as_hashdb_mut()).unwrap();
+		spec.ensure_db_good(&mut db).unwrap();
 		let last_hashes = Arc::new(vec![genesis_header.hash()]);
 		let b = OpenBlock::new(engine, Default::default(), false, db, &genesis_header, last_hashes.clone(), Address::zero(), (3141562.into(), 31415620.into()), vec![]).unwrap()
 			.close_and_lock().seal(engine, vec![]).unwrap();
 		let orig_bytes = b.rlp_bytes();
 		let orig_db = b.drain();
 
-		let mut db_result = get_temp_journal_db();
+		let mut db_result = get_temp_state_db();
 		let mut db = db_result.take();
-		spec.ensure_db_good(db.as_hashdb_mut()).unwrap();
+		spec.ensure_db_good(&mut db).unwrap();
 		let e = enact_and_seal(&orig_bytes, engine, false, db, &genesis_header, last_hashes, Default::default()).unwrap();
 
 		assert_eq!(e.rlp_bytes(), orig_bytes);
 
 		let db = e.drain();
-		assert_eq!(orig_db.keys(), db.keys());
+		assert_eq!(orig_db.journal_db().keys(), db.journal_db().keys());
-		assert!(orig_db.keys().iter().filter(|k| orig_db.get(k.0) != db.get(k.0)).next() == None);
+		assert!(orig_db.journal_db().keys().iter().filter(|k| orig_db.journal_db().get(k.0) != db.journal_db().get(k.0)).next() == None);
 	}
 
 	#[test]
@@ -632,9 +668,9 @@ mod tests {
 		let engine = &*spec.engine;
 		let genesis_header = spec.genesis_header();
 
-		let mut db_result = get_temp_journal_db();
+		let mut db_result = get_temp_state_db();
 		let mut db = db_result.take();
-		spec.ensure_db_good(db.as_hashdb_mut()).unwrap();
+		spec.ensure_db_good(&mut db).unwrap();
 		let last_hashes = Arc::new(vec![genesis_header.hash()]);
 		let mut open_block = OpenBlock::new(engine, Default::default(), false, db, &genesis_header, last_hashes.clone(), Address::zero(), (3141562.into(), 31415620.into()), vec![]).unwrap();
 		let mut uncle1_header = Header::new();
@@ -648,9 +684,9 @@ mod tests {
 		let orig_bytes = b.rlp_bytes();
 		let orig_db = b.drain();
 
-		let mut db_result = get_temp_journal_db();
+		let mut db_result = get_temp_state_db();
 		let mut db = db_result.take();
-		spec.ensure_db_good(db.as_hashdb_mut()).unwrap();
+		spec.ensure_db_good(&mut db).unwrap();
 		let e = enact_and_seal(&orig_bytes, engine, false, db, &genesis_header, last_hashes, Default::default()).unwrap();
 
 		let bytes = e.rlp_bytes();
@@ -659,7 +695,7 @@ mod tests {
 		assert_eq!(uncles[1].extra_data(), b"uncle2");
 
 		let db = e.drain();
-		assert_eq!(orig_db.keys(), db.keys());
+		assert_eq!(orig_db.journal_db().keys(), db.journal_db().keys());
-		assert!(orig_db.keys().iter().filter(|k| orig_db.get(k.0) != db.get(k.0)).next() == None);
+		assert!(orig_db.journal_db().keys().iter().filter(|k| orig_db.journal_db().get(k.0) != db.journal_db().get(k.0)).next() == None);
 	}
 }

ethcore/src/blockchain/blockchain.rs

@@ -181,7 +181,7 @@ pub struct BlockChain {
 
 	pending_best_block: RwLock<Option<BestBlock>>,
 	pending_block_hashes: RwLock<HashMap<BlockNumber, H256>>,
-	pending_transaction_addresses: RwLock<HashMap<H256, TransactionAddress>>,
+	pending_transaction_addresses: RwLock<HashMap<H256, Option<TransactionAddress>>>,
 }
 
 impl BlockProvider for BlockChain {
@@ -331,11 +331,12 @@ impl BlockProvider for BlockChain {
 			.filter_map(|number| self.block_hash(number).map(|hash| (number, hash)))
 			.filter_map(|(number, hash)| self.block_receipts(&hash).map(|r| (number, hash, r.receipts)))
 			.filter_map(|(number, hash, receipts)| self.block_body(&hash).map(|ref b| (number, hash, receipts, BodyView::new(b).transaction_hashes())))
-			.flat_map(|(number, hash, mut receipts, hashes)| {
+			.flat_map(|(number, hash, mut receipts, mut hashes)| {
 				assert_eq!(receipts.len(), hashes.len());
 				log_index = receipts.iter().fold(0, |sum, receipt| sum + receipt.logs.len());
 
 				let receipts_len = receipts.len();
+				hashes.reverse();
 				receipts.reverse();
 				receipts.into_iter()
 					.map(|receipt| receipt.logs)
@@ -680,8 +681,8 @@ impl BlockChain {
 				block_hashes: self.prepare_block_hashes_update(bytes, &info),
 				block_details: self.prepare_block_details_update(bytes, &info),
 				block_receipts: self.prepare_block_receipts_update(receipts, &info),
-				transactions_addresses: self.prepare_transaction_addresses_update(bytes, &info),
 				blocks_blooms: self.prepare_block_blooms_update(bytes, &info),
+				transactions_addresses: self.prepare_transaction_addresses_update(bytes, &info),
 				info: info,
 				block: bytes
 			}, is_best);
@@ -714,8 +715,8 @@ impl BlockChain {
 				block_hashes: self.prepare_block_hashes_update(bytes, &info),
 				block_details: update,
 				block_receipts: self.prepare_block_receipts_update(receipts, &info),
-				transactions_addresses: self.prepare_transaction_addresses_update(bytes, &info),
 				blocks_blooms: self.prepare_block_blooms_update(bytes, &info),
+				transactions_addresses: self.prepare_transaction_addresses_update(bytes, &info),
 				info: info,
 				block: bytes,
 			}, is_best);
@@ -783,8 +784,8 @@ impl BlockChain {
 			block_hashes: self.prepare_block_hashes_update(bytes, &info),
 			block_details: self.prepare_block_details_update(bytes, &info),
 			block_receipts: self.prepare_block_receipts_update(receipts, &info),
-			transactions_addresses: self.prepare_transaction_addresses_update(bytes, &info),
 			blocks_blooms: self.prepare_block_blooms_update(bytes, &info),
+			transactions_addresses: self.prepare_transaction_addresses_update(bytes, &info),
 			info: info.clone(),
 			block: bytes,
 		}, true);
@@ -877,7 +878,7 @@ impl BlockChain {
 			let mut write_txs = self.pending_transaction_addresses.write();
 
 			batch.extend_with_cache(db::COL_EXTRA, &mut *write_hashes, update.block_hashes, CacheUpdatePolicy::Overwrite);
-			batch.extend_with_cache(db::COL_EXTRA, &mut *write_txs, update.transactions_addresses, CacheUpdatePolicy::Overwrite);
+			batch.extend_with_option_cache(db::COL_EXTRA, &mut *write_txs, update.transactions_addresses, CacheUpdatePolicy::Overwrite);
 		}
 	}
 
@@ -895,18 +896,25 @@ impl BlockChain {
 			*best_block = block;
 		}
 
+		let pending_txs = mem::replace(&mut *pending_write_txs, HashMap::new());
+		let (retracted_txs, enacted_txs) = pending_txs.into_iter().partition::<HashMap<_, _>, _>(|&(_, ref value)| value.is_none());
+
 		let pending_hashes_keys: Vec<_> = pending_write_hashes.keys().cloned().collect();
-		let pending_txs_keys: Vec<_> = pending_write_txs.keys().cloned().collect();
+		let enacted_txs_keys: Vec<_> = enacted_txs.keys().cloned().collect();
 
 		write_hashes.extend(mem::replace(&mut *pending_write_hashes, HashMap::new()));
-		write_txs.extend(mem::replace(&mut *pending_write_txs, HashMap::new()));
+		write_txs.extend(enacted_txs.into_iter().map(|(k, v)| (k, v.expect("Transactions were partitioned; qed"))));
+
+		for hash in retracted_txs.keys() {
+			write_txs.remove(hash);
+		}
 
 		let mut cache_man = self.cache_man.lock();
 		for n in pending_hashes_keys {
 			cache_man.note_used(CacheID::BlockHashes(n));
 		}
 
-		for hash in pending_txs_keys {
+		for hash in enacted_txs_keys {
 			cache_man.note_used(CacheID::TransactionAddresses(hash));
 		}
 	}
@@ -1008,7 +1016,7 @@ impl BlockChain {
 	}
 
 	/// This function returns modified transaction addresses.
-	fn prepare_transaction_addresses_update(&self, block_bytes: &[u8], info: &BlockInfo) -> HashMap<H256, TransactionAddress> {
+	fn prepare_transaction_addresses_update(&self, block_bytes: &[u8], info: &BlockInfo) -> HashMap<H256, Option<TransactionAddress>> {
 		let block = BlockView::new(block_bytes);
 		let transaction_hashes = block.transaction_hashes();
 
@@ -1017,10 +1025,10 @@ impl BlockChain {
 				transaction_hashes.into_iter()
 					.enumerate()
 					.map(|(i ,tx_hash)| {
-						(tx_hash, TransactionAddress {
+						(tx_hash, Some(TransactionAddress {
 							block_hash: info.hash.clone(),
 							index: i
-						})
+						}))
 					})
 					.collect()
 			},
@@ -1031,23 +1039,30 @@ impl BlockChain {
 						let hashes = BodyView::new(&bytes).transaction_hashes();
 						hashes.into_iter()
 							.enumerate()
-							.map(|(i, tx_hash)| (tx_hash, TransactionAddress {
+							.map(|(i, tx_hash)| (tx_hash, Some(TransactionAddress {
 								block_hash: hash.clone(),
 								index: i,
-							}))
+							})))
-							.collect::<HashMap<H256, TransactionAddress>>()
+							.collect::<HashMap<H256, Option<TransactionAddress>>>()
 					});
 
 				let current_addresses = transaction_hashes.into_iter()
 					.enumerate()
 					.map(|(i ,tx_hash)| {
-						(tx_hash, TransactionAddress {
+						(tx_hash, Some(TransactionAddress {
 							block_hash: info.hash.clone(),
 							index: i
-						})
+						}))
 					});
 
-				addresses.chain(current_addresses).collect()
+				let retracted = data.retracted.iter().flat_map(|hash| {
+					let bytes = self.block_body(hash).expect("Retracted block must be in database.");
+					let hashes = BodyView::new(&bytes).transaction_hashes();
+					hashes.into_iter().map(|hash| (hash, None)).collect::<HashMap<H256, Option<TransactionAddress>>>()
+				});
+
+				// The order here is important! Don't remove transaction if it was part of enacted blocks as well.
+				retracted.chain(addresses).chain(current_addresses).collect()
 			},
 			BlockLocation::Branch => HashMap::new(),
 		}
@@ -1345,6 +1360,70 @@ mod tests {
 		// TODO: insert block that already includes one of them as an uncle to check it's not allowed.
 	}
 
+	#[test]
+	fn test_fork_transaction_addresses() {
+		let mut canon_chain = ChainGenerator::default();
+		let mut finalizer = BlockFinalizer::default();
+		let genesis = canon_chain.generate(&mut finalizer).unwrap();
+		let mut fork_chain = canon_chain.fork(1);
+		let mut fork_finalizer = finalizer.fork();
+
+		let t1 = Transaction {
+			nonce: 0.into(),
+			gas_price: 0.into(),
+			gas: 100_000.into(),
+			action: Action::Create,
+			value: 100.into(),
+			data: "601080600c6000396000f3006000355415600957005b60203560003555".from_hex().unwrap(),
+		}.sign(&"".sha3());
+
+
+		let b1a = canon_chain
+			.with_transaction(t1.clone())
+			.generate(&mut finalizer).unwrap();
+
+		// Empty block
+		let b1b = fork_chain
+			.generate(&mut fork_finalizer).unwrap();
+
+		let b2 = fork_chain
+			.generate(&mut fork_finalizer).unwrap();
+
+		let b1a_hash = BlockView::new(&b1a).header_view().sha3();
+		let b2_hash = BlockView::new(&b2).header_view().sha3();
+
+		let t1_hash = t1.hash();
+
+		let temp = RandomTempPath::new();
+		let db = new_db(temp.as_str());
+		let bc = BlockChain::new(Config::default(), &genesis, db.clone());
+
+		let mut batch = db.transaction();
+		let _ = bc.insert_block(&mut batch, &b1a, vec![]);
+		bc.commit();
+		let _ = bc.insert_block(&mut batch, &b1b, vec![]);
+		bc.commit();
+		db.write(batch).unwrap();
+
+		assert_eq!(bc.best_block_hash(), b1a_hash);
+		assert_eq!(bc.transaction_address(&t1_hash), Some(TransactionAddress {
+			block_hash: b1a_hash.clone(),
+			index: 0,
+		}));
+
+		// now let's make forked chain the canon chain
+		let mut batch = db.transaction();
+		let _ = bc.insert_block(&mut batch, &b2, vec![]);
+		bc.commit();
+		db.write(batch).unwrap();
+
+		// Transaction should be retracted
+		assert_eq!(bc.best_block_hash(), b2_hash);
+		assert_eq!(bc.transaction_address(&t1_hash), None);
+	}
+
+
+
 	#[test]
 	fn test_overwriting_transaction_addresses() {
 		let mut canon_chain = ChainGenerator::default();
@@ -1415,14 +1494,14 @@ mod tests {
 		db.write(batch).unwrap();
 
 		assert_eq!(bc.best_block_hash(), b1a_hash);
-		assert_eq!(bc.transaction_address(&t1_hash).unwrap(), TransactionAddress {
+		assert_eq!(bc.transaction_address(&t1_hash), Some(TransactionAddress {
 			block_hash: b1a_hash.clone(),
 			index: 0,
-		});
+		}));
-		assert_eq!(bc.transaction_address(&t2_hash).unwrap(), TransactionAddress {
+		assert_eq!(bc.transaction_address(&t2_hash), Some(TransactionAddress {
 			block_hash: b1a_hash.clone(),
 			index: 1,
-		});
+		}));
 
 		// now let's make forked chain the canon chain
 		let mut batch = db.transaction();
@@ -1431,18 +1510,18 @@ mod tests {
 		db.write(batch).unwrap();
 
 		assert_eq!(bc.best_block_hash(), b2_hash);
-		assert_eq!(bc.transaction_address(&t1_hash).unwrap(), TransactionAddress {
+		assert_eq!(bc.transaction_address(&t1_hash), Some(TransactionAddress {
 			block_hash: b1b_hash.clone(),
 			index: 1,
-		});
+		}));
-		assert_eq!(bc.transaction_address(&t2_hash).unwrap(), TransactionAddress {
+		assert_eq!(bc.transaction_address(&t2_hash), Some(TransactionAddress {
 			block_hash: b1b_hash.clone(),
 			index: 0,
-		});
+		}));
-		assert_eq!(bc.transaction_address(&t3_hash).unwrap(), TransactionAddress {
+		assert_eq!(bc.transaction_address(&t3_hash), Some(TransactionAddress {
 			block_hash: b2_hash.clone(),
 			index: 0,
-		});
+		}));
 	}
 
 	#[test]
@@ -1682,7 +1761,7 @@ mod tests {
 			gas_price: 0.into(),
 			gas: 100_000.into(),
 			action: Action::Create,
-			value: 100.into(),
+			value: 101.into(),
 			data: "601080600c6000396000f3006000355415600957005b60203560003555".from_hex().unwrap(),
 		}.sign(&"".sha3());
 		let t2 = Transaction {
@@ -1690,7 +1769,7 @@ mod tests {
 			gas_price: 0.into(),
 			gas: 100_000.into(),
 			action: Action::Create,
-			value: 100.into(),
+			value: 102.into(),
 			data: "601080600c6000396000f3006000355415600957005b60203560003555".from_hex().unwrap(),
 		}.sign(&"".sha3());
 		let t3 = Transaction {
@@ -1698,7 +1777,7 @@ mod tests {
 			gas_price: 0.into(),
 			gas: 100_000.into(),
 			action: Action::Create,
-			value: 100.into(),
+			value: 103.into(),
 			data: "601080600c6000396000f3006000355415600957005b60203560003555".from_hex().unwrap(),
 		}.sign(&"".sha3());
 		let tx_hash1 = t1.hash();

ethcore/src/blockchain/update.rs

@@ -17,8 +17,8 @@ pub struct ExtrasUpdate<'a> {
 	pub block_details: HashMap<H256, BlockDetails>,
 	/// Modified block receipts.
 	pub block_receipts: HashMap<H256, BlockReceipts>,
-	/// Modified transaction addresses.
-	pub transactions_addresses: HashMap<H256, TransactionAddress>,
 	/// Modified blocks blooms.
 	pub blocks_blooms: HashMap<LogGroupPosition, BloomGroup>,
+	/// Modified transaction addresses (None signifies removed transactions).
+	pub transactions_addresses: HashMap<H256, Option<TransactionAddress>>,
 }

ethcore/src/client/chain_notify.rs

@@ -18,7 +18,7 @@ use ipc::IpcConfig;
 use util::H256;
 
 /// Represents what has to be handled by actor listening to chain events
-#[derive(Ipc)]
+#[ipc]
 pub trait ChainNotify : Send + Sync {
 	/// fires when chain has new blocks.
 	fn new_blocks(&self,

ethcore/src/client/client.rs

@@ -23,15 +23,14 @@ use time::precise_time_ns;
 
 // util
 use util::{Bytes, PerfTimer, Itertools, Mutex, RwLock};
-use util::journaldb::{self, JournalDB};
+use util::{journaldb, TrieFactory, Trie};
-use util::{U256, H256, Address, H2048, Uint};
+use util::trie::TrieSpec;
-use util::sha3::*;
+use util::{U256, H256, Address, H2048, Uint, FixedHash};
-use util::TrieFactory;
 use util::kvdb::*;
 
 // other
 use io::*;
-use views::{BlockView, HeaderView, BodyView};
+use views::{HeaderView, BodyView};
 use error::{ImportError, ExecutionError, CallError, BlockError, ImportResult, Error as EthcoreError};
 use header::BlockNumber;
 use state::State;
@@ -47,12 +46,12 @@ use transaction::{LocalizedTransaction, SignedTransaction, Action};
 use blockchain::extras::TransactionAddress;
 use types::filter::Filter;
 use log_entry::LocalizedLogEntry;
-use block_queue::{BlockQueue, BlockQueueInfo};
+use verification::queue::{BlockQueue, QueueInfo as BlockQueueInfo};
 use blockchain::{BlockChain, BlockProvider, TreeRoute, ImportRoute};
 use client::{
 	BlockID, TransactionID, UncleID, TraceId, ClientConfig, BlockChainClient,
 	MiningBlockChainClient, TraceFilter, CallAnalytics, BlockImportError, Mode,
-	ChainNotify
+	ChainNotify,
 };
 use client::Error as ClientError;
 use env_info::EnvInfo;
@@ -66,7 +65,7 @@ use miner::{Miner, MinerService};
 use snapshot::{self, io as snapshot_io};
 use factory::Factories;
 use rlp::{View, UntrustedRlp};
-
+use state_db::StateDB;
 
 // re-export
 pub use types::blockchain_info::BlockChainInfo;
@@ -126,9 +125,9 @@ pub struct Client {
 	tracedb: RwLock<TraceDB<BlockChain>>,
 	engine: Arc<Engine>,
 	config: ClientConfig,
-	db: RwLock<Arc<Database>>,
 	pruning: journaldb::Algorithm,
-	state_db: RwLock<Box<JournalDB>>,
+	db: RwLock<Arc<Database>>,
+	state_db: Mutex<StateDB>,
 	block_queue: BlockQueue,
 	report: RwLock<ClientReport>,
 	import_lock: Mutex<()>,
@@ -148,13 +147,6 @@ pub struct Client {
 /// assume finality of a given candidate.
 pub const HISTORY: u64 = 1200;
 
-/// Append a path element to the given path and return the string.
-pub fn append_path<P>(path: P, item: &str) -> String where P: AsRef<Path> {
-	let mut p = path.as_ref().to_path_buf();
-	p.push(item);
-	p.to_str().unwrap().to_owned()
-}
-
 impl Client {
 	/// Create a new client with given spec and DB path and custom verifier.
 	pub fn new(
@@ -168,18 +160,24 @@ impl Client {
 		let path = path.to_path_buf();
 		let gb = spec.genesis_block();
 
-		let db = Arc::new(try!(Database::open(&db_config, &path.to_str().unwrap()).map_err(ClientError::Database)));
+		let db = Arc::new(try!(Database::open(&db_config, &path.to_str().expect("DB path could not be converted to string.")).map_err(ClientError::Database)));
 		let chain = Arc::new(BlockChain::new(config.blockchain.clone(), &gb, db.clone()));
-		let tracedb = RwLock::new(try!(TraceDB::new(config.tracing.clone(), db.clone(), chain.clone())));
+		let tracedb = RwLock::new(TraceDB::new(config.tracing.clone(), db.clone(), chain.clone()));
 
-		let mut state_db = journaldb::new(db.clone(), config.pruning, ::db::COL_STATE);
+		let trie_spec = match config.fat_db {
-		if state_db.is_empty() && try!(spec.ensure_db_good(state_db.as_hashdb_mut())) {
+			true => TrieSpec::Fat,
+			false => TrieSpec::Secure,
+		};
+
+		let journal_db = journaldb::new(db.clone(), config.pruning, ::db::COL_STATE);
+		let mut state_db = StateDB::new(journal_db, config.state_cache_size);
+		if state_db.journal_db().is_empty() && try!(spec.ensure_db_good(&mut state_db)) {
 			let mut batch = DBTransaction::new(&db);
 			try!(state_db.journal_under(&mut batch, 0, &spec.genesis_header().hash()));
 			try!(db.write(batch).map_err(ClientError::Database));
 		}
 
-		if !chain.block_header(&chain.best_block_hash()).map_or(true, |h| state_db.contains(h.state_root())) {
+		if !chain.block_header(&chain.best_block_hash()).map_or(true, |h| state_db.journal_db().contains(h.state_root())) {
 			warn!("State root not found for block #{} ({})", chain.best_block_number(), chain.best_block_hash().hex());
 		}
 
@@ -192,8 +190,8 @@ impl Client {
 		let awake = match config.mode { Mode::Dark(..) => false, _ => true };
 
 		let factories = Factories {
-			vm: EvmFactory::new(config.vm_type.clone()),
+			vm: EvmFactory::new(config.vm_type.clone(), config.jump_table_size),
-			trie: TrieFactory::new(config.trie_spec.clone()),
+			trie: TrieFactory::new(trie_spec),
 			accountdb: Default::default(),
 		};
 
@@ -208,7 +206,7 @@ impl Client {
 			verifier: verification::new(config.verifier_type.clone()),
 			config: config,
 			db: RwLock::new(db),
-			state_db: RwLock::new(state_db),
+			state_db: Mutex::new(state_db),
 			block_queue: block_queue,
 			report: RwLock::new(Default::default()),
 			import_lock: Mutex::new(()),
@@ -291,30 +289,27 @@ impl Client {
 
 		// Check if Parent is in chain
 		let chain_has_parent = chain.block_header(header.parent_hash());
-		if let None = chain_has_parent {
+		if let Some(parent) = chain_has_parent {
-			warn!(target: "client", "Block import failed for #{} ({}): Parent not found ({}) ", header.number(), header.hash(), header.parent_hash());
-			return Err(());
-		};
-
 			// Enact Verified Block
-		let parent = chain_has_parent.unwrap();
 			let last_hashes = self.build_last_hashes(header.parent_hash().clone());
-		let db = self.state_db.read().boxed_clone();
+			let db = self.state_db.lock().boxed_clone_canon(&header.parent_hash());
 
 			let enact_result = enact_verified(block, engine, self.tracedb.read().tracing_enabled(), db, &parent, last_hashes, self.factories.clone());
-		if let Err(e) = enact_result {
+			let locked_block = try!(enact_result.map_err(|e| {
 				warn!(target: "client", "Block import failed for #{} ({})\nError: {:?}", header.number(), header.hash(), e);
-			return Err(());
+			}));
-		};
 
 			// Final Verification
-		let locked_block = enact_result.unwrap();
 			if let Err(e) = self.verifier.verify_block_final(header, locked_block.block().header()) {
 				warn!(target: "client", "Stage 4 block verification failed for #{} ({})\nError: {:?}", header.number(), header.hash(), e);
 				return Err(());
 			}
 
 			Ok(locked_block)
+		} else {
+			warn!(target: "client", "Block import failed for #{} ({}): Parent not found ({}) ", header.number(), header.hash(), header.parent_hash());
+			Err(())
+		}
 	}
 
 	fn calculate_enacted_retracted(&self, import_results: &[ImportRoute]) -> (Vec<H256>, Vec<H256>) {
@@ -358,23 +353,21 @@ impl Client {
 
 			for block in blocks {
 				let header = &block.header;
-				if invalid_blocks.contains(header.parent_hash()) {
+				let is_invalid = invalid_blocks.contains(header.parent_hash());
+				if is_invalid {
 					invalid_blocks.insert(header.hash());
 					continue;
 				}
-				let closed_block = self.check_and_close_block(&block);
+				if let Ok(closed_block) = self.check_and_close_block(&block) {
-				if let Err(_) = closed_block {
-					invalid_blocks.insert(header.hash());
-					continue;
-				}
-
-				let closed_block = closed_block.unwrap();
 					imported_blocks.push(header.hash());
 
 					let route = self.commit_block(closed_block, &header.hash(), &block.bytes);
 					import_results.push(route);
 
 					self.report.write().accrue_block(&block);
+				} else {
+					invalid_blocks.insert(header.hash());
+				}
 			}
 
 			let imported = imported_blocks.len();
@@ -435,13 +428,13 @@ impl Client {
 		// CHECK! I *think* this is fine, even if the state_root is equal to another
 		// already-imported block of the same number.
 		// TODO: Prove it with a test.
-		let mut db = block.drain();
+		let mut state = block.drain();
 
-		db.journal_under(&mut batch, number, hash).expect("DB commit failed");
+		state.journal_under(&mut batch, number, hash).expect("DB commit failed");
 
 		if number >= HISTORY {
 			let n = number - HISTORY;
-			db.mark_canonical(&mut batch, n, &chain.block_hash(n).unwrap()).expect("DB commit failed");
+			state.mark_canonical(&mut batch, n, &chain.block_hash(n).unwrap()).expect("DB commit failed");
 		}
 
 		let route = chain.insert_block(&mut batch, block_data, receipts);
@@ -452,10 +445,12 @@ impl Client {
 			enacted: route.enacted.clone(),
 			retracted: route.retracted.len()
 		});
+
+		let is_canon = route.enacted.last().map_or(false, |h| h == hash);
+		state.sync_cache(&route.enacted, &route.retracted, is_canon);
 		// Final commit to the DB
 		self.db.read().write_buffered(batch);
 		chain.commit();
-
 		self.update_last_hashes(&parent, hash);
 		route
 	}
@@ -497,7 +492,7 @@ impl Client {
 		};
 
 		self.block_header(id).and_then(|header| {
-			let db = self.state_db.read().boxed_clone();
+			let db = self.state_db.lock().boxed_clone();
 
 			// early exit for pruned blocks
 			if db.is_pruned() && self.chain.read().best_block_number() >= block_number + HISTORY {
@@ -527,9 +522,11 @@ impl Client {
 
 	/// Get a copy of the best block's state.
 	pub fn state(&self) -> State {
+		let header = self.best_block_header();
+		let header = HeaderView::new(&header);
 		State::from_existing(
-			self.state_db.read().boxed_clone(),
+			self.state_db.lock().boxed_clone_canon(&header.hash()),
-			HeaderView::new(&self.best_block_header()).state_root(),
+			header.state_root(),
 			self.engine.account_start_nonce(),
 			self.factories.clone())
 		.expect("State root of best block header always valid.")
@@ -543,7 +540,7 @@ impl Client {
 	/// Get the report.
 	pub fn report(&self) -> ClientReport {
 		let mut report = self.report.read().clone();
-		report.state_db_mem = self.state_db.read().mem_used();
+		report.state_db_mem = self.state_db.lock().mem_used();
 		report
 	}
 
@@ -599,7 +596,7 @@ impl Client {
 	/// Take a snapshot at the given block.
 	/// If the ID given is "latest", this will default to 1000 blocks behind.
 	pub fn take_snapshot<W: snapshot_io::SnapshotWriter + Send>(&self, writer: W, at: BlockID, p: &snapshot::Progress) -> Result<(), EthcoreError> {
-		let db = self.state_db.read().boxed_clone();
+		let db = self.state_db.lock().journal_db().boxed_clone();
 		let best_block_number = self.chain_info().best_block_number;
 		let block_number = try!(self.block_number(at).ok_or(snapshot::Error::InvalidStartingBlock(at)));
 
@@ -678,16 +675,17 @@ impl snapshot::DatabaseRestore for Client {
 		trace!(target: "snapshot", "Replacing client database with {:?}", new_db);
 
 		let _import_lock = self.import_lock.lock();
-		let mut state_db = self.state_db.write();
+		let mut state_db = self.state_db.lock();
 		let mut chain = self.chain.write();
 		let mut tracedb = self.tracedb.write();
 		self.miner.clear();
 		let db = self.db.write();
 		try!(db.restore(new_db));
 
-		*state_db = journaldb::new(db.clone(), self.pruning, ::db::COL_STATE);
+		let cache_size = state_db.cache_size();
+		*state_db = StateDB::new(journaldb::new(db.clone(), self.pruning, ::db::COL_STATE), cache_size);
 		*chain = Arc::new(BlockChain::new(self.config.blockchain.clone(), &[], db.clone()));
-		*tracedb = try!(TraceDB::new(self.config.tracing.clone(), db.clone(), chain.clone()).map_err(ClientError::from));
+		*tracedb = TraceDB::new(self.config.tracing.clone(), db.clone(), chain.clone());
 		Ok(())
 	}
 }
@@ -805,7 +803,7 @@ impl BlockChainClient for Client {
 		let chain = self.chain.read();
 		match Self::block_hash(&chain, id) {
 			Some(ref hash) if chain.is_known(hash) => BlockStatus::InChain,
-			Some(hash) => self.block_queue.block_status(&hash),
+			Some(hash) => self.block_queue.status(&hash).into(),
 			None => BlockStatus::Unknown
 		}
 	}
@@ -830,7 +828,7 @@ impl BlockChainClient for Client {
 	}
 
 	fn code(&self, address: &Address, id: BlockID) -> Option<Option<Bytes>> {
-		self.state_at(id).map(|s| s.code(address))
+		self.state_at(id).map(|s| s.code(address).map(|c| (*c).clone()))
 	}
 
 	fn balance(&self, address: &Address, id: BlockID) -> Option<U256> {
@@ -841,6 +839,38 @@ impl BlockChainClient for Client {
 		self.state_at(id).map(|s| s.storage_at(address, position))
 	}
 
+	fn list_accounts(&self, id: BlockID) -> Option<Vec<Address>> {
+		if !self.factories.trie.is_fat() {
+			trace!(target: "fatdb", "list_accounts: Not a fat DB");
+			return None;
+		}
+
+		let state = match self.state_at(id) {
+			Some(state) => state,
+			_ => return None,
+		};
+
+		let (root, db) = state.drop();
+		let trie = match self.factories.trie.readonly(db.as_hashdb(), &root) {
+			Ok(trie) => trie,
+			_ => {
+				trace!(target: "fatdb", "list_accounts: Couldn't open the DB");
+				return None;
+			}
+		};
+
+		let iter = match trie.iter() {
+			Ok(iter) => iter,
+			_ => return None,
+		};
+
+		let accounts = iter.filter_map(|item| {
+			item.ok().map(|(addr, _)| Address::from_slice(&addr))
+		}).collect();
+
+		Some(accounts)
+	}
+
 	fn transaction(&self, id: TransactionID) -> Option<LocalizedTransaction> {
 		self.transaction_address(id).and_then(|address| self.chain.read().transaction(&address))
 	}
@@ -852,12 +882,17 @@ impl BlockChainClient for Client {
 
 	fn transaction_receipt(&self, id: TransactionID) -> Option<LocalizedReceipt> {
 		let chain = self.chain.read();
-		self.transaction_address(id).and_then(|address| chain.block_number(&address.block_hash).and_then(|block_number| {
+		self.transaction_address(id)
+			.and_then(|address| chain.block_number(&address.block_hash).and_then(|block_number| {
 			let t = chain.block_body(&address.block_hash)
-				.and_then(|block| BodyView::new(&block).localized_transaction_at(&address.block_hash, block_number, address.index));
+				.and_then(|block| {
+					BodyView::new(&block).localized_transaction_at(&address.block_hash, block_number, address.index)
+				});
 
-			match (t, chain.transaction_receipt(&address)) {
+			let tx_and_sender = t.and_then(|tx| tx.sender().ok().map(|sender| (tx, sender)));
-				(Some(tx), Some(receipt)) => {
+
+			match (tx_and_sender, chain.transaction_receipt(&address)) {
+				(Some((tx, sender)), Some(receipt)) => {
 					let block_hash = tx.block_hash.clone();
 					let block_number = tx.block_number.clone();
 					let transaction_hash = tx.hash();
@@ -879,7 +914,7 @@ impl BlockChainClient for Client {
 						gas_used: receipt.gas_used - prior_gas_used,
 						contract_address: match tx.action {
 							Action::Call(_) => None,
-							Action::Create => Some(contract_address(&tx.sender().unwrap(), &tx.nonce))
+							Action::Create => Some(contract_address(&sender, &tx.nonce))
 						},
 						logs: receipt.logs.into_iter().enumerate().map(|(i, log)| LocalizedLogEntry {
 							entry: log,
@@ -909,7 +944,7 @@ impl BlockChainClient for Client {
 	}
 
 	fn state_data(&self, hash: &H256) -> Option<Bytes> {
-		self.state_db.read().state(hash)
+		self.state_db.lock().journal_db().state(hash)
 	}
 
 	fn block_receipts(&self, hash: &H256) -> Option<Bytes> {
@@ -917,16 +952,21 @@ impl BlockChainClient for Client {
 	}
 
 	fn import_block(&self, bytes: Bytes) -> Result<H256, BlockImportError> {
+		use verification::queue::kind::HasHash;
+		use verification::queue::kind::blocks::Unverified;
+
+		// create unverified block here so the `sha3` calculation can be cached.
+		let unverified = Unverified::new(bytes);
+
 		{
-			let header = BlockView::new(&bytes).header_view();
+			if self.chain.read().is_known(&unverified.hash()) {
-			if self.chain.read().is_known(&header.sha3()) {
 				return Err(BlockImportError::Import(ImportError::AlreadyInChain));
 			}
-			if self.block_status(BlockID::Hash(header.parent_hash())) == BlockStatus::Unknown {
+			if self.block_status(BlockID::Hash(unverified.parent_hash())) == BlockStatus::Unknown {
-				return Err(BlockImportError::Block(BlockError::UnknownParent(header.parent_hash())));
+				return Err(BlockImportError::Block(BlockError::UnknownParent(unverified.parent_hash())));
 			}
 		}
-		Ok(try!(self.block_queue.import_block(bytes)))
+		Ok(try!(self.block_queue.import(unverified)))
 	}
 
 	fn queue_info(&self) -> BlockQueueInfo {
@@ -975,17 +1015,18 @@ impl BlockChainClient for Client {
 		let start = self.block_number(filter.range.start);
 		let end = self.block_number(filter.range.end);
 
-		if start.is_some() && end.is_some() {
+		match (start, end) {
+			(Some(s), Some(e)) => {
 				let filter = trace::Filter {
-				range: start.unwrap() as usize..end.unwrap() as usize,
+					range: s as usize..e as usize,
 					from_address: From::from(filter.from_address),
 					to_address: From::from(filter.to_address),
 				};
 
 				let traces = self.tracedb.read().filter(&filter);
 				Some(traces)
-		} else {
+			},
-			None
+			_ => None,
 		}
 	}
 
@@ -1032,7 +1073,7 @@ impl BlockChainClient for Client {
 	}
 
 	fn pending_transactions(&self) -> Vec<SignedTransaction> {
-		self.miner.pending_transactions()
+		self.miner.pending_transactions(self.chain.read().best_block_number())
 	}
 }
 
@@ -1046,7 +1087,7 @@ impl MiningBlockChainClient for Client {
 			engine,
 			self.factories.clone(),
 			false,	// TODO: this will need to be parameterised once we want to do immediate mining insertion.
-			self.state_db.read().boxed_clone(),
+			self.state_db.lock().boxed_clone_canon(&h),
 			&chain.block_header(&h).expect("h is best block hash: so its header must exist: qed"),
 			self.build_last_hashes(h.clone()),
 			author,
@@ -1057,11 +1098,15 @@ impl MiningBlockChainClient for Client {
 		// Add uncles
 		chain
 			.find_uncle_headers(&h, engine.maximum_uncle_age())
-			.unwrap()
+			.unwrap_or_else(Vec::new)
 			.into_iter()
 			.take(engine.maximum_uncle_count())
 			.foreach(|h| {
-				open_block.push_uncle(h).unwrap();
+				open_block.push_uncle(h).expect("pushing maximum_uncle_count;
+												open_block was just created;
+												push_uncle is not ok only if more than maximum_uncle_count is pushed;
+												so all push_uncle are Ok;
+												qed");
 			});
 
 		open_block
@@ -1082,6 +1127,7 @@ impl MiningBlockChainClient for Client {
 		let block_data = block.rlp_bytes();
 		let route = self.commit_block(block, &h, &block_data);
 		trace!(target: "client", "Imported sealed block #{} ({})", number, h);
+		self.state_db.lock().sync_cache(&route.enacted, &route.retracted, false);
 
 		let (enacted, retracted) = self.calculate_enacted_retracted(&[route]);
 		self.miner.chain_new_blocks(self, &[h.clone()], &[], &enacted, &retracted);

ethcore/src/client/config.rs

@@ -16,13 +16,12 @@
 
 use std::str::FromStr;
 pub use std::time::Duration;
-pub use block_queue::BlockQueueConfig;
 pub use blockchain::Config as BlockChainConfig;
-pub use trace::{Config as TraceConfig, Switch};
+pub use trace::Config as TraceConfig;
 pub use evm::VMType;
-pub use verification::VerifierType;
+
+use verification::{VerifierType, QueueConfig};
 use util::{journaldb, CompactionProfile};
-use util::trie::TrieSpec;
 
 /// Client state db compaction profile
 #[derive(Debug, PartialEq)]
@@ -84,20 +83,20 @@ impl Default for Mode {
 #[derive(Debug, PartialEq, Default)]
 pub struct ClientConfig {
 	/// Block queue configuration.
-	pub queue: BlockQueueConfig,
+	pub queue: QueueConfig,
 	/// Blockchain configuration.
 	pub blockchain: BlockChainConfig,
 	/// Trace configuration.
 	pub tracing: TraceConfig,
 	/// VM type.
 	pub vm_type: VMType,
-	/// Trie type.
+	/// Fat DB enabled?
-	pub trie_spec: TrieSpec,
+	pub fat_db: bool,
 	/// The JournalDB ("pruning") algorithm to use.
 	pub pruning: journaldb::Algorithm,
 	/// The name of the client instance.
 	pub name: String,
-	/// State db cache-size if not default
+	/// RocksDB state column cache-size if not default
 	pub db_cache_size: Option<usize>,
 	/// State db compaction profile
 	pub db_compaction: DatabaseCompactionProfile,
@@ -107,6 +106,10 @@ pub struct ClientConfig {
 	pub mode: Mode,
 	/// Type of block verifier used by client.
 	pub verifier_type: VerifierType,
+	/// State db cache-size.
+	pub state_cache_size: usize,
+	/// EVM jump-tables cache size.
+	pub jump_table_size: usize,
 }
 
 #[cfg(test)]

ethcore/src/client/mod.rs

@@ -23,20 +23,27 @@ mod trace;
 mod client;
 
 pub use self::client::*;
-pub use self::config::{Mode, ClientConfig, DatabaseCompactionProfile, BlockQueueConfig, BlockChainConfig, Switch, VMType};
+pub use self::config::{Mode, ClientConfig, DatabaseCompactionProfile, BlockChainConfig, VMType};
 pub use self::error::Error;
 pub use types::ids::*;
 pub use self::test_client::{TestBlockChainClient, EachBlockWith};
 pub use types::trace_filter::Filter as TraceFilter;
 pub use executive::{Executed, Executive, TransactOptions};
 pub use env_info::{LastHashes, EnvInfo};
-pub use self::chain_notify::{ChainNotify, ChainNotifyClient};
+pub use self::chain_notify::ChainNotify;
 
 pub use types::call_analytics::CallAnalytics;
 pub use block_import_error::BlockImportError;
 pub use transaction_import::TransactionImportResult;
 pub use transaction_import::TransactionImportError;
-pub use self::traits::{BlockChainClient, MiningBlockChainClient, RemoteClient};
+pub use self::traits::{BlockChainClient, MiningBlockChainClient};
+
+/// IPC interfaces
+#[cfg(feature="ipc")]
+pub mod remote {
+	pub use super::traits::RemoteClient;
+	pub use super::chain_notify::ChainNotifyClient;
+}
 
 mod traits {
 	#![allow(dead_code, unused_assignments, unused_variables, missing_docs)] // codegen issues

ethcore/src/client/test_client.rs

@@ -25,8 +25,9 @@ use transaction::{Transaction, LocalizedTransaction, SignedTransaction, Action};
 use blockchain::TreeRoute;
 use client::{
 	BlockChainClient, MiningBlockChainClient, BlockChainInfo, BlockStatus, BlockID,
-	TransactionID, UncleID, TraceId, TraceFilter, LastHashes, CallAnalytics, BlockImportError
+	TransactionID, UncleID, TraceId, TraceFilter, LastHashes, CallAnalytics, BlockImportError,
 };
+use db::{NUM_COLUMNS, COL_STATE};
 use header::{Header as BlockHeader, BlockNumber};
 use filter::Filter;
 use log_entry::LocalizedLogEntry;
@@ -37,11 +38,12 @@ use evm::{Factory as EvmFactory, VMType};
 use miner::{Miner, MinerService, TransactionImportResult};
 use spec::Spec;
 
-use block_queue::BlockQueueInfo;
+use verification::queue::QueueInfo;
 use block::{OpenBlock, SealedBlock};
 use executive::Executed;
 use error::CallError;
 use trace::LocalizedTrace;
+use state_db::StateDB;
 
 /// Test client.
 pub struct TestBlockChainClient {
@@ -53,6 +55,8 @@ pub struct TestBlockChainClient {
 	pub genesis_hash: H256,
 	/// Last block hash.
 	pub last_hash: RwLock<H256>,
+	/// Extra data do set for each block
+	pub extra_data: Bytes,
 	/// Difficulty.
 	pub difficulty: RwLock<U256>,
 	/// Balances.
@@ -103,11 +107,17 @@ impl Default for TestBlockChainClient {
 impl TestBlockChainClient {
 	/// Creates new test client.
 	pub fn new() -> Self {
+		Self::new_with_extra_data(Bytes::new())
+	}
+
+	/// Creates new test client with specified extra data for each block
+	pub fn new_with_extra_data(extra_data: Bytes) -> Self {
 		let spec = Spec::new_test();
 		let mut client = TestBlockChainClient {
 			blocks: RwLock::new(HashMap::new()),
 			numbers: RwLock::new(HashMap::new()),
 			genesis_hash: H256::new(),
+			extra_data: extra_data,
 			last_hash: RwLock::new(H256::new()),
 			difficulty: RwLock::new(From::from(0)),
 			balances: RwLock::new(HashMap::new()),
@@ -120,7 +130,7 @@ impl TestBlockChainClient {
 			queue_size: AtomicUsize::new(0),
 			miner: Arc::new(Miner::with_spec(&spec)),
 			spec: spec,
-			vm_factory: EvmFactory::new(VMType::Interpreter),
+			vm_factory: EvmFactory::new(VMType::Interpreter, 1024 * 1024),
 			latest_block_timestamp: RwLock::new(10_000_000),
 		};
 		client.add_blocks(1, EachBlockWith::Nothing); // add genesis block
@@ -182,6 +192,7 @@ impl TestBlockChainClient {
 			header.set_parent_hash(self.last_hash.read().clone());
 			header.set_number(n as BlockNumber);
 			header.set_gas_limit(U256::from(1_000_000));
+			header.set_extra_data(self.extra_data.clone());
 			let uncles = match with {
 				EachBlockWith::Uncle | EachBlockWith::UncleAndTransaction => {
 					let mut uncles = RlpStream::new_list(1);
@@ -283,13 +294,14 @@ impl TestBlockChainClient {
 	}
 }
 
-pub fn get_temp_journal_db() -> GuardedTempResult<Box<JournalDB>> {
+pub fn get_temp_state_db() -> GuardedTempResult<StateDB> {
 	let temp = RandomTempPath::new();
-	let db = Database::open_default(temp.as_str()).unwrap();
+	let db = Database::open(&DatabaseConfig::with_columns(NUM_COLUMNS), temp.as_str()).unwrap();
-	let journal_db = journaldb::new(Arc::new(db), journaldb::Algorithm::EarlyMerge, None);
+	let journal_db = journaldb::new(Arc::new(db), journaldb::Algorithm::EarlyMerge, COL_STATE);
+	let state_db = StateDB::new(journal_db, 1024 * 1024);
 	GuardedTempResult {
 		_temp: temp,
-		result: Some(journal_db)
+		result: Some(state_db)
 	}
 }
 
@@ -297,9 +309,9 @@ impl MiningBlockChainClient for TestBlockChainClient {
 	fn prepare_open_block(&self, author: Address, gas_range_target: (U256, U256), extra_data: Bytes) -> OpenBlock {
 		let engine = &*self.spec.engine;
 		let genesis_header = self.spec.genesis_header();
-		let mut db_result = get_temp_journal_db();
+		let mut db_result = get_temp_state_db();
 		let mut db = db_result.take();
-		self.spec.ensure_db_good(db.as_hashdb_mut()).unwrap();
+		self.spec.ensure_db_good(&mut db).unwrap();
 
 		let last_hashes = vec![genesis_header.hash()];
 		let mut open_block = OpenBlock::new(
@@ -382,12 +394,16 @@ impl BlockChainClient for TestBlockChainClient {
 		}
 	}
 
+	fn list_accounts(&self, _id: BlockID) -> Option<Vec<Address>> {
+		None
+	}
+
 	fn transaction(&self, _id: TransactionID) -> Option<LocalizedTransaction> {
-		unimplemented!();
+		None	// Simple default.
 	}
 
 	fn uncle(&self, _id: UncleID) -> Option<Bytes> {
-		unimplemented!();
+		None	// Simple default.
 	}
 
 	fn transaction_receipt(&self, id: TransactionID) -> Option<LocalizedReceipt> {
@@ -544,8 +560,8 @@ impl BlockChainClient for TestBlockChainClient {
 		Ok(h)
 	}
 
-	fn queue_info(&self) -> BlockQueueInfo {
+	fn queue_info(&self) -> QueueInfo {
-		BlockQueueInfo {
+		QueueInfo {
 			verified_queue_size: self.queue_size.load(AtomicOrder::Relaxed),
 			unverified_queue_size: 0,
 			verifying_queue_size: 0,
@@ -595,6 +611,6 @@ impl BlockChainClient for TestBlockChainClient {
 	}
 
 	fn pending_transactions(&self) -> Vec<SignedTransaction> {
-		self.miner.pending_transactions()
+		self.miner.pending_transactions(self.chain_info().best_block_number)
 	}
 }

ethcore/src/client/traits.rs

@@ -17,7 +17,7 @@
 use std::collections::BTreeMap;
 use util::{U256, Address, H256, H2048, Bytes, Itertools};
 use blockchain::TreeRoute;
-use block_queue::BlockQueueInfo;
+use verification::queue::QueueInfo as BlockQueueInfo;
 use block::{OpenBlock, SealedBlock};
 use header::{BlockNumber};
 use transaction::{LocalizedTransaction, SignedTransaction};
@@ -38,7 +38,6 @@ use ipc::IpcConfig;
 use types::blockchain_info::BlockChainInfo;
 use types::block_status::BlockStatus;
 
-#[derive(Ipc)]
 #[ipc(client_ident="RemoteClient")]
 /// Blockchain database client. Owns and manages a blockchain and a block queue.
 pub trait BlockChainClient : Sync + Send {
@@ -112,6 +111,9 @@ pub trait BlockChainClient : Sync + Send {
 			Therefore storage_at has returned Some; qed")
 	}
 
+	/// Get a list of all accounts in the block `id`, if fat DB is in operation, otherwise `None`.
+	fn list_accounts(&self, id: BlockID) -> Option<Vec<Address>>;
+
 	/// Get transaction with given hash.
 	fn transaction(&self, id: TransactionID) -> Option<LocalizedTransaction>;
 

ethcore/src/db.rs

@@ -34,8 +34,10 @@ pub const COL_BODIES: Option<u32> = Some(2);
 pub const COL_EXTRA: Option<u32> = Some(3);
 /// Column for Traces
 pub const COL_TRACE: Option<u32> = Some(4);
+/// Column for Traces
+pub const COL_ACCOUNT_BLOOM: Option<u32> = Some(5);
 /// Number of columns in DB
-pub const NUM_COLUMNS: Option<u32> = Some(5);
+pub const NUM_COLUMNS: Option<u32> = Some(6);
 
 /// Modes for updating caches.
 #[derive(Clone, Copy)]
@@ -86,6 +88,9 @@ pub trait Writable {
 	/// Writes the value into the database.
 	fn write<T, R>(&mut self, col: Option<u32>, key: &Key<T, Target = R>, value: &T) where T: rlp::Encodable, R: Deref<Target = [u8]>;
 
+	/// Deletes key from the databse.
+	fn delete<T, R>(&mut self, col: Option<u32>, key: &Key<T, Target = R>) where T: rlp::Encodable, R: Deref<Target = [u8]>;
+
 	/// Writes the value into the database and updates the cache.
 	fn write_with_cache<K, T, R>(&mut self, col: Option<u32>, cache: &mut Cache<K, T>, key: K, value: T, policy: CacheUpdatePolicy) where
 	K: Key<T, Target = R> + Hash + Eq,
@@ -122,6 +127,34 @@ pub trait Writable {
 			},
 		}
 	}
+
+	/// Writes and removes the values into the database and updates the cache.
+	fn extend_with_option_cache<K, T, R>(&mut self, col: Option<u32>, cache: &mut Cache<K, Option<T>>, values: HashMap<K, Option<T>>, policy: CacheUpdatePolicy) where
+	K: Key<T, Target = R> + Hash + Eq,
+	T: rlp::Encodable,
+	R: Deref<Target = [u8]> {
+		match policy {
+			CacheUpdatePolicy::Overwrite => {
+				for (key, value) in values.into_iter() {
+					match value {
+						Some(ref v) => self.write(col, &key, v),
+						None => self.delete(col, &key),
+					}
+					cache.insert(key, value);
+				}
+			},
+			CacheUpdatePolicy::Remove => {
+				for (key, value) in values.into_iter() {
+					match value {
+						Some(v) => self.write(col, &key, &v),
+						None => self.delete(col, &key),
+					}
+					cache.remove(&key);
+				}
+			},
+		}
+	}
+
 }
 
 /// Should be used to read values from database.
@@ -173,6 +206,10 @@ impl Writable for DBTransaction {
 	fn write<T, R>(&mut self, col: Option<u32>, key: &Key<T, Target = R>, value: &T) where T: rlp::Encodable, R: Deref<Target = [u8]> {
 		self.put(col, &key.key(), &rlp::encode(value));
 	}
+
+	fn delete<T, R>(&mut self, col: Option<u32>, key: &Key<T, Target = R>) where T: rlp::Encodable, R: Deref<Target = [u8]> {
+		self.delete(col, &key.key());
+	}
 }
 
 impl Readable for Database {

ethcore/src/engines/basic_authority.rs

@@ -252,9 +252,9 @@ mod tests {
 		let spec = new_test_authority();
 		let engine = &*spec.engine;
 		let genesis_header = spec.genesis_header();
-		let mut db_result = get_temp_journal_db();
+		let mut db_result = get_temp_state_db();
 		let mut db = db_result.take();
-		spec.ensure_db_good(db.as_hashdb_mut()).unwrap();
+		spec.ensure_db_good(&mut db).unwrap();
 		let last_hashes = Arc::new(vec![genesis_header.hash()]);
 		let b = OpenBlock::new(engine, Default::default(), false, db, &genesis_header, last_hashes, addr, (3141562.into(), 31415620.into()), vec![]).unwrap();
 		let b = b.close_and_lock();

ethcore/src/engines/instant_seal.rs

@@ -81,9 +81,9 @@ mod tests {
 		let spec = Spec::new_test_instant();
 		let engine = &*spec.engine;
 		let genesis_header = spec.genesis_header();
-		let mut db_result = get_temp_journal_db();
+		let mut db_result = get_temp_state_db();
 		let mut db = db_result.take();
-		spec.ensure_db_good(db.as_hashdb_mut()).unwrap();
+		spec.ensure_db_good(&mut db).unwrap();
 		let last_hashes = Arc::new(vec![genesis_header.hash()]);
 		let b = OpenBlock::new(engine, Default::default(), false, db, &genesis_header, last_hashes, addr, (3141562.into(), 31415620.into()), vec![]).unwrap();
 		let b = b.close_and_lock();

ethcore/src/ethereum/ethash.rs

@@ -14,7 +14,7 @@
 // You should have received a copy of the GNU General Public License
 // along with Parity.  If not, see <http://www.gnu.org/licenses/>.
 
-use ethash::{quick_get_difficulty, EthashManager, H256 as EH256};
+use ethash::{quick_get_difficulty, slow_get_seedhash, EthashManager, H256 as EH256};
 use common::*;
 use block::*;
 use spec::CommonParams;
@@ -32,6 +32,8 @@ pub struct EthashParams {
 	pub minimum_difficulty: U256,
 	/// Difficulty bound divisor.
 	pub difficulty_bound_divisor: U256,
+	/// Difficulty increment divisor.
+	pub difficulty_increment_divisor: u64,
 	/// Block duration.
 	pub duration_limit: u64,
 	/// Block reward.
@@ -46,6 +48,12 @@ pub struct EthashParams {
 	pub dao_hardfork_beneficiary: Address,
 	/// DAO hard-fork DAO accounts list (L)
 	pub dao_hardfork_accounts: Vec<Address>,
+	/// Transition block for a change of difficulty params (currently just bound_divisor).
+	pub difficulty_hardfork_transition: u64,
+	/// Difficulty param after the difficulty transition.
+	pub difficulty_hardfork_bound_divisor: U256,
+	/// Block on which there is no additional difficulty from the exponential bomb.
+	pub bomb_defuse_transition: u64,
 }
 
 impl From<ethjson::spec::EthashParams> for EthashParams {
@@ -54,6 +62,7 @@ impl From<ethjson::spec::EthashParams> for EthashParams {
 			gas_limit_bound_divisor: p.gas_limit_bound_divisor.into(),
 			minimum_difficulty: p.minimum_difficulty.into(),
 			difficulty_bound_divisor: p.difficulty_bound_divisor.into(),
+			difficulty_increment_divisor: p.difficulty_increment_divisor.map_or(10, Into::into),
 			duration_limit: p.duration_limit.into(),
 			block_reward: p.block_reward.into(),
 			registrar: p.registrar.map_or_else(Address::new, Into::into),
@@ -61,6 +70,9 @@ impl From<ethjson::spec::EthashParams> for EthashParams {
 			dao_hardfork_transition: p.dao_hardfork_transition.map_or(0x7fffffffffffffff, Into::into),
 			dao_hardfork_beneficiary: p.dao_hardfork_beneficiary.map_or_else(Address::new, Into::into),
 			dao_hardfork_accounts: p.dao_hardfork_accounts.unwrap_or_else(Vec::new).into_iter().map(Into::into).collect(),
+			difficulty_hardfork_transition: p.difficulty_hardfork_transition.map_or(0x7fffffffffffffff, Into::into),
+			difficulty_hardfork_bound_divisor: p.difficulty_hardfork_bound_divisor.map_or(p.difficulty_bound_divisor.into(), Into::into),
+			bomb_defuse_transition: p.bomb_defuse_transition.map_or(0x7fffffffffffffff, Into::into),
 		}
 	}
 }
@@ -168,6 +180,10 @@ impl Engine for Ethash {
 			fields.state.add_balance(u.author(), &(reward * U256::from(8 + u.number() - current_number) / U256::from(8)));
 		}
 
+		// Commit state so that we can actually figure out the state root.
+		if let Err(e) = fields.state.commit() {
+			warn!("Encountered error on state commit: {}", e);		
+		}		
 	}
 
 	fn verify_block_basic(&self, header: &Header, _block: Option<&[u8]>) -> result::Result<(), Error> {
@@ -217,6 +233,7 @@ impl Engine for Ethash {
 		let result = self.pow.compute_light(header.number() as u64, &Ethash::to_ethash(header.bare_hash()), header.nonce().low_u64());
 		let mix = Ethash::from_ethash(result.mix_hash);
 		let difficulty = Ethash::boundary_to_difficulty(&Ethash::from_ethash(result.value));
+		trace!(target: "miner", "num: {}, seed: {}, h: {}, non: {}, mix: {}, res: {}" , header.number() as u64, Ethash::from_ethash(slow_get_seedhash(header.number() as u64)), header.bare_hash(), header.nonce().low_u64(), Ethash::from_ethash(result.mix_hash), Ethash::from_ethash(result.value));
 		if mix != header.mix_hash() {
 			return Err(From::from(BlockError::MismatchedH256SealElement(Mismatch { expected: mix, found: header.mix_hash() })));
 		}
@@ -267,7 +284,11 @@ impl Ethash {
 		}
 
 		let min_difficulty = self.ethash_params.minimum_difficulty;
-		let difficulty_bound_divisor = self.ethash_params.difficulty_bound_divisor;
+		let difficulty_hardfork = header.number() >= self.ethash_params.difficulty_hardfork_transition;
+		let difficulty_bound_divisor = match difficulty_hardfork {
+			true => self.ethash_params.difficulty_hardfork_bound_divisor,
+			false => self.ethash_params.difficulty_bound_divisor,
+		};
 		let duration_limit = self.ethash_params.duration_limit;
 		let frontier_limit = self.ethash_params.frontier_compatibility_mode_limit;
 
@@ -281,18 +302,20 @@ impl Ethash {
 		else {
 			trace!(target: "ethash", "Calculating difficulty parent.difficulty={}, header.timestamp={}, parent.timestamp={}", parent.difficulty(), header.timestamp(), parent.timestamp());
 			//block_diff = parent_diff + parent_diff // 2048 * max(1 - (block_timestamp - parent_timestamp) // 10, -99)
-			let diff_inc = (header.timestamp() - parent.timestamp()) / 10;
+			let diff_inc = (header.timestamp() - parent.timestamp()) / self.ethash_params.difficulty_increment_divisor;
 			if diff_inc <= 1 {
-				parent.difficulty().clone() + parent.difficulty().clone() / From::from(2048) * From::from(1 - diff_inc)
+				parent.difficulty().clone() + parent.difficulty().clone() / From::from(difficulty_bound_divisor) * From::from(1 - diff_inc)
 			} else {
-				parent.difficulty().clone() - parent.difficulty().clone() / From::from(2048) * From::from(min(diff_inc - 1, 99))
+				parent.difficulty().clone() - parent.difficulty().clone() / From::from(difficulty_bound_divisor) * From::from(min(diff_inc - 1, 99))
 			}
 		};
 		target = max(min_difficulty, target);
+		if header.number() < self.ethash_params.bomb_defuse_transition {
 			let period = ((parent.number() + 1) / EXP_DIFF_PERIOD) as usize;
 			if period > 1 {
 				target = max(min_difficulty, target + (U256::from(1) << (period - 2)));
 			}
+		}
 		target
 	}
 
@@ -355,9 +378,9 @@ mod tests {
 		let spec = new_morden();
 		let engine = &*spec.engine;
 		let genesis_header = spec.genesis_header();
-		let mut db_result = get_temp_journal_db();
+		let mut db_result = get_temp_state_db();
 		let mut db = db_result.take();
-		spec.ensure_db_good(db.as_hashdb_mut()).unwrap();
+		spec.ensure_db_good(&mut db).unwrap();
 		let last_hashes = Arc::new(vec![genesis_header.hash()]);
 		let b = OpenBlock::new(engine, Default::default(), false, db, &genesis_header, last_hashes, Address::zero(), (3141562.into(), 31415620.into()), vec![]).unwrap();
 		let b = b.close();
@@ -369,9 +392,9 @@ mod tests {
 		let spec = new_morden();
 		let engine = &*spec.engine;
 		let genesis_header = spec.genesis_header();
-		let mut db_result = get_temp_journal_db();
+		let mut db_result = get_temp_state_db();
 		let mut db = db_result.take();
-		spec.ensure_db_good(db.as_hashdb_mut()).unwrap();
+		spec.ensure_db_good(&mut db).unwrap();
 		let last_hashes = Arc::new(vec![genesis_header.hash()]);
 		let mut b = OpenBlock::new(engine, Default::default(), false, db, &genesis_header, last_hashes, Address::zero(), (3141562.into(), 31415620.into()), vec![]).unwrap();
 		let mut uncle = Header::new();

ethcore/src/ethereum/mod.rs

@@ -42,6 +42,9 @@ pub fn new_frontier() -> Spec { load(include_bytes!("../../res/ethereum/frontier
 /// Create a new Frontier mainnet chain spec without the DAO hardfork.
 pub fn new_classic() -> Spec { load(include_bytes!("../../res/ethereum/classic.json")) }
 
+/// Create a new Frontier mainnet chain spec without the DAO hardfork.
+pub fn new_expanse() -> Spec { load(include_bytes!("../../res/ethereum/expanse.json")) }
+
 /// Create a new Frontier chain spec as though it never changes to Homestead.
 pub fn new_frontier_test() -> Spec { load(include_bytes!("../../res/ethereum/frontier_test.json")) }
 
@@ -69,9 +72,9 @@ mod tests {
 		let spec = new_morden();
 		let engine = &spec.engine;
 		let genesis_header = spec.genesis_header();
-		let mut db_result = get_temp_journal_db();
+		let mut db_result = get_temp_state_db();
 		let mut db = db_result.take();
-		spec.ensure_db_good(db.as_hashdb_mut()).unwrap();
+		spec.ensure_db_good(&mut db).unwrap();
 		let s = State::from_existing(db, genesis_header.state_root().clone(), engine.account_start_nonce(), Default::default()).unwrap();
 		assert_eq!(s.balance(&"0000000000000000000000000000000000000001".into()), 1u64.into());
 		assert_eq!(s.balance(&"0000000000000000000000000000000000000002".into()), 1u64.into());

ethcore/src/evm/ext.rs

@@ -81,7 +81,7 @@ pub trait Ext {
 	) -> MessageCallResult;
 
 	/// Returns code at given address
-	fn extcode(&self, address: &Address) -> Bytes;
+	fn extcode(&self, address: &Address) -> Arc<Bytes>;
 
 	/// Returns code size at given address
 	fn extcodesize(&self, address: &Address) -> usize;

ethcore/src/evm/factory.rs

@@ -18,8 +18,10 @@
 //!
 //! TODO: consider spliting it into two separate files.
 use std::fmt;
+use std::sync::Arc;
 use evm::Evm;
 use util::{U256, Uint};
+use super::interpreter::SharedCache;
 
 #[derive(Debug, PartialEq, Clone)]
 /// Type of EVM to use.
@@ -82,7 +84,8 @@ impl VMType {
 /// Evm factory. Creates appropriate Evm.
 #[derive(Clone)]
 pub struct Factory {
-	evm: VMType
+	evm: VMType,
+	evm_cache: Arc<SharedCache>,
 }
 
 impl Factory {
@@ -95,9 +98,9 @@ impl Factory {
 				Box::new(super::jit::JitEvm::default())
 			},
 			VMType::Interpreter => if Self::can_fit_in_usize(gas) {
-				Box::new(super::interpreter::Interpreter::<usize>::default())
+				Box::new(super::interpreter::Interpreter::<usize>::new(self.evm_cache.clone()))
 			} else {
-				Box::new(super::interpreter::Interpreter::<U256>::default())
+				Box::new(super::interpreter::Interpreter::<U256>::new(self.evm_cache.clone()))
 			}
 		}
 	}
@@ -108,17 +111,19 @@ impl Factory {
 	pub fn create(&self, gas: U256) -> Box<Evm> {
 		match self.evm {
 			VMType::Interpreter => if Self::can_fit_in_usize(gas) {
-				Box::new(super::interpreter::Interpreter::<usize>::default())
+				Box::new(super::interpreter::Interpreter::<usize>::new(self.evm_cache.clone()))
 			} else {
-				Box::new(super::interpreter::Interpreter::<U256>::default())
+				Box::new(super::interpreter::Interpreter::<U256>::new(self.evm_cache.clone()))
 			}
 		}
 	}
 
-	/// Create new instance of specific `VMType` factory
+	/// Create new instance of specific `VMType` factory, with a size in bytes
-	pub fn new(evm: VMType) -> Self {
+	/// for caching jump destinations.
+	pub fn new(evm: VMType, cache_size: usize) -> Self {
 		Factory {
-			evm: evm
+			evm: evm,
+			evm_cache: Arc::new(SharedCache::new(cache_size)),
 		}
 	}
 
@@ -132,7 +137,8 @@ impl Default for Factory {
 	#[cfg(all(feature = "jit", not(test)))]
 	fn default() -> Factory {
 		Factory {
-			evm: VMType::Jit
+			evm: VMType::Jit,
+			evm_cache: Arc::new(SharedCache::default()),
 		}
 	}
 
@@ -140,7 +146,8 @@ impl Default for Factory {
 	#[cfg(any(not(feature = "jit"), test))]
 	fn default() -> Factory {
 		Factory {
-			evm: VMType::Interpreter
+			evm: VMType::Interpreter,
+			evm_cache: Arc::new(SharedCache::default()),
 		}
 	}
 }
@@ -158,22 +165,22 @@ macro_rules! evm_test(
 		#[ignore]
 		#[cfg(feature = "jit")]
 		fn $name_jit() {
-			$name_test(Factory::new(VMType::Jit));
+			$name_test(Factory::new(VMType::Jit, 1024 * 32));
 		}
 		#[test]
 		fn $name_int() {
-			$name_test(Factory::new(VMType::Interpreter));
+			$name_test(Factory::new(VMType::Interpreter, 1024 * 32));
 		}
 	};
 	($name_test: ident: $name_jit: ident, $name_int: ident) => {
 		#[test]
 		#[cfg(feature = "jit")]
 		fn $name_jit() {
-			$name_test(Factory::new(VMType::Jit));
+			$name_test(Factory::new(VMType::Jit, 1024 * 32));
 		}
 		#[test]
 		fn $name_int() {
-			$name_test(Factory::new(VMType::Interpreter));
+			$name_test(Factory::new(VMType::Interpreter, 1024 * 32));
 		}
 	}
 );
@@ -187,13 +194,13 @@ macro_rules! evm_test_ignore(
 		#[cfg(feature = "jit")]
 		#[cfg(feature = "ignored-tests")]
 		fn $name_jit() {
-			$name_test(Factory::new(VMType::Jit));
+			$name_test(Factory::new(VMType::Jit, 1024 * 32));
 		}
 		#[test]
 		#[ignore]
 		#[cfg(feature = "ignored-tests")]
 		fn $name_int() {
-			$name_test(Factory::new(VMType::Interpreter));
+			$name_test(Factory::new(VMType::Interpreter, 1024 * 32));
 		}
 	}
 );

ethcore/src/evm/interpreter/informant.rs

@@ -0,0 +1,164 @@
+// Copyright 2015, 2016 Ethcore (UK) Ltd.
+// This file is part of Parity.
+
+// Parity is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Parity is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Parity.  If not, see <http://www.gnu.org/licenses/>.
+
+pub use self::inner::*;
+
+#[macro_use]
+#[cfg(not(feature = "evm-debug"))]
+mod inner {
+	macro_rules! evm_debug {
+		($x: expr) => {}
+	}
+
+	pub struct EvmInformant;
+	impl EvmInformant {
+		pub fn new(_depth: usize) -> Self {
+			EvmInformant {}
+		}
+		pub fn done(&mut self) {}
+	}
+}
+
+#[macro_use]
+#[cfg(feature = "evm-debug")]
+mod inner {
+	use std::iter;
+	use std::collections::HashMap;
+	use std::time::{Instant, Duration};
+
+	use evm::interpreter::stack::Stack;
+	use evm::instructions::{Instruction, InstructionInfo, INSTRUCTIONS};
+	use evm::{CostType};
+
+	use util::U256;
+
+	macro_rules! evm_debug {
+		($x: expr) => {
+			$x
+		}
+	}
+
+	fn print(data: String) {
+		if cfg!(feature = "evm-debug-tests") {
+			println!("{}", data);
+		} else {
+			debug!(target: "evm", "{}", data);
+		}
+	}
+
+	pub struct EvmInformant {
+		spacing: String,
+		last_instruction: Instant,
+		stats: HashMap<Instruction, Stats>,
+	}
+
+	impl EvmInformant {
+
+		fn color(instruction: Instruction, name: &str) -> String {
+			let c = instruction as usize % 6;
+			let colors = [31, 34, 33, 32, 35, 36];
+			format!("\x1B[1;{}m{}\x1B[0m", colors[c], name)
+		}
+
+		fn as_micro(duration: &Duration) -> u64 {
+			let mut sec = duration.as_secs();
+			let subsec = duration.subsec_nanos() as u64;
+			sec = sec.saturating_mul(1_000_000u64);
+			sec += subsec / 1_000;
+			sec
+		}
+
+		pub fn new(depth: usize) -> Self {
+			EvmInformant {
+				spacing: iter::repeat(".").take(depth).collect(),
+				last_instruction: Instant::now(),
+				stats: HashMap::new(),
+			}
+		}
+
+		pub fn before_instruction<Cost: CostType>(&mut self, pc: usize, instruction: Instruction, info: &InstructionInfo, current_gas: &Cost, stack: &Stack<U256>) {
+			let time = self.last_instruction.elapsed();
+			self.last_instruction = Instant::now();
+
+			print(format!("{}[0x{:<3x}][{:>19}(0x{:<2x}) Gas Left: {:6?} (Previous took: {:10}μs)",
+				&self.spacing,
+				pc,
+				Self::color(instruction, info.name),
+				instruction,
+				current_gas,
+				Self::as_micro(&time),
+			));
+
+			if info.args > 0 {
+				for (idx, item) in stack.peek_top(info.args).iter().enumerate() {
+					print(format!("{}       |{:2}: {:?}", self.spacing, idx, item));
+				}
+			}
+		}
+
+		pub fn after_instruction(&mut self, instruction: Instruction) {
+			let mut stats = self.stats.entry(instruction).or_insert_with(|| Stats::default());
+			let took = self.last_instruction.elapsed();
+			stats.note(took);
+		}
+
+		pub fn done(&mut self) {
+			// Print out stats
+			let infos = &*INSTRUCTIONS;
+
+			let mut stats: Vec<(_,_)> = self.stats.drain().collect();
+			stats.sort_by(|ref a, ref b| b.1.avg().cmp(&a.1.avg()));
+
+			print(format!("\n{}-------OPCODE STATS:", self.spacing));
+			for (instruction, stats) in stats.into_iter() {
+				let info = infos[instruction as usize];
+				print(format!("{}-------{:>19}(0x{:<2x}) count: {:4}, avg: {:10}μs",
+					self.spacing,
+					Self::color(instruction, info.name),
+					instruction,
+					stats.count,
+					stats.avg(),
+				));
+			}
+		}
+
+	}
+
+	struct Stats {
+		count: u64,
+		total_duration: Duration,
+	}
+
+	impl Default for Stats {
+		fn default() -> Self {
+			Stats {
+				count: 0,
+				total_duration: Duration::from_secs(0),
+			}
+		}
+	}
+
+	impl Stats {
+		fn note(&mut self, took: Duration) {
+			self.count += 1;
+			self.total_duration += took;
+		}
+
+		fn avg(&self) -> u64 {
+			EvmInformant::as_micro(&self.total_duration) / self.count
+		}
+	}
+}

ethcore/src/evm/interpreter/mod.rs

@@ -16,25 +16,17 @@
 
 //! Rust VM implementation
 
-#[cfg(not(feature = "evm-debug"))]
+#[macro_use]
-macro_rules! evm_debug {
+mod informant;
-	($x: expr) => {}
-}
-
-#[cfg(feature = "evm-debug")]
-macro_rules! evm_debug {
-	($x: expr) => {
-		$x
-	}
-}
-
 mod gasometer;
 mod stack;
 mod memory;
+mod shared_cache;
 
 use self::gasometer::Gasometer;
 use self::stack::{Stack, VecStack};
 use self::memory::Memory;
+pub use self::shared_cache::SharedCache;
 
 use std::marker::PhantomData;
 use common::*;
@@ -43,16 +35,20 @@ use super::instructions::{self, Instruction, InstructionInfo};
 use evm::{self, MessageCallResult, ContractCreateResult, GasLeft, CostType};
 use bit_set::BitSet;
 
-#[cfg(feature = "evm-debug")]
-fn color(instruction: Instruction, name: &'static str) -> String {
-	let c = instruction as usize % 6;
-	let colors = [31, 34, 33, 32, 35, 36];
-	format!("\x1B[1;{}m{}\x1B[0m", colors[c], name)
-}
-
 type CodePosition = usize;
 type ProgramCounter = usize;
 
+const ONE: U256 = U256([1, 0, 0, 0]);
+const TWO: U256 = U256([2, 0, 0, 0]);
+const TWO_POW_5: U256 = U256([0x20, 0, 0, 0]);
+const TWO_POW_8: U256 = U256([0x100, 0, 0, 0]);
+const TWO_POW_16: U256 = U256([0x10000, 0, 0, 0]);
+const TWO_POW_24: U256 = U256([0x1000000, 0, 0, 0]);
+const TWO_POW_64: U256 = U256([0, 0x1, 0, 0]); // 0x1 00000000 00000000
+const TWO_POW_96: U256 = U256([0, 0x100000000, 0, 0]); //0x1 00000000 00000000 00000000
+const TWO_POW_224: U256 = U256([0, 0, 0, 0x100000000]); //0x1 00000000 00000000 00000000 00000000 00000000 00000000 00000000
+const TWO_POW_248: U256 = U256([0, 0, 0, 0x100000000000000]); //0x1 00000000 00000000 00000000 00000000 00000000 00000000 00000000 000000
+
 /// Abstraction over raw vector of Bytes. Easier state management of PC.
 struct CodeReader<'a> {
 	position: ProgramCounter,
@@ -61,6 +57,15 @@ struct CodeReader<'a> {
 
 #[cfg_attr(feature="dev", allow(len_without_is_empty))]
 impl<'a> CodeReader<'a> {
+
+	/// Create new code reader - starting at position 0.
+	fn new(code: &'a Bytes) -> Self {
+		CodeReader {
+			position: 0,
+			code: code,
+		}
+	}
+
 	/// Get `no_of_bytes` from code and convert to U256. Move PC
 	fn read(&mut self, no_of_bytes: usize) -> U256 {
 		let pos = self.position;
@@ -87,9 +92,9 @@ enum InstructionResult<Gas> {
 
 
 /// Intepreter EVM implementation
-#[derive(Default)]
 pub struct Interpreter<Cost: CostType> {
 	mem: Vec<u8>,
+	cache: Arc<SharedCache>,
 	_type: PhantomData<Cost>,
 }
 
@@ -97,26 +102,25 @@ impl<Cost: CostType> evm::Evm for Interpreter<Cost> {
 	fn exec(&mut self, params: ActionParams, ext: &mut evm::Ext) -> evm::Result<GasLeft> {
 		self.mem.clear();
 
+		let mut informant = informant::EvmInformant::new(ext.depth());
+
 		let code = &params.code.as_ref().unwrap();
-		let valid_jump_destinations = self.find_jump_destinations(code);
+		let valid_jump_destinations = self.cache.jump_destinations(&params.code_hash, code);
 
 		let mut gasometer = Gasometer::<Cost>::new(try!(Cost::from_u256(params.gas)));
 		let mut stack = VecStack::with_capacity(ext.schedule().stack_limit, U256::zero());
-		let mut reader = CodeReader {
+		let mut reader = CodeReader::new(code);
-			position: 0,
-			code: code
-		};
 		let infos = &*instructions::INSTRUCTIONS;
 
 		while reader.position < code.len() {
 			let instruction = code[reader.position];
 			reader.position += 1;
 
-			let info = infos[instruction as usize];
+			let info = &infos[instruction as usize];
-			try!(self.verify_instruction(ext, instruction, &info, &stack));
+			try!(self.verify_instruction(ext, instruction, info, &stack));
 
 			// Calculate gas cost
-			let (gas_cost, mem_gas, mem_size) = try!(gasometer.get_gas_cost_mem(ext, instruction, &info, &stack, self.mem.size()));
+			let (gas_cost, mem_gas, mem_size) = try!(gasometer.get_gas_cost_mem(ext, instruction, info, &stack, self.mem.size()));
 			// TODO: make compile-time removable if too much of a performance hit.
 			let trace_executed = ext.trace_prepare_execute(reader.position - 1, instruction, &gas_cost.as_u256());
 
@@ -125,15 +129,7 @@ impl<Cost: CostType> evm::Evm for Interpreter<Cost> {
 			gasometer.current_mem_gas = mem_gas;
 			gasometer.current_gas = gasometer.current_gas - gas_cost;
 
-			evm_debug!({
+			evm_debug!({ informant.before_instruction(reader.position, instruction, info, &gasometer.current_gas, &stack) });
-				println!("[0x{:x}][{}(0x{:x}) Gas: {:?}\n  Gas Before: {:?}",
-					reader.position,
-					color(instruction, info.name),
-					instruction,
-					gas_cost,
-					gasometer.current_gas + gas_cost
-				);
-			});
 
 			let (mem_written, store_written) = match trace_executed {
 				true => (Self::mem_written(instruction, &stack), Self::store_written(instruction, &stack)),
@@ -145,6 +141,8 @@ impl<Cost: CostType> evm::Evm for Interpreter<Cost> {
 				gasometer.current_gas, &params, ext, instruction, &mut reader, &mut stack
 			));
 
+			evm_debug!({ informant.after_instruction(instruction) });
+
 			if trace_executed {
 				ext.trace_executed(gasometer.current_gas.as_u256(), stack.peek_top(info.ret), mem_written.map(|(o, s)| (o, &(self.mem[o..(o + s)]))), store_written);
 			}
@@ -166,17 +164,26 @@ impl<Cost: CostType> evm::Evm for Interpreter<Cost> {
 					reader.position = pos;
 				},
 				InstructionResult::StopExecutionNeedsReturn(gas, off, size) => {
+					informant.done();
 					return Ok(GasLeft::NeedsReturn(gas.as_u256(), self.mem.read_slice(off, size)));
 				},
 				InstructionResult::StopExecution => break,
 			}
 		}
-
+		informant.done();
 		Ok(GasLeft::Known(gasometer.current_gas.as_u256()))
 	}
 }
 
 impl<Cost: CostType> Interpreter<Cost> {
+	/// Create a new `Interpreter` instance with shared cache.
+	pub fn new(cache: Arc<SharedCache>) -> Interpreter<Cost> {
+		Interpreter {
+			mem: Vec::new(),
+			cache: cache,
+			_type: PhantomData::default(),
+		}
+	}
 
 	fn verify_instruction(&self, ext: &evm::Ext, instruction: Instruction, info: &InstructionInfo, stack: &Stack<U256>) -> evm::Result<()> {
 		let schedule = ext.schedule();
@@ -475,10 +482,10 @@ impl<Cost: CostType> Interpreter<Cost> {
 				stack.push(U256::from(len));
 			},
 			instructions::CALLDATACOPY => {
-				self.copy_data_to_memory(stack, &params.data.clone().unwrap_or_else(|| vec![]));
+				self.copy_data_to_memory(stack, params.data.as_ref().map_or_else(|| &[] as &[u8], |d| &*d as &[u8]));
 			},
 			instructions::CODECOPY => {
-				self.copy_data_to_memory(stack, &params.code.clone().unwrap_or_else(|| vec![]));
+				self.copy_data_to_memory(stack, params.code.as_ref().map_or_else(|| &[] as &[u8], |c| &**c as &[u8]));
 			},
 			instructions::EXTCODECOPY => {
 				let address = u256_to_address(&stack.pop_back());
@@ -599,7 +606,19 @@ impl<Cost: CostType> Interpreter<Cost> {
 				let a = stack.pop_back();
 				let b = stack.pop_back();
 				stack.push(if !self.is_zero(&b) {
-					a.overflowing_div(b).0
+					match b {
+						ONE => a,
+						TWO => a >> 1,
+						TWO_POW_5 => a >> 5,
+						TWO_POW_8 => a >> 8,
+						TWO_POW_16 => a >> 16,
+						TWO_POW_24 => a >> 24,
+						TWO_POW_64 => a >> 64,
+						TWO_POW_96 => a >> 96,
+						TWO_POW_224 => a >> 224,
+						TWO_POW_248 => a >> 248,
+						_ => a.overflowing_div(b).0,
+					}
 				} else {
 					U256::zero()
 				});
@@ -767,23 +786,6 @@ impl<Cost: CostType> Interpreter<Cost> {
 		Ok(())
 	}
 
-	fn find_jump_destinations(&self, code: &[u8]) -> BitSet {
-		let mut jump_dests = BitSet::with_capacity(code.len());
-		let mut position = 0;
-
-		while position < code.len() {
-			let instruction = code[position];
-
-			if instruction == instructions::JUMPDEST {
-				jump_dests.insert(position);
-			} else if instructions::is_push(instruction) {
-				position += instructions::get_push_bytes(instruction);
-			}
-			position += 1;
-		}
-
-		jump_dests
-	}
 }
 
 fn get_and_reset_sign(value: U256) -> (U256, bool) {
@@ -810,15 +812,3 @@ fn address_to_u256(value: Address) -> U256 {
 	U256::from(&*H256::from(value))
 }
 
-#[test]
-fn test_find_jump_destinations() {
-	// given
-	let interpreter = Interpreter::<U256>::default();
-	let code = "7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff5b01600055".from_hex().unwrap();
-
-	// when
-	let valid_jump_destinations = interpreter.find_jump_destinations(&code);
-
-	// then
-	assert!(valid_jump_destinations.contains(66));
-}

ethcore/src/evm/interpreter/shared_cache.rs

@@ -0,0 +1,125 @@
+// Copyright 2015, 2016 Ethcore (UK) Ltd.
+// This file is part of Parity.
+
+// Parity is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Parity is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Parity.  If not, see <http://www.gnu.org/licenses/>.
+
+use std::sync::Arc;
+use lru_cache::LruCache;
+use util::{H256, Mutex};
+use util::sha3::*;
+use bit_set::BitSet;
+use super::super::instructions;
+
+const INITIAL_CAPACITY: usize = 32;
+const DEFAULT_CACHE_SIZE: usize = 4 * 1024 * 1024;
+
+/// Global cache for EVM interpreter
+pub struct SharedCache {
+	jump_destinations: Mutex<LruCache<H256, Arc<BitSet>>>,
+	max_size: usize,
+	cur_size: Mutex<usize>,
+}
+
+impl SharedCache {
+	/// Create a jump destinations cache with a maximum size in bytes
+	/// to cache.
+	pub fn new(max_size: usize) -> Self {
+		SharedCache {
+			jump_destinations: Mutex::new(LruCache::new(INITIAL_CAPACITY)),
+			max_size: max_size * 8, // dealing with bits here.
+			cur_size: Mutex::new(0),
+		}
+	}
+
+	/// Get jump destinations bitmap for a contract.
+	pub fn jump_destinations(&self, code_hash: &H256, code: &[u8]) -> Arc<BitSet> {
+		if code_hash == &SHA3_EMPTY {
+			return Self::find_jump_destinations(code);
+		}
+
+		if let Some(d) = self.jump_destinations.lock().get_mut(code_hash) {
+			return d.clone();
+		}
+
+		let d = Self::find_jump_destinations(code);
+
+		{
+			let mut cur_size = self.cur_size.lock();
+			*cur_size += d.capacity();
+
+			let mut jump_dests = self.jump_destinations.lock();
+			let cap = jump_dests.capacity();
+
+			// grow the cache as necessary; it operates on amount of items
+			// but we're working based on memory usage.
+			if jump_dests.len() == cap && *cur_size < self.max_size {
+				jump_dests.set_capacity(cap * 2);
+			}
+
+			// account for any element displaced from the cache.
+			if let Some(lru) = jump_dests.insert(code_hash.clone(), d.clone()) {
+				*cur_size -= lru.capacity();
+			}
+
+			// remove elements until we are below the memory target.
+			while *cur_size > self.max_size {
+				match jump_dests.remove_lru() {
+					Some((_, v)) => *cur_size -= v.capacity(),
+					_ => break,
+				}
+			}
+		}
+
+		d
+	}
+
+	fn find_jump_destinations(code: &[u8]) -> Arc<BitSet> {
+		let mut jump_dests = BitSet::with_capacity(code.len());
+		let mut position = 0;
+
+		while position < code.len() {
+			let instruction = code[position];
+
+			if instruction == instructions::JUMPDEST {
+				jump_dests.insert(position);
+			} else if instructions::is_push(instruction) {
+				position += instructions::get_push_bytes(instruction);
+			}
+			position += 1;
+		}
+
+		jump_dests.shrink_to_fit();
+		Arc::new(jump_dests)
+	}
+}
+
+impl Default for SharedCache {
+	fn default() -> Self {
+		SharedCache::new(DEFAULT_CACHE_SIZE)
+	}
+}
+
+
+#[test]
+fn test_find_jump_destinations() {
+	use util::FromHex;
+	// given
+	let code = "7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff5b01600055".from_hex().unwrap();
+
+	// when
+	let valid_jump_destinations = SharedCache::find_jump_destinations(&code);
+
+	// then
+	assert!(valid_jump_destinations.contains(66));
+}

ethcore/src/evm/interpreter/stack.rs

@@ -34,7 +34,7 @@ pub trait Stack<T> {
 	/// Get number of elements on Stack
 	fn size(&self) -> usize;
 	/// Returns all data on stack.
-	fn peek_top(&mut self, no_of_elems: usize) -> &[T];
+	fn peek_top(&self, no_of_elems: usize) -> &[T];
 }
 
 pub struct VecStack<S> {
@@ -68,12 +68,7 @@ impl<S : fmt::Display> Stack<S> for VecStack<S> {
 	fn pop_back(&mut self) -> S {
 		let val = self.stack.pop();
 		match val {
-			Some(x) => {
+			Some(x) => x,
-				evm_debug!({
-					println!("   POP: {}", x)
-				});
-				x
-			},
 			None => panic!("Tried to pop from empty stack.")
 		}
 	}
@@ -88,9 +83,6 @@ impl<S : fmt::Display> Stack<S> for VecStack<S> {
 	}
 
 	fn push(&mut self, elem: S) {
-		evm_debug!({
-			println!("  PUSH: {}", elem)
-		});
 		self.stack.push(elem);
 	}
 
@@ -98,7 +90,7 @@ impl<S : fmt::Display> Stack<S> for VecStack<S> {
 		self.stack.len()
 	}
 
-	fn peek_top(&mut self, no_from_top: usize) -> &[S] {
+	fn peek_top(&self, no_from_top: usize) -> &[S] {
 		assert!(self.stack.len() >= no_from_top, "peek_top asked for more items than exist.");
 		&self.stack[self.stack.len() - no_from_top .. self.stack.len()]
 	}

ethcore/src/evm/jit.rs

@@ -18,6 +18,7 @@
 use common::*;
 use evmjit;
 use evm::{self, GasLeft};
+use types::executed::CallType;
 
 /// Should be used to convert jit types to ethcore
 trait FromJit<T>: Sized {
@@ -77,10 +78,11 @@ impl IntoJit<evmjit::I256> for U256 {
 impl IntoJit<evmjit::I256> for H256 {
 	fn into_jit(self) -> evmjit::I256 {
 		let mut ret = [0; 4];
-		for i in 0..self.bytes().len() {
+		let len = self.len();
-			let rev = self.bytes().len() - 1 - i;
+		for i in 0..len {
+			let rev = len - 1 - i;
 			let pos = rev / 8;
-			ret[pos] += (self.bytes()[i] as u64) << ((rev % 8) * 8);
+			ret[pos] += (self[i] as u64) << ((rev % 8) * 8);
 		}
 		evmjit::I256 { words: ret }
 	}
@@ -194,6 +196,7 @@ impl<'a> evmjit::Ext for ExtAdapter<'a> {
 				receive_address: *const evmjit::H256,
 				code_address: *const evmjit::H256,
 				transfer_value: *const evmjit::I256,
+				// We are ignoring apparent value - it's handled in externalities.
 				_apparent_value: *const evmjit::I256,
 				in_beg: *const u8,
 				in_size: u64,
@@ -207,10 +210,12 @@ impl<'a> evmjit::Ext for ExtAdapter<'a> {
 		let receive_address = unsafe { Address::from_jit(&*receive_address) };
 		let code_address = unsafe { Address::from_jit(&*code_address) };
 		let transfer_value = unsafe { U256::from_jit(&*transfer_value) };
-		let value = Some(transfer_value);
 
 		// receive address and code address are the same in normal calls
 		let is_callcode = receive_address != code_address;
+		let is_delegatecall = is_callcode && sender_address != receive_address;
+
+		let value = if is_delegatecall { None } else { Some(transfer_value) };
 
 		if !is_callcode && !self.ext.exists(&code_address) {
 			gas_cost = gas_cost + U256::from(self.ext.schedule().call_new_account_gas);
@@ -239,6 +244,12 @@ impl<'a> evmjit::Ext for ExtAdapter<'a> {
 			}
 		}
 
+		let call_type = match (is_callcode, is_delegatecall) {
+			(_, true) => CallType::DelegateCall,
+			(true, false) => CallType::CallCode,
+			(false, false) => CallType::Call,
+		};
+
 		match self.ext.call(
 					  &call_gas,
 					  &sender_address,
@@ -246,7 +257,9 @@ impl<'a> evmjit::Ext for ExtAdapter<'a> {
 					  value,
 					  unsafe { slice::from_raw_parts(in_beg, in_size as usize) },
 					  &code_address,
-					  unsafe { slice::from_raw_parts_mut(out_beg, out_size as usize) }) {
+					  unsafe { slice::from_raw_parts_mut(out_beg, out_size as usize) },
+					  call_type,
+					  ) {
 			evm::MessageCallResult::Success(gas_left) => unsafe {
 				*io_gas = (gas + gas_left).low_u64();
 				true

ethcore/src/evm/tests.rs

@@ -49,7 +49,7 @@ pub struct FakeExt {
 	depth: usize,
 	store: HashMap<H256, H256>,
 	blockhashes: HashMap<U256, H256>,
-	codes: HashMap<Address, Bytes>,
+	codes: HashMap<Address, Arc<Bytes>>,
 	logs: Vec<FakeLogEntry>,
 	_suicides: HashSet<Address>,
 	info: EnvInfo,
@@ -136,12 +136,12 @@ impl Ext for FakeExt {
 		MessageCallResult::Success(*gas)
 	}
 
-	fn extcode(&self, address: &Address) -> Bytes {
+	fn extcode(&self, address: &Address) -> Arc<Bytes> {
-		self.codes.get(address).unwrap_or(&Bytes::new()).clone()
+		self.codes.get(address).unwrap_or(&Arc::new(Bytes::new())).clone()
 	}
 
 	fn extcodesize(&self, address: &Address) -> usize {
-		self.codes.get(address).map(|v| v.len()).unwrap_or(0)
+		self.codes.get(address).map_or(0, |c| c.len())
 	}
 
 	fn log(&mut self, topics: Vec<H256>, data: &[u8]) {
@@ -184,11 +184,11 @@ fn test_stack_underflow() {
 	let mut params = ActionParams::default();
 	params.address = address.clone();
 	params.gas = U256::from(100_000);
-	params.code = Some(code);
+	params.code = Some(Arc::new(code));
 	let mut ext = FakeExt::new();
 
 	let err = {
-		let mut vm : Box<evm::Evm> = Box::new(super::interpreter::Interpreter::<usize>::default());
+		let mut vm : Box<evm::Evm> = Box::new(super::interpreter::Interpreter::<usize>::new(Arc::new(super::interpreter::SharedCache::default())));
 		test_finalize(vm.exec(params, &mut ext)).unwrap_err()
 	};
 
@@ -211,7 +211,7 @@ fn test_add(factory: super::Factory) {
 	let mut params = ActionParams::default();
 	params.address = address.clone();
 	params.gas = U256::from(100_000);
-	params.code = Some(code);
+	params.code = Some(Arc::new(code));
 	let mut ext = FakeExt::new();
 
 	let gas_left = {
@@ -231,7 +231,7 @@ fn test_sha3(factory: super::Factory) {
 	let mut params = ActionParams::default();
 	params.address = address.clone();
 	params.gas = U256::from(100_000);
-	params.code = Some(code);
+	params.code = Some(Arc::new(code));
 	let mut ext = FakeExt::new();
 
 	let gas_left = {
@@ -251,7 +251,7 @@ fn test_address(factory: super::Factory) {
 	let mut params = ActionParams::default();
 	params.address = address.clone();
 	params.gas = U256::from(100_000);
-	params.code = Some(code);
+	params.code = Some(Arc::new(code));
 	let mut ext = FakeExt::new();
 
 	let gas_left = {
@@ -273,7 +273,7 @@ fn test_origin(factory: super::Factory) {
 	params.address = address.clone();
 	params.origin = origin.clone();
 	params.gas = U256::from(100_000);
-	params.code = Some(code);
+	params.code = Some(Arc::new(code));
 	let mut ext = FakeExt::new();
 
 	let gas_left = {
@@ -295,7 +295,7 @@ fn test_sender(factory: super::Factory) {
 	params.address = address.clone();
 	params.sender = sender.clone();
 	params.gas = U256::from(100_000);
-	params.code = Some(code);
+	params.code = Some(Arc::new(code));
 	let mut ext = FakeExt::new();
 
 	let gas_left = {
@@ -329,9 +329,9 @@ fn test_extcodecopy(factory: super::Factory) {
 	params.address = address.clone();
 	params.sender = sender.clone();
 	params.gas = U256::from(100_000);
-	params.code = Some(code);
+	params.code = Some(Arc::new(code));
 	let mut ext = FakeExt::new();
-	ext.codes.insert(sender, sender_code);
+	ext.codes.insert(sender, Arc::new(sender_code));
 
 	let gas_left = {
 		let mut vm = factory.create(params.gas);
@@ -350,7 +350,7 @@ fn test_log_empty(factory: super::Factory) {
 	let mut params = ActionParams::default();
 	params.address = address.clone();
 	params.gas = U256::from(100_000);
-	params.code = Some(code);
+	params.code = Some(Arc::new(code));
 	let mut ext = FakeExt::new();
 
 	let gas_left = {
@@ -382,7 +382,7 @@ fn test_log_sender(factory: super::Factory) {
 	params.address = address.clone();
 	params.sender = sender.clone();
 	params.gas = U256::from(100_000);
-	params.code = Some(code);
+	params.code = Some(Arc::new(code));
 	let mut ext = FakeExt::new();
 
 	let gas_left = {
@@ -406,7 +406,7 @@ fn test_blockhash(factory: super::Factory) {
 	let mut params = ActionParams::default();
 	params.address = address.clone();
 	params.gas = U256::from(100_000);
-	params.code = Some(code);
+	params.code = Some(Arc::new(code));
 	let mut ext = FakeExt::new();
 	ext.blockhashes.insert(U256::zero(), blockhash.clone());
 
@@ -428,7 +428,7 @@ fn test_calldataload(factory: super::Factory) {
 	let mut params = ActionParams::default();
 	params.address = address.clone();
 	params.gas = U256::from(100_000);
-	params.code = Some(code);
+	params.code = Some(Arc::new(code));
 	params.data = Some(data);
 	let mut ext = FakeExt::new();
 
@@ -449,7 +449,7 @@ fn test_author(factory: super::Factory) {
 
 	let mut params = ActionParams::default();
 	params.gas = U256::from(100_000);
-	params.code = Some(code);
+	params.code = Some(Arc::new(code));
 	let mut ext = FakeExt::new();
 	ext.info.author = author;
 
@@ -469,7 +469,7 @@ fn test_timestamp(factory: super::Factory) {
 
 	let mut params = ActionParams::default();
 	params.gas = U256::from(100_000);
-	params.code = Some(code);
+	params.code = Some(Arc::new(code));
 	let mut ext = FakeExt::new();
 	ext.info.timestamp = timestamp;
 
@@ -489,7 +489,7 @@ fn test_number(factory: super::Factory) {
 
 	let mut params = ActionParams::default();
 	params.gas = U256::from(100_000);
-	params.code = Some(code);
+	params.code = Some(Arc::new(code));
 	let mut ext = FakeExt::new();
 	ext.info.number = number;
 
@@ -509,7 +509,7 @@ fn test_difficulty(factory: super::Factory) {
 
 	let mut params = ActionParams::default();
 	params.gas = U256::from(100_000);
-	params.code = Some(code);
+	params.code = Some(Arc::new(code));
 	let mut ext = FakeExt::new();
 	ext.info.difficulty = difficulty;
 
@@ -529,7 +529,7 @@ fn test_gas_limit(factory: super::Factory) {
 
 	let mut params = ActionParams::default();
 	params.gas = U256::from(100_000);
-	params.code = Some(code);
+	params.code = Some(Arc::new(code));
 	let mut ext = FakeExt::new();
 	ext.info.gas_limit = gas_limit;
 
@@ -548,7 +548,7 @@ fn test_mul(factory: super::Factory) {
 
 	let mut params = ActionParams::default();
 	params.gas = U256::from(100_000);
-	params.code = Some(code);
+	params.code = Some(Arc::new(code));
 	let mut ext = FakeExt::new();
 
 	let gas_left = {
@@ -566,7 +566,7 @@ fn test_sub(factory: super::Factory) {
 
 	let mut params = ActionParams::default();
 	params.gas = U256::from(100_000);
-	params.code = Some(code);
+	params.code = Some(Arc::new(code));
 	let mut ext = FakeExt::new();
 
 	let gas_left = {
@@ -584,7 +584,7 @@ fn test_div(factory: super::Factory) {
 
 	let mut params = ActionParams::default();
 	params.gas = U256::from(100_000);
-	params.code = Some(code);
+	params.code = Some(Arc::new(code));
 	let mut ext = FakeExt::new();
 
 	let gas_left = {
@@ -602,7 +602,7 @@ fn test_div_zero(factory: super::Factory) {
 
 	let mut params = ActionParams::default();
 	params.gas = U256::from(100_000);
-	params.code = Some(code);
+	params.code = Some(Arc::new(code));
 	let mut ext = FakeExt::new();
 
 	let gas_left = {
@@ -620,7 +620,7 @@ fn test_mod(factory: super::Factory) {
 
 	let mut params = ActionParams::default();
 	params.gas = U256::from(100_000);
-	params.code = Some(code);
+	params.code = Some(Arc::new(code));
 	let mut ext = FakeExt::new();
 
 	let gas_left = {
@@ -639,7 +639,7 @@ fn test_smod(factory: super::Factory) {
 
 	let mut params = ActionParams::default();
 	params.gas = U256::from(100_000);
-	params.code = Some(code);
+	params.code = Some(Arc::new(code));
 	let mut ext = FakeExt::new();
 
 	let gas_left = {
@@ -658,7 +658,7 @@ fn test_sdiv(factory: super::Factory) {
 
 	let mut params = ActionParams::default();
 	params.gas = U256::from(100_000);
-	params.code = Some(code);
+	params.code = Some(Arc::new(code));
 	let mut ext = FakeExt::new();
 
 	let gas_left = {
@@ -677,7 +677,7 @@ fn test_exp(factory: super::Factory) {
 
 	let mut params = ActionParams::default();
 	params.gas = U256::from(100_000);
-	params.code = Some(code);
+	params.code = Some(Arc::new(code));
 	let mut ext = FakeExt::new();
 
 	let gas_left = {
@@ -697,7 +697,7 @@ fn test_comparison(factory: super::Factory) {
 
 	let mut params = ActionParams::default();
 	params.gas = U256::from(100_000);
-	params.code = Some(code);
+	params.code = Some(Arc::new(code));
 	let mut ext = FakeExt::new();
 
 	let gas_left = {
@@ -718,7 +718,7 @@ fn test_signed_comparison(factory: super::Factory) {
 
 	let mut params = ActionParams::default();
 	params.gas = U256::from(100_000);
-	params.code = Some(code);
+	params.code = Some(Arc::new(code));
 	let mut ext = FakeExt::new();
 
 	let gas_left = {
@@ -739,7 +739,7 @@ fn test_bitops(factory: super::Factory) {
 
 	let mut params = ActionParams::default();
 	params.gas = U256::from(150_000);
-	params.code = Some(code);
+	params.code = Some(Arc::new(code));
 	let mut ext = FakeExt::new();
 
 	let gas_left = {
@@ -762,7 +762,7 @@ fn test_addmod_mulmod(factory: super::Factory) {
 
 	let mut params = ActionParams::default();
 	params.gas = U256::from(100_000);
-	params.code = Some(code);
+	params.code = Some(Arc::new(code));
 	let mut ext = FakeExt::new();
 
 	let gas_left = {
@@ -783,7 +783,7 @@ fn test_byte(factory: super::Factory) {
 
 	let mut params = ActionParams::default();
 	params.gas = U256::from(100_000);
-	params.code = Some(code);
+	params.code = Some(Arc::new(code));
 	let mut ext = FakeExt::new();
 
 	let gas_left = {
@@ -802,7 +802,7 @@ fn test_signextend(factory: super::Factory) {
 
 	let mut params = ActionParams::default();
 	params.gas = U256::from(100_000);
-	params.code = Some(code);
+	params.code = Some(Arc::new(code));
 	let mut ext = FakeExt::new();
 
 	let gas_left = {
@@ -817,12 +817,12 @@ fn test_signextend(factory: super::Factory) {
 
 #[test] // JIT just returns out of gas
 fn test_badinstruction_int() {
-	let factory = super::Factory::new(VMType::Interpreter);
+	let factory = super::Factory::new(VMType::Interpreter, 1024 * 32);
 	let code = "af".from_hex().unwrap();
 
 	let mut params = ActionParams::default();
 	params.gas = U256::from(100_000);
-	params.code = Some(code);
+	params.code = Some(Arc::new(code));
 	let mut ext = FakeExt::new();
 
 	let err = {
@@ -842,7 +842,7 @@ fn test_pop(factory: super::Factory) {
 
 	let mut params = ActionParams::default();
 	params.gas = U256::from(100_000);
-	params.code = Some(code);
+	params.code = Some(Arc::new(code));
 	let mut ext = FakeExt::new();
 
 	let gas_left = {
@@ -862,7 +862,7 @@ fn test_extops(factory: super::Factory) {
 	params.gas = U256::from(150_000);
 	params.gas_price = U256::from(0x32);
 	params.value = ActionValue::Transfer(U256::from(0x99));
-	params.code = Some(code);
+	params.code = Some(Arc::new(code));
 	let mut ext = FakeExt::new();
 
 	let gas_left = {
@@ -885,7 +885,7 @@ fn test_jumps(factory: super::Factory) {
 
 	let mut params = ActionParams::default();
 	params.gas = U256::from(150_000);
-	params.code = Some(code);
+	params.code = Some(Arc::new(code));
 	let mut ext = FakeExt::new();
 
 	let gas_left = {
@@ -908,7 +908,7 @@ fn test_calls(factory: super::Factory) {
 	let code_address = Address::from(0x998);
 	let mut params = ActionParams::default();
 	params.gas = U256::from(150_000);
-	params.code = Some(code);
+	params.code = Some(Arc::new(code));
 	params.address = address.clone();
 	let mut ext = FakeExt::new();
 	ext.balances = {

ethcore/src/executive.rs

@@ -25,10 +25,10 @@ use trace::{FlatTrace, Tracer, NoopTracer, ExecutiveTracer, VMTrace, VMTracer, E
 use crossbeam;
 pub use types::executed::{Executed, ExecutionResult};
 
-/// Max depth to avoid stack overflow (when it's reached we start a new thread with VM)
+/// Roughly estimate what stack size each level of evm depth will use
 /// TODO [todr] We probably need some more sophisticated calculations here (limit on my machine 132)
 /// Maybe something like here: `https://github.com/ethereum/libethereum/blob/4db169b8504f2b87f7d5a481819cfb959fc65f6c/libethereum/ExtVM.cpp`
-const MAX_VM_DEPTH_FOR_THREAD: usize = 64;
+const STACK_SIZE_PER_DEPTH: usize = 24*1024;
 
 /// Returns new address created from address and given nonce.
 pub fn contract_address(address: &Address, nonce: &U256) -> Address {
@@ -149,12 +149,13 @@ impl<'a> Executive<'a> {
 
 		// TODO: we might need bigints here, or at least check overflows.
 		let balance = self.state.balance(&sender);
-		let gas_cost = U512::from(t.gas) * U512::from(t.gas_price);
+		let gas_cost = t.gas.full_mul(t.gas_price);
 		let total_cost = U512::from(t.value) + gas_cost;
 
 		// avoid unaffordable transactions
-		if U512::from(balance) < total_cost {
+		let balance512 = U512::from(balance);
-			return Err(From::from(ExecutionError::NotEnoughCash { required: total_cost, got: U512::from(balance) }));
+		if balance512 < total_cost {
+			return Err(From::from(ExecutionError::NotEnoughCash { required: total_cost, got: balance512 }));
 		}
 
 		// NOTE: there can be no invalid transactions from this point.
@@ -168,13 +169,14 @@ impl<'a> Executive<'a> {
 				let new_address = contract_address(&sender, &nonce);
 				let params = ActionParams {
 					code_address: new_address.clone(),
+					code_hash: t.data.sha3(),
 					address: new_address,
 					sender: sender.clone(),
 					origin: sender.clone(),
 					gas: init_gas,
 					gas_price: t.gas_price,
 					value: ActionValue::Transfer(t.value),
-					code: Some(t.data.clone()),
+					code: Some(Arc::new(t.data.clone())),
 					data: None,
 					call_type: CallType::None,
 				};
@@ -190,6 +192,7 @@ impl<'a> Executive<'a> {
 					gas_price: t.gas_price,
 					value: ActionValue::Transfer(t.value),
 					code: self.state.code(address),
+					code_hash: self.state.code_hash(address),
 					data: Some(t.data.clone()),
 					call_type: CallType::Call,
 				};
@@ -210,8 +213,11 @@ impl<'a> Executive<'a> {
 		tracer: &mut T,
 		vm_tracer: &mut V
 	) -> evm::Result<U256> where T: Tracer, V: VMTracer {
+
+		let depth_threshold = ::io::LOCAL_STACK_SIZE.with(|sz| sz.get() / STACK_SIZE_PER_DEPTH);
+
 		// Ordinary execution - keep VM in same thread
-		if (self.depth + 1) % MAX_VM_DEPTH_FOR_THREAD != 0 {
+		if (self.depth + 1) % depth_threshold != 0 {
 			let vm_factory = self.vm_factory;
 			let mut ext = self.as_externalities(OriginInfo::from(&params), unconfirmed_substate, output_policy, tracer, vm_tracer);
 			trace!(target: "executive", "ext.schedule.have_delegate_call: {}", ext.schedule().have_delegate_call);
@@ -263,7 +269,7 @@ impl<'a> Executive<'a> {
 			let cost = self.engine.cost_of_builtin(&params.code_address, data);
 			if cost <= params.gas {
 				self.engine.execute_builtin(&params.code_address, data, &mut output);
-				self.state.clear_snapshot();
+				self.state.discard_snapshot();
 
 				// trace only top level calls to builtins to avoid DDoS attacks
 				if self.depth == 0 {
@@ -283,7 +289,7 @@ impl<'a> Executive<'a> {
 				Ok(params.gas - cost)
 			} else {
 				// just drain the whole gas
-				self.state.revert_snapshot();
+				self.state.revert_to_snapshot();
 
 				tracer.trace_failed_call(trace_info, vec![], evm::Error::OutOfGas.into());
 
@@ -329,7 +335,7 @@ impl<'a> Executive<'a> {
 				res
 			} else {
 				// otherwise it's just a basic transaction, only do tracing, if necessary.
-				self.state.clear_snapshot();
+				self.state.discard_snapshot();
 
 				tracer.trace_call(trace_info, U256::zero(), trace_output, vec![]);
 				Ok(params.gas)
@@ -411,7 +417,7 @@ impl<'a> Executive<'a> {
 
 		// real ammount to refund
 		let gas_left_prerefund = match result { Ok(x) => x, _ => 0.into() };
-		let refunded = cmp::min(refunds_bound, (t.gas - gas_left_prerefund) / U256::from(2));
+		let refunded = cmp::min(refunds_bound, (t.gas - gas_left_prerefund) >> 1);
 		let gas_left = gas_left_prerefund + refunded;
 
 		let gas_used = t.gas - gas_left;
@@ -471,10 +477,10 @@ impl<'a> Executive<'a> {
 				| Err(evm::Error::BadInstruction {.. })
 				| Err(evm::Error::StackUnderflow {..})
 				| Err(evm::Error::OutOfStack {..}) => {
-					self.state.revert_snapshot();
+					self.state.revert_to_snapshot();
 			},
 			Ok(_) | Err(evm::Error::Internal) => {
-				self.state.clear_snapshot();
+				self.state.discard_snapshot();
 				substate.accrue(un_substate);
 			}
 		}
@@ -511,7 +517,7 @@ mod tests {
 		params.address = address.clone();
 		params.sender = sender.clone();
 		params.gas = U256::from(100_000);
-		params.code = Some("3331600055".from_hex().unwrap());
+		params.code = Some(Arc::new("3331600055".from_hex().unwrap()));
 		params.value = ActionValue::Transfer(U256::from(0x7));
 		let mut state_result = get_temp_state();
 		let mut state = state_result.reference_mut();
@@ -570,7 +576,7 @@ mod tests {
 		params.sender = sender.clone();
 		params.origin = sender.clone();
 		params.gas = U256::from(100_000);
-		params.code = Some(code.clone());
+		params.code = Some(Arc::new(code));
 		params.value = ActionValue::Transfer(U256::from(100));
 		let mut state_result = get_temp_state();
 		let mut state = state_result.reference_mut();
@@ -589,8 +595,11 @@ mod tests {
 		assert_eq!(substate.contracts_created.len(), 0);
 	}
 
-	evm_test!{test_call_to_create: test_call_to_create_jit, test_call_to_create_int}
+	#[test]
-	fn test_call_to_create(factory: Factory) {
+	// Tracing is not suported in JIT
+	fn test_call_to_create() {
+		let factory = Factory::new(VMType::Interpreter, 1024 * 32);
+
 		// code:
 		//
 		// 7c 601080600c6000396000f3006000355415600957005b60203560003555 - push 29 bytes?
@@ -625,7 +634,7 @@ mod tests {
 		params.sender = sender.clone();
 		params.origin = sender.clone();
 		params.gas = U256::from(100_000);
-		params.code = Some(code.clone());
+		params.code = Some(Arc::new(code));
 		params.value = ActionValue::Transfer(U256::from(100));
 		params.call_type = CallType::Call;
 		let mut state_result = get_temp_state();
@@ -712,8 +721,10 @@ mod tests {
 		assert_eq!(vm_tracer.drain().unwrap(), expected_vm_trace);
 	}
 
-	evm_test!{test_create_contract: test_create_contract_jit, test_create_contract_int}
+	#[test]
-	fn test_create_contract(factory: Factory) {
+	fn test_create_contract() {
+		// Tracing is not supported in JIT
+		let factory = Factory::new(VMType::Interpreter, 1024 * 32);
 		// code:
 		//
 		// 60 10 - push 16
@@ -735,7 +746,7 @@ mod tests {
 		params.sender = sender.clone();
 		params.origin = sender.clone();
 		params.gas = U256::from(100_000);
-		params.code = Some(code.clone());
+		params.code = Some(Arc::new(code));
 		params.value = ActionValue::Transfer(100.into());
 		let mut state_result = get_temp_state();
 		let mut state = state_result.reference_mut();
@@ -823,7 +834,7 @@ mod tests {
 		params.sender = sender.clone();
 		params.origin = sender.clone();
 		params.gas = U256::from(100_000);
-		params.code = Some(code.clone());
+		params.code = Some(Arc::new(code));
 		params.value = ActionValue::Transfer(U256::from(100));
 		let mut state_result = get_temp_state();
 		let mut state = state_result.reference_mut();
@@ -875,7 +886,7 @@ mod tests {
 		params.sender = sender.clone();
 		params.origin = sender.clone();
 		params.gas = U256::from(100_000);
-		params.code = Some(code.clone());
+		params.code = Some(Arc::new(code));
 		params.value = ActionValue::Transfer(U256::from(100));
 		let mut state_result = get_temp_state();
 		let mut state = state_result.reference_mut();
@@ -932,7 +943,7 @@ mod tests {
 		params.address = address_a.clone();
 		params.sender = sender.clone();
 		params.gas = U256::from(100_000);
-		params.code = Some(code_a.clone());
+		params.code = Some(Arc::new(code_a.clone()));
 		params.value = ActionValue::Transfer(U256::from(100_000));
 
 		let mut state_result = get_temp_state();
@@ -982,10 +993,10 @@ mod tests {
 		let mut params = ActionParams::default();
 		params.address = address.clone();
 		params.gas = U256::from(100_000);
-		params.code = Some(code.clone());
+		params.code = Some(Arc::new(code.clone()));
 		let mut state_result = get_temp_state();
 		let mut state = state_result.reference_mut();
-		state.init_code(&address, code.clone());
+		state.init_code(&address, code);
 		let info = EnvInfo::default();
 		let engine = TestEngine::new(0);
 		let mut substate = Substate::new();
@@ -1183,7 +1194,7 @@ mod tests {
 		params.sender = sender.clone();
 		params.origin = sender.clone();
 		params.gas = U256::from(0x0186a0);
-		params.code = Some(code.clone());
+		params.code = Some(Arc::new(code));
 		params.value = ActionValue::Transfer(U256::from_str("0de0b6b3a7640000").unwrap());
 		let mut state_result = get_temp_state();
 		let mut state = state_result.reference_mut();

ethcore/src/externalities.rs

@@ -146,7 +146,8 @@ impl<'a, T, V> Ext for Externalities<'a, T, V> where T: 'a + Tracer, V: 'a + VMT
 			gas: *gas,
 			gas_price: self.origin_info.gas_price,
 			value: ActionValue::Transfer(*value),
-			code: Some(code.to_vec()),
+			code: Some(Arc::new(code.to_vec())),
+			code_hash: code.sha3(),
 			data: None,
 			call_type: CallType::None,
 		};
@@ -185,6 +186,7 @@ impl<'a, T, V> Ext for Externalities<'a, T, V> where T: 'a + Tracer, V: 'a + VMT
 			gas: *gas,
 			gas_price: self.origin_info.gas_price,
 			code: self.state.code(code_address),
+			code_hash: self.state.code_hash(code_address),
 			data: Some(data.to_vec()),
 			call_type: call_type,
 		};
@@ -201,15 +203,14 @@ impl<'a, T, V> Ext for Externalities<'a, T, V> where T: 'a + Tracer, V: 'a + VMT
 		}
 	}
 
-	fn extcode(&self, address: &Address) -> Bytes {
+	fn extcode(&self, address: &Address) -> Arc<Bytes> {
-		self.state.code(address).unwrap_or_else(|| vec![])
+		self.state.code(address).unwrap_or_else(|| Arc::new(vec![]))
 	}
 
 	fn extcodesize(&self, address: &Address) -> usize {
 		self.state.code_size(address).unwrap_or(0)
 	}
 
-
 	#[cfg_attr(feature="dev", allow(match_ref_pats))]
 	fn ret(mut self, gas: &U256, data: &[u8]) -> evm::Result<U256>
 		where Self: Sized {

ethcore/src/header.rs

@@ -295,6 +295,12 @@ impl Encodable for Header {
 	}
 }
 
+impl HeapSizeOf for Header {
+	fn heap_size_of_children(&self) -> usize {
+		self.extra_data.heap_size_of_children() + self.seal.heap_size_of_children()
+	}
+}
+
 #[cfg(test)]
 mod tests {
 	use rustc_serialize::hex::FromHex;

ethcore/src/json_tests/executive.rs

@@ -127,7 +127,7 @@ impl<'a, T, V> Ext for TestExt<'a, T, V> where T: Tracer, V: VMTracer {
 		MessageCallResult::Success(*gas)
 	}
 
-	fn extcode(&self, address: &Address) -> Bytes  {
+	fn extcode(&self, address: &Address) -> Arc<Bytes>  {
 		self.ext.extcode(address)
 	}
 
@@ -191,7 +191,7 @@ fn do_json_test_for(vm_type: &VMType, json_data: &[u8]) -> Vec<String> {
 		state.populate_from(From::from(vm.pre_state.clone()));
 		let info = From::from(vm.env);
 		let engine = TestEngine::new(1);
-		let vm_factory = Factory::new(vm_type.clone());
+		let vm_factory = Factory::new(vm_type.clone(), 1024 * 32);
 		let params = ActionParams::from(vm.transaction);
 
 		let mut substate = Substate::new();
@@ -232,7 +232,7 @@ fn do_json_test_for(vm_type: &VMType, json_data: &[u8]) -> Vec<String> {
 				for (address, account) in vm.post_state.unwrap().into_iter() {
 					let address = address.into();
 					let code: Vec<u8> = account.code.into();
-					fail_unless(state.code(&address).unwrap_or_else(Vec::new) == code, "code is incorrect");
+					fail_unless(state.code(&address).as_ref().map_or_else(|| code.is_empty(), |c| &**c == &code), "code is incorrect");
 					fail_unless(state.balance(&address) == account.balance.into(), "balance is incorrect");
 					fail_unless(state.nonce(&address) == account.nonce.into(), "nonce is incorrect");
 					account.storage.into_iter().foreach(|(k, v)| {

ethcore/src/json_tests/homestead_chain.rs

@@ -36,3 +36,7 @@ declare_test!{BlockchainTests_Homestead_bcUncleHeaderValiditiy, "BlockchainTests
 declare_test!{BlockchainTests_Homestead_bcUncleTest, "BlockchainTests/Homestead/bcUncleTest"}
 declare_test!{BlockchainTests_Homestead_bcValidBlockTest, "BlockchainTests/Homestead/bcValidBlockTest"}
 declare_test!{BlockchainTests_Homestead_bcWalletTest, "BlockchainTests/Homestead/bcWalletTest"}
+declare_test!{BlockchainTests_Homestead_bcShanghaiLove, "BlockchainTests/Homestead/bcShanghaiLove"}
+// Uncomment once the test is correct.
+// declare_test!{BlockchainTests_Homestead_bcSuicideIssue, "BlockchainTests/Homestead/bcSuicideIssue"}
+declare_test!{BlockchainTests_Homestead_bcExploitTest, "BlockchainTests/Homestead/bcExploitTest"}

ethcore/src/json_tests/state.rs

@@ -96,7 +96,6 @@ pub fn json_chain_test(json_data: &[u8], era: ChainEra) -> Vec<String> {
 declare_test!{StateTests_stBlockHashTest, "StateTests/stBlockHashTest"}
 declare_test!{StateTests_stCallCodes, "StateTests/stCallCodes"}
 declare_test!{StateTests_stCallCreateCallCodeTest, "StateTests/stCallCreateCallCodeTest"}
-declare_test!{StateTests_stDelegatecallTest, "StateTests/stDelegatecallTest"}
 declare_test!{StateTests_stExample, "StateTests/stExample"}
 declare_test!{StateTests_stInitCodeTest, "StateTests/stInitCodeTest"}
 declare_test!{StateTests_stLogTests, "StateTests/stLogTests"}

ethcore/src/lib.rs

@@ -99,6 +99,8 @@ extern crate ethcore_devtools as devtools;
 extern crate rand;
 extern crate bit_set;
 extern crate rlp;
+extern crate ethcore_bloom_journal as bloom_journal;
+extern crate byteorder;
 
 #[macro_use]
 extern crate log;
@@ -110,6 +112,7 @@ extern crate lazy_static;
 extern crate heapsize;
 #[macro_use]
 extern crate ethcore_ipc as ipc;
+extern crate lru_cache;
 
 #[cfg(feature = "jit" )]
 extern crate evmjit;
@@ -119,7 +122,6 @@ pub extern crate ethstore;
 pub mod account_provider;
 pub mod engines;
 pub mod block;
-pub mod block_queue;
 pub mod client;
 pub mod error;
 pub mod ethereum;
@@ -143,6 +145,7 @@ mod basic_types;
 mod env_info;
 mod pod_account;
 mod state;
+mod state_db;
 mod account_db;
 mod builtin;
 mod executive;

ethcore/src/migrations/mod.rs

@@ -23,3 +23,6 @@ pub mod extras;
 mod v9;
 pub use self::v9::ToV9;
 pub use self::v9::Extract;
+
+mod v10;
+pub use self::v10::ToV10;

ethcore/src/migrations/state/v7.rs

@@ -24,6 +24,7 @@ use util::{Address, FixedHash, H256};
 use util::kvdb::Database;
 use util::migration::{Batch, Config, Error, Migration, SimpleMigration, Progress};
 use util::sha3::Hashable;
+use std::sync::Arc;
 
 use rlp::{decode, Rlp, RlpStream, Stream, View};
 
@@ -107,7 +108,7 @@ pub struct OverlayRecentV7 {
 impl OverlayRecentV7 {
 	// walk all journal entries in the database backwards.
 	// find migrations for any possible inserted keys.
-	fn walk_journal(&mut self, source: &Database) -> Result<(), Error> {
+	fn walk_journal(&mut self, source: Arc<Database>) -> Result<(), Error> {
 		if let Some(val) = try!(source.get(None, V7_LATEST_ERA_KEY).map_err(Error::Custom)) {
 			let mut era = decode::<u64>(&val);
 			loop {
@@ -151,7 +152,7 @@ impl OverlayRecentV7 {
 	// walk all journal entries in the database backwards.
 	// replace all possible inserted/deleted keys with their migrated counterparts
 	// and commit the altered entries.
-	fn migrate_journal(&self, source: &Database, mut batch: Batch, dest: &mut Database) -> Result<(), Error> {
+	fn migrate_journal(&self, source: Arc<Database>, mut batch: Batch, dest: &mut Database) -> Result<(), Error> {
 		if let Some(val) = try!(source.get(None, V7_LATEST_ERA_KEY).map_err(Error::Custom)) {
 			try!(batch.insert(V7_LATEST_ERA_KEY.into(), val.to_owned(), dest));
 
@@ -228,7 +229,7 @@ impl Migration for OverlayRecentV7 {
 	// walk all records in the database, attempting to migrate any possible and
 	// keeping records of those that we do. then migrate the journal using
 	// this information.
-	fn migrate(&mut self, source: &Database, config: &Config, dest: &mut Database, col: Option<u32>) -> Result<(), Error> {
+	fn migrate(&mut self, source: Arc<Database>, config: &Config, dest: &mut Database, col: Option<u32>) -> Result<(), Error> {
 		let mut batch = Batch::new(config, col);
 
 		// check version metadata.
@@ -257,7 +258,7 @@ impl Migration for OverlayRecentV7 {
 			try!(batch.insert(key, value.into_vec(), dest));
 		}
 
-		try!(self.walk_journal(source));
+		try!(self.walk_journal(source.clone()));
 		self.migrate_journal(source, batch, dest)
 	}
 }

ethcore/src/migrations/v10.rs

@@ -0,0 +1,117 @@
+// Copyright 2015, 2016 Ethcore (UK) Ltd.
+// This file is part of Parity.
+
+// Parity is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Parity is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Parity.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Bloom upgrade
+
+use std::sync::Arc;
+use db::{COL_EXTRA, COL_HEADERS, COL_STATE};
+use state_db::{ACCOUNT_BLOOM_SPACE, DEFAULT_ACCOUNT_PRESET, StateDB};
+use util::trie::TrieDB;
+use views::HeaderView;
+use bloom_journal::Bloom;
+use util::migration::{Error, Migration, Progress, Batch, Config};
+use util::journaldb;
+use util::{H256, FixedHash, Trie};
+use util::{Database, DBTransaction};
+
+/// Account bloom upgrade routine. If bloom already present, does nothing.
+/// If database empty (no best block), does nothing.
+/// Can be called on upgraded database with no issues (will do nothing).
+pub fn generate_bloom(source: Arc<Database>, dest: &mut Database) -> Result<(), Error> {
+	trace!(target: "migration", "Account bloom upgrade started");
+	let best_block_hash = match try!(source.get(COL_EXTRA, b"best")) {
+		// no migration needed
+		None => {
+			trace!(target: "migration", "No best block hash, skipping");
+			return Ok(());
+		},
+		Some(hash) => hash,
+	};
+	let best_block_header = match try!(source.get(COL_HEADERS, &best_block_hash)) {
+		// no best block, nothing to do
+		None => {
+			trace!(target: "migration", "No best block header, skipping");
+			return Ok(())
+		},
+		Some(x) => x,
+	};
+	let state_root = HeaderView::new(&best_block_header).state_root();
+
+	trace!("Adding accounts bloom (one-time upgrade)");
+	let bloom_journal = {
+		let mut bloom = Bloom::new(ACCOUNT_BLOOM_SPACE, DEFAULT_ACCOUNT_PRESET);
+		// no difference what algorithm is passed, since there will be no writes
+		let state_db = journaldb::new(
+			source.clone(),
+			journaldb::Algorithm::OverlayRecent,
+			COL_STATE);
+		let account_trie = try!(TrieDB::new(state_db.as_hashdb(), &state_root).map_err(|e| Error::Custom(format!("Cannot open trie: {:?}", e))));
+		for item in try!(account_trie.iter().map_err(|_| Error::MigrationImpossible)) {
+			let (ref account_key, _) = try!(item.map_err(|_| Error::MigrationImpossible));
+			let account_key_hash = H256::from_slice(&account_key);
+			bloom.set(&*account_key_hash);
+		}
+
+		bloom.drain_journal()
+	};
+
+	trace!(target: "migration", "Generated {} bloom updates", bloom_journal.entries.len());
+
+	let mut batch = DBTransaction::new(dest);
+	try!(StateDB::commit_bloom(&mut batch, bloom_journal).map_err(|_| Error::Custom("Failed to commit bloom".to_owned())));
+	try!(dest.write(batch));
+
+	trace!(target: "migration", "Finished bloom update");
+
+
+	Ok(())
+}
+
+/// Account bloom migration.
+#[derive(Default)]
+pub struct ToV10 {
+	progress: Progress,
+}
+
+impl ToV10 {
+	/// New v10 migration
+	pub fn new() -> ToV10 { ToV10 { progress: Progress::default() } }
+}
+
+impl Migration for ToV10 {
+	fn version(&self) -> u32 {
+		10
+	}
+
+	fn pre_columns(&self) -> Option<u32> { Some(5) }
+
+	fn columns(&self) -> Option<u32> { Some(6) }
+
+	fn migrate(&mut self, source: Arc<Database>, config: &Config, dest: &mut Database, col: Option<u32>) -> Result<(), Error> {
+		let mut batch = Batch::new(config, col);
+		for (key, value) in source.iter(col) {
+			self.progress.tick();
+			try!(batch.insert(key.to_vec(), value.to_vec(), dest));
+		}
+		try!(batch.commit(dest));
+
+		if col == COL_STATE {
+			try!(generate_bloom(source, dest));
+		}
+
+		Ok(())
+	}
+}

ethcore/src/migrations/v9.rs

@@ -20,6 +20,7 @@
 use rlp::{Rlp, RlpStream, View, Stream};
 use util::kvdb::Database;
 use util::migration::{Batch, Config, Error, Migration, Progress};
+use std::sync::Arc;
 
 /// Which part of block to preserve
 pub enum Extract {
@@ -55,7 +56,7 @@ impl Migration for ToV9 {
 
 	fn version(&self) -> u32 { 9 }
 
-	fn migrate(&mut self, source: &Database, config: &Config, dest: &mut Database, col: Option<u32>) -> Result<(), Error> {
+	fn migrate(&mut self, source: Arc<Database>, config: &Config, dest: &mut Database, col: Option<u32>) -> Result<(), Error> {
 		let mut batch = Batch::new(config, self.column);
 
 		for (key, value) in source.iter(col) {

ethcore/src/miner/miner.rs

@@ -48,6 +48,17 @@ pub enum PendingSet {
 	SealingOrElseQueue,
 }
 
+/// Type of the gas limit to apply to the transaction queue.
+#[derive(Debug, PartialEq)]
+pub enum GasLimit {
+	/// Depends on the block gas limit and is updated with every block.
+	Auto,
+	/// No limit.
+	None,
+	/// Set to a fixed gas value.
+	Fixed(U256),
+}
+
 /// Configures the behaviour of the miner.
 #[derive(Debug, PartialEq)]
 pub struct MinerOptions {
@@ -71,6 +82,8 @@ pub struct MinerOptions {
 	pub work_queue_size: usize,
 	/// Can we submit two different solutions for the same block and expect both to result in an import?
 	pub enable_resubmission: bool,
+	/// Global gas limit for all transaction in the queue except for local and retracted.
+	pub tx_queue_gas_limit: GasLimit,
 }
 
 impl Default for MinerOptions {
@@ -81,11 +94,12 @@ impl Default for MinerOptions {
 			reseal_on_external_tx: false,
 			reseal_on_own_tx: true,
 			tx_gas_limit: !U256::zero(),
-			tx_queue_size: 1024,
+			tx_queue_size: 2048,
 			pending_set: PendingSet::AlwaysQueue,
 			reseal_min_period: Duration::from_secs(2),
 			work_queue_size: 20,
 			enable_resubmission: true,
+			tx_queue_gas_limit: GasLimit::Auto,
 		}
 	}
 }
@@ -194,7 +208,11 @@ impl Miner {
 			true => None,
 			false => Some(WorkPoster::new(&options.new_work_notify))
 		};
-		let txq = Arc::new(Mutex::new(TransactionQueue::with_limits(options.tx_queue_size, options.tx_gas_limit)));
+		let gas_limit = match options.tx_queue_gas_limit {
+			GasLimit::Fixed(ref limit) => *limit,
+			_ => !U256::zero(),
+		};
+		let txq = Arc::new(Mutex::new(TransactionQueue::with_limits(options.tx_queue_size, gas_limit, options.tx_gas_limit)));
 		Miner {
 			transaction_queue: txq,
 			next_allowed_reseal: Mutex::new(Instant::now()),
@@ -292,6 +310,7 @@ impl Miner {
 		};
 
 		let mut invalid_transactions = HashSet::new();
+		let mut transactions_to_penalize = HashSet::new();
 		let block_number = open_block.block().fields().header.number();
 		// TODO: push new uncles, too.
 		for tx in transactions {
@@ -299,6 +318,12 @@ impl Miner {
 			match open_block.push_transaction(tx, None) {
 				Err(Error::Execution(ExecutionError::BlockGasLimitReached { gas_limit, gas_used, gas })) => {
 					debug!(target: "miner", "Skipping adding transaction to block because of gas limit: {:?} (limit: {:?}, used: {:?}, gas: {:?})", hash, gas_limit, gas_used, gas);
+
+					// Penalize transaction if it's above current gas limit
+					if gas > gas_limit {
+						transactions_to_penalize.insert(hash);
+					}
+
 					// Exit early if gas left is smaller then min_tx_gas
 					let min_tx_gas: U256 = 21000.into();	// TODO: figure this out properly.
 					if gas_limit - gas_used < min_tx_gas {
@@ -334,6 +359,9 @@ impl Miner {
 			for hash in invalid_transactions.into_iter() {
 				queue.remove_invalid(&hash, &fetch_account);
 			}
+			for hash in transactions_to_penalize {
+				queue.penalize(&hash);
+			}
 		}
 		(block, original_work_hash)
 	}
@@ -433,6 +461,10 @@ impl Miner {
 		let gas_limit = HeaderView::new(&chain.best_block_header()).gas_limit();
 		let mut queue = self.transaction_queue.lock();
 		queue.set_gas_limit(gas_limit);
+		if let GasLimit::Auto = self.options.tx_queue_gas_limit {
+			// Set total tx queue gas limit to be 2x the block gas limit.
+			queue.set_total_gas_limit(gas_limit << 1);
+		}
 	}
 
 	/// Returns true if we had to prepare new pending block.
@@ -483,6 +515,21 @@ impl Miner {
 
 	/// Are we allowed to do a non-mandatory reseal?
 	fn tx_reseal_allowed(&self) -> bool { Instant::now() > *self.next_allowed_reseal.lock() }
+
+	fn from_pending_block<H, F, G>(&self, latest_block_number: BlockNumber, from_chain: F, map_block: G) -> H
+		where F: Fn() -> H, G: Fn(&ClosedBlock) -> H {
+		let sealing_work = self.sealing_work.lock();
+		sealing_work.queue.peek_last_ref().map_or_else(
+			|| from_chain(),
+			|b| {
+				if b.block().header().number() > latest_block_number {
+					map_block(b)
+				} else {
+					from_chain()
+				}
+			}
+		)
+	}
 }
 
 const SEALING_TIMEOUT_IN_BLOCKS : u64 = 5;
@@ -555,29 +602,35 @@ impl MinerService for Miner {
 	}
 
 	fn balance(&self, chain: &MiningBlockChainClient, address: &Address) -> U256 {
-		let sealing_work = self.sealing_work.lock();
+		self.from_pending_block(
-		sealing_work.queue.peek_last_ref().map_or_else(
+			chain.chain_info().best_block_number,
 			|| chain.latest_balance(address),
 			|b| b.block().fields().state.balance(address)
 		)
 	}
 
 	fn storage_at(&self, chain: &MiningBlockChainClient, address: &Address, position: &H256) -> H256 {
-		let sealing_work = self.sealing_work.lock();
+		self.from_pending_block(
-		sealing_work.queue.peek_last_ref().map_or_else(
+			chain.chain_info().best_block_number,
 			|| chain.latest_storage_at(address, position),
 			|b| b.block().fields().state.storage_at(address, position)
 		)
 	}
 
 	fn nonce(&self, chain: &MiningBlockChainClient, address: &Address) -> U256 {
-		let sealing_work = self.sealing_work.lock();
+		self.from_pending_block(
-		sealing_work.queue.peek_last_ref().map_or_else(|| chain.latest_nonce(address), |b| b.block().fields().state.nonce(address))
+			chain.chain_info().best_block_number,
+			|| chain.latest_nonce(address),
+			|b| b.block().fields().state.nonce(address)
+		)
 	}
 
 	fn code(&self, chain: &MiningBlockChainClient, address: &Address) -> Option<Bytes> {
-		let sealing_work = self.sealing_work.lock();
+		self.from_pending_block(
-		sealing_work.queue.peek_last_ref().map_or_else(|| chain.latest_code(address), |b| b.block().fields().state.code(address))
+			chain.chain_info().best_block_number,
+			|| chain.latest_code(address),
+			|b| b.block().fields().state.code(address).map(|c| (*c).clone())
+		)
 	}
 
 	fn set_author(&self, author: Address) {
@@ -727,50 +780,74 @@ impl MinerService for Miner {
 		queue.top_transactions()
 	}
 
-	fn pending_transactions(&self) -> Vec<SignedTransaction> {
+	fn pending_transactions(&self, best_block: BlockNumber) -> Vec<SignedTransaction> {
 		let queue = self.transaction_queue.lock();
-		let sw = self.sealing_work.lock();
+		match self.options.pending_set {
-		// TODO: should only use the sealing_work when it's current (it could be an old block)
+			PendingSet::AlwaysQueue => queue.top_transactions(),
-		let sealing_set = match sw.enabled {
+			PendingSet::SealingOrElseQueue => {
-			true => sw.queue.peek_last_ref(),
+				self.from_pending_block(
-			false => None,
+					best_block,
-		};
+					|| queue.top_transactions(),
-		match (&self.options.pending_set, sealing_set) {
+					|sealing| sealing.transactions().to_owned()
-			(&PendingSet::AlwaysQueue, _) | (&PendingSet::SealingOrElseQueue, None) => queue.top_transactions(),
+				)
-			(_, sealing) => sealing.map_or_else(Vec::new, |s| s.transactions().to_owned()),
+			},
+			PendingSet::AlwaysSealing => {
+				self.from_pending_block(
+					best_block,
+					|| vec![],
+					|sealing| sealing.transactions().to_owned()
+				)
+			},
 		}
 	}
 
-	fn pending_transactions_hashes(&self) -> Vec<H256> {
+	fn pending_transactions_hashes(&self, best_block: BlockNumber) -> Vec<H256> {
 		let queue = self.transaction_queue.lock();
-		let sw = self.sealing_work.lock();
+		match self.options.pending_set {
-		let sealing_set = match sw.enabled {
+			PendingSet::AlwaysQueue => queue.pending_hashes(),
-			true => sw.queue.peek_last_ref(),
+			PendingSet::SealingOrElseQueue => {
-			false => None,
+				self.from_pending_block(
-		};
+					best_block,
-		match (&self.options.pending_set, sealing_set) {
+					|| queue.pending_hashes(),
-			(&PendingSet::AlwaysQueue, _) | (&PendingSet::SealingOrElseQueue, None) => queue.pending_hashes(),
+					|sealing| sealing.transactions().iter().map(|t| t.hash()).collect()
-			(_, sealing) => sealing.map_or_else(Vec::new, |s| s.transactions().iter().map(|t| t.hash()).collect()),
+				)
+			},
+			PendingSet::AlwaysSealing => {
+				self.from_pending_block(
+					best_block,
+					|| vec![],
+					|sealing| sealing.transactions().iter().map(|t| t.hash()).collect()
+				)
+			},
 		}
 	}
 
-	fn transaction(&self, hash: &H256) -> Option<SignedTransaction> {
+	fn transaction(&self, best_block: BlockNumber, hash: &H256) -> Option<SignedTransaction> {
 		let queue = self.transaction_queue.lock();
-		let sw = self.sealing_work.lock();
+		match self.options.pending_set {
-		let sealing_set = match sw.enabled {
+			PendingSet::AlwaysQueue => queue.find(hash),
-			true => sw.queue.peek_last_ref(),
+			PendingSet::SealingOrElseQueue => {
-			false => None,
+				self.from_pending_block(
-		};
+					best_block,
-		match (&self.options.pending_set, sealing_set) {
+					|| queue.find(hash),
-			(&PendingSet::AlwaysQueue, _) | (&PendingSet::SealingOrElseQueue, None) => queue.find(hash),
+					|sealing| sealing.transactions().iter().find(|t| &t.hash() == hash).cloned()
-			(_, sealing) => sealing.and_then(|s| s.transactions().iter().find(|t| &t.hash() == hash).cloned()),
+				)
+			},
+			PendingSet::AlwaysSealing => {
+				self.from_pending_block(
+					best_block,
+					|| None,
+					|sealing| sealing.transactions().iter().find(|t| &t.hash() == hash).cloned()
+				)
+			},
 		}
 	}
 
-	fn pending_receipt(&self, hash: &H256) -> Option<RichReceipt> {
+	fn pending_receipt(&self, best_block: BlockNumber, hash: &H256) -> Option<RichReceipt> {
-		let sealing_work = self.sealing_work.lock();
+		self.from_pending_block(
-		match (sealing_work.enabled, sealing_work.queue.peek_last_ref()) {
+			best_block,
-			(true, Some(pending)) => {
+			|| None,
+			|pending| {
 				let txs = pending.transactions();
 				txs.iter()
 					.map(|t| t.hash())
@@ -791,15 +868,15 @@ impl MinerService for Miner {
 							logs: receipt.logs.clone(),
 						}
 					})
-			},
-			_ => None
 			}
+		)
 	}
 
-	fn pending_receipts(&self) -> BTreeMap<H256, Receipt> {
+	fn pending_receipts(&self, best_block: BlockNumber) -> BTreeMap<H256, Receipt> {
-		let sealing_work = self.sealing_work.lock();
+		self.from_pending_block(
-		match (sealing_work.enabled, sealing_work.queue.peek_last_ref()) {
+			best_block,
-			(true, Some(pending)) => {
+			|| BTreeMap::new(),
+			|pending| {
 				let hashes = pending.transactions()
 					.iter()
 					.map(|t| t.hash());
@@ -807,9 +884,8 @@ impl MinerService for Miner {
 				let receipts = pending.receipts().iter().cloned();
 
 				hashes.zip(receipts).collect()
-			},
-			_ => BTreeMap::new()
 			}
+		)
 	}
 
 	fn last_nonce(&self, address: &Address) -> Option<U256> {
@@ -854,9 +930,18 @@ impl MinerService for Miner {
 	}
 
 	fn submit_seal(&self, chain: &MiningBlockChainClient, pow_hash: H256, seal: Vec<Bytes>) -> Result<(), Error> {
-		let result = if let Some(b) = self.sealing_work.lock().queue.get_used_if(if self.options.enable_resubmission { GetAction::Clone } else { GetAction::Take }, |b| &b.hash() == &pow_hash) {
+		let result =
-			b.lock().try_seal(&*self.engine, seal).or_else(|_| {
+			if let Some(b) = self.sealing_work.lock().queue.get_used_if(
-				warn!(target: "miner", "Mined solution rejected: Invalid.");
+				if self.options.enable_resubmission {
+					GetAction::Clone
+				} else {
+					GetAction::Take
+				},
+				|b| &b.hash() == &pow_hash
+			) {
+				trace!(target: "miner", "Sealing block {}={}={} with seal {:?}", pow_hash, b.hash(), b.header().bare_hash(), seal);
+				b.lock().try_seal(&*self.engine, seal).or_else(|(e, _)| {
+					warn!(target: "miner", "Mined solution rejected: {}", e);
 					Err(Error::PowInvalid)
 				})
 			} else {
@@ -905,7 +990,7 @@ impl MinerService for Miner {
 			out_of_chain.for_each(|txs| {
 				let mut transaction_queue = self.transaction_queue.lock();
 				let _ = self.add_transactions_to_queue(
-					chain, txs, TransactionOrigin::External, &mut transaction_queue
+					chain, txs, TransactionOrigin::RetractedBlock, &mut transaction_queue
 				);
 			});
 		}
@@ -997,6 +1082,7 @@ mod tests {
 				reseal_min_period: Duration::from_secs(5),
 				tx_gas_limit: !U256::zero(),
 				tx_queue_size: 1024,
+				tx_queue_gas_limit: GasLimit::None,
 				pending_set: PendingSet::AlwaysSealing,
 				work_queue_size: 5,
 				enable_resubmission: true,
@@ -1025,34 +1111,54 @@ mod tests {
 		let client = TestBlockChainClient::default();
 		let miner = miner();
 		let transaction = transaction();
+		let best_block = 0;
 		// when
 		let res = miner.import_own_transaction(&client, transaction);
 
 		// then
 		assert_eq!(res.unwrap(), TransactionImportResult::Current);
 		assert_eq!(miner.all_transactions().len(), 1);
-		assert_eq!(miner.pending_transactions().len(), 1);
+		assert_eq!(miner.pending_transactions(best_block).len(), 1);
-		assert_eq!(miner.pending_transactions_hashes().len(), 1);
+		assert_eq!(miner.pending_transactions_hashes(best_block).len(), 1);
-		assert_eq!(miner.pending_receipts().len(), 1);
+		assert_eq!(miner.pending_receipts(best_block).len(), 1);
 		// This method will let us know if pending block was created (before calling that method)
 		assert!(!miner.prepare_work_sealing(&client));
 	}
 
+	#[test]
+	fn should_not_use_pending_block_if_best_block_is_higher() {
+		// given
+		let client = TestBlockChainClient::default();
+		let miner = miner();
+		let transaction = transaction();
+		let best_block = 10;
+		// when
+		let res = miner.import_own_transaction(&client, transaction);
+
+		// then
+		assert_eq!(res.unwrap(), TransactionImportResult::Current);
+		assert_eq!(miner.all_transactions().len(), 1);
+		assert_eq!(miner.pending_transactions(best_block).len(), 0);
+		assert_eq!(miner.pending_transactions_hashes(best_block).len(), 0);
+		assert_eq!(miner.pending_receipts(best_block).len(), 0);
+	}
+
 	#[test]
 	fn should_import_external_transaction() {
 		// given
 		let client = TestBlockChainClient::default();
 		let miner = miner();
 		let transaction = transaction();
+		let best_block = 0;
 		// when
 		let res = miner.import_external_transactions(&client, vec![transaction]).pop().unwrap();
 
 		// then
 		assert_eq!(res.unwrap(), TransactionImportResult::Current);
 		assert_eq!(miner.all_transactions().len(), 1);
-		assert_eq!(miner.pending_transactions_hashes().len(), 0);
+		assert_eq!(miner.pending_transactions_hashes(best_block).len(), 0);
-		assert_eq!(miner.pending_transactions().len(), 0);
+		assert_eq!(miner.pending_transactions(best_block).len(), 0);
-		assert_eq!(miner.pending_receipts().len(), 0);
+		assert_eq!(miner.pending_receipts(best_block).len(), 0);
 		// This method will let us know if pending block was created (before calling that method)
 		assert!(miner.prepare_work_sealing(&client));
 	}

ethcore/src/miner/mod.rs

@@ -48,7 +48,7 @@ mod work_notify;
 mod price_info;
 
 pub use self::transaction_queue::{TransactionQueue, AccountDetails, TransactionOrigin};
-pub use self::miner::{Miner, MinerOptions, PendingSet, GasPricer, GasPriceCalibratorOptions};
+pub use self::miner::{Miner, MinerOptions, PendingSet, GasPricer, GasPriceCalibratorOptions, GasLimit};
 pub use self::external::{ExternalMiner, ExternalMinerService};
 pub use client::TransactionImportResult;
 
@@ -56,6 +56,7 @@ use std::collections::BTreeMap;
 use util::{H256, U256, Address, Bytes};
 use client::{MiningBlockChainClient, Executed, CallAnalytics};
 use block::ClosedBlock;
+use header::BlockNumber;
 use receipt::{RichReceipt, Receipt};
 use error::{Error, CallError};
 use transaction::SignedTransaction;
@@ -115,7 +116,7 @@ pub trait MinerService : Send + Sync {
 		Result<TransactionImportResult, Error>;
 
 	/// Returns hashes of transactions currently in pending
-	fn pending_transactions_hashes(&self) -> Vec<H256>;
+	fn pending_transactions_hashes(&self, best_block: BlockNumber) -> Vec<H256>;
 
 	/// Removes all transactions from the queue and restart mining operation.
 	fn clear_and_reset(&self, chain: &MiningBlockChainClient);
@@ -135,19 +136,19 @@ pub trait MinerService : Send + Sync {
 		where F: FnOnce(&ClosedBlock) -> T, Self: Sized;
 
 	/// Query pending transactions for hash.
-	fn transaction(&self, hash: &H256) -> Option<SignedTransaction>;
+	fn transaction(&self, best_block: BlockNumber, hash: &H256) -> Option<SignedTransaction>;
 
 	/// Get a list of all transactions.
 	fn all_transactions(&self) -> Vec<SignedTransaction>;
 
 	/// Get a list of all pending transactions.
-	fn pending_transactions(&self) -> Vec<SignedTransaction>;
+	fn pending_transactions(&self, best_block: BlockNumber) -> Vec<SignedTransaction>;
 
 	/// Get a list of all pending receipts.
-	fn pending_receipts(&self) -> BTreeMap<H256, Receipt>;
+	fn pending_receipts(&self, best_block: BlockNumber) -> BTreeMap<H256, Receipt>;
 
 	/// Get a particular reciept.
-	fn pending_receipt(&self, hash: &H256) -> Option<RichReceipt>;
+	fn pending_receipt(&self, best_block: BlockNumber, hash: &H256) -> Option<RichReceipt>;
 
 	/// Returns highest transaction nonce for given address.
 	fn last_nonce(&self, address: &Address) -> Option<U256>;

ethcore/src/miner/transaction_queue.rs

@@ -98,6 +98,8 @@ pub enum TransactionOrigin {
 	Local,
 	/// External transaction received from network
 	External,
+	/// Transactions from retracted blocks
+	RetractedBlock,
 }
 
 impl PartialOrd for TransactionOrigin {
@@ -112,10 +114,11 @@ impl Ord for TransactionOrigin {
 			return Ordering::Equal;
 		}
 
-		if *self == TransactionOrigin::Local {
+		match (*self, *other) {
-			Ordering::Less
+			(TransactionOrigin::RetractedBlock, _) => Ordering::Less,
-		} else {
+			(_, TransactionOrigin::RetractedBlock) => Ordering::Greater,
-			Ordering::Greater
+			(TransactionOrigin::Local, _) => Ordering::Less,
+			_ => Ordering::Greater,
 		}
 	}
 }
@@ -127,6 +130,8 @@ struct TransactionOrder {
 	/// (e.g. Tx(nonce:5), State(nonce:0) -> height: 5)
 	/// High nonce_height = Low priority (processed later)
 	nonce_height: U256,
+	/// Gas specified in the transaction.
+	gas: U256,
 	/// Gas Price of the transaction.
 	/// Low gas price = Low priority (processed later)
 	gas_price: U256,
@@ -134,6 +139,8 @@ struct TransactionOrder {
 	hash: H256,
 	/// Origin of the transaction
 	origin: TransactionOrigin,
+	/// Penalties
+	penalties: usize,
 }
 
 
@@ -141,9 +148,11 @@ impl TransactionOrder {
 	fn for_transaction(tx: &VerifiedTransaction, base_nonce: U256) -> Self {
 		TransactionOrder {
 			nonce_height: tx.nonce() - base_nonce,
+			gas: tx.transaction.gas.clone(),
 			gas_price: tx.transaction.gas_price,
 			hash: tx.hash(),
 			origin: tx.origin,
+			penalties: 0,
 		}
 	}
 
@@ -151,6 +160,11 @@ impl TransactionOrder {
 		self.nonce_height = nonce - base_nonce;
 		self
 	}
+
+	fn penalize(mut self) -> Self {
+		self.penalties = self.penalties.saturating_add(1);
+		self
+	}
 }
 
 impl Eq for TransactionOrder {}
@@ -167,6 +181,11 @@ impl PartialOrd for TransactionOrder {
 
 impl Ord for TransactionOrder {
 	fn cmp(&self, b: &TransactionOrder) -> Ordering {
+		// First check number of penalties
+		if self.penalties != b.penalties {
+			return self.penalties.cmp(&b.penalties);
+		}
+
 		// First check nonce_height
 		if self.nonce_height != b.nonce_height {
 			return self.nonce_height.cmp(&b.nonce_height);
@@ -271,6 +290,7 @@ struct TransactionSet {
 	by_address: Table<Address, U256, TransactionOrder>,
 	by_gas_price: GasPriceQueue,
 	limit: usize,
+	gas_limit: U256,
 }
 
 impl TransactionSet {
@@ -301,15 +321,20 @@ impl TransactionSet {
 	/// It drops transactions from this set but also removes associated `VerifiedTransaction`.
 	/// Returns addresses and lowest nonces of transactions removed because of limit.
 	fn enforce_limit(&mut self, by_hash: &mut HashMap<H256, VerifiedTransaction>) -> Option<HashMap<Address, U256>> {
-		let len = self.by_priority.len();
+		let mut count = 0;
-		if len <= self.limit {
+		let mut gas: U256 = 0.into();
-			return None;
-		}
-
 		let to_drop : Vec<(Address, U256)> = {
 			self.by_priority
 				.iter()
-				.skip(self.limit)
+				.skip_while(|order| {
+					count = count + 1;
+					let r = gas.overflowing_add(order.gas);
+					if r.1 { return false }
+					gas = r.0;
+					// Own and retracted transactions are allowed to go above the gas limit, bot not above the count limit.
+					(gas <= self.gas_limit || order.origin == TransactionOrigin::Local || order.origin == TransactionOrigin::RetractedBlock) &&
+						count <= self.limit
+				})
 				.map(|order| by_hash.get(&order.hash)
 					.expect("All transactions in `self.by_priority` and `self.by_address` are kept in sync with `by_hash`."))
 				.map(|tx| (tx.sender(), tx.nonce()))
@@ -387,7 +412,7 @@ pub struct AccountDetails {
 }
 
 /// Transactions with `gas > (gas_limit + gas_limit * Factor(in percents))` are not imported to the queue.
-const GAS_LIMIT_HYSTERESIS: usize = 10; // %
+const GAS_LIMIT_HYSTERESIS: usize = 10; // (100/GAS_LIMIT_HYSTERESIS) %
 
 /// `TransactionQueue` implementation
 pub struct TransactionQueue {
@@ -416,16 +441,17 @@ impl Default for TransactionQueue {
 impl TransactionQueue {
 	/// Creates new instance of this Queue
 	pub fn new() -> Self {
-		Self::with_limits(1024, !U256::zero())
+		Self::with_limits(1024, !U256::zero(), !U256::zero())
 	}
 
 	/// Create new instance of this Queue with specified limits
-	pub fn with_limits(limit: usize, tx_gas_limit: U256) -> Self {
+	pub fn with_limits(limit: usize, gas_limit: U256, tx_gas_limit: U256) -> Self {
 		let current = TransactionSet {
 			by_priority: BTreeSet::new(),
 			by_address: Table::new(),
 			by_gas_price: Default::default(),
 			limit: limit,
+			gas_limit: gas_limit,
 		};
 
 		let future = TransactionSet {
@@ -433,6 +459,7 @@ impl TransactionQueue {
 			by_address: Table::new(),
 			by_gas_price: Default::default(),
 			limit: limit,
+			gas_limit: gas_limit,
 		};
 
 		TransactionQueue {
@@ -488,6 +515,13 @@ impl TransactionQueue {
 		};
 	}
 
+	/// Sets new total gas limit.
+	pub fn set_total_gas_limit(&mut self, gas_limit: U256) {
+		self.future.gas_limit = gas_limit;
+		self.current.gas_limit = gas_limit;
+		self.future.enforce_limit(&mut self.by_hash);
+	}
+
 	/// Set the new limit for the amount of gas any individual transaction may have.
 	/// Any transaction already imported to the queue is not affected.
 	pub fn set_tx_gas_limit(&mut self, limit: U256) {
@@ -506,8 +540,6 @@ impl TransactionQueue {
 	pub fn add<T>(&mut self, tx: SignedTransaction, fetch_account: &T, origin: TransactionOrigin) -> Result<TransactionImportResult, Error>
 	where T: Fn(&Address) -> AccountDetails {
 
-		trace!(target: "txqueue", "Importing: {:?}", tx.hash());
-
 		if tx.gas_price < self.minimal_gas_price && origin != TransactionOrigin::Local {
 			trace!(target: "txqueue",
 				"Dropping transaction below minimal gas price threshold: {:?} (gp: {} < {})",
@@ -593,6 +625,39 @@ impl TransactionQueue {
 		assert_eq!(self.future.by_priority.len() + self.current.by_priority.len(), self.by_hash.len());
 	}
 
+	/// Penalize transactions from sender of transaction with given hash.
+	/// I.e. it should change the priority of the transaction in the queue.
+	///
+	/// NOTE: We need to penalize all transactions from particular sender
+	/// to avoid breaking invariants in queue (ordered by nonces).
+	/// Consecutive transactions from this sender would fail otherwise (because of invalid nonce).
+	pub fn penalize(&mut self, transaction_hash: &H256) {
+		let transaction = match self.by_hash.get(transaction_hash) {
+			None => return,
+			Some(t) => t,
+		};
+		let sender = transaction.sender();
+
+		// Penalize all transactions from this sender
+		let nonces_from_sender = match self.current.by_address.row(&sender) {
+			Some(row_map) => row_map.keys().cloned().collect::<Vec<U256>>(),
+			None => vec![],
+		};
+		for k in nonces_from_sender {
+			let order = self.current.drop(&sender, &k).unwrap();
+			self.current.insert(sender, k, order.penalize());
+		}
+		// Same thing for future
+		let nonces_from_sender = match self.future.by_address.row(&sender) {
+			Some(row_map) => row_map.keys().cloned().collect::<Vec<U256>>(),
+			None => vec![],
+		};
+		for k in nonces_from_sender {
+			let order = self.future.drop(&sender, &k).unwrap();
+			self.future.insert(sender, k, order.penalize());
+		}
+	}
+
 	/// Removes invalid transaction identified by hash from queue.
 	/// Assumption is that this transaction nonce is not related to client nonce,
 	/// so transactions left in queue are processed according to client nonce.
@@ -667,7 +732,10 @@ impl TransactionQueue {
 			let order = self.current.drop(sender, &k).expect("iterating over a collection that has been retrieved above;
 															 qed");
 			if k >= current_nonce {
-				self.future.insert(*sender, k, order.update_height(k, current_nonce));
+				let order = order.update_height(k, current_nonce);
+				if let Some(old) = self.future.insert(*sender, k, order.clone()) {
+					Self::replace_orders(*sender, k, old, order, &mut self.future, &mut self.by_hash);
+				}
 			} else {
 				trace!(target: "txqueue", "Removing old transaction: {:?} (nonce: {} < {})", order.hash, k, current_nonce);
 				self.by_hash.remove(&order.hash).expect("All transactions in `future` are also in `by_hash`");
@@ -685,6 +753,15 @@ impl TransactionQueue {
 			.collect()
 	}
 
+	#[cfg(test)]
+	fn future_transactions(&self) -> Vec<SignedTransaction> {
+		self.future.by_priority
+			.iter()
+			.map(|t| self.by_hash.get(&t.hash).expect("All transactions in `current` and `future` are always included in `by_hash`"))
+			.map(|t| t.transaction.clone())
+			.collect()
+	}
+
 	/// Returns hashes of all transactions from current, ordered by priority.
 	pub fn pending_hashes(&self) -> Vec<H256> {
 		self.current.by_priority
@@ -732,7 +809,9 @@ impl TransactionQueue {
 				self.future.by_gas_price.remove(&order.gas_price, &order.hash);
 				// Put to current
 				let order = order.update_height(current_nonce, first_nonce);
-				self.current.insert(address, current_nonce, order);
+				if let Some(old) = self.current.insert(address, current_nonce, order.clone()) {
+					Self::replace_orders(address, current_nonce, old, order, &mut self.current, &mut self.by_hash);
+				}
 				update_last_nonce_to = Some(current_nonce);
 				current_nonce = current_nonce + U256::one();
 			}
@@ -764,45 +843,61 @@ impl TransactionQueue {
 
 		let address = tx.sender();
 		let nonce = tx.nonce();
+		let hash = tx.hash();
 
-		let next_nonce = self.last_nonces
+		{
-			.get(&address)
+			// Rough size sanity check
-			.cloned()
+			let gas = &tx.transaction.gas;
-			.map_or(state_nonce, |n| n + U256::one());
+			if U256::from(tx.transaction.data.len()) > *gas {
+				// Droping transaction
+				trace!(target: "txqueue", "Dropping oversized transaction: {:?} (gas: {} < size {})", hash, gas, tx.transaction.data.len());
+				return Err(TransactionError::LimitReached);
+			}
+		}
 
 		// The transaction might be old, let's check that.
 		// This has to be the first test, otherwise calculating
 		// nonce height would result in overflow.
 		if nonce < state_nonce {
 			// Droping transaction
-			trace!(target: "txqueue", "Dropping old transaction: {:?} (nonce: {} < {})", tx.hash(), nonce, next_nonce);
+			trace!(target: "txqueue", "Dropping old transaction: {:?} (nonce: {} < {})", tx.hash(), nonce, state_nonce);
 			return Err(TransactionError::Old);
-		} else if nonce > next_nonce {
+		}
-			// We have a gap - put to future.
+
 		// Update nonces of transactions in future (remove old transactions)
 		self.update_future(&address, state_nonce);
+		// State nonce could be updated. Maybe there are some more items waiting in future?
+		self.move_matching_future_to_current(address, state_nonce, state_nonce);
+		// Check the next expected nonce (might be updated by move above)
+		let next_nonce = self.last_nonces
+			.get(&address)
+			.cloned()
+			.map_or(state_nonce, |n| n + U256::one());
+
+		// Future transaction
+		if nonce > next_nonce {
+			// We have a gap - put to future.
 			// Insert transaction (or replace old one with lower gas price)
 			try!(check_too_cheap(Self::replace_transaction(tx, state_nonce, &mut self.future, &mut self.by_hash)));
-			// Return an error if this transaction is not imported because of limit.
+			// Enforce limit in Future
-			try!(check_if_removed(&address, &nonce, self.future.enforce_limit(&mut self.by_hash)));
+			let removed = self.future.enforce_limit(&mut self.by_hash);
+			// Return an error if this transaction was not imported because of limit.
+			try!(check_if_removed(&address, &nonce, removed));
+
+			debug!(target: "txqueue", "Importing transaction to future: {:?}", hash);
+			debug!(target: "txqueue", "status: {:?}", self.status());
 			return Ok(TransactionImportResult::Future);
 		}
+
+		// We might have filled a gap - move some more transactions from future
+		self.move_matching_future_to_current(address, nonce, state_nonce);
+		self.move_matching_future_to_current(address, nonce + U256::one(), state_nonce);
+
+		// Replace transaction if any
 		try!(check_too_cheap(Self::replace_transaction(tx, state_nonce, &mut self.current, &mut self.by_hash)));
 		// Keep track of highest nonce stored in current
 		let new_max = self.last_nonces.get(&address).map_or(nonce, |n| cmp::max(nonce, *n));
 		self.last_nonces.insert(address, new_max);
-		// Update nonces of transactions in future
-		self.update_future(&address, state_nonce);
-		// Maybe there are some more items waiting in future?
-		self.move_matching_future_to_current(address, nonce + U256::one(), state_nonce);
-		// There might be exactly the same transaction waiting in future
-		// same (sender, nonce), but above function would not move it.
-		if let Some(order) = self.future.drop(&address, &nonce) {
-			// Let's insert that transaction to current (if it has higher gas_price)
-			let future_tx = self.by_hash.remove(&order.hash).expect("All transactions in `future` are always in `by_hash`.");
-			// if transaction in `current` (then one we are importing) is replaced it means that it has to low gas_price
-			try!(check_too_cheap(!Self::replace_transaction(future_tx, state_nonce, &mut self.current, &mut self.by_hash)));
-		}
 
 		// Also enforce the limit
 		let removed = self.current.enforce_limit(&mut self.by_hash);
@@ -811,7 +906,8 @@ impl TransactionQueue {
 		// Trigger error if the transaction we are importing was removed.
 		try!(check_if_removed(&address, &nonce, removed));
 
-		trace!(target: "txqueue", "status: {:?}", self.status());
+		debug!(target: "txqueue", "Imported transaction to current: {:?}", hash);
+		debug!(target: "txqueue", "status: {:?}", self.status());
 		Ok(TransactionImportResult::Current)
 	}
 
@@ -846,6 +942,13 @@ impl TransactionQueue {
 
 
 		if let Some(old) = set.insert(address, nonce, order.clone()) {
+			Self::replace_orders(address, nonce, old, order, set, by_hash)
+		} else {
+			true
+		}
+	}
+
+	fn replace_orders(address: Address, nonce: U256, old: TransactionOrder, order: TransactionOrder, set: &mut TransactionSet, by_hash: &mut HashMap<H256, VerifiedTransaction>) -> bool {
 		// There was already transaction in queue. Let's check which one should stay
 		let old_fee = old.gas_price;
 		let new_fee = order.gas_price;
@@ -853,16 +956,13 @@ impl TransactionQueue {
 			// Put back old transaction since it has greater priority (higher gas_price)
 			set.insert(address, nonce, old);
 			// and remove new one
-				by_hash.remove(&hash).expect("The hash has been just inserted and no other line is altering `by_hash`.");
+			by_hash.remove(&order.hash).expect("The hash has been just inserted and no other line is altering `by_hash`.");
 			false
 		} else {
 			// Make sure we remove old transaction entirely
 			by_hash.remove(&old.hash).expect("The hash is coming from `future` so it has to be in `by_hash`.");
 			true
 		}
-		} else {
-			true
-		}
 	}
 }
 
@@ -907,6 +1007,7 @@ mod test {
 	}
 
 	fn default_nonce() -> U256 { 123.into() }
+	fn default_gas_val() -> U256 { 100_000.into() }
 	fn default_gas_price() -> U256 { 1.into() }
 
 	fn new_unsigned_tx(nonce: U256, gas_price: U256) -> Transaction {
@@ -914,7 +1015,7 @@ mod test {
 			action: Action::Create,
 			value: U256::from(100),
 			data: "3331600055".from_hex().unwrap(),
-			gas: U256::from(100_000),
+			gas: default_gas_val(),
 			gas_price: gas_price,
 			nonce: nonce
 		}
@@ -945,6 +1046,17 @@ mod test {
 		(tx1.sign(secret), tx2.sign(secret))
 	}
 
+	/// Returns two consecutive transactions, both with increased gas price
+	fn new_tx_pair_with_gas_price_increment(gas_price_increment: U256) -> (SignedTransaction, SignedTransaction) {
+		let gas = default_gas_price() + gas_price_increment;
+		let tx1 = new_unsigned_tx(default_nonce(), gas);
+		let tx2 = new_unsigned_tx(default_nonce() + 1.into(), gas);
+
+		let keypair = Random.generate().unwrap();
+		let secret = &keypair.secret();
+		(tx1.sign(secret), tx2.sign(secret))
+	}
+
 	fn new_tx_pair_default(nonce_increment: U256, gas_price_increment: U256) -> (SignedTransaction, SignedTransaction) {
 		new_tx_pair(default_nonce(), default_gas_price(), nonce_increment, gas_price_increment)
 	}
@@ -954,10 +1066,21 @@ mod test {
 		new_tx_pair_default(0.into(), 1.into())
 	}
 
+	#[test]
+	fn test_ordering() {
+		assert_eq!(TransactionOrigin::Local.cmp(&TransactionOrigin::External), Ordering::Less);
+		assert_eq!(TransactionOrigin::RetractedBlock.cmp(&TransactionOrigin::Local), Ordering::Less);
+		assert_eq!(TransactionOrigin::RetractedBlock.cmp(&TransactionOrigin::External), Ordering::Less);
+
+		assert_eq!(TransactionOrigin::External.cmp(&TransactionOrigin::Local), Ordering::Greater);
+		assert_eq!(TransactionOrigin::Local.cmp(&TransactionOrigin::RetractedBlock), Ordering::Greater);
+		assert_eq!(TransactionOrigin::External.cmp(&TransactionOrigin::RetractedBlock), Ordering::Greater);
+	}
+
 	#[test]
 	fn should_return_correct_nonces_when_dropped_because_of_limit() {
 		// given
-		let mut txq = TransactionQueue::with_limits(2, !U256::zero());
+		let mut txq = TransactionQueue::with_limits(2, !U256::zero(), !U256::zero());
 		let (tx1, tx2) = new_tx_pair(123.into(), 1.into(), 1.into(), 0.into());
 		let sender = tx1.sender().unwrap();
 		let nonce = tx1.nonce;
@@ -995,7 +1118,8 @@ mod test {
 			by_priority: BTreeSet::new(),
 			by_address: Table::new(),
 			by_gas_price: Default::default(),
-			limit: 1
+			limit: 1,
+			gas_limit: !U256::zero(),
 		};
 		let (tx1, tx2) = new_tx_pair_default(1.into(), 0.into());
 		let tx1 = VerifiedTransaction::new(tx1, TransactionOrigin::External).unwrap();
@@ -1035,7 +1159,8 @@ mod test {
 			by_priority: BTreeSet::new(),
 			by_address: Table::new(),
 			by_gas_price: Default::default(),
-			limit: 1
+			limit: 1,
+			gas_limit: !U256::zero(),
 		};
 		// Create two transactions with same nonce
 		// (same hash)
@@ -1083,7 +1208,8 @@ mod test {
 			by_priority: BTreeSet::new(),
 			by_address: Table::new(),
 			by_gas_price: Default::default(),
-			limit: 2
+			limit: 2,
+			gas_limit: !U256::zero(),
 		};
 		let tx = new_tx_default();
 		let tx1 = VerifiedTransaction::new(tx.clone(), TransactionOrigin::External).unwrap();
@@ -1100,7 +1226,8 @@ mod test {
 			by_priority: BTreeSet::new(),
 			by_address: Table::new(),
 			by_gas_price: Default::default(),
-			limit: 1
+			limit: 1,
+			gas_limit: !U256::zero(),
 		};
 
 		assert_eq!(set.gas_price_entry_limit(), 0.into());
@@ -1136,6 +1263,31 @@ mod test {
 		assert_eq!(txq.top_transactions()[0], tx2);
 	}
 
+	#[test]
+	fn should_move_all_transactions_from_future() {
+		// given
+		let mut txq = TransactionQueue::new();
+		let (tx, tx2) = new_tx_pair_default(1.into(), 1.into());
+		let prev_nonce = |a: &Address| AccountDetails{ nonce: default_account_details(a).nonce - U256::one(), balance:
+			!U256::zero() };
+
+		// First insert one transaction to future
+		let res = txq.add(tx.clone(), &prev_nonce, TransactionOrigin::External);
+		assert_eq!(res.unwrap(), TransactionImportResult::Future);
+		assert_eq!(txq.status().future, 1);
+
+		// now import second transaction to current
+		let res = txq.add(tx2.clone(), &default_account_details, TransactionOrigin::External);
+
+		// then
+		assert_eq!(res.unwrap(), TransactionImportResult::Current);
+		assert_eq!(txq.status().pending, 2);
+		assert_eq!(txq.status().future, 0);
+		assert_eq!(txq.current.by_priority.len(), 2);
+		assert_eq!(txq.current.by_address.len(), 2);
+		assert_eq!(txq.top_transactions()[0], tx);
+		assert_eq!(txq.top_transactions()[1], tx2);
+	}
 
 	#[test]
 	fn should_import_tx() {
@@ -1315,6 +1467,27 @@ mod test {
 		assert_eq!(top.len(), 2);
 	}
 
+	#[test]
+	fn should_prioritize_reimported_transactions_within_same_nonce_height() {
+		// given
+		let mut txq = TransactionQueue::new();
+		let tx = new_tx_default();
+		// the second one has same nonce but higher `gas_price`
+		let (_, tx2) = new_similar_tx_pair();
+
+		// when
+		// first insert local one with higher gas price
+		txq.add(tx2.clone(), &default_account_details, TransactionOrigin::Local).unwrap();
+		// then the one with lower gas price, but from retracted block
+		txq.add(tx.clone(), &default_account_details, TransactionOrigin::RetractedBlock).unwrap();
+
+		// then
+		let top = txq.top_transactions();
+		assert_eq!(top[0], tx); // retracted should be first
+		assert_eq!(top[1], tx2);
+		assert_eq!(top.len(), 2);
+	}
+
 	#[test]
 	fn should_not_prioritize_local_transactions_with_different_nonce_height() {
 		// given
@@ -1332,6 +1505,69 @@ mod test {
 		assert_eq!(top.len(), 2);
 	}
 
+	#[test]
+	fn should_penalize_transactions_from_sender_in_future() {
+		// given
+		let prev_nonce = |a: &Address| AccountDetails{ nonce: default_account_details(a).nonce - U256::one(), balance: !U256::zero() };
+		let mut txq = TransactionQueue::new();
+		// txa, txb - slightly bigger gas price to have consistent ordering
+		let (txa, txb) = new_tx_pair_default(1.into(), 0.into());
+		let (tx1, tx2) = new_tx_pair_with_gas_price_increment(3.into());
+
+		// insert everything
+		txq.add(txa.clone(), &prev_nonce, TransactionOrigin::External).unwrap();
+		txq.add(txb.clone(), &prev_nonce, TransactionOrigin::External).unwrap();
+		txq.add(tx1.clone(), &prev_nonce, TransactionOrigin::External).unwrap();
+		txq.add(tx2.clone(), &prev_nonce, TransactionOrigin::External).unwrap();
+
+		assert_eq!(txq.status().future, 4);
+
+		// when
+		txq.penalize(&tx1.hash());
+
+		// then
+		let top = txq.future_transactions();
+		assert_eq!(top[0], txa);
+		assert_eq!(top[1], txb);
+		assert_eq!(top[2], tx1);
+		assert_eq!(top[3], tx2);
+		assert_eq!(top.len(), 4);
+	}
+
+
+	#[test]
+	fn should_penalize_transactions_from_sender() {
+		// given
+		let mut txq = TransactionQueue::new();
+		// txa, txb - slightly bigger gas price to have consistent ordering
+		let (txa, txb) = new_tx_pair_default(1.into(), 0.into());
+		let (tx1, tx2) = new_tx_pair_with_gas_price_increment(3.into());
+
+		// insert everything
+		txq.add(txa.clone(), &default_account_details, TransactionOrigin::External).unwrap();
+		txq.add(txb.clone(), &default_account_details, TransactionOrigin::External).unwrap();
+		txq.add(tx1.clone(), &default_account_details, TransactionOrigin::External).unwrap();
+		txq.add(tx2.clone(), &default_account_details, TransactionOrigin::External).unwrap();
+
+		let top = txq.top_transactions();
+		assert_eq!(top[0], tx1);
+		assert_eq!(top[1], txa);
+		assert_eq!(top[2], tx2);
+		assert_eq!(top[3], txb);
+		assert_eq!(top.len(), 4);
+
+		// when
+		txq.penalize(&tx1.hash());
+
+		// then
+		let top = txq.top_transactions();
+		assert_eq!(top[0], txa);
+		assert_eq!(top[1], txb);
+		assert_eq!(top[2], tx1);
+		assert_eq!(top[3], tx2);
+		assert_eq!(top.len(), 4);
+	}
+
 	#[test]
 	fn should_return_pending_hashes() {
 			// given
@@ -1487,7 +1723,7 @@ mod test {
 	#[test]
 	fn should_drop_old_transactions_when_hitting_the_limit() {
 		// given
-		let mut txq = TransactionQueue::with_limits(1, !U256::zero());
+		let mut txq = TransactionQueue::with_limits(1, !U256::zero(), !U256::zero());
 		let (tx, tx2) = new_tx_pair_default(1.into(), 0.into());
 		let sender = tx.sender().unwrap();
 		let nonce = tx.nonce;
@@ -1508,7 +1744,7 @@ mod test {
 
 	#[test]
 	fn should_limit_future_transactions() {
-		let mut txq = TransactionQueue::with_limits(1, !U256::zero());
+		let mut txq = TransactionQueue::with_limits(1, !U256::zero(), !U256::zero());
 		txq.current.set_limit(10);
 		let (tx1, tx2) = new_tx_pair_default(4.into(), 1.into());
 		let (tx3, tx4) = new_tx_pair_default(4.into(), 2.into());
@@ -1525,6 +1761,30 @@ mod test {
 		assert_eq!(txq.status().future, 1);
 	}
 
+	#[test]
+	fn should_limit_by_gas() {
+		let mut txq = TransactionQueue::with_limits(100, default_gas_val() * U256::from(2), !U256::zero());
+		let (tx1, tx2) = new_tx_pair_default(U256::from(1), U256::from(1));
+		let (tx3, tx4) = new_tx_pair_default(U256::from(1), U256::from(2));
+		txq.add(tx1.clone(), &default_account_details, TransactionOrigin::External).ok();
+		txq.add(tx2.clone(), &default_account_details, TransactionOrigin::External).ok();
+		txq.add(tx3.clone(), &default_account_details, TransactionOrigin::External).ok();
+		txq.add(tx4.clone(), &default_account_details, TransactionOrigin::External).ok();
+		assert_eq!(txq.status().pending, 2);
+	}
+
+	#[test]
+	fn should_keep_own_transactions_above_gas_limit() {
+		let mut txq = TransactionQueue::with_limits(100, default_gas_val() * U256::from(2), !U256::zero());
+		let (tx1, tx2) = new_tx_pair_default(U256::from(1), U256::from(1));
+		let (tx3, tx4) = new_tx_pair_default(U256::from(1), U256::from(2));
+		txq.add(tx1.clone(), &default_account_details, TransactionOrigin::Local).unwrap();
+		txq.add(tx2.clone(), &default_account_details, TransactionOrigin::Local).unwrap();
+		txq.add(tx3.clone(), &default_account_details, TransactionOrigin::Local).unwrap();
+		txq.add(tx4.clone(), &default_account_details, TransactionOrigin::Local).unwrap();
+		assert_eq!(txq.status().pending, 4);
+	}
+
 	#[test]
 	fn should_drop_transactions_with_old_nonces() {
 		let mut txq = TransactionQueue::new();
@@ -1768,7 +2028,7 @@ mod test {
 	#[test]
 	fn should_keep_right_order_in_future() {
 		// given
-		let mut txq = TransactionQueue::with_limits(1, !U256::zero());
+		let mut txq = TransactionQueue::with_limits(1, !U256::zero(), !U256::zero());
 		let (tx1, tx2) = new_tx_pair_default(1.into(), 0.into());
 		let prev_nonce = |a: &Address| AccountDetails { nonce: default_account_details(a).nonce - U256::one(), balance:
 			default_account_details(a).balance };

ethcore/src/pod_account.rs

@@ -48,7 +48,7 @@ impl PodAccount {
 		PodAccount {
 			balance: *acc.balance(),
 			nonce: *acc.nonce(),
-			storage: acc.storage_overlay().iter().fold(BTreeMap::new(), |mut m, (k, &(_, ref v))| {m.insert(k.clone(), v.clone()); m}),
+			storage: acc.storage_changes().iter().fold(BTreeMap::new(), |mut m, (k, v)| {m.insert(k.clone(), v.clone()); m}),
 			code: acc.code().map(|x| x.to_vec()),
 		}
 	}

ethcore/src/service.rs

@@ -80,7 +80,13 @@ impl ClientService {
 		}
 
 		let mut db_config = DatabaseConfig::with_columns(::db::NUM_COLUMNS);
-		db_config.cache_size = config.db_cache_size;
+
+		// give all rocksdb cache to state column; everything else has its
+		// own caches.
+		if let Some(size) = config.db_cache_size {
+			db_config.set_cache(::db::COL_STATE, size);
+		}
+
 		db_config.compaction = config.db_compaction.compaction_profile();
 		db_config.wal = config.db_wal;
 
@@ -90,7 +96,7 @@ impl ClientService {
 		let snapshot_params = SnapServiceParams {
 			engine: spec.engine.clone(),
 			genesis_block: spec.genesis_block(),
-			db_config: db_config,
+			db_config: db_config.clone(),
 			pruning: pruning,
 			channel: io_service.channel(),
 			snapshot_root: snapshot_path.into(),

ethcore/src/snapshot/account.rs

@@ -205,7 +205,7 @@ impl Account {
 #[cfg(test)]
 mod tests {
 	use account_db::{AccountDB, AccountDBMut};
-	use tests::helpers::get_temp_journal_db;
+	use tests::helpers::get_temp_state_db;
 	use snapshot::tests::helpers::fill_storage;
 
 	use util::sha3::{SHA3_EMPTY, SHA3_NULL_RLP};
@@ -218,8 +218,7 @@ mod tests {
 
 	#[test]
 	fn encoding_basic() {
-		let mut db = get_temp_journal_db();
+		let mut db = get_temp_state_db();
-		let mut db = &mut **db;
 		let addr = Address::random();
 
 		let account = Account {
@@ -239,8 +238,7 @@ mod tests {
 
 	#[test]
 	fn encoding_storage() {
-		let mut db = get_temp_journal_db();
+		let mut db = get_temp_state_db();
-		let mut db = &mut **db;
 		let addr = Address::random();
 
 		let account = {
@@ -265,8 +263,7 @@ mod tests {
 
 	#[test]
 	fn encoding_code() {
-		let mut db = get_temp_journal_db();
+		let mut db = get_temp_state_db();
-		let mut db = &mut **db;
 
 		let addr1 = Address::random();
 		let addr2 = Address::random();

ethcore/src/snapshot/mod.rs

@@ -43,13 +43,15 @@ use self::account::Account;
 use self::block::AbridgedBlock;
 use self::io::SnapshotWriter;
 
+use super::state_db::StateDB;
+
 use crossbeam::{scope, ScopedJoinHandle};
 use rand::{Rng, OsRng};
 
 pub use self::error::Error;
 
 pub use self::service::{Service, DatabaseRestore};
-pub use self::traits::{SnapshotService, RemoteSnapshotService};
+pub use self::traits::SnapshotService;
 pub use self::watcher::Watcher;
 pub use types::snapshot_manifest::ManifestData;
 pub use types::restoration_status::RestorationStatus;
@@ -65,6 +67,12 @@ mod watcher;
 #[cfg(test)]
 mod tests;
 
+/// IPC interfaces
+#[cfg(feature="ipc")]
+pub mod remote {
+	pub use super::traits::RemoteSnapshotService;
+}
+
 mod traits {
 	#![allow(dead_code, unused_assignments, unused_variables, missing_docs)] // codegen issues
 	include!(concat!(env!("OUT_DIR"), "/snapshot_service_trait.rs"));
@@ -454,6 +462,10 @@ impl StateRebuilder {
 			self.code_map.insert(code_hash, code);
 		}
 
+		let backing = self.db.backing().clone();
+
+		// bloom has to be updated
+		let mut bloom = StateDB::load_bloom(&backing);
 
 		// batch trie writes
 		{
@@ -464,12 +476,14 @@ impl StateRebuilder {
 			};
 
 			for (hash, thin_rlp) in pairs {
+				bloom.set(&*hash);
 				try!(account_trie.insert(&hash, &thin_rlp));
 			}
 		}
 
-		let backing = self.db.backing().clone();
+		let bloom_journal = bloom.drain_journal();
 		let mut batch = backing.transaction();
+		try!(StateDB::commit_bloom(&mut batch, bloom_journal));
 		try!(self.db.inject(&mut batch));
 		try!(backing.write(batch).map_err(::util::UtilError::SimpleString));
 		trace!(target: "snapshot", "current state root: {:?}", self.state_root);

ethcore/src/snapshot/snapshot_service_trait.rs

@@ -22,7 +22,6 @@ use ipc::IpcConfig;
 /// This handles:
 ///    - restoration of snapshots to temporary databases.
 ///    - responding to queries for snapshot manifests and chunks
-#[derive(Ipc)]
 #[ipc(client_ident="RemoteSnapshotService")]
 pub trait SnapshotService : Sync + Send {
 	/// Query the most recent manifest data.

ethcore/src/spec/spec.rs

@@ -20,6 +20,7 @@ use common::*;
 use engines::{Engine, NullEngine, InstantSeal, BasicAuthority};
 use pod_state::*;
 use account_db::*;
+use state_db::StateDB;
 use super::genesis::Genesis;
 use super::seal::Generic as GenericSeal;
 use ethereum;
@@ -36,6 +37,8 @@ pub struct CommonParams {
 	pub maximum_extra_data_size: usize,
 	/// Network id.
 	pub network_id: U256,
+	/// Main subprotocol name.
+	pub subprotocol_name: String,
 	/// Minimum gas limit.
 	pub min_gas_limit: U256,
 	/// Fork block to check.
@@ -48,6 +51,7 @@ impl From<ethjson::spec::Params> for CommonParams {
 			account_start_nonce: p.account_start_nonce.into(),
 			maximum_extra_data_size: p.maximum_extra_data_size.into(),
 			network_id: p.network_id.into(),
+			subprotocol_name: p.subprotocol_name.unwrap_or_else(|| "eth".to_owned()),
 			min_gas_limit: p.min_gas_limit.into(),
 			fork_block: if let (Some(n), Some(h)) = (p.fork_block, p.fork_hash) { Some((n.into(), h.into())) } else { None },
 		}
@@ -155,6 +159,9 @@ impl Spec {
 	/// Get the configured Network ID.
 	pub fn network_id(&self) -> U256 { self.params.network_id }
 
+	/// Get the configured Network ID.
+	pub fn subprotocol_name(&self) -> String { self.params.subprotocol_name.clone() }
+
 	/// Get the configured network fork block.
 	pub fn fork_block(&self) -> Option<(BlockNumber, H256)> { self.params.fork_block }
 
@@ -168,7 +175,7 @@ impl Spec {
 		header.set_transactions_root(self.transactions_root.clone());
 		header.set_uncles_hash(RlpStream::new_list(0).out().sha3());
 		header.set_extra_data(self.extra_data.clone());
-		header.set_state_root(self.state_root().clone());
+		header.set_state_root(self.state_root());
 		header.set_receipts_root(self.receipts_root.clone());
 		header.set_log_bloom(H2048::new().clone());
 		header.set_gas_used(self.gas_used.clone());
@@ -183,6 +190,7 @@ impl Spec {
 			let r = Rlp::new(&seal);
 			(0..self.seal_fields).map(|i| r.at(i).as_raw().to_vec()).collect()
 		});
+		trace!(target: "spec", "Header hash is {}", header.hash());
 		header
 	}
 
@@ -226,19 +234,23 @@ impl Spec {
 	}
 
 	/// Ensure that the given state DB has the trie nodes in for the genesis state.
-	pub fn ensure_db_good(&self, db: &mut HashDB) -> Result<bool, Box<TrieError>> {
+	pub fn ensure_db_good(&self, db: &mut StateDB) -> Result<bool, Box<TrieError>> {
-		if !db.contains(&self.state_root()) {
+		if !db.as_hashdb().contains(&self.state_root()) {
+			trace!(target: "spec", "ensure_db_good: Fresh database? Cannot find state root {}", self.state_root());
 			let mut root = H256::new();
+
 			{
-				let mut t = SecTrieDBMut::new(db, &mut root);
+				let mut t = SecTrieDBMut::new(db.as_hashdb_mut(), &mut root);
 				for (address, account) in self.genesis_state.get().iter() {
 					try!(t.insert(&**address, &account.rlp()));
 				}
 			}
+			trace!(target: "spec", "ensure_db_good: Populated sec trie; root is {}", root);
 			for (address, account) in self.genesis_state.get().iter() {
-				account.insert_additional(&mut AccountDBMut::new(db, address));
+				db.note_account_bloom(address);
+				account.insert_additional(&mut AccountDBMut::new(db.as_hashdb_mut(), address));
 			}
-			assert!(db.contains(&self.state_root()));
+			assert!(db.as_hashdb().contains(&self.state_root()));
 			Ok(true)
 		} else { Ok(false) }
 	}

ethcore/src/state/account.rs

@@ -16,15 +16,18 @@
 
 //! Single account in the system.
 
-use std::collections::hash_map::Entry;
 use util::*;
 use pod_account::*;
 use rlp::*;
+use lru_cache::LruCache;
 
-use std::cell::{Ref, RefCell, Cell};
+use std::cell::{RefCell, Cell};
+
+const STORAGE_CACHE_ITEMS: usize = 8192;
 
 /// Single account in the system.
-#[derive(Clone)]
+/// Keeps track of changes to the code and storage.
+/// The changes are applied in `commit_storage` and `commit_code`
 pub struct Account {
 	// Balance of the account.
 	balance: U256,
@@ -32,14 +35,20 @@ pub struct Account {
 	nonce: U256,
 	// Trie-backed storage.
 	storage_root: H256,
-	// Overlay on trie-backed storage - tuple is (<clean>, <value>).
+	// LRU Cache of the trie-backed storage.
-	storage_overlay: RefCell<HashMap<H256, (Filth, H256)>>,
+	// This is limited to `STORAGE_CACHE_ITEMS` recent queries
-	// Code hash of the account. If None, means that it's a contract whose code has not yet been set.
+	storage_cache: RefCell<LruCache<H256, H256>>,
-	code_hash: Option<H256>,
+	// Modified storage. Accumulates changes to storage made in `set_storage`
+	// Takes precedence over `storage_cache`.
+	storage_changes: HashMap<H256, H256>,
+	// Code hash of the account.
+	code_hash: H256,
+	// Size of the accoun code.
+	code_size: Option<usize>,
 	// Code cache of the account.
-	code_cache: Bytes,
+	code_cache: Arc<Bytes>,
-	// Account is new or has been modified
+	// Account code new or has been modified.
-	filth: Filth,
+	code_filth: Filth,
 	// Cached address hash.
 	address_hash: Cell<Option<H256>>,
 }
@@ -52,24 +61,32 @@ impl Account {
 			balance: balance,
 			nonce: nonce,
 			storage_root: SHA3_NULL_RLP,
-			storage_overlay: RefCell::new(storage.into_iter().map(|(k, v)| (k, (Filth::Dirty, v))).collect()),
+			storage_cache: Self::empty_storage_cache(),
-			code_hash: Some(code.sha3()),
+			storage_changes: storage,
-			code_cache: code,
+			code_hash: code.sha3(),
-			filth: Filth::Dirty,
+			code_size: Some(code.len()),
+			code_cache: Arc::new(code),
+			code_filth: Filth::Dirty,
 			address_hash: Cell::new(None),
 		}
 	}
 
+	fn empty_storage_cache() -> RefCell<LruCache<H256, H256>> {
+		RefCell::new(LruCache::new(STORAGE_CACHE_ITEMS))
+	}
+
 	/// General constructor.
 	pub fn from_pod(pod: PodAccount) -> Account {
 		Account {
 			balance: pod.balance,
 			nonce: pod.nonce,
 			storage_root: SHA3_NULL_RLP,
-			storage_overlay: RefCell::new(pod.storage.into_iter().map(|(k, v)| (k, (Filth::Dirty, v))).collect()),
+			storage_cache: Self::empty_storage_cache(),
-			code_hash: pod.code.as_ref().map(|c| c.sha3()),
+			storage_changes: pod.storage.into_iter().collect(),
-			code_cache: pod.code.as_ref().map_or_else(|| { warn!("POD account with unknown code is being created! Assuming no code."); vec![] }, |c| c.clone()),
+			code_hash: pod.code.as_ref().map_or(SHA3_EMPTY, |c| c.sha3()),
-			filth: Filth::Dirty,
+			code_filth: Filth::Dirty,
+			code_size: Some(pod.code.as_ref().map_or(0, |c| c.len())),
+			code_cache: Arc::new(pod.code.map_or_else(|| { warn!("POD account with unknown code is being created! Assuming no code."); vec![] }, |c| c)),
 			address_hash: Cell::new(None),
 		}
 	}
@@ -80,10 +97,12 @@ impl Account {
 			balance: balance,
 			nonce: nonce,
 			storage_root: SHA3_NULL_RLP,
-			storage_overlay: RefCell::new(HashMap::new()),
+			storage_cache: Self::empty_storage_cache(),
-			code_hash: Some(SHA3_EMPTY),
+			storage_changes: HashMap::new(),
-			code_cache: vec![],
+			code_hash: SHA3_EMPTY,
-			filth: Filth::Dirty,
+			code_cache: Arc::new(vec![]),
+			code_size: Some(0),
+			code_filth: Filth::Clean,
 			address_hash: Cell::new(None),
 		}
 	}
@@ -95,10 +114,12 @@ impl Account {
 			nonce: r.val_at(0),
 			balance: r.val_at(1),
 			storage_root: r.val_at(2),
-			storage_overlay: RefCell::new(HashMap::new()),
+			storage_cache: Self::empty_storage_cache(),
-			code_hash: Some(r.val_at(3)),
+			storage_changes: HashMap::new(),
-			code_cache: vec![],
+			code_hash: r.val_at(3),
-			filth: Filth::Clean,
+			code_cache: Arc::new(vec![]),
+			code_size: None,
+			code_filth: Filth::Clean,
 			address_hash: Cell::new(None),
 		}
 	}
@@ -110,10 +131,12 @@ impl Account {
 			balance: balance,
 			nonce: nonce,
 			storage_root: SHA3_NULL_RLP,
-			storage_overlay: RefCell::new(HashMap::new()),
+			storage_cache: Self::empty_storage_cache(),
-			code_hash: None,
+			storage_changes: HashMap::new(),
-			code_cache: vec![],
+			code_hash: SHA3_EMPTY,
-			filth: Filth::Dirty,
+			code_cache: Arc::new(vec![]),
+			code_size: None,
+			code_filth: Filth::Clean,
 			address_hash: Cell::new(None),
 		}
 	}
@@ -121,35 +144,28 @@ impl Account {
 	/// Set this account's code to the given code.
 	/// NOTE: Account should have been created with `new_contract()`
 	pub fn init_code(&mut self, code: Bytes) {
-		assert!(self.code_hash.is_none());
+		self.code_hash = code.sha3();
-		self.code_cache = code;
+		self.code_cache = Arc::new(code);
-		self.filth = Filth::Dirty;
+		self.code_size = Some(self.code_cache.len());
+		self.code_filth = Filth::Dirty;
 	}
 
 	/// Reset this account's code to the given code.
 	pub fn reset_code(&mut self, code: Bytes) {
-		self.code_hash = None;
 		self.init_code(code);
 	}
 
 	/// Set (and cache) the contents of the trie's storage at `key` to `value`.
 	pub fn set_storage(&mut self, key: H256, value: H256) {
-		match self.storage_overlay.borrow_mut().entry(key) {
+		self.storage_changes.insert(key, value);
-			Entry::Occupied(ref mut entry) if entry.get().1 != value => {
-				entry.insert((Filth::Dirty, value));
-				self.filth = Filth::Dirty;
-			},
-			Entry::Vacant(entry) => {
-				entry.insert((Filth::Dirty, value));
-				self.filth = Filth::Dirty;
-			},
-			_ => (),
-		}
 	}
 
 	/// Get (and cache) the contents of the trie's storage at `key`.
+	/// Takes modifed storage into account.
 	pub fn storage_at(&self, db: &HashDB, key: &H256) -> H256 {
-		self.storage_overlay.borrow_mut().entry(key.clone()).or_insert_with(||{
+		if let Some(value) = self.cached_storage_at(key) {
+			return value;
+		}
 		let db = SecTrieDB::new(db, &self.storage_root)
 			.expect("Account storage_root initially set to zero (valid) and only altered by SecTrieDBMut. \
 			SecTrieDBMut would not set it to an invalid state root. Therefore the root is valid and DB creation \
@@ -159,8 +175,21 @@ impl Account {
 			Ok(x) => x.map_or_else(U256::zero, decode),
 			Err(e) => panic!("Encountered potential DB corruption: {}", e),
 		};
-			(Filth::Clean, item.into())
+		let value: H256 = item.into();
-		}).1.clone()
+		self.storage_cache.borrow_mut().insert(key.clone(), value.clone());
+		value
+	}
+
+	/// Get cached storage value if any. Returns `None` if the
+	/// key is not in the cache.
+	pub fn cached_storage_at(&self, key: &H256) -> Option<H256> {
+		if let Some(value) = self.storage_changes.get(key) {
+			return Some(value.clone())
+		}
+		if let Some(value) = self.storage_cache.borrow_mut().get_mut(key) {
+			return Some(value.clone())
+		}
+		None
 	}
 
 	/// return the balance associated with this account.
@@ -169,10 +198,9 @@ impl Account {
 	/// return the nonce associated with this account.
 	pub fn nonce(&self) -> &U256 { &self.nonce }
 
-	#[cfg(test)]
 	/// return the code hash associated with this account.
 	pub fn code_hash(&self) -> H256 {
-		self.code_hash.clone().unwrap_or(SHA3_EMPTY)
+		self.code_hash.clone()
 	}
 
 	/// return the code hash associated with this account.
@@ -187,41 +215,35 @@ impl Account {
 
 	/// returns the account's code. If `None` then the code cache isn't available -
 	/// get someone who knows to call `note_code`.
-	pub fn code(&self) -> Option<&[u8]> {
+	pub fn code(&self) -> Option<Arc<Bytes>> {
-		match self.code_hash {
+		if self.code_hash != SHA3_EMPTY && self.code_cache.is_empty() {
-			Some(c) if c == SHA3_EMPTY && self.code_cache.is_empty() => Some(&self.code_cache),
+			return None;
-			Some(_) if !self.code_cache.is_empty() => Some(&self.code_cache),
-			None => Some(&self.code_cache),
-			_ => None,
 		}
+		Some(self.code_cache.clone())
+	}
+
+	/// returns the account's code size. If `None` then the code cache or code size cache isn't available -
+	/// get someone who knows to call `note_code`.
+	pub fn code_size(&self) -> Option<usize> {
+		self.code_size.clone()
 	}
 
 	#[cfg(test)]
 	/// Provide a byte array which hashes to the `code_hash`. returns the hash as a result.
 	pub fn note_code(&mut self, code: Bytes) -> Result<(), H256> {
 		let h = code.sha3();
-		match self.code_hash {
+		if self.code_hash == h {
-			Some(ref i) if h == *i => {
+			self.code_cache = Arc::new(code);
-				self.code_cache = code;
+			self.code_size = Some(self.code_cache.len());
 			Ok(())
-			},
+		} else {
-			_ => Err(h)
+			Err(h)
 		}
 	}
 
 	/// Is `code_cache` valid; such that code is going to return Some?
 	pub fn is_cached(&self) -> bool {
-		!self.code_cache.is_empty() || (self.code_cache.is_empty() && self.code_hash == Some(SHA3_EMPTY))
+		!self.code_cache.is_empty() || (self.code_cache.is_empty() && self.code_hash == SHA3_EMPTY)
-	}
-
-	/// Is this a new or modified account?
-	pub fn is_dirty(&self) -> bool {
-		self.filth == Filth::Dirty
-	}
-
-	/// Mark account as clean.
-	pub fn set_clean(&mut self) {
-		self.filth = Filth::Clean
 	}
 
 	/// Provide a database to get `code_hash`. Should not be called if it is a contract without code.
@@ -229,83 +251,100 @@ impl Account {
 		// TODO: fill out self.code_cache;
 		trace!("Account::cache_code: ic={}; self.code_hash={:?}, self.code_cache={}", self.is_cached(), self.code_hash, self.code_cache.pretty());
 		self.is_cached() ||
-			match self.code_hash {
+		match db.get(&self.code_hash) {
-				Some(ref h) => match db.get(h) {
+			Some(x) => {
-					Some(x) => { self.code_cache = x.to_vec(); true },
+				self.code_cache = Arc::new(x.to_vec());
+				self.code_size = Some(x.len());
+				true
+			},
 			_ => {
-						warn!("Failed reverse get of {}", h);
+				warn!("Failed reverse get of {}", self.code_hash);
 				false
 			},
-				},
-				_ => false,
 		}
 	}
 
-	#[cfg(test)]
+	/// Provide a database to get `code_size`. Should not be called if it is a contract without code.
+	pub fn cache_code_size(&mut self, db: &HashDB) -> bool {
+		// TODO: fill out self.code_cache;
+		trace!("Account::cache_code_size: ic={}; self.code_hash={:?}, self.code_cache={}", self.is_cached(), self.code_hash, self.code_cache.pretty());
+		self.code_size.is_some() ||
+			if self.code_hash != SHA3_EMPTY {
+				match db.get(&self.code_hash) {
+					Some(x) => {
+						self.code_size = Some(x.len());
+						true
+					},
+					_ => {
+						warn!("Failed reverse get of {}", self.code_hash);
+						false
+					},
+				}
+			} else {
+				false
+			}
+	}
+
 	/// Determine whether there are any un-`commit()`-ed storage-setting operations.
-	pub fn storage_is_clean(&self) -> bool { self.storage_overlay.borrow().iter().find(|&(_, &(f, _))| f == Filth::Dirty).is_none() }
+	pub fn storage_is_clean(&self) -> bool { self.storage_changes.is_empty() }
 
 	#[cfg(test)]
 	/// return the storage root associated with this account or None if it has been altered via the overlay.
 	pub fn storage_root(&self) -> Option<&H256> { if self.storage_is_clean() {Some(&self.storage_root)} else {None} }
 
 	/// return the storage overlay.
-	pub fn storage_overlay(&self) -> Ref<HashMap<H256, (Filth, H256)>> { self.storage_overlay.borrow() }
+	pub fn storage_changes(&self) -> &HashMap<H256, H256> { &self.storage_changes }
 
 	/// Increment the nonce of the account by one.
 	pub fn inc_nonce(&mut self) {
 		self.nonce = self.nonce + U256::from(1u8);
-		self.filth = Filth::Dirty;
 	}
 
-	/// Increment the nonce of the account by one.
+	/// Increase account balance.
 	pub fn add_balance(&mut self, x: &U256) {
-		if !x.is_zero() {
 		self.balance = self.balance + *x;
-			self.filth = Filth::Dirty;
-		}
 	}
 
-	/// Increment the nonce of the account by one.
+	/// Decrease account balance.
 	/// Panics if balance is less than `x`
 	pub fn sub_balance(&mut self, x: &U256) {
-		if !x.is_zero() {
 		assert!(self.balance >= *x);
 		self.balance = self.balance - *x;
-			self.filth = Filth::Dirty;
-		}
 	}
 
-	/// Commit the `storage_overlay` to the backing DB and update `storage_root`.
+	/// Commit the `storage_changes` to the backing DB and update `storage_root`.
 	pub fn commit_storage(&mut self, trie_factory: &TrieFactory, db: &mut HashDB) {
 		let mut t = trie_factory.from_existing(db, &mut self.storage_root)
 			.expect("Account storage_root initially set to zero (valid) and only altered by SecTrieDBMut. \
 				SecTrieDBMut would not set it to an invalid state root. Therefore the root is valid and DB creation \
 				using it will not fail.");
-		for (k, &mut (ref mut f, ref mut v)) in self.storage_overlay.borrow_mut().iter_mut() {
+		for (k, v) in self.storage_changes.drain() {
-			if f == &Filth::Dirty {
 			// cast key and value to trait type,
 			// so we can call overloaded `to_bytes` method
 			let res = match v.is_zero() {
-					true => t.remove(k),
+				true => t.remove(&k),
-					false => t.insert(k, &encode(&U256::from(&*v))),
+				false => t.insert(&k, &encode(&U256::from(&*v))),
 			};
 
 			if let Err(e) = res {
 				warn!("Encountered potential DB corruption: {}", e);
 			}
-				*f = Filth::Clean;
+			self.storage_cache.borrow_mut().insert(k, v);
-			}
 		}
 	}
 
 	/// Commit any unsaved code. `code_hash` will always return the hash of the `code_cache` after this.
 	pub fn commit_code(&mut self, db: &mut HashDB) {
-		trace!("Commiting code of {:?} - {:?}, {:?}", self, self.code_hash.is_none(), self.code_cache.is_empty());
+		trace!("Commiting code of {:?} - {:?}, {:?}", self, self.code_filth == Filth::Dirty, self.code_cache.is_empty());
-		match (self.code_hash.is_none(), self.code_cache.is_empty()) {
+		match (self.code_filth == Filth::Dirty, self.code_cache.is_empty()) {
-			(true, true) => self.code_hash = Some(SHA3_EMPTY),
+			(true, true) => {
+				self.code_size = Some(0);
+				self.code_filth = Filth::Clean;
+			},
 			(true, false) => {
-				self.code_hash = Some(db.insert(&self.code_cache));
+				db.emplace(self.code_hash.clone(), (*self.code_cache).clone());
+				self.code_size = Some(self.code_cache.len());
+				self.code_filth = Filth::Clean;
 			},
 			(false, _) => {},
 		}
@@ -317,9 +356,59 @@ impl Account {
 		stream.append(&self.nonce);
 		stream.append(&self.balance);
 		stream.append(&self.storage_root);
-		stream.append(self.code_hash.as_ref().expect("Cannot form RLP of contract account without code."));
+		stream.append(&self.code_hash);
 		stream.out()
 	}
+
+	/// Clone basic account data
+	pub fn clone_basic(&self) -> Account {
+		Account {
+			balance: self.balance.clone(),
+			nonce: self.nonce.clone(),
+			storage_root: self.storage_root.clone(),
+			storage_cache: Self::empty_storage_cache(),
+			storage_changes: HashMap::new(),
+			code_hash: self.code_hash.clone(),
+			code_size: self.code_size.clone(),
+			code_cache: self.code_cache.clone(),
+			code_filth: self.code_filth,
+			address_hash: self.address_hash.clone(),
+		}
+	}
+
+	/// Clone account data and dirty storage keys
+	pub fn clone_dirty(&self) -> Account {
+		let mut account = self.clone_basic();
+		account.storage_changes = self.storage_changes.clone();
+		account.code_cache = self.code_cache.clone();
+		account
+	}
+
+	/// Clone account data, dirty storage keys and cached storage keys.
+	pub fn clone_all(&self) -> Account {
+		let mut account = self.clone_dirty();
+		account.storage_cache = self.storage_cache.clone();
+		account
+	}
+
+	/// Replace self with the data from other account merging storage cache.
+	/// Basic account data and all modifications are overwritten
+	/// with new values.
+	pub fn overwrite_with(&mut self, other: Account) {
+		self.balance = other.balance;
+		self.nonce = other.nonce;
+		self.storage_root = other.storage_root;
+		self.code_hash = other.code_hash;
+		self.code_filth = other.code_filth;
+		self.code_cache = other.code_cache;
+		self.code_size = other.code_size;
+		self.address_hash = other.address_hash;
+		let mut cache = self.storage_cache.borrow_mut();
+		for (k, v) in other.storage_cache.into_inner().into_iter() {
+			cache.insert(k.clone() , v.clone()); //TODO: cloning should not be required here
+		}
+		self.storage_changes = other.storage_changes;
+	}
 }
 
 impl fmt::Debug for Account {
@@ -415,7 +504,8 @@ mod tests {
 		let mut db = MemoryDB::new();
 		let mut db = AccountDBMut::new(&mut db, &Address::new());
 		a.init_code(vec![0x55, 0x44, 0xffu8]);
-		assert_eq!(a.code_hash(), SHA3_EMPTY);
+		assert_eq!(a.code_filth, Filth::Dirty);
+		assert_eq!(a.code_size(), Some(3));
 		a.commit_code(&mut db);
 		assert_eq!(a.code_hash().hex(), "af231e631776a517ca23125370d542873eca1fb4d613ed9b5d5335a46ae5b7eb");
 	}
@@ -426,11 +516,12 @@ mod tests {
 		let mut db = MemoryDB::new();
 		let mut db = AccountDBMut::new(&mut db, &Address::new());
 		a.init_code(vec![0x55, 0x44, 0xffu8]);
-		assert_eq!(a.code_hash(), SHA3_EMPTY);
+		assert_eq!(a.code_filth, Filth::Dirty);
 		a.commit_code(&mut db);
+		assert_eq!(a.code_filth, Filth::Clean);
 		assert_eq!(a.code_hash().hex(), "af231e631776a517ca23125370d542873eca1fb4d613ed9b5d5335a46ae5b7eb");
 		a.reset_code(vec![0x55]);
-		assert_eq!(a.code_hash(), SHA3_EMPTY);
+		assert_eq!(a.code_filth, Filth::Dirty);
 		a.commit_code(&mut db);
 		assert_eq!(a.code_hash().hex(), "37bf2238b11b68cdc8382cece82651b59d3c3988873b6e0f33d79694aa45f1be");
 	}

ethcore/src/state/mod.rs

@@ -15,6 +15,7 @@
 // along with Parity.  If not, see <http://www.gnu.org/licenses/>.
 
 use std::cell::{RefCell, RefMut};
+use std::collections::hash_map::Entry;
 use common::*;
 use engines::Engine;
 use executive::{Executive, TransactOptions};
@@ -23,6 +24,7 @@ use trace::FlatTrace;
 use pod_account::*;
 use pod_state::{self, PodState};
 use types::state_diff::StateDiff;
+use state_db::StateDB;
 
 mod account;
 mod substate;
@@ -41,23 +43,166 @@ pub struct ApplyOutcome {
 /// Result type for the execution ("application") of a transaction.
 pub type ApplyResult = Result<ApplyOutcome, Error>;
 
+#[derive(Eq, PartialEq, Clone, Copy, Debug)]
+/// Account modification state. Used to check if the account was
+/// Modified in between commits and overall.
+enum AccountState {
+	/// Account was loaded from disk and never modified in this state object.
+	CleanFresh,
+	/// Account was loaded from the global cache and never modified.
+	CleanCached,
+	/// Account has been modified and is not committed to the trie yet.
+	/// This is set if any of the account data is changed, including
+	/// storage and code.
+	Dirty,
+	/// Account was modified and committed to the trie.
+	Committed,
+}
+
+#[derive(Debug)]
+/// In-memory copy of the account data. Holds the optional account
+/// and the modification status.
+/// Account entry can contain existing (`Some`) or non-existing
+/// account (`None`)
+struct AccountEntry {
+	account: Option<Account>,
+	state: AccountState,
+}
+
+// Account cache item. Contains account data and
+// modification state
+impl AccountEntry {
+	fn is_dirty(&self) -> bool {
+		self.state == AccountState::Dirty
+	}
+
+	/// Clone dirty data into new `AccountEntry`. This includes
+	/// basic account data and modified storage keys.
+	/// Returns None if clean.
+	fn clone_if_dirty(&self) -> Option<AccountEntry> {
+		match self.is_dirty() {
+			true => Some(self.clone_dirty()),
+			false => None,
+		}
+	}
+
+	/// Clone dirty data into new `AccountEntry`. This includes
+	/// basic account data and modified storage keys.
+	fn clone_dirty(&self) -> AccountEntry {
+		AccountEntry {
+			account: self.account.as_ref().map(Account::clone_dirty),
+			state: self.state,
+		}
+	}
+
+	// Create a new account entry and mark it as dirty.
+	fn new_dirty(account: Option<Account>) -> AccountEntry {
+		AccountEntry {
+			account: account,
+			state: AccountState::Dirty,
+		}
+	}
+
+	// Create a new account entry and mark it as clean.
+	fn new_clean(account: Option<Account>) -> AccountEntry {
+		AccountEntry {
+			account: account,
+			state: AccountState::CleanFresh,
+		}
+	}
+
+	// Create a new account entry and mark it as clean and cached.
+	fn new_clean_cached(account: Option<Account>) -> AccountEntry {
+		AccountEntry {
+			account: account,
+			state: AccountState::CleanCached,
+		}
+	}
+
+	// Replace data with another entry but preserve storage cache.
+	fn overwrite_with(&mut self, other: AccountEntry) {
+		self.state = other.state;
+		match other.account {
+			Some(acc) => match self.account {
+				Some(ref mut ours) => {
+					ours.overwrite_with(acc);
+				},
+				None => {},
+			},
+			None => self.account = None,
+		}
+	}
+}
+
 /// Representation of the entire state of all accounts in the system.
+///
+/// `State` can work together with `StateDB` to share account cache.
+///
+/// Local cache contains changes made locally and changes accumulated
+/// locally from previous commits. Global cache reflects the database
+/// state and never contains any changes.
+///
+/// Cache items contains account data, or the flag that account does not exist
+/// and modification state (see `AccountState`)
+///
+/// Account data can be in the following cache states:
+/// * In global but not local - something that was queried from the database,
+/// but never modified
+/// * In local but not global - something that was just added (e.g. new account)
+/// * In both with the same value - something that was changed to a new value,
+/// but changed back to a previous block in the same block (same State instance)
+/// * In both with different values - something that was overwritten with a
+/// new value.
+///
+/// All read-only state queries check local cache/modifications first,
+/// then global state cache. If data is not found in any of the caches
+/// it is loaded from the DB to the local cache.
+///
+/// **** IMPORTANT *************************************************************
+/// All the modifications to the account data must set the `Dirty` state in the
+/// `AccountEntry`. This is done in `require` and `require_or_from`. So just
+/// use that.
+/// ****************************************************************************
+///
+/// Upon destruction all the local cache data propagated into the global cache.
+/// Propagated items might be rejected if current state is non-canonical.
+///
+/// State snapshotting.
+///
+/// A new snapshot can be created with `snapshot()`. Snapshots can be
+/// created in a hierarchy.
+/// When a snapshot is active all changes are applied directly into
+/// `cache` and the original value is copied into an active snapshot.
+/// Reverting a snapshot with `revert_to_snapshot` involves copying
+/// original values from the latest snapshot back into `cache`. The code
+/// takes care not to overwrite cached storage while doing that.
+/// Snapshot can be discateded with `discard_snapshot`. All of the orignal
+/// backed-up values are moved into a parent snapshot (if any).
+///
 pub struct State {
-	db: Box<JournalDB>,
+	db: StateDB,
 	root: H256,
-	cache: RefCell<HashMap<Address, Option<Account>>>,
+	cache: RefCell<HashMap<Address, AccountEntry>>,
-	snapshots: RefCell<Vec<HashMap<Address, Option<Option<Account>>>>>,
+	// The original account is preserved in
+	snapshots: RefCell<Vec<HashMap<Address, Option<AccountEntry>>>>,
 	account_start_nonce: U256,
 	factories: Factories,
 }
 
+#[derive(Copy, Clone)]
+enum RequireCache {
+	None,
+	CodeSize,
+	Code,
+}
+
 const SEC_TRIE_DB_UNWRAP_STR: &'static str = "A state can only be created with valid root. Creating a SecTrieDB with a valid root will not fail. \
 			 Therefore creating a SecTrieDB with this state's root will not fail.";
 
 impl State {
 	/// Creates new state with empty state root
 	#[cfg(test)]
-	pub fn new(mut db: Box<JournalDB>, account_start_nonce: U256, factories: Factories) -> State {
+	pub fn new(mut db: StateDB, account_start_nonce: U256, factories: Factories) -> State {
 		let mut root = H256::new();
 		{
 			// init trie and reset root too null
@@ -75,7 +220,7 @@ impl State {
 	}
 
 	/// Creates new state with existing state root
-	pub fn from_existing(db: Box<JournalDB>, root: H256, account_start_nonce: U256, factories: Factories) -> Result<State, TrieError> {
+	pub fn from_existing(db: StateDB, root: H256, account_start_nonce: U256, factories: Factories) -> Result<State, TrieError> {
 		if !db.as_hashdb().contains(&root) {
 			return Err(TrieError::InvalidStateRoot(root));
 		}
@@ -92,60 +237,88 @@ impl State {
 		Ok(state)
 	}
 
-	/// Create a recoverable snaphot of this state
+	/// Create a recoverable snaphot of this state.
 	pub fn snapshot(&mut self) {
 		self.snapshots.borrow_mut().push(HashMap::new());
 	}
 
-	/// Merge last snapshot with previous
+	/// Merge last snapshot with previous.
-	pub fn clear_snapshot(&mut self) {
+	pub fn discard_snapshot(&mut self) {
 		// merge with previous snapshot
 		let last = self.snapshots.borrow_mut().pop();
 		if let Some(mut snapshot) = last {
 			if let Some(ref mut prev) = self.snapshots.borrow_mut().last_mut() {
+				if prev.is_empty() {
+					**prev = snapshot;
+				} else {
 					for (k, v) in snapshot.drain() {
 						prev.entry(k).or_insert(v);
 					}
 				}
 			}
 		}
+	}
 
-	/// Revert to snapshot
+	/// Revert to the last snapshot and discard it.
-	pub fn revert_snapshot(&mut self) {
+	pub fn revert_to_snapshot(&mut self) {
 		if let Some(mut snapshot) = self.snapshots.borrow_mut().pop() {
 			for (k, v) in snapshot.drain() {
 				match v {
 					Some(v) => {
-						self.cache.borrow_mut().insert(k, v);
+						match self.cache.borrow_mut().entry(k) {
+							Entry::Occupied(mut e) => {
+								// Merge snapshotted changes back into the main account
+								// storage preserving the cache.
+								e.get_mut().overwrite_with(v);
+							},
+							Entry::Vacant(e) => {
+								e.insert(v);
+							}
+						}
 					},
 					None => {
-						self.cache.borrow_mut().remove(&k);
+						match self.cache.borrow_mut().entry(k) {
+							Entry::Occupied(e) => {
+								if e.get().is_dirty() {
+									e.remove();
+								}
+							},
+							_ => {}
+						}
 					}
 				}
 			}
 		}
 	}
 
-	fn insert_cache(&self, address: &Address, account: Option<Account>) {
+	fn insert_cache(&self, address: &Address, account: AccountEntry) {
+		// Dirty account which is not in the cache means this is a new account.
+		// It goes directly into the snapshot as there's nothing to rever to.
+		//
+		// In all other cases account is read as clean first, and after that made
+		// dirty in and added to the snapshot with `note_cache`.
+		if account.is_dirty() {
 			if let Some(ref mut snapshot) = self.snapshots.borrow_mut().last_mut() {
 				if !snapshot.contains_key(address) {
 					snapshot.insert(address.clone(), self.cache.borrow_mut().insert(address.clone(), account));
 					return;
 				}
 			}
+		}
 		self.cache.borrow_mut().insert(address.clone(), account);
 	}
 
 	fn note_cache(&self, address: &Address) {
 		if let Some(ref mut snapshot) = self.snapshots.borrow_mut().last_mut() {
 			if !snapshot.contains_key(address) {
-				snapshot.insert(address.clone(), self.cache.borrow().get(address).cloned());
+				snapshot.insert(address.clone(), self.cache.borrow().get(address).map(AccountEntry::clone_dirty));
 			}
 		}
 	}
 
 	/// Destroy the current object and return root and database.
-	pub fn drop(self) -> (H256, Box<JournalDB>) {
+	pub fn drop(mut self) -> (H256, StateDB) {
+		self.propagate_to_global_cache();
 		(self.root, self.db)
 	}
 
@@ -157,63 +330,121 @@ impl State {
 	/// Create a new contract at address `contract`. If there is already an account at the address
 	/// it will have its code reset, ready for `init_code()`.
 	pub fn new_contract(&mut self, contract: &Address, balance: U256) {
-		self.insert_cache(contract, Some(Account::new_contract(balance, self.account_start_nonce)));
+		self.insert_cache(contract, AccountEntry::new_dirty(Some(Account::new_contract(balance, self.account_start_nonce))));
 	}
 
 	/// Remove an existing account.
 	pub fn kill_account(&mut self, account: &Address) {
-		self.insert_cache(account, None);
+		self.insert_cache(account, AccountEntry::new_dirty(None));
 	}
 
 	/// Determine whether an account exists.
 	pub fn exists(&self, a: &Address) -> bool {
-		self.ensure_cached(a, false, |a| a.is_some())
+		self.ensure_cached(a, RequireCache::None, |a| a.is_some())
 	}
 
 	/// Get the balance of account `a`.
 	pub fn balance(&self, a: &Address) -> U256 {
-		self.ensure_cached(a, false,
+		self.ensure_cached(a, RequireCache::None,
 			|a| a.as_ref().map_or(U256::zero(), |account| *account.balance()))
 	}
 
 	/// Get the nonce of account `a`.
 	pub fn nonce(&self, a: &Address) -> U256 {
-		self.ensure_cached(a, false,
+		self.ensure_cached(a, RequireCache::None,
 			|a| a.as_ref().map_or(self.account_start_nonce, |account| *account.nonce()))
 	}
 
 	/// Mutate storage of account `address` so that it is `value` for `key`.
 	pub fn storage_at(&self, address: &Address, key: &H256) -> H256 {
-		self.ensure_cached(address, false, |a| a.as_ref().map_or(H256::new(), |a| {
+		// Storage key search and update works like this:
-			let addr_hash = a.address_hash(address);
+		// 1. If there's an entry for the account in the local cache check for the key and return it if found.
-			let db = self.factories.accountdb.readonly(self.db.as_hashdb(), addr_hash);
+		// 2. If there's an entry for the account in the global cache check for the key or load it into that account.
-			a.storage_at(db.as_hashdb(), key)
+		// 3. If account is missing in the global cache load it into the local cache and cache the key there.
-		}))
+
+		// check local cache first without updating
+		{
+			let local_cache = self.cache.borrow_mut();
+			let mut local_account = None;
+			if let Some(maybe_acc) = local_cache.get(address) {
+				match maybe_acc.account {
+					Some(ref account) => {
+						if let Some(value) = account.cached_storage_at(key) {
+							return value;
+						} else {
+							local_account = Some(maybe_acc);
+						}
+					},
+					_ => return H256::new(),
+				}
+			}
+			// check the global cache and and cache storage key there if found,
+			// otherwise cache the account localy and cache storage key there.
+			if let Some(result) = self.db.get_cached(address, |acc| acc.map_or(H256::new(), |a| {
+					let account_db = self.factories.accountdb.readonly(self.db.as_hashdb(), a.address_hash(address));
+					a.storage_at(account_db.as_hashdb(), key)
+				})) {
+				return result;
+			}
+			if let Some(ref mut acc) = local_account {
+				if let Some(ref account) = acc.account {
+					let account_db = self.factories.accountdb.readonly(self.db.as_hashdb(), account.address_hash(address));
+					return account.storage_at(account_db.as_hashdb(), key)
+				} else {
+					return H256::new()
+				}
+			}
 		}
 
-	/// Get the code of account `a`.
+		// check bloom before any requests to trie
-	pub fn code(&self, a: &Address) -> Option<Bytes> {
+		if !self.db.check_account_bloom(address) { return H256::zero() }
-		self.ensure_cached(a, true,
+
-			|a| a.as_ref().map_or(None, |a| a.code().map(|x| x.to_vec())))
+		// account is not found in the global cache, get from the DB and insert into local
+		let db = self.factories.trie.readonly(self.db.as_hashdb(), &self.root).expect(SEC_TRIE_DB_UNWRAP_STR);
+		let maybe_acc = match db.get(address) {
+			Ok(acc) => acc.map(Account::from_rlp),
+			Err(e) => panic!("Potential DB corruption encountered: {}", e),
+		};
+		let r = maybe_acc.as_ref().map_or(H256::new(), |a| {
+			let account_db = self.factories.accountdb.readonly(self.db.as_hashdb(), a.address_hash(address));
+			a.storage_at(account_db.as_hashdb(), key)
+		});
+		self.insert_cache(address, AccountEntry::new_clean(maybe_acc));
+		r
 	}
 
-	/// Get the code size of account `a`.
+	/// Get accounts' code.
+	pub fn code(&self, a: &Address) -> Option<Arc<Bytes>> {
+		self.ensure_cached(a, RequireCache::Code,
+			|a| a.as_ref().map_or(None, |a| a.code().clone()))
+	}
+
+	pub fn code_hash(&self, a: &Address) -> H256 {
+		self.ensure_cached(a, RequireCache::None,
+			|a| a.as_ref().map_or(SHA3_EMPTY, |a| a.code_hash()))
+	}
+
+	/// Get accounts' code size.
 	pub fn code_size(&self, a: &Address) -> Option<usize> {
-		self.ensure_cached(a, true,
+		self.ensure_cached(a, RequireCache::CodeSize,
-			|a| a.as_ref().map_or(None, |a| a.code().map(|x| x.len())))
+			|a| a.as_ref().and_then(|a| a.code_size()))
 	}
 
 	/// Add `incr` to the balance of account `a`.
 	pub fn add_balance(&mut self, a: &Address, incr: &U256) {
 		trace!(target: "state", "add_balance({}, {}): {}", a, incr, self.balance(a));
+		if !incr.is_zero() || !self.exists(a) {
 			self.require(a, false).add_balance(incr);
 		}
+	}
 
 	/// Subtract `decr` from the balance of account `a`.
 	pub fn sub_balance(&mut self, a: &Address, decr: &U256) {
 		trace!(target: "state", "sub_balance({}, {}): {}", a, decr, self.balance(a));
+		if !decr.is_zero() || !self.exists(a) {
 			self.require(a, false).sub_balance(decr);
 		}
+	}
 
 	/// Subtracts `by` from the balance of `from` and adds it to that of `to`.
 	pub fn transfer_balance(&mut self, from: &Address, to: &Address, by: &U256) {
@@ -228,8 +459,10 @@ impl State {
 
 	/// Mutate storage of account `a` so that it is `value` for `key`.
 	pub fn set_storage(&mut self, a: &Address, key: H256, value: H256) {
+		if self.storage_at(a, &key) != value {
 			self.require(a, false).set_storage(key, value)
 		}
+	}
 
 	/// Initialise the code of account `a` so that it is `code`.
 	/// NOTE: Account should have been created with `new_contract`.
@@ -262,19 +495,19 @@ impl State {
 	/// Commit accounts to SecTrieDBMut. This is similar to cpp-ethereum's dev::eth::commit.
 	/// `accounts` is mutable because we may need to commit the code or storage and record that.
 	#[cfg_attr(feature="dev", allow(match_ref_pats))]
-	pub fn commit_into(
+	fn commit_into(
 		factories: &Factories,
-		db: &mut HashDB,
+		db: &mut StateDB,
 		root: &mut H256,
-		accounts: &mut HashMap<Address, Option<Account>>
+		accounts: &mut HashMap<Address, AccountEntry>
 	) -> Result<(), Error> {
 		// first, commit the sub trees.
-		// TODO: is this necessary or can we dispense with the `ref mut a` for just `a`?
+		for (address, ref mut a) in accounts.iter_mut().filter(|&(_, ref a)| a.is_dirty()) {
-		for (address, ref mut a) in accounts.iter_mut() {
+			match a.account {
-			match a {
+				Some(ref mut account) => {
-				&mut&mut Some(ref mut account) if account.is_dirty() => {
+					db.note_account_bloom(&address);
 					let addr_hash = account.address_hash(address);
-					let mut account_db = factories.accountdb.create(db, addr_hash);
+					let mut account_db = factories.accountdb.create(db.as_hashdb_mut(), addr_hash);
 					account.commit_storage(&factories.trie, account_db.as_hashdb_mut());
 					account.commit_code(account_db.as_hashdb_mut());
 				}
@@ -283,15 +516,16 @@ impl State {
 		}
 
 		{
-			let mut trie = factories.trie.from_existing(db, root).unwrap();
+			let mut trie = factories.trie.from_existing(db.as_hashdb_mut(), root).unwrap();
-			for (address, ref mut a) in accounts.iter_mut() {
+			for (address, ref mut a) in accounts.iter_mut().filter(|&(_, ref a)| a.is_dirty()) {
-				match **a {
+				a.state = AccountState::Committed;
-					Some(ref mut account) if account.is_dirty() => {
+				match a.account {
-						account.set_clean();
+					Some(ref mut account) => {
-						try!(trie.insert(address, &account.rlp()))
+						try!(trie.insert(address, &account.rlp()));
+					},
+					None => {
+						try!(trie.remove(address));
 					},
-					None => try!(trie.remove(address)),
-					_ => (),
 				}
 			}
 		}
@@ -299,10 +533,19 @@ impl State {
 		Ok(())
 	}
 
+	/// Propagate local cache into shared canonical state cache.
+	fn propagate_to_global_cache(&mut self) {
+		let mut addresses = self.cache.borrow_mut();
+		trace!("Committing cache {:?} entries", addresses.len());
+		for (address, a) in addresses.drain().filter(|&(_, ref a)| a.state == AccountState::Committed || a.state == AccountState::CleanFresh) {
+			self.db.add_to_account_cache(address, a.account, a.state == AccountState::Committed);
+		}
+	}
+
 	/// Commits our cached account changes into the trie.
 	pub fn commit(&mut self) -> Result<(), Error> {
 		assert!(self.snapshots.borrow().is_empty());
-		Self::commit_into(&self.factories, self.db.as_hashdb_mut(), &mut self.root, &mut *self.cache.borrow_mut())
+		Self::commit_into(&self.factories, &mut self.db, &mut self.root, &mut *self.cache.borrow_mut())
 	}
 
 	/// Clear state cache
@@ -316,7 +559,8 @@ impl State {
 	pub fn populate_from(&mut self, accounts: PodState) {
 		assert!(self.snapshots.borrow().is_empty());
 		for (add, acc) in accounts.drain().into_iter() {
-			self.cache.borrow_mut().insert(add, Some(Account::from_pod(acc)));
+			self.db.note_account_bloom(&add);
+			self.cache.borrow_mut().insert(add, AccountEntry::new_dirty(Some(Account::from_pod(acc))));
 		}
 	}
 
@@ -326,7 +570,7 @@ impl State {
 		// TODO: handle database rather than just the cache.
 		// will need fat db.
 		PodState::from(self.cache.borrow().iter().fold(BTreeMap::new(), |mut m, (add, opt)| {
-			if let Some(ref acc) = *opt {
+			if let Some(ref acc) = opt.account {
 				m.insert(add.clone(), PodAccount::from_account(acc));
 			}
 			m
@@ -335,7 +579,7 @@ impl State {
 
 	fn query_pod(&mut self, query: &PodState) {
 		for (address, pod_account) in query.get() {
-			self.ensure_cached(address, true, |a| {
+			self.ensure_cached(address, RequireCache::Code, |a| {
 				if a.is_some() {
 					for key in pod_account.storage.keys() {
 						self.storage_at(address, key);
@@ -354,28 +598,61 @@ impl State {
 		pod_state::diff_pod(&state_pre.to_pod(), &pod_state_post)
 	}
 
-	/// Ensure account `a` is in our cache of the trie DB and return a handle for getting it.
+	fn update_account_cache(require: RequireCache, account: &mut Account, db: &HashDB) {
-	/// `require_code` requires that the code be cached, too.
+		match require {
-	fn ensure_cached<'a, F, U>(&'a self, a: &'a Address, require_code: bool, f: F) -> U
+			RequireCache::None => {},
-		where F: FnOnce(&Option<Account>) -> U {
+			RequireCache::Code => {
-		let have_key = self.cache.borrow().contains_key(a);
+				account.cache_code(db);
-		if !have_key {
+			}
-			let db = self.factories.trie.readonly(self.db.as_hashdb(), &self.root).expect(SEC_TRIE_DB_UNWRAP_STR);
+			RequireCache::CodeSize => {
-			let maybe_acc = match db.get(a) {
+				account.cache_code_size(db);
-				Ok(acc) => acc.map(Account::from_rlp),
-				Err(e) => panic!("Potential DB corruption encountered: {}", e),
-			};
-			self.insert_cache(a, maybe_acc);
 			}
-		if require_code {
-			if let Some(ref mut account) = self.cache.borrow_mut().get_mut(a).unwrap().as_mut() {
-				let addr_hash = account.address_hash(a);
-				let accountdb = self.factories.accountdb.readonly(self.db.as_hashdb(), addr_hash);
-				account.cache_code(accountdb.as_hashdb());
 		}
 	}
 
-		f(self.cache.borrow().get(a).unwrap())
+	/// Check caches for required data
+	/// First searches for account in the local, then the shared cache.
+	/// Populates local cache if nothing found.
+	fn ensure_cached<F, U>(&self, a: &Address, require: RequireCache, f: F) -> U
+		where F: Fn(Option<&Account>) -> U {
+		// check local cache first
+		if let Some(ref mut maybe_acc) = self.cache.borrow_mut().get_mut(a) {
+			if let Some(ref mut account) = maybe_acc.account {
+				let accountdb = self.factories.accountdb.readonly(self.db.as_hashdb(), account.address_hash(a));
+				Self::update_account_cache(require, account, accountdb.as_hashdb());
+				return f(Some(account));
+			}
+			return f(None);
+		}
+		// check global cache
+		let result = self.db.get_cached(a, |mut acc| {
+			if let Some(ref mut account) = acc {
+				let accountdb = self.factories.accountdb.readonly(self.db.as_hashdb(), account.address_hash(a));
+				Self::update_account_cache(require, account, accountdb.as_hashdb());
+			}
+			f(acc.map(|a| &*a))
+		});
+		match result {
+			Some(r) => r,
+			None => {
+				// first check bloom if it is not in database for sure
+				if !self.db.check_account_bloom(a) { return f(None); }
+
+				// not found in the global cache, get from the DB and insert into local
+				let db = self.factories.trie.readonly(self.db.as_hashdb(), &self.root).expect(SEC_TRIE_DB_UNWRAP_STR);
+				let mut maybe_acc = match db.get(a) {
+					Ok(acc) => acc.map(Account::from_rlp),
+					Err(e) => panic!("Potential DB corruption encountered: {}", e),
+				};
+				if let Some(ref mut account) = maybe_acc.as_mut() {
+					let accountdb = self.factories.accountdb.readonly(self.db.as_hashdb(), account.address_hash(a));
+					Self::update_account_cache(require, account, accountdb.as_hashdb());
+				}
+				let r = f(maybe_acc.as_ref());
+				self.insert_cache(a, AccountEntry::new_clean(maybe_acc));
+				r
+			}
+		}
 	}
 
 	/// Pull account `a` in our cache from the trie DB. `require_code` requires that the code be cached, too.
@@ -390,30 +667,48 @@ impl State {
 	{
 		let contains_key = self.cache.borrow().contains_key(a);
 		if !contains_key {
+			match self.db.get_cached_account(a) {
+				Some(acc) => self.insert_cache(a, AccountEntry::new_clean_cached(acc)),
+				None => {
+					let maybe_acc = if self.db.check_account_bloom(a) {
 						let db = self.factories.trie.readonly(self.db.as_hashdb(), &self.root).expect(SEC_TRIE_DB_UNWRAP_STR);
 						let maybe_acc = match db.get(a) {
-				Ok(acc) => acc.map(Account::from_rlp),
+							Ok(Some(acc)) => AccountEntry::new_clean(Some(Account::from_rlp(acc))),
+							Ok(None) => AccountEntry::new_clean(None),
 							Err(e) => panic!("Potential DB corruption encountered: {}", e),
 						};
-
+						maybe_acc
+					}
+					else {
+						AccountEntry::new_clean(None)
+					};
 					self.insert_cache(a, maybe_acc);
-		} else {
+				}
+			}
+		}
 		self.note_cache(a);
-		}
 
-		match self.cache.borrow_mut().get_mut(a).unwrap() {
+		match &mut self.cache.borrow_mut().get_mut(a).unwrap().account {
 			&mut Some(ref mut acc) => not_default(acc),
-			slot @ &mut None => *slot = Some(default()),
+			slot => *slot = Some(default()),
 		}
 
+		// at this point the account is guaranteed to be in the cache.
 		RefMut::map(self.cache.borrow_mut(), |c| {
-			let account = c.get_mut(a).unwrap().as_mut().unwrap();
+			let mut entry = c.get_mut(a).unwrap();
+			// set the dirty flag after changing account data.
+			entry.state = AccountState::Dirty;
+			match entry.account {
+				Some(ref mut account) => {
 					if require_code {
 						let addr_hash = account.address_hash(a);
 						let accountdb = self.factories.accountdb.readonly(self.db.as_hashdb(), addr_hash);
 						account.cache_code(accountdb.as_hashdb());
 					}
 					account
+				},
+				_ => panic!("Required account must always exist; qed"),
+			}
 		})
 	}
 }
@@ -427,17 +722,10 @@ impl fmt::Debug for State {
 impl Clone for State {
 	fn clone(&self) -> State {
 		let cache = {
-			let mut cache = HashMap::new();
+			let mut cache: HashMap<Address, AccountEntry> = HashMap::new();
 			for (key, val) in self.cache.borrow().iter() {
-				let key = key.clone();
+				if let Some(entry) = val.clone_if_dirty() {
-				match *val {
+					cache.insert(key.clone(), entry);
-					Some(ref acc) if acc.is_dirty() => {
-						cache.insert(key, Some(acc.clone()));
-					},
-					None => {
-						cache.insert(key, None);
-					},
-					_ => {},
 				}
 			}
 			cache
@@ -447,7 +735,7 @@ impl Clone for State {
 			db: self.db.boxed_clone(),
 			root: self.root.clone(),
 			cache: RefCell::new(cache),
-			snapshots: RefCell::new(self.snapshots.borrow().clone()),
+			snapshots: RefCell::new(Vec::new()),
 			account_start_nonce: self.account_start_nonce.clone(),
 			factories: self.factories.clone(),
 		}
@@ -457,6 +745,7 @@ impl Clone for State {
 #[cfg(test)]
 mod tests {
 
+use std::sync::Arc;
 use std::str::FromStr;
 use rustc_serialize::hex::FromHex;
 use super::*;
@@ -1321,14 +1610,14 @@ fn code_from_database() {
 		let mut state = get_temp_state_in(temp.as_path());
 		state.require_or_from(&a, false, ||Account::new_contract(42.into(), 0.into()), |_|{});
 		state.init_code(&a, vec![1, 2, 3]);
-		assert_eq!(state.code(&a), Some([1u8, 2, 3].to_vec()));
+		assert_eq!(state.code(&a), Some(Arc::new([1u8, 2, 3].to_vec())));
 		state.commit().unwrap();
-		assert_eq!(state.code(&a), Some([1u8, 2, 3].to_vec()));
+		assert_eq!(state.code(&a), Some(Arc::new([1u8, 2, 3].to_vec())));
 		state.drop()
 	};
 
 	let state = State::from_existing(db, root, U256::from(0u8), Default::default()).unwrap();
-	assert_eq!(state.code(&a), Some([1u8, 2, 3].to_vec()));
+	assert_eq!(state.code(&a), Some(Arc::new([1u8, 2, 3].to_vec())));
 }
 
 #[test]
@@ -1476,12 +1765,12 @@ fn snapshot_basic() {
 	state.snapshot();
 	state.add_balance(&a, &U256::from(69u64));
 	assert_eq!(state.balance(&a), U256::from(69u64));
-	state.clear_snapshot();
+	state.discard_snapshot();
 	assert_eq!(state.balance(&a), U256::from(69u64));
 	state.snapshot();
 	state.add_balance(&a, &U256::from(1u64));
 	assert_eq!(state.balance(&a), U256::from(70u64));
-	state.revert_snapshot();
+	state.revert_to_snapshot();
 	assert_eq!(state.balance(&a), U256::from(69u64));
 }
 
@@ -1494,9 +1783,9 @@ fn snapshot_nested() {
 	state.snapshot();
 	state.add_balance(&a, &U256::from(69u64));
 	assert_eq!(state.balance(&a), U256::from(69u64));
-	state.clear_snapshot();
+	state.discard_snapshot();
 	assert_eq!(state.balance(&a), U256::from(69u64));
-	state.revert_snapshot();
+	state.revert_to_snapshot();
 	assert_eq!(state.balance(&a), U256::from(0));
 }
 

ethcore/src/state_db.rs

@@ -0,0 +1,503 @@
+// Copyright 2015, 2016 Ethcore (UK) Ltd.
+// This file is part of Parity.
+
+// Parity is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Parity is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Parity.  If not, see <http://www.gnu.org/licenses/>.
+
+use std::collections::{VecDeque, HashSet};
+use lru_cache::LruCache;
+use util::journaldb::JournalDB;
+use util::hash::{H256};
+use util::hashdb::HashDB;
+use state::Account;
+use header::BlockNumber;
+use util::{Arc, Address, Database, DBTransaction, UtilError, Mutex, Hashable};
+use bloom_journal::{Bloom, BloomJournal};
+use db::COL_ACCOUNT_BLOOM;
+use byteorder::{LittleEndian, ByteOrder};
+
+pub const ACCOUNT_BLOOM_SPACE: usize = 1048576;
+pub const DEFAULT_ACCOUNT_PRESET: usize = 1000000;
+
+pub const ACCOUNT_BLOOM_HASHCOUNT_KEY: &'static [u8] = b"account_hash_count";
+
+const STATE_CACHE_BLOCKS: usize = 8;
+
+
+/// Shared canonical state cache.
+struct AccountCache {
+	/// DB Account cache. `None` indicates that account is known to be missing.
+	// When changing the type of the values here, be sure to update `mem_used` and
+	// `new`.
+	accounts: LruCache<Address, Option<Account>>,
+	/// Information on the modifications in recently committed blocks; specifically which addresses
+	/// changed in which block. Ordered by block number.
+	modifications: VecDeque<BlockChanges>,
+}
+
+/// Buffered account cache item.
+struct CacheQueueItem {
+	/// Account address.
+	address: Address,
+	/// Acccount data or `None` if account does not exist.
+	account: Option<Account>,
+	/// Indicates that the account was modified before being
+	/// added to the cache.
+	modified: bool,
+}
+
+#[derive(Debug)]
+/// Accumulates a list of accounts changed in a block.
+struct BlockChanges {
+	/// Block number.
+	number: BlockNumber,
+	/// Block hash.
+	hash: H256,
+	/// Parent block hash.
+	parent: H256,
+	/// A set of modified account addresses.
+	accounts: HashSet<Address>,
+	/// Block is part of the canonical chain.
+	is_canon: bool,
+}
+
+/// State database abstraction.
+/// Manages shared global state cache which reflects the canonical
+/// state as it is on the disk. All the entries in the cache are clean.
+/// A clone of `StateDB` may be created as canonical or not.
+/// For canonical clones local cache is accumulated and applied
+/// in `sync_cache`
+/// For non-canonical clones local cache is dropped.
+///
+/// Global cache propagation.
+/// After a `State` object has been committed to the trie it
+/// propagates its local cache into the `StateDB` local cache
+/// using `add_to_account_cache` function.
+/// Then, after the block has been added to the chain the local cache in the
+/// `StateDB` is propagated into the global cache.
+pub struct StateDB {
+	/// Backing database.
+	db: Box<JournalDB>,
+	/// Shared canonical state cache.
+	account_cache: Arc<Mutex<AccountCache>>,
+	/// Local dirty cache.
+	local_cache: Vec<CacheQueueItem>,
+	/// Shared account bloom. Does not handle chain reorganizations.
+	account_bloom: Arc<Mutex<Bloom>>,
+	cache_size: usize,
+	/// Hash of the block on top of which this instance was created or
+	/// `None` if cache is disabled
+	parent_hash: Option<H256>,
+	/// Hash of the committing block or `None` if not committed yet.
+	commit_hash: Option<H256>,
+	/// Number of the committing block or `None` if not committed yet.
+	commit_number: Option<BlockNumber>,
+}
+
+impl StateDB {
+
+	/// Create a new instance wrapping `JournalDB` and the maximum allowed size
+	/// of the LRU cache in bytes. Actual used memory may (read: will) be higher due to bookkeeping.
+	// TODO: make the cache size actually accurate by moving the account storage cache
+	// into the `AccountCache` structure as its own `LruCache<(Address, H256), H256>`.
+	pub fn new(db: Box<JournalDB>, cache_size: usize) -> StateDB {
+		let bloom = Self::load_bloom(db.backing());
+		let cache_items = cache_size / ::std::mem::size_of::<Option<Account>>();
+
+		StateDB {
+			db: db,
+			account_cache: Arc::new(Mutex::new(AccountCache {
+				accounts: LruCache::new(cache_items),
+				modifications: VecDeque::new(),
+			})),
+			local_cache: Vec::new(),
+			account_bloom: Arc::new(Mutex::new(bloom)),
+			cache_size: cache_size,
+			parent_hash: None,
+			commit_hash: None,
+			commit_number: None,
+		}
+	}
+
+	/// Loads accounts bloom from the database
+	/// This bloom is used to handle request for the non-existant account fast
+	pub fn load_bloom(db: &Database) -> Bloom {
+		let hash_count_entry = db.get(COL_ACCOUNT_BLOOM, ACCOUNT_BLOOM_HASHCOUNT_KEY)
+			.expect("Low-level database error");
+
+		if hash_count_entry.is_none() {
+			return Bloom::new(ACCOUNT_BLOOM_SPACE, DEFAULT_ACCOUNT_PRESET);
+		}
+		let hash_count_bytes = hash_count_entry.unwrap();
+		assert_eq!(hash_count_bytes.len(), 1);
+		let hash_count = hash_count_bytes[0];
+
+		let mut bloom_parts = vec![0u64; ACCOUNT_BLOOM_SPACE / 8];
+		let mut key = [0u8; 8];
+		for i in 0..ACCOUNT_BLOOM_SPACE / 8 {
+			LittleEndian::write_u64(&mut key, i as u64);
+			bloom_parts[i] = db.get(COL_ACCOUNT_BLOOM, &key).expect("low-level database error")
+				.and_then(|val| Some(LittleEndian::read_u64(&val[..])))
+				.unwrap_or(0u64);
+		}
+
+		let bloom = Bloom::from_parts(&bloom_parts, hash_count as u32);
+		trace!(target: "account_bloom", "Bloom is {:?} full, hash functions count = {:?}", bloom.saturation(), hash_count);
+		bloom
+	}
+
+	pub fn check_account_bloom(&self, address: &Address) -> bool {
+		trace!(target: "account_bloom", "Check account bloom: {:?}", address);
+		let bloom = self.account_bloom.lock();
+		bloom.check(&*address.sha3())
+	}
+
+	pub fn note_account_bloom(&self, address: &Address) {
+		trace!(target: "account_bloom", "Note account bloom: {:?}", address);
+		let mut bloom = self.account_bloom.lock();
+		bloom.set(&*address.sha3());
+	}
+
+	pub fn commit_bloom(batch: &mut DBTransaction, journal: BloomJournal) -> Result<(), UtilError> {
+		assert!(journal.hash_functions <= 255);
+		batch.put(COL_ACCOUNT_BLOOM, ACCOUNT_BLOOM_HASHCOUNT_KEY, &vec![journal.hash_functions as u8]);
+		let mut key = [0u8; 8];
+		let mut val = [0u8; 8];
+
+		for (bloom_part_index, bloom_part_value) in journal.entries {
+			LittleEndian::write_u64(&mut key, bloom_part_index as u64);
+			LittleEndian::write_u64(&mut val, bloom_part_value);
+			batch.put(COL_ACCOUNT_BLOOM, &key, &val);
+		}
+		Ok(())
+	}
+
+	/// Journal all recent operations under the given era and ID.
+	pub fn journal_under(&mut self, batch: &mut DBTransaction, now: u64, id: &H256) -> Result<u32, UtilError> {
+		{
+ 			let mut bloom_lock = self.account_bloom.lock();
+ 			try!(Self::commit_bloom(batch, bloom_lock.drain_journal()));
+ 		}
+		let records = try!(self.db.journal_under(batch, now, id));
+		self.commit_hash = Some(id.clone());
+		self.commit_number = Some(now);
+		Ok(records)
+	}
+
+	/// Mark a given candidate from an ancient era as canonical, enacting its removals from the
+	/// backing database and reverting any non-canonical historical commit's insertions.
+	pub fn mark_canonical(&mut self, batch: &mut DBTransaction, end_era: u64, canon_id: &H256) -> Result<u32, UtilError> {
+		self.db.mark_canonical(batch, end_era, canon_id)
+	}
+
+	/// Propagate local cache into the global cache and synchonize
+	/// the global cache with the best block state.
+	/// This function updates the global cache by removing entries
+	/// that are invalidated by chain reorganization. `sync_cache`
+	/// should be called after the block has been committed and the
+	/// blockchain route has ben calculated.
+	pub fn sync_cache(&mut self, enacted: &[H256], retracted: &[H256], is_best: bool) {
+		trace!("sync_cache id = (#{:?}, {:?}), parent={:?}, best={}", self.commit_number, self.commit_hash, self.parent_hash, is_best);
+		let mut cache = self.account_cache.lock();
+		let mut cache = &mut *cache;
+
+		// Purge changes from re-enacted and retracted blocks.
+		// Filter out commiting block if any.
+		let mut clear = false;
+		for block in enacted.iter().filter(|h| self.commit_hash.as_ref().map_or(true, |p| *h != p)) {
+			clear = clear || {
+				if let Some(ref mut m) = cache.modifications.iter_mut().find(|ref m| &m.hash == block) {
+					trace!("Reverting enacted block {:?}", block);
+					m.is_canon = true;
+					for a in &m.accounts {
+						trace!("Reverting enacted address {:?}", a);
+						cache.accounts.remove(a);
+					}
+					false
+				} else {
+					true
+				}
+			};
+		}
+
+		for block in retracted {
+			clear = clear || {
+				if let Some(ref mut m) = cache.modifications.iter_mut().find(|ref m| &m.hash == block) {
+					trace!("Retracting block {:?}", block);
+					m.is_canon = false;
+					for a in &m.accounts {
+						trace!("Retracted address {:?}", a);
+						cache.accounts.remove(a);
+					}
+					false
+				} else {
+					true
+				}
+			};
+		}
+		if clear {
+			// We don't know anything about the block; clear everything
+			trace!("Wiping cache");
+			cache.accounts.clear();
+			cache.modifications.clear();
+		}
+
+		// Propagate cache only if committing on top of the latest canonical state
+		// blocks are ordered by number and only one block with a given number is marked as canonical
+		// (contributed to canonical state cache)
+		if let (Some(ref number), Some(ref hash), Some(ref parent)) = (self.commit_number, self.commit_hash, self.parent_hash) {
+			if cache.modifications.len() == STATE_CACHE_BLOCKS {
+				cache.modifications.pop_back();
+			}
+			let mut modifications = HashSet::new();
+			trace!("committing {} cache entries", self.local_cache.len());
+			for account in self.local_cache.drain(..) {
+				if account.modified {
+					modifications.insert(account.address.clone());
+				}
+				if is_best {
+					if let Some(&mut Some(ref mut existing)) = cache.accounts.get_mut(&account.address) {
+						if let Some(new) = account.account {
+							if account.modified {
+								existing.overwrite_with(new);
+							}
+							continue;
+						}
+					}
+					cache.accounts.insert(account.address, account.account);
+				}
+			}
+
+			// Save modified accounts. These are ordered by the block number.
+			let block_changes = BlockChanges {
+				accounts: modifications,
+				number: *number,
+				hash: hash.clone(),
+				is_canon: is_best,
+				parent: parent.clone(),
+			};
+			let insert_at = cache.modifications.iter().enumerate().find(|&(_, ref m)| m.number < *number).map(|(i, _)| i);
+			trace!("inserting modifications at {:?}", insert_at);
+			if let Some(insert_at) = insert_at {
+				cache.modifications.insert(insert_at, block_changes);
+			} else {
+				cache.modifications.push_back(block_changes);
+			}
+		}
+	}
+
+	/// Returns an interface to HashDB.
+	pub fn as_hashdb(&self) -> &HashDB {
+		self.db.as_hashdb()
+	}
+
+	/// Returns an interface to mutable HashDB.
+	pub fn as_hashdb_mut(&mut self) -> &mut HashDB {
+		self.db.as_hashdb_mut()
+	}
+
+	/// Clone the database.
+	pub fn boxed_clone(&self) -> StateDB {
+		StateDB {
+			db: self.db.boxed_clone(),
+			account_cache: self.account_cache.clone(),
+			local_cache: Vec::new(),
+			account_bloom: self.account_bloom.clone(),
+			cache_size: self.cache_size,
+			parent_hash: None,
+			commit_hash: None,
+			commit_number: None,
+		}
+	}
+
+	/// Clone the database for a canonical state.
+	pub fn boxed_clone_canon(&self, parent: &H256) -> StateDB {
+		StateDB {
+			db: self.db.boxed_clone(),
+			account_cache: self.account_cache.clone(),
+			local_cache: Vec::new(),
+			account_bloom: self.account_bloom.clone(),
+			cache_size: self.cache_size,
+			parent_hash: Some(parent.clone()),
+			commit_hash: None,
+			commit_number: None,
+		}
+	}
+
+	/// Check if pruning is enabled on the database.
+	pub fn is_pruned(&self) -> bool {
+		self.db.is_pruned()
+	}
+
+	/// Heap size used.
+	pub fn mem_used(&self) -> usize {
+		// TODO: account for LRU-cache overhead; this is a close approximation.
+		self.db.mem_used() + self.account_cache.lock().accounts.len() * ::std::mem::size_of::<Option<Account>>()
+	}
+
+	/// Returns underlying `JournalDB`.
+	pub fn journal_db(&self) -> &JournalDB {
+		&*self.db
+	}
+
+	/// Add a local cache entry.
+	/// The entry will be propagated to the global cache in `sync_cache`.
+	/// `modified` indicates that the entry was changed since being read from disk or global cache.
+	/// `data` can be set to an existing (`Some`), or non-existing account (`None`).
+	pub fn add_to_account_cache(&mut self, addr: Address, data: Option<Account>, modified: bool) {
+		self.local_cache.push(CacheQueueItem {
+			address: addr,
+			account: data,
+			modified: modified,
+		})
+	}
+
+	/// Get basic copy of the cached account. Does not include storage.
+	/// Returns 'None' if cache is disabled or if the account is not cached.
+	pub fn get_cached_account(&self, addr: &Address) -> Option<Option<Account>> {
+		let mut cache = self.account_cache.lock();
+		if !Self::is_allowed(addr, &self.parent_hash, &cache.modifications) {
+			return None;
+		}
+		cache.accounts.get_mut(&addr).map(|a| a.as_ref().map(|a| a.clone_basic()))
+	}
+
+	/// Get value from a cached account.
+	/// Returns 'None' if cache is disabled or if the account is not cached.
+	pub fn get_cached<F, U>(&self, a: &Address, f: F) -> Option<U>
+		where F: FnOnce(Option<&mut Account>) -> U {
+		let mut cache = self.account_cache.lock();
+		if !Self::is_allowed(a, &self.parent_hash, &cache.modifications) {
+			return None;
+		}
+		cache.accounts.get_mut(a).map(|c| f(c.as_mut()))
+	}
+
+	/// Query how much memory is set aside for the accounts cache (in bytes).
+	pub fn cache_size(&self) -> usize {
+		self.cache_size
+	}
+
+	/// Check if the account can be returned from cache by matching current block parent hash against canonical
+	/// state and filtering out account modified in later blocks.
+	fn is_allowed(addr: &Address, parent_hash: &Option<H256>, modifications: &VecDeque<BlockChanges>) -> bool {
+		let mut parent = match *parent_hash {
+			None => {
+				trace!("Cache lookup skipped for {:?}: no parent hash", addr);
+				return false;
+			}
+			Some(ref parent) => parent,
+		};
+		if modifications.is_empty() {
+			return true;
+		}
+		// Ignore all accounts modified in later blocks
+		// Modifications contains block ordered by the number
+		// We search for our parent in that list first and then for
+		// all its parent until we hit the canonical block,
+		// checking against all the intermediate modifications.
+		let mut iter = modifications.iter();
+		while let Some(ref m) = iter.next() {
+			if &m.hash == parent {
+				if m.is_canon {
+					return true;
+				}
+				parent = &m.parent;
+			}
+			if m.accounts.contains(addr) {
+				trace!("Cache lookup skipped for {:?}: modified in a later block", addr);
+				return false;
+			}
+		}
+		trace!("Cache lookup skipped for {:?}: parent hash is unknown", addr);
+		return false;
+	}
+}
+
+#[cfg(test)]
+mod tests {
+
+	use util::{U256, H256, FixedHash, Address, DBTransaction};
+	use tests::helpers::*;
+	use state::Account;
+	use util::log::init_log;
+
+	#[test]
+	fn state_db_smoke() {
+		init_log();
+
+		let mut state_db_result = get_temp_state_db();
+		let state_db = state_db_result.take();
+		let root_parent = H256::random();
+		let address = Address::random();
+		let h0 = H256::random();
+		let h1a = H256::random();
+		let h1b = H256::random();
+		let h2a = H256::random();
+		let h2b = H256::random();
+		let h3a = H256::random();
+		let h3b = H256::random();
+		let mut batch = DBTransaction::new(state_db.journal_db().backing());
+
+		// blocks  [ 3a(c) 2a(c) 2b 1b 1a(c) 0 ]
+	    // balance [ 5     5     4  3  2     2 ]
+		let mut s = state_db.boxed_clone_canon(&root_parent);
+		s.add_to_account_cache(address, Some(Account::new_basic(2.into(), 0.into())), false);
+		s.journal_under(&mut batch, 0, &h0).unwrap();
+		s.sync_cache(&[], &[], true);
+
+		let mut s = state_db.boxed_clone_canon(&h0);
+		s.journal_under(&mut batch, 1, &h1a).unwrap();
+		s.sync_cache(&[], &[], true);
+
+		let mut s = state_db.boxed_clone_canon(&h0);
+		s.add_to_account_cache(address, Some(Account::new_basic(3.into(), 0.into())), true);
+		s.journal_under(&mut batch, 1, &h1b).unwrap();
+		s.sync_cache(&[], &[], false);
+
+		let mut s = state_db.boxed_clone_canon(&h1b);
+		s.add_to_account_cache(address, Some(Account::new_basic(4.into(), 0.into())), true);
+		s.journal_under(&mut batch, 2, &h2b).unwrap();
+		s.sync_cache(&[], &[], false);
+
+		let mut s = state_db.boxed_clone_canon(&h1a);
+		s.add_to_account_cache(address, Some(Account::new_basic(5.into(), 0.into())), true);
+		s.journal_under(&mut batch, 2, &h2a).unwrap();
+		s.sync_cache(&[], &[], true);
+
+		let mut s = state_db.boxed_clone_canon(&h2a);
+		s.journal_under(&mut batch, 3, &h3a).unwrap();
+		s.sync_cache(&[], &[], true);
+
+		let s = state_db.boxed_clone_canon(&h3a);
+		assert_eq!(s.get_cached_account(&address).unwrap().unwrap().balance(), &U256::from(5));
+
+		let s = state_db.boxed_clone_canon(&h1a);
+		assert!(s.get_cached_account(&address).is_none());
+
+		let s = state_db.boxed_clone_canon(&h2b);
+		assert!(s.get_cached_account(&address).is_none());
+
+		let s = state_db.boxed_clone_canon(&h1b);
+		assert!(s.get_cached_account(&address).is_none());
+
+		// reorg to 3b
+		// blocks  [ 3b(c) 3a 2a 2b(c) 1b 1a 0 ]
+		let mut s = state_db.boxed_clone_canon(&h2b);
+		s.journal_under(&mut batch, 3, &h3b).unwrap();
+		s.sync_cache(&[h1b.clone(), h2b.clone(), h3b.clone()], &[h1a.clone(), h2a.clone(), h3a.clone()], true);
+		let s = state_db.boxed_clone_canon(&h3a);
+		assert!(s.get_cached_account(&address).is_none());
+	}
+}
+

ethcore/src/tests/client.rs

@@ -57,7 +57,11 @@ fn should_return_registrar() {
 		IoChannel::disconnected(),
 		&db_config
 	).unwrap();
-	assert_eq!(client.additional_params().get("registrar"), Some(&"8e4e9b13d4b45cb0befc93c3061b1408f67316b2".to_owned()));
+	let params = client.additional_params();
+	let address = params.get("registrar").unwrap();
+
+	assert_eq!(address.len(), 40);
+	assert!(U256::from_str(address).is_ok());
 }
 
 #[test]

ethcore/src/tests/helpers.rs

@@ -19,6 +19,7 @@ use io::*;
 use client::{BlockChainClient, Client, ClientConfig};
 use common::*;
 use spec::*;
+use state_db::StateDB;
 use block::{OpenBlock, Drain};
 use blockchain::{BlockChain, Config as BlockChainConfig};
 use state::*;
@@ -28,6 +29,7 @@ use ethereum;
 use devtools::*;
 use miner::Miner;
 use rlp::{self, RlpStream, Stream};
+use db::COL_STATE;
 
 #[cfg(feature = "json-tests")]
 pub enum ChainEra {
@@ -146,9 +148,9 @@ pub fn generate_dummy_client_with_spec_and_data<F>(get_test_spec: F, block_numbe
 	).unwrap();
 	let test_engine = &*test_spec.engine;
 
-	let mut db_result = get_temp_journal_db();
+	let mut db_result = get_temp_state_db();
 	let mut db = db_result.take();
-	test_spec.ensure_db_good(db.as_hashdb_mut()).unwrap();
+	test_spec.ensure_db_good(&mut db).unwrap();
 	let genesis_header = test_spec.genesis_header();
 
 	let mut rolling_timestamp = 40;
@@ -321,9 +323,9 @@ pub fn generate_dummy_empty_blockchain() -> GuardedTempResult<BlockChain> {
 	}
 }
 
-pub fn get_temp_journal_db() -> GuardedTempResult<Box<JournalDB>> {
+pub fn get_temp_state_db() -> GuardedTempResult<StateDB> {
 	let temp = RandomTempPath::new();
-	let journal_db = get_temp_journal_db_in(temp.as_path());
+	let journal_db = get_temp_state_db_in(temp.as_path());
 
 	GuardedTempResult {
 		_temp: temp,
@@ -333,7 +335,7 @@ pub fn get_temp_journal_db() -> GuardedTempResult<Box<JournalDB>> {
 
 pub fn get_temp_state() -> GuardedTempResult<State> {
 	let temp = RandomTempPath::new();
-	let journal_db = get_temp_journal_db_in(temp.as_path());
+	let journal_db = get_temp_state_db_in(temp.as_path());
 
 	GuardedTempResult {
 	    _temp: temp,
@@ -341,13 +343,14 @@ pub fn get_temp_state() -> GuardedTempResult<State> {
 	}
 }
 
-pub fn get_temp_journal_db_in(path: &Path) -> Box<JournalDB> {
+pub fn get_temp_state_db_in(path: &Path) -> StateDB {
 	let db = new_db(path.to_str().expect("Only valid utf8 paths for tests."));
-	journaldb::new(db.clone(), journaldb::Algorithm::EarlyMerge, None)
+	let journal_db = journaldb::new(db.clone(), journaldb::Algorithm::EarlyMerge, COL_STATE);
+	StateDB::new(journal_db, 5 * 1024 * 1024)
 }
 
 pub fn get_temp_state_in(path: &Path) -> State {
-	let journal_db = get_temp_journal_db_in(path);
+	let journal_db = get_temp_state_db_in(path);
 	State::new(journal_db, U256::from(0), Default::default())
 }
 

ethcore/src/tests/mod.rs

@@ -16,4 +16,5 @@
 
 pub mod helpers;
 mod client;
+#[cfg(feature="ipc")]
 mod rpc;

ethcore/src/tests/rpc.rs

@@ -19,7 +19,8 @@
 use nanoipc;
 use std::sync::Arc;
 use std::sync::atomic::{Ordering, AtomicBool};
-use client::{Client, BlockChainClient, ClientConfig, RemoteClient, BlockID};
+use client::{Client, BlockChainClient, ClientConfig, BlockID};
+use client::remote::RemoteClient;
 use tests::helpers::*;
 use devtools::*;
 use miner::Miner;

ethcore/src/trace/config.rs

@@ -15,57 +15,14 @@
 // along with Parity.  If not, see <http://www.gnu.org/licenses/>.
 
 //! Traces config.
-use std::str::FromStr;
 use bloomchain::Config as BloomConfig;
-use trace::Error;
-
-/// 3-value enum.
-#[derive(Debug, Clone, Copy, PartialEq)]
-pub enum Switch {
-	/// True.
-	On,
-	/// False.
-	Off,
-	/// Auto.
-	Auto,
-}
-
-impl Default for Switch {
-	fn default() -> Self {
-		Switch::Auto
-	}
-}
-
-impl FromStr for Switch {
-	type Err = String;
-
-	fn from_str(s: &str) -> Result<Self, Self::Err> {
-		match s {
-			"on" => Ok(Switch::On),
-			"off" => Ok(Switch::Off),
-			"auto" => Ok(Switch::Auto),
-			other => Err(format!("Invalid switch value: {}", other))
-		}
-	}
-}
-
-impl Switch {
-	/// Tries to turn old switch to new value.
-	pub fn turn_to(&self, to: Switch) -> Result<bool, Error> {
-		match (*self, to) {
-			(Switch::On, Switch::On) | (Switch::On, Switch::Auto) | (Switch::Auto, Switch::On) => Ok(true),
-			(Switch::Off, Switch::On) => Err(Error::ResyncRequired),
-			_ => Ok(false),
-		}
-	}
-}
 
 /// Traces config.
 #[derive(Debug, PartialEq, Clone)]
 pub struct Config {
 	/// Indicates if tracing should be enabled or not.
 	/// If it's None, it will be automatically configured.
-	pub enabled: Switch,
+	pub enabled: bool,
 	/// Traces blooms configuration.
 	pub blooms: BloomConfig,
 	/// Preferef cache-size.
@@ -77,7 +34,7 @@ pub struct Config {
 impl Default for Config {
 	fn default() -> Self {
 		Config {
-			enabled: Switch::default(),
+			enabled: false,
 			blooms: BloomConfig {
 				levels: 3,
 				elements_per_index: 16,
@@ -87,20 +44,3 @@ impl Default for Config {
 		}
 	}
 }
-
-#[cfg(test)]
-mod tests {
-	use super::Switch;
-
-	#[test]
-	fn test_switch_parsing() {
-		assert_eq!(Switch::On, "on".parse().unwrap());
-		assert_eq!(Switch::Off, "off".parse().unwrap());
-		assert_eq!(Switch::Auto, "auto".parse().unwrap());
-	}
-
-	#[test]
-	fn test_switch_default() {
-		assert_eq!(Switch::default(), Switch::Auto);
-	}
-}

ethcore/src/trace/db.rs

@@ -22,7 +22,7 @@ use bloomchain::{Number, Config as BloomConfig};
 use bloomchain::group::{BloomGroupDatabase, BloomGroupChain, GroupPosition, BloomGroup};
 use util::{H256, H264, Database, DBTransaction, RwLock, HeapSizeOf};
 use header::BlockNumber;
-use trace::{LocalizedTrace, Config, Switch, Filter, Database as TraceDatabase, ImportRequest, DatabaseExtras, Error};
+use trace::{LocalizedTrace, Config, Filter, Database as TraceDatabase, ImportRequest, DatabaseExtras};
 use db::{self, Key, Writable, Readable, CacheUpdatePolicy};
 use blooms;
 use super::flat::{FlatTrace, FlatBlockTraces, FlatTransactionTraces};
@@ -126,38 +126,20 @@ impl<T> BloomGroupDatabase for TraceDB<T> where T: DatabaseExtras {
 
 impl<T> TraceDB<T> where T: DatabaseExtras {
 	/// Creates new instance of `TraceDB`.
-	pub fn new(config: Config, tracesdb: Arc<Database>, extras: Arc<T>) -> Result<Self, Error> {
+	pub fn new(config: Config, tracesdb: Arc<Database>, extras: Arc<T>) -> Self {
-		// check if in previously tracing was enabled
-		let old_tracing = match tracesdb.get(db::COL_TRACE, b"enabled").unwrap() {
-			Some(ref value) if value as &[u8] == &[0x1] => Switch::On,
-			Some(ref value) if value as &[u8] == &[0x0] => Switch::Off,
-			Some(_) => { panic!("tracesdb is corrupted") },
-			None => Switch::Auto,
-		};
-
-		let enabled = try!(old_tracing.turn_to(config.enabled));
-
-		let encoded_tracing = match enabled {
-			true => [0x1],
-			false => [0x0]
-		};
-
 		let mut batch = DBTransaction::new(&tracesdb);
-		batch.put(db::COL_TRACE, b"enabled", &encoded_tracing);
 		batch.put(db::COL_TRACE, b"version", TRACE_DB_VER);
 		tracesdb.write(batch).unwrap();
 
-		let db = TraceDB {
+		TraceDB {
 			traces: RwLock::new(HashMap::new()),
 			blooms: RwLock::new(HashMap::new()),
 			cache_manager: RwLock::new(CacheManager::new(config.pref_cache_size, config.max_cache_size, 10 * 1024)),
 			tracesdb: tracesdb,
 			bloom_config: config.blooms,
-			enabled: enabled,
+			enabled: config.enabled,
 			extras: extras,
-		};
+		}
-
-		Ok(db)
 	}
 
 	fn cache_size(&self) -> usize {
@@ -274,16 +256,6 @@ impl<T> TraceDatabase for TraceDB<T> where T: DatabaseExtras {
 			return;
 		}
 
-		// at first, let's insert new block traces
-		{
-			let mut traces = self.traces.write();
-			// it's important to use overwrite here,
-			// cause this value might be queried by hash later
-			batch.write_with_cache(db::COL_TRACE, &mut *traces, request.block_hash, request.traces, CacheUpdatePolicy::Overwrite);
-			// note_used must be called after locking traces to avoid cache/traces deadlock on garbage collection
-			self.note_used(CacheID::Trace(request.block_hash.clone()));
-		}
-
 		// now let's rebuild the blooms
 		if !request.enacted.is_empty() {
 			let range_start = request.block_number as Number + 1 - request.enacted.len();
@@ -294,12 +266,25 @@ impl<T> TraceDatabase for TraceDB<T> where T: DatabaseExtras {
 				// all traces are expected to be found here. That's why `expect` has been used
 				// instead of `filter_map`. If some traces haven't been found, it meens that
 				// traces database is corrupted or incomplete.
-				.map(|block_hash| self.traces(block_hash).expect("Traces database is incomplete."))
+				.map(|block_hash| if block_hash == &request.block_hash {
-				.map(|block_traces| block_traces.bloom())
+					request.traces.bloom()
+				} else {
+					self.traces(block_hash).expect("Traces database is incomplete.").bloom()
+				})
 				.map(blooms::Bloom::from)
 				.map(Into::into)
 				.collect();
 
+			// insert new block traces into the cache and the database
+			{
+				let mut traces = self.traces.write();
+				// it's important to use overwrite here,
+				// cause this value might be queried by hash later
+				batch.write_with_cache(db::COL_TRACE, &mut *traces, request.block_hash, request.traces, CacheUpdatePolicy::Overwrite);
+				// note_used must be called after locking traces to avoid cache/traces deadlock on garbage collection
+				self.note_used(CacheID::Trace(request.block_hash.clone()));
+			}
+
 			let chain = BloomGroupChain::new(self.bloom_config, self);
 			let trace_blooms = chain.replace(&replaced_range, enacted_blooms);
 			let blooms_to_insert = trace_blooms.into_iter()
@@ -419,7 +404,7 @@ mod tests {
 	use util::{Address, U256, H256, Database, DatabaseConfig, DBTransaction};
 	use devtools::RandomTempPath;
 	use header::BlockNumber;
-	use trace::{Config, Switch, TraceDB, Database as TraceDatabase, DatabaseExtras, ImportRequest};
+	use trace::{Config, TraceDB, Database as TraceDatabase, DatabaseExtras, ImportRequest};
 	use trace::{Filter, LocalizedTrace, AddressesFilter, TraceError};
 	use trace::trace::{Call, Action, Res};
 	use trace::flat::{FlatTrace, FlatBlockTraces, FlatTransactionTraces};
@@ -474,22 +459,10 @@ mod tests {
 		let mut config = Config::default();
 
 		// set autotracing
-		config.enabled = Switch::Auto;
+		config.enabled = false;
 
 		{
-			let tracedb = TraceDB::new(config.clone(), db.clone(), Arc::new(NoopExtras)).unwrap();
+			let tracedb = TraceDB::new(config.clone(), db.clone(), Arc::new(NoopExtras));
-			assert_eq!(tracedb.tracing_enabled(), false);
-		}
-
-		{
-			let tracedb = TraceDB::new(config.clone(), db.clone(), Arc::new(NoopExtras)).unwrap();
-			assert_eq!(tracedb.tracing_enabled(), false);
-		}
-
-		config.enabled = Switch::Off;
-
-		{
-			let tracedb = TraceDB::new(config.clone(), db.clone(), Arc::new(NoopExtras)).unwrap();
 			assert_eq!(tracedb.tracing_enabled(), false);
 		}
 	}
@@ -501,50 +474,12 @@ mod tests {
 		let mut config = Config::default();
 
 		// set tracing on
-		config.enabled = Switch::On;
+		config.enabled = true;
 
 		{
-			let tracedb = TraceDB::new(config.clone(), db.clone(), Arc::new(NoopExtras)).unwrap();
+			let tracedb = TraceDB::new(config.clone(), db.clone(), Arc::new(NoopExtras));
 			assert_eq!(tracedb.tracing_enabled(), true);
 		}
-
-		{
-			let tracedb = TraceDB::new(config.clone(), db.clone(), Arc::new(NoopExtras)).unwrap();
-			assert_eq!(tracedb.tracing_enabled(), true);
-		}
-
-		config.enabled = Switch::Auto;
-
-		{
-			let tracedb = TraceDB::new(config.clone(), db.clone(), Arc::new(NoopExtras)).unwrap();
-			assert_eq!(tracedb.tracing_enabled(), true);
-		}
-
-		config.enabled = Switch::Off;
-
-		{
-			let tracedb = TraceDB::new(config.clone(), db.clone(), Arc::new(NoopExtras)).unwrap();
-			assert_eq!(tracedb.tracing_enabled(), false);
-		}
-	}
-
-	#[test]
-	#[should_panic]
-	fn test_invalid_reopening_db() {
-		let temp = RandomTempPath::new();
-		let db = new_db(temp.as_str());
-		let mut config = Config::default();
-
-		// set tracing on
-		config.enabled = Switch::Off;
-
-		{
-			let tracedb = TraceDB::new(config.clone(), db.clone(), Arc::new(NoopExtras)).unwrap();
-			assert_eq!(tracedb.tracing_enabled(), true);
-		}
-
-		config.enabled = Switch::On;
-		TraceDB::new(config.clone(), db.clone(), Arc::new(NoopExtras)).unwrap(); // should panic!
 	}
 
 	fn create_simple_import_request(block_number: BlockNumber, block_hash: H256) -> ImportRequest {
@@ -595,7 +530,7 @@ mod tests {
 		let temp = RandomTempPath::new();
 		let db = Arc::new(Database::open(&DatabaseConfig::with_columns(::db::NUM_COLUMNS), temp.as_str()).unwrap());
 		let mut config = Config::default();
-		config.enabled = Switch::On;
+		config.enabled = true;
 		let block_0 = H256::from(0xa1);
 		let block_1 = H256::from(0xa2);
 		let tx_0 = H256::from(0xff);
@@ -607,7 +542,7 @@ mod tests {
 		extras.transaction_hashes.insert(0, vec![tx_0.clone()]);
 		extras.transaction_hashes.insert(1, vec![tx_1.clone()]);
 
-		let tracedb = TraceDB::new(config, db.clone(), Arc::new(extras)).unwrap();
+		let tracedb = TraceDB::new(config, db.clone(), Arc::new(extras));
 
 		// import block 0
 		let request = create_simple_import_request(0, block_0.clone());
@@ -679,10 +614,10 @@ mod tests {
 		extras.transaction_hashes.insert(0, vec![tx_0.clone()]);
 
 		// set tracing on
-		config.enabled = Switch::On;
+		config.enabled = true;
 
 		{
-			let tracedb = TraceDB::new(config.clone(), db.clone(), Arc::new(extras.clone())).unwrap();
+			let tracedb = TraceDB::new(config.clone(), db.clone(), Arc::new(extras.clone()));
 
 			// import block 0
 			let request = create_simple_import_request(0, block_0.clone());
@@ -692,7 +627,7 @@ mod tests {
 		}
 
 		{
-			let tracedb = TraceDB::new(config.clone(), db.clone(), Arc::new(extras)).unwrap();
+			let tracedb = TraceDB::new(config.clone(), db.clone(), Arc::new(extras));
 			let traces = tracedb.transaction_traces(0, 0);
 			assert_eq!(traces.unwrap(), vec![create_simple_localized_trace(0, block_0, tx_0)]);
 		}

ethcore/src/trace/executive_tracer.rs

@@ -31,7 +31,7 @@ fn top_level_subtraces(traces: &[FlatTrace]) -> usize {
 	traces.iter().filter(|t| t.trace_address.is_empty()).count()
 }
 
-fn update_trace_address(traces: Vec<FlatTrace>) -> Vec<FlatTrace> {
+fn prefix_subtrace_addresses(mut traces: Vec<FlatTrace>) -> Vec<FlatTrace> {
 	// input traces are expected to be ordered like
 	// []
 	// [0]
@@ -48,24 +48,36 @@ fn update_trace_address(traces: Vec<FlatTrace>) -> Vec<FlatTrace> {
 	// [0, 0, 1]
 	// [1]
 	// [1, 0]
-	let mut top_subtrace_index = 0;
+	let mut current_subtrace_index = 0;
-	let mut subtrace_subtraces_left = 0;
+	let mut first = true;
-	traces.into_iter().map(|mut trace| {
+	for trace in traces.iter_mut() {
-		let is_top_subtrace = trace.trace_address.is_empty();
+		match (first, trace.trace_address.is_empty()) {
-		let is_subtrace = trace.trace_address.len() == 1;
+			(true, _) => first = false,
-		trace.trace_address.push_front(top_subtrace_index);
+			(_, true) => current_subtrace_index += 1,
-
+			_ => {}
-		if is_top_subtrace {
+		}			
-			subtrace_subtraces_left = trace.subtraces;
+		trace.trace_address.push_front(current_subtrace_index);
-		} else if is_subtrace {
+	}
-			subtrace_subtraces_left -= 1;
+	traces
 }
 
-		if subtrace_subtraces_left == 0 {
+#[test]
-			top_subtrace_index += 1;
+fn should_prefix_address_properly() {
-		}
+	use super::trace::{Action, Res, Suicide};
-		trace
+
-	}).collect()
+	let f = |v: Vec<usize>| FlatTrace {
+		action: Action::Suicide(Suicide {
+			address: Default::default(),
+			balance: Default::default(),
+			refund_address: Default::default(),
+		}),
+		result: Res::None,
+		subtraces: 0,
+		trace_address: v.into_iter().collect(),
+	};
+	let t = vec![vec![], vec![0], vec![0, 0], vec![0], vec![], vec![], vec![0], vec![]].into_iter().map(&f).collect();
+	let t = prefix_subtrace_addresses(t);
+	assert_eq!(t, vec![vec![0], vec![0, 0], vec![0, 0, 0], vec![0, 0], vec![1], vec![2], vec![2, 0], vec![3]].into_iter().map(&f).collect::<Vec<_>>());
 }  
 
 impl Tracer for ExecutiveTracer {
@@ -93,7 +105,7 @@ impl Tracer for ExecutiveTracer {
 		};
 		debug!(target: "trace", "Traced call {:?}", trace);
 		self.traces.push(trace);
-		self.traces.extend(update_trace_address(subs));
+		self.traces.extend(prefix_subtrace_addresses(subs));
 	}
 
 	fn trace_create(&mut self, create: Option<Create>, gas_used: U256, code: Option<Bytes>, address: Address, subs: Vec<FlatTrace>) {
@@ -109,7 +121,7 @@ impl Tracer for ExecutiveTracer {
 		};
 		debug!(target: "trace", "Traced create {:?}", trace);
 		self.traces.push(trace);
-		self.traces.extend(update_trace_address(subs));
+		self.traces.extend(prefix_subtrace_addresses(subs));
 	}
 
 	fn trace_failed_call(&mut self, call: Option<Call>, subs: Vec<FlatTrace>, error: TraceError) {
@@ -121,7 +133,7 @@ impl Tracer for ExecutiveTracer {
 		};
 		debug!(target: "trace", "Traced failed call {:?}", trace);
 		self.traces.push(trace);
-		self.traces.extend(update_trace_address(subs));
+		self.traces.extend(prefix_subtrace_addresses(subs));
 	}
 
 	fn trace_failed_create(&mut self, create: Option<Create>, subs: Vec<FlatTrace>, error: TraceError) {
@@ -133,7 +145,7 @@ impl Tracer for ExecutiveTracer {
 		};
 		debug!(target: "trace", "Traced failed create {:?}", trace);
 		self.traces.push(trace);
-		self.traces.extend(update_trace_address(subs));
+		self.traces.extend(prefix_subtrace_addresses(subs));
 	}
 
 	fn trace_suicide(&mut self, address: Address, balance: U256, refund_address: Address) {

ethcore/src/trace/mod.rs

@@ -26,7 +26,7 @@ mod noop_tracer;
 
 pub use types::trace_types::{filter, flat, localized, trace};
 pub use types::trace_types::error::Error as TraceError;
-pub use self::config::{Config, Switch};
+pub use self::config::Config;
 pub use self::db::TraceDB;
 pub use self::error::Error;
 pub use types::trace_types::trace::{VMTrace, VMOperation, VMExecutedOperation, MemoryDiff, StorageDiff};

ethcore/src/types/block_status.rs

@@ -15,6 +15,7 @@
 // along with Parity.  If not, see <http://www.gnu.org/licenses/>.
 
 //! Block status description module
+use verification::queue::Status as QueueStatus;
 
 /// General block status
 #[derive(Debug, Eq, PartialEq, Binary)]
@@ -28,3 +29,13 @@ pub enum BlockStatus {
 	/// Unknown.
 	Unknown,
 }
+
+impl From<QueueStatus> for BlockStatus {
+	fn from(status: QueueStatus) -> Self {
+		match status {
+			QueueStatus::Queued => BlockStatus::Queued,
+			QueueStatus::Bad => BlockStatus::Bad,
+			QueueStatus::Unknown => BlockStatus::Unknown,
+		}
+	}
+}

ethcore/src/types/filter.rs

@@ -22,7 +22,7 @@ use client::BlockID;
 use log_entry::LogEntry;
 
 /// Blockchain Filter.
-#[derive(Binary)]
+#[derive(Binary, Debug, PartialEq)]
 pub struct Filter {
 	/// Blockchain will be searched from this block.
 	pub from_block: BlockID,

ethcore/src/types/mod.rs.in

@@ -25,7 +25,7 @@ pub mod executed;
 pub mod block_status;
 pub mod account_diff;
 pub mod state_diff;
-pub mod block_queue_info;
+pub mod verification_queue_info;
 pub mod filter;
 pub mod trace_filter;
 pub mod call_analytics;

ethcore/src/types/trace_types/trace.rs

@@ -181,7 +181,7 @@ impl From<ActionParams> for Create {
 			from: p.sender,
 			value: p.value.value(),
 			gas: p.gas,
-			init: p.code.unwrap_or_else(Vec::new),
+			init: p.code.map_or_else(Vec::new, |c| (*c).clone()),
 		}
 	}
 }

ethcore/src/types/transaction.rs

@@ -20,7 +20,7 @@ use std::ops::Deref;
 use std::cell::*;
 use rlp::*;
 use util::sha3::Hashable;
-use util::{H256, Address, U256, Bytes};
+use util::{H256, Address, U256, Bytes, HeapSizeOf};
 use ethkey::{Signature, sign, Secret, Public, recover, public_to_address, Error as EthkeyError};
 use error::*;
 use evm::Schedule;
@@ -86,6 +86,12 @@ impl Transaction {
 	}
 }
 
+impl HeapSizeOf for Transaction {
+	fn heap_size_of_children(&self) -> usize {
+		self.data.heap_size_of_children()
+	}
+}
+
 impl From<ethjson::state::Transaction> for SignedTransaction {
 	fn from(t: ethjson::state::Transaction) -> Self {
 		let to: Option<ethjson::hash::Address> = t.to.into();
@@ -251,6 +257,12 @@ impl Encodable for SignedTransaction {
 	fn rlp_append(&self, s: &mut RlpStream) { self.rlp_append_sealed_transaction(s) }
 }
 
+impl HeapSizeOf for SignedTransaction {
+	fn heap_size_of_children(&self) -> usize {
+		self.unsigned.heap_size_of_children()
+	}
+}
+
 impl SignedTransaction {
 	/// Append object with a signature into RLP stream
 	pub fn rlp_append_sealed_transaction(&self, s: &mut RlpStream) {

ethcore/src/types/verification_queue_info.rs

@@ -0,0 +1,53 @@
+// Copyright 2015, 2016 Ethcore (UK) Ltd.
+// This file is part of Parity.
+
+// Parity is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Parity is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Parity.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Verification queue info types
+
+/// Verification queue status
+#[derive(Debug, Binary)]
+pub struct VerificationQueueInfo {
+	/// Number of queued items pending verification
+	pub unverified_queue_size: usize,
+	/// Number of verified queued items pending import
+	pub verified_queue_size: usize,
+	/// Number of items being verified
+	pub verifying_queue_size: usize,
+	/// Configured maximum number of items in the queue
+	pub max_queue_size: usize,
+	/// Configured maximum number of bytes to use
+	pub max_mem_use: usize,
+	/// Heap memory used in bytes
+	pub mem_used: usize,
+}
+
+impl VerificationQueueInfo {
+	/// The total size of the queues.
+	pub fn total_queue_size(&self) -> usize { self.unverified_queue_size + self.verified_queue_size + self.verifying_queue_size }
+
+	/// The size of the unverified and verifying queues.
+	pub fn incomplete_queue_size(&self) -> usize { self.unverified_queue_size + self.verifying_queue_size }
+
+	/// Indicates that queue is full
+	pub fn is_full(&self) -> bool {
+		self.unverified_queue_size + self.verified_queue_size + self.verifying_queue_size > self.max_queue_size ||
+			self.mem_used > self.max_mem_use
+	}
+
+	/// Indicates that queue is empty
+	pub fn is_empty(&self) -> bool {
+		self.unverified_queue_size + self.verified_queue_size + self.verifying_queue_size == 0
+	}
+}

ethcore/src/verification/mod.rs

@@ -16,6 +16,7 @@
 
 pub mod verification;
 pub mod verifier;
+pub mod queue;
 mod canon_verifier;
 mod noop_verifier;
 
@@ -23,6 +24,7 @@ pub use self::verification::*;
 pub use self::verifier::Verifier;
 pub use self::canon_verifier::CanonVerifier;
 pub use self::noop_verifier::NoopVerifier;
+pub use self::queue::{BlockQueue, Config as QueueConfig, VerificationQueue, QueueInfo};
 
 /// Verifier type.
 #[derive(Debug, PartialEq, Clone)]

ethcore/src/verification/queue/kind.rs

@@ -0,0 +1,183 @@
+// Copyright 2015, 2016 Ethcore (UK) Ltd.
+// This file is part of Parity.
+
+// Parity is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Parity is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Parity.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Definition of valid items for the verification queue.
+
+use engines::Engine;
+use error::Error;
+
+use util::{HeapSizeOf, H256};
+
+pub use self::blocks::Blocks;
+pub use self::headers::Headers;
+
+/// Something which can produce a hash and a parent hash.
+pub trait HasHash {
+	/// Get the hash of this item.
+	fn hash(&self) -> H256;
+
+	/// Get the hash of this item's parent.
+	fn parent_hash(&self) -> H256;
+}
+
+/// Defines transitions between stages of verification.
+///
+/// It starts with a fallible transformation from an "input" into the unverified item.
+/// This consists of quick, simply done checks as well as extracting particular data.
+///
+/// Then, there is a `verify` function which performs more expensive checks and
+/// produces the verified output.
+///
+/// For correctness, the hashes produced by each stage of the pipeline should be
+/// consistent.
+pub trait Kind: 'static + Sized + Send + Sync {
+	/// The first stage: completely unverified.
+	type Input: Sized + Send + HasHash + HeapSizeOf;
+
+	/// The second stage: partially verified.
+	type Unverified: Sized + Send + HasHash + HeapSizeOf;
+
+	/// The third stage: completely verified.
+	type Verified: Sized + Send + HasHash + HeapSizeOf;
+
+	/// Attempt to create the `Unverified` item from the input.
+	fn create(input: Self::Input, engine: &Engine) -> Result<Self::Unverified, Error>;
+
+	/// Attempt to verify the `Unverified` item using the given engine.
+	fn verify(unverified: Self::Unverified, engine: &Engine) -> Result<Self::Verified, Error>;
+}
+
+/// The blocks verification module.
+pub mod blocks {
+	use super::{Kind, HasHash};
+
+	use engines::Engine;
+	use error::Error;
+	use header::Header;
+	use verification::{PreverifiedBlock, verify_block_basic, verify_block_unordered};
+
+	use util::{Bytes, HeapSizeOf, H256};
+
+	/// A mode for verifying blocks.
+	pub struct Blocks;
+
+	impl Kind for Blocks {
+		type Input = Unverified;
+		type Unverified = Unverified;
+		type Verified = PreverifiedBlock;
+
+		fn create(input: Self::Input, engine: &Engine) -> Result<Self::Unverified, Error> {
+			match verify_block_basic(&input.header, &input.bytes, engine) {
+				Ok(()) => Ok(input),
+				Err(e) => {
+					warn!(target: "client", "Stage 1 block verification failed for {}: {:?}", input.hash(), e);
+					Err(e)
+				}
+			}
+		}
+
+		fn verify(un: Self::Unverified, engine: &Engine) -> Result<Self::Verified, Error> {
+			let hash = un.hash();
+			match verify_block_unordered(un.header, un.bytes, engine) {
+				Ok(verified) => Ok(verified),
+				Err(e) => {
+					warn!(target: "client", "Stage 2 block verification failed for {}: {:?}", hash, e);
+					Err(e)
+				}
+			}
+		}
+	}
+
+	/// An unverified block.
+	pub struct Unverified {
+		header: Header,
+		bytes: Bytes,
+	}
+
+	impl Unverified {
+		/// Create an `Unverified` from raw bytes.
+		pub fn new(bytes: Bytes) -> Self {
+			use views::BlockView;
+
+			let header = BlockView::new(&bytes).header();
+			Unverified {
+				header: header,
+				bytes: bytes,
+			}
+		}
+	}
+
+	impl HeapSizeOf for Unverified {
+		fn heap_size_of_children(&self) -> usize {
+			self.header.heap_size_of_children() + self.bytes.heap_size_of_children()
+		}
+	}
+
+	impl HasHash for Unverified {
+		fn hash(&self) -> H256 {
+			self.header.hash()
+		}
+
+		fn parent_hash(&self) -> H256 {
+			self.header.parent_hash().clone()
+		}
+	}
+
+	impl HasHash for PreverifiedBlock {
+		fn hash(&self) -> H256 {
+			self.header.hash()
+		}
+
+		fn parent_hash(&self) -> H256 {
+			self.header.parent_hash().clone()
+		}
+	}
+}
+
+/// Verification for headers.
+pub mod headers {
+	use super::{Kind, HasHash};
+
+	use engines::Engine;
+	use error::Error;
+	use header::Header;
+	use verification::verify_header_params;
+
+	use util::hash::H256;
+
+	impl HasHash for Header {
+		fn hash(&self) -> H256 { self.hash() }
+		fn parent_hash(&self) -> H256 { self.parent_hash().clone() }
+	}
+
+	/// A mode for verifying headers.
+	#[allow(dead_code)]
+	pub struct Headers;
+
+	impl Kind for Headers {
+		type Input = Header;
+		type Unverified = Header;
+		type Verified = Header;
+
+		fn create(input: Self::Input, engine: &Engine) -> Result<Self::Unverified, Error> {
+			verify_header_params(&input, engine).map(|_| input)
+		}
+
+		fn verify(unverified: Self::Unverified, engine: &Engine) -> Result<Self::Verified, Error> {
+			engine.verify_block_unordered(&unverified, None).map(|_| unverified)
+		}
+	}
+}

ethcore/src/verification/queue/mod.rs

@@ -16,30 +16,35 @@
 
 //! A queue of blocks. Sits between network or other I/O and the `BlockChain`.
 //! Sorts them ready for blockchain insertion.
+
 use std::thread::{JoinHandle, self};
 use std::sync::atomic::{AtomicBool, Ordering as AtomicOrdering};
 use std::sync::{Condvar as SCondvar, Mutex as SMutex};
 use util::*;
 use io::*;
-use verification::*;
 use error::*;
 use engines::Engine;
-use views::*;
-use header::*;
 use service::*;
-use client::BlockStatus;
 
-pub use types::block_queue_info::BlockQueueInfo;
+use self::kind::{HasHash, Kind};
 
-known_heap_size!(0, UnverifiedBlock, VerifyingBlock, PreverifiedBlock);
+pub use types::verification_queue_info::VerificationQueueInfo as QueueInfo;
+
+pub mod kind;
 
 const MIN_MEM_LIMIT: usize = 16384;
 const MIN_QUEUE_LIMIT: usize = 512;
 
-/// Block queue configuration
+/// Type alias for block queue convenience.
+pub type BlockQueue = VerificationQueue<self::kind::Blocks>;
+
+/// Type alias for header queue convenience.
+pub type HeaderQueue = VerificationQueue<self::kind::Headers>;
+
+/// Verification queue configuration
 #[derive(Debug, PartialEq, Clone)]
-pub struct BlockQueueConfig {
+pub struct Config {
-	/// Maximum number of blocks to keep in unverified queue.
+	/// Maximum number of items to keep in unverified queue.
 	/// When the limit is reached, is_full returns true.
 	pub max_queue_size: usize,
 	/// Maximum heap memory to use.
@@ -47,42 +52,44 @@ pub struct BlockQueueConfig {
 	pub max_mem_use: usize,
 }
 
-impl Default for BlockQueueConfig {
+impl Default for Config {
 	fn default() -> Self {
-		BlockQueueConfig {
+		Config {
 			max_queue_size: 30000,
 			max_mem_use: 50 * 1024 * 1024,
 		}
 	}
 }
 
-
+/// An item which is in the process of being verified.
-impl BlockQueueInfo {
+pub struct Verifying<K: Kind> {
-	/// The total size of the queues.
+	hash: H256,
-	pub fn total_queue_size(&self) -> usize { self.unverified_queue_size + self.verified_queue_size + self.verifying_queue_size }
+	output: Option<K::Verified>,
-
-	/// The size of the unverified and verifying queues.
-	pub fn incomplete_queue_size(&self) -> usize { self.unverified_queue_size + self.verifying_queue_size }
-
-	/// Indicates that queue is full
-	pub fn is_full(&self) -> bool {
-		self.unverified_queue_size + self.verified_queue_size + self.verifying_queue_size > self.max_queue_size ||
-			self.mem_used > self.max_mem_use
 }
 
-	/// Indicates that queue is empty
+impl<K: Kind> HeapSizeOf for Verifying<K> {
-	pub fn is_empty(&self) -> bool {
+	fn heap_size_of_children(&self) -> usize {
-		self.unverified_queue_size + self.verified_queue_size + self.verifying_queue_size == 0
+		self.output.heap_size_of_children()
 	}
 }
 
-/// A queue of blocks. Sits between network or other I/O and the `BlockChain`.
+/// Status of items in the queue.
-/// Sorts them ready for blockchain insertion.
+pub enum Status {
-pub struct BlockQueue {
+	/// Currently queued.
+	Queued,
+	/// Known to be bad.
+	Bad,
+	/// Unknown.
+	Unknown,
+}
+
+/// A queue of items to be verified. Sits between network or other I/O and the `BlockChain`.
+/// Keeps them in the same order as inserted, minus invalid items.
+pub struct VerificationQueue<K: Kind> {
 	panic_handler: Arc<PanicHandler>,
 	engine: Arc<Engine>,
 	more_to_verify: Arc<SCondvar>,
-	verification: Arc<Verification>,
+	verification: Arc<Verification<K>>,
 	verifiers: Vec<JoinHandle<()>>,
 	deleting: Arc<AtomicBool>,
 	ready_signal: Arc<QueueSignal>,
@@ -92,16 +99,6 @@ pub struct BlockQueue {
 	max_mem_use: usize,
 }
 
-struct UnverifiedBlock {
-	header: Header,
-	bytes: Bytes,
-}
-
-struct VerifyingBlock {
-	hash: H256,
-	block: Option<PreverifiedBlock>,
-}
-
 struct QueueSignal {
 	deleting: Arc<AtomicBool>,
 	signalled: AtomicBool,
@@ -128,19 +125,19 @@ impl QueueSignal {
 	}
 }
 
-struct Verification {
+struct Verification<K: Kind> {
 	// All locks must be captured in the order declared here.
-	unverified: Mutex<VecDeque<UnverifiedBlock>>,
+	unverified: Mutex<VecDeque<K::Unverified>>,
-	verified: Mutex<VecDeque<PreverifiedBlock>>,
+	verified: Mutex<VecDeque<K::Verified>>,
-	verifying: Mutex<VecDeque<VerifyingBlock>>,
+	verifying: Mutex<VecDeque<Verifying<K>>>,
 	bad: Mutex<HashSet<H256>>,
 	more_to_verify: SMutex<()>,
 	empty: SMutex<()>,
 }
 
-impl BlockQueue {
+impl<K: Kind> VerificationQueue<K> {
 	/// Creates a new queue instance.
-	pub fn new(config: BlockQueueConfig, engine: Arc<Engine>, message_channel: IoChannel<ClientIoMessage>) -> BlockQueue {
+	pub fn new(config: Config, engine: Arc<Engine>, message_channel: IoChannel<ClientIoMessage>) -> Self {
 		let verification = Arc::new(Verification {
 			unverified: Mutex::new(VecDeque::new()),
 			verified: Mutex::new(VecDeque::new()),
@@ -175,13 +172,13 @@ impl BlockQueue {
 				.name(format!("Verifier #{}", i))
 				.spawn(move || {
 					panic_handler.catch_panic(move || {
-						BlockQueue::verify(verification, engine, more_to_verify, ready_signal, deleting, empty)
+						VerificationQueue::verify(verification, engine, more_to_verify, ready_signal, deleting, empty)
 					}).unwrap()
 				})
 				.expect("Error starting block verification thread")
 			);
 		}
-		BlockQueue {
+		VerificationQueue {
 			engine: engine,
 			panic_handler: panic_handler,
 			ready_signal: ready_signal.clone(),
@@ -196,7 +193,7 @@ impl BlockQueue {
 		}
 	}
 
-	fn verify(verification: Arc<Verification>, engine: Arc<Engine>, wait: Arc<SCondvar>, ready: Arc<QueueSignal>, deleting: Arc<AtomicBool>, empty: Arc<SCondvar>) {
+	fn verify(verification: Arc<Verification<K>>, engine: Arc<Engine>, wait: Arc<SCondvar>, ready: Arc<QueueSignal>, deleting: Arc<AtomicBool>, empty: Arc<SCondvar>) {
 		while !deleting.load(AtomicOrdering::Acquire) {
 			{
 				let mut more_to_verify = verification.more_to_verify.lock().unwrap();
@@ -214,57 +211,66 @@ impl BlockQueue {
 				}
 			}
 
-			let block = {
+			let item = {
+				// acquire these locks before getting the item to verify.
 				let mut unverified = verification.unverified.lock();
-				if unverified.is_empty() {
-					continue;
-				}
 				let mut verifying = verification.verifying.lock();
-				let block = unverified.pop_front().unwrap();
+
-				verifying.push_back(VerifyingBlock{ hash: block.header.hash(), block: None });
+				let item = match unverified.pop_front() {
-				block
+					Some(item) => item,
+					None => continue,
 				};
 
-			let block_hash = block.header.hash();
+				verifying.push_back(Verifying { hash: item.hash(), output: None });
-			match verify_block_unordered(block.header, block.bytes, &*engine) {
+				item
+			};
+
+			let hash = item.hash();
+			match K::verify(item, &*engine) {
 				Ok(verified) => {
 					let mut verifying = verification.verifying.lock();
-					for e in verifying.iter_mut() {
+					let mut idx = None;
-						if e.hash == block_hash {
+					for (i, e) in verifying.iter_mut().enumerate() {
-							e.block = Some(verified);
+						if e.hash == hash {
+							idx = Some(i);
+							e.output = Some(verified);
 							break;
 						}
 					}
-					if !verifying.is_empty() && verifying.front().unwrap().hash == block_hash {
+
+					if idx == Some(0) {
 						// we're next!
 						let mut verified = verification.verified.lock();
 						let mut bad = verification.bad.lock();
-						BlockQueue::drain_verifying(&mut verifying, &mut verified, &mut bad);
+						VerificationQueue::drain_verifying(&mut verifying, &mut verified, &mut bad);
 						ready.set();
 					}
 				},
-				Err(err) => {
+				Err(_) => {
 					let mut verifying = verification.verifying.lock();
 					let mut verified = verification.verified.lock();
 					let mut bad = verification.bad.lock();
-					warn!(target: "client", "Stage 2 block verification failed for {}\nError: {:?}", block_hash, err);
+
-					bad.insert(block_hash.clone());
+					bad.insert(hash.clone());
-					verifying.retain(|e| e.hash != block_hash);
+					verifying.retain(|e| e.hash != hash);
-					BlockQueue::drain_verifying(&mut verifying, &mut verified, &mut bad);
+
+					if verifying.front().map_or(false, |x| x.output.is_some()) {
+						VerificationQueue::drain_verifying(&mut verifying, &mut verified, &mut bad);
 						ready.set();
 					}
 				}
 			}
 		}
+	}
 
-	fn drain_verifying(verifying: &mut VecDeque<VerifyingBlock>, verified: &mut VecDeque<PreverifiedBlock>, bad: &mut HashSet<H256>) {
+	fn drain_verifying(verifying: &mut VecDeque<Verifying<K>>, verified: &mut VecDeque<K::Verified>, bad: &mut HashSet<H256>) {
-		while !verifying.is_empty() && verifying.front().unwrap().block.is_some() {
+		while let Some(output) = verifying.front_mut().and_then(|x| x.output.take()) {
-			let block = verifying.pop_front().unwrap().block.unwrap();
+			assert!(verifying.pop_front().is_some());
-			if bad.contains(block.header.parent_hash()) {
+
-				bad.insert(block.header.hash());
+			if bad.contains(&output.parent_hash()) {
-			}
+				bad.insert(output.hash());
-			else {
+			} else {
-				verified.push_back(block);
+				verified.push_back(output);
 			}
 		}
 	}
@@ -288,21 +294,20 @@ impl BlockQueue {
 		}
 	}
 
-	/// Check if the block is currently in the queue
+	/// Check if the item is currently in the queue
-	pub fn block_status(&self, hash: &H256) -> BlockStatus {
+	pub fn status(&self, hash: &H256) -> Status {
 		if self.processing.read().contains(hash) {
-			return BlockStatus::Queued;
+			return Status::Queued;
 		}
 		if self.verification.bad.lock().contains(hash) {
-			return BlockStatus::Bad;
+			return Status::Bad;
 		}
-		BlockStatus::Unknown
+		Status::Unknown
 	}
 
 	/// Add a block to the queue.
-	pub fn import_block(&self, bytes: Bytes) -> ImportResult {
+	pub fn import(&self, input: K::Input) -> ImportResult {
-		let header = BlockView::new(&bytes).header();
+		let h = input.hash();
-		let h = header.hash();
 		{
 			if self.processing.read().contains(&h) {
 				return Err(ImportError::AlreadyQueued.into());
@@ -313,74 +318,71 @@ impl BlockQueue {
 				return Err(ImportError::KnownBad.into());
 			}
 
-			if bad.contains(header.parent_hash()) {
+			if bad.contains(&input.parent_hash()) {
 				bad.insert(h.clone());
 				return Err(ImportError::KnownBad.into());
 			}
 		}
 
-		match verify_block_basic(&header, &bytes, &*self.engine) {
+		match K::create(input, &*self.engine) {
-			Ok(()) => {
+			Ok(item) => {
 				self.processing.write().insert(h.clone());
-				self.verification.unverified.lock().push_back(UnverifiedBlock { header: header, bytes: bytes });
+				self.verification.unverified.lock().push_back(item);
 				self.more_to_verify.notify_all();
 				Ok(h)
 			},
 			Err(err) => {
-				warn!(target: "client", "Stage 1 block verification failed for {}\nError: {:?}", BlockView::new(&bytes).header_view().sha3(), err);
 				self.verification.bad.lock().insert(h.clone());
 				Err(err)
 			}
 		}
 	}
 
-	/// Mark given block and all its children as bad. Stops verification.
+	/// Mark given item and all its children as bad. pauses verification
-	pub fn mark_as_bad(&self, block_hashes: &[H256]) {
+	/// until complete.
-		if block_hashes.is_empty() {
+	pub fn mark_as_bad(&self, hashes: &[H256]) {
+		if hashes.is_empty() {
 			return;
 		}
 		let mut verified_lock = self.verification.verified.lock();
 		let mut verified = &mut *verified_lock;
 		let mut bad = self.verification.bad.lock();
 		let mut processing = self.processing.write();
-		bad.reserve(block_hashes.len());
+		bad.reserve(hashes.len());
-		for hash in block_hashes {
+		for hash in hashes {
 			bad.insert(hash.clone());
 			processing.remove(hash);
 		}
 
 		let mut new_verified = VecDeque::new();
-		for block in verified.drain(..) {
+		for output in verified.drain(..) {
-			if bad.contains(block.header.parent_hash()) {
+			if bad.contains(&output.parent_hash()) {
-				bad.insert(block.header.hash());
+				bad.insert(output.hash());
-				processing.remove(&block.header.hash());
+				processing.remove(&output.hash());
 			} else {
-				new_verified.push_back(block);
+				new_verified.push_back(output);
 			}
 		}
 		*verified = new_verified;
 	}
 
-	/// Mark given block as processed
+	/// Mark given item as processed
-	pub fn mark_as_good(&self, block_hashes: &[H256]) {
+	pub fn mark_as_good(&self, hashes: &[H256]) {
-		if block_hashes.is_empty() {
+		if hashes.is_empty() {
 			return;
 		}
 		let mut processing = self.processing.write();
-		for hash in block_hashes {
+		for hash in hashes {
 			processing.remove(hash);
 		}
 	}
 
-	/// Removes up to `max` verified blocks from the queue
+	/// Removes up to `max` verified items from the queue
-	pub fn drain(&self, max: usize) -> Vec<PreverifiedBlock> {
+	pub fn drain(&self, max: usize) -> Vec<K::Verified> {
 		let mut verified = self.verification.verified.lock();
 		let count = min(max, verified.len());
-		let mut result = Vec::with_capacity(count);
+		let result = verified.drain(..count).collect::<Vec<_>>();
-		for _ in 0..count {
+
-			let block = verified.pop_front().unwrap();
-			result.push(block);
-		}
 		self.ready_signal.reset();
 		if !verified.is_empty() {
 			self.ready_signal.set();
@@ -389,7 +391,7 @@ impl BlockQueue {
 	}
 
 	/// Get queue status.
-	pub fn queue_info(&self) -> BlockQueueInfo {
+	pub fn queue_info(&self) -> QueueInfo {
 		let (unverified_len, unverified_bytes) = {
 			let v = self.verification.unverified.lock();
 			(v.len(), v.heap_size_of_children())
@@ -402,7 +404,8 @@ impl BlockQueue {
 			let v = self.verification.verified.lock();
 			(v.len(), v.heap_size_of_children())
 		};
-		BlockQueueInfo {
+
+		QueueInfo {
 			unverified_queue_size: unverified_len,
 			verifying_queue_size: verifying_len,
 			verified_queue_size: verified_len,
@@ -428,22 +431,22 @@ impl BlockQueue {
 	}
 }
 
-impl MayPanic for BlockQueue {
+impl<K: Kind> MayPanic for VerificationQueue<K> {
 	fn on_panic<F>(&self, closure: F) where F: OnPanicListener {
 		self.panic_handler.on_panic(closure);
 	}
 }
 
-impl Drop for BlockQueue {
+impl<K: Kind> Drop for VerificationQueue<K> {
 	fn drop(&mut self) {
-		trace!(target: "shutdown", "[BlockQueue] Closing...");
+		trace!(target: "shutdown", "[VerificationQueue] Closing...");
 		self.clear();
 		self.deleting.store(true, AtomicOrdering::Release);
 		self.more_to_verify.notify_all();
 		for t in self.verifiers.drain(..) {
 			t.join().unwrap();
 		}
-		trace!(target: "shutdown", "[BlockQueue] Closed.");
+		trace!(target: "shutdown", "[VerificationQueue] Closed.");
 	}
 }
 
@@ -452,7 +455,8 @@ mod tests {
 	use util::*;
 	use io::*;
 	use spec::*;
-	use block_queue::*;
+	use super::{BlockQueue, Config};
+	use super::kind::blocks::Unverified;
 	use tests::helpers::*;
 	use error::*;
 	use views::*;
@@ -460,7 +464,7 @@ mod tests {
 	fn get_test_queue() -> BlockQueue {
 		let spec = get_test_spec();
 		let engine = spec.engine;
-		BlockQueue::new(BlockQueueConfig::default(), engine, IoChannel::disconnected())
+		BlockQueue::new(Config::default(), engine, IoChannel::disconnected())
 	}
 
 	#[test]
@@ -468,13 +472,13 @@ mod tests {
 		// TODO better test
 		let spec = Spec::new_test();
 		let engine = spec.engine;
-		let _ = BlockQueue::new(BlockQueueConfig::default(), engine, IoChannel::disconnected());
+		let _ = BlockQueue::new(Config::default(), engine, IoChannel::disconnected());
 	}
 
 	#[test]
 	fn can_import_blocks() {
 		let queue = get_test_queue();
-		if let Err(e) = queue.import_block(get_good_dummy_block()) {
+		if let Err(e) = queue.import(Unverified::new(get_good_dummy_block())) {
 			panic!("error importing block that is valid by definition({:?})", e);
 		}
 	}
@@ -482,11 +486,11 @@ mod tests {
 	#[test]
 	fn returns_error_for_duplicates() {
 		let queue = get_test_queue();
-		if let Err(e) = queue.import_block(get_good_dummy_block()) {
+		if let Err(e) = queue.import(Unverified::new(get_good_dummy_block())) {
 			panic!("error importing block that is valid by definition({:?})", e);
 		}
 
-		let duplicate_import = queue.import_block(get_good_dummy_block());
+		let duplicate_import = queue.import(Unverified::new(get_good_dummy_block()));
 		match duplicate_import {
 			Err(e) => {
 				match e {
@@ -503,14 +507,14 @@ mod tests {
 		let queue = get_test_queue();
 		let block = get_good_dummy_block();
 		let hash = BlockView::new(&block).header().hash().clone();
-		if let Err(e) = queue.import_block(block) {
+		if let Err(e) = queue.import(Unverified::new(block)) {
 			panic!("error importing block that is valid by definition({:?})", e);
 		}
 		queue.flush();
 		queue.drain(10);
 		queue.mark_as_good(&[ hash ]);
 
-		if let Err(e) = queue.import_block(get_good_dummy_block()) {
+		if let Err(e) = queue.import(Unverified::new(get_good_dummy_block())) {
 			panic!("error importing block that has already been drained ({:?})", e);
 		}
 	}
@@ -518,7 +522,8 @@ mod tests {
 	#[test]
 	fn returns_empty_once_finished() {
 		let queue = get_test_queue();
-		queue.import_block(get_good_dummy_block()).expect("error importing block that is valid by definition");
+		queue.import(Unverified::new(get_good_dummy_block()))
+			.expect("error importing block that is valid by definition");
 		queue.flush();
 		queue.drain(1);
 
@@ -529,13 +534,13 @@ mod tests {
 	fn test_mem_limit() {
 		let spec = get_test_spec();
 		let engine = spec.engine;
-		let mut config = BlockQueueConfig::default();
+		let mut config = Config::default();
 		config.max_mem_use = super::MIN_MEM_LIMIT;  // empty queue uses about 15000
 		let queue = BlockQueue::new(config, engine, IoChannel::disconnected());
 		assert!(!queue.queue_info().is_full());
 		let mut blocks = get_good_dummy_block_seq(50);
 		for b in blocks.drain(..) {
-			queue.import_block(b).unwrap();
+			queue.import(Unverified::new(b)).unwrap();
 		}
 		assert!(queue.queue_info().is_full());
 	}

ethcore/src/verification/verification.rs

@@ -36,14 +36,22 @@ pub struct PreverifiedBlock {
 	pub bytes: Bytes,
 }
 
+impl HeapSizeOf for PreverifiedBlock {
+	fn heap_size_of_children(&self) -> usize {
+		self.header.heap_size_of_children()
+			+ self.transactions.heap_size_of_children()
+			+ self.bytes.heap_size_of_children()
+	}
+}
+
 /// Phase 1 quick block verification. Only does checks that are cheap. Operates on a single block
 pub fn verify_block_basic(header: &Header, bytes: &[u8], engine: &Engine) -> Result<(), Error> {
-	try!(verify_header(&header, engine));
+	try!(verify_header_params(&header, engine));
 	try!(verify_block_integrity(bytes, &header.transactions_root(), &header.uncles_hash()));
 	try!(engine.verify_block_basic(&header, Some(bytes)));
 	for u in try!(UntrustedRlp::new(bytes).at(2)).iter().map(|rlp| rlp.as_val::<Header>()) {
 		let u = try!(u);
-		try!(verify_header(&u, engine));
+		try!(verify_header_params(&u, engine));
 		try!(engine.verify_block_basic(&u, None));
 	}
 	// Verify transactions.
@@ -179,7 +187,7 @@ pub fn verify_block_final(expected: &Header, got: &Header) -> Result<(), Error>
 }
 
 /// Check basic header parameters.
-fn verify_header(header: &Header, engine: &Engine) -> Result<(), Error> {
+pub fn verify_header_params(header: &Header, engine: &Engine) -> Result<(), Error> {
 	if header.number() >= From::from(BlockNumber::max_value()) {
 		return Err(From::from(BlockError::RidiculousNumber(OutOfBounds { max: Some(From::from(BlockNumber::max_value())), min: None, found: header.number() })))
 	}

ethkey/src/signature.rs

@@ -25,7 +25,6 @@ use bigint::hash::{H520, H256, FixedHash};
 use {Secret, Public, SECP256K1, Error, Message, public_to_address, Address};
 
 #[repr(C)]
-#[derive(Eq)]
 pub struct Signature([u8; 65]);
 
 impl Signature {
@@ -76,6 +75,9 @@ impl PartialEq for Signature {
 	}
 }
 
+// manual implementation required in Rust 1.13+, see `std::cmp::AssertParamIsEq`.
+impl Eq for Signature { }
+
 // also manual for the same reason, but the pretty printing might be useful.
 impl fmt::Debug for Signature {
 	fn fmt(&self, f: &mut fmt::Formatter) -> Result<(), fmt::Error> {

ethstore/src/account/safe_account.rs

@@ -16,14 +16,14 @@
 
 use ethkey::{KeyPair, sign, Address, Secret, Signature, Message};
 use {json, Error, crypto};
-use crypto::{Keccak256};
+use crypto::Keccak256;
 use random::Random;
 use account::{Version, Cipher, Kdf, Aes128Ctr, Pbkdf2, Prf};
 
 #[derive(Debug, PartialEq, Clone)]
 pub struct Crypto {
 	pub cipher: Cipher,
-	pub ciphertext: [u8; 32],
+	pub ciphertext: Vec<u8>,
 	pub kdf: Kdf,
 	pub mac: [u8; 32],
 }
@@ -95,7 +95,7 @@ impl Crypto {
 			cipher: Cipher::Aes128Ctr(Aes128Ctr {
 				iv: iv,
 			}),
-			ciphertext: ciphertext,
+			ciphertext: ciphertext.to_vec(),
 			kdf: Kdf::Pbkdf2(Pbkdf2 {
 				dklen: crypto::KEY_LENGTH as u32,
 				salt: salt,
@@ -107,6 +107,10 @@ impl Crypto {
 	}
 
 	pub fn secret(&self, password: &str) -> Result<Secret, Error> {
+		if self.ciphertext.len() > 32 {
+			return Err(Error::InvalidSecret);
+		}
+
 		let (derived_left_bits, derived_right_bits) = match self.kdf {
 			Kdf::Pbkdf2(ref params) => crypto::derive_key_iterations(password, &params.salt, params.c),
 			Kdf::Scrypt(ref params) => crypto::derive_key_scrypt(password, &params.salt, params.n, params.p, params.r),
@@ -122,7 +126,8 @@ impl Crypto {
 
 		match self.cipher {
 			Cipher::Aes128Ctr(ref params) => {
-				crypto::aes::decrypt(&derived_left_bits, &params.iv, &self.ciphertext, &mut *secret)
+				let from = 32 - self.ciphertext.len();
+				crypto::aes::decrypt(&derived_left_bits, &params.iv, &self.ciphertext, &mut (&mut *secret)[from..])
 			},
 		}
 

ethstore/src/dir/disk.rs

@@ -76,15 +76,14 @@ impl DiskDirectory {
 			.map(|entry| entry.path())
 			.collect::<Vec<PathBuf>>();
 
-		let files: Result<Vec<_>, _> = paths.iter()
+		paths
-			.map(fs::File::open)
+			.iter()
-			.collect();
+			.map(|p| (
-
+				fs::File::open(p)
-		let files = try!(files);
+					.map_err(Error::from)
-
+					.and_then(|r| json::KeyFile::load(r).map_err(|e| Error::Custom(format!("{:?}", e)))),
-		files.into_iter()
+				p
-			.map(json::KeyFile::load)
+			))
-			.zip(paths.into_iter())
 			.map(|(file, path)| match file {
 				Ok(file) => Ok((path.clone(), SafeAccount::from_file(
 					file, Some(path.file_name().and_then(|n| n.to_str()).expect("Keys have valid UTF8 names only.").to_owned())

ethstore/src/json/bytes.rs

@@ -0,0 +1,58 @@
+use std::{ops, str};
+use serde::{Deserialize, Deserializer, Error, Serialize, Serializer};
+use rustc_serialize::hex::{ToHex, FromHex, FromHexError};
+
+#[derive(Debug, PartialEq)]
+pub struct Bytes(Vec<u8>);
+
+impl ops::Deref for Bytes {
+	type Target = [u8];
+
+	fn deref(&self) -> &Self::Target {
+		&self.0
+	}
+}
+
+impl Deserialize for Bytes {
+	fn deserialize<D>(deserializer: &mut D) -> Result<Self, D::Error>
+		where D: Deserializer
+	{
+		let s = try!(String::deserialize(deserializer));
+		let data = try!(s.from_hex().map_err(|e| Error::custom(format!("Invalid hex value {}", e))));
+		Ok(Bytes(data))
+	}
+}
+
+impl Serialize for Bytes {
+	fn serialize<S>(&self, serializer: &mut S) -> Result<(), S::Error>
+	where S: Serializer {
+		serializer.serialize_str(&self.0.to_hex())
+	}
+}
+
+impl str::FromStr for Bytes {
+	type Err = FromHexError;
+
+	fn from_str(s: &str) -> Result<Self, Self::Err> {
+		s.from_hex().map(Bytes)
+	}
+}
+
+impl From<&'static str> for Bytes {
+	fn from(s: &'static str) -> Self {
+		s.parse().expect(&format!("invalid string literal for {}: '{}'", stringify!(Self), s))
+	}
+}
+
+impl From<Vec<u8>> for Bytes {
+	fn from(v: Vec<u8>) -> Self {
+		Bytes(v)
+	}
+}
+
+impl From<Bytes> for Vec<u8> {
+	fn from(b: Bytes) -> Self {
+		b.0
+	}
+}
+

ethstore/src/json/crypto.rs

@@ -16,12 +16,14 @@
 
 use serde::{Deserialize, Deserializer, Serialize, Serializer, Error};
 use serde::de::{Visitor, MapVisitor};
-use super::{Cipher, CipherSer, CipherSerParams, Kdf, KdfSer, KdfSerParams, H256};
+use super::{Cipher, CipherSer, CipherSerParams, Kdf, KdfSer, KdfSerParams, H256, Bytes};
+
+pub type CipherText = Bytes;
 
 #[derive(Debug, PartialEq)]
 pub struct Crypto {
 	pub cipher: Cipher,
-	pub ciphertext: H256,
+	pub ciphertext: CipherText,
 	pub kdf: Kdf,
 	pub mac: H256,
 }

ethstore/src/json/hash.rs

@@ -14,9 +14,7 @@
 // You should have received a copy of the GNU General Public License
 // along with Parity.  If not, see <http://www.gnu.org/licenses/>.
 
-use std::fmt;
+use std::{ops, fmt, str};
-use std::ops;
-use std::str::FromStr;
 use rustc_serialize::hex::{FromHex, ToHex};
 use serde::{Serialize, Serializer, Deserialize, Deserializer, Error as SerdeError};
 use serde::de::Visitor;
@@ -65,7 +63,7 @@ macro_rules! impl_hash {
 					type Value = $name;
 
 					fn visit_str<E>(&mut self, value: &str) -> Result<Self::Value, E> where E: SerdeError {
-						FromStr::from_str(value).map_err(SerdeError::custom)
+						value.parse().map_err(SerdeError::custom)
 					}
 
 					fn visit_string<E>(&mut self, value: String) -> Result<Self::Value, E> where E: SerdeError {
@@ -77,7 +75,7 @@ macro_rules! impl_hash {
 			}
 		}
 
-		impl FromStr for $name {
+		impl str::FromStr for $name {
 			type Err = Error;
 
 			fn from_str(value: &str) -> Result<Self, Self::Err> {
@@ -92,6 +90,12 @@ macro_rules! impl_hash {
 			}
 		}
 
+		impl From<&'static str> for $name {
+			fn from(s: &'static str) -> Self {
+				s.parse().expect(&format!("invalid string literal for {}: '{}'", stringify!(Self), s))
+			}
+		}
+
 		impl From<[u8; $size]> for $name {
 			fn from(bytes: [u8; $size]) -> Self {
 				$name(bytes)

ethstore/src/json/id.rs

@@ -15,8 +15,7 @@
 // along with Parity.  If not, see <http://www.gnu.org/licenses/>.
 
 //! Universaly unique identifier.
-use std::str::FromStr;
+use std::{fmt, str};
-use std::fmt;
 use rustc_serialize::hex::{ToHex, FromHex};
 use serde::{Deserialize, Serialize, Deserializer, Serializer, Error as SerdeError};
 use serde::de::Visitor;
@@ -73,7 +72,7 @@ fn copy_into(from: &str, into: &mut [u8]) -> Result<(), Error> {
 	Ok(())
 }
 
-impl FromStr for UUID {
+impl str::FromStr for UUID {
 	type Err = Error;
 
 	fn from_str(s: &str) -> Result<Self, Self::Err> {
@@ -95,6 +94,12 @@ impl FromStr for UUID {
 	}
 }
 
+impl From<&'static str> for UUID {
+	fn from(s: &'static str) -> Self {
+		s.parse().expect(&format!("invalid string literal for {}: '{}'", stringify!(Self), s))
+	}
+}
+
 impl Serialize for UUID {
 	fn serialize<S>(&self, serializer: &mut S) -> Result<(), S::Error>
 	where S: Serializer {
@@ -116,7 +121,7 @@ impl Visitor for UUIDVisitor {
 	type Value = UUID;
 
 	fn visit_str<E>(&mut self, value: &str) -> Result<Self::Value, E> where E: SerdeError {
-		UUID::from_str(value).map_err(SerdeError::custom)
+		value.parse().map_err(SerdeError::custom)
 	}
 
 	fn visit_string<E>(&mut self, value: String) -> Result<Self::Value, E> where E: SerdeError {
@@ -126,19 +131,18 @@ impl Visitor for UUIDVisitor {
 
 #[cfg(test)]
 mod tests {
-	use std::str::FromStr;
 	use super::UUID;
 
 	#[test]
 	fn uuid_from_str() {
-		let uuid = UUID::from_str("3198bc9c-6672-5ab3-d995-4942343ae5b6").unwrap();
+		let uuid: UUID = "3198bc9c-6672-5ab3-d995-4942343ae5b6".into();
 		assert_eq!(uuid, UUID::from([0x31, 0x98, 0xbc, 0x9c, 0x66, 0x72, 0x5a, 0xb3, 0xd9, 0x95, 0x49, 0x42, 0x34, 0x3a, 0xe5, 0xb6]));
 	}
 
 	#[test]
 	fn uuid_from_and_to_str() {
 		let from = "3198bc9c-6672-5ab3-d995-4942343ae5b6";
-		let uuid = UUID::from_str(from).unwrap();
+		let uuid: UUID = from.into();
 		let to: String = uuid.into();
 		assert_eq!(from, &to);
 	}

ethstore/src/json/key_file.rs

@@ -153,7 +153,7 @@ impl KeyFile {
 mod tests {
 	use std::str::FromStr;
 	use serde_json;
-	use json::{KeyFile, UUID, Version, Crypto, Cipher, Aes128Ctr, Kdf, Scrypt, H128, H160, H256};
+	use json::{KeyFile, UUID, Version, Crypto, Cipher, Aes128Ctr, Kdf, Scrypt};
 
 	#[test]
 	fn basic_keyfile() {
@@ -185,20 +185,20 @@ mod tests {
 		let expected = KeyFile {
 			id: UUID::from_str("8777d9f6-7860-4b9b-88b7-0b57ee6b3a73").unwrap(),
 			version: Version::V3,
-			address: H160::from_str("6edddfc6349aff20bc6467ccf276c5b52487f7a8").unwrap(),
+			address: "6edddfc6349aff20bc6467ccf276c5b52487f7a8".into(),
 			crypto: Crypto {
 				cipher: Cipher::Aes128Ctr(Aes128Ctr {
-					iv: H128::from_str("b5a7ec855ec9e2c405371356855fec83").unwrap(),
+					iv: "b5a7ec855ec9e2c405371356855fec83".into(),
 				}),
-				ciphertext: H256::from_str("7203da0676d141b138cd7f8e1a4365f59cc1aa6978dc5443f364ca943d7cb4bc").unwrap(),
+				ciphertext: "7203da0676d141b138cd7f8e1a4365f59cc1aa6978dc5443f364ca943d7cb4bc".into(),
 				kdf: Kdf::Scrypt(Scrypt {
 					n: 262144,
 					dklen: 32,
 					p: 1,
 					r: 8,
-					salt: H256::from_str("1e8642fdf1f87172492c1412fc62f8db75d796cdfa9c53c3f2b11e44a2a1b209").unwrap(),
+					salt: "1e8642fdf1f87172492c1412fc62f8db75d796cdfa9c53c3f2b11e44a2a1b209".into(),
 				}),
-				mac: H256::from_str("46325c5d4e8c991ad2683d525c7854da387138b6ca45068985aa4959fa2b8c8f").unwrap(),
+				mac: "46325c5d4e8c991ad2683d525c7854da387138b6ca45068985aa4959fa2b8c8f".into(),
 			},
 			name: Some("Test".to_owned()),
 			meta: Some("{}".to_owned()),
@@ -234,22 +234,22 @@ mod tests {
 		}"#;
 
 		let expected = KeyFile {
-			id: UUID::from_str("8777d9f6-7860-4b9b-88b7-0b57ee6b3a73").unwrap(),
+			id: "8777d9f6-7860-4b9b-88b7-0b57ee6b3a73".into(),
 			version: Version::V3,
-			address: H160::from_str("6edddfc6349aff20bc6467ccf276c5b52487f7a8").unwrap(),
+			address: "6edddfc6349aff20bc6467ccf276c5b52487f7a8".into(),
 			crypto: Crypto {
 				cipher: Cipher::Aes128Ctr(Aes128Ctr {
-					iv: H128::from_str("b5a7ec855ec9e2c405371356855fec83").unwrap(),
+					iv: "b5a7ec855ec9e2c405371356855fec83".into(),
 				}),
-				ciphertext: H256::from_str("7203da0676d141b138cd7f8e1a4365f59cc1aa6978dc5443f364ca943d7cb4bc").unwrap(),
+				ciphertext: "7203da0676d141b138cd7f8e1a4365f59cc1aa6978dc5443f364ca943d7cb4bc".into(),
 				kdf: Kdf::Scrypt(Scrypt {
 					n: 262144,
 					dklen: 32,
 					p: 1,
 					r: 8,
-					salt: H256::from_str("1e8642fdf1f87172492c1412fc62f8db75d796cdfa9c53c3f2b11e44a2a1b209").unwrap(),
+					salt: "1e8642fdf1f87172492c1412fc62f8db75d796cdfa9c53c3f2b11e44a2a1b209".into(),
 				}),
-				mac: H256::from_str("46325c5d4e8c991ad2683d525c7854da387138b6ca45068985aa4959fa2b8c8f").unwrap(),
+				mac: "46325c5d4e8c991ad2683d525c7854da387138b6ca45068985aa4959fa2b8c8f".into(),
 			},
 			name: None,
 			meta: None,
@@ -262,22 +262,22 @@ mod tests {
 	#[test]
 	fn to_and_from_json() {
 		let file = KeyFile {
-			id: UUID::from_str("8777d9f6-7860-4b9b-88b7-0b57ee6b3a73").unwrap(),
+			id: "8777d9f6-7860-4b9b-88b7-0b57ee6b3a73".into(),
 			version: Version::V3,
-			address: H160::from_str("6edddfc6349aff20bc6467ccf276c5b52487f7a8").unwrap(),
+			address: "6edddfc6349aff20bc6467ccf276c5b52487f7a8".into(),
 			crypto: Crypto {
 				cipher: Cipher::Aes128Ctr(Aes128Ctr {
-					iv: H128::from_str("b5a7ec855ec9e2c405371356855fec83").unwrap(),
+					iv: "b5a7ec855ec9e2c405371356855fec83".into(),
 				}),
-				ciphertext: H256::from_str("7203da0676d141b138cd7f8e1a4365f59cc1aa6978dc5443f364ca943d7cb4bc").unwrap(),
+				ciphertext: "7203da0676d141b138cd7f8e1a4365f59cc1aa6978dc5443f364ca943d7cb4bc".into(),
 				kdf: Kdf::Scrypt(Scrypt {
 					n: 262144,
 					dklen: 32,
 					p: 1,
 					r: 8,
-					salt: H256::from_str("1e8642fdf1f87172492c1412fc62f8db75d796cdfa9c53c3f2b11e44a2a1b209").unwrap(),
+					salt: "1e8642fdf1f87172492c1412fc62f8db75d796cdfa9c53c3f2b11e44a2a1b209".into(),
 				}),
-				mac: H256::from_str("46325c5d4e8c991ad2683d525c7854da387138b6ca45068985aa4959fa2b8c8f").unwrap(),
+				mac: "46325c5d4e8c991ad2683d525c7854da387138b6ca45068985aa4959fa2b8c8f".into(),
 			},
 			name: Some("Test".to_owned()),
 			meta: None,

ethstore/src/json/mod.rs.in

@@ -1,3 +1,4 @@
+mod bytes;
 mod cipher;
 mod crypto;
 mod error;
@@ -8,8 +9,9 @@ mod key_file;
 mod presale;
 mod version;
 
+pub use self::bytes::Bytes;
 pub use self::cipher::{Cipher, CipherSer, CipherSerParams, Aes128Ctr};
-pub use self::crypto::Crypto;
+pub use self::crypto::{Crypto, CipherText};
 pub use self::error::Error;
 pub use self::hash::{H128, H160, H256};
 pub use self::id::UUID;

ethstore/src/json/presale.rs

@@ -1,30 +1,8 @@
 use std::io::Read;
-use std::ops::Deref;
 use serde_json;
-use serde::{Deserialize, Deserializer, Error};
+use super::{H160, Bytes};
-use rustc_serialize::hex::FromHex;
-use super::{H160};
 
-#[derive(Debug, PartialEq)]
+pub type Encseed = Bytes;
-pub struct Encseed(Vec<u8>);
-
-impl Deref for Encseed {
-	type Target = [u8];
-
-	fn deref(&self) -> &Self::Target {
-		&self.0
-	}
-}
-
-impl Deserialize for Encseed {
-	fn deserialize<D>(deserializer: &mut D) -> Result<Self, D::Error>
-		where D: Deserializer
-	{
-		let s = try!(String::deserialize(deserializer));
-		let data = try!(s.from_hex().map_err(|e| Error::custom(format!("Invalid hex value {}", e))));
-		Ok(Encseed(data))
-	}
-}
 
 #[derive(Debug, PartialEq, Deserialize)]
 pub struct PresaleWallet {
@@ -43,8 +21,7 @@ impl PresaleWallet {
 mod tests {
 	use std::str::FromStr;
 	use serde_json;
-	use rustc_serialize::hex::FromHex;
+	use json::{PresaleWallet, H160};
-	use json::{PresaleWallet, H160, Encseed};
 
 	#[test]
 	fn presale_wallet() {
@@ -57,7 +34,7 @@ mod tests {
 		} "#;
 
 		let expected = PresaleWallet {
-			encseed: Encseed("137103c28caeebbcea5d7f95edb97a289ded151b72159137cb7b2671f394f54cff8c121589dcb373e267225547b3c71cbdb54f6e48ec85cd549f96cf0dedb3bc0a9ac6c79b9c426c5878ca2c9d06ff42a23cb648312fc32ba83649de0928e066".from_hex().unwrap()),
+			encseed: "137103c28caeebbcea5d7f95edb97a289ded151b72159137cb7b2671f394f54cff8c121589dcb373e267225547b3c71cbdb54f6e48ec85cd549f96cf0dedb3bc0a9ac6c79b9c426c5878ca2c9d06ff42a23cb648312fc32ba83649de0928e066".into(),
 			address: H160::from_str("ede84640d1a1d3e06902048e67aa7db8d52c2ce1").unwrap(),
 		};
 
@@ -77,7 +54,7 @@ mod tests {
 		} "#;
 
 		let expected = PresaleWallet {
-			encseed: Encseed("137103c28caeebbcea5d7f95edb97a289ded151b72159137cb7b2671f394f54cff8c121589dcb373e267225547b3c71cbdb54f6e48ec85cd549f96cf0dedb3bc0a9ac6c79b9c426c5878ca2c9d06ff42a23cb648312fc32ba83649de0928e066137103c28caeebbcea5d7f95edb97a289ded151b72159137cb7b2671f394f54cff8c121589dcb373e267225547b3c71cbdb54f6e48ec85cd549f96cf0dedb3bc0a9ac6c79b9c426c5878ca2c9d06ff42a23cb648312fc32ba83649de0928e066137103c28caeebbcea5d7f95edb97a289ded151b72159137cb7b2671f394f54cff8c121589dcb373e267225547b3c71cbdb54f6e48ec85cd549f96cf0dedb3bc0a9ac6c79b9c426c5878ca2c9d06ff42a23cb648312fc32ba83649de0928e066137103c28caeebbcea5d7f95edb97a289ded151b72159137cb7b2671f394f54cff8c121589dcb373e267225547b3c71cbdb54f6e48ec85cd549f96cf0dedb3bc0a9ac6c79b9c426c5878ca2c9d06ff42a23cb648312fc32ba83649de0928e066137103c28caeebbcea5d7f95edb97a289ded151b72159137cb7b2671f394f54cff8c121589dcb373e267225547b3c71cbdb54f6e48ec85cd549f96cf0dedb3bc0a9ac6c79b9c426c5878ca2c9d06ff42a23cb648312fc32ba83649de0928e066137103c28caeebbcea5d7f95edb97a289ded151b72159137cb7b2671f394f54cff8c121589dcb373e267225547b3c71cbdb54f6e48ec85cd549f96cf0dcf0dcf0dcf0dcf0dcf0dcf0dcf0dcf0dcf0dcf0dcf0dcf0dcf0dcf0dcf0dcf0dcf0dcf0dcf0dcf0dcf0dcf0dcf0dcf0dcf0dcf0dcf0dcf0dcf0dcf0dcf0dcf0dcf0dcf0dcf0dcf0dcf0dcf0dcf0dcf0dcf0dcf0d".from_hex().unwrap()),
+			encseed: "137103c28caeebbcea5d7f95edb97a289ded151b72159137cb7b2671f394f54cff8c121589dcb373e267225547b3c71cbdb54f6e48ec85cd549f96cf0dedb3bc0a9ac6c79b9c426c5878ca2c9d06ff42a23cb648312fc32ba83649de0928e066137103c28caeebbcea5d7f95edb97a289ded151b72159137cb7b2671f394f54cff8c121589dcb373e267225547b3c71cbdb54f6e48ec85cd549f96cf0dedb3bc0a9ac6c79b9c426c5878ca2c9d06ff42a23cb648312fc32ba83649de0928e066137103c28caeebbcea5d7f95edb97a289ded151b72159137cb7b2671f394f54cff8c121589dcb373e267225547b3c71cbdb54f6e48ec85cd549f96cf0dedb3bc0a9ac6c79b9c426c5878ca2c9d06ff42a23cb648312fc32ba83649de0928e066137103c28caeebbcea5d7f95edb97a289ded151b72159137cb7b2671f394f54cff8c121589dcb373e267225547b3c71cbdb54f6e48ec85cd549f96cf0dedb3bc0a9ac6c79b9c426c5878ca2c9d06ff42a23cb648312fc32ba83649de0928e066137103c28caeebbcea5d7f95edb97a289ded151b72159137cb7b2671f394f54cff8c121589dcb373e267225547b3c71cbdb54f6e48ec85cd549f96cf0dedb3bc0a9ac6c79b9c426c5878ca2c9d06ff42a23cb648312fc32ba83649de0928e066137103c28caeebbcea5d7f95edb97a289ded151b72159137cb7b2671f394f54cff8c121589dcb373e267225547b3c71cbdb54f6e48ec85cd549f96cf0dcf0dcf0dcf0dcf0dcf0dcf0dcf0dcf0dcf0dcf0dcf0dcf0dcf0dcf0dcf0dcf0dcf0dcf0dcf0dcf0dcf0dcf0dcf0dcf0dcf0dcf0dcf0dcf0dcf0dcf0dcf0dcf0dcf0dcf0dcf0dcf0dcf0dcf0dcf0dcf0dcf0dcf0d".into(),
 			address: H160::from_str("ede84640d1a1d3e06902048e67aa7db8d52c2ce1").unwrap(),
 		};
 

ethstore/tests/api.rs

@@ -19,9 +19,8 @@ extern crate ethstore;
 
 mod util;
 
-use std::str::FromStr;
 use ethstore::{SecretStore, EthStore};
-use ethstore::ethkey::{Random, Generator, Secret, Address};
+use ethstore::ethkey::{Random, Generator, Secret, KeyPair, verify_address};
 use ethstore::dir::DiskDirectory;
 use util::TransientDir;
 
@@ -103,14 +102,21 @@ fn pat_path() -> &'static str {
 	}
 }
 
+fn ciphertext_path() -> &'static str {
+	match ::std::fs::metadata("ethstore") {
+		Ok(_) => "ethstore/tests/res/ciphertext",
+		Err(_) => "tests/res/ciphertext",
+	}
+}
+
 #[test]
 fn secret_store_laod_geth_files() {
 	let dir = DiskDirectory::at(test_path());
 	let store = EthStore::open(Box::new(dir)).unwrap();
 	assert_eq!(store.accounts().unwrap(), vec![
-		Address::from_str("3f49624084b67849c7b4e805c5988c21a430f9d9").unwrap(),
+		"3f49624084b67849c7b4e805c5988c21a430f9d9".into(),
-		Address::from_str("5ba4dcf897e97c2bdf8315b9ef26c13c085988cf").unwrap(),
+		"5ba4dcf897e97c2bdf8315b9ef26c13c085988cf".into(),
-		Address::from_str("63121b431a52f8043c16fcf0d1df9cb7b5f66649").unwrap(),
+		"63121b431a52f8043c16fcf0d1df9cb7b5f66649".into(),
 	]);
 }
 
@@ -119,9 +125,30 @@ fn secret_store_load_pat_files() {
 	let dir = DiskDirectory::at(pat_path());
 	let store = EthStore::open(Box::new(dir)).unwrap();
 	assert_eq!(store.accounts().unwrap(), vec![
-		Address::from_str("3f49624084b67849c7b4e805c5988c21a430f9d9").unwrap(),
+		"3f49624084b67849c7b4e805c5988c21a430f9d9".into(),
-		Address::from_str("5ba4dcf897e97c2bdf8315b9ef26c13c085988cf").unwrap(),
+		"5ba4dcf897e97c2bdf8315b9ef26c13c085988cf".into(),
 	]);
 }
 
+#[test]
+fn test_decrypting_files_with_short_ciphertext() {
+	// 31e9d1e6d844bd3a536800ef8d8be6a9975db509, 30
+	let kp1 = KeyPair::from_secret("000081c29e8142bb6a81bef5a92bda7a8328a5c85bb2f9542e76f9b0f94fc018".into()).unwrap();
+	// d1e64e5480bfaf733ba7d48712decb8227797a4e , 31
+	let kp2 = KeyPair::from_secret("00fa7b3db73dc7dfdf8c5fbdb796d741e4488628c41fc4febd9160a866ba0f35".into()).unwrap();
+	let dir = DiskDirectory::at(ciphertext_path());
+	let store = EthStore::open(Box::new(dir)).unwrap();
+	let accounts = store.accounts().unwrap();
+	assert_eq!(accounts, vec![
+		"31e9d1e6d844bd3a536800ef8d8be6a9975db509".into(),
+		"d1e64e5480bfaf733ba7d48712decb8227797a4e".into(),
+	]);
 
+	let message = Default::default();
+
+	let s1 = store.sign(&accounts[0], "foo", &message).unwrap();
+	let s2 = store.sign(&accounts[1], "foo", &message).unwrap();
+	assert!(verify_address(&accounts[0], &s1, &message).unwrap());
+	assert!(verify_address(&kp1.address(), &s1, &message).unwrap());
+	assert!(verify_address(&kp2.address(), &s2, &message).unwrap());
+}

ethstore/tests/res/ciphertext/30.json

@@ -0,0 +1,21 @@
+{
+	"address" : "31e9d1e6d844bd3a536800ef8d8be6a9975db509",
+	"crypto" : {
+		"cipher" : "aes-128-ctr",
+		"cipherparams" : {
+			"iv" : "3ca92af36ad7c2cd92454c59cea5ef00"
+		},
+		"ciphertext" : "108b7d34f3442fc26ab1ab90ca91476ba6bfa8c00975a49ef9051dc675aa",
+		"kdf" : "scrypt",
+		"kdfparams" : {
+			"dklen" : 32,
+			"n" : 2,
+			"r" : 8,
+			"p" : 1,
+			"salt" : "d0769e608fb86cda848065642a9c6fa046845c928175662b8e356c77f914cd3b"
+		},
+		"mac" : "75d0e6759f7b3cefa319c3be41680ab6beea7d8328653474bd06706d4cc67420"
+	},
+	"id" : "a37e1559-5955-450d-8075-7b8931b392b2",
+	"version" : 3
+}

ethstore/tests/res/ciphertext/31.json

@@ -0,0 +1,21 @@
+{
+	"address" : "d1e64e5480bfaf733ba7d48712decb8227797a4e",
+	"crypto" : {
+		"cipher" : "aes-128-ctr",
+		"cipherparams" : {
+			"iv" : "e0c41130a323adc1446fc82f724bca2f"
+		},
+		"ciphertext" : "9517cd5bdbe69076f9bf5057248c6c050141e970efa36ce53692d5d59a3984",
+		"kdf" : "scrypt",
+		"kdfparams" : {
+			"dklen" : 32,
+			"n" : 2,
+			"r" : 8,
+			"p" : 1,
+			"salt" : "711f816911c92d649fb4c84b047915679933555030b3552c1212609b38208c63"
+		},
+		"mac" : "d5e116151c6aa71470e67a7d42c9620c75c4d23229847dcc127794f0732b0db5"
+	},
+	"id" : "fecfc4ce-e956-48fd-953b-30f8b52ed66c",
+	"version" : 3
+}