diff --git a/.travis.yml b/.travis.yml index 167dc2622..8d2349dae 100644 --- a/.travis.yml +++ b/.travis.yml @@ -44,12 +44,12 @@ after_success: | wget https://github.com/SimonKagstrom/kcov/archive/master.tar.gz && tar xzf master.tar.gz && mkdir kcov-master/build && cd kcov-master/build && cmake .. && make && make install DESTDIR=../tmp && cd ../.. && cargo test --no-run ${KCOV_FEATURES} ${TARGETS} && - ./kcov-master/tmp/usr/local/bin/kcov --exclude-pattern /.cargo,/root/.multirust,src/tests,util/json-tests,util/src/network/tests,sync/src/tests,ethcore/src/tests,ethcore/src/evm/tests target/kcov target/debug/deps/ethcore_util-* && - ./kcov-master/tmp/usr/local/bin/kcov --exclude-pattern /.cargo,/root/.multirust,src/tests,util/json-tests,util/src/network/tests,sync/src/tests,ethcore/src/tests,ethcore/src/evm/tests target/kcov target/debug/deps/ethash-* && - ./kcov-master/tmp/usr/local/bin/kcov --exclude-pattern /.cargo,/root/.multirust,src/tests,util/json-tests,util/src/network/tests,sync/src/tests,ethcore/src/tests,ethcore/src/evm/tests target/kcov target/debug/deps/ethcore-* && - ./kcov-master/tmp/usr/local/bin/kcov --exclude-pattern /.cargo,/root/.multirust,src/tests,util/json-tests,util/src/network/tests,sync/src/tests,ethcore/src/tests,ethcore/src/evm/tests target/kcov target/debug/deps/ethsync-* && - ./kcov-master/tmp/usr/local/bin/kcov --exclude-pattern /.cargo,/root/.multirust,src/tests,util/json-tests,util/src/network/tests,sync/src/tests,ethcore/src/tests,ethcore/src/evm/tests target/kcov target/debug/deps/ethcore_rpc-* && - ./kcov-master/tmp/usr/local/bin/kcov --coveralls-id=${TRAVIS_JOB_ID} --exclude-pattern /.cargo,/root/.multirust target/kcov target/debug/parity-* && + ./kcov-master/tmp/usr/local/bin/kcov --exclude-pattern /usr/,/.cargo,/root/.multirust,src/tests,util/json-tests,util/src/network/tests,sync/src/tests,ethcore/src/tests,ethcore/src/evm/tests target/kcov target/debug/deps/ethcore_util-* && + ./kcov-master/tmp/usr/local/bin/kcov --exclude-pattern /usr/,/.cargo,/root/.multirust,src/tests,util/json-tests,util/src/network/tests,sync/src/tests,ethcore/src/tests,ethcore/src/evm/tests target/kcov target/debug/deps/ethash-* && + ./kcov-master/tmp/usr/local/bin/kcov --exclude-pattern /usr/,/.cargo,/root/.multirust,src/tests,util/json-tests,util/src/network/tests,sync/src/tests,ethcore/src/tests,ethcore/src/evm/tests target/kcov target/debug/deps/ethcore-* && + ./kcov-master/tmp/usr/local/bin/kcov --exclude-pattern /usr/,/.cargo,/root/.multirust,src/tests,util/json-tests,util/src/network/tests,sync/src/tests,ethcore/src/tests,ethcore/src/evm/tests target/kcov target/debug/deps/ethsync-* && + ./kcov-master/tmp/usr/local/bin/kcov --exclude-pattern /usr/,/.cargo,/root/.multirust,src/tests,util/json-tests,util/src/network/tests,sync/src/tests,ethcore/src/tests,ethcore/src/evm/tests target/kcov target/debug/deps/ethcore_rpc-* && + ./kcov-master/tmp/usr/local/bin/kcov --coveralls-id=${TRAVIS_JOB_ID} --exclude-pattern /usr/,/.cargo,/root/.multirust target/kcov target/debug/parity-* && [ $TRAVIS_BRANCH = master ] && [ $TRAVIS_PULL_REQUEST = false ] && [ $TRAVIS_RUST_VERSION = beta ] && diff --git a/Cargo.lock b/Cargo.lock index d37c641b6..bca236813 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -14,6 +14,7 @@ dependencies = [ "ethsync 0.9.99", "fdlimit 0.1.0", "log 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)", + "number_prefix 0.2.5 (registry+https://github.com/rust-lang/crates.io-index)", "rustc-serialize 0.3.18 (registry+https://github.com/rust-lang/crates.io-index)", "time 0.1.34 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -73,7 +74,7 @@ name = "clippy" version = "0.0.44" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "regex-syntax 0.2.4 (registry+https://github.com/rust-lang/crates.io-index)", + "regex-syntax 0.2.5 (registry+https://github.com/rust-lang/crates.io-index)", "semver 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)", "unicode-normalization 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -129,7 +130,7 @@ name = "docopt" version = "0.6.78" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "regex 0.1.53 (registry+https://github.com/rust-lang/crates.io-index)", + "regex 0.1.54 (registry+https://github.com/rust-lang/crates.io-index)", "rustc-serialize 0.3.18 (registry+https://github.com/rust-lang/crates.io-index)", "strsim 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -145,21 +146,21 @@ version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "log 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)", - "regex 0.1.53 (registry+https://github.com/rust-lang/crates.io-index)", + "regex 0.1.54 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] name = "eth-secp256k1" version = "0.5.4" -source = "git+https://github.com/arkpar/rust-secp256k1.git#45503e1de68d909b1862e3f2bdb9e1cdfdff3f1e" +source = "git+https://github.com/ethcore/rust-secp256k1#283a0677d8327536be58a87e0494d7e0e7b1d1d8" dependencies = [ "arrayvec 0.3.15 (registry+https://github.com/rust-lang/crates.io-index)", - "gcc 0.3.24 (registry+https://github.com/rust-lang/crates.io-index)", + "gcc 0.3.25 (registry+https://github.com/rust-lang/crates.io-index)", "libc 0.1.12 (registry+https://github.com/rust-lang/crates.io-index)", "rand 0.3.14 (registry+https://github.com/rust-lang/crates.io-index)", "rustc-serialize 0.3.18 (registry+https://github.com/rust-lang/crates.io-index)", - "serde 0.6.15 (registry+https://github.com/rust-lang/crates.io-index)", - "serde_json 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)", + "serde 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)", + "serde_json 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -185,7 +186,6 @@ dependencies = [ "lazy_static 0.1.15 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)", "num_cpus 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)", - "rocksdb 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)", "rust-crypto 0.2.34 (registry+https://github.com/rust-lang/crates.io-index)", "rustc-serialize 0.3.18 (registry+https://github.com/rust-lang/crates.io-index)", "time 0.1.34 (registry+https://github.com/rust-lang/crates.io-index)", @@ -206,12 +206,12 @@ dependencies = [ "ethcore 0.9.99", "ethcore-util 0.9.99", "ethsync 0.9.99", - "jsonrpc-core 1.1.4 (registry+https://github.com/rust-lang/crates.io-index)", - "jsonrpc-http-server 2.0.0 (registry+https://github.com/rust-lang/crates.io-index)", + "jsonrpc-core 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)", + "jsonrpc-http-server 2.1.0 (registry+https://github.com/rust-lang/crates.io-index)", "rustc-serialize 0.3.18 (registry+https://github.com/rust-lang/crates.io-index)", - "serde 0.6.15 (registry+https://github.com/rust-lang/crates.io-index)", - "serde_codegen 0.6.14 (registry+https://github.com/rust-lang/crates.io-index)", - "serde_json 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)", + "serde 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)", + "serde_codegen 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)", + "serde_json 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)", "syntex 0.29.0 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -224,22 +224,22 @@ dependencies = [ "crossbeam 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)", "elastic-array 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", "env_logger 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)", - "eth-secp256k1 0.5.4 (git+https://github.com/arkpar/rust-secp256k1.git)", + "eth-secp256k1 0.5.4 (git+https://github.com/ethcore/rust-secp256k1)", "ethcore-devtools 0.9.99", "heapsize 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)", "igd 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", - "itertools 0.4.9 (registry+https://github.com/rust-lang/crates.io-index)", + "itertools 0.4.10 (registry+https://github.com/rust-lang/crates.io-index)", "json-tests 0.1.0", "lazy_static 0.1.15 (registry+https://github.com/rust-lang/crates.io-index)", "libc 0.2.7 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)", "mio 0.5.0 (registry+https://github.com/rust-lang/crates.io-index)", "rand 0.3.14 (registry+https://github.com/rust-lang/crates.io-index)", - "rocksdb 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)", + "rocksdb 0.4.1 (git+https://github.com/arkpar/rust-rocksdb.git)", "rust-crypto 0.2.34 (registry+https://github.com/rust-lang/crates.io-index)", "rustc-serialize 0.3.18 (registry+https://github.com/rust-lang/crates.io-index)", "rustc_version 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)", - "serde 0.6.15 (registry+https://github.com/rust-lang/crates.io-index)", + "serde 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)", "sha3 0.1.0", "slab 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)", "target_info 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", @@ -256,6 +256,7 @@ dependencies = [ "env_logger 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)", "ethcore 0.9.99", "ethcore-util 0.9.99", + "heapsize 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)", "rand 0.3.14 (registry+https://github.com/rust-lang/crates.io-index)", "time 0.1.34 (registry+https://github.com/rust-lang/crates.io-index)", @@ -270,7 +271,7 @@ dependencies = [ [[package]] name = "gcc" -version = "0.3.24" +version = "0.3.25" source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] @@ -289,7 +290,7 @@ version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "kernel32-sys 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", - "regex 0.1.53 (registry+https://github.com/rust-lang/crates.io-index)", + "regex 0.1.54 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -321,7 +322,7 @@ dependencies = [ "time 0.1.34 (registry+https://github.com/rust-lang/crates.io-index)", "traitobject 0.0.1 (registry+https://github.com/rust-lang/crates.io-index)", "typeable 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", - "unicase 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)", + "unicase 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)", "url 0.2.38 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -341,7 +342,7 @@ dependencies = [ "time 0.1.34 (registry+https://github.com/rust-lang/crates.io-index)", "traitobject 0.0.1 (registry+https://github.com/rust-lang/crates.io-index)", "typeable 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", - "unicase 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)", + "unicase 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)", "url 0.5.5 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -352,14 +353,14 @@ source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "hyper 0.6.16 (registry+https://github.com/rust-lang/crates.io-index)", "rand 0.3.14 (registry+https://github.com/rust-lang/crates.io-index)", - "regex 0.1.53 (registry+https://github.com/rust-lang/crates.io-index)", + "regex 0.1.54 (registry+https://github.com/rust-lang/crates.io-index)", "xml-rs 0.1.26 (registry+https://github.com/rust-lang/crates.io-index)", "xmltree 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] name = "itertools" -version = "0.4.9" +version = "0.4.10" source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] @@ -372,23 +373,23 @@ dependencies = [ [[package]] name = "jsonrpc-core" -version = "1.1.4" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "serde 0.6.15 (registry+https://github.com/rust-lang/crates.io-index)", - "serde_codegen 0.6.14 (registry+https://github.com/rust-lang/crates.io-index)", - "serde_json 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)", + "serde 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)", + "serde_codegen 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)", + "serde_json 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)", "syntex 0.29.0 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] name = "jsonrpc-http-server" -version = "2.0.0" +version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "hyper 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)", - "jsonrpc-core 1.1.4 (registry+https://github.com/rust-lang/crates.io-index)", - "unicase 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)", + "jsonrpc-core 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)", + "unicase 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -425,6 +426,15 @@ name = "libc" version = "0.2.7" source = "registry+https://github.com/rust-lang/crates.io-index" +[[package]] +name = "librocksdb-sys" +version = "0.2.1" +source = "git+https://github.com/arkpar/rust-rocksdb.git#2156621f583bda95c1c07e89e79e4019f75158ee" +dependencies = [ + "libc 0.2.7 (registry+https://github.com/rust-lang/crates.io-index)", + "pkg-config 0.3.7 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "log" version = "0.3.5" @@ -464,7 +474,7 @@ dependencies = [ "libc 0.1.12 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)", "miow 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", - "net2 0.2.21 (registry+https://github.com/rust-lang/crates.io-index)", + "net2 0.2.22 (registry+https://github.com/rust-lang/crates.io-index)", "nix 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", "slab 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)", "time 0.1.34 (registry+https://github.com/rust-lang/crates.io-index)", @@ -477,14 +487,14 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "kernel32-sys 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", - "net2 0.2.21 (registry+https://github.com/rust-lang/crates.io-index)", + "net2 0.2.22 (registry+https://github.com/rust-lang/crates.io-index)", "winapi 0.2.5 (registry+https://github.com/rust-lang/crates.io-index)", "ws2_32-sys 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] name = "net2" -version = "0.2.21" +version = "0.2.22" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "cfg-if 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", @@ -513,7 +523,7 @@ dependencies = [ [[package]] name = "nom" -version = "1.2.0" +version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] @@ -533,11 +543,24 @@ dependencies = [ "libc 0.2.7 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "number_prefix" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "num 0.1.31 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "odds" version = "0.2.12" source = "registry+https://github.com/rust-lang/crates.io-index" +[[package]] +name = "pkg-config" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" + [[package]] name = "primal" version = "0.2.3" @@ -607,26 +630,27 @@ dependencies = [ [[package]] name = "regex" -version = "0.1.53" +version = "0.1.54" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "aho-corasick 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)", "memchr 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)", - "regex-syntax 0.2.4 (registry+https://github.com/rust-lang/crates.io-index)", + "regex-syntax 0.2.5 (registry+https://github.com/rust-lang/crates.io-index)", "utf8-ranges 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] name = "regex-syntax" -version = "0.2.4" +version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "rocksdb" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" +version = "0.4.1" +source = "git+https://github.com/arkpar/rust-rocksdb.git#2156621f583bda95c1c07e89e79e4019f75158ee" dependencies = [ - "libc 0.1.12 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.7 (registry+https://github.com/rust-lang/crates.io-index)", + "librocksdb-sys 0.2.1 (git+https://github.com/arkpar/rust-rocksdb.git)", ] [[package]] @@ -634,7 +658,7 @@ name = "rust-crypto" version = "0.2.34" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "gcc 0.3.24 (registry+https://github.com/rust-lang/crates.io-index)", + "gcc 0.3.25 (registry+https://github.com/rust-lang/crates.io-index)", "libc 0.1.12 (registry+https://github.com/rust-lang/crates.io-index)", "rand 0.3.14 (registry+https://github.com/rust-lang/crates.io-index)", "rustc-serialize 0.3.18 (registry+https://github.com/rust-lang/crates.io-index)", @@ -664,7 +688,7 @@ name = "semver" version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "nom 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)", + "nom 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -675,9 +699,14 @@ dependencies = [ "num 0.1.31 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "serde" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + [[package]] name = "serde_codegen" -version = "0.6.14" +version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "aster 0.13.1 (registry+https://github.com/rust-lang/crates.io-index)", @@ -689,18 +718,18 @@ dependencies = [ [[package]] name = "serde_json" -version = "0.6.0" +version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "num 0.1.31 (registry+https://github.com/rust-lang/crates.io-index)", - "serde 0.6.15 (registry+https://github.com/rust-lang/crates.io-index)", + "serde 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] name = "sha3" version = "0.1.0" dependencies = [ - "gcc 0.3.24 (registry+https://github.com/rust-lang/crates.io-index)", + "gcc 0.3.25 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -784,7 +813,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "unicase" -version = "1.2.1" +version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "rustc_version 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)", diff --git a/Cargo.toml b/Cargo.toml index 2590008b1..25b7caa85 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -20,6 +20,7 @@ ethcore-rpc = { path = "rpc", optional = true } fdlimit = { path = "util/fdlimit" } daemonize = "0.2" ethcore-devtools = { path = "devtools" } +number_prefix = "0.2" [features] default = ["rpc"] diff --git a/README.md b/README.md index 4df7cad34..f8b24f088 100644 --- a/README.md +++ b/README.md @@ -15,77 +15,28 @@ ### Building from source -##### Ubuntu 14.04, 15.04, 15.10 +First (if you don't already have it) get multirust: +- Linux: ```bash -# install rocksdb -add-apt-repository ppa:ethcore/ethcore -apt-get update -apt-get install -y --force-yes librocksdb-dev - -# install multirust -curl -sf https://raw.githubusercontent.com/brson/multirust/master/blastoff.sh | sh -s -- --yes - -# install beta -multirust update beta - -# download and build parity -git clone https://github.com/ethcore/parity -cd parity - -# parity should be build with rust beta -multirust override beta - -# build in release -cargo build --release -``` - -##### Other Linux - -```bash -# install rocksdb -git clone --tag v4.1 --depth=1 https://github.com/facebook/rocksdb.git -cd rocksdb -make shared_lib -sudo cp -a librocksdb.so* /usr/lib -sudo ldconfig -cd .. - -# install rust beta curl -sf https://raw.githubusercontent.com/brson/multirust/master/blastoff.sh | sudo sh -s -- --yes - -# install beta -multirust update beta - -# download and build parity -git clone https://github.com/ethcore/parity -cd parity - -# parity should be build with rust beta -multirust override beta - -# build in release -cargo build --release ``` -##### OSX with Homebrew +- OSX with Homebrew: +```bash +brew update && brew install multirust +``` + +Then, download and build Parity: ```bash -# install rocksdb && multirust -brew update -brew install rocksdb -brew install multirust - -# install beta -multirust update beta - -# download and build parity +# download Parity code git clone https://github.com/ethcore/parity cd parity -# use rust beta for building parity +# parity should be built with rust beta multirust override beta +# build in release mode cargo build --release ``` - diff --git a/ethcore/Cargo.toml b/ethcore/Cargo.toml index 660fdb1b2..c3a3d32dc 100644 --- a/ethcore/Cargo.toml +++ b/ethcore/Cargo.toml @@ -10,7 +10,6 @@ authors = ["Ethcore "] log = "0.3" env_logger = "0.3" rustc-serialize = "0.3" -rocksdb = "0.3" heapsize = "0.3" rust-crypto = "0.2.34" time = "0.1" diff --git a/ethcore/res/ethereum/morden.json b/ethcore/res/ethereum/morden.json index 2e45c2edd..e9f8e0e99 100644 --- a/ethcore/res/ethereum/morden.json +++ b/ethcore/res/ethereum/morden.json @@ -3,7 +3,7 @@ "engineName": "Ethash", "params": { "accountStartNonce": "0x0100000", - "frontierCompatibilityModeLimit": "0x10c8e0", + "frontierCompatibilityModeLimit": "0x789b0", "maximumExtraDataSize": "0x20", "tieBreakingGas": false, "minGasLimit": "0x1388", diff --git a/ethcore/src/block.rs b/ethcore/src/block.rs index a1194c665..70c6c2fb2 100644 --- a/ethcore/src/block.rs +++ b/ethcore/src/block.rs @@ -144,20 +144,20 @@ impl IsBlock for ExecutedBlock { /// Block that is ready for transactions to be added. /// -/// It's a bit like a Vec, eccept that whenever a transaction is pushed, we execute it and +/// It's a bit like a Vec, except that whenever a transaction is pushed, we execute it and /// maintain the system `state()`. We also archive execution receipts in preparation for later block creation. -pub struct OpenBlock<'x, 'y> { +pub struct OpenBlock<'x> { block: ExecutedBlock, engine: &'x Engine, - last_hashes: &'y LastHashes, + last_hashes: LastHashes, } /// Just like OpenBlock, except that we've applied `Engine::on_close_block`, finished up the non-seal header fields, /// and collected the uncles. /// /// There is no function available to push a transaction. If you want that you'll need to `reopen()` it. -pub struct ClosedBlock<'x, 'y> { - open_block: OpenBlock<'x, 'y>, +pub struct ClosedBlock<'x> { + open_block: OpenBlock<'x>, uncle_bytes: Bytes, } @@ -169,9 +169,9 @@ pub struct SealedBlock { uncle_bytes: Bytes, } -impl<'x, 'y> OpenBlock<'x, 'y> { +impl<'x> OpenBlock<'x> { /// Create a new OpenBlock ready for transaction pushing. - pub fn new(engine: &'x Engine, db: JournalDB, parent: &Header, last_hashes: &'y LastHashes, author: Address, extra_data: Bytes) -> Self { + pub fn new(engine: &'x Engine, db: JournalDB, parent: &Header, last_hashes: LastHashes, author: Address, extra_data: Bytes) -> Self { let mut r = OpenBlock { block: ExecutedBlock::new(State::from_existing(db, parent.state_root().clone(), engine.account_start_nonce())), engine: engine, @@ -259,7 +259,7 @@ impl<'x, 'y> OpenBlock<'x, 'y> { } /// Turn this into a `ClosedBlock`. A BlockChain must be provided in order to figure out the uncles. - pub fn close(self) -> ClosedBlock<'x, 'y> { + pub fn close(self) -> ClosedBlock<'x> { let mut s = self; s.engine.on_close_block(&mut s.block); s.block.base.header.transactions_root = ordered_trie_root(s.block.base.transactions.iter().map(|ref e| e.rlp_bytes().to_vec()).collect()); @@ -275,16 +275,16 @@ impl<'x, 'y> OpenBlock<'x, 'y> { } } -impl<'x, 'y> IsBlock for OpenBlock<'x, 'y> { +impl<'x> IsBlock for OpenBlock<'x> { fn block(&self) -> &ExecutedBlock { &self.block } } -impl<'x, 'y> IsBlock for ClosedBlock<'x, 'y> { +impl<'x> IsBlock for ClosedBlock<'x> { fn block(&self) -> &ExecutedBlock { &self.open_block.block } } -impl<'x, 'y> ClosedBlock<'x, 'y> { - fn new(open_block: OpenBlock<'x, 'y>, uncle_bytes: Bytes) -> Self { +impl<'x> ClosedBlock<'x> { + fn new(open_block: OpenBlock<'x>, uncle_bytes: Bytes) -> Self { ClosedBlock { open_block: open_block, uncle_bytes: uncle_bytes, @@ -307,7 +307,7 @@ impl<'x, 'y> ClosedBlock<'x, 'y> { } /// Turn this back into an `OpenBlock`. - pub fn reopen(self) -> OpenBlock<'x, 'y> { self.open_block } + pub fn reopen(self) -> OpenBlock<'x> { self.open_block } /// Drop this object and return the underlieing database. pub fn drain(self) -> JournalDB { self.open_block.block.state.drop().1 } @@ -332,7 +332,7 @@ impl IsBlock for SealedBlock { } /// Enact the block given by block header, transactions and uncles -pub fn enact<'x, 'y>(header: &Header, transactions: &[SignedTransaction], uncles: &[Header], engine: &'x Engine, db: JournalDB, parent: &Header, last_hashes: &'y LastHashes) -> Result, Error> { +pub fn enact<'x>(header: &Header, transactions: &[SignedTransaction], uncles: &[Header], engine: &'x Engine, db: JournalDB, parent: &Header, last_hashes: LastHashes) -> Result, Error> { { if ::log::max_log_level() >= ::log::LogLevel::Trace { let s = State::from_existing(db.clone(), parent.state_root().clone(), engine.account_start_nonce()); @@ -350,20 +350,20 @@ pub fn enact<'x, 'y>(header: &Header, transactions: &[SignedTransaction], uncles } /// Enact the block given by `block_bytes` using `engine` on the database `db` with given `parent` block header -pub fn enact_bytes<'x, 'y>(block_bytes: &[u8], engine: &'x Engine, db: JournalDB, parent: &Header, last_hashes: &'y LastHashes) -> Result, Error> { +pub fn enact_bytes<'x>(block_bytes: &[u8], engine: &'x Engine, db: JournalDB, parent: &Header, last_hashes: LastHashes) -> Result, Error> { let block = BlockView::new(block_bytes); let header = block.header(); enact(&header, &block.transactions(), &block.uncles(), engine, db, parent, last_hashes) } /// Enact the block given by `block_bytes` using `engine` on the database `db` with given `parent` block header -pub fn enact_verified<'x, 'y>(block: &PreVerifiedBlock, engine: &'x Engine, db: JournalDB, parent: &Header, last_hashes: &'y LastHashes) -> Result, Error> { +pub fn enact_verified<'x>(block: &PreVerifiedBlock, engine: &'x Engine, db: JournalDB, parent: &Header, last_hashes: LastHashes) -> Result, Error> { let view = BlockView::new(&block.bytes); enact(&block.header, &block.transactions, &view.uncles(), engine, db, parent, last_hashes) } /// Enact the block given by `block_bytes` using `engine` on the database `db` with given `parent` block header. Seal the block aferwards -pub fn enact_and_seal(block_bytes: &[u8], engine: &Engine, db: JournalDB, parent: &Header, last_hashes: &LastHashes) -> Result { +pub fn enact_and_seal(block_bytes: &[u8], engine: &Engine, db: JournalDB, parent: &Header, last_hashes: LastHashes) -> Result { let header = BlockView::new(block_bytes).header_view(); Ok(try!(try!(enact_bytes(block_bytes, engine, db, parent, last_hashes)).seal(header.seal()))) } @@ -384,7 +384,7 @@ mod tests { let mut db = db_result.take(); engine.spec().ensure_db_good(&mut db); let last_hashes = vec![genesis_header.hash()]; - let b = OpenBlock::new(engine.deref(), db, &genesis_header, &last_hashes, Address::zero(), vec![]); + let b = OpenBlock::new(engine.deref(), db, &genesis_header, last_hashes, Address::zero(), vec![]); let b = b.close(); let _ = b.seal(vec![]); } @@ -398,14 +398,14 @@ mod tests { let mut db_result = get_temp_journal_db(); let mut db = db_result.take(); engine.spec().ensure_db_good(&mut db); - let b = OpenBlock::new(engine.deref(), db, &genesis_header, &vec![genesis_header.hash()], Address::zero(), vec![]).close().seal(vec![]).unwrap(); + let b = OpenBlock::new(engine.deref(), db, &genesis_header, vec![genesis_header.hash()], Address::zero(), vec![]).close().seal(vec![]).unwrap(); let orig_bytes = b.rlp_bytes(); let orig_db = b.drain(); let mut db_result = get_temp_journal_db(); let mut db = db_result.take(); engine.spec().ensure_db_good(&mut db); - let e = enact_and_seal(&orig_bytes, engine.deref(), db, &genesis_header, &vec![genesis_header.hash()]).unwrap(); + let e = enact_and_seal(&orig_bytes, engine.deref(), db, &genesis_header, vec![genesis_header.hash()]).unwrap(); assert_eq!(e.rlp_bytes(), orig_bytes); diff --git a/ethcore/src/block_queue.rs b/ethcore/src/block_queue.rs index ba9867966..f61eb565d 100644 --- a/ethcore/src/block_queue.rs +++ b/ethcore/src/block_queue.rs @@ -28,6 +28,31 @@ use service::*; use client::BlockStatus; use util::panics::*; +known_heap_size!(0, UnVerifiedBlock, VerifyingBlock, PreVerifiedBlock); + +const MIN_MEM_LIMIT: usize = 16384; +const MIN_QUEUE_LIMIT: usize = 512; + +/// Block queue configuration +#[derive(Debug)] +pub struct BlockQueueConfig { + /// Maximum number of blocks to keep in unverified queue. + /// When the limit is reached, is_full returns true. + pub max_queue_size: usize, + /// Maximum heap memory to use. + /// When the limit is reached, is_full returns true. + pub max_mem_use: usize, +} + +impl Default for BlockQueueConfig { + fn default() -> Self { + BlockQueueConfig { + max_queue_size: 30000, + max_mem_use: 50 * 1024 * 1024, + } + } +} + /// Block queue status #[derive(Debug)] pub struct BlockQueueInfo { @@ -37,6 +62,12 @@ pub struct BlockQueueInfo { pub verified_queue_size: usize, /// Number of blocks being verified pub verifying_queue_size: usize, + /// Configured maximum number of blocks in the queue + pub max_queue_size: usize, + /// Configured maximum number of bytes to use + pub max_mem_use: usize, + /// Heap memory used in bytes + pub mem_used: usize, } impl BlockQueueInfo { @@ -48,7 +79,8 @@ impl BlockQueueInfo { /// Indicates that queue is full pub fn is_full(&self) -> bool { - self.unverified_queue_size + self.verified_queue_size + self.verifying_queue_size > MAX_UNVERIFIED_QUEUE_SIZE + self.unverified_queue_size + self.verified_queue_size + self.verifying_queue_size > self.max_queue_size || + self.mem_used > self.max_mem_use } /// Indicates that queue is empty @@ -68,7 +100,9 @@ pub struct BlockQueue { deleting: Arc, ready_signal: Arc, empty: Arc, - processing: RwLock> + processing: RwLock>, + max_queue_size: usize, + max_mem_use: usize, } struct UnVerifiedBlock { @@ -106,11 +140,9 @@ struct Verification { bad: Mutex>, } -const MAX_UNVERIFIED_QUEUE_SIZE: usize = 50000; - impl BlockQueue { /// Creates a new queue instance. - pub fn new(engine: Arc>, message_channel: IoChannel) -> BlockQueue { + pub fn new(config: BlockQueueConfig, engine: Arc>, message_channel: IoChannel) -> BlockQueue { let verification = Arc::new(Verification { unverified: Mutex::new(VecDeque::new()), verified: Mutex::new(VecDeque::new()), @@ -154,6 +186,8 @@ impl BlockQueue { deleting: deleting.clone(), processing: RwLock::new(HashSet::new()), empty: empty.clone(), + max_queue_size: max(config.max_queue_size, MIN_QUEUE_LIMIT), + max_mem_use: max(config.max_mem_use, MIN_MEM_LIMIT), } } @@ -297,19 +331,23 @@ impl BlockQueue { } /// Mark given block and all its children as bad. Stops verification. - pub fn mark_as_bad(&self, hash: &H256) { + pub fn mark_as_bad(&self, block_hashes: &[H256]) { let mut verified_lock = self.verification.verified.lock().unwrap(); let mut verified = verified_lock.deref_mut(); let mut bad = self.verification.bad.lock().unwrap(); - bad.insert(hash.clone()); - self.processing.write().unwrap().remove(&hash); + let mut processing = self.processing.write().unwrap(); + bad.reserve(block_hashes.len()); + for hash in block_hashes { + bad.insert(hash.clone()); + processing.remove(&hash); + } + let mut new_verified = VecDeque::new(); for block in verified.drain(..) { if bad.contains(&block.header.parent_hash) { bad.insert(block.header.hash()); - self.processing.write().unwrap().remove(&block.header.hash()); - } - else { + processing.remove(&block.header.hash()); + } else { new_verified.push_back(block); } } @@ -317,10 +355,10 @@ impl BlockQueue { } /// Mark given block as processed - pub fn mark_as_good(&self, hashes: &[H256]) { + pub fn mark_as_good(&self, block_hashes: &[H256]) { let mut processing = self.processing.write().unwrap(); - for h in hashes { - processing.remove(&h); + for hash in block_hashes { + processing.remove(&hash); } } @@ -342,12 +380,41 @@ impl BlockQueue { /// Get queue status. pub fn queue_info(&self) -> BlockQueueInfo { + let (unverified_len, unverified_bytes) = { + let v = self.verification.unverified.lock().unwrap(); + (v.len(), v.heap_size_of_children()) + }; + let (verifying_len, verifying_bytes) = { + let v = self.verification.verifying.lock().unwrap(); + (v.len(), v.heap_size_of_children()) + }; + let (verified_len, verified_bytes) = { + let v = self.verification.verified.lock().unwrap(); + (v.len(), v.heap_size_of_children()) + }; BlockQueueInfo { - unverified_queue_size: self.verification.unverified.lock().unwrap().len(), - verifying_queue_size: self.verification.verifying.lock().unwrap().len(), - verified_queue_size: self.verification.verified.lock().unwrap().len(), + unverified_queue_size: unverified_len, + verifying_queue_size: verifying_len, + verified_queue_size: verified_len, + max_queue_size: self.max_queue_size, + max_mem_use: self.max_mem_use, + mem_used: + unverified_bytes + + verifying_bytes + + verified_bytes + // TODO: https://github.com/servo/heapsize/pull/50 + //+ self.processing.read().unwrap().heap_size_of_children(), } } + + pub fn collect_garbage(&self) { + { + self.verification.unverified.lock().unwrap().shrink_to_fit(); + self.verification.verifying.lock().unwrap().shrink_to_fit(); + self.verification.verified.lock().unwrap().shrink_to_fit(); + } + self.processing.write().unwrap().shrink_to_fit(); + } } impl MayPanic for BlockQueue { @@ -379,7 +446,7 @@ mod tests { fn get_test_queue() -> BlockQueue { let spec = get_test_spec(); let engine = spec.to_engine().unwrap(); - BlockQueue::new(Arc::new(engine), IoChannel::disconnected()) + BlockQueue::new(BlockQueueConfig::default(), Arc::new(engine), IoChannel::disconnected()) } #[test] @@ -387,7 +454,7 @@ mod tests { // TODO better test let spec = Spec::new_test(); let engine = spec.to_engine().unwrap(); - let _ = BlockQueue::new(Arc::new(engine), IoChannel::disconnected()); + let _ = BlockQueue::new(BlockQueueConfig::default(), Arc::new(engine), IoChannel::disconnected()); } #[test] @@ -443,4 +510,19 @@ mod tests { assert!(queue.queue_info().is_empty()); } + + #[test] + fn test_mem_limit() { + let spec = get_test_spec(); + let engine = spec.to_engine().unwrap(); + let mut config = BlockQueueConfig::default(); + config.max_mem_use = super::MIN_MEM_LIMIT; // empty queue uses about 15000 + let mut queue = BlockQueue::new(config, Arc::new(engine), IoChannel::disconnected()); + assert!(!queue.queue_info().is_full()); + let mut blocks = get_good_dummy_block_seq(50); + for b in blocks.drain(..) { + queue.import_block(b).unwrap(); + } + assert!(queue.queue_info().is_full()); + } } diff --git a/ethcore/src/blockchain/best_block.rs b/ethcore/src/blockchain/best_block.rs new file mode 100644 index 000000000..cbb219617 --- /dev/null +++ b/ethcore/src/blockchain/best_block.rs @@ -0,0 +1,30 @@ +// Copyright 2015, 2016 Ethcore (UK) Ltd. +// This file is part of Parity. + +// Parity is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Parity is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Parity. If not, see . + +use util::hash::H256; +use util::uint::U256; +use header::BlockNumber; + +/// Best block info. +#[derive(Default)] +pub struct BestBlock { + /// Best block hash. + pub hash: H256, + /// Best block number. + pub number: BlockNumber, + /// Best block total difficulty. + pub total_difficulty: U256 +} diff --git a/ethcore/src/blockchain/block_info.rs b/ethcore/src/blockchain/block_info.rs new file mode 100644 index 000000000..98dac648d --- /dev/null +++ b/ethcore/src/blockchain/block_info.rs @@ -0,0 +1,48 @@ +// Copyright 2015, 2016 Ethcore (UK) Ltd. +// This file is part of Parity. + +// Parity is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Parity is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Parity. If not, see . + +use util::hash::H256; +use util::uint::U256; +use header::BlockNumber; + +/// Brief info about inserted block. +pub struct BlockInfo { + /// Block hash. + pub hash: H256, + /// Block number. + pub number: BlockNumber, + /// Total block difficulty. + pub total_difficulty: U256, + /// Block location in blockchain. + pub location: BlockLocation +} + +/// Describes location of newly inserted block. +pub enum BlockLocation { + /// It's part of the canon chain. + CanonChain, + /// It's not a part of the canon chain. + Branch, + /// It's part of the fork which should become canon chain, + /// because it's total difficulty is higher than current + /// canon chain difficulty. + BranchBecomingCanonChain { + /// Hash of the newest common ancestor with old canon chain. + ancestor: H256, + /// Hashes of the blocks between ancestor and this block. + route: Vec + } +} diff --git a/ethcore/src/blockchain.rs b/ethcore/src/blockchain/blockchain.rs similarity index 83% rename from ethcore/src/blockchain.rs rename to ethcore/src/blockchain/blockchain.rs index 54d482d62..7339f3a1a 100644 --- a/ethcore/src/blockchain.rs +++ b/ethcore/src/blockchain/blockchain.rs @@ -18,116 +18,36 @@ use std::sync::atomic::{AtomicUsize, Ordering as AtomicOrder}; use util::*; -use rocksdb::{DB, WriteBatch, Writable}; use header::*; use extras::*; use transaction::*; use views::*; use receipt::Receipt; use chainfilter::{ChainFilter, BloomIndex, FilterDataSource}; +use blockchain::block_info::{BlockInfo, BlockLocation}; +use blockchain::best_block::BestBlock; +use blockchain::bloom_indexer::BloomIndexer; +use blockchain::tree_route::TreeRoute; +use blockchain::update::ExtrasUpdate; +use blockchain::CacheSize; const BLOOM_INDEX_SIZE: usize = 16; const BLOOM_LEVELS: u8 = 3; -/// Represents a tree route between `from` block and `to` block: -pub struct TreeRoute { - /// A vector of hashes of all blocks, ordered from `from` to `to`. - pub blocks: Vec, - /// Best common ancestor of these blocks. - pub ancestor: H256, - /// An index where best common ancestor would be. - pub index: usize, -} - -/// Represents blockchain's in-memory cache size in bytes. +/// Blockchain configuration. #[derive(Debug)] -pub struct CacheSize { - /// Blocks cache size. - pub blocks: usize, - /// BlockDetails cache size. - pub block_details: usize, - /// Transaction addresses cache size. - pub transaction_addresses: usize, - /// Logs cache size. - pub block_logs: usize, - /// Blooms cache size. - pub blocks_blooms: usize, - /// Block receipts size. - pub block_receipts: usize, +pub struct BlockChainConfig { + /// Preferred cache size in bytes. + pub pref_cache_size: usize, + /// Maximum cache size in bytes. + pub max_cache_size: usize, } -struct BloomIndexer { - index_size: usize, - levels: u8, -} - -impl BloomIndexer { - fn new(index_size: usize, levels: u8) -> Self { - BloomIndexer { - index_size: index_size, - levels: levels - } - } - - /// Calculates bloom's position in database. - fn location(&self, bloom_index: &BloomIndex) -> BlocksBloomLocation { - use std::{mem, ptr}; - - let hash = unsafe { - let mut hash: H256 = mem::zeroed(); - ptr::copy(&[bloom_index.index / self.index_size] as *const usize as *const u8, hash.as_mut_ptr(), 8); - hash[8] = bloom_index.level; - hash.reverse(); - hash - }; - - BlocksBloomLocation { - hash: hash, - index: bloom_index.index % self.index_size - } - } - - fn index_size(&self) -> usize { - self.index_size - } - - fn levels(&self) -> u8 { - self.levels - } -} - -/// Blockchain update info. -struct ExtrasUpdate { - /// Block hash. - hash: H256, - /// DB update batch. - batch: WriteBatch, - /// Inserted block familial details. - details: BlockDetails, - /// New best block (if it has changed). - new_best: Option, - /// Changed blocks bloom location hashes. - bloom_hashes: HashSet, -} - -impl CacheSize { - /// Total amount used by the cache. - fn total(&self) -> usize { self.blocks + self.block_details + self.transaction_addresses + self.block_logs + self.blocks_blooms } -} - -/// Information about best block gathered together -struct BestBlock { - pub hash: H256, - pub number: BlockNumber, - pub total_difficulty: U256 -} - -impl BestBlock { - fn new() -> BestBlock { - BestBlock { - hash: H256::new(), - number: 0, - total_difficulty: U256::from(0) +impl Default for BlockChainConfig { + fn default() -> Self { + BlockChainConfig { + pref_cache_size: 1 << 14, + max_cache_size: 1 << 20, } } } @@ -232,8 +152,8 @@ pub struct BlockChain { blocks_blooms: RwLock>, block_receipts: RwLock>, - extras_db: DB, - blocks_db: DB, + extras_db: Database, + blocks_db: Database, cache_man: RwLock, @@ -313,50 +233,24 @@ const COLLECTION_QUEUE_SIZE: usize = 8; impl BlockChain { /// Create new instance of blockchain from given Genesis - /// - /// ```rust - /// extern crate ethcore_util as util; - /// extern crate ethcore; - /// use std::env; - /// use std::str::FromStr; - /// use ethcore::spec::*; - /// use ethcore::blockchain::*; - /// use ethcore::ethereum; - /// use util::hash::*; - /// use util::uint::*; - /// - /// fn main() { - /// let spec = ethereum::new_frontier(); - /// - /// let mut dir = env::temp_dir(); - /// dir.push(H32::random().hex()); - /// - /// let bc = BlockChain::new(&spec.genesis_block(), &dir); - /// - /// let genesis_hash = "d4e56740f876aef8c010b86a40d5f56745a118d0906a34e69aec8c0db1cb8fa3"; - /// assert_eq!(bc.genesis_hash(), H256::from_str(genesis_hash).unwrap()); - /// assert!(bc.is_known(&bc.genesis_hash())); - /// assert_eq!(bc.genesis_hash(), bc.block_hash(0).unwrap()); - /// } - /// ``` - pub fn new(genesis: &[u8], path: &Path) -> BlockChain { + pub fn new(config: BlockChainConfig, genesis: &[u8], path: &Path) -> BlockChain { // open extras db let mut extras_path = path.to_path_buf(); extras_path.push("extras"); - let extras_db = DB::open_default(extras_path.to_str().unwrap()).unwrap(); + let extras_db = Database::open_default(extras_path.to_str().unwrap()).unwrap(); // open blocks db let mut blocks_path = path.to_path_buf(); blocks_path.push("blocks"); - let blocks_db = DB::open_default(blocks_path.to_str().unwrap()).unwrap(); + let blocks_db = Database::open_default(blocks_path.to_str().unwrap()).unwrap(); let mut cache_man = CacheManager{cache_usage: VecDeque::new(), in_use: HashSet::new()}; (0..COLLECTION_QUEUE_SIZE).foreach(|_| cache_man.cache_usage.push_back(HashSet::new())); let bc = BlockChain { - pref_cache_size: AtomicUsize::new(1 << 14), - max_cache_size: AtomicUsize::new(1 << 20), - best_block: RwLock::new(BestBlock::new()), + pref_cache_size: AtomicUsize::new(config.pref_cache_size), + max_cache_size: AtomicUsize::new(config.max_cache_size), + best_block: RwLock::new(BestBlock::default()), blocks: RwLock::new(HashMap::new()), block_details: RwLock::new(HashMap::new()), block_hashes: RwLock::new(HashMap::new()), @@ -390,7 +284,7 @@ impl BlockChain { bc.blocks_db.put(&hash, genesis).unwrap(); - let batch = WriteBatch::new(); + let batch = DBTransaction::new(); batch.put_extras(&hash, &details); batch.put_extras(&header.number(), &hash); batch.put(b"best", &hash).unwrap(); @@ -458,40 +352,26 @@ impl BlockChain { /// ```json /// { blocks: [B4, B3, A3, A4], ancestor: A2, index: 2 } /// ``` - pub fn tree_route(&self, from: H256, to: H256) -> Option { - let from_details = match self.block_details(&from) { - Some(h) => h, - None => return None, - }; - let to_details = match self.block_details(&to) { - Some(h) => h, - None => return None, - }; - Some(self.tree_route_aux((&from_details, &from), (&to_details, &to))) - } - - /// Similar to `tree_route` function, but can be used to return a route - /// between blocks which may not be in database yet. - fn tree_route_aux(&self, from: (&BlockDetails, &H256), to: (&BlockDetails, &H256)) -> TreeRoute { + pub fn tree_route(&self, from: H256, to: H256) -> TreeRoute { let mut from_branch = vec![]; let mut to_branch = vec![]; - let mut from_details = from.0.clone(); - let mut to_details = to.0.clone(); - let mut current_from = from.1.clone(); - let mut current_to = to.1.clone(); + let mut from_details = self.block_details(&from).expect(&format!("0. Expected to find details for block {:?}", from)); + let mut to_details = self.block_details(&to).expect(&format!("1. Expected to find details for block {:?}", to)); + let mut current_from = from; + let mut current_to = to; // reset from && to to the same level while from_details.number > to_details.number { from_branch.push(current_from); current_from = from_details.parent.clone(); - from_details = self.block_details(&from_details.parent).expect(&format!("1. Expected to find details for block {:?}", from_details.parent)); + from_details = self.block_details(&from_details.parent).expect(&format!("2. Expected to find details for block {:?}", from_details.parent)); } while to_details.number > from_details.number { to_branch.push(current_to); current_to = to_details.parent.clone(); - to_details = self.block_details(&to_details.parent).expect(&format!("2. Expected to find details for block {:?}", to_details.parent)); + to_details = self.block_details(&to_details.parent).expect(&format!("3. Expected to find details for block {:?}", to_details.parent)); } assert_eq!(from_details.number, to_details.number); @@ -500,11 +380,11 @@ impl BlockChain { while current_from != current_to { from_branch.push(current_from); current_from = from_details.parent.clone(); - from_details = self.block_details(&from_details.parent).expect(&format!("3. Expected to find details for block {:?}", from_details.parent)); + from_details = self.block_details(&from_details.parent).expect(&format!("4. Expected to find details for block {:?}", from_details.parent)); to_branch.push(current_to); current_to = to_details.parent.clone(); - to_details = self.block_details(&to_details.parent).expect(&format!("4. Expected to find details for block {:?}", from_details.parent)); + to_details = self.block_details(&to_details.parent).expect(&format!("5. Expected to find details for block {:?}", from_details.parent)); } let index = from_branch.len(); @@ -534,170 +414,239 @@ impl BlockChain { let _lock = self.insert_lock.lock(); // store block in db self.blocks_db.put(&hash, &bytes).unwrap(); - let update = self.block_to_extras_update(bytes, receipts); - self.apply_update(update); + + let info = self.block_info(bytes); + + self.apply_update(ExtrasUpdate { + block_hashes: self.prepare_block_hashes_update(bytes, &info), + block_details: self.prepare_block_details_update(bytes, &info), + block_receipts: self.prepare_block_receipts_update(receipts, &info), + transactions_addresses: self.prepare_transaction_addresses_update(bytes, &info), + blocks_blooms: self.prepare_block_blooms_update(bytes, &info), + info: info + }); } /// Applies extras update. fn apply_update(&self, update: ExtrasUpdate) { + let batch = DBTransaction::new(); + batch.put(b"best", &update.info.hash).unwrap(); + + // These cached values must be updated atomically + let mut best_block = self.best_block.write().unwrap(); + let mut write_hashes = self.block_hashes.write().unwrap(); + let mut write_txs = self.transaction_addresses.write().unwrap(); + // update best block - { - let mut best_block = self.best_block.write().unwrap(); - if let Some(b) = update.new_best { - *best_block = b; + match update.info.location { + BlockLocation::Branch => (), + _ => { + *best_block = BestBlock { + hash: update.info.hash, + number: update.info.number, + total_difficulty: update.info.total_difficulty + }; } } - { - // update details cache - let mut write_details = self.block_details.write().unwrap(); - write_details.remove(&update.details.parent); - write_details.insert(update.hash.clone(), update.details); - self.note_used(CacheID::Block(update.hash)); + for (number, hash) in &update.block_hashes { + batch.put_extras(number, hash); + write_hashes.remove(number); } - { - // update blocks blooms cache - let mut write_blocks_blooms = self.blocks_blooms.write().unwrap(); - for bloom_hash in &update.bloom_hashes { - write_blocks_blooms.remove(bloom_hash); - } + let mut write_details = self.block_details.write().unwrap(); + for (hash, details) in update.block_details.into_iter() { + batch.put_extras(&hash, &details); + write_details.insert(hash, details); } + + let mut write_receipts = self.block_receipts.write().unwrap(); + for (hash, receipt) in &update.block_receipts { + batch.put_extras(hash, receipt); + write_receipts.remove(hash); + } + + for (hash, tx_address) in &update.transactions_addresses { + batch.put_extras(hash, tx_address); + write_txs.remove(hash); + } + + let mut write_blocks_blooms = self.blocks_blooms.write().unwrap(); + for (bloom_hash, blocks_bloom) in &update.blocks_blooms { + batch.put_extras(bloom_hash, blocks_bloom); + write_blocks_blooms.remove(bloom_hash); + } + // update extras database - self.extras_db.write(update.batch).unwrap(); + self.extras_db.write(batch).unwrap(); } - /// Transforms block into WriteBatch that may be written into database - /// Additionally, if it's new best block it returns new best block object. - fn block_to_extras_update(&self, bytes: &[u8], receipts: Vec) -> ExtrasUpdate { - // create views onto rlp - let block = BlockView::new(bytes); + /// Get inserted block info which is critical to preapre extras updates. + fn block_info(&self, block_bytes: &[u8]) -> BlockInfo { + let block = BlockView::new(block_bytes); let header = block.header_view(); - - // prepare variables let hash = block.sha3(); - let mut parent_details = self.block_details(&header.parent_hash()).expect(format!("Invalid parent hash: {:?}", header.parent_hash()).as_ref()); + let number = header.number(); + let parent_hash = header.parent_hash(); + let parent_details = self.block_details(&parent_hash).expect(format!("Invalid parent hash: {:?}", parent_hash).as_ref()); let total_difficulty = parent_details.total_difficulty + header.difficulty(); let is_new_best = total_difficulty > self.best_block_total_difficulty(); + + BlockInfo { + hash: hash, + number: number, + total_difficulty: total_difficulty, + location: if is_new_best { + // on new best block we need to make sure that all ancestors + // are moved to "canon chain" + // find the route between old best block and the new one + let best_hash = self.best_block_hash(); + let route = self.tree_route(best_hash, parent_hash); + + assert_eq!(number, parent_details.number + 1); + + match route.blocks.len() { + 0 => BlockLocation::CanonChain, + _ => BlockLocation::BranchBecomingCanonChain { + ancestor: route.ancestor, + route: route.blocks.into_iter().skip(route.index).collect() + } + } + } else { + BlockLocation::Branch + } + } + } + + /// This function returns modified block hashes. + fn prepare_block_hashes_update(&self, block_bytes: &[u8], info: &BlockInfo) -> HashMap { + let mut block_hashes = HashMap::new(); + let block = BlockView::new(block_bytes); + let header = block.header_view(); + let number = header.number(); + + match info.location { + BlockLocation::Branch => (), + BlockLocation::CanonChain => { + block_hashes.insert(number, info.hash.clone()); + }, + BlockLocation::BranchBecomingCanonChain { ref ancestor, ref route } => { + let ancestor_number = self.block_number(ancestor).unwrap(); + let start_number = ancestor_number + 1; + + for (index, hash) in route.iter().cloned().enumerate() { + block_hashes.insert(start_number + index as BlockNumber, hash); + } + + block_hashes.insert(number, info.hash.clone()); + } + } + + block_hashes + } + + /// This function returns modified block details. + fn prepare_block_details_update(&self, block_bytes: &[u8], info: &BlockInfo) -> HashMap { + let block = BlockView::new(block_bytes); + let header = block.header_view(); let parent_hash = header.parent_hash(); + // update parent + let mut parent_details = self.block_details(&parent_hash).expect(format!("Invalid parent hash: {:?}", parent_hash).as_ref()); + parent_details.children.push(info.hash.clone()); + // create current block details let details = BlockDetails { number: header.number(), - total_difficulty: total_difficulty, + total_difficulty: info.total_difficulty, parent: parent_hash.clone(), children: vec![] }; - // prepare the batch - let batch = WriteBatch::new(); + // write to batch + let mut block_details = HashMap::new(); + block_details.insert(parent_hash, parent_details); + block_details.insert(info.hash.clone(), details); + block_details + } - // insert new block details - batch.put_extras(&hash, &details); + /// This function returns modified block receipts. + fn prepare_block_receipts_update(&self, receipts: Vec, info: &BlockInfo) -> HashMap { + let mut block_receipts = HashMap::new(); + block_receipts.insert(info.hash.clone(), BlockReceipts::new(receipts)); + block_receipts + } - // update parent details - parent_details.children.push(hash.clone()); - batch.put_extras(&parent_hash, &parent_details); + /// This function returns modified transaction addresses. + fn prepare_transaction_addresses_update(&self, block_bytes: &[u8], info: &BlockInfo) -> HashMap { + let block = BlockView::new(block_bytes); + let transaction_hashes = block.transaction_hashes(); - // update transaction addresses - for (i, tx_hash) in block.transaction_hashes().iter().enumerate() { - batch.put_extras(tx_hash, &TransactionAddress { - block_hash: hash.clone(), - index: i - }); - } + transaction_hashes.into_iter() + .enumerate() + .fold(HashMap::new(), |mut acc, (i ,tx_hash)| { + acc.insert(tx_hash, TransactionAddress { + block_hash: info.hash.clone(), + index: i + }); + acc + }) + } - // update block receipts - batch.put_extras(&hash, &BlockReceipts::new(receipts)); + /// This functions returns modified blocks blooms. + /// + /// To accelerate blooms lookups, blomms are stored in multiple + /// layers (BLOOM_LEVELS, currently 3). + /// ChainFilter is responsible for building and rebuilding these layers. + /// It returns them in HashMap, where values are Blooms and + /// keys are BloomIndexes. BloomIndex represents bloom location on one + /// of these layers. + /// + /// To reduce number of queries to databse, block blooms are stored + /// in BlocksBlooms structure which contains info about several + /// (BLOOM_INDEX_SIZE, currently 16) consecutive blocks blooms. + /// + /// Later, BloomIndexer is used to map bloom location on filter layer (BloomIndex) + /// to bloom location in database (BlocksBloomLocation). + /// + fn prepare_block_blooms_update(&self, block_bytes: &[u8], info: &BlockInfo) -> HashMap { + let block = BlockView::new(block_bytes); + let header = block.header_view(); - // if it's not new best block, just return - if !is_new_best { - return ExtrasUpdate { - hash: hash.clone(), - batch: batch, - details: details, - new_best: None, - bloom_hashes: HashSet::new() - }; - } - - // if its new best block we need to make sure that all ancestors - // are moved to "canon chain" - // find the route between old best block and the new one - let best_hash = self.best_block_hash(); - let best_details = self.block_details(&best_hash).expect("best block hash is invalid!"); - let route = self.tree_route_aux((&best_details, &best_hash), (&details, &hash)); - - let modified_blooms; - - match route.blocks.len() { - // its our parent - 1 => { - batch.put_extras(&header.number(), &hash); - - // update block blooms - modified_blooms = ChainFilter::new(self, self.bloom_indexer.index_size(), self.bloom_indexer.levels()) - .add_bloom(&header.log_bloom(), header.number() as usize); + let modified_blooms = match info.location { + BlockLocation::Branch => HashMap::new(), + BlockLocation::CanonChain => { + ChainFilter::new(self, self.bloom_indexer.index_size(), self.bloom_indexer.levels()) + .add_bloom(&header.log_bloom(), header.number() as usize) }, - // it is a fork - i if i > 1 => { - let ancestor_number = self.block_number(&route.ancestor).unwrap(); + BlockLocation::BranchBecomingCanonChain { ref ancestor, ref route } => { + let ancestor_number = self.block_number(ancestor).unwrap(); let start_number = ancestor_number + 1; - for (index, hash) in route.blocks.iter().skip(route.index).enumerate() { - batch.put_extras(&(start_number + index as BlockNumber), hash); - } - // get all blocks that are not part of canon chain (TODO: optimize it to one query) - let blooms: Vec = route.blocks.iter() - .skip(route.index) + let mut blooms: Vec = route.iter() .map(|hash| self.block(hash).unwrap()) .map(|bytes| BlockView::new(&bytes).header_view().log_bloom()) .collect(); - // reset blooms chain head - modified_blooms = ChainFilter::new(self, self.bloom_indexer.index_size(), self.bloom_indexer.levels()) - .reset_chain_head(&blooms, start_number as usize, self.best_block_number() as usize); - }, - // route.blocks.len() could be 0 only if inserted block is best block, - // and this is not possible at this stage - _ => { unreachable!(); } + blooms.push(header.log_bloom()); + + ChainFilter::new(self, self.bloom_indexer.index_size(), self.bloom_indexer.levels()) + .reset_chain_head(&blooms, start_number as usize, self.best_block_number() as usize) + } }; - let bloom_hashes = modified_blooms.iter() - .map(|(bloom_index, _)| self.bloom_indexer.location(&bloom_index).hash) - .collect(); - - for (bloom_hash, blocks_blooms) in modified_blooms.into_iter() + modified_blooms.into_iter() .fold(HashMap::new(), | mut acc, (bloom_index, bloom) | { { let location = self.bloom_indexer.location(&bloom_index); let mut blocks_blooms = acc .entry(location.hash.clone()) .or_insert_with(|| self.blocks_blooms(&location.hash).unwrap_or_else(BlocksBlooms::new)); - assert_eq!(self.bloom_indexer.index_size, blocks_blooms.blooms.len()); + assert_eq!(self.bloom_indexer.index_size(), blocks_blooms.blooms.len()); blocks_blooms.blooms[location.index] = bloom; } acc - }) { - batch.put_extras(&bloom_hash, &blocks_blooms); - } - - // this is new best block - batch.put(b"best", &hash).unwrap(); - - let best_block = BestBlock { - hash: hash.clone(), - number: header.number(), - total_difficulty: total_difficulty - }; - - ExtrasUpdate { - hash: hash, - batch: batch, - new_best: Some(best_block), - details: details, - bloom_hashes: bloom_hashes - } + }) } /// Get best block hash. @@ -825,7 +774,7 @@ mod tests { use std::str::FromStr; use rustc_serialize::hex::FromHex; use util::hash::*; - use blockchain::{BlockProvider, BlockChain}; + use blockchain::{BlockProvider, BlockChain, BlockChainConfig}; use tests::helpers::*; use devtools::*; @@ -834,7 +783,7 @@ mod tests { let genesis = "f901fcf901f7a00000000000000000000000000000000000000000000000000000000000000000a01dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347948888f1f195afa192cfee860698584c030f4c9db1a0925002c3260b44e44c3edebad1cc442142b03020209df1ab8bb86752edbd2cd7a056e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421a056e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421b90100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000008302000080832fefd8808454c98c8142a0363659b251bf8b819179874c8cce7b9b983d7f3704cbb58a3b334431f7032871889032d09c281e1236c0c0".from_hex().unwrap(); let temp = RandomTempPath::new(); - let bc = BlockChain::new(&genesis, temp.as_path()); + let bc = BlockChain::new(BlockChainConfig::default(), &genesis, temp.as_path()); let genesis_hash = H256::from_str("3caa2203f3d7c136c0295ed128a7d31cea520b1ca5e27afe17d0853331798942").unwrap(); @@ -879,7 +828,7 @@ mod tests { let best_block_hash = H256::from_str("c208f88c9f5bf7e00840439742c12e5226d9752981f3ec0521bdcb6dd08af277").unwrap(); let temp = RandomTempPath::new(); - let bc = BlockChain::new(&genesis, temp.as_path()); + let bc = BlockChain::new(BlockChainConfig::default(), &genesis, temp.as_path()); bc.insert_block(&b1, vec![]); bc.insert_block(&b2, vec![]); bc.insert_block(&b3a, vec![]); @@ -898,52 +847,52 @@ mod tests { assert_eq!(bc.block_hash(3).unwrap(), b3a_hash); // test trie route - let r0_1 = bc.tree_route(genesis_hash.clone(), b1_hash.clone()).unwrap(); + let r0_1 = bc.tree_route(genesis_hash.clone(), b1_hash.clone()); assert_eq!(r0_1.ancestor, genesis_hash); assert_eq!(r0_1.blocks, [b1_hash.clone()]); assert_eq!(r0_1.index, 0); - let r0_2 = bc.tree_route(genesis_hash.clone(), b2_hash.clone()).unwrap(); + let r0_2 = bc.tree_route(genesis_hash.clone(), b2_hash.clone()); assert_eq!(r0_2.ancestor, genesis_hash); assert_eq!(r0_2.blocks, [b1_hash.clone(), b2_hash.clone()]); assert_eq!(r0_2.index, 0); - let r1_3a = bc.tree_route(b1_hash.clone(), b3a_hash.clone()).unwrap(); + let r1_3a = bc.tree_route(b1_hash.clone(), b3a_hash.clone()); assert_eq!(r1_3a.ancestor, b1_hash); assert_eq!(r1_3a.blocks, [b2_hash.clone(), b3a_hash.clone()]); assert_eq!(r1_3a.index, 0); - let r1_3b = bc.tree_route(b1_hash.clone(), b3b_hash.clone()).unwrap(); + let r1_3b = bc.tree_route(b1_hash.clone(), b3b_hash.clone()); assert_eq!(r1_3b.ancestor, b1_hash); assert_eq!(r1_3b.blocks, [b2_hash.clone(), b3b_hash.clone()]); assert_eq!(r1_3b.index, 0); - let r3a_3b = bc.tree_route(b3a_hash.clone(), b3b_hash.clone()).unwrap(); + let r3a_3b = bc.tree_route(b3a_hash.clone(), b3b_hash.clone()); assert_eq!(r3a_3b.ancestor, b2_hash); assert_eq!(r3a_3b.blocks, [b3a_hash.clone(), b3b_hash.clone()]); assert_eq!(r3a_3b.index, 1); - let r1_0 = bc.tree_route(b1_hash.clone(), genesis_hash.clone()).unwrap(); + let r1_0 = bc.tree_route(b1_hash.clone(), genesis_hash.clone()); assert_eq!(r1_0.ancestor, genesis_hash); assert_eq!(r1_0.blocks, [b1_hash.clone()]); assert_eq!(r1_0.index, 1); - let r2_0 = bc.tree_route(b2_hash.clone(), genesis_hash.clone()).unwrap(); + let r2_0 = bc.tree_route(b2_hash.clone(), genesis_hash.clone()); assert_eq!(r2_0.ancestor, genesis_hash); assert_eq!(r2_0.blocks, [b2_hash.clone(), b1_hash.clone()]); assert_eq!(r2_0.index, 2); - let r3a_1 = bc.tree_route(b3a_hash.clone(), b1_hash.clone()).unwrap(); + let r3a_1 = bc.tree_route(b3a_hash.clone(), b1_hash.clone()); assert_eq!(r3a_1.ancestor, b1_hash); assert_eq!(r3a_1.blocks, [b3a_hash.clone(), b2_hash.clone()]); assert_eq!(r3a_1.index, 2); - let r3b_1 = bc.tree_route(b3b_hash.clone(), b1_hash.clone()).unwrap(); + let r3b_1 = bc.tree_route(b3b_hash.clone(), b1_hash.clone()); assert_eq!(r3b_1.ancestor, b1_hash); assert_eq!(r3b_1.blocks, [b3b_hash.clone(), b2_hash.clone()]); assert_eq!(r3b_1.index, 2); - let r3b_3a = bc.tree_route(b3b_hash.clone(), b3a_hash.clone()).unwrap(); + let r3b_3a = bc.tree_route(b3b_hash.clone(), b3a_hash.clone()); assert_eq!(r3b_3a.ancestor, b2_hash); assert_eq!(r3b_3a.blocks, [b3b_hash.clone(), b3a_hash.clone()]); assert_eq!(r3b_3a.index, 1); @@ -958,14 +907,14 @@ mod tests { let temp = RandomTempPath::new(); { - let bc = BlockChain::new(&genesis, temp.as_path()); + let bc = BlockChain::new(BlockChainConfig::default(), &genesis, temp.as_path()); assert_eq!(bc.best_block_hash(), genesis_hash); bc.insert_block(&b1, vec![]); assert_eq!(bc.best_block_hash(), b1_hash); } { - let bc = BlockChain::new(&genesis, temp.as_path()); + let bc = BlockChain::new(BlockChainConfig::default(), &genesis, temp.as_path()); assert_eq!(bc.best_block_hash(), b1_hash); } } @@ -1018,7 +967,7 @@ mod tests { let b1_hash = H256::from_str("f53f268d23a71e85c7d6d83a9504298712b84c1a2ba220441c86eeda0bf0b6e3").unwrap(); let temp = RandomTempPath::new(); - let bc = BlockChain::new(&genesis, temp.as_path()); + let bc = BlockChain::new(BlockChainConfig::default(), &genesis, temp.as_path()); bc.insert_block(&b1, vec![]); let transactions = bc.transactions(&b1_hash).unwrap(); @@ -1054,7 +1003,7 @@ mod tests { let bloom_ba = H2048::from_str("00000000000000000000000000000000000000000000020000000800000000000000000000000000000000000000000000000000000000000000000000000000100000000000000000008000000000000000000000000000000000040000000000000000000000000000000000000000000000000000000000000000000000800000000000000000000000000000000000000000000000000000000000008000000000000000000000000000000000000000000000000000000000000000000000000000000000000002000000000000000000040000000000000000000000000000000000000000000000000000000000000000000000000000000000000000").unwrap(); let temp = RandomTempPath::new(); - let bc = BlockChain::new(&genesis, temp.as_path()); + let bc = BlockChain::new(BlockChainConfig::default(), &genesis, temp.as_path()); let blocks_b1 = bc.blocks_with_bloom(&bloom_b1, 0, 5); let blocks_b2 = bc.blocks_with_bloom(&bloom_b2, 0, 5); @@ -1101,30 +1050,5 @@ mod tests { assert_eq!(blocks_ba, vec![3]); } - #[test] - fn test_bloom_indexer() { - use chainfilter::BloomIndex; - use blockchain::BloomIndexer; - use extras::BlocksBloomLocation; - let bi = BloomIndexer::new(16, 3); - - let index = BloomIndex::new(0, 0); - assert_eq!(bi.location(&index), BlocksBloomLocation { - hash: H256::new(), - index: 0 - }); - - let index = BloomIndex::new(1, 0); - assert_eq!(bi.location(&index), BlocksBloomLocation { - hash: H256::from_str("0000000000000000000000000000000000000000000000010000000000000000").unwrap(), - index: 0 - }); - - let index = BloomIndex::new(0, 299_999); - assert_eq!(bi.location(&index), BlocksBloomLocation { - hash: H256::from_str("000000000000000000000000000000000000000000000000000000000000493d").unwrap(), - index: 15 - }); - } } diff --git a/ethcore/src/blockchain/bloom_indexer.rs b/ethcore/src/blockchain/bloom_indexer.rs new file mode 100644 index 000000000..74c679ba8 --- /dev/null +++ b/ethcore/src/blockchain/bloom_indexer.rs @@ -0,0 +1,102 @@ +// Copyright 2015, 2016 Ethcore (UK) Ltd. +// This file is part of Parity. + +// Parity is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Parity is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Parity. If not, see . + +use util::hash::H256; +use chainfilter::BloomIndex; + +/// Represents location of block bloom in extras database. +#[derive(Debug, PartialEq)] +pub struct BlocksBloomLocation { + /// Unique hash of BlocksBloom + pub hash: H256, + /// Index within BlocksBloom + pub index: usize, +} + +/// Should be used to localize blocks blooms in extras database. +pub struct BloomIndexer { + index_size: usize, + levels: u8, +} + +impl BloomIndexer { + /// Plain constructor. + pub fn new(index_size: usize, levels: u8) -> Self { + BloomIndexer { + index_size: index_size, + levels: levels + } + } + + /// Calculates bloom's position in database. + pub fn location(&self, bloom_index: &BloomIndex) -> BlocksBloomLocation { + use std::{mem, ptr}; + + let hash = unsafe { + let mut hash: H256 = mem::zeroed(); + ptr::copy(&[bloom_index.index / self.index_size] as *const usize as *const u8, hash.as_mut_ptr(), 8); + hash[8] = bloom_index.level; + hash.reverse(); + hash + }; + + BlocksBloomLocation { + hash: hash, + index: bloom_index.index % self.index_size + } + } + + /// Returns index size. + pub fn index_size(&self) -> usize { + self.index_size + } + + /// Returns number of cache levels. + pub fn levels(&self) -> u8 { + self.levels + } +} + +#[cfg(test)] +mod tests { + use std::str::FromStr; + use util::hash::{H256, FixedHash}; + use chainfilter::BloomIndex; + use blockchain::bloom_indexer::{BloomIndexer, BlocksBloomLocation}; + + #[test] + fn test_bloom_indexer() { + let bi = BloomIndexer::new(16, 3); + + let index = BloomIndex::new(0, 0); + assert_eq!(bi.location(&index), BlocksBloomLocation { + hash: H256::new(), + index: 0 + }); + + let index = BloomIndex::new(1, 0); + assert_eq!(bi.location(&index), BlocksBloomLocation { + hash: H256::from_str("0000000000000000000000000000000000000000000000010000000000000000").unwrap(), + index: 0 + }); + + let index = BloomIndex::new(0, 299_999); + assert_eq!(bi.location(&index), BlocksBloomLocation { + hash: H256::from_str("000000000000000000000000000000000000000000000000000000000000493d").unwrap(), + index: 15 + }); + } +} diff --git a/ethcore/src/blockchain/cache.rs b/ethcore/src/blockchain/cache.rs new file mode 100644 index 000000000..722f83c16 --- /dev/null +++ b/ethcore/src/blockchain/cache.rs @@ -0,0 +1,37 @@ +// Copyright 2015, 2016 Ethcore (UK) Ltd. +// This file is part of Parity. + +// Parity is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Parity is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Parity. If not, see . + +/// Represents blockchain's in-memory cache size in bytes. +#[derive(Debug)] +pub struct CacheSize { + /// Blocks cache size. + pub blocks: usize, + /// BlockDetails cache size. + pub block_details: usize, + /// Transaction addresses cache size. + pub transaction_addresses: usize, + /// Logs cache size. + pub block_logs: usize, + /// Blooms cache size. + pub blocks_blooms: usize, + /// Block receipts size. + pub block_receipts: usize, +} + +impl CacheSize { + /// Total amount used by the cache. + pub fn total(&self) -> usize { self.blocks + self.block_details + self.transaction_addresses + self.block_logs + self.blocks_blooms } +} diff --git a/ethcore/src/blockchain/mod.rs b/ethcore/src/blockchain/mod.rs new file mode 100644 index 000000000..c1046d960 --- /dev/null +++ b/ethcore/src/blockchain/mod.rs @@ -0,0 +1,29 @@ +// Copyright 2015, 2016 Ethcore (UK) Ltd. +// This file is part of Parity. + +// Parity is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Parity is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Parity. If not, see . + +//! Blockchain database. + +pub mod blockchain; +mod best_block; +mod block_info; +mod bloom_indexer; +mod cache; +mod tree_route; +mod update; + +pub use self::blockchain::{BlockProvider, BlockChain, BlockChainConfig}; +pub use self::cache::CacheSize; +pub use self::tree_route::TreeRoute; diff --git a/ethcore/src/blockchain/tree_route.rs b/ethcore/src/blockchain/tree_route.rs new file mode 100644 index 000000000..1bd0e6f75 --- /dev/null +++ b/ethcore/src/blockchain/tree_route.rs @@ -0,0 +1,29 @@ +// Copyright 2015, 2016 Ethcore (UK) Ltd. +// This file is part of Parity. + +// Parity is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Parity is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Parity. If not, see . + +use util::hash::H256; + +/// Represents a tree route between `from` block and `to` block: +#[derive(Debug)] +pub struct TreeRoute { + /// A vector of hashes of all blocks, ordered from `from` to `to`. + pub blocks: Vec, + /// Best common ancestor of these blocks. + pub ancestor: H256, + /// An index where best common ancestor would be. + pub index: usize, +} + diff --git a/ethcore/src/blockchain/update.rs b/ethcore/src/blockchain/update.rs new file mode 100644 index 000000000..f8ca06e66 --- /dev/null +++ b/ethcore/src/blockchain/update.rs @@ -0,0 +1,21 @@ +use std::collections::HashMap; +use util::hash::H256; +use header::BlockNumber; +use blockchain::block_info::BlockInfo; +use extras::{BlockDetails, BlockReceipts, TransactionAddress, BlocksBlooms}; + +/// Block extras update info. +pub struct ExtrasUpdate { + /// Block info. + pub info: BlockInfo, + /// Modified block hashes. + pub block_hashes: HashMap, + /// Modified block details. + pub block_details: HashMap, + /// Modified block receipts. + pub block_receipts: HashMap, + /// Modified transaction addresses. + pub transactions_addresses: HashMap, + /// Modified blocks blooms. + pub blocks_blooms: HashMap, +} diff --git a/ethcore/src/client.rs b/ethcore/src/client.rs index 4e29433b3..65db6e0e6 100644 --- a/ethcore/src/client.rs +++ b/ethcore/src/client.rs @@ -18,16 +18,15 @@ use util::*; use util::panics::*; -use rocksdb::{Options, DB, DBCompactionStyle}; -use blockchain::{BlockChain, BlockProvider, CacheSize}; +use blockchain::{BlockChain, BlockProvider}; use views::BlockView; use error::*; -use header::BlockNumber; +use header::{BlockNumber, Header}; use state::State; use spec::Spec; use engine::Engine; use views::HeaderView; -use block_queue::{BlockQueue, BlockQueueInfo}; +use block_queue::BlockQueue; use service::{NetSyncMessage, SyncMessage}; use env_info::LastHashes; use verification::*; @@ -36,7 +35,8 @@ use transaction::LocalizedTransaction; use extras::TransactionAddress; use filter::Filter; use log_entry::LocalizedLogEntry; -pub use blockchain::TreeRoute; +pub use block_queue::{BlockQueueConfig, BlockQueueInfo}; +pub use blockchain::{TreeRoute, BlockChainConfig, CacheSize as BlockChainCacheSize}; /// Uniquely identifies block. #[derive(Debug, PartialEq, Clone)] @@ -75,7 +75,16 @@ pub enum BlockStatus { Unknown, } -/// Information about the blockchain gthered together. +/// Client configuration. Includes configs for all sub-systems. +#[derive(Debug, Default)] +pub struct ClientConfig { + /// Block queue configuration. + pub queue: BlockQueueConfig, + /// Blockchain configuration. + pub blockchain: BlockChainConfig, +} + +/// Information about the blockchain gathered together. #[derive(Debug)] pub struct BlockChainInfo { /// Blockchain difficulty. @@ -187,51 +196,28 @@ pub struct Client { } const HISTORY: u64 = 1000; -const CLIENT_DB_VER_STR: &'static str = "3"; +const CLIENT_DB_VER_STR: &'static str = "4.0"; impl Client { /// Create a new client with given spec and DB path. - pub fn new(spec: Spec, path: &Path, message_channel: IoChannel ) -> Result, Error> { + pub fn new(config: ClientConfig, spec: Spec, path: &Path, message_channel: IoChannel ) -> Result, Error> { let mut dir = path.to_path_buf(); dir.push(H64::from(spec.genesis_header().hash()).hex()); //TODO: sec/fat: pruned/full versioning dir.push(format!("v{}-sec-pruned", CLIENT_DB_VER_STR)); let path = dir.as_path(); let gb = spec.genesis_block(); - let chain = Arc::new(BlockChain::new(&gb, path)); - let mut opts = Options::new(); - opts.set_max_open_files(256); - opts.create_if_missing(true); - opts.set_use_fsync(false); - opts.set_compaction_style(DBCompactionStyle::DBUniversalCompaction); - /* - opts.set_bytes_per_sync(8388608); - opts.set_disable_data_sync(false); - opts.set_block_cache_size_mb(1024); - opts.set_table_cache_num_shard_bits(6); - opts.set_max_write_buffer_number(32); - opts.set_write_buffer_size(536870912); - opts.set_target_file_size_base(1073741824); - opts.set_min_write_buffer_number_to_merge(4); - opts.set_level_zero_stop_writes_trigger(2000); - opts.set_level_zero_slowdown_writes_trigger(0); - opts.set_compaction_style(DBUniversalCompaction); - opts.set_max_background_compactions(4); - opts.set_max_background_flushes(4); - opts.set_filter_deletes(false); - opts.set_disable_auto_compactions(false);*/ - + let chain = Arc::new(BlockChain::new(config.blockchain, &gb, path)); let mut state_path = path.to_path_buf(); state_path.push("state"); - let db = Arc::new(DB::open(&opts, state_path.to_str().unwrap()).unwrap()); let engine = Arc::new(try!(spec.to_engine())); - let mut state_db = JournalDB::new_with_arc(db.clone()); + let mut state_db = JournalDB::new(state_path.to_str().unwrap()); if state_db.is_empty() && engine.spec().ensure_db_good(&mut state_db) { state_db.commit(0, &engine.spec().genesis_header().hash(), None).expect("Error commiting genesis state to state DB"); } - let block_queue = BlockQueue::new(engine.clone(), message_channel); + let block_queue = BlockQueue::new(config.queue, engine.clone(), message_channel); let panic_handler = PanicHandler::new_in_arc(); panic_handler.forward_from(&block_queue); @@ -251,85 +237,123 @@ impl Client { self.block_queue.flush(); } + fn build_last_hashes(&self, header: &Header) -> LastHashes { + let mut last_hashes = LastHashes::new(); + last_hashes.resize(256, H256::new()); + last_hashes[0] = header.parent_hash.clone(); + for i in 0..255 { + match self.chain.block_details(&last_hashes[i]) { + Some(details) => { + last_hashes[i + 1] = details.parent.clone(); + }, + None => break, + } + } + last_hashes + } + + fn check_and_close_block(&self, block: &PreVerifiedBlock) -> Result { + let engine = self.engine.deref().deref(); + let header = &block.header; + + // Verify Block Family + let verify_family_result = verify_block_family(&header, &block.bytes, engine, self.chain.deref()); + if let Err(e) = verify_family_result { + warn!(target: "client", "Stage 3 block verification failed for #{} ({})\nError: {:?}", header.number(), header.hash(), e); + return Err(()); + }; + + // Check if Parent is in chain + let chain_has_parent = self.chain.block_header(&header.parent_hash); + if let None = chain_has_parent { + warn!(target: "client", "Block import failed for #{} ({}): Parent not found ({}) ", header.number(), header.hash(), header.parent_hash); + return Err(()); + }; + + // Enact Verified Block + let parent = chain_has_parent.unwrap(); + let last_hashes = self.build_last_hashes(header); + let db = self.state_db.lock().unwrap().clone(); + + let enact_result = enact_verified(&block, engine, db, &parent, last_hashes); + if let Err(e) = enact_result { + warn!(target: "client", "Block import failed for #{} ({})\nError: {:?}", header.number(), header.hash(), e); + return Err(()); + }; + + // Final Verification + let closed_block = enact_result.unwrap(); + if let Err(e) = verify_block_final(&header, closed_block.block().header()) { + warn!(target: "client", "Stage 4 block verification failed for #{} ({})\nError: {:?}", header.number(), header.hash(), e); + return Err(()); + } + + Ok(closed_block) + } + /// This is triggered by a message coming from a block queue when the block is ready for insertion pub fn import_verified_blocks(&self, io: &IoChannel) -> usize { - let mut ret = 0; - let mut bad = HashSet::new(); + let max_blocks_to_import = 128; + + let mut good_blocks = Vec::with_capacity(max_blocks_to_import); + let mut bad_blocks = HashSet::new(); + let _import_lock = self.import_lock.lock(); - let blocks = self.block_queue.drain(128); - let mut good_blocks = Vec::with_capacity(128); + let blocks = self.block_queue.drain(max_blocks_to_import); + for block in blocks { - if bad.contains(&block.header.parent_hash) { - self.block_queue.mark_as_bad(&block.header.hash()); - bad.insert(block.header.hash()); + let header = &block.header; + + if bad_blocks.contains(&header.parent_hash) { + bad_blocks.insert(header.hash()); continue; } - let header = &block.header; - if let Err(e) = verify_block_family(&header, &block.bytes, self.engine.deref().deref(), self.chain.deref()) { - warn!(target: "client", "Stage 3 block verification failed for #{} ({})\nError: {:?}", header.number(), header.hash(), e); - self.block_queue.mark_as_bad(&header.hash()); - bad.insert(block.header.hash()); - break; - }; - let parent = match self.chain.block_header(&header.parent_hash) { - Some(p) => p, - None => { - warn!(target: "client", "Block import failed for #{} ({}): Parent not found ({}) ", header.number(), header.hash(), header.parent_hash); - self.block_queue.mark_as_bad(&header.hash()); - bad.insert(block.header.hash()); - break; - }, - }; - // build last hashes - let mut last_hashes = LastHashes::new(); - last_hashes.resize(256, H256::new()); - last_hashes[0] = header.parent_hash.clone(); - for i in 0..255 { - match self.chain.block_details(&last_hashes[i]) { - Some(details) => { - last_hashes[i + 1] = details.parent.clone(); - }, - None => break, - } - } - - let db = self.state_db.lock().unwrap().clone(); - let result = match enact_verified(&block, self.engine.deref().deref(), db, &parent, &last_hashes) { - Ok(b) => b, - Err(e) => { - warn!(target: "client", "Block import failed for #{} ({})\nError: {:?}", header.number(), header.hash(), e); - bad.insert(block.header.hash()); - self.block_queue.mark_as_bad(&header.hash()); - break; - } - }; - if let Err(e) = verify_block_final(&header, result.block().header()) { - warn!(target: "client", "Stage 4 block verification failed for #{} ({})\nError: {:?}", header.number(), header.hash(), e); - self.block_queue.mark_as_bad(&header.hash()); + let closed_block = self.check_and_close_block(&block); + if let Err(_) = closed_block { + bad_blocks.insert(header.hash()); break; } - good_blocks.push(header.hash().clone()); + // Insert block + let closed_block = closed_block.unwrap(); + self.chain.insert_block(&block.bytes, closed_block.block().receipts().clone()); + good_blocks.push(header.hash()); + + let ancient = if header.number() >= HISTORY { + let n = header.number() - HISTORY; + Some((n, self.chain.block_hash(n).unwrap())) + } else { + None + }; + + // Commit results + closed_block.drain() + .commit(header.number(), &header.hash(), ancient) + .expect("State DB commit failed."); - self.chain.insert_block(&block.bytes, result.block().receipts().clone()); //TODO: err here? - let ancient = if header.number() >= HISTORY { Some(header.number() - HISTORY) } else { None }; - match result.drain().commit(header.number(), &header.hash(), ancient.map(|n|(n, self.chain.block_hash(n).unwrap()))) { - Ok(_) => (), - Err(e) => { - warn!(target: "client", "State DB commit failed: {:?}", e); - break; - } - } self.report.write().unwrap().accrue_block(&block); trace!(target: "client", "Imported #{} ({})", header.number(), header.hash()); - ret += 1; } - self.block_queue.mark_as_good(&good_blocks); - if !good_blocks.is_empty() && self.block_queue.queue_info().is_empty() { - io.send(NetworkIoMessage::User(SyncMessage::BlockVerified)).unwrap(); + + let imported = good_blocks.len(); + let bad_blocks = bad_blocks.into_iter().collect::>(); + + { + self.block_queue.mark_as_bad(&bad_blocks); + self.block_queue.mark_as_good(&good_blocks); } - ret + + { + if !good_blocks.is_empty() && self.block_queue.queue_info().is_empty() { + io.send(NetworkIoMessage::User(SyncMessage::NewChainBlocks { + good: good_blocks, + bad: bad_blocks, + })).unwrap(); + } + } + + imported } /// Get a copy of the best block's state. @@ -338,7 +362,7 @@ impl Client { } /// Get info on the cache. - pub fn cache_info(&self) -> CacheSize { + pub fn blockchain_cache_info(&self) -> BlockChainCacheSize { self.chain.cache_size() } @@ -350,6 +374,7 @@ impl Client { /// Tick the client. pub fn tick(&self) { self.chain.collect_garbage(); + self.block_queue.collect_garbage(); } /// Set up the cache behaviour. @@ -406,7 +431,7 @@ impl BlockChainClient for Client { None => BlockStatus::Unknown } } - + fn block_total_difficulty(&self, id: BlockId) -> Option { Self::block_hash(&self.chain, id).and_then(|hash| self.chain.block_details(&hash)).map(|d| d.total_difficulty) } @@ -426,7 +451,10 @@ impl BlockChainClient for Client { } fn tree_route(&self, from: &H256, to: &H256) -> Option { - self.chain.tree_route(from.clone(), to.clone()) + match self.chain.is_known(from) && self.chain.is_known(to) { + true => Some(self.chain.tree_route(from.clone(), to.clone())), + false => None + } } fn state_data(&self, _hash: &H256) -> Option { @@ -508,7 +536,7 @@ impl BlockChainClient for Client { .collect::>() }) .collect::>() - + }) .collect() } diff --git a/ethcore/src/ethereum/ethash.rs b/ethcore/src/ethereum/ethash.rs index 8c411e7f0..54f02524d 100644 --- a/ethcore/src/ethereum/ethash.rs +++ b/ethcore/src/ethereum/ethash.rs @@ -282,7 +282,7 @@ mod tests { let mut db = db_result.take(); engine.spec().ensure_db_good(&mut db); let last_hashes = vec![genesis_header.hash()]; - let b = OpenBlock::new(engine.deref(), db, &genesis_header, &last_hashes, Address::zero(), vec![]); + let b = OpenBlock::new(engine.deref(), db, &genesis_header, last_hashes, Address::zero(), vec![]); let b = b.close(); assert_eq!(b.state().balance(&Address::zero()), U256::from_str("4563918244f40000").unwrap()); } @@ -295,7 +295,7 @@ mod tests { let mut db = db_result.take(); engine.spec().ensure_db_good(&mut db); let last_hashes = vec![genesis_header.hash()]; - let mut b = OpenBlock::new(engine.deref(), db, &genesis_header, &last_hashes, Address::zero(), vec![]); + let mut b = OpenBlock::new(engine.deref(), db, &genesis_header, last_hashes, Address::zero(), vec![]); let mut uncle = Header::new(); let uncle_author = address_from_hex("ef2d6d194084c2de36e0dabfce45d046b37d1106"); uncle.author = uncle_author.clone(); diff --git a/ethcore/src/extras.rs b/ethcore/src/extras.rs index 64d8afcb9..f4759b040 100644 --- a/ethcore/src/extras.rs +++ b/ethcore/src/extras.rs @@ -16,7 +16,6 @@ //! Blockchain DB extras. -use rocksdb::{DB, Writable}; use util::*; use header::BlockNumber; use receipt::Receipt; @@ -59,7 +58,7 @@ pub trait ExtrasReadable { K: ExtrasSliceConvertable; } -impl ExtrasWritable for W where W: Writable { +impl ExtrasWritable for DBTransaction { fn put_extras(&self, hash: &K, value: &T) where T: ExtrasIndexable + Encodable, K: ExtrasSliceConvertable { @@ -68,7 +67,7 @@ impl ExtrasWritable for W where W: Writable { } } -impl ExtrasReadable for DB { +impl ExtrasReadable for Database { fn get_extras(&self, hash: &K) -> Option where T: ExtrasIndexable + Decodable, K: ExtrasSliceConvertable { @@ -263,15 +262,6 @@ impl Encodable for BlocksBlooms { } } -/// Represents location of bloom in database. -#[derive(Debug, PartialEq)] -pub struct BlocksBloomLocation { - /// Unique hash of BlocksBloom - pub hash: H256, - /// Index within BlocksBloom - pub index: usize, -} - /// Represents address of certain transaction within block #[derive(Clone)] pub struct TransactionAddress { diff --git a/ethcore/src/json_tests/chain.rs b/ethcore/src/json_tests/chain.rs index a386e2854..89bd5da2b 100644 --- a/ethcore/src/json_tests/chain.rs +++ b/ethcore/src/json_tests/chain.rs @@ -15,7 +15,7 @@ // along with Parity. If not, see . use super::test_common::*; -use client::{BlockChainClient,Client}; +use client::{BlockChainClient, Client, ClientConfig}; use pod_state::*; use block::Block; use ethereum; @@ -53,7 +53,7 @@ pub fn json_chain_test(json_data: &[u8], era: ChainEra) -> Vec { let temp = RandomTempPath::new(); { - let client = Client::new(spec, temp.as_path(), IoChannel::disconnected()).unwrap(); + let client = Client::new(ClientConfig::default(), spec, temp.as_path(), IoChannel::disconnected()).unwrap(); assert_eq!(client.chain_info().best_block_hash, genesis_hash); for (b, is_valid) in blocks.into_iter() { if Block::is_good(&b) { diff --git a/ethcore/src/json_tests/state.rs b/ethcore/src/json_tests/state.rs index b5f28444a..f6b5751a7 100644 --- a/ethcore/src/json_tests/state.rs +++ b/ethcore/src/json_tests/state.rs @@ -115,7 +115,7 @@ declare_test!{StateTests_stSolidityTest, "StateTests/stSolidityTest"} declare_test!{StateTests_stSpecialTest, "StateTests/stSpecialTest"} declare_test!{StateTests_stSystemOperationsTest, "StateTests/stSystemOperationsTest"} declare_test!{StateTests_stTransactionTest, "StateTests/stTransactionTest"} -//declare_test!{StateTests_stTransitionTest, "StateTests/stTransitionTest"} +declare_test!{StateTests_stTransitionTest, "StateTests/stTransitionTest"} declare_test!{StateTests_stWalletTest, "StateTests/stWalletTest"} diff --git a/ethcore/src/lib.rs b/ethcore/src/lib.rs index d7efae774..938da02a0 100644 --- a/ethcore/src/lib.rs +++ b/ethcore/src/lib.rs @@ -40,23 +40,17 @@ //! - Ubuntu 14.04 and later: //! //! ```bash -//! # install rocksdb -//! add-apt-repository "deb http://ppa.launchpad.net/giskou/librocksdb/ubuntu trusty main" -//! apt-get update -//! apt-get install -y --force-yes librocksdb //! //! # install multirust //! curl -sf https://raw.githubusercontent.com/brson/multirust/master/blastoff.sh | sh -s -- --yes //! -//! # install nightly and make it default -//! multirust update nightly && multirust default nightly -//! //! # export rust LIBRARY_PATH //! export LIBRARY_PATH=/usr/local/lib //! //! # download and build parity //! git clone https://github.com/ethcore/parity //! cd parity +//! multirust override beta //! cargo build --release //! ``` //! @@ -65,18 +59,15 @@ //! ```bash //! # install rocksdb && multirust //! brew update -//! brew install rocksdb //! brew install multirust //! -//! # install nightly and make it default -//! multirust update nightly && multirust default nightly -//! //! # export rust LIBRARY_PATH //! export LIBRARY_PATH=/usr/local/lib //! //! # download and build parity //! git clone https://github.com/ethcore/parity //! cd parity +//! multirust override beta //! cargo build --release //! ``` @@ -84,8 +75,7 @@ #[macro_use] extern crate ethcore_util as util; #[macro_use] extern crate lazy_static; extern crate rustc_serialize; -extern crate rocksdb; -extern crate heapsize; +#[macro_use] extern crate heapsize; extern crate crypto; extern crate time; extern crate env_logger; @@ -96,8 +86,6 @@ extern crate crossbeam; #[cfg(feature = "jit" )] extern crate evmjit; pub mod block; -pub mod blockchain; -pub mod block_queue; pub mod client; pub mod error; pub mod ethereum; @@ -131,6 +119,8 @@ mod substate; mod executive; mod externalities; mod verification; +mod block_queue; +mod blockchain; #[cfg(test)] mod tests; diff --git a/ethcore/src/service.rs b/ethcore/src/service.rs index 534aab49d..756d02407 100644 --- a/ethcore/src/service.rs +++ b/ethcore/src/service.rs @@ -20,13 +20,18 @@ use util::*; use util::panics::*; use spec::Spec; use error::*; -use client::Client; +use client::{Client, ClientConfig}; /// Message type for external and internal events #[derive(Clone)] pub enum SyncMessage { /// New block has been imported into the blockchain - NewChainBlock(Bytes), //TODO: use Cow + NewChainBlocks { + /// Hashes of blocks imported to blockchain + good: Vec, + /// Hashes of blocks not imported to blockchain + bad: Vec, + }, /// A block is ready BlockVerified, } @@ -43,14 +48,14 @@ pub struct ClientService { impl ClientService { /// Start the service in a separate thread. - pub fn start(spec: Spec, net_config: NetworkConfiguration, db_path: &Path) -> Result { + pub fn start(config: ClientConfig, spec: Spec, net_config: NetworkConfiguration, db_path: &Path) -> Result { let panic_handler = PanicHandler::new_in_arc(); let mut net_service = try!(NetworkService::start(net_config)); panic_handler.forward_from(&net_service); info!("Starting {}", net_service.host_info()); info!("Configured for {} using {} engine", spec.name, spec.engine_name); - let client = try!(Client::new(spec, db_path, net_service.io().channel())); + let client = try!(Client::new(config, spec, db_path, net_service.io().channel())); panic_handler.forward_from(client.deref()); let client_io = Arc::new(ClientIoHandler { client: client.clone() @@ -130,12 +135,13 @@ mod tests { use tests::helpers::*; use util::network::*; use devtools::*; + use client::ClientConfig; #[test] fn it_can_be_started() { let spec = get_test_spec(); let temp_path = RandomTempPath::new(); - let service = ClientService::start(spec, NetworkConfiguration::new_with_port(40456), &temp_path.as_path()); + let service = ClientService::start(ClientConfig::default(), spec, NetworkConfiguration::new_with_port(40456), &temp_path.as_path()); assert!(service.is_ok()); } } diff --git a/ethcore/src/spec.rs b/ethcore/src/spec.rs index 5714ca734..38a0dda53 100644 --- a/ethcore/src/spec.rs +++ b/ethcore/src/spec.rs @@ -58,6 +58,8 @@ pub struct Spec { /// Known nodes on the network in enode format. pub nodes: Vec, + /// Network ID + pub network_id: U256, /// Parameters concerning operation of the specific engine we're using. /// Maps the parameter name to an RLP-encoded value. @@ -120,6 +122,9 @@ impl Spec { /// Get the known knodes of the network in enode format. pub fn nodes(&self) -> &Vec { &self.nodes } + /// Get the configured Network ID. + pub fn network_id(&self) -> U256 { self.network_id } + /// Get the header of the genesis block. pub fn genesis_header(&self) -> Header { Header { @@ -250,6 +255,7 @@ impl FromJson for Spec { engine_name: json["engineName"].as_string().unwrap().to_owned(), engine_params: json_to_rlp_map(&json["params"]), nodes: nodes, + network_id: U256::from_str(&json["params"]["networkID"].as_string().unwrap()[2..]).unwrap(), builtins: builtins, parent_hash: H256::from_str(&genesis["parentHash"].as_string().unwrap()[2..]).unwrap(), author: Address::from_str(&genesis["author"].as_string().unwrap()[2..]).unwrap(), diff --git a/ethcore/src/tests/client.rs b/ethcore/src/tests/client.rs index af25d1b72..83df81fa2 100644 --- a/ethcore/src/tests/client.rs +++ b/ethcore/src/tests/client.rs @@ -14,7 +14,7 @@ // You should have received a copy of the GNU General Public License // along with Parity. If not, see . -use client::{BlockChainClient, Client, BlockId}; +use client::{BlockChainClient, Client, ClientConfig, BlockId}; use tests::helpers::*; use common::*; use devtools::*; @@ -22,14 +22,14 @@ use devtools::*; #[test] fn created() { let dir = RandomTempPath::new(); - let client_result = Client::new(get_test_spec(), dir.as_path(), IoChannel::disconnected()); + let client_result = Client::new(ClientConfig::default(), get_test_spec(), dir.as_path(), IoChannel::disconnected()); assert!(client_result.is_ok()); } #[test] fn imports_from_empty() { let dir = RandomTempPath::new(); - let client = Client::new(get_test_spec(), dir.as_path(), IoChannel::disconnected()).unwrap(); + let client = Client::new(ClientConfig::default(), get_test_spec(), dir.as_path(), IoChannel::disconnected()).unwrap(); client.import_verified_blocks(&IoChannel::disconnected()); client.flush_queue(); } @@ -37,7 +37,7 @@ fn imports_from_empty() { #[test] fn imports_good_block() { let dir = RandomTempPath::new(); - let client = Client::new(get_test_spec(), dir.as_path(), IoChannel::disconnected()).unwrap(); + let client = Client::new(ClientConfig::default(), get_test_spec(), dir.as_path(), IoChannel::disconnected()).unwrap(); let good_block = get_good_dummy_block(); if let Err(_) = client.import_block(good_block) { panic!("error importing block being good by definition"); @@ -52,7 +52,7 @@ fn imports_good_block() { #[test] fn query_none_block() { let dir = RandomTempPath::new(); - let client = Client::new(get_test_spec(), dir.as_path(), IoChannel::disconnected()).unwrap(); + let client = Client::new(ClientConfig::default(), get_test_spec(), dir.as_path(), IoChannel::disconnected()).unwrap(); let non_existant = client.block_header(BlockId::Number(188)); assert!(non_existant.is_none()); @@ -104,5 +104,5 @@ fn can_collect_garbage() { let client_result = generate_dummy_client(100); let client = client_result.reference(); client.tick(); - assert!(client.cache_info().blocks < 100 * 1024); + assert!(client.blockchain_cache_info().blocks < 100 * 1024); } diff --git a/ethcore/src/tests/helpers.rs b/ethcore/src/tests/helpers.rs index 2f90731f0..44ad667b9 100644 --- a/ethcore/src/tests/helpers.rs +++ b/ethcore/src/tests/helpers.rs @@ -14,12 +14,11 @@ // You should have received a copy of the GNU General Public License // along with Parity. If not, see . -use client::{BlockChainClient, Client}; +use client::{BlockChainClient, Client, ClientConfig}; use common::*; use spec::*; -use blockchain::{BlockChain}; +use blockchain::{BlockChain, BlockChainConfig}; use state::*; -use rocksdb::*; use evm::{Schedule, Factory}; use engine::*; use ethereum; @@ -135,7 +134,7 @@ pub fn create_test_block_with_data(header: &Header, transactions: &[&SignedTrans pub fn generate_dummy_client(block_number: u32) -> GuardedTempResult> { let dir = RandomTempPath::new(); - let client = Client::new(get_test_spec(), dir.as_path(), IoChannel::disconnected()).unwrap(); + let client = Client::new(ClientConfig::default(), get_test_spec(), dir.as_path(), IoChannel::disconnected()).unwrap(); let test_spec = get_test_spec(); let test_engine = test_spec.to_engine().unwrap(); let state_root = test_engine.spec().genesis_header().state_root; @@ -173,7 +172,7 @@ pub fn generate_dummy_client(block_number: u32) -> GuardedTempResult pub fn get_test_client_with_blocks(blocks: Vec) -> GuardedTempResult> { let dir = RandomTempPath::new(); - let client = Client::new(get_test_spec(), dir.as_path(), IoChannel::disconnected()).unwrap(); + let client = Client::new(ClientConfig::default(), get_test_spec(), dir.as_path(), IoChannel::disconnected()).unwrap(); for block in &blocks { if let Err(_) = client.import_block(block.clone()) { panic!("panic importing block which is well-formed"); @@ -190,7 +189,7 @@ pub fn get_test_client_with_blocks(blocks: Vec) -> GuardedTempResult GuardedTempResult { let temp = RandomTempPath::new(); - let bc = BlockChain::new(&create_unverifiable_block(0, H256::zero()), temp.as_path()); + let bc = BlockChain::new(BlockChainConfig::default(), &create_unverifiable_block(0, H256::zero()), temp.as_path()); for block_order in 1..block_number { bc.insert_block(&create_unverifiable_block(block_order, bc.best_block_hash()), vec![]); } @@ -203,7 +202,7 @@ pub fn generate_dummy_blockchain(block_number: u32) -> GuardedTempResult GuardedTempResult { let temp = RandomTempPath::new(); - let bc = BlockChain::new(&create_unverifiable_block(0, H256::zero()), temp.as_path()); + let bc = BlockChain::new(BlockChainConfig::default(), &create_unverifiable_block(0, H256::zero()), temp.as_path()); for block_order in 1..block_number { bc.insert_block(&create_unverifiable_block_with_extra(block_order, bc.best_block_hash(), None), vec![]); } @@ -216,7 +215,7 @@ pub fn generate_dummy_blockchain_with_extra(block_number: u32) -> GuardedTempRes pub fn generate_dummy_empty_blockchain() -> GuardedTempResult { let temp = RandomTempPath::new(); - let bc = BlockChain::new(&create_unverifiable_block(0, H256::zero()), temp.as_path()); + let bc = BlockChain::new(BlockChainConfig::default(), &create_unverifiable_block(0, H256::zero()), temp.as_path()); GuardedTempResult:: { _temp: temp, @@ -226,8 +225,7 @@ pub fn generate_dummy_empty_blockchain() -> GuardedTempResult { pub fn get_temp_journal_db() -> GuardedTempResult { let temp = RandomTempPath::new(); - let db = DB::open_default(temp.as_str()).unwrap(); - let journal_db = JournalDB::new(db); + let journal_db = JournalDB::new(temp.as_str()); GuardedTempResult { _temp: temp, result: Some(journal_db) @@ -244,8 +242,7 @@ pub fn get_temp_state() -> GuardedTempResult { } pub fn get_temp_journal_db_in(path: &Path) -> JournalDB { - let db = DB::open_default(path.to_str().unwrap()).unwrap(); - JournalDB::new(db) + JournalDB::new(path.to_str().unwrap()) } pub fn get_temp_state_in(path: &Path) -> State { @@ -253,6 +250,25 @@ pub fn get_temp_state_in(path: &Path) -> State { State::new(journal_db, U256::from(0u8)) } +pub fn get_good_dummy_block_seq(count: usize) -> Vec { + let test_spec = get_test_spec(); + let test_engine = test_spec.to_engine().unwrap(); + let mut parent = test_engine.spec().genesis_header().hash(); + let mut r = Vec::new(); + for i in 1 .. count + 1 { + let mut block_header = Header::new(); + block_header.gas_limit = decode(test_engine.spec().engine_params.get("minGasLimit").unwrap()); + block_header.difficulty = decode(test_engine.spec().engine_params.get("minimumDifficulty").unwrap()); + block_header.timestamp = i as u64; + block_header.number = i as u64; + block_header.parent_hash = parent; + block_header.state_root = test_engine.spec().genesis_header().state_root; + parent = block_header.hash(); + r.push(create_test_block(&block_header)); + } + r +} + pub fn get_good_dummy_block() -> Bytes { let mut block_header = Header::new(); let test_spec = get_test_spec(); diff --git a/install-deps.sh b/install-deps.sh index f5e1e44cf..6cc7de002 100755 --- a/install-deps.sh +++ b/install-deps.sh @@ -342,8 +342,6 @@ function run_installer() exe brew update echo - info "Installing rocksdb" - exe brew install rocksdb info "Installing multirust" exe brew install multirust sudo multirust default beta @@ -391,7 +389,6 @@ function run_installer() linux_version find_multirust - find_rocksdb find_curl find_git @@ -402,21 +399,6 @@ function run_installer() find_sudo } - function find_rocksdb() - { - depCount=$((depCount+1)) - if [[ $(ldconfig -v 2>/dev/null | grep rocksdb | wc -l) == 1 ]]; then - depFound=$((depFound+1)) - check "apt-get" - isRocksDB=true - INSTALL_FILES+="${blue}${dim}==> librocksdb:${reset}$n" - else - uncheck "librocksdb is missing" - isRocksDB=false - INSTALL_FILES+="${blue}${dim}==> librocksdb:${reset}$n" - fi - } - function find_multirust() { depCount=$((depCount+2)) @@ -562,34 +544,6 @@ function run_installer() fi } - function ubuntu_rocksdb_installer() - { - sudo apt-get update -qq - sudo apt-get install -qq -y software-properties-common - sudo apt-add-repository -y ppa:ethcore/ethcore - sudo apt-get -f -y install - sudo apt-get update -qq - sudo apt-get install -qq -y librocksdb-dev librocksdb - } - - function linux_rocksdb_installer() - { - if [[ $isUbuntu == true ]]; then - ubuntu_rocksdb_installer - else - oldpwd=`pwd` - cd /tmp - exe git clone --branch v4.2 --depth=1 https://github.com/facebook/rocksdb.git - cd rocksdb - exe make shared_lib - sudo cp -a librocksdb.so* /usr/lib - sudo ldconfig - cd /tmp - rm -rf /tmp/rocksdb - cd $oldpwd - fi - } - function linux_installer() { if [[ $isGCC == false || $isGit == false || $isMake == false || $isCurl == false ]]; then @@ -610,12 +564,6 @@ function run_installer() echo fi - if [[ $isRocksDB == false ]]; then - info "Installing rocksdb..." - linux_rocksdb_installer - echo - fi - if [[ $isMultirust == false ]]; then info "Installing multirust..." if [[ $isSudo == false ]]; then @@ -655,10 +603,9 @@ function run_installer() find_git find_make find_gcc - find_rocksdb find_multirust - if [[ $isCurl == false || $isGit == false || $isMake == false || $isGCC == false || $isRocksDB == false || $isMultirustBeta == false ]]; then + if [[ $isCurl == false || $isGit == false || $isMake == false || $isGCC == false || $isMultirustBeta == false ]]; then abort_install fi fi diff --git a/install-parity.sh b/install-parity.sh index 60d3471d5..439700306 100755 --- a/install-parity.sh +++ b/install-parity.sh @@ -236,14 +236,29 @@ function run_installer() { linux_version - find_rocksdb - find_curl find_apt find_sudo } + function find_git() + { + depCount=$((depCount+1)) + GIT_PATH=`which git 2>/dev/null` + + if [[ -f $GIT_PATH ]] + then + depFound=$((depFound+1)) + check "git" + isGit=true + else + uncheck "git is missing" + isGit=false + INSTALL_FILES+="${blue}${dim}==> git:${reset}${n}" + fi + } + function find_brew() { BREW_PATH=`which brew 2>/dev/null` @@ -333,20 +348,6 @@ function run_installer() fi } - function find_rocksdb() - { - depCount=$((depCount+1)) - if [[ $(ldconfig -v 2>/dev/null | grep rocksdb | wc -l) == 1 ]]; then - depFound=$((depFound+1)) - check "librocksdb" - isRocksDB=true - else - uncheck "librocksdb is missing" - isRocksDB=false - INSTALL_FILES+="${blue}${dim}==>${reset}\tlibrocksdb${n}" - fi - } - function find_apt() { depCount=$((depCount+1)) @@ -386,10 +387,9 @@ function run_installer() info "Verifying installation" if [[ $OS_TYPE == "linux" ]]; then - find_rocksdb find_apt - if [[ $isRocksDB == false || $isApt == false ]]; then + if [[ $isApt == false ]]; then abortInstall fi fi @@ -397,23 +397,11 @@ function run_installer() function linux_deps_installer() { - if [[ $isRocksDB == false || $isCurl == false ]]; then + if [[ $isCurl == false ]]; then info "Preparing apt..." sudo apt-get update -qq echo fi - - if [[ $isRocksDB == false ]]; then - info "Installing rocksdb..." - - sudo apt-get install -qq -y software-properties-common - sudo apt-add-repository -y ppa:ethcore/ethcore - sudo apt-get -f -y install - sudo apt-get update -qq - sudo apt-get install -qq -y librocksdb - - echo - fi if [[ $isCurl == false ]]; then info "Installing curl..." diff --git a/parity/main.rs b/parity/main.rs index 964529303..3d0d183a3 100644 --- a/parity/main.rs +++ b/parity/main.rs @@ -31,6 +31,7 @@ extern crate ctrlc; extern crate fdlimit; extern crate daemonize; extern crate time; +extern crate number_prefix; #[cfg(feature = "rpc")] extern crate ethcore_rpc as rpc; @@ -47,10 +48,10 @@ use ethcore::spec::*; use ethcore::client::*; use ethcore::service::{ClientService, NetSyncMessage}; use ethcore::ethereum; -use ethcore::blockchain::CacheSize; -use ethsync::EthSync; +use ethsync::{EthSync, SyncConfig}; use docopt::Docopt; use daemonize::Daemonize; +use number_prefix::{binary_prefix, Standalone, Prefixed}; const USAGE: &'static str = r#" Parity. Ethereum Client. @@ -71,13 +72,14 @@ Options: --listen-address URL Specify the IP/port on which to listen for peers [default: 0.0.0.0:30304]. --public-address URL Specify the IP/port on which peers may connect. --address URL Equivalent to --listen-address URL --public-address URL. - --peers NUM Try to manintain that many peers [default: 25]. + --peers NUM Try to maintain that many peers [default: 25]. --no-discovery Disable new peer discovery. - --upnp Use UPnP to try to figure out the correct network settings. + --no-upnp Disable trying to figure out the correct public adderss over UPnP. --node-key KEY Specify node secret key, either as 64-character hex string or input to SHA3 operation. --cache-pref-size BYTES Specify the prefered size of the blockchain cache in bytes [default: 16384]. --cache-max-size BYTES Specify the maximum size of the blockchain cache in bytes [default: 262144]. + --queue-max-size BYTES Specify the maximum size of memory to use for block queue [default: 52428800]. -j --jsonrpc Enable the JSON-RPC API sever. --jsonrpc-url URL Specify URL for JSON-RPC API server [default: 127.0.0.1:8545]. @@ -102,10 +104,11 @@ struct Args { flag_address: Option, flag_peers: u32, flag_no_discovery: bool, - flag_upnp: bool, + flag_no_upnp: bool, flag_node_key: Option, flag_cache_pref_size: usize, flag_cache_max_size: usize, + flag_queue_max_size: usize, flag_jsonrpc: bool, flag_jsonrpc_url: String, flag_jsonrpc_cors: String, @@ -145,9 +148,9 @@ fn setup_rpc_server(client: Arc, sync: Arc, url: &str, cors_dom let mut server = rpc::HttpServer::new(1); server.add_delegate(Web3Client::new().to_delegate()); - server.add_delegate(EthClient::new(client.clone(), sync.clone()).to_delegate()); - server.add_delegate(EthFilterClient::new(client).to_delegate()); - server.add_delegate(NetClient::new(sync).to_delegate()); + server.add_delegate(EthClient::new(&client, &sync).to_delegate()); + server.add_delegate(EthFilterClient::new(&client).to_delegate()); + server.add_delegate(NetClient::new(&sync).to_delegate()); server.start_async(url, cors_domain); } @@ -246,7 +249,7 @@ impl Configuration { fn net_settings(&self, spec: &Spec) -> NetworkConfiguration { let mut ret = NetworkConfiguration::new(); - ret.nat_enabled = self.args.flag_upnp; + ret.nat_enabled = !self.args.flag_no_upnp; ret.boot_nodes = self.init_nodes(spec); let (listen, public) = self.net_addresses(); ret.listen_address = listen; @@ -283,14 +286,19 @@ impl Configuration { let spec = self.spec(); let net_settings = self.net_settings(&spec); + let mut sync_config = SyncConfig::default(); + sync_config.network_id = spec.network_id(); // Build client - let mut service = ClientService::start(spec, net_settings, &Path::new(&self.path())).unwrap(); + let mut client_config = ClientConfig::default(); + client_config.blockchain.pref_cache_size = self.args.flag_cache_pref_size; + client_config.blockchain.max_cache_size = self.args.flag_cache_max_size; + client_config.queue.max_mem_use = self.args.flag_queue_max_size; + let mut service = ClientService::start(client_config, spec, net_settings, &Path::new(&self.path())).unwrap(); let client = service.client().clone(); - client.configure_cache(self.args.flag_cache_pref_size, self.args.flag_cache_max_size); // Sync - let sync = EthSync::register(service.network(), client); + let sync = EthSync::register(service.network(), sync_config, client); // Setup rpc if self.args.flag_jsonrpc { @@ -331,7 +339,7 @@ fn main() { struct Informant { chain_info: RwLock>, - cache_info: RwLock>, + cache_info: RwLock>, report: RwLock>, } @@ -346,18 +354,26 @@ impl Default for Informant { } impl Informant { + + fn format_bytes(b: usize) -> String { + match binary_prefix(b as f64) { + Standalone(bytes) => format!("{} bytes", bytes), + Prefixed(prefix, n) => format!("{:.0} {}B", n, prefix), + } + } + pub fn tick(&self, client: &Client, sync: &EthSync) { // 5 seconds betwen calls. TODO: calculate this properly. let dur = 5usize; let chain_info = client.chain_info(); let queue_info = client.queue_info(); - let cache_info = client.cache_info(); + let cache_info = client.blockchain_cache_info(); let report = client.report(); let sync_info = sync.status(); - if let (_, &Some(ref last_cache_info), &Some(ref last_report)) = (self.chain_info.read().unwrap().deref(), self.cache_info.read().unwrap().deref(), self.report.read().unwrap().deref()) { - println!("[ #{} {} ]---[ {} blk/s | {} tx/s | {} gas/s //··· {}/{} peers, #{}, {}+{} queued ···// {} ({}) bl {} ({}) ex ]", + if let (_, _, &Some(ref last_report)) = (self.chain_info.read().unwrap().deref(), self.cache_info.read().unwrap().deref(), self.report.read().unwrap().deref()) { + println!("[ #{} {} ]---[ {} blk/s | {} tx/s | {} gas/s //··· {}/{} peers, #{}, {}+{} queued ···// mem: {} chain, {} queue, {} sync ]", chain_info.best_block_number, chain_info.best_block_hash, (report.blocks_imported - last_report.blocks_imported) / dur, @@ -370,10 +386,9 @@ impl Informant { queue_info.unverified_queue_size, queue_info.verified_queue_size, - cache_info.blocks, - cache_info.blocks as isize - last_cache_info.blocks as isize, - cache_info.block_details, - cache_info.block_details as isize - last_cache_info.block_details as isize + Informant::format_bytes(cache_info.total()), + Informant::format_bytes(queue_info.mem_used), + Informant::format_bytes(sync_info.mem_used), ); } diff --git a/rpc/Cargo.toml b/rpc/Cargo.toml index c91549b86..b9ca9deac 100644 --- a/rpc/Cargo.toml +++ b/rpc/Cargo.toml @@ -9,19 +9,19 @@ build = "build.rs" [lib] [dependencies] -serde = "0.6.7" -serde_json = "0.6.0" -jsonrpc-core = "1.1" -jsonrpc-http-server = "2.0" +serde = "0.7.0" +serde_json = "0.7.0" +jsonrpc-core = "1.2" +jsonrpc-http-server = "2.1" ethcore-util = { path = "../util" } ethcore = { path = "../ethcore" } ethsync = { path = "../sync" } clippy = { version = "0.0.44", optional = true } rustc-serialize = "0.3" -serde_macros = { version = "0.6.13", optional = true } +serde_macros = { version = "0.7.0", optional = true } [build-dependencies] -serde_codegen = { version = "0.6.13", optional = true } +serde_codegen = { version = "0.7.0", optional = true } syntex = "0.29.0" [features] diff --git a/rpc/src/lib.rs b/rpc/src/lib.rs index d7b5bdc3b..8dd58d0b8 100644 --- a/rpc/src/lib.rs +++ b/rpc/src/lib.rs @@ -16,8 +16,8 @@ //! Ethcore rpc. #![warn(missing_docs)] -#![cfg_attr(nightly, feature(custom_derive, custom_attribute, plugin))] -#![cfg_attr(nightly, plugin(serde_macros, clippy))] +#![cfg_attr(feature="nightly", feature(custom_derive, custom_attribute, plugin))] +#![cfg_attr(feature="nightly", plugin(serde_macros, clippy))] extern crate rustc_serialize; extern crate serde; diff --git a/rpc/src/v1/impls/eth.rs b/rpc/src/v1/impls/eth.rs index 689afd019..00bce5437 100644 --- a/rpc/src/v1/impls/eth.rs +++ b/rpc/src/v1/impls/eth.rs @@ -15,7 +15,7 @@ // along with Parity. If not, see . //! Eth rpc implementation. -use std::sync::Arc; +use std::sync::{Arc, Weak}; use ethsync::{EthSync, SyncState}; use jsonrpc_core::*; use util::hash::*; @@ -29,21 +29,22 @@ use v1::types::{Block, BlockTransactions, BlockNumber, Bytes, SyncStatus, SyncIn /// Eth rpc implementation. pub struct EthClient { - client: Arc, - sync: Arc + client: Weak, + sync: Weak } impl EthClient { /// Creates new EthClient. - pub fn new(client: Arc, sync: Arc) -> Self { + pub fn new(client: &Arc, sync: &Arc) -> Self { EthClient { - client: client, - sync: sync + client: Arc::downgrade(client), + sync: Arc::downgrade(sync) } } fn block(&self, id: BlockId, include_txs: bool) -> Result { - match (self.client.block(id.clone()), self.client.block_total_difficulty(id)) { + let client = take_weak!(self.client); + match (client.block(id.clone()), client.block_total_difficulty(id)) { (Some(bytes), Some(total_difficulty)) => { let block_view = BlockView::new(&bytes); let view = block_view.header_view(); @@ -78,9 +79,9 @@ impl EthClient { _ => Ok(Value::Null) } } - + fn transaction(&self, id: TransactionId) -> Result { - match self.client.transaction(id) { + match take_weak!(self.client).transaction(id) { Some(t) => to_value(&Transaction::from(t)), None => Ok(Value::Null) } @@ -90,7 +91,7 @@ impl EthClient { impl Eth for EthClient { fn protocol_version(&self, params: Params) -> Result { match params { - Params::None => to_value(&U256::from(self.sync.status().protocol_version)), + Params::None => to_value(&U256::from(take_weak!(self.sync).status().protocol_version)), _ => Err(Error::invalid_params()) } } @@ -98,12 +99,12 @@ impl Eth for EthClient { fn syncing(&self, params: Params) -> Result { match params { Params::None => { - let status = self.sync.status(); + let status = take_weak!(self.sync).status(); let res = match status.state { SyncState::NotSynced | SyncState::Idle => SyncStatus::None, SyncState::Waiting | SyncState::Blocks | SyncState::NewBlocks => SyncStatus::Info(SyncInfo { starting_block: U256::from(status.start_block_number), - current_block: U256::from(self.client.chain_info().best_block_number), + current_block: U256::from(take_weak!(self.client).chain_info().best_block_number), highest_block: U256::from(status.highest_block_number.unwrap_or(status.start_block_number)) }) }; @@ -146,14 +147,14 @@ impl Eth for EthClient { fn block_number(&self, params: Params) -> Result { match params { - Params::None => to_value(&U256::from(self.client.chain_info().best_block_number)), + Params::None => to_value(&U256::from(take_weak!(self.client).chain_info().best_block_number)), _ => Err(Error::invalid_params()) } } fn block_transaction_count(&self, params: Params) -> Result { from_params::<(H256,)>(params) - .and_then(|(hash,)| match self.client.block(BlockId::Hash(hash)) { + .and_then(|(hash,)| match take_weak!(self.client).block(BlockId::Hash(hash)) { Some(bytes) => to_value(&BlockView::new(&bytes).transactions_count()), None => Ok(Value::Null) }) @@ -161,7 +162,7 @@ impl Eth for EthClient { fn block_uncles_count(&self, params: Params) -> Result { from_params::<(H256,)>(params) - .and_then(|(hash,)| match self.client.block(BlockId::Hash(hash)) { + .and_then(|(hash,)| match take_weak!(self.client).block(BlockId::Hash(hash)) { Some(bytes) => to_value(&BlockView::new(&bytes).uncles_count()), None => Ok(Value::Null) }) @@ -170,7 +171,7 @@ impl Eth for EthClient { // TODO: do not ignore block number param fn code_at(&self, params: Params) -> Result { from_params::<(Address, BlockNumber)>(params) - .and_then(|(address, _block_number)| to_value(&self.client.code(&address).map_or_else(Bytes::default, Bytes::new))) + .and_then(|(address, _block_number)| to_value(&take_weak!(self.client).code(&address).map_or_else(Bytes::default, Bytes::new))) } fn block_by_hash(&self, params: Params) -> Result { @@ -201,7 +202,7 @@ impl Eth for EthClient { fn logs(&self, params: Params) -> Result { from_params::<(Filter,)>(params) .and_then(|(filter,)| { - let logs = self.client.logs(filter.into()) + let logs = take_weak!(self.client).logs(filter.into()) .into_iter() .map(From::from) .collect::>(); @@ -212,14 +213,14 @@ impl Eth for EthClient { /// Eth filter rpc implementation. pub struct EthFilterClient { - client: Arc + client: Weak } impl EthFilterClient { /// Creates new Eth filter client. - pub fn new(client: Arc) -> Self { + pub fn new(client: &Arc) -> Self { EthFilterClient { - client: client + client: Arc::downgrade(client) } } } @@ -234,6 +235,6 @@ impl EthFilter for EthFilterClient { } fn filter_changes(&self, _: Params) -> Result { - to_value(&self.client.chain_info().best_block_hash).map(|v| Value::Array(vec![v])) + to_value(&take_weak!(self.client).chain_info().best_block_hash).map(|v| Value::Array(vec![v])) } } diff --git a/rpc/src/v1/impls/mod.rs b/rpc/src/v1/impls/mod.rs index bc8b436fd..d9102b1db 100644 --- a/rpc/src/v1/impls/mod.rs +++ b/rpc/src/v1/impls/mod.rs @@ -15,6 +15,16 @@ // along with Parity. If not, see . //! Ethereum rpc interface implementation. + +macro_rules! take_weak { + ($weak: expr) => { + match $weak.upgrade() { + Some(arc) => arc, + None => return Err(Error::internal_error()) + } + } +} + mod web3; mod eth; mod net; diff --git a/rpc/src/v1/impls/net.rs b/rpc/src/v1/impls/net.rs index b36042dba..9e24caad2 100644 --- a/rpc/src/v1/impls/net.rs +++ b/rpc/src/v1/impls/net.rs @@ -15,31 +15,31 @@ // along with Parity. If not, see . //! Net rpc implementation. -use std::sync::Arc; +use std::sync::{Arc, Weak}; use jsonrpc_core::*; use ethsync::EthSync; use v1::traits::Net; /// Net rpc implementation. pub struct NetClient { - sync: Arc + sync: Weak } impl NetClient { /// Creates new NetClient. - pub fn new(sync: Arc) -> Self { + pub fn new(sync: &Arc) -> Self { NetClient { - sync: sync + sync: Arc::downgrade(sync) } } } impl Net for NetClient { fn version(&self, _: Params) -> Result { - Ok(Value::U64(self.sync.status().protocol_version as u64)) + Ok(Value::U64(take_weak!(self.sync).status().protocol_version as u64)) } fn peer_count(&self, _params: Params) -> Result { - Ok(Value::U64(self.sync.status().num_peers as u64)) + Ok(Value::U64(take_weak!(self.sync).status().num_peers as u64)) } } diff --git a/rpc/src/v1/types/block_number.rs b/rpc/src/v1/types/block_number.rs index bb563de99..c955ec895 100644 --- a/rpc/src/v1/types/block_number.rs +++ b/rpc/src/v1/types/block_number.rs @@ -30,7 +30,7 @@ pub enum BlockNumber { impl Deserialize for BlockNumber { fn deserialize(deserializer: &mut D) -> Result where D: Deserializer { - deserializer.visit(BlockNumberVisitor) + deserializer.deserialize(BlockNumberVisitor) } } @@ -44,8 +44,8 @@ impl Visitor for BlockNumberVisitor { "latest" => Ok(BlockNumber::Latest), "earliest" => Ok(BlockNumber::Earliest), "pending" => Ok(BlockNumber::Pending), - _ if value.starts_with("0x") => u64::from_str_radix(&value[2..], 16).map(BlockNumber::Num).map_err(|_| Error::syntax("invalid block number")), - _ => value.parse::().map(BlockNumber::Num).map_err(|_| Error::syntax("invalid block number")) + _ if value.starts_with("0x") => u64::from_str_radix(&value[2..], 16).map(BlockNumber::Num).map_err(|_| Error::custom("invalid block number")), + _ => value.parse::().map(BlockNumber::Num).map_err(|_| Error::custom("invalid block number")) } } diff --git a/rpc/src/v1/types/bytes.rs b/rpc/src/v1/types/bytes.rs index d6a648d7c..f09f24e4d 100644 --- a/rpc/src/v1/types/bytes.rs +++ b/rpc/src/v1/types/bytes.rs @@ -40,7 +40,7 @@ impl Serialize for Bytes { where S: Serializer { let mut serialized = "0x".to_owned(); serialized.push_str(self.0.to_hex().as_ref()); - serializer.visit_str(serialized.as_ref()) + serializer.serialize_str(serialized.as_ref()) } } diff --git a/rpc/src/v1/types/filter.rs b/rpc/src/v1/types/filter.rs index c5c3604dd..01d322cce 100644 --- a/rpc/src/v1/types/filter.rs +++ b/rpc/src/v1/types/filter.rs @@ -40,7 +40,7 @@ impl Deserialize for VariadicValue where T: Deserialize { Deserialize::deserialize(&mut value::Deserializer::new(v.clone())).map(VariadicValue::Single) .or_else(|_| Deserialize::deserialize(&mut value::Deserializer::new(v.clone())).map(VariadicValue::Multiple)) - .map_err(|_| Error::syntax("")) // unreachable, but types must match + .map_err(|_| Error::custom("")) // unreachable, but types must match } } @@ -48,6 +48,7 @@ pub type FilterAddress = VariadicValue
; pub type Topic = VariadicValue; #[derive(Debug, PartialEq, Deserialize)] +#[serde(deny_unknown_fields)] pub struct Filter { #[serde(rename="fromBlock")] pub from_block: Option, diff --git a/rpc/src/v1/types/index.rs b/rpc/src/v1/types/index.rs index a77096fbf..e7cbbd255 100644 --- a/rpc/src/v1/types/index.rs +++ b/rpc/src/v1/types/index.rs @@ -30,7 +30,7 @@ impl Index { impl Deserialize for Index { fn deserialize(deserializer: &mut D) -> Result where D: Deserializer { - deserializer.visit(IndexVisitor) + deserializer.deserialize(IndexVisitor) } } @@ -41,8 +41,8 @@ impl Visitor for IndexVisitor { fn visit_str(&mut self, value: &str) -> Result where E: Error { match value { - _ if value.starts_with("0x") => usize::from_str_radix(&value[2..], 16).map(Index).map_err(|_| Error::syntax("invalid index")), - _ => value.parse::().map(Index).map_err(|_| Error::syntax("invalid index")) + _ if value.starts_with("0x") => usize::from_str_radix(&value[2..], 16).map(Index).map_err(|_| Error::custom("invalid index")), + _ => value.parse::().map(Index).map_err(|_| Error::custom("invalid index")) } } diff --git a/sync/Cargo.toml b/sync/Cargo.toml index 7a81c7d97..26a7d463c 100644 --- a/sync/Cargo.toml +++ b/sync/Cargo.toml @@ -15,6 +15,7 @@ log = "0.3" env_logger = "0.3" time = "0.1.34" rand = "0.3.13" +heapsize = "0.3" [features] default = [] diff --git a/sync/src/chain.rs b/sync/src/chain.rs index 930518007..dd89cdbf8 100644 --- a/sync/src/chain.rs +++ b/sync/src/chain.rs @@ -39,7 +39,9 @@ use ethcore::error::*; use ethcore::block::Block; use io::SyncIo; use time; -use std::option::Option; +use super::SyncConfig; + +known_heap_size!(0, PeerInfo, Header, HeaderId); impl ToUsize for BlockNumber { fn to_usize(&self) -> usize { @@ -80,9 +82,7 @@ const NODE_DATA_PACKET: u8 = 0x0e; const GET_RECEIPTS_PACKET: u8 = 0x0f; const RECEIPTS_PACKET: u8 = 0x10; -const NETWORK_ID: U256 = ONE_U256; //TODO: get this from parent - -const CONNECTION_TIMEOUT_SEC: f64 = 10f64; +const CONNECTION_TIMEOUT_SEC: f64 = 5f64; struct Header { /// Header data @@ -135,6 +135,8 @@ pub struct SyncStatus { pub num_peers: usize, /// Total number of active peers pub num_active_peers: usize, + /// Heap memory used in bytes + pub mem_used: usize, } #[derive(PartialEq, Eq, Debug, Clone)] @@ -203,13 +205,17 @@ pub struct ChainSync { have_common_block: bool, /// Last propagated block number last_send_block_number: BlockNumber, + /// Max blocks to download ahead + max_download_ahead_blocks: usize, + /// Network ID + network_id: U256, } type RlpResponseResult = Result, PacketDecodeError>; impl ChainSync { /// Create a new instance of syncing strategy. - pub fn new() -> ChainSync { + pub fn new(config: SyncConfig) -> ChainSync { ChainSync { state: SyncState::NotSynced, starting_block: 0, @@ -226,6 +232,8 @@ impl ChainSync { syncing_difficulty: U256::from(0u64), have_common_block: false, last_send_block_number: 0, + max_download_ahead_blocks: max(MAX_HEADERS_TO_REQUEST, config.max_download_ahead_blocks), + network_id: config.network_id, } } @@ -241,6 +249,15 @@ impl ChainSync { blocks_total: match self.highest_block { None => 0, Some(x) => x - self.starting_block }, num_peers: self.peers.len(), num_active_peers: self.peers.values().filter(|p| p.asking != PeerAsking::Nothing).count(), + mem_used: + // TODO: https://github.com/servo/heapsize/pull/50 + // self.downloading_hashes.heap_size_of_children() + //+ self.downloading_bodies.heap_size_of_children() + //+ self.downloading_hashes.heap_size_of_children() + self.headers.heap_size_of_children() + + self.bodies.heap_size_of_children() + + self.peers.heap_size_of_children() + + self.header_ids.heap_size_of_children(), } } @@ -275,7 +292,6 @@ impl ChainSync { self.starting_block = 0; self.highest_block = None; self.have_common_block = false; - io.chain().clear_queue(); self.starting_block = io.chain().chain_info().best_block_number; self.state = SyncState::NotSynced; } @@ -307,7 +323,7 @@ impl ChainSync { trace!(target: "sync", "Peer {} genesis hash not matched", peer_id); return Ok(()); } - if peer.network_id != NETWORK_ID { + if peer.network_id != self.network_id { io.disable_peer(peer_id); trace!(target: "sync", "Peer {} network id not matched", peer_id); return Ok(()); @@ -436,7 +452,7 @@ impl ChainSync { trace!(target: "sync", "Got body {}", n); } None => { - debug!(target: "sync", "Ignored unknown block body"); + trace!(target: "sync", "Ignored unknown/stale block body"); } } } @@ -611,7 +627,7 @@ impl ChainSync { self.request_headers_by_hash(io, peer_id, &peer_latest, 1, 0, false); } else if self.state == SyncState::Blocks && io.chain().block_status(BlockId::Hash(peer_latest)) == BlockStatus::Unknown { - self.request_blocks(io, peer_id); + self.request_blocks(io, peer_id, false); } } @@ -620,7 +636,7 @@ impl ChainSync { } /// Find some headers or blocks to download for a peer. - fn request_blocks(&mut self, io: &mut SyncIo, peer_id: PeerId) { + fn request_blocks(&mut self, io: &mut SyncIo, peer_id: PeerId, ignore_others: bool) { self.clear_peer_download(peer_id); if io.chain().queue_info().is_full() { @@ -640,28 +656,34 @@ impl ChainSync { let mut index: BlockNumber = 0; while index != items.len() as BlockNumber && needed_bodies.len() < MAX_BODIES_TO_REQUEST { let block = start + index; - if !self.downloading_bodies.contains(&block) && !self.bodies.have_item(&block) { + if ignore_others || (!self.downloading_bodies.contains(&block) && !self.bodies.have_item(&block)) { needed_bodies.push(items[index as usize].hash.clone()); needed_numbers.push(block); - self.downloading_bodies.insert(block); } index += 1; } } } if !needed_bodies.is_empty() { + let (head, _) = self.headers.range_iter().next().unwrap(); + if needed_numbers.first().unwrap() - head > self.max_download_ahead_blocks as BlockNumber { + trace!(target: "sync", "{}: Stalled download ({} vs {}), helping with downloading block bodies", peer_id, needed_numbers.first().unwrap(), head); + self.request_blocks(io, peer_id, true); + return; + } + self.downloading_bodies.extend(needed_numbers.iter()); replace(&mut self.peers.get_mut(&peer_id).unwrap().asking_blocks, needed_numbers); self.request_bodies(io, peer_id, needed_bodies); } else { // check if need to download headers - let mut start = 0usize; + let mut start = 0; if !self.have_common_block { // download backwards until common block is found 1 header at a time let chain_info = io.chain().chain_info(); - start = chain_info.best_block_number as usize; + start = chain_info.best_block_number; if !self.headers.is_empty() { - start = min(start, self.headers.range_iter().next().unwrap().0 as usize - 1); + start = min(start, self.headers.range_iter().next().unwrap().0 - 1); } if start == 0 { self.have_common_block = true; //reached genesis @@ -672,6 +694,7 @@ impl ChainSync { if self.have_common_block { let mut headers: Vec = Vec::new(); let mut prev = self.current_base_block() + 1; + let head = self.headers.range_iter().next().map(|(h, _)| h); for (next, ref items) in self.headers.range_iter() { if !headers.is_empty() { break; @@ -682,9 +705,8 @@ impl ChainSync { } let mut block = prev; while block < next && headers.len() < MAX_HEADERS_TO_REQUEST { - if !self.downloading_headers.contains(&(block as BlockNumber)) { + if ignore_others || !self.downloading_headers.contains(&(block as BlockNumber)) { headers.push(block as BlockNumber); - self.downloading_headers.insert(block as BlockNumber); } block += 1; } @@ -692,17 +714,23 @@ impl ChainSync { } if !headers.is_empty() { - start = headers[0] as usize; + start = headers[0]; + if head.is_some() && start > head.unwrap() && start - head.unwrap() > self.max_download_ahead_blocks as BlockNumber { + trace!(target: "sync", "{}: Stalled download ({} vs {}), helping with downloading headers", peer_id, start, head.unwrap()); + self.request_blocks(io, peer_id, true); + return; + } let count = headers.len(); + self.downloading_headers.extend(headers.iter()); replace(&mut self.peers.get_mut(&peer_id).unwrap().asking_blocks, headers); - assert!(!self.headers.have_item(&(start as BlockNumber))); - self.request_headers_by_number(io, peer_id, start as BlockNumber, count, 0, false); + assert!(!self.headers.have_item(&start)); + self.request_headers_by_number(io, peer_id, start, count, 0, false); } } else { // continue search for common block - self.downloading_headers.insert(start as BlockNumber); - self.request_headers_by_number(io, peer_id, start as BlockNumber, 1, 0, false); + self.downloading_headers.insert(start); + self.request_headers_by_number(io, peer_id, start, 1, 0, false); } } } @@ -894,7 +922,7 @@ impl ChainSync { let mut packet = RlpStream::new_list(5); let chain = io.chain().chain_info(); packet.append(&(PROTOCOL_VERSION as u32)); - packet.append(&NETWORK_ID); //TODO: network id + packet.append(&self.network_id); packet.append(&chain.total_difficulty); packet.append(&chain.best_block_hash); packet.append(&chain.genesis_hash); @@ -1220,6 +1248,7 @@ impl ChainSync { mod tests { use tests::helpers::*; use super::*; + use ::SyncConfig; use util::*; use super::{PeerInfo, PeerAsking}; use ethcore::header::*; @@ -1333,7 +1362,7 @@ mod tests { } fn dummy_sync_with_peer(peer_latest_hash: H256) -> ChainSync { - let mut sync = ChainSync::new(); + let mut sync = ChainSync::new(SyncConfig::default()); sync.peers.insert(0, PeerInfo { protocol_version: 0, diff --git a/sync/src/lib.rs b/sync/src/lib.rs index fd586409a..6f28fc320 100644 --- a/sync/src/lib.rs +++ b/sync/src/lib.rs @@ -34,15 +34,15 @@ //! use std::env; //! use std::sync::Arc; //! use util::network::{NetworkService, NetworkConfiguration}; -//! use ethcore::client::Client; -//! use ethsync::EthSync; +//! use ethcore::client::{Client, ClientConfig}; +//! use ethsync::{EthSync, SyncConfig}; //! use ethcore::ethereum; //! //! fn main() { //! let mut service = NetworkService::start(NetworkConfiguration::new()).unwrap(); //! let dir = env::temp_dir(); -//! let client = Client::new(ethereum::new_frontier(), &dir, service.io().channel()).unwrap(); -//! EthSync::register(&mut service, client); +//! let client = Client::new(ClientConfig::default(), ethereum::new_frontier(), &dir, service.io().channel()).unwrap(); +//! EthSync::register(&mut service, SyncConfig::default(), client); //! } //! ``` @@ -54,12 +54,15 @@ extern crate ethcore; extern crate env_logger; extern crate time; extern crate rand; +#[macro_use] +extern crate heapsize; use std::ops::*; use std::sync::*; use ethcore::client::Client; use util::network::{NetworkProtocolHandler, NetworkService, NetworkContext, PeerId}; use util::TimerToken; +use util::{U256, ONE_U256}; use chain::ChainSync; use ethcore::service::SyncMessage; use io::NetSyncIo; @@ -71,6 +74,23 @@ mod range_collection; #[cfg(test)] mod tests; +/// Sync configuration +pub struct SyncConfig { + /// Max blocks to download ahead + pub max_download_ahead_blocks: usize, + /// Network ID + pub network_id: U256, +} + +impl Default for SyncConfig { + fn default() -> SyncConfig { + SyncConfig { + max_download_ahead_blocks: 20000, + network_id: ONE_U256, + } + } +} + /// Ethereum network protocol handler pub struct EthSync { /// Shared blockchain client. TODO: this should evetually become an IPC endpoint @@ -83,10 +103,10 @@ pub use self::chain::{SyncStatus, SyncState}; impl EthSync { /// Creates and register protocol with the network service - pub fn register(service: &mut NetworkService, chain: Arc) -> Arc { + pub fn register(service: &mut NetworkService, config: SyncConfig, chain: Arc) -> Arc { let sync = Arc::new(EthSync { chain: chain, - sync: RwLock::new(ChainSync::new()), + sync: RwLock::new(ChainSync::new(config)), }); service.register_protocol(sync.clone(), "eth", &[62u8, 63u8]).expect("Error registering eth protocol handler"); sync diff --git a/sync/src/tests/helpers.rs b/sync/src/tests/helpers.rs index 7f2928ccd..63a0b88bb 100644 --- a/sync/src/tests/helpers.rs +++ b/sync/src/tests/helpers.rs @@ -15,12 +15,12 @@ // along with Parity. If not, see . use util::*; -use ethcore::client::{BlockChainClient, BlockStatus, TreeRoute, BlockChainInfo, TransactionId, BlockId}; -use ethcore::block_queue::BlockQueueInfo; +use ethcore::client::{BlockChainClient, BlockStatus, TreeRoute, BlockChainInfo, TransactionId, BlockId, BlockQueueInfo}; use ethcore::header::{Header as BlockHeader, BlockNumber}; use ethcore::error::*; use io::SyncIo; -use chain::{ChainSync}; +use chain::ChainSync; +use ::SyncConfig; use ethcore::receipt::Receipt; use ethcore::transaction::LocalizedTransaction; use ethcore::filter::Filter; @@ -251,6 +251,9 @@ impl BlockChainClient for TestBlockChainClient { verified_queue_size: 0, unverified_queue_size: 0, verifying_queue_size: 0, + max_queue_size: 0, + max_mem_use: 0, + mem_used: 0, } } @@ -340,7 +343,7 @@ impl TestNet { for _ in 0..n { net.peers.push(TestPeer { chain: TestBlockChainClient::new(), - sync: ChainSync::new(), + sync: ChainSync::new(SyncConfig::default()), queue: VecDeque::new(), }); } diff --git a/util/Cargo.toml b/util/Cargo.toml index 6d2ebcd9b..6a18f80de 100644 --- a/util/Cargo.toml +++ b/util/Cargo.toml @@ -16,9 +16,9 @@ mio = "0.5.0" rand = "0.3.12" time = "0.1.34" tiny-keccak = "1.0" -rocksdb = "0.3" +rocksdb = { git = "https://github.com/arkpar/rust-rocksdb.git" } lazy_static = "0.1" -eth-secp256k1 = { git = "https://github.com/arkpar/rust-secp256k1.git" } +eth-secp256k1 = { git = "https://github.com/ethcore/rust-secp256k1" } rust-crypto = "0.2.34" elastic-array = "0.4" heapsize = "0.3" @@ -26,7 +26,7 @@ itertools = "0.4" crossbeam = "0.2" slab = "0.1" sha3 = { path = "sha3" } -serde = "0.6.7" +serde = "0.7.0" clippy = { version = "0.0.44", optional = true } json-tests = { path = "json-tests" } rustc_version = "0.1.0" @@ -39,6 +39,7 @@ target_info = "0.1" [features] default = [] dev = ["clippy"] +x64asm = [] [build-dependencies] vergen = "*" diff --git a/util/benches/bigint.rs b/util/benches/bigint.rs new file mode 100644 index 000000000..da82084b8 --- /dev/null +++ b/util/benches/bigint.rs @@ -0,0 +1,84 @@ +// Copyright 2015, 2016 Ethcore (UK) Ltd. +// This file is part of Parity. + +// Parity is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Parity is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Parity. If not, see . + +//! benchmarking for rlp +//! should be started with: +//! ```bash +//! multirust run nightly cargo bench +//! ``` + +#![feature(test)] +#![feature(asm)] + +extern crate test; +extern crate ethcore_util; +extern crate rand; + +use test::{Bencher, black_box}; +use ethcore_util::uint::*; + +#[bench] +fn u256_add(b: &mut Bencher) { + b.iter(|| { + let n = black_box(10000); + (0..n).fold(U256([rand::random::(), rand::random::(), rand::random::(), rand::random::()]), |old, new| { old.overflowing_add(U256::from(new)).0 }) + }); +} + +#[bench] +fn u256_sub(b: &mut Bencher) { + b.iter(|| { + let n = black_box(10000); + (0..n).fold(U256([rand::random::(), rand::random::(), rand::random::(), rand::random::()]), |old, new| { old.overflowing_sub(U256::from(new)).0 }) + }); +} + +#[bench] +fn u512_sub(b: &mut Bencher) { + b.iter(|| { + let n = black_box(10000); + (0..n).fold(U512([rand::random::(), rand::random::(), rand::random::(), rand::random::(), + rand::random::(), rand::random::(), rand::random::(), rand::random::()]), + |old, new| { old.overflowing_sub(U512([0, 0, 0, 0, 0, 0, 0, new])).0 }) + }); +} + +#[bench] +fn u512_add(b: &mut Bencher) { + b.iter(|| { + let n = black_box(10000); + (0..n).fold(U512([0, 0, 0, 0, 0, 0, 0, 0]), + |old, new| { old.overflowing_add(U512([new, new, new, new, new, new, new, new])).0 }) + }); +} + +#[bench] +fn u256_mul(b: &mut Bencher) { + b.iter(|| { + let n = black_box(10000); + (0..n).fold(U256([rand::random::(), rand::random::(), rand::random::(), rand::random::()]), |old, new| { old.overflowing_mul(U256::from(new)).0 }) + }); +} + + +#[bench] +fn u128_mul(b: &mut Bencher) { + b.iter(|| { + let n = black_box(10000); + (0..n).fold(U128([12345u64, 0u64]), |old, new| { old.overflowing_mul(U128::from(new)).0 }) + }); +} + diff --git a/util/res/geth_keystore/UTC--2016-02-17T09-20-45.721400158Z--3f49624084b67849c7b4e805c5988c21a430f9d9 b/util/res/geth_keystore/UTC--2016-02-17T09-20-45.721400158Z--3f49624084b67849c7b4e805c5988c21a430f9d9 new file mode 100644 index 000000000..afc376774 --- /dev/null +++ b/util/res/geth_keystore/UTC--2016-02-17T09-20-45.721400158Z--3f49624084b67849c7b4e805c5988c21a430f9d9 @@ -0,0 +1,21 @@ +{ + "address": "3f49624084b67849c7b4e805c5988c21a430f9d9", + "Crypto": { + "cipher": "aes-128-ctr", + "ciphertext": "9f27e3dd4fc73e7103ed61e5493662189a3eb52223ae49e3d1deacc04c889eae", + "cipherparams": { + "iv": "457494bf05f2618c397dc74dbb5181c0" + }, + "kdf": "scrypt", + "kdfparams": { + "dklen": 32, + "n": 262144, + "p": 1, + "r": 8, + "salt": "db14edb18c41ee7f5ec4397df89c3a2ae4d0af60884c52bb54ce490574f8df33" + }, + "mac": "572d24532438d31fdf513c744a3ff26c933ffda5744ee42bc71661cbe3f2112e" + }, + "id": "62a0ad73-556d-496a-8e1c-0783d30d3ace", + "version": 3 +} diff --git a/util/res/geth_keystore/UTC--2016-02-20T09-33-03.984382741Z--5ba4dcf897e97c2bdf8315b9ef26c13c085988cf b/util/res/geth_keystore/UTC--2016-02-20T09-33-03.984382741Z--5ba4dcf897e97c2bdf8315b9ef26c13c085988cf new file mode 100644 index 000000000..b14922037 --- /dev/null +++ b/util/res/geth_keystore/UTC--2016-02-20T09-33-03.984382741Z--5ba4dcf897e97c2bdf8315b9ef26c13c085988cf @@ -0,0 +1,21 @@ +{ + "address": "5ba4dcf897e97c2bdf8315b9ef26c13c085988cf", + "Crypto": { + "cipher": "aes-128-ctr", + "ciphertext": "d4a08ec930163778273920f6ad1d49b71836337be6fd9863993ac700a612fddd", + "cipherparams": { + "iv": "89ce5ec129fc27cd5bcbeb8c92bdad50" + }, + "kdf": "scrypt", + "kdfparams": { + "dklen": 32, + "n": 262144, + "p": 1, + "r": 8, + "salt": "612ab108dc37e69ee8af37a7b24bf7f2234086d7bbf945bacdeccce331f7f84a" + }, + "mac": "4152caa7444e06784223d735cea80cd2690b4c587ad8db3d5529442227b25695" + }, + "id": "35086353-fb12-4029-b56b-033cd61ce35b", + "version": 3 +} diff --git a/util/src/crypto.rs b/util/src/crypto.rs index 596265cd0..2d5516b6f 100644 --- a/util/src/crypto.rs +++ b/util/src/crypto.rs @@ -254,7 +254,7 @@ pub mod ecies { use crypto::*; /// Encrypt a message with a public key - pub fn encrypt(public: &Public, plain: &[u8]) -> Result { + pub fn encrypt(public: &Public, shared_mac: &[u8], plain: &[u8]) -> Result { use ::rcrypto::digest::Digest; use ::rcrypto::sha2::Sha256; use ::rcrypto::hmac::Hmac; @@ -284,13 +284,14 @@ pub mod ecies { let cipher_iv = &msgd[64..(64 + 16 + plain.len())]; hmac.input(cipher_iv); } + hmac.input(shared_mac); hmac.raw_result(&mut msgd[(64 + 16 + plain.len())..]); } Ok(msg) } /// Decrypt a message with a secret key - pub fn decrypt(secret: &Secret, encrypted: &[u8]) -> Result { + pub fn decrypt(secret: &Secret, shared_mac: &[u8], encrypted: &[u8]) -> Result { use ::rcrypto::digest::Digest; use ::rcrypto::sha2::Sha256; use ::rcrypto::hmac::Hmac; @@ -322,6 +323,7 @@ pub mod ecies { // Verify tag let mut hmac = Hmac::new(Sha256::new(), &mkey); hmac.input(cipher_with_iv); + hmac.input(shared_mac); let mut mac = H256::new(); hmac.raw_result(&mut mac); if &mac[..] != msg_mac { @@ -405,4 +407,20 @@ mod tests { let pair = KeyPair::from_secret(h256_from_hex("6f7b0d801bc7b5ce7bbd930b84fd0369b3eb25d09be58d64ba811091046f3aa2")).unwrap(); assert_eq!(pair.public().hex(), "101b3ef5a4ea7a1c7928e24c4c75fd053c235d7b80c22ae5c03d145d0ac7396e2a4ffff9adee3133a7b05044a5cee08115fd65145e5165d646bde371010d803c"); } + + #[test] + fn ecies_shared() { + let kp = KeyPair::create().unwrap(); + let message = b"So many books, so little time"; + + let shared = b"shared"; + let wrong_shared = b"incorrect"; + let encrypted = ecies::encrypt(kp.public(), shared, message).unwrap(); + assert!(encrypted[..] != message[..]); + assert_eq!(encrypted[0], 0x04); + + assert!(ecies::decrypt(kp.secret(), wrong_shared, &encrypted).is_err()); + let decrypted = ecies::decrypt(kp.secret(), shared, &encrypted).unwrap(); + assert_eq!(decrypted[..message.len()], message[..]); + } } diff --git a/util/src/hash.rs b/util/src/hash.rs index 71c690ef6..88c57371e 100644 --- a/util/src/hash.rs +++ b/util/src/hash.rs @@ -235,11 +235,11 @@ macro_rules! impl_hash { } impl serde::Serialize for $from { - fn serialize(&self, serializer: &mut S) -> Result<(), S::Error> + fn serialize(&self, serializer: &mut S) -> Result<(), S::Error> where S: serde::Serializer { let mut hex = "0x".to_owned(); hex.push_str(self.to_hex().as_ref()); - serializer.visit_str(hex.as_ref()) + serializer.serialize_str(hex.as_ref()) } } @@ -250,14 +250,14 @@ macro_rules! impl_hash { impl serde::de::Visitor for HashVisitor { type Value = $from; - + fn visit_str(&mut self, value: &str) -> Result where E: serde::Error { // 0x + len if value.len() != 2 + $size * 2 { - return Err(serde::Error::syntax("Invalid length.")); + return Err(serde::Error::custom("Invalid length.")); } - value[2..].from_hex().map(|ref v| $from::from_slice(v)).map_err(|_| serde::Error::syntax("Invalid valid hex.")) + value[2..].from_hex().map(|ref v| $from::from_slice(v)).map_err(|_| serde::Error::custom("Invalid valid hex.")) } fn visit_string(&mut self, value: String) -> Result where E: serde::Error { @@ -265,7 +265,7 @@ macro_rules! impl_hash { } } - deserializer.visit(HashVisitor) + deserializer.deserialize(HashVisitor) } } @@ -719,4 +719,3 @@ mod tests { assert_eq!(r, u); } } - diff --git a/util/src/journaldb.rs b/util/src/journaldb.rs index 7b810639b..cfcff6ea4 100644 --- a/util/src/journaldb.rs +++ b/util/src/journaldb.rs @@ -20,7 +20,7 @@ use common::*; use rlp::*; use hashdb::*; use memorydb::*; -use rocksdb::{DB, Writable, WriteBatch, IteratorMode}; +use kvdb::{Database, DBTransaction, DatabaseConfig}; #[cfg(test)] use std::env; @@ -33,7 +33,7 @@ use std::env; /// the removals actually take effect. pub struct JournalDB { overlay: MemoryDB, - backing: Arc, + backing: Arc, counters: Arc>>, } @@ -47,21 +47,25 @@ impl Clone for JournalDB { } } -const LATEST_ERA_KEY : [u8; 4] = [ b'l', b'a', b's', b't' ]; -const VERSION_KEY : [u8; 4] = [ b'j', b'v', b'e', b'r' ]; +// all keys must be at least 12 bytes +const LATEST_ERA_KEY : [u8; 12] = [ b'l', b'a', b's', b't', 0, 0, 0, 0, 0, 0, 0, 0 ]; +const VERSION_KEY : [u8; 12] = [ b'j', b'v', b'e', b'r', 0, 0, 0, 0, 0, 0, 0, 0 ]; -const DB_VERSION: u32 = 2; +const DB_VERSION: u32 = 3; + +const PADDING : [u8; 10] = [ 0u8; 10 ]; impl JournalDB { - /// Create a new instance given a `backing` database. - pub fn new(backing: DB) -> JournalDB { - let db = Arc::new(backing); - JournalDB::new_with_arc(db) - } - /// Create a new instance given a shared `backing` database. - pub fn new_with_arc(backing: Arc) -> JournalDB { - if backing.iterator(IteratorMode::Start).next().is_some() { + /// Create a new instance from file + pub fn new(path: &str) -> JournalDB { + let opts = DatabaseConfig { + prefix_size: Some(12) //use 12 bytes as prefix, this must match account_db prefix + }; + let backing = Database::open(&opts, path).unwrap_or_else(|e| { + panic!("Error opening state db: {}", e); + }); + if !backing.is_empty() { match backing.get(&VERSION_KEY).map(|d| d.map(|v| decode::(&v))) { Ok(Some(DB_VERSION)) => {}, v => panic!("Incompatible DB version, expected {}, got {:?}", DB_VERSION, v) @@ -72,7 +76,7 @@ impl JournalDB { let counters = JournalDB::read_counters(&backing); JournalDB { overlay: MemoryDB::new(), - backing: backing, + backing: Arc::new(backing), counters: Arc::new(RwLock::new(counters)), } } @@ -82,7 +86,7 @@ impl JournalDB { pub fn new_temp() -> JournalDB { let mut dir = env::temp_dir(); dir.push(H32::random().hex()); - Self::new(DB::open_default(dir.to_str().unwrap()).unwrap()) + Self::new(dir.to_str().unwrap()) } /// Check if this database has any commits @@ -117,16 +121,17 @@ impl JournalDB { // and the key is safe to delete. // record new commit's details. - let batch = WriteBatch::new(); + let batch = DBTransaction::new(); let mut counters = self.counters.write().unwrap(); { let mut index = 0usize; let mut last; while try!(self.backing.get({ - let mut r = RlpStream::new_list(2); + let mut r = RlpStream::new_list(3); r.append(&now); r.append(&index); + r.append(&&PADDING[..]); last = r.drain(); &last })).is_some() { @@ -154,9 +159,10 @@ impl JournalDB { let mut to_remove: Vec = Vec::new(); let mut canon_inserts: Vec = Vec::new(); while let Some(rlp_data) = try!(self.backing.get({ - let mut r = RlpStream::new_list(2); + let mut r = RlpStream::new_list(3); r.append(&end_era); r.append(&index); + r.append(&&PADDING[..]); last = r.drain(); &last })) { @@ -226,16 +232,17 @@ impl JournalDB { self.backing.get(&key.bytes()).expect("Low-level database error. Some issue with your hard disk?").map(|v| v.to_vec()) } - fn read_counters(db: &DB) -> HashMap { + fn read_counters(db: &Database) -> HashMap { let mut res = HashMap::new(); if let Some(val) = db.get(&LATEST_ERA_KEY).expect("Low-level database error.") { let mut era = decode::(&val); loop { let mut index = 0usize; while let Some(rlp_data) = db.get({ - let mut r = RlpStream::new_list(2); + let mut r = RlpStream::new_list(3); r.append(&era); r.append(&index); + r.append(&&PADDING[..]); &r.drain() }).expect("Low-level database error.") { let rlp = Rlp::new(&rlp_data); @@ -259,7 +266,7 @@ impl JournalDB { impl HashDB for JournalDB { fn keys(&self) -> HashMap { let mut ret: HashMap = HashMap::new(); - for (key, _) in self.backing.iterator(IteratorMode::Start) { + for (key, _) in self.backing.iter() { let h = H256::from_slice(key.deref()); ret.insert(h, 1); } @@ -429,12 +436,11 @@ mod tests { #[test] fn reopen() { - use rocksdb::DB; let mut dir = ::std::env::temp_dir(); dir.push(H32::random().hex()); let foo = { - let mut jdb = JournalDB::new(DB::open_default(dir.to_str().unwrap()).unwrap()); + let mut jdb = JournalDB::new(dir.to_str().unwrap()); // history is 1 let foo = jdb.insert(b"foo"); jdb.commit(0, &b"0".sha3(), None).unwrap(); @@ -442,13 +448,13 @@ mod tests { }; { - let mut jdb = JournalDB::new(DB::open_default(dir.to_str().unwrap()).unwrap()); + let mut jdb = JournalDB::new(dir.to_str().unwrap()); jdb.remove(&foo); jdb.commit(1, &b"1".sha3(), Some((0, b"0".sha3()))).unwrap(); } { - let mut jdb = JournalDB::new(DB::open_default(dir.to_str().unwrap()).unwrap()); + let mut jdb = JournalDB::new(dir.to_str().unwrap()); assert!(jdb.exists(&foo)); jdb.commit(2, &b"2".sha3(), Some((1, b"1".sha3()))).unwrap(); assert!(!jdb.exists(&foo)); diff --git a/util/src/keys/directory.rs b/util/src/keys/directory.rs index d7db8ac11..7178508a2 100644 --- a/util/src/keys/directory.rs +++ b/util/src/keys/directory.rs @@ -333,7 +333,9 @@ pub struct KeyFileContent { /// Holds cypher and decrypt function settings. pub crypto: KeyFileCrypto, /// The identifier. - pub id: Uuid + pub id: Uuid, + /// Account (if present) + pub account: Option
, } #[derive(Debug)] @@ -374,7 +376,19 @@ impl KeyFileContent { KeyFileContent { id: new_uuid(), version: KeyFileVersion::V3(3), - crypto: crypto + crypto: crypto, + account: None + } + } + + /// Loads key from valid json, returns error and records warning if key is mallformed + pub fn load(json: &Json) -> Result { + match Self::from_json(json) { + Ok(key_file) => Ok(key_file), + Err(e) => { + warn!(target: "sstore", "Error parsing json for key: {:?}", e); + Err(()) + } } } @@ -407,6 +421,9 @@ impl KeyFileContent { Ok(id) => id }; + let account = as_object.get("address").and_then(|json| json.as_string()).and_then( + |account_text| match Address::from_str(account_text) { Ok(account) => Some(account), Err(_) => None }); + let crypto = match as_object.get("crypto") { None => { return Err(KeyFileParseError::NoCryptoSection); } Some(crypto_json) => match KeyFileCrypto::from_json(crypto_json) { @@ -418,7 +435,8 @@ impl KeyFileContent { Ok(KeyFileContent { version: version, id: id.clone(), - crypto: crypto + crypto: crypto, + account: account }) } @@ -427,6 +445,7 @@ impl KeyFileContent { map.insert("id".to_owned(), Json::String(uuid_to_string(&self.id))); map.insert("version".to_owned(), Json::U64(CURRENT_DECLARED_VERSION)); map.insert("crypto".to_owned(), self.crypto.to_json()); + if let Some(ref address) = self.account { map.insert("address".to_owned(), Json::String(format!("{:?}", address))); } Json::Object(map) } } @@ -599,6 +618,8 @@ impl KeyDirectory { Err(_) => Err(KeyFileLoadError::ParseError(KeyFileParseError::InvalidJson)) } } + + } @@ -653,7 +674,7 @@ mod file_tests { } #[test] - fn can_read_scrypt_krf() { + fn can_read_scrypt_kdf() { let json = Json::from_str( r#" { @@ -689,6 +710,47 @@ mod file_tests { } } + #[test] + fn can_read_scrypt_kdf_params() { + let json = Json::from_str( + r#" + { + "crypto" : { + "cipher" : "aes-128-ctr", + "cipherparams" : { + "iv" : "83dbcc02d8ccb40e466191a123791e0e" + }, + "ciphertext" : "d172bf743a674da9cdad04534d56926ef8358534d458fffccd4e6ad2fbde479c", + "kdf" : "scrypt", + "kdfparams" : { + "dklen" : 32, + "n" : 262144, + "r" : 1, + "p" : 8, + "salt" : "ab0c7876052600dd703518d6fc3fe8984592145b591fc8fb5c6d43190334ba19" + }, + "mac" : "2103ac29920d71da29f15d75b4a16dbe95cfd7ff8faea1056c33131d846e3097" + }, + "id" : "3198bc9c-6672-5ab3-d995-4942343ae5b6", + "version" : 3 + } + "#).unwrap(); + + match KeyFileContent::from_json(&json) { + Ok(key_file) => { + match key_file.crypto.kdf { + KeyFileKdf::Scrypt(scrypt_params) => { + assert_eq!(262144, scrypt_params.n); + assert_eq!(1, scrypt_params.r); + assert_eq!(8, scrypt_params.p); + }, + _ => { panic!("expected kdf params of crypto to be of scrypt type" ); } + } + }, + Err(e) => panic!("Error parsing valid file: {:?}", e) + } + } + #[test] fn can_return_error_no_id() { let json = Json::from_str( @@ -844,7 +906,7 @@ mod file_tests { panic!("Should be error of no identifier, got ok"); }, Err(KeyFileParseError::Crypto(CryptoParseError::Scrypt(_))) => { }, - Err(other_error) => { panic!("should be error of no identifier, got {:?}", other_error); } + Err(other_error) => { panic!("should be scrypt parse error, got {:?}", other_error); } } } diff --git a/util/src/keys/geth_import.rs b/util/src/keys/geth_import.rs new file mode 100644 index 000000000..2ee3c987c --- /dev/null +++ b/util/src/keys/geth_import.rs @@ -0,0 +1,165 @@ +// Copyright 2015, 2016 Ethcore (UK) Ltd. +// This file is part of Parity. + +// Parity is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Parity is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Parity. If not, see . + +//! Geth keys import/export tool + +use common::*; +use keys::store::SecretStore; +use keys::directory::KeyFileContent; + +/// Enumerates all geth keys in the directory and returns collection of tuples `(accountId, filename)` +pub fn enumerate_geth_keys(path: &Path) -> Result, io::Error> { + let mut entries = Vec::new(); + for entry in try!(fs::read_dir(path)) { + let entry = try!(entry); + if !try!(fs::metadata(entry.path())).is_dir() { + match entry.file_name().to_str() { + Some(name) => { + let parts: Vec<&str> = name.split("--").collect(); + if parts.len() != 3 { continue; } + match Address::from_str(parts[2]) { + Ok(account_id) => { entries.push((account_id, name.to_owned())); } + Err(e) => { panic!("error: {:?}", e); } + } + }, + None => { continue; } + }; + } + } + Ok(entries) +} + +/// Geth import error +#[derive(Debug)] +pub enum ImportError { + /// Io error reading geth file + IoError(io::Error), + /// format error + FormatError, +} + +impl From for ImportError { + fn from (err: io::Error) -> ImportError { + ImportError::IoError(err) + } +} + +/// Imports one geth key to the store +pub fn import_geth_key(secret_store: &mut SecretStore, geth_keyfile_path: &Path) -> Result<(), ImportError> { + let mut file = try!(fs::File::open(geth_keyfile_path)); + let mut buf = String::new(); + try!(file.read_to_string(&mut buf)); + + let mut json_result = Json::from_str(&buf); + let mut json = match json_result { + Ok(ref mut parsed_json) => try!(parsed_json.as_object_mut().ok_or(ImportError::FormatError)), + Err(_) => { return Err(ImportError::FormatError); } + }; + let crypto_object = try!(json.get("Crypto").and_then(|crypto| crypto.as_object()).ok_or(ImportError::FormatError)).clone(); + json.insert("crypto".to_owned(), Json::Object(crypto_object)); + json.remove("Crypto"); + match KeyFileContent::load(&Json::Object(json.clone())) { + Ok(key_file) => try!(secret_store.import_key(key_file)), + Err(_) => { return Err(ImportError::FormatError); } + }; + Ok(()) +} + +/// Imports all geth keys in the directory +pub fn import_geth_keys(secret_store: &mut SecretStore, geth_keyfiles_directory: &Path) -> Result<(), ImportError> { + use std::path::PathBuf; + let geth_files = try!(enumerate_geth_keys(geth_keyfiles_directory)); + for &(ref address, ref file_path) in geth_files.iter() { + let mut path = PathBuf::new(); + path.push(geth_keyfiles_directory); + path.push(file_path); + if let Err(e) = import_geth_key(secret_store, Path::new(&path)) { + warn!("Skipped geth address {}, error importing: {:?}", address, e) + } + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + use common::*; + use keys::store::SecretStore; + + + #[test] + fn can_enumerate() { + let keys = enumerate_geth_keys(Path::new("res/geth_keystore")).unwrap(); + assert_eq!(2, keys.len()); + } + + #[test] + fn can_import() { + let temp = ::devtools::RandomTempPath::create_dir(); + let mut secret_store = SecretStore::new_in(temp.as_path()); + import_geth_key(&mut secret_store, Path::new("res/geth_keystore/UTC--2016-02-17T09-20-45.721400158Z--3f49624084b67849c7b4e805c5988c21a430f9d9")).unwrap(); + let key = secret_store.account(&Address::from_str("3f49624084b67849c7b4e805c5988c21a430f9d9").unwrap()); + assert!(key.is_some()); + } + + #[test] + fn can_import_directory() { + let temp = ::devtools::RandomTempPath::create_dir(); + let mut secret_store = SecretStore::new_in(temp.as_path()); + import_geth_keys(&mut secret_store, Path::new("res/geth_keystore")).unwrap(); + + let key = secret_store.account(&Address::from_str("3f49624084b67849c7b4e805c5988c21a430f9d9").unwrap()); + assert!(key.is_some()); + + let key = secret_store.account(&Address::from_str("5ba4dcf897e97c2bdf8315b9ef26c13c085988cf").unwrap()); + assert!(key.is_some()); + } + + #[test] + fn imports_as_scrypt_keys() { + use keys::directory::{KeyDirectory, KeyFileKdf}; + let temp = ::devtools::RandomTempPath::create_dir(); + { + let mut secret_store = SecretStore::new_in(temp.as_path()); + import_geth_keys(&mut secret_store, Path::new("res/geth_keystore")).unwrap(); + } + + let key_directory = KeyDirectory::new(&temp.as_path()); + let key_file = key_directory.get(&H128::from_str("62a0ad73556d496a8e1c0783d30d3ace").unwrap()).unwrap(); + + match key_file.crypto.kdf { + KeyFileKdf::Scrypt(scrypt_params) => { + assert_eq!(262144, scrypt_params.n); + assert_eq!(8, scrypt_params.r); + assert_eq!(1, scrypt_params.p); + }, + _ => { panic!("expected kdf params of crypto to be of scrypt type" ); } + } + } + + #[test] + fn can_decrypt_with_imported() { + use keys::store::EncryptedHashMap; + + let temp = ::devtools::RandomTempPath::create_dir(); + let mut secret_store = SecretStore::new_in(temp.as_path()); + import_geth_keys(&mut secret_store, Path::new("res/geth_keystore")).unwrap(); + + let val = secret_store.get::(&H128::from_str("62a0ad73556d496a8e1c0783d30d3ace").unwrap(), "123"); + assert!(val.is_ok()); + assert_eq!(32, val.unwrap().len()); + } +} diff --git a/util/src/keys/mod.rs b/util/src/keys/mod.rs index fd52136d7..b9c9dcb08 100644 --- a/util/src/keys/mod.rs +++ b/util/src/keys/mod.rs @@ -18,3 +18,4 @@ pub mod directory; pub mod store; +mod geth_import; diff --git a/util/src/keys/store.rs b/util/src/keys/store.rs index ae44d567a..c4fa377f9 100644 --- a/util/src/keys/store.rs +++ b/util/src/keys/store.rs @@ -19,11 +19,12 @@ use keys::directory::*; use common::*; use rcrypto::pbkdf2::*; +use rcrypto::scrypt::*; use rcrypto::hmac::*; use crypto; const KEY_LENGTH: u32 = 32; -const KEY_ITERATIONS: u32 = 4096; +const KEY_ITERATIONS: u32 = 10240; const KEY_LENGTH_AES: u32 = KEY_LENGTH/2; const KEY_LENGTH_USIZE: usize = KEY_LENGTH as usize; @@ -60,15 +61,62 @@ pub struct SecretStore { } impl SecretStore { - /// new instance of Secret Store + /// new instance of Secret Store in default home directory pub fn new() -> SecretStore { let mut path = ::std::env::home_dir().expect("Failed to get home dir"); - path.push(".keys"); + path.push(".parity"); + path.push("keys"); + Self::new_in(&path) + } + + /// new instance of Secret Store in specific directory + pub fn new_in(path: &Path) -> SecretStore { SecretStore { - directory: KeyDirectory::new(&path) + directory: KeyDirectory::new(path) } } + /// trys to import keys in the known locations + pub fn try_import_existing(&mut self) { + use std::path::PathBuf; + use keys::geth_import; + + let mut import_path = PathBuf::new(); + import_path.push(::std::env::home_dir().expect("Failed to get home dir")); + import_path.push(".ethereum"); + import_path.push("keystore"); + if let Err(e) = geth_import::import_geth_keys(self, &import_path) { + warn!(target: "sstore", "Error retrieving geth keys: {:?}", e) + } + } + + /// Lists all accounts and corresponding key ids + pub fn accounts(&self) -> Result, ::std::io::Error> { + let accounts = try!(self.directory.list()).iter().map(|key_id| self.directory.get(key_id)) + .filter(|key| key.is_some()) + .map(|key| { let some_key = key.unwrap(); (some_key.account, some_key.id) }) + .filter(|&(ref account, _)| account.is_some()) + .map(|(account, id)| (account.unwrap(), id)) + .collect::>(); + Ok(accounts) + } + + /// Resolves key_id by account address + pub fn account(&self, account: &Address) -> Option { + let mut accounts = match self.accounts() { + Ok(accounts) => accounts, + Err(e) => { warn!(target: "sstore", "Failed to load accounts: {}", e); return None; } + }; + accounts.retain(|&(ref store_account, _)| account == store_account); + accounts.first().and_then(|&(_, ref key_id)| Some(key_id.clone())) + } + + /// Imports pregenerated key, returns error if not saved correctly + pub fn import_key(&mut self, key_file: KeyFileContent) -> Result<(), ::std::io::Error> { + try!(self.directory.save(key_file)); + Ok(()) + } + #[cfg(test)] fn new_test(path: &::devtools::RandomTempPath) -> SecretStore { SecretStore { @@ -90,6 +138,15 @@ fn derive_key(password: &str, salt: &H256) -> (Bytes, Bytes) { derive_key_iterations(password, salt, KEY_ITERATIONS) } +fn derive_key_scrypt(password: &str, salt: &H256, n: u32, p: u32, r: u32) -> (Bytes, Bytes) { + let mut derived_key = vec![0u8; KEY_LENGTH_USIZE]; + let scrypt_params = ScryptParams::new(n.trailing_zeros() as u8, r, p); + scrypt(password.as_bytes(), &salt.as_slice(), &scrypt_params, &mut derived_key); + let derived_right_bits = &derived_key[0..KEY_LENGTH_AES_USIZE]; + let derived_left_bits = &derived_key[KEY_LENGTH_AES_USIZE..KEY_LENGTH_USIZE]; + (derived_right_bits.to_vec(), derived_left_bits.to_vec()) +} + fn derive_mac(derived_left_bits: &[u8], cipher_text: &[u8]) -> Bytes { let mut mac = vec![0u8; KEY_LENGTH_AES_USIZE + cipher_text.len()]; mac[0..KEY_LENGTH_AES_USIZE].clone_from_slice(derived_left_bits); @@ -101,24 +158,22 @@ impl EncryptedHashMap for SecretStore { fn get(&self, key: &H128, password: &str) -> Result { match self.directory.get(key) { Some(key_file) => { - let decrypted_bytes = match key_file.crypto.kdf { - KeyFileKdf::Pbkdf2(ref params) => { - let (derived_left_bits, derived_right_bits) = derive_key_iterations(password, ¶ms.salt, params.c); - if derive_mac(&derived_right_bits, &key_file.crypto.cipher_text) - .sha3() != key_file.crypto.mac { return Err(EncryptedHashMapError::InvalidPassword); } - - let mut val = vec![0u8; key_file.crypto.cipher_text.len()]; - match key_file.crypto.cipher_type { - CryptoCipherType::Aes128Ctr(ref iv) => { - crypto::aes::decrypt(&derived_left_bits, &iv.as_slice(), &key_file.crypto.cipher_text, &mut val); - } - } - val - } - _ => { unimplemented!(); } + let (derived_left_bits, derived_right_bits) = match key_file.crypto.kdf { + KeyFileKdf::Pbkdf2(ref params) => derive_key_iterations(password, ¶ms.salt, params.c), + KeyFileKdf::Scrypt(ref params) => derive_key_scrypt(password, ¶ms.salt, params.n, params.p, params.r) }; - match Value::from_bytes(&decrypted_bytes) { + if derive_mac(&derived_right_bits, &key_file.crypto.cipher_text) + .sha3() != key_file.crypto.mac { return Err(EncryptedHashMapError::InvalidPassword); } + + let mut val = vec![0u8; key_file.crypto.cipher_text.len()]; + match key_file.crypto.cipher_type { + CryptoCipherType::Aes128Ctr(ref iv) => { + crypto::aes::decrypt(&derived_left_bits, &iv.as_slice(), &key_file.crypto.cipher_text, &mut val); + } + }; + + match Value::from_bytes(&val) { Ok(value) => Ok(value), Err(bytes_error) => Err(EncryptedHashMapError::InvalidValueFormat(bytes_error)) } @@ -259,6 +314,27 @@ mod tests { result } + fn pregenerate_accounts(temp: &RandomTempPath, count: usize) -> Vec { + use keys::directory::{KeyFileContent, KeyFileCrypto}; + let mut write_sstore = SecretStore::new_test(&temp); + let mut result = Vec::new(); + for i in 0..count { + let mut key_file = + KeyFileContent::new( + KeyFileCrypto::new_pbkdf2( + FromHex::from_hex("5318b4d5bcd28de64ee5559e671353e16f075ecae9f99c7a79a38af5f869aa46").unwrap(), + H128::from_str("6087dab2f9fdbbfaddc31a909735c1e6").unwrap(), + H256::from_str("ae3cd4e7013836a3df6bd7241b12db061dbe2c6785853cce422d148a624ce0bd").unwrap(), + H256::from_str("517ead924a9d0dc3124507e3393d175ce3ff7c1e96529c6c555ce9e51205e9b2").unwrap(), + 262144, + 32)); + key_file.account = Some(x!(i as u64)); + result.push(key_file.id.clone()); + write_sstore.import_key(key_file).unwrap(); + } + result + } + #[test] fn can_get() { let temp = RandomTempPath::create_dir(); @@ -293,5 +369,35 @@ mod tests { assert_eq!(4, sstore.directory.list().unwrap().len()) } + #[test] + fn can_import_account() { + use keys::directory::{KeyFileContent, KeyFileCrypto}; + let temp = RandomTempPath::create_dir(); + let mut key_file = + KeyFileContent::new( + KeyFileCrypto::new_pbkdf2( + FromHex::from_hex("5318b4d5bcd28de64ee5559e671353e16f075ecae9f99c7a79a38af5f869aa46").unwrap(), + H128::from_str("6087dab2f9fdbbfaddc31a909735c1e6").unwrap(), + H256::from_str("ae3cd4e7013836a3df6bd7241b12db061dbe2c6785853cce422d148a624ce0bd").unwrap(), + H256::from_str("517ead924a9d0dc3124507e3393d175ce3ff7c1e96529c6c555ce9e51205e9b2").unwrap(), + 262144, + 32)); + key_file.account = Some(Address::from_str("3f49624084b67849c7b4e805c5988c21a430f9d9").unwrap()); + let mut sstore = SecretStore::new_test(&temp); + + sstore.import_key(key_file).unwrap(); + + assert_eq!(1, sstore.accounts().unwrap().len()); + assert!(sstore.account(&Address::from_str("3f49624084b67849c7b4e805c5988c21a430f9d9").unwrap()).is_some()); + } + + #[test] + fn can_list_accounts() { + let temp = RandomTempPath::create_dir(); + pregenerate_accounts(&temp, 30); + let sstore = SecretStore::new_test(&temp); + let accounts = sstore.accounts().unwrap(); + assert_eq!(30, accounts.len()); + } } diff --git a/util/src/kvdb.rs b/util/src/kvdb.rs new file mode 100644 index 000000000..43a9fc532 --- /dev/null +++ b/util/src/kvdb.rs @@ -0,0 +1,206 @@ +// Copyright 2015, 2016 Ethcore (UK) Ltd. +// This file is part of Parity. + +// Parity is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Parity is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Parity. If not, see . + +//! Key-Value store abstraction with RocksDB backend. + +use rocksdb::{DB, Writable, WriteBatch, IteratorMode, DBVector, DBIterator, + IndexType, Options, DBCompactionStyle, BlockBasedOptions, Direction}; + +/// Write transaction. Batches a sequence of put/delete operations for efficiency. +pub struct DBTransaction { + batch: WriteBatch, +} + +impl DBTransaction { + /// Create new transaction. + pub fn new() -> DBTransaction { + DBTransaction { batch: WriteBatch::new() } + } + + /// Insert a key-value pair in the transaction. Any existing value value will be overwritten upon write. + pub fn put(&self, key: &[u8], value: &[u8]) -> Result<(), String> { + self.batch.put(key, value) + } + + /// Delete value by key. + pub fn delete(&self, key: &[u8]) -> Result<(), String> { + self.batch.delete(key) + } +} + +/// Database configuration +pub struct DatabaseConfig { + /// Optional prefix size in bytes. Allows lookup by partial key. + pub prefix_size: Option +} + +/// Database iterator +pub struct DatabaseIterator<'a> { + iter: DBIterator<'a>, +} + +impl<'a> Iterator for DatabaseIterator<'a> { + type Item = (Box<[u8]>, Box<[u8]>); + + #[cfg_attr(feature="dev", allow(type_complexity))] + fn next(&mut self) -> Option<(Box<[u8]>, Box<[u8]>)> { + self.iter.next() + } +} + +/// Key-Value database. +pub struct Database { + db: DB, +} + +impl Database { + /// Open database with default settings. + pub fn open_default(path: &str) -> Result { + Database::open(&DatabaseConfig { prefix_size: None }, path) + } + + /// Open database file. Creates if it does not exist. + pub fn open(config: &DatabaseConfig, path: &str) -> Result { + let mut opts = Options::new(); + opts.set_max_open_files(256); + opts.create_if_missing(true); + opts.set_use_fsync(false); + opts.set_compaction_style(DBCompactionStyle::DBUniversalCompaction); + /* + opts.set_bytes_per_sync(8388608); + opts.set_disable_data_sync(false); + opts.set_block_cache_size_mb(1024); + opts.set_table_cache_num_shard_bits(6); + opts.set_max_write_buffer_number(32); + opts.set_write_buffer_size(536870912); + opts.set_target_file_size_base(1073741824); + opts.set_min_write_buffer_number_to_merge(4); + opts.set_level_zero_stop_writes_trigger(2000); + opts.set_level_zero_slowdown_writes_trigger(0); + opts.set_compaction_style(DBUniversalCompaction); + opts.set_max_background_compactions(4); + opts.set_max_background_flushes(4); + opts.set_filter_deletes(false); + opts.set_disable_auto_compactions(false); + */ + + if let Some(size) = config.prefix_size { + let mut block_opts = BlockBasedOptions::new(); + block_opts.set_index_type(IndexType::HashSearch); + opts.set_block_based_table_factory(&block_opts); + opts.set_prefix_extractor_fixed_size(size); + } + let db = try!(DB::open(&opts, path)); + Ok(Database { db: db }) + } + + /// Insert a key-value pair in the transaction. Any existing value value will be overwritten. + pub fn put(&self, key: &[u8], value: &[u8]) -> Result<(), String> { + self.db.put(key, value) + } + + /// Delete value by key. + pub fn delete(&self, key: &[u8]) -> Result<(), String> { + self.db.delete(key) + } + + /// Commit transaction to database. + pub fn write(&self, tr: DBTransaction) -> Result<(), String> { + self.db.write(tr.batch) + } + + /// Get value by key. + pub fn get(&self, key: &[u8]) -> Result, String> { + self.db.get(key) + } + + /// Get value by partial key. Prefix size should match configured prefix size. + pub fn get_by_prefix(&self, prefix: &[u8]) -> Option> { + let mut iter = self.db.iterator(IteratorMode::From(prefix, Direction::forward)); + match iter.next() { + // TODO: use prefix_same_as_start read option (not availabele in C API currently) + Some((k, v)) => if k[0 .. prefix.len()] == prefix[..] { Some(v) } else { None }, + _ => None + } + } + + /// Check if there is anything in the database. + pub fn is_empty(&self) -> bool { + self.db.iterator(IteratorMode::Start).next().is_none() + } + + /// Check if there is anything in the database. + pub fn iter(&self) -> DatabaseIterator { + DatabaseIterator { iter: self.db.iterator(IteratorMode::Start) } + } +} + +#[cfg(test)] +mod tests { + use hash::*; + use super::*; + use devtools::*; + use std::str::FromStr; + use std::ops::Deref; + + fn test_db(config: &DatabaseConfig) { + let path = RandomTempPath::create_dir(); + let db = Database::open(config, path.as_path().to_str().unwrap()).unwrap(); + let key1 = H256::from_str("02c69be41d0b7e40352fc85be1cd65eb03d40ef8427a0ca4596b1ead9a00e9fc").unwrap(); + let key2 = H256::from_str("03c69be41d0b7e40352fc85be1cd65eb03d40ef8427a0ca4596b1ead9a00e9fc").unwrap(); + let key3 = H256::from_str("01c69be41d0b7e40352fc85be1cd65eb03d40ef8427a0ca4596b1ead9a00e9fc").unwrap(); + + db.put(&key1, b"cat").unwrap(); + db.put(&key2, b"dog").unwrap(); + + assert_eq!(db.get(&key1).unwrap().unwrap().deref(), b"cat"); + + let contents: Vec<_> = db.iter().collect(); + assert_eq!(contents.len(), 2); + assert_eq!(&*contents[0].0, key1.deref()); + assert_eq!(&*contents[0].1, b"cat"); + assert_eq!(&*contents[1].0, key2.deref()); + assert_eq!(&*contents[1].1, b"dog"); + + db.delete(&key1).unwrap(); + assert!(db.get(&key1).unwrap().is_none()); + db.put(&key1, b"cat").unwrap(); + + let transaction = DBTransaction::new(); + transaction.put(&key3, b"elephant").unwrap(); + transaction.delete(&key1).unwrap(); + db.write(transaction).unwrap(); + assert!(db.get(&key1).unwrap().is_none()); + assert_eq!(db.get(&key3).unwrap().unwrap().deref(), b"elephant"); + + if config.prefix_size.is_some() { + assert_eq!(db.get_by_prefix(&key3).unwrap().deref(), b"elephant"); + assert_eq!(db.get_by_prefix(&key2).unwrap().deref(), b"dog"); + } + } + + #[test] + fn kvdb() { + let path = RandomTempPath::create_dir(); + let smoke = Database::open_default(path.as_path().to_str().unwrap()).unwrap(); + assert!(smoke.is_empty()); + test_db(&DatabaseConfig { prefix_size: None }); + test_db(&DatabaseConfig { prefix_size: Some(1) }); + test_db(&DatabaseConfig { prefix_size: Some(8) }); + test_db(&DatabaseConfig { prefix_size: Some(32) }); + } +} + diff --git a/util/src/lib.rs b/util/src/lib.rs index 1fba48e00..be1e788c9 100644 --- a/util/src/lib.rs +++ b/util/src/lib.rs @@ -16,6 +16,7 @@ #![warn(missing_docs)] #![cfg_attr(feature="dev", feature(plugin))] +#![cfg_attr(feature="x64asm", feature(asm))] #![cfg_attr(feature="dev", plugin(clippy))] // Clippy settings @@ -129,6 +130,7 @@ pub mod hashdb; pub mod memorydb; pub mod overlaydb; pub mod journaldb; +pub mod kvdb; mod math; pub mod crypto; pub mod triehash; @@ -161,4 +163,5 @@ pub use semantic_version::*; pub use network::*; pub use io::*; pub use log::*; +pub use kvdb::*; diff --git a/util/src/network/connection.rs b/util/src/network/connection.rs index a7396ff33..55e688c91 100644 --- a/util/src/network/connection.rs +++ b/util/src/network/connection.rs @@ -279,7 +279,7 @@ impl EncryptedConnection { /// Create an encrypted connection out of the handshake. Consumes a handshake object. pub fn new(handshake: &mut Handshake) -> Result { - let shared = try!(crypto::ecdh::agree(handshake.ecdhe.secret(), &handshake.remote_public)); + let shared = try!(crypto::ecdh::agree(handshake.ecdhe.secret(), &handshake.remote_ephemeral)); let mut nonce_material = H512::new(); if handshake.originated { handshake.remote_nonce.copy_to(&mut nonce_material[0..32]); diff --git a/util/src/network/discovery.rs b/util/src/network/discovery.rs index e2fd1c269..e52d5d25f 100644 --- a/util/src/network/discovery.rs +++ b/util/src/network/discovery.rs @@ -85,7 +85,8 @@ pub struct Discovery { discovery_id: NodeId, discovery_nodes: HashSet, node_buckets: Vec, - send_queue: VecDeque + send_queue: VecDeque, + check_timestamps: bool, } pub struct TableUpdates { @@ -107,6 +108,7 @@ impl Discovery { node_buckets: (0..NODE_BINS).map(|_| NodeBucket::new()).collect(), udp_socket: socket, send_queue: VecDeque::new(), + check_timestamps: true, } } @@ -344,20 +346,20 @@ impl Discovery { } } + fn check_timestamp(&self, timestamp: u64) -> Result<(), NetworkError> { + if self.check_timestamps && timestamp < time::get_time().sec as u64{ + debug!(target: "discovery", "Expired packet"); + return Err(NetworkError::Expired); + } + Ok(()) + } + fn on_ping(&mut self, rlp: &UntrustedRlp, node: &NodeId, from: &SocketAddr) -> Result, NetworkError> { trace!(target: "discovery", "Got Ping from {:?}", &from); - let version: u32 = try!(rlp.val_at(0)); - if version != PROTOCOL_VERSION { - debug!(target: "discovery", "Unexpected protocol version: {}", version); - return Err(NetworkError::BadProtocol); - } let source = try!(NodeEndpoint::from_rlp(&try!(rlp.at(1)))); let dest = try!(NodeEndpoint::from_rlp(&try!(rlp.at(2)))); let timestamp: u64 = try!(rlp.val_at(3)); - if timestamp < time::get_time().sec as u64{ - debug!(target: "discovery", "Expired ping"); - return Err(NetworkError::Expired); - } + try!(self.check_timestamp(timestamp)); let mut added_map = HashMap::new(); let entry = NodeEntry { id: node.clone(), endpoint: source.clone() }; if !entry.endpoint.is_valid() || !entry.endpoint.is_global() { @@ -381,9 +383,7 @@ impl Discovery { // TODO: validate pong packet let dest = try!(NodeEndpoint::from_rlp(&try!(rlp.at(0)))); let timestamp: u64 = try!(rlp.val_at(2)); - if timestamp < time::get_time().sec as u64 { - return Err(NetworkError::Expired); - } + try!(self.check_timestamp(timestamp)); let mut entry = NodeEntry { id: node.clone(), endpoint: dest }; if !entry.endpoint.is_valid() { debug!(target: "discovery", "Bad address: {:?}", entry); @@ -399,10 +399,7 @@ impl Discovery { trace!(target: "discovery", "Got FindNode from {:?}", &from); let target: NodeId = try!(rlp.val_at(0)); let timestamp: u64 = try!(rlp.val_at(1)); - if timestamp < time::get_time().sec as u64 { - return Err(NetworkError::Expired); - } - + try!(self.check_timestamp(timestamp)); let limit = (MAX_DATAGRAM_SIZE - 109) / 90; let nearest = Discovery::nearest_node_entries(&target, &self.node_buckets); if nearest.is_empty() { @@ -501,6 +498,7 @@ mod tests { use network::node_table::*; use crypto::KeyPair; use std::str::FromStr; + use rustc_serialize::hex::FromHex; #[test] fn discovery() { @@ -540,7 +538,7 @@ mod tests { #[test] fn removes_expired() { let key = KeyPair::create().unwrap(); - let ep = NodeEndpoint { address: SocketAddr::from_str("127.0.0.1:40446").unwrap(), udp_port: 40444 }; + let ep = NodeEndpoint { address: SocketAddr::from_str("127.0.0.1:40446").unwrap(), udp_port: 40447 }; let mut discovery = Discovery::new(&key, ep.address.clone(), ep.clone(), 0); for _ in 0..1200 { discovery.add_node(NodeEntry { id: NodeId::random(), endpoint: ep.clone() }); @@ -549,4 +547,70 @@ mod tests { let removed = discovery.check_expired(true).len(); assert!(removed > 0); } + + #[test] + fn packets() { + let key = KeyPair::create().unwrap(); + let ep = NodeEndpoint { address: SocketAddr::from_str("127.0.0.1:40447").unwrap(), udp_port: 40447 }; + let mut discovery = Discovery::new(&key, ep.address.clone(), ep.clone(), 0); + discovery.check_timestamps = false; + let from = SocketAddr::from_str("99.99.99.99:40445").unwrap(); + + let packet = "\ + e9614ccfd9fc3e74360018522d30e1419a143407ffcce748de3e22116b7e8dc92ff74788c0b6663a\ + aa3d67d641936511c8f8d6ad8698b820a7cf9e1be7155e9a241f556658c55428ec0563514365799a\ + 4be2be5a685a80971ddcfa80cb422cdd0101ec04cb847f000001820cfa8215a8d790000000000000\ + 000000000000000000018208ae820d058443b9a3550102\ + ".from_hex().unwrap(); + assert!(discovery.on_packet(&packet, from.clone()).is_ok()); + + let packet = "\ + 577be4349c4dd26768081f58de4c6f375a7a22f3f7adda654d1428637412c3d7fe917cadc56d4e5e\ + 7ffae1dbe3efffb9849feb71b262de37977e7c7a44e677295680e9e38ab26bee2fcbae207fba3ff3\ + d74069a50b902a82c9903ed37cc993c50001f83e82022bd79020010db83c4d001500000000abcdef\ + 12820cfa8215a8d79020010db885a308d313198a2e037073488208ae82823a8443b9a355c5010203\ + 040531b9019afde696e582a78fa8d95ea13ce3297d4afb8ba6433e4154caa5ac6431af1b80ba7602\ + 3fa4090c408f6b4bc3701562c031041d4702971d102c9ab7fa5eed4cd6bab8f7af956f7d565ee191\ + 7084a95398b6a21eac920fe3dd1345ec0a7ef39367ee69ddf092cbfe5b93e5e568ebc491983c09c7\ + 6d922dc3\ + ".from_hex().unwrap(); + assert!(discovery.on_packet(&packet, from.clone()).is_ok()); + + let packet = "\ + 09b2428d83348d27cdf7064ad9024f526cebc19e4958f0fdad87c15eb598dd61d08423e0bf66b206\ + 9869e1724125f820d851c136684082774f870e614d95a2855d000f05d1648b2d5945470bc187c2d2\ + 216fbe870f43ed0909009882e176a46b0102f846d79020010db885a308d313198a2e037073488208\ + ae82823aa0fbc914b16819237dcd8801d7e53f69e9719adecb3cc0e790c57e91ca4461c9548443b9\ + a355c6010203c2040506a0c969a58f6f9095004c0177a6b47f451530cab38966a25cca5cb58f0555 + 42124e\ + ".from_hex().unwrap(); + assert!(discovery.on_packet(&packet, from.clone()).is_ok()); + + let packet = "\ + c7c44041b9f7c7e41934417ebac9a8e1a4c6298f74553f2fcfdcae6ed6fe53163eb3d2b52e39fe91\ + 831b8a927bf4fc222c3902202027e5e9eb812195f95d20061ef5cd31d502e47ecb61183f74a504fe\ + 04c51e73df81f25c4d506b26db4517490103f84eb840ca634cae0d49acb401d8a4c6b6fe8c55b70d\ + 115bf400769cc1400f3258cd31387574077f301b421bc84df7266c44e9e6d569fc56be0081290476\ + 7bf5ccd1fc7f8443b9a35582999983999999280dc62cc8255c73471e0a61da0c89acdc0e035e260a\ + dd7fc0c04ad9ebf3919644c91cb247affc82b69bd2ca235c71eab8e49737c937a2c396\ + ".from_hex().unwrap(); + assert!(discovery.on_packet(&packet, from.clone()).is_ok()); + + let packet = "\ + c679fc8fe0b8b12f06577f2e802d34f6fa257e6137a995f6f4cbfc9ee50ed3710faf6e66f932c4c8\ + d81d64343f429651328758b47d3dbc02c4042f0fff6946a50f4a49037a72bb550f3a7872363a83e1\ + b9ee6469856c24eb4ef80b7535bcf99c0004f9015bf90150f84d846321163782115c82115db84031\ + 55e1427f85f10a5c9a7755877748041af1bcd8d474ec065eb33df57a97babf54bfd2103575fa8291\ + 15d224c523596b401065a97f74010610fce76382c0bf32f84984010203040101b840312c55512422\ + cf9b8a4097e9a6ad79402e87a15ae909a4bfefa22398f03d20951933beea1e4dfa6f968212385e82\ + 9f04c2d314fc2d4e255e0d3bc08792b069dbf8599020010db83c4d001500000000abcdef12820d05\ + 820d05b84038643200b172dcfef857492156971f0e6aa2c538d8b74010f8e140811d53b98c765dd2\ + d96126051913f44582e8c199ad7c6d6819e9a56483f637feaac9448aacf8599020010db885a308d3\ + 13198a2e037073488203e78203e8b8408dcab8618c3253b558d459da53bd8fa68935a719aff8b811\ + 197101a4b2b47dd2d47295286fc00cc081bb542d760717d1bdd6bec2c37cd72eca367d6dd3b9df73\ + 8443b9a355010203b525a138aa34383fec3d2719a0\ + ".from_hex().unwrap(); + assert!(discovery.on_packet(&packet, from.clone()).is_ok()); + } + } diff --git a/util/src/network/handshake.rs b/util/src/network/handshake.rs index 087de63b3..cca133ba9 100644 --- a/util/src/network/handshake.rs +++ b/util/src/network/handshake.rs @@ -15,9 +15,11 @@ // along with Parity. If not, see . use std::sync::Arc; +use rand::random; use mio::*; use mio::tcp::*; use hash::*; +use rlp::*; use sha3::Hashable; use bytes::Bytes; use crypto::*; @@ -36,8 +38,12 @@ enum HandshakeState { New, /// Waiting for auth packet ReadingAuth, + /// Waiting for extended auth packet + ReadingAuthEip8, /// Waiting for ack packet ReadingAck, + /// Waiting for extended ack packet + ReadingAckEip8, /// Ready to start a session StartSession, } @@ -57,9 +63,11 @@ pub struct Handshake { /// Connection nonce pub nonce: H256, /// Handshake public key - pub remote_public: Public, + pub remote_ephemeral: Public, /// Remote connection nonce. pub remote_nonce: H256, + /// Remote RLPx protocol version. + pub remote_version: u64, /// A copy of received encryped auth packet pub auth_cipher: Bytes, /// A copy of received encryped ack packet @@ -68,9 +76,12 @@ pub struct Handshake { pub expired: bool, } -const AUTH_PACKET_SIZE: usize = 307; -const ACK_PACKET_SIZE: usize = 210; +const V4_AUTH_PACKET_SIZE: usize = 307; +const V4_ACK_PACKET_SIZE: usize = 210; const HANDSHAKE_TIMEOUT: u64 = 5000; +const PROTOCOL_VERSION: u64 = 4; +// Amount of bytes added when encrypting with encryptECIES. +const ECIES_OVERHEAD: usize = 113; impl Handshake { /// Create a new handshake object @@ -82,8 +93,9 @@ impl Handshake { state: HandshakeState::New, ecdhe: try!(KeyPair::create()), nonce: nonce.clone(), - remote_public: Public::new(), + remote_ephemeral: Public::new(), remote_nonce: H256::new(), + remote_version: PROTOCOL_VERSION, auth_cipher: Bytes::new(), ack_cipher: Bytes::new(), expired: false, @@ -115,11 +127,11 @@ impl Handshake { self.originated = originated; io.register_timer(self.connection.token, HANDSHAKE_TIMEOUT).ok(); if originated { - try!(self.write_auth(host)); + try!(self.write_auth(host.secret(), host.id())); } else { self.state = HandshakeState::ReadingAuth; - self.connection.expect(AUTH_PACKET_SIZE); + self.connection.expect(V4_AUTH_PACKET_SIZE); }; Ok(()) } @@ -134,20 +146,28 @@ impl Handshake { if !self.expired() { io.clear_timer(self.connection.token).unwrap(); match self.state { + HandshakeState::New => {} HandshakeState::ReadingAuth => { if let Some(data) = try!(self.connection.readable()) { - try!(self.read_auth(host, &data)); - try!(self.write_ack()); + try!(self.read_auth(host.secret(), &data)); + }; + }, + HandshakeState::ReadingAuthEip8 => { + if let Some(data) = try!(self.connection.readable()) { + try!(self.read_auth_eip8(host.secret(), &data)); }; }, HandshakeState::ReadingAck => { if let Some(data) = try!(self.connection.readable()) { - try!(self.read_ack(host, &data)); - self.state = HandshakeState::StartSession; + try!(self.read_ack(host.secret(), &data)); + }; + }, + HandshakeState::ReadingAckEip8 => { + if let Some(data) = try!(self.connection.readable()) { + try!(self.read_ack_eip8(host.secret(), &data)); }; }, HandshakeState::StartSession => {}, - _ => { panic!("Unexpected state"); } } if self.state != HandshakeState::StartSession { try!(io.update_registration(self.connection.token)); @@ -190,48 +210,105 @@ impl Handshake { Ok(()) } + fn set_auth(&mut self, host_secret: &Secret, sig: &[u8], remote_public: &[u8], remote_nonce: &[u8], remote_version: u64) -> Result<(), UtilError> { + self.id.clone_from_slice(remote_public); + self.remote_nonce.clone_from_slice(remote_nonce); + self.remote_version = remote_version; + let shared = try!(ecdh::agree(host_secret, &self.id)); + let signature = Signature::from_slice(sig); + self.remote_ephemeral = try!(ec::recover(&signature, &(&shared ^ &self.remote_nonce))); + Ok(()) + } + /// Parse, validate and confirm auth message - fn read_auth(&mut self, host: &HostInfo, data: &[u8]) -> Result<(), UtilError> { - trace!(target:"net", "Received handshake auth to {:?}", self.connection.socket.peer_addr()); - if data.len() != AUTH_PACKET_SIZE { + fn read_auth(&mut self, secret: &Secret, data: &[u8]) -> Result<(), UtilError> { + trace!(target:"net", "Received handshake auth from {:?}", self.connection.socket.peer_addr()); + if data.len() != V4_AUTH_PACKET_SIZE { debug!(target:"net", "Wrong auth packet size"); return Err(From::from(NetworkError::BadProtocol)); } self.auth_cipher = data.to_vec(); - let auth = try!(ecies::decrypt(host.secret(), data)); - let (sig, rest) = auth.split_at(65); - let (hepubk, rest) = rest.split_at(32); - let (pubk, rest) = rest.split_at(64); - let (nonce, _) = rest.split_at(32); - self.id.clone_from_slice(pubk); - self.remote_nonce.clone_from_slice(nonce); - let shared = try!(ecdh::agree(host.secret(), &self.id)); - let signature = Signature::from_slice(sig); - let spub = try!(ec::recover(&signature, &(&shared ^ &self.remote_nonce))); - self.remote_public = spub.clone(); - if &spub.sha3()[..] != hepubk { - trace!(target:"net", "Handshake hash mismath with {:?}", self.connection.socket.peer_addr()); - return Err(From::from(NetworkError::Auth)); - }; + match ecies::decrypt(secret, &[], data) { + Ok(auth) => { + let (sig, rest) = auth.split_at(65); + let (_, rest) = rest.split_at(32); + let (pubk, rest) = rest.split_at(64); + let (nonce, _) = rest.split_at(32); + try!(self.set_auth(secret, sig, pubk, nonce, PROTOCOL_VERSION)); + try!(self.write_ack()); + } + Err(_) => { + // Try to interpret as EIP-8 packet + let total = (((data[0] as u16) << 8 | (data[1] as u16)) as usize) + 2; + if total < V4_AUTH_PACKET_SIZE { + debug!(target:"net", "Wrong EIP8 auth packet size"); + return Err(From::from(NetworkError::BadProtocol)); + } + let rest = total - data.len(); + self.state = HandshakeState::ReadingAuthEip8; + self.connection.expect(rest); + } + } + Ok(()) + } + + fn read_auth_eip8(&mut self, secret: &Secret, data: &[u8]) -> Result<(), UtilError> { + trace!(target:"net", "Received EIP8 handshake auth from {:?}", self.connection.socket.peer_addr()); + self.auth_cipher.extend_from_slice(data); + let auth = try!(ecies::decrypt(secret, &self.auth_cipher[0..2], &self.auth_cipher[2..])); + let rlp = UntrustedRlp::new(&auth); + let signature: Signature = try!(rlp.val_at(0)); + let remote_public: Public = try!(rlp.val_at(1)); + let remote_nonce: H256 = try!(rlp.val_at(2)); + let remote_version: u64 = try!(rlp.val_at(3)); + try!(self.set_auth(secret, &signature, &remote_public, &remote_nonce, remote_version)); + try!(self.write_ack_eip8()); Ok(()) } /// Parse and validate ack message - fn read_ack(&mut self, host: &HostInfo, data: &[u8]) -> Result<(), UtilError> { + fn read_ack(&mut self, secret: &Secret, data: &[u8]) -> Result<(), UtilError> { trace!(target:"net", "Received handshake auth to {:?}", self.connection.socket.peer_addr()); - if data.len() != ACK_PACKET_SIZE { + if data.len() != V4_ACK_PACKET_SIZE { debug!(target:"net", "Wrong ack packet size"); return Err(From::from(NetworkError::BadProtocol)); } self.ack_cipher = data.to_vec(); - let ack = try!(ecies::decrypt(host.secret(), data)); - self.remote_public.clone_from_slice(&ack[0..64]); - self.remote_nonce.clone_from_slice(&ack[64..(64+32)]); + match ecies::decrypt(secret, &[], data) { + Ok(ack) => { + self.remote_ephemeral.clone_from_slice(&ack[0..64]); + self.remote_nonce.clone_from_slice(&ack[64..(64+32)]); + self.state = HandshakeState::StartSession; + } + Err(_) => { + // Try to interpret as EIP-8 packet + let total = (((data[0] as u16) << 8 | (data[1] as u16)) as usize) + 2; + if total < V4_ACK_PACKET_SIZE { + debug!(target:"net", "Wrong EIP8 ack packet size"); + return Err(From::from(NetworkError::BadProtocol)); + } + let rest = total - data.len(); + self.state = HandshakeState::ReadingAckEip8; + self.connection.expect(rest); + } + } + Ok(()) + } + + fn read_ack_eip8(&mut self, secret: &Secret, data: &[u8]) -> Result<(), UtilError> { + trace!(target:"net", "Received EIP8 handshake auth from {:?}", self.connection.socket.peer_addr()); + self.ack_cipher.extend_from_slice(data); + let ack = try!(ecies::decrypt(secret, &self.ack_cipher[0..2], &self.ack_cipher[2..])); + let rlp = UntrustedRlp::new(&ack); + self.remote_ephemeral = try!(rlp.val_at(0)); + self.remote_nonce = try!(rlp.val_at(1)); + self.remote_version = try!(rlp.val_at(2)); + self.state = HandshakeState::StartSession; Ok(()) } /// Sends auth message - fn write_auth(&mut self, host: &HostInfo) -> Result<(), UtilError> { + fn write_auth(&mut self, secret: &Secret, public: &Public) -> Result<(), UtilError> { trace!(target:"net", "Sending handshake auth to {:?}", self.connection.socket.peer_addr()); let mut data = [0u8; /*Signature::SIZE*/ 65 + /*H256::SIZE*/ 32 + /*Public::SIZE*/ 64 + /*H256::SIZE*/ 32 + 1]; //TODO: use associated constants let len = data.len(); @@ -243,16 +320,16 @@ impl Handshake { let (nonce, _) = rest.split_at_mut(32); // E(remote-pubk, S(ecdhe-random, ecdh-shared-secret^nonce) || H(ecdhe-random-pubk) || pubk || nonce || 0x0) - let shared = try!(crypto::ecdh::agree(host.secret(), &self.id)); + let shared = try!(crypto::ecdh::agree(secret, &self.id)); try!(crypto::ec::sign(self.ecdhe.secret(), &(&shared ^ &self.nonce))).copy_to(sig); self.ecdhe.public().sha3_into(hepubk); - host.id().copy_to(pubk); + public.copy_to(pubk); self.nonce.copy_to(nonce); } - let message = try!(crypto::ecies::encrypt(&self.id, &data)); + let message = try!(crypto::ecies::encrypt(&self.id, &[], &data)); self.auth_cipher = message.clone(); self.connection.send(message); - self.connection.expect(ACK_PACKET_SIZE); + self.connection.expect(V4_ACK_PACKET_SIZE); self.state = HandshakeState::ReadingAck; Ok(()) } @@ -269,10 +346,222 @@ impl Handshake { self.ecdhe.public().copy_to(epubk); self.nonce.copy_to(nonce); } - let message = try!(crypto::ecies::encrypt(&self.id, &data)); + let message = try!(crypto::ecies::encrypt(&self.id, &[], &data)); self.ack_cipher = message.clone(); self.connection.send(message); self.state = HandshakeState::StartSession; Ok(()) } + + /// Sends EIP8 ack message + fn write_ack_eip8(&mut self) -> Result<(), UtilError> { + trace!(target:"net", "Sending EIP8 handshake ack to {:?}", self.connection.socket.peer_addr()); + let mut rlp = RlpStream::new_list(3); + rlp.append(self.ecdhe.public()); + rlp.append(&self.nonce); + rlp.append(&PROTOCOL_VERSION); + + let pad_array = [0u8; 200]; + let pad = &pad_array[0 .. 100 + random::() % 100]; + rlp.append_raw(pad, 0); + + let encoded = rlp.drain(); + let len = (encoded.len() + ECIES_OVERHEAD) as u16; + let prefix = [ (len >> 8) as u8, (len & 0xff) as u8 ]; + let message = try!(crypto::ecies::encrypt(&self.id, &prefix, &encoded)); + self.ack_cipher.extend_from_slice(&prefix); + self.ack_cipher.extend_from_slice(&message); + self.connection.send(self.ack_cipher.clone()); + self.state = HandshakeState::StartSession; + Ok(()) + } } + +#[cfg(test)] +mod test { + use std::sync::Arc; + use std::str::FromStr; + use rustc_serialize::hex::FromHex; + use super::*; + use crypto::*; + use hash::*; + use std::net::SocketAddr; + use mio::tcp::TcpStream; + use network::stats::NetworkStats; + + fn check_auth(h: &Handshake, version: u64) { + assert_eq!(h.id, Public::from_str("fda1cff674c90c9a197539fe3dfb53086ace64f83ed7c6eabec741f7f381cc803e52ab2cd55d5569bce4347107a310dfd5f88a010cd2ffd1005ca406f1842877").unwrap()); + assert_eq!(h.remote_nonce, H256::from_str("7e968bba13b6c50e2c4cd7f241cc0d64d1ac25c7f5952df231ac6a2bda8ee5d6").unwrap()); + assert_eq!(h.remote_ephemeral, Public::from_str("654d1044b69c577a44e5f01a1209523adb4026e70c62d1c13a067acabc09d2667a49821a0ad4b634554d330a15a58fe61f8a8e0544b310c6de7b0c8da7528a8d").unwrap()); + assert_eq!(h.remote_version, version); + } + + fn check_ack(h: &Handshake, version: u64) { + assert_eq!(h.remote_nonce, H256::from_str("559aead08264d5795d3909718cdd05abd49572e84fe55590eef31a88a08fdffd").unwrap()); + assert_eq!(h.remote_ephemeral, Public::from_str("b6d82fa3409da933dbf9cb0140c5dde89f4e64aec88d476af648880f4a10e1e49fe35ef3e69e93dd300b4797765a747c6384a6ecf5db9c2690398607a86181e4").unwrap()); + assert_eq!(h.remote_version, version); + } + + fn create_handshake(to: Option<&Public>) -> Handshake { + let addr = SocketAddr::from_str("127.0.0.1:50556").unwrap(); + let socket = TcpStream::connect(&addr).unwrap(); + let nonce = H256::new(); + Handshake::new(0, to, socket, &nonce, Arc::new(NetworkStats::new())).unwrap() + } + + #[test] + fn test_handshake_auth_plain() { + let mut h = create_handshake(None); + let secret = Secret::from_str("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291").unwrap(); + let auth = + "\ + 048ca79ad18e4b0659fab4853fe5bc58eb83992980f4c9cc147d2aa31532efd29a3d3dc6a3d89eaf\ + 913150cfc777ce0ce4af2758bf4810235f6e6ceccfee1acc6b22c005e9e3a49d6448610a58e98744\ + ba3ac0399e82692d67c1f58849050b3024e21a52c9d3b01d871ff5f210817912773e610443a9ef14\ + 2e91cdba0bd77b5fdf0769b05671fc35f83d83e4d3b0b000c6b2a1b1bba89e0fc51bf4e460df3105\ + c444f14be226458940d6061c296350937ffd5e3acaceeaaefd3c6f74be8e23e0f45163cc7ebd7622\ + 0f0128410fd05250273156d548a414444ae2f7dea4dfca2d43c057adb701a715bf59f6fb66b2d1d2\ + 0f2c703f851cbf5ac47396d9ca65b6260bd141ac4d53e2de585a73d1750780db4c9ee4cd4d225173\ + a4592ee77e2bd94d0be3691f3b406f9bba9b591fc63facc016bfa8\ + ".from_hex().unwrap(); + + h.read_auth(&secret, &auth).unwrap(); + assert_eq!(h.state, super::HandshakeState::StartSession); + check_auth(&h, 4); + } + + #[test] + fn test_handshake_auth_eip8() { + let mut h = create_handshake(None); + let secret = Secret::from_str("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291").unwrap(); + let auth = + "\ + 01b304ab7578555167be8154d5cc456f567d5ba302662433674222360f08d5f1534499d3678b513b\ + 0fca474f3a514b18e75683032eb63fccb16c156dc6eb2c0b1593f0d84ac74f6e475f1b8d56116b84\ + 9634a8c458705bf83a626ea0384d4d7341aae591fae42ce6bd5c850bfe0b999a694a49bbbaf3ef6c\ + da61110601d3b4c02ab6c30437257a6e0117792631a4b47c1d52fc0f8f89caadeb7d02770bf999cc\ + 147d2df3b62e1ffb2c9d8c125a3984865356266bca11ce7d3a688663a51d82defaa8aad69da39ab6\ + d5470e81ec5f2a7a47fb865ff7cca21516f9299a07b1bc63ba56c7a1a892112841ca44b6e0034dee\ + 70c9adabc15d76a54f443593fafdc3b27af8059703f88928e199cb122362a4b35f62386da7caad09\ + c001edaeb5f8a06d2b26fb6cb93c52a9fca51853b68193916982358fe1e5369e249875bb8d0d0ec3\ + 6f917bc5e1eafd5896d46bd61ff23f1a863a8a8dcd54c7b109b771c8e61ec9c8908c733c0263440e\ + 2aa067241aaa433f0bb053c7b31a838504b148f570c0ad62837129e547678c5190341e4f1693956c\ + 3bf7678318e2d5b5340c9e488eefea198576344afbdf66db5f51204a6961a63ce072c8926c\ + ".from_hex().unwrap(); + + h.read_auth(&secret, &auth[0..super::V4_AUTH_PACKET_SIZE]).unwrap(); + assert_eq!(h.state, super::HandshakeState::ReadingAuthEip8); + h.read_auth_eip8(&secret, &auth[super::V4_AUTH_PACKET_SIZE..]).unwrap(); + assert_eq!(h.state, super::HandshakeState::StartSession); + check_auth(&h, 4); + } + + #[test] + fn test_handshake_auth_eip8_2() { + let mut h = create_handshake(None); + let secret = Secret::from_str("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291").unwrap(); + let auth = + "\ + 01b8044c6c312173685d1edd268aa95e1d495474c6959bcdd10067ba4c9013df9e40ff45f5bfd6f7\ + 2471f93a91b493f8e00abc4b80f682973de715d77ba3a005a242eb859f9a211d93a347fa64b597bf\ + 280a6b88e26299cf263b01b8dfdb712278464fd1c25840b995e84d367d743f66c0e54a586725b7bb\ + f12acca27170ae3283c1073adda4b6d79f27656993aefccf16e0d0409fe07db2dc398a1b7e8ee93b\ + cd181485fd332f381d6a050fba4c7641a5112ac1b0b61168d20f01b479e19adf7fdbfa0905f63352\ + bfc7e23cf3357657455119d879c78d3cf8c8c06375f3f7d4861aa02a122467e069acaf513025ff19\ + 6641f6d2810ce493f51bee9c966b15c5043505350392b57645385a18c78f14669cc4d960446c1757\ + 1b7c5d725021babbcd786957f3d17089c084907bda22c2b2675b4378b114c601d858802a55345a15\ + 116bc61da4193996187ed70d16730e9ae6b3bb8787ebcaea1871d850997ddc08b4f4ea668fbf3740\ + 7ac044b55be0908ecb94d4ed172ece66fd31bfdadf2b97a8bc690163ee11f5b575a4b44e36e2bfb2\ + f0fce91676fd64c7773bac6a003f481fddd0bae0a1f31aa27504e2a533af4cef3b623f4791b2cca6\ + d490\ + ".from_hex().unwrap(); + + h.read_auth(&secret, &auth[0..super::V4_AUTH_PACKET_SIZE]).unwrap(); + assert_eq!(h.state, super::HandshakeState::ReadingAuthEip8); + h.read_auth_eip8(&secret, &auth[super::V4_AUTH_PACKET_SIZE..]).unwrap(); + assert_eq!(h.state, super::HandshakeState::StartSession); + check_auth(&h, 56); + let ack = h.ack_cipher.clone(); + let total = (((ack[0] as u16) << 8 | (ack[1] as u16)) as usize) + 2; + assert_eq!(ack.len(), total); + } + + #[test] + fn test_handshake_ack_plain() { + let remote = Public::from_str("fda1cff674c90c9a197539fe3dfb53086ace64f83ed7c6eabec741f7f381cc803e52ab2cd55d5569bce4347107a310dfd5f88a010cd2ffd1005ca406f1842877").unwrap(); + let mut h = create_handshake(Some(&remote)); + let secret = Secret::from_str("49a7b37aa6f6645917e7b807e9d1c00d4fa71f18343b0d4122a4d2df64dd6fee").unwrap(); + let ack = + "\ + 049f8abcfa9c0dc65b982e98af921bc0ba6e4243169348a236abe9df5f93aa69d99cadddaa387662\ + b0ff2c08e9006d5a11a278b1b3331e5aaabf0a32f01281b6f4ede0e09a2d5f585b26513cb794d963\ + 5a57563921c04a9090b4f14ee42be1a5461049af4ea7a7f49bf4c97a352d39c8d02ee4acc416388c\ + 1c66cec761d2bc1c72da6ba143477f049c9d2dde846c252c111b904f630ac98e51609b3b1f58168d\ + dca6505b7196532e5f85b259a20c45e1979491683fee108e9660edbf38f3add489ae73e3dda2c71b\ + d1497113d5c755e942d1\ + ".from_hex().unwrap(); + + h.read_ack(&secret, &ack).unwrap(); + assert_eq!(h.state, super::HandshakeState::StartSession); + check_ack(&h, 4); + } + + #[test] + fn test_handshake_ack_eip8() { + let remote = Public::from_str("fda1cff674c90c9a197539fe3dfb53086ace64f83ed7c6eabec741f7f381cc803e52ab2cd55d5569bce4347107a310dfd5f88a010cd2ffd1005ca406f1842877").unwrap(); + let mut h = create_handshake(Some(&remote)); + let secret = Secret::from_str("49a7b37aa6f6645917e7b807e9d1c00d4fa71f18343b0d4122a4d2df64dd6fee").unwrap(); + let ack = + "\ + 01ea0451958701280a56482929d3b0757da8f7fbe5286784beead59d95089c217c9b917788989470\ + b0e330cc6e4fb383c0340ed85fab836ec9fb8a49672712aeabbdfd1e837c1ff4cace34311cd7f4de\ + 05d59279e3524ab26ef753a0095637ac88f2b499b9914b5f64e143eae548a1066e14cd2f4bd7f814\ + c4652f11b254f8a2d0191e2f5546fae6055694aed14d906df79ad3b407d94692694e259191cde171\ + ad542fc588fa2b7333313d82a9f887332f1dfc36cea03f831cb9a23fea05b33deb999e85489e645f\ + 6aab1872475d488d7bd6c7c120caf28dbfc5d6833888155ed69d34dbdc39c1f299be1057810f34fb\ + e754d021bfca14dc989753d61c413d261934e1a9c67ee060a25eefb54e81a4d14baff922180c395d\ + 3f998d70f46f6b58306f969627ae364497e73fc27f6d17ae45a413d322cb8814276be6ddd13b885b\ + 201b943213656cde498fa0e9ddc8e0b8f8a53824fbd82254f3e2c17e8eaea009c38b4aa0a3f306e8\ + 797db43c25d68e86f262e564086f59a2fc60511c42abfb3057c247a8a8fe4fb3ccbadde17514b7ac\ + 8000cdb6a912778426260c47f38919a91f25f4b5ffb455d6aaaf150f7e5529c100ce62d6d92826a7\ + 1778d809bdf60232ae21ce8a437eca8223f45ac37f6487452ce626f549b3b5fdee26afd2072e4bc7\ + 5833c2464c805246155289f4\ + ".from_hex().unwrap(); + + h.read_ack(&secret, &ack[0..super::V4_ACK_PACKET_SIZE]).unwrap(); + assert_eq!(h.state, super::HandshakeState::ReadingAckEip8); + h.read_ack_eip8(&secret, &ack[super::V4_ACK_PACKET_SIZE..]).unwrap(); + assert_eq!(h.state, super::HandshakeState::StartSession); + check_ack(&h, 4); + } + + #[test] + fn test_handshake_ack_eip8_2() { + let remote = Public::from_str("fda1cff674c90c9a197539fe3dfb53086ace64f83ed7c6eabec741f7f381cc803e52ab2cd55d5569bce4347107a310dfd5f88a010cd2ffd1005ca406f1842877").unwrap(); + let mut h = create_handshake(Some(&remote)); + let secret = Secret::from_str("49a7b37aa6f6645917e7b807e9d1c00d4fa71f18343b0d4122a4d2df64dd6fee").unwrap(); + let ack = + "\ + 01f004076e58aae772bb101ab1a8e64e01ee96e64857ce82b1113817c6cdd52c09d26f7b90981cd7\ + ae835aeac72e1573b8a0225dd56d157a010846d888dac7464baf53f2ad4e3d584531fa203658fab0\ + 3a06c9fd5e35737e417bc28c1cbf5e5dfc666de7090f69c3b29754725f84f75382891c561040ea1d\ + dc0d8f381ed1b9d0d4ad2a0ec021421d847820d6fa0ba66eaf58175f1b235e851c7e2124069fbc20\ + 2888ddb3ac4d56bcbd1b9b7eab59e78f2e2d400905050f4a92dec1c4bdf797b3fc9b2f8e84a482f3\ + d800386186712dae00d5c386ec9387a5e9c9a1aca5a573ca91082c7d68421f388e79127a5177d4f8\ + 590237364fd348c9611fa39f78dcdceee3f390f07991b7b47e1daa3ebcb6ccc9607811cb17ce51f1\ + c8c2c5098dbdd28fca547b3f58c01a424ac05f869f49c6a34672ea2cbbc558428aa1fe48bbfd6115\ + 8b1b735a65d99f21e70dbc020bfdface9f724a0d1fb5895db971cc81aa7608baa0920abb0a565c9c\ + 436e2fd13323428296c86385f2384e408a31e104670df0791d93e743a3a5194ee6b076fb6323ca59\ + 3011b7348c16cf58f66b9633906ba54a2ee803187344b394f75dd2e663a57b956cb830dd7a908d4f\ + 39a2336a61ef9fda549180d4ccde21514d117b6c6fd07a9102b5efe710a32af4eeacae2cb3b1dec0\ + 35b9593b48b9d3ca4c13d245d5f04169b0b1\ + ".from_hex().unwrap(); + + h.read_ack(&secret, &ack[0..super::V4_ACK_PACKET_SIZE]).unwrap(); + assert_eq!(h.state, super::HandshakeState::ReadingAckEip8); + h.read_ack_eip8(&secret, &ack[super::V4_ACK_PACKET_SIZE..]).unwrap(); + assert_eq!(h.state, super::HandshakeState::StartSession); + check_ack(&h, 57); + } +} + diff --git a/util/src/network/host.rs b/util/src/network/host.rs index d50f2d86d..f54c85855 100644 --- a/util/src/network/host.rs +++ b/util/src/network/host.rs @@ -106,6 +106,7 @@ const IDLE: usize = LAST_HANDSHAKE + 2; const DISCOVERY: usize = LAST_HANDSHAKE + 3; const DISCOVERY_REFRESH: usize = LAST_HANDSHAKE + 4; const DISCOVERY_ROUND: usize = LAST_HANDSHAKE + 5; +const INIT_PUBLIC: usize = LAST_HANDSHAKE + 6; const FIRST_SESSION: usize = 0; const LAST_SESSION: usize = FIRST_SESSION + MAX_SESSIONS - 1; const FIRST_HANDSHAKE: usize = LAST_SESSION + 1; @@ -261,7 +262,9 @@ pub struct HostInfo { /// TCP connection port. pub listen_port: u16, /// Registered capabilities (handlers) - pub capabilities: Vec + pub capabilities: Vec, + /// Public address + discovery port + public_endpoint: NodeEndpoint, } impl HostInfo { @@ -294,16 +297,15 @@ struct ProtocolTimer { /// Root IO handler. Manages protocol handlers, IO timers and network connections. pub struct Host where Message: Send + Sync + Clone { pub info: RwLock, - tcp_listener: Mutex, + tcp_listener: Mutex>, handshakes: Arc>>, sessions: Arc>>, - discovery: Option>, + discovery: Mutex>, nodes: RwLock, handlers: RwLock>>>, timers: RwLock>, timer_counter: RwLock, stats: Arc, - public_endpoint: NodeEndpoint, pinned_nodes: Vec, } @@ -316,27 +318,6 @@ impl Host where Message: Send + Sync + Clone { }; let udp_port = config.udp_port.unwrap_or(listen_address.port()); - let public_endpoint = match config.public_address { - None => { - let public_address = select_public_address(listen_address.port()); - let local_endpoint = NodeEndpoint { address: public_address, udp_port: udp_port }; - if config.nat_enabled { - match map_external_address(&local_endpoint) { - Some(endpoint) => { - info!("NAT Mappped to external address {}", endpoint.address); - endpoint - }, - None => local_endpoint - } - } else { - local_endpoint - } - } - Some(addr) => NodeEndpoint { address: addr, udp_port: udp_port } - }; - - // Setup the server socket - let tcp_listener = TcpListener::bind(&listen_address).unwrap(); let keys = if let Some(ref secret) = config.use_secret { KeyPair::from_secret(secret.clone()).unwrap() } else { @@ -350,10 +331,8 @@ impl Host where Message: Send + Sync + Clone { }, |s| KeyPair::from_secret(s).expect("Error creating node secret key")) }; - let discovery = if config.discovery_enabled && !config.pin { - Some(Discovery::new(&keys, listen_address.clone(), public_endpoint.clone(), DISCOVERY)) - } else { None }; let path = config.config_path.clone(); + let local_endpoint = NodeEndpoint { address: listen_address, udp_port: udp_port }; let mut host = Host:: { info: RwLock::new(HostInfo { keys: keys, @@ -363,9 +342,10 @@ impl Host where Message: Send + Sync + Clone { client_version: version(), listen_port: 0, capabilities: Vec::new(), + public_endpoint: local_endpoint, // will be replaced by public once it is resolved }), - discovery: discovery.map(Mutex::new), - tcp_listener: Mutex::new(tcp_listener), + discovery: Mutex::new(None), + tcp_listener: Mutex::new(None), handshakes: Arc::new(RwLock::new(Slab::new_starting_at(FIRST_HANDSHAKE, MAX_HANDSHAKES))), sessions: Arc::new(RwLock::new(Slab::new_starting_at(FIRST_SESSION, MAX_SESSIONS))), nodes: RwLock::new(NodeTable::new(path)), @@ -373,16 +353,12 @@ impl Host where Message: Send + Sync + Clone { timers: RwLock::new(HashMap::new()), timer_counter: RwLock::new(USER_TIMER), stats: Arc::new(NetworkStats::default()), - public_endpoint: public_endpoint, pinned_nodes: Vec::new(), }; let port = listen_address.port(); host.info.write().unwrap().deref_mut().listen_port = port; let boot_nodes = host.info.read().unwrap().config.boot_nodes.clone(); - if let Some(ref mut discovery) = host.discovery { - discovery.lock().unwrap().init_node_list(host.nodes.read().unwrap().unordered_entries()); - } for n in boot_nodes { host.add_node(&n); } @@ -400,8 +376,8 @@ impl Host where Message: Send + Sync + Clone { let entry = NodeEntry { endpoint: n.endpoint.clone(), id: n.id.clone() }; self.pinned_nodes.push(n.id.clone()); self.nodes.write().unwrap().add_node(n); - if let Some(ref mut discovery) = self.discovery { - discovery.lock().unwrap().add_node(entry); + if let &mut Some(ref mut discovery) = self.discovery.lock().unwrap().deref_mut() { + discovery.add_node(entry); } } } @@ -412,7 +388,61 @@ impl Host where Message: Send + Sync + Clone { } pub fn client_url(&self) -> String { - format!("{}", Node::new(self.info.read().unwrap().id().clone(), self.public_endpoint.clone())) + format!("{}", Node::new(self.info.read().unwrap().id().clone(), self.info.read().unwrap().public_endpoint.clone())) + } + + fn init_public_interface(&self, io: &IoContext>) { + io.clear_timer(INIT_PUBLIC).unwrap(); + let mut tcp_listener = self.tcp_listener.lock().unwrap(); + if tcp_listener.is_some() { + return; + } + // public_endpoint in host info contains local adderss at this point + let listen_address = self.info.read().unwrap().public_endpoint.address.clone(); + let udp_port = self.info.read().unwrap().config.udp_port.unwrap_or(listen_address.port()); + let public_endpoint = match self.info.read().unwrap().config.public_address { + None => { + let public_address = select_public_address(listen_address.port()); + let local_endpoint = NodeEndpoint { address: public_address, udp_port: udp_port }; + if self.info.read().unwrap().config.nat_enabled { + match map_external_address(&local_endpoint) { + Some(endpoint) => { + info!("NAT mappped to external address {}", endpoint.address); + endpoint + }, + None => local_endpoint + } + } else { + local_endpoint + } + } + Some(addr) => NodeEndpoint { address: addr, udp_port: udp_port } + }; + + // Setup the server socket + *tcp_listener = Some(TcpListener::bind(&listen_address).unwrap()); + self.info.write().unwrap().public_endpoint = public_endpoint.clone(); + io.register_stream(TCP_ACCEPT).expect("Error registering TCP listener"); + info!("Public node URL: {}", self.client_url()); + + // Initialize discovery. + let discovery = { + let info = self.info.read().unwrap(); + if info.config.discovery_enabled && !info.config.pin { + Some(Discovery::new(&info.keys, listen_address.clone(), public_endpoint, DISCOVERY)) + } else { None } + }; + + if let Some(mut discovery) = discovery { + discovery.init_node_list(self.nodes.read().unwrap().unordered_entries()); + for n in self.nodes.read().unwrap().unordered_entries() { + discovery.add_node(n.clone()); + } + io.register_stream(DISCOVERY).expect("Error registering UDP listener"); + io.register_timer(DISCOVERY_REFRESH, 7200).expect("Error registering discovery timer"); + io.register_timer(DISCOVERY_ROUND, 300).expect("Error registering discovery timer"); + *self.discovery.lock().unwrap().deref_mut() = Some(discovery); + } } fn maintain_network(&self, io: &IoContext>) { @@ -526,7 +556,7 @@ impl Host where Message: Send + Sync + Clone { fn accept(&self, io: &IoContext>) { trace!(target: "network", "Accepting incoming connection"); loop { - let socket = match self.tcp_listener.lock().unwrap().accept() { + let socket = match self.tcp_listener.lock().unwrap().as_ref().unwrap().accept() { Ok(None) => break, Ok(Some((sock, _addr))) => sock, Err(e) => { @@ -666,8 +696,9 @@ impl Host where Message: Send + Sync + Clone { if let Ok(address) = session.remote_addr() { let entry = NodeEntry { id: session.id().clone(), endpoint: NodeEndpoint { address: address, udp_port: address.port() } }; self.nodes.write().unwrap().add_node(Node::new(entry.id.clone(), entry.endpoint.clone())); - if let Some(ref discovery) = self.discovery { - discovery.lock().unwrap().add_node(entry); + let mut discovery = self.discovery.lock().unwrap(); + if let &mut Some(ref mut discovery) = discovery.deref_mut() { + discovery.add_node(entry); } } } @@ -764,13 +795,8 @@ impl Host where Message: Send + Sync + Clone { impl IoHandler> for Host where Message: Send + Sync + Clone + 'static { /// Initialize networking fn initialize(&self, io: &IoContext>) { - io.register_stream(TCP_ACCEPT).expect("Error registering TCP listener"); io.register_timer(IDLE, MAINTENANCE_TIMEOUT).expect("Error registering Network idle timer"); - if self.discovery.is_some() { - io.register_stream(DISCOVERY).expect("Error registering UDP listener"); - io.register_timer(DISCOVERY_REFRESH, 7200).expect("Error registering discovery timer"); - io.register_timer(DISCOVERY_ROUND, 300).expect("Error registering discovery timer"); - } + io.register_timer(INIT_PUBLIC, 0).expect("Error registering initialization timer"); self.maintain_network(io) } @@ -788,7 +814,7 @@ impl IoHandler> for Host where Messa FIRST_SESSION ... LAST_SESSION => self.session_readable(stream, io), FIRST_HANDSHAKE ... LAST_HANDSHAKE => self.handshake_readable(stream, io), DISCOVERY => { - let node_changes = { self.discovery.as_ref().unwrap().lock().unwrap().readable() }; + let node_changes = { self.discovery.lock().unwrap().as_mut().unwrap().readable() }; if let Some(node_changes) = node_changes { self.update_nodes(io, node_changes); } @@ -804,7 +830,7 @@ impl IoHandler> for Host where Messa FIRST_SESSION ... LAST_SESSION => self.session_writable(stream, io), FIRST_HANDSHAKE ... LAST_HANDSHAKE => self.handshake_writable(stream, io), DISCOVERY => { - self.discovery.as_ref().unwrap().lock().unwrap().writable(); + self.discovery.lock().unwrap().as_mut().unwrap().writable(); io.update_registration(DISCOVERY).expect("Error updating discovery registration"); } _ => panic!("Received unknown writable token"), @@ -814,14 +840,15 @@ impl IoHandler> for Host where Messa fn timeout(&self, io: &IoContext>, token: TimerToken) { match token { IDLE => self.maintain_network(io), + INIT_PUBLIC => self.init_public_interface(io), FIRST_SESSION ... LAST_SESSION => self.connection_timeout(token, io), FIRST_HANDSHAKE ... LAST_HANDSHAKE => self.connection_timeout(token, io), DISCOVERY_REFRESH => { - self.discovery.as_ref().unwrap().lock().unwrap().refresh(); + self.discovery.lock().unwrap().as_mut().unwrap().refresh(); io.update_registration(DISCOVERY).expect("Error updating discovery registration"); }, DISCOVERY_ROUND => { - let node_changes = { self.discovery.as_ref().unwrap().lock().unwrap().round() }; + let node_changes = { self.discovery.lock().unwrap().as_mut().unwrap().round() }; if let Some(node_changes) = node_changes { self.update_nodes(io, node_changes); } @@ -896,8 +923,8 @@ impl IoHandler> for Host where Messa connection.lock().unwrap().register_socket(reg, event_loop).expect("Error registering socket"); } } - DISCOVERY => self.discovery.as_ref().unwrap().lock().unwrap().register_socket(event_loop).expect("Error registering discovery socket"), - TCP_ACCEPT => event_loop.register(self.tcp_listener.lock().unwrap().deref(), Token(TCP_ACCEPT), EventSet::all(), PollOpt::edge()).expect("Error registering stream"), + DISCOVERY => self.discovery.lock().unwrap().as_ref().unwrap().register_socket(event_loop).expect("Error registering discovery socket"), + TCP_ACCEPT => event_loop.register(self.tcp_listener.lock().unwrap().as_ref().unwrap(), Token(TCP_ACCEPT), EventSet::all(), PollOpt::edge()).expect("Error registering stream"), _ => warn!("Unexpected stream registration") } } @@ -919,7 +946,6 @@ impl IoHandler> for Host where Messa } } DISCOVERY => (), - TCP_ACCEPT => event_loop.deregister(self.tcp_listener.lock().unwrap().deref()).unwrap(), _ => warn!("Unexpected stream deregistration") } } @@ -938,8 +964,8 @@ impl IoHandler> for Host where Messa connection.lock().unwrap().update_socket(reg, event_loop).expect("Error updating socket"); } } - DISCOVERY => self.discovery.as_ref().unwrap().lock().unwrap().update_registration(event_loop).expect("Error reregistering discovery socket"), - TCP_ACCEPT => event_loop.reregister(self.tcp_listener.lock().unwrap().deref(), Token(TCP_ACCEPT), EventSet::all(), PollOpt::edge()).expect("Error reregistering stream"), + DISCOVERY => self.discovery.lock().unwrap().as_ref().unwrap().update_registration(event_loop).expect("Error reregistering discovery socket"), + TCP_ACCEPT => event_loop.reregister(self.tcp_listener.lock().unwrap().as_ref().unwrap(), Token(TCP_ACCEPT), EventSet::all(), PollOpt::edge()).expect("Error reregistering stream"), _ => warn!("Unexpected stream update") } } diff --git a/util/src/network/service.rs b/util/src/network/service.rs index 1cd48abe1..7b9388e85 100644 --- a/util/src/network/service.rs +++ b/util/src/network/service.rs @@ -42,7 +42,6 @@ impl NetworkService where Message: Send + Sync + Clone + 'stat let host = Arc::new(Host::new(config)); let stats = host.stats().clone(); let host_info = host.client_version(); - info!("Node URL: {}", host.client_url()); try!(io_service.register_handler(host)); Ok(NetworkService { io_service: io_service, diff --git a/util/src/overlaydb.rs b/util/src/overlaydb.rs index f8e9c3eee..f4ed2d5d6 100644 --- a/util/src/overlaydb.rs +++ b/util/src/overlaydb.rs @@ -26,7 +26,7 @@ use std::ops::*; use std::sync::*; use std::env; use std::collections::HashMap; -use rocksdb::{DB, Writable, IteratorMode}; +use kvdb::{Database}; /// Implementation of the HashDB trait for a disk-backed database with a memory overlay. /// @@ -38,15 +38,15 @@ use rocksdb::{DB, Writable, IteratorMode}; /// queries have an immediate effect in terms of these functions. pub struct OverlayDB { overlay: MemoryDB, - backing: Arc, + backing: Arc, } impl OverlayDB { /// Create a new instance of OverlayDB given a `backing` database. - pub fn new(backing: DB) -> OverlayDB { Self::new_with_arc(Arc::new(backing)) } + pub fn new(backing: Database) -> OverlayDB { Self::new_with_arc(Arc::new(backing)) } /// Create a new instance of OverlayDB given a `backing` database. - pub fn new_with_arc(backing: Arc) -> OverlayDB { + pub fn new_with_arc(backing: Arc) -> OverlayDB { OverlayDB{ overlay: MemoryDB::new(), backing: backing } } @@ -54,7 +54,7 @@ impl OverlayDB { pub fn new_temp() -> OverlayDB { let mut dir = env::temp_dir(); dir.push(H32::random().hex()); - Self::new(DB::open_default(dir.to_str().unwrap()).unwrap()) + Self::new(Database::open_default(dir.to_str().unwrap()).unwrap()) } /// Commit all memory operations to the backing database. @@ -164,7 +164,7 @@ impl OverlayDB { impl HashDB for OverlayDB { fn keys(&self) -> HashMap { let mut ret: HashMap = HashMap::new(); - for (key, _) in self.backing.iterator(IteratorMode::Start) { + for (key, _) in self.backing.iter() { let h = H256::from_slice(key.deref()); let r = self.payload(&h).unwrap().1; ret.insert(h, r as i32); @@ -318,7 +318,7 @@ fn overlaydb_complex() { fn playpen() { use std::fs; { - let db: DB = DB::open_default("/tmp/test").unwrap(); + let db: Database = Database::open_default("/tmp/test").unwrap(); db.put(b"test", b"test2").unwrap(); match db.get(b"test") { Ok(Some(value)) => println!("Got value {:?}", value.deref()), diff --git a/util/src/trie/standardmap.rs b/util/src/trie/standardmap.rs index 98a5ae0f0..b7f3a9500 100644 --- a/util/src/trie/standardmap.rs +++ b/util/src/trie/standardmap.rs @@ -20,6 +20,7 @@ extern crate rand; use bytes::*; use sha3::*; use hash::*; +use rlp::encode; /// Alphabet to use when creating words for insertion into tries. pub enum Alphabet { @@ -39,6 +40,8 @@ pub enum ValueMode { Mirror, /// Randomly (50:50) 1 or 32 byte randomly string. Random, + /// RLP-encoded index. + Index, } /// Standard test map for profiling tries. @@ -89,19 +92,27 @@ impl StandardMap { /// Create the standard map (set of keys and values) for the object's fields. pub fn make(&self) -> Vec<(Bytes, Bytes)> { + self.make_with(&mut H256::new()) + } + + /// Create the standard map (set of keys and values) for the object's fields, using the given seed. + pub fn make_with(&self, seed: &mut H256) -> Vec<(Bytes, Bytes)> { let low = b"abcdef"; let mid = b"@QWERTYUIOPASDFGHJKLZXCVBNM[/]^_"; let mut d: Vec<(Bytes, Bytes)> = Vec::new(); - let mut seed = H256::new(); - for _ in 0..self.count { + for index in 0..self.count { let k = match self.alphabet { - Alphabet::All => Self::random_bytes(self.min_key, self.journal_key, &mut seed), - Alphabet::Low => Self::random_word(low, self.min_key, self.journal_key, &mut seed), - Alphabet::Mid => Self::random_word(mid, self.min_key, self.journal_key, &mut seed), - Alphabet::Custom(ref a) => Self::random_word(&a, self.min_key, self.journal_key, &mut seed), + Alphabet::All => Self::random_bytes(self.min_key, self.journal_key, seed), + Alphabet::Low => Self::random_word(low, self.min_key, self.journal_key, seed), + Alphabet::Mid => Self::random_word(mid, self.min_key, self.journal_key, seed), + Alphabet::Custom(ref a) => Self::random_word(&a, self.min_key, self.journal_key, seed), + }; + let v = match self.value_mode { + ValueMode::Mirror => k.clone(), + ValueMode::Random => Self::random_value(seed), + ValueMode::Index => encode(&index).to_vec(), }; - let v = match self.value_mode { ValueMode::Mirror => k.clone(), ValueMode::Random => Self::random_value(&mut seed) }; d.push((k, v)) } d diff --git a/util/src/trie/triedbmut.rs b/util/src/trie/triedbmut.rs index 2b4567264..829c1e518 100644 --- a/util/src/trie/triedbmut.rs +++ b/util/src/trie/triedbmut.rs @@ -687,31 +687,10 @@ mod tests { use super::*; use nibbleslice::*; use rlp::*; - use rand::random; - use std::collections::HashSet; - use bytes::{ToPretty,Bytes,Populatable}; + use bytes::ToPretty; use super::super::node::*; use super::super::trietraits::*; - - fn random_key(alphabet: &[u8], min_count: usize, journal_count: usize) -> Vec { - let mut ret: Vec = Vec::new(); - let r = min_count + if journal_count > 0 {random::() % journal_count} else {0}; - for _ in 0..r { - ret.push(alphabet[random::() % alphabet.len()]); - } - ret - } - - fn random_value_indexed(j: usize) -> Bytes { - match random::() % 2 { - 0 => encode(&j).to_vec(), - _ => { - let mut h = H256::new(); - h.as_slice_mut()[31] = j as u8; - encode(&h).to_vec() - }, - } - } + use super::super::standardmap::*; fn populate_trie<'db>(db: &'db mut HashDB, root: &'db mut H256, v: &[(Vec, Vec)]) -> TrieDBMut<'db> { let mut t = TrieDBMut::new(db, root); @@ -756,20 +735,18 @@ mod tests { };*/ // panic!(); + let mut seed = H256::new(); for test_i in 0..1 { if test_i % 50 == 0 { debug!("{:?} of 10000 stress tests done", test_i); } - let mut x: Vec<(Vec, Vec)> = Vec::new(); - let mut got: HashSet> = HashSet::new(); - let alphabet = b"@QWERTYUIOPASDFGHJKLZXCVBNM[/]^_"; - for j in 0..100usize { - let key = random_key(alphabet, 5, 0); - if !got.contains(&key) { - x.push((key.clone(), random_value_indexed(j))); - got.insert(key); - } - } + let x = StandardMap { + alphabet: Alphabet::Custom(b"@QWERTYUIOPASDFGHJKLZXCVBNM[/]^_".to_vec()), + min_key: 5, + journal_key: 0, + value_mode: ValueMode::Index, + count: 100, + }.make_with(&mut seed); let real = trie_root(x.clone()); let mut memdb = MemoryDB::new(); @@ -1049,13 +1026,16 @@ mod tests { #[test] fn stress() { + let mut seed = H256::new(); for _ in 0..50 { - let mut x: Vec<(Vec, Vec)> = Vec::new(); - let alphabet = b"@QWERTYUIOPASDFGHJKLZXCVBNM[/]^_"; - for j in 0..4u32 { - let key = random_key(alphabet, 5, 1); - x.push((key, encode(&j).to_vec())); - } + let x = StandardMap { + alphabet: Alphabet::Custom(b"@QWERTYUIOPASDFGHJKLZXCVBNM[/]^_".to_vec()), + min_key: 5, + journal_key: 0, + value_mode: ValueMode::Index, + count: 4, + }.make_with(&mut seed); + let real = trie_root(x.clone()); let mut memdb = MemoryDB::new(); let mut root = H256::new(); diff --git a/util/src/uint.rs b/util/src/uint.rs index 6490cbd9b..67a6f2ff4 100644 --- a/util/src/uint.rs +++ b/util/src/uint.rs @@ -51,6 +51,346 @@ macro_rules! impl_map_from { } } +#[cfg(not(all(feature="x64asm", target_arch="x86_64")))] +macro_rules! uint_overflowing_add { + ($name:ident, $n_words:expr, $self_expr: expr, $other: expr) => ({ + uint_overflowing_add_reg!($name, $n_words, $self_expr, $other) + }) +} + +macro_rules! uint_overflowing_add_reg { + ($name:ident, $n_words:expr, $self_expr: expr, $other: expr) => ({ + let $name(ref me) = $self_expr; + let $name(ref you) = $other; + let mut ret = [0u64; $n_words]; + let mut carry = [0u64; $n_words]; + let mut b_carry = false; + let mut overflow = false; + + for i in 0..$n_words { + ret[i] = me[i].wrapping_add(you[i]); + + if ret[i] < me[i] { + if i < $n_words - 1 { + carry[i + 1] = 1; + b_carry = true; + } else { + overflow = true; + } + } + } + if b_carry { + let ret = overflowing!($name(ret).overflowing_add($name(carry)), overflow); + (ret, overflow) + } else { + ($name(ret), overflow) + } + }) +} + + +#[cfg(all(feature="x64asm", target_arch="x86_64"))] +macro_rules! uint_overflowing_add { + (U256, $n_words: expr, $self_expr: expr, $other: expr) => ({ + let mut result: [u64; 4] = unsafe { mem::uninitialized() }; + let self_t: &[u64; 4] = unsafe { &mem::transmute($self_expr) }; + let other_t: &[u64; 4] = unsafe { &mem::transmute($other) }; + + let overflow: u8; + unsafe { + asm!(" + add $9, $0 + adc $10, $1 + adc $11, $2 + adc $12, $3 + setc %al + " + : "=r"(result[0]), "=r"(result[1]), "=r"(result[2]), "=r"(result[3]), "={al}"(overflow) + : "0"(self_t[0]), "1"(self_t[1]), "2"(self_t[2]), "3"(self_t[3]), + "mr"(other_t[0]), "mr"(other_t[1]), "mr"(other_t[2]), "mr"(other_t[3]) + : + : + ); + } + (U256(result), overflow != 0) + }); + (U512, $n_words: expr, $self_expr: expr, $other: expr) => ({ + let mut result: [u64; 8] = unsafe { mem::uninitialized() }; + let self_t: &[u64; 8] = unsafe { &mem::transmute($self_expr) }; + let other_t: &[u64; 8] = unsafe { &mem::transmute($other) }; + + let overflow: u8; + + unsafe { + asm!(" + add $15, $0 + adc $16, $1 + adc $17, $2 + adc $18, $3 + lodsq + adc $11, %rax + stosq + lodsq + adc $12, %rax + stosq + lodsq + adc $13, %rax + stosq + lodsq + adc $14, %rax + stosq + setc %al + + ": "=r"(result[0]), "=r"(result[1]), "=r"(result[2]), "=r"(result[3]), + + "={al}"(overflow) /* $0 - $4 */ + + : "{rdi}"(&result[4] as *const u64) /* $5 */ + "{rsi}"(&other_t[4] as *const u64) /* $6 */ + "0"(self_t[0]), "1"(self_t[1]), "2"(self_t[2]), "3"(self_t[3]), + "m"(self_t[4]), "m"(self_t[5]), "m"(self_t[6]), "m"(self_t[7]), + /* $7 - $14 */ + + "mr"(other_t[0]), "mr"(other_t[1]), "mr"(other_t[2]), "mr"(other_t[3]), + "m"(other_t[4]), "m"(other_t[5]), "m"(other_t[6]), "m"(other_t[7]) /* $15 - $22 */ + : "rdi", "rsi" + : + ); + } + (U512(result), overflow != 0) + }); + + ($name:ident, $n_words:expr, $self_expr: expr, $other: expr) => ( + uint_overflowing_add_reg!($name, $n_words, $self_expr, $other) + ) +} + +#[cfg(not(all(feature="x64asm", target_arch="x86_64")))] +macro_rules! uint_overflowing_sub { + ($name:ident, $n_words: expr, $self_expr: expr, $other: expr) => ({ + let res = overflowing!((!$other).overflowing_add(From::from(1u64))); + let res = overflowing!($self_expr.overflowing_add(res)); + (res, $self_expr < $other) + }) +} + +#[cfg(all(feature="x64asm", target_arch="x86_64"))] +macro_rules! uint_overflowing_sub { + (U256, $n_words: expr, $self_expr: expr, $other: expr) => ({ + let mut result: [u64; 4] = unsafe { mem::uninitialized() }; + let self_t: &[u64; 4] = unsafe { &mem::transmute($self_expr) }; + let other_t: &[u64; 4] = unsafe { &mem::transmute($other) }; + + let overflow: u8; + unsafe { + asm!(" + sub $9, $0 + sbb $10, $1 + sbb $11, $2 + sbb $12, $3 + setb %al + " + : "=r"(result[0]), "=r"(result[1]), "=r"(result[2]), "=r"(result[3]), "={al}"(overflow) + : "0"(self_t[0]), "1"(self_t[1]), "2"(self_t[2]), "3"(self_t[3]), "mr"(other_t[0]), "mr"(other_t[1]), "mr"(other_t[2]), "mr"(other_t[3]) + : + : + ); + } + (U256(result), overflow != 0) + }); + (U512, $n_words: expr, $self_expr: expr, $other: expr) => ({ + let mut result: [u64; 8] = unsafe { mem::uninitialized() }; + let self_t: &[u64; 8] = unsafe { &mem::transmute($self_expr) }; + let other_t: &[u64; 8] = unsafe { &mem::transmute($other) }; + + let overflow: u8; + + unsafe { + asm!(" + sub $15, $0 + sbb $16, $1 + sbb $17, $2 + sbb $18, $3 + lodsq + sbb $19, %rax + stosq + lodsq + sbb $20, %rax + stosq + lodsq + sbb $21, %rax + stosq + lodsq + sbb $22, %rax + stosq + setb %al + " + : "=r"(result[0]), "=r"(result[1]), "=r"(result[2]), "=r"(result[3]), + + "={al}"(overflow) /* $0 - $4 */ + + : "{rdi}"(&result[4] as *const u64) /* $5 */ + "{rsi}"(&self_t[4] as *const u64) /* $6 */ + "0"(self_t[0]), "1"(self_t[1]), "2"(self_t[2]), "3"(self_t[3]), + "m"(self_t[4]), "m"(self_t[5]), "m"(self_t[6]), "m"(self_t[7]), + /* $7 - $14 */ + + "m"(other_t[0]), "m"(other_t[1]), "m"(other_t[2]), "m"(other_t[3]), + "m"(other_t[4]), "m"(other_t[5]), "m"(other_t[6]), "m"(other_t[7]) /* $15 - $22 */ + : "rdi", "rsi" + : + ); + } + (U512(result), overflow != 0) + }); + ($name:ident, $n_words: expr, $self_expr: expr, $other: expr) => ({ + let res = overflowing!((!$other).overflowing_add(From::from(1u64))); + let res = overflowing!($self_expr.overflowing_add(res)); + (res, $self_expr < $other) + }) +} + +#[cfg(all(feature="x64asm", target_arch="x86_64"))] +macro_rules! uint_overflowing_mul { + (U256, $n_words: expr, $self_expr: expr, $other: expr) => ({ + let mut result: [u64; 4] = unsafe { mem::uninitialized() }; + let self_t: &[u64; 4] = unsafe { &mem::transmute($self_expr) }; + let other_t: &[u64; 4] = unsafe { &mem::transmute($other) }; + + let overflow: u64; + unsafe { + asm!(" + mov $5, %rax + mulq $9 + mov %rax, $0 + mov %rdx, $1 + + mov $5, %rax + mulq $10 + add %rax, $1 + adc $$0, %rdx + mov %rdx, $2 + + mov $5, %rax + mulq $11 + add %rax, $2 + adc $$0, %rdx + mov %rdx, $3 + + mov $5, %rax + mulq $12 + add %rax, $3 + adc $$0, %rdx + mov %rdx, %rcx + + mov $6, %rax + mulq $9 + add %rax, $1 + adc %rdx, $2 + adc $$0, $3 + adc $$0, %rcx + + mov $6, %rax + mulq $10 + add %rax, $2 + adc %rdx, $3 + adc $$0, %rcx + adc $$0, $3 + adc $$0, %rcx + + mov $6, %rax + mulq $11 + add %rax, $3 + adc $$0, %rdx + or %rdx, %rcx + + mov $7, %rax + mulq $9 + add %rax, $2 + adc %rdx, $3 + adc $$0, %rcx + + mov $7, %rax + mulq $10 + add %rax, $3 + adc $$0, %rdx + or %rdx, %rcx + + mov $8, %rax + mulq $9 + add %rax, $3 + or %rdx, %rcx + + cmpq $$0, %rcx + jne 2f + + popcnt $8, %rcx + jrcxz 12f + + popcnt $12, %rcx + popcnt $11, %rax + add %rax, %rcx + popcnt $10, %rax + add %rax, %rcx + jmp 2f + + 12: + popcnt $12, %rcx + jrcxz 11f + + popcnt $7, %rcx + popcnt $6, %rax + add %rax, %rcx + + cmpq $$0, %rcx + jne 2f + + 11: + popcnt $11, %rcx + jrcxz 2f + popcnt $7, %rcx + + 2: + " + : /* $0 */ "={r8}"(result[0]), /* $1 */ "={r9}"(result[1]), /* $2 */ "={r10}"(result[2]), + /* $3 */ "={r11}"(result[3]), /* $4 */ "={rcx}"(overflow) + + : /* $5 */ "m"(self_t[0]), /* $6 */ "m"(self_t[1]), /* $7 */ "m"(self_t[2]), + /* $8 */ "m"(self_t[3]), /* $9 */ "m"(other_t[0]), /* $10 */ "m"(other_t[1]), + /* $11 */ "m"(other_t[2]), /* $12 */ "m"(other_t[3]) + : "rax", "rdx" + : + + ); + } + (U256(result), overflow > 0) + }); + ($name:ident, $n_words:expr, $self_expr: expr, $other: expr) => ( + uint_overflowing_mul_reg!($name, $n_words, $self_expr, $other) + ) +} + +#[cfg(not(all(feature="x64asm", target_arch="x86_64")))] +macro_rules! uint_overflowing_mul { + ($name:ident, $n_words: expr, $self_expr: expr, $other: expr) => ({ + uint_overflowing_mul_reg!($name, $n_words, $self_expr, $other) + }) +} + +macro_rules! uint_overflowing_mul_reg { + ($name:ident, $n_words: expr, $self_expr: expr, $other: expr) => ({ + let mut res = $name::from(0u64); + let mut overflow = false; + // TODO: be more efficient about this + for i in 0..(2 * $n_words) { + let v = overflowing!($self_expr.overflowing_mul_u32(($other >> (32 * i)).low_u32()), overflow); + let res2 = overflowing!(v.overflowing_shl(32 * i as u32), overflow); + res = overflowing!(res.overflowing_add(res2), overflow); + } + (res, overflow) + }) +} + macro_rules! overflowing { ($op: expr, $overflow: expr) => ( { @@ -297,50 +637,20 @@ macro_rules! construct_uint { (res, overflow) } + /// Optimized instructions + #[inline(always)] fn overflowing_add(self, other: $name) -> ($name, bool) { - let $name(ref me) = self; - let $name(ref you) = other; - let mut ret = [0u64; $n_words]; - let mut carry = [0u64; $n_words]; - let mut b_carry = false; - let mut overflow = false; - - for i in 0..$n_words { - ret[i] = me[i].wrapping_add(you[i]); - - if ret[i] < me[i] { - if i < $n_words - 1 { - carry[i + 1] = 1; - b_carry = true; - } else { - overflow = true; - } - } - } - if b_carry { - let ret = overflowing!($name(ret).overflowing_add($name(carry)), overflow); - (ret, overflow) - } else { - ($name(ret), overflow) - } + uint_overflowing_add!($name, $n_words, self, other) } + #[inline(always)] fn overflowing_sub(self, other: $name) -> ($name, bool) { - let res = overflowing!((!other).overflowing_add(From::from(1u64))); - let res = overflowing!(self.overflowing_add(res)); - (res, self < other) + uint_overflowing_sub!($name, $n_words, self, other) } + #[inline(always)] fn overflowing_mul(self, other: $name) -> ($name, bool) { - let mut res = $name::from(0u64); - let mut overflow = false; - // TODO: be more efficient about this - for i in 0..(2 * $n_words) { - let v = overflowing!(self.overflowing_mul_u32((other >> (32 * i)).low_u32()), overflow); - let res2 = overflowing!(v.overflowing_shl(32 * i as u32), overflow); - res = overflowing!(res.overflowing_add(res2), overflow); - } - (res, overflow) + uint_overflowing_mul!($name, $n_words, self, other) } fn overflowing_div(self, other: $name) -> ($name, bool) { @@ -391,6 +701,7 @@ macro_rules! construct_uint { } impl $name { + #[allow(dead_code)] // not used when multiplied with inline assembly /// Multiplication by u32 fn mul_u32(self, other: u32) -> Self { let $name(ref arr) = self; @@ -412,6 +723,7 @@ macro_rules! construct_uint { $name(ret) + $name(carry) } + #[allow(dead_code)] // not used when multiplied with inline assembly /// Overflowing multiplication by u32 fn overflowing_mul_u32(self, other: u32) -> (Self, bool) { let $name(ref arr) = self; @@ -455,7 +767,7 @@ macro_rules! construct_uint { self.to_bytes(&mut bytes); let len = cmp::max((self.bits() + 7) / 8, 1); hex.push_str(bytes[bytes.len() - len..].to_hex().as_ref()); - serializer.visit_str(hex.as_ref()) + serializer.serialize_str(hex.as_ref()) } } @@ -535,23 +847,9 @@ macro_rules! construct_uint { type Output = $name; fn add(self, other: $name) -> $name { - let $name(ref me) = self; - let $name(ref you) = other; - let mut ret = [0u64; $n_words]; - let mut carry = [0u64; $n_words]; - let mut b_carry = false; - for i in 0..$n_words { - if i < $n_words - 1 { - ret[i] = me[i].wrapping_add(you[i]); - if ret[i] < me[i] { - carry[i + 1] = 1; - b_carry = true; - } - } else { - ret[i] = me[i] + you[i]; - } - } - if b_carry { $name(ret) + $name(carry) } else { $name(ret) } + let (result, overflow) = self.overflowing_add(other); + panic_on_overflow!(overflow); + result } } @@ -560,9 +858,9 @@ macro_rules! construct_uint { #[inline] fn sub(self, other: $name) -> $name { - panic_on_overflow!(self < other); - let res = overflowing!((!other).overflowing_add(From::from(1u64))); - overflowing!(self.overflowing_add(res)) + let (result, overflow) = self.overflowing_sub(other); + panic_on_overflow!(overflow); + result } } @@ -570,15 +868,9 @@ macro_rules! construct_uint { type Output = $name; fn mul(self, other: $name) -> $name { - let mut res = $name::from(0u64); - // TODO: be more efficient about this - for i in 0..(2 * $n_words) { - let v = self.mul_u32((other >> (32 * i)).low_u32()); - let (r, overflow) = v.overflowing_shl(32 * i as u32); - panic_on_overflow!(overflow); - res = res + r; - } - res + let (result, overflow) = self.overflowing_mul(other); + panic_on_overflow!(overflow); + result } } @@ -1171,8 +1463,6 @@ mod tests { ); } - - #[test] #[should_panic] pub fn uint256_mul_overflow_panic() { @@ -1291,5 +1581,252 @@ mod tests { fn display_uint_zero() { assert_eq!(format!("{}", U256::from(0)), "0"); } + + #[test] + fn u512_multi_adds() { + let (result, _) = U512([0, 0, 0, 0, 0, 0, 0, 0]).overflowing_add(U512([0, 0, 0, 0, 0, 0, 0, 0])); + assert_eq!(result, U512([0, 0, 0, 0, 0, 0, 0, 0])); + + let (result, _) = U512([1, 0, 0, 0, 0, 0, 0, 1]).overflowing_add(U512([1, 0, 0, 0, 0, 0, 0, 1])); + assert_eq!(result, U512([2, 0, 0, 0, 0, 0, 0, 2])); + + let (result, _) = U512([0, 0, 0, 0, 0, 0, 0, 1]).overflowing_add(U512([0, 0, 0, 0, 0, 0, 0, 1])); + assert_eq!(result, U512([0, 0, 0, 0, 0, 0, 0, 2])); + + let (result, _) = U512([0, 0, 0, 0, 0, 0, 2, 1]).overflowing_add(U512([0, 0, 0, 0, 0, 0, 3, 1])); + assert_eq!(result, U512([0, 0, 0, 0, 0, 0, 5, 2])); + + let (result, _) = U512([1, 2, 3, 4, 5, 6, 7, 8]).overflowing_add(U512([9, 10, 11, 12, 13, 14, 15, 16])); + assert_eq!(result, U512([10, 12, 14, 16, 18, 20, 22, 24])); + + let (_, overflow) = U512([0, 0, 0, 0, 0, 0, 2, 1]).overflowing_add(U512([0, 0, 0, 0, 0, 0, 3, 1])); + assert!(!overflow); + + let (_, overflow) = U512([::std::u64::MAX, ::std::u64::MAX, ::std::u64::MAX, ::std::u64::MAX, ::std::u64::MAX, ::std::u64::MAX, ::std::u64::MAX, ::std::u64::MAX]) + .overflowing_add(U512([::std::u64::MAX, ::std::u64::MAX, ::std::u64::MAX, ::std::u64::MAX, ::std::u64::MAX, ::std::u64::MAX, ::std::u64::MAX, ::std::u64::MAX])); + assert!(overflow); + + let (_, overflow) = U512([0, 0, 0, 0, 0, 0, 0, ::std::u64::MAX]) + .overflowing_add(U512([0, 0, 0, 0, 0, 0, 0, ::std::u64::MAX])); + assert!(overflow); + + let (_, overflow) = U512([0, 0, 0, 0, 0, 0, 0, ::std::u64::MAX]) + .overflowing_add(U512([0, 0, 0, 0, 0, 0, 0, 0])); + assert!(!overflow); + } + + #[test] + fn u256_multi_adds() { + let (result, _) = U256([0, 0, 0, 0]).overflowing_add(U256([0, 0, 0, 0])); + assert_eq!(result, U256([0, 0, 0, 0])); + + let (result, _) = U256([0, 0, 0, 1]).overflowing_add(U256([0, 0, 0, 1])); + assert_eq!(result, U256([0, 0, 0, 2])); + + let (result, overflow) = U256([0, 0, 2, 1]).overflowing_add(U256([0, 0, 3, 1])); + assert_eq!(result, U256([0, 0, 5, 2])); + assert!(!overflow); + + let (_, overflow) = U256([::std::u64::MAX, ::std::u64::MAX, ::std::u64::MAX, ::std::u64::MAX]) + .overflowing_add(U256([::std::u64::MAX, ::std::u64::MAX, ::std::u64::MAX, ::std::u64::MAX])); + assert!(overflow); + + let (_, overflow) = U256([0, 0, 0, ::std::u64::MAX]).overflowing_add(U256([0, 0, 0, ::std::u64::MAX])); + assert!(overflow); + } + + + #[test] + fn u256_multi_subs() { + let (result, _) = U256([0, 0, 0, 0]).overflowing_sub(U256([0, 0, 0, 0])); + assert_eq!(result, U256([0, 0, 0, 0])); + + let (result, _) = U256([0, 0, 0, 1]).overflowing_sub(U256([0, 0, 0, 1])); + assert_eq!(result, U256([0, 0, 0, 0])); + + let (_, overflow) = U256([0, 0, 2, 1]).overflowing_sub(U256([0, 0, 3, 1])); + assert!(overflow); + + let (result, overflow) = U256([::std::u64::MAX, ::std::u64::MAX, ::std::u64::MAX, ::std::u64::MAX]) + .overflowing_sub(U256([::std::u64::MAX/2, ::std::u64::MAX/2, ::std::u64::MAX/2, ::std::u64::MAX/2])); + assert!(!overflow); + assert_eq!(U256([::std::u64::MAX/2+1, ::std::u64::MAX/2+1, ::std::u64::MAX/2+1, ::std::u64::MAX/2+1]), result); + + let (result, overflow) = U256([0, 0, 0, 1]).overflowing_sub(U256([0, 0, 1, 0])); + assert!(!overflow); + assert_eq!(U256([0, 0, ::std::u64::MAX, 0]), result); + + let (result, overflow) = U256([0, 0, 0, 1]).overflowing_sub(U256([1, 0, 0, 0])); + assert!(!overflow); + assert_eq!(U256([::std::u64::MAX, ::std::u64::MAX, ::std::u64::MAX, 0]), result); + } + + #[test] + fn u512_multi_subs() { + let (result, _) = U512([0, 0, 0, 0, 0, 0, 0, 0]).overflowing_sub(U512([0, 0, 0, 0, 0, 0, 0, 0])); + assert_eq!(result, U512([0, 0, 0, 0, 0, 0, 0, 0])); + + let (result, _) = U512([10, 9, 8, 7, 6, 5, 4, 3]).overflowing_sub(U512([9, 8, 7, 6, 5, 4, 3, 2])); + assert_eq!(result, U512([1, 1, 1, 1, 1, 1, 1, 1])); + + let (_, overflow) = U512([10, 9, 8, 7, 6, 5, 4, 3]).overflowing_sub(U512([9, 8, 7, 6, 5, 4, 3, 2])); + assert!(!overflow); + + let (_, overflow) = U512([9, 8, 7, 6, 5, 4, 3, 2]).overflowing_sub(U512([10, 9, 8, 7, 6, 5, 4, 3])); + assert!(overflow); + } + + #[test] + fn u256_multi_carry_all() { + let (result, _) = U256([::std::u64::MAX, 0, 0, 0]).overflowing_mul(U256([::std::u64::MAX, 0, 0, 0])); + assert_eq!(U256([1, ::std::u64::MAX-1, 0, 0]), result); + + let (result, _) = U256([0, ::std::u64::MAX, 0, 0]).overflowing_mul(U256([::std::u64::MAX, 0, 0, 0])); + assert_eq!(U256([0, 1, ::std::u64::MAX-1, 0]), result); + + let (result, _) = U256([::std::u64::MAX, ::std::u64::MAX, 0, 0]).overflowing_mul(U256([::std::u64::MAX, 0, 0, 0])); + assert_eq!(U256([1, ::std::u64::MAX, ::std::u64::MAX-1, 0]), result); + + let (result, _) = U256([::std::u64::MAX, 0, 0, 0]).overflowing_mul(U256([::std::u64::MAX, ::std::u64::MAX, 0, 0])); + assert_eq!(U256([1, ::std::u64::MAX, ::std::u64::MAX-1, 0]), result); + + let (result, _) = U256([::std::u64::MAX, ::std::u64::MAX, 0, 0]) + .overflowing_mul(U256([::std::u64::MAX, ::std::u64::MAX, 0, 0])); + assert_eq!(U256([1, 0, ::std::u64::MAX-1, ::std::u64::MAX]), result); + + let (result, _) = U256([::std::u64::MAX, 0, 0, 0]).overflowing_mul(U256([::std::u64::MAX, ::std::u64::MAX, ::std::u64::MAX, 0])); + assert_eq!(U256([1, ::std::u64::MAX, ::std::u64::MAX, ::std::u64::MAX-1]), result); + + let (result, _) = U256([::std::u64::MAX, ::std::u64::MAX, ::std::u64::MAX, 0]).overflowing_mul(U256([::std::u64::MAX, 0, 0, 0])); + assert_eq!(U256([1, ::std::u64::MAX, ::std::u64::MAX, ::std::u64::MAX-1]), result); + + let (result, _) = U256([::std::u64::MAX, 0, 0, 0]).overflowing_mul( + U256([::std::u64::MAX, ::std::u64::MAX, ::std::u64::MAX, ::std::u64::MAX])); + assert_eq!(U256([1, ::std::u64::MAX, ::std::u64::MAX, ::std::u64::MAX]), result); + + let (result, _) = U256([::std::u64::MAX, ::std::u64::MAX, ::std::u64::MAX, ::std::u64::MAX]) + .overflowing_mul(U256([::std::u64::MAX, 0, 0, 0])); + assert_eq!(U256([1, ::std::u64::MAX, ::std::u64::MAX, ::std::u64::MAX]), result); + + let (result, _) = U256([::std::u64::MAX, ::std::u64::MAX, ::std::u64::MAX, 0]) + .overflowing_mul(U256([::std::u64::MAX, ::std::u64::MAX, 0, 0])); + assert_eq!(U256([1, 0, ::std::u64::MAX, ::std::u64::MAX-1]), result); + + let (result, _) = U256([::std::u64::MAX, ::std::u64::MAX, 0, 0]) + .overflowing_mul(U256([::std::u64::MAX, ::std::u64::MAX, ::std::u64::MAX, 0])); + assert_eq!(U256([1, 0, ::std::u64::MAX, ::std::u64::MAX-1]), result); + + let (result, _) = U256([::std::u64::MAX, ::std::u64::MAX, ::std::u64::MAX, ::std::u64::MAX]) + .overflowing_mul(U256([::std::u64::MAX, ::std::u64::MAX, 0, 0])); + assert_eq!(U256([1, 0, ::std::u64::MAX, ::std::u64::MAX]), result); + + let (result, _) = U256([::std::u64::MAX, ::std::u64::MAX, 0, 0]) + .overflowing_mul(U256([::std::u64::MAX, ::std::u64::MAX, ::std::u64::MAX, ::std::u64::MAX])); + assert_eq!(U256([1, 0, ::std::u64::MAX, ::std::u64::MAX]), result); + + let (result, _) = U256([::std::u64::MAX, ::std::u64::MAX, ::std::u64::MAX, 0]) + .overflowing_mul(U256([::std::u64::MAX, ::std::u64::MAX, ::std::u64::MAX, 0])); + assert_eq!(U256([1, 0, 0, ::std::u64::MAX-1]), result); + + let (result, _) = U256([::std::u64::MAX, ::std::u64::MAX, ::std::u64::MAX, 0]) + .overflowing_mul(U256([::std::u64::MAX, ::std::u64::MAX, ::std::u64::MAX, ::std::u64::MAX])); + assert_eq!(U256([1, 0, 0, ::std::u64::MAX]), result); + + let (result, _) = U256([::std::u64::MAX, ::std::u64::MAX, ::std::u64::MAX, ::std::u64::MAX]) + .overflowing_mul(U256([::std::u64::MAX, ::std::u64::MAX, ::std::u64::MAX, 0])); + assert_eq!(U256([1, 0, 0, ::std::u64::MAX]), result); + + let (result, _) = U256([0, 0, 0, ::std::u64::MAX]).overflowing_mul(U256([0, 0, 0, ::std::u64::MAX])); + assert_eq!(U256([0, 0, 0, 0]), result); + + let (result, _) = U256([1, 0, 0, 0]).overflowing_mul(U256([0, 0, 0, ::std::u64::MAX])); + assert_eq!(U256([0, 0, 0, ::std::u64::MAX]), result); + + let (result, _) = U256([::std::u64::MAX, ::std::u64::MAX, ::std::u64::MAX, ::std::u64::MAX]) + .overflowing_mul(U256([::std::u64::MAX, ::std::u64::MAX, ::std::u64::MAX, ::std::u64::MAX])); + assert_eq!(U256([1, 0, 0, 0]), result); + } + + #[test] + fn u256_multi_muls() { + use hash::*; + + let (result, _) = U256([0, 0, 0, 0]).overflowing_mul(U256([0, 0, 0, 0])); + assert_eq!(U256([0, 0, 0, 0]), result); + + let (result, _) = U256([1, 0, 0, 0]).overflowing_mul(U256([1, 0, 0, 0])); + assert_eq!(U256([1, 0, 0, 0]), result); + + let (result, _) = U256([5, 0, 0, 0]).overflowing_mul(U256([5, 0, 0, 0])); + assert_eq!(U256([25, 0, 0, 0]), result); + + let (result, _) = U256([0, 5, 0, 0]).overflowing_mul(U256([0, 5, 0, 0])); + assert_eq!(U256([0, 0, 25, 0]), result); + + let (result, _) = U256([0, 0, 0, 1]).overflowing_mul(U256([1, 0, 0, 0])); + assert_eq!(U256([0, 0, 0, 1]), result); + + let (result, _) = U256([0, 0, 0, 5]).overflowing_mul(U256([2, 0, 0, 0])); + assert_eq!(U256([0, 0, 0, 10]), result); + + let (result, _) = U256([0, 0, 1, 0]).overflowing_mul(U256([0, 5, 0, 0])); + assert_eq!(U256([0, 0, 0, 5]), result); + + let (result, _) = U256([0, 0, 8, 0]).overflowing_mul(U256([0, 0, 7, 0])); + assert_eq!(U256([0, 0, 0, 0]), result); + + let (result, _) = U256([2, 0, 0, 0]).overflowing_mul(U256([0, 5, 0, 0])); + assert_eq!(U256([0, 10, 0, 0]), result); + + let (result, _) = U256([1, 0, 0, 0]).overflowing_mul(U256([0, 0, 0, ::std::u64::MAX])); + assert_eq!(U256([0, 0, 0, ::std::u64::MAX]), result); + + let x1 = U256::from_str("0000000000000000000000000000000000000000000000000000012365124623").unwrap(); + let x2sqr_right = U256::from_str("000000000000000000000000000000000000000000014baeef72e0378e2328c9").unwrap(); + let x1sqr = x1 * x1; + assert_eq!(H256::from(x2sqr_right), H256::from(x1sqr)); + let x1cube = x1sqr * x1; + let x1cube_right = U256::from_str("0000000000000000000000000000000001798acde139361466f712813717897b").unwrap(); + assert_eq!(H256::from(x1cube_right), H256::from(x1cube)); + let x1quad = x1cube * x1; + let x1quad_right = U256::from_str("000000000000000000000001adbdd6bd6ff027485484b97f8a6a4c7129756dd1").unwrap(); + assert_eq!(H256::from(x1quad_right), H256::from(x1quad)); + let x1penta = x1quad * x1; + let x1penta_right = U256::from_str("00000000000001e92875ac24be246e1c57e0507e8c46cc8d233b77f6f4c72993").unwrap(); + assert_eq!(H256::from(x1penta_right), H256::from(x1penta)); + let x1septima = x1penta * x1; + let x1septima_right = U256::from_str("00022cca1da3f6e5722b7d3cc5bbfb486465ebc5a708dd293042f932d7eee119").unwrap(); + assert_eq!(H256::from(x1septima_right), H256::from(x1septima)); + } + + #[test] + fn u256_multi_muls_overflow() { + let (_, overflow) = U256([1, 0, 0, 0]).overflowing_mul(U256([0, 0, 0, 0])); + assert!(!overflow); + + let (_, overflow) = U256([1, 0, 0, 0]).overflowing_mul(U256([0, 0, 0, ::std::u64::MAX])); + assert!(!overflow); + + let (_, overflow) = U256([0, 1, 0, 0]).overflowing_mul(U256([0, 0, 0, ::std::u64::MAX])); + assert!(overflow); + + let (_, overflow) = U256([0, 1, 0, 0]).overflowing_mul(U256([0, 1, 0, 0])); + assert!(!overflow); + + let (_, overflow) = U256([0, 1, 0, ::std::u64::MAX]).overflowing_mul(U256([0, 1, 0, ::std::u64::MAX])); + assert!(overflow); + + let (_, overflow) = U256([0, ::std::u64::MAX, 0, 0]).overflowing_mul(U256([0, ::std::u64::MAX, 0, 0])); + assert!(!overflow); + + let (_, overflow) = U256([1, 0, 0, 0]).overflowing_mul(U256([10, 0, 0, 0])); + assert!(!overflow); + + let (_, overflow) = U256([2, 0, 0, 0]).overflowing_mul(U256([0, 0, 0, ::std::u64::MAX / 2])); + assert!(!overflow); + + let (_, overflow) = U256([0, 0, 8, 0]).overflowing_mul(U256([0, 0, 7, 0])); + assert!(overflow); + } }