grassrootseconomics/openethereum
8
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

merge accountdb migration

Browse Source
This commit is contained in:
Robert Habermeier
2016-07-09 17:33:14 +02:00
parent 465aae71e7 1f885254e1
commit 36dfa4743e
234 changed files with 5015 additions and 30967 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
util/Cargo.toml
View File

@@ -27,16 +27,16 @@ itertools = "0.4"
crossbeam = "0.2"
slab = "0.2"
sha3 = { path = "sha3" }
serde = "0.7.0"
clippy = { version = "0.0.77", optional = true}
json-tests = { path = "json-tests" }
igd = "0.4.2"
clippy = { version = "0.0.78", optional = true}
igd = "0.5.0"
ethcore-devtools = { path = "../devtools" }
libc = "0.2.7"
vergen = "0.1"
target_info = "0.1"
bigint = { path = "bigint" }
chrono = "0.2"
using_queue = { path = "using_queue" }
table = { path = "table" }
ansi_term = "0.7"
[features]

1
util/bigint/Cargo.toml
View File

@@ -12,7 +12,6 @@ rustc_version = "0.1"
[dependencies]
rustc-serialize = "0.3"
serde = "0.7.0"
heapsize = "0.3"
[features]

1
util/bigint/src/lib.rs
View File

@@ -17,7 +17,6 @@
#![cfg_attr(asm_available, feature(asm))]
extern crate rustc_serialize;
extern crate serde;
#[macro_use] extern crate heapsize;
pub mod uint;

91
util/bigint/src/uint.rs
View File

@@ -39,22 +39,22 @@
#[cfg(all(asm_available, target_arch="x86_64"))]
use std::mem;
use std::fmt;
use std::cmp;
use std::str::{FromStr};
use std::convert::From;
use std::hash::{Hash, Hasher};
use std::hash::Hash;
use std::ops::*;
use std::cmp::*;
use serde;
use rustc_serialize::hex::{FromHex, FromHexError, ToHex};
use rustc_serialize::hex::{FromHex, FromHexError};
/// Conversion from decimal string error
#[derive(Debug, PartialEq)]
pub enum FromDecStrErr {
/// Char not from range 0-9
InvalidCharacter,
/// Value does not fit into type
InvalidLength
InvalidLength,
}
macro_rules! impl_map_from {
@@ -562,8 +562,11 @@ macro_rules! construct_uint {
impl Uint for $name {
/// TODO: optimize, throw appropriate err
fn from_dec_str(value: &str) -> Result<Self, FromDecStrErr> {
if value.bytes().any(|b| b < 48 && b > 57) {
return Err(FromDecStrErr::InvalidCharacter)
}
let mut res = Self::default();
for b in value.bytes().map(|b| b - 48) {
let (r, overflow) = res.overflowing_mul_u32(10);
@@ -649,7 +652,7 @@ macro_rules! construct_uint {
fn exp10(n: usize) -> Self {
match n {
0 => Self::from(1u64),
_ => Self::exp10(n - 1) * Self::from(10u64)
_ => Self::exp10(n - 1).mul_u32(10)
}
}
@@ -757,16 +760,16 @@ macro_rules! construct_uint {
}
impl $name {
#[allow(dead_code)] // not used when multiplied with inline assembly
/// Multiplication by u32
#[allow(dead_code)] // not used when multiplied with inline assembly
fn mul_u32(self, other: u32) -> Self {
let (ret, overflow) = self.overflowing_mul_u32(other);
panic_on_overflow!(overflow);
ret
}
#[allow(dead_code)] // not used when multiplied with inline assembly
/// Overflowing multiplication by u32
#[allow(dead_code)] // not used when multiplied with inline assembly
fn overflowing_mul_u32(self, other: u32) -> (Self, bool) {
let $name(ref arr) = self;
let mut ret = [0u64; $n_words];
@@ -789,44 +792,6 @@ macro_rules! construct_uint {
}
}
impl serde::Serialize for $name {
fn serialize<S>(&self, serializer: &mut S) -> Result<(), S::Error>
where S: serde::Serializer {
let mut hex = "0x".to_owned();
let mut bytes = [0u8; 8 * $n_words];
self.to_raw_bytes(&mut bytes);
let len = cmp::max((self.bits() + 7) / 8, 1);
hex.push_str(bytes[bytes.len() - len..].to_hex().as_ref());
serializer.serialize_str(hex.as_ref())
}
}
impl serde::Deserialize for $name {
fn deserialize<D>(deserializer: &mut D) -> Result<$name, D::Error>
where D: serde::Deserializer {
struct UintVisitor;
impl serde::de::Visitor for UintVisitor {
type Value = $name;
fn visit_str<E>(&mut self, value: &str) -> Result<Self::Value, E> where E: serde::Error {
// 0x + len
if value.len() > 2 + $n_words * 16 || value.len() < 2 {
return Err(serde::Error::custom("Invalid length."));
}
$name::from_str(&value[2..]).map_err(|_| serde::Error::custom("Invalid hex value."))
}
fn visit_string<E>(&mut self, value: String) -> Result<Self::Value, E> where E: serde::Error {
self.visit_str(value.as_ref())
}
}
deserializer.deserialize(UintVisitor)
}
}
impl From<u64> for $name {
fn from(value: u64) -> $name {
let mut ret = [0; $n_words];
@@ -959,8 +924,6 @@ macro_rules! construct_uint {
}
}
// TODO: optimise and traitify.
impl BitAnd<$name> for $name {
type Output = $name;
@@ -1031,7 +994,7 @@ macro_rules! construct_uint {
// shift
for i in word_shift..$n_words {
ret[i] += original[i - word_shift] << bit_shift;
ret[i] = original[i - word_shift] << bit_shift;
}
// carry
if bit_shift > 0 {
@@ -1052,14 +1015,18 @@ macro_rules! construct_uint {
let word_shift = shift / 64;
let bit_shift = shift % 64;
// shift
for i in word_shift..$n_words {
// Shift
ret[i - word_shift] += original[i] >> bit_shift;
// Carry
if bit_shift > 0 && i < $n_words - 1 {
ret[i - word_shift] += original[i + 1] << (64 - bit_shift);
ret[i - word_shift] = original[i] >> bit_shift;
}
// Carry
if bit_shift > 0 {
for i in word_shift+1..$n_words {
ret[i - word_shift - 1] += original[i] << (64 - bit_shift);
}
}
$name(ret)
}
}
@@ -1430,12 +1397,6 @@ impl From<U256> for u32 {
}
}
/// Constant value of `U256::zero()` that can be used for a reference saving an additional instance creation.
pub const ZERO_U256: U256 = U256([0x00u64; 4]);
/// Constant value of `U256::one()` that can be used for a reference saving an additional instance creation.
pub const ONE_U256: U256 = U256([0x01u64, 0x00u64, 0x00u64, 0x00u64]);
known_heap_size!(0, U128, U256);
#[cfg(test)]
@@ -1578,7 +1539,13 @@ mod tests {
assert_eq!(U256::from(105u8) / U256::from(5u8), U256::from(21u8));
let div = mult / U256::from(300u16);
assert_eq!(div, U256([0x9F30411021524112u64, 0x0001BD5B7DDFBD5A, 0, 0]));
//// TODO: bit inversion
let a = U256::from_str("ff000000000000000000000000000000000000000000000000000000000000d1").unwrap();
let b = U256::from_str("00ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff2e").unwrap();
println!("{:x}", a);
println!("{:x}", b);
assert_eq!(!a, b);
assert_eq!(a, !b);
}
#[test]

15
util/json-tests/README.md
View File

@@ -1,15 +0,0 @@
# How to write json test file?
Cause it's very hard to write generic json test files, each subdirectory should follow its own
convention. BUT all json files `within` same directory should be consistent.
### Test files should always contain a single test with input and output.
```json
{
input: ...,
output: ...
}
```
As a reference, please use trietests.

BIN
util/json-tests/json/.DS_Store vendored
View File

Binary file not shown.

39
util/json-tests/json/rlp/README.md
View File

@@ -1,39 +0,0 @@
# Rlp tests guideline
Rlp can be tested in various ways. It can encode/decode a value or an array of values. Let's start with encoding.
Each operation must have field:
- `operation` - `append`, `append_list`, `append_empty` or `append_raw`
Additionally `append` and `append_raw` must additionally define a `value` field:
- `value` - data
Also `append_raw` and `append_list` requires `len` field
- `len` - integer
### Encoding Test Example
```json
{
"input":
[
{
"operation": "append_list",
"len": 2
},
{
"operation": "append",
"value": "cat"
},
{
"operation": "append",
"value": "dog"
}
],
"output": "0xc88363617183646f67"
}
```

10
util/json-tests/json/rlp/stream/bytestring0.json
View File

@@ -1,10 +0,0 @@
{
"input":
[
{
"operation": "append",
"value": "\u0000"
}
],
"output": "0x00"
}

10
util/json-tests/json/rlp/stream/bytestring1.json
View File

@@ -1,10 +0,0 @@
{
"input":
[
{
"operation": "append",
"value": "\u0001"
}
],
"output": "0x01"
}

10
util/json-tests/json/rlp/stream/bytestring7.json
View File

@@ -1,10 +0,0 @@
{
"input":
[
{
"operation": "append",
"value": "\u007f"
}
],
"output": "0x7f"
}

10
util/json-tests/json/rlp/stream/catdog.json
View File

@@ -1,10 +0,0 @@
{
"input":
[
{
"operation": "append",
"value": "\u0000"
}
],
"output": "0x00"
}

9
util/json-tests/json/rlp/stream/empty.json
View File

@@ -1,9 +0,0 @@
{
"input":
[
{
"operation": "append_empty"
}
],
"output": "0x80"
}

38
util/json-tests/json/rlp/stream/empty_lists.json
View File

@@ -1,38 +0,0 @@
{
"input":
[
{
"operation": "append_list",
"len": 3
},
{
"operation": "append_list",
"len": 0
},
{
"operation": "append_list",
"len": 1
},
{
"operation": "append_list",
"len": 0
},
{
"operation": "append_list",
"len": 2
},
{
"operation": "append_list",
"len": 0
},
{
"operation": "append_list",
"len": 1
},
{
"operation": "append_list",
"len": 0
}
],
"output": "0xc7c0c1c0c3c0c1c0"
}

10
util/json-tests/json/rlp/stream/integer.json
View File

@@ -1,10 +0,0 @@
{
"input":
[
{
"operation": "append",
"value": "0x0400"
}
],
"output": "0x820400"
}

22
util/json-tests/json/rlp/stream/list_of_empty_data.json
View File

@@ -1,22 +0,0 @@
{
"input":
[
{
"operation": "append_list",
"len": 3
},
{
"operation": "append",
"value": ""
},
{
"operation": "append",
"value": ""
},
{
"operation": "append",
"value": ""
}
],
"output": "0xc3808080"
}

19
util/json-tests/json/rlp/stream/list_of_empty_data2.json
View File

@@ -1,19 +0,0 @@
{
"input":
[
{
"operation": "append_list",
"len": 3
},
{
"operation": "append_empty"
},
{
"operation": "append_empty"
},
{
"operation": "append_empty"
}
],
"output": "0xc3808080"
}

521
util/json-tests/json/rlp/stream/longlist.json
View File

@@ -1,521 +0,0 @@
{
"input": [
{
"operation": "append_list",
"len": 32
},
{
"operation": "append_list",
"len": 3
},
{
"operation": "append",
"value": "asdf"
},
{
"operation": "append",
"value": "qwer"
},
{
"operation": "append",
"value": "zxcv"
},
{
"operation": "append_list",
"len": 3
},
{
"operation": "append",
"value": "asdf"
},
{
"operation": "append",
"value": "qwer"
},
{
"operation": "append",
"value": "zxcv"
},
{
"operation": "append_list",
"len": 3
},
{
"operation": "append",
"value": "asdf"
},
{
"operation": "append",
"value": "qwer"
},
{
"operation": "append",
"value": "zxcv"
},
{
"operation": "append_list",
"len": 3
},
{
"operation": "append",
"value": "asdf"
},
{
"operation": "append",
"value": "qwer"
},
{
"operation": "append",
"value": "zxcv"
},
{
"operation": "append_list",
"len": 3
},
{
"operation": "append",
"value": "asdf"
},
{
"operation": "append",
"value": "qwer"
},
{
"operation": "append",
"value": "zxcv"
},
{
"operation": "append_list",
"len": 3
},
{
"operation": "append",
"value": "asdf"
},
{
"operation": "append",
"value": "qwer"
},
{
"operation": "append",
"value": "zxcv"
},
{
"operation": "append_list",
"len": 3
},
{
"operation": "append",
"value": "asdf"
},
{
"operation": "append",
"value": "qwer"
},
{
"operation": "append",
"value": "zxcv"
},
{
"operation": "append_list",
"len": 3
},
{
"operation": "append",
"value": "asdf"
},
{
"operation": "append",
"value": "qwer"
},
{
"operation": "append",
"value": "zxcv"
},
{
"operation": "append_list",
"len": 3
},
{
"operation": "append",
"value": "asdf"
},
{
"operation": "append",
"value": "qwer"
},
{
"operation": "append",
"value": "zxcv"
},
{
"operation": "append_list",
"len": 3
},
{
"operation": "append",
"value": "asdf"
},
{
"operation": "append",
"value": "qwer"
},
{
"operation": "append",
"value": "zxcv"
},
{
"operation": "append_list",
"len": 3
},
{
"operation": "append",
"value": "asdf"
},
{
"operation": "append",
"value": "qwer"
},
{
"operation": "append",
"value": "zxcv"
},
{
"operation": "append_list",
"len": 3
},
{
"operation": "append",
"value": "asdf"
},
{
"operation": "append",
"value": "qwer"
},
{
"operation": "append",
"value": "zxcv"
},
{
"operation": "append_list",
"len": 3
},
{
"operation": "append",
"value": "asdf"
},
{
"operation": "append",
"value": "qwer"
},
{
"operation": "append",
"value": "zxcv"
},
{
"operation": "append_list",
"len": 3
},
{
"operation": "append",
"value": "asdf"
},
{
"operation": "append",
"value": "qwer"
},
{
"operation": "append",
"value": "zxcv"
},
{
"operation": "append_list",
"len": 3
},
{
"operation": "append",
"value": "asdf"
},
{
"operation": "append",
"value": "qwer"
},
{
"operation": "append",
"value": "zxcv"
},
{
"operation": "append_list",
"len": 3
},
{
"operation": "append",
"value": "asdf"
},
{
"operation": "append",
"value": "qwer"
},
{
"operation": "append",
"value": "zxcv"
},
{
"operation": "append_list",
"len": 3
},
{
"operation": "append",
"value": "asdf"
},
{
"operation": "append",
"value": "qwer"
},
{
"operation": "append",
"value": "zxcv"
},
{
"operation": "append_list",
"len": 3
},
{
"operation": "append",
"value": "asdf"
},
{
"operation": "append",
"value": "qwer"
},
{
"operation": "append",
"value": "zxcv"
},
{
"operation": "append_list",
"len": 3
},
{
"operation": "append",
"value": "asdf"
},
{
"operation": "append",
"value": "qwer"
},
{
"operation": "append",
"value": "zxcv"
},
{
"operation": "append_list",
"len": 3
},
{
"operation": "append",
"value": "asdf"
},
{
"operation": "append",
"value": "qwer"
},
{
"operation": "append",
"value": "zxcv"
},
{
"operation": "append_list",
"len": 3
},
{
"operation": "append",
"value": "asdf"
},
{
"operation": "append",
"value": "qwer"
},
{
"operation": "append",
"value": "zxcv"
},
{
"operation": "append_list",
"len": 3
},
{
"operation": "append",
"value": "asdf"
},
{
"operation": "append",
"value": "qwer"
},
{
"operation": "append",
"value": "zxcv"
},
{
"operation": "append_list",
"len": 3
},
{
"operation": "append",
"value": "asdf"
},
{
"operation": "append",
"value": "qwer"
},
{
"operation": "append",
"value": "zxcv"
},
{
"operation": "append_list",
"len": 3
},
{
"operation": "append",
"value": "asdf"
},
{
"operation": "append",
"value": "qwer"
},
{
"operation": "append",
"value": "zxcv"
},
{
"operation": "append_list",
"len": 3
},
{
"operation": "append",
"value": "asdf"
},
{
"operation": "append",
"value": "qwer"
},
{
"operation": "append",
"value": "zxcv"
},
{
"operation": "append_list",
"len": 3
},
{
"operation": "append",
"value": "asdf"
},
{
"operation": "append",
"value": "qwer"
},
{
"operation": "append",
"value": "zxcv"
},
{
"operation": "append_list",
"len": 3
},
{
"operation": "append",
"value": "asdf"
},
{
"operation": "append",
"value": "qwer"
},
{
"operation": "append",
"value": "zxcv"
},
{
"operation": "append_list",
"len": 3
},
{
"operation": "append",
"value": "asdf"
},
{
"operation": "append",
"value": "qwer"
},
{
"operation": "append",
"value": "zxcv"
},
{
"operation": "append_list",
"len": 3
},
{
"operation": "append",
"value": "asdf"
},
{
"operation": "append",
"value": "qwer"
},
{
"operation": "append",
"value": "zxcv"
},
{
"operation": "append_list",
"len": 3
},
{
"operation": "append",
"value": "asdf"
},
{
"operation": "append",
"value": "qwer"
},
{
"operation": "append",
"value": "zxcv"
},
{
"operation": "append_list",
"len": 3
},
{
"operation": "append",
"value": "asdf"
},
{
"operation": "append",
"value": "qwer"
},
{
"operation": "append",
"value": "zxcv"
},
{
"operation": "append_list",
"len": 3
},
{
"operation": "append",
"value": "asdf"
},
{
"operation": "append",
"value": "qwer"
},
{
"operation": "append",
"value": "zxcv"
}],
"output": "0xf90200cf84617364668471776572847a786376cf84617364668471776572847a786376cf84617364668471776572847a786376cf84617364668471776572847a786376cf84617364668471776572847a786376cf84617364668471776572847a786376cf84617364668471776572847a786376cf84617364668471776572847a786376cf84617364668471776572847a786376cf84617364668471776572847a786376cf84617364668471776572847a786376cf84617364668471776572847a786376cf84617364668471776572847a786376cf84617364668471776572847a786376cf84617364668471776572847a786376cf84617364668471776572847a786376cf84617364668471776572847a786376cf84617364668471776572847a786376cf84617364668471776572847a786376cf84617364668471776572847a786376cf84617364668471776572847a786376cf84617364668471776572847a786376cf84617364668471776572847a786376cf84617364668471776572847a786376cf84617364668471776572847a786376cf84617364668471776572847a786376cf84617364668471776572847a786376cf84617364668471776572847a786376cf84617364668471776572847a786376cf84617364668471776572847a786376cf84617364668471776572847a786376cf84617364668471776572847a786376"
}

10
util/json-tests/json/rlp/stream/longstring.json
View File

@@ -1,10 +0,0 @@
{
"input":
[
{
"operation": "append",
"value": "Lorem ipsum dolor sit amet, consectetur adipiscing elit. Curabitur mauris magna, suscipit sed vehicula non, iaculis faucibus tortor. Proin suscipit ultricies malesuada. Duis tortor elit, dictum quis tristique eu, ultrices at risus. Morbi a est imperdiet mi ullamcorper aliquet suscipit nec lorem. Aenean quis leo mollis, vulputate elit varius, consequat enim. Nulla ultrices turpis justo, et posuere urna consectetur nec. Proin non convallis metus. Donec tempor ipsum in mauris congue sollicitudin. Vestibulum ante ipsum primis in faucibus orci luctus et ultrices posuere cubilia Curae; Suspendisse convallis sem vel massa faucibus, eget lacinia lacus tempor. Nulla quis ultricies purus. Proin auctor rhoncus nibh condimentum mollis. Aliquam consequat enim at metus luctus, a eleifend purus egestas. Curabitur at nibh metus. Nam bibendum, neque at auctor tristique, lorem libero aliquet arcu, non interdum tellus lectus sit amet eros. Cras rhoncus, metus ac ornare cursus, dolor justo ultrices metus, at ullamcorper volutpat"
}
],
"output": "0xb904004c6f72656d20697073756d20646f6c6f722073697420616d65742c20636f6e73656374657475722061646970697363696e6720656c69742e20437572616269747572206d6175726973206d61676e612c20737573636970697420736564207665686963756c61206e6f6e2c20696163756c697320666175636962757320746f72746f722e2050726f696e20737573636970697420756c74726963696573206d616c6573756164612e204475697320746f72746f7220656c69742c2064696374756d2071756973207472697374697175652065752c20756c7472696365732061742072697375732e204d6f72626920612065737420696d70657264696574206d6920756c6c616d636f7270657220616c6971756574207375736369706974206e6563206c6f72656d2e2041656e65616e2071756973206c656f206d6f6c6c69732c2076756c70757461746520656c6974207661726975732c20636f6e73657175617420656e696d2e204e756c6c6120756c74726963657320747572706973206a7573746f2c20657420706f73756572652075726e6120636f6e7365637465747572206e65632e2050726f696e206e6f6e20636f6e76616c6c6973206d657475732e20446f6e65632074656d706f7220697073756d20696e206d617572697320636f6e67756520736f6c6c696369747564696e2e20566573746962756c756d20616e746520697073756d207072696d697320696e206661756369627573206f726369206c756374757320657420756c74726963657320706f737565726520637562696c69612043757261653b2053757370656e646973736520636f6e76616c6c69732073656d2076656c206d617373612066617563696275732c2065676574206c6163696e6961206c616375732074656d706f722e204e756c6c61207175697320756c747269636965732070757275732e2050726f696e20617563746f722072686f6e637573206e69626820636f6e64696d656e74756d206d6f6c6c69732e20416c697175616d20636f6e73657175617420656e696d206174206d65747573206c75637475732c206120656c656966656e6420707572757320656765737461732e20437572616269747572206174206e696268206d657475732e204e616d20626962656e64756d2c206e6571756520617420617563746f72207472697374697175652c206c6f72656d206c696265726f20616c697175657420617263752c206e6f6e20696e74657264756d2074656c6c7573206c65637475732073697420616d65742065726f732e20437261732072686f6e6375732c206d65747573206163206f726e617265206375727375732c20646f6c6f72206a7573746f20756c747269636573206d657475732c20617420756c6c616d636f7270657220766f6c7574706174"
}

35
util/json-tests/json/trie/README.md
View File

@@ -1,35 +0,0 @@
# Trie tests guideline
Trie test input is an array of operations. Each operation must have 2 fields:
- `operation` - string, either `insert` or `remove`
- `key` - string, or hex value prefixed with `0x`
And optional field:
- `value`- which is used by `insert` operation
### Example
```json
{
"input":
[
{
"operation": "insert",
"key": "world",
"value": "hello"
},
{
"operation": "insert",
"key": "0x1234",
"value": "ooooops"
},
{
"operation": "remove",
"key": "0x1234"
}
],
"output": "0x5991bb8c6514148a29db676a14ac506cd2cd5775ace63c30a4fe457715e9ac84"
}
```

11
util/json-tests/json/trie/basic.json
View File

@@ -1,11 +0,0 @@
{
"input":
[
{
"operation": "insert",
"key": "A",
"value": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
}
],
"output": "0xd23786fb4a010da3ce639d66d5e904a11dbc02746d1ce25029e53290cabf28ab"
}

229
util/json-tests/json/trie/branching.json
View File

@@ -1,229 +0,0 @@
{
"input": [
{
"operation": "insert",
"key": "0x04110d816c380812a427968ece99b1c963dfbce6",
"value": "something"
},
{
"operation": "insert",
"key": "0x095e7baea6a6c7c4c2dfeb977efac326af552d87",
"value": "something"
},
{
"operation": "insert",
"key": "0x0a517d755cebbf66312b30fff713666a9cb917e0",
"value": "something"
},
{
"operation": "insert",
"key": "0x24dd378f51adc67a50e339e8031fe9bd4aafab36",
"value": "something"
},
{
"operation": "insert",
"key": "0x293f982d000532a7861ab122bdc4bbfd26bf9030",
"value": "something"
},
{
"operation": "insert",
"key": "0x2cf5732f017b0cf1b1f13a1478e10239716bf6b5",
"value": "something"
},
{
"operation": "insert",
"key": "0x31c640b92c21a1f1465c91070b4b3b4d6854195f",
"value": "something"
},
{
"operation": "insert",
"key": "0x37f998764813b136ddf5a754f34063fd03065e36",
"value": "something"
},
{
"operation": "insert",
"key": "0x37fa399a749c121f8a15ce77e3d9f9bec8020d7a",
"value": "something"
},
{
"operation": "insert",
"key": "0x4f36659fa632310b6ec438dea4085b522a2dd077",
"value": "something"
},
{
"operation": "insert",
"key": "0x62c01474f089b07dae603491675dc5b5748f7049",
"value": "something"
},
{
"operation": "insert",
"key": "0x729af7294be595a0efd7d891c9e51f89c07950c7",
"value": "something"
},
{
"operation": "insert",
"key": "0x83e3e5a16d3b696a0314b30b2534804dd5e11197",
"value": "something"
},
{
"operation": "insert",
"key": "0x8703df2417e0d7c59d063caa9583cb10a4d20532",
"value": "something"
},
{
"operation": "insert",
"key": "0x8dffcd74e5b5923512916c6a64b502689cfa65e1",
"value": "something"
},
{
"operation": "insert",
"key": "0x95a4d7cccb5204733874fa87285a176fe1e9e240",
"value": "something"
},
{
"operation": "insert",
"key": "0x99b2fcba8120bedd048fe79f5262a6690ed38c39",
"value": "something"
},
{
"operation": "insert",
"key": "0xa4202b8b8afd5354e3e40a219bdc17f6001bf2cf",
"value": "something"
},
{
"operation": "insert",
"key": "0xa94f5374fce5edbc8e2a8697c15331677e6ebf0b",
"value": "something"
},
{
"operation": "insert",
"key": "0xa9647f4a0a14042d91dc33c0328030a7157c93ae",
"value": "something"
},
{
"operation": "insert",
"key": "0xaa6cffe5185732689c18f37a7f86170cb7304c2a",
"value": "something"
},
{
"operation": "insert",
"key": "0xaae4a2e3c51c04606dcb3723456e58f3ed214f45",
"value": "something"
},
{
"operation": "insert",
"key": "0xc37a43e940dfb5baf581a0b82b351d48305fc885",
"value": "something"
},
{
"operation": "insert",
"key": "0xd2571607e241ecf590ed94b12d87c94babe36db6",
"value": "something"
},
{
"operation": "insert",
"key": "0xf735071cbee190d76b704ce68384fc21e389fbe7",
"value": "something"
},
{
"operation": "remove",
"key": "0x04110d816c380812a427968ece99b1c963dfbce6"
},
{
"operation": "remove",
"key": "0x095e7baea6a6c7c4c2dfeb977efac326af552d87"
},
{
"operation": "remove",
"key": "0x0a517d755cebbf66312b30fff713666a9cb917e0"
},
{
"operation": "remove",
"key": "0x24dd378f51adc67a50e339e8031fe9bd4aafab36"
},
{
"operation": "remove",
"key": "0x293f982d000532a7861ab122bdc4bbfd26bf9030"
},
{
"operation": "remove",
"key": "0x2cf5732f017b0cf1b1f13a1478e10239716bf6b5"
},
{
"operation": "remove",
"key": "0x31c640b92c21a1f1465c91070b4b3b4d6854195f"
},
{
"operation": "remove",
"key": "0x37f998764813b136ddf5a754f34063fd03065e36"
},
{
"operation": "remove",
"key": "0x37fa399a749c121f8a15ce77e3d9f9bec8020d7a"
},
{
"operation": "remove",
"key": "0x4f36659fa632310b6ec438dea4085b522a2dd077"
},
{
"operation": "remove",
"key": "0x62c01474f089b07dae603491675dc5b5748f7049"
},
{
"operation": "remove",
"key": "0x729af7294be595a0efd7d891c9e51f89c07950c7"
},
{
"operation": "remove",
"key": "0x83e3e5a16d3b696a0314b30b2534804dd5e11197"
},
{
"operation": "remove",
"key": "0x8703df2417e0d7c59d063caa9583cb10a4d20532"
},
{
"operation": "remove",
"key": "0x8dffcd74e5b5923512916c6a64b502689cfa65e1"
},
{
"operation": "remove",
"key": "0x95a4d7cccb5204733874fa87285a176fe1e9e240"
},
{
"operation": "remove",
"key": "0x99b2fcba8120bedd048fe79f5262a6690ed38c39"
},
{
"operation": "remove",
"key": "0xa4202b8b8afd5354e3e40a219bdc17f6001bf2cf"
},
{
"operation": "remove",
"key": "0xa94f5374fce5edbc8e2a8697c15331677e6ebf0b"
},
{
"operation": "remove",
"key": "0xa9647f4a0a14042d91dc33c0328030a7157c93ae"
},
{
"operation": "remove",
"key": "0xaa6cffe5185732689c18f37a7f86170cb7304c2a"
},
{
"operation": "remove",
"key": "0xaae4a2e3c51c04606dcb3723456e58f3ed214f45"
},
{
"operation": "remove",
"key": "0xc37a43e940dfb5baf581a0b82b351d48305fc885"
},
{
"operation": "remove",
"key": "0xd2571607e241ecf590ed94b12d87c94babe36db6"
},
{
"operation": "remove",
"key": "0xf735071cbee190d76b704ce68384fc21e389fbe7"
}],
"output": "0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421"
}

21
util/json-tests/json/trie/dogs.json
View File

@@ -1,21 +0,0 @@
{
"input":
[
{
"operation": "insert",
"key": "doe",
"value": "reindeer"
},
{
"operation": "insert",
"key": "dogglesworth",
"value": "cat"
},
{
"operation": "insert",
"key": "dog",
"value": "puppy"
}
],
"output": "0x8aad789dff2f538bca5d8ea56e8abe10f4c7ba3a5dea95fea4cd6e7c3a1168d3"
}

4
util/json-tests/json/trie/empty.json
View File

@@ -1,4 +0,0 @@
{
"input": [],
"output": "0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421"
}

44
util/json-tests/json/trie/empty_values.json
View File

@@ -1,44 +0,0 @@
{
"input":
[
{
"operation": "insert",
"key": "do",
"value": "verb"
},
{
"operation": "insert",
"key": "ether",
"value": "wookiedoo"
},
{
"operation": "insert",
"key": "horse",
"value": "stallion"
},
{
"operation": "insert",
"key": "shaman",
"value": "horse"
},
{
"operation": "insert",
"key": "doge",
"value": "coin"
},
{
"operation": "remove",
"key": "ether"
},
{
"operation": "insert",
"key": "dog",
"value": "puppy"
},
{
"operation": "remove",
"key": "shaman"
}
],
"output": "0x5991bb8c6514148a29db676a14ac506cd2cd5775ace63c30a4fe457715e9ac84"
}

16
util/json-tests/json/trie/foo.json
View File

@@ -1,16 +0,0 @@
{
"input":
[
{
"operation": "insert",
"key": "foo",
"value": "bar"
},
{
"operation": "insert",
"key": "food",
"value": "bass"
}
],
"output": "0x17beaa1648bafa633cda809c90c04af50fc8aed3cb40d16efbddee6fdf63c4c3"
}

58
util/json-tests/json/trie/jeff.json
View File

@@ -1,58 +0,0 @@
{
"input": [
{
"operation": "insert",
"key": "0x0000000000000000000000000000000000000000000000000000000000000045",
"value": "0x22b224a1420a802ab51d326e29fa98e34c4f24ea"
},
{
"operation": "insert",
"key": "0x0000000000000000000000000000000000000000000000000000000000000046",
"value": "0x67706c2076330000000000000000000000000000000000000000000000000000"
},
{
"operation": "insert",
"key": "0x0000000000000000000000000000000000000000000000000000001234567890",
"value": "0x697c7b8c961b56f675d570498424ac8de1a918f6"
},
{
"operation": "insert",
"key": "0x000000000000000000000000697c7b8c961b56f675d570498424ac8de1a918f6",
"value": "0x1234567890"
},
{
"operation": "insert",
"key": "0x0000000000000000000000007ef9e639e2733cb34e4dfc576d4b23f72db776b2",
"value": "0x4655474156000000000000000000000000000000000000000000000000000000"
},
{
"operation": "insert",
"key": "0x000000000000000000000000ec4f34c97e43fbb2816cfd95e388353c7181dab1",
"value": "0x4e616d6552656700000000000000000000000000000000000000000000000000"
},
{
"operation": "insert",
"key": "0x4655474156000000000000000000000000000000000000000000000000000000",
"value": "0x7ef9e639e2733cb34e4dfc576d4b23f72db776b2"
},
{
"operation": "insert",
"key": "0x4e616d6552656700000000000000000000000000000000000000000000000000",
"value": "0xec4f34c97e43fbb2816cfd95e388353c7181dab1"
},
{
"operation": "remove",
"key": "0x0000000000000000000000000000000000000000000000000000001234567890"
},
{
"operation": "insert",
"key": "0x000000000000000000000000697c7b8c961b56f675d570498424ac8de1a918f6",
"value": "0x6f6f6f6820736f2067726561742c207265616c6c6c793f000000000000000000"
},
{
"operation": "insert",
"key": "0x6f6f6f6820736f2067726561742c207265616c6c6c793f000000000000000000",
"value": "0x697c7b8c961b56f675d570498424ac8de1a918f6"
}],
"output": "0x9f6221ebb8efe7cff60a716ecb886e67dd042014be444669f0159d8e68b42100"
}

82
util/json-tests/src/lib.rs
View File

@@ -1,82 +0,0 @@
// Copyright 2015, 2016 Ethcore (UK) Ltd.
// This file is part of Parity.
// Parity is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// Parity is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
// You should have received a copy of the GNU General Public License
// along with Parity. If not, see <http://www.gnu.org/licenses/>.
extern crate rustc_serialize;
extern crate glob;
use std::str::from_utf8;
use std::path::*;
use std::io::prelude::*;
use std::fs::File;
use glob::glob;
use rustc_serialize::*;
mod util;
pub mod trie;
pub mod rlp;
pub trait JsonTest: Sized {
type Input;
type Output;
fn new(data: &[u8]) -> Self;
fn input(&self) -> Self::Input;
fn output(&self) -> Self::Output;
}
pub struct JsonLoader {
json: json::Json
}
impl JsonTest for JsonLoader {
type Input = json::Json;
type Output = json::Json;
fn new(data: &[u8]) -> Self {
JsonLoader {
json: json::Json::from_str(from_utf8(data).unwrap()).unwrap()
}
}
fn input(&self) -> Self::Input {
self.json.as_object().unwrap()["input"].clone()
}
fn output(&self) -> Self::Output {
self.json.as_object().unwrap()["output"].clone()
}
}
pub fn execute_test<T, F>(data: &[u8], f: &mut F) where T: JsonTest, F: FnMut(T::Input, T::Output) {
let test = T::new(data);
f(test.input(), test.output())
}
pub fn execute_test_from_file<T, F>(path: &Path, f: &mut F) where T: JsonTest, F: FnMut(T::Input, T::Output) {
let mut file = File::open(path).unwrap();
let mut buffer = vec![];
let _ = file.read_to_end(&mut buffer);
let test = T::new(&buffer);
f(test.input(), test.output())
}
pub fn execute_tests_from_directory<T, F>(pattern: &str, f: &mut F) where T: JsonTest, F: FnMut(String, T::Input, T::Output) {
for path in glob(pattern).unwrap().filter_map(Result::ok) {
execute_test_from_file::<T, _>(&path, &mut | input, output | {
f(path.to_str().unwrap().to_string(), input, output);
});
}
}

68
util/json-tests/src/rlp.rs
View File

@@ -1,68 +0,0 @@
// Copyright 2015, 2016 Ethcore (UK) Ltd.
// This file is part of Parity.
// Parity is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// Parity is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
// You should have received a copy of the GNU General Public License
// along with Parity. If not, see <http://www.gnu.org/licenses/>.
//! json rlp tests
use rustc_serialize::*;
use super::{JsonTest, JsonLoader};
use util::*;
pub enum Operation {
Append(Vec<u8>),
AppendList(usize),
AppendRaw(Vec<u8>, usize),
AppendEmpty
}
impl Into<Operation> for json::Json {
fn into(self) -> Operation {
let obj = self.as_object().unwrap();
match obj["operation"].as_string().unwrap().as_ref() {
"append" => Operation::Append(hex_or_string(obj["value"].as_string().unwrap())),
"append_list" => Operation::AppendList(obj["len"].as_u64().unwrap() as usize),
"append_raw" => Operation::AppendRaw(hex_or_string(obj["value"].as_string().unwrap()), obj["len"].as_u64().unwrap() as usize),
"append_empty" => Operation::AppendEmpty,
other => { panic!("Unsupported opertation: {}", other); }
}
}
}
pub struct RlpStreamTest {
loader: JsonLoader
}
impl JsonTest for RlpStreamTest {
type Input = Vec<Operation>;
type Output = Vec<u8>;
fn new(data: &[u8]) -> Self {
RlpStreamTest {
loader: JsonLoader::new(data)
}
}
fn input(&self) -> Self::Input {
self.loader.input().as_array().unwrap()
.iter()
.cloned()
.map(|i| i.into())
.collect()
}
fn output(&self) -> Self::Output {
hex_or_string(self.loader.output().as_string().unwrap())
}
}

105
util/json-tests/src/trie.rs
View File

@@ -1,105 +0,0 @@
// Copyright 2015, 2016 Ethcore (UK) Ltd.
// This file is part of Parity.
// Parity is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// Parity is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
// You should have received a copy of the GNU General Public License
// along with Parity. If not, see <http://www.gnu.org/licenses/>.
//! json trie tests
use std::collections::HashMap;
use rustc_serialize::*;
use super::{JsonTest, JsonLoader};
use util::*;
#[derive(RustcDecodable)]
struct RawOperation {
operation: String,
key: String,
value: Option<String>
}
pub enum Operation {
Insert(Vec<u8>, Vec<u8>),
Remove(Vec<u8>)
}
impl Into<Operation> for RawOperation {
fn into(self) -> Operation {
match self.operation.as_ref() {
"insert" => Operation::Insert(hex_or_string(&self.key), hex_or_string(&self.value.unwrap())),
"remove" => Operation::Remove(hex_or_string(&self.key)),
other => panic!("invalid operation type: {}", other)
}
}
}
pub struct TrieTest {
loader: JsonLoader
}
impl JsonTest for TrieTest {
type Input = Vec<Operation>;
type Output = Vec<u8>;
fn new(data: &[u8]) -> Self {
TrieTest {
loader: JsonLoader::new(data)
}
}
fn input(&self) -> Self::Input {
let mut decoder = json::Decoder::new(self.loader.input());
let raw: Vec<RawOperation> = Decodable::decode(&mut decoder).unwrap();
raw.into_iter()
.map(|i| i.into())
.collect()
}
fn output(&self) -> Self::Output {
hex_or_string(self.loader.output().as_string().unwrap())
}
}
pub struct TriehashTest {
trietest: TrieTest
}
impl JsonTest for TriehashTest {
type Input = Vec<(Vec<u8>, Vec<u8>)>;
type Output = Vec<u8>;
fn new(data: &[u8]) -> Self {
TriehashTest {
trietest: TrieTest::new(data)
}
}
fn input(&self) -> Self::Input {
self.trietest.input()
.into_iter()
.fold(HashMap::new(), | mut map, o | {
match o {
Operation::Insert(k, v) => map.insert(k, v),
Operation::Remove(k) => map.remove(&k)
};
map
})
.into_iter()
.map(|p| { p })
.collect()
}
fn output(&self) -> Self::Output {
self.trietest.output()
}
}

24
util/json-tests/src/util.rs
View File

@@ -1,24 +0,0 @@
// Copyright 2015, 2016 Ethcore (UK) Ltd.
// This file is part of Parity.
// Parity is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// Parity is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
// You should have received a copy of the GNU General Public License
// along with Parity. If not, see <http://www.gnu.org/licenses/>.
use rustc_serialize::hex::FromHex;
pub fn hex_or_string(s: &str) -> Vec<u8> {
match s.starts_with("0x") {
true => s[2..].from_hex().unwrap(),
false => From::from(s)
}
}

16
util/src/bytes.rs
View File

@@ -24,7 +24,7 @@
//! use util::bytes::BytesConvertable;
//!
//! let arr = [0; 5];
//! let slice: &[u8] = arr.bytes();
//! let slice: &[u8] = arr.as_slice();
//! }
//!
//! fn main() {
@@ -120,12 +120,12 @@ impl<'a> ToPretty for &'a [u8] {
impl<'a> ToPretty for &'a Bytes {
fn pretty(&self) -> PrettySlice {
PrettySlice(self.bytes())
PrettySlice(self.as_slice())
}
}
impl ToPretty for Bytes {
fn pretty(&self) -> PrettySlice {
PrettySlice(self.bytes())
PrettySlice(self.as_slice())
}
}
@@ -162,23 +162,19 @@ pub type Bytes = Vec<u8>;
/// Slice of bytes to underlying memory
pub trait BytesConvertable {
// TODO: rename to as_slice
/// Get the underlying byte-wise representation of the value.
/// Deprecated - use `as_slice` instead.
fn bytes(&self) -> &[u8];
/// Get the underlying byte-wise representation of the value.
fn as_slice(&self) -> &[u8] { self.bytes() }
fn as_slice(&self) -> &[u8];
/// Get a copy of the underlying byte-wise representation.
fn to_bytes(&self) -> Bytes { self.as_slice().to_vec() }
}
impl<T> BytesConvertable for T where T: AsRef<[u8]> {
fn bytes(&self) -> &[u8] { self.as_ref() }
fn as_slice(&self) -> &[u8] { self.as_ref() }
}
#[test]
fn bytes_convertable() {
assert_eq!(vec![0x12u8, 0x34].bytes(), &[0x12u8, 0x34]);
assert_eq!(vec![0x12u8, 0x34].as_slice(), &[0x12u8, 0x34]);
assert!([0u8; 0].as_slice().is_empty());
}

2
util/src/error.rs
View File

@@ -104,7 +104,7 @@ impl<T: fmt::Debug + fmt::Display> fmt::Display for Mismatch<T> {
}
}
#[derive(Debug, PartialEq, Eq)]
#[derive(Debug, PartialEq, Eq, Clone)]
/// Error indicating value found is outside of a valid range.
pub struct OutOfBounds<T: fmt::Debug> {
/// Minimum allowed value.

80
util/src/hash.rs
View File

@@ -16,16 +16,18 @@
//! General hash types, a fixed-size raw-data type used as the output of hash functions.
use standard::*;
use rustc_serialize::hex::FromHex;
use std::{ops, fmt, cmp};
use std::cmp::*;
use std::ops::*;
use std::hash::{Hash, Hasher};
use std::str::FromStr;
use math::log2;
use error::UtilError;
use rand::Rng;
use rand::os::OsRng;
use bytes::{BytesConvertable,Populatable};
use from_json::*;
use bigint::uint::{Uint, U256};
use rustc_serialize::hex::ToHex;
use serde;
/// Trait for a fixed-size byte array to be used as the output of hash functions.
///
@@ -228,55 +230,6 @@ macro_rules! impl_hash {
}
}
impl serde::Serialize for $from {
fn serialize<S>(&self, serializer: &mut S) -> Result<(), S::Error>
where S: serde::Serializer {
let mut hex = "0x".to_owned();
hex.push_str(self.to_hex().as_ref());
serializer.serialize_str(hex.as_ref())
}
}
impl serde::Deserialize for $from {
fn deserialize<D>(deserializer: &mut D) -> Result<$from, D::Error>
where D: serde::Deserializer {
struct HashVisitor;
impl serde::de::Visitor for HashVisitor {
type Value = $from;
fn visit_str<E>(&mut self, value: &str) -> Result<Self::Value, E> where E: serde::Error {
// 0x + len
if value.len() != 2 + $size * 2 {
return Err(serde::Error::custom("Invalid length."));
}
value[2..].from_hex().map(|ref v| $from::from_slice(v)).map_err(|_| serde::Error::custom("Invalid hex value."))
}
fn visit_string<E>(&mut self, value: String) -> Result<Self::Value, E> where E: serde::Error {
self.visit_str(value.as_ref())
}
}
deserializer.deserialize(HashVisitor)
}
}
impl FromJson for $from {
fn from_json(json: &Json) -> Self {
match *json {
Json::String(ref s) => {
match s.len() % 2 {
0 => FromStr::from_str(clean_0x(s)).unwrap(),
_ => FromStr::from_str(&("0".to_owned() + &(clean_0x(s).to_owned()))[..]).unwrap()
}
},
_ => Default::default(),
}
}
}
impl fmt::Debug for $from {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
for i in &self.0[..] {
@@ -285,6 +238,7 @@ macro_rules! impl_hash {
Ok(())
}
}
impl fmt::Display for $from {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
for i in &self.0[0..2] {
@@ -506,13 +460,13 @@ impl<'a> From<&'a U256> for H256 {
impl From<H256> for U256 {
fn from(value: H256) -> U256 {
U256::from(value.bytes())
U256::from(value.as_slice())
}
}
impl<'a> From<&'a H256> for U256 {
fn from(value: &'a H256) -> U256 {
U256::from(value.bytes())
U256::from(value.as_slice())
}
}
@@ -532,17 +486,6 @@ impl From<H256> for H64 {
}
}
/*
impl<'a> From<&'a H256> for Address {
fn from(value: &'a H256) -> Address {
let mut ret = Address::new();
ret.0.copy_from_slice(&value[12..32]);
ret
}
}
}
*/
impl From<Address> for H256 {
fn from(value: Address) -> H256 {
let mut ret = H256::new();
@@ -596,11 +539,6 @@ impl_hash!(H520, 65);
impl_hash!(H1024, 128);
impl_hash!(H2048, 256);
/// Constant address for point 0. Often used as a default.
pub static ZERO_ADDRESS: Address = Address([0x00; 20]);
/// Constant 256-bit datum for 0. Often used as a default.
pub static ZERO_H256: H256 = H256([0x00; 32]);
#[cfg(test)]
mod tests {
use hash::*;

7
util/src/io/service.rs
View File

@@ -248,6 +248,13 @@ impl<Message> Handler for IoManager<Message> where Message: Send + Clone + Sync
IoMessage::RemoveHandler { handler_id } => {
// TODO: flush event loop
self.handlers.remove(handler_id);
// unregister timers
let mut timers = self.timers.write().unwrap();
let to_remove: Vec<_> = timers.keys().cloned().filter(|timer_id| timer_id / TOKENS_PER_HANDLER == handler_id).collect();
for timer_id in to_remove {
let timer = timers.remove(&timer_id).expect("to_remove only contains keys from timers; qed");
event_loop.clear_timeout(timer.timeout);
}
},
IoMessage::AddTimer { handler_id, token, delay } => {
let timer_id = token + handler_id * TOKENS_PER_HANDLER;

6
util/src/journaldb/archivedb.rs
View File

@@ -79,7 +79,7 @@ impl ArchiveDB {
}
fn payload(&self, key: &H256) -> Option<Bytes> {
self.backing.get(&key.bytes()).expect("Low-level database error. Some issue with your hard disk?").map(|v| v.to_vec())
self.backing.get(key).expect("Low-level database error. Some issue with your hard disk?").map(|v| v.to_vec())
}
}
@@ -177,7 +177,7 @@ impl JournalDB for ArchiveDB {
let (key, (value, rc)) = i;
if rc > 0 {
assert!(rc == 1);
batch.put(&key.bytes(), &value).expect("Low-level database error. Some issue with your hard disk?");
batch.put(&key, &value).expect("Low-level database error. Some issue with your hard disk?");
inserts += 1;
}
if rc < 0 {
@@ -202,7 +202,7 @@ impl JournalDB for ArchiveDB {
fn latest_era(&self) -> Option<u64> { self.latest_era }
fn state(&self, id: &H256) -> Option<Bytes> {
self.backing.get_by_prefix(&id.bytes()[0..12]).and_then(|b| Some(b.to_vec()))
self.backing.get_by_prefix(&id[0..12]).and_then(|b| Some(b.to_vec()))
}
fn is_pruned(&self) -> bool { false }

12
util/src/journaldb/earlymergedb.rs
View File

@@ -106,7 +106,7 @@ impl EarlyMergeDB {
}
fn morph_key(key: &H256, index: u8) -> Bytes {
let mut ret = key.bytes().to_owned();
let mut ret = key.to_bytes();
ret.push(index);
ret
}
@@ -130,7 +130,7 @@ impl EarlyMergeDB {
}
// this is the first entry for this node in the journal.
if backing.get(&h.bytes()).expect("Low-level database error. Some issue with your hard disk?").is_some() {
if backing.get(h).expect("Low-level database error. Some issue with your hard disk?").is_some() {
// already in the backing DB. start counting, and remember it was already in.
Self::set_already_in(batch, &h);
refs.insert(h.clone(), RefInfo{queue_refs: 1, in_archive: true});
@@ -143,7 +143,7 @@ impl EarlyMergeDB {
// Gets removed when a key leaves the journal, so should never be set when we're placing a new key.
//Self::reset_already_in(&h);
assert!(!Self::is_already_in(backing, &h));
batch.put(&h.bytes(), d).expect("Low-level database error. Some issue with your hard disk?");
batch.put(h, d).expect("Low-level database error. Some issue with your hard disk?");
refs.insert(h.clone(), RefInfo{queue_refs: 1, in_archive: false});
if trace {
trace!(target: "jdb.fine", " insert({}): New to queue, not in DB: Inserting into queue and DB", h);
@@ -204,7 +204,7 @@ impl EarlyMergeDB {
}
Some(RefInfo{queue_refs: 1, in_archive: false}) => {
refs.remove(h);
batch.delete(&h.bytes()).expect("Low-level database error. Some issue with your hard disk?");
batch.delete(h).expect("Low-level database error. Some issue with your hard disk?");
if trace {
trace!(target: "jdb.fine", " remove({}): Not in archive, only 1 ref in queue: Removing from queue and DB", h);
}
@@ -212,7 +212,7 @@ impl EarlyMergeDB {
None => {
// Gets removed when moving from 1 to 0 additional refs. Should never be here at 0 additional refs.
//assert!(!Self::is_already_in(db, &h));
batch.delete(&h.bytes()).expect("Low-level database error. Some issue with your hard disk?");
batch.delete(h).expect("Low-level database error. Some issue with your hard disk?");
if trace {
trace!(target: "jdb.fine", " remove({}): Not in queue - MUST BE IN ARCHIVE: Removing from DB", h);
}
@@ -237,7 +237,7 @@ impl EarlyMergeDB {
}
fn payload(&self, key: &H256) -> Option<Bytes> {
self.backing.get(&key.bytes()).expect("Low-level database error. Some issue with your hard disk?").map(|v| v.to_vec())
self.backing.get(key).expect("Low-level database error. Some issue with your hard disk?").map(|v| v.to_vec())
}
fn read_refs(db: &Database) -> (Option<u64>, HashMap<H256, RefInfo>) {

2
util/src/journaldb/overlayrecentdb.rs
View File

@@ -141,7 +141,7 @@ impl OverlayRecentDB {
}
fn payload(&self, key: &H256) -> Option<Bytes> {
self.backing.get(&key.bytes()).expect("Low-level database error. Some issue with your hard disk?").map(|v| v.to_vec())
self.backing.get(key).expect("Low-level database error. Some issue with your hard disk?").map(|v| v.to_vec())
}
fn read_overlay(db: &Database) -> JournalOverlay {

154
util/src/json_aid.rs
View File

@@ -1,154 +0,0 @@
// Copyright 2015, 2016 Ethcore (UK) Ltd.
// This file is part of Parity.
// Parity is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// Parity is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
// You should have received a copy of the GNU General Public License
// along with Parity. If not, see <http://www.gnu.org/licenses/>.
use common::*;
/// Remove the `"0x"`, if present, from the left of `s`, returning the remaining slice.
pub fn clean(s: &str) -> &str {
if s.len() >= 2 && &s[0..2] == "0x" {
&s[2..]
} else {
s
}
}
fn u256_from_str(s: &str) -> U256 {
if s.len() >= 2 && &s[0..2] == "0x" {
U256::from_str(&s[2..]).unwrap_or_else(|_| U256::zero())
} else {
U256::from_dec_str(s).unwrap_or_else(|_| U256::zero())
}
}
impl FromJson for Bytes {
fn from_json(json: &Json) -> Self {
match *json {
Json::String(ref s) => match s.len() % 2 {
0 => FromHex::from_hex(clean(s)).unwrap_or_else(|_| vec![]),
_ => FromHex::from_hex(&("0".to_owned() + &(clean(s).to_owned()))[..]).unwrap_or_else(|_| vec![]),
},
_ => vec![],
}
}
}
impl FromJson for BTreeMap<H256, H256> {
fn from_json(json: &Json) -> Self {
match *json {
Json::Object(ref o) => o.iter().map(|(key, value)| (u256_from_str(key).into(), U256::from_json(value).into())).collect(),
_ => BTreeMap::new(),
}
}
}
impl<T> FromJson for Vec<T> where T: FromJson {
fn from_json(json: &Json) -> Self {
match *json {
Json::Array(ref o) => o.iter().map(|x|T::from_json(x)).collect(),
_ => Vec::new(),
}
}
}
impl<T> FromJson for Option<T> where T: FromJson {
fn from_json(json: &Json) -> Self {
match *json {
Json::String(ref o) if o.is_empty() => None,
Json::Null => None,
_ => Some(FromJson::from_json(json)),
}
}
}
impl FromJson for u64 {
fn from_json(json: &Json) -> Self {
U256::from_json(json).low_u64()
}
}
impl FromJson for u32 {
fn from_json(json: &Json) -> Self {
U256::from_json(json).low_u64() as u32
}
}
impl FromJson for u16 {
fn from_json(json: &Json) -> Self {
U256::from_json(json).low_u64() as u16
}
}
#[test]
fn u256_from_json() {
let j = Json::from_str("{ \"dec\": \"10\", \"hex\": \"0x0a\", \"int\": 10 }").unwrap();
let v: U256 = xjson!(&j["dec"]);
assert_eq!(U256::from(10), v);
let v: U256 = xjson!(&j["hex"]);
assert_eq!(U256::from(10), v);
let v: U256 = xjson!(&j["int"]);
assert_eq!(U256::from(10), v);
}
#[test]
fn h256_from_json() {
let j = Json::from_str("{ \"with\": \"0x1234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef\", \"without\": \"0x1234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef\" }").unwrap();
let v: H256 = xjson!(&j["with"]);
assert_eq!(H256::from_str("1234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef").unwrap(), v);
let v: H256 = xjson!(&j["without"]);
assert_eq!(H256::from_str("1234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef").unwrap(), v);
}
#[test]
fn vec_u256_from_json() {
let j = Json::from_str("{ \"array\": [ \"10\", \"0x0a\", 10] }").unwrap();
let v: Vec<U256> = xjson!(&j["array"]);
assert_eq!(vec![U256::from(10); 3], v);
}
#[test]
fn vec_h256_from_json() {
let j = Json::from_str("{ \"array\": [ \"1234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef\", \"0x1234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef\"] }").unwrap();
let v: Vec<H256> = xjson!(&j["array"]);
assert_eq!(vec![H256::from_str("1234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef").unwrap(); 2], v);
}
#[test]
fn simple_types() {
let j = Json::from_str("{ \"null\": null, \"empty\": \"\", \"int\": 42, \"dec\": \"42\", \"hex\": \"0x2a\" }").unwrap();
let v: u16 = xjson!(&j["int"]);
assert_eq!(42u16, v);
let v: u32 = xjson!(&j["dec"]);
assert_eq!(42u32, v);
let v: u64 = xjson!(&j["hex"]);
assert_eq!(42u64, v);
}
#[test]
fn option_types() {
let j = Json::from_str("{ \"null\": null, \"empty\": \"\", \"int\": 42, \"dec\": \"42\", \"hex\": \"0x2a\" }").unwrap();
let v: Option<u16> = xjson!(&j["int"]);
assert_eq!(Some(42u16), v);
let v: Option<u16> = xjson!(&j["dec"]);
assert_eq!(Some(42u16), v);
let v: Option<u16> = xjson!(&j["null"]);
assert_eq!(None, v);
let v: Option<u16> = xjson!(&j["empty"]);
assert_eq!(None, v);
}

11
util/src/lib.rs
View File

@@ -108,7 +108,6 @@ extern crate secp256k1;
extern crate arrayvec;
extern crate elastic_array;
extern crate crossbeam;
extern crate serde;
#[macro_use]
extern crate log as rlog;
extern crate igd;
@@ -117,6 +116,8 @@ extern crate libc;
extern crate target_info;
extern crate bigint;
extern crate chrono;
pub extern crate using_queue;
pub extern crate table;
extern crate ansi_term;
pub mod standard;
@@ -130,8 +131,6 @@ pub mod hash;
pub mod bytes;
pub mod rlp;
pub mod misc;
pub mod using_queue;
mod json_aid;
pub mod vector;
pub mod sha3;
pub mod hashdb;
@@ -145,14 +144,13 @@ pub mod crypto;
pub mod triehash;
pub mod trie;
pub mod nibbleslice;
pub mod nibblevec;
mod heapsizeof;
pub mod squeeze;
pub mod semantic_version;
pub mod io;
pub mod network;
pub mod log;
pub mod panics;
pub mod table;
pub mod network_settings;
pub mod path;
pub mod snappy;
@@ -160,8 +158,6 @@ mod timer;
pub use common::*;
pub use misc::*;
pub use using_queue::*;
pub use json_aid::*;
pub use rlp::*;
pub use hashdb::*;
pub use memorydb::*;
@@ -172,7 +168,6 @@ pub use crypto::*;
pub use triehash::*;
pub use trie::*;
pub use nibbleslice::*;
pub use squeeze::*;
pub use semantic_version::*;
pub use network::*;
pub use io::*;

21
util/src/log.rs
View File

@@ -17,6 +17,7 @@
//! Common log helper functions
use std::env;
use std::borrow::Cow;
use rlog::{LogLevelFilter};
use env_logger::LogBuilder;
use std::sync::{RwLock, RwLockReadGuard};
@@ -28,12 +29,20 @@ lazy_static! {
static ref USE_COLOR: AtomicBool = AtomicBool::new(false);
}
/// Paint, using colour if desired.
pub fn paint(c: Style, t: String) -> String {
match USE_COLOR.load(Ordering::Relaxed) {
true => format!("{}", c.paint(t)),
false => t,
}
/// Something which can be apply()ed.
pub trait Applyable: AsRef<str> {
/// Apply the style `c` to ourself, returning us styled in that manner.
fn apply(&self, c: Style) -> Cow<str>;
}
impl<T: AsRef<str>> Applyable for T {
fn apply(&self, c: Style) -> Cow<str> {
let s = self.as_ref();
match USE_COLOR.load(Ordering::Relaxed) {
true => Cow::Owned(format!("{}", c.paint(s))),
false => Cow::Borrowed(s),
}
}
}
lazy_static! {

8
util/src/memorydb.rs
View File

@@ -198,14 +198,14 @@ impl HashDB for MemoryDB {
let key = value.sha3();
if match self.data.get_mut(&key) {
Some(&mut (ref mut old_value, ref mut rc @ -0x80000000i32 ... 0)) => {
*old_value = From::from(value.bytes());
*old_value = From::from(value);
*rc += 1;
false
},
Some(&mut (_, ref mut x)) => { *x += 1; false } ,
None => true,
}{ // ... None falls through into...
self.data.insert(key.clone(), (From::from(value.bytes()), 1));
self.data.insert(key.clone(), (From::from(value), 1));
}
key
}
@@ -262,8 +262,8 @@ fn memorydb_denote() {
for _ in 0..1000 {
let r = H256::random();
let k = r.sha3();
let &(ref v, ref rc) = m.denote(&k, r.bytes().to_vec());
assert_eq!(v, &r.bytes());
let &(ref v, ref rc) = m.denote(&k, r.to_bytes());
assert_eq!(v.as_slice(), r.as_slice());
assert_eq!(*rc, 0);
}

55
util/src/migration/db_impl.rs
View File

@@ -1,55 +0,0 @@
// Copyright 2015, 2016 Ethcore (UK) Ltd.
// This file is part of Parity.
// Parity is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// Parity is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
// You should have received a copy of the GNU General Public License
// along with Parity. If not, see <http://www.gnu.org/licenses/>.
//! `kvdb::Database` as `migration::Destination`
use std::collections::BTreeMap;
use kvdb::{Database, DatabaseIterator, DBTransaction};
use migration::{Destination, Error};
/// Database iterator with `Item` complient with migration `Manager` interface.
pub struct MigrationIterator {
iter: DatabaseIterator,
}
impl From<DatabaseIterator> for MigrationIterator {
fn from(iter: DatabaseIterator) -> Self {
MigrationIterator {
iter: iter
}
}
}
impl Iterator for MigrationIterator {
type Item = (Vec<u8>, Vec<u8>);
fn next(&mut self) -> Option<Self::Item> {
self.iter.next().map(|(k, v)| (k.to_vec(), v.to_vec()))
}
}
impl Destination for Database {
fn commit(&mut self, batch: BTreeMap<Vec<u8>, Vec<u8>>) -> Result<(), Error> {
let transaction = DBTransaction::new();
for keypair in &batch {
try!(transaction.put(&keypair.0, &keypair.1).map_err(Error::Custom))
}
self.write(transaction).map_err(Error::Custom)
}
}

117
util/src/migration/manager.rs
View File

@@ -1,117 +0,0 @@
// Copyright 2015, 2016 Ethcore (UK) Ltd.
// This file is part of Parity.
// Parity is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// Parity is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
// You should have received a copy of the GNU General Public License
// along with Parity. If not, see <http://www.gnu.org/licenses/>.
//! Migration manager
use std::collections::BTreeMap;
use migration::{Migration, Destination};
/// Migration error.
#[derive(Debug)]
pub enum Error {
/// Error returned when it is impossible to add new migration rules.
CannotAddMigration,
/// Error returned when migration from specific version can not be performed.
MigrationImpossible,
/// Custom error.
Custom(String),
}
/// Migration config.
pub struct Config {
/// Defines how many elements should be migrated at once.
pub batch_size: usize,
}
impl Default for Config {
fn default() -> Self {
Config {
batch_size: 1024,
}
}
}
/// Manages database migration.
pub struct Manager {
config: Config,
migrations: Vec<Box<Migration>>,
}
impl Manager {
/// Creates new migration manager with given configuration.
pub fn new(config: Config) -> Self {
Manager {
config: config,
migrations: vec![]
}
}
/// Adds new migration rules.
pub fn add_migration<T>(&mut self, migration: T) -> Result<(), Error> where T: Migration {
let version_match = match self.migrations.last() {
Some(last) => last.version() + 1 == migration.version(),
None => true,
};
match version_match {
true => Ok(self.migrations.push(Box::new(migration))),
false => Err(Error::CannotAddMigration),
}
}
/// Performs migration to destination.
pub fn execute<D>(&self, db_iter: D, version: u32, destination: &mut Destination) -> Result<(), Error> where
D: Iterator<Item = (Vec<u8>, Vec<u8>)> {
let migrations = try!(self.migrations_from(version).ok_or(Error::MigrationImpossible));
let mut batch: BTreeMap<Vec<u8>, Vec<u8>> = BTreeMap::new();
for keypair in db_iter {
let migrated = migrations.iter().fold(Some(keypair), |migrated, migration| {
migrated.and_then(|(key, value)| migration.simple_migrate(key, value))
});
if let Some((key, value)) = migrated {
batch.insert(key, value);
}
if batch.len() == self.config.batch_size {
try!(destination.commit(batch));
batch = BTreeMap::new();
}
}
try!(destination.commit(batch));
Ok(())
}
/// Returns true if migration is needed.
pub fn is_needed(&self, version: u32) -> bool {
match self.migrations.last() {
Some(last) => version < last.version(),
None => false,
}
}
fn migrations_from(&self, version: u32) -> Option<&[Box<Migration>]> {
// index of the first required migration
let position = self.migrations.iter().position(|m| m.version() == version + 1);
position.map(|p| &self.migrations[p..])
}
}

227
util/src/migration/mod.rs
View File

@@ -15,28 +15,231 @@
// along with Parity. If not, see <http://www.gnu.org/licenses/>.
//! DB Migration module.
mod db_impl;
mod manager;
#[cfg(test)]
mod tests;
pub use self::manager::{Error, Config, Manager};
pub use self::db_impl::MigrationIterator;
use std::collections::BTreeMap;
use std::fs;
use std::path::{Path, PathBuf};
/// Single migration.
use ::kvdb::{CompactionProfile, Database, DatabaseConfig, DBTransaction};
/// Migration config.
#[derive(Clone)]
pub struct Config {
/// Defines how many elements should be migrated at once.
pub batch_size: usize,
}
impl Default for Config {
fn default() -> Self {
Config {
batch_size: 1024,
}
}
}
/// A batch of key-value pairs to be written into the database.
pub struct Batch {
inner: BTreeMap<Vec<u8>, Vec<u8>>,
batch_size: usize,
}
impl Batch {
/// Make a new batch with the given config.
pub fn new(config: &Config) -> Self {
Batch {
inner: BTreeMap::new(),
batch_size: config.batch_size,
}
}
/// Insert a value into the batch, committing if necessary.
pub fn insert(&mut self, key: Vec<u8>, value: Vec<u8>, dest: &mut Database) -> Result<(), Error> {
self.inner.insert(key, value);
if self.inner.len() == self.batch_size {
try!(self.commit(dest));
}
Ok(())
}
/// Commit all the items in the batch to the given database.
pub fn commit(&mut self, dest: &mut Database) -> Result<(), Error> {
if self.inner.is_empty() { return Ok(()) }
let transaction = DBTransaction::new();
for keypair in &self.inner {
try!(transaction.put(&keypair.0, &keypair.1).map_err(Error::Custom));
}
self.inner.clear();
dest.write(transaction).map_err(Error::Custom)
}
}
/// Migration error.
#[derive(Debug)]
pub enum Error {
/// Error returned when it is impossible to add new migration rules.
CannotAddMigration,
/// Error returned when migration from specific version can not be performed.
MigrationImpossible,
/// Io Error.
Io(::std::io::Error),
/// Custom error.
Custom(String),
}
impl From<::std::io::Error> for Error {
fn from(e: ::std::io::Error) -> Self {
Error::Io(e)
}
}
/// A generalized migration from the given db to a destination db.
pub trait Migration: 'static {
/// Version of the database after the migration.
fn version(&self) -> u32;
/// Migrate a source to a destination.
fn migrate(&mut self, source: &Database, config: &Config, destination: &mut Database) -> Result<(), Error>;
}
/// A simple migration over key-value pairs.
pub trait SimpleMigration: 'static {
/// Version of database after the migration.
fn version(&self) -> u32;
/// Should migrate existing object to new database.
/// Returns `None` if the object does not exist in new version of database.
fn simple_migrate(&self, key: Vec<u8>, value: Vec<u8>) -> Option<(Vec<u8>, Vec<u8>)>;
fn simple_migrate(&mut self, key: Vec<u8>, value: Vec<u8>) -> Option<(Vec<u8>, Vec<u8>)>;
}
/// Migration destination.
pub trait Destination {
/// Called on destination to commit batch of migrated entries.
fn commit(&mut self, batch: BTreeMap<Vec<u8>, Vec<u8>>) -> Result<(), Error>;
impl<T: SimpleMigration> Migration for T {
fn version(&self) -> u32 { SimpleMigration::version(self) }
fn migrate(&mut self, source: &Database, config: &Config, dest: &mut Database) -> Result<(), Error> {
let mut batch = Batch::new(config);
for (key, value) in source.iter() {
if let Some((key, value)) = self.simple_migrate(key.to_vec(), value.to_vec()) {
try!(batch.insert(key, value, dest));
}
}
batch.commit(dest)
}
}
/// Get the path where all databases reside.
fn database_path(path: &Path) -> PathBuf {
let mut temp_path = path.to_owned();
temp_path.pop();
temp_path
}
enum TempIndex {
One,
Two,
}
impl TempIndex {
fn swap(&mut self) {
match *self {
TempIndex::One => *self = TempIndex::Two,
TempIndex::Two => *self = TempIndex::One,
}
}
// given the path to the old database, get the path of this one.
fn path(&self, db_root: &Path) -> PathBuf {
let mut buf = db_root.to_owned();
match *self {
TempIndex::One => buf.push("temp_migration_1"),
TempIndex::Two => buf.push("temp_migration_2"),
};
buf
}
}
/// Manages database migration.
pub struct Manager {
config: Config,
migrations: Vec<Box<Migration>>,
}
impl Manager {
/// Creates new migration manager with given configuration.
pub fn new(config: Config) -> Self {
Manager {
config: config,
migrations: vec![],
}
}
/// Adds new migration rules.
pub fn add_migration<T>(&mut self, migration: T) -> Result<(), Error> where T: Migration {
let version_match = match self.migrations.last() {
Some(last) => last.version() + 1 == migration.version(),
None => true,
};
match version_match {
true => Ok(self.migrations.push(Box::new(migration))),
false => Err(Error::CannotAddMigration),
}
}
/// Performs migration in order, starting with a source path, migrating between two temporary databases,
/// and producing a path where the final migration lives.
pub fn execute(&mut self, old_path: &Path, version: u32) -> Result<PathBuf, Error> {
let config = self.config.clone();
let migrations = try!(self.migrations_from(version).ok_or(Error::MigrationImpossible));
let db_config = DatabaseConfig {
prefix_size: None,
max_open_files: 64,
cache_size: None,
compaction: CompactionProfile::default(),
};
let db_root = database_path(old_path);
let mut temp_idx = TempIndex::One;
let mut temp_path = temp_idx.path(&db_root);
// start with the old db.
let old_path_str = try!(old_path.to_str().ok_or(Error::MigrationImpossible));
let mut cur_db = try!(Database::open(&db_config, old_path_str).map_err(|s| Error::Custom(s)));
for migration in migrations {
// open the target temporary database.
temp_path = temp_idx.path(&db_root);
let temp_path_str = try!(temp_path.to_str().ok_or(Error::MigrationImpossible));
let mut new_db = try!(Database::open(&db_config, temp_path_str).map_err(|s| Error::Custom(s)));
// perform the migration from cur_db to new_db.
try!(migration.migrate(&cur_db, &config, &mut new_db));
// next iteration, we will migrate from this db into the other temp.
cur_db = new_db;
temp_idx.swap();
// remove the other temporary migration database.
let _ = fs::remove_dir_all(temp_idx.path(&db_root));
}
Ok(temp_path)
}
/// Returns true if migration is needed.
pub fn is_needed(&self, version: u32) -> bool {
match self.migrations.last() {
Some(last) => version < last.version(),
None => false,
}
}
fn migrations_from(&mut self, version: u32) -> Option<&mut [Box<Migration>]> {
// index of the first required migration
let position = self.migrations.iter().position(|m| m.version() == version + 1);
position.map(move |p| &mut self.migrations[p..])
}
}

111
util/src/migration/tests.rs
View File

@@ -14,24 +14,55 @@
// You should have received a copy of the GNU General Public License
// along with Parity. If not, see <http://www.gnu.org/licenses/>.
use std::collections::BTreeMap;
use migration::{Error, Destination, Migration, Manager, Config};
//! Tests for migrations.
//! A random temp directory is created. A database is created within it, and migrations
//! are performed in temp sub-directories.
impl Destination for BTreeMap<Vec<u8>, Vec<u8>> {
fn commit(&mut self, batch: BTreeMap<Vec<u8>, Vec<u8>>) -> Result<(), Error> {
self.extend(batch);
Ok(())
use common::*;
use migration::{Config, SimpleMigration, Manager};
use kvdb::{Database, DBTransaction};
use devtools::RandomTempPath;
use std::path::PathBuf;
fn db_path(path: &Path) -> PathBuf {
let mut p = path.to_owned();
p.push("db");
p
}
// initialize a database at the given directory with the given values.
fn make_db(path: &Path, pairs: BTreeMap<Vec<u8>, Vec<u8>>) {
let db = Database::open_default(path.to_str().unwrap()).expect("failed to open temp database");
{
let transaction = DBTransaction::new();
for (k, v) in pairs {
transaction.put(&k, &v).expect("failed to add pair to transaction");
}
db.write(transaction).expect("failed to write db transaction");
}
}
// helper for verifying a migrated database.
fn verify_migration(path: &Path, pairs: BTreeMap<Vec<u8>, Vec<u8>>) {
let db = Database::open_default(path.to_str().unwrap()).unwrap();
for (k, v) in pairs {
let x = db.get(&k).unwrap().unwrap();
assert_eq!(&x[..], &v[..]);
}
}
struct Migration0;
impl Migration for Migration0 {
impl SimpleMigration for Migration0 {
fn version(&self) -> u32 {
1
}
fn simple_migrate(&self, key: Vec<u8>, value: Vec<u8>) -> Option<(Vec<u8>, Vec<u8>)> {
fn simple_migrate(&mut self, key: Vec<u8>, value: Vec<u8>) -> Option<(Vec<u8>, Vec<u8>)> {
let mut key = key;
key.push(0x11);
let mut value = value;
@@ -42,80 +73,70 @@ impl Migration for Migration0 {
struct Migration1;
impl Migration for Migration1 {
impl SimpleMigration for Migration1 {
fn version(&self) -> u32 {
2
}
fn simple_migrate(&self, key: Vec<u8>, _value: Vec<u8>) -> Option<(Vec<u8>, Vec<u8>)> {
fn simple_migrate(&mut self, key: Vec<u8>, _value: Vec<u8>) -> Option<(Vec<u8>, Vec<u8>)> {
Some((key, vec![]))
}
}
#[test]
fn one_simple_migration() {
let dir = RandomTempPath::create_dir();
let db_path = db_path(dir.as_path());
let mut manager = Manager::new(Config::default());
let keys = vec![vec![], vec![1u8]];
let values = vec![vec![], vec![1u8]];
let db = keys.into_iter().zip(values.into_iter());
make_db(&db_path, map![vec![] => vec![], vec![1] => vec![1]]);
let expected = map![vec![0x11] => vec![0x22], vec![1, 0x11] => vec![1, 0x22]];
let expected_keys = vec![vec![0x11u8], vec![1, 0x11]];
let expected_values = vec![vec![0x22u8], vec![1, 0x22]];
let expected_db = expected_keys.into_iter().zip(expected_values.into_iter()).collect::<BTreeMap<_, _>>();
let mut result = BTreeMap::new();
manager.add_migration(Migration0).unwrap();
manager.execute(db, 0, &mut result).unwrap();
assert_eq!(expected_db, result);
let end_path = manager.execute(&db_path, 0).unwrap();
verify_migration(&end_path, expected);
}
#[test]
#[should_panic]
fn no_migration_needed() {
let dir = RandomTempPath::create_dir();
let db_path = db_path(dir.as_path());
let mut manager = Manager::new(Config::default());
let keys = vec![vec![], vec![1u8]];
let values = vec![vec![], vec![1u8]];
let db = keys.into_iter().zip(values.into_iter());
make_db(&db_path, map![vec![] => vec![], vec![1] => vec![1]]);
let mut result = BTreeMap::new();
manager.add_migration(Migration0).unwrap();
manager.execute(db, 1, &mut result).unwrap();
manager.execute(&db_path, 1).unwrap();
}
#[test]
fn multiple_migrations() {
let dir = RandomTempPath::create_dir();
let db_path = db_path(dir.as_path());
let mut manager = Manager::new(Config::default());
let keys = vec![vec![], vec![1u8]];
let values = vec![vec![], vec![1u8]];
let db = keys.into_iter().zip(values.into_iter());
make_db(&db_path, map![vec![] => vec![], vec![1] => vec![1]]);
let expected = map![vec![0x11] => vec![], vec![1, 0x11] => vec![]];
let expected_keys = vec![vec![0x11u8], vec![1, 0x11]];
let expected_values = vec![vec![], vec![]];
let expected_db = expected_keys.into_iter().zip(expected_values.into_iter()).collect::<BTreeMap<_, _>>();
let mut result = BTreeMap::new();
manager.add_migration(Migration0).unwrap();
manager.add_migration(Migration1).unwrap();
manager.execute(db, 0, &mut result).unwrap();
assert_eq!(expected_db, result);
let end_path = manager.execute(&db_path, 0).unwrap();
verify_migration(&end_path, expected);
}
#[test]
fn second_migration() {
let dir = RandomTempPath::create_dir();
let db_path = db_path(dir.as_path());
let mut manager = Manager::new(Config::default());
let keys = vec![vec![], vec![1u8]];
let values = vec![vec![], vec![1u8]];
let db = keys.into_iter().zip(values.into_iter());
make_db(&db_path, map![vec![] => vec![], vec![1] => vec![1]]);
let expected = map![vec![] => vec![], vec![1] => vec![]];
let expected_keys = vec![vec![], vec![1u8]];
let expected_values = vec![vec![], vec![]];
let expected_db = expected_keys.into_iter().zip(expected_values.into_iter()).collect::<BTreeMap<_, _>>();
let mut result = BTreeMap::new();
manager.add_migration(Migration0).unwrap();
manager.add_migration(Migration1).unwrap();
manager.execute(db, 1, &mut result).unwrap();
assert_eq!(expected_db, result);
let end_path = manager.execute(&db_path, 1).unwrap();
verify_migration(&end_path, expected);
}
#[test]

14
util/src/network/host.rs
View File

@@ -32,8 +32,6 @@ use misc::version;
use crypto::*;
use sha3::Hashable;
use rlp::*;
use log::Colour::White;
use log::paint;
use network::session::{Session, SessionData};
use error::*;
use io::*;
@@ -162,6 +160,8 @@ pub enum NetworkIoMessage<Message> where Message: Send + Sync + Clone {
Disconnect(PeerId),
/// Disconnect and temporary disable peer.
DisablePeer(PeerId),
/// Network has been started with the host as the given enode.
NetworkStarted(String),
/// User message
User(Message),
}
@@ -345,12 +345,13 @@ pub struct Host<Message> where Message: Send + Sync + Clone {
reserved_nodes: RwLock<HashSet<NodeId>>,
num_sessions: AtomicUsize,
stopping: AtomicBool,
first_time: AtomicBool,
}
impl<Message> Host<Message> where Message: Send + Sync + Clone {
/// Create a new instance
pub fn new(config: NetworkConfiguration, stats: Arc<NetworkStats>) -> Result<Host<Message>, UtilError> {
trace!(target: "host", "Creating new Host object");
let mut listen_address = match config.listen_address {
None => SocketAddr::from_str("0.0.0.0:30304").unwrap(),
Some(addr) => addr,
@@ -401,7 +402,6 @@ impl<Message> Host<Message> where Message: Send + Sync + Clone {
reserved_nodes: RwLock::new(HashSet::new()),
num_sessions: AtomicUsize::new(0),
stopping: AtomicBool::new(false),
first_time: AtomicBool::new(true),
};
for n in boot_nodes {
@@ -538,9 +538,8 @@ impl<Message> Host<Message> where Message: Send + Sync + Clone {
self.info.write().unwrap().public_endpoint = Some(public_endpoint.clone());
if self.first_time.load(AtomicOrdering::Relaxed) {
info!("Public node URL: {}", paint(White.bold(), format!("{}", self.external_url().unwrap())));
self.first_time.store(false, AtomicOrdering::Relaxed);
if let Some(url) = self.external_url() {
io.message(NetworkIoMessage::NetworkStarted(url)).unwrap_or_else(|e| warn!("Error sending IO notification: {:?}", e));
}
// Initialize discovery.
@@ -1038,6 +1037,7 @@ impl<Message> IoHandler<NetworkIoMessage<Message>> for Host<Message> where Messa
h.message(&NetworkContext::new(io, p, None, self.sessions.clone(), &reserved), &message);
}
}
_ => {} // ignore others.
}
}

5
util/src/network/service.rs
View File

@@ -78,6 +78,11 @@ impl<Message> NetworkService<Message> where Message: Send + Sync + Clone + 'stat
&self.stats
}
/// Returns network configuration.
pub fn config(&self) -> &NetworkConfiguration {
&self.config
}
/// Returns external url if available.
pub fn external_url(&self) -> Option<String> {
let host = self.host.read().unwrap();

130
util/src/nibblevec.rs Normal file
View File

@@ -0,0 +1,130 @@
//! An owning, nibble-oriented byte vector.
use ::NibbleSlice;
#[derive(PartialEq, Eq, PartialOrd, Ord, Debug)]
/// Owning, nibble-oriented byte vector. Counterpart to NibbleSlice.
pub struct NibbleVec {
inner: Vec<u8>,
len: usize,
}
impl NibbleVec {
/// Make a new NibbleVec
pub fn new() -> Self {
NibbleVec {
inner: Vec::new(),
len: 0
}
}
/// Make a NibbleVec with capacity for `n` nibbles.
pub fn with_capacity(n: usize) -> Self {
NibbleVec {
inner: Vec::with_capacity((n / 2) + (n % 2)),
len: 0
}
}
/// Length of the NibbleVec
pub fn len(&self) -> usize { self.len }
/// Capacity of the NibbleVec.
pub fn capacity(&self) -> usize { self.inner.capacity() * 2 }
/// Try to get the nibble at the given offset.
pub fn at(&self, idx: usize) -> u8 {
if idx % 2 == 0 {
self.inner[idx / 2] >> 4
} else {
self.inner[idx / 2] & 0x0F
}
}
/// Push a nibble onto the NibbleVec. Ignores the high 4 bits.
pub fn push(&mut self, nibble: u8) {
let nibble = nibble & 0x0F;
if self.len % 2 == 0 {
self.inner.push(nibble << 4);
} else {
*self.inner.last_mut().expect("len != 0 since len % 2 != 0; inner has a last element; qed") |= nibble;
}
self.len += 1;
}
/// Try to pop a nibble off the NibbleVec. Fails if len == 0.
pub fn pop(&mut self) -> Option<u8> {
if self.len == 0 {
return None;
}
let byte = self.inner.pop().expect("len != 0; inner has last elem; qed");
let nibble = if self.len % 2 == 0 {
self.inner.push(byte & 0xF0);
byte & 0x0F
} else {
byte >> 4
};
self.len -= 1;
Some(nibble)
}
/// Try to treat this NibbleVec as a NibbleSlice. Works only if len is even.
pub fn as_nibbleslice(&self) -> Option<NibbleSlice> {
if self.len % 2 == 0 {
Some(NibbleSlice::new(self.inner()))
} else {
None
}
}
/// Get the underlying byte slice.
pub fn inner(&self) -> &[u8] {
&self.inner[..]
}
}
impl<'a> From<NibbleSlice<'a>> for NibbleVec {
fn from(s: NibbleSlice<'a>) -> Self {
let mut v = NibbleVec::with_capacity(s.len());
for i in 0..s.len() {
v.push(s.at(i));
}
v
}
}
#[cfg(test)]
mod tests {
use super::NibbleVec;
#[test]
fn push_pop() {
let mut v = NibbleVec::new();
for i in 0..16 {
v.push(i);
assert_eq!(v.len() - 1, i as usize);
assert_eq!(v.at(i as usize), i);
}
for i in (0..16).rev() {
assert_eq!(v.pop(), Some(i));
assert_eq!(v.len(), i as usize);
}
}
#[test]
fn nibbleslice_conv() {
let mut v = NibbleVec::new();
for i in 0..10 {
v.push(i);
}
let v2: NibbleVec = v.as_nibbleslice().unwrap().into();
assert_eq!(v, v2);
}
}

12
util/src/overlaydb.rs
View File

@@ -172,10 +172,10 @@ impl OverlayDB {
/// Get the refs and value of the given key.
fn payload(&self, key: &H256) -> Option<(Bytes, u32)> {
self.backing.get(&key.bytes())
self.backing.get(key)
.expect("Low-level database error. Some issue with your hard disk?")
.map(|d| {
let r = Rlp::new(d.deref());
let r = Rlp::new(&d);
(r.at(1).as_val(), r.at(0).as_val())
})
}
@@ -186,10 +186,10 @@ impl OverlayDB {
let mut s = RlpStream::new_list(2);
s.append(&payload.1);
s.append(&payload.0);
batch.put(&key.bytes(), s.as_raw()).expect("Low-level database error. Some issue with your hard disk?");
batch.put(key, s.as_raw()).expect("Low-level database error. Some issue with your hard disk?");
false
} else {
batch.delete(&key.bytes()).expect("Low-level database error. Some issue with your hard disk?");
batch.delete(key).expect("Low-level database error. Some issue with your hard disk?");
true
}
}
@@ -200,10 +200,10 @@ impl OverlayDB {
let mut s = RlpStream::new_list(2);
s.append(&payload.1);
s.append(&payload.0);
self.backing.put(&key.bytes(), s.as_raw()).expect("Low-level database error. Some issue with your hard disk?");
self.backing.put(key, s.as_raw()).expect("Low-level database error. Some issue with your hard disk?");
false
} else {
self.backing.delete(&key.bytes()).expect("Low-level database error. Some issue with your hard disk?");
self.backing.delete(key).expect("Low-level database error. Some issue with your hard disk?");
true
}
}

4
util/src/rlp/bytes.rs
View File

@@ -148,9 +148,9 @@ impl_uint_to_bytes!(U128);
impl <T>ToBytes for T where T: FixedHash {
fn to_bytes<V: VecLike<u8>>(&self, out: &mut V) {
out.vec_extend(self.bytes());
out.vec_extend(self.as_slice());
}
fn to_bytes_len(&self) -> usize { self.bytes().len() }
fn to_bytes_len(&self) -> usize { self.as_slice().len() }
}
/// Error returned when `FromBytes` conversation goes wrong

23
util/src/rlp/tests.rs
View File

@@ -14,9 +14,6 @@
// You should have received a copy of the GNU General Public License
// along with Parity. If not, see <http://www.gnu.org/licenses/>.
extern crate json_tests;
use self::json_tests::execute_tests_from_directory;
use self::json_tests::rlp as rlptest;
use std::{fmt, cmp};
use std::str::FromStr;
use rlp;
@@ -340,26 +337,6 @@ fn decode_untrusted_vector_of_vectors_str() {
run_decode_tests(tests);
}
#[test]
fn test_rlp_json() {
println!("Json rlp test: ");
execute_tests_from_directory::<rlptest::RlpStreamTest, _>("json-tests/json/rlp/stream/*.json", &mut | file, input, output | {
println!("file: {}", file);
let mut stream = RlpStream::new();
for operation in input.into_iter() {
match operation {
rlptest::Operation::Append(ref v) => stream.append(v),
rlptest::Operation::AppendList(len) => stream.begin_list(len),
rlptest::Operation::AppendRaw(ref raw, len) => stream.append_raw(raw, len),
rlptest::Operation::AppendEmpty => stream.append_empty_data()
};
}
assert_eq!(stream.out(), output);
});
}
#[test]
fn test_decoding_array() {
let v = vec![5u16, 2u16];

2
util/src/sha3.rs
View File

@@ -58,7 +58,7 @@ impl<T> Hashable for T where T: BytesConvertable {
}
fn sha3_into(&self, dest: &mut [u8]) {
unsafe {
let input: &[u8] = self.bytes();
let input: &[u8] = self.as_slice();
sha3_256(dest.as_mut_ptr(), dest.len(), input.as_ptr(), input.len());
}
}

84
util/src/squeeze.rs
View File

@@ -1,84 +0,0 @@
// Copyright 2015, 2016 Ethcore (UK) Ltd.
// This file is part of Parity.
// Parity is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// Parity is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
// You should have received a copy of the GNU General Public License
// along with Parity. If not, see <http://www.gnu.org/licenses/>.
//! Helper module that should be used to randomly squeeze
//! caches to a given size in bytes
//!
//! ```
//! extern crate heapsize;
//! extern crate ethcore_util as util;
//! use std::collections::HashMap;
//! use std::mem::size_of;
//! use heapsize::HeapSizeOf;
//! use util::squeeze::Squeeze;
//!
//! fn main() {
//! let initial_size = 60;
//! let mut map: HashMap<u8, u8> = HashMap::with_capacity(initial_size);
//! assert!(map.capacity() >= initial_size);
//! for i in 0..initial_size {
//! map.insert(i as u8, i as u8);
//! }
//!
//! assert_eq!(map.heap_size_of_children(), map.capacity() * 2 * size_of::<u8>());
//! assert_eq!(map.len(), initial_size);
//! let initial_heap_size = map.heap_size_of_children();
//!
//! // squeeze it to size of key and value
//! map.squeeze(2 * size_of::<u8>());
//! assert_eq!(map.len(), 1);
//!
//! // its likely that heap size was reduced, but we can't be 100% sure
//! assert!(initial_heap_size >= map.heap_size_of_children());
//! }
//! ```
use std::collections::HashMap;
use std::hash::Hash;
use heapsize::HeapSizeOf;
/// Should be used to squeeze collections to certain size in bytes
pub trait Squeeze {
/// Try to reduce collection size to `size` bytes
fn squeeze(&mut self, size: usize);
}
impl<K, T> Squeeze for HashMap<K, T> where K: Eq + Hash + Clone + HeapSizeOf, T: HeapSizeOf {
fn squeeze(&mut self, size: usize) {
if self.is_empty() {
return
}
let size_of_entry = self.heap_size_of_children() / self.capacity();
let all_entries = size_of_entry * self.len();
let mut shrinked_size = all_entries;
while !self.is_empty() && shrinked_size > size {
// could be optimized
let key = self.keys().next().unwrap().clone();
self.remove(&key);
shrinked_size -= size_of_entry;
}
self.shrink_to_fit();
// if we squeezed something, but not enough, squeeze again
if all_entries != shrinked_size && self.heap_size_of_children() > size {
self.squeeze(size);
}
}
}

3
util/src/trie/fatdb.rs
View File

@@ -18,6 +18,7 @@ use hash::H256;
use sha3::Hashable;
use hashdb::HashDB;
use super::{TrieDB, Trie, TrieDBIterator, TrieError};
use trie::trietraits::TrieItem;
/// A `Trie` implementation which hashes keys and uses a generic `HashDB` backing database.
/// Additionaly it stores inserted hash-key mappings for later retrieval.
@@ -51,7 +52,7 @@ impl<'db> FatDB<'db> {
}
impl<'db> Trie for FatDB<'db> {
fn iter<'a>(&'a self) -> Box<Iterator<Item = (Vec<u8>, &[u8])> + 'a> {
fn iter<'a>(&'a self) -> Box<Iterator<Item = TrieItem> + 'a> {
Box::new(FatDB::iter(self))
}

3
util/src/trie/fatdbmut.rs
View File

@@ -17,7 +17,7 @@
use hash::H256;
use sha3::Hashable;
use hashdb::HashDB;
use super::{TrieDBMut, Trie, TrieMut, TrieError};
use super::{TrieDBMut, TrieMut, TrieError};
/// A mutable `Trie` implementation which hashes keys and uses a generic `HashDB` backing database.
/// Additionaly it stores inserted hash-key mappings for later retrieval.
@@ -82,6 +82,7 @@ impl<'db> TrieMut for FatDBMut<'db> {
fn fatdb_to_trie() {
use memorydb::MemoryDB;
use super::TrieDB;
use super::Trie;
let mut memdb = MemoryDB::new();
let mut root = H256::default();

1
util/src/trie/mod.rs
View File

@@ -86,6 +86,7 @@ pub struct TrieFactory {
spec: TrieSpec,
}
#[cfg_attr(feature="dev", allow(wrong_self_convention))]
impl TrieFactory {
/// Creates new factory.
pub fn new(spec: TrieSpec) -> Self {

4
util/src/trie/sectriedb.rs
View File

@@ -18,7 +18,7 @@ use hash::H256;
use sha3::Hashable;
use hashdb::HashDB;
use super::triedb::TrieDB;
use super::trietraits::Trie;
use super::trietraits::{Trie, TrieItem};
use super::TrieError;
/// A `Trie` implementation which hashes keys and uses a generic `HashDB` backing database.
@@ -50,7 +50,7 @@ impl<'db> SecTrieDB<'db> {
}
impl<'db> Trie for SecTrieDB<'db> {
fn iter<'a>(&'a self) -> Box<Iterator<Item = (Vec<u8>, &[u8])> + 'a> {
fn iter<'a>(&'a self) -> Box<Iterator<Item = TrieItem> + 'a> {
Box::new(TrieDB::iter(&self.raw))
}

3
util/src/trie/sectriedbmut.rs
View File

@@ -18,7 +18,7 @@ use hash::H256;
use sha3::Hashable;
use hashdb::HashDB;
use super::triedbmut::TrieDBMut;
use super::trietraits::{Trie, TrieMut};
use super::trietraits::TrieMut;
use super::TrieError;
/// A mutable `Trie` implementation which hashes keys and uses a generic `HashDB` backing database.
@@ -74,6 +74,7 @@ impl<'db> TrieMut for SecTrieDBMut<'db> {
fn sectrie_to_trie() {
use memorydb::*;
use super::triedb::*;
use super::Trie;
let mut memdb = MemoryDB::new();
let mut root = H256::default();

14
util/src/trie/standardmap.rs
View File

@@ -64,16 +64,16 @@ impl StandardMap {
fn random_bytes(min_count: usize, journal_count: usize, seed: &mut H256) -> Vec<u8> {
assert!(min_count + journal_count <= 32);
*seed = seed.sha3();
let r = min_count + (seed.bytes()[31] as usize % (journal_count + 1));
seed.bytes()[0..r].to_vec()
let r = min_count + (seed[31] as usize % (journal_count + 1));
seed[0..r].to_vec()
}
/// Get a random value. Equal chance of being 1 byte as of 32. `seed` is mutated pseudoramdonly and used.
fn random_value(seed: &mut H256) -> Bytes {
*seed = seed.sha3();
match seed.bytes()[0] % 2 {
1 => vec![seed.bytes()[31];1],
_ => seed.bytes().to_vec(),
match seed[0] % 2 {
1 => vec![seed[31];1],
_ => seed.to_vec(),
}
}
@@ -82,10 +82,10 @@ impl StandardMap {
fn random_word(alphabet: &[u8], min_count: usize, journal_count: usize, seed: &mut H256) -> Vec<u8> {
assert!(min_count + journal_count <= 32);
*seed = seed.sha3();
let r = min_count + (seed.bytes()[31] as usize % (journal_count + 1));
let r = min_count + (seed[31] as usize % (journal_count + 1));
let mut ret: Vec<u8> = Vec::with_capacity(r);
for i in 0..r {
ret.push(alphabet[seed.bytes()[i] as usize % alphabet.len()]);
ret.push(alphabet[seed[i] as usize % alphabet.len()]);
}
ret
}

4
util/src/trie/triedb.rs
View File

@@ -18,7 +18,7 @@ use common::*;
use hashdb::*;
use nibbleslice::*;
use rlp::*;
use super::trietraits::{Trie};
use super::trietraits::{Trie, TrieItem};
use super::node::Node;
use super::TrieError;
@@ -337,7 +337,7 @@ impl<'db> TrieDB<'db> {
}
impl<'db> Trie for TrieDB<'db> {
fn iter<'a>(&'a self) -> Box<Iterator<Item = (Vec<u8>, &[u8])> + 'a> {
fn iter<'a>(&'a self) -> Box<Iterator<Item = TrieItem> + 'a> {
Box::new(TrieDB::iter(self))
}

92
util/src/trie/triedbmut.rs
View File

@@ -20,7 +20,7 @@ use nibbleslice::*;
use rlp::*;
use super::node::Node;
use super::journal::Journal;
use super::trietraits::{Trie, TrieMut};
use super::trietraits::TrieMut;
use super::TrieError;
/// A `Trie` implementation using a generic `HashDB` backing database.
@@ -676,8 +676,6 @@ impl<'db> fmt::Debug for TrieDBMut<'db> {
#[cfg(test)]
mod tests {
extern crate json_tests;
use self::json_tests::{trie, execute_tests_from_directory};
use triehash::*;
use hash::*;
use hashdb::*;
@@ -858,6 +856,21 @@ mod tests {
]));
}
#[test]
fn insert_out_of_order() {
let mut memdb = MemoryDB::new();
let mut root = H256::new();
let mut t = TrieDBMut::new(&mut memdb, &mut root);
t.insert(&[0xf1u8, 0x23], &[0xf1u8, 0x23]);
t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]);
t.insert(&[0x81u8, 0x23], &[0x81u8, 0x23]);
assert_eq!(*t.root(), trie_root(vec![
(vec![0x01u8, 0x23], vec![0x01u8, 0x23]),
(vec![0x81u8, 0x23], vec![0x81u8, 0x23]),
(vec![0xf1u8, 0x23], vec![0xf1u8, 0x23]),
]));
}
#[test]
fn insert_value_into_branch_root() {
let mut memdb = MemoryDB::new();
@@ -1063,23 +1076,64 @@ mod tests {
}
#[test]
fn test_trie_json() {
println!("Json trie test: ");
execute_tests_from_directory::<trie::TrieTest, _>("json-tests/json/trie/*.json", &mut | file, input, output | {
println!("file: {}", file);
fn branching_test() {
use std::str::FromStr;
use rustc_serialize::hex::FromHex;
let mut memdb = MemoryDB::new();
let mut root = H256::new();
let mut t = TrieDBMut::new(&mut memdb, &mut root);
for operation in input.into_iter() {
match operation {
trie::Operation::Insert(key, value) => t.insert(&key, &value),
trie::Operation::Remove(key) => t.remove(&key)
}
}
assert_eq!(*t.root(), H256::from_slice(&output));
});
let mut memdb = MemoryDB::new();
let mut root = H256::new();
let mut t = TrieDBMut::new(&mut memdb, &mut root);
t.insert(&"04110d816c380812a427968ece99b1c963dfbce6".from_hex().unwrap(), b"something");
t.insert(&"095e7baea6a6c7c4c2dfeb977efac326af552d87".from_hex().unwrap(), b"something");
t.insert(&"0a517d755cebbf66312b30fff713666a9cb917e0".from_hex().unwrap(), b"something");
t.insert(&"24dd378f51adc67a50e339e8031fe9bd4aafab36".from_hex().unwrap(), b"something");
t.insert(&"293f982d000532a7861ab122bdc4bbfd26bf9030".from_hex().unwrap(), b"something");
t.insert(&"2cf5732f017b0cf1b1f13a1478e10239716bf6b5".from_hex().unwrap(), b"something");
t.insert(&"31c640b92c21a1f1465c91070b4b3b4d6854195f".from_hex().unwrap(), b"something");
t.insert(&"37f998764813b136ddf5a754f34063fd03065e36".from_hex().unwrap(), b"something");
t.insert(&"37fa399a749c121f8a15ce77e3d9f9bec8020d7a".from_hex().unwrap(), b"something");
t.insert(&"4f36659fa632310b6ec438dea4085b522a2dd077".from_hex().unwrap(), b"something");
t.insert(&"62c01474f089b07dae603491675dc5b5748f7049".from_hex().unwrap(), b"something");
t.insert(&"729af7294be595a0efd7d891c9e51f89c07950c7".from_hex().unwrap(), b"something");
t.insert(&"83e3e5a16d3b696a0314b30b2534804dd5e11197".from_hex().unwrap(), b"something");
t.insert(&"8703df2417e0d7c59d063caa9583cb10a4d20532".from_hex().unwrap(), b"something");
t.insert(&"8dffcd74e5b5923512916c6a64b502689cfa65e1".from_hex().unwrap(), b"something");
t.insert(&"95a4d7cccb5204733874fa87285a176fe1e9e240".from_hex().unwrap(), b"something");
t.insert(&"99b2fcba8120bedd048fe79f5262a6690ed38c39".from_hex().unwrap(), b"something");
t.insert(&"a4202b8b8afd5354e3e40a219bdc17f6001bf2cf".from_hex().unwrap(), b"something");
t.insert(&"a94f5374fce5edbc8e2a8697c15331677e6ebf0b".from_hex().unwrap(), b"something");
t.insert(&"a9647f4a0a14042d91dc33c0328030a7157c93ae".from_hex().unwrap(), b"something");
t.insert(&"aa6cffe5185732689c18f37a7f86170cb7304c2a".from_hex().unwrap(), b"something");
t.insert(&"aae4a2e3c51c04606dcb3723456e58f3ed214f45".from_hex().unwrap(), b"something");
t.insert(&"c37a43e940dfb5baf581a0b82b351d48305fc885".from_hex().unwrap(), b"something");
t.insert(&"d2571607e241ecf590ed94b12d87c94babe36db6".from_hex().unwrap(), b"something");
t.insert(&"f735071cbee190d76b704ce68384fc21e389fbe7".from_hex().unwrap(), b"something");
t.insert(&"04110d816c380812a427968ece99b1c963dfbce6".from_hex().unwrap(), &[]);
t.insert(&"095e7baea6a6c7c4c2dfeb977efac326af552d87".from_hex().unwrap(), &[]);
t.insert(&"0a517d755cebbf66312b30fff713666a9cb917e0".from_hex().unwrap(), &[]);
t.insert(&"24dd378f51adc67a50e339e8031fe9bd4aafab36".from_hex().unwrap(), &[]);
t.insert(&"293f982d000532a7861ab122bdc4bbfd26bf9030".from_hex().unwrap(), &[]);
t.insert(&"2cf5732f017b0cf1b1f13a1478e10239716bf6b5".from_hex().unwrap(), &[]);
t.insert(&"31c640b92c21a1f1465c91070b4b3b4d6854195f".from_hex().unwrap(), &[]);
t.insert(&"37f998764813b136ddf5a754f34063fd03065e36".from_hex().unwrap(), &[]);
t.insert(&"37fa399a749c121f8a15ce77e3d9f9bec8020d7a".from_hex().unwrap(), &[]);
t.insert(&"4f36659fa632310b6ec438dea4085b522a2dd077".from_hex().unwrap(), &[]);
t.insert(&"62c01474f089b07dae603491675dc5b5748f7049".from_hex().unwrap(), &[]);
t.insert(&"729af7294be595a0efd7d891c9e51f89c07950c7".from_hex().unwrap(), &[]);
t.insert(&"83e3e5a16d3b696a0314b30b2534804dd5e11197".from_hex().unwrap(), &[]);
t.insert(&"8703df2417e0d7c59d063caa9583cb10a4d20532".from_hex().unwrap(), &[]);
t.insert(&"8dffcd74e5b5923512916c6a64b502689cfa65e1".from_hex().unwrap(), &[]);
t.insert(&"95a4d7cccb5204733874fa87285a176fe1e9e240".from_hex().unwrap(), &[]);
t.insert(&"99b2fcba8120bedd048fe79f5262a6690ed38c39".from_hex().unwrap(), &[]);
t.insert(&"a4202b8b8afd5354e3e40a219bdc17f6001bf2cf".from_hex().unwrap(), &[]);
t.insert(&"a94f5374fce5edbc8e2a8697c15331677e6ebf0b".from_hex().unwrap(), &[]);
t.insert(&"a9647f4a0a14042d91dc33c0328030a7157c93ae".from_hex().unwrap(), &[]);
t.insert(&"aa6cffe5185732689c18f37a7f86170cb7304c2a".from_hex().unwrap(), &[]);
t.insert(&"aae4a2e3c51c04606dcb3723456e58f3ed214f45".from_hex().unwrap(), &[]);
t.insert(&"c37a43e940dfb5baf581a0b82b351d48305fc885".from_hex().unwrap(), &[]);
t.insert(&"d2571607e241ecf590ed94b12d87c94babe36db6".from_hex().unwrap(), &[]);
t.insert(&"f735071cbee190d76b704ce68384fc21e389fbe7".from_hex().unwrap(), &[]);
assert_eq!(*t.root(), H256::from_str("56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421").unwrap());
}
#[test]

5
util/src/trie/trietraits.rs
View File

@@ -17,6 +17,9 @@
use hash::H256;
use rlp::SHA3_NULL_RLP;
/// Trie-Item type.
pub type TrieItem<'a> = (Vec<u8>, &'a[u8]);
/// A key-value datastore implemented as a database-backed modified Merkle tree.
pub trait Trie {
/// Return the root of the trie.
@@ -32,7 +35,7 @@ pub trait Trie {
fn get<'a, 'key>(&'a self, key: &'key [u8]) -> Option<&'a [u8]> where 'a: 'key;
/// Returns an iterator over elements of trie.
fn iter<'a>(&'a self) -> Box<Iterator<Item = (Vec<u8>, &[u8])> + 'a>;
fn iter<'a>(&'a self) -> Box<Iterator<Item = TrieItem> + 'a>;
}
/// A key-value datastore implemented as a database-backed modified Merkle tree.

37
util/src/triehash.rs
View File

@@ -46,7 +46,8 @@ pub fn ordered_trie_root(input: Vec<Vec<u8>>) -> H256 {
// optimize it later
.into_iter()
.enumerate()
.fold(BTreeMap::new(), | mut acc, (i, vec) | { acc.insert(rlp::encode(&i).to_vec(), vec); acc })
.map(|(i, vec)| (rlp::encode(&i).to_vec(), vec))
.collect::<BTreeMap<_, _>>()
// then move them to a vector
.into_iter()
.map(|(k, v)| (as_nibbles(&k), v) )
@@ -78,10 +79,7 @@ pub fn trie_root(input: Vec<(Vec<u8>, Vec<u8>)>) -> H256 {
let gen_input = input
// first put elements into btree to sort them and to remove duplicates
.into_iter()
.fold(BTreeMap::new(), | mut acc, (k, v) | {
acc.insert(k, v);
acc
})
.collect::<BTreeMap<_, _>>()
// then move them to a vector
.into_iter()
.map(|(k, v)| (as_nibbles(&k), v) )
@@ -97,7 +95,7 @@ pub fn trie_root(input: Vec<(Vec<u8>, Vec<u8>)>) -> H256 {
/// use std::str::FromStr;
/// use util::triehash::*;
/// use util::hash::*;
///
///
/// fn main() {
/// let v = vec![
/// (From::from("doe"), From::from("reindeer")),
@@ -113,10 +111,8 @@ pub fn sec_trie_root(input: Vec<(Vec<u8>, Vec<u8>)>) -> H256 {
let gen_input = input
// first put elements into btree to sort them and to remove duplicates
.into_iter()
.fold(BTreeMap::new(), | mut acc, (k, v) | {
acc.insert(k.sha3().to_vec(), v);
acc
})
.map(|(k, v)| (k.sha3().to_vec(), v))
.collect::<BTreeMap<_, _>>()
// then move them to a vector
.into_iter()
.map(|(k, v)| (as_nibbles(&k), v) )
@@ -324,10 +320,16 @@ fn test_hex_prefix_encode() {
#[cfg(test)]
mod tests {
extern crate json_tests;
use self::json_tests::*;
use hash::*;
use triehash::*;
use std::str::FromStr;
use hash::H256;
use super::trie_root;
#[test]
fn simple_test() {
assert_eq!(trie_root(vec![
(b"A".to_vec(), b"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa".to_vec())
]), H256::from_str("d23786fb4a010da3ce639d66d5e904a11dbc02746d1ce25029e53290cabf28ab").unwrap());
}
#[test]
fn test_triehash_out_of_order() {
@@ -343,11 +345,4 @@ mod tests {
]));
}
#[test]
fn test_triehash_json() {
execute_tests_from_directory::<trie::TriehashTest, _>("json-tests/json/trie/*.json", &mut | file, input, output | {
println!("file: {}, output: {:?}", file, output);
assert_eq!(trie_root(input), H256::from_slice(&output));
});
}
}

4
util/json-tests/Cargo.toml → util/table/Cargo.toml
View File

@@ -1,8 +1,6 @@
[package]
name = "json-tests"
name = "table"
version = "0.1.0"
authors = ["debris <marek.kotewicz@gmail.com>"]
[dependencies]
rustc-serialize = "0.3"
glob = "*"

0
util/src/table.rs → util/table/src/lib.rs
View File

6
util/using_queue/Cargo.toml Normal file
View File

@@ -0,0 +1,6 @@
[package]
name = "using_queue"
version = "0.1.0"
authors = ["debris <marek.kotewicz@gmail.com>"]
[dependencies]

3
util/src/using_queue.rs → util/using_queue/src/lib.rs
View File

@@ -13,6 +13,7 @@
// You should have received a copy of the GNU General Public License
// along with Parity. If not, see <http://www.gnu.org/licenses/>.
//! Queue-like datastructure including notion of usage.
/// Special queue-like datastructure that includes the notion of
@@ -50,7 +51,7 @@ impl<T> UsingQueue<T> where T: Clone {
pub fn peek_last_ref(&self) -> Option<&T> {
self.pending.as_ref().or(self.in_use.last())
}
/// Return a reference to the item at the top of the queue (or `None` if the queue is empty);
/// this constitutes using the item and will remain in the queue for at least another
/// `max_size` invocations of `push()`.