grassrootseconomics/openethereum
8
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Merge branch 'master' into rustfmt

Browse Source 
Conflicts:
	ethash/src/sizes.rs
	util/src/uint.rs
This commit is contained in:
Tomasz Drwięga
2016-03-23 16:55:54 +01:00
parent 22e2458ce5 038b67cbd4
commit a0db1d5416
252 changed files with 25911 additions and 6611 deletions
Download Patch File Download Diff File Expand all files Collapse all files

30
util/Cargo.toml
View File

@@ -3,8 +3,9 @@ description = "Ethcore utility library"
homepage = "http://ethcore.io"
license = "GPL-3.0"
name = "ethcore-util"
version = "0.9.0"
version = "1.1.0"
authors = ["Ethcore <admin@ethcore.io>"]
build = "build.rs"
[dependencies]
log = "0.3"
@@ -15,18 +16,31 @@ mio = "0.5.0"
rand = "0.3.12"
time = "0.1.34"
tiny-keccak = "1.0"
rocksdb = "0.3"
rocksdb = { git = "https://github.com/arkpar/rust-rocksdb.git" }
lazy_static = "0.1"
eth-secp256k1 = { git = "https://github.com/arkpar/rust-secp256k1.git" }
eth-secp256k1 = { git = "https://github.com/ethcore/rust-secp256k1" }
rust-crypto = "0.2.34"
elastic-array = "0.4"
heapsize = "0.2"
heapsize = "0.3"
itertools = "0.4"
crossbeam = "0.2"
slab = { git = "https://github.com/arkpar/slab.git" }
slab = "0.1"
sha3 = { path = "sha3" }
serde = "0.6.7"
clippy = "0.0.41"
serde = "0.7.0"
clippy = { version = "0.0.54", optional = true }
json-tests = { path = "json-tests" }
target_info = "0.1.0"
igd = "0.4.2"
ethcore-devtools = { path = "../devtools" }
libc = "0.2.7"
vergen = "0.1"
target_info = "0.1"
bigint = { path = "bigint" }
chrono = "0.2"
[features]
default = []
dev = ["clippy"]
[build-dependencies]
vergen = "*"
rustc_version = "0.1.0"

104
util/benches/bigint.rs Normal file
View File

@@ -0,0 +1,104 @@
// Copyright 2015, 2016 Ethcore (UK) Ltd.
// This file is part of Parity.
// Parity is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// Parity is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
// You should have received a copy of the GNU General Public License
// along with Parity. If not, see <http://www.gnu.org/licenses/>.
//! benchmarking for rlp
//! should be started with:
//! ```bash
//! multirust run nightly cargo bench
//! ```
#![feature(test)]
#![feature(asm)]
extern crate test;
extern crate ethcore_util;
extern crate rand;
use test::{Bencher, black_box};
use ethcore_util::numbers::*;
#[bench]
fn u256_add(b: &mut Bencher) {
b.iter(|| {
let n = black_box(10000);
(0..n).fold(U256([rand::random::<u64>(), rand::random::<u64>(), rand::random::<u64>(), rand::random::<u64>()]), |old, new| { old.overflowing_add(U256::from(new)).0 })
});
}
#[bench]
fn u256_sub(b: &mut Bencher) {
b.iter(|| {
let n = black_box(10000);
(0..n).fold(U256([rand::random::<u64>(), rand::random::<u64>(), rand::random::<u64>(), rand::random::<u64>()]), |old, new| { old.overflowing_sub(U256::from(new)).0 })
});
}
#[bench]
fn u512_sub(b: &mut Bencher) {
b.iter(|| {
let n = black_box(10000);
(0..n).fold(
U512([
rand::random::<u64>(), rand::random::<u64>(), rand::random::<u64>(), rand::random::<u64>(),
rand::random::<u64>(), rand::random::<u64>(), rand::random::<u64>(), rand::random::<u64>()
]),
|old, new| {
let p = new % 2;
old.overflowing_sub(U512([p, p, p, p, p, p, p, new])).0
}
)
});
}
#[bench]
fn u512_add(b: &mut Bencher) {
b.iter(|| {
let n = black_box(10000);
(0..n).fold(U512([0, 0, 0, 0, 0, 0, 0, 0]),
|old, new| { old.overflowing_add(U512([new, new, new, new, new, new, new, new])).0 })
});
}
#[bench]
fn u256_mul(b: &mut Bencher) {
b.iter(|| {
let n = black_box(10000);
(0..n).fold(U256([rand::random::<u64>(), rand::random::<u64>(), rand::random::<u64>(), rand::random::<u64>()]), |old, new| { old.overflowing_mul(U256::from(new)).0 })
});
}
#[bench]
fn u256_full_mul(b: &mut Bencher) {
b.iter(|| {
let n = black_box(10000);
(0..n).fold(U256([rand::random::<u64>(), rand::random::<u64>(), rand::random::<u64>(), rand::random::<u64>()]),
|old, _new| {
let U512(ref u512words) = old.full_mul(U256([rand::random::<u64>(), rand::random::<u64>(), rand::random::<u64>(), rand::random::<u64>()]));
U256([u512words[0], u512words[2], u512words[2], u512words[3]])
})
});
}
#[bench]
fn u128_mul(b: &mut Bencher) {
b.iter(|| {
let n = black_box(10000);
(0..n).fold(U128([12345u64, 0u64]), |old, new| { old.overflowing_mul(U128::from(new)).0 })
});
}

2
util/benches/rlp.rs
View File

@@ -28,7 +28,7 @@ extern crate ethcore_util;
use test::Bencher;
use std::str::FromStr;
use ethcore_util::rlp::*;
use ethcore_util::uint::U256;
use ethcore_util::numbers::U256;
#[bench]
fn bench_stream_u64_value(b: &mut Bencher) {

22
util/bigint/Cargo.toml Normal file
View File

@@ -0,0 +1,22 @@
[package]
description = "Rust-assembler implementation of big integers arithmetic"
homepage = "http://ethcore.io"
license = "GPL-3.0"
name = "bigint"
version = "0.1.0"
authors = ["Ethcore <admin@ethcore.io>"]
build = "build.rs"
[build-dependencies]
rustc_version = "0.1"
[dependencies]
rustc-serialize = "0.3"
arrayvec = "0.3"
rand = "0.3.12"
serde = "0.7.0"
heapsize = "0.3"
[features]
x64asm_arithmetic=[]
rust_arithmetic=[]

25
util/bigint/build.rs Normal file
View File

@@ -0,0 +1,25 @@
// Copyright 2015, 2016 Ethcore (UK) Ltd.
// This file is part of Parity.
// Parity is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// Parity is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
// You should have received a copy of the GNU General Public License
// along with Parity. If not, see <http://www.gnu.org/licenses/>.
extern crate rustc_version;
use rustc_version::{version_meta, Channel};
fn main() {
if let Channel::Nightly = version_meta().channel {
println!("cargo:rustc-cfg=asm_available");
}
}

23
util/bigint/src/lib.rs Normal file
View File

@@ -0,0 +1,23 @@
// Copyright 2015, 2016 Ethcore (UK) Ltd.
// This file is part of Parity.
// Parity is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// Parity is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
// You should have received a copy of the GNU General Public License
// along with Parity. If not, see <http://www.gnu.org/licenses/>.
#![cfg_attr(asm_available, feature(asm))]
extern crate rustc_serialize;
extern crate serde;
#[macro_use] extern crate heapsize;
pub mod uint;

2152
util/bigint/src/uint.rs Normal file
View File

File diff suppressed because it is too large Load Diff

37
util/build.rs Normal file
View File

@@ -0,0 +1,37 @@
// Copyright 2015, 2016 Ethcore (UK) Ltd.
// This file is part of Parity.
// Parity is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// Parity is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
// You should have received a copy of the GNU General Public License
// along with Parity. If not, see <http://www.gnu.org/licenses/>.
extern crate vergen;
extern crate rustc_version;
use vergen::*;
use std::env;
use std::fs::File;
use std::io::Write;
use std::path::Path;
fn main() {
vergen(OutputFns::all()).unwrap();
let out_dir = env::var("OUT_DIR").unwrap();
let dest_path = Path::new(&out_dir).join("rustc_version.rs");
let mut f = File::create(&dest_path).unwrap();
f.write_all(format!("
/// Returns compiler version.
pub fn rustc_version() -> &'static str {{
\"{}\"
}}
", rustc_version::version()).as_bytes()).unwrap();
}

25
util/fdlimit/src/raise_fd_limit.rs
View File

@@ -57,5 +57,28 @@ pub unsafe fn raise_fd_limit() {
}
}
#[cfg(not(any(target_os = "macos", target_os = "ios")))]
#[cfg(any(target_os = "linux"))]
#[allow(non_camel_case_types)]
pub unsafe fn raise_fd_limit() {
use libc;
use std::io;
// Fetch the current resource limits
let mut rlim = libc::rlimit{rlim_cur: 0, rlim_max: 0};
if libc::getrlimit(libc::RLIMIT_NOFILE, &mut rlim) != 0 {
let err = io::Error::last_os_error();
panic!("raise_fd_limit: error calling getrlimit: {}", err);
}
// Set soft limit to hard imit
rlim.rlim_cur = rlim.rlim_max;
// Set our newly-increased resource limit
if libc::setrlimit(libc::RLIMIT_NOFILE, &rlim) != 0 {
let err = io::Error::last_os_error();
panic!("raise_fd_limit: error calling setrlimit: {}", err);
}
}
#[cfg(not(any(target_os = "macos", target_os = "ios", target_os = "linux")))]
pub unsafe fn raise_fd_limit() {}

21
util/res/geth_keystore/UTC--2016-02-17T09-20-45.721400158Z--3f49624084b67849c7b4e805c5988c21a430f9d9 Normal file
View File

@@ -0,0 +1,21 @@
{
"address": "3f49624084b67849c7b4e805c5988c21a430f9d9",
"Crypto": {
"cipher": "aes-128-ctr",
"ciphertext": "9f27e3dd4fc73e7103ed61e5493662189a3eb52223ae49e3d1deacc04c889eae",
"cipherparams": {
"iv": "457494bf05f2618c397dc74dbb5181c0"
},
"kdf": "scrypt",
"kdfparams": {
"dklen": 32,
"n": 262144,
"p": 1,
"r": 8,
"salt": "db14edb18c41ee7f5ec4397df89c3a2ae4d0af60884c52bb54ce490574f8df33"
},
"mac": "572d24532438d31fdf513c744a3ff26c933ffda5744ee42bc71661cbe3f2112e"
},
"id": "62a0ad73-556d-496a-8e1c-0783d30d3ace",
"version": 3
}

21
util/res/geth_keystore/UTC--2016-02-20T09-33-03.984382741Z--5ba4dcf897e97c2bdf8315b9ef26c13c085988cf Normal file
View File

@@ -0,0 +1,21 @@
{
"address": "5ba4dcf897e97c2bdf8315b9ef26c13c085988cf",
"Crypto": {
"cipher": "aes-128-ctr",
"ciphertext": "d4a08ec930163778273920f6ad1d49b71836337be6fd9863993ac700a612fddd",
"cipherparams": {
"iv": "89ce5ec129fc27cd5bcbeb8c92bdad50"
},
"kdf": "scrypt",
"kdfparams": {
"dklen": 32,
"n": 262144,
"p": 1,
"r": 8,
"salt": "612ab108dc37e69ee8af37a7b24bf7f2234086d7bbf945bacdeccce331f7f84a"
},
"mac": "4152caa7444e06784223d735cea80cd2690b4c587ad8db3d5529442227b25695"
},
"id": "35086353-fb12-4029-b56b-033cd61ce35b",
"version": 3
}

73
util/src/bytes.rs
View File

@@ -141,7 +141,7 @@ impl<'a> Deref for BytesRef<'a> {
fn deref(&self) -> &[u8] {
match *self {
BytesRef::Flexible(ref bytes) => bytes,
BytesRef::Fixed(ref bytes) => bytes
BytesRef::Fixed(ref bytes) => bytes,
}
}
}
@@ -150,7 +150,7 @@ impl <'a> DerefMut for BytesRef<'a> {
fn deref_mut(&mut self) -> &mut [u8] {
match *self {
BytesRef::Flexible(ref mut bytes) => bytes,
BytesRef::Fixed(ref mut bytes) => bytes
BytesRef::Fixed(ref mut bytes) => bytes,
}
}
}
@@ -170,34 +170,14 @@ pub trait BytesConvertable {
fn to_bytes(&self) -> Bytes { self.as_slice().to_vec() }
}
impl<'a> BytesConvertable for &'a [u8] {
fn bytes(&self) -> &[u8] { self }
}
impl BytesConvertable for Vec<u8> {
fn bytes(&self) -> &[u8] { self }
}
macro_rules! impl_bytes_convertable_for_array {
($zero: expr) => ();
($len: expr, $($idx: expr),*) => {
impl BytesConvertable for [u8; $len] {
fn bytes(&self) -> &[u8] { self }
}
impl_bytes_convertable_for_array! { $($idx),* }
}
}
// -1 at the end is not expanded
impl_bytes_convertable_for_array! {
32, 31, 30, 29, 28, 27, 26, 25, 24, 23, 22, 21, 20, 19, 18, 17, 16,
15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0, -1
impl<T> BytesConvertable for T where T: AsRef<[u8]> {
fn bytes(&self) -> &[u8] { self.as_ref() }
}
#[test]
fn bytes_convertable() {
assert_eq!(vec![0x12u8, 0x34].bytes(), &[0x12u8, 0x34]);
assert_eq!([0u8; 0].bytes(), &[]);
assert!([0u8; 0].as_slice().is_empty());
}
/// Simple trait to allow for raw population of a Sized object from a byte slice.
@@ -252,6 +232,49 @@ impl<T> Populatable for [T] where T: Sized {
}
}
#[derive(Debug)]
/// Bytes array deserialization error
pub enum FromBytesError {
/// Not enough bytes for the requested type
NotLongEnough,
/// Too many bytes for the requested type
TooLong,
}
/// Value that can be serialized from bytes array
pub trait FromRawBytes : Sized {
/// function that will instantiate and initialize object from slice
fn from_bytes(d: &[u8]) -> Result<Self, FromBytesError>;
}
impl<T> FromRawBytes for T where T: Sized + FixedHash {
fn from_bytes(bytes: &[u8]) -> Result<Self, FromBytesError> {
use std::mem;
use std::cmp::Ordering;
match bytes.len().cmp(&mem::size_of::<T>()) {
Ordering::Less => return Err(FromBytesError::NotLongEnough),
Ordering::Greater => return Err(FromBytesError::TooLong),
Ordering::Equal => ()
};
let mut res: Self = unsafe { mem::uninitialized() };
res.copy_raw(bytes);
Ok(res)
}
}
impl FromRawBytes for String {
fn from_bytes(bytes: &[u8]) -> Result<String, FromBytesError> {
Ok(::std::str::from_utf8(bytes).unwrap().to_owned())
}
}
impl FromRawBytes for Vec<u8> {
fn from_bytes(bytes: &[u8]) -> Result<Vec<u8>, FromBytesError> {
Ok(bytes.clone().to_vec())
}
}
#[test]
fn fax_raw() {
let mut x = [255u8; 4];

469
util/src/chainfilter.rs
View File

@@ -1,469 +0,0 @@
// Copyright 2015, 2016 Ethcore (UK) Ltd.
// This file is part of Parity.
// Parity is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// Parity is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
// You should have received a copy of the GNU General Public License
// along with Parity. If not, see <http://www.gnu.org/licenses/>.
//! Multilevel blockchain bloom filter.
//!
//! ```
//! extern crate ethcore_util as util;
//! use std::str::FromStr;
//! use util::chainfilter::*;
//! use util::sha3::*;
//! use util::hash::*;
//!
//! fn main() {
//! let (index_size, bloom_levels) = (16, 3);
//! let mut cache = MemoryCache::new();
//!
//! let address = Address::from_str("ef2d6d194084c2de36e0dabfce45d046b37d1106").unwrap();
//!
//! // borrow cache for reading inside the scope
//! let modified_blooms = {
//! let filter = ChainFilter::new(&cache, index_size, bloom_levels);
//! let block_number = 39;
//! let mut bloom = H2048::new();
//! bloom.shift_bloomed(&address.sha3());
//! filter.add_bloom(&bloom, block_number)
//! };
//!
//! // number of updated blooms is equal number of levels
//! assert_eq!(modified_blooms.len(), bloom_levels as usize);
//!
//! // lets inserts modified blooms into the cache
//! cache.insert_blooms(modified_blooms);
//!
//! // borrow cache for another reading operations
//! {
//! let filter = ChainFilter::new(&cache, index_size, bloom_levels);
//! let blocks = filter.blocks_with_address(&address, 10, 40);
//! assert_eq!(blocks.len(), 1);
//! assert_eq!(blocks[0], 39);
//! }
//! }
//! ```
//!
use std::collections::{HashMap};
use hash::*;
use sha3::*;
/// Represents bloom index in cache
///
/// On cache level 0, every block bloom is represented by different index.
/// On higher cache levels, multiple block blooms are represented by one
/// index. Their `BloomIndex` can be created from block number and given level.
#[derive(Eq, PartialEq, Hash, Clone, Debug)]
pub struct BloomIndex {
/// Bloom level
pub level: u8,
/// Filter Index
pub index: usize,
}
impl BloomIndex {
/// Default constructor for `BloomIndex`
pub fn new(level: u8, index: usize) -> BloomIndex {
BloomIndex {
level: level,
index: index,
}
}
}
/// Types implementing this trait should provide read access for bloom filters database.
pub trait FilterDataSource {
/// returns reference to log at given position if it exists
fn bloom_at_index(&self, index: &BloomIndex) -> Option<&H2048>;
}
/// In memory cache for blooms.
///
/// Stores all blooms in HashMap, which indexes them by `BloomIndex`.
pub struct MemoryCache {
blooms: HashMap<BloomIndex, H2048>,
}
impl MemoryCache {
/// Default constructor for MemoryCache
pub fn new() -> MemoryCache {
MemoryCache { blooms: HashMap::new() }
}
/// inserts all blooms into cache
///
/// if bloom at given index already exists, overwrites it
pub fn insert_blooms(&mut self, blooms: HashMap<BloomIndex, H2048>) {
self.blooms.extend(blooms);
}
}
impl FilterDataSource for MemoryCache {
fn bloom_at_index(&self, index: &BloomIndex) -> Option<&H2048> {
self.blooms.get(index)
}
}
/// Should be used for search operations on blockchain.
pub struct ChainFilter<'a, D>
where D: FilterDataSource + 'a
{
data_source: &'a D,
index_size: usize,
level_sizes: Vec<usize>,
}
impl<'a, D> ChainFilter<'a, D> where D: FilterDataSource
{
/// Creates new filter instance.
///
/// Borrows `FilterDataSource` for reading.
pub fn new(data_source: &'a D, index_size: usize, levels: u8) -> Self {
if levels == 0 {
panic!("ChainFilter requires at least 1 level");
}
let mut filter = ChainFilter {
data_source: data_source,
index_size: index_size,
// 0 level has always a size of 1
level_sizes: vec![1]
};
// cache level sizes, so we do not have to calculate them all the time
// eg. if levels == 3, index_size = 16
// level_sizes = [1, 16, 256]
let additional: Vec<usize> = (1..).into_iter()
.scan(1, |acc, _| {
*acc = *acc * index_size;
Some(*acc)
})
.take(levels as usize - 1)
.collect();
filter.level_sizes.extend(additional);
filter
}
/// unsafely get level size
fn level_size(&self, level: u8) -> usize {
self.level_sizes[level as usize]
}
/// converts block number and level to `BloomIndex`
fn bloom_index(&self, block_number: usize, level: u8) -> BloomIndex {
BloomIndex {
level: level,
index: block_number / self.level_size(level),
}
}
/// return bloom which are dependencies for given index
///
/// bloom indexes are ordered from lowest to highest
fn lower_level_bloom_indexes(&self, index: &BloomIndex) -> Vec<BloomIndex> {
// this is the lowest level
if index.level == 0 {
return vec![];
}
let new_level = index.level - 1;
let offset = self.index_size * index.index;
(0..self.index_size).map(|i| BloomIndex::new(new_level, offset + i)).collect()
}
/// return number of levels
fn levels(&self) -> u8 {
self.level_sizes.len() as u8
}
/// returns max filter level
fn max_level(&self) -> u8 {
self.level_sizes.len() as u8 - 1
}
/// internal function which does bloom search recursively
fn blocks(&self, bloom: &H2048, from_block: usize, to_block: usize, level: u8, offset: usize) -> Option<Vec<usize>> {
let index = self.bloom_index(offset, level);
match self.data_source.bloom_at_index(&index) {
None => return None,
Some(level_bloom) => match level {
// if we are on the lowest level
// take the value, exclude to_block
0 if offset < to_block => return Some(vec![offset]),
// return None if it is is equal to to_block
0 => return None,
// return None if current level doesnt contain given bloom
_ if !level_bloom.contains(bloom) => return None,
// continue processing && go down
_ => ()
}
};
let level_size = self.level_size(level - 1);
let from_index = self.bloom_index(from_block, level - 1);
let to_index = self.bloom_index(to_block, level - 1);
let res: Vec<usize> = self.lower_level_bloom_indexes(&index).into_iter()
// chose only blooms in range
.filter(|li| li.index >= from_index.index && li.index <= to_index.index)
// map them to offsets
.map(|li| li.index * level_size)
// get all blocks that may contain our bloom
.map(|off| self.blocks(bloom, from_block, to_block, level - 1, off))
// filter existing ones
.filter_map(|x| x)
// flatten nested structures
.flat_map(|v| v)
.collect();
Some(res)
}
/// Adds new bloom to all filter levels
pub fn add_bloom(&self, bloom: &H2048, block_number: usize) -> HashMap<BloomIndex, H2048> {
let mut result: HashMap<BloomIndex, H2048> = HashMap::new();
for level in 0..self.levels() {
let bloom_index = self.bloom_index(block_number, level);
let new_bloom = match self.data_source.bloom_at_index(&bloom_index) {
Some(old_bloom) => old_bloom | bloom,
None => bloom.clone(),
};
result.insert(bloom_index, new_bloom);
}
result
}
/// Adds new blooms starting from block number.
pub fn add_blooms(&self, blooms: &[H2048], block_number: usize) -> HashMap<BloomIndex, H2048> {
let mut result: HashMap<BloomIndex, H2048> = HashMap::new();
for level in 0..self.levels() {
for i in 0..blooms.len() {
let bloom_index = self.bloom_index(block_number + i, level);
let is_new_bloom = match result.get_mut(&bloom_index) {
// it was already modified
Some(to_shift) => {
*to_shift = &blooms[i] | to_shift;
false
}
None => true,
};
// it hasn't been modified yet
if is_new_bloom {
let new_bloom = match self.data_source.bloom_at_index(&bloom_index) {
Some(old_bloom) => old_bloom | &blooms[i],
None => blooms[i].clone(),
};
result.insert(bloom_index, new_bloom);
}
}
}
result
}
/// Resets bloom at level 0 and forces rebuild on higher levels.
pub fn reset_bloom(&self, bloom: &H2048, block_number: usize) -> HashMap<BloomIndex, H2048> {
let mut result: HashMap<BloomIndex, H2048> = HashMap::new();
let mut reset_index = self.bloom_index(block_number, 0);
result.insert(reset_index.clone(), bloom.clone());
for level in 1..self.levels() {
let index = self.bloom_index(block_number, level);
// get all bloom indexes that were used to construct this bloom
let lower_indexes = self.lower_level_bloom_indexes(&index);
let new_bloom = lower_indexes.into_iter()
// skip reseted one
.filter(|li| li != &reset_index)
// get blooms for these indexes
.map(|li| self.data_source.bloom_at_index(&li))
// filter existing ones
.filter_map(|b| b)
// BitOr all of them
.fold(H2048::new(), |acc, bloom| &acc | bloom);
reset_index = index.clone();
result.insert(index, &new_bloom | bloom);
}
result
}
/// Sets lowest level bloom to 0 and forces rebuild on higher levels.
pub fn clear_bloom(&self, block_number: usize) -> HashMap<BloomIndex, H2048> {
self.reset_bloom(&H2048::new(), block_number)
}
/// Returns numbers of blocks that may contain Address.
pub fn blocks_with_address(&self, address: &Address, from_block: usize, to_block: usize) -> Vec<usize> {
let mut bloom = H2048::new();
bloom.shift_bloomed(&address.sha3());
self.blocks_with_bloom(&bloom, from_block, to_block)
}
/// Returns numbers of blocks that may contain Topic.
pub fn blocks_with_topic(&self, topic: &H256, from_block: usize, to_block: usize) -> Vec<usize> {
let mut bloom = H2048::new();
bloom.shift_bloomed(&topic.sha3());
self.blocks_with_bloom(&bloom, from_block, to_block)
}
/// Returns numbers of blocks that may log bloom.
pub fn blocks_with_bloom(&self, bloom: &H2048, from_block: usize, to_block: usize) -> Vec<usize> {
let mut result = vec![];
// lets start from highest level
let max_level = self.max_level();
let level_size = self.level_size(max_level);
let from_index = self.bloom_index(from_block, max_level);
let to_index = self.bloom_index(to_block, max_level);
for index in from_index.index..to_index.index + 1 {
// offset will be used to calculate where we are right now
let offset = level_size * index;
// go doooown!
if let Some(blocks) = self.blocks(bloom, from_block, to_block, max_level, offset) {
result.extend(blocks);
}
}
result
}
}
#[cfg(test)]
mod tests {
use hash::*;
use chainfilter::*;
use sha3::*;
use std::str::FromStr;
#[test]
fn test_level_size() {
let cache = MemoryCache::new();
let filter = ChainFilter::new(&cache, 16, 3);
assert_eq!(filter.level_size(0), 1);
assert_eq!(filter.level_size(1), 16);
assert_eq!(filter.level_size(2), 256);
}
#[test]
fn test_bloom_index() {
let cache = MemoryCache::new();
let filter = ChainFilter::new(&cache, 16, 3);
let bi0 = filter.bloom_index(0, 0);
assert_eq!(bi0.level, 0);
assert_eq!(bi0.index, 0);
let bi1 = filter.bloom_index(1, 0);
assert_eq!(bi1.level, 0);
assert_eq!(bi1.index, 1);
let bi2 = filter.bloom_index(2, 0);
assert_eq!(bi2.level, 0);
assert_eq!(bi2.index, 2);
let bi3 = filter.bloom_index(3, 1);
assert_eq!(bi3.level, 1);
assert_eq!(bi3.index, 0);
let bi4 = filter.bloom_index(15, 1);
assert_eq!(bi4.level, 1);
assert_eq!(bi4.index, 0);
let bi5 = filter.bloom_index(16, 1);
assert_eq!(bi5.level, 1);
assert_eq!(bi5.index, 1);
let bi6 = filter.bloom_index(255, 2);
assert_eq!(bi6.level, 2);
assert_eq!(bi6.index, 0);
let bi7 = filter.bloom_index(256, 2);
assert_eq!(bi7.level, 2);
assert_eq!(bi7.index, 1);
}
#[test]
fn test_lower_level_bloom_indexes() {
let cache = MemoryCache::new();
let filter = ChainFilter::new(&cache, 16, 3);
let bi = filter.bloom_index(256, 2);
assert_eq!(bi.level, 2);
assert_eq!(bi.index, 1);
let mut ebis = vec![];
for i in 16..32 {
ebis.push(BloomIndex::new(1, i));
}
let bis = filter.lower_level_bloom_indexes(&bi);
assert_eq!(ebis, bis);
}
#[test]
fn test_topic_basic_search() {
let index_size = 16;
let bloom_levels = 3;
let mut cache = MemoryCache::new();
let topic = H256::from_str("8d936b1bd3fc635710969ccfba471fb17d598d9d1971b538dd712e1e4b4f4dba").unwrap();
let modified_blooms = {
let filter = ChainFilter::new(&cache, index_size, bloom_levels);
let block_number = 23;
let mut bloom = H2048::new();
bloom.shift_bloomed(&topic.sha3());
filter.add_bloom(&bloom, block_number)
};
// number of modified blooms should always be equal number of levels
assert_eq!(modified_blooms.len(), bloom_levels as usize);
cache.insert_blooms(modified_blooms);
{
let filter = ChainFilter::new(&cache, index_size, bloom_levels);
let blocks = filter.blocks_with_topic(&topic, 0, 100);
assert_eq!(blocks.len(), 1);
assert_eq!(blocks[0], 23);
}
{
let filter = ChainFilter::new(&cache, index_size, bloom_levels);
let blocks = filter.blocks_with_topic(&topic, 0, 23);
assert_eq!(blocks.len(), 0);
}
{
let filter = ChainFilter::new(&cache, index_size, bloom_levels);
let blocks = filter.blocks_with_topic(&topic, 23, 24);
assert_eq!(blocks.len(), 1);
assert_eq!(blocks[0], 23);
}
{
let filter = ChainFilter::new(&cache, index_size, bloom_levels);
let blocks = filter.blocks_with_topic(&topic, 24, 100);
assert_eq!(blocks.len(), 0);
}
}
}

3
util/src/common.rs
View File

@@ -19,10 +19,9 @@
pub use standard::*;
pub use from_json::*;
pub use error::*;
pub use hash::*;
pub use uint::*;
pub use bytes::*;
pub use vector::*;
pub use numbers::*;
pub use sha3::*;
#[macro_export]

36
util/src/crypto.rs
View File

@@ -16,11 +16,11 @@
//! Ethcore crypto.
use hash::*;
use numbers::*;
use bytes::*;
use uint::*;
use secp256k1::{key, Secp256k1};
use rand::os::OsRng;
use sha3::Hashable;
/// Secret key for secp256k1 EC operations. 256 bit generic "hash" data.
pub type Secret = H256;
@@ -136,23 +136,29 @@ impl KeyPair {
public: p,
})
}
/// Returns public key
pub fn public(&self) -> &Public {
&self.public
}
/// Returns private key
pub fn secret(&self) -> &Secret {
&self.secret
}
/// Returns address.
pub fn address(&self) -> Address {
Address::from(self.public.sha3())
}
/// Sign a message with our secret key.
pub fn sign(&self, message: &H256) -> Result<Signature, CryptoError> { ec::sign(&self.secret, message) }
}
/// EC functions
pub mod ec {
use hash::*;
use uint::*;
use numbers::*;
use standard::*;
use crypto::*;
use crypto::{self};
@@ -254,7 +260,7 @@ pub mod ecies {
use crypto::*;
/// Encrypt a message with a public key
pub fn encrypt(public: &Public, plain: &[u8]) -> Result<Bytes, CryptoError> {
pub fn encrypt(public: &Public, shared_mac: &[u8], plain: &[u8]) -> Result<Bytes, CryptoError> {
use ::rcrypto::digest::Digest;
use ::rcrypto::sha2::Sha256;
use ::rcrypto::hmac::Hmac;
@@ -284,13 +290,14 @@ pub mod ecies {
let cipher_iv = &msgd[64..(64 + 16 + plain.len())];
hmac.input(cipher_iv);
}
hmac.input(shared_mac);
hmac.raw_result(&mut msgd[(64 + 16 + plain.len())..]);
}
Ok(msg)
}
/// Decrypt a message with a secret key
pub fn decrypt(secret: &Secret, encrypted: &[u8]) -> Result<Bytes, CryptoError> {
pub fn decrypt(secret: &Secret, shared_mac: &[u8], encrypted: &[u8]) -> Result<Bytes, CryptoError> {
use ::rcrypto::digest::Digest;
use ::rcrypto::sha2::Sha256;
use ::rcrypto::hmac::Hmac;
@@ -322,6 +329,7 @@ pub mod ecies {
// Verify tag
let mut hmac = Hmac::new(Sha256::new(), &mkey);
hmac.input(cipher_with_iv);
hmac.input(shared_mac);
let mut mac = H256::new();
hmac.raw_result(&mut mac);
if &mac[..] != msg_mac {
@@ -405,4 +413,20 @@ mod tests {
let pair = KeyPair::from_secret(h256_from_hex("6f7b0d801bc7b5ce7bbd930b84fd0369b3eb25d09be58d64ba811091046f3aa2")).unwrap();
assert_eq!(pair.public().hex(), "101b3ef5a4ea7a1c7928e24c4c75fd053c235d7b80c22ae5c03d145d0ac7396e2a4ffff9adee3133a7b05044a5cee08115fd65145e5165d646bde371010d803c");
}
#[test]
fn ecies_shared() {
let kp = KeyPair::create().unwrap();
let message = b"So many books, so little time";
let shared = b"shared";
let wrong_shared = b"incorrect";
let encrypted = ecies::encrypt(kp.public(), shared, message).unwrap();
assert!(encrypted[..] != message[..]);
assert_eq!(encrypted[0], 0x04);
assert!(ecies::decrypt(kp.secret(), wrong_shared, &encrypted).is_err());
let decrypted = ecies::decrypt(kp.secret(), shared, &encrypted).unwrap();
assert_eq!(decrypted[..message.len()], message[..]);
}
}

3
util/src/error.rs
View File

@@ -21,12 +21,13 @@ use network::NetworkError;
use rlp::DecoderError;
use io;
use std::fmt;
use hash::H256;
#[derive(Debug)]
/// Error in database subsystem.
pub enum BaseDataError {
/// An entry was removed more times than inserted.
NegativelyReferencedHash,
NegativelyReferencedHash(H256),
}
#[derive(Debug)]

18
util/src/from_json.rs
View File

@@ -17,6 +17,7 @@
//! Coversion from json.
use standard::*;
use bigint::uint::*;
#[macro_export]
macro_rules! xjson {
@@ -30,3 +31,20 @@ pub trait FromJson {
/// Convert a JSON value to an instance of this type.
fn from_json(json: &Json) -> Self;
}
impl FromJson for U256 {
fn from_json(json: &Json) -> Self {
match *json {
Json::String(ref s) => {
if s.len() >= 2 && &s[0..2] == "0x" {
FromStr::from_str(&s[2..]).unwrap_or_else(|_| Default::default())
} else {
Uint::from_dec_str(s).unwrap_or_else(|_| Default::default())
}
},
Json::U64(u) => From::from(u),
Json::I64(i) => From::from(i as u64),
_ => Uint::zero(),
}
}
}

48
util/src/hash.rs
View File

@@ -23,7 +23,7 @@ use rand::Rng;
use rand::os::OsRng;
use bytes::{BytesConvertable,Populatable};
use from_json::*;
use uint::{Uint, U256};
use bigint::uint::{Uint, U256};
use rustc_serialize::hex::ToHex;
use serde;
@@ -63,7 +63,8 @@ pub trait FixedHash: Sized + BytesConvertable + Populatable + FromStr + Default
fn low_u64(&self) -> u64;
}
fn clean_0x(s: &str) -> &str {
/// Return `s` without the `0x` at the beginning of it, if any.
pub fn clean_0x(s: &str) -> &str {
if s.len() >= 2 && &s[0..2] == "0x" {
&s[2..]
} else {
@@ -77,12 +78,6 @@ macro_rules! impl_hash {
/// Unformatted binary data of fixed length.
pub struct $from (pub [u8; $size]);
impl BytesConvertable for $from {
fn bytes(&self) -> &[u8] {
&self.0
}
}
impl Deref for $from {
type Target = [u8];
@@ -92,6 +87,13 @@ macro_rules! impl_hash {
}
}
impl AsRef<[u8]> for $from {
#[inline]
fn as_ref(&self) -> &[u8] {
&self.0
}
}
impl DerefMut for $from {
#[inline]
fn deref_mut(&mut self) -> &mut [u8] {
@@ -234,11 +236,11 @@ macro_rules! impl_hash {
}
impl serde::Serialize for $from {
fn serialize<S>(&self, serializer: &mut S) -> Result<(), S::Error>
fn serialize<S>(&self, serializer: &mut S) -> Result<(), S::Error>
where S: serde::Serializer {
let mut hex = "0x".to_owned();
hex.push_str(self.to_hex().as_ref());
serializer.visit_str(hex.as_ref())
serializer.serialize_str(hex.as_ref())
}
}
@@ -249,14 +251,14 @@ macro_rules! impl_hash {
impl serde::de::Visitor for HashVisitor {
type Value = $from;
fn visit_str<E>(&mut self, value: &str) -> Result<Self::Value, E> where E: serde::Error {
// 0x + len
if value.len() != 2 + $size * 2 {
return Err(serde::Error::syntax("Invalid length."));
return Err(serde::Error::custom("Invalid length."));
}
value[2..].from_hex().map(|ref v| $from::from_slice(v)).map_err(|_| serde::Error::syntax("Invalid valid hex."))
value[2..].from_hex().map(|ref v| $from::from_slice(v)).map_err(|_| serde::Error::custom("Invalid hex value."))
}
fn visit_string<E>(&mut self, value: String) -> Result<Self::Value, E> where E: serde::Error {
@@ -264,7 +266,7 @@ macro_rules! impl_hash {
}
}
deserializer.visit(HashVisitor)
deserializer.deserialize(HashVisitor)
}
}
@@ -303,6 +305,8 @@ macro_rules! impl_hash {
}
}
impl Copy for $from {}
#[cfg_attr(feature="dev", allow(expl_impl_clone_on_copy))]
impl Clone for $from {
fn clone(&self) -> $from {
unsafe {
@@ -413,15 +417,6 @@ macro_rules! impl_hash {
}
}
/// Moving BitOrAssign
impl<'a> BitOrAssign<&'a $from> for $from {
fn bitor_assign(&mut self, rhs: &'a Self) {
for i in 0..$size {
self.0[i] = self.0[i] | rhs.0[i];
}
}
}
/// BitAnd on references
impl <'a> BitAnd for &'a $from {
type Output = $from;
@@ -603,7 +598,7 @@ pub fn h256_from_hex(s: &str) -> H256 {
/// Convert `n` to an `H256`, setting the rightmost 8 bytes.
pub fn h256_from_u64(n: u64) -> H256 {
use uint::U256;
use bigint::uint::U256;
H256::from(&U256::from(n))
}
@@ -639,11 +634,11 @@ pub static ZERO_H256: H256 = H256([0x00; 32]);
#[cfg(test)]
mod tests {
use hash::*;
use uint::*;
use bigint::uint::*;
use std::str::FromStr;
#[test]
#[allow(eq_op)]
#[cfg_attr(feature="dev", allow(eq_op))]
fn hash() {
let h = H64([0x01, 0x23, 0x45, 0x67, 0x89, 0xab, 0xcd, 0xef]);
assert_eq!(H64::from_str("0123456789abcdef").unwrap(), h);
@@ -727,4 +722,3 @@ mod tests {
assert_eq!(r, u);
}
}

15
util/src/hashdb.rs
View File

@@ -20,7 +20,7 @@ use bytes::*;
use std::collections::HashMap;
/// Trait modelling datastore keyed by a 32-byte Keccak hash.
pub trait HashDB {
pub trait HashDB : AsHashDB {
/// Get the keys in the database together with number of underlying references.
fn keys(&self) -> HashMap<H256, i32>;
@@ -111,3 +111,16 @@ pub trait HashDB {
/// ```
fn remove(&mut self, key: &H256) { self.kill(key) }
}
/// Upcast trait.
pub trait AsHashDB {
/// Perform upcast to HashDB for anything that derives from HashDB.
fn as_hashdb(&self) -> &HashDB;
/// Perform mutable upcast to HashDB for anything that derives from HashDB.
fn as_hashdb_mut(&mut self) -> &mut HashDB;
}
impl<T: HashDB> AsHashDB for T {
fn as_hashdb(&self) -> &HashDB { self }
fn as_hashdb_mut(&mut self) -> &mut HashDB { self }
}

3
util/src/heapsizeof.rs
View File

@@ -16,8 +16,7 @@
//! Calculates heapsize of util types.
use uint::*;
use hash::*;
known_heap_size!(0, H32, H64, H128, Address, H256, H264, H512, H520, H1024, H2048);
known_heap_size!(0, U128, U256);

14
util/src/io/service.rs
View File

@@ -153,7 +153,7 @@ struct UserTimer {
pub struct IoManager<Message> where Message: Send + Sync {
timers: Arc<RwLock<HashMap<HandlerId, UserTimer>>>,
handlers: Vec<Arc<IoHandler<Message>>>,
_workers: Vec<Worker>,
workers: Vec<Worker>,
worker_channel: chase_lev::Worker<Work<Message>>,
work_ready: Arc<Condvar>,
}
@@ -180,7 +180,7 @@ impl<Message> IoManager<Message> where Message: Send + Sync + Clone + 'static {
timers: Arc::new(RwLock::new(HashMap::new())),
handlers: Vec::new(),
worker_channel: worker,
_workers: workers,
workers: workers,
work_ready: work_ready,
};
try!(event_loop.run(&mut io));
@@ -230,7 +230,10 @@ impl<Message> Handler for IoManager<Message> where Message: Send + Clone + Sync
fn notify(&mut self, event_loop: &mut EventLoop<Self>, msg: Self::Message) {
match msg {
IoMessage::Shutdown => event_loop.shutdown(),
IoMessage::Shutdown => {
self.workers.clear();
event_loop.shutdown();
},
IoMessage::AddHandler { handler } => {
let handler_id = {
self.handlers.push(handler.clone());
@@ -256,6 +259,11 @@ impl<Message> Handler for IoManager<Message> where Message: Send + Clone + Sync
IoMessage::DeregisterStream { handler_id, token } => {
let handler = self.handlers.get(handler_id).expect("Unknown handler id").clone();
handler.deregister_stream(token, event_loop);
// unregister a timer associated with the token (if any)
let timer_id = token + handler_id * TOKENS_PER_HANDLER;
if let Some(timer) = self.timers.write().unwrap().remove(&timer_id) {
event_loop.clear_timeout(timer.timeout);
}
},
IoMessage::UpdateStreamRegistration { handler_id, token } => {
let handler = self.handlers.get(handler_id).expect("Unknown handler id").clone();

15
util/src/io/worker.rs
View File

@@ -44,6 +44,7 @@ pub struct Worker {
thread: Option<JoinHandle<()>>,
wait: Arc<Condvar>,
deleting: Arc<AtomicBool>,
wait_mutex: Arc<Mutex<()>>,
}
impl Worker {
@@ -61,6 +62,7 @@ impl Worker {
thread: None,
wait: wait.clone(),
deleting: deleting.clone(),
wait_mutex: wait_mutex.clone(),
};
worker.thread = Some(thread::Builder::new().name(format!("IO Worker #{}", index)).spawn(
move || {
@@ -77,13 +79,17 @@ impl Worker {
wait_mutex: Arc<Mutex<()>>,
deleting: Arc<AtomicBool>)
where Message: Send + Sync + Clone + 'static {
while !deleting.load(AtomicOrdering::Relaxed) {
loop {
{
let lock = wait_mutex.lock().unwrap();
let _ = wait.wait(lock).unwrap();
if deleting.load(AtomicOrdering::Relaxed) {
if deleting.load(AtomicOrdering::Acquire) {
return;
}
let _ = wait.wait(lock).unwrap();
}
if deleting.load(AtomicOrdering::Acquire) {
return;
}
while let chase_lev::Steal::Data(work) = stealer.steal() {
Worker::do_work(work, channel.clone());
@@ -114,7 +120,8 @@ impl Worker {
impl Drop for Worker {
fn drop(&mut self) {
self.deleting.store(true, AtomicOrdering::Relaxed);
let _ = self.wait_mutex.lock();
self.deleting.store(true, AtomicOrdering::Release);
self.wait.notify_all();
let thread = mem::replace(&mut self.thread, None).unwrap();
thread.join().ok();

457
util/src/journaldb.rs
View File

@@ -1,457 +0,0 @@
// Copyright 2015, 2016 Ethcore (UK) Ltd.
// This file is part of Parity.
// Parity is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// Parity is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
// You should have received a copy of the GNU General Public License
// along with Parity. If not, see <http://www.gnu.org/licenses/>.
//! Disk-backed HashDB implementation.
use common::*;
use rlp::*;
use hashdb::*;
use memorydb::*;
use rocksdb::{DB, Writable, WriteBatch, IteratorMode};
#[cfg(test)]
use std::env;
/// Implementation of the HashDB trait for a disk-backed database with a memory overlay
/// and latent-removal semantics.
///
/// Like OverlayDB, there is a memory overlay; `commit()` must be called in order to
/// write operations out to disk. Unlike OverlayDB, `remove()` operations do not take effect
/// immediately. Rather some age (based on a linear but arbitrary metric) must pass before
/// the removals actually take effect.
pub struct JournalDB {
overlay: MemoryDB,
backing: Arc<DB>,
counters: Arc<RwLock<HashMap<H256, i32>>>,
}
impl Clone for JournalDB {
fn clone(&self) -> JournalDB {
JournalDB {
overlay: MemoryDB::new(),
backing: self.backing.clone(),
counters: self.counters.clone(),
}
}
}
const LATEST_ERA_KEY : [u8; 4] = [ b'l', b'a', b's', b't' ];
const VERSION_KEY : [u8; 4] = [ b'j', b'v', b'e', b'r' ];
const DB_VERSION: u32 = 2;
impl JournalDB {
/// Create a new instance given a `backing` database.
pub fn new(backing: DB) -> JournalDB {
let db = Arc::new(backing);
JournalDB::new_with_arc(db)
}
/// Create a new instance given a shared `backing` database.
pub fn new_with_arc(backing: Arc<DB>) -> JournalDB {
if backing.iterator(IteratorMode::Start).next().is_some() {
match backing.get(&VERSION_KEY).map(|d| d.map(|v| decode::<u32>(&v))) {
Ok(Some(DB_VERSION)) => {},
v => panic!("Incompatible DB version, expected {}, got {:?}", DB_VERSION, v)
}
} else {
backing.put(&VERSION_KEY, &encode(&DB_VERSION)).expect("Error writing version to database");
}
let counters = JournalDB::read_counters(&backing);
JournalDB {
overlay: MemoryDB::new(),
backing: backing,
counters: Arc::new(RwLock::new(counters)),
}
}
/// Create a new instance with an anonymous temporary database.
#[cfg(test)]
pub fn new_temp() -> JournalDB {
let mut dir = env::temp_dir();
dir.push(H32::random().hex());
Self::new(DB::open_default(dir.to_str().unwrap()).unwrap())
}
/// Check if this database has any commits
pub fn is_empty(&self) -> bool {
self.backing.get(&LATEST_ERA_KEY).expect("Low level database error").is_none()
}
/// Commit all recent insert operations and historical removals from the old era
/// to the backing database.
pub fn commit(&mut self, now: u64, id: &H256, end: Option<(u64, H256)>) -> Result<u32, UtilError> {
// journal format:
// [era, 0] => [ id, [insert_0, ...], [remove_0, ...] ]
// [era, 1] => [ id, [insert_0, ...], [remove_0, ...] ]
// [era, n] => [ ... ]
// TODO: store reclaim_period.
// when we make a new commit, we journal the inserts and removes.
// for each end_era that we journaled that we are no passing by,
// we remove all of its removes assuming it is canonical and all
// of its inserts otherwise.
//
// We also keep reference counters for each key inserted in the journal to handle
// the following cases where key K must not be deleted from the DB when processing removals :
// Given H is the journal size in eras, 0 <= C <= H.
// Key K is removed in era A(N) and re-inserted in canonical era B(N + C).
// Key K is removed in era A(N) and re-inserted in non-canonical era B`(N + C).
// Key K is added in non-canonical era A'(N) canonical B(N + C).
//
// The counter is encreased each time a key is inserted in the journal in the commit. The list of insertions
// is saved with the era record. When the era becomes end_era and goes out of journal the counter is decreased
// and the key is safe to delete.
// record new commit's details.
let batch = WriteBatch::new();
let mut counters = self.counters.write().unwrap();
{
let mut index = 0usize;
let mut last;
while try!(self.backing.get({
let mut r = RlpStream::new_list(2);
r.append(&now);
r.append(&index);
last = r.drain();
&last
})).is_some() {
index += 1;
}
let mut r = RlpStream::new_list(3);
let inserts: Vec<H256> = self.overlay.keys().iter().filter(|&(_, &c)| c > 0).map(|(key, _)| key.clone()).collect();
// Increase counter for each inserted key no matter if the block is canonical or not.
for i in &inserts {
*counters.entry(i.clone()).or_insert(0) += 1;
}
let removes: Vec<H256> = self.overlay.keys().iter().filter(|&(_, &c)| c < 0).map(|(key, _)| key.clone()).collect();
r.append(id);
r.append(&inserts);
r.append(&removes);
try!(batch.put(&last, r.as_raw()));
try!(batch.put(&LATEST_ERA_KEY, &encode(&now)));
}
// apply old commits' details
if let Some((end_era, canon_id)) = end {
let mut index = 0usize;
let mut last;
let mut to_remove: Vec<H256> = Vec::new();
let mut canon_inserts: Vec<H256> = Vec::new();
while let Some(rlp_data) = try!(self.backing.get({
let mut r = RlpStream::new_list(2);
r.append(&end_era);
r.append(&index);
last = r.drain();
&last
})) {
let rlp = Rlp::new(&rlp_data);
let inserts: Vec<H256> = rlp.val_at(1);
JournalDB::decrease_counters(&inserts, &mut counters);
// Collect keys to be removed. These are removed keys for canonical block, inserted for non-canonical
if canon_id == rlp.val_at(0) {
to_remove.extend(rlp.at(2).iter().map(|r| r.as_val::<H256>()));
canon_inserts = inserts;
}
else {
to_remove.extend(inserts);
}
try!(batch.delete(&last));
index += 1;
}
let canon_inserts = canon_inserts.drain(..).collect::<HashSet<_>>();
// Purge removed keys if they are not referenced and not re-inserted in the canon commit
let mut deletes = 0;
for h in to_remove.iter().filter(|h| !counters.contains_key(h) && !canon_inserts.contains(h)) {
try!(batch.delete(&h));
deletes += 1;
}
trace!("JournalDB: delete journal for time #{}.{}, (canon was {}): {} entries", end_era, index, canon_id, deletes);
}
// Commit overlay insertions
let mut ret = 0u32;
let mut deletes = 0usize;
for i in self.overlay.drain().into_iter() {
let (key, (value, rc)) = i;
if rc > 0 {
assert!(rc == 1);
batch.put(&key.bytes(), &value).expect("Low-level database error. Some issue with your hard disk?");
ret += 1;
}
if rc < 0 {
assert!(rc == -1);
ret += 1;
deletes += 1;
}
}
try!(self.backing.write(batch));
trace!("JournalDB::commit() deleted {} nodes", deletes);
Ok(ret)
}
// Decrease counters for given keys. Deletes obsolete counters
fn decrease_counters(keys: &[H256], counters: &mut HashMap<H256, i32>) {
for i in keys.iter() {
let delete_counter = {
let cnt = counters.get_mut(i).expect("Missing key counter");
*cnt -= 1;
*cnt == 0
};
if delete_counter {
counters.remove(i);
}
}
}
fn payload(&self, key: &H256) -> Option<Bytes> {
self.backing.get(&key.bytes()).expect("Low-level database error. Some issue with your hard disk?").map(|v| v.to_vec())
}
fn read_counters(db: &DB) -> HashMap<H256, i32> {
let mut res = HashMap::new();
if let Some(val) = db.get(&LATEST_ERA_KEY).expect("Low-level database error.") {
let mut era = decode::<u64>(&val);
loop {
let mut index = 0usize;
while let Some(rlp_data) = db.get({
let mut r = RlpStream::new_list(2);
r.append(&era);
r.append(&index);
&r.drain()
}).expect("Low-level database error.") {
let rlp = Rlp::new(&rlp_data);
let to_add: Vec<H256> = rlp.val_at(1);
for h in to_add {
*res.entry(h).or_insert(0) += 1;
}
index += 1;
};
if index == 0 || era == 0 {
break;
}
era -= 1;
}
}
trace!("Recovered {} counters", res.len());
res
}
}
impl HashDB for JournalDB {
fn keys(&self) -> HashMap<H256, i32> {
let mut ret: HashMap<H256, i32> = HashMap::new();
for (key, _) in self.backing.iterator(IteratorMode::Start) {
let h = H256::from_slice(key.deref());
ret.insert(h, 1);
}
for (key, refs) in self.overlay.keys().into_iter() {
let refs = *ret.get(&key).unwrap_or(&0) + refs;
ret.insert(key, refs);
}
ret
}
fn lookup(&self, key: &H256) -> Option<&[u8]> {
let k = self.overlay.raw(key);
match k {
Some(&(ref d, rc)) if rc > 0 => Some(d),
_ => {
if let Some(x) = self.payload(key) {
Some(&self.overlay.denote(key, x).0)
}
else {
None
}
}
}
}
fn exists(&self, key: &H256) -> bool {
self.lookup(key).is_some()
}
fn insert(&mut self, value: &[u8]) -> H256 {
self.overlay.insert(value)
}
fn emplace(&mut self, key: H256, value: Bytes) {
self.overlay.emplace(key, value);
}
fn kill(&mut self, key: &H256) {
self.overlay.kill(key);
}
}
#[cfg(test)]
mod tests {
use common::*;
use super::*;
use hashdb::*;
#[test]
fn long_history() {
// history is 3
let mut jdb = JournalDB::new_temp();
let h = jdb.insert(b"foo");
jdb.commit(0, &b"0".sha3(), None).unwrap();
assert!(jdb.exists(&h));
jdb.remove(&h);
jdb.commit(1, &b"1".sha3(), None).unwrap();
assert!(jdb.exists(&h));
jdb.commit(2, &b"2".sha3(), None).unwrap();
assert!(jdb.exists(&h));
jdb.commit(3, &b"3".sha3(), Some((0, b"0".sha3()))).unwrap();
assert!(jdb.exists(&h));
jdb.commit(4, &b"4".sha3(), Some((1, b"1".sha3()))).unwrap();
assert!(!jdb.exists(&h));
}
#[test]
fn complex() {
// history is 1
let mut jdb = JournalDB::new_temp();
let foo = jdb.insert(b"foo");
let bar = jdb.insert(b"bar");
jdb.commit(0, &b"0".sha3(), None).unwrap();
assert!(jdb.exists(&foo));
assert!(jdb.exists(&bar));
jdb.remove(&foo);
jdb.remove(&bar);
let baz = jdb.insert(b"baz");
jdb.commit(1, &b"1".sha3(), Some((0, b"0".sha3()))).unwrap();
assert!(jdb.exists(&foo));
assert!(jdb.exists(&bar));
assert!(jdb.exists(&baz));
let foo = jdb.insert(b"foo");
jdb.remove(&baz);
jdb.commit(2, &b"2".sha3(), Some((1, b"1".sha3()))).unwrap();
assert!(jdb.exists(&foo));
assert!(!jdb.exists(&bar));
assert!(jdb.exists(&baz));
jdb.remove(&foo);
jdb.commit(3, &b"3".sha3(), Some((2, b"2".sha3()))).unwrap();
assert!(jdb.exists(&foo));
assert!(!jdb.exists(&bar));
assert!(!jdb.exists(&baz));
jdb.commit(4, &b"4".sha3(), Some((3, b"3".sha3()))).unwrap();
assert!(!jdb.exists(&foo));
assert!(!jdb.exists(&bar));
assert!(!jdb.exists(&baz));
}
#[test]
fn fork() {
// history is 1
let mut jdb = JournalDB::new_temp();
let foo = jdb.insert(b"foo");
let bar = jdb.insert(b"bar");
jdb.commit(0, &b"0".sha3(), None).unwrap();
assert!(jdb.exists(&foo));
assert!(jdb.exists(&bar));
jdb.remove(&foo);
let baz = jdb.insert(b"baz");
jdb.commit(1, &b"1a".sha3(), Some((0, b"0".sha3()))).unwrap();
jdb.remove(&bar);
jdb.commit(1, &b"1b".sha3(), Some((0, b"0".sha3()))).unwrap();
assert!(jdb.exists(&foo));
assert!(jdb.exists(&bar));
assert!(jdb.exists(&baz));
jdb.commit(2, &b"2b".sha3(), Some((1, b"1b".sha3()))).unwrap();
assert!(jdb.exists(&foo));
assert!(!jdb.exists(&baz));
assert!(!jdb.exists(&bar));
}
#[test]
fn overwrite() {
// history is 1
let mut jdb = JournalDB::new_temp();
let foo = jdb.insert(b"foo");
jdb.commit(0, &b"0".sha3(), None).unwrap();
assert!(jdb.exists(&foo));
jdb.remove(&foo);
jdb.commit(1, &b"1".sha3(), Some((0, b"0".sha3()))).unwrap();
jdb.insert(b"foo");
assert!(jdb.exists(&foo));
jdb.commit(2, &b"2".sha3(), Some((1, b"1".sha3()))).unwrap();
assert!(jdb.exists(&foo));
jdb.commit(3, &b"2".sha3(), Some((0, b"2".sha3()))).unwrap();
assert!(jdb.exists(&foo));
}
#[test]
fn fork_same_key() {
// history is 1
let mut jdb = JournalDB::new_temp();
jdb.commit(0, &b"0".sha3(), None).unwrap();
let foo = jdb.insert(b"foo");
jdb.commit(1, &b"1a".sha3(), Some((0, b"0".sha3()))).unwrap();
jdb.insert(b"foo");
jdb.commit(1, &b"1b".sha3(), Some((0, b"0".sha3()))).unwrap();
assert!(jdb.exists(&foo));
jdb.commit(2, &b"2a".sha3(), Some((1, b"1a".sha3()))).unwrap();
assert!(jdb.exists(&foo));
}
#[test]
fn reopen() {
use rocksdb::DB;
let mut dir = ::std::env::temp_dir();
dir.push(H32::random().hex());
let foo = {
let mut jdb = JournalDB::new(DB::open_default(dir.to_str().unwrap()).unwrap());
// history is 1
let foo = jdb.insert(b"foo");
jdb.commit(0, &b"0".sha3(), None).unwrap();
foo
};
{
let mut jdb = JournalDB::new(DB::open_default(dir.to_str().unwrap()).unwrap());
jdb.remove(&foo);
jdb.commit(1, &b"1".sha3(), Some((0, b"0".sha3()))).unwrap();
}
{
let mut jdb = JournalDB::new(DB::open_default(dir.to_str().unwrap()).unwrap());
assert!(jdb.exists(&foo));
jdb.commit(2, &b"2".sha3(), Some((1, b"1".sha3()))).unwrap();
assert!(!jdb.exists(&foo));
}
}
}

420
util/src/journaldb/archivedb.rs Normal file
View File

@@ -0,0 +1,420 @@
// Copyright 2015, 2016 Ethcore (UK) Ltd.
// This file is part of Parity.
// Parity is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// Parity is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
// You should have received a copy of the GNU General Public License
// along with Parity. If not, see <http://www.gnu.org/licenses/>.
//! Disk-backed HashDB implementation.
use common::*;
use rlp::*;
use hashdb::*;
use memorydb::*;
use super::traits::JournalDB;
use kvdb::{Database, DBTransaction, DatabaseConfig};
#[cfg(test)]
use std::env;
/// Implementation of the HashDB trait for a disk-backed database with a memory overlay
/// and latent-removal semantics.
///
/// Like OverlayDB, there is a memory overlay; `commit()` must be called in order to
/// write operations out to disk. Unlike OverlayDB, `remove()` operations do not take effect
/// immediately. Rather some age (based on a linear but arbitrary metric) must pass before
/// the removals actually take effect.
pub struct ArchiveDB {
overlay: MemoryDB,
backing: Arc<Database>,
latest_era: Option<u64>,
}
// all keys must be at least 12 bytes
const LATEST_ERA_KEY : [u8; 12] = [ b'l', b'a', b's', b't', 0, 0, 0, 0, 0, 0, 0, 0 ];
const VERSION_KEY : [u8; 12] = [ b'j', b'v', b'e', b'r', 0, 0, 0, 0, 0, 0, 0, 0 ];
const DB_VERSION : u32 = 259;
impl ArchiveDB {
/// Create a new instance from file
pub fn new(path: &str) -> ArchiveDB {
let opts = DatabaseConfig {
prefix_size: Some(12) //use 12 bytes as prefix, this must match account_db prefix
};
let backing = Database::open(&opts, path).unwrap_or_else(|e| {
panic!("Error opening state db: {}", e);
});
if !backing.is_empty() {
match backing.get(&VERSION_KEY).map(|d| d.map(|v| decode::<u32>(&v))) {
Ok(Some(DB_VERSION)) => {},
v => panic!("Incompatible DB version, expected {}, got {:?}", DB_VERSION, v)
}
} else {
backing.put(&VERSION_KEY, &encode(&DB_VERSION)).expect("Error writing version to database");
}
let latest_era = backing.get(&LATEST_ERA_KEY).expect("Low-level database error.").map(|val| decode::<u64>(&val));
ArchiveDB {
overlay: MemoryDB::new(),
backing: Arc::new(backing),
latest_era: latest_era,
}
}
/// Create a new instance with an anonymous temporary database.
#[cfg(test)]
fn new_temp() -> ArchiveDB {
let mut dir = env::temp_dir();
dir.push(H32::random().hex());
Self::new(dir.to_str().unwrap())
}
fn payload(&self, key: &H256) -> Option<Bytes> {
self.backing.get(&key.bytes()).expect("Low-level database error. Some issue with your hard disk?").map(|v| v.to_vec())
}
}
impl HashDB for ArchiveDB {
fn keys(&self) -> HashMap<H256, i32> {
let mut ret: HashMap<H256, i32> = HashMap::new();
for (key, _) in self.backing.iter() {
let h = H256::from_slice(key.deref());
ret.insert(h, 1);
}
for (key, refs) in self.overlay.keys().into_iter() {
let refs = *ret.get(&key).unwrap_or(&0) + refs;
ret.insert(key, refs);
}
ret
}
fn lookup(&self, key: &H256) -> Option<&[u8]> {
let k = self.overlay.raw(key);
match k {
Some(&(ref d, rc)) if rc > 0 => Some(d),
_ => {
if let Some(x) = self.payload(key) {
Some(&self.overlay.denote(key, x).0)
}
else {
None
}
}
}
}
fn exists(&self, key: &H256) -> bool {
self.lookup(key).is_some()
}
fn insert(&mut self, value: &[u8]) -> H256 {
self.overlay.insert(value)
}
fn emplace(&mut self, key: H256, value: Bytes) {
self.overlay.emplace(key, value);
}
fn kill(&mut self, key: &H256) {
self.overlay.kill(key);
}
}
impl JournalDB for ArchiveDB {
fn spawn(&self) -> Box<JournalDB> {
Box::new(ArchiveDB {
overlay: MemoryDB::new(),
backing: self.backing.clone(),
latest_era: self.latest_era,
})
}
fn mem_used(&self) -> usize {
self.overlay.mem_used()
}
fn is_empty(&self) -> bool {
self.latest_era.is_none()
}
fn commit(&mut self, now: u64, _: &H256, _: Option<(u64, H256)>) -> Result<u32, UtilError> {
let batch = DBTransaction::new();
let mut inserts = 0usize;
let mut deletes = 0usize;
for i in self.overlay.drain().into_iter() {
let (key, (value, rc)) = i;
if rc > 0 {
assert!(rc == 1);
batch.put(&key.bytes(), &value).expect("Low-level database error. Some issue with your hard disk?");
inserts += 1;
}
if rc < 0 {
assert!(rc == -1);
deletes += 1;
}
}
if self.latest_era.map_or(true, |e| now > e) {
try!(batch.put(&LATEST_ERA_KEY, &encode(&now)));
self.latest_era = Some(now);
}
try!(self.backing.write(batch));
Ok((inserts + deletes) as u32)
}
fn state(&self, id: &H256) -> Option<Bytes> {
self.backing.get_by_prefix(&id.bytes()[0..12]).and_then(|b| Some(b.to_vec()))
}
}
#[cfg(test)]
mod tests {
#![cfg_attr(feature="dev", allow(blacklisted_name))]
use common::*;
use super::*;
use hashdb::*;
use journaldb::traits::JournalDB;
#[test]
fn insert_same_in_fork() {
// history is 1
let mut jdb = ArchiveDB::new_temp();
let x = jdb.insert(b"X");
jdb.commit(1, &b"1".sha3(), None).unwrap();
jdb.commit(2, &b"2".sha3(), None).unwrap();
jdb.commit(3, &b"1002a".sha3(), Some((1, b"1".sha3()))).unwrap();
jdb.commit(4, &b"1003a".sha3(), Some((2, b"2".sha3()))).unwrap();
jdb.remove(&x);
jdb.commit(3, &b"1002b".sha3(), Some((1, b"1".sha3()))).unwrap();
let x = jdb.insert(b"X");
jdb.commit(4, &b"1003b".sha3(), Some((2, b"2".sha3()))).unwrap();
jdb.commit(5, &b"1004a".sha3(), Some((3, b"1002a".sha3()))).unwrap();
jdb.commit(6, &b"1005a".sha3(), Some((4, b"1003a".sha3()))).unwrap();
assert!(jdb.exists(&x));
}
#[test]
fn long_history() {
// history is 3
let mut jdb = ArchiveDB::new_temp();
let h = jdb.insert(b"foo");
jdb.commit(0, &b"0".sha3(), None).unwrap();
assert!(jdb.exists(&h));
jdb.remove(&h);
jdb.commit(1, &b"1".sha3(), None).unwrap();
assert!(jdb.exists(&h));
jdb.commit(2, &b"2".sha3(), None).unwrap();
assert!(jdb.exists(&h));
jdb.commit(3, &b"3".sha3(), Some((0, b"0".sha3()))).unwrap();
assert!(jdb.exists(&h));
jdb.commit(4, &b"4".sha3(), Some((1, b"1".sha3()))).unwrap();
}
#[test]
fn complex() {
// history is 1
let mut jdb = ArchiveDB::new_temp();
let foo = jdb.insert(b"foo");
let bar = jdb.insert(b"bar");
jdb.commit(0, &b"0".sha3(), None).unwrap();
assert!(jdb.exists(&foo));
assert!(jdb.exists(&bar));
jdb.remove(&foo);
jdb.remove(&bar);
let baz = jdb.insert(b"baz");
jdb.commit(1, &b"1".sha3(), Some((0, b"0".sha3()))).unwrap();
assert!(jdb.exists(&foo));
assert!(jdb.exists(&bar));
assert!(jdb.exists(&baz));
let foo = jdb.insert(b"foo");
jdb.remove(&baz);
jdb.commit(2, &b"2".sha3(), Some((1, b"1".sha3()))).unwrap();
assert!(jdb.exists(&foo));
assert!(jdb.exists(&baz));
jdb.remove(&foo);
jdb.commit(3, &b"3".sha3(), Some((2, b"2".sha3()))).unwrap();
assert!(jdb.exists(&foo));
jdb.commit(4, &b"4".sha3(), Some((3, b"3".sha3()))).unwrap();
}
#[test]
fn fork() {
// history is 1
let mut jdb = ArchiveDB::new_temp();
let foo = jdb.insert(b"foo");
let bar = jdb.insert(b"bar");
jdb.commit(0, &b"0".sha3(), None).unwrap();
assert!(jdb.exists(&foo));
assert!(jdb.exists(&bar));
jdb.remove(&foo);
let baz = jdb.insert(b"baz");
jdb.commit(1, &b"1a".sha3(), Some((0, b"0".sha3()))).unwrap();
jdb.remove(&bar);
jdb.commit(1, &b"1b".sha3(), Some((0, b"0".sha3()))).unwrap();
assert!(jdb.exists(&foo));
assert!(jdb.exists(&bar));
assert!(jdb.exists(&baz));
jdb.commit(2, &b"2b".sha3(), Some((1, b"1b".sha3()))).unwrap();
assert!(jdb.exists(&foo));
}
#[test]
fn overwrite() {
// history is 1
let mut jdb = ArchiveDB::new_temp();
let foo = jdb.insert(b"foo");
jdb.commit(0, &b"0".sha3(), None).unwrap();
assert!(jdb.exists(&foo));
jdb.remove(&foo);
jdb.commit(1, &b"1".sha3(), Some((0, b"0".sha3()))).unwrap();
jdb.insert(b"foo");
assert!(jdb.exists(&foo));
jdb.commit(2, &b"2".sha3(), Some((1, b"1".sha3()))).unwrap();
assert!(jdb.exists(&foo));
jdb.commit(3, &b"2".sha3(), Some((0, b"2".sha3()))).unwrap();
assert!(jdb.exists(&foo));
}
#[test]
fn fork_same_key() {
// history is 1
let mut jdb = ArchiveDB::new_temp();
jdb.commit(0, &b"0".sha3(), None).unwrap();
let foo = jdb.insert(b"foo");
jdb.commit(1, &b"1a".sha3(), Some((0, b"0".sha3()))).unwrap();
jdb.insert(b"foo");
jdb.commit(1, &b"1b".sha3(), Some((0, b"0".sha3()))).unwrap();
assert!(jdb.exists(&foo));
jdb.commit(2, &b"2a".sha3(), Some((1, b"1a".sha3()))).unwrap();
assert!(jdb.exists(&foo));
}
#[test]
fn reopen() {
let mut dir = ::std::env::temp_dir();
dir.push(H32::random().hex());
let bar = H256::random();
let foo = {
let mut jdb = ArchiveDB::new(dir.to_str().unwrap());
// history is 1
let foo = jdb.insert(b"foo");
jdb.emplace(bar.clone(), b"bar".to_vec());
jdb.commit(0, &b"0".sha3(), None).unwrap();
foo
};
{
let mut jdb = ArchiveDB::new(dir.to_str().unwrap());
jdb.remove(&foo);
jdb.commit(1, &b"1".sha3(), Some((0, b"0".sha3()))).unwrap();
}
{
let mut jdb = ArchiveDB::new(dir.to_str().unwrap());
assert!(jdb.exists(&foo));
assert!(jdb.exists(&bar));
jdb.commit(2, &b"2".sha3(), Some((1, b"1".sha3()))).unwrap();
}
}
#[test]
fn reopen_remove() {
let mut dir = ::std::env::temp_dir();
dir.push(H32::random().hex());
let foo = {
let mut jdb = ArchiveDB::new(dir.to_str().unwrap());
// history is 1
let foo = jdb.insert(b"foo");
jdb.commit(0, &b"0".sha3(), None).unwrap();
jdb.commit(1, &b"1".sha3(), Some((0, b"0".sha3()))).unwrap();
// foo is ancient history.
jdb.insert(b"foo");
jdb.commit(2, &b"2".sha3(), Some((1, b"1".sha3()))).unwrap();
foo
};
{
let mut jdb = ArchiveDB::new(dir.to_str().unwrap());
jdb.remove(&foo);
jdb.commit(3, &b"3".sha3(), Some((2, b"2".sha3()))).unwrap();
assert!(jdb.exists(&foo));
jdb.remove(&foo);
jdb.commit(4, &b"4".sha3(), Some((3, b"3".sha3()))).unwrap();
jdb.commit(5, &b"5".sha3(), Some((4, b"4".sha3()))).unwrap();
}
}
#[test]
fn reopen_fork() {
let mut dir = ::std::env::temp_dir();
dir.push(H32::random().hex());
let (foo, _, _) = {
let mut jdb = ArchiveDB::new(dir.to_str().unwrap());
// history is 1
let foo = jdb.insert(b"foo");
let bar = jdb.insert(b"bar");
jdb.commit(0, &b"0".sha3(), None).unwrap();
jdb.remove(&foo);
let baz = jdb.insert(b"baz");
jdb.commit(1, &b"1a".sha3(), Some((0, b"0".sha3()))).unwrap();
jdb.remove(&bar);
jdb.commit(1, &b"1b".sha3(), Some((0, b"0".sha3()))).unwrap();
(foo, bar, baz)
};
{
let mut jdb = ArchiveDB::new(dir.to_str().unwrap());
jdb.commit(2, &b"2b".sha3(), Some((1, b"1b".sha3()))).unwrap();
assert!(jdb.exists(&foo));
}
}
#[test]
fn returns_state() {
let temp = ::devtools::RandomTempPath::new();
let key = {
let mut jdb = ArchiveDB::new(temp.as_str());
let key = jdb.insert(b"foo");
jdb.commit(0, &b"0".sha3(), None).unwrap();
key
};
{
let jdb = ArchiveDB::new(temp.as_str());
let state = jdb.state(&key);
assert!(state.is_some());
}
}
}

1064
util/src/journaldb/earlymergedb.rs Normal file
View File

File diff suppressed because it is too large Load Diff

81
util/src/journaldb/mod.rs Normal file
View File

@@ -0,0 +1,81 @@
// Copyright 2015, 2016 Ethcore (UK) Ltd.
// This file is part of Parity.
// Parity is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// Parity is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
// You should have received a copy of the GNU General Public License
// along with Parity. If not, see <http://www.gnu.org/licenses/>.
//! JournalDB interface and implementation.
use common::*;
/// Export the journaldb module.
pub mod traits;
mod archivedb;
mod earlymergedb;
mod overlayrecentdb;
mod refcounteddb;
/// Export the JournalDB trait.
pub use self::traits::JournalDB;
/// A journal database algorithm.
#[derive(Debug)]
pub enum Algorithm {
/// Keep all keys forever.
Archive,
/// Ancient and recent history maintained separately; recent history lasts for particular
/// number of blocks.
///
/// Inserts go into backing database, journal retains knowledge of whether backing DB key is
/// ancient or recent. Non-canon inserts get explicitly reverted and removed from backing DB.
EarlyMerge,
/// Ancient and recent history maintained separately; recent history lasts for particular
/// number of blocks.
///
/// Inserts go into memory overlay, which is tried for key fetches. Memory overlay gets
/// flushed in backing only at end of recent history.
OverlayRecent,
/// Ancient and recent history maintained separately; recent history lasts for particular
/// number of blocks.
///
/// References are counted in disk-backed DB.
RefCounted,
}
impl Default for Algorithm {
fn default() -> Algorithm { Algorithm::Archive }
}
impl fmt::Display for Algorithm {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
write!(f, "{}", match self {
&Algorithm::Archive => "archive",
&Algorithm::EarlyMerge => "earlymerge",
&Algorithm::OverlayRecent => "overlayrecent",
&Algorithm::RefCounted => "refcounted",
})
}
}
/// Create a new JournalDB trait object.
pub fn new(path: &str, algorithm: Algorithm) -> Box<JournalDB> {
match algorithm {
Algorithm::Archive => Box::new(archivedb::ArchiveDB::new(path)),
Algorithm::EarlyMerge => Box::new(earlymergedb::EarlyMergeDB::new(path)),
Algorithm::OverlayRecent => Box::new(overlayrecentdb::OverlayRecentDB::new(path)),
Algorithm::RefCounted => Box::new(refcounteddb::RefCountedDB::new(path)),
}
}

895
util/src/journaldb/overlayrecentdb.rs Normal file
View File

@@ -0,0 +1,895 @@
// Copyright 2015, 2016 Ethcore (UK) Ltd.
// This file is part of Parity.
// Parity is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// Parity is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
// You should have received a copy of the GNU General Public License
// along with Parity. If not, see <http://www.gnu.org/licenses/>.
//! JournalDB over in-memory overlay
use common::*;
use rlp::*;
use hashdb::*;
use memorydb::*;
use kvdb::{Database, DBTransaction, DatabaseConfig};
#[cfg(test)]
use std::env;
use super::JournalDB;
/// Implementation of the JournalDB trait for a disk-backed database with a memory overlay
/// and, possibly, latent-removal semantics.
///
/// Like OverlayDB, there is a memory overlay; `commit()` must be called in order to
/// write operations out to disk. Unlike OverlayDB, `remove()` operations do not take effect
/// immediately. Rather some age (based on a linear but arbitrary metric) must pass before
/// the removals actually take effect.
///
/// There are two memory overlays:
/// - Transaction overlay contains current transaction data. It is merged with with history
/// overlay on each `commit()`
/// - History overlay contains all data inserted during the history period. When the node
/// in the overlay becomes ancient it is written to disk on `commit()`
///
/// There is also a journal maintained in memory and on the disk as well which lists insertions
/// and removals for each commit during the history period. This is used to track
/// data nodes that go out of history scope and must be written to disk.
///
/// Commit workflow:
/// 1. Create a new journal record from the transaction overlay.
/// 2. Inseart each node from the transaction overlay into the History overlay increasing reference
/// count if it is already there. Note that the reference counting is managed by `MemoryDB`
/// 3. Clear the transaction overlay.
/// 4. For a canonical journal record that becomes ancient inserts its insertions into the disk DB
/// 5. For each journal record that goes out of the history scope (becomes ancient) remove its
/// insertions from the history overlay, decreasing the reference counter and removing entry if
/// if reaches zero.
/// 6. For a canonical journal record that becomes ancient delete its removals from the disk only if
/// the removed key is not present in the history overlay.
/// 7. Delete ancient record from memory and disk.
pub struct OverlayRecentDB {
transaction_overlay: MemoryDB,
backing: Arc<Database>,
journal_overlay: Arc<RwLock<JournalOverlay>>,
}
#[derive(PartialEq)]
struct JournalOverlay {
backing_overlay: MemoryDB,
journal: HashMap<u64, Vec<JournalEntry>>,
latest_era: Option<u64>,
}
#[derive(PartialEq)]
struct JournalEntry {
id: H256,
insertions: Vec<H256>,
deletions: Vec<H256>,
}
impl HeapSizeOf for JournalEntry {
fn heap_size_of_children(&self) -> usize {
self.insertions.heap_size_of_children() + self.deletions.heap_size_of_children()
}
}
impl Clone for OverlayRecentDB {
fn clone(&self) -> OverlayRecentDB {
OverlayRecentDB {
transaction_overlay: MemoryDB::new(),
backing: self.backing.clone(),
journal_overlay: self.journal_overlay.clone(),
}
}
}
// all keys must be at least 12 bytes
const LATEST_ERA_KEY : [u8; 12] = [ b'l', b'a', b's', b't', 0, 0, 0, 0, 0, 0, 0, 0 ];
const VERSION_KEY : [u8; 12] = [ b'j', b'v', b'e', b'r', 0, 0, 0, 0, 0, 0, 0, 0 ];
const DB_VERSION : u32 = 0x200 + 3;
const PADDING : [u8; 10] = [ 0u8; 10 ];
impl OverlayRecentDB {
/// Create a new instance from file
pub fn new(path: &str) -> OverlayRecentDB {
Self::from_prefs(path)
}
/// Create a new instance from file
pub fn from_prefs(path: &str) -> OverlayRecentDB {
let opts = DatabaseConfig {
prefix_size: Some(12) //use 12 bytes as prefix, this must match account_db prefix
};
let backing = Database::open(&opts, path).unwrap_or_else(|e| {
panic!("Error opening state db: {}", e);
});
if !backing.is_empty() {
match backing.get(&VERSION_KEY).map(|d| d.map(|v| decode::<u32>(&v))) {
Ok(Some(DB_VERSION)) => {}
v => panic!("Incompatible DB version, expected {}, got {:?}", DB_VERSION, v)
}
} else {
backing.put(&VERSION_KEY, &encode(&DB_VERSION)).expect("Error writing version to database");
}
let journal_overlay = Arc::new(RwLock::new(OverlayRecentDB::read_overlay(&backing)));
OverlayRecentDB {
transaction_overlay: MemoryDB::new(),
backing: Arc::new(backing),
journal_overlay: journal_overlay,
}
}
/// Create a new instance with an anonymous temporary database.
#[cfg(test)]
pub fn new_temp() -> OverlayRecentDB {
let mut dir = env::temp_dir();
dir.push(H32::random().hex());
Self::new(dir.to_str().unwrap())
}
#[cfg(test)]
fn can_reconstruct_refs(&self) -> bool {
let reconstructed = Self::read_overlay(&self.backing);
let journal_overlay = self.journal_overlay.read().unwrap();
*journal_overlay == reconstructed
}
fn payload(&self, key: &H256) -> Option<Bytes> {
self.backing.get(&key.bytes()).expect("Low-level database error. Some issue with your hard disk?").map(|v| v.to_vec())
}
fn read_overlay(db: &Database) -> JournalOverlay {
let mut journal = HashMap::new();
let mut overlay = MemoryDB::new();
let mut count = 0;
let mut latest_era = None;
if let Some(val) = db.get(&LATEST_ERA_KEY).expect("Low-level database error.") {
let mut era = decode::<u64>(&val);
latest_era = Some(era);
loop {
let mut index = 0usize;
while let Some(rlp_data) = db.get({
let mut r = RlpStream::new_list(3);
r.append(&era);
r.append(&index);
r.append(&&PADDING[..]);
&r.drain()
}).expect("Low-level database error.") {
trace!("read_overlay: era={}, index={}", era, index);
let rlp = Rlp::new(&rlp_data);
let id: H256 = rlp.val_at(0);
let insertions = rlp.at(1);
let deletions: Vec<H256> = rlp.val_at(2);
let mut inserted_keys = Vec::new();
for r in insertions.iter() {
let k: H256 = r.val_at(0);
let v: Bytes = r.val_at(1);
overlay.emplace(k.clone(), v);
inserted_keys.push(k);
count += 1;
}
journal.entry(era).or_insert_with(Vec::new).push(JournalEntry {
id: id,
insertions: inserted_keys,
deletions: deletions,
});
index += 1;
};
if index == 0 || era == 0 {
break;
}
era -= 1;
}
}
trace!("Recovered {} overlay entries, {} journal entries", count, journal.len());
JournalOverlay { backing_overlay: overlay, journal: journal, latest_era: latest_era }
}
}
impl JournalDB for OverlayRecentDB {
fn spawn(&self) -> Box<JournalDB> {
Box::new(self.clone())
}
fn mem_used(&self) -> usize {
let mut mem = self.transaction_overlay.mem_used();
let overlay = self.journal_overlay.read().unwrap();
mem += overlay.backing_overlay.mem_used();
mem += overlay.journal.heap_size_of_children();
mem
}
fn is_empty(&self) -> bool {
self.backing.get(&LATEST_ERA_KEY).expect("Low level database error").is_none()
}
fn commit(&mut self, now: u64, id: &H256, end: Option<(u64, H256)>) -> Result<u32, UtilError> {
// record new commit's details.
trace!("commit: #{} ({}), end era: {:?}", now, id, end);
let mut journal_overlay = self.journal_overlay.write().unwrap();
let batch = DBTransaction::new();
{
let mut r = RlpStream::new_list(3);
let mut tx = self.transaction_overlay.drain();
let inserted_keys: Vec<_> = tx.iter().filter_map(|(k, &(_, c))| if c > 0 { Some(k.clone()) } else { None }).collect();
let removed_keys: Vec<_> = tx.iter().filter_map(|(k, &(_, c))| if c < 0 { Some(k.clone()) } else { None }).collect();
// Increase counter for each inserted key no matter if the block is canonical or not.
let insertions = tx.drain().filter_map(|(k, (v, c))| if c > 0 { Some((k, v)) } else { None });
r.append(id);
r.begin_list(inserted_keys.len());
for (k, v) in insertions {
r.begin_list(2);
r.append(&k);
r.append(&v);
journal_overlay.backing_overlay.emplace(k, v);
}
r.append(&removed_keys);
let mut k = RlpStream::new_list(3);
let index = journal_overlay.journal.get(&now).map_or(0, |j| j.len());
k.append(&now);
k.append(&index);
k.append(&&PADDING[..]);
try!(batch.put(&k.drain(), r.as_raw()));
if journal_overlay.latest_era.map_or(true, |e| now > e) {
try!(batch.put(&LATEST_ERA_KEY, &encode(&now)));
journal_overlay.latest_era = Some(now);
}
journal_overlay.journal.entry(now).or_insert_with(Vec::new).push(JournalEntry { id: id.clone(), insertions: inserted_keys, deletions: removed_keys });
}
let journal_overlay = journal_overlay.deref_mut();
// apply old commits' details
if let Some((end_era, canon_id)) = end {
if let Some(ref mut records) = journal_overlay.journal.get_mut(&end_era) {
let mut canon_insertions: Vec<(H256, Bytes)> = Vec::new();
let mut canon_deletions: Vec<H256> = Vec::new();
let mut overlay_deletions: Vec<H256> = Vec::new();
let mut index = 0usize;
for mut journal in records.drain(..) {
//delete the record from the db
let mut r = RlpStream::new_list(3);
r.append(&end_era);
r.append(&index);
r.append(&&PADDING[..]);
try!(batch.delete(&r.drain()));
trace!("commit: Delete journal for time #{}.{}: {}, (canon was {}): +{} -{} entries", end_era, index, journal.id, canon_id, journal.insertions.len(), journal.deletions.len());
{
if canon_id == journal.id {
for h in &journal.insertions {
if let Some(&(ref d, rc)) = journal_overlay.backing_overlay.raw(h) {
if rc > 0 {
canon_insertions.push((h.clone(), d.clone())); //TODO: optimize this to avoid data copy
}
}
}
canon_deletions = journal.deletions;
}
overlay_deletions.append(&mut journal.insertions);
}
index += 1;
}
// apply canon inserts first
for (k, v) in canon_insertions {
try!(batch.put(&k, &v));
}
// update the overlay
for k in overlay_deletions {
journal_overlay.backing_overlay.kill(&k);
}
// apply canon deletions
for k in canon_deletions {
if !journal_overlay.backing_overlay.exists(&k) {
try!(batch.delete(&k));
}
}
journal_overlay.backing_overlay.purge();
}
journal_overlay.journal.remove(&end_era);
}
try!(self.backing.write(batch));
Ok(0)
}
}
impl HashDB for OverlayRecentDB {
fn keys(&self) -> HashMap<H256, i32> {
let mut ret: HashMap<H256, i32> = HashMap::new();
for (key, _) in self.backing.iter() {
let h = H256::from_slice(key.deref());
ret.insert(h, 1);
}
for (key, refs) in self.transaction_overlay.keys().into_iter() {
let refs = *ret.get(&key).unwrap_or(&0) + refs;
ret.insert(key, refs);
}
ret
}
fn lookup(&self, key: &H256) -> Option<&[u8]> {
let k = self.transaction_overlay.raw(key);
match k {
Some(&(ref d, rc)) if rc > 0 => Some(d),
_ => {
let v = self.journal_overlay.read().unwrap().backing_overlay.lookup(key).map(|v| v.to_vec());
match v {
Some(x) => {
Some(&self.transaction_overlay.denote(key, x).0)
}
_ => {
if let Some(x) = self.payload(key) {
Some(&self.transaction_overlay.denote(key, x).0)
}
else {
None
}
}
}
}
}
}
fn exists(&self, key: &H256) -> bool {
self.lookup(key).is_some()
}
fn insert(&mut self, value: &[u8]) -> H256 {
self.transaction_overlay.insert(value)
}
fn emplace(&mut self, key: H256, value: Bytes) {
self.transaction_overlay.emplace(key, value);
}
fn kill(&mut self, key: &H256) {
self.transaction_overlay.kill(key);
}
}
#[cfg(test)]
mod tests {
#![cfg_attr(feature="dev", allow(blacklisted_name))]
use common::*;
use super::*;
use hashdb::*;
use log::init_log;
use journaldb::JournalDB;
#[test]
fn insert_same_in_fork() {
// history is 1
let mut jdb = OverlayRecentDB::new_temp();
let x = jdb.insert(b"X");
jdb.commit(1, &b"1".sha3(), None).unwrap();
assert!(jdb.can_reconstruct_refs());
jdb.commit(2, &b"2".sha3(), None).unwrap();
assert!(jdb.can_reconstruct_refs());
jdb.commit(3, &b"1002a".sha3(), Some((1, b"1".sha3()))).unwrap();
assert!(jdb.can_reconstruct_refs());
jdb.commit(4, &b"1003a".sha3(), Some((2, b"2".sha3()))).unwrap();
assert!(jdb.can_reconstruct_refs());
jdb.remove(&x);
jdb.commit(3, &b"1002b".sha3(), Some((1, b"1".sha3()))).unwrap();
assert!(jdb.can_reconstruct_refs());
let x = jdb.insert(b"X");
jdb.commit(4, &b"1003b".sha3(), Some((2, b"2".sha3()))).unwrap();
assert!(jdb.can_reconstruct_refs());
jdb.commit(5, &b"1004a".sha3(), Some((3, b"1002a".sha3()))).unwrap();
assert!(jdb.can_reconstruct_refs());
jdb.commit(6, &b"1005a".sha3(), Some((4, b"1003a".sha3()))).unwrap();
assert!(jdb.can_reconstruct_refs());
assert!(jdb.exists(&x));
}
#[test]
fn long_history() {
// history is 3
let mut jdb = OverlayRecentDB::new_temp();
let h = jdb.insert(b"foo");
jdb.commit(0, &b"0".sha3(), None).unwrap();
assert!(jdb.can_reconstruct_refs());
assert!(jdb.exists(&h));
jdb.remove(&h);
jdb.commit(1, &b"1".sha3(), None).unwrap();
assert!(jdb.can_reconstruct_refs());
assert!(jdb.exists(&h));
jdb.commit(2, &b"2".sha3(), None).unwrap();
assert!(jdb.can_reconstruct_refs());
assert!(jdb.exists(&h));
jdb.commit(3, &b"3".sha3(), Some((0, b"0".sha3()))).unwrap();
assert!(jdb.can_reconstruct_refs());
assert!(jdb.exists(&h));
jdb.commit(4, &b"4".sha3(), Some((1, b"1".sha3()))).unwrap();
assert!(jdb.can_reconstruct_refs());
assert!(!jdb.exists(&h));
}
#[test]
fn complex() {
// history is 1
let mut jdb = OverlayRecentDB::new_temp();
let foo = jdb.insert(b"foo");
let bar = jdb.insert(b"bar");
jdb.commit(0, &b"0".sha3(), None).unwrap();
assert!(jdb.can_reconstruct_refs());
assert!(jdb.exists(&foo));
assert!(jdb.exists(&bar));
jdb.remove(&foo);
jdb.remove(&bar);
let baz = jdb.insert(b"baz");
jdb.commit(1, &b"1".sha3(), Some((0, b"0".sha3()))).unwrap();
assert!(jdb.can_reconstruct_refs());
assert!(jdb.exists(&foo));
assert!(jdb.exists(&bar));
assert!(jdb.exists(&baz));
let foo = jdb.insert(b"foo");
jdb.remove(&baz);
jdb.commit(2, &b"2".sha3(), Some((1, b"1".sha3()))).unwrap();
assert!(jdb.can_reconstruct_refs());
assert!(jdb.exists(&foo));
assert!(!jdb.exists(&bar));
assert!(jdb.exists(&baz));
jdb.remove(&foo);
jdb.commit(3, &b"3".sha3(), Some((2, b"2".sha3()))).unwrap();
assert!(jdb.can_reconstruct_refs());
assert!(jdb.exists(&foo));
assert!(!jdb.exists(&bar));
assert!(!jdb.exists(&baz));
jdb.commit(4, &b"4".sha3(), Some((3, b"3".sha3()))).unwrap();
assert!(jdb.can_reconstruct_refs());
assert!(!jdb.exists(&foo));
assert!(!jdb.exists(&bar));
assert!(!jdb.exists(&baz));
}
#[test]
fn fork() {
// history is 1
let mut jdb = OverlayRecentDB::new_temp();
let foo = jdb.insert(b"foo");
let bar = jdb.insert(b"bar");
jdb.commit(0, &b"0".sha3(), None).unwrap();
assert!(jdb.can_reconstruct_refs());
assert!(jdb.exists(&foo));
assert!(jdb.exists(&bar));
jdb.remove(&foo);
let baz = jdb.insert(b"baz");
jdb.commit(1, &b"1a".sha3(), Some((0, b"0".sha3()))).unwrap();
assert!(jdb.can_reconstruct_refs());
jdb.remove(&bar);
jdb.commit(1, &b"1b".sha3(), Some((0, b"0".sha3()))).unwrap();
assert!(jdb.can_reconstruct_refs());
assert!(jdb.exists(&foo));
assert!(jdb.exists(&bar));
assert!(jdb.exists(&baz));
jdb.commit(2, &b"2b".sha3(), Some((1, b"1b".sha3()))).unwrap();
assert!(jdb.can_reconstruct_refs());
assert!(jdb.exists(&foo));
assert!(!jdb.exists(&baz));
assert!(!jdb.exists(&bar));
}
#[test]
fn overwrite() {
// history is 1
let mut jdb = OverlayRecentDB::new_temp();
let foo = jdb.insert(b"foo");
jdb.commit(0, &b"0".sha3(), None).unwrap();
assert!(jdb.can_reconstruct_refs());
assert!(jdb.exists(&foo));
jdb.remove(&foo);
jdb.commit(1, &b"1".sha3(), Some((0, b"0".sha3()))).unwrap();
assert!(jdb.can_reconstruct_refs());
jdb.insert(b"foo");
assert!(jdb.exists(&foo));
jdb.commit(2, &b"2".sha3(), Some((1, b"1".sha3()))).unwrap();
assert!(jdb.can_reconstruct_refs());
assert!(jdb.exists(&foo));
jdb.commit(3, &b"2".sha3(), Some((0, b"2".sha3()))).unwrap();
assert!(jdb.can_reconstruct_refs());
assert!(jdb.exists(&foo));
}
#[test]
fn fork_same_key_one() {
let mut dir = ::std::env::temp_dir();
dir.push(H32::random().hex());
let mut jdb = OverlayRecentDB::new(dir.to_str().unwrap());
jdb.commit(0, &b"0".sha3(), None).unwrap();
assert!(jdb.can_reconstruct_refs());
let foo = jdb.insert(b"foo");
jdb.commit(1, &b"1a".sha3(), Some((0, b"0".sha3()))).unwrap();
assert!(jdb.can_reconstruct_refs());
jdb.insert(b"foo");
jdb.commit(1, &b"1b".sha3(), Some((0, b"0".sha3()))).unwrap();
assert!(jdb.can_reconstruct_refs());
jdb.insert(b"foo");
jdb.commit(1, &b"1c".sha3(), Some((0, b"0".sha3()))).unwrap();
assert!(jdb.can_reconstruct_refs());
assert!(jdb.exists(&foo));
jdb.commit(2, &b"2a".sha3(), Some((1, b"1a".sha3()))).unwrap();
assert!(jdb.can_reconstruct_refs());
assert!(jdb.exists(&foo));
}
#[test]
fn fork_same_key_other() {
let mut dir = ::std::env::temp_dir();
dir.push(H32::random().hex());
let mut jdb = OverlayRecentDB::new(dir.to_str().unwrap());
jdb.commit(0, &b"0".sha3(), None).unwrap();
assert!(jdb.can_reconstruct_refs());
let foo = jdb.insert(b"foo");
jdb.commit(1, &b"1a".sha3(), Some((0, b"0".sha3()))).unwrap();
assert!(jdb.can_reconstruct_refs());
jdb.insert(b"foo");
jdb.commit(1, &b"1b".sha3(), Some((0, b"0".sha3()))).unwrap();
assert!(jdb.can_reconstruct_refs());
jdb.insert(b"foo");
jdb.commit(1, &b"1c".sha3(), Some((0, b"0".sha3()))).unwrap();
assert!(jdb.can_reconstruct_refs());
assert!(jdb.exists(&foo));
jdb.commit(2, &b"2b".sha3(), Some((1, b"1b".sha3()))).unwrap();
assert!(jdb.can_reconstruct_refs());
assert!(jdb.exists(&foo));
}
#[test]
fn fork_ins_del_ins() {
let mut dir = ::std::env::temp_dir();
dir.push(H32::random().hex());
let mut jdb = OverlayRecentDB::new(dir.to_str().unwrap());
jdb.commit(0, &b"0".sha3(), None).unwrap();
assert!(jdb.can_reconstruct_refs());
let foo = jdb.insert(b"foo");
jdb.commit(1, &b"1".sha3(), None).unwrap();
assert!(jdb.can_reconstruct_refs());
jdb.remove(&foo);
jdb.commit(2, &b"2a".sha3(), Some((0, b"0".sha3()))).unwrap();
assert!(jdb.can_reconstruct_refs());
jdb.remove(&foo);
jdb.commit(2, &b"2b".sha3(), Some((0, b"0".sha3()))).unwrap();
assert!(jdb.can_reconstruct_refs());
jdb.insert(b"foo");
jdb.commit(3, &b"3a".sha3(), Some((1, b"1".sha3()))).unwrap();
assert!(jdb.can_reconstruct_refs());
jdb.insert(b"foo");
jdb.commit(3, &b"3b".sha3(), Some((1, b"1".sha3()))).unwrap();
assert!(jdb.can_reconstruct_refs());
jdb.commit(4, &b"4a".sha3(), Some((2, b"2a".sha3()))).unwrap();
assert!(jdb.can_reconstruct_refs());
jdb.commit(5, &b"5a".sha3(), Some((3, b"3a".sha3()))).unwrap();
assert!(jdb.can_reconstruct_refs());
}
#[test]
fn reopen() {
let mut dir = ::std::env::temp_dir();
dir.push(H32::random().hex());
let bar = H256::random();
let foo = {
let mut jdb = OverlayRecentDB::new(dir.to_str().unwrap());
// history is 1
let foo = jdb.insert(b"foo");
jdb.emplace(bar.clone(), b"bar".to_vec());
jdb.commit(0, &b"0".sha3(), None).unwrap();
assert!(jdb.can_reconstruct_refs());
foo
};
{
let mut jdb = OverlayRecentDB::new(dir.to_str().unwrap());
jdb.remove(&foo);
jdb.commit(1, &b"1".sha3(), Some((0, b"0".sha3()))).unwrap();
assert!(jdb.can_reconstruct_refs());
}
{
let mut jdb = OverlayRecentDB::new(dir.to_str().unwrap());
assert!(jdb.exists(&foo));
assert!(jdb.exists(&bar));
jdb.commit(2, &b"2".sha3(), Some((1, b"1".sha3()))).unwrap();
assert!(jdb.can_reconstruct_refs());
assert!(!jdb.exists(&foo));
}
}
#[test]
fn insert_delete_insert_delete_insert_expunge() {
init_log();
let mut dir = ::std::env::temp_dir();
dir.push(H32::random().hex());
let mut jdb = OverlayRecentDB::new(dir.to_str().unwrap());
// history is 4
let foo = jdb.insert(b"foo");
jdb.commit(0, &b"0".sha3(), None).unwrap();
assert!(jdb.can_reconstruct_refs());
jdb.remove(&foo);
jdb.commit(1, &b"1".sha3(), None).unwrap();
assert!(jdb.can_reconstruct_refs());
jdb.insert(b"foo");
jdb.commit(2, &b"2".sha3(), None).unwrap();
assert!(jdb.can_reconstruct_refs());
jdb.remove(&foo);
jdb.commit(3, &b"3".sha3(), None).unwrap();
assert!(jdb.can_reconstruct_refs());
jdb.insert(b"foo");
jdb.commit(4, &b"4".sha3(), Some((0, b"0".sha3()))).unwrap();
assert!(jdb.can_reconstruct_refs());
// expunge foo
jdb.commit(5, &b"5".sha3(), Some((1, b"1".sha3()))).unwrap();
assert!(jdb.can_reconstruct_refs());
}
#[test]
fn forked_insert_delete_insert_delete_insert_expunge() {
init_log();
let mut dir = ::std::env::temp_dir();
dir.push(H32::random().hex());
let mut jdb = OverlayRecentDB::new(dir.to_str().unwrap());
// history is 4
let foo = jdb.insert(b"foo");
jdb.commit(0, &b"0".sha3(), None).unwrap();
assert!(jdb.can_reconstruct_refs());
jdb.remove(&foo);
jdb.commit(1, &b"1a".sha3(), None).unwrap();
assert!(jdb.can_reconstruct_refs());
jdb.remove(&foo);
jdb.commit(1, &b"1b".sha3(), None).unwrap();
assert!(jdb.can_reconstruct_refs());
jdb.insert(b"foo");
jdb.commit(2, &b"2a".sha3(), None).unwrap();
assert!(jdb.can_reconstruct_refs());
jdb.insert(b"foo");
jdb.commit(2, &b"2b".sha3(), None).unwrap();
assert!(jdb.can_reconstruct_refs());
jdb.remove(&foo);
jdb.commit(3, &b"3a".sha3(), None).unwrap();
assert!(jdb.can_reconstruct_refs());
jdb.remove(&foo);
jdb.commit(3, &b"3b".sha3(), None).unwrap();
assert!(jdb.can_reconstruct_refs());
jdb.insert(b"foo");
jdb.commit(4, &b"4a".sha3(), Some((0, b"0".sha3()))).unwrap();
assert!(jdb.can_reconstruct_refs());
jdb.insert(b"foo");
jdb.commit(4, &b"4b".sha3(), Some((0, b"0".sha3()))).unwrap();
assert!(jdb.can_reconstruct_refs());
// expunge foo
jdb.commit(5, &b"5".sha3(), Some((1, b"1a".sha3()))).unwrap();
assert!(jdb.can_reconstruct_refs());
}
#[test]
fn broken_assert() {
let mut dir = ::std::env::temp_dir();
dir.push(H32::random().hex());
let mut jdb = OverlayRecentDB::new(dir.to_str().unwrap());
// history is 1
let foo = jdb.insert(b"foo");
jdb.commit(1, &b"1".sha3(), Some((0, b"0".sha3()))).unwrap();
assert!(jdb.can_reconstruct_refs());
// foo is ancient history.
jdb.remove(&foo);
jdb.commit(2, &b"2".sha3(), Some((1, b"1".sha3()))).unwrap();
assert!(jdb.can_reconstruct_refs());
jdb.insert(b"foo");
jdb.commit(3, &b"3".sha3(), Some((2, b"2".sha3()))).unwrap(); // BROKEN
assert!(jdb.can_reconstruct_refs());
assert!(jdb.exists(&foo));
jdb.remove(&foo);
jdb.commit(4, &b"4".sha3(), Some((3, b"3".sha3()))).unwrap();
assert!(jdb.can_reconstruct_refs());
jdb.commit(5, &b"5".sha3(), Some((4, b"4".sha3()))).unwrap();
assert!(jdb.can_reconstruct_refs());
assert!(!jdb.exists(&foo));
}
#[test]
fn reopen_test() {
let mut dir = ::std::env::temp_dir();
dir.push(H32::random().hex());
let mut jdb = OverlayRecentDB::new(dir.to_str().unwrap());
// history is 4
let foo = jdb.insert(b"foo");
jdb.commit(0, &b"0".sha3(), None).unwrap();
assert!(jdb.can_reconstruct_refs());
jdb.commit(1, &b"1".sha3(), None).unwrap();
assert!(jdb.can_reconstruct_refs());
jdb.commit(2, &b"2".sha3(), None).unwrap();
assert!(jdb.can_reconstruct_refs());
jdb.commit(3, &b"3".sha3(), None).unwrap();
assert!(jdb.can_reconstruct_refs());
jdb.commit(4, &b"4".sha3(), Some((0, b"0".sha3()))).unwrap();
assert!(jdb.can_reconstruct_refs());
// foo is ancient history.
jdb.insert(b"foo");
let bar = jdb.insert(b"bar");
jdb.commit(5, &b"5".sha3(), Some((1, b"1".sha3()))).unwrap();
assert!(jdb.can_reconstruct_refs());
jdb.remove(&foo);
jdb.remove(&bar);
jdb.commit(6, &b"6".sha3(), Some((2, b"2".sha3()))).unwrap();
assert!(jdb.can_reconstruct_refs());
jdb.insert(b"foo");
jdb.insert(b"bar");
jdb.commit(7, &b"7".sha3(), Some((3, b"3".sha3()))).unwrap();
assert!(jdb.can_reconstruct_refs());
}
#[test]
fn reopen_remove_three() {
init_log();
let mut dir = ::std::env::temp_dir();
dir.push(H32::random().hex());
let foo = b"foo".sha3();
{
let mut jdb = OverlayRecentDB::new(dir.to_str().unwrap());
// history is 1
jdb.insert(b"foo");
jdb.commit(0, &b"0".sha3(), None).unwrap();
assert!(jdb.can_reconstruct_refs());
jdb.commit(1, &b"1".sha3(), None).unwrap();
assert!(jdb.can_reconstruct_refs());
// foo is ancient history.
jdb.remove(&foo);
jdb.commit(2, &b"2".sha3(), Some((0, b"0".sha3()))).unwrap();
assert!(jdb.can_reconstruct_refs());
assert!(jdb.exists(&foo));
jdb.insert(b"foo");
jdb.commit(3, &b"3".sha3(), Some((1, b"1".sha3()))).unwrap();
assert!(jdb.can_reconstruct_refs());
assert!(jdb.exists(&foo));
// incantation to reopen the db
}; { let mut jdb = OverlayRecentDB::new(dir.to_str().unwrap());
jdb.remove(&foo);
jdb.commit(4, &b"4".sha3(), Some((2, b"2".sha3()))).unwrap();
assert!(jdb.can_reconstruct_refs());
assert!(jdb.exists(&foo));
// incantation to reopen the db
}; { let mut jdb = OverlayRecentDB::new(dir.to_str().unwrap());
jdb.commit(5, &b"5".sha3(), Some((3, b"3".sha3()))).unwrap();
assert!(jdb.can_reconstruct_refs());
assert!(jdb.exists(&foo));
// incantation to reopen the db
}; { let mut jdb = OverlayRecentDB::new(dir.to_str().unwrap());
jdb.commit(6, &b"6".sha3(), Some((4, b"4".sha3()))).unwrap();
assert!(jdb.can_reconstruct_refs());
assert!(!jdb.exists(&foo));
}
}
#[test]
fn reopen_fork() {
let mut dir = ::std::env::temp_dir();
dir.push(H32::random().hex());
let (foo, bar, baz) = {
let mut jdb = OverlayRecentDB::new(dir.to_str().unwrap());
// history is 1
let foo = jdb.insert(b"foo");
let bar = jdb.insert(b"bar");
jdb.commit(0, &b"0".sha3(), None).unwrap();
assert!(jdb.can_reconstruct_refs());
jdb.remove(&foo);
let baz = jdb.insert(b"baz");
jdb.commit(1, &b"1a".sha3(), Some((0, b"0".sha3()))).unwrap();
assert!(jdb.can_reconstruct_refs());
jdb.remove(&bar);
jdb.commit(1, &b"1b".sha3(), Some((0, b"0".sha3()))).unwrap();
assert!(jdb.can_reconstruct_refs());
(foo, bar, baz)
};
{
let mut jdb = OverlayRecentDB::new(dir.to_str().unwrap());
jdb.commit(2, &b"2b".sha3(), Some((1, b"1b".sha3()))).unwrap();
assert!(jdb.can_reconstruct_refs());
assert!(jdb.exists(&foo));
assert!(!jdb.exists(&baz));
assert!(!jdb.exists(&bar));
}
}
#[test]
fn insert_older_era() {
let mut jdb = OverlayRecentDB::new_temp();
let foo = jdb.insert(b"foo");
jdb.commit(0, &b"0a".sha3(), None).unwrap();
assert!(jdb.can_reconstruct_refs());
let bar = jdb.insert(b"bar");
jdb.commit(1, &b"1".sha3(), Some((0, b"0a".sha3()))).unwrap();
assert!(jdb.can_reconstruct_refs());
jdb.remove(&bar);
jdb.commit(0, &b"0b".sha3(), None).unwrap();
assert!(jdb.can_reconstruct_refs());
jdb.commit(2, &b"2".sha3(), Some((1, b"1".sha3()))).unwrap();
assert!(jdb.exists(&foo));
assert!(jdb.exists(&bar));
}
}

287
util/src/journaldb/refcounteddb.rs Normal file
View File

@@ -0,0 +1,287 @@
// Copyright 2015, 2016 Ethcore (UK) Ltd.
// This file is part of Parity.
// Parity is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// Parity is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
// You should have received a copy of the GNU General Public License
// along with Parity. If not, see <http://www.gnu.org/licenses/>.
//! Disk-backed, ref-counted JournalDB implementation.
use common::*;
use rlp::*;
use hashdb::*;
use overlaydb::*;
use super::traits::JournalDB;
use kvdb::{Database, DBTransaction, DatabaseConfig};
#[cfg(test)]
use std::env;
/// Implementation of the HashDB trait for a disk-backed database with a memory overlay
/// and latent-removal semantics.
///
/// Like OverlayDB, there is a memory overlay; `commit()` must be called in order to
/// write operations out to disk. Unlike OverlayDB, `remove()` operations do not take effect
/// immediately. Rather some age (based on a linear but arbitrary metric) must pass before
/// the removals actually take effect.
pub struct RefCountedDB {
forward: OverlayDB,
backing: Arc<Database>,
latest_era: Option<u64>,
inserts: Vec<H256>,
removes: Vec<H256>,
}
const LATEST_ERA_KEY : [u8; 12] = [ b'l', b'a', b's', b't', 0, 0, 0, 0, 0, 0, 0, 0 ];
const VERSION_KEY : [u8; 12] = [ b'j', b'v', b'e', b'r', 0, 0, 0, 0, 0, 0, 0, 0 ];
const DB_VERSION : u32 = 512;
const PADDING : [u8; 10] = [ 0u8; 10 ];
impl RefCountedDB {
/// Create a new instance given a `backing` database.
pub fn new(path: &str) -> RefCountedDB {
let opts = DatabaseConfig {
prefix_size: Some(12) //use 12 bytes as prefix, this must match account_db prefix
};
let backing = Database::open(&opts, path).unwrap_or_else(|e| {
panic!("Error opening state db: {}", e);
});
if !backing.is_empty() {
match backing.get(&VERSION_KEY).map(|d| d.map(|v| decode::<u32>(&v))) {
Ok(Some(DB_VERSION)) => {},
v => panic!("Incompatible DB version, expected {}, got {:?}", DB_VERSION, v)
}
} else {
backing.put(&VERSION_KEY, &encode(&DB_VERSION)).expect("Error writing version to database");
}
let backing = Arc::new(backing);
let latest_era = backing.get(&LATEST_ERA_KEY).expect("Low-level database error.").map(|val| decode::<u64>(&val));
RefCountedDB {
forward: OverlayDB::new_with_arc(backing.clone()),
backing: backing,
inserts: vec![],
removes: vec![],
latest_era: latest_era,
}
}
/// Create a new instance with an anonymous temporary database.
#[cfg(test)]
fn new_temp() -> RefCountedDB {
let mut dir = env::temp_dir();
dir.push(H32::random().hex());
Self::new(dir.to_str().unwrap())
}
}
impl HashDB for RefCountedDB {
fn keys(&self) -> HashMap<H256, i32> { self.forward.keys() }
fn lookup(&self, key: &H256) -> Option<&[u8]> { self.forward.lookup(key) }
fn exists(&self, key: &H256) -> bool { self.forward.exists(key) }
fn insert(&mut self, value: &[u8]) -> H256 { let r = self.forward.insert(value); self.inserts.push(r.clone()); r }
fn emplace(&mut self, key: H256, value: Bytes) { self.inserts.push(key.clone()); self.forward.emplace(key, value); }
fn kill(&mut self, key: &H256) { self.removes.push(key.clone()); }
}
impl JournalDB for RefCountedDB {
fn spawn(&self) -> Box<JournalDB> {
Box::new(RefCountedDB {
forward: self.forward.clone(),
backing: self.backing.clone(),
latest_era: self.latest_era,
inserts: self.inserts.clone(),
removes: self.removes.clone(),
})
}
fn mem_used(&self) -> usize {
self.inserts.heap_size_of_children() + self.removes.heap_size_of_children()
}
fn is_empty(&self) -> bool {
self.latest_era.is_none()
}
fn commit(&mut self, now: u64, id: &H256, end: Option<(u64, H256)>) -> Result<u32, UtilError> {
// journal format:
// [era, 0] => [ id, [insert_0, ...], [remove_0, ...] ]
// [era, 1] => [ id, [insert_0, ...], [remove_0, ...] ]
// [era, n] => [ ... ]
// TODO: store last_era, reclaim_period.
// when we make a new commit, we journal the inserts and removes.
// for each end_era that we journaled that we are no passing by,
// we remove all of its removes assuming it is canonical and all
// of its inserts otherwise.
// record new commit's details.
let batch = DBTransaction::new();
{
let mut index = 0usize;
let mut last;
while try!(self.backing.get({
let mut r = RlpStream::new_list(3);
r.append(&now);
r.append(&index);
r.append(&&PADDING[..]);
last = r.drain();
&last
})).is_some() {
index += 1;
}
let mut r = RlpStream::new_list(3);
r.append(id);
r.append(&self.inserts);
r.append(&self.removes);
try!(batch.put(&last, r.as_raw()));
trace!(target: "rcdb", "new journal for time #{}.{} => {}: inserts={:?}, removes={:?}", now, index, id, self.inserts, self.removes);
self.inserts.clear();
self.removes.clear();
if self.latest_era.map_or(true, |e| now > e) {
try!(batch.put(&LATEST_ERA_KEY, &encode(&now)));
self.latest_era = Some(now);
}
}
// apply old commits' details
if let Some((end_era, canon_id)) = end {
let mut index = 0usize;
let mut last;
while let Some(rlp_data) = {
// trace!(target: "rcdb", "checking for journal #{}.{}", end_era, index);
try!(self.backing.get({
let mut r = RlpStream::new_list(3);
r.append(&end_era);
r.append(&index);
r.append(&&PADDING[..]);
last = r.drain();
&last
}))
} {
let rlp = Rlp::new(&rlp_data);
let our_id: H256 = rlp.val_at(0);
let to_remove: Vec<H256> = rlp.val_at(if canon_id == our_id {2} else {1});
trace!(target: "rcdb", "delete journal for time #{}.{}=>{}, (canon was {}): deleting {:?}", end_era, index, our_id, canon_id, to_remove);
for i in &to_remove {
self.forward.remove(i);
}
try!(batch.delete(&last));
index += 1;
}
}
let r = try!(self.forward.commit_to_batch(&batch));
try!(self.backing.write(batch));
Ok(r)
}
}
#[cfg(test)]
mod tests {
#![cfg_attr(feature="dev", allow(blacklisted_name))]
use common::*;
use super::*;
use super::super::traits::JournalDB;
use hashdb::*;
#[test]
fn long_history() {
// history is 3
let mut jdb = RefCountedDB::new_temp();
let h = jdb.insert(b"foo");
jdb.commit(0, &b"0".sha3(), None).unwrap();
assert!(jdb.exists(&h));
jdb.remove(&h);
jdb.commit(1, &b"1".sha3(), None).unwrap();
assert!(jdb.exists(&h));
jdb.commit(2, &b"2".sha3(), None).unwrap();
assert!(jdb.exists(&h));
jdb.commit(3, &b"3".sha3(), Some((0, b"0".sha3()))).unwrap();
assert!(jdb.exists(&h));
jdb.commit(4, &b"4".sha3(), Some((1, b"1".sha3()))).unwrap();
assert!(!jdb.exists(&h));
}
#[test]
fn complex() {
// history is 1
let mut jdb = RefCountedDB::new_temp();
let foo = jdb.insert(b"foo");
let bar = jdb.insert(b"bar");
jdb.commit(0, &b"0".sha3(), None).unwrap();
assert!(jdb.exists(&foo));
assert!(jdb.exists(&bar));
jdb.remove(&foo);
jdb.remove(&bar);
let baz = jdb.insert(b"baz");
jdb.commit(1, &b"1".sha3(), Some((0, b"0".sha3()))).unwrap();
assert!(jdb.exists(&foo));
assert!(jdb.exists(&bar));
assert!(jdb.exists(&baz));
let foo = jdb.insert(b"foo");
jdb.remove(&baz);
jdb.commit(2, &b"2".sha3(), Some((1, b"1".sha3()))).unwrap();
assert!(jdb.exists(&foo));
assert!(!jdb.exists(&bar));
assert!(jdb.exists(&baz));
jdb.remove(&foo);
jdb.commit(3, &b"3".sha3(), Some((2, b"2".sha3()))).unwrap();
assert!(jdb.exists(&foo));
assert!(!jdb.exists(&bar));
assert!(!jdb.exists(&baz));
jdb.commit(4, &b"4".sha3(), Some((3, b"3".sha3()))).unwrap();
assert!(!jdb.exists(&foo));
assert!(!jdb.exists(&bar));
assert!(!jdb.exists(&baz));
}
#[test]
fn fork() {
// history is 1
let mut jdb = RefCountedDB::new_temp();
let foo = jdb.insert(b"foo");
let bar = jdb.insert(b"bar");
jdb.commit(0, &b"0".sha3(), None).unwrap();
assert!(jdb.exists(&foo));
assert!(jdb.exists(&bar));
jdb.remove(&foo);
let baz = jdb.insert(b"baz");
jdb.commit(1, &b"1a".sha3(), Some((0, b"0".sha3()))).unwrap();
jdb.remove(&bar);
jdb.commit(1, &b"1b".sha3(), Some((0, b"0".sha3()))).unwrap();
assert!(jdb.exists(&foo));
assert!(jdb.exists(&bar));
assert!(jdb.exists(&baz));
jdb.commit(2, &b"2b".sha3(), Some((1, b"1b".sha3()))).unwrap();
assert!(jdb.exists(&foo));
assert!(!jdb.exists(&baz));
assert!(!jdb.exists(&bar));
}
}

42
util/src/journaldb/traits.rs Normal file
View File

@@ -0,0 +1,42 @@
// Copyright 2015, 2016 Ethcore (UK) Ltd.
// This file is part of Parity.
// Parity is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// Parity is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
// You should have received a copy of the GNU General Public License
// along with Parity. If not, see <http://www.gnu.org/licenses/>.
//! Disk-backed HashDB implementation.
use common::*;
use hashdb::*;
/// A HashDB which can manage a short-term journal potentially containing many forks of mutually
/// exclusive actions.
pub trait JournalDB : HashDB + Send + Sync {
/// Return a copy of ourself, in a box.
fn spawn(&self) -> Box<JournalDB>;
/// Returns heap memory size used
fn mem_used(&self) -> usize;
/// Check if this database has any commits
fn is_empty(&self) -> bool;
/// Commit all recent insert operations and canonical historical commits' removals from the
/// old era to the backing database, reverting any non-canonical historical commit's inserts.
fn commit(&mut self, now: u64, id: &H256, end: Option<(u64, H256)>) -> Result<u32, UtilError>;
/// State data query
fn state(&self, _id: &H256) -> Option<Bytes> {
None
}
}

244
util/src/keys/directory.rs
View File

@@ -27,7 +27,7 @@ const MAX_CACHE_USAGE_TRACK: usize = 128;
#[derive(PartialEq, Debug, Clone)]
pub enum CryptoCipherType {
/// aes-128-ctr with 128-bit initialisation vector(iv)
Aes128Ctr(U128)
Aes128Ctr(H128)
}
#[derive(PartialEq, Debug, Clone)]
@@ -168,6 +168,8 @@ pub struct KeyFileCrypto {
pub cipher_text: Bytes,
/// Password derived key generator function settings.
pub kdf: KeyFileKdf,
/// Mac
pub mac: H256
}
impl KeyFileCrypto {
@@ -182,7 +184,7 @@ impl KeyFileCrypto {
Some("aes-128-ctr") => CryptoCipherType::Aes128Ctr(
match try!(as_object.get("cipherparams").ok_or(CryptoParseError::NoCipherParameters)).as_object() {
None => { return Err(CryptoParseError::NoCipherParameters); },
Some(cipher_param) => match U128::from_str(match cipher_param["iv"].as_string() {
Some(cipher_param) => match H128::from_str(match cipher_param["iv"].as_string() {
None => { return Err(CryptoParseError::NoInitialVector); },
Some(iv_hex_string) => iv_hex_string
})
@@ -216,22 +218,31 @@ impl KeyFileCrypto {
}
};
let cipher_text = match as_object["ciphertext"].as_string() {
None => { return Err(CryptoParseError::NoCipherText); }
let cipher_text = match try!(as_object.get("ciphertext").ok_or(CryptoParseError::NoCipherText)).as_string() {
None => { return Err(CryptoParseError::InvalidCipherText); }
Some(text) => text
};
let mac: H256 = match try!(as_object.get("mac").ok_or(CryptoParseError::NoMac)).as_string() {
None => { return Err(CryptoParseError::InvalidMacFormat(None)) },
Some(salt_value) => match H256::from_str(salt_value) {
Ok(salt_hex_value) => salt_hex_value,
Err(from_hex_error) => { return Err(CryptoParseError::InvalidMacFormat(Some(from_hex_error))); },
}
};
Ok(KeyFileCrypto {
cipher_text: Bytes::from(cipher_text),
cipher_text: match FromHex::from_hex(cipher_text) { Ok(bytes) => bytes, Err(_) => { return Err(CryptoParseError::InvalidCipherText); } },
cipher_type: cipher_type,
kdf: kdf,
mac: mac,
})
}
fn to_json(&self) -> Json {
let mut map = BTreeMap::new();
match self.cipher_type {
CryptoCipherType::Aes128Ctr(iv) => {
CryptoCipherType::Aes128Ctr(ref iv) => {
map.insert("cipher".to_owned(), Json::String("aes-128-ctr".to_owned()));
let mut cipher_params = BTreeMap::new();
cipher_params.insert("iv".to_owned(), Json::String(format!("{:?}", iv)));
@@ -251,6 +262,8 @@ impl KeyFileCrypto {
KeyFileKdf::Scrypt(ref scrypt_params) => scrypt_params.to_json()
});
map.insert("mac".to_owned(), Json::String(format!("{:?}", self.mac)));
Json::Object(map)
}
@@ -260,7 +273,7 @@ impl KeyFileCrypto {
/// `c` - number of iterations for derived key.
/// `salt` - cryptographic site, random 256-bit hash (ensure it's crypto-random).
/// `iv` - initialisation vector.
pub fn new_pbkdf2(cipher_text: Bytes, iv: U128, salt: H256, c: u32, dk_len: u32) -> KeyFileCrypto {
pub fn new_pbkdf2(cipher_text: Bytes, iv: H128, salt: H256, mac: H256, c: u32, dk_len: u32) -> KeyFileCrypto {
KeyFileCrypto {
cipher_type: CryptoCipherType::Aes128Ctr(iv),
cipher_text: cipher_text,
@@ -270,6 +283,7 @@ impl KeyFileCrypto {
c: c,
prf: Pbkdf2CryptoFunction::HMacSha256
}),
mac: mac,
}
}
}
@@ -291,7 +305,7 @@ fn uuid_to_string(uuid: &Uuid) -> String {
}
fn uuid_from_string(s: &str) -> Result<Uuid, UtilError> {
let parts: Vec<&str> = s.split("-").collect();
let parts: Vec<&str> = s.split('-').collect();
if parts.len() != 5 { return Err(UtilError::BadSize); }
let mut uuid = H128::zero();
@@ -319,19 +333,24 @@ pub struct KeyFileContent {
/// Holds cypher and decrypt function settings.
pub crypto: KeyFileCrypto,
/// The identifier.
pub id: Uuid
pub id: Uuid,
/// Account (if present)
pub account: Option<Address>,
}
#[derive(Debug)]
enum CryptoParseError {
InvalidMacFormat(Option<UtilError>),
NoMac,
NoCipherText,
InvalidCipherText,
NoCipherType,
InvalidJsonFormat,
InvalidKdfType(Mismatch<String>),
InvalidCipherType(Mismatch<String>),
NoInitialVector,
NoCipherParameters,
InvalidInitialVector(FromHexError),
InvalidInitialVector(UtilError),
NoKdf,
NoKdfType,
Scrypt(ScryptParseError),
@@ -357,7 +376,19 @@ impl KeyFileContent {
KeyFileContent {
id: new_uuid(),
version: KeyFileVersion::V3(3),
crypto: crypto
crypto: crypto,
account: None
}
}
/// Loads key from valid json, returns error and records warning if key is mallformed
pub fn load(json: &Json) -> Result<KeyFileContent, ()> {
match Self::from_json(json) {
Ok(key_file) => Ok(key_file),
Err(e) => {
warn!(target: "sstore", "Error parsing json for key: {:?}", e);
Err(())
}
}
}
@@ -390,6 +421,9 @@ impl KeyFileContent {
Ok(id) => id
};
let account = as_object.get("address").and_then(|json| json.as_string()).and_then(
|account_text| match Address::from_str(account_text) { Ok(account) => Some(account), Err(_) => None });
let crypto = match as_object.get("crypto") {
None => { return Err(KeyFileParseError::NoCryptoSection); }
Some(crypto_json) => match KeyFileCrypto::from_json(crypto_json) {
@@ -401,7 +435,8 @@ impl KeyFileContent {
Ok(KeyFileContent {
version: version,
id: id.clone(),
crypto: crypto
crypto: crypto,
account: account
})
}
@@ -410,6 +445,7 @@ impl KeyFileContent {
map.insert("id".to_owned(), Json::String(uuid_to_string(&self.id)));
map.insert("version".to_owned(), Json::U64(CURRENT_DECLARED_VERSION));
map.insert("crypto".to_owned(), self.crypto.to_json());
if let Some(ref address) = self.account { map.insert("address".to_owned(), Json::String(format!("{:?}", address))); }
Json::Object(map)
}
}
@@ -425,17 +461,17 @@ enum KeyFileLoadError {
pub struct KeyDirectory {
/// Directory path for key management.
path: String,
cache: HashMap<Uuid, KeyFileContent>,
cache_usage: VecDeque<Uuid>,
cache: RwLock<HashMap<Uuid, KeyFileContent>>,
cache_usage: RwLock<VecDeque<Uuid>>,
}
impl KeyDirectory {
/// Initializes new cache directory context with a given `path`
pub fn new(path: &Path) -> KeyDirectory {
KeyDirectory {
cache: HashMap::new(),
cache: RwLock::new(HashMap::new()),
path: path.to_str().expect("Initialized key directory with empty path").to_owned(),
cache_usage: VecDeque::new(),
cache_usage: RwLock::new(VecDeque::new()),
}
}
@@ -448,25 +484,37 @@ impl KeyDirectory {
let json_bytes = json_text.into_bytes();
try!(file.write(&json_bytes));
}
let mut cache = self.cache.write().unwrap();
let id = key_file.id.clone();
self.cache.insert(id.clone(), key_file);
cache.insert(id.clone(), key_file);
Ok(id.clone())
}
/// Returns key given by id if corresponding file exists and no load error occured.
/// Warns if any error occured during the key loading
pub fn get(&mut self, id: &Uuid) -> Option<&KeyFileContent> {
pub fn get(&self, id: &Uuid) -> Option<KeyFileContent> {
let path = self.key_path(id);
self.cache_usage.push_back(id.clone());
Some(self.cache.entry(id.to_owned()).or_insert(
{
let mut usage = self.cache_usage.write().unwrap();
usage.push_back(id.clone());
}
if !self.cache.read().unwrap().contains_key(id) {
match KeyDirectory::load_key(&path) {
Ok(loaded_key) => loaded_key,
Ok(loaded_key) => {
self.cache.write().unwrap().insert(id.to_owned(), loaded_key);
}
Err(error) => {
warn!(target: "sstore", "error loading key {:?}: {:?}", id, error);
return None;
}
}
))
}
// todo: replace with Ref::map when it stabilized to avoid copies
Some(self.cache.read().unwrap().get(id)
.expect("Key should be there, we have just inserted or checked it.")
.clone())
}
/// Returns current path to the directory with keys
@@ -476,29 +524,65 @@ impl KeyDirectory {
/// Removes keys that never been requested during last `MAX_USAGE_TRACK` times
pub fn collect_garbage(&mut self) {
let total_usages = self.cache_usage.len();
let mut cache_usage = self.cache_usage.write().unwrap();
let total_usages = cache_usage.len();
let untracked_usages = max(total_usages as i64 - MAX_CACHE_USAGE_TRACK as i64, 0) as usize;
if untracked_usages > 0 {
self.cache_usage.drain(..untracked_usages);
cache_usage.drain(..untracked_usages);
}
if self.cache.len() <= MAX_CACHE_USAGE_TRACK { return; }
if self.cache.read().unwrap().len() <= MAX_CACHE_USAGE_TRACK { return; }
let uniqs: HashSet<&Uuid> = self.cache_usage.iter().collect();
let mut removes = HashSet::new();
let uniqs: HashSet<&Uuid> = cache_usage.iter().collect();
let removes:Vec<Uuid> = {
let cache = self.cache.read().unwrap();
cache.keys().cloned().filter(|key| !uniqs.contains(key)).collect()
};
if removes.is_empty() { return; }
let mut cache = self.cache.write().unwrap();
for key in removes { cache.remove(&key); }
for key in self.cache.keys() {
if !uniqs.contains(key) {
removes.insert(key.clone());
}
}
for removed_key in removes { self.cache.remove(&removed_key); }
cache.shrink_to_fit();
}
/// Reports how many keys are currently cached.
pub fn cache_size(&self) -> usize {
self.cache.len()
self.cache.read().unwrap().len()
}
/// Removes key file from key directory
pub fn delete(&mut self, id: &Uuid) -> Result<(), ::std::io::Error> {
let path = self.key_path(id);
if !self.cache.read().unwrap().contains_key(id) {
return match fs::remove_file(&path) {
Ok(_) => {
self.cache.write().unwrap().remove(&id);
Ok(())
},
Err(e) => Err(e)
};
}
Ok(())
}
/// Enumerates all keys in the directory
pub fn list(&self) -> Result<Vec<Uuid>, ::std::io::Error> {
let mut result = Vec::new();
for entry in try!(fs::read_dir(&self.path)) {
let entry = try!(entry);
if !try!(fs::metadata(entry.path())).is_dir() {
match entry.file_name().to_str() {
Some(ref name) => {
if let Ok(uuid) = uuid_from_string(name) { result.push(uuid); }
},
None => { continue; }
};
}
}
Ok(result)
}
fn key_path(&self, id: &Uuid) -> PathBuf {
@@ -536,6 +620,8 @@ impl KeyDirectory {
Err(_) => Err(KeyFileLoadError::ParseError(KeyFileParseError::InvalidJson))
}
}
}
@@ -590,7 +676,7 @@ mod file_tests {
}
#[test]
fn can_read_scrypt_krf() {
fn can_read_scrypt_kdf() {
let json = Json::from_str(
r#"
{
@@ -626,6 +712,47 @@ mod file_tests {
}
}
#[test]
fn can_read_scrypt_kdf_params() {
let json = Json::from_str(
r#"
{
"crypto" : {
"cipher" : "aes-128-ctr",
"cipherparams" : {
"iv" : "83dbcc02d8ccb40e466191a123791e0e"
},
"ciphertext" : "d172bf743a674da9cdad04534d56926ef8358534d458fffccd4e6ad2fbde479c",
"kdf" : "scrypt",
"kdfparams" : {
"dklen" : 32,
"n" : 262144,
"r" : 1,
"p" : 8,
"salt" : "ab0c7876052600dd703518d6fc3fe8984592145b591fc8fb5c6d43190334ba19"
},
"mac" : "2103ac29920d71da29f15d75b4a16dbe95cfd7ff8faea1056c33131d846e3097"
},
"id" : "3198bc9c-6672-5ab3-d995-4942343ae5b6",
"version" : 3
}
"#).unwrap();
match KeyFileContent::from_json(&json) {
Ok(key_file) => {
match key_file.crypto.kdf {
KeyFileKdf::Scrypt(scrypt_params) => {
assert_eq!(262144, scrypt_params.n);
assert_eq!(1, scrypt_params.r);
assert_eq!(8, scrypt_params.p);
},
_ => { panic!("expected kdf params of crypto to be of scrypt type" ); }
}
},
Err(e) => panic!("Error parsing valid file: {:?}", e)
}
}
#[test]
fn can_return_error_no_id() {
let json = Json::from_str(
@@ -781,7 +908,7 @@ mod file_tests {
panic!("Should be error of no identifier, got ok");
},
Err(KeyFileParseError::Crypto(CryptoParseError::Scrypt(_))) => { },
Err(other_error) => { panic!("should be error of no identifier, got {:?}", other_error); }
Err(other_error) => { panic!("should be scrypt parse error, got {:?}", other_error); }
}
}
@@ -820,14 +947,14 @@ mod file_tests {
#[test]
fn can_create_key_with_new_id() {
let cipher_text: Bytes = FromHex::from_hex("a0f05555").unwrap();
let key = KeyFileContent::new(KeyFileCrypto::new_pbkdf2(cipher_text, U128::zero(), H256::random(), 32, 32));
let key = KeyFileContent::new(KeyFileCrypto::new_pbkdf2(cipher_text, H128::zero(), H256::random(), H256::random(), 32, 32));
assert!(!uuid_to_string(&key.id).is_empty());
}
#[test]
fn can_load_json_from_itself() {
let cipher_text: Bytes = FromHex::from_hex("aaaaaaaaaaaaaaaaaaaaaaaaaaa22222222222222222222222").unwrap();
let key = KeyFileContent::new(KeyFileCrypto::new_pbkdf2(cipher_text, U128::zero(), H256::random(), 32, 32));
let key = KeyFileContent::new(KeyFileCrypto::new_pbkdf2(cipher_text, H128::zero(), H256::random(), H256::random(), 32, 32));
let json = key.to_json();
let loaded_key = KeyFileContent::from_json(&json).unwrap();
@@ -967,7 +1094,7 @@ mod file_tests {
mod directory_tests {
use super::{KeyDirectory, new_uuid, uuid_to_string, KeyFileContent, KeyFileCrypto, MAX_CACHE_USAGE_TRACK};
use common::*;
use tests::helpers::*;
use devtools::*;
#[test]
fn key_directory_locates_keys() {
@@ -985,7 +1112,7 @@ mod directory_tests {
let cipher_text: Bytes = FromHex::from_hex("a0f05555").unwrap();
let temp_path = RandomTempPath::create_dir();
let mut directory = KeyDirectory::new(&temp_path.as_path());
let uuid = directory.save(KeyFileContent::new(KeyFileCrypto::new_pbkdf2(cipher_text, U128::zero(), H256::random(), 32, 32))).unwrap();
let uuid = directory.save(KeyFileContent::new(KeyFileCrypto::new_pbkdf2(cipher_text, H128::zero(), H256::random(), H256::random(), 32, 32))).unwrap();
let path = directory.key_path(&uuid);
let key = KeyDirectory::load_key(&path).unwrap();
@@ -1001,7 +1128,7 @@ mod directory_tests {
let cipher_text: Bytes = FromHex::from_hex("a0f05555").unwrap();
let mut keys = Vec::new();
for _ in 0..1000 {
let key = KeyFileContent::new(KeyFileCrypto::new_pbkdf2(cipher_text.clone(), U128::zero(), H256::random(), 32, 32));
let key = KeyFileContent::new(KeyFileCrypto::new_pbkdf2(cipher_text.clone(), H128::zero(), H256::random(), H256::random(), 32, 32));
keys.push(directory.save(key).unwrap());
}
@@ -1021,7 +1148,7 @@ mod directory_tests {
let cipher_text: Bytes = FromHex::from_hex("a0f05555").unwrap();
let mut keys = Vec::new();
for _ in 0..1000 {
let key = KeyFileContent::new(KeyFileCrypto::new_pbkdf2(cipher_text.clone(), U128::zero(), H256::random(), 32, 32));
let key = KeyFileContent::new(KeyFileCrypto::new_pbkdf2(cipher_text.clone(), H128::zero(), H256::random(), H256::random(), 32, 32));
keys.push(directory.save(key).unwrap());
}
@@ -1033,13 +1160,21 @@ mod directory_tests {
// since all keys are different, should be exactly MAX_CACHE_USAGE_TRACK
assert_eq!(MAX_CACHE_USAGE_TRACK, directory.cache_size())
}
#[test]
fn collects_garbage_on_empty() {
let temp_path = RandomTempPath::create_dir();
let mut directory = KeyDirectory::new(&temp_path.as_path());
directory.collect_garbage();
assert_eq!(0, directory.cache_size())
}
}
#[cfg(test)]
mod specs {
use super::*;
use common::*;
use tests::helpers::*;
use devtools::*;
#[test]
fn can_initiate_key_directory() {
@@ -1054,7 +1189,7 @@ mod specs {
let temp_path = RandomTempPath::create_dir();
let mut directory = KeyDirectory::new(&temp_path.as_path());
let uuid = directory.save(KeyFileContent::new(KeyFileCrypto::new_pbkdf2(cipher_text, U128::zero(), H256::random(), 32, 32)));
let uuid = directory.save(KeyFileContent::new(KeyFileCrypto::new_pbkdf2(cipher_text, H128::zero(), H256::random(), H256::random(), 32, 32)));
assert!(uuid.is_ok());
}
@@ -1064,7 +1199,7 @@ mod specs {
let cipher_text: Bytes = FromHex::from_hex("a0f05555").unwrap();
let temp_path = RandomTempPath::create_dir();
let mut directory = KeyDirectory::new(&temp_path.as_path());
let uuid = directory.save(KeyFileContent::new(KeyFileCrypto::new_pbkdf2(cipher_text.clone(), U128::zero(), H256::random(), 32, 32))).unwrap();
let uuid = directory.save(KeyFileContent::new(KeyFileCrypto::new_pbkdf2(cipher_text.clone(), H128::zero(), H256::random(), H256::random(), 32, 32))).unwrap();
let key = directory.get(&uuid).unwrap();
@@ -1079,10 +1214,25 @@ mod specs {
let cipher_text: Bytes = FromHex::from_hex("a0f05555").unwrap();
let mut keys = Vec::new();
for _ in 0..10 {
let key = KeyFileContent::new(KeyFileCrypto::new_pbkdf2(cipher_text.clone(), U128::zero(), H256::random(), 32, 32));
let key = KeyFileContent::new(KeyFileCrypto::new_pbkdf2(cipher_text.clone(), H128::zero(), H256::random(), H256::random(), 32, 32));
keys.push(directory.save(key).unwrap());
}
assert_eq!(10, keys.len())
}
#[test]
fn can_list_keys() {
let temp_path = RandomTempPath::create_dir();
let mut directory = KeyDirectory::new(&temp_path.as_path());
let cipher_text: Bytes = FromHex::from_hex("a0f05555").unwrap();
let mut keys = Vec::new();
for _ in 0..33 {
let key = KeyFileContent::new(KeyFileCrypto::new_pbkdf2(cipher_text.clone(), H128::zero(), H256::random(), H256::random(), 32, 32));
keys.push(directory.save(key).unwrap());
}
assert_eq!(33, directory.list().unwrap().len());
}
}

176
util/src/keys/geth_import.rs Normal file
View File

@@ -0,0 +1,176 @@
// Copyright 2015, 2016 Ethcore (UK) Ltd.
// This file is part of Parity.
// Parity is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// Parity is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
// You should have received a copy of the GNU General Public License
// along with Parity. If not, see <http://www.gnu.org/licenses/>.
//! Geth keys import/export tool
use common::*;
use keys::store::SecretStore;
use keys::directory::KeyFileContent;
/// Enumerates all geth keys in the directory and returns collection of tuples `(accountId, filename)`
pub fn enumerate_geth_keys(path: &Path) -> Result<Vec<(Address, String)>, io::Error> {
let mut entries = Vec::new();
for entry in try!(fs::read_dir(path)) {
let entry = try!(entry);
if !try!(fs::metadata(entry.path())).is_dir() {
match entry.file_name().to_str() {
Some(name) => {
let parts: Vec<&str> = name.split("--").collect();
if parts.len() != 3 { continue; }
match Address::from_str(parts[2]) {
Ok(account_id) => { entries.push((account_id, name.to_owned())); }
Err(e) => { panic!("error: {:?}", e); }
}
},
None => { continue; }
};
}
}
Ok(entries)
}
/// Geth import error
#[derive(Debug)]
pub enum ImportError {
/// Io error reading geth file
Io(io::Error),
/// format error
Format,
}
impl From<io::Error> for ImportError {
fn from (err: io::Error) -> ImportError {
ImportError::Io(err)
}
}
/// Imports one geth key to the store
pub fn import_geth_key(secret_store: &mut SecretStore, geth_keyfile_path: &Path) -> Result<(), ImportError> {
let mut file = try!(fs::File::open(geth_keyfile_path));
let mut buf = String::new();
try!(file.read_to_string(&mut buf));
let mut json_result = Json::from_str(&buf);
let mut json = match json_result {
Ok(ref mut parsed_json) => try!(parsed_json.as_object_mut().ok_or(ImportError::Format)),
Err(_) => { return Err(ImportError::Format); }
};
let crypto_object = try!(json.get("Crypto").and_then(|crypto| crypto.as_object()).ok_or(ImportError::Format)).clone();
json.insert("crypto".to_owned(), Json::Object(crypto_object));
json.remove("Crypto");
match KeyFileContent::load(&Json::Object(json.clone())) {
Ok(key_file) => try!(secret_store.import_key(key_file)),
Err(_) => { return Err(ImportError::Format); }
};
Ok(())
}
/// Imports all geth keys in the directory
pub fn import_geth_keys(secret_store: &mut SecretStore, geth_keyfiles_directory: &Path) -> Result<(), ImportError> {
use std::path::PathBuf;
let geth_files = try!(enumerate_geth_keys(geth_keyfiles_directory));
for &(ref address, ref file_path) in &geth_files {
let mut path = PathBuf::new();
path.push(geth_keyfiles_directory);
path.push(file_path);
if let Err(e) = import_geth_key(secret_store, Path::new(&path)) {
warn!("Skipped geth address {}, error importing: {:?}", address, e)
}
}
Ok(())
}
#[cfg(test)]
mod tests {
use super::*;
use common::*;
use keys::store::SecretStore;
fn test_path() -> &'static str {
match ::std::fs::metadata("res") {
Ok(_) => "res/geth_keystore",
Err(_) => "util/res/geth_keystore"
}
}
fn test_path_param(param_val: &'static str) -> String {
test_path().to_owned() + param_val
}
#[test]
fn can_enumerate() {
let keys = enumerate_geth_keys(Path::new(test_path())).unwrap();
assert_eq!(2, keys.len());
}
#[test]
fn can_import() {
let temp = ::devtools::RandomTempPath::create_dir();
let mut secret_store = SecretStore::new_in(temp.as_path());
import_geth_key(&mut secret_store, Path::new(&test_path_param("/UTC--2016-02-17T09-20-45.721400158Z--3f49624084b67849c7b4e805c5988c21a430f9d9"))).unwrap();
let key = secret_store.account(&Address::from_str("3f49624084b67849c7b4e805c5988c21a430f9d9").unwrap());
assert!(key.is_some());
}
#[test]
fn can_import_directory() {
let temp = ::devtools::RandomTempPath::create_dir();
let mut secret_store = SecretStore::new_in(temp.as_path());
import_geth_keys(&mut secret_store, Path::new(test_path())).unwrap();
let key = secret_store.account(&Address::from_str("3f49624084b67849c7b4e805c5988c21a430f9d9").unwrap());
assert!(key.is_some());
let key = secret_store.account(&Address::from_str("5ba4dcf897e97c2bdf8315b9ef26c13c085988cf").unwrap());
assert!(key.is_some());
}
#[test]
fn imports_as_scrypt_keys() {
use keys::directory::{KeyDirectory, KeyFileKdf};
let temp = ::devtools::RandomTempPath::create_dir();
{
let mut secret_store = SecretStore::new_in(temp.as_path());
import_geth_keys(&mut secret_store, Path::new(test_path())).unwrap();
}
let key_directory = KeyDirectory::new(&temp.as_path());
let key_file = key_directory.get(&H128::from_str("62a0ad73556d496a8e1c0783d30d3ace").unwrap()).unwrap();
match key_file.crypto.kdf {
KeyFileKdf::Scrypt(scrypt_params) => {
assert_eq!(262144, scrypt_params.n);
assert_eq!(8, scrypt_params.r);
assert_eq!(1, scrypt_params.p);
},
_ => { panic!("expected kdf params of crypto to be of scrypt type" ); }
}
}
#[test]
#[cfg(feature="heavy-tests")]
fn can_decrypt_with_imported() {
use keys::store::EncryptedHashMap;
let temp = ::devtools::RandomTempPath::create_dir();
let mut secret_store = SecretStore::new_in(temp.as_path());
import_geth_keys(&mut secret_store, Path::new(test_path())).unwrap();
let val = secret_store.get::<Bytes>(&H128::from_str("62a0ad73556d496a8e1c0783d30d3ace").unwrap(), "123");
assert!(val.is_ok());
assert_eq!(32, val.unwrap().len());
}
}

2
util/src/keys/mod.rs
View File

@@ -17,3 +17,5 @@
//! Key management module
pub mod directory;
pub mod store;
mod geth_import;

639
util/src/keys/store.rs Normal file
View File

@@ -0,0 +1,639 @@
// Copyright 2015, 2016 Ethcore (UK) Ltd.
// This file is part of Parity.
// Parity is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// Parity is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
// You should have received a copy of the GNU General Public License
// along with Parity. If not, see <http://www.gnu.org/licenses/>.
//! Secret Store
use keys::directory::*;
use common::*;
use rcrypto::pbkdf2::*;
use rcrypto::scrypt::*;
use rcrypto::hmac::*;
use crypto;
use chrono::*;
const KEY_LENGTH: u32 = 32;
const KEY_ITERATIONS: u32 = 10240;
const KEY_LENGTH_AES: u32 = KEY_LENGTH/2;
const KEY_LENGTH_USIZE: usize = KEY_LENGTH as usize;
const KEY_LENGTH_AES_USIZE: usize = KEY_LENGTH_AES as usize;
/// Encrypted hash-map, each request should contain password
pub trait EncryptedHashMap<Key: Hash + Eq> {
/// Returns existing value for the key, if any
fn get<Value: FromRawBytes + BytesConvertable>(&self, key: &Key, password: &str) -> Result<Value, EncryptedHashMapError>;
/// Insert new encrypted key-value and returns previous if there was any
fn insert<Value: FromRawBytes + BytesConvertable>(&mut self, key: Key, value: Value, password: &str) -> Option<Value>;
/// Removes key-value by key and returns the removed one, if any exists and password was provided
fn remove<Value: FromRawBytes + BytesConvertable> (&mut self, key: &Key, password: Option<&str>) -> Option<Value>;
/// Deletes key-value by key and returns if the key-value existed
fn delete(&mut self, key: &Key) -> bool {
self.remove::<Bytes>(key, None).is_some()
}
}
/// Error retrieving value from encrypted hashmap
#[derive(Debug)]
pub enum EncryptedHashMapError {
/// Encryption failed
InvalidPassword,
/// No key in the hashmap
UnknownIdentifier,
/// Stored value is not well formed for the requested type
InvalidValueFormat(FromBytesError),
}
/// Error retrieving value from encrypted hashmap
#[derive(Debug)]
pub enum SigningError {
/// Account passed does not exist
NoAccount,
/// Account passed is not unlocked
AccountNotUnlocked,
/// Invalid secret in store
InvalidSecret
}
/// Represent service for storing encrypted arbitrary data
pub struct SecretStore {
directory: KeyDirectory,
unlocks: RwLock<HashMap<Address, AccountUnlock>>,
}
struct AccountUnlock {
secret: H256,
expires: DateTime<UTC>,
}
/// Basic account management trait
pub trait AccountProvider : Send + Sync {
/// Lists all accounts
fn accounts(&self) -> Result<Vec<Address>, ::std::io::Error>;
/// Unlocks account with the password provided
fn unlock_account(&self, account: &Address, pass: &str) -> Result<(), EncryptedHashMapError>;
/// Creates account
fn new_account(&self, pass: &str) -> Result<Address, ::std::io::Error>;
/// Returns secret for unlocked account
fn account_secret(&self, account: &Address) -> Result<crypto::Secret, SigningError>;
/// Returns secret for unlocked account
fn sign(&self, account: &Address, message: &H256) -> Result<crypto::Signature, SigningError>;
}
/// Thread-safe accounts management
pub struct AccountService {
secret_store: RwLock<SecretStore>,
}
impl AccountProvider for AccountService {
/// Lists all accounts
fn accounts(&self) -> Result<Vec<Address>, ::std::io::Error> {
Ok(try!(self.secret_store.read().unwrap().accounts()).iter().map(|&(addr, _)| addr).collect::<Vec<Address>>())
}
/// Unlocks account with the password provided
fn unlock_account(&self, account: &Address, pass: &str) -> Result<(), EncryptedHashMapError> {
self.secret_store.read().unwrap().unlock_account(account, pass)
}
/// Creates account
fn new_account(&self, pass: &str) -> Result<Address, ::std::io::Error> {
self.secret_store.write().unwrap().new_account(pass)
}
/// Returns secret for unlocked account
fn account_secret(&self, account: &Address) -> Result<crypto::Secret, SigningError> {
self.secret_store.read().unwrap().account_secret(account)
}
/// Returns secret for unlocked account
fn sign(&self, account: &Address, message: &H256) -> Result<crypto::Signature, SigningError> {
self.secret_store.read().unwrap().sign(account, message)
}
}
impl Default for AccountService {
fn default() -> Self {
AccountService::new()
}
}
impl AccountService {
/// New account service with the default location
pub fn new() -> Self {
let secret_store = RwLock::new(SecretStore::new());
secret_store.write().unwrap().try_import_existing();
AccountService {
secret_store: secret_store
}
}
#[cfg(test)]
fn new_test(temp: &::devtools::RandomTempPath) -> Self {
let secret_store = RwLock::new(SecretStore::new_test(temp));
AccountService {
secret_store: secret_store
}
}
/// Ticks the account service
pub fn tick(&self) {
self.secret_store.write().unwrap().collect_garbage();
}
}
impl Default for SecretStore {
fn default() -> Self {
SecretStore::new()
}
}
impl SecretStore {
/// new instance of Secret Store in default home directory
pub fn new() -> Self {
let mut path = ::std::env::home_dir().expect("Failed to get home dir");
path.push(".parity");
path.push("keys");
::std::fs::create_dir_all(&path).expect("Should panic since it is critical to be able to access home dir");
Self::new_in(&path)
}
/// new instance of Secret Store in specific directory
pub fn new_in(path: &Path) -> Self {
SecretStore {
directory: KeyDirectory::new(path),
unlocks: RwLock::new(HashMap::new()),
}
}
/// trys to import keys in the known locations
pub fn try_import_existing(&mut self) {
use std::path::PathBuf;
use keys::geth_import;
let mut import_path = PathBuf::new();
import_path.push(::std::env::home_dir().expect("Failed to get home dir"));
import_path.push(".ethereum");
import_path.push("keystore");
if let Err(e) = geth_import::import_geth_keys(self, &import_path) {
trace!(target: "sstore", "Geth key not imported: {:?}", e);
}
}
/// Lists all accounts and corresponding key ids
pub fn accounts(&self) -> Result<Vec<(Address, H128)>, ::std::io::Error> {
let accounts = try!(self.directory.list()).iter().map(|key_id| self.directory.get(key_id))
.filter(|key| key.is_some())
.map(|key| { let some_key = key.unwrap(); (some_key.account, some_key.id) })
.filter(|&(ref account, _)| account.is_some())
.map(|(account, id)| (account.unwrap(), id))
.collect::<Vec<(Address, H128)>>();
Ok(accounts)
}
/// Resolves key_id by account address
pub fn account(&self, account: &Address) -> Option<H128> {
let mut accounts = match self.accounts() {
Ok(accounts) => accounts,
Err(e) => { warn!(target: "sstore", "Failed to load accounts: {}", e); return None; }
};
accounts.retain(|&(ref store_account, _)| account == store_account);
accounts.first().and_then(|&(_, ref key_id)| Some(key_id.clone()))
}
/// Imports pregenerated key, returns error if not saved correctly
pub fn import_key(&mut self, key_file: KeyFileContent) -> Result<(), ::std::io::Error> {
try!(self.directory.save(key_file));
Ok(())
}
#[cfg(test)]
fn new_test(path: &::devtools::RandomTempPath) -> SecretStore {
SecretStore {
directory: KeyDirectory::new(path.as_path()),
unlocks: RwLock::new(HashMap::new()),
}
}
/// Unlocks account for use
pub fn unlock_account(&self, account: &Address, pass: &str) -> Result<(), EncryptedHashMapError> {
let secret_id = try!(self.account(&account).ok_or(EncryptedHashMapError::UnknownIdentifier));
let secret = try!(self.get(&secret_id, pass));
{
let mut write_lock = self.unlocks.write().unwrap();
let mut unlock = write_lock.entry(*account)
.or_insert_with(|| AccountUnlock { secret: secret, expires: UTC::now() });
unlock.secret = secret;
unlock.expires = UTC::now() + Duration::minutes(20);
}
Ok(())
}
/// Creates new account
pub fn new_account(&mut self, pass: &str) -> Result<Address, ::std::io::Error> {
let key_pair = crypto::KeyPair::create().expect("Error creating key-pair. Something wrong with crypto libraries?");
let address = Address::from(key_pair.public().sha3());
let key_id = H128::random();
self.insert(key_id.clone(), key_pair.secret().clone(), pass);
let mut key_file = self.directory.get(&key_id).expect("the key was just inserted");
key_file.account = Some(address);
try!(self.directory.save(key_file));
Ok(address)
}
/// Signs message with unlocked account
pub fn sign(&self, account: &Address, message: &H256) -> Result<crypto::Signature, SigningError> {
let read_lock = self.unlocks.read().unwrap();
let unlock = try!(read_lock.get(account).ok_or(SigningError::AccountNotUnlocked));
match crypto::KeyPair::from_secret(unlock.secret) {
Ok(pair) => match pair.sign(message) {
Ok(signature) => Ok(signature),
Err(_) => Err(SigningError::InvalidSecret)
},
Err(_) => Err(SigningError::InvalidSecret)
}
}
/// Returns secret for unlocked account
pub fn account_secret(&self, account: &Address) -> Result<crypto::Secret, SigningError> {
let read_lock = self.unlocks.read().unwrap();
let unlock = try!(read_lock.get(account).ok_or(SigningError::AccountNotUnlocked));
Ok(unlock.secret as crypto::Secret)
}
/// Makes account unlocks expire and removes unused key files from memory
pub fn collect_garbage(&mut self) {
let mut garbage_lock = self.unlocks.write().unwrap();
self.directory.collect_garbage();
let utc = UTC::now();
let expired_addresses = garbage_lock.iter()
.filter(|&(_, unlock)| unlock.expires < utc)
.map(|(address, _)| address.clone()).collect::<Vec<Address>>();
for expired in expired_addresses { garbage_lock.remove(&expired); }
garbage_lock.shrink_to_fit();
}
}
fn derive_key_iterations(password: &str, salt: &H256, c: u32) -> (Bytes, Bytes) {
let mut h_mac = Hmac::new(::rcrypto::sha2::Sha256::new(), password.as_bytes());
let mut derived_key = vec![0u8; KEY_LENGTH_USIZE];
pbkdf2(&mut h_mac, &salt.as_slice(), c, &mut derived_key);
let derived_right_bits = &derived_key[0..KEY_LENGTH_AES_USIZE];
let derived_left_bits = &derived_key[KEY_LENGTH_AES_USIZE..KEY_LENGTH_USIZE];
(derived_right_bits.to_vec(), derived_left_bits.to_vec())
}
fn derive_key(password: &str, salt: &H256) -> (Bytes, Bytes) {
derive_key_iterations(password, salt, KEY_ITERATIONS)
}
fn derive_key_scrypt(password: &str, salt: &H256, n: u32, p: u32, r: u32) -> (Bytes, Bytes) {
let mut derived_key = vec![0u8; KEY_LENGTH_USIZE];
let scrypt_params = ScryptParams::new(n.trailing_zeros() as u8, r, p);
scrypt(password.as_bytes(), &salt.as_slice(), &scrypt_params, &mut derived_key);
let derived_right_bits = &derived_key[0..KEY_LENGTH_AES_USIZE];
let derived_left_bits = &derived_key[KEY_LENGTH_AES_USIZE..KEY_LENGTH_USIZE];
(derived_right_bits.to_vec(), derived_left_bits.to_vec())
}
fn derive_mac(derived_left_bits: &[u8], cipher_text: &[u8]) -> Bytes {
let mut mac = vec![0u8; KEY_LENGTH_AES_USIZE + cipher_text.len()];
mac[0..KEY_LENGTH_AES_USIZE].clone_from_slice(derived_left_bits);
mac[KEY_LENGTH_AES_USIZE..cipher_text.len()+KEY_LENGTH_AES_USIZE].clone_from_slice(cipher_text);
mac
}
impl EncryptedHashMap<H128> for SecretStore {
fn get<Value: FromRawBytes + BytesConvertable>(&self, key: &H128, password: &str) -> Result<Value, EncryptedHashMapError> {
match self.directory.get(key) {
Some(key_file) => {
let (derived_left_bits, derived_right_bits) = match key_file.crypto.kdf {
KeyFileKdf::Pbkdf2(ref params) => derive_key_iterations(password, &params.salt, params.c),
KeyFileKdf::Scrypt(ref params) => derive_key_scrypt(password, &params.salt, params.n, params.p, params.r)
};
if derive_mac(&derived_right_bits, &key_file.crypto.cipher_text)
.sha3() != key_file.crypto.mac { return Err(EncryptedHashMapError::InvalidPassword); }
let mut val = vec![0u8; key_file.crypto.cipher_text.len()];
match key_file.crypto.cipher_type {
CryptoCipherType::Aes128Ctr(ref iv) => {
crypto::aes::decrypt(&derived_left_bits, &iv.as_slice(), &key_file.crypto.cipher_text, &mut val);
}
};
match Value::from_bytes(&val) {
Ok(value) => Ok(value),
Err(bytes_error) => Err(EncryptedHashMapError::InvalidValueFormat(bytes_error))
}
},
None => Err(EncryptedHashMapError::UnknownIdentifier)
}
}
fn insert<Value: FromRawBytes + BytesConvertable>(&mut self, key: H128, value: Value, password: &str) -> Option<Value> {
let previous = if let Ok(previous_value) = self.get(&key, password) { Some(previous_value) } else { None };
// crypto random initiators
let salt = H256::random();
let iv = H128::random();
// two parts of derived key
// DK = [ DK[0..15] DK[16..31] ] = [derived_left_bits, derived_right_bits]
let (derived_left_bits, derived_right_bits) = derive_key(password, &salt);
let mut cipher_text = vec![0u8; value.as_slice().len()];
// aes-128-ctr with initial vector of iv
crypto::aes::encrypt(&derived_left_bits, &iv.clone(), &value.as_slice(), &mut cipher_text);
// KECCAK(DK[16..31] ++ <ciphertext>), where DK[16..31] - derived_right_bits
let mac = derive_mac(&derived_right_bits, &cipher_text.clone()).sha3();
let mut key_file = KeyFileContent::new(
KeyFileCrypto::new_pbkdf2(
cipher_text,
iv,
salt,
mac,
KEY_ITERATIONS,
KEY_LENGTH));
key_file.id = key;
if let Err(io_error) = self.directory.save(key_file) {
warn!("Error saving key file: {:?}", io_error);
}
previous
}
fn remove<Value: FromRawBytes + BytesConvertable>(&mut self, key: &H128, password: Option<&str>) -> Option<Value> {
let previous = if let Some(pass) = password {
if let Ok(previous_value) = self.get(&key, pass) { Some(previous_value) } else { None }
}
else { None };
if let Err(io_error) = self.directory.delete(key) {
warn!("Error saving key file: {:?}", io_error);
}
previous
}
}
#[cfg(all(test, feature="heavy-tests"))]
mod vector_tests {
use super::{derive_mac,derive_key_iterations};
use common::*;
#[test]
fn mac_vector() {
let password = "testpassword";
let salt = H256::from_str("ae3cd4e7013836a3df6bd7241b12db061dbe2c6785853cce422d148a624ce0bd").unwrap();
let cipher_text = FromHex::from_hex("5318b4d5bcd28de64ee5559e671353e16f075ecae9f99c7a79a38af5f869aa46").unwrap();
let iterations = 262144u32;
let (derived_left_bits, derived_right_bits) = derive_key_iterations(password, &salt, iterations);
assert_eq!("f06d69cdc7da0faffb1008270bca38f5", derived_left_bits.to_hex());
assert_eq!("e31891a3a773950e6d0fea48a7188551", derived_right_bits.to_hex());
let mac_body = derive_mac(&derived_right_bits, &cipher_text);
assert_eq!("e31891a3a773950e6d0fea48a71885515318b4d5bcd28de64ee5559e671353e16f075ecae9f99c7a79a38af5f869aa46", mac_body.to_hex());
let mac = mac_body.sha3();
assert_eq!("517ead924a9d0dc3124507e3393d175ce3ff7c1e96529c6c555ce9e51205e9b2", format!("{:?}", mac));
}
}
#[cfg(test)]
mod tests {
use super::*;
use devtools::*;
use common::*;
use crypto::KeyPair;
use chrono::*;
#[test]
fn can_insert() {
let temp = RandomTempPath::create_dir();
let mut sstore = SecretStore::new_test(&temp);
let id = H128::random();
sstore.insert(id.clone(), "Cat".to_owned(), "pass");
assert!(sstore.get::<String>(&id, "pass").is_ok());
}
#[test]
fn can_get_fail() {
let temp = RandomTempPath::create_dir();
{
use keys::directory::{KeyFileContent, KeyFileCrypto};
let mut write_sstore = SecretStore::new_test(&temp);
write_sstore.directory.save(
KeyFileContent::new(
KeyFileCrypto::new_pbkdf2(
FromHex::from_hex("5318b4d5bcd28de64ee5559e671353e16f075ecae9f99c7a79a38af5f869aa46").unwrap(),
H128::from_str("6087dab2f9fdbbfaddc31a909735c1e6").unwrap(),
H256::from_str("ae3cd4e7013836a3df6bd7241b12db061dbe2c6785853cce422d148a624ce0bd").unwrap(),
H256::from_str("517ead924a9d0dc3124507e3393d175ce3ff7c1e96529c6c555ce9e51205e9b2").unwrap(),
262144,
32)))
.unwrap();
}
let sstore = SecretStore::new_test(&temp);
if let Ok(_) = sstore.get::<Bytes>(&H128::from_str("3198bc9c66725ab3d9954942343ae5b6").unwrap(), "testpassword") {
panic!("should be error loading key, we requested the wrong key");
}
}
fn pregenerate_keys(temp: &RandomTempPath, count: usize) -> Vec<H128> {
use keys::directory::{KeyFileContent, KeyFileCrypto};
let mut write_sstore = SecretStore::new_test(&temp);
let mut result = Vec::new();
for _ in 0..count {
result.push(write_sstore.directory.save(
KeyFileContent::new(
KeyFileCrypto::new_pbkdf2(
FromHex::from_hex("5318b4d5bcd28de64ee5559e671353e16f075ecae9f99c7a79a38af5f869aa46").unwrap(),
H128::from_str("6087dab2f9fdbbfaddc31a909735c1e6").unwrap(),
H256::from_str("ae3cd4e7013836a3df6bd7241b12db061dbe2c6785853cce422d148a624ce0bd").unwrap(),
H256::from_str("517ead924a9d0dc3124507e3393d175ce3ff7c1e96529c6c555ce9e51205e9b2").unwrap(),
262144,
32)))
.unwrap());
}
result
}
fn pregenerate_accounts(temp: &RandomTempPath, count: usize) -> Vec<H128> {
use keys::directory::{KeyFileContent, KeyFileCrypto};
let mut write_sstore = SecretStore::new_test(&temp);
let mut result = Vec::new();
for i in 0..count {
let mut key_file =
KeyFileContent::new(
KeyFileCrypto::new_pbkdf2(
FromHex::from_hex("5318b4d5bcd28de64ee5559e671353e16f075ecae9f99c7a79a38af5f869aa46").unwrap(),
H128::from_str("6087dab2f9fdbbfaddc31a909735c1e6").unwrap(),
H256::from_str("ae3cd4e7013836a3df6bd7241b12db061dbe2c6785853cce422d148a624ce0bd").unwrap(),
H256::from_str("517ead924a9d0dc3124507e3393d175ce3ff7c1e96529c6c555ce9e51205e9b2").unwrap(),
262144,
32));
key_file.account = Some(x!(i as u64));
result.push(key_file.id.clone());
write_sstore.import_key(key_file).unwrap();
}
result
}
#[test]
#[cfg(feature="heavy-tests")]
fn can_get() {
let temp = RandomTempPath::create_dir();
let key_id = {
use keys::directory::{KeyFileContent, KeyFileCrypto};
let mut write_sstore = SecretStore::new_test(&temp);
write_sstore.directory.save(
KeyFileContent::new(
KeyFileCrypto::new_pbkdf2(
FromHex::from_hex("5318b4d5bcd28de64ee5559e671353e16f075ecae9f99c7a79a38af5f869aa46").unwrap(),
H128::from_str("6087dab2f9fdbbfaddc31a909735c1e6").unwrap(),
H256::from_str("ae3cd4e7013836a3df6bd7241b12db061dbe2c6785853cce422d148a624ce0bd").unwrap(),
H256::from_str("517ead924a9d0dc3124507e3393d175ce3ff7c1e96529c6c555ce9e51205e9b2").unwrap(),
262144,
32)))
.unwrap()
};
let sstore = SecretStore::new_test(&temp);
if let Err(e) = sstore.get::<Bytes>(&key_id, "testpassword") {
panic!("got no key: {:?}", e);
}
}
#[test]
fn can_delete() {
let temp = RandomTempPath::create_dir();
let keys = pregenerate_keys(&temp, 5);
let mut sstore = SecretStore::new_test(&temp);
sstore.delete(&keys[2]);
assert_eq!(4, sstore.directory.list().unwrap().len())
}
#[test]
fn can_create_account() {
let temp = RandomTempPath::create_dir();
let mut sstore = SecretStore::new_test(&temp);
sstore.new_account("123").unwrap();
assert_eq!(1, sstore.accounts().unwrap().len());
}
#[test]
fn can_unlock_account() {
let temp = RandomTempPath::create_dir();
let mut sstore = SecretStore::new_test(&temp);
let address = sstore.new_account("123").unwrap();
let secret = sstore.unlock_account(&address, "123");
assert!(secret.is_ok());
}
#[test]
fn can_sign_data() {
let temp = RandomTempPath::create_dir();
let address = {
let mut sstore = SecretStore::new_test(&temp);
sstore.new_account("334").unwrap()
};
let signature = {
let sstore = SecretStore::new_test(&temp);
sstore.unlock_account(&address, "334").unwrap();
sstore.sign(&address, &H256::random()).unwrap()
};
assert!(signature != x!(0));
}
#[test]
fn can_import_account() {
use keys::directory::{KeyFileContent, KeyFileCrypto};
let temp = RandomTempPath::create_dir();
let mut key_file =
KeyFileContent::new(
KeyFileCrypto::new_pbkdf2(
FromHex::from_hex("5318b4d5bcd28de64ee5559e671353e16f075ecae9f99c7a79a38af5f869aa46").unwrap(),
H128::from_str("6087dab2f9fdbbfaddc31a909735c1e6").unwrap(),
H256::from_str("ae3cd4e7013836a3df6bd7241b12db061dbe2c6785853cce422d148a624ce0bd").unwrap(),
H256::from_str("517ead924a9d0dc3124507e3393d175ce3ff7c1e96529c6c555ce9e51205e9b2").unwrap(),
262144,
32));
key_file.account = Some(Address::from_str("3f49624084b67849c7b4e805c5988c21a430f9d9").unwrap());
let mut sstore = SecretStore::new_test(&temp);
sstore.import_key(key_file).unwrap();
assert_eq!(1, sstore.accounts().unwrap().len());
assert!(sstore.account(&Address::from_str("3f49624084b67849c7b4e805c5988c21a430f9d9").unwrap()).is_some());
}
#[test]
fn can_list_accounts() {
let temp = RandomTempPath::create_dir();
pregenerate_accounts(&temp, 30);
let sstore = SecretStore::new_test(&temp);
let accounts = sstore.accounts().unwrap();
assert_eq!(30, accounts.len());
}
#[test]
fn validate_generated_addresses() {
let temp = RandomTempPath::create_dir();
let mut sstore = SecretStore::new_test(&temp);
let addr = sstore.new_account("test").unwrap();
sstore.unlock_account(&addr, "test").unwrap();
let secret = sstore.account_secret(&addr).unwrap();
let kp = KeyPair::from_secret(secret).unwrap();
assert_eq!(Address::from(kp.public().sha3()), addr);
}
#[test]
fn can_create_service() {
let temp = RandomTempPath::create_dir();
let svc = AccountService::new_test(&temp);
assert!(svc.accounts().unwrap().is_empty());
}
#[test]
fn accounts_expire() {
use std::collections::hash_map::*;
let temp = RandomTempPath::create_dir();
let svc = AccountService::new_test(&temp);
let address = svc.new_account("pass").unwrap();
svc.unlock_account(&address, "pass").unwrap();
assert!(svc.account_secret(&address).is_ok());
{
let ss_rw = svc.secret_store.write().unwrap();
let mut ua_rw = ss_rw.unlocks.write().unwrap();
let entry = ua_rw.entry(address);
if let Entry::Occupied(mut occupied) = entry { occupied.get_mut().expires = UTC::now() - Duration::minutes(1); }
}
svc.tick();
assert!(svc.account_secret(&address).is_err());
}
}

212
util/src/kvdb.rs Normal file
View File

@@ -0,0 +1,212 @@
// Copyright 2015, 2016 Ethcore (UK) Ltd.
// This file is part of Parity.
// Parity is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// Parity is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
// You should have received a copy of the GNU General Public License
// along with Parity. If not, see <http://www.gnu.org/licenses/>.
//! Key-Value store abstraction with RocksDB backend.
use std::default::Default;
use rocksdb::{DB, Writable, WriteBatch, IteratorMode, DBVector, DBIterator,
IndexType, Options, DBCompactionStyle, BlockBasedOptions, Direction};
/// Write transaction. Batches a sequence of put/delete operations for efficiency.
pub struct DBTransaction {
batch: WriteBatch,
}
impl Default for DBTransaction {
fn default() -> Self {
DBTransaction::new()
}
}
impl DBTransaction {
/// Create new transaction.
pub fn new() -> DBTransaction {
DBTransaction { batch: WriteBatch::new() }
}
/// Insert a key-value pair in the transaction. Any existing value value will be overwritten upon write.
pub fn put(&self, key: &[u8], value: &[u8]) -> Result<(), String> {
self.batch.put(key, value)
}
/// Delete value by key.
pub fn delete(&self, key: &[u8]) -> Result<(), String> {
self.batch.delete(key)
}
}
/// Database configuration
pub struct DatabaseConfig {
/// Optional prefix size in bytes. Allows lookup by partial key.
pub prefix_size: Option<usize>
}
/// Database iterator
pub struct DatabaseIterator<'a> {
iter: DBIterator<'a>,
}
impl<'a> Iterator for DatabaseIterator<'a> {
type Item = (Box<[u8]>, Box<[u8]>);
fn next(&mut self) -> Option<Self::Item> {
self.iter.next()
}
}
/// Key-Value database.
pub struct Database {
db: DB,
}
impl Database {
/// Open database with default settings.
pub fn open_default(path: &str) -> Result<Database, String> {
Database::open(&DatabaseConfig { prefix_size: None }, path)
}
/// Open database file. Creates if it does not exist.
pub fn open(config: &DatabaseConfig, path: &str) -> Result<Database, String> {
let mut opts = Options::new();
opts.set_max_open_files(256);
opts.create_if_missing(true);
opts.set_use_fsync(false);
opts.set_compaction_style(DBCompactionStyle::DBUniversalCompaction);
/*
opts.set_bytes_per_sync(8388608);
opts.set_disable_data_sync(false);
opts.set_block_cache_size_mb(1024);
opts.set_table_cache_num_shard_bits(6);
opts.set_max_write_buffer_number(32);
opts.set_write_buffer_size(536870912);
opts.set_target_file_size_base(1073741824);
opts.set_min_write_buffer_number_to_merge(4);
opts.set_level_zero_stop_writes_trigger(2000);
opts.set_level_zero_slowdown_writes_trigger(0);
opts.set_compaction_style(DBUniversalCompaction);
opts.set_max_background_compactions(4);
opts.set_max_background_flushes(4);
opts.set_filter_deletes(false);
opts.set_disable_auto_compactions(false);
*/
if let Some(size) = config.prefix_size {
let mut block_opts = BlockBasedOptions::new();
block_opts.set_index_type(IndexType::HashSearch);
opts.set_block_based_table_factory(&block_opts);
opts.set_prefix_extractor_fixed_size(size);
}
let db = try!(DB::open(&opts, path));
Ok(Database { db: db })
}
/// Insert a key-value pair in the transaction. Any existing value value will be overwritten.
pub fn put(&self, key: &[u8], value: &[u8]) -> Result<(), String> {
self.db.put(key, value)
}
/// Delete value by key.
pub fn delete(&self, key: &[u8]) -> Result<(), String> {
self.db.delete(key)
}
/// Commit transaction to database.
pub fn write(&self, tr: DBTransaction) -> Result<(), String> {
self.db.write(tr.batch)
}
/// Get value by key.
pub fn get(&self, key: &[u8]) -> Result<Option<DBVector>, String> {
self.db.get(key)
}
/// Get value by partial key. Prefix size should match configured prefix size.
pub fn get_by_prefix(&self, prefix: &[u8]) -> Option<Box<[u8]>> {
let mut iter = self.db.iterator(IteratorMode::From(prefix, Direction::forward));
match iter.next() {
// TODO: use prefix_same_as_start read option (not availabele in C API currently)
Some((k, v)) => if k[0 .. prefix.len()] == prefix[..] { Some(v) } else { None },
_ => None
}
}
/// Check if there is anything in the database.
pub fn is_empty(&self) -> bool {
self.db.iterator(IteratorMode::Start).next().is_none()
}
/// Check if there is anything in the database.
pub fn iter(&self) -> DatabaseIterator {
DatabaseIterator { iter: self.db.iterator(IteratorMode::Start) }
}
}
#[cfg(test)]
mod tests {
use hash::*;
use super::*;
use devtools::*;
use std::str::FromStr;
use std::ops::Deref;
fn test_db(config: &DatabaseConfig) {
let path = RandomTempPath::create_dir();
let db = Database::open(config, path.as_path().to_str().unwrap()).unwrap();
let key1 = H256::from_str("02c69be41d0b7e40352fc85be1cd65eb03d40ef8427a0ca4596b1ead9a00e9fc").unwrap();
let key2 = H256::from_str("03c69be41d0b7e40352fc85be1cd65eb03d40ef8427a0ca4596b1ead9a00e9fc").unwrap();
let key3 = H256::from_str("01c69be41d0b7e40352fc85be1cd65eb03d40ef8427a0ca4596b1ead9a00e9fc").unwrap();
db.put(&key1, b"cat").unwrap();
db.put(&key2, b"dog").unwrap();
assert_eq!(db.get(&key1).unwrap().unwrap().deref(), b"cat");
let contents: Vec<_> = db.iter().collect();
assert_eq!(contents.len(), 2);
assert_eq!(&*contents[0].0, key1.deref());
assert_eq!(&*contents[0].1, b"cat");
assert_eq!(&*contents[1].0, key2.deref());
assert_eq!(&*contents[1].1, b"dog");
db.delete(&key1).unwrap();
assert!(db.get(&key1).unwrap().is_none());
db.put(&key1, b"cat").unwrap();
let transaction = DBTransaction::new();
transaction.put(&key3, b"elephant").unwrap();
transaction.delete(&key1).unwrap();
db.write(transaction).unwrap();
assert!(db.get(&key1).unwrap().is_none());
assert_eq!(db.get(&key3).unwrap().unwrap().deref(), b"elephant");
if config.prefix_size.is_some() {
assert_eq!(db.get_by_prefix(&key3).unwrap().deref(), b"elephant");
assert_eq!(db.get_by_prefix(&key2).unwrap().deref(), b"dog");
}
}
#[test]
fn kvdb() {
let path = RandomTempPath::create_dir();
let smoke = Database::open_default(path.as_path().to_str().unwrap()).unwrap();
assert!(smoke.is_empty());
test_db(&DatabaseConfig { prefix_size: None });
test_db(&DatabaseConfig { prefix_size: Some(1) });
test_db(&DatabaseConfig { prefix_size: Some(8) });
test_db(&DatabaseConfig { prefix_size: Some(32) });
}
}

90
util/src/lib.rs
View File

@@ -15,21 +15,20 @@
// along with Parity. If not, see <http://www.gnu.org/licenses/>.
#![warn(missing_docs)]
#![feature(op_assign_traits)]
#![feature(augmented_assignments)]
#![feature(associated_consts)]
#![feature(plugin)]
#![feature(catch_panic)]
#![cfg_attr(feature="dev", feature(plugin))]
#![cfg_attr(feature="dev", plugin(clippy))]
// Clippy settings
#![plugin(clippy)]
// TODO [todr] not really sure
#![allow(needless_range_loop)]
#![cfg_attr(feature="dev", allow(needless_range_loop))]
// Shorter than if-else
#![allow(match_bool)]
#![cfg_attr(feature="dev", allow(match_bool))]
// We use that to be more explicit about handled cases
#![allow(match_same_arms)]
#![cfg_attr(feature="dev", allow(match_same_arms))]
// Keeps consistency (all lines with `.clone()`) and helpful when changing ref to non-ref.
#![allow(clone_on_copy)]
#![cfg_attr(feature="dev", allow(clone_on_copy))]
// In most cases it expresses function flow better
#![cfg_attr(feature="dev", allow(if_not_else))]
//! Ethcore-util library
//!
@@ -85,7 +84,6 @@
//! cargo build --release
//! ```
extern crate target_info;
extern crate slab;
extern crate rustc_serialize;
extern crate mio;
@@ -109,15 +107,20 @@ extern crate serde;
#[macro_use]
extern crate log as rlog;
extern crate igd;
extern crate ethcore_devtools as devtools;
extern crate libc;
extern crate target_info;
extern crate bigint;
extern crate chrono;
pub mod standard;
#[macro_use]
pub mod from_json;
#[macro_use]
pub mod common;
pub mod numbers;
pub mod error;
pub mod hash;
pub mod uint;
pub mod bytes;
pub mod rlp;
pub mod misc;
@@ -128,8 +131,8 @@ pub mod hashdb;
pub mod memorydb;
pub mod overlaydb;
pub mod journaldb;
pub mod kvdb;
mod math;
pub mod chainfilter;
pub mod crypto;
pub mod triehash;
pub mod trie;
@@ -142,6 +145,7 @@ pub mod network;
pub mod log;
pub mod panics;
pub mod keys;
pub mod table;
pub use common::*;
pub use misc::*;
@@ -150,9 +154,8 @@ pub use rlp::*;
pub use hashdb::*;
pub use memorydb::*;
pub use overlaydb::*;
pub use journaldb::*;
pub use journaldb::JournalDB;
pub use math::*;
pub use chainfilter::*;
pub use crypto::*;
pub use triehash::*;
pub use trie::*;
@@ -162,6 +165,61 @@ pub use semantic_version::*;
pub use network::*;
pub use io::*;
pub use log::*;
pub use kvdb::*;
#[cfg(test)]
mod tests;
mod tests {
use super::numbers::*;
use std::str::FromStr;
#[test]
fn u256_multi_muls() {
let (result, _) = U256([0, 0, 0, 0]).overflowing_mul(U256([0, 0, 0, 0]));
assert_eq!(U256([0, 0, 0, 0]), result);
let (result, _) = U256([1, 0, 0, 0]).overflowing_mul(U256([1, 0, 0, 0]));
assert_eq!(U256([1, 0, 0, 0]), result);
let (result, _) = U256([5, 0, 0, 0]).overflowing_mul(U256([5, 0, 0, 0]));
assert_eq!(U256([25, 0, 0, 0]), result);
let (result, _) = U256([0, 5, 0, 0]).overflowing_mul(U256([0, 5, 0, 0]));
assert_eq!(U256([0, 0, 25, 0]), result);
let (result, _) = U256([0, 0, 0, 1]).overflowing_mul(U256([1, 0, 0, 0]));
assert_eq!(U256([0, 0, 0, 1]), result);
let (result, _) = U256([0, 0, 0, 5]).overflowing_mul(U256([2, 0, 0, 0]));
assert_eq!(U256([0, 0, 0, 10]), result);
let (result, _) = U256([0, 0, 1, 0]).overflowing_mul(U256([0, 5, 0, 0]));
assert_eq!(U256([0, 0, 0, 5]), result);
let (result, _) = U256([0, 0, 8, 0]).overflowing_mul(U256([0, 0, 7, 0]));
assert_eq!(U256([0, 0, 0, 0]), result);
let (result, _) = U256([2, 0, 0, 0]).overflowing_mul(U256([0, 5, 0, 0]));
assert_eq!(U256([0, 10, 0, 0]), result);
let (result, _) = U256([1, 0, 0, 0]).overflowing_mul(U256([0, 0, 0, ::std::u64::MAX]));
assert_eq!(U256([0, 0, 0, ::std::u64::MAX]), result);
let x1 = U256::from_str("0000000000000000000000000000000000000000000000000000012365124623").unwrap();
let x2sqr_right = U256::from_str("000000000000000000000000000000000000000000014baeef72e0378e2328c9").unwrap();
let x1sqr = x1 * x1;
assert_eq!(H256::from(x2sqr_right), H256::from(x1sqr));
let x1cube = x1sqr * x1;
let x1cube_right = U256::from_str("0000000000000000000000000000000001798acde139361466f712813717897b").unwrap();
assert_eq!(H256::from(x1cube_right), H256::from(x1cube));
let x1quad = x1cube * x1;
let x1quad_right = U256::from_str("000000000000000000000001adbdd6bd6ff027485484b97f8a6a4c7129756dd1").unwrap();
assert_eq!(H256::from(x1quad_right), H256::from(x1quad));
let x1penta = x1quad * x1;
let x1penta_right = U256::from_str("00000000000001e92875ac24be246e1c57e0507e8c46cc8d233b77f6f4c72993").unwrap();
assert_eq!(H256::from(x1penta_right), H256::from(x1penta));
let x1septima = x1penta * x1;
let x1septima_right = U256::from_str("00022cca1da3f6e5722b7d3cc5bbfb486465ebc5a708dd293042f932d7eee119").unwrap();
assert_eq!(H256::from(x1septima_right), H256::from(x1septima));
}
}

18
util/src/memorydb.rs
View File

@@ -21,8 +21,10 @@ use bytes::*;
use rlp::*;
use sha3::*;
use hashdb::*;
use heapsize::*;
use std::mem;
use std::collections::HashMap;
use std::default::Default;
#[derive(Debug,Clone)]
/// Reference-counted memory-based HashDB implementation.
@@ -31,7 +33,7 @@ use std::collections::HashMap;
/// with `kill()`, check for existance with `exists()` and lookup a hash to derive
/// the data with `lookup()`. Clear with `clear()` and purge the portions of the data
/// that have no references with `purge()`.
///
///
/// # Example
/// ```rust
/// extern crate ethcore_util;
@@ -68,11 +70,18 @@ use std::collections::HashMap;
/// assert!(!m.exists(&k));
/// }
/// ```
#[derive(PartialEq)]
pub struct MemoryDB {
data: HashMap<H256, (Bytes, i32)>,
static_null_rlp: (Bytes, i32),
}
impl Default for MemoryDB {
fn default() -> Self {
MemoryDB::new()
}
}
impl MemoryDB {
/// Create a new instance of the memory DB.
pub fn new() -> MemoryDB {
@@ -132,7 +141,7 @@ impl MemoryDB {
/// Denote than an existing value has the given key. Used when a key gets removed without
/// a prior insert and thus has a negative reference with no value.
///
///
/// May safely be called even if the key's value is known, in which case it will be a no-op.
pub fn denote(&self, key: &H256, value: Bytes) -> &(Bytes, i32) {
if self.raw(key) == None {
@@ -143,6 +152,11 @@ impl MemoryDB {
}
self.raw(key).unwrap()
}
/// Returns the size of allocated heap memory
pub fn mem_used(&self) -> usize {
self.data.heap_size_of_children()
}
}
static NULL_RLP_STATIC: [u8; 1] = [0x80; 1];

30
util/src/misc.rs
View File

@@ -18,6 +18,11 @@
use std::fs::File;
use common::*;
use rlp::{Stream, RlpStream};
use target_info::Target;
include!(concat!(env!("OUT_DIR"), "/version.rs"));
include!(concat!(env!("OUT_DIR"), "/rustc_version.rs"));
#[derive(Debug,Clone,PartialEq,Eq)]
/// Diff type for specifying a change (or not).
@@ -62,3 +67,28 @@ pub fn contents(name: &str) -> Result<Bytes, UtilError> {
try!(file.read_to_end(&mut ret));
Ok(ret)
}
/// Get the standard version string for this software.
pub fn version() -> String {
let sha3 = short_sha();
let sha3_dash = if sha3.is_empty() { "" } else { "-" };
let commit_date = commit_date().replace("-", "");
let date_dash = if commit_date.is_empty() { "" } else { "-" };
let env = Target::env();
let env_dash = if env.is_empty() { "" } else { "-" };
format!("Parity/v{}-unstable{}{}{}{}/{}-{}{}{}/rustc{}", env!("CARGO_PKG_VERSION"), sha3_dash, sha3, date_dash, commit_date, Target::arch(), Target::os(), env_dash, env, rustc_version())
}
/// Get the standard version data for this software.
pub fn version_data() -> Bytes {
let mut s = RlpStream::new_list(4);
let v =
(u32::from_str(env!("CARGO_PKG_VERSION_MAJOR")).unwrap() << 16) +
(u32::from_str(env!("CARGO_PKG_VERSION_MINOR")).unwrap() << 8) +
u32::from_str(env!("CARGO_PKG_VERSION_PATCH")).unwrap();
s.append(&v);
s.append(&"Parity");
s.append(&rustc_version());
s.append(&&Target::os()[0..2]);
s.out()
}

102
util/src/network/connection.rs
View File

@@ -16,6 +16,7 @@
use std::sync::Arc;
use std::collections::VecDeque;
use std::net::SocketAddr;
use mio::{Handler, Token, EventSet, EventLoop, PollOpt, TryRead, TryWrite};
use mio::tcp::*;
use hash::*;
@@ -159,27 +160,55 @@ impl Connection {
}
}
/// Get socket token
pub fn token(&self) -> StreamToken {
self.token
}
/// Replace socket token
pub fn set_token(&mut self, token: StreamToken) {
self.token = token;
}
/// Get remote peer address
pub fn remote_addr(&self) -> io::Result<SocketAddr> {
self.socket.peer_addr()
}
/// Clone this connection. Clears the receiving buffer of the returned connection.
pub fn try_clone(&self) -> io::Result<Self> {
Ok(Connection {
token: self.token,
socket: try!(self.socket.try_clone()),
rec_buf: Vec::new(),
rec_size: 0,
send_queue: self.send_queue.clone(),
interest: EventSet::hup() | EventSet::readable(),
stats: self.stats.clone(),
})
}
/// Register this connection with the IO event loop.
pub fn register_socket<Host: Handler>(&self, reg: Token, event_loop: &mut EventLoop<Host>) -> io::Result<()> {
trace!(target: "net", "connection register; token={:?}", reg);
event_loop.register(&self.socket, reg, self.interest, PollOpt::edge() | PollOpt::oneshot()).or_else(|e| {
debug!("Failed to register {:?}, {:?}", reg, e);
Ok(())
})
trace!(target: "network", "connection register; token={:?}", reg);
if let Err(e) = event_loop.register(&self.socket, reg, self.interest, PollOpt::edge() | PollOpt::oneshot()) {
trace!(target: "network", "Failed to register {:?}, {:?}", reg, e);
}
Ok(())
}
/// Update connection registration. Should be called at the end of the IO handler.
pub fn update_socket<Host: Handler>(&self, reg: Token, event_loop: &mut EventLoop<Host>) -> io::Result<()> {
trace!(target: "net", "connection reregister; token={:?}", reg);
trace!(target: "network", "connection reregister; token={:?}", reg);
event_loop.reregister( &self.socket, reg, self.interest, PollOpt::edge() | PollOpt::oneshot()).or_else(|e| {
debug!("Failed to reregister {:?}, {:?}", reg, e);
trace!(target: "network", "Failed to reregister {:?}, {:?}", reg, e);
Ok(())
})
}
/// Delete connection registration. Should be called at the end of the IO handler.
pub fn deregister_socket<Host: Handler>(&self, event_loop: &mut EventLoop<Host>) -> io::Result<()> {
trace!(target: "net", "connection deregister; token={:?}", self.token);
trace!(target: "network", "connection deregister; token={:?}", self.token);
event_loop.deregister(&self.socket).ok(); // ignore errors here
Ok(())
}
@@ -232,15 +261,25 @@ pub struct EncryptedConnection {
}
impl EncryptedConnection {
/// Get socket token
/// Get socket token
pub fn token(&self) -> StreamToken {
self.connection.token
}
/// Replace socket token
pub fn set_token(&mut self, token: StreamToken) {
self.connection.set_token(token);
}
/// Get remote peer address
pub fn remote_addr(&self) -> io::Result<SocketAddr> {
self.connection.remote_addr()
}
/// Create an encrypted connection out of the handshake. Consumes a handshake object.
pub fn new(mut handshake: Handshake) -> Result<EncryptedConnection, UtilError> {
let shared = try!(crypto::ecdh::agree(handshake.ecdhe.secret(), &handshake.remote_public));
pub fn new(handshake: &mut Handshake) -> Result<EncryptedConnection, UtilError> {
let shared = try!(crypto::ecdh::agree(handshake.ecdhe.secret(), &handshake.remote_ephemeral));
let mut nonce_material = H512::new();
if handshake.originated {
handshake.remote_nonce.copy_to(&mut nonce_material[0..32]);
@@ -274,9 +313,8 @@ impl EncryptedConnection {
ingress_mac.update(&mac_material);
ingress_mac.update(if handshake.originated { &handshake.ack_cipher } else { &handshake.auth_cipher });
handshake.connection.expect(ENCRYPTED_HEADER_LEN);
Ok(EncryptedConnection {
connection: handshake.connection,
let mut enc = EncryptedConnection {
connection: try!(handshake.connection.try_clone()),
encoder: encoder,
decoder: decoder,
mac_encoder: mac_encoder,
@@ -285,7 +323,9 @@ impl EncryptedConnection {
read_state: EncryptedConnectionState::Header,
protocol_id: 0,
payload_len: 0
})
};
enc.connection.expect(ENCRYPTED_HEADER_LEN);
Ok(enc)
}
/// Send a packet
@@ -414,6 +454,12 @@ impl EncryptedConnection {
Ok(())
}
/// Register socket with the event lpop. This should be called at the end of the event loop.
pub fn register_socket<Host:Handler>(&self, reg: Token, event_loop: &mut EventLoop<Host>) -> Result<(), UtilError> {
try!(self.connection.register_socket(reg, event_loop));
Ok(())
}
/// Update connection registration. This should be called at the end of the event loop.
pub fn update_socket<Host:Handler>(&self, reg: Token, event_loop: &mut EventLoop<Host>) -> Result<(), UtilError> {
try!(self.connection.update_socket(reg, event_loop));
@@ -467,8 +513,14 @@ mod tests {
buf_size: usize,
}
impl Default for TestSocket {
fn default() -> Self {
TestSocket::new()
}
}
impl TestSocket {
fn new() -> TestSocket {
fn new() -> Self {
TestSocket {
read_buffer: vec![],
write_buffer: vec![],
@@ -547,8 +599,14 @@ mod tests {
type TestConnection = GenericConnection<TestSocket>;
impl Default for TestConnection {
fn default() -> Self {
TestConnection::new()
}
}
impl TestConnection {
pub fn new() -> TestConnection {
pub fn new() -> Self {
TestConnection {
token: 999998888usize,
socket: TestSocket::new(),
@@ -563,8 +621,14 @@ mod tests {
type TestBrokenConnection = GenericConnection<TestBrokenSocket>;
impl Default for TestBrokenConnection {
fn default() -> Self {
TestBrokenConnection::new()
}
}
impl TestBrokenConnection {
pub fn new() -> TestBrokenConnection {
pub fn new() -> Self {
TestBrokenConnection {
token: 999998888usize,
socket: TestBrokenSocket { error: "test broken socket".to_owned() },

680
util/src/network/discovery.rs
View File

@@ -14,115 +14,188 @@
// You should have received a copy of the GNU General Public License
// along with Parity. If not, see <http://www.gnu.org/licenses/>.
// This module is a work in progress
#![allow(dead_code)] //TODO: remove this after everything is done
use std::collections::{HashSet, BTreeMap};
use std::cell::{RefCell};
use std::ops::{DerefMut};
use bytes::Bytes;
use std::net::SocketAddr;
use std::collections::{HashSet, HashMap, BTreeMap, VecDeque};
use std::mem;
use std::default::Default;
use mio::*;
use mio::udp::*;
use sha3::*;
use time;
use hash::*;
use sha3::Hashable;
use crypto::*;
use network::node::*;
use rlp::*;
use network::node_table::*;
use network::error::NetworkError;
use io::StreamToken;
const ADDRESS_BYTES_SIZE: u32 = 32; ///< Size of address type in bytes.
const ADDRESS_BITS: u32 = 8 * ADDRESS_BYTES_SIZE; ///< Denoted by n in [Kademlia].
const NODE_BINS: u32 = ADDRESS_BITS - 1; ///< Size of m_state (excludes root, which is us).
const DISCOVERY_MAX_STEPS: u16 = 8; ///< Max iterations of discovery. (discover)
const BUCKET_SIZE: u32 = 16; ///< Denoted by k in [Kademlia]. Number of nodes stored in each bucket.
const ALPHA: usize = 3; ///< Denoted by \alpha in [Kademlia]. Number of concurrent FindNode requests.
use network::PROTOCOL_VERSION;
const ADDRESS_BYTES_SIZE: u32 = 32; // Size of address type in bytes.
const ADDRESS_BITS: u32 = 8 * ADDRESS_BYTES_SIZE; // Denoted by n in [Kademlia].
const NODE_BINS: u32 = ADDRESS_BITS - 1; // Size of m_state (excludes root, which is us).
const DISCOVERY_MAX_STEPS: u16 = 8; // Max iterations of discovery. (discover)
const BUCKET_SIZE: usize = 16; // Denoted by k in [Kademlia]. Number of nodes stored in each bucket.
const ALPHA: usize = 3; // Denoted by \alpha in [Kademlia]. Number of concurrent FindNode requests.
const MAX_DATAGRAM_SIZE: usize = 1280;
const PACKET_PING: u8 = 1;
const PACKET_PONG: u8 = 2;
const PACKET_FIND_NODE: u8 = 3;
const PACKET_NEIGHBOURS: u8 = 4;
const PING_TIMEOUT_MS: u64 = 300;
#[derive(Clone, Debug)]
pub struct NodeEntry {
pub id: NodeId,
pub endpoint: NodeEndpoint,
}
pub struct BucketEntry {
pub address: NodeEntry,
pub timeout: Option<u64>,
}
struct NodeBucket {
distance: u32,
nodes: Vec<NodeId>
nodes: VecDeque<BucketEntry>, //sorted by last active
}
impl Default for NodeBucket {
fn default() -> Self {
NodeBucket::new()
}
}
impl NodeBucket {
fn new(distance: u32) -> NodeBucket {
fn new() -> Self {
NodeBucket {
distance: distance,
nodes: Vec::new()
nodes: VecDeque::new()
}
}
}
struct Discovery {
struct Datagramm {
payload: Bytes,
address: SocketAddr,
}
pub struct Discovery {
id: NodeId,
secret: Secret,
public_endpoint: NodeEndpoint,
udp_socket: UdpSocket,
token: StreamToken,
discovery_round: u16,
discovery_id: NodeId,
discovery_nodes: HashSet<NodeId>,
node_buckets: Vec<NodeBucket>,
send_queue: VecDeque<Datagramm>,
check_timestamps: bool,
}
struct FindNodePacket;
impl FindNodePacket {
fn new(_endpoint: &NodeEndpoint, _id: &NodeId) -> FindNodePacket {
FindNodePacket
}
fn sign(&mut self, _secret: &Secret) {
}
fn send(& self, _socket: &mut UdpSocket) {
}
pub struct TableUpdates {
pub added: HashMap<NodeId, NodeEntry>,
pub removed: HashSet<NodeId>,
}
impl Discovery {
pub fn new(id: &NodeId) -> Discovery {
pub fn new(key: &KeyPair, listen: SocketAddr, public: NodeEndpoint, token: StreamToken) -> Discovery {
let socket = UdpSocket::bound(&listen).expect("Error binding UDP socket");
Discovery {
id: id.clone(),
id: key.public().clone(),
secret: key.secret().clone(),
public_endpoint: public,
token: token,
discovery_round: 0,
discovery_id: NodeId::new(),
discovery_nodes: HashSet::new(),
node_buckets: (0..NODE_BINS).map(NodeBucket::new).collect(),
node_buckets: (0..NODE_BINS).map(|_| NodeBucket::new()).collect(),
udp_socket: socket,
send_queue: VecDeque::new(),
check_timestamps: true,
}
}
pub fn add_node(&mut self, id: &NodeId) {
self.node_buckets[Discovery::distance(&self.id, &id) as usize].nodes.push(id.clone());
/// Add a new node to discovery table. Pings the node.
pub fn add_node(&mut self, e: NodeEntry) {
let endpoint = e.endpoint.clone();
self.update_node(e);
self.ping(&endpoint);
}
fn start_node_discovery<Host:Handler>(&mut self, event_loop: &mut EventLoop<Host>) {
/// Add a list of known nodes to the table.
pub fn init_node_list(&mut self, mut nodes: Vec<NodeEntry>) {
for n in nodes.drain(..) {
self.update_node(n);
}
}
fn update_node(&mut self, e: NodeEntry) {
trace!(target: "discovery", "Inserting {:?}", &e);
let ping = {
let mut bucket = self.node_buckets.get_mut(Discovery::distance(&self.id, &e.id) as usize).unwrap();
let updated = if let Some(node) = bucket.nodes.iter_mut().find(|n| n.address.id == e.id) {
node.address = e.clone();
node.timeout = None;
true
} else { false };
if !updated {
bucket.nodes.push_front(BucketEntry { address: e, timeout: None });
}
if bucket.nodes.len() > BUCKET_SIZE {
//ping least active node
bucket.nodes.back_mut().unwrap().timeout = Some(time::precise_time_ns());
Some(bucket.nodes.back().unwrap().address.endpoint.clone())
} else { None }
};
if let Some(endpoint) = ping {
self.ping(&endpoint);
}
}
fn clear_ping(&mut self, id: &NodeId) {
let mut bucket = self.node_buckets.get_mut(Discovery::distance(&self.id, &id) as usize).unwrap();
if let Some(node) = bucket.nodes.iter_mut().find(|n| &n.address.id == id) {
node.timeout = None;
}
}
fn start(&mut self) {
trace!(target: "discovery", "Starting discovery");
self.discovery_round = 0;
self.discovery_id.randomize();
self.discovery_id.randomize(); //TODO: use cryptographic nonce
self.discovery_nodes.clear();
self.discover(event_loop);
}
fn discover<Host:Handler>(&mut self, event_loop: &mut EventLoop<Host>) {
if self.discovery_round == DISCOVERY_MAX_STEPS
{
debug!("Restarting discovery");
self.start_node_discovery(event_loop);
fn discover(&mut self) {
if self.discovery_round == DISCOVERY_MAX_STEPS {
return;
}
trace!(target: "discovery", "Starting round {:?}", self.discovery_round);
let mut tried_count = 0;
{
let nearest = Discovery::nearest_node_entries(&self.id, &self.discovery_id, &self.node_buckets).into_iter();
let nodes = RefCell::new(&mut self.discovery_nodes);
let nearest = nearest.filter(|x| nodes.borrow().contains(&x)).take(ALPHA);
let nearest = Discovery::nearest_node_entries(&self.discovery_id, &self.node_buckets).into_iter();
let nearest = nearest.filter(|x| !self.discovery_nodes.contains(&x.id)).take(ALPHA).collect::<Vec<_>>();
for r in nearest {
//let mut p = FindNodePacket::new(&r.endpoint, &self.discovery_id);
//p.sign(&self.secret);
//p.send(&mut self.udp_socket);
let mut borrowed = nodes.borrow_mut();
borrowed.deref_mut().insert(r.clone());
let rlp = encode(&(&[self.discovery_id.clone()][..]));
self.send_packet(PACKET_FIND_NODE, &r.endpoint.udp_address(), &rlp);
self.discovery_nodes.insert(r.id.clone());
tried_count += 1;
trace!(target: "discovery", "Sent FindNode to {:?}", &r.endpoint);
}
}
if tried_count == 0
{
debug!("Restarting discovery");
self.start_node_discovery(event_loop);
if tried_count == 0 {
trace!(target: "discovery", "Completing discovery");
self.discovery_round = DISCOVERY_MAX_STEPS;
self.discovery_nodes.clear();
return;
}
self.discovery_round += 1;
//event_loop.timeout_ms(Token(NODETABLE_DISCOVERY), 1200).unwrap();
}
fn distance(a: &NodeId, b: &NodeId) -> u32 {
@@ -138,86 +211,437 @@ impl Discovery {
ret
}
#[allow(cyclomatic_complexity)]
fn nearest_node_entries<'b>(source: &NodeId, target: &NodeId, buckets: &'b [NodeBucket]) -> Vec<&'b NodeId>
{
// send ALPHA FindNode packets to nodes we know, closest to target
const LAST_BIN: u32 = NODE_BINS - 1;
let mut head = Discovery::distance(source, target);
let mut tail = if head == 0 { LAST_BIN } else { (head - 1) % NODE_BINS };
fn ping(&mut self, node: &NodeEndpoint) {
let mut rlp = RlpStream::new_list(3);
rlp.append(&PROTOCOL_VERSION);
self.public_endpoint.to_rlp_list(&mut rlp);
node.to_rlp_list(&mut rlp);
trace!(target: "discovery", "Sent Ping to {:?}", &node);
self.send_packet(PACKET_PING, &node.udp_address(), &rlp.drain());
}
let mut found: BTreeMap<u32, Vec<&'b NodeId>> = BTreeMap::new();
fn send_packet(&mut self, packet_id: u8, address: &SocketAddr, payload: &[u8]) {
let mut rlp = RlpStream::new();
rlp.append_raw(&[packet_id], 1);
let source = Rlp::new(payload);
rlp.begin_list(source.item_count() + 1);
for i in 0 .. source.item_count() {
rlp.append_raw(source.at(i).as_raw(), 1);
}
let timestamp = time::get_time().sec as u32 + 60;
rlp.append(&timestamp);
let bytes = rlp.drain();
let hash = bytes.as_ref().sha3();
let signature = match ec::sign(&self.secret, &hash) {
Ok(s) => s,
Err(_) => {
warn!("Error signing UDP packet");
return;
}
};
let mut packet = Bytes::with_capacity(bytes.len() + 32 + 65);
packet.extend(hash.iter());
packet.extend(signature.iter());
packet.extend(bytes.iter());
let signed_hash = (&packet[32..]).sha3();
packet[0..32].clone_from_slice(&signed_hash);
self.send_to(packet, address.clone());
}
#[cfg_attr(feature="dev", allow(map_clone))]
fn nearest_node_entries(target: &NodeId, buckets: &[NodeBucket]) -> Vec<NodeEntry> {
let mut found: BTreeMap<u32, Vec<&NodeEntry>> = BTreeMap::new();
let mut count = 0;
// if d is 0, then we roll look forward, if last, we reverse, else, spread from d
if head > 1 && tail != LAST_BIN {
while head != tail && head < NODE_BINS && count < BUCKET_SIZE
{
for n in &buckets[head as usize].nodes
{
if count < BUCKET_SIZE {
count += 1;
found.entry(Discovery::distance(target, &n)).or_insert_with(Vec::new).push(n);
}
else {
break;
}
}
if count < BUCKET_SIZE && tail != 0 {
for n in &buckets[tail as usize].nodes {
if count < BUCKET_SIZE {
count += 1;
found.entry(Discovery::distance(target, &n)).or_insert_with(Vec::new).push(n);
}
else {
break;
}
// Sort nodes by distance to target
for bucket in buckets {
for node in &bucket.nodes {
let distance = Discovery::distance(target, &node.address.id);
found.entry(distance).or_insert_with(Vec::new).push(&node.address);
if count == BUCKET_SIZE {
// delete the most distant element
let remove = {
let (_, last) = found.iter_mut().next_back().unwrap();
last.pop();
last.is_empty()
};
if remove {
found.remove(&distance);
}
}
head += 1;
if tail > 0 {
tail -= 1;
else {
count += 1;
}
}
}
else if head < 2 {
while head < NODE_BINS && count < BUCKET_SIZE {
for n in &buckets[head as usize].nodes {
if count < BUCKET_SIZE {
count += 1;
found.entry(Discovery::distance(target, &n)).or_insert_with(Vec::new).push(n);
}
else {
break;
}
}
head += 1;
}
}
else {
while tail > 0 && count < BUCKET_SIZE {
for n in &buckets[tail as usize].nodes {
if count < BUCKET_SIZE {
count += 1;
found.entry(Discovery::distance(target, &n)).or_insert_with(Vec::new).push(n);
}
else {
break;
}
}
tail -= 1;
}
}
let mut ret:Vec<&NodeId> = Vec::new();
let mut ret:Vec<NodeEntry> = Vec::new();
for nodes in found.values() {
for n in nodes {
if ret.len() < BUCKET_SIZE as usize /* && n->endpoint && n->endpoint.isAllowed() */ {
ret.push(n);
}
}
ret.extend(nodes.iter().map(|&n| n.clone()));
}
ret
}
pub fn writable(&mut self) {
while !self.send_queue.is_empty() {
let data = self.send_queue.pop_front().unwrap();
match self.udp_socket.send_to(&data.payload, &data.address) {
Ok(Some(size)) if size == data.payload.len() => {
},
Ok(Some(_)) => {
warn!("UDP sent incomplete datagramm");
},
Ok(None) => {
self.send_queue.push_front(data);
return;
}
Err(e) => {
debug!("UDP send error: {:?}, address: {:?}", e, &data.address);
return;
}
}
}
}
fn send_to(&mut self, payload: Bytes, address: SocketAddr) {
self.send_queue.push_back(Datagramm { payload: payload, address: address });
}
pub fn readable(&mut self) -> Option<TableUpdates> {
let mut buf: [u8; MAX_DATAGRAM_SIZE] = unsafe { mem::uninitialized() };
match self.udp_socket.recv_from(&mut buf) {
Ok(Some((len, address))) => self.on_packet(&buf[0..len], address).unwrap_or_else(|e| {
debug!("Error processing UDP packet: {:?}", e);
None
}),
Ok(_) => None,
Err(e) => {
debug!("Error reading UPD socket: {:?}", e);
None
}
}
}
fn on_packet(&mut self, packet: &[u8], from: SocketAddr) -> Result<Option<TableUpdates>, NetworkError> {
// validate packet
if packet.len() < 32 + 65 + 4 + 1 {
return Err(NetworkError::BadProtocol);
}
let hash_signed = (&packet[32..]).sha3();
if hash_signed[..] != packet[0..32] {
return Err(NetworkError::BadProtocol);
}
let signed = &packet[(32 + 65)..];
let signature = Signature::from_slice(&packet[32..(32 + 65)]);
let node_id = try!(ec::recover(&signature, &signed.sha3()));
let packet_id = signed[0];
let rlp = UntrustedRlp::new(&signed[1..]);
match packet_id {
PACKET_PING => self.on_ping(&rlp, &node_id, &from),
PACKET_PONG => self.on_pong(&rlp, &node_id, &from),
PACKET_FIND_NODE => self.on_find_node(&rlp, &node_id, &from),
PACKET_NEIGHBOURS => self.on_neighbours(&rlp, &node_id, &from),
_ => {
debug!("Unknown UDP packet: {}", packet_id);
Ok(None)
}
}
}
fn check_timestamp(&self, timestamp: u64) -> Result<(), NetworkError> {
if self.check_timestamps && timestamp < time::get_time().sec as u64{
debug!(target: "discovery", "Expired packet");
return Err(NetworkError::Expired);
}
Ok(())
}
fn on_ping(&mut self, rlp: &UntrustedRlp, node: &NodeId, from: &SocketAddr) -> Result<Option<TableUpdates>, NetworkError> {
trace!(target: "discovery", "Got Ping from {:?}", &from);
let source = try!(NodeEndpoint::from_rlp(&try!(rlp.at(1))));
let dest = try!(NodeEndpoint::from_rlp(&try!(rlp.at(2))));
let timestamp: u64 = try!(rlp.val_at(3));
try!(self.check_timestamp(timestamp));
let mut added_map = HashMap::new();
let entry = NodeEntry { id: node.clone(), endpoint: source.clone() };
if !entry.endpoint.is_valid() || !entry.endpoint.is_global() {
debug!(target: "discovery", "Got bad address: {:?}", entry);
}
else {
self.update_node(entry.clone());
added_map.insert(node.clone(), entry);
}
let hash = rlp.as_raw().sha3();
let mut response = RlpStream::new_list(2);
dest.to_rlp_list(&mut response);
response.append(&hash);
self.send_packet(PACKET_PONG, from, &response.drain());
Ok(Some(TableUpdates { added: added_map, removed: HashSet::new() }))
}
fn on_pong(&mut self, rlp: &UntrustedRlp, node: &NodeId, from: &SocketAddr) -> Result<Option<TableUpdates>, NetworkError> {
trace!(target: "discovery", "Got Pong from {:?}", &from);
// TODO: validate pong packet
let dest = try!(NodeEndpoint::from_rlp(&try!(rlp.at(0))));
let timestamp: u64 = try!(rlp.val_at(2));
try!(self.check_timestamp(timestamp));
let mut entry = NodeEntry { id: node.clone(), endpoint: dest };
if !entry.endpoint.is_valid() {
debug!(target: "discovery", "Bad address: {:?}", entry);
entry.endpoint.address = from.clone();
}
self.clear_ping(node);
let mut added_map = HashMap::new();
added_map.insert(node.clone(), entry);
Ok(None)
}
fn on_find_node(&mut self, rlp: &UntrustedRlp, _node: &NodeId, from: &SocketAddr) -> Result<Option<TableUpdates>, NetworkError> {
trace!(target: "discovery", "Got FindNode from {:?}", &from);
let target: NodeId = try!(rlp.val_at(0));
let timestamp: u64 = try!(rlp.val_at(1));
try!(self.check_timestamp(timestamp));
let nearest = Discovery::nearest_node_entries(&target, &self.node_buckets);
if nearest.is_empty() {
return Ok(None);
}
let mut packets = Discovery::prepare_neighbours_packets(&nearest);
for p in packets.drain(..) {
self.send_packet(PACKET_NEIGHBOURS, &from, &p);
}
trace!(target: "discovery", "Sent {} Neighbours to {:?}", nearest.len(), &from);
Ok(None)
}
fn prepare_neighbours_packets(nearest: &[NodeEntry]) -> Vec<Bytes> {
let limit = (MAX_DATAGRAM_SIZE - 109) / 90;
let chunks = nearest.chunks(limit);
let packets = chunks.map(|c| {
let mut rlp = RlpStream::new_list(1);
rlp.begin_list(c.len());
for n in 0 .. c.len() {
rlp.begin_list(4);
c[n].endpoint.to_rlp(&mut rlp);
rlp.append(&c[n].id);
}
rlp.out()
});
packets.collect()
}
fn on_neighbours(&mut self, rlp: &UntrustedRlp, _node: &NodeId, from: &SocketAddr) -> Result<Option<TableUpdates>, NetworkError> {
// TODO: validate packet
let mut added = HashMap::new();
trace!(target: "discovery", "Got {} Neighbours from {:?}", try!(rlp.at(0)).item_count(), &from);
for r in try!(rlp.at(0)).iter() {
let endpoint = try!(NodeEndpoint::from_rlp(&r));
if !endpoint.is_valid() {
debug!(target: "discovery", "Bad address: {:?}", endpoint);
continue;
}
let node_id: NodeId = try!(r.val_at(3));
if node_id == self.id {
continue;
}
let entry = NodeEntry { id: node_id.clone(), endpoint: endpoint };
added.insert(node_id, entry.clone());
self.ping(&entry.endpoint);
self.update_node(entry);
}
Ok(Some(TableUpdates { added: added, removed: HashSet::new() }))
}
fn check_expired(&mut self, force: bool) -> HashSet<NodeId> {
let now = time::precise_time_ns();
let mut removed: HashSet<NodeId> = HashSet::new();
for bucket in &mut self.node_buckets {
bucket.nodes.retain(|node| {
if let Some(timeout) = node.timeout {
if !force && now - timeout < PING_TIMEOUT_MS * 1000_0000 {
true
}
else {
trace!(target: "discovery", "Removed expired node {:?}", &node.address);
removed.insert(node.address.id.clone());
false
}
} else { true }
});
}
removed
}
pub fn round(&mut self) -> Option<TableUpdates> {
let removed = self.check_expired(false);
self.discover();
if !removed.is_empty() {
Some(TableUpdates { added: HashMap::new(), removed: removed })
} else { None }
}
pub fn refresh(&mut self) {
self.start();
}
pub fn register_socket<Host:Handler>(&self, event_loop: &mut EventLoop<Host>) -> Result<(), NetworkError> {
event_loop.register(&self.udp_socket, Token(self.token), EventSet::all(), PollOpt::edge()).expect("Error registering UDP socket");
Ok(())
}
pub fn update_registration<Host:Handler>(&self, event_loop: &mut EventLoop<Host>) -> Result<(), NetworkError> {
let mut registration = EventSet::readable();
if !self.send_queue.is_empty() {
registration = registration | EventSet::writable();
}
event_loop.reregister(&self.udp_socket, Token(self.token), registration, PollOpt::edge()).expect("Error reregistering UDP socket");
Ok(())
}
}
#[cfg(test)]
mod tests {
use super::*;
use hash::*;
use std::net::*;
use network::node_table::*;
use crypto::KeyPair;
use std::str::FromStr;
use rustc_serialize::hex::FromHex;
use rlp::*;
#[test]
fn find_node() {
let mut nearest = Vec::new();
let node = Node::from_str("enode://a979fb575495b8d6db44f750317d0f4622bf4c2aa3365d6af7c284339968eef29b69ad0dce72a4d8db5ebb4968de0e3bec910127f134779fbcb0cb6d3331163c@127.0.0.1:7770").unwrap();
for _ in 0..1000 {
nearest.push( NodeEntry { id: node.id.clone(), endpoint: node.endpoint.clone() });
}
let packets = Discovery::prepare_neighbours_packets(&nearest);
assert_eq!(packets.len(), 77);
for p in &packets[0..76] {
assert!(p.len() > 1280/2);
assert!(p.len() <= 1280);
}
assert!(packets.last().unwrap().len() > 0);
}
#[test]
fn discovery() {
let key1 = KeyPair::create().unwrap();
let key2 = KeyPair::create().unwrap();
let ep1 = NodeEndpoint { address: SocketAddr::from_str("127.0.0.1:40444").unwrap(), udp_port: 40444 };
let ep2 = NodeEndpoint { address: SocketAddr::from_str("127.0.0.1:40445").unwrap(), udp_port: 40445 };
let mut discovery1 = Discovery::new(&key1, ep1.address.clone(), ep1.clone(), 0);
let mut discovery2 = Discovery::new(&key2, ep2.address.clone(), ep2.clone(), 0);
let node1 = Node::from_str("enode://a979fb575495b8d6db44f750317d0f4622bf4c2aa3365d6af7c284339968eef29b69ad0dce72a4d8db5ebb4968de0e3bec910127f134779fbcb0cb6d3331163c@127.0.0.1:7770").unwrap();
let node2 = Node::from_str("enode://b979fb575495b8d6db44f750317d0f4622bf4c2aa3365d6af7c284339968eef29b69ad0dce72a4d8db5ebb4968de0e3bec910127f134779fbcb0cb6d3331163c@127.0.0.1:7771").unwrap();
discovery1.add_node(NodeEntry { id: node1.id.clone(), endpoint: node1.endpoint.clone() });
discovery1.add_node(NodeEntry { id: node2.id.clone(), endpoint: node2.endpoint.clone() });
discovery2.add_node(NodeEntry { id: key1.public().clone(), endpoint: ep1.clone() });
discovery2.refresh();
for _ in 0 .. 10 {
while !discovery1.send_queue.is_empty() {
let datagramm = discovery1.send_queue.pop_front().unwrap();
if datagramm.address == ep2.address {
discovery2.on_packet(&datagramm.payload, ep1.address.clone()).ok();
}
}
while !discovery2.send_queue.is_empty() {
let datagramm = discovery2.send_queue.pop_front().unwrap();
if datagramm.address == ep1.address {
discovery1.on_packet(&datagramm.payload, ep2.address.clone()).ok();
}
}
discovery2.round();
}
assert_eq!(Discovery::nearest_node_entries(&NodeId::new(), &discovery2.node_buckets).len(), 3)
}
#[test]
fn removes_expired() {
let key = KeyPair::create().unwrap();
let ep = NodeEndpoint { address: SocketAddr::from_str("127.0.0.1:40446").unwrap(), udp_port: 40447 };
let mut discovery = Discovery::new(&key, ep.address.clone(), ep.clone(), 0);
for _ in 0..1200 {
discovery.add_node(NodeEntry { id: NodeId::random(), endpoint: ep.clone() });
}
assert!(Discovery::nearest_node_entries(&NodeId::new(), &discovery.node_buckets).len() <= 16);
let removed = discovery.check_expired(true).len();
assert!(removed > 0);
}
#[test]
fn packets() {
let key = KeyPair::create().unwrap();
let ep = NodeEndpoint { address: SocketAddr::from_str("127.0.0.1:40447").unwrap(), udp_port: 40447 };
let mut discovery = Discovery::new(&key, ep.address.clone(), ep.clone(), 0);
discovery.check_timestamps = false;
let from = SocketAddr::from_str("99.99.99.99:40445").unwrap();
let packet = "\
e9614ccfd9fc3e74360018522d30e1419a143407ffcce748de3e22116b7e8dc92ff74788c0b6663a\
aa3d67d641936511c8f8d6ad8698b820a7cf9e1be7155e9a241f556658c55428ec0563514365799a\
4be2be5a685a80971ddcfa80cb422cdd0101ec04cb847f000001820cfa8215a8d790000000000000\
000000000000000000018208ae820d058443b9a3550102\
".from_hex().unwrap();
assert!(discovery.on_packet(&packet, from.clone()).is_ok());
let packet = "\
577be4349c4dd26768081f58de4c6f375a7a22f3f7adda654d1428637412c3d7fe917cadc56d4e5e\
7ffae1dbe3efffb9849feb71b262de37977e7c7a44e677295680e9e38ab26bee2fcbae207fba3ff3\
d74069a50b902a82c9903ed37cc993c50001f83e82022bd79020010db83c4d001500000000abcdef\
12820cfa8215a8d79020010db885a308d313198a2e037073488208ae82823a8443b9a355c5010203\
040531b9019afde696e582a78fa8d95ea13ce3297d4afb8ba6433e4154caa5ac6431af1b80ba7602\
3fa4090c408f6b4bc3701562c031041d4702971d102c9ab7fa5eed4cd6bab8f7af956f7d565ee191\
7084a95398b6a21eac920fe3dd1345ec0a7ef39367ee69ddf092cbfe5b93e5e568ebc491983c09c7\
6d922dc3\
".from_hex().unwrap();
assert!(discovery.on_packet(&packet, from.clone()).is_ok());
let packet = "\
09b2428d83348d27cdf7064ad9024f526cebc19e4958f0fdad87c15eb598dd61d08423e0bf66b206\
9869e1724125f820d851c136684082774f870e614d95a2855d000f05d1648b2d5945470bc187c2d2\
216fbe870f43ed0909009882e176a46b0102f846d79020010db885a308d313198a2e037073488208\
ae82823aa0fbc914b16819237dcd8801d7e53f69e9719adecb3cc0e790c57e91ca4461c9548443b9\
a355c6010203c2040506a0c969a58f6f9095004c0177a6b47f451530cab38966a25cca5cb58f0555
42124e\
".from_hex().unwrap();
assert!(discovery.on_packet(&packet, from.clone()).is_ok());
let packet = "\
c7c44041b9f7c7e41934417ebac9a8e1a4c6298f74553f2fcfdcae6ed6fe53163eb3d2b52e39fe91\
831b8a927bf4fc222c3902202027e5e9eb812195f95d20061ef5cd31d502e47ecb61183f74a504fe\
04c51e73df81f25c4d506b26db4517490103f84eb840ca634cae0d49acb401d8a4c6b6fe8c55b70d\
115bf400769cc1400f3258cd31387574077f301b421bc84df7266c44e9e6d569fc56be0081290476\
7bf5ccd1fc7f8443b9a35582999983999999280dc62cc8255c73471e0a61da0c89acdc0e035e260a\
dd7fc0c04ad9ebf3919644c91cb247affc82b69bd2ca235c71eab8e49737c937a2c396\
".from_hex().unwrap();
assert!(discovery.on_packet(&packet, from.clone()).is_ok());
let packet = "\
c679fc8fe0b8b12f06577f2e802d34f6fa257e6137a995f6f4cbfc9ee50ed3710faf6e66f932c4c8\
d81d64343f429651328758b47d3dbc02c4042f0fff6946a50f4a49037a72bb550f3a7872363a83e1\
b9ee6469856c24eb4ef80b7535bcf99c0004f9015bf90150f84d846321163782115c82115db84031\
55e1427f85f10a5c9a7755877748041af1bcd8d474ec065eb33df57a97babf54bfd2103575fa8291\
15d224c523596b401065a97f74010610fce76382c0bf32f84984010203040101b840312c55512422\
cf9b8a4097e9a6ad79402e87a15ae909a4bfefa22398f03d20951933beea1e4dfa6f968212385e82\
9f04c2d314fc2d4e255e0d3bc08792b069dbf8599020010db83c4d001500000000abcdef12820d05\
820d05b84038643200b172dcfef857492156971f0e6aa2c538d8b74010f8e140811d53b98c765dd2\
d96126051913f44582e8c199ad7c6d6819e9a56483f637feaac9448aacf8599020010db885a308d3\
13198a2e037073488203e78203e8b8408dcab8618c3253b558d459da53bd8fa68935a719aff8b811\
197101a4b2b47dd2d47295286fc00cc081bb542d760717d1bdd6bec2c37cd72eca367d6dd3b9df73\
8443b9a355010203b525a138aa34383fec3d2719a0\
".from_hex().unwrap();
assert!(discovery.on_packet(&packet, from.clone()).is_ok());
}
}

69
util/src/network/error.rs
View File

@@ -15,23 +15,45 @@
// along with Parity. If not, see <http://www.gnu.org/licenses/>.
use io::IoError;
use crypto::CryptoError;
use rlp::*;
#[derive(Debug, Copy, Clone)]
#[derive(Debug, Copy, Clone, PartialEq, Eq)]
pub enum DisconnectReason
{
DisconnectRequested,
_TCPError,
_BadProtocol,
TCPError,
BadProtocol,
UselessPeer,
_TooManyPeers,
_DuplicatePeer,
_IncompatibleProtocol,
_NullIdentity,
_ClientQuit,
_UnexpectedIdentity,
_LocalIdentity,
TooManyPeers,
DuplicatePeer,
IncompatibleProtocol,
NullIdentity,
ClientQuit,
UnexpectedIdentity,
LocalIdentity,
PingTimeout,
Unknown,
}
impl DisconnectReason {
pub fn from_u8(n: u8) -> DisconnectReason {
match n {
0 => DisconnectReason::DisconnectRequested,
1 => DisconnectReason::TCPError,
2 => DisconnectReason::BadProtocol,
3 => DisconnectReason::UselessPeer,
4 => DisconnectReason::TooManyPeers,
5 => DisconnectReason::DuplicatePeer,
6 => DisconnectReason::IncompatibleProtocol,
7 => DisconnectReason::NullIdentity,
8 => DisconnectReason::ClientQuit,
9 => DisconnectReason::UnexpectedIdentity,
10 => DisconnectReason::LocalIdentity,
11 => DisconnectReason::PingTimeout,
_ => DisconnectReason::Unknown,
}
}
}
#[derive(Debug)]
@@ -41,6 +63,8 @@ pub enum NetworkError {
Auth,
/// Unrecognised protocol.
BadProtocol,
/// Message expired.
Expired,
/// Peer not found.
PeerNotFound,
/// Peer is diconnected.
@@ -61,3 +85,28 @@ impl From<IoError> for NetworkError {
}
}
impl From<CryptoError> for NetworkError {
fn from(_err: CryptoError) -> NetworkError {
NetworkError::Auth
}
}
#[test]
fn test_errors() {
assert_eq!(DisconnectReason::ClientQuit, DisconnectReason::from_u8(8));
let mut r = DisconnectReason::DisconnectRequested;
for i in 0 .. 20 {
r = DisconnectReason::from_u8(i);
}
assert_eq!(DisconnectReason::Unknown, r);
match <NetworkError as From<DecoderError>>::from(DecoderError::RlpIsTooBig) {
NetworkError::Auth => {},
_ => panic!("Unexpeceted error"),
}
match <NetworkError as From<CryptoError>>::from(CryptoError::InvalidSecret) {
NetworkError::Auth => {},
_ => panic!("Unexpeceted error"),
}
}

453
util/src/network/handshake.rs
View File

@@ -15,16 +15,18 @@
// along with Parity. If not, see <http://www.gnu.org/licenses/>.
use std::sync::Arc;
use rand::random;
use mio::*;
use mio::tcp::*;
use hash::*;
use rlp::*;
use sha3::Hashable;
use bytes::Bytes;
use crypto::*;
use crypto;
use network::connection::{Connection};
use network::host::{HostInfo};
use network::node::NodeId;
use network::node_table::NodeId;
use error::*;
use network::error::NetworkError;
use network::stats::NetworkStats;
@@ -36,8 +38,12 @@ enum HandshakeState {
New,
/// Waiting for auth packet
ReadingAuth,
/// Waiting for extended auth packet
ReadingAuthEip8,
/// Waiting for ack packet
ReadingAck,
/// Waiting for extended ack packet
ReadingAckEip8,
/// Ready to start a session
StartSession,
}
@@ -57,18 +63,25 @@ pub struct Handshake {
/// Connection nonce
pub nonce: H256,
/// Handshake public key
pub remote_public: Public,
pub remote_ephemeral: Public,
/// Remote connection nonce.
pub remote_nonce: H256,
/// Remote RLPx protocol version.
pub remote_version: u64,
/// A copy of received encryped auth packet
pub auth_cipher: Bytes,
/// A copy of received encryped ack packet
pub ack_cipher: Bytes
pub ack_cipher: Bytes,
/// This Handshake is marked for deleteion flag
pub expired: bool,
}
const AUTH_PACKET_SIZE: usize = 307;
const ACK_PACKET_SIZE: usize = 210;
const HANDSHAKE_TIMEOUT: u64 = 30000;
const V4_AUTH_PACKET_SIZE: usize = 307;
const V4_ACK_PACKET_SIZE: usize = 210;
const HANDSHAKE_TIMEOUT: u64 = 5000;
const PROTOCOL_VERSION: u64 = 4;
// Amount of bytes added when encrypting with encryptECIES.
const ECIES_OVERHEAD: usize = 113;
impl Handshake {
/// Create a new handshake object
@@ -80,23 +93,45 @@ impl Handshake {
state: HandshakeState::New,
ecdhe: try!(KeyPair::create()),
nonce: nonce.clone(),
remote_public: Public::new(),
remote_ephemeral: Public::new(),
remote_nonce: H256::new(),
remote_version: PROTOCOL_VERSION,
auth_cipher: Bytes::new(),
ack_cipher: Bytes::new(),
expired: false,
})
}
/// Get id of the remote node if known
pub fn id(&self) -> &NodeId {
&self.id
}
/// Get stream token id
pub fn token(&self) -> StreamToken {
self.connection.token()
}
/// Mark this handshake as inactive to be deleted lated.
pub fn set_expired(&mut self) {
self.expired = true;
}
/// Check if this handshake is expired.
pub fn expired(&self) -> bool {
self.expired
}
/// Start a handhsake
pub fn start<Message>(&mut self, io: &IoContext<Message>, host: &HostInfo, originated: bool) -> Result<(), UtilError> where Message: Send + Clone{
self.originated = originated;
io.register_timer(self.connection.token, HANDSHAKE_TIMEOUT).ok();
if originated {
try!(self.write_auth(host));
try!(self.write_auth(host.secret(), host.id()));
}
else {
self.state = HandshakeState::ReadingAuth;
self.connection.expect(AUTH_PACKET_SIZE);
self.connection.expect(V4_AUTH_PACKET_SIZE);
};
Ok(())
}
@@ -108,47 +143,64 @@ impl Handshake {
/// Readable IO handler. Drives the state change.
pub fn readable<Message>(&mut self, io: &IoContext<Message>, host: &HostInfo) -> Result<(), UtilError> where Message: Send + Clone {
io.clear_timer(self.connection.token).unwrap();
match self.state {
HandshakeState::ReadingAuth => {
if let Some(data) = try!(self.connection.readable()) {
try!(self.read_auth(host, &data));
try!(self.write_ack());
};
},
HandshakeState::ReadingAck => {
if let Some(data) = try!(self.connection.readable()) {
try!(self.read_ack(host, &data));
self.state = HandshakeState::StartSession;
};
},
HandshakeState::StartSession => {},
_ => { panic!("Unexpected state"); }
}
if self.state != HandshakeState::StartSession {
try!(io.update_registration(self.connection.token));
if !self.expired() {
io.clear_timer(self.connection.token).unwrap();
match self.state {
HandshakeState::New => {}
HandshakeState::ReadingAuth => {
if let Some(data) = try!(self.connection.readable()) {
try!(self.read_auth(host.secret(), &data));
};
},
HandshakeState::ReadingAuthEip8 => {
if let Some(data) = try!(self.connection.readable()) {
try!(self.read_auth_eip8(host.secret(), &data));
};
},
HandshakeState::ReadingAck => {
if let Some(data) = try!(self.connection.readable()) {
try!(self.read_ack(host.secret(), &data));
};
},
HandshakeState::ReadingAckEip8 => {
if let Some(data) = try!(self.connection.readable()) {
try!(self.read_ack_eip8(host.secret(), &data));
};
},
HandshakeState::StartSession => {},
}
if self.state != HandshakeState::StartSession {
try!(io.update_registration(self.connection.token));
}
}
Ok(())
}
/// Writabe IO handler.
pub fn writable<Message>(&mut self, io: &IoContext<Message>, _host: &HostInfo) -> Result<(), UtilError> where Message: Send + Clone {
io.clear_timer(self.connection.token).unwrap();
try!(self.connection.writable());
if self.state != HandshakeState::StartSession {
io.update_registration(self.connection.token).unwrap();
if !self.expired() {
io.clear_timer(self.connection.token).unwrap();
try!(self.connection.writable());
if self.state != HandshakeState::StartSession {
io.update_registration(self.connection.token).unwrap();
}
}
Ok(())
}
/// Register the socket with the event loop
pub fn register_socket<Host:Handler<Timeout=Token>>(&self, reg: Token, event_loop: &mut EventLoop<Host>) -> Result<(), UtilError> {
try!(self.connection.register_socket(reg, event_loop));
if !self.expired() {
try!(self.connection.register_socket(reg, event_loop));
}
Ok(())
}
/// Update socket registration with the event loop.
pub fn update_socket<Host:Handler<Timeout=Token>>(&self, reg: Token, event_loop: &mut EventLoop<Host>) -> Result<(), UtilError> {
try!(self.connection.update_socket(reg, event_loop));
if !self.expired() {
try!(self.connection.update_socket(reg, event_loop));
}
Ok(())
}
@@ -158,49 +210,106 @@ impl Handshake {
Ok(())
}
fn set_auth(&mut self, host_secret: &Secret, sig: &[u8], remote_public: &[u8], remote_nonce: &[u8], remote_version: u64) -> Result<(), UtilError> {
self.id.clone_from_slice(remote_public);
self.remote_nonce.clone_from_slice(remote_nonce);
self.remote_version = remote_version;
let shared = try!(ecdh::agree(host_secret, &self.id));
let signature = Signature::from_slice(sig);
self.remote_ephemeral = try!(ec::recover(&signature, &(&shared ^ &self.remote_nonce)));
Ok(())
}
/// Parse, validate and confirm auth message
fn read_auth(&mut self, host: &HostInfo, data: &[u8]) -> Result<(), UtilError> {
trace!(target:"net", "Received handshake auth to {:?}", self.connection.socket.peer_addr());
if data.len() != AUTH_PACKET_SIZE {
fn read_auth(&mut self, secret: &Secret, data: &[u8]) -> Result<(), UtilError> {
trace!(target:"network", "Received handshake auth from {:?}", self.connection.socket.peer_addr());
if data.len() != V4_AUTH_PACKET_SIZE {
debug!(target:"net", "Wrong auth packet size");
return Err(From::from(NetworkError::BadProtocol));
}
self.auth_cipher = data.to_vec();
let auth = try!(ecies::decrypt(host.secret(), data));
let (sig, rest) = auth.split_at(65);
let (hepubk, rest) = rest.split_at(32);
let (pubk, rest) = rest.split_at(64);
let (nonce, _) = rest.split_at(32);
self.id.clone_from_slice(pubk);
self.remote_nonce.clone_from_slice(nonce);
let shared = try!(ecdh::agree(host.secret(), &self.id));
let signature = Signature::from_slice(sig);
let spub = try!(ec::recover(&signature, &(&shared ^ &self.remote_nonce)));
self.remote_public = spub.clone();
if &spub.sha3()[..] != hepubk {
trace!(target:"net", "Handshake hash mismath with {:?}", self.connection.socket.peer_addr());
return Err(From::from(NetworkError::Auth));
};
match ecies::decrypt(secret, &[], data) {
Ok(auth) => {
let (sig, rest) = auth.split_at(65);
let (_, rest) = rest.split_at(32);
let (pubk, rest) = rest.split_at(64);
let (nonce, _) = rest.split_at(32);
try!(self.set_auth(secret, sig, pubk, nonce, PROTOCOL_VERSION));
try!(self.write_ack());
}
Err(_) => {
// Try to interpret as EIP-8 packet
let total = (((data[0] as u16) << 8 | (data[1] as u16)) as usize) + 2;
if total < V4_AUTH_PACKET_SIZE {
debug!(target:"net", "Wrong EIP8 auth packet size");
return Err(From::from(NetworkError::BadProtocol));
}
let rest = total - data.len();
self.state = HandshakeState::ReadingAuthEip8;
self.connection.expect(rest);
}
}
Ok(())
}
fn read_auth_eip8(&mut self, secret: &Secret, data: &[u8]) -> Result<(), UtilError> {
trace!(target:"network", "Received EIP8 handshake auth from {:?}", self.connection.socket.peer_addr());
self.auth_cipher.extend_from_slice(data);
let auth = try!(ecies::decrypt(secret, &self.auth_cipher[0..2], &self.auth_cipher[2..]));
let rlp = UntrustedRlp::new(&auth);
let signature: Signature = try!(rlp.val_at(0));
let remote_public: Public = try!(rlp.val_at(1));
let remote_nonce: H256 = try!(rlp.val_at(2));
let remote_version: u64 = try!(rlp.val_at(3));
try!(self.set_auth(secret, &signature, &remote_public, &remote_nonce, remote_version));
try!(self.write_ack_eip8());
Ok(())
}
/// Parse and validate ack message
fn read_ack(&mut self, host: &HostInfo, data: &[u8]) -> Result<(), UtilError> {
trace!(target:"net", "Received handshake auth to {:?}", self.connection.socket.peer_addr());
if data.len() != ACK_PACKET_SIZE {
fn read_ack(&mut self, secret: &Secret, data: &[u8]) -> Result<(), UtilError> {
trace!(target:"network", "Received handshake auth to {:?}", self.connection.socket.peer_addr());
if data.len() != V4_ACK_PACKET_SIZE {
debug!(target:"net", "Wrong ack packet size");
return Err(From::from(NetworkError::BadProtocol));
}
self.ack_cipher = data.to_vec();
let ack = try!(ecies::decrypt(host.secret(), data));
self.remote_public.clone_from_slice(&ack[0..64]);
self.remote_nonce.clone_from_slice(&ack[64..(64+32)]);
match ecies::decrypt(secret, &[], data) {
Ok(ack) => {
self.remote_ephemeral.clone_from_slice(&ack[0..64]);
self.remote_nonce.clone_from_slice(&ack[64..(64+32)]);
self.state = HandshakeState::StartSession;
}
Err(_) => {
// Try to interpret as EIP-8 packet
let total = (((data[0] as u16) << 8 | (data[1] as u16)) as usize) + 2;
if total < V4_ACK_PACKET_SIZE {
debug!(target:"net", "Wrong EIP8 ack packet size");
return Err(From::from(NetworkError::BadProtocol));
}
let rest = total - data.len();
self.state = HandshakeState::ReadingAckEip8;
self.connection.expect(rest);
}
}
Ok(())
}
fn read_ack_eip8(&mut self, secret: &Secret, data: &[u8]) -> Result<(), UtilError> {
trace!(target:"network", "Received EIP8 handshake auth from {:?}", self.connection.socket.peer_addr());
self.ack_cipher.extend_from_slice(data);
let ack = try!(ecies::decrypt(secret, &self.ack_cipher[0..2], &self.ack_cipher[2..]));
let rlp = UntrustedRlp::new(&ack);
self.remote_ephemeral = try!(rlp.val_at(0));
self.remote_nonce = try!(rlp.val_at(1));
self.remote_version = try!(rlp.val_at(2));
self.state = HandshakeState::StartSession;
Ok(())
}
/// Sends auth message
fn write_auth(&mut self, host: &HostInfo) -> Result<(), UtilError> {
trace!(target:"net", "Sending handshake auth to {:?}", self.connection.socket.peer_addr());
fn write_auth(&mut self, secret: &Secret, public: &Public) -> Result<(), UtilError> {
trace!(target:"network", "Sending handshake auth to {:?}", self.connection.socket.peer_addr());
let mut data = [0u8; /*Signature::SIZE*/ 65 + /*H256::SIZE*/ 32 + /*Public::SIZE*/ 64 + /*H256::SIZE*/ 32 + 1]; //TODO: use associated constants
let len = data.len();
{
@@ -211,23 +320,23 @@ impl Handshake {
let (nonce, _) = rest.split_at_mut(32);
// E(remote-pubk, S(ecdhe-random, ecdh-shared-secret^nonce) || H(ecdhe-random-pubk) || pubk || nonce || 0x0)
let shared = try!(crypto::ecdh::agree(host.secret(), &self.id));
let shared = try!(crypto::ecdh::agree(secret, &self.id));
try!(crypto::ec::sign(self.ecdhe.secret(), &(&shared ^ &self.nonce))).copy_to(sig);
self.ecdhe.public().sha3_into(hepubk);
host.id().copy_to(pubk);
public.copy_to(pubk);
self.nonce.copy_to(nonce);
}
let message = try!(crypto::ecies::encrypt(&self.id, &data));
let message = try!(crypto::ecies::encrypt(&self.id, &[], &data));
self.auth_cipher = message.clone();
self.connection.send(message);
self.connection.expect(ACK_PACKET_SIZE);
self.connection.expect(V4_ACK_PACKET_SIZE);
self.state = HandshakeState::ReadingAck;
Ok(())
}
/// Sends ack message
fn write_ack(&mut self) -> Result<(), UtilError> {
trace!(target:"net", "Sending handshake ack to {:?}", self.connection.socket.peer_addr());
trace!(target:"network", "Sending handshake ack to {:?}", self.connection.socket.peer_addr());
let mut data = [0u8; 1 + /*Public::SIZE*/ 64 + /*H256::SIZE*/ 32]; //TODO: use associated constants
let len = data.len();
{
@@ -237,10 +346,222 @@ impl Handshake {
self.ecdhe.public().copy_to(epubk);
self.nonce.copy_to(nonce);
}
let message = try!(crypto::ecies::encrypt(&self.id, &data));
let message = try!(crypto::ecies::encrypt(&self.id, &[], &data));
self.ack_cipher = message.clone();
self.connection.send(message);
self.state = HandshakeState::StartSession;
Ok(())
}
/// Sends EIP8 ack message
fn write_ack_eip8(&mut self) -> Result<(), UtilError> {
trace!(target:"network", "Sending EIP8 handshake ack to {:?}", self.connection.socket.peer_addr());
let mut rlp = RlpStream::new_list(3);
rlp.append(self.ecdhe.public());
rlp.append(&self.nonce);
rlp.append(&PROTOCOL_VERSION);
let pad_array = [0u8; 200];
let pad = &pad_array[0 .. 100 + random::<usize>() % 100];
rlp.append_raw(pad, 0);
let encoded = rlp.drain();
let len = (encoded.len() + ECIES_OVERHEAD) as u16;
let prefix = [ (len >> 8) as u8, (len & 0xff) as u8 ];
let message = try!(crypto::ecies::encrypt(&self.id, &prefix, &encoded));
self.ack_cipher.extend_from_slice(&prefix);
self.ack_cipher.extend_from_slice(&message);
self.connection.send(self.ack_cipher.clone());
self.state = HandshakeState::StartSession;
Ok(())
}
}
#[cfg(test)]
mod test {
use std::sync::Arc;
use std::str::FromStr;
use rustc_serialize::hex::FromHex;
use super::*;
use crypto::*;
use hash::*;
use std::net::SocketAddr;
use mio::tcp::TcpStream;
use network::stats::NetworkStats;
fn check_auth(h: &Handshake, version: u64) {
assert_eq!(h.id, Public::from_str("fda1cff674c90c9a197539fe3dfb53086ace64f83ed7c6eabec741f7f381cc803e52ab2cd55d5569bce4347107a310dfd5f88a010cd2ffd1005ca406f1842877").unwrap());
assert_eq!(h.remote_nonce, H256::from_str("7e968bba13b6c50e2c4cd7f241cc0d64d1ac25c7f5952df231ac6a2bda8ee5d6").unwrap());
assert_eq!(h.remote_ephemeral, Public::from_str("654d1044b69c577a44e5f01a1209523adb4026e70c62d1c13a067acabc09d2667a49821a0ad4b634554d330a15a58fe61f8a8e0544b310c6de7b0c8da7528a8d").unwrap());
assert_eq!(h.remote_version, version);
}
fn check_ack(h: &Handshake, version: u64) {
assert_eq!(h.remote_nonce, H256::from_str("559aead08264d5795d3909718cdd05abd49572e84fe55590eef31a88a08fdffd").unwrap());
assert_eq!(h.remote_ephemeral, Public::from_str("b6d82fa3409da933dbf9cb0140c5dde89f4e64aec88d476af648880f4a10e1e49fe35ef3e69e93dd300b4797765a747c6384a6ecf5db9c2690398607a86181e4").unwrap());
assert_eq!(h.remote_version, version);
}
fn create_handshake(to: Option<&Public>) -> Handshake {
let addr = SocketAddr::from_str("127.0.0.1:50556").unwrap();
let socket = TcpStream::connect(&addr).unwrap();
let nonce = H256::new();
Handshake::new(0, to, socket, &nonce, Arc::new(NetworkStats::new())).unwrap()
}
#[test]
fn test_handshake_auth_plain() {
let mut h = create_handshake(None);
let secret = Secret::from_str("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291").unwrap();
let auth =
"\
048ca79ad18e4b0659fab4853fe5bc58eb83992980f4c9cc147d2aa31532efd29a3d3dc6a3d89eaf\
913150cfc777ce0ce4af2758bf4810235f6e6ceccfee1acc6b22c005e9e3a49d6448610a58e98744\
ba3ac0399e82692d67c1f58849050b3024e21a52c9d3b01d871ff5f210817912773e610443a9ef14\
2e91cdba0bd77b5fdf0769b05671fc35f83d83e4d3b0b000c6b2a1b1bba89e0fc51bf4e460df3105\
c444f14be226458940d6061c296350937ffd5e3acaceeaaefd3c6f74be8e23e0f45163cc7ebd7622\
0f0128410fd05250273156d548a414444ae2f7dea4dfca2d43c057adb701a715bf59f6fb66b2d1d2\
0f2c703f851cbf5ac47396d9ca65b6260bd141ac4d53e2de585a73d1750780db4c9ee4cd4d225173\
a4592ee77e2bd94d0be3691f3b406f9bba9b591fc63facc016bfa8\
".from_hex().unwrap();
h.read_auth(&secret, &auth).unwrap();
assert_eq!(h.state, super::HandshakeState::StartSession);
check_auth(&h, 4);
}
#[test]
fn test_handshake_auth_eip8() {
let mut h = create_handshake(None);
let secret = Secret::from_str("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291").unwrap();
let auth =
"\
01b304ab7578555167be8154d5cc456f567d5ba302662433674222360f08d5f1534499d3678b513b\
0fca474f3a514b18e75683032eb63fccb16c156dc6eb2c0b1593f0d84ac74f6e475f1b8d56116b84\
9634a8c458705bf83a626ea0384d4d7341aae591fae42ce6bd5c850bfe0b999a694a49bbbaf3ef6c\
da61110601d3b4c02ab6c30437257a6e0117792631a4b47c1d52fc0f8f89caadeb7d02770bf999cc\
147d2df3b62e1ffb2c9d8c125a3984865356266bca11ce7d3a688663a51d82defaa8aad69da39ab6\
d5470e81ec5f2a7a47fb865ff7cca21516f9299a07b1bc63ba56c7a1a892112841ca44b6e0034dee\
70c9adabc15d76a54f443593fafdc3b27af8059703f88928e199cb122362a4b35f62386da7caad09\
c001edaeb5f8a06d2b26fb6cb93c52a9fca51853b68193916982358fe1e5369e249875bb8d0d0ec3\
6f917bc5e1eafd5896d46bd61ff23f1a863a8a8dcd54c7b109b771c8e61ec9c8908c733c0263440e\
2aa067241aaa433f0bb053c7b31a838504b148f570c0ad62837129e547678c5190341e4f1693956c\
3bf7678318e2d5b5340c9e488eefea198576344afbdf66db5f51204a6961a63ce072c8926c\
".from_hex().unwrap();
h.read_auth(&secret, &auth[0..super::V4_AUTH_PACKET_SIZE]).unwrap();
assert_eq!(h.state, super::HandshakeState::ReadingAuthEip8);
h.read_auth_eip8(&secret, &auth[super::V4_AUTH_PACKET_SIZE..]).unwrap();
assert_eq!(h.state, super::HandshakeState::StartSession);
check_auth(&h, 4);
}
#[test]
fn test_handshake_auth_eip8_2() {
let mut h = create_handshake(None);
let secret = Secret::from_str("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291").unwrap();
let auth =
"\
01b8044c6c312173685d1edd268aa95e1d495474c6959bcdd10067ba4c9013df9e40ff45f5bfd6f7\
2471f93a91b493f8e00abc4b80f682973de715d77ba3a005a242eb859f9a211d93a347fa64b597bf\
280a6b88e26299cf263b01b8dfdb712278464fd1c25840b995e84d367d743f66c0e54a586725b7bb\
f12acca27170ae3283c1073adda4b6d79f27656993aefccf16e0d0409fe07db2dc398a1b7e8ee93b\
cd181485fd332f381d6a050fba4c7641a5112ac1b0b61168d20f01b479e19adf7fdbfa0905f63352\
bfc7e23cf3357657455119d879c78d3cf8c8c06375f3f7d4861aa02a122467e069acaf513025ff19\
6641f6d2810ce493f51bee9c966b15c5043505350392b57645385a18c78f14669cc4d960446c1757\
1b7c5d725021babbcd786957f3d17089c084907bda22c2b2675b4378b114c601d858802a55345a15\
116bc61da4193996187ed70d16730e9ae6b3bb8787ebcaea1871d850997ddc08b4f4ea668fbf3740\
7ac044b55be0908ecb94d4ed172ece66fd31bfdadf2b97a8bc690163ee11f5b575a4b44e36e2bfb2\
f0fce91676fd64c7773bac6a003f481fddd0bae0a1f31aa27504e2a533af4cef3b623f4791b2cca6\
d490\
".from_hex().unwrap();
h.read_auth(&secret, &auth[0..super::V4_AUTH_PACKET_SIZE]).unwrap();
assert_eq!(h.state, super::HandshakeState::ReadingAuthEip8);
h.read_auth_eip8(&secret, &auth[super::V4_AUTH_PACKET_SIZE..]).unwrap();
assert_eq!(h.state, super::HandshakeState::StartSession);
check_auth(&h, 56);
let ack = h.ack_cipher.clone();
let total = (((ack[0] as u16) << 8 | (ack[1] as u16)) as usize) + 2;
assert_eq!(ack.len(), total);
}
#[test]
fn test_handshake_ack_plain() {
let remote = Public::from_str("fda1cff674c90c9a197539fe3dfb53086ace64f83ed7c6eabec741f7f381cc803e52ab2cd55d5569bce4347107a310dfd5f88a010cd2ffd1005ca406f1842877").unwrap();
let mut h = create_handshake(Some(&remote));
let secret = Secret::from_str("49a7b37aa6f6645917e7b807e9d1c00d4fa71f18343b0d4122a4d2df64dd6fee").unwrap();
let ack =
"\
049f8abcfa9c0dc65b982e98af921bc0ba6e4243169348a236abe9df5f93aa69d99cadddaa387662\
b0ff2c08e9006d5a11a278b1b3331e5aaabf0a32f01281b6f4ede0e09a2d5f585b26513cb794d963\
5a57563921c04a9090b4f14ee42be1a5461049af4ea7a7f49bf4c97a352d39c8d02ee4acc416388c\
1c66cec761d2bc1c72da6ba143477f049c9d2dde846c252c111b904f630ac98e51609b3b1f58168d\
dca6505b7196532e5f85b259a20c45e1979491683fee108e9660edbf38f3add489ae73e3dda2c71b\
d1497113d5c755e942d1\
".from_hex().unwrap();
h.read_ack(&secret, &ack).unwrap();
assert_eq!(h.state, super::HandshakeState::StartSession);
check_ack(&h, 4);
}
#[test]
fn test_handshake_ack_eip8() {
let remote = Public::from_str("fda1cff674c90c9a197539fe3dfb53086ace64f83ed7c6eabec741f7f381cc803e52ab2cd55d5569bce4347107a310dfd5f88a010cd2ffd1005ca406f1842877").unwrap();
let mut h = create_handshake(Some(&remote));
let secret = Secret::from_str("49a7b37aa6f6645917e7b807e9d1c00d4fa71f18343b0d4122a4d2df64dd6fee").unwrap();
let ack =
"\
01ea0451958701280a56482929d3b0757da8f7fbe5286784beead59d95089c217c9b917788989470\
b0e330cc6e4fb383c0340ed85fab836ec9fb8a49672712aeabbdfd1e837c1ff4cace34311cd7f4de\
05d59279e3524ab26ef753a0095637ac88f2b499b9914b5f64e143eae548a1066e14cd2f4bd7f814\
c4652f11b254f8a2d0191e2f5546fae6055694aed14d906df79ad3b407d94692694e259191cde171\
ad542fc588fa2b7333313d82a9f887332f1dfc36cea03f831cb9a23fea05b33deb999e85489e645f\
6aab1872475d488d7bd6c7c120caf28dbfc5d6833888155ed69d34dbdc39c1f299be1057810f34fb\
e754d021bfca14dc989753d61c413d261934e1a9c67ee060a25eefb54e81a4d14baff922180c395d\
3f998d70f46f6b58306f969627ae364497e73fc27f6d17ae45a413d322cb8814276be6ddd13b885b\
201b943213656cde498fa0e9ddc8e0b8f8a53824fbd82254f3e2c17e8eaea009c38b4aa0a3f306e8\
797db43c25d68e86f262e564086f59a2fc60511c42abfb3057c247a8a8fe4fb3ccbadde17514b7ac\
8000cdb6a912778426260c47f38919a91f25f4b5ffb455d6aaaf150f7e5529c100ce62d6d92826a7\
1778d809bdf60232ae21ce8a437eca8223f45ac37f6487452ce626f549b3b5fdee26afd2072e4bc7\
5833c2464c805246155289f4\
".from_hex().unwrap();
h.read_ack(&secret, &ack[0..super::V4_ACK_PACKET_SIZE]).unwrap();
assert_eq!(h.state, super::HandshakeState::ReadingAckEip8);
h.read_ack_eip8(&secret, &ack[super::V4_ACK_PACKET_SIZE..]).unwrap();
assert_eq!(h.state, super::HandshakeState::StartSession);
check_ack(&h, 4);
}
#[test]
fn test_handshake_ack_eip8_2() {
let remote = Public::from_str("fda1cff674c90c9a197539fe3dfb53086ace64f83ed7c6eabec741f7f381cc803e52ab2cd55d5569bce4347107a310dfd5f88a010cd2ffd1005ca406f1842877").unwrap();
let mut h = create_handshake(Some(&remote));
let secret = Secret::from_str("49a7b37aa6f6645917e7b807e9d1c00d4fa71f18343b0d4122a4d2df64dd6fee").unwrap();
let ack =
"\
01f004076e58aae772bb101ab1a8e64e01ee96e64857ce82b1113817c6cdd52c09d26f7b90981cd7\
ae835aeac72e1573b8a0225dd56d157a010846d888dac7464baf53f2ad4e3d584531fa203658fab0\
3a06c9fd5e35737e417bc28c1cbf5e5dfc666de7090f69c3b29754725f84f75382891c561040ea1d\
dc0d8f381ed1b9d0d4ad2a0ec021421d847820d6fa0ba66eaf58175f1b235e851c7e2124069fbc20\
2888ddb3ac4d56bcbd1b9b7eab59e78f2e2d400905050f4a92dec1c4bdf797b3fc9b2f8e84a482f3\
d800386186712dae00d5c386ec9387a5e9c9a1aca5a573ca91082c7d68421f388e79127a5177d4f8\
590237364fd348c9611fa39f78dcdceee3f390f07991b7b47e1daa3ebcb6ccc9607811cb17ce51f1\
c8c2c5098dbdd28fca547b3f58c01a424ac05f869f49c6a34672ea2cbbc558428aa1fe48bbfd6115\
8b1b735a65d99f21e70dbc020bfdface9f724a0d1fb5895db971cc81aa7608baa0920abb0a565c9c\
436e2fd13323428296c86385f2384e408a31e104670df0791d93e743a3a5194ee6b076fb6323ca59\
3011b7348c16cf58f66b9633906ba54a2ee803187344b394f75dd2e663a57b956cb830dd7a908d4f\
39a2336a61ef9fda549180d4ccde21514d117b6c6fd07a9102b5efe710a32af4eeacae2cb3b1dec0\
35b9593b48b9d3ca4c13d245d5f04169b0b1\
".from_hex().unwrap();
h.read_ack(&secret, &ack[0..super::V4_ACK_PACKET_SIZE]).unwrap();
assert_eq!(h.state, super::HandshakeState::ReadingAckEip8);
h.read_ack_eip8(&secret, &ack[super::V4_ACK_PACKET_SIZE..]).unwrap();
assert_eq!(h.state, super::HandshakeState::StartSession);
check_ack(&h, 57);
}
}

943
util/src/network/host.rs
View File

File diff suppressed because it is too large Load Diff

271
util/src/network/ip_utils.rs Normal file
View File

@@ -0,0 +1,271 @@
// Copyright 2015, 2016 Ethcore (UK) Ltd.
// This file is part of Parity.
// Parity is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// Parity is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
// You should have received a copy of the GNU General Public License
// along with Parity. If not, see <http://www.gnu.org/licenses/>.
// Based on original work by David Levy https://raw.githubusercontent.com/dlevy47/rust-interfaces
use std::net::{Ipv4Addr, Ipv6Addr, SocketAddr, SocketAddrV4, SocketAddrV6};
use std::io;
use igd::{PortMappingProtocol, search_gateway_from_timeout};
use std::time::Duration;
use network::node_table::{NodeEndpoint};
pub enum IpAddr{
V4(Ipv4Addr),
V6(Ipv6Addr),
}
/// Socket address extension for rustc beta. To be replaces with now unstable API
pub trait SocketAddrExt {
/// Returns true for the special 'unspecified' address 0.0.0.0.
fn is_unspecified_s(&self) -> bool;
/// Returns true if the address appears to be globally routable.
fn is_global_s(&self) -> bool;
}
impl SocketAddrExt for Ipv4Addr {
fn is_unspecified_s(&self) -> bool {
self.octets() == [0, 0, 0, 0]
}
fn is_global_s(&self) -> bool {
!self.is_private() && !self.is_loopback() && !self.is_link_local() &&
!self.is_broadcast() && !self.is_documentation()
}
}
impl SocketAddrExt for Ipv6Addr {
fn is_unspecified_s(&self) -> bool {
self.segments() == [0, 0, 0, 0, 0, 0, 0, 0]
}
fn is_global_s(&self) -> bool {
if self.is_multicast() {
self.segments()[0] & 0x000f == 14
} else {
!self.is_loopback() && !((self.segments()[0] & 0xffc0) == 0xfe80) &&
!((self.segments()[0] & 0xffc0) == 0xfec0) && !((self.segments()[0] & 0xfe00) == 0xfc00)
}
}
}
#[cfg(not(windows))]
mod getinterfaces {
use std::{mem, io, ptr};
use libc::{AF_INET, AF_INET6};
use libc::{getifaddrs, freeifaddrs, ifaddrs, sockaddr, sockaddr_in, sockaddr_in6};
use std::net::{Ipv4Addr, Ipv6Addr};
use super::IpAddr;
fn convert_sockaddr (sa: *mut sockaddr) -> Option<IpAddr> {
if sa == ptr::null_mut() { return None; }
let (addr, _) = match unsafe { *sa }.sa_family as i32 {
AF_INET => {
let sa: *const sockaddr_in = unsafe { mem::transmute(sa) };
let sa = & unsafe { *sa };
let (addr, port) = (sa.sin_addr.s_addr, sa.sin_port);
(IpAddr::V4(Ipv4Addr::new(
(addr & 0x000000FF) as u8,
((addr & 0x0000FF00) >> 8) as u8,
((addr & 0x00FF0000) >> 16) as u8,
((addr & 0xFF000000) >> 24) as u8)),
port)
},
AF_INET6 => {
let sa: *const sockaddr_in6 = unsafe { mem::transmute(sa) };
let sa = & unsafe { *sa };
let (addr, port) = (sa.sin6_addr.s6_addr, sa.sin6_port);
let addr: [u16; 8] = unsafe { mem::transmute(addr) };
(IpAddr::V6(Ipv6Addr::new(
addr[0],
addr[1],
addr[2],
addr[3],
addr[4],
addr[5],
addr[6],
addr[7])),
port)
},
_ => return None,
};
Some(addr)
}
fn convert_ifaddrs (ifa: *mut ifaddrs) -> Option<IpAddr> {
let ifa = unsafe { &mut *ifa };
convert_sockaddr(ifa.ifa_addr)
}
pub fn get_all() -> io::Result<Vec<IpAddr>> {
let mut ifap: *mut ifaddrs = unsafe { mem::zeroed() };
if unsafe { getifaddrs(&mut ifap as *mut _) } != 0 {
return Err(io::Error::last_os_error());
}
let mut ret = Vec::new();
let mut cur: *mut ifaddrs = ifap;
while cur != ptr::null_mut() {
if let Some(ip_addr) = convert_ifaddrs(cur) {
ret.push(ip_addr);
}
//TODO: do something else maybe?
cur = unsafe { (*cur).ifa_next };
}
unsafe { freeifaddrs(ifap) };
Ok(ret)
}
}
#[cfg(not(windows))]
fn get_if_addrs() -> io::Result<Vec<IpAddr>> {
getinterfaces::get_all()
}
#[cfg(windows)]
fn get_if_addrs() -> io::Result<Vec<IpAddr>> {
Ok(Vec::new())
}
/// Select the best available public address
pub fn select_public_address(port: u16) -> SocketAddr {
match get_if_addrs() {
Ok(list) => {
//prefer IPV4 bindings
for addr in &list { //TODO: use better criteria than just the first in the list
match *addr {
IpAddr::V4(a) if !a.is_unspecified_s() && !a.is_loopback() && !a.is_link_local() => {
return SocketAddr::V4(SocketAddrV4::new(a, port));
},
_ => {},
}
}
for addr in list {
match addr {
IpAddr::V6(a) if !a.is_unspecified_s() && !a.is_loopback() => {
return SocketAddr::V6(SocketAddrV6::new(a, port, 0, 0));
},
_ => {},
}
}
},
Err(e) => debug!("Error listing public interfaces: {:?}", e)
}
SocketAddr::V4(SocketAddrV4::new(Ipv4Addr::new(127, 0, 0, 1), port))
}
pub fn map_external_address(local: &NodeEndpoint) -> Option<NodeEndpoint> {
if let SocketAddr::V4(ref local_addr) = local.address {
match search_gateway_from_timeout(local_addr.ip().clone(), Duration::new(5, 0)) {
Err(ref err) => debug!("Gateway search error: {}", err),
Ok(gateway) => {
match gateway.get_external_ip() {
Err(ref err) => {
debug!("IP request error: {}", err);
},
Ok(external_addr) => {
match gateway.add_any_port(PortMappingProtocol::TCP, SocketAddrV4::new(local_addr.ip().clone(), local_addr.port()), 0, "Parity Node/TCP") {
Err(ref err) => {
debug!("Port mapping error: {}", err);
},
Ok(tcp_port) => {
match gateway.add_any_port(PortMappingProtocol::UDP, SocketAddrV4::new(local_addr.ip().clone(), local.udp_port), 0, "Parity Node/UDP") {
Err(ref err) => {
debug!("Port mapping error: {}", err);
},
Ok(udp_port) => {
return Some(NodeEndpoint { address: SocketAddr::V4(SocketAddrV4::new(external_addr, tcp_port)), udp_port: udp_port });
},
}
},
}
},
}
},
}
}
None
}
#[test]
fn can_select_public_address() {
let pub_address = select_public_address(40477);
assert!(pub_address.port() == 40477);
}
#[ignore]
#[test]
fn can_map_external_address_or_fail() {
let pub_address = select_public_address(40478);
let _ = map_external_address(&NodeEndpoint { address: pub_address, udp_port: 40478 });
}
#[test]
fn ipv4_properties() {
#![cfg_attr(feature="dev", allow(too_many_arguments))]
fn check(octets: &[u8; 4], unspec: bool, loopback: bool,
private: bool, link_local: bool, global: bool,
multicast: bool, broadcast: bool, documentation: bool) {
let ip = Ipv4Addr::new(octets[0], octets[1], octets[2], octets[3]);
assert_eq!(octets, &ip.octets());
assert_eq!(ip.is_unspecified_s(), unspec);
assert_eq!(ip.is_loopback(), loopback);
assert_eq!(ip.is_private(), private);
assert_eq!(ip.is_link_local(), link_local);
assert_eq!(ip.is_global_s(), global);
assert_eq!(ip.is_multicast(), multicast);
assert_eq!(ip.is_broadcast(), broadcast);
assert_eq!(ip.is_documentation(), documentation);
}
// address unspec loopbk privt linloc global multicast brdcast doc
check(&[0, 0, 0, 0], true, false, false, false, true, false, false, false);
check(&[0, 0, 0, 1], false, false, false, false, true, false, false, false);
check(&[1, 0, 0, 0], false, false, false, false, true, false, false, false);
check(&[10, 9, 8, 7], false, false, true, false, false, false, false, false);
check(&[127, 1, 2, 3], false, true, false, false, false, false, false, false);
check(&[172, 31, 254, 253], false, false, true, false, false, false, false, false);
check(&[169, 254, 253, 242], false, false, false, true, false, false, false, false);
check(&[192, 0, 2, 183], false, false, false, false, false, false, false, true);
check(&[192, 1, 2, 183], false, false, false, false, true, false, false, false);
check(&[192, 168, 254, 253], false, false, true, false, false, false, false, false);
check(&[198, 51, 100, 0], false, false, false, false, false, false, false, true);
check(&[203, 0, 113, 0], false, false, false, false, false, false, false, true);
check(&[203, 2, 113, 0], false, false, false, false, true, false, false, false);
check(&[224, 0, 0, 0], false, false, false, false, true, true, false, false);
check(&[239, 255, 255, 255], false, false, false, false, true, true, false, false);
check(&[255, 255, 255, 255], false, false, false, false, false, false, true, false);
}
#[test]
fn ipv6_properties() {
fn check(str_addr: &str, unspec: bool, loopback: bool, global: bool) {
let ip: Ipv6Addr = str_addr.parse().unwrap();
assert_eq!(str_addr, ip.to_string());
assert_eq!(ip.is_unspecified_s(), unspec);
assert_eq!(ip.is_loopback(), loopback);
assert_eq!(ip.is_global_s(), global);
}
// unspec loopbk global
check("::", true, false, true);
check("::1", false, true, false);
}

8
util/src/network/mod.rs
View File

@@ -56,7 +56,7 @@
//! }
//!
//! fn main () {
//! let mut service = NetworkService::<MyMessage>::start(NetworkConfiguration::new()).expect("Error creating network service");
//! let mut service = NetworkService::<MyMessage>::start(NetworkConfiguration::new_local()).expect("Error creating network service");
//! service.register_protocol(Arc::new(MyHandler), "myproto", &[1u8]);
//!
//! // Wait for quit condition
@@ -71,8 +71,9 @@ mod session;
mod discovery;
mod service;
mod error;
mod node;
mod node_table;
mod stats;
mod ip_utils;
#[cfg(test)]
mod tests;
@@ -88,6 +89,9 @@ pub use network::host::NetworkConfiguration;
pub use network::stats::NetworkStats;
use io::TimerToken;
pub use network::node_table::is_valid_node_url;
const PROTOCOL_VERSION: u32 = 4;
/// Network IO protocol handler. This needs to be implemented for each new subprotocol.
/// All the handler function are called from within IO event loop.

134
util/src/network/node.rs
View File

@@ -1,134 +0,0 @@
// Copyright 2015, 2016 Ethcore (UK) Ltd.
// This file is part of Parity.
// Parity is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// Parity is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
// You should have received a copy of the GNU General Public License
// along with Parity. If not, see <http://www.gnu.org/licenses/>.
use std::net::{SocketAddr, ToSocketAddrs};
use std::hash::{Hash, Hasher};
use std::str::{FromStr};
use hash::*;
use rlp::*;
use time::Tm;
use error::*;
/// Node public key
pub type NodeId = H512;
#[derive(Debug)]
/// Noe address info
pub struct NodeEndpoint {
/// IP(V4 or V6) address
pub address: SocketAddr,
/// Address as string (can be host name).
pub address_str: String,
/// Conneciton port.
pub udp_port: u16
}
impl FromStr for NodeEndpoint {
type Err = UtilError;
/// Create endpoint from string. Performs name resolution if given a host name.
fn from_str(s: &str) -> Result<NodeEndpoint, UtilError> {
let address = s.to_socket_addrs().map(|mut i| i.next());
match address {
Ok(Some(a)) => Ok(NodeEndpoint {
address: a,
address_str: s.to_owned(),
udp_port: a.port()
}),
Ok(_) => Err(UtilError::AddressResolve(None)),
Err(e) => Err(UtilError::AddressResolve(Some(e)))
}
}
}
#[derive(PartialEq, Eq, Copy, Clone)]
pub enum PeerType {
Required,
Optional
}
pub struct Node {
pub id: NodeId,
pub endpoint: NodeEndpoint,
pub peer_type: PeerType,
pub last_attempted: Option<Tm>,
}
impl FromStr for Node {
type Err = UtilError;
fn from_str(s: &str) -> Result<Self, Self::Err> {
let (id, endpoint) = if &s[0..8] == "enode://" && s.len() > 136 && &s[136..137] == "@" {
(try!(NodeId::from_str(&s[8..136])), try!(NodeEndpoint::from_str(&s[137..])))
}
else {
(NodeId::new(), try!(NodeEndpoint::from_str(s)))
};
Ok(Node {
id: id,
endpoint: endpoint,
peer_type: PeerType::Optional,
last_attempted: None,
})
}
}
impl PartialEq for Node {
fn eq(&self, other: &Self) -> bool {
self.id == other.id
}
}
impl Eq for Node { }
impl Hash for Node {
fn hash<H>(&self, state: &mut H) where H: Hasher {
self.id.hash(state)
}
}
#[cfg(test)]
mod tests {
use super::*;
use std::str::FromStr;
use std::net::*;
use hash::*;
#[test]
fn endpoint_parse() {
let endpoint = NodeEndpoint::from_str("123.99.55.44:7770");
assert!(endpoint.is_ok());
let v4 = match endpoint.unwrap().address {
SocketAddr::V4(v4address) => v4address,
_ => panic!("should ve v4 address")
};
assert_eq!(SocketAddrV4::new(Ipv4Addr::new(123, 99, 55, 44), 7770), v4);
}
#[test]
fn node_parse() {
let node = Node::from_str("enode://a979fb575495b8d6db44f750317d0f4622bf4c2aa3365d6af7c284339968eef29b69ad0dce72a4d8db5ebb4968de0e3bec910127f134779fbcb0cb6d3331163c@22.99.55.44:7770");
assert!(node.is_ok());
let node = node.unwrap();
let v4 = match node.endpoint.address {
SocketAddr::V4(v4address) => v4address,
_ => panic!("should ve v4 address")
};
assert_eq!(SocketAddrV4::new(Ipv4Addr::new(22, 99, 55, 44), 7770), v4);
assert_eq!(
H512::from_str("a979fb575495b8d6db44f750317d0f4622bf4c2aa3365d6af7c284339968eef29b69ad0dce72a4d8db5ebb4968de0e3bec910127f134779fbcb0cb6d3331163c").unwrap(),
node.id);
}
}

420
util/src/network/node_table.rs Normal file
View File

@@ -0,0 +1,420 @@
// Copyright 2015, 2016 Ethcore (UK) Ltd.
// This file is part of Parity.
// Parity is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// Parity is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
// You should have received a copy of the GNU General Public License
// along with Parity. If not, see <http://www.gnu.org/licenses/>.
use std::mem;
use std::slice::from_raw_parts;
use std::net::{SocketAddr, ToSocketAddrs, SocketAddrV4, SocketAddrV6, Ipv4Addr, Ipv6Addr};
use std::hash::{Hash, Hasher};
use std::str::{FromStr};
use std::collections::HashMap;
use std::fmt::{Display, Formatter};
use std::path::{PathBuf};
use std::fmt;
use std::fs;
use std::io::{Read, Write};
use hash::*;
use rlp::*;
use time::Tm;
use error::*;
use network::discovery::{TableUpdates, NodeEntry};
use network::ip_utils::*;
pub use rustc_serialize::json::Json;
/// Node public key
pub type NodeId = H512;
#[derive(Debug, Clone)]
/// Node address info
pub struct NodeEndpoint {
/// IP(V4 or V6) address
pub address: SocketAddr,
/// Conneciton port.
pub udp_port: u16
}
impl NodeEndpoint {
pub fn udp_address(&self) -> SocketAddr {
match self.address {
SocketAddr::V4(a) => SocketAddr::V4(SocketAddrV4::new(a.ip().clone(), self.udp_port)),
SocketAddr::V6(a) => SocketAddr::V6(SocketAddrV6::new(a.ip().clone(), self.udp_port, a.flowinfo(), a.scope_id())),
}
}
}
impl NodeEndpoint {
pub fn from_rlp(rlp: &UntrustedRlp) -> Result<Self, DecoderError> {
let tcp_port = try!(rlp.val_at::<u16>(2));
let udp_port = try!(rlp.val_at::<u16>(1));
let addr_bytes = try!(try!(rlp.at(0)).data());
let address = try!(match addr_bytes.len() {
4 => Ok(SocketAddr::V4(SocketAddrV4::new(Ipv4Addr::new(addr_bytes[0], addr_bytes[1], addr_bytes[2], addr_bytes[3]), tcp_port))),
16 => unsafe {
let o: *const u16 = mem::transmute(addr_bytes.as_ptr());
let o = from_raw_parts(o, 8);
Ok(SocketAddr::V6(SocketAddrV6::new(Ipv6Addr::new(o[0], o[1], o[2], o[3], o[4], o[5], o[6], o[7]), tcp_port, 0, 0)))
},
_ => Err(DecoderError::RlpInconsistentLengthAndData)
});
Ok(NodeEndpoint { address: address, udp_port: udp_port })
}
pub fn to_rlp(&self, rlp: &mut RlpStream) {
match self.address {
SocketAddr::V4(a) => {
rlp.append(&(&a.ip().octets()[..]));
}
SocketAddr::V6(a) => unsafe {
let o: *const u8 = mem::transmute(a.ip().segments().as_ptr());
rlp.append(&from_raw_parts(o, 16));
}
};
rlp.append(&self.udp_port);
rlp.append(&self.address.port());
}
pub fn to_rlp_list(&self, rlp: &mut RlpStream) {
rlp.begin_list(3);
self.to_rlp(rlp);
}
pub fn is_valid(&self) -> bool {
self.udp_port != 0 && self.address.port() != 0 &&
match self.address {
SocketAddr::V4(a) => !a.ip().is_unspecified_s(),
SocketAddr::V6(a) => !a.ip().is_unspecified_s()
}
}
pub fn is_global(&self) -> bool {
match self.address {
SocketAddr::V4(a) => a.ip().is_global_s(),
SocketAddr::V6(a) => a.ip().is_global_s()
}
}
}
impl FromStr for NodeEndpoint {
type Err = UtilError;
/// Create endpoint from string. Performs name resolution if given a host name.
fn from_str(s: &str) -> Result<NodeEndpoint, UtilError> {
let address = s.to_socket_addrs().map(|mut i| i.next());
match address {
Ok(Some(a)) => Ok(NodeEndpoint {
address: a,
udp_port: a.port()
}),
Ok(_) => Err(UtilError::AddressResolve(None)),
Err(e) => Err(UtilError::AddressResolve(Some(e)))
}
}
}
#[derive(PartialEq, Eq, Copy, Clone)]
pub enum PeerType {
_Required,
Optional
}
pub struct Node {
pub id: NodeId,
pub endpoint: NodeEndpoint,
pub peer_type: PeerType,
pub failures: u32,
pub last_attempted: Option<Tm>,
}
impl Node {
pub fn new(id: NodeId, endpoint: NodeEndpoint) -> Node {
Node {
id: id,
endpoint: endpoint,
peer_type: PeerType::Optional,
failures: 0,
last_attempted: None,
}
}
}
impl Display for Node {
fn fmt(&self, f: &mut Formatter) -> fmt::Result {
if self.endpoint.udp_port != self.endpoint.address.port() {
try!(write!(f, "enode://{}@{}+{}", self.id.hex(), self.endpoint.address, self.endpoint.udp_port));
} else {
try!(write!(f, "enode://{}@{}", self.id.hex(), self.endpoint.address));
}
Ok(())
}
}
impl FromStr for Node {
type Err = UtilError;
fn from_str(s: &str) -> Result<Self, Self::Err> {
let (id, endpoint) = if s.len() > 136 && &s[0..8] == "enode://" && &s[136..137] == "@" {
(try!(NodeId::from_str(&s[8..136])), try!(NodeEndpoint::from_str(&s[137..])))
}
else {
(NodeId::new(), try!(NodeEndpoint::from_str(s)))
};
Ok(Node {
id: id,
endpoint: endpoint,
peer_type: PeerType::Optional,
last_attempted: None,
failures: 0,
})
}
}
impl PartialEq for Node {
fn eq(&self, other: &Self) -> bool {
self.id == other.id
}
}
impl Eq for Node {}
impl Hash for Node {
fn hash<H>(&self, state: &mut H) where H: Hasher {
self.id.hash(state)
}
}
/// Node table backed by disk file.
pub struct NodeTable {
nodes: HashMap<NodeId, Node>,
path: Option<String>,
}
impl NodeTable {
pub fn new(path: Option<String>) -> NodeTable {
NodeTable {
path: path.clone(),
nodes: NodeTable::load(path),
}
}
/// Add a node to table
pub fn add_node(&mut self, mut node: Node) {
// preserve failure counter
let failures = self.nodes.get(&node.id).map_or(0, |n| n.failures);
node.failures = failures;
self.nodes.insert(node.id.clone(), node);
}
/// Returns node ids sorted by number of failures
pub fn nodes(&self) -> Vec<NodeId> {
let mut refs: Vec<&Node> = self.nodes.values().collect();
refs.sort_by(|a, b| a.failures.cmp(&b.failures));
refs.iter().map(|n| n.id.clone()).collect()
}
/// Unordered list of all entries
pub fn unordered_entries(&self) -> Vec<NodeEntry> {
// preserve failure counter
self.nodes.values().map(|n| NodeEntry { endpoint: n.endpoint.clone(), id: n.id.clone() }).collect()
}
/// Get particular node
pub fn get_mut(&mut self, id: &NodeId) -> Option<&mut Node> {
self.nodes.get_mut(id)
}
/// Apply table changes coming from discovery
pub fn update(&mut self, mut update: TableUpdates) {
for (_, node) in update.added.drain() {
let mut entry = self.nodes.entry(node.id.clone()).or_insert_with(|| Node::new(node.id.clone(), node.endpoint.clone()));
entry.endpoint = node.endpoint;
}
for r in update.removed {
self.nodes.remove(&r);
}
}
/// Increase failure counte for a node
pub fn note_failure(&mut self, id: &NodeId) {
if let Some(node) = self.nodes.get_mut(id) {
node.failures += 1;
}
}
fn save(&self) {
if let Some(ref path) = self.path {
let mut path_buf = PathBuf::from(path);
if let Err(e) = fs::create_dir_all(path_buf.as_path()) {
warn!("Error creating node table directory: {:?}", e);
return;
};
path_buf.push("nodes.json");
let mut json = String::new();
json.push_str("{\n");
json.push_str("\"nodes\": [\n");
let node_ids = self.nodes();
for i in 0 .. node_ids.len() {
let node = self.nodes.get(&node_ids[i]).unwrap();
json.push_str(&format!("\t{{ \"url\": \"{}\", \"failures\": {} }}{}\n", node, node.failures, if i == node_ids.len() - 1 {""} else {","}))
}
json.push_str("]\n");
json.push_str("}");
let mut file = match fs::File::create(path_buf.as_path()) {
Ok(file) => file,
Err(e) => {
warn!("Error creating node table file: {:?}", e);
return;
}
};
if let Err(e) = file.write(&json.into_bytes()) {
warn!("Error writing node table file: {:?}", e);
}
}
}
fn load(path: Option<String>) -> HashMap<NodeId, Node> {
let mut nodes: HashMap<NodeId, Node> = HashMap::new();
if let Some(path) = path {
let mut path_buf = PathBuf::from(path);
path_buf.push("nodes.json");
let mut file = match fs::File::open(path_buf.as_path()) {
Ok(file) => file,
Err(e) => {
debug!("Error opening node table file: {:?}", e);
return nodes;
}
};
let mut buf = String::new();
match file.read_to_string(&mut buf) {
Ok(_) => {},
Err(e) => {
warn!("Error reading node table file: {:?}", e);
return nodes;
}
}
let json = match Json::from_str(&buf) {
Ok(json) => json,
Err(e) => {
warn!("Error parsing node table file: {:?}", e);
return nodes;
}
};
if let Some(list) = json.as_object().and_then(|o| o.get("nodes")).and_then(|n| n.as_array()) {
for n in list.iter().filter_map(|n| n.as_object()) {
if let Some(url) = n.get("url").and_then(|u| u.as_string()) {
if let Ok(mut node) = Node::from_str(url) {
if let Some(failures) = n.get("failures").and_then(|f| f.as_u64()) {
node.failures = failures as u32;
}
nodes.insert(node.id.clone(), node);
}
}
}
}
}
nodes
}
}
impl Drop for NodeTable {
fn drop(&mut self) {
self.save();
}
}
/// Check if node url is valid
pub fn is_valid_node_url(url: &str) -> bool {
use std::str::FromStr;
Node::from_str(url).is_ok()
}
#[cfg(test)]
mod tests {
use super::*;
use std::str::FromStr;
use std::net::*;
use hash::*;
use devtools::*;
#[test]
fn endpoint_parse() {
let endpoint = NodeEndpoint::from_str("123.99.55.44:7770");
assert!(endpoint.is_ok());
let v4 = match endpoint.unwrap().address {
SocketAddr::V4(v4address) => v4address,
_ => panic!("should ve v4 address")
};
assert_eq!(SocketAddrV4::new(Ipv4Addr::new(123, 99, 55, 44), 7770), v4);
}
#[test]
fn node_parse() {
assert!(is_valid_node_url("enode://a979fb575495b8d6db44f750317d0f4622bf4c2aa3365d6af7c284339968eef29b69ad0dce72a4d8db5ebb4968de0e3bec910127f134779fbcb0cb6d3331163c@22.99.55.44:7770"));
let node = Node::from_str("enode://a979fb575495b8d6db44f750317d0f4622bf4c2aa3365d6af7c284339968eef29b69ad0dce72a4d8db5ebb4968de0e3bec910127f134779fbcb0cb6d3331163c@22.99.55.44:7770");
assert!(node.is_ok());
let node = node.unwrap();
let v4 = match node.endpoint.address {
SocketAddr::V4(v4address) => v4address,
_ => panic!("should ve v4 address")
};
assert_eq!(SocketAddrV4::new(Ipv4Addr::new(22, 99, 55, 44), 7770), v4);
assert_eq!(
H512::from_str("a979fb575495b8d6db44f750317d0f4622bf4c2aa3365d6af7c284339968eef29b69ad0dce72a4d8db5ebb4968de0e3bec910127f134779fbcb0cb6d3331163c").unwrap(),
node.id);
}
#[test]
fn table_failure_order() {
let node1 = Node::from_str("enode://a979fb575495b8d6db44f750317d0f4622bf4c2aa3365d6af7c284339968eef29b69ad0dce72a4d8db5ebb4968de0e3bec910127f134779fbcb0cb6d3331163c@22.99.55.44:7770").unwrap();
let node2 = Node::from_str("enode://b979fb575495b8d6db44f750317d0f4622bf4c2aa3365d6af7c284339968eef29b69ad0dce72a4d8db5ebb4968de0e3bec910127f134779fbcb0cb6d3331163c@22.99.55.44:7770").unwrap();
let node3 = Node::from_str("enode://c979fb575495b8d6db44f750317d0f4622bf4c2aa3365d6af7c284339968eef29b69ad0dce72a4d8db5ebb4968de0e3bec910127f134779fbcb0cb6d3331163c@22.99.55.44:7770").unwrap();
let id1 = H512::from_str("a979fb575495b8d6db44f750317d0f4622bf4c2aa3365d6af7c284339968eef29b69ad0dce72a4d8db5ebb4968de0e3bec910127f134779fbcb0cb6d3331163c").unwrap();
let id2 = H512::from_str("b979fb575495b8d6db44f750317d0f4622bf4c2aa3365d6af7c284339968eef29b69ad0dce72a4d8db5ebb4968de0e3bec910127f134779fbcb0cb6d3331163c").unwrap();
let id3 = H512::from_str("c979fb575495b8d6db44f750317d0f4622bf4c2aa3365d6af7c284339968eef29b69ad0dce72a4d8db5ebb4968de0e3bec910127f134779fbcb0cb6d3331163c").unwrap();
let mut table = NodeTable::new(None);
table.add_node(node3);
table.add_node(node1);
table.add_node(node2);
table.note_failure(&id1);
table.note_failure(&id1);
table.note_failure(&id2);
let r = table.nodes();
assert_eq!(r[0][..], id3[..]);
assert_eq!(r[1][..], id2[..]);
assert_eq!(r[2][..], id1[..]);
}
#[test]
fn table_save_load() {
let temp_path = RandomTempPath::create_dir();
let node1 = Node::from_str("enode://a979fb575495b8d6db44f750317d0f4622bf4c2aa3365d6af7c284339968eef29b69ad0dce72a4d8db5ebb4968de0e3bec910127f134779fbcb0cb6d3331163c@22.99.55.44:7770").unwrap();
let node2 = Node::from_str("enode://b979fb575495b8d6db44f750317d0f4622bf4c2aa3365d6af7c284339968eef29b69ad0dce72a4d8db5ebb4968de0e3bec910127f134779fbcb0cb6d3331163c@22.99.55.44:7770").unwrap();
let id1 = H512::from_str("a979fb575495b8d6db44f750317d0f4622bf4c2aa3365d6af7c284339968eef29b69ad0dce72a4d8db5ebb4968de0e3bec910127f134779fbcb0cb6d3331163c").unwrap();
let id2 = H512::from_str("b979fb575495b8d6db44f750317d0f4622bf4c2aa3365d6af7c284339968eef29b69ad0dce72a4d8db5ebb4968de0e3bec910127f134779fbcb0cb6d3331163c").unwrap();
{
let mut table = NodeTable::new(Some(temp_path.as_path().to_str().unwrap().to_owned()));
table.add_node(node1);
table.add_node(node2);
table.note_failure(&id2);
}
{
let table = NodeTable::new(Some(temp_path.as_path().to_str().unwrap().to_owned()));
let r = table.nodes();
assert_eq!(r[0][..], id1[..]);
assert_eq!(r[1][..], id2[..]);
}
}
}

22
util/src/network/service.rs
View File

@@ -28,6 +28,7 @@ use io::*;
pub struct NetworkService<Message> where Message: Send + Sync + Clone + 'static {
io_service: IoService<NetworkIoMessage<Message>>,
host_info: String,
host: Arc<Host<Message>>,
stats: Arc<NetworkStats>,
panic_handler: Arc<PanicHandler>
}
@@ -39,16 +40,16 @@ impl<Message> NetworkService<Message> where Message: Send + Sync + Clone + 'stat
let mut io_service = try!(IoService::<NetworkIoMessage<Message>>::start());
panic_handler.forward_from(&io_service);
let host = Arc::new(Host::new(config));
let host = Arc::new(try!(Host::new(config)));
let stats = host.stats().clone();
let host_info = host.client_version();
info!("Host ID={:?}", host.client_id());
try!(io_service.register_handler(host));
try!(io_service.register_handler(host.clone()));
Ok(NetworkService {
io_service: io_service,
host_info: host_info,
stats: stats,
panic_handler: panic_handler
panic_handler: panic_handler,
host: host,
})
}
@@ -72,12 +73,21 @@ impl<Message> NetworkService<Message> where Message: Send + Sync + Clone + 'stat
&mut self.io_service
}
/// Returns underlying io service.
/// Returns network statistics.
pub fn stats(&self) -> &NetworkStats {
&self.stats
}
}
/// Returns external url if available.
pub fn external_url(&self) -> Option<String> {
self.host.external_url()
}
/// Returns external url if available.
pub fn local_url(&self) -> String {
self.host.local_url()
}
}
impl<Message> MayPanic for NetworkService<Message> where Message: Send + Sync + Clone + 'static {
fn on_panic<F>(&self, closure: F) where F: OnPanicListener {

79
util/src/network/session.rs
View File

@@ -14,6 +14,8 @@
// You should have received a copy of the GNU General Public License
// along with Parity. If not, see <http://www.gnu.org/licenses/>.
use std::net::SocketAddr;
use std::io;
use mio::*;
use hash::*;
use rlp::*;
@@ -23,7 +25,7 @@ use error::*;
use io::{IoContext, StreamToken};
use network::error::{NetworkError, DisconnectReason};
use network::host::*;
use network::node::NodeId;
use network::node_table::NodeId;
use time;
const PING_TIMEOUT_SEC: u64 = 30;
@@ -39,6 +41,8 @@ pub struct Session {
connection: EncryptedConnection,
/// Session ready flag. Set after successfull Hello packet exchange
had_hello: bool,
/// Session is no longer active flag.
expired: bool,
ping_time_ns: u64,
pong_time_ns: Option<u64>,
}
@@ -89,7 +93,7 @@ impl Decodable for PeerCapabilityInfo {
}
}
#[derive(Debug, PartialEq, Eq)]
#[derive(Debug)]
struct SessionCapabilityInfo {
pub protocol: &'static str,
pub version: u8,
@@ -107,8 +111,9 @@ const PACKET_USER: u8 = 0x10;
const PACKET_LAST: u8 = 0x7f;
impl Session {
/// Create a new session out of comepleted handshake. Consumes handshake object.
pub fn new<Message>(h: Handshake, _io: &IoContext<Message>, host: &HostInfo) -> Result<Session, UtilError> where Message: Send + Sync + Clone {
/// Create a new session out of comepleted handshake. This clones the handshake connection object
/// and leaves the handhsake in limbo to be deregistered from the event loop.
pub fn new(h: &mut Handshake, host: &HostInfo) -> Result<Session, UtilError> {
let id = h.id.clone();
let connection = try!(EncryptedConnection::new(h));
let mut session = Session {
@@ -123,19 +128,48 @@ impl Session {
},
ping_time_ns: 0,
pong_time_ns: None,
expired: false,
};
try!(session.write_hello(host));
try!(session.send_ping());
Ok(session)
}
/// Get id of the remote peer
pub fn id(&self) -> &NodeId {
&self.info.id
}
/// Check if session is ready to send/receive data
pub fn is_ready(&self) -> bool {
self.had_hello
}
/// Mark this session as inactive to be deleted lated.
pub fn set_expired(&mut self) {
self.expired = true;
}
/// Check if this session is expired.
pub fn expired(&self) -> bool {
self.expired
}
/// Replace socket token
pub fn set_token(&mut self, token: StreamToken) {
self.connection.set_token(token);
}
/// Get remote peer address
pub fn remote_addr(&self) -> io::Result<SocketAddr> {
self.connection.remote_addr()
}
/// Readable IO handler. Returns packet data if available.
pub fn readable<Message>(&mut self, io: &IoContext<Message>, host: &HostInfo) -> Result<SessionData, UtilError> where Message: Send + Sync + Clone {
if self.expired() {
return Ok(SessionData::None)
}
match try!(self.connection.readable(io)) {
Some(data) => Ok(try!(self.read_packet(data, host))),
None => Ok(SessionData::None)
@@ -144,6 +178,9 @@ impl Session {
/// Writable IO handler. Sends pending packets.
pub fn writable<Message>(&mut self, io: &IoContext<Message>, _host: &HostInfo) -> Result<(), UtilError> where Message: Send + Sync + Clone {
if self.expired() {
return Ok(())
}
self.connection.writable(io)
}
@@ -152,8 +189,20 @@ impl Session {
self.info.capabilities.iter().any(|c| c.protocol == protocol)
}
/// Register the session socket with the event loop
pub fn register_socket<Host:Handler<Timeout = Token>>(&self, reg: Token, event_loop: &mut EventLoop<Host>) -> Result<(), UtilError> {
if self.expired() {
return Ok(());
}
try!(self.connection.register_socket(reg, event_loop));
Ok(())
}
/// Update registration with the event loop. Should be called at the end of the IO handler.
pub fn update_socket<Host:Handler>(&self, reg:Token, event_loop: &mut EventLoop<Host>) -> Result<(), UtilError> {
if self.expired() {
return Ok(());
}
self.connection.update_socket(reg, event_loop)
}
@@ -164,6 +213,13 @@ impl Session {
/// Send a protocol packet to peer.
pub fn send_packet(&mut self, protocol: &str, packet_id: u8, data: &[u8]) -> Result<(), UtilError> {
if self.info.capabilities.is_empty() || !self.had_hello {
debug!(target: "network", "Sending to unconfirmed session {}, protocol: {}, packet: {}", self.token(), protocol, packet_id);
return Err(From::from(NetworkError::BadProtocol));
}
if self.expired() {
return Err(From::from(NetworkError::Expired));
}
let mut i = 0usize;
while protocol != self.info.capabilities[i].protocol {
i += 1;
@@ -214,7 +270,11 @@ impl Session {
try!(self.read_hello(&rlp, host));
Ok(SessionData::Ready)
},
PACKET_DISCONNECT => Err(From::from(NetworkError::Disconnect(DisconnectReason::DisconnectRequested))),
PACKET_DISCONNECT => {
let rlp = UntrustedRlp::new(&packet.data[1..]);
let reason: u8 = try!(rlp.val_at(0));
Err(From::from(NetworkError::Disconnect(DisconnectReason::from_u8(reason))))
}
PACKET_PING => {
try!(self.send_pong());
Ok(SessionData::None)
@@ -255,7 +315,7 @@ impl Session {
.append(&host.protocol_version)
.append(&host.client_version)
.append(&host.capabilities)
.append(&host.listen_port)
.append(&host.local_endpoint.address.port())
.append(host.id());
self.connection.send_packet(&rlp.out())
}
@@ -298,10 +358,15 @@ impl Session {
offset += caps[i].packet_count;
i += 1;
}
trace!(target: "net", "Hello: {} v{} {} {:?}", client_version, protocol, id, caps);
trace!(target: "network", "Hello: {} v{} {} {:?}", client_version, protocol, id, caps);
self.info.client_version = client_version;
self.info.capabilities = caps;
if self.info.capabilities.is_empty() {
trace!(target: "network", "No common capabilities with peer.");
return Err(From::from(self.disconnect(DisconnectReason::UselessPeer)));
}
if protocol != host.protocol_version {
trace!(target: "network", "Peer protocol version mismatch: {}", protocol);
return Err(From::from(self.disconnect(DisconnectReason::UselessPeer)));
}
self.had_hello = true;

81
util/src/network/tests.rs
View File

@@ -23,17 +23,10 @@ use io::TimerToken;
use crypto::KeyPair;
pub struct TestProtocol {
drop_session: bool,
pub packet: Mutex<Bytes>,
pub got_timeout: AtomicBool,
}
impl Default for TestProtocol {
fn default() -> Self {
TestProtocol {
packet: Mutex::new(Vec::new()),
got_timeout: AtomicBool::new(false),
}
}
pub got_disconnect: AtomicBool,
}
#[derive(Clone)]
@@ -42,9 +35,17 @@ pub struct TestProtocolMessage {
}
impl TestProtocol {
pub fn new(drop_session: bool) -> Self {
TestProtocol {
packet: Mutex::new(Vec::new()),
got_timeout: AtomicBool::new(false),
got_disconnect: AtomicBool::new(false),
drop_session: drop_session,
}
}
/// Creates and register protocol with the network service
pub fn register(service: &mut NetworkService<TestProtocolMessage>) -> Arc<TestProtocol> {
let handler = Arc::new(TestProtocol::default());
pub fn register(service: &mut NetworkService<TestProtocolMessage>, drop_session: bool) -> Arc<TestProtocol> {
let handler = Arc::new(TestProtocol::new(drop_session));
service.register_protocol(handler.clone(), "test", &[42u8, 43u8]).expect("Error registering test protocol handler");
handler
}
@@ -56,6 +57,10 @@ impl TestProtocol {
pub fn got_timeout(&self) -> bool {
self.got_timeout.load(AtomicOrdering::Relaxed)
}
pub fn got_disconnect(&self) -> bool {
self.got_disconnect.load(AtomicOrdering::Relaxed)
}
}
impl NetworkProtocolHandler<TestProtocolMessage> for TestProtocol {
@@ -68,15 +73,22 @@ impl NetworkProtocolHandler<TestProtocolMessage> for TestProtocol {
self.packet.lock().unwrap().extend(data);
}
fn connected(&self, io: &NetworkContext<TestProtocolMessage>, _peer: &PeerId) {
io.respond(33, "hello".to_owned().into_bytes()).unwrap();
fn connected(&self, io: &NetworkContext<TestProtocolMessage>, peer: &PeerId) {
assert!(io.peer_info(*peer).contains("Parity"));
if self.drop_session {
io.disconnect_peer(*peer)
} else {
io.respond(33, "hello".to_owned().into_bytes()).unwrap();
}
}
fn disconnected(&self, _io: &NetworkContext<TestProtocolMessage>, _peer: &PeerId) {
self.got_disconnect.store(true, AtomicOrdering::Relaxed);
}
/// Timer function called after a timeout created with `NetworkContext::timeout`.
fn timeout(&self, _io: &NetworkContext<TestProtocolMessage>, timer: TimerToken) {
fn timeout(&self, io: &NetworkContext<TestProtocolMessage>, timer: TimerToken) {
io.message(TestProtocolMessage { payload: 22 });
assert_eq!(timer, 0);
self.got_timeout.store(true, AtomicOrdering::Relaxed);
}
@@ -85,34 +97,55 @@ impl NetworkProtocolHandler<TestProtocolMessage> for TestProtocol {
#[test]
fn net_service() {
let mut service = NetworkService::<TestProtocolMessage>::start(NetworkConfiguration::new()).expect("Error creating network service");
service.register_protocol(Arc::new(TestProtocol::default()), "myproto", &[1u8]).unwrap();
let mut service = NetworkService::<TestProtocolMessage>::start(NetworkConfiguration::new_local()).expect("Error creating network service");
service.register_protocol(Arc::new(TestProtocol::new(false)), "myproto", &[1u8]).unwrap();
}
#[test]
fn net_connect() {
::log::init_log();
let key1 = KeyPair::create().unwrap();
let mut config1 = NetworkConfiguration::new_with_port(30344);
let mut config1 = NetworkConfiguration::new_local();
config1.use_secret = Some(key1.secret().clone());
config1.boot_nodes = vec![ ];
let mut config2 = NetworkConfiguration::new_with_port(30345);
config2.boot_nodes = vec![ format!("enode://{}@127.0.0.1:30344", key1.public().hex()) ];
let mut service1 = NetworkService::<TestProtocolMessage>::start(config1).unwrap();
let handler1 = TestProtocol::register(&mut service1, false);
let mut config2 = NetworkConfiguration::new_local();
info!("net_connect: local URL: {}", service1.local_url());
config2.boot_nodes = vec![ service1.local_url() ];
let mut service2 = NetworkService::<TestProtocolMessage>::start(config2).unwrap();
let handler1 = TestProtocol::register(&mut service1);
let handler2 = TestProtocol::register(&mut service2);
while !handler1.got_packet() && !handler2.got_packet() {
let handler2 = TestProtocol::register(&mut service2, false);
while !handler1.got_packet() && !handler2.got_packet() && (service1.stats().sessions() == 0 || service2.stats().sessions() == 0) {
thread::sleep(Duration::from_millis(50));
}
assert!(service1.stats().sessions() >= 1);
assert!(service2.stats().sessions() >= 1);
}
#[test]
fn net_disconnect() {
let key1 = KeyPair::create().unwrap();
let mut config1 = NetworkConfiguration::new_local();
config1.use_secret = Some(key1.secret().clone());
config1.boot_nodes = vec![ ];
let mut service1 = NetworkService::<TestProtocolMessage>::start(config1).unwrap();
let handler1 = TestProtocol::register(&mut service1, false);
let mut config2 = NetworkConfiguration::new_local();
config2.boot_nodes = vec![ service1.local_url() ];
let mut service2 = NetworkService::<TestProtocolMessage>::start(config2).unwrap();
let handler2 = TestProtocol::register(&mut service2, true);
while !(handler1.got_disconnect() && handler2.got_disconnect()) {
thread::sleep(Duration::from_millis(50));
}
assert!(handler1.got_disconnect());
assert!(handler2.got_disconnect());
}
#[test]
fn net_timeout() {
let config = NetworkConfiguration::new_with_port(30346);
let config = NetworkConfiguration::new_local();
let mut service = NetworkService::<TestProtocolMessage>::start(config).unwrap();
let handler = TestProtocol::register(&mut service);
let handler = TestProtocol::register(&mut service, false);
while !handler.got_timeout() {
thread::sleep(Duration::from_millis(50));
}

6
util/src/nibbleslice.rs
View File

@@ -69,13 +69,13 @@ impl<'a> Iterator for NibbleSliceIterator<'a> {
impl<'a, 'view> NibbleSlice<'a> where 'a: 'view {
/// Create a new nibble slice with the given byte-slice.
pub fn new(data: &[u8]) -> NibbleSlice { NibbleSlice::new_offset(data, 0) }
pub fn new(data: &'a [u8]) -> Self { NibbleSlice::new_offset(data, 0) }
/// Create a new nibble slice with the given byte-slice with a nibble offset.
pub fn new_offset(data: &'a [u8], offset: usize) -> NibbleSlice { NibbleSlice{data: data, offset: offset, data_encode_suffix: &b""[..], offset_encode_suffix: 0} }
pub fn new_offset(data: &'a [u8], offset: usize) -> Self { NibbleSlice{data: data, offset: offset, data_encode_suffix: &b""[..], offset_encode_suffix: 0} }
/// Create a composed nibble slice; one followed by the other.
pub fn new_composed(a: &'a NibbleSlice, b: &'a NibbleSlice) -> NibbleSlice<'a> { NibbleSlice{data: a.data, offset: a.offset, data_encode_suffix: b.data, offset_encode_suffix: b.offset} }
pub fn new_composed(a: &'a NibbleSlice, b: &'a NibbleSlice) -> Self { NibbleSlice{data: a.data, offset: a.offset, data_encode_suffix: b.data, offset_encode_suffix: b.offset} }
/*pub fn new_composed_bytes_offset(a: &NibbleSlice, b: &NibbleSlice) -> (Bytes, usize) {
let r: Vec<u8>::with_capacity((a.len() + b.len() + 1) / 2);

20
util/src/numbers.rs Normal file
View File

@@ -0,0 +1,20 @@
// Copyright 2015, 2016 Ethcore (UK) Ltd.
// This file is part of Parity.
// Parity is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// Parity is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
// You should have received a copy of the GNU General Public License
// along with Parity. If not, see <http://www.gnu.org/licenses/>.
//! Utils number types.
pub use hash::*;
pub use bigint::uint::*;

68
util/src/overlaydb.rs
View File

@@ -26,7 +26,7 @@ use std::ops::*;
use std::sync::*;
use std::env;
use std::collections::HashMap;
use rocksdb::{DB, Writable, IteratorMode};
use kvdb::{Database, DBTransaction};
/// Implementation of the HashDB trait for a disk-backed database with a memory overlay.
///
@@ -36,17 +36,18 @@ use rocksdb::{DB, Writable, IteratorMode};
///
/// `lookup()` and `contains()` maintain normal behaviour - all `insert()` and `remove()`
/// queries have an immediate effect in terms of these functions.
#[derive(Clone)]
pub struct OverlayDB {
overlay: MemoryDB,
backing: Arc<DB>,
backing: Arc<Database>,
}
impl OverlayDB {
/// Create a new instance of OverlayDB given a `backing` database.
pub fn new(backing: DB) -> OverlayDB { Self::new_with_arc(Arc::new(backing)) }
pub fn new(backing: Database) -> OverlayDB { Self::new_with_arc(Arc::new(backing)) }
/// Create a new instance of OverlayDB given a `backing` database.
pub fn new_with_arc(backing: Arc<DB>) -> OverlayDB {
pub fn new_with_arc(backing: Arc<Database>) -> OverlayDB {
OverlayDB{ overlay: MemoryDB::new(), backing: backing }
}
@@ -54,7 +55,37 @@ impl OverlayDB {
pub fn new_temp() -> OverlayDB {
let mut dir = env::temp_dir();
dir.push(H32::random().hex());
Self::new(DB::open_default(dir.to_str().unwrap()).unwrap())
Self::new(Database::open_default(dir.to_str().unwrap()).unwrap())
}
/// Commit all operations to given batch.
pub fn commit_to_batch(&mut self, batch: &DBTransaction) -> Result<u32, UtilError> {
let mut ret = 0u32;
let mut deletes = 0usize;
for i in self.overlay.drain().into_iter() {
let (key, (value, rc)) = i;
if rc != 0 {
match self.payload(&key) {
Some(x) => {
let (back_value, back_rc) = x;
let total_rc: i32 = back_rc as i32 + rc;
if total_rc < 0 {
return Err(From::from(BaseDataError::NegativelyReferencedHash(key)));
}
deletes += if self.put_payload_in_batch(batch, &key, (back_value, total_rc as u32)) {1} else {0};
}
None => {
if rc < 0 {
return Err(From::from(BaseDataError::NegativelyReferencedHash(key)));
}
self.put_payload_in_batch(batch, &key, (value, rc as u32));
}
};
ret += 1;
}
}
trace!("OverlayDB::commit() deleted {} nodes", deletes);
Ok(ret)
}
/// Commit all memory operations to the backing database.
@@ -95,13 +126,13 @@ impl OverlayDB {
let (back_value, back_rc) = x;
let total_rc: i32 = back_rc as i32 + rc;
if total_rc < 0 {
return Err(From::from(BaseDataError::NegativelyReferencedHash));
return Err(From::from(BaseDataError::NegativelyReferencedHash(key)));
}
deletes += if self.put_payload(&key, (back_value, total_rc as u32)) {1} else {0};
}
None => {
if rc < 0 {
return Err(From::from(BaseDataError::NegativelyReferencedHash));
return Err(From::from(BaseDataError::NegativelyReferencedHash(key)));
}
self.put_payload(&key, (value, rc as u32));
}
@@ -136,6 +167,9 @@ impl OverlayDB {
/// ```
pub fn revert(&mut self) { self.overlay.clear(); }
/// Get the number of references that would be committed.
pub fn commit_refs(&self, key: &H256) -> i32 { self.overlay.raw(&key).map_or(0, |&(_, refs)| refs) }
/// Get the refs and value of the given key.
fn payload(&self, key: &H256) -> Option<(Bytes, u32)> {
self.backing.get(&key.bytes())
@@ -146,7 +180,21 @@ impl OverlayDB {
})
}
/// Get the refs and value of the given key.
/// Put the refs and value of the given key, possibly deleting it from the db.
fn put_payload_in_batch(&self, batch: &DBTransaction, key: &H256, payload: (Bytes, u32)) -> bool {
if payload.1 > 0 {
let mut s = RlpStream::new_list(2);
s.append(&payload.1);
s.append(&payload.0);
batch.put(&key.bytes(), s.as_raw()).expect("Low-level database error. Some issue with your hard disk?");
false
} else {
batch.delete(&key.bytes()).expect("Low-level database error. Some issue with your hard disk?");
true
}
}
/// Put the refs and value of the given key, possibly deleting it from the db.
fn put_payload(&self, key: &H256, payload: (Bytes, u32)) -> bool {
if payload.1 > 0 {
let mut s = RlpStream::new_list(2);
@@ -164,7 +212,7 @@ impl OverlayDB {
impl HashDB for OverlayDB {
fn keys(&self) -> HashMap<H256, i32> {
let mut ret: HashMap<H256, i32> = HashMap::new();
for (key, _) in self.backing.iterator(IteratorMode::Start) {
for (key, _) in self.backing.iter() {
let h = H256::from_slice(key.deref());
let r = self.payload(&h).unwrap().1;
ret.insert(h, r as i32);
@@ -318,7 +366,7 @@ fn overlaydb_complex() {
fn playpen() {
use std::fs;
{
let db: DB = DB::open_default("/tmp/test").unwrap();
let db: Database = Database::open_default("/tmp/test").unwrap();
db.put(b"test", b"test2").unwrap();
match db.get(b"test") {
Ok(Some(value)) => println!("Got value {:?}", value.deref()),

50
util/src/panics.rs
View File

@@ -18,8 +18,8 @@
use std::thread;
use std::ops::DerefMut;
use std::any::Any;
use std::sync::{Arc, Mutex};
use std::default::Default;
/// Thread-safe closure for handling possible panics
pub trait OnPanicListener: Send + Sync + 'static {
@@ -40,19 +40,37 @@ pub trait MayPanic {
fn on_panic<F>(&self, closure: F) where F: OnPanicListener;
}
struct PanicGuard<'a> {
handler: &'a PanicHandler,
}
impl<'a> Drop for PanicGuard<'a> {
fn drop(&mut self) {
if thread::panicking() {
self.handler.notify_all("Panic!".to_owned());
}
}
}
/// Structure that allows to catch panics and notify listeners
pub struct PanicHandler {
listeners: Mutex<Vec<Box<OnPanicListener>>>
}
impl Default for PanicHandler {
fn default() -> Self {
PanicHandler::new()
}
}
impl PanicHandler {
/// Creates new `PanicHandler` wrapped in `Arc`
pub fn new_in_arc() -> Arc<PanicHandler> {
pub fn new_in_arc() -> Arc<Self> {
Arc::new(Self::new())
}
/// Creates new `PanicHandler`
pub fn new() -> PanicHandler {
pub fn new() -> Self {
PanicHandler {
listeners: Mutex::new(vec![])
}
@@ -60,19 +78,11 @@ impl PanicHandler {
/// Invoke closure and catch any possible panics.
/// In case of panic notifies all listeners about it.
#[allow(deprecated)]
// TODO [todr] catch_panic is deprecated but panic::recover has different bounds (not allowing mutex)
#[cfg_attr(feature="dev", allow(deprecated))]
pub fn catch_panic<G, R>(&self, g: G) -> thread::Result<R> where G: FnOnce() -> R + Send + 'static {
let result = thread::catch_panic(g);
if let Err(ref e) = result {
let res = convert_to_string(e);
if let Some(r) = res {
self.notify_all(r);
}
}
result
let _guard = PanicGuard { handler: self };
let result = g();
Ok(result)
}
fn notify_all(&self, r: String) {
@@ -103,14 +113,8 @@ impl<F> OnPanicListener for F
}
}
fn convert_to_string(t: &Box<Any + Send>) -> Option<String> {
let as_str = t.downcast_ref::<&'static str>().cloned().map(|t| t.to_owned());
let as_string = t.downcast_ref::<String>().cloned();
as_str.or(as_string)
}
#[test]
#[ignore] // panic forwarding doesnt work on the same thread in beta
fn should_notify_listeners_about_panic () {
use std::sync::RwLock;
// given
@@ -127,6 +131,7 @@ fn should_notify_listeners_about_panic () {
}
#[test]
#[ignore] // panic forwarding doesnt work on the same thread in beta
fn should_notify_listeners_about_panic_when_string_is_dynamic () {
use std::sync::RwLock;
// given
@@ -164,6 +169,7 @@ fn should_notify_listeners_about_panic_in_other_thread () {
}
#[test]
#[ignore] // panic forwarding doesnt work on the same thread in beta
fn should_forward_panics () {
use std::sync::RwLock;
// given

10
util/src/rlp/bytes.rs
View File

@@ -21,7 +21,7 @@ use std::mem;
use std::fmt;
use std::cmp::Ordering;
use std::error::Error as StdError;
use uint::{Uint, U128, U256};
use bigint::uint::{Uint, U128, U256};
use hash::FixedHash;
use elastic_array::*;
@@ -232,12 +232,12 @@ impl_uint_from_bytes!(u64);
impl_uint_from_bytes!(usize);
macro_rules! impl_uint_from_bytes {
($name: ident) => {
($name: ident, $size: expr) => {
impl FromBytes for $name {
fn from_bytes(bytes: &[u8]) -> FromBytesResult<$name> {
if !bytes.is_empty() && bytes[0] == 0 {
Err(FromBytesError::ZeroPrefixedInt)
} else if bytes.len() <= $name::SIZE {
} else if bytes.len() <= $size {
Ok($name::from(bytes))
} else {
Err(FromBytesError::DataIsTooLong)
@@ -247,8 +247,8 @@ macro_rules! impl_uint_from_bytes {
}
}
impl_uint_from_bytes!(U256);
impl_uint_from_bytes!(U128);
impl_uint_from_bytes!(U256, 32);
impl_uint_from_bytes!(U128, 16);
impl <T>FromBytes for T where T: FixedHash {
fn from_bytes(bytes: &[u8]) -> FromBytesResult<T> {

2
util/src/rlp/rlpin.rs
View File

@@ -116,7 +116,7 @@ impl<'a, 'view> View<'a, 'view> for Rlp<'a> where 'a: 'view {
impl <'a, 'view> Rlp<'a> where 'a: 'view {
fn view_as_val<T, R>(r: &R) -> T where R: View<'a, 'view>, T: RlpDecodable {
let res: Result<T, DecoderError> = r.as_val();
res.unwrap_or_else(|_| panic!())
res.unwrap_or_else(|e| panic!("DecodeError: {}", e))
}
/// Decode into an object

17
util/src/rlp/rlpstream.rs
View File

@@ -15,6 +15,7 @@
// along with Parity. If not, see <http://www.gnu.org/licenses/>.
use std::ops::Deref;
use std::default::Default;
use elastic_array::*;
use rlp::bytes::{ToBytes, VecLike};
use rlp::{Stream, Encoder, Encodable};
@@ -44,6 +45,12 @@ pub struct RlpStream {
finished_list: bool,
}
impl Default for RlpStream {
fn default() -> Self {
RlpStream::new()
}
}
impl Stream for RlpStream {
fn new() -> Self {
RlpStream {
@@ -190,8 +197,14 @@ struct BasicEncoder {
bytes: ElasticArray1024<u8>,
}
impl Default for BasicEncoder {
fn default() -> Self {
BasicEncoder::new()
}
}
impl BasicEncoder {
fn new() -> BasicEncoder {
fn new() -> Self {
BasicEncoder { bytes: ElasticArray1024::new() }
}
@@ -222,7 +235,7 @@ impl Encoder for BasicEncoder {
// just 0
0 => self.bytes.push(0x80u8),
// byte is its own encoding if < 0x80
1 => {
1 => {
value.to_bytes(&mut self.bytes);
let len = self.bytes.len();
let last_byte = self.bytes[len - 1];

2
util/src/rlp/tests.rs
View File

@@ -21,7 +21,7 @@ use std::{fmt, cmp};
use std::str::FromStr;
use rlp;
use rlp::{UntrustedRlp, RlpStream, View, Stream, DecoderError};
use uint::U256;
use bigint::uint::U256;
#[test]
fn rlp_at() {

1
util/src/rlp/untrusted_rlp.rs
View File

@@ -429,7 +429,6 @@ impl<T> Decodable for Option<T> where T: Decodable {
macro_rules! impl_array_decodable {
($index_type:ty, $len:expr ) => (
impl<T> Decodable for [T; $len] where T: Decodable {
#[allow(len_zero)]
fn decode<D>(decoder: &D) -> Result<Self, DecoderError> where D: Decoder {
let decoders = decoder.as_rlp();

6
util/src/semantic_version.rs
View File

@@ -16,13 +16,13 @@
//! Semantic version formatting and comparing.
/// A version value with strict meaning. Use `to_u32` to convert to a simple integer.
///
/// A version value with strict meaning. Use `as_u32` to convert to a simple integer.
///
/// # Example
/// ```
/// extern crate ethcore_util as util;
/// use util::semantic_version::*;
///
///
/// fn main() {
/// assert_eq!(SemanticVersion::new(1, 2, 3).as_u32(), 0x010203);
/// }

265
util/src/table.rs Normal file
View File

@@ -0,0 +1,265 @@
// Copyright 2015, 2016 Ethcore (UK) Ltd.
// This file is part of Parity.
// Parity is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// Parity is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
// You should have received a copy of the GNU General Public License
// along with Parity. If not, see <http://www.gnu.org/licenses/>.
//! A collection associating pair of keys (row and column) with a single value.
use std::default::Default;
use std::hash::Hash;
use std::collections::HashMap;
/// Structure to hold double-indexed values
///
/// You can obviously use `HashMap<(Row,Col), Val>`, but this structure gives
/// you better access to all `Columns` in Specific `Row`. Namely you can get sub-hashmap
/// `HashMap<Col, Val>` for specific `Row`
pub struct Table<Row, Col, Val>
where Row: Eq + Hash + Clone,
Col: Eq + Hash {
map: HashMap<Row, HashMap<Col, Val>>,
}
impl<Row, Col, Val> Default for Table<Row, Col, Val>
where Row: Eq + Hash + Clone,
Col: Eq + Hash {
fn default() -> Self {
Table::new()
}
}
// There is default but clippy does not detect it?
#[cfg_attr(feature="dev", allow(new_without_default))]
impl<Row, Col, Val> Table<Row, Col, Val>
where Row: Eq + Hash + Clone,
Col: Eq + Hash {
/// Creates new Table
pub fn new() -> Self {
Table {
map: HashMap::new(),
}
}
/// Removes all elements from this Table
pub fn clear(&mut self) {
self.map.clear();
}
/// Returns length of the Table (number of (row, col, val) tuples)
pub fn len(&self) -> usize {
self.map.values().fold(0, |acc, v| acc + v.len())
}
/// Check if there is any element in this Table
pub fn is_empty(&self) -> bool {
self.map.is_empty() || self.map.values().all(|v| v.is_empty())
}
/// Get mutable reference for single Table row.
pub fn row_mut(&mut self, row: &Row) -> Option<&mut HashMap<Col, Val>> {
self.map.get_mut(row)
}
/// Checks if row is defined for that table (note that even if defined it might be empty)
pub fn has_row(&self, row: &Row) -> bool {
self.map.contains_key(row)
}
/// Get immutable reference for single row in this Table
pub fn row(&self, row: &Row) -> Option<&HashMap<Col, Val>> {
self.map.get(row)
}
/// Get element in cell described by `(row, col)`
pub fn get(&self, row: &Row, col: &Col) -> Option<&Val> {
self.map.get(row).and_then(|r| r.get(col))
}
/// Remove value from specific cell
///
/// It will remove the row if it's the last value in it
pub fn remove(&mut self, row: &Row, col: &Col) -> Option<Val> {
let (val, is_empty) = {
let row_map = self.map.get_mut(row);
if let None = row_map {
return None;
}
let mut row_map = row_map.unwrap();
let val = row_map.remove(col);
(val, row_map.is_empty())
};
// Clean row
if is_empty {
self.map.remove(row);
}
val
}
/// Remove given row from Table if there are no values defined in it
///
/// When using `#row_mut` it may happen that all values from some row are drained.
/// Table however will not be aware that row is empty.
/// You can use this method to explicitly remove row entry from the Table.
pub fn clear_if_empty(&mut self, row: &Row) {
let is_empty = self.map.get(row).map_or(false, |m| m.is_empty());
if is_empty {
self.map.remove(row);
}
}
/// Inserts new value to specified cell
///
/// Returns previous value (if any)
pub fn insert(&mut self, row: Row, col: Col, val: Val) -> Option<Val> {
self.map.entry(row).or_insert_with(HashMap::new).insert(col, val)
}
}
#[cfg(test)]
mod test {
use super::*;
#[test]
fn should_create_empty_table() {
// when
let table : Table<usize, usize, bool> = Table::new();
// then
assert!(table.is_empty());
assert_eq!(table.len(), 0);
}
#[test]
fn should_insert_elements_and_return_previous_if_any() {
// given
let mut table = Table::new();
// when
let r1 = table.insert(5, 4, true);
let r2 = table.insert(10, 4, true);
let r3 = table.insert(10, 10, true);
let r4 = table.insert(10, 10, false);
// then
assert!(r1.is_none());
assert!(r2.is_none());
assert!(r3.is_none());
assert!(r4.is_some());
assert!(!table.is_empty());
assert_eq!(r4.unwrap(), true);
assert_eq!(table.len(), 3);
}
#[test]
fn should_remove_element() {
// given
let mut table = Table::new();
table.insert(5, 4, true);
assert!(!table.is_empty());
assert_eq!(table.len(), 1);
// when
let r = table.remove(&5, &4);
// then
assert!(table.is_empty());
assert_eq!(table.len() ,0);
assert_eq!(r.unwrap(), true);
}
#[test]
fn should_return_none_if_trying_to_remove_non_existing_element() {
// given
let mut table : Table<usize, usize, usize> = Table::new();
assert!(table.is_empty());
// when
let r = table.remove(&5, &4);
// then
assert!(r.is_none());
}
#[test]
fn should_clear_row_if_removing_last_element() {
// given
let mut table = Table::new();
table.insert(5, 4, true);
assert!(table.has_row(&5));
// when
let r = table.remove(&5, &4);
// then
assert!(r.is_some());
assert!(!table.has_row(&5));
}
#[test]
fn should_return_element_given_row_and_col() {
// given
let mut table = Table::new();
table.insert(1551, 1234, 123);
// when
let r1 = table.get(&1551, &1234);
let r2 = table.get(&5, &4);
// then
assert!(r1.is_some());
assert!(r2.is_none());
assert_eq!(r1.unwrap(), &123);
}
#[test]
fn should_clear_table() {
// given
let mut table = Table::new();
table.insert(1, 1, true);
table.insert(1, 2, false);
table.insert(2, 2, false);
assert_eq!(table.len(), 3);
// when
table.clear();
// then
assert!(table.is_empty());
assert_eq!(table.len(), 0);
assert_eq!(table.has_row(&1), false);
assert_eq!(table.has_row(&2), false);
}
#[test]
fn should_return_mutable_row() {
// given
let mut table = Table::new();
table.insert(1, 1, true);
table.insert(1, 2, false);
table.insert(2, 2, false);
// when
{
let mut row = table.row_mut(&1).unwrap();
row.remove(&1);
row.remove(&2);
}
assert!(table.has_row(&1));
table.clear_if_empty(&1);
// then
assert!(!table.has_row(&1));
assert_eq!(table.len(), 1);
}
}

31
util/src/tests/helpers.rs
View File

@@ -1,31 +0,0 @@
use common::*;
use std::path::PathBuf;
use std::fs::{remove_dir_all};
use std::env;
pub struct RandomTempPath {
path: PathBuf
}
impl RandomTempPath {
pub fn create_dir() -> RandomTempPath {
let mut dir = env::temp_dir();
dir.push(H32::random().hex());
fs::create_dir_all(dir.as_path()).unwrap();
RandomTempPath {
path: dir.clone()
}
}
pub fn as_path(&self) -> &PathBuf {
&self.path
}
}
impl Drop for RandomTempPath {
fn drop(&mut self) {
if let Err(e) = remove_dir_all(self.as_path()) {
panic!("failed to remove temp directory, probably something failed to destroyed ({})", e);
}
}
}

1
util/src/tests/mod.rs
View File

@@ -1 +0,0 @@
pub mod helpers;

7
util/src/trie/journal.rs
View File

@@ -14,6 +14,7 @@
// You should have received a copy of the GNU General Public License
// along with Parity. If not, see <http://www.gnu.org/licenses/>.
use std::default::Default;
use sha3::*;
use hash::H256;
use bytes::*;
@@ -39,6 +40,12 @@ pub struct Score {
#[derive(Debug)]
pub struct Journal (Vec<Operation>);
impl Default for Journal {
fn default() -> Self {
Journal::new()
}
}
impl Journal {
/// Create a new, empty, object.
pub fn new() -> Journal { Journal(vec![]) }

25
util/src/trie/standardmap.rs
View File

@@ -20,6 +20,7 @@ extern crate rand;
use bytes::*;
use sha3::*;
use hash::*;
use rlp::encode;
/// Alphabet to use when creating words for insertion into tries.
pub enum Alphabet {
@@ -39,6 +40,8 @@ pub enum ValueMode {
Mirror,
/// Randomly (50:50) 1 or 32 byte randomly string.
Random,
/// RLP-encoded index.
Index,
}
/// Standard test map for profiling tries.
@@ -89,19 +92,27 @@ impl StandardMap {
/// Create the standard map (set of keys and values) for the object's fields.
pub fn make(&self) -> Vec<(Bytes, Bytes)> {
self.make_with(&mut H256::new())
}
/// Create the standard map (set of keys and values) for the object's fields, using the given seed.
pub fn make_with(&self, seed: &mut H256) -> Vec<(Bytes, Bytes)> {
let low = b"abcdef";
let mid = b"@QWERTYUIOPASDFGHJKLZXCVBNM[/]^_";
let mut d: Vec<(Bytes, Bytes)> = Vec::new();
let mut seed = H256::new();
for _ in 0..self.count {
for index in 0..self.count {
let k = match self.alphabet {
Alphabet::All => Self::random_bytes(self.min_key, self.journal_key, &mut seed),
Alphabet::Low => Self::random_word(low, self.min_key, self.journal_key, &mut seed),
Alphabet::Mid => Self::random_word(mid, self.min_key, self.journal_key, &mut seed),
Alphabet::Custom(ref a) => Self::random_word(&a, self.min_key, self.journal_key, &mut seed),
Alphabet::All => Self::random_bytes(self.min_key, self.journal_key, seed),
Alphabet::Low => Self::random_word(low, self.min_key, self.journal_key, seed),
Alphabet::Mid => Self::random_word(mid, self.min_key, self.journal_key, seed),
Alphabet::Custom(ref a) => Self::random_word(&a, self.min_key, self.journal_key, seed),
};
let v = match self.value_mode {
ValueMode::Mirror => k.clone(),
ValueMode::Random => Self::random_value(seed),
ValueMode::Index => encode(&index).to_vec(),
};
let v = match self.value_mode { ValueMode::Mirror => k.clone(), ValueMode::Random => Self::random_value(&mut seed) };
d.push((k, v))
}
d

18
util/src/trie/triedb.rs
View File

@@ -22,7 +22,7 @@ use super::trietraits::*;
use super::node::*;
/// A `Trie` implementation using a generic `HashDB` backing database.
///
///
/// Use it as a `Trie` trait object. You can use `db()` to get the backing database object, `keys`
/// to get the keys belonging to the trie in the backing database, and `db_items_remaining()` to get
/// which items in the backing database do not belong to this trie. If this is the only trie in the
@@ -54,7 +54,7 @@ pub struct TrieDB<'db> {
pub hash_count: usize,
}
#[allow(wrong_self_convention)]
#[cfg_attr(feature="dev", allow(wrong_self_convention))]
impl<'db> TrieDB<'db> {
/// Create a new trie with the backing database `db` and `root`
/// Panics, if `root` does not exist
@@ -63,16 +63,16 @@ impl<'db> TrieDB<'db> {
flushln!("TrieDB::new({}): Trie root not found!", root);
panic!("Trie root not found!");
}
TrieDB {
db: db,
TrieDB {
db: db,
root: root,
hash_count: 0
hash_count: 0
}
}
/// Get the backing database.
pub fn db(&'db self) -> &'db HashDB {
self.db
pub fn db(&'db self) -> &'db HashDB {
self.db
}
/// Determine all the keys in the backing database that belong to the trie.
@@ -142,7 +142,7 @@ impl<'db> TrieDB<'db> {
/// Indentation helper for `formal_all`.
fn fmt_indent(&self, f: &mut fmt::Formatter, size: usize) -> fmt::Result {
for _ in 0..size {
for _ in 0..size {
try!(write!(f, " "));
}
Ok(())
@@ -358,7 +358,7 @@ impl<'db> fmt::Debug for TrieDB<'db> {
fn iterator() {
use memorydb::*;
use super::triedbmut::*;
let d = vec![ &b"A"[..], &b"AA"[..], &b"AB"[..], &b"B"[..] ];
let mut memdb = MemoryDB::new();

94
util/src/trie/triedbmut.rs
View File

@@ -23,7 +23,7 @@ use super::journal::*;
use super::trietraits::*;
/// A `Trie` implementation using a generic `HashDB` backing database.
///
///
/// Use it as a `Trie` trait object. You can use `db()` to get the backing database object, `keys`
/// to get the keys belonging to the trie in the backing database, and `db_items_remaining()` to get
/// which items in the backing database do not belong to this trie. If this is the only trie in the
@@ -66,21 +66,21 @@ enum MaybeChanged<'a> {
Changed(Bytes),
}
#[allow(wrong_self_convention)]
#[cfg_attr(feature="dev", allow(wrong_self_convention))]
impl<'db> TrieDBMut<'db> {
/// Create a new trie with the backing database `db` and empty `root`
/// Initialise to the state entailed by the genesis block.
/// This guarantees the trie is built correctly.
pub fn new(db: &'db mut HashDB, root: &'db mut H256) -> Self {
pub fn new(db: &'db mut HashDB, root: &'db mut H256) -> Self {
let mut r = TrieDBMut{
db: db,
db: db,
root: root,
hash_count: 0
};
hash_count: 0
};
// set root rlp
*r.root = SHA3_NULL_RLP.clone();
r
*r.root = SHA3_NULL_RLP.clone();
r
}
/// Create a new trie with the backing database `db` and `root`.
@@ -91,21 +91,21 @@ impl<'db> TrieDBMut<'db> {
flushln!("Trie root not found {}", root);
panic!("Trie root not found!");
}
TrieDBMut {
db: db,
TrieDBMut {
db: db,
root: root,
hash_count: 0
hash_count: 0
}
}
/// Get the backing database.
pub fn db(&'db self) -> &'db HashDB {
self.db
pub fn db(&'db self) -> &'db HashDB {
self.db
}
/// Get the backing database.
pub fn db_mut(&'db mut self) -> &'db mut HashDB {
self.db
pub fn db_mut(&'db mut self) -> &'db mut HashDB {
self.db
}
/// Determine all the keys in the backing database that belong to the trie.
@@ -184,7 +184,7 @@ impl<'db> TrieDBMut<'db> {
/// Indentation helper for `formal_all`.
fn fmt_indent(&self, f: &mut fmt::Formatter, size: usize) -> fmt::Result {
for _ in 0..size {
for _ in 0..size {
try!(write!(f, " "));
}
Ok(())
@@ -350,7 +350,7 @@ impl<'db> TrieDBMut<'db> {
}
}
#[allow(cyclomatic_complexity)]
#[cfg_attr(feature="dev", allow(cyclomatic_complexity))]
/// Determine the RLP of the node, assuming we're inserting `partial` into the
/// node currently of data `old`. This will *not* delete any hash of `old` from the database;
/// it will just return the new RLP that includes the new node.
@@ -378,7 +378,7 @@ impl<'db> TrieDBMut<'db> {
// original had empty slot - place a leaf there.
true if old_rlp.at(i).is_empty() => journal.new_node(Self::compose_leaf(&partial.mid(1), value), &mut s),
// original has something there already; augment.
true => {
true => {
let new = self.augmented(self.take_node(&old_rlp.at(i), journal), &partial.mid(1), value, journal);
journal.new_node(new, &mut s);
}
@@ -687,31 +687,10 @@ mod tests {
use super::*;
use nibbleslice::*;
use rlp::*;
use rand::random;
use std::collections::HashSet;
use bytes::{ToPretty,Bytes,Populatable};
use bytes::ToPretty;
use super::super::node::*;
use super::super::trietraits::*;
fn random_key(alphabet: &[u8], min_count: usize, journal_count: usize) -> Vec<u8> {
let mut ret: Vec<u8> = Vec::new();
let r = min_count + if journal_count > 0 {random::<usize>() % journal_count} else {0};
for _ in 0..r {
ret.push(alphabet[random::<usize>() % alphabet.len()]);
}
ret
}
fn random_value_indexed(j: usize) -> Bytes {
match random::<usize>() % 2 {
0 => encode(&j).to_vec(),
_ => {
let mut h = H256::new();
h.as_slice_mut()[31] = j as u8;
encode(&h).to_vec()
},
}
}
use super::super::standardmap::*;
fn populate_trie<'db>(db: &'db mut HashDB, root: &'db mut H256, v: &[(Vec<u8>, Vec<u8>)]) -> TrieDBMut<'db> {
let mut t = TrieDBMut::new(db, root);
@@ -756,20 +735,18 @@ mod tests {
};*/
// panic!();
let mut seed = H256::new();
for test_i in 0..1 {
if test_i % 50 == 0 {
debug!("{:?} of 10000 stress tests done", test_i);
}
let mut x: Vec<(Vec<u8>, Vec<u8>)> = Vec::new();
let mut got: HashSet<Vec<u8>> = HashSet::new();
let alphabet = b"@QWERTYUIOPASDFGHJKLZXCVBNM[/]^_";
for j in 0..100usize {
let key = random_key(alphabet, 5, 0);
if !got.contains(&key) {
x.push((key.clone(), random_value_indexed(j)));
got.insert(key);
}
}
let x = StandardMap {
alphabet: Alphabet::Custom(b"@QWERTYUIOPASDFGHJKLZXCVBNM[/]^_".to_vec()),
min_key: 5,
journal_key: 0,
value_mode: ValueMode::Index,
count: 100,
}.make_with(&mut seed);
let real = trie_root(x.clone());
let mut memdb = MemoryDB::new();
@@ -1049,13 +1026,16 @@ mod tests {
#[test]
fn stress() {
let mut seed = H256::new();
for _ in 0..50 {
let mut x: Vec<(Vec<u8>, Vec<u8>)> = Vec::new();
let alphabet = b"@QWERTYUIOPASDFGHJKLZXCVBNM[/]^_";
for j in 0..4u32 {
let key = random_key(alphabet, 5, 1);
x.push((key, encode(&j).to_vec()));
}
let x = StandardMap {
alphabet: Alphabet::Custom(b"@QWERTYUIOPASDFGHJKLZXCVBNM[/]^_".to_vec()),
min_key: 5,
journal_key: 0,
value_mode: ValueMode::Index,
count: 4,
}.make_with(&mut seed);
let real = trie_root(x.clone());
let mut memdb = MemoryDB::new();
let mut root = H256::new();

1390
util/src/uint.rs
View File

File diff suppressed because it is too large Load Diff