grassrootseconomics/openethereum
8
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

serde is no longer util dependency (#1534)

Browse Source 
* removed old json-tests

* simplify folds in triehash.rs

* removed unused json_aid

* removed unused squeeze.rs

* json branching tests for trie

* removing todos from util

* separated UsingQueue and Table

* further cleanup, removing unused code

* serde serialization of hash moved to rpc module

* uint wrapper for rpc in progress

* serialization of uint moved to rpc module

* updated eth-secp256k1

* updated igd, serde is no longer dependency of util

* loading trie consensus tests

* renamed aliases in rpc imports
This commit is contained in:
Marek Kotewicz
2016-07-06 11:23:29 +02:00
committed by Gav Wood
parent cb1808d53d
commit bcb63bce12
55 changed files with 854 additions and 717 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
util/Cargo.toml
View File

@@ -27,15 +27,16 @@ itertools = "0.4"
crossbeam = "0.2"
slab = "0.2"
sha3 = { path = "sha3" }
serde = "0.7.0"
clippy = { version = "0.0.78", optional = true}
igd = "0.4.2"
igd = "0.5.0"
ethcore-devtools = { path = "../devtools" }
libc = "0.2.7"
vergen = "0.1"
target_info = "0.1"
bigint = { path = "bigint" }
chrono = "0.2"
using_queue = { path = "using_queue" }
table = { path = "table" }
ansi_term = "0.7"
[features]

1
util/bigint/Cargo.toml
View File

@@ -12,7 +12,6 @@ rustc_version = "0.1"
[dependencies]
rustc-serialize = "0.3"
serde = "0.7.0"
heapsize = "0.3"
[features]

1
util/bigint/src/lib.rs
View File

@@ -17,7 +17,6 @@
#![cfg_attr(asm_available, feature(asm))]
extern crate rustc_serialize;
extern crate serde;
#[macro_use] extern crate heapsize;
pub mod uint;

73
util/bigint/src/uint.rs
View File

@@ -39,7 +39,6 @@
#[cfg(all(asm_available, target_arch="x86_64"))]
use std::mem;
use std::fmt;
use std::cmp;
use std::str::{FromStr};
use std::convert::From;
@@ -47,14 +46,15 @@ use std::hash::Hash;
use std::ops::*;
use std::cmp::*;
use serde;
use rustc_serialize::hex::{FromHex, FromHexError, ToHex};
use rustc_serialize::hex::{FromHex, FromHexError};
/// Conversion from decimal string error
#[derive(Debug, PartialEq)]
pub enum FromDecStrErr {
/// Char not from range 0-9
InvalidCharacter,
/// Value does not fit into type
InvalidLength
InvalidLength,
}
macro_rules! impl_map_from {
@@ -562,8 +562,11 @@ macro_rules! construct_uint {
impl Uint for $name {
/// TODO: optimize, throw appropriate err
fn from_dec_str(value: &str) -> Result<Self, FromDecStrErr> {
if value.bytes().any(|b| b < 48 && b > 57) {
return Err(FromDecStrErr::InvalidCharacter)
}
let mut res = Self::default();
for b in value.bytes().map(|b| b - 48) {
let (r, overflow) = res.overflowing_mul_u32(10);
@@ -649,7 +652,7 @@ macro_rules! construct_uint {
fn exp10(n: usize) -> Self {
match n {
0 => Self::from(1u64),
_ => Self::exp10(n - 1) * Self::from(10u64)
_ => Self::exp10(n - 1).mul_u32(10)
}
}
@@ -757,16 +760,16 @@ macro_rules! construct_uint {
}
impl $name {
#[allow(dead_code)] // not used when multiplied with inline assembly
/// Multiplication by u32
#[allow(dead_code)] // not used when multiplied with inline assembly
fn mul_u32(self, other: u32) -> Self {
let (ret, overflow) = self.overflowing_mul_u32(other);
panic_on_overflow!(overflow);
ret
}
#[allow(dead_code)] // not used when multiplied with inline assembly
/// Overflowing multiplication by u32
#[allow(dead_code)] // not used when multiplied with inline assembly
fn overflowing_mul_u32(self, other: u32) -> (Self, bool) {
let $name(ref arr) = self;
let mut ret = [0u64; $n_words];
@@ -789,44 +792,6 @@ macro_rules! construct_uint {
}
}
impl serde::Serialize for $name {
fn serialize<S>(&self, serializer: &mut S) -> Result<(), S::Error>
where S: serde::Serializer {
let mut hex = "0x".to_owned();
let mut bytes = [0u8; 8 * $n_words];
self.to_raw_bytes(&mut bytes);
let len = cmp::max((self.bits() + 7) / 8, 1);
hex.push_str(bytes[bytes.len() - len..].to_hex().as_ref());
serializer.serialize_str(hex.as_ref())
}
}
impl serde::Deserialize for $name {
fn deserialize<D>(deserializer: &mut D) -> Result<$name, D::Error>
where D: serde::Deserializer {
struct UintVisitor;
impl serde::de::Visitor for UintVisitor {
type Value = $name;
fn visit_str<E>(&mut self, value: &str) -> Result<Self::Value, E> where E: serde::Error {
// 0x + len
if value.len() > 2 + $n_words * 16 || value.len() < 2 {
return Err(serde::Error::custom("Invalid length."));
}
$name::from_str(&value[2..]).map_err(|_| serde::Error::custom("Invalid hex value."))
}
fn visit_string<E>(&mut self, value: String) -> Result<Self::Value, E> where E: serde::Error {
self.visit_str(value.as_ref())
}
}
deserializer.deserialize(UintVisitor)
}
}
impl From<u64> for $name {
fn from(value: u64) -> $name {
let mut ret = [0; $n_words];
@@ -959,8 +924,6 @@ macro_rules! construct_uint {
}
}
// TODO: optimise and traitify.
impl BitAnd<$name> for $name {
type Output = $name;
@@ -1434,12 +1397,6 @@ impl From<U256> for u32 {
}
}
/// Constant value of `U256::zero()` that can be used for a reference saving an additional instance creation.
pub const ZERO_U256: U256 = U256([0x00u64; 4]);
/// Constant value of `U256::one()` that can be used for a reference saving an additional instance creation.
pub const ONE_U256: U256 = U256([0x01u64, 0x00u64, 0x00u64, 0x00u64]);
known_heap_size!(0, U128, U256);
#[cfg(test)]
@@ -1582,7 +1539,13 @@ mod tests {
assert_eq!(U256::from(105u8) / U256::from(5u8), U256::from(21u8));
let div = mult / U256::from(300u16);
assert_eq!(div, U256([0x9F30411021524112u64, 0x0001BD5B7DDFBD5A, 0, 0]));
//// TODO: bit inversion
let a = U256::from_str("ff000000000000000000000000000000000000000000000000000000000000d1").unwrap();
let b = U256::from_str("00ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff2e").unwrap();
println!("{:x}", a);
println!("{:x}", b);
assert_eq!(!a, b);
assert_eq!(a, !b);
}
#[test]

16
util/src/bytes.rs
View File

@@ -24,7 +24,7 @@
//! use util::bytes::BytesConvertable;
//!
//! let arr = [0; 5];
//! let slice: &[u8] = arr.bytes();
//! let slice: &[u8] = arr.as_slice();
//! }
//!
//! fn main() {
@@ -120,12 +120,12 @@ impl<'a> ToPretty for &'a [u8] {
impl<'a> ToPretty for &'a Bytes {
fn pretty(&self) -> PrettySlice {
PrettySlice(self.bytes())
PrettySlice(self.as_slice())
}
}
impl ToPretty for Bytes {
fn pretty(&self) -> PrettySlice {
PrettySlice(self.bytes())
PrettySlice(self.as_slice())
}
}
@@ -162,23 +162,19 @@ pub type Bytes = Vec<u8>;
/// Slice of bytes to underlying memory
pub trait BytesConvertable {
// TODO: rename to as_slice
/// Get the underlying byte-wise representation of the value.
/// Deprecated - use `as_slice` instead.
fn bytes(&self) -> &[u8];
/// Get the underlying byte-wise representation of the value.
fn as_slice(&self) -> &[u8] { self.bytes() }
fn as_slice(&self) -> &[u8];
/// Get a copy of the underlying byte-wise representation.
fn to_bytes(&self) -> Bytes { self.as_slice().to_vec() }
}
impl<T> BytesConvertable for T where T: AsRef<[u8]> {
fn bytes(&self) -> &[u8] { self.as_ref() }
fn as_slice(&self) -> &[u8] { self.as_ref() }
}
#[test]
fn bytes_convertable() {
assert_eq!(vec![0x12u8, 0x34].bytes(), &[0x12u8, 0x34]);
assert_eq!(vec![0x12u8, 0x34].as_slice(), &[0x12u8, 0x34]);
assert!([0u8; 0].as_slice().is_empty());
}

80
util/src/hash.rs
View File

@@ -16,16 +16,18 @@
//! General hash types, a fixed-size raw-data type used as the output of hash functions.
use standard::*;
use rustc_serialize::hex::FromHex;
use std::{ops, fmt, cmp};
use std::cmp::*;
use std::ops::*;
use std::hash::{Hash, Hasher};
use std::str::FromStr;
use math::log2;
use error::UtilError;
use rand::Rng;
use rand::os::OsRng;
use bytes::{BytesConvertable,Populatable};
use from_json::*;
use bigint::uint::{Uint, U256};
use rustc_serialize::hex::ToHex;
use serde;
/// Trait for a fixed-size byte array to be used as the output of hash functions.
///
@@ -228,55 +230,6 @@ macro_rules! impl_hash {
}
}
impl serde::Serialize for $from {
fn serialize<S>(&self, serializer: &mut S) -> Result<(), S::Error>
where S: serde::Serializer {
let mut hex = "0x".to_owned();
hex.push_str(self.to_hex().as_ref());
serializer.serialize_str(hex.as_ref())
}
}
impl serde::Deserialize for $from {
fn deserialize<D>(deserializer: &mut D) -> Result<$from, D::Error>
where D: serde::Deserializer {
struct HashVisitor;
impl serde::de::Visitor for HashVisitor {
type Value = $from;
fn visit_str<E>(&mut self, value: &str) -> Result<Self::Value, E> where E: serde::Error {
// 0x + len
if value.len() != 2 + $size * 2 {
return Err(serde::Error::custom("Invalid length."));
}
value[2..].from_hex().map(|ref v| $from::from_slice(v)).map_err(|_| serde::Error::custom("Invalid hex value."))
}
fn visit_string<E>(&mut self, value: String) -> Result<Self::Value, E> where E: serde::Error {
self.visit_str(value.as_ref())
}
}
deserializer.deserialize(HashVisitor)
}
}
impl FromJson for $from {
fn from_json(json: &Json) -> Self {
match *json {
Json::String(ref s) => {
match s.len() % 2 {
0 => FromStr::from_str(clean_0x(s)).unwrap(),
_ => FromStr::from_str(&("0".to_owned() + &(clean_0x(s).to_owned()))[..]).unwrap()
}
},
_ => Default::default(),
}
}
}
impl fmt::Debug for $from {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
for i in &self.0[..] {
@@ -285,6 +238,7 @@ macro_rules! impl_hash {
Ok(())
}
}
impl fmt::Display for $from {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
for i in &self.0[0..2] {
@@ -506,13 +460,13 @@ impl<'a> From<&'a U256> for H256 {
impl From<H256> for U256 {
fn from(value: H256) -> U256 {
U256::from(value.bytes())
U256::from(value.as_slice())
}
}
impl<'a> From<&'a H256> for U256 {
fn from(value: &'a H256) -> U256 {
U256::from(value.bytes())
U256::from(value.as_slice())
}
}
@@ -532,17 +486,6 @@ impl From<H256> for H64 {
}
}
/*
impl<'a> From<&'a H256> for Address {
fn from(value: &'a H256) -> Address {
let mut ret = Address::new();
ret.0.copy_from_slice(&value[12..32]);
ret
}
}
}
*/
impl From<Address> for H256 {
fn from(value: Address) -> H256 {
let mut ret = H256::new();
@@ -596,11 +539,6 @@ impl_hash!(H520, 65);
impl_hash!(H1024, 128);
impl_hash!(H2048, 256);
/// Constant address for point 0. Often used as a default.
pub static ZERO_ADDRESS: Address = Address([0x00; 20]);
/// Constant 256-bit datum for 0. Often used as a default.
pub static ZERO_H256: H256 = H256([0x00; 32]);
#[cfg(test)]
mod tests {
use hash::*;

6
util/src/journaldb/archivedb.rs
View File

@@ -79,7 +79,7 @@ impl ArchiveDB {
}
fn payload(&self, key: &H256) -> Option<Bytes> {
self.backing.get(&key.bytes()).expect("Low-level database error. Some issue with your hard disk?").map(|v| v.to_vec())
self.backing.get(key).expect("Low-level database error. Some issue with your hard disk?").map(|v| v.to_vec())
}
}
@@ -177,7 +177,7 @@ impl JournalDB for ArchiveDB {
let (key, (value, rc)) = i;
if rc > 0 {
assert!(rc == 1);
batch.put(&key.bytes(), &value).expect("Low-level database error. Some issue with your hard disk?");
batch.put(&key, &value).expect("Low-level database error. Some issue with your hard disk?");
inserts += 1;
}
if rc < 0 {
@@ -202,7 +202,7 @@ impl JournalDB for ArchiveDB {
fn latest_era(&self) -> Option<u64> { self.latest_era }
fn state(&self, id: &H256) -> Option<Bytes> {
self.backing.get_by_prefix(&id.bytes()[0..12]).and_then(|b| Some(b.to_vec()))
self.backing.get_by_prefix(&id[0..12]).and_then(|b| Some(b.to_vec()))
}
fn is_pruned(&self) -> bool { false }

12
util/src/journaldb/earlymergedb.rs
View File

@@ -106,7 +106,7 @@ impl EarlyMergeDB {
}
fn morph_key(key: &H256, index: u8) -> Bytes {
let mut ret = key.bytes().to_owned();
let mut ret = key.to_bytes();
ret.push(index);
ret
}
@@ -130,7 +130,7 @@ impl EarlyMergeDB {
}
// this is the first entry for this node in the journal.
if backing.get(&h.bytes()).expect("Low-level database error. Some issue with your hard disk?").is_some() {
if backing.get(h).expect("Low-level database error. Some issue with your hard disk?").is_some() {
// already in the backing DB. start counting, and remember it was already in.
Self::set_already_in(batch, &h);
refs.insert(h.clone(), RefInfo{queue_refs: 1, in_archive: true});
@@ -143,7 +143,7 @@ impl EarlyMergeDB {
// Gets removed when a key leaves the journal, so should never be set when we're placing a new key.
//Self::reset_already_in(&h);
assert!(!Self::is_already_in(backing, &h));
batch.put(&h.bytes(), d).expect("Low-level database error. Some issue with your hard disk?");
batch.put(h, d).expect("Low-level database error. Some issue with your hard disk?");
refs.insert(h.clone(), RefInfo{queue_refs: 1, in_archive: false});
if trace {
trace!(target: "jdb.fine", " insert({}): New to queue, not in DB: Inserting into queue and DB", h);
@@ -204,7 +204,7 @@ impl EarlyMergeDB {
}
Some(RefInfo{queue_refs: 1, in_archive: false}) => {
refs.remove(h);
batch.delete(&h.bytes()).expect("Low-level database error. Some issue with your hard disk?");
batch.delete(h).expect("Low-level database error. Some issue with your hard disk?");
if trace {
trace!(target: "jdb.fine", " remove({}): Not in archive, only 1 ref in queue: Removing from queue and DB", h);
}
@@ -212,7 +212,7 @@ impl EarlyMergeDB {
None => {
// Gets removed when moving from 1 to 0 additional refs. Should never be here at 0 additional refs.
//assert!(!Self::is_already_in(db, &h));
batch.delete(&h.bytes()).expect("Low-level database error. Some issue with your hard disk?");
batch.delete(h).expect("Low-level database error. Some issue with your hard disk?");
if trace {
trace!(target: "jdb.fine", " remove({}): Not in queue - MUST BE IN ARCHIVE: Removing from DB", h);
}
@@ -237,7 +237,7 @@ impl EarlyMergeDB {
}
fn payload(&self, key: &H256) -> Option<Bytes> {
self.backing.get(&key.bytes()).expect("Low-level database error. Some issue with your hard disk?").map(|v| v.to_vec())
self.backing.get(key).expect("Low-level database error. Some issue with your hard disk?").map(|v| v.to_vec())
}
fn read_refs(db: &Database) -> (Option<u64>, HashMap<H256, RefInfo>) {

2
util/src/journaldb/overlayrecentdb.rs
View File

@@ -141,7 +141,7 @@ impl OverlayRecentDB {
}
fn payload(&self, key: &H256) -> Option<Bytes> {
self.backing.get(&key.bytes()).expect("Low-level database error. Some issue with your hard disk?").map(|v| v.to_vec())
self.backing.get(key).expect("Low-level database error. Some issue with your hard disk?").map(|v| v.to_vec())
}
fn read_overlay(db: &Database) -> JournalOverlay {

6
util/src/lib.rs
View File

@@ -108,7 +108,6 @@ extern crate secp256k1;
extern crate arrayvec;
extern crate elastic_array;
extern crate crossbeam;
extern crate serde;
#[macro_use]
extern crate log as rlog;
extern crate igd;
@@ -117,6 +116,8 @@ extern crate libc;
extern crate target_info;
extern crate bigint;
extern crate chrono;
pub extern crate using_queue;
pub extern crate table;
extern crate ansi_term;
pub mod standard;
@@ -130,7 +131,6 @@ pub mod hash;
pub mod bytes;
pub mod rlp;
pub mod misc;
pub mod using_queue;
pub mod vector;
pub mod sha3;
pub mod hashdb;
@@ -151,14 +151,12 @@ pub mod io;
pub mod network;
pub mod log;
pub mod panics;
pub mod table;
pub mod network_settings;
pub mod path;
mod timer;
pub use common::*;
pub use misc::*;
pub use using_queue::*;
pub use rlp::*;
pub use hashdb::*;
pub use memorydb::*;

8
util/src/memorydb.rs
View File

@@ -198,14 +198,14 @@ impl HashDB for MemoryDB {
let key = value.sha3();
if match self.data.get_mut(&key) {
Some(&mut (ref mut old_value, ref mut rc @ -0x80000000i32 ... 0)) => {
*old_value = From::from(value.bytes());
*old_value = From::from(value);
*rc += 1;
false
},
Some(&mut (_, ref mut x)) => { *x += 1; false } ,
None => true,
}{ // ... None falls through into...
self.data.insert(key.clone(), (From::from(value.bytes()), 1));
self.data.insert(key.clone(), (From::from(value), 1));
}
key
}
@@ -262,8 +262,8 @@ fn memorydb_denote() {
for _ in 0..1000 {
let r = H256::random();
let k = r.sha3();
let &(ref v, ref rc) = m.denote(&k, r.bytes().to_vec());
assert_eq!(v, &r.bytes());
let &(ref v, ref rc) = m.denote(&k, r.to_bytes());
assert_eq!(v.as_slice(), r.as_slice());
assert_eq!(*rc, 0);
}

12
util/src/overlaydb.rs
View File

@@ -172,10 +172,10 @@ impl OverlayDB {
/// Get the refs and value of the given key.
fn payload(&self, key: &H256) -> Option<(Bytes, u32)> {
self.backing.get(&key.bytes())
self.backing.get(key)
.expect("Low-level database error. Some issue with your hard disk?")
.map(|d| {
let r = Rlp::new(d.deref());
let r = Rlp::new(&d);
(r.at(1).as_val(), r.at(0).as_val())
})
}
@@ -186,10 +186,10 @@ impl OverlayDB {
let mut s = RlpStream::new_list(2);
s.append(&payload.1);
s.append(&payload.0);
batch.put(&key.bytes(), s.as_raw()).expect("Low-level database error. Some issue with your hard disk?");
batch.put(key, s.as_raw()).expect("Low-level database error. Some issue with your hard disk?");
false
} else {
batch.delete(&key.bytes()).expect("Low-level database error. Some issue with your hard disk?");
batch.delete(key).expect("Low-level database error. Some issue with your hard disk?");
true
}
}
@@ -200,10 +200,10 @@ impl OverlayDB {
let mut s = RlpStream::new_list(2);
s.append(&payload.1);
s.append(&payload.0);
self.backing.put(&key.bytes(), s.as_raw()).expect("Low-level database error. Some issue with your hard disk?");
self.backing.put(key, s.as_raw()).expect("Low-level database error. Some issue with your hard disk?");
false
} else {
self.backing.delete(&key.bytes()).expect("Low-level database error. Some issue with your hard disk?");
self.backing.delete(key).expect("Low-level database error. Some issue with your hard disk?");
true
}
}

4
util/src/rlp/bytes.rs
View File

@@ -148,9 +148,9 @@ impl_uint_to_bytes!(U128);
impl <T>ToBytes for T where T: FixedHash {
fn to_bytes<V: VecLike<u8>>(&self, out: &mut V) {
out.vec_extend(self.bytes());
out.vec_extend(self.as_slice());
}
fn to_bytes_len(&self) -> usize { self.bytes().len() }
fn to_bytes_len(&self) -> usize { self.as_slice().len() }
}
/// Error returned when `FromBytes` conversation goes wrong

2
util/src/sha3.rs
View File

@@ -58,7 +58,7 @@ impl<T> Hashable for T where T: BytesConvertable {
}
fn sha3_into(&self, dest: &mut [u8]) {
unsafe {
let input: &[u8] = self.bytes();
let input: &[u8] = self.as_slice();
sha3_256(dest.as_mut_ptr(), dest.len(), input.as_ptr(), input.len());
}
}

14
util/src/trie/standardmap.rs
View File

@@ -64,16 +64,16 @@ impl StandardMap {
fn random_bytes(min_count: usize, journal_count: usize, seed: &mut H256) -> Vec<u8> {
assert!(min_count + journal_count <= 32);
*seed = seed.sha3();
let r = min_count + (seed.bytes()[31] as usize % (journal_count + 1));
seed.bytes()[0..r].to_vec()
let r = min_count + (seed[31] as usize % (journal_count + 1));
seed[0..r].to_vec()
}
/// Get a random value. Equal chance of being 1 byte as of 32. `seed` is mutated pseudoramdonly and used.
fn random_value(seed: &mut H256) -> Bytes {
*seed = seed.sha3();
match seed.bytes()[0] % 2 {
1 => vec![seed.bytes()[31];1],
_ => seed.bytes().to_vec(),
match seed[0] % 2 {
1 => vec![seed[31];1],
_ => seed.to_vec(),
}
}
@@ -82,10 +82,10 @@ impl StandardMap {
fn random_word(alphabet: &[u8], min_count: usize, journal_count: usize, seed: &mut H256) -> Vec<u8> {
assert!(min_count + journal_count <= 32);
*seed = seed.sha3();
let r = min_count + (seed.bytes()[31] as usize % (journal_count + 1));
let r = min_count + (seed[31] as usize % (journal_count + 1));
let mut ret: Vec<u8> = Vec::with_capacity(r);
for i in 0..r {
ret.push(alphabet[seed.bytes()[i] as usize % alphabet.len()]);
ret.push(alphabet[seed[i] as usize % alphabet.len()]);
}
ret
}

6
util/table/Cargo.toml Normal file
View File

@@ -0,0 +1,6 @@
[package]
name = "table"
version = "0.1.0"
authors = ["debris <marek.kotewicz@gmail.com>"]
[dependencies]

0
util/src/table.rs → util/table/src/lib.rs
View File

6
util/using_queue/Cargo.toml Normal file
View File

@@ -0,0 +1,6 @@
[package]
name = "using_queue"
version = "0.1.0"
authors = ["debris <marek.kotewicz@gmail.com>"]
[dependencies]

3
util/src/using_queue.rs → util/using_queue/src/lib.rs
View File

@@ -13,6 +13,7 @@
// You should have received a copy of the GNU General Public License
// along with Parity. If not, see <http://www.gnu.org/licenses/>.
//! Queue-like datastructure including notion of usage.
/// Special queue-like datastructure that includes the notion of
@@ -50,7 +51,7 @@ impl<T> UsingQueue<T> where T: Clone {
pub fn peek_last_ref(&self) -> Option<&T> {
self.pending.as_ref().or(self.in_use.last())
}
/// Return a reference to the item at the top of the queue (or `None` if the queue is empty);
/// this constitutes using the item and will remain in the queue for at least another
/// `max_size` invocations of `push()`.