From a555686bcd3ab02e2a3189487378415381c2fb6a Mon Sep 17 00:00:00 2001 From: Marek Kotewicz Date: Mon, 20 Mar 2017 19:14:29 +0100 Subject: [PATCH] rlp serialization refactor (#4873) * fixed naming of rlp modules * RlpStream cleanup * appending short rlp lists (0...55 bytes) is 25% faster * RlpStream does not use bytes module, nor trait Stream * removed unused code from rlp module * compiling ethcore-util with new rlp serialization * compiling parity with new rlp serialization * fixed compiling ethcore-light with new rlp serialization * fixed compiling ethsync with new rlp serialization * removed redundant comment, print * removed redundant double-space * replace usage of WriteBytesExt with ByteOrder --- Cargo.lock | 1 + ethcore/light/src/cht.rs | 2 +- ethcore/light/src/net/mod.rs | 2 +- ethcore/light/src/net/request_credits.rs | 2 +- ethcore/light/src/net/status.rs | 6 +- ethcore/light/src/on_demand/mod.rs | 2 +- ethcore/light/src/on_demand/request.rs | 6 +- ethcore/light/src/provider.rs | 4 +- ethcore/src/block.rs | 8 +- ethcore/src/blockchain/extras.rs | 4 +- ethcore/src/blockchain/generator/block.rs | 4 +- ethcore/src/blooms/bloom_group.rs | 2 +- ethcore/src/engines/tendermint/message.rs | 5 +- ethcore/src/engines/tendermint/mod.rs | 8 +- ethcore/src/engines/vote_collector.rs | 8 +- ethcore/src/executive.rs | 2 +- ethcore/src/migrations/state/v7.rs | 4 +- ethcore/src/migrations/v9.rs | 2 +- ethcore/src/pod_account.rs | 2 +- ethcore/src/snapshot/account.rs | 2 +- ethcore/src/snapshot/block.rs | 8 +- ethcore/src/snapshot/io.rs | 8 +- ethcore/src/snapshot/mod.rs | 2 +- ethcore/src/snapshot/tests/blocks.rs | 2 +- ethcore/src/snapshot/tests/state.rs | 2 +- ethcore/src/spec/seal.rs | 9 +- ethcore/src/spec/spec.rs | 2 +- ethcore/src/tests/helpers.rs | 4 +- ethcore/src/trace/bloom.rs | 2 +- ethcore/src/types/log_entry.rs | 2 +- ethcore/src/types/receipt.rs | 2 +- ethcore/src/types/snapshot_manifest.rs | 4 +- ethcore/src/types/trace_types/error.rs | 5 +- ethcore/src/types/trace_types/flat.rs | 6 +- ethcore/src/types/trace_types/trace.rs | 6 +- ethcore/src/verification/verification.rs | 4 +- ipfs/src/route.rs | 2 +- rpc/src/v1/helpers/dispatch.rs | 2 +- sync/src/chain.rs | 2 +- util/network/src/discovery.rs | 2 +- util/network/src/session.rs | 16 +- util/rlp/Cargo.toml | 1 + util/rlp/src/bytes.rs | 153 +------- util/rlp/src/{commonrlps.rs => common.rs} | 4 +- .../src/{rlpcompression.rs => compression.rs} | 11 +- util/rlp/src/{rlperrors.rs => error.rs} | 0 util/rlp/src/impls.rs | 120 ++++++ util/rlp/src/lib.rs | 49 ++- util/rlp/src/rlpin.rs | 2 +- util/rlp/src/rlpstream.rs | 357 ------------------ util/rlp/src/stream.rs | 335 ++++++++++++++++ util/rlp/src/tests.rs | 42 ++- util/rlp/src/{rlptraits.rs => traits.rs} | 130 +------ util/src/journaldb/earlymergedb.rs | 2 +- util/src/journaldb/overlayrecentdb.rs | 2 +- util/src/journaldb/refcounteddb.rs | 4 +- util/src/misc.rs | 2 +- util/src/trie/triedbmut.rs | 3 +- util/src/triehash.rs | 2 +- 59 files changed, 620 insertions(+), 767 deletions(-) rename util/rlp/src/{commonrlps.rs => common.rs} (98%) rename util/rlp/src/{rlpcompression.rs => compression.rs} (97%) rename util/rlp/src/{rlperrors.rs => error.rs} (100%) create mode 100644 util/rlp/src/impls.rs delete mode 100644 util/rlp/src/rlpstream.rs create mode 100644 util/rlp/src/stream.rs rename util/rlp/src/{rlptraits.rs => traits.rs} (64%) diff --git a/Cargo.lock b/Cargo.lock index 4068a8e42..3ab71f092 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1924,6 +1924,7 @@ dependencies = [ name = "rlp" version = "0.1.0" dependencies = [ + "byteorder 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", "elastic-array 0.6.0 (git+https://github.com/ethcore/elastic-array)", "ethcore-bigint 0.1.2", "lazy_static 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", diff --git a/ethcore/light/src/cht.rs b/ethcore/light/src/cht.rs index 1fcb7b26a..94b9946c0 100644 --- a/ethcore/light/src/cht.rs +++ b/ethcore/light/src/cht.rs @@ -23,7 +23,7 @@ use ethcore::ids::BlockId; use util::{Bytes, H256, U256, HashDB, MemoryDB}; use util::trie::{self, TrieMut, TrieDBMut, Trie, TrieDB, Recorder}; -use rlp::{Stream, RlpStream, UntrustedRlp, View}; +use rlp::{RlpStream, UntrustedRlp, View}; // encode a key. macro_rules! key { diff --git a/ethcore/light/src/net/mod.rs b/ethcore/light/src/net/mod.rs index 181f95e95..4749dc281 100644 --- a/ethcore/light/src/net/mod.rs +++ b/ethcore/light/src/net/mod.rs @@ -24,7 +24,7 @@ use ethcore::receipt::Receipt; use io::TimerToken; use network::{NetworkProtocolHandler, NetworkContext, PeerId}; -use rlp::{RlpStream, Stream, UntrustedRlp, View}; +use rlp::{RlpStream, UntrustedRlp, View}; use util::hash::H256; use util::{Bytes, DBValue, Mutex, RwLock, U256}; use time::{Duration, SteadyTime}; diff --git a/ethcore/light/src/net/request_credits.rs b/ethcore/light/src/net/request_credits.rs index 97aa9b431..7f1960fc5 100644 --- a/ethcore/light/src/net/request_credits.rs +++ b/ethcore/light/src/net/request_credits.rs @@ -105,7 +105,7 @@ impl Default for CostTable { } } -impl RlpEncodable for CostTable { +impl Encodable for CostTable { fn rlp_append(&self, s: &mut RlpStream) { fn append_cost(s: &mut RlpStream, msg_id: u8, cost: &Cost) { s.begin_list(3) diff --git a/ethcore/light/src/net/status.rs b/ethcore/light/src/net/status.rs index 3a09be3c2..f5464c036 100644 --- a/ethcore/light/src/net/status.rs +++ b/ethcore/light/src/net/status.rs @@ -16,7 +16,7 @@ //! Peer status and capabilities. -use rlp::{DecoderError, RlpDecodable, RlpEncodable, RlpStream, Stream, UntrustedRlp, View}; +use rlp::{DecoderError, RlpDecodable, Encodable, RlpStream, UntrustedRlp, View}; use util::{H256, U256}; use super::request_credits::FlowParams; @@ -126,7 +126,7 @@ impl<'a> Parser<'a> { } // Helper for encoding a key-value pair -fn encode_pair(key: Key, val: &T) -> Vec { +fn encode_pair(key: Key, val: &T) -> Vec { let mut s = RlpStream::new_list(2); s.append(&key.as_str()).append(val); s.out() @@ -374,7 +374,7 @@ mod tests { use super::*; use super::super::request_credits::FlowParams; use util::{U256, H256}; - use rlp::{RlpStream, Stream ,UntrustedRlp, View}; + use rlp::{RlpStream, UntrustedRlp, View}; #[test] fn full_handshake() { diff --git a/ethcore/light/src/on_demand/mod.rs b/ethcore/light/src/on_demand/mod.rs index 25cde402b..2ef01a0b1 100644 --- a/ethcore/light/src/on_demand/mod.rs +++ b/ethcore/light/src/on_demand/mod.rs @@ -30,7 +30,7 @@ use ethcore::executed::{Executed, ExecutionError}; use futures::{Async, Poll, Future}; use futures::sync::oneshot::{self, Sender, Receiver}; use network::PeerId; -use rlp::{RlpStream, Stream}; +use rlp::RlpStream; use util::{Bytes, DBValue, RwLock, Mutex, U256}; use util::sha3::{SHA3_NULL_RLP, SHA3_EMPTY_LIST_RLP}; diff --git a/ethcore/light/src/on_demand/request.rs b/ethcore/light/src/on_demand/request.rs index 01792543c..0b16d091c 100644 --- a/ethcore/light/src/on_demand/request.rs +++ b/ethcore/light/src/on_demand/request.rs @@ -26,7 +26,7 @@ use ethcore::receipt::Receipt; use ethcore::state::{self, ProvedExecution}; use ethcore::transaction::SignedTransaction; -use rlp::{RlpStream, Stream, UntrustedRlp, View}; +use rlp::{RlpStream, UntrustedRlp, View}; use util::{Address, Bytes, DBValue, HashDB, H256, U256}; use util::memorydb::MemoryDB; use util::sha3::Hashable; @@ -323,7 +323,7 @@ mod tests { #[test] fn check_body() { - use rlp::{RlpStream, Stream}; + use rlp::RlpStream; let header = Header::new(); let mut body_stream = RlpStream::new_list(2); @@ -360,7 +360,7 @@ mod tests { #[test] fn check_state_proof() { - use rlp::{RlpStream, Stream}; + use rlp::RlpStream; let mut root = H256::default(); let mut db = MemoryDB::new(); diff --git a/ethcore/light/src/provider.rs b/ethcore/light/src/provider.rs index 3f55a6b99..d9f3937da 100644 --- a/ethcore/light/src/provider.rs +++ b/ethcore/light/src/provider.rs @@ -128,7 +128,7 @@ pub trait Provider: Send + Sync { /// /// Returns a vector of RLP-encoded lists satisfying the requests. fn proofs(&self, req: request::StateProofs) -> Vec { - use rlp::{RlpStream, Stream}; + use rlp::RlpStream; let mut results = Vec::with_capacity(req.requests.len()); @@ -166,7 +166,7 @@ pub trait Provider: Send + Sync { /// The first element is a block header and the second a merkle proof of /// the header in a requested CHT. fn header_proofs(&self, req: request::HeaderProofs) -> Vec { - use rlp::{self, RlpStream, Stream}; + use rlp::{self, RlpStream}; req.requests.into_iter() .map(|req| self.header_proof(req)) diff --git a/ethcore/src/block.rs b/ethcore/src/block.rs index 14f3df799..c0ae90bb4 100644 --- a/ethcore/src/block.rs +++ b/ethcore/src/block.rs @@ -20,7 +20,7 @@ use std::cmp; use std::sync::Arc; use std::collections::HashSet; -use rlp::{UntrustedRlp, RlpStream, Encodable, Decodable, Decoder, DecoderError, View, Stream}; +use rlp::{UntrustedRlp, RlpStream, Encodable, Decodable, Decoder, DecoderError, View}; use util::{Bytes, Address, Uint, Hashable, U256, H256, ordered_trie_root, SHA3_NULL_RLP}; use util::error::{Mismatch, OutOfBounds}; @@ -59,8 +59,8 @@ impl Block { pub fn rlp_bytes(&self, seal: Seal) -> Bytes { let mut block_rlp = RlpStream::new_list(3); self.header.stream_rlp(&mut block_rlp, seal); - block_rlp.append(&self.transactions); - block_rlp.append(&self.uncles); + block_rlp.append_list(&self.transactions); + block_rlp.append_list(&self.uncles); block_rlp.out() } } @@ -507,7 +507,7 @@ impl SealedBlock { pub fn rlp_bytes(&self) -> Bytes { let mut block_rlp = RlpStream::new_list(3); self.block.header.stream_rlp(&mut block_rlp, Seal::With); - block_rlp.append(&self.block.transactions); + block_rlp.append_list(&self.block.transactions); block_rlp.append_raw(&self.uncle_bytes, 1); block_rlp.out() } diff --git a/ethcore/src/blockchain/extras.rs b/ethcore/src/blockchain/extras.rs index 0e6dadbfe..3122a78d7 100644 --- a/ethcore/src/blockchain/extras.rs +++ b/ethcore/src/blockchain/extras.rs @@ -172,7 +172,7 @@ impl Encodable for BlockDetails { s.append(&self.number); s.append(&self.total_difficulty); s.append(&self.parent); - s.append(&self.children); + s.append_list(&self.children); } } @@ -233,7 +233,7 @@ impl Decodable for BlockReceipts { impl Encodable for BlockReceipts { fn rlp_append(&self, s: &mut RlpStream) { - Encodable::rlp_append(&self.receipts, s); + s.append_list(&self.receipts); } } diff --git a/ethcore/src/blockchain/generator/block.rs b/ethcore/src/blockchain/generator/block.rs index d7c958430..9dacb070e 100644 --- a/ethcore/src/blockchain/generator/block.rs +++ b/ethcore/src/blockchain/generator/block.rs @@ -37,8 +37,8 @@ impl Encodable for Block { fn rlp_append(&self, s: &mut RlpStream) { s.begin_list(3); s.append(&self.header); - s.append(&self.transactions); - s.append(&self.uncles); + s.append_list(&self.transactions); + s.append_list(&self.uncles); } } diff --git a/ethcore/src/blooms/bloom_group.rs b/ethcore/src/blooms/bloom_group.rs index c2961a7eb..6d8c40f75 100644 --- a/ethcore/src/blooms/bloom_group.rs +++ b/ethcore/src/blooms/bloom_group.rs @@ -63,7 +63,7 @@ impl Decodable for BloomGroup { impl Encodable for BloomGroup { fn rlp_append(&self, s: &mut RlpStream) { - Encodable::rlp_append(&self.blooms, s) + s.append_list(&self.blooms); } } diff --git a/ethcore/src/engines/tendermint/message.rs b/ethcore/src/engines/tendermint/message.rs index 1f6359ad4..ead58ba98 100644 --- a/ethcore/src/engines/tendermint/message.rs +++ b/ethcore/src/engines/tendermint/message.rs @@ -20,7 +20,7 @@ use util::*; use super::{Height, View, BlockHash, Step}; use error::Error; use header::Header; -use rlp::{Rlp, UntrustedRlp, RlpStream, Stream, RlpEncodable, Encodable, Decodable, Decoder, DecoderError, View as RlpView}; +use rlp::{Rlp, UntrustedRlp, RlpStream, Encodable, Decodable, Decoder, DecoderError, View as RlpView}; use ethkey::{recover, public_to_address}; use super::super::vote_collector::Message; @@ -162,7 +162,7 @@ impl Decodable for Step { impl Encodable for Step { fn rlp_append(&self, s: &mut RlpStream) { - RlpEncodable::rlp_append(&self.number(), s); + s.append_internal(&self.number()); } } @@ -278,6 +278,7 @@ mod tests { ::rlp::encode(&H520::default()).to_vec(), Vec::new() ]; + header.set_seal(seal); let message = ConsensusMessage::new_proposal(&header).unwrap(); assert_eq!( diff --git a/ethcore/src/engines/tendermint/mod.rs b/ethcore/src/engines/tendermint/mod.rs index aac101447..2d2714e97 100644 --- a/ethcore/src/engines/tendermint/mod.rs +++ b/ethcore/src/engines/tendermint/mod.rs @@ -243,7 +243,7 @@ impl Tendermint { let seal = vec![ ::rlp::encode(&view).to_vec(), ::rlp::encode(&seal.proposal).to_vec(), - ::rlp::encode(&seal.votes).to_vec() + ::rlp::encode_list(&seal.votes).to_vec() ]; self.submit_seal(block_hash, seal); self.to_next_height(height); @@ -825,7 +825,7 @@ mod tests { let vote_info = message_info_rlp(&VoteStep::new(2, 0, Step::Precommit), Some(header.bare_hash())); let signature1 = tap.sign(proposer, None, vote_info.sha3()).unwrap(); - seal[2] = ::rlp::encode(&vec![H520::from(signature1.clone())]).to_vec(); + seal[2] = ::rlp::encode_list(&vec![H520::from(signature1.clone())]).to_vec(); header.set_seal(seal.clone()); // One good signature is not enough. @@ -837,7 +837,7 @@ mod tests { let voter = insert_and_unlock(&tap, "0"); let signature0 = tap.sign(voter, None, vote_info.sha3()).unwrap(); - seal[2] = ::rlp::encode(&vec![H520::from(signature1.clone()), H520::from(signature0.clone())]).to_vec(); + seal[2] = ::rlp::encode_list(&vec![H520::from(signature1.clone()), H520::from(signature0.clone())]).to_vec(); header.set_seal(seal.clone()); assert!(engine.verify_block_family(&header, &parent_header, None).is_ok()); @@ -845,7 +845,7 @@ mod tests { let bad_voter = insert_and_unlock(&tap, "101"); let bad_signature = tap.sign(bad_voter, None, vote_info.sha3()).unwrap(); - seal[2] = ::rlp::encode(&vec![H520::from(signature1), H520::from(bad_signature)]).to_vec(); + seal[2] = ::rlp::encode_list(&vec![H520::from(signature1), H520::from(bad_signature)]).to_vec(); header.set_seal(seal); // One good and one bad signature. diff --git a/ethcore/src/engines/vote_collector.rs b/ethcore/src/engines/vote_collector.rs index 3f1354ad5..a2969db3a 100644 --- a/ethcore/src/engines/vote_collector.rs +++ b/ethcore/src/engines/vote_collector.rs @@ -245,7 +245,7 @@ mod tests { #[test] fn seal_retrieval() { - let collector = VoteCollector::default(); + let collector = VoteCollector::default(); let bh = Some("1".sha3()); let mut signatures = Vec::new(); for _ in 0..5 { @@ -284,7 +284,7 @@ mod tests { #[test] fn count_votes() { - let collector = VoteCollector::default(); + let collector = VoteCollector::default(); let round1 = 1; let round3 = 3; // good 1 @@ -318,7 +318,7 @@ mod tests { #[test] fn remove_old() { - let collector = VoteCollector::default(); + let collector = VoteCollector::default(); let vote = |round, hash| { random_vote(&collector, H520::random(), round, hash); }; @@ -334,7 +334,7 @@ mod tests { #[test] fn malicious_authority() { - let collector = VoteCollector::default(); + let collector = VoteCollector::default(); let round = 3; // Vote is inserted fine. assert!(full_vote(&collector, H520::random(), round, Some("0".sha3()), &Address::default()).is_none()); diff --git a/ethcore/src/executive.rs b/ethcore/src/executive.rs index 924f3353c..db1fe6a4e 100644 --- a/ethcore/src/executive.rs +++ b/ethcore/src/executive.rs @@ -36,7 +36,7 @@ const STACK_SIZE_PER_DEPTH: usize = 24*1024; /// Returns new address created from address and given nonce. pub fn contract_address(address: &Address, nonce: &U256) -> Address { - use rlp::{RlpStream, Stream}; + use rlp::RlpStream; let mut stream = RlpStream::new_list(2); stream.append(address); diff --git a/ethcore/src/migrations/state/v7.rs b/ethcore/src/migrations/state/v7.rs index bcfaf8c5d..9e2d3c110 100644 --- a/ethcore/src/migrations/state/v7.rs +++ b/ethcore/src/migrations/state/v7.rs @@ -26,7 +26,7 @@ use util::migration::{Batch, Config, Error, Migration, SimpleMigration, Progress use util::sha3::Hashable; use std::sync::Arc; -use rlp::{decode, Rlp, RlpStream, Stream, View}; +use rlp::{decode, Rlp, RlpStream, View}; // attempt to migrate a key, value pair. None if migration not possible. @@ -199,7 +199,7 @@ impl OverlayRecentV7 { stream.begin_list(2).append(&k).append(&v); } - stream.append(&deleted_keys); + stream.append_list(&deleted_keys); // and insert it into the new database. batch.insert(entry_key, stream.out(), dest)?; diff --git a/ethcore/src/migrations/v9.rs b/ethcore/src/migrations/v9.rs index 68c1e0655..7e469fb7b 100644 --- a/ethcore/src/migrations/v9.rs +++ b/ethcore/src/migrations/v9.rs @@ -17,7 +17,7 @@ //! This migration consolidates all databases into single one using Column Families. -use rlp::{Rlp, RlpStream, View, Stream}; +use rlp::{Rlp, RlpStream, View}; use util::kvdb::Database; use util::migration::{Batch, Config, Error, Migration, Progress}; use std::sync::Arc; diff --git a/ethcore/src/pod_account.rs b/ethcore/src/pod_account.rs index 98321639f..4b5dc8921 100644 --- a/ethcore/src/pod_account.rs +++ b/ethcore/src/pod_account.rs @@ -19,7 +19,7 @@ use state::Account; use account_db::AccountDBMut; use ethjson; use types::account_diff::*; -use rlp::{self, RlpStream, Stream}; +use rlp::{self, RlpStream}; #[derive(Debug, Clone, PartialEq, Eq)] /// An account, expressed as Plain-Old-Data (hence the name). diff --git a/ethcore/src/snapshot/account.rs b/ethcore/src/snapshot/account.rs index 44c8093b7..62ee28f89 100644 --- a/ethcore/src/snapshot/account.rs +++ b/ethcore/src/snapshot/account.rs @@ -22,7 +22,7 @@ use snapshot::Error; use util::{U256, H256, Bytes, HashDB, SHA3_EMPTY, SHA3_NULL_RLP}; use util::trie::{TrieDB, Trie}; -use rlp::{RlpStream, Stream, UntrustedRlp, View}; +use rlp::{RlpStream, UntrustedRlp, View}; use std::collections::HashSet; diff --git a/ethcore/src/snapshot/block.rs b/ethcore/src/snapshot/block.rs index 048724ec9..0eee05575 100644 --- a/ethcore/src/snapshot/block.rs +++ b/ethcore/src/snapshot/block.rs @@ -20,7 +20,7 @@ use block::Block; use header::Header; use views::BlockView; -use rlp::{DecoderError, RlpStream, Stream, UntrustedRlp, View}; +use rlp::{DecoderError, RlpStream, UntrustedRlp, View}; use util::{Bytes, Hashable, H256}; use util::triehash::ordered_trie_root; @@ -69,7 +69,9 @@ impl AbridgedBlock { .append(&header.extra_data()); // write block values. - stream.append(&block_view.transactions()).append(&block_view.uncles()); + stream + .append_list(&block_view.transactions()) + .append_list(&block_view.uncles()); // write seal fields. for field in seal_fields { @@ -108,7 +110,7 @@ impl AbridgedBlock { header.set_receipts_root(receipts_root); let mut uncles_rlp = RlpStream::new(); - uncles_rlp.append(&uncles); + uncles_rlp.append_list(&uncles); header.set_uncles_hash(uncles_rlp.as_raw().sha3()); let mut seal_fields = Vec::new(); diff --git a/ethcore/src/snapshot/io.rs b/ethcore/src/snapshot/io.rs index 362faf80b..45f3ec4df 100644 --- a/ethcore/src/snapshot/io.rs +++ b/ethcore/src/snapshot/io.rs @@ -27,7 +27,7 @@ use std::path::{Path, PathBuf}; use util::Bytes; use util::hash::H256; -use rlp::{self, Encodable, RlpStream, UntrustedRlp, Stream, View}; +use rlp::{self, Encodable, RlpStream, UntrustedRlp, View}; use super::ManifestData; @@ -122,8 +122,8 @@ impl SnapshotWriter for PackedWriter { // they are consistent with ours. let mut stream = RlpStream::new_list(5); stream - .append(&self.state_hashes) - .append(&self.block_hashes) + .append_list(&self.state_hashes) + .append_list(&self.block_hashes) .append(&manifest.state_root) .append(&manifest.block_number) .append(&manifest.block_hash); @@ -428,4 +428,4 @@ mod tests { reader.chunk(hash.clone()).unwrap(); } } -} \ No newline at end of file +} diff --git a/ethcore/src/snapshot/mod.rs b/ethcore/src/snapshot/mod.rs index e8001e54b..473c91e4d 100644 --- a/ethcore/src/snapshot/mod.rs +++ b/ethcore/src/snapshot/mod.rs @@ -37,7 +37,7 @@ use util::journaldb::{self, Algorithm, JournalDB}; use util::kvdb::Database; use util::trie::{TrieDB, TrieDBMut, Trie, TrieMut}; use util::sha3::SHA3_NULL_RLP; -use rlp::{RlpStream, Stream, UntrustedRlp, View}; +use rlp::{RlpStream, UntrustedRlp, View}; use bloom_journal::Bloom; use self::block::AbridgedBlock; diff --git a/ethcore/src/snapshot/tests/blocks.rs b/ethcore/src/snapshot/tests/blocks.rs index 89a01fbd7..f2ee40a7b 100644 --- a/ethcore/src/snapshot/tests/blocks.rs +++ b/ethcore/src/snapshot/tests/blocks.rs @@ -99,7 +99,7 @@ fn chunk_and_restore_40k() { chunk_and_restore(40000) } #[test] fn checks_flag() { - use ::rlp::{RlpStream, Stream}; + use rlp::RlpStream; use util::H256; let mut stream = RlpStream::new_list(5); diff --git a/ethcore/src/snapshot/tests/state.rs b/ethcore/src/snapshot/tests/state.rs index 809e9fe21..854cd07d4 100644 --- a/ethcore/src/snapshot/tests/state.rs +++ b/ethcore/src/snapshot/tests/state.rs @@ -95,7 +95,7 @@ fn snap_and_restore() { #[test] fn get_code_from_prev_chunk() { use std::collections::HashSet; - use rlp::{RlpStream, Stream}; + use rlp::RlpStream; use util::{HashDB, H256, U256, Hashable}; use account_db::{AccountDBMut, AccountDB}; diff --git a/ethcore/src/spec/seal.rs b/ethcore/src/spec/seal.rs index 6e301a9ac..a1e929604 100644 --- a/ethcore/src/spec/seal.rs +++ b/ethcore/src/spec/seal.rs @@ -64,9 +64,12 @@ impl Into for AuthorityRound { impl Into for Tendermint { fn into(self) -> Generic { - let mut s = RlpStream::new_list(3); - s.append(&self.round).append(&self.proposal).append(&self.precommits); - Generic(s.out()) + let mut stream = RlpStream::new_list(3); + stream + .append(&self.round) + .append(&self.proposal) + .append_list(&self.precommits); + Generic(stream.out()) } } diff --git a/ethcore/src/spec/spec.rs b/ethcore/src/spec/spec.rs index 3b7f84acf..1bc693a4f 100644 --- a/ethcore/src/spec/spec.rs +++ b/ethcore/src/spec/spec.rs @@ -34,7 +34,7 @@ use super::genesis::Genesis; use super::seal::Generic as GenericSeal; use ethereum; use ethjson; -use rlp::{Rlp, RlpStream, View, Stream}; +use rlp::{Rlp, RlpStream, View}; /// Parameters common to all engines. #[derive(Debug, PartialEq, Clone, Default)] diff --git a/ethcore/src/tests/helpers.rs b/ethcore/src/tests/helpers.rs index 7de46b0b3..dcb173b63 100644 --- a/ethcore/src/tests/helpers.rs +++ b/ethcore/src/tests/helpers.rs @@ -34,7 +34,7 @@ use devtools::*; use miner::Miner; use header::Header; use transaction::{Action, Transaction, SignedTransaction}; -use rlp::{self, RlpStream, Stream}; +use rlp::{self, RlpStream}; use views::BlockView; #[cfg(feature = "json-tests")] @@ -129,7 +129,7 @@ pub fn create_test_block_with_data(header: &Header, transactions: &[SignedTransa for t in transactions { rlp.append_raw(&rlp::encode(t).to_vec(), 1); } - rlp.append(&uncles); + rlp.append_list(&uncles); rlp.out() } diff --git a/ethcore/src/trace/bloom.rs b/ethcore/src/trace/bloom.rs index 4e2bd4eca..e0a32d5ae 100644 --- a/ethcore/src/trace/bloom.rs +++ b/ethcore/src/trace/bloom.rs @@ -83,7 +83,7 @@ impl Decodable for BlockTracesBloomGroup { impl Encodable for BlockTracesBloomGroup { fn rlp_append(&self, s: &mut RlpStream) { - Encodable::rlp_append(&self.blooms, s) + s.append_list(&self.blooms); } } diff --git a/ethcore/src/types/log_entry.rs b/ethcore/src/types/log_entry.rs index 23155148a..3eac1dad5 100644 --- a/ethcore/src/types/log_entry.rs +++ b/ethcore/src/types/log_entry.rs @@ -41,7 +41,7 @@ impl Encodable for LogEntry { fn rlp_append(&self, s: &mut RlpStream) { s.begin_list(3); s.append(&self.address); - s.append(&self.topics); + s.append_list(&self.topics); s.append(&self.data); } } diff --git a/ethcore/src/types/receipt.rs b/ethcore/src/types/receipt.rs index f9e478b6a..5db74b0b6 100644 --- a/ethcore/src/types/receipt.rs +++ b/ethcore/src/types/receipt.rs @@ -60,7 +60,7 @@ impl Encodable for Receipt { } s.append(&self.gas_used); s.append(&self.log_bloom); - s.append(&self.logs); + s.append_list(&self.logs); } } diff --git a/ethcore/src/types/snapshot_manifest.rs b/ethcore/src/types/snapshot_manifest.rs index 24f56efeb..910a038bd 100644 --- a/ethcore/src/types/snapshot_manifest.rs +++ b/ethcore/src/types/snapshot_manifest.rs @@ -40,8 +40,8 @@ impl ManifestData { /// Encode the manifest data to rlp. pub fn into_rlp(self) -> Bytes { let mut stream = RlpStream::new_list(5); - stream.append(&self.state_hashes); - stream.append(&self.block_hashes); + stream.append_list(&self.state_hashes); + stream.append_list(&self.block_hashes); stream.append(&self.state_root); stream.append(&self.block_number); stream.append(&self.block_hash); diff --git a/ethcore/src/types/trace_types/error.rs b/ethcore/src/types/trace_types/error.rs index ea3d32679..33ccf2bb7 100644 --- a/ethcore/src/types/trace_types/error.rs +++ b/ethcore/src/types/trace_types/error.rs @@ -17,7 +17,7 @@ //! Trace errors. use std::fmt; -use rlp::{RlpEncodable, Encodable, RlpStream, Decodable, Decoder, DecoderError, View}; +use rlp::{Encodable, RlpStream, Decodable, Decoder, DecoderError, View}; use evm::Error as EvmError; /// Trace evm errors. @@ -85,7 +85,8 @@ impl Encodable for Error { OutOfStack => 4, Internal => 5, }; - RlpEncodable::rlp_append(&value, s); + + s.append_internal(&value); } } diff --git a/ethcore/src/types/trace_types/flat.rs b/ethcore/src/types/trace_types/flat.rs index 870f13802..832e7d055 100644 --- a/ethcore/src/types/trace_types/flat.rs +++ b/ethcore/src/types/trace_types/flat.rs @@ -59,7 +59,7 @@ impl Encodable for FlatTrace { s.append(&self.action); s.append(&self.result); s.append(&self.subtraces); - s.append(&self.trace_address.clone().into_iter().collect::>()); + s.append_list::(&self.trace_address.iter().collect::>()); } } @@ -103,7 +103,7 @@ impl FlatTransactionTraces { impl Encodable for FlatTransactionTraces { fn rlp_append(&self, s: &mut RlpStream) { - Encodable::rlp_append(&self.0, s); + s.append_list(&self.0); } } @@ -144,7 +144,7 @@ impl FlatBlockTraces { impl Encodable for FlatBlockTraces { fn rlp_append(&self, s: &mut RlpStream) { - Encodable::rlp_append(&self.0, s); + s.append_list(&self.0); } } diff --git a/ethcore/src/types/trace_types/trace.rs b/ethcore/src/types/trace_types/trace.rs index 12df1de25..9c3377dac 100644 --- a/ethcore/src/types/trace_types/trace.rs +++ b/ethcore/src/types/trace_types/trace.rs @@ -475,7 +475,7 @@ impl Encodable for VMExecutedOperation { fn rlp_append(&self, s: &mut RlpStream) { s.begin_list(4); s.append(&self.gas_used); - s.append(&self.stack_push); + s.append_list(&self.stack_push); s.append(&self.mem_diff); s.append(&self.store_diff); } @@ -551,8 +551,8 @@ impl Encodable for VMTrace { s.begin_list(4); s.append(&self.parent_step); s.append(&self.code); - s.append(&self.operations); - s.append(&self.subs); + s.append_list(&self.operations); + s.append_list(&self.subs); } } diff --git a/ethcore/src/verification/verification.rs b/ethcore/src/verification/verification.rs index c2c932066..7fbeb29bb 100644 --- a/ethcore/src/verification/verification.rs +++ b/ethcore/src/verification/verification.rs @@ -395,7 +395,7 @@ mod tests { #[test] #[cfg_attr(feature="dev", allow(similar_names))] fn test_verify_block() { - use rlp::{RlpStream, Stream}; + use rlp::RlpStream; // Test against morden let mut good = Header::new(); @@ -460,7 +460,7 @@ mod tests { let good_uncles = vec![ good_uncle1.clone(), good_uncle2.clone() ]; let mut uncles_rlp = RlpStream::new(); - uncles_rlp.append(&good_uncles); + uncles_rlp.append_list(&good_uncles); let good_uncles_hash = uncles_rlp.as_raw().sha3(); let good_transactions_root = ordered_trie_root(good_transactions.iter().map(|t| ::rlp::encode::(t).to_vec())); diff --git a/ipfs/src/route.rs b/ipfs/src/route.rs index 5b571885d..fecafb264 100644 --- a/ipfs/src/route.rs +++ b/ipfs/src/route.rs @@ -79,7 +79,7 @@ impl IpfsHandler { fn block_list(&self, hash: H256) -> Result { let uncles = self.client().find_uncles(&hash).ok_or(Error::BlockNotFound)?; - Ok(Out::OctetStream(rlp::encode(&uncles).to_vec())) + Ok(Out::OctetStream(rlp::encode_list(&uncles).to_vec())) } /// Get transaction by hash and return as raw binary. diff --git a/rpc/src/v1/helpers/dispatch.rs b/rpc/src/v1/helpers/dispatch.rs index b11ada048..8ff08b965 100644 --- a/rpc/src/v1/helpers/dispatch.rs +++ b/rpc/src/v1/helpers/dispatch.rs @@ -25,7 +25,7 @@ use light::cache::Cache as LightDataCache; use light::client::LightChainClient; use light::on_demand::{request, OnDemand}; use light::TransactionQueue as LightTransactionQueue; -use rlp::{self, Stream as StreamRlp}; +use rlp; use util::{Address, H520, H256, U256, Uint, Bytes, Mutex, RwLock}; use util::sha3::Hashable; use stats::Corpus; diff --git a/sync/src/chain.rs b/sync/src/chain.rs index 2212876d1..415b44a39 100644 --- a/sync/src/chain.rs +++ b/sync/src/chain.rs @@ -2177,7 +2177,7 @@ mod tests { use util::sha3::Hashable; use util::hash::H256; use util::bytes::Bytes; - use rlp::{Rlp, RlpStream, UntrustedRlp, View, Stream}; + use rlp::{Rlp, RlpStream, UntrustedRlp, View}; use super::*; use ::SyncConfig; use super::{PeerInfo, PeerAsking}; diff --git a/util/network/src/discovery.rs b/util/network/src/discovery.rs index f3a8aa523..ae4c7e565 100644 --- a/util/network/src/discovery.rs +++ b/util/network/src/discovery.rs @@ -213,7 +213,7 @@ impl Discovery { let nearest = Discovery::nearest_node_entries(&self.discovery_id, &self.node_buckets).into_iter(); let nearest = nearest.filter(|x| !self.discovery_nodes.contains(&x.id)).take(ALPHA).collect::>(); for r in nearest { - let rlp = encode(&(&[self.discovery_id.clone()][..])); + let rlp = encode_list(&(&[self.discovery_id.clone()][..])); self.send_packet(PACKET_FIND_NODE, &r.endpoint.udp_address(), &rlp); self.discovery_nodes.insert(r.id.clone()); tried_count += 1; diff --git a/util/network/src/session.rs b/util/network/src/session.rs index dc4a5464c..bc416b8d8 100644 --- a/util/network/src/session.rs +++ b/util/network/src/session.rs @@ -59,8 +59,8 @@ pub struct Session { ping_time_ns: u64, pong_time_ns: Option, state: State, - // Protocol states -- accumulates pending packets until signaled as ready. - protocol_states: HashMap, + // Protocol states -- accumulates pending packets until signaled as ready. + protocol_states: HashMap, } enum State { @@ -198,7 +198,7 @@ impl Session { ping_time_ns: 0, pong_time_ns: None, expired: false, - protocol_states: HashMap::new(), + protocol_states: HashMap::new(), }) } @@ -374,16 +374,16 @@ impl Session { self.connection().token() } - /// Signal that a subprotocol has handled the connection successfully and + /// Signal that a subprotocol has handled the connection successfully and /// get all pending packets in order received. pub fn mark_connected(&mut self, protocol: ProtocolId) -> Vec<(ProtocolId, u8, Vec)> { match self.protocol_states.insert(protocol, ProtocolState::Connected) { - None => Vec::new(), + None => Vec::new(), Some(ProtocolState::Connected) => { debug!(target: "network", "Protocol {:?} marked as connected more than once", protocol); Vec::new() } - Some(ProtocolState::Pending(pending)) => + Some(ProtocolState::Pending(pending)) => pending.into_iter().map(|(data, id)| (protocol, id, data)).collect(), } } @@ -463,7 +463,7 @@ impl Session { rlp.begin_list(5) .append(&host.protocol_version) .append(&host.client_version) - .append(&host.capabilities) + .append_list(&host.capabilities) .append(&host.local_endpoint.address.port()) .append(host.id()); self.send(io, rlp) @@ -515,7 +515,7 @@ impl Session { self.info.protocol_version = protocol; self.info.client_version = client_version; self.info.capabilities = caps; - self.info.peer_capabilities = peer_caps; + self.info.peer_capabilities = peer_caps; if self.info.capabilities.is_empty() { trace!(target: "network", "No common capabilities with peer."); return Err(From::from(self.disconnect(io, DisconnectReason::UselessPeer))); diff --git a/util/rlp/Cargo.toml b/util/rlp/Cargo.toml index ae6b4accd..4f31e9467 100644 --- a/util/rlp/Cargo.toml +++ b/util/rlp/Cargo.toml @@ -10,3 +10,4 @@ elastic-array = { git = "https://github.com/ethcore/elastic-array" } ethcore-bigint = { path = "../bigint" } lazy_static = "0.2" rustc-serialize = "0.3" +byteorder = "1.0" diff --git a/util/rlp/src/bytes.rs b/util/rlp/src/bytes.rs index 30d434203..e5f266f92 100644 --- a/util/rlp/src/bytes.rs +++ b/util/rlp/src/bytes.rs @@ -17,152 +17,9 @@ //! Unified interfaces for RLP bytes operations on basic types //! -use std::mem; -use std::fmt; -use std::cmp::Ordering; +use std::{mem, fmt, cmp}; use std::error::Error as StdError; -use bigint::prelude::{Uint, U128, U256, H64, H128, H160, H256, H512, H520, H2048}; -use elastic_array::*; - -/// Vector like object -pub trait VecLike { - /// Add an element to the collection - fn vec_push(&mut self, value: T); - - /// Add a slice to the collection - fn vec_extend(&mut self, slice: &[T]); -} - -impl VecLike for Vec where T: Copy { - fn vec_push(&mut self, value: T) { - Vec::::push(self, value) - } - - fn vec_extend(&mut self, slice: &[T]) { - Vec::::extend_from_slice(self, slice) - } -} - -macro_rules! impl_veclike_for_elastic_array { - ($from: ident) => { - impl VecLike for $from where T: Copy { - fn vec_push(&mut self, value: T) { - $from::::push(self, value) - } - fn vec_extend(&mut self, slice: &[T]) { - $from::::append_slice(self, slice) - - } - } - } -} - -impl_veclike_for_elastic_array!(ElasticArray16); -impl_veclike_for_elastic_array!(ElasticArray32); -impl_veclike_for_elastic_array!(ElasticArray1024); - -/// Converts given type to its shortest representation in bytes -/// -/// TODO: optimise some conversations -pub trait ToBytes { - /// Serialize self to byte array - fn to_bytes>(&self, out: &mut V); - /// Get length of serialized data in bytes - fn to_bytes_len(&self) -> usize; -} - -impl <'a> ToBytes for &'a str { - fn to_bytes>(&self, out: &mut V) { - out.vec_extend(self.as_bytes()); - } - - fn to_bytes_len(&self) -> usize { - self.as_bytes().len() - } -} - -impl ToBytes for String { - fn to_bytes>(&self, out: &mut V) { - out.vec_extend(self.as_bytes()); - } - - fn to_bytes_len(&self) -> usize { - self.len() - } -} - -impl ToBytes for u64 { - fn to_bytes>(&self, out: &mut V) { - let count = self.to_bytes_len(); - for i in 0..count { - let j = count - 1 - i; - out.vec_push((*self >> (j * 8)) as u8); - } - } - - fn to_bytes_len(&self) -> usize { 8 - self.leading_zeros() as usize / 8 } -} - -impl ToBytes for bool { - fn to_bytes>(&self, out: &mut V) { - out.vec_push(if *self { 1u8 } else { 0u8 }) - } - - fn to_bytes_len(&self) -> usize { 1 } -} - -macro_rules! impl_map_to_bytes { - ($from: ident, $to: ty) => { - impl ToBytes for $from { - fn to_bytes>(&self, out: &mut V) { - (*self as $to).to_bytes(out) - } - - fn to_bytes_len(&self) -> usize { (*self as $to).to_bytes_len() } - } - } -} - -impl_map_to_bytes!(usize, u64); -impl_map_to_bytes!(u16, u64); -impl_map_to_bytes!(u32, u64); - -macro_rules! impl_uint_to_bytes { - ($name: ident) => { - impl ToBytes for $name { - fn to_bytes>(&self, out: &mut V) { - let count = self.to_bytes_len(); - for i in 0..count { - let j = count - 1 - i; - out.vec_push(self.byte(j)); - } - } - fn to_bytes_len(&self) -> usize { (self.bits() + 7) / 8 } - } - } -} - -impl_uint_to_bytes!(U256); -impl_uint_to_bytes!(U128); - -macro_rules! impl_hash_to_bytes { - ($name: ident) => { - impl ToBytes for $name { - fn to_bytes>(&self, out: &mut V) { - out.vec_extend(&self); - } - fn to_bytes_len(&self) -> usize { self.len() } - } - } -} - -impl_hash_to_bytes!(H64); -impl_hash_to_bytes!(H128); -impl_hash_to_bytes!(H160); -impl_hash_to_bytes!(H256); -impl_hash_to_bytes!(H512); -impl_hash_to_bytes!(H520); -impl_hash_to_bytes!(H2048); +use bigint::prelude::{U128, U256, H64, H128, H160, H256, H512, H520, H2048}; /// Error returned when `FromBytes` conversation goes wrong #[derive(Debug, PartialEq, Eq)] @@ -268,9 +125,9 @@ macro_rules! impl_hash_from_bytes { impl FromBytes for $name { fn from_bytes(bytes: &[u8]) -> FromBytesResult<$name> { match bytes.len().cmp(&$size) { - Ordering::Less => Err(FromBytesError::DataIsTooShort), - Ordering::Greater => Err(FromBytesError::DataIsTooLong), - Ordering::Equal => { + cmp::Ordering::Less => Err(FromBytesError::DataIsTooShort), + cmp::Ordering::Greater => Err(FromBytesError::DataIsTooLong), + cmp::Ordering::Equal => { let mut t = [0u8; $size]; t.copy_from_slice(bytes); Ok($name(t)) diff --git a/util/rlp/src/commonrlps.rs b/util/rlp/src/common.rs similarity index 98% rename from util/rlp/src/commonrlps.rs rename to util/rlp/src/common.rs index 4dd8788cd..0be63864a 100644 --- a/util/rlp/src/commonrlps.rs +++ b/util/rlp/src/common.rs @@ -16,7 +16,7 @@ //! Contains RLPs used for compression. -use rlpcompression::InvalidRlpSwapper; +use compression::InvalidRlpSwapper; lazy_static! { /// Swapper for snapshot compression. @@ -47,4 +47,4 @@ static COMMON_RLPS: &'static [&'static [u8]] = &[ &[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] ]; -static INVALID_RLPS: &'static [&'static [u8]] = &[&[0x81, 0x0], &[0x81, 0x1], &[0x81, 0x2], &[0x81, 0x3], &[0x81, 0x4], &[0x81, 0x5], &[0x81, 0x6], &[0x81, 0x7], &[0x81, 0x8], &[0x81, 0x9], &[0x81, 0xa], &[0x81, 0xb], &[0x81, 0xc], &[0x81, 0xd], &[0x81, 0xe], &[0x81, 0xf], &[0x81, 0x10], &[0x81, 0x11], &[0x81, 0x12], &[0x81, 0x13], &[0x81, 0x14], &[0x81, 0x15], &[0x81, 0x16], &[0x81, 0x17], &[0x81, 0x18], &[0x81, 0x19], &[0x81, 0x1a], &[0x81, 0x1b], &[0x81, 0x1c], &[0x81, 0x1d], &[0x81, 0x1e], &[0x81, 0x1f], &[0x81, 0x20], &[0x81, 0x21], &[0x81, 0x22], &[0x81, 0x23], &[0x81, 0x24], &[0x81, 0x25], &[0x81, 0x26], &[0x81, 0x27], &[0x81, 0x28], &[0x81, 0x29], &[0x81, 0x2a], &[0x81, 0x2b], &[0x81, 0x2c], &[0x81, 0x2d], &[0x81, 0x2e], &[0x81, 0x2f], &[0x81, 0x30], &[0x81, 0x31], &[0x81, 0x32], &[0x81, 0x33], &[0x81, 0x34], &[0x81, 0x35], &[0x81, 0x36], &[0x81, 0x37], &[0x81, 0x38], &[0x81, 0x39], &[0x81, 0x3a], &[0x81, 0x3b], &[0x81, 0x3c], &[0x81, 0x3d], &[0x81, 0x3e], &[0x81, 0x3f], &[0x81, 0x40], &[0x81, 0x41], &[0x81, 0x42], &[0x81, 0x43], &[0x81, 0x44], &[0x81, 0x45], &[0x81, 0x46], &[0x81, 0x47], &[0x81, 0x48], &[0x81, 0x49], &[0x81, 0x4a], &[0x81, 0x4b], &[0x81, 0x4c], &[0x81, 0x4d], &[0x81, 0x4e], &[0x81, 0x4f], &[0x81, 0x50], &[0x81, 0x51], &[0x81, 0x52], &[0x81, 0x53], &[0x81, 0x54], &[0x81, 0x55], &[0x81, 0x56], &[0x81, 0x57], &[0x81, 0x58], &[0x81, 0x59], &[0x81, 0x5a], &[0x81, 0x5b], &[0x81, 0x5c], &[0x81, 0x5d], &[0x81, 0x5e], &[0x81, 0x5f], &[0x81, 0x60], &[0x81, 0x61], &[0x81, 0x62], &[0x81, 0x63], &[0x81, 0x64], &[0x81, 0x65], &[0x81, 0x66], &[0x81, 0x67], &[0x81, 0x68], &[0x81, 0x69], &[0x81, 0x6a], &[0x81, 0x6b], &[0x81, 0x6c], &[0x81, 0x6d], &[0x81, 0x6e], &[0x81, 0x6f], &[0x81, 0x70], &[0x81, 0x71], &[0x81, 0x72], &[0x81, 0x73], &[0x81, 0x74], &[0x81, 0x75], &[0x81, 0x76], &[0x81, 0x77], &[0x81, 0x78], &[0x81, 0x79], &[0x81, 0x7a], &[0x81, 0x7b], &[0x81, 0x7c], &[0x81, 0x7d], &[0x81, 0x7e]]; \ No newline at end of file +static INVALID_RLPS: &'static [&'static [u8]] = &[&[0x81, 0x0], &[0x81, 0x1], &[0x81, 0x2], &[0x81, 0x3], &[0x81, 0x4], &[0x81, 0x5], &[0x81, 0x6], &[0x81, 0x7], &[0x81, 0x8], &[0x81, 0x9], &[0x81, 0xa], &[0x81, 0xb], &[0x81, 0xc], &[0x81, 0xd], &[0x81, 0xe], &[0x81, 0xf], &[0x81, 0x10], &[0x81, 0x11], &[0x81, 0x12], &[0x81, 0x13], &[0x81, 0x14], &[0x81, 0x15], &[0x81, 0x16], &[0x81, 0x17], &[0x81, 0x18], &[0x81, 0x19], &[0x81, 0x1a], &[0x81, 0x1b], &[0x81, 0x1c], &[0x81, 0x1d], &[0x81, 0x1e], &[0x81, 0x1f], &[0x81, 0x20], &[0x81, 0x21], &[0x81, 0x22], &[0x81, 0x23], &[0x81, 0x24], &[0x81, 0x25], &[0x81, 0x26], &[0x81, 0x27], &[0x81, 0x28], &[0x81, 0x29], &[0x81, 0x2a], &[0x81, 0x2b], &[0x81, 0x2c], &[0x81, 0x2d], &[0x81, 0x2e], &[0x81, 0x2f], &[0x81, 0x30], &[0x81, 0x31], &[0x81, 0x32], &[0x81, 0x33], &[0x81, 0x34], &[0x81, 0x35], &[0x81, 0x36], &[0x81, 0x37], &[0x81, 0x38], &[0x81, 0x39], &[0x81, 0x3a], &[0x81, 0x3b], &[0x81, 0x3c], &[0x81, 0x3d], &[0x81, 0x3e], &[0x81, 0x3f], &[0x81, 0x40], &[0x81, 0x41], &[0x81, 0x42], &[0x81, 0x43], &[0x81, 0x44], &[0x81, 0x45], &[0x81, 0x46], &[0x81, 0x47], &[0x81, 0x48], &[0x81, 0x49], &[0x81, 0x4a], &[0x81, 0x4b], &[0x81, 0x4c], &[0x81, 0x4d], &[0x81, 0x4e], &[0x81, 0x4f], &[0x81, 0x50], &[0x81, 0x51], &[0x81, 0x52], &[0x81, 0x53], &[0x81, 0x54], &[0x81, 0x55], &[0x81, 0x56], &[0x81, 0x57], &[0x81, 0x58], &[0x81, 0x59], &[0x81, 0x5a], &[0x81, 0x5b], &[0x81, 0x5c], &[0x81, 0x5d], &[0x81, 0x5e], &[0x81, 0x5f], &[0x81, 0x60], &[0x81, 0x61], &[0x81, 0x62], &[0x81, 0x63], &[0x81, 0x64], &[0x81, 0x65], &[0x81, 0x66], &[0x81, 0x67], &[0x81, 0x68], &[0x81, 0x69], &[0x81, 0x6a], &[0x81, 0x6b], &[0x81, 0x6c], &[0x81, 0x6d], &[0x81, 0x6e], &[0x81, 0x6f], &[0x81, 0x70], &[0x81, 0x71], &[0x81, 0x72], &[0x81, 0x73], &[0x81, 0x74], &[0x81, 0x75], &[0x81, 0x76], &[0x81, 0x77], &[0x81, 0x78], &[0x81, 0x79], &[0x81, 0x7a], &[0x81, 0x7b], &[0x81, 0x7c], &[0x81, 0x7d], &[0x81, 0x7e]]; diff --git a/util/rlp/src/rlpcompression.rs b/util/rlp/src/compression.rs similarity index 97% rename from util/rlp/src/rlpcompression.rs rename to util/rlp/src/compression.rs index e26105a24..b7cf72a4f 100644 --- a/util/rlp/src/rlpcompression.rs +++ b/util/rlp/src/compression.rs @@ -14,11 +14,10 @@ // You should have received a copy of the GNU General Public License // along with Parity. If not, see . -use ::{UntrustedRlp, View, Compressible, encode, Stream, RlpStream}; -use commonrlps::{BLOCKS_RLP_SWAPPER, SNAPSHOT_RLP_SWAPPER}; - use std::collections::HashMap; use elastic_array::ElasticArray1024; +use common::{BLOCKS_RLP_SWAPPER, SNAPSHOT_RLP_SWAPPER}; +use {UntrustedRlp, View, Compressible, encode, RlpStream}; /// Stores RLPs used for compression pub struct InvalidRlpSwapper<'a> { @@ -149,8 +148,6 @@ fn deep_decompress(rlp: &UntrustedRlp, swapper: &InvalidRlpSwapper) -> Option Compressible for UntrustedRlp<'a> { type DataType = RlpType; @@ -171,8 +168,8 @@ impl<'a> Compressible for UntrustedRlp<'a> { #[cfg(test)] mod tests { - use ::{UntrustedRlp, Compressible, View, RlpType}; - use rlpcompression::InvalidRlpSwapper; + use compression::InvalidRlpSwapper; + use {UntrustedRlp, Compressible, View, RlpType}; #[test] fn invalid_rlp_swapper() { diff --git a/util/rlp/src/rlperrors.rs b/util/rlp/src/error.rs similarity index 100% rename from util/rlp/src/rlperrors.rs rename to util/rlp/src/error.rs diff --git a/util/rlp/src/impls.rs b/util/rlp/src/impls.rs new file mode 100644 index 000000000..affac1ddc --- /dev/null +++ b/util/rlp/src/impls.rs @@ -0,0 +1,120 @@ +use byteorder::{ByteOrder, BigEndian}; +use bigint::prelude::{Uint, U128, U256, H64, H128, H160, H256, H512, H520, H2048}; +use traits::Encodable; +use stream::RlpStream; + +impl Encodable for bool { + fn rlp_append(&self, s: &mut RlpStream) { + if *self { + s.encoder().encode_value(&[1]); + } else { + s.encoder().encode_value(&[0]); + } + } +} + +impl<'a> Encodable for &'a [u8] { + fn rlp_append(&self, s: &mut RlpStream) { + s.encoder().encode_value(self); + } +} + +impl Encodable for Vec { + fn rlp_append(&self, s: &mut RlpStream) { + s.encoder().encode_value(self); + } +} + +impl Encodable for Option where T: Encodable { + fn rlp_append(&self, s: &mut RlpStream) { + match *self { + None => { + s.begin_list(0); + }, + Some(ref value) => { + s.begin_list(1); + s.append(value); + } + } + } +} + +impl Encodable for u8 { + fn rlp_append(&self, s: &mut RlpStream) { + if *self != 0 { + s.encoder().encode_value(&[*self]); + } else { + s.encoder().encode_value(&[]); + } + } +} + +macro_rules! impl_encodable_for_u { + ($name: ident, $func: ident, $size: expr) => { + impl Encodable for $name { + fn rlp_append(&self, s: &mut RlpStream) { + let leading_empty_bytes = self.leading_zeros() as usize / 8; + let mut buffer = [0u8; $size]; + BigEndian::$func(&mut buffer, *self); + s.encoder().encode_value(&buffer[leading_empty_bytes..]); + } + } + } +} + +impl_encodable_for_u!(u16, write_u16, 2); +impl_encodable_for_u!(u32, write_u32, 4); +impl_encodable_for_u!(u64, write_u64, 8); + +impl Encodable for usize { + fn rlp_append(&self, s: &mut RlpStream) { + (*self as u64).rlp_append(s); + } +} + +macro_rules! impl_encodable_for_hash { + ($name: ident) => { + impl Encodable for $name { + fn rlp_append(&self, s: &mut RlpStream) { + s.encoder().encode_value(self); + } + } + } +} + +impl_encodable_for_hash!(H64); +impl_encodable_for_hash!(H128); +impl_encodable_for_hash!(H160); +impl_encodable_for_hash!(H256); +impl_encodable_for_hash!(H512); +impl_encodable_for_hash!(H520); +impl_encodable_for_hash!(H2048); + +macro_rules! impl_encodable_for_uint { + ($name: ident, $size: expr) => { + impl Encodable for $name { + fn rlp_append(&self, s: &mut RlpStream) { + let leading_empty_bytes = $size - (self.bits() + 7) / 8; + let mut buffer = [0u8; $size]; + self.to_big_endian(&mut buffer); + s.encoder().encode_value(&buffer[leading_empty_bytes..]); + } + } + } +} + +impl_encodable_for_uint!(U256, 32); +impl_encodable_for_uint!(U128, 16); + +impl<'a> Encodable for &'a str { + fn rlp_append(&self, s: &mut RlpStream) { + s.encoder().encode_value(self.as_bytes()); + } +} + +impl Encodable for String { + fn rlp_append(&self, s: &mut RlpStream) { + s.encoder().encode_value(self.as_bytes()); + } +} + diff --git a/util/rlp/src/lib.rs b/util/rlp/src/lib.rs index d060a2d23..406501b1e 100644 --- a/util/rlp/src/lib.rs +++ b/util/rlp/src/lib.rs @@ -46,25 +46,7 @@ //! * You want to get view onto rlp-slice. //! * You don't want to decode whole rlp at once. -pub mod rlptraits; -mod rlperrors; -mod rlpin; -mod untrusted_rlp; -mod rlpstream; -mod rlpcompression; -mod commonrlps; -mod bytes; - -#[cfg(test)] -mod tests; - -pub use self::rlperrors::DecoderError; -pub use self::rlptraits::{Decoder, Decodable, View, Stream, Encodable, Encoder, RlpEncodable, RlpDecodable, Compressible}; -pub use self::untrusted_rlp::{UntrustedRlp, UntrustedRlpIterator, PayloadInfo, Prototype}; -pub use self::rlpin::{Rlp, RlpIterator}; -pub use self::rlpstream::RlpStream; -pub use self::rlpcompression::RlpType; - +extern crate byteorder; extern crate ethcore_bigint as bigint; extern crate elastic_array; extern crate rustc_serialize; @@ -72,8 +54,29 @@ extern crate rustc_serialize; #[macro_use] extern crate lazy_static; +mod traits; +mod error; +mod rlpin; +mod untrusted_rlp; +mod stream; +mod compression; +mod common; +mod bytes; +mod impls; + +#[cfg(test)] +mod tests; + +use std::borrow::Borrow; use elastic_array::ElasticArray1024; +pub use error::DecoderError; +pub use traits::{Decoder, Decodable, View, Encodable, RlpDecodable, Compressible}; +pub use untrusted_rlp::{UntrustedRlp, UntrustedRlpIterator, PayloadInfo, Prototype}; +pub use rlpin::{Rlp, RlpIterator}; +pub use stream::RlpStream; +pub use compression::RlpType; + /// The RLP encoded empty data (used to mean "null value"). pub const NULL_RLP: [u8; 1] = [0x80; 1]; /// The RLP encoded empty list. @@ -106,8 +109,14 @@ pub fn decode(bytes: &[u8]) -> T where T: RlpDecodable { /// assert_eq!(out, vec![0x83, b'c', b'a', b't']); /// } /// ``` -pub fn encode(object: &E) -> ElasticArray1024 where E: RlpEncodable { +pub fn encode(object: &E) -> ElasticArray1024 where E: Encodable { let mut stream = RlpStream::new(); stream.append(object); stream.drain() } + +pub fn encode_list(object: &[K]) -> ElasticArray1024 where E: Encodable, K: Borrow { + let mut stream = RlpStream::new(); + stream.append_list(object); + stream.drain() +} diff --git a/util/rlp/src/rlpin.rs b/util/rlp/src/rlpin.rs index 81941a89b..7ae715649 100644 --- a/util/rlp/src/rlpin.rs +++ b/util/rlp/src/rlpin.rs @@ -16,7 +16,7 @@ use std::fmt; use rustc_serialize::hex::ToHex; -use ::{View, DecoderError, UntrustedRlp, PayloadInfo, Prototype, RlpDecodable}; +use {View, DecoderError, UntrustedRlp, PayloadInfo, Prototype, RlpDecodable}; impl<'a> From> for Rlp<'a> { fn from(rlp: UntrustedRlp<'a>) -> Rlp<'a> { diff --git a/util/rlp/src/rlpstream.rs b/util/rlp/src/rlpstream.rs deleted file mode 100644 index 190d69e31..000000000 --- a/util/rlp/src/rlpstream.rs +++ /dev/null @@ -1,357 +0,0 @@ -// Copyright 2015-2017 Parity Technologies (UK) Ltd. -// This file is part of Parity. - -// Parity is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// Parity is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. - -// You should have received a copy of the GNU General Public License -// along with Parity. If not, see . - -use elastic_array::*; - -use ::{Stream, Encoder, Encodable}; -use bytes::{ToBytes, VecLike}; -use rlptraits::{ByteEncodable, RlpEncodable}; - -#[derive(Debug, Copy, Clone)] -struct ListInfo { - position: usize, - current: usize, - max: usize, -} - -impl ListInfo { - fn new(position: usize, max: usize) -> ListInfo { - ListInfo { - position: position, - current: 0, - max: max, - } - } -} - -/// Appendable rlp encoder. -pub struct RlpStream { - unfinished_lists: ElasticArray16, - encoder: BasicEncoder, - finished_list: bool, -} - -impl Default for RlpStream { - fn default() -> Self { - RlpStream::new() - } -} - -impl Stream for RlpStream { - fn new() -> Self { - RlpStream { - unfinished_lists: ElasticArray16::new(), - encoder: BasicEncoder::new(), - finished_list: false, - } - } - - fn new_list(len: usize) -> Self { - let mut stream = RlpStream::new(); - stream.begin_list(len); - stream - } - - fn append<'a, E>(&'a mut self, value: &E) -> &'a mut Self where E: RlpEncodable { - self.finished_list = false; - value.rlp_append(self); - if !self.finished_list { - self.note_appended(1); - } - self - } - - fn begin_list(&mut self, len: usize) -> &mut RlpStream { - self.finished_list = false; - match len { - 0 => { - // we may finish, if the appended list len is equal 0 - self.encoder.bytes.push(0xc0u8); - self.note_appended(1); - self.finished_list = true; - }, - _ => { - let position = self.encoder.bytes.len(); - self.unfinished_lists.push(ListInfo::new(position, len)); - }, - } - - // return chainable self - self - } - - fn append_empty_data(&mut self) -> &mut RlpStream { - // self push raw item - self.encoder.bytes.push(0x80); - - // try to finish and prepend the length - self.note_appended(1); - - // return chainable self - self - } - - fn append_raw<'a>(&'a mut self, bytes: &[u8], item_count: usize) -> &'a mut RlpStream { - // push raw items - self.encoder.bytes.append_slice(bytes); - - // try to finish and prepend the length - self.note_appended(item_count); - - // return chainable self - self - } - - fn clear(&mut self) { - // clear bytes - self.encoder.bytes.clear(); - - // clear lists - self.unfinished_lists.clear(); - } - - fn is_finished(&self) -> bool { - self.unfinished_lists.len() == 0 - } - - fn as_raw(&self) -> &[u8] { - &self.encoder.bytes - } - - fn out(self) -> Vec { - match self.is_finished() { - true => self.encoder.out().to_vec(), - false => panic!() - } - } -} - -impl RlpStream { - - /// Appends primitive value to the end of stream - fn append_value(&mut self, object: &E) where E: ByteEncodable { - // encode given value and add it at the end of the stream - self.encoder.emit_value(object); - } - - fn append_internal<'a, E>(&'a mut self, value: &E) -> &'a mut Self where E: Encodable { - self.finished_list = false; - value.rlp_append(self); - if !self.finished_list { - self.note_appended(1); - } - self - } - - /// Try to finish lists - fn note_appended(&mut self, inserted_items: usize) -> () { - if self.unfinished_lists.len() == 0 { - return; - } - - let back = self.unfinished_lists.len() - 1; - let should_finish = match self.unfinished_lists.get_mut(back) { - None => false, - Some(ref mut x) => { - x.current += inserted_items; - if x.current > x.max { - panic!("You cannot append more items then you expect!"); - } - x.current == x.max - } - }; - - if should_finish { - let x = self.unfinished_lists.pop().unwrap(); - let len = self.encoder.bytes.len() - x.position; - self.encoder.insert_list_len_at_pos(len, x.position); - self.note_appended(1); - } - self.finished_list = should_finish; - } - - /// Drain the object and return the underlying ElasticArray. - pub fn drain(self) -> ElasticArray1024 { - match self.is_finished() { - true => self.encoder.bytes, - false => panic!() - } - } -} - -struct BasicEncoder { - bytes: ElasticArray1024, -} - -impl Default for BasicEncoder { - fn default() -> Self { - BasicEncoder::new() - } -} - -impl BasicEncoder { - fn new() -> Self { - BasicEncoder { bytes: ElasticArray1024::new() } - } - - /// inserts list prefix at given position - /// TODO: optimise it further? - fn insert_list_len_at_pos(&mut self, len: usize, pos: usize) -> () { - let mut res = ElasticArray16::new(); - match len { - 0...55 => res.push(0xc0u8 + len as u8), - _ => { - res.push(0xf7u8 + len.to_bytes_len() as u8); - ToBytes::to_bytes(&len, &mut res); - } - }; - - self.bytes.insert_slice(pos, &res); - } - - /// get encoded value - fn out(self) -> ElasticArray1024 { - self.bytes - } -} - -impl Encoder for BasicEncoder { - fn emit_value(&mut self, value: &E) { - match value.bytes_len() { - // just 0 - 0 => self.bytes.push(0x80u8), - // byte is its own encoding if < 0x80 - 1 => { - value.to_bytes(&mut self.bytes); - let len = self.bytes.len(); - let last_byte = self.bytes[len - 1]; - if last_byte >= 0x80 { - self.bytes.push(last_byte); - self.bytes[len - 1] = 0x81; - } - } - // (prefix + length), followed by the string - len @ 2 ... 55 => { - self.bytes.push(0x80u8 + len as u8); - value.to_bytes(&mut self.bytes); - } - // (prefix + length of length), followed by the length, followd by the string - len => { - self.bytes.push(0xb7 + len.to_bytes_len() as u8); - ToBytes::to_bytes(&len, &mut self.bytes); - value.to_bytes(&mut self.bytes); - } - } - } - - fn emit_raw(&mut self, bytes: &[u8]) -> () { - self.bytes.append_slice(bytes); - } -} - -impl ByteEncodable for T where T: ToBytes { - fn to_bytes>(&self, out: &mut V) { - ToBytes::to_bytes(self, out) - } - - fn bytes_len(&self) -> usize { - ToBytes::to_bytes_len(self) - } -} - -struct U8Slice<'a>(&'a [u8]); - -impl<'a> ByteEncodable for U8Slice<'a> { - fn to_bytes>(&self, out: &mut V) { - out.vec_extend(self.0) - } - - fn bytes_len(&self) -> usize { - self.0.len() - } -} - -impl<'a> Encodable for &'a[u8] { - fn rlp_append(&self, s: &mut RlpStream) { - s.append_value(&U8Slice(self)) - } -} - -impl Encodable for Vec { - fn rlp_append(&self, s: &mut RlpStream) { - s.append_value(&U8Slice(self)) - } -} - -impl Encodable for T where T: ByteEncodable { - fn rlp_append(&self, s: &mut RlpStream) { - s.append_value(self) - } -} - -struct EncodableU8 (u8); - -impl ByteEncodable for EncodableU8 { - fn to_bytes>(&self, out: &mut V) { - if self.0 != 0 { - out.vec_push(self.0) - } - } - - fn bytes_len(&self) -> usize { - match self.0 { 0 => 0, _ => 1 } - } -} - -impl RlpEncodable for u8 { - fn rlp_append(&self, s: &mut RlpStream) { - s.append_value(&EncodableU8(*self)) - } -} - -impl<'a, T> Encodable for &'a[T] where T: Encodable { - fn rlp_append(&self, s: &mut RlpStream) { - s.begin_list(self.len()); - for el in self.iter() { - s.append_internal(el); - } - } -} - -impl Encodable for Vec where T: Encodable { - fn rlp_append(&self, s: &mut RlpStream) { - Encodable::rlp_append(&self.as_slice(), s); - } -} - -impl Encodable for Option where T: Encodable { - fn rlp_append(&self, s: &mut RlpStream) { - match *self { - None => { s.begin_list(0); }, - Some(ref x) => { - s.begin_list(1); - s.append_internal(x); - } - } - } -} - -impl RlpEncodable for T where T: Encodable { - fn rlp_append(&self, s: &mut RlpStream) { - Encodable::rlp_append(self, s) - } -} - diff --git a/util/rlp/src/stream.rs b/util/rlp/src/stream.rs new file mode 100644 index 000000000..318e019fc --- /dev/null +++ b/util/rlp/src/stream.rs @@ -0,0 +1,335 @@ +// Copyright 2015-2017 Parity Technologies (UK) Ltd. +// This file is part of Parity. + +// Parity is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Parity is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Parity. If not, see . + +use std::borrow::Borrow; +use byteorder::{ByteOrder, BigEndian}; +use elastic_array::{ElasticArray16, ElasticArray1024}; +use traits::Encodable; + +#[derive(Debug, Copy, Clone)] +struct ListInfo { + position: usize, + current: usize, + max: usize, +} + +impl ListInfo { + fn new(position: usize, max: usize) -> ListInfo { + ListInfo { + position: position, + current: 0, + max: max, + } + } +} + +/// Appendable rlp encoder. +pub struct RlpStream { + unfinished_lists: ElasticArray16, + buffer: ElasticArray1024, + finished_list: bool, +} + +impl Default for RlpStream { + fn default() -> Self { + RlpStream::new() + } +} + +impl RlpStream { + /// Initializes instance of empty `Stream`. + pub fn new() -> Self { + RlpStream { + unfinished_lists: ElasticArray16::new(), + buffer: ElasticArray1024::new(), + finished_list: false, + } + } + + /// Initializes the `Stream` as a list. + pub fn new_list(len: usize) -> Self { + let mut stream = RlpStream::new(); + stream.begin_list(len); + stream + } + + /// Appends value to the end of stream, chainable. + /// + /// ```rust + /// extern crate rlp; + /// use rlp::*; + /// + /// fn main () { + /// let mut stream = RlpStream::new_list(2); + /// stream.append(&"cat").append(&"dog"); + /// let out = stream.out(); + /// assert_eq!(out, vec![0xc8, 0x83, b'c', b'a', b't', 0x83, b'd', b'o', b'g']); + /// } + /// ``` + pub fn append<'a, E>(&'a mut self, value: &E) -> &'a mut Self where E: Encodable { + self.finished_list = false; + value.rlp_append(self); + if !self.finished_list { + self.note_appended(1); + } + self + } + + /// Appends list of values to the end of stream, chainable. + pub fn append_list<'a, E, K>(&'a mut self, values: &[K]) -> &'a mut Self where E: Encodable, K: Borrow { + self.begin_list(values.len()); + for value in values { + self.append(value.borrow()); + } + self + } + + /// Appends value to the end of stream, but do not count it as an appended item. + /// It's useful for wrapper types + pub fn append_internal<'a, E>(&'a mut self, value: &E) -> &'a mut Self where E: Encodable { + value.rlp_append(self); + self + } + + /// Declare appending the list of given size, chainable. + /// + /// ```rust + /// extern crate rlp; + /// use rlp::*; + /// + /// fn main () { + /// let mut stream = RlpStream::new_list(2); + /// stream.begin_list(2).append(&"cat").append(&"dog"); + /// stream.append(&""); + /// let out = stream.out(); + /// assert_eq!(out, vec![0xca, 0xc8, 0x83, b'c', b'a', b't', 0x83, b'd', b'o', b'g', 0x80]); + /// } + /// ``` + pub fn begin_list(&mut self, len: usize) -> &mut RlpStream { + self.finished_list = false; + match len { + 0 => { + // we may finish, if the appended list len is equal 0 + self.buffer.push(0xc0u8); + self.note_appended(1); + self.finished_list = true; + }, + _ => { + // payload is longer than 1 byte only for lists > 55 bytes + // by pushing always this 1 byte we may avoid unnecessary shift of data + self.buffer.push(0); + + let position = self.buffer.len(); + self.unfinished_lists.push(ListInfo::new(position, len)); + }, + } + + // return chainable self + self + } + + /// Apends null to the end of stream, chainable. + /// + /// ```rust + /// extern crate rlp; + /// use rlp::*; + /// + /// fn main () { + /// let mut stream = RlpStream::new_list(2); + /// stream.append_empty_data().append_empty_data(); + /// let out = stream.out(); + /// assert_eq!(out, vec![0xc2, 0x80, 0x80]); + /// } + /// ``` + pub fn append_empty_data(&mut self) -> &mut RlpStream { + // self push raw item + self.buffer.push(0x80); + + // try to finish and prepend the length + self.note_appended(1); + + // return chainable self + self + } + + /// Appends raw (pre-serialised) RLP data. Use with caution. Chainable. + pub fn append_raw<'a>(&'a mut self, bytes: &[u8], item_count: usize) -> &'a mut RlpStream { + // push raw items + self.buffer.append_slice(bytes); + + // try to finish and prepend the length + self.note_appended(item_count); + + // return chainable self + self + } + + /// Clear the output stream so far. + /// + /// ```rust + /// extern crate rlp; + /// use rlp::*; + /// + /// fn main () { + /// let mut stream = RlpStream::new_list(3); + /// stream.append(&"cat"); + /// stream.clear(); + /// stream.append(&"dog"); + /// let out = stream.out(); + /// assert_eq!(out, vec![0x83, b'd', b'o', b'g']); + /// } + pub fn clear(&mut self) { + // clear bytes + self.buffer.clear(); + + // clear lists + self.unfinished_lists.clear(); + } + + /// Returns true if stream doesnt expect any more items. + /// + /// ```rust + /// extern crate rlp; + /// use rlp::*; + /// + /// fn main () { + /// let mut stream = RlpStream::new_list(2); + /// stream.append(&"cat"); + /// assert_eq!(stream.is_finished(), false); + /// stream.append(&"dog"); + /// assert_eq!(stream.is_finished(), true); + /// let out = stream.out(); + /// assert_eq!(out, vec![0xc8, 0x83, b'c', b'a', b't', 0x83, b'd', b'o', b'g']); + /// } + pub fn is_finished(&self) -> bool { + self.unfinished_lists.len() == 0 + } + + /// Get raw encoded bytes + pub fn as_raw(&self) -> &[u8] { + //&self.encoder.bytes + &self.buffer + } + + /// Streams out encoded bytes. + /// + /// panic! if stream is not finished. + pub fn out(self) -> Vec { + match self.is_finished() { + //true => self.encoder.out().to_vec(), + true => self.buffer.to_vec(), + false => panic!() + } + } + + /// Try to finish lists + fn note_appended(&mut self, inserted_items: usize) -> () { + if self.unfinished_lists.len() == 0 { + return; + } + + let back = self.unfinished_lists.len() - 1; + let should_finish = match self.unfinished_lists.get_mut(back) { + None => false, + Some(ref mut x) => { + x.current += inserted_items; + if x.current > x.max { + panic!("You cannot append more items then you expect!"); + } + x.current == x.max + } + }; + + if should_finish { + let x = self.unfinished_lists.pop().unwrap(); + let len = self.buffer.len() - x.position; + self.encoder().insert_list_payload(len, x.position); + self.note_appended(1); + } + self.finished_list = should_finish; + } + + pub fn encoder(&mut self) -> BasicEncoder { + BasicEncoder::new(self) + } + + /// Drain the object and return the underlying ElasticArray. + pub fn drain(self) -> ElasticArray1024 { + match self.is_finished() { + true => self.buffer, + false => panic!() + } + } +} + +pub struct BasicEncoder<'a> { + buffer: &'a mut ElasticArray1024, +} + +impl<'a> BasicEncoder<'a> { + fn new(stream: &'a mut RlpStream) -> Self { + BasicEncoder { + buffer: &mut stream.buffer + } + } + + fn insert_size(&mut self, size: usize, position: usize) -> u8 { + let size = size as u32; + let leading_empty_bytes = size.leading_zeros() as usize / 8; + let size_bytes = 4 - leading_empty_bytes as u8; + let mut buffer = [0u8; 4]; + BigEndian::write_u32(&mut buffer, size); + self.buffer.insert_slice(position, &buffer[leading_empty_bytes..]); + size_bytes as u8 + } + + /// Inserts list prefix at given position + fn insert_list_payload(&mut self, len: usize, pos: usize) { + // 1 byte was already reserved for payload earlier + match len { + 0...55 => { + self.buffer[pos - 1] = 0xc0u8 + len as u8; + }, + _ => { + let inserted_bytes = self.insert_size(len, pos); + self.buffer[pos - 1] = 0xf7u8 + inserted_bytes; + } + }; + } + + /// Pushes encoded value to the end of buffer + pub fn encode_value(&mut self, value: &[u8]) { + match value.len() { + // just 0 + 0 => self.buffer.push(0x80u8), + // byte is its own encoding if < 0x80 + 1 if value[0] < 0x80 => self.buffer.push(value[0]), + // (prefix + length), followed by the string + len @ 1 ... 55 => { + self.buffer.push(0x80u8 + len as u8); + self.buffer.append_slice(value); + } + // (prefix + length of length), followed by the length, followd by the string + len => { + self.buffer.push(0); + let position = self.buffer.len(); + let inserted_bytes = self.insert_size(len, position); + self.buffer[position - 1] = 0xb7 + inserted_bytes; + self.buffer.append_slice(value); + } + } + } +} diff --git a/util/rlp/src/tests.rs b/util/rlp/src/tests.rs index 151619571..bd3abfe63 100644 --- a/util/rlp/src/tests.rs +++ b/util/rlp/src/tests.rs @@ -15,9 +15,8 @@ // along with Parity. If not, see . use std::{fmt, cmp}; -use std::str::FromStr; -use ::{Encodable, RlpDecodable, UntrustedRlp, RlpStream, View, Stream, DecoderError}; use bigint::prelude::U256; +use {Encodable, RlpDecodable, UntrustedRlp, RlpStream, View, DecoderError}; #[test] fn rlp_at() { @@ -95,6 +94,17 @@ fn run_encode_tests(tests: Vec>) } } +struct VETestPair(Vec, Vec) where T: Encodable; + +fn run_encode_tests_list(tests: Vec>) + where T: Encodable +{ + for t in &tests { + let res = super::encode_list(&t.0); + assert_eq!(&res[..], &t.1[..]); + } +} + #[test] fn encode_u16() { let tests = vec![ @@ -131,9 +141,8 @@ fn encode_u256() { ETestPair(U256::from(0x1000000u64), vec![0x84, 0x01, 0x00, 0x00, 0x00]), ETestPair(U256::from(0xffffffffu64), vec![0x84, 0xff, 0xff, 0xff, 0xff]), - ETestPair(U256::from_str("8090a0b0c0d0e0f00910203040506077000000000000\ - 000100000000000012f0") - .unwrap(), + ETestPair(("8090a0b0c0d0e0f00910203040506077000000000000\ + 000100000000000012f0").into(), vec![0xa0, 0x80, 0x90, 0xa0, 0xb0, 0xc0, 0xd0, 0xe0, 0xf0, 0x09, 0x10, 0x20, 0x30, 0x40, 0x50, 0x60, 0x77, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, @@ -186,19 +195,19 @@ fn encode_vector_u8() { #[test] fn encode_vector_u64() { let tests = vec![ - ETestPair(vec![], vec![0xc0]), - ETestPair(vec![15u64], vec![0xc1, 0x0f]), - ETestPair(vec![1, 2, 3, 7, 0xff], vec![0xc6, 1, 2, 3, 7, 0x81, 0xff]), - ETestPair(vec![0xffffffff, 1, 2, 3, 7, 0xff], vec![0xcb, 0x84, 0xff, 0xff, 0xff, 0xff, 1, 2, 3, 7, 0x81, 0xff]), + VETestPair(vec![], vec![0xc0]), + VETestPair(vec![15u64], vec![0xc1, 0x0f]), + VETestPair(vec![1, 2, 3, 7, 0xff], vec![0xc6, 1, 2, 3, 7, 0x81, 0xff]), + VETestPair(vec![0xffffffff, 1, 2, 3, 7, 0xff], vec![0xcb, 0x84, 0xff, 0xff, 0xff, 0xff, 1, 2, 3, 7, 0x81, 0xff]), ]; - run_encode_tests(tests); + run_encode_tests_list(tests); } #[test] fn encode_vector_str() { - let tests = vec![ETestPair(vec!["cat", "dog"], + let tests = vec![VETestPair(vec!["cat", "dog"], vec![0xc8, 0x83, b'c', b'a', b't', 0x83, b'd', b'o', b'g'])]; - run_encode_tests(tests); + run_encode_tests_list(tests); } struct DTestPair(T, Vec) where T: RlpDecodable + fmt::Debug + cmp::Eq; @@ -265,9 +274,8 @@ fn decode_untrusted_u256() { DTestPair(U256::from(0x1000000u64), vec![0x84, 0x01, 0x00, 0x00, 0x00]), DTestPair(U256::from(0xffffffffu64), vec![0x84, 0xff, 0xff, 0xff, 0xff]), - DTestPair(U256::from_str("8090a0b0c0d0e0f00910203040506077000000000000\ - 000100000000000012f0") - .unwrap(), + DTestPair(("8090a0b0c0d0e0f00910203040506077000000000000\ + 000100000000000012f0").into(), vec![0xa0, 0x80, 0x90, 0xa0, 0xb0, 0xc0, 0xd0, 0xe0, 0xf0, 0x09, 0x10, 0x20, 0x30, 0x40, 0x50, 0x60, 0x77, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, @@ -335,7 +343,7 @@ fn decode_untrusted_vector_of_vectors_str() { #[test] fn test_decoding_array() { let v = vec![5u16, 2u16]; - let res = super::encode(&v); + let res = super::encode_list(&v); let arr: [u16; 2] = super::decode(&res); assert_eq!(arr[0], 5); assert_eq!(arr[1], 2); @@ -396,7 +404,7 @@ fn test_rlp_2bytes_data_length_check() #[test] fn test_rlp_nested_empty_list_encode() { let mut stream = RlpStream::new_list(2); - stream.append(&(Vec::new() as Vec)); + stream.append_list(&(Vec::new() as Vec)); stream.append(&40u32); assert_eq!(stream.drain()[..], [0xc2u8, 0xc0u8, 40u8][..]); } diff --git a/util/rlp/src/rlptraits.rs b/util/rlp/src/traits.rs similarity index 64% rename from util/rlp/src/rlptraits.rs rename to util/rlp/src/traits.rs index ca7da2ec7..3f79e9cab 100644 --- a/util/rlp/src/rlptraits.rs +++ b/util/rlp/src/traits.rs @@ -15,11 +15,9 @@ // along with Parity. If not, see . //! Common RLP traits -use ::{DecoderError, UntrustedRlp}; -use bytes::VecLike; -use rlpstream::RlpStream; - use elastic_array::ElasticArray1024; +use stream::RlpStream; +use {DecoderError, UntrustedRlp}; /// Type is able to decode RLP. pub trait Decoder: Sized { @@ -226,23 +224,7 @@ pub trait View<'a, 'view>: Sized { fn val_at(&self, index: usize) -> Result where T: RlpDecodable; } -/// Raw RLP encoder -pub trait Encoder { - /// Write a value represented as bytes - fn emit_value(&mut self, value: &E); - /// Write raw preencoded data to the output - fn emit_raw(&mut self, bytes: &[u8]) -> (); -} - -/// Primitive data type encodable to RLP -pub trait ByteEncodable { - /// Serialize this object to given byte container - fn to_bytes>(&self, out: &mut V); - /// Get size of serialised data in bytes - fn bytes_len(&self) -> usize; -} - -/// Structure encodable to RLP. Implement this trait for +/// Structure encodable to RLP pub trait Encodable { /// Append a value to the stream fn rlp_append(&self, s: &mut RlpStream); @@ -255,112 +237,6 @@ pub trait Encodable { } } -/// Encodable wrapper trait required to handle special case of encoding a &[u8] as string and not as list -pub trait RlpEncodable { - /// Append a value to the stream - fn rlp_append(&self, s: &mut RlpStream); -} - -/// RLP encoding stream -pub trait Stream: Sized { - - /// Initializes instance of empty `Stream`. - fn new() -> Self; - - /// Initializes the `Stream` as a list. - fn new_list(len: usize) -> Self; - - /// Apends value to the end of stream, chainable. - /// - /// ```rust - /// extern crate rlp; - /// use rlp::*; - /// - /// fn main () { - /// let mut stream = RlpStream::new_list(2); - /// stream.append(&"cat").append(&"dog"); - /// let out = stream.out(); - /// assert_eq!(out, vec![0xc8, 0x83, b'c', b'a', b't', 0x83, b'd', b'o', b'g']); - /// } - /// ``` - fn append<'a, E>(&'a mut self, value: &E) -> &'a mut Self where E: RlpEncodable; - - /// Declare appending the list of given size, chainable. - /// - /// ```rust - /// extern crate rlp; - /// use rlp::*; - /// - /// fn main () { - /// let mut stream = RlpStream::new_list(2); - /// stream.begin_list(2).append(&"cat").append(&"dog"); - /// stream.append(&""); - /// let out = stream.out(); - /// assert_eq!(out, vec![0xca, 0xc8, 0x83, b'c', b'a', b't', 0x83, b'd', b'o', b'g', 0x80]); - /// } - /// ``` - fn begin_list(&mut self, len: usize) -> &mut Self; - - /// Apends null to the end of stream, chainable. - /// - /// ```rust - /// extern crate rlp; - /// use rlp::*; - /// - /// fn main () { - /// let mut stream = RlpStream::new_list(2); - /// stream.append_empty_data().append_empty_data(); - /// let out = stream.out(); - /// assert_eq!(out, vec![0xc2, 0x80, 0x80]); - /// } - /// ``` - fn append_empty_data(&mut self) -> &mut Self; - - /// Appends raw (pre-serialised) RLP data. Use with caution. Chainable. - fn append_raw<'a>(&'a mut self, bytes: &[u8], item_count: usize) -> &'a mut Self; - - /// Clear the output stream so far. - /// - /// ```rust - /// extern crate rlp; - /// use rlp::*; - /// - /// fn main () { - /// let mut stream = RlpStream::new_list(3); - /// stream.append(&"cat"); - /// stream.clear(); - /// stream.append(&"dog"); - /// let out = stream.out(); - /// assert_eq!(out, vec![0x83, b'd', b'o', b'g']); - /// } - fn clear(&mut self); - - /// Returns true if stream doesnt expect any more items. - /// - /// ```rust - /// extern crate rlp; - /// use rlp::*; - /// - /// fn main () { - /// let mut stream = RlpStream::new_list(2); - /// stream.append(&"cat"); - /// assert_eq!(stream.is_finished(), false); - /// stream.append(&"dog"); - /// assert_eq!(stream.is_finished(), true); - /// let out = stream.out(); - /// assert_eq!(out, vec![0xc8, 0x83, b'c', b'a', b't', 0x83, b'd', b'o', b'g']); - /// } - fn is_finished(&self) -> bool; - - /// Get raw encoded bytes - fn as_raw(&self) -> &[u8]; - - /// Streams out encoded bytes. - /// - /// panic! if stream is not finished. - fn out(self) -> Vec; -} - /// Trait for compressing and decompressing RLP by replacement of common terms. pub trait Compressible: Sized { /// Indicates the origin of RLP to be compressed. diff --git a/util/src/journaldb/earlymergedb.rs b/util/src/journaldb/earlymergedb.rs index 9644a60ac..f484a623a 100644 --- a/util/src/journaldb/earlymergedb.rs +++ b/util/src/journaldb/earlymergedb.rs @@ -427,7 +427,7 @@ impl JournalDB for EarlyMergeDB { r.begin_list(inserts.len()); inserts.iter().foreach(|&(k, _)| {r.append(&k);}); - r.append(&removes); + r.append_list(&removes); Self::insert_keys(&inserts, &*self.backing, self.column, &mut refs, batch, trace); let ins = inserts.iter().map(|&(k, _)| k).collect::>(); diff --git a/util/src/journaldb/overlayrecentdb.rs b/util/src/journaldb/overlayrecentdb.rs index ed39905a2..90f7acfcf 100644 --- a/util/src/journaldb/overlayrecentdb.rs +++ b/util/src/journaldb/overlayrecentdb.rs @@ -278,7 +278,7 @@ impl JournalDB for OverlayRecentDB { journal_overlay.backing_overlay.emplace(short_key, v); } - r.append(&removed_keys); + r.append_list(&removed_keys); let mut k = RlpStream::new_list(3); let index = journal_overlay.journal.get(&now).map_or(0, |j| j.len()); diff --git a/util/src/journaldb/refcounteddb.rs b/util/src/journaldb/refcounteddb.rs index 57d7aa7c0..b18954eb3 100644 --- a/util/src/journaldb/refcounteddb.rs +++ b/util/src/journaldb/refcounteddb.rs @@ -136,8 +136,8 @@ impl JournalDB for RefCountedDB { let mut r = RlpStream::new_list(3); r.append(id); - r.append(&self.inserts); - r.append(&self.removes); + r.append_list(&self.inserts); + r.append_list(&self.removes); batch.put(self.column, &last, r.as_raw()); let ops = self.inserts.len() + self.removes.len(); diff --git a/util/src/misc.rs b/util/src/misc.rs index 489628a30..6dc2fba12 100644 --- a/util/src/misc.rs +++ b/util/src/misc.rs @@ -17,7 +17,7 @@ //! Diff misc. use common::*; -use rlp::{Stream, RlpStream}; +use rlp::RlpStream; use target_info::Target; include!(concat!(env!("OUT_DIR"), "/version.rs")); diff --git a/util/src/trie/triedbmut.rs b/util/src/trie/triedbmut.rs index 0df752756..10d7f3aa6 100644 --- a/util/src/trie/triedbmut.rs +++ b/util/src/trie/triedbmut.rs @@ -24,7 +24,7 @@ use super::node::NodeKey; use ::{HashDB, H256}; use ::bytes::ToPretty; use ::nibbleslice::NibbleSlice; -use ::rlp::{Rlp, RlpStream, View, Stream}; +use ::rlp::{Rlp, RlpStream, View}; use ::sha3::SHA3_NULL_RLP; use hashdb::DBValue; @@ -931,7 +931,6 @@ impl<'a> Drop for TrieDBMut<'a> { #[cfg(test)] mod tests { use triehash::trie_root; - use hash::*; use hashdb::*; use memorydb::*; use super::*; diff --git a/util/src/triehash.rs b/util/src/triehash.rs index 9884794ca..772f3f475 100644 --- a/util/src/triehash.rs +++ b/util/src/triehash.rs @@ -23,7 +23,7 @@ use std::cmp; use hash::*; use sha3::*; use rlp; -use rlp::{RlpStream, Stream}; +use rlp::RlpStream; use vector::SharedPrefix; /// Generates a trie root hash for a vector of values