rlp serialization refactor (#4873)

* fixed naming of rlp modules

* RlpStream cleanup

* appending short rlp lists (0...55 bytes) is 25% faster

* RlpStream does not use bytes module, nor trait Stream

* removed unused code from rlp module

* compiling ethcore-util with new rlp serialization

* compiling parity with new rlp serialization

* fixed compiling ethcore-light with new rlp serialization

* fixed compiling ethsync with new rlp serialization

* removed redundant comment, print

* removed redundant double-space

* replace usage of WriteBytesExt with ByteOrder

ethcore/src/block.rs

@@ -20,7 +20,7 @@ use std::cmp;
 use std::sync::Arc;
 use std::collections::HashSet;
 
-use rlp::{UntrustedRlp, RlpStream, Encodable, Decodable, Decoder, DecoderError, View, Stream};
+use rlp::{UntrustedRlp, RlpStream, Encodable, Decodable, Decoder, DecoderError, View};
 use util::{Bytes, Address, Uint, Hashable, U256, H256, ordered_trie_root, SHA3_NULL_RLP};
 use util::error::{Mismatch, OutOfBounds};
 
@@ -59,8 +59,8 @@ impl Block {
 	pub fn rlp_bytes(&self, seal: Seal) -> Bytes {
 		let mut block_rlp = RlpStream::new_list(3);
 		self.header.stream_rlp(&mut block_rlp, seal);
-		block_rlp.append(&self.transactions);
-		block_rlp.append(&self.uncles);
+		block_rlp.append_list(&self.transactions);
+		block_rlp.append_list(&self.uncles);
 		block_rlp.out()
 	}
 }
@@ -507,7 +507,7 @@ impl SealedBlock {
 	pub fn rlp_bytes(&self) -> Bytes {
 		let mut block_rlp = RlpStream::new_list(3);
 		self.block.header.stream_rlp(&mut block_rlp, Seal::With);
-		block_rlp.append(&self.block.transactions);
+		block_rlp.append_list(&self.block.transactions);
 		block_rlp.append_raw(&self.uncle_bytes, 1);
 		block_rlp.out()
 	}

ethcore/src/blockchain/extras.rs

@@ -172,7 +172,7 @@ impl Encodable for BlockDetails {
 		s.append(&self.number);
 		s.append(&self.total_difficulty);
 		s.append(&self.parent);
-		s.append(&self.children);
+		s.append_list(&self.children);
 	}
 }
 
@@ -233,7 +233,7 @@ impl Decodable for BlockReceipts {
 
 impl Encodable for BlockReceipts {
 	fn rlp_append(&self, s: &mut RlpStream) {
-		Encodable::rlp_append(&self.receipts, s);
+		s.append_list(&self.receipts);
 	}
 }
 

ethcore/src/blockchain/generator/block.rs

@@ -37,8 +37,8 @@ impl Encodable for Block {
 	fn rlp_append(&self, s: &mut RlpStream) {
 		s.begin_list(3);
 		s.append(&self.header);
-		s.append(&self.transactions);
-		s.append(&self.uncles);
+		s.append_list(&self.transactions);
+		s.append_list(&self.uncles);
 	}
 }
 

ethcore/src/blooms/bloom_group.rs

@@ -63,7 +63,7 @@ impl Decodable for BloomGroup {
 
 impl Encodable for BloomGroup {
 	fn rlp_append(&self, s: &mut RlpStream) {
-		Encodable::rlp_append(&self.blooms, s)
+		s.append_list(&self.blooms);
 	}
 }
 

ethcore/src/engines/tendermint/message.rs

@@ -20,7 +20,7 @@ use util::*;
 use super::{Height, View, BlockHash, Step};
 use error::Error;
 use header::Header;
-use rlp::{Rlp, UntrustedRlp, RlpStream, Stream, RlpEncodable, Encodable, Decodable, Decoder, DecoderError, View as RlpView};
+use rlp::{Rlp, UntrustedRlp, RlpStream, Encodable, Decodable, Decoder, DecoderError, View as RlpView};
 use ethkey::{recover, public_to_address};
 use super::super::vote_collector::Message;
 
@@ -162,7 +162,7 @@ impl Decodable for Step {
 
 impl Encodable for Step {
 	fn rlp_append(&self, s: &mut RlpStream) {
-		RlpEncodable::rlp_append(&self.number(), s);
+		s.append_internal(&self.number());
 	}
 }
 
@@ -278,6 +278,7 @@ mod tests {
 			::rlp::encode(&H520::default()).to_vec(),
 			Vec::new()
 		];
+
 		header.set_seal(seal);
 		let message = ConsensusMessage::new_proposal(&header).unwrap();
 		assert_eq!(

ethcore/src/engines/tendermint/mod.rs

@@ -243,7 +243,7 @@ impl Tendermint {
 							let seal = vec![
 								::rlp::encode(&view).to_vec(),
 								::rlp::encode(&seal.proposal).to_vec(),
-								::rlp::encode(&seal.votes).to_vec()
+								::rlp::encode_list(&seal.votes).to_vec()
 							];
 							self.submit_seal(block_hash, seal);
 							self.to_next_height(height);
@@ -825,7 +825,7 @@ mod tests {
 		let vote_info = message_info_rlp(&VoteStep::new(2, 0, Step::Precommit), Some(header.bare_hash()));
 		let signature1 = tap.sign(proposer, None, vote_info.sha3()).unwrap();
 
-		seal[2] = ::rlp::encode(&vec![H520::from(signature1.clone())]).to_vec();
+		seal[2] = ::rlp::encode_list(&vec![H520::from(signature1.clone())]).to_vec();
 		header.set_seal(seal.clone());
 
 		// One good signature is not enough.
@@ -837,7 +837,7 @@ mod tests {
 		let voter = insert_and_unlock(&tap, "0");
 		let signature0 = tap.sign(voter, None, vote_info.sha3()).unwrap();
 
-		seal[2] = ::rlp::encode(&vec![H520::from(signature1.clone()), H520::from(signature0.clone())]).to_vec();
+		seal[2] = ::rlp::encode_list(&vec![H520::from(signature1.clone()), H520::from(signature0.clone())]).to_vec();
 		header.set_seal(seal.clone());
 
 		assert!(engine.verify_block_family(&header, &parent_header, None).is_ok());
@@ -845,7 +845,7 @@ mod tests {
 		let bad_voter = insert_and_unlock(&tap, "101");
 		let bad_signature = tap.sign(bad_voter, None, vote_info.sha3()).unwrap();
 
-		seal[2] = ::rlp::encode(&vec![H520::from(signature1), H520::from(bad_signature)]).to_vec();
+		seal[2] = ::rlp::encode_list(&vec![H520::from(signature1), H520::from(bad_signature)]).to_vec();
 		header.set_seal(seal);
 
 		// One good and one bad signature.

ethcore/src/engines/vote_collector.rs

@@ -245,7 +245,7 @@ mod tests {
 
 	#[test]
 	fn seal_retrieval() {
-		let collector = VoteCollector::default();	
+		let collector = VoteCollector::default();
 		let bh = Some("1".sha3());
 		let mut signatures = Vec::new();
 		for _ in 0..5 {
@@ -284,7 +284,7 @@ mod tests {
 
 	#[test]
 	fn count_votes() {
-		let collector = VoteCollector::default();	
+		let collector = VoteCollector::default();
 		let round1 = 1;
 		let round3 = 3;
 		// good 1
@@ -318,7 +318,7 @@ mod tests {
 
 	#[test]
 	fn remove_old() {
-		let collector = VoteCollector::default();	
+		let collector = VoteCollector::default();
 		let vote = |round, hash| {
 			random_vote(&collector, H520::random(), round, hash);
 		};
@@ -334,7 +334,7 @@ mod tests {
 
 	#[test]
 	fn malicious_authority() {
-		let collector = VoteCollector::default();	
+		let collector = VoteCollector::default();
 		let round = 3;
 		// Vote is inserted fine.
 		assert!(full_vote(&collector, H520::random(), round, Some("0".sha3()), &Address::default()).is_none());

ethcore/src/executive.rs

@@ -36,7 +36,7 @@ const STACK_SIZE_PER_DEPTH: usize = 24*1024;
 
 /// Returns new address created from address and given nonce.
 pub fn contract_address(address: &Address, nonce: &U256) -> Address {
-	use rlp::{RlpStream, Stream};
+	use rlp::RlpStream;
 
 	let mut stream = RlpStream::new_list(2);
 	stream.append(address);

ethcore/src/migrations/state/v7.rs

@@ -26,7 +26,7 @@ use util::migration::{Batch, Config, Error, Migration, SimpleMigration, Progress
 use util::sha3::Hashable;
 use std::sync::Arc;
 
-use rlp::{decode, Rlp, RlpStream, Stream, View};
+use rlp::{decode, Rlp, RlpStream, View};
 
 
 // attempt to migrate a key, value pair. None if migration not possible.
@@ -199,7 +199,7 @@ impl OverlayRecentV7 {
 							stream.begin_list(2).append(&k).append(&v);
 						}
 
-						stream.append(&deleted_keys);
+						stream.append_list(&deleted_keys);
 
 						// and insert it into the new database.
 						batch.insert(entry_key, stream.out(), dest)?;

ethcore/src/migrations/v9.rs

@@ -17,7 +17,7 @@
 
 //! This migration consolidates all databases into single one using Column Families.
 
-use rlp::{Rlp, RlpStream, View, Stream};
+use rlp::{Rlp, RlpStream, View};
 use util::kvdb::Database;
 use util::migration::{Batch, Config, Error, Migration, Progress};
 use std::sync::Arc;

ethcore/src/pod_account.rs

@@ -19,7 +19,7 @@ use state::Account;
 use account_db::AccountDBMut;
 use ethjson;
 use types::account_diff::*;
-use rlp::{self, RlpStream, Stream};
+use rlp::{self, RlpStream};
 
 #[derive(Debug, Clone, PartialEq, Eq)]
 /// An account, expressed as Plain-Old-Data (hence the name).

ethcore/src/snapshot/account.rs

@@ -22,7 +22,7 @@ use snapshot::Error;
 
 use util::{U256, H256, Bytes, HashDB, SHA3_EMPTY, SHA3_NULL_RLP};
 use util::trie::{TrieDB, Trie};
-use rlp::{RlpStream, Stream, UntrustedRlp, View};
+use rlp::{RlpStream, UntrustedRlp, View};
 
 use std::collections::HashSet;
 

ethcore/src/snapshot/block.rs

@@ -20,7 +20,7 @@ use block::Block;
 use header::Header;
 
 use views::BlockView;
-use rlp::{DecoderError, RlpStream, Stream, UntrustedRlp, View};
+use rlp::{DecoderError, RlpStream, UntrustedRlp, View};
 use util::{Bytes, Hashable, H256};
 use util::triehash::ordered_trie_root;
 
@@ -69,7 +69,9 @@ impl AbridgedBlock {
 			.append(&header.extra_data());
 
 		// write block values.
-		stream.append(&block_view.transactions()).append(&block_view.uncles());
+		stream
+			.append_list(&block_view.transactions())
+			.append_list(&block_view.uncles());
 
 		// write seal fields.
 		for field in seal_fields {
@@ -108,7 +110,7 @@ impl AbridgedBlock {
 		header.set_receipts_root(receipts_root);
 
 		let mut uncles_rlp = RlpStream::new();
-		uncles_rlp.append(&uncles);
+		uncles_rlp.append_list(&uncles);
 		header.set_uncles_hash(uncles_rlp.as_raw().sha3());
 
 		let mut seal_fields = Vec::new();

ethcore/src/snapshot/io.rs

@@ -27,7 +27,7 @@ use std::path::{Path, PathBuf};
 
 use util::Bytes;
 use util::hash::H256;
-use rlp::{self, Encodable, RlpStream, UntrustedRlp, Stream, View};
+use rlp::{self, Encodable, RlpStream, UntrustedRlp, View};
 
 use super::ManifestData;
 
@@ -122,8 +122,8 @@ impl SnapshotWriter for PackedWriter {
 		// they are consistent with ours.
 		let mut stream = RlpStream::new_list(5);
 		stream
-			.append(&self.state_hashes)
-			.append(&self.block_hashes)
+			.append_list(&self.state_hashes)
+			.append_list(&self.block_hashes)
 			.append(&manifest.state_root)
 			.append(&manifest.block_number)
 			.append(&manifest.block_hash);
@@ -428,4 +428,4 @@ mod tests {
 			reader.chunk(hash.clone()).unwrap();
 		}
 	}
-}
+}

ethcore/src/snapshot/mod.rs

@@ -37,7 +37,7 @@ use util::journaldb::{self, Algorithm, JournalDB};
 use util::kvdb::Database;
 use util::trie::{TrieDB, TrieDBMut, Trie, TrieMut};
 use util::sha3::SHA3_NULL_RLP;
-use rlp::{RlpStream, Stream, UntrustedRlp, View};
+use rlp::{RlpStream, UntrustedRlp, View};
 use bloom_journal::Bloom;
 
 use self::block::AbridgedBlock;

ethcore/src/snapshot/tests/blocks.rs

@@ -99,7 +99,7 @@ fn chunk_and_restore_40k() { chunk_and_restore(40000) }
 
 #[test]
 fn checks_flag() {
-	use ::rlp::{RlpStream, Stream};
+	use rlp::RlpStream;
 	use util::H256;
 
 	let mut stream = RlpStream::new_list(5);

ethcore/src/snapshot/tests/state.rs

@@ -95,7 +95,7 @@ fn snap_and_restore() {
 #[test]
 fn get_code_from_prev_chunk() {
 	use std::collections::HashSet;
-	use rlp::{RlpStream, Stream};
+	use rlp::RlpStream;
 	use util::{HashDB, H256, U256, Hashable};
 
 	use account_db::{AccountDBMut, AccountDB};

ethcore/src/spec/seal.rs

@@ -64,9 +64,12 @@ impl Into<Generic> for AuthorityRound {
 
 impl Into<Generic> for Tendermint {
 	fn into(self) -> Generic {
-		let mut s = RlpStream::new_list(3);
-		s.append(&self.round).append(&self.proposal).append(&self.precommits);
-		Generic(s.out())
+		let mut stream = RlpStream::new_list(3);
+		stream
+			.append(&self.round)
+			.append(&self.proposal)
+			.append_list(&self.precommits);
+		Generic(stream.out())
 	}
 }
 

ethcore/src/spec/spec.rs

@@ -34,7 +34,7 @@ use super::genesis::Genesis;
 use super::seal::Generic as GenericSeal;
 use ethereum;
 use ethjson;
-use rlp::{Rlp, RlpStream, View, Stream};
+use rlp::{Rlp, RlpStream, View};
 
 /// Parameters common to all engines.
 #[derive(Debug, PartialEq, Clone, Default)]

ethcore/src/tests/helpers.rs

@@ -34,7 +34,7 @@ use devtools::*;
 use miner::Miner;
 use header::Header;
 use transaction::{Action, Transaction, SignedTransaction};
-use rlp::{self, RlpStream, Stream};
+use rlp::{self, RlpStream};
 use views::BlockView;
 
 #[cfg(feature = "json-tests")]
@@ -129,7 +129,7 @@ pub fn create_test_block_with_data(header: &Header, transactions: &[SignedTransa
 	for t in transactions {
 		rlp.append_raw(&rlp::encode(t).to_vec(), 1);
 	}
-	rlp.append(&uncles);
+	rlp.append_list(&uncles);
 	rlp.out()
 }
 

ethcore/src/trace/bloom.rs

@@ -83,7 +83,7 @@ impl Decodable for BlockTracesBloomGroup {
 
 impl Encodable for BlockTracesBloomGroup {
 	fn rlp_append(&self, s: &mut RlpStream) {
-		Encodable::rlp_append(&self.blooms, s)
+		s.append_list(&self.blooms);
 	}
 }
 

ethcore/src/types/log_entry.rs

@@ -41,7 +41,7 @@ impl Encodable for LogEntry {
 	fn rlp_append(&self, s: &mut RlpStream) {
 		s.begin_list(3);
 		s.append(&self.address);
-		s.append(&self.topics);
+		s.append_list(&self.topics);
 		s.append(&self.data);
 	}
 }

ethcore/src/types/receipt.rs

@@ -60,7 +60,7 @@ impl Encodable for Receipt {
 		}
 		s.append(&self.gas_used);
 		s.append(&self.log_bloom);
-		s.append(&self.logs);
+		s.append_list(&self.logs);
 	}
 }
 

ethcore/src/types/snapshot_manifest.rs

@@ -40,8 +40,8 @@ impl ManifestData {
 	/// Encode the manifest data to rlp.
 	pub fn into_rlp(self) -> Bytes {
 		let mut stream = RlpStream::new_list(5);
-		stream.append(&self.state_hashes);
-		stream.append(&self.block_hashes);
+		stream.append_list(&self.state_hashes);
+		stream.append_list(&self.block_hashes);
 		stream.append(&self.state_root);
 		stream.append(&self.block_number);
 		stream.append(&self.block_hash);

ethcore/src/types/trace_types/error.rs

@@ -17,7 +17,7 @@
 //! Trace errors.
 
 use std::fmt;
-use rlp::{RlpEncodable, Encodable, RlpStream, Decodable, Decoder, DecoderError, View};
+use rlp::{Encodable, RlpStream, Decodable, Decoder, DecoderError, View};
 use evm::Error as EvmError;
 
 /// Trace evm errors.
@@ -85,7 +85,8 @@ impl Encodable for Error {
 			OutOfStack => 4,
 			Internal => 5,
 		};
-		RlpEncodable::rlp_append(&value, s);
+
+		s.append_internal(&value);
 	}
 }
 

ethcore/src/types/trace_types/flat.rs

@@ -59,7 +59,7 @@ impl Encodable for FlatTrace {
 		s.append(&self.action);
 		s.append(&self.result);
 		s.append(&self.subtraces);
-		s.append(&self.trace_address.clone().into_iter().collect::<Vec<_>>());
+		s.append_list::<usize, &usize>(&self.trace_address.iter().collect::<Vec<_>>());
 	}
 }
 
@@ -103,7 +103,7 @@ impl FlatTransactionTraces {
 
 impl Encodable for FlatTransactionTraces {
 	fn rlp_append(&self, s: &mut RlpStream) {
-		Encodable::rlp_append(&self.0, s);
+		s.append_list(&self.0);
 	}
 }
 
@@ -144,7 +144,7 @@ impl FlatBlockTraces {
 
 impl Encodable for FlatBlockTraces {
 	fn rlp_append(&self, s: &mut RlpStream) {
-		Encodable::rlp_append(&self.0, s);
+		s.append_list(&self.0);
 	}
 }
 

ethcore/src/types/trace_types/trace.rs

@@ -475,7 +475,7 @@ impl Encodable for VMExecutedOperation {
 	fn rlp_append(&self, s: &mut RlpStream) {
 		s.begin_list(4);
 		s.append(&self.gas_used);
-		s.append(&self.stack_push);
+		s.append_list(&self.stack_push);
 		s.append(&self.mem_diff);
 		s.append(&self.store_diff);
 	}
@@ -551,8 +551,8 @@ impl Encodable for VMTrace {
 		s.begin_list(4);
 		s.append(&self.parent_step);
 		s.append(&self.code);
-		s.append(&self.operations);
-		s.append(&self.subs);
+		s.append_list(&self.operations);
+		s.append_list(&self.subs);
 	}
 }
 

ethcore/src/verification/verification.rs

@@ -395,7 +395,7 @@ mod tests {
 	#[test]
 	#[cfg_attr(feature="dev", allow(similar_names))]
 	fn test_verify_block() {
-		use rlp::{RlpStream, Stream};
+		use rlp::RlpStream;
 
 		// Test against morden
 		let mut good = Header::new();
@@ -460,7 +460,7 @@ mod tests {
 
 		let good_uncles = vec![ good_uncle1.clone(), good_uncle2.clone() ];
 		let mut uncles_rlp = RlpStream::new();
-		uncles_rlp.append(&good_uncles);
+		uncles_rlp.append_list(&good_uncles);
 		let good_uncles_hash = uncles_rlp.as_raw().sha3();
 		let good_transactions_root = ordered_trie_root(good_transactions.iter().map(|t| ::rlp::encode::<UnverifiedTransaction>(t).to_vec()));