simplify compression and move it out of rlp crate (#7957)

* simplify compression and move it out of rlp crate

* removed lazy_static dependency from rlp

Cargo.lock

@@ -499,6 +499,7 @@ dependencies = [
  "rand 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)",
  "rayon 0.8.2 (registry+https://github.com/rust-lang/crates.io-index)",
  "rlp 0.2.1",
+ "rlp_compress 0.1.0",
  "rlp_derive 0.1.0",
  "rust-crypto 0.2.36 (registry+https://github.com/rust-lang/crates.io-index)",
  "rustc-hex 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -1391,7 +1392,6 @@ version = "0.1.0"
 dependencies = [
  "kvdb 0.1.0",
  "parking_lot 0.5.3 (registry+https://github.com/rust-lang/crates.io-index)",
- "rlp 0.2.1",
 ]
 
 [[package]]
@@ -1406,7 +1406,6 @@ dependencies = [
  "num_cpus 1.7.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "parking_lot 0.5.3 (registry+https://github.com/rust-lang/crates.io-index)",
  "regex 0.2.5 (registry+https://github.com/rust-lang/crates.io-index)",
- "rlp 0.2.1",
  "rocksdb 0.4.5 (git+https://github.com/paritytech/rust-rocksdb)",
  "tempdir 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
@@ -2664,10 +2663,18 @@ dependencies = [
  "byteorder 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)",
  "elastic-array 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "ethereum-types 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)",
- "lazy_static 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "rustc-hex 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
+[[package]]
+name = "rlp_compress"
+version = "0.1.0"
+dependencies = [
+ "elastic-array 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "lazy_static 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "rlp 0.2.1",
+]
+
 [[package]]
 name = "rlp_derive"
 version = "0.1.0"

Cargo.toml

@@ -131,4 +131,5 @@ members = [
 	"miner",
 	"transaction-pool",
 	"whisper",
+	"util/rlp_compress"
 ]

ethcore/Cargo.toml

@@ -47,6 +47,7 @@ price-info = { path = "../price-info" }
 rayon = "0.8"
 rand = "0.4"
 rlp = { path = "../util/rlp" }
+rlp_compress = { path = "../util/rlp_compress" }
 rlp_derive = { path = "../util/rlp_derive" }
 kvdb = { path = "../util/kvdb" }
 kvdb-rocksdb = { path = "../util/kvdb-rocksdb" }

ethcore/src/blockchain/blockchain.rs

@@ -25,6 +25,7 @@ use ethereum_types::{H256, Bloom, U256};
 use parking_lot::{Mutex, RwLock};
 use bytes::Bytes;
 use rlp::*;
+use rlp_compress::{compress, decompress, blocks_swapper};
 use header::*;
 use transaction::*;
 use views::*;
@@ -254,7 +255,7 @@ impl BlockProvider for BlockChain {
 
 		let result = match opt {
 			Some(b) => {
-				let bytes: Bytes = UntrustedRlp::new(&b).decompress(RlpType::Blocks).into_vec();
+				let bytes = decompress(&b, blocks_swapper()).into_vec();
 				let mut write = self.block_headers.write();
 				write.insert(*hash, bytes.clone());
 				Some(encoded::Header::new(bytes))
@@ -290,7 +291,7 @@ impl BlockProvider for BlockChain {
 
 		let result = match opt {
 			Some(b) => {
-				let bytes: Bytes = UntrustedRlp::new(&b).decompress(RlpType::Blocks).into_vec();
+				let bytes = decompress(&b, blocks_swapper()).into_vec();
 				let mut write = self.block_bodies.write();
 				write.insert(*hash, bytes.clone());
 				Some(encoded::Body::new(bytes))
@@ -702,9 +703,8 @@ impl BlockChain {
 
 		assert!(self.pending_best_block.read().is_none());
 
-		let block_rlp = UntrustedRlp::new(bytes);
+		let compressed_header = compress(block.header_rlp().as_raw(), blocks_swapper());
-		let compressed_header = block_rlp.at(0).unwrap().compress(RlpType::Blocks);
+		let compressed_body = compress(&Self::block_to_body(bytes), blocks_swapper());
-		let compressed_body = UntrustedRlp::new(&Self::block_to_body(bytes)).compress(RlpType::Blocks);
 
 		// store block in db
 		batch.put(db::COL_HEADERS, &hash, &compressed_header);
@@ -901,9 +901,12 @@ impl BlockChain {
 
 		assert!(self.pending_best_block.read().is_none());
 
+		let compressed_header = compress(block.header_rlp().as_raw(), blocks_swapper());
+		let compressed_body = compress(&Self::block_to_body(bytes), blocks_swapper());
+
 		// store block in db
-		batch.put_compressed(db::COL_HEADERS, &hash, block.header_rlp().as_raw().to_vec());
+		batch.put(db::COL_HEADERS, &hash, &compressed_header);
-		batch.put_compressed(db::COL_BODIES, &hash, Self::block_to_body(bytes));
+		batch.put(db::COL_BODIES, &hash, &compressed_body);
 
 		let info = self.block_info(&header);
 

ethcore/src/lib.rs

@@ -83,6 +83,7 @@ extern crate price_info;
 extern crate rand;
 extern crate rayon;
 extern crate rlp;
+extern crate rlp_compress;
 extern crate keccak_hash as hash;
 extern crate heapsize;
 extern crate memorydb;

ethcore/src/state/account.rs

@@ -485,7 +485,7 @@ impl fmt::Debug for Account {
 
 #[cfg(test)]
 mod tests {
-	use rlp::{UntrustedRlp, RlpType, Compressible};
+	use rlp_compress::{compress, decompress, snapshot_swapper};
 	use ethereum_types::{H256, Address};
 	use memorydb::MemoryDB;
 	use bytes::Bytes;
@@ -495,10 +495,9 @@ mod tests {
 	#[test]
 	fn account_compress() {
 		let raw = Account::new_basic(2.into(), 4.into()).rlp();
-		let rlp = UntrustedRlp::new(&raw);
+		let compact_vec = compress(&raw, snapshot_swapper());
-		let compact_vec = rlp.compress(RlpType::Snapshot).into_vec();
 		assert!(raw.len() > compact_vec.len());
-		let again_raw = UntrustedRlp::new(&compact_vec).decompress(RlpType::Snapshot);
+		let again_raw = decompress(&compact_vec, snapshot_swapper());
 		assert_eq!(raw, again_raw.into_vec());
     }
 

util/kvdb-memorydb/Cargo.toml

@@ -5,5 +5,4 @@ authors = ["Parity Technologies <admin@parity.io>"]
 
 [dependencies]
 parking_lot = "0.5"
-rlp = { path = "../rlp" }
 kvdb = { path = "../kvdb" }

util/kvdb-memorydb/src/lib.rs

@@ -16,12 +16,10 @@
 
 extern crate parking_lot;
 extern crate kvdb;
-extern crate rlp;
 
 use std::collections::{BTreeMap, HashMap};
 use parking_lot::RwLock;
 use kvdb::{DBValue, DBTransaction, KeyValueDB, DBOp, Result};
-use rlp::{RlpType, UntrustedRlp, Compressible};
 
 /// A key-value database fulfilling the `KeyValueDB` trait, living in memory.
 /// This is generally intended for tests and is not particularly optimized.
@@ -75,14 +73,6 @@ impl KeyValueDB for InMemory {
 						col.insert(key.into_vec(), value);
 					}
 				},
-				DBOp::InsertCompressed { col, key, value } => {
-					if let Some(col) = columns.get_mut(&col) {
-						let compressed = UntrustedRlp::new(&value).compress(RlpType::Blocks);
-						let mut value = DBValue::new();
-						value.append_slice(&compressed);
-						col.insert(key.into_vec(), value);
-					}
-				},
 				DBOp::Delete { col, key } => {
 					if let Some(col) = columns.get_mut(&col) {
 						col.remove(&*key);

util/kvdb-rocksdb/Cargo.toml

@@ -11,7 +11,6 @@ log = "0.3"
 num_cpus = "1.0"
 parking_lot = "0.5"
 regex = "0.2"
-rlp = { path = "../rlp" }
 rocksdb = { git = "https://github.com/paritytech/rust-rocksdb" }
 interleaved-ordered = "0.1.0"
 

util/kvdb-rocksdb/src/lib.rs

@@ -26,7 +26,6 @@ extern crate rocksdb;
 
 extern crate ethereum_types;
 extern crate kvdb;
-extern crate rlp;
 
 use std::cmp;
 use std::collections::HashMap;
@@ -42,7 +41,6 @@ use rocksdb::{
 use interleaved_ordered::{interleave_ordered, InterleaveOrdered};
 
 use elastic_array::ElasticArray32;
-use rlp::{UntrustedRlp, RlpType, Compressible};
 use kvdb::{KeyValueDB, DBTransaction, DBValue, DBOp, Result};
 
 #[cfg(target_os = "linux")]
@@ -56,7 +54,6 @@ const DB_DEFAULT_MEMORY_BUDGET_MB: usize = 128;
 
 enum KeyState {
 	Insert(DBValue),
-	InsertCompressed(DBValue),
 	Delete,
 }
 
@@ -406,10 +403,6 @@ impl Database {
 					let c = Self::to_overlay_column(col);
 					overlay[c].insert(key, KeyState::Insert(value));
 				},
-				DBOp::InsertCompressed { col, key, value } => {
-					let c = Self::to_overlay_column(col);
-					overlay[c].insert(key, KeyState::InsertCompressed(value));
-				},
 				DBOp::Delete { col, key } => {
 					let c = Self::to_overlay_column(col);
 					overlay[c].insert(key, KeyState::Delete);
@@ -442,14 +435,6 @@ impl Database {
 										batch.put(&key, &value)?;
 									}
 								},
-								KeyState::InsertCompressed(ref value) => {
-									let compressed = UntrustedRlp::new(&value).compress(RlpType::Blocks);
-									if c > 0 {
-										batch.put_cf(cfs[c - 1], &key, &compressed)?;
-									} else {
-										batch.put(&key, &value)?;
-									}
-								}
 							}
 						}
 					}
@@ -498,10 +483,6 @@ impl Database {
 						DBOp::Insert { col, key, value } => {
 							col.map_or_else(|| batch.put(&key, &value), |c| batch.put_cf(cfs[c as usize], &key, &value))?
 						},
-						DBOp::InsertCompressed { col, key, value } => {
-							let compressed = UntrustedRlp::new(&value).compress(RlpType::Blocks);
-							col.map_or_else(|| batch.put(&key, &compressed), |c| batch.put_cf(cfs[c as usize], &key, &compressed))?
-						},
 						DBOp::Delete { col, key } => {
 							col.map_or_else(|| batch.delete(&key), |c| batch.delete_cf(cfs[c as usize], &key))?
 						},
@@ -522,12 +503,12 @@ impl Database {
 			Some(DBAndColumns { ref db, ref cfs }) => {
 				let overlay = &self.overlay.read()[Self::to_overlay_column(col)];
 				match overlay.get(key) {
-					Some(&KeyState::Insert(ref value)) | Some(&KeyState::InsertCompressed(ref value)) => Ok(Some(value.clone())),
+					Some(&KeyState::Insert(ref value)) => Ok(Some(value.clone())),
 					Some(&KeyState::Delete) => Ok(None),
 					None => {
 						let flushing = &self.flushing.read()[Self::to_overlay_column(col)];
 						match flushing.get(key) {
-							Some(&KeyState::Insert(ref value)) | Some(&KeyState::InsertCompressed(ref value)) => Ok(Some(value.clone())),
+							Some(&KeyState::Insert(ref value)) => Ok(Some(value.clone())),
 							Some(&KeyState::Delete) => Ok(None),
 							None => {
 								col.map_or_else(
@@ -562,8 +543,7 @@ impl Database {
 				let overlay = &self.overlay.read()[Self::to_overlay_column(col)];
 				let mut overlay_data = overlay.iter()
 					.filter_map(|(k, v)| match *v {
-						KeyState::Insert(ref value) |
+						KeyState::Insert(ref value) =>
-						KeyState::InsertCompressed(ref value) =>
 							Some((k.clone().into_vec().into_boxed_slice(), value.clone().into_vec().into_boxed_slice())),
 						KeyState::Delete => None,
 					}).collect::<Vec<_>>();

util/kvdb/src/lib.rs

@@ -56,11 +56,6 @@ pub enum DBOp {
 		key: ElasticArray32<u8>,
 		value: DBValue,
 	},
-	InsertCompressed {
-		col: Option<u32>,
-		key: ElasticArray32<u8>,
-		value: DBValue,
-	},
 	Delete {
 		col: Option<u32>,
 		key: ElasticArray32<u8>,
@@ -72,7 +67,6 @@ impl DBOp {
 	pub fn key(&self) -> &[u8] {
 		match *self {
 			DBOp::Insert { ref key, .. } => key,
-			DBOp::InsertCompressed { ref key, .. } => key,
 			DBOp::Delete { ref key, .. } => key,
 		}
 	}
@@ -81,7 +75,6 @@ impl DBOp {
 	pub fn col(&self) -> Option<u32> {
 		match *self {
 			DBOp::Insert { col, .. } => col,
-			DBOp::InsertCompressed { col, .. } => col,
 			DBOp::Delete { col, .. } => col,
 		}
 	}
@@ -122,18 +115,6 @@ impl DBTransaction {
 		});
 	}
 
-	/// Insert a key-value pair in the transaction. Any existing value will be overwritten upon write.
-	/// Value will be RLP-compressed on flush
-	pub fn put_compressed(&mut self, col: Option<u32>, key: &[u8], value: Bytes) {
-		let mut ekey = ElasticArray32::new();
-		ekey.append_slice(key);
-		self.ops.push(DBOp::InsertCompressed {
-			col: col,
-			key: ekey,
-			value: DBValue::from_vec(value),
-		});
-	}
-
 	/// Delete value by key.
 	pub fn delete(&mut self, col: Option<u32>, key: &[u8]) {
 		let mut ekey = ElasticArray32::new();

util/rlp/Cargo.toml

@@ -9,6 +9,5 @@ authors = ["Parity Technologies <admin@parity.io>"]
 [dependencies]
 elastic-array = "0.9"
 ethereum-types = "0.2"
-lazy_static = "1.0"
 rustc-hex = "1.0"
 byteorder = "1.0"

util/rlp/src/lib.rs

@@ -43,27 +43,21 @@ extern crate ethereum_types as bigint;
 extern crate elastic_array;
 extern crate rustc_hex;
 
-#[macro_use]
-extern crate lazy_static;
-
 mod traits;
 mod error;
 mod rlpin;
 mod untrusted_rlp;
 mod stream;
-mod compression;
-mod common;
 mod impls;
 
 use std::borrow::Borrow;
 use elastic_array::ElasticArray1024;
 
 pub use error::DecoderError;
-pub use traits::{Decodable, Encodable, Compressible};
+pub use traits::{Decodable, Encodable};
 pub use untrusted_rlp::{UntrustedRlp, UntrustedRlpIterator, PayloadInfo, Prototype};
 pub use rlpin::{Rlp, RlpIterator};
 pub use stream::RlpStream;
-pub use compression::RlpType;
 
 /// The RLP encoded empty data (used to mean "null value").
 pub const NULL_RLP: [u8; 1] = [0x80; 1];

util/rlp/src/traits.rs

@@ -28,14 +28,3 @@ pub trait Encodable {
 		s.drain()
 	}
 }
-
-/// Trait for compressing and decompressing RLP by replacement of common terms.
-pub trait Compressible: Sized {
-	/// Indicates the origin of RLP to be compressed.
-	type DataType;
-
-	/// Compress given RLP type using appropriate methods.
-	fn compress(&self, t: Self::DataType) -> ElasticArray1024<u8>;
-	/// Decompress given RLP type using appropriate methods.
-	fn decompress(&self, t: Self::DataType) -> ElasticArray1024<u8>;
-}

util/rlp_compress/Cargo.toml

@@ -0,0 +1,9 @@
+[package]
+name = "rlp_compress"
+version = "0.1.0"
+authors = ["Parity Technologies <admin@parity.io>"]
+
+[dependencies]
+rlp = { path = "../rlp" }
+elastic-array = "0.9"
+lazy_static = "1.0"

util/rlp_compress/src/common.rs

@@ -8,16 +8,16 @@
 
 //! Contains RLPs used for compression.
 
-use compression::InvalidRlpSwapper;
+use Swapper;
 
 lazy_static! {
 	/// Swapper for snapshot compression.
-	pub static ref SNAPSHOT_RLP_SWAPPER: InvalidRlpSwapper<'static> = InvalidRlpSwapper::new(EMPTY_RLPS, INVALID_RLPS);
+	pub static ref SNAPSHOT_SWAPPER: Swapper<'static> = Swapper::new(EMPTY_RLPS, INVALID_RLPS);
 }
 
 lazy_static! {
 	/// Swapper with common long RLPs, up to 127 can be added.
-	pub static ref BLOCKS_RLP_SWAPPER: InvalidRlpSwapper<'static> = InvalidRlpSwapper::new(COMMON_RLPS, INVALID_RLPS);
+	pub static ref BLOCKS_SWAPPER: Swapper<'static> = Swapper::new(COMMON_RLPS, INVALID_RLPS);
 }
 
 static EMPTY_RLPS: &'static [&'static [u8]] = &[

util/rlp_compress/src/lib.rs

@@ -0,0 +1,110 @@
+// Copyright 2015-2018 Parity Technologies
+//
+// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+// option. This file may not be copied, modified, or distributed
+// except according to those terms.
+
+extern crate elastic_array;
+#[macro_use]
+extern crate lazy_static;
+extern crate rlp;
+
+mod common;
+
+use std::cmp;
+use std::collections::HashMap;
+use elastic_array::ElasticArray1024;
+use rlp::{UntrustedRlp, RlpStream};
+use common::{SNAPSHOT_SWAPPER, BLOCKS_SWAPPER};
+
+pub fn snapshot_swapper() -> &'static Swapper<'static> {
+	&SNAPSHOT_SWAPPER as &Swapper
+}
+
+pub fn blocks_swapper() -> &'static Swapper<'static> {
+	&BLOCKS_SWAPPER as &Swapper
+}
+
+/// A trait used to compress rlp.
+pub trait Compressor {
+	/// Get compressed version of given rlp.
+	fn compressed(&self, rlp: &[u8]) -> Option<&[u8]>;
+}
+
+/// A trait used to convert compressed rlp into it's original version.
+pub trait Decompressor {
+	/// Get decompressed rlp.
+	fn decompressed(&self, compressed: &[u8]) -> Option<&[u8]>;
+}
+
+/// Call this function to compress rlp.
+pub fn compress(c: &[u8], swapper: &Compressor) -> ElasticArray1024<u8> {
+	let rlp = UntrustedRlp::new(c);
+	if rlp.is_data() {
+		ElasticArray1024::from_slice(swapper.compressed(rlp.as_raw()).unwrap_or_else(|| rlp.as_raw()))
+	} else {
+		map_rlp(&rlp, |r| compress(r.as_raw(), swapper))
+	}
+}
+
+/// Call this function to decompress rlp.
+pub fn decompress(c: &[u8], swapper: &Decompressor) -> ElasticArray1024<u8> {
+	let rlp = UntrustedRlp::new(c);
+	if rlp.is_data() {
+		ElasticArray1024::from_slice(swapper.decompressed(rlp.as_raw()).unwrap_or_else(|| rlp.as_raw()))
+	} else {
+		map_rlp(&rlp, |r| decompress(r.as_raw(), swapper))
+	}
+}
+
+fn map_rlp<F: Fn(&UntrustedRlp) -> ElasticArray1024<u8>>(rlp: &UntrustedRlp, f: F) -> ElasticArray1024<u8> {
+	let mut stream = RlpStream::new_list(rlp.item_count().unwrap_or_default());
+	for subrlp in rlp.iter() {
+		stream.append_raw(&f(&subrlp), 1);
+	}
+	stream.drain()
+}
+
+/// Stores RLPs used for compression
+pub struct Swapper<'a> {
+	compressed_to_rlp: HashMap<&'a [u8], &'a [u8]>,
+	rlp_to_compressed: HashMap<&'a [u8], &'a [u8]>,
+}
+
+impl<'a> Swapper<'a> {
+	/// Construct a swapper from a list of common RLPs
+	pub fn new(rlps_to_swap: &[&'a [u8]], compressed: &[&'a [u8]]) -> Self {
+		if rlps_to_swap.len() > 0x7e {
+			panic!("Invalid usage, only 127 RLPs can be swappable.");
+		}
+
+		let items = cmp::min(rlps_to_swap.len(), compressed.len());
+		let mut compressed_to_rlp = HashMap::with_capacity(items);
+		let mut rlp_to_compressed = HashMap::with_capacity(items);
+
+		for (&rlp, &compressed) in rlps_to_swap.iter().zip(compressed.iter()) {
+			compressed_to_rlp.insert(compressed, rlp);
+			rlp_to_compressed.insert(rlp, compressed);
+		}
+
+		Swapper {
+			compressed_to_rlp,
+			rlp_to_compressed,
+		}
+	}
+}
+
+impl<'a> Decompressor for Swapper<'a> {
+	fn decompressed(&self, compressed: &[u8]) -> Option<&[u8]> {
+		self.compressed_to_rlp.get(compressed).cloned()
+	}
+}
+
+impl<'a> Compressor for Swapper<'a> {
+	fn compressed(&self, rlp: &[u8]) -> Option<&[u8]> {
+		self.rlp_to_compressed.get(rlp).cloned()
+	}
+}
+