Blocks and snapshot compression (#1687)

* new Compressible rlp trait

* new Compressible rlp trait

* make compressed rlp iterable

* make compressed rlp iterable

* invalid rlp slice swapper

* switch compress to swapper, add reverse swapper test case

* add basic account compression test

* add new rlp trait

* new Compressible rlp trait

* make compressed rlp iterable

* invalid rlp slice swapper

* invalid rlp slice swapper

* switch compress to swapper, add reverse swapper test case

* switch compress to swapper, add reverse swapper test case

* add account compress/ decompress test

* make compressor cleaner, use hashmaps for swapper

* improve compression tests

* add a DecompressingDecoder, change Decoder to take refernce

* separate rlp compression related stuff

* new Compressible rlp trait

* new Compressible rlp trait

* new Compressible rlp trait

* make compressed rlp iterable

* make compressed rlp iterable

* make compressed rlp iterable

* invalid rlp slice swapper

* invalid rlp slice swapper

* invalid rlp slice swapper

* switch compress to swapper, add reverse swapper test case

* switch compress to swapper, add reverse swapper test case

* switch compress to swapper, add reverse swapper test case

* add basic account compression test

* add new rlp trait

* add account compress/ decompress test

* make compressor cleaner, use hashmaps for swapper

* improve compression tests

* add a DecompressingDecoder, change Decoder to take refernce

* separate rlp compression related stuff

* DecompressingDecoder test

* initial compressing HashDB wrapper

* remove unused test

* change CompressedDB to struct wrapper with overlay

* simplify compressor

* failed RefCell attempt

* use denote to return reference

* compiled compresseddb

* compressdb test, add overlay emplace

* fix overlay reference count handling

* add immutable compresseddb, make account use hashdb

* simplify using trait objects

* enable hashdb for account

* initial state compression attempt

* wrap state db

* add tests for analyzing db

* add account predicate

* try to compress data fields as rlp too

* remove compression for storage trie

* add a compressing migration

* more compression stats tests

* fix migration import

* nested encoding compression test

* fix decompression, move db stats tests to rlpcompression

* added malformed rlp tests, cover a few edge cases

* new CompressingEncoder struct

* extend migrations to state

* first version working on the whole db

* clean up Compressible impl

* tests cleanup

* add a testing migration

* refactor deep compression using option, add simple compression

* put tests in a module

* fix compressed overlay loading

* simple compression for snapshots

* remove unused DecompressingDecoder

* add a general compressing migration

* add more common rlps to compress

* use static slices for swapper

* add precomputed hashes and invalid rlps

* make decoder private again

* cover more cases with tests

* style

* fix weird indentation

* remove possible panic in payload_info

* make prefix checking safe

* fix db existence check

* remove db dir from test

* pass usize by value [ci skip]

* Improve comment on panic removal.

* add common blocks db rlps

* add compression to blockchain db

* add blocks db migration

* fix the migrations

* remove state compression

* add a separate snapshot swapper

* ability to use different swappers and traversal

* update tests to new interface

* clean up code ordering

* update usage

* fix compilation

* remove unnecessary changes

* move methods to functions to reduce interface

* move test to module

* update common rlps to blocks db

* move tests to tests modules

* remove redundant &

ethcore/src/account.rs

@@ -288,6 +288,16 @@ mod tests {
 	use super::*;
 	use account_db::*;
 
+	#[test]
+	fn account_compress() {
+		let raw = Account::new_basic(2.into(), 4.into()).rlp();
+		let rlp = UntrustedRlp::new(&raw);
+		let compact_vec = rlp.compress(RlpType::Snapshot).to_vec();
+		assert!(raw.len() > compact_vec.len());
+		let again_raw = UntrustedRlp::new(&compact_vec).decompress(RlpType::Snapshot);
+		assert_eq!(raw, again_raw.to_vec());
+    }
+
 	#[test]
 	fn storage_at() {
 		let mut db = MemoryDB::new();

ethcore/src/blockchain/blockchain.rs

@@ -183,7 +183,7 @@ impl BlockProvider for BlockChain {
 
 		match opt {
 			Some(b) => {
-				let bytes: Bytes = b.to_vec();
+				let bytes: Bytes = UntrustedRlp::new(&b).decompress(RlpType::Blocks).to_vec();
 				let mut write = self.blocks.write();
 				write.insert(hash.clone(), bytes.clone());
 				Some(bytes)
@@ -510,9 +510,11 @@ impl BlockChain {
 			return ImportRoute::none();
 		}
 
+		let compressed = UntrustedRlp::new(bytes).compress(RlpType::Blocks).to_vec();
+
 		let _lock = self.insert_lock.lock();
 		// store block in db
-		self.blocks_db.put(&hash, bytes).unwrap();
+		self.blocks_db.put(&hash, &compressed).unwrap();
 
 		let info = self.block_info(bytes);
 

ethcore/src/migrations/blocks/mod.rs

@@ -0,0 +1,21 @@
+// Copyright 2015, 2016 Ethcore (UK) Ltd.
+// This file is part of Parity.
+
+// Parity is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Parity is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Parity.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Blocks database migrations.
+
+mod v8;
+
+pub use self::v8::V8;

ethcore/src/migrations/blocks/v8.rs

@@ -0,0 +1,34 @@
+// Copyright 2015, 2016 Ethcore (UK) Ltd.
+// This file is part of Parity.
+
+// Parity is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Parity is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Parity.  If not, see <http://www.gnu.org/licenses/>.
+
+//! This migration compresses the state db.
+
+use util::migration::SimpleMigration;
+use util::rlp::{Compressible, UntrustedRlp, View, RlpType};
+
+/// Compressing migration.
+#[derive(Default)]
+pub struct V8;
+
+impl SimpleMigration for V8 {
+	fn version(&self) -> u32 {
+		8
+	}
+
+	fn simple_migrate(&mut self, key: Vec<u8>, value: Vec<u8>) -> Option<(Vec<u8>, Vec<u8>)> {
+		Some((key,UntrustedRlp::new(&value).compress(RlpType::Blocks).to_vec()))
+	}
+}

ethcore/src/migrations/mod.rs

@@ -1,4 +1,5 @@
 //! Database migrations.
 
-pub mod extras;
 pub mod state;
+pub mod blocks;
+pub mod extras;

ethcore/src/snapshot/mod.rs

@@ -29,7 +29,7 @@ use views::{BlockView, HeaderView};
 
 use util::{Bytes, Hashable, HashDB, JournalDB, snappy, TrieDB, TrieDBMut, TrieMut};
 use util::hash::{FixedHash, H256};
-use util::rlp::{DecoderError, RlpStream, Stream, UntrustedRlp, View};
+use util::rlp::{DecoderError, RlpStream, Stream, UntrustedRlp, View, Compressible, RlpType};
 
 use self::account::Account;
 use self::block::AbridgedBlock;
@@ -261,7 +261,8 @@ pub fn chunk_state(db: &HashDB, root: &H256, path: &Path) -> Result<Vec<H256>, E
 		let account_db = AccountDB::from_hash(db, account_key_hash);
 
 		let fat_rlp = try!(account.to_fat_rlp(&account_db));
-		try!(chunker.push(account_key, fat_rlp));
+		let compressed_rlp = UntrustedRlp::new(&fat_rlp).compress(RlpType::Snapshot).to_vec();
+		try!(chunker.push(account_key, compressed_rlp));
 	}
 
 	if chunker.cur_size != 0 {
@@ -400,7 +401,8 @@ fn rebuild_account_trie(db: &mut HashDB, account_chunk: &[&[u8]], out_chunk: &mu
 		let account_rlp = UntrustedRlp::new(account_pair);
 
 		let hash: H256 = try!(account_rlp.val_at(0));
-		let fat_rlp = try!(account_rlp.at(1));
+		let decompressed = try!(account_rlp.at(1)).decompress(RlpType::Snapshot);
+		let fat_rlp = UntrustedRlp::new(&decompressed[..]);
 
 		let thin_rlp = {
 			let mut acct_db = AccountDBMut::from_hash(db.as_hashdb_mut(), hash);