Blocks and snapshot compression (#1687)

* new Compressible rlp trait

* new Compressible rlp trait

* make compressed rlp iterable

* make compressed rlp iterable

* invalid rlp slice swapper

* switch compress to swapper, add reverse swapper test case

* add basic account compression test

* add new rlp trait

* new Compressible rlp trait

* make compressed rlp iterable

* invalid rlp slice swapper

* invalid rlp slice swapper

* switch compress to swapper, add reverse swapper test case

* switch compress to swapper, add reverse swapper test case

* add account compress/ decompress test

* make compressor cleaner, use hashmaps for swapper

* improve compression tests

* add a DecompressingDecoder, change Decoder to take refernce

* separate rlp compression related stuff

* new Compressible rlp trait

* new Compressible rlp trait

* new Compressible rlp trait

* make compressed rlp iterable

* make compressed rlp iterable

* make compressed rlp iterable

* invalid rlp slice swapper

* invalid rlp slice swapper

* invalid rlp slice swapper

* switch compress to swapper, add reverse swapper test case

* switch compress to swapper, add reverse swapper test case

* switch compress to swapper, add reverse swapper test case

* add basic account compression test

* add new rlp trait

* add account compress/ decompress test

* make compressor cleaner, use hashmaps for swapper

* improve compression tests

* add a DecompressingDecoder, change Decoder to take refernce

* separate rlp compression related stuff

* DecompressingDecoder test

* initial compressing HashDB wrapper

* remove unused test

* change CompressedDB to struct wrapper with overlay

* simplify compressor

* failed RefCell attempt

* use denote to return reference

* compiled compresseddb

* compressdb test, add overlay emplace

* fix overlay reference count handling

* add immutable compresseddb, make account use hashdb

* simplify using trait objects

* enable hashdb for account

* initial state compression attempt

* wrap state db

* add tests for analyzing db

* add account predicate

* try to compress data fields as rlp too

* remove compression for storage trie

* add a compressing migration

* more compression stats tests

* fix migration import

* nested encoding compression test

* fix decompression, move db stats tests to rlpcompression

* added malformed rlp tests, cover a few edge cases

* new CompressingEncoder struct

* extend migrations to state

* first version working on the whole db

* clean up Compressible impl

* tests cleanup

* add a testing migration

* refactor deep compression using option, add simple compression

* put tests in a module

* fix compressed overlay loading

* simple compression for snapshots

* remove unused DecompressingDecoder

* add a general compressing migration

* add more common rlps to compress

* use static slices for swapper

* add precomputed hashes and invalid rlps

* make decoder private again

* cover more cases with tests

* style

* fix weird indentation

* remove possible panic in payload_info

* make prefix checking safe

* fix db existence check

* remove db dir from test

* pass usize by value [ci skip]

* Improve comment on panic removal.

* add common blocks db rlps

* add compression to blockchain db

* add blocks db migration

* fix the migrations

* remove state compression

* add a separate snapshot swapper

* ability to use different swappers and traversal

* update tests to new interface

* clean up code ordering

* update usage

* fix compilation

* remove unnecessary changes

* move methods to functions to reduce interface

* move test to module

* update common rlps to blocks db

* move tests to tests modules

* remove redundant &

parity/migration.rs

@@ -26,7 +26,7 @@ use ethcore::migrations;
 /// Database is assumed to be at default version, when no version file is found.
 const DEFAULT_VERSION: u32 = 5;
 /// Current version of database models.
-const CURRENT_VERSION: u32 = 7;
+const CURRENT_VERSION: u32 = 8;
 /// Defines how many items are migrated to the new version of database at once.
 const BATCH_SIZE: usize = 1024;
 /// Version file name.
@@ -110,6 +110,13 @@ fn update_version(path: &Path) -> Result<(), Error> {
 	Ok(())
 }
 
+/// State database path.
+fn state_database_path(path: &Path) -> PathBuf {
+	let mut state_path = path.to_owned();
+	state_path.push("state");
+	state_path
+}
+
 /// Blocks database path.
 fn blocks_database_path(path: &Path) -> PathBuf {
 	let mut blocks_path = path.to_owned();
@@ -124,13 +131,6 @@ fn extras_database_path(path: &Path) -> PathBuf {
 	extras_path
 }
 
-/// State database path.
-fn state_database_path(path: &Path) -> PathBuf {
-	let mut state_path = path.to_owned();
-	state_path.push("state");
-	state_path
-}
-
 /// Database backup
 fn backup_database_path(path: &Path) -> PathBuf {
 	let mut backup_path = path.to_owned();
@@ -148,7 +148,8 @@ fn default_migration_settings() -> MigrationConfig {
 
 /// Migrations on the blocks database.
 fn blocks_database_migrations() -> Result<MigrationManager, Error> {
-	let manager = MigrationManager::new(default_migration_settings());
+	let mut manager = MigrationManager::new(default_migration_settings());
+	try!(manager.add_migration(migrations::blocks::V8::default()).map_err(|_| Error::MigrationImpossible));
 	Ok(manager)
 }
 
@@ -167,8 +168,8 @@ fn state_database_migrations(pruning: Algorithm) -> Result<MigrationManager, Err
 		Algorithm::OverlayRecent => manager.add_migration(migrations::state::OverlayRecentV7::default()),
 		_ => return Err(Error::UnsuportedPruningMethod),
 	};
-
 	try!(res.map_err(|_| Error::MigrationImpossible));
+
 	Ok(manager)
 }