openethereum/util/rlp/src/compression.rs

// Copyright 2015-2017 Parity Technologies
//
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.

use std::collections::HashMap;
use elastic_array::ElasticArray1024;
use common::{BLOCKS_RLP_SWAPPER, SNAPSHOT_RLP_SWAPPER};
use {UntrustedRlp, Compressible, encode, RlpStream};

/// Stores RLPs used for compression
pub struct InvalidRlpSwapper<'a> {
	invalid_to_valid: HashMap<&'a [u8], &'a [u8]>,
	valid_to_invalid: HashMap<&'a [u8], &'a [u8]>,
}

impl<'a> InvalidRlpSwapper<'a> {
	/// Construct a swapper from a list of common RLPs
	pub fn new(rlps_to_swap: &[&'a [u8]], invalid_rlps: &[&'a [u8]]) -> Self {
		if rlps_to_swap.len() > 0x7e {
			panic!("Invalid usage, only 127 RLPs can be swappable.");
		}
		let mut invalid_to_valid = HashMap::new();
		let mut valid_to_invalid = HashMap::new();
		for (&rlp, &invalid) in rlps_to_swap.iter().zip(invalid_rlps.iter()) {
			invalid_to_valid.insert(invalid, rlp);
			valid_to_invalid.insert(rlp, invalid);
		}
		InvalidRlpSwapper {
			invalid_to_valid: invalid_to_valid,
			valid_to_invalid: valid_to_invalid
		}
	}
	/// Get a valid RLP corresponding to an invalid one
	fn get_valid(&self, invalid_rlp: &[u8]) -> Option<&[u8]> {
		self.invalid_to_valid.get(invalid_rlp).cloned()
	}
	/// Get an invalid RLP corresponding to a valid one
	fn get_invalid(&self, valid_rlp: &[u8]) -> Option<&[u8]> {
		self.valid_to_invalid.get(valid_rlp).cloned()
	}
}

/// Type of RLP indicating its origin database.
pub enum RlpType {
	/// RLP used in blocks database.
	Blocks,
	/// RLP used in snapshots.
	Snapshot,
}

fn to_elastic(slice: &[u8]) -> ElasticArray1024<u8> {
	let mut out = ElasticArray1024::new();
	out.append_slice(slice);
	out
}

fn map_rlp<F>(rlp: &UntrustedRlp, f: F) -> Option<ElasticArray1024<u8>> where
	F: Fn(&UntrustedRlp) -> Option<ElasticArray1024<u8>> {
	match rlp.iter()
		.fold((false, RlpStream::new_list(rlp.item_count().unwrap_or(0))),
		|(is_some, mut acc), subrlp| {
  		let new = f(&subrlp);
  		if let Some(ref insert) = new {
  			acc.append_raw(&insert[..], 1);
  		} else {
  			acc.append_raw(subrlp.as_raw(), 1);
  		}
  		(is_some || new.is_some(), acc)
  	  }) {
  	(true, s) => Some(s.drain()),
  	_ => None,
  }
}

/// Replace common RLPs with invalid shorter ones.
fn simple_compress(rlp: &UntrustedRlp, swapper: &InvalidRlpSwapper) -> ElasticArray1024<u8> {
	if rlp.is_data() {
		to_elastic(swapper.get_invalid(rlp.as_raw()).unwrap_or_else(|| rlp.as_raw()))
	} else {
		map_rlp(rlp, |r| Some(simple_compress(r, swapper))).unwrap_or_else(|| to_elastic(rlp.as_raw()))
	}
}

/// Recover valid RLP from a compressed form.
fn simple_decompress(rlp: &UntrustedRlp, swapper: &InvalidRlpSwapper) -> ElasticArray1024<u8> {
	if rlp.is_data() {
		to_elastic(swapper.get_valid(rlp.as_raw()).unwrap_or_else(|| rlp.as_raw()))
	} else {
		map_rlp(rlp, |r| Some(simple_decompress(r, swapper))).unwrap_or_else(|| to_elastic(rlp.as_raw()))
	}
}

/// Recover valid RLP from a compressed form, None if no decompression achieved.
/// Tries to decompress compressed data insides.
fn deep_decompress(rlp: &UntrustedRlp, swapper: &InvalidRlpSwapper) -> Option<ElasticArray1024<u8>> {
	let simple_swap = ||
		swapper.get_valid(rlp.as_raw()).map(to_elastic);
	// Simply decompress data.
	if rlp.is_data() { return simple_swap(); }
	match rlp.item_count().unwrap_or(0) {
		// Look for special compressed list, which contains nested data.
		2 if rlp.at(0).map(|r| r.as_raw() == &[0x81, 0x7f]).unwrap_or(false) =>
			rlp.at(1).ok().map_or(simple_swap(),
			|r| deep_decompress(&r, swapper).map(|d| { let v = d.to_vec(); encode(&v) })),
		// Iterate through RLP while checking if it has been compressed.
		_ => map_rlp(rlp, |r| deep_decompress(r, swapper)),
  	}
}

impl<'a> Compressible for UntrustedRlp<'a> {
	type DataType = RlpType;

	fn compress(&self, t: RlpType) -> ElasticArray1024<u8> {
		match t {
			RlpType::Snapshot => simple_compress(self, &SNAPSHOT_RLP_SWAPPER),
			RlpType::Blocks => simple_compress(self, &BLOCKS_RLP_SWAPPER),
		}
	}

	fn decompress(&self, t: RlpType) -> ElasticArray1024<u8> {
		match t {
			RlpType::Snapshot => simple_decompress(self, &SNAPSHOT_RLP_SWAPPER),
			RlpType::Blocks => deep_decompress(self, &BLOCKS_RLP_SWAPPER).unwrap_or_else(|| to_elastic(self.as_raw())),
		}
	}
}

#[cfg(test)]
mod tests {
	use compression::InvalidRlpSwapper;
	use {UntrustedRlp, Compressible, RlpType};

	#[test]
	fn invalid_rlp_swapper() {
		let to_swap: &[&[u8]] = &[&[0x83, b'c', b'a', b't'], &[0x83, b'd', b'o', b'g']];
		let invalid_rlp: &[&[u8]] = &[&[0x81, 0x00], &[0x81, 0x01]];
		let swapper = InvalidRlpSwapper::new(to_swap, invalid_rlp);
		assert_eq!(Some(invalid_rlp[0]), swapper.get_invalid(&[0x83, b'c', b'a', b't']));
		assert_eq!(None, swapper.get_invalid(&[0x83, b'b', b'a', b't']));
		assert_eq!(Some(to_swap[1]), swapper.get_valid(invalid_rlp[1]));
	}

	#[test]
	fn simple_compression() {
		let basic_account_rlp = vec![248, 68, 4, 2, 160, 86, 232, 31, 23, 27, 204, 85, 166, 255, 131, 69, 230, 146, 192, 248, 110, 91, 72, 224, 27, 153, 108, 173, 192, 1, 98, 47, 181, 227, 99, 180, 33, 160, 197, 210, 70, 1, 134, 247, 35, 60, 146, 126, 125, 178, 220, 199, 3, 192, 229, 0, 182, 83, 202, 130, 39, 59, 123, 250, 216, 4, 93, 133, 164, 112];
		let rlp = UntrustedRlp::new(&basic_account_rlp);
		let compressed = rlp.compress(RlpType::Snapshot).to_vec();
		assert_eq!(compressed, vec![198, 4, 2, 129, 0, 129, 1]);
		let compressed_rlp = UntrustedRlp::new(&compressed);
		assert_eq!(compressed_rlp.decompress(RlpType::Snapshot).to_vec(), basic_account_rlp);
	}

	#[test]
	fn data_compression() {
		let data_basic_account_rlp = vec![184, 70, 248, 68, 4, 2, 160, 86, 232, 31, 23, 27, 204, 85, 166, 255, 131, 69, 230, 146, 192, 248, 110, 91, 72, 224, 27, 153, 108, 173, 192, 1, 98, 47, 181, 227, 99, 180, 33, 160, 197, 210, 70, 1, 134, 247, 35, 60, 146, 126, 125, 178, 220, 199, 3, 192, 229, 0, 182, 83, 202, 130, 39, 59, 123, 250, 216, 4, 93, 133, 164, 112];
		let compressed = vec![201, 129, 127, 198, 4, 2, 129, 0, 129, 1];
		let compressed_rlp = UntrustedRlp::new(&compressed);
		assert_eq!(compressed_rlp.decompress(RlpType::Blocks).to_vec(), data_basic_account_rlp);
	}

	#[test]
	fn nested_list_rlp() {
		let nested_basic_account_rlp = vec![228, 4, 226, 2, 160, 86, 232, 31, 23, 27, 204, 85, 166, 255, 131, 69, 230, 146, 192, 248, 110, 91, 72, 224, 27, 153, 108, 173, 192, 1, 98, 47, 181, 227, 99, 180, 33];
		let nested_rlp = UntrustedRlp::new(&nested_basic_account_rlp);
		let compressed = nested_rlp.compress(RlpType::Blocks).to_vec();
		assert_eq!(compressed, vec![197, 4, 195, 2, 129, 0]);
		let compressed_rlp = UntrustedRlp::new(&compressed);
		assert_eq!(compressed_rlp.decompress(RlpType::Blocks).to_vec(), nested_basic_account_rlp);
		let compressed = nested_rlp.compress(RlpType::Snapshot).to_vec();
		assert_eq!(compressed, vec![197, 4, 195, 2, 129, 0]);
		let compressed_rlp = UntrustedRlp::new(&compressed);
		assert_eq!(compressed_rlp.decompress(RlpType::Snapshot).to_vec(), nested_basic_account_rlp);
	}

	#[test]
	fn malformed_rlp() {
		let malformed = vec![248, 81, 128, 128, 128, 128, 128, 160, 12, 51, 241, 93, 69, 218, 74, 138, 79, 115, 227, 44, 216, 81, 46, 132, 85, 235, 96, 45, 252, 48, 181, 29, 75, 141, 217, 215, 86, 160, 109, 130, 160, 140, 36, 93, 200, 109, 215, 100, 241, 246, 99, 135, 92, 168, 149, 170, 114, 9, 143, 4, 93, 25, 76, 54, 176, 119, 230, 170, 154, 105, 47, 121, 10, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128];
		let malformed_rlp = UntrustedRlp::new(&malformed);
		assert_eq!(malformed_rlp.decompress(RlpType::Blocks).to_vec(), malformed);
	}

	#[test]
	fn large_block() {
		let block = vec![249, 97, 87, 249, 2, 19, 160, 137, 152, 36, 115, 234, 67, 89, 207, 44, 42, 186, 128, 91, 242, 10, 16, 42, 193, 195, 2, 129, 60, 181, 150, 192, 178, 117, 15, 18, 100, 174, 249, 160, 29, 204, 77, 232, 222, 199, 93, 122, 171, 133, 181, 103, 182, 204, 212, 26, 211, 18, 69, 27, 148, 138, 116, 19, 240, 161, 66, 253, 64, 212, 147, 71, 148, 223, 125, 126, 5, 57, 51, 181, 204, 36, 55, 47, 135, 140, 144, 230, 45, 173, 173, 93, 66, 160, 93, 42, 52, 28, 156, 139, 242, 60, 121, 90, 117, 99, 92, 182, 196, 25, 131, 16, 155, 186, 239, 137, 33, 118, 105, 232, 230, 239, 213, 240, 207, 6, 160, 59, 72, 35, 216, 124, 37, 62, 178, 34, 97, 180, 254, 212, 103, 179, 45, 247, 168, 205, 145, 7, 157, 75, 247, 83, 230, 233, 248, 97, 132, 232, 161, 160, 122, 167, 249, 196, 203, 2, 173, 180, 106, 203, 129, 214, 232, 181, 87, 39, 60, 99, 135, 6, 40, 34, 163, 118, 140, 149, 79, 241, 238, 230, 201, 194, 185, 1, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 32, 0, 0, 0, 0, 0, 0, 0, 0, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 32, 0, 0, 0, 0, 64, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 32, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 134, 36, 160, 31, 187, 182, 29, 131, 58, 212, 207, 131, 71, 168, 13, 131, 45, 60, 7, 132, 89, 53, 51, 233, 147, 69, 84, 67, 32, 101, 116, 104, 101, 114, 109, 105, 110, 101, 32, 45, 32, 69, 85, 49, 160, 204, 49, 229, 99, 26, 47, 30, 50, 223, 117, 111, 168, 102, 158, 12, 186, 140, 98, 193, 196, 214, 29, 13, 87, 44, 31, 216, 48, 250, 251, 148, 69, 136, 225, 206, 131, 96, 2, 39, 50, 46, 249, 95, 61, 248, 109, 1, 133, 4, 227, 178, 146, 0, 130, 82, 8, 148, 123, 45, 95, 28, 247, 139, 190, 196, 52, 39, 251, 53, 226, 79, 251, 1, 98, 34, 68, 240, 136, 18, 81, 1, 26, 180, 46, 168, 0, 128, 129, 157, 160, 31, 164, 62, 186, 38, 56, 118, 133, 24, 180, 239, 139, 254, 154, 196, 115, 8, 246, 45, 233, 227, 165, 192, 193, 7, 111, 1, 169, 2, 204, 2, 144, 160, 42, 60, 78, 200, 5, 113, 98, 65, 250, 105, 0, 164, 152, 81, 235, 154, 100, 204, 182, 141, 174, 39, 107, 127, 219, 120, 63, 221, 237, 87, 57, 9, 249, 94, 203, 52, 133, 2, 84, 11, 228, 0, 131, 44, 233, 255, 148, 137, 61, 196, 25, 119, 102, 53, 248, 253, 27, 31, 169, 147, 75, 245, 41, 174, 242, 86, 7, 128, 185, 94, 100, 194, 32, 154, 203, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 60, 131, 110, 76, 27, 36, 34, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 138, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 35, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 91, 96, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 92, 224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 248, 249, 1, 245, 160, 89, 97, 73, 3, 84, 63, 105, 163, 169, 7, 237, 254, 220, 243, 13, 27, 0, 45, 215, 39, 235, 232, 237, 22, 162, 83, 99, 164, 247, 240, 48, 66, 160, 29, 204, 77, 232, 222, 199, 93, 122, 171, 133, 181, 103, 182, 204, 212, 26, 211, 18, 69, 27, 148, 138, 116, 19, 240, 161, 66, 253, 64, 212, 147, 71, 148, 137, 61, 196, 25, 119, 102, 53, 248, 253, 27, 31, 169, 147, 75, 245, 41, 174, 242, 86, 7, 160, 243, 100, 253, 32, 5
		let block_rlp = UntrustedRlp::new(&block);
		let compressed = block_rlp.compress(RlpType::Blocks).to_vec();
		let compressed_rlp = UntrustedRlp::new(&compressed);
		assert_eq!(compressed_rlp.decompress(RlpType::Blocks).to_vec(), block);
	}
}