openethereum/util/rlp/src/untrusted_rlp.rs

406 lines
11 KiB
Rust
Raw Normal View History

// Copyright 2015-2017 Parity Technologies (UK) Ltd.
2016-02-05 13:40:41 +01:00
// This file is part of Parity.
// Parity is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// Parity is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
// You should have received a copy of the GNU General Public License
// along with Parity. If not, see <http://www.gnu.org/licenses/>.
2015-12-07 16:32:06 +01:00
use std::cell::Cell;
2016-01-15 21:03:38 +01:00
use std::fmt;
use rustc_serialize::hex::ToHex;
use impls::decode_usize;
use {Decodable, DecoderError};
2015-12-07 16:32:06 +01:00
/// rlp offset
#[derive(Copy, Clone, Debug)]
struct OffsetCache {
index: usize,
offset: usize,
}
impl OffsetCache {
fn new(index: usize, offset: usize) -> OffsetCache {
OffsetCache {
index: index,
offset: offset,
}
}
}
#[derive(Debug)]
2016-02-03 16:43:48 +01:00
/// RLP prototype
2015-12-07 16:32:06 +01:00
pub enum Prototype {
2016-02-03 16:43:48 +01:00
/// Empty
2015-12-07 16:32:06 +01:00
Null,
2016-02-03 16:43:48 +01:00
/// Value
2015-12-07 16:32:06 +01:00
Data(usize),
2016-02-03 16:43:48 +01:00
/// List
2015-12-07 16:32:06 +01:00
List(usize),
}
/// Stores basic information about item
pub struct PayloadInfo {
2016-02-03 16:43:48 +01:00
/// Header length in bytes
2015-12-07 16:32:06 +01:00
pub header_len: usize,
2016-02-03 16:43:48 +01:00
/// Value length in bytes
2015-12-07 16:32:06 +01:00
pub value_len: usize,
}
Blocks and snapshot compression (#1687) * new Compressible rlp trait * new Compressible rlp trait * make compressed rlp iterable * make compressed rlp iterable * invalid rlp slice swapper * switch compress to swapper, add reverse swapper test case * add basic account compression test * add new rlp trait * new Compressible rlp trait * make compressed rlp iterable * invalid rlp slice swapper * invalid rlp slice swapper * switch compress to swapper, add reverse swapper test case * switch compress to swapper, add reverse swapper test case * add account compress/ decompress test * make compressor cleaner, use hashmaps for swapper * improve compression tests * add a DecompressingDecoder, change Decoder to take refernce * separate rlp compression related stuff * new Compressible rlp trait * new Compressible rlp trait * new Compressible rlp trait * make compressed rlp iterable * make compressed rlp iterable * make compressed rlp iterable * invalid rlp slice swapper * invalid rlp slice swapper * invalid rlp slice swapper * switch compress to swapper, add reverse swapper test case * switch compress to swapper, add reverse swapper test case * switch compress to swapper, add reverse swapper test case * add basic account compression test * add new rlp trait * add account compress/ decompress test * make compressor cleaner, use hashmaps for swapper * improve compression tests * add a DecompressingDecoder, change Decoder to take refernce * separate rlp compression related stuff * DecompressingDecoder test * initial compressing HashDB wrapper * remove unused test * change CompressedDB to struct wrapper with overlay * simplify compressor * failed RefCell attempt * use denote to return reference * compiled compresseddb * compressdb test, add overlay emplace * fix overlay reference count handling * add immutable compresseddb, make account use hashdb * simplify using trait objects * enable hashdb for account * initial state compression attempt * wrap state db * add tests for analyzing db * add account predicate * try to compress data fields as rlp too * remove compression for storage trie * add a compressing migration * more compression stats tests * fix migration import * nested encoding compression test * fix decompression, move db stats tests to rlpcompression * added malformed rlp tests, cover a few edge cases * new CompressingEncoder struct * extend migrations to state * first version working on the whole db * clean up Compressible impl * tests cleanup * add a testing migration * refactor deep compression using option, add simple compression * put tests in a module * fix compressed overlay loading * simple compression for snapshots * remove unused DecompressingDecoder * add a general compressing migration * add more common rlps to compress * use static slices for swapper * add precomputed hashes and invalid rlps * make decoder private again * cover more cases with tests * style * fix weird indentation * remove possible panic in payload_info * make prefix checking safe * fix db existence check * remove db dir from test * pass usize by value [ci skip] * Improve comment on panic removal. * add common blocks db rlps * add compression to blockchain db * add blocks db migration * fix the migrations * remove state compression * add a separate snapshot swapper * ability to use different swappers and traversal * update tests to new interface * clean up code ordering * update usage * fix compilation * remove unnecessary changes * move methods to functions to reduce interface * move test to module * update common rlps to blocks db * move tests to tests modules * remove redundant &
2016-07-27 17:11:41 +02:00
fn calculate_payload_info(header_bytes: &[u8], len_of_len: usize) -> Result<PayloadInfo, DecoderError> {
let header_len = 1 + len_of_len;
match header_bytes.get(1) {
Some(&0) => return Err(DecoderError::RlpDataLenWithZeroPrefix),
None => return Err(DecoderError::RlpIsTooShort),
_ => (),
}
if header_bytes.len() < header_len { return Err(DecoderError::RlpIsTooShort); }
let value_len = decode_usize(&header_bytes[1..header_len])?;
Blocks and snapshot compression (#1687) * new Compressible rlp trait * new Compressible rlp trait * make compressed rlp iterable * make compressed rlp iterable * invalid rlp slice swapper * switch compress to swapper, add reverse swapper test case * add basic account compression test * add new rlp trait * new Compressible rlp trait * make compressed rlp iterable * invalid rlp slice swapper * invalid rlp slice swapper * switch compress to swapper, add reverse swapper test case * switch compress to swapper, add reverse swapper test case * add account compress/ decompress test * make compressor cleaner, use hashmaps for swapper * improve compression tests * add a DecompressingDecoder, change Decoder to take refernce * separate rlp compression related stuff * new Compressible rlp trait * new Compressible rlp trait * new Compressible rlp trait * make compressed rlp iterable * make compressed rlp iterable * make compressed rlp iterable * invalid rlp slice swapper * invalid rlp slice swapper * invalid rlp slice swapper * switch compress to swapper, add reverse swapper test case * switch compress to swapper, add reverse swapper test case * switch compress to swapper, add reverse swapper test case * add basic account compression test * add new rlp trait * add account compress/ decompress test * make compressor cleaner, use hashmaps for swapper * improve compression tests * add a DecompressingDecoder, change Decoder to take refernce * separate rlp compression related stuff * DecompressingDecoder test * initial compressing HashDB wrapper * remove unused test * change CompressedDB to struct wrapper with overlay * simplify compressor * failed RefCell attempt * use denote to return reference * compiled compresseddb * compressdb test, add overlay emplace * fix overlay reference count handling * add immutable compresseddb, make account use hashdb * simplify using trait objects * enable hashdb for account * initial state compression attempt * wrap state db * add tests for analyzing db * add account predicate * try to compress data fields as rlp too * remove compression for storage trie * add a compressing migration * more compression stats tests * fix migration import * nested encoding compression test * fix decompression, move db stats tests to rlpcompression * added malformed rlp tests, cover a few edge cases * new CompressingEncoder struct * extend migrations to state * first version working on the whole db * clean up Compressible impl * tests cleanup * add a testing migration * refactor deep compression using option, add simple compression * put tests in a module * fix compressed overlay loading * simple compression for snapshots * remove unused DecompressingDecoder * add a general compressing migration * add more common rlps to compress * use static slices for swapper * add precomputed hashes and invalid rlps * make decoder private again * cover more cases with tests * style * fix weird indentation * remove possible panic in payload_info * make prefix checking safe * fix db existence check * remove db dir from test * pass usize by value [ci skip] * Improve comment on panic removal. * add common blocks db rlps * add compression to blockchain db * add blocks db migration * fix the migrations * remove state compression * add a separate snapshot swapper * ability to use different swappers and traversal * update tests to new interface * clean up code ordering * update usage * fix compilation * remove unnecessary changes * move methods to functions to reduce interface * move test to module * update common rlps to blocks db * move tests to tests modules * remove redundant &
2016-07-27 17:11:41 +02:00
Ok(PayloadInfo::new(header_len, value_len))
}
2015-12-07 16:32:06 +01:00
impl PayloadInfo {
fn new(header_len: usize, value_len: usize) -> PayloadInfo {
PayloadInfo {
header_len: header_len,
value_len: value_len,
}
}
/// Total size of the RLP.
pub fn total(&self) -> usize { self.header_len + self.value_len }
/// Create a new object from the given bytes RLP. The bytes
pub fn from(header_bytes: &[u8]) -> Result<PayloadInfo, DecoderError> {
Blocks and snapshot compression (#1687) * new Compressible rlp trait * new Compressible rlp trait * make compressed rlp iterable * make compressed rlp iterable * invalid rlp slice swapper * switch compress to swapper, add reverse swapper test case * add basic account compression test * add new rlp trait * new Compressible rlp trait * make compressed rlp iterable * invalid rlp slice swapper * invalid rlp slice swapper * switch compress to swapper, add reverse swapper test case * switch compress to swapper, add reverse swapper test case * add account compress/ decompress test * make compressor cleaner, use hashmaps for swapper * improve compression tests * add a DecompressingDecoder, change Decoder to take refernce * separate rlp compression related stuff * new Compressible rlp trait * new Compressible rlp trait * new Compressible rlp trait * make compressed rlp iterable * make compressed rlp iterable * make compressed rlp iterable * invalid rlp slice swapper * invalid rlp slice swapper * invalid rlp slice swapper * switch compress to swapper, add reverse swapper test case * switch compress to swapper, add reverse swapper test case * switch compress to swapper, add reverse swapper test case * add basic account compression test * add new rlp trait * add account compress/ decompress test * make compressor cleaner, use hashmaps for swapper * improve compression tests * add a DecompressingDecoder, change Decoder to take refernce * separate rlp compression related stuff * DecompressingDecoder test * initial compressing HashDB wrapper * remove unused test * change CompressedDB to struct wrapper with overlay * simplify compressor * failed RefCell attempt * use denote to return reference * compiled compresseddb * compressdb test, add overlay emplace * fix overlay reference count handling * add immutable compresseddb, make account use hashdb * simplify using trait objects * enable hashdb for account * initial state compression attempt * wrap state db * add tests for analyzing db * add account predicate * try to compress data fields as rlp too * remove compression for storage trie * add a compressing migration * more compression stats tests * fix migration import * nested encoding compression test * fix decompression, move db stats tests to rlpcompression * added malformed rlp tests, cover a few edge cases * new CompressingEncoder struct * extend migrations to state * first version working on the whole db * clean up Compressible impl * tests cleanup * add a testing migration * refactor deep compression using option, add simple compression * put tests in a module * fix compressed overlay loading * simple compression for snapshots * remove unused DecompressingDecoder * add a general compressing migration * add more common rlps to compress * use static slices for swapper * add precomputed hashes and invalid rlps * make decoder private again * cover more cases with tests * style * fix weird indentation * remove possible panic in payload_info * make prefix checking safe * fix db existence check * remove db dir from test * pass usize by value [ci skip] * Improve comment on panic removal. * add common blocks db rlps * add compression to blockchain db * add blocks db migration * fix the migrations * remove state compression * add a separate snapshot swapper * ability to use different swappers and traversal * update tests to new interface * clean up code ordering * update usage * fix compilation * remove unnecessary changes * move methods to functions to reduce interface * move test to module * update common rlps to blocks db * move tests to tests modules * remove redundant &
2016-07-27 17:11:41 +02:00
match header_bytes.first().cloned() {
None => Err(DecoderError::RlpIsTooShort),
Some(0...0x7f) => Ok(PayloadInfo::new(0, 1)),
Some(l @ 0x80...0xb7) => Ok(PayloadInfo::new(1, l as usize - 0x80)),
Some(l @ 0xb8...0xbf) => {
let len_of_len = l as usize - 0xb7;
Blocks and snapshot compression (#1687) * new Compressible rlp trait * new Compressible rlp trait * make compressed rlp iterable * make compressed rlp iterable * invalid rlp slice swapper * switch compress to swapper, add reverse swapper test case * add basic account compression test * add new rlp trait * new Compressible rlp trait * make compressed rlp iterable * invalid rlp slice swapper * invalid rlp slice swapper * switch compress to swapper, add reverse swapper test case * switch compress to swapper, add reverse swapper test case * add account compress/ decompress test * make compressor cleaner, use hashmaps for swapper * improve compression tests * add a DecompressingDecoder, change Decoder to take refernce * separate rlp compression related stuff * new Compressible rlp trait * new Compressible rlp trait * new Compressible rlp trait * make compressed rlp iterable * make compressed rlp iterable * make compressed rlp iterable * invalid rlp slice swapper * invalid rlp slice swapper * invalid rlp slice swapper * switch compress to swapper, add reverse swapper test case * switch compress to swapper, add reverse swapper test case * switch compress to swapper, add reverse swapper test case * add basic account compression test * add new rlp trait * add account compress/ decompress test * make compressor cleaner, use hashmaps for swapper * improve compression tests * add a DecompressingDecoder, change Decoder to take refernce * separate rlp compression related stuff * DecompressingDecoder test * initial compressing HashDB wrapper * remove unused test * change CompressedDB to struct wrapper with overlay * simplify compressor * failed RefCell attempt * use denote to return reference * compiled compresseddb * compressdb test, add overlay emplace * fix overlay reference count handling * add immutable compresseddb, make account use hashdb * simplify using trait objects * enable hashdb for account * initial state compression attempt * wrap state db * add tests for analyzing db * add account predicate * try to compress data fields as rlp too * remove compression for storage trie * add a compressing migration * more compression stats tests * fix migration import * nested encoding compression test * fix decompression, move db stats tests to rlpcompression * added malformed rlp tests, cover a few edge cases * new CompressingEncoder struct * extend migrations to state * first version working on the whole db * clean up Compressible impl * tests cleanup * add a testing migration * refactor deep compression using option, add simple compression * put tests in a module * fix compressed overlay loading * simple compression for snapshots * remove unused DecompressingDecoder * add a general compressing migration * add more common rlps to compress * use static slices for swapper * add precomputed hashes and invalid rlps * make decoder private again * cover more cases with tests * style * fix weird indentation * remove possible panic in payload_info * make prefix checking safe * fix db existence check * remove db dir from test * pass usize by value [ci skip] * Improve comment on panic removal. * add common blocks db rlps * add compression to blockchain db * add blocks db migration * fix the migrations * remove state compression * add a separate snapshot swapper * ability to use different swappers and traversal * update tests to new interface * clean up code ordering * update usage * fix compilation * remove unnecessary changes * move methods to functions to reduce interface * move test to module * update common rlps to blocks db * move tests to tests modules * remove redundant &
2016-07-27 17:11:41 +02:00
calculate_payload_info(header_bytes, len_of_len)
}
Blocks and snapshot compression (#1687) * new Compressible rlp trait * new Compressible rlp trait * make compressed rlp iterable * make compressed rlp iterable * invalid rlp slice swapper * switch compress to swapper, add reverse swapper test case * add basic account compression test * add new rlp trait * new Compressible rlp trait * make compressed rlp iterable * invalid rlp slice swapper * invalid rlp slice swapper * switch compress to swapper, add reverse swapper test case * switch compress to swapper, add reverse swapper test case * add account compress/ decompress test * make compressor cleaner, use hashmaps for swapper * improve compression tests * add a DecompressingDecoder, change Decoder to take refernce * separate rlp compression related stuff * new Compressible rlp trait * new Compressible rlp trait * new Compressible rlp trait * make compressed rlp iterable * make compressed rlp iterable * make compressed rlp iterable * invalid rlp slice swapper * invalid rlp slice swapper * invalid rlp slice swapper * switch compress to swapper, add reverse swapper test case * switch compress to swapper, add reverse swapper test case * switch compress to swapper, add reverse swapper test case * add basic account compression test * add new rlp trait * add account compress/ decompress test * make compressor cleaner, use hashmaps for swapper * improve compression tests * add a DecompressingDecoder, change Decoder to take refernce * separate rlp compression related stuff * DecompressingDecoder test * initial compressing HashDB wrapper * remove unused test * change CompressedDB to struct wrapper with overlay * simplify compressor * failed RefCell attempt * use denote to return reference * compiled compresseddb * compressdb test, add overlay emplace * fix overlay reference count handling * add immutable compresseddb, make account use hashdb * simplify using trait objects * enable hashdb for account * initial state compression attempt * wrap state db * add tests for analyzing db * add account predicate * try to compress data fields as rlp too * remove compression for storage trie * add a compressing migration * more compression stats tests * fix migration import * nested encoding compression test * fix decompression, move db stats tests to rlpcompression * added malformed rlp tests, cover a few edge cases * new CompressingEncoder struct * extend migrations to state * first version working on the whole db * clean up Compressible impl * tests cleanup * add a testing migration * refactor deep compression using option, add simple compression * put tests in a module * fix compressed overlay loading * simple compression for snapshots * remove unused DecompressingDecoder * add a general compressing migration * add more common rlps to compress * use static slices for swapper * add precomputed hashes and invalid rlps * make decoder private again * cover more cases with tests * style * fix weird indentation * remove possible panic in payload_info * make prefix checking safe * fix db existence check * remove db dir from test * pass usize by value [ci skip] * Improve comment on panic removal. * add common blocks db rlps * add compression to blockchain db * add blocks db migration * fix the migrations * remove state compression * add a separate snapshot swapper * ability to use different swappers and traversal * update tests to new interface * clean up code ordering * update usage * fix compilation * remove unnecessary changes * move methods to functions to reduce interface * move test to module * update common rlps to blocks db * move tests to tests modules * remove redundant &
2016-07-27 17:11:41 +02:00
Some(l @ 0xc0...0xf7) => Ok(PayloadInfo::new(1, l as usize - 0xc0)),
Some(l @ 0xf8...0xff) => {
let len_of_len = l as usize - 0xf7;
Blocks and snapshot compression (#1687) * new Compressible rlp trait * new Compressible rlp trait * make compressed rlp iterable * make compressed rlp iterable * invalid rlp slice swapper * switch compress to swapper, add reverse swapper test case * add basic account compression test * add new rlp trait * new Compressible rlp trait * make compressed rlp iterable * invalid rlp slice swapper * invalid rlp slice swapper * switch compress to swapper, add reverse swapper test case * switch compress to swapper, add reverse swapper test case * add account compress/ decompress test * make compressor cleaner, use hashmaps for swapper * improve compression tests * add a DecompressingDecoder, change Decoder to take refernce * separate rlp compression related stuff * new Compressible rlp trait * new Compressible rlp trait * new Compressible rlp trait * make compressed rlp iterable * make compressed rlp iterable * make compressed rlp iterable * invalid rlp slice swapper * invalid rlp slice swapper * invalid rlp slice swapper * switch compress to swapper, add reverse swapper test case * switch compress to swapper, add reverse swapper test case * switch compress to swapper, add reverse swapper test case * add basic account compression test * add new rlp trait * add account compress/ decompress test * make compressor cleaner, use hashmaps for swapper * improve compression tests * add a DecompressingDecoder, change Decoder to take refernce * separate rlp compression related stuff * DecompressingDecoder test * initial compressing HashDB wrapper * remove unused test * change CompressedDB to struct wrapper with overlay * simplify compressor * failed RefCell attempt * use denote to return reference * compiled compresseddb * compressdb test, add overlay emplace * fix overlay reference count handling * add immutable compresseddb, make account use hashdb * simplify using trait objects * enable hashdb for account * initial state compression attempt * wrap state db * add tests for analyzing db * add account predicate * try to compress data fields as rlp too * remove compression for storage trie * add a compressing migration * more compression stats tests * fix migration import * nested encoding compression test * fix decompression, move db stats tests to rlpcompression * added malformed rlp tests, cover a few edge cases * new CompressingEncoder struct * extend migrations to state * first version working on the whole db * clean up Compressible impl * tests cleanup * add a testing migration * refactor deep compression using option, add simple compression * put tests in a module * fix compressed overlay loading * simple compression for snapshots * remove unused DecompressingDecoder * add a general compressing migration * add more common rlps to compress * use static slices for swapper * add precomputed hashes and invalid rlps * make decoder private again * cover more cases with tests * style * fix weird indentation * remove possible panic in payload_info * make prefix checking safe * fix db existence check * remove db dir from test * pass usize by value [ci skip] * Improve comment on panic removal. * add common blocks db rlps * add compression to blockchain db * add blocks db migration * fix the migrations * remove state compression * add a separate snapshot swapper * ability to use different swappers and traversal * update tests to new interface * clean up code ordering * update usage * fix compilation * remove unnecessary changes * move methods to functions to reduce interface * move test to module * update common rlps to blocks db * move tests to tests modules * remove redundant &
2016-07-27 17:11:41 +02:00
calculate_payload_info(header_bytes, len_of_len)
},
// we cant reach this place, but rust requires _ to be implemented
_ => { unreachable!(); }
Blocks and snapshot compression (#1687) * new Compressible rlp trait * new Compressible rlp trait * make compressed rlp iterable * make compressed rlp iterable * invalid rlp slice swapper * switch compress to swapper, add reverse swapper test case * add basic account compression test * add new rlp trait * new Compressible rlp trait * make compressed rlp iterable * invalid rlp slice swapper * invalid rlp slice swapper * switch compress to swapper, add reverse swapper test case * switch compress to swapper, add reverse swapper test case * add account compress/ decompress test * make compressor cleaner, use hashmaps for swapper * improve compression tests * add a DecompressingDecoder, change Decoder to take refernce * separate rlp compression related stuff * new Compressible rlp trait * new Compressible rlp trait * new Compressible rlp trait * make compressed rlp iterable * make compressed rlp iterable * make compressed rlp iterable * invalid rlp slice swapper * invalid rlp slice swapper * invalid rlp slice swapper * switch compress to swapper, add reverse swapper test case * switch compress to swapper, add reverse swapper test case * switch compress to swapper, add reverse swapper test case * add basic account compression test * add new rlp trait * add account compress/ decompress test * make compressor cleaner, use hashmaps for swapper * improve compression tests * add a DecompressingDecoder, change Decoder to take refernce * separate rlp compression related stuff * DecompressingDecoder test * initial compressing HashDB wrapper * remove unused test * change CompressedDB to struct wrapper with overlay * simplify compressor * failed RefCell attempt * use denote to return reference * compiled compresseddb * compressdb test, add overlay emplace * fix overlay reference count handling * add immutable compresseddb, make account use hashdb * simplify using trait objects * enable hashdb for account * initial state compression attempt * wrap state db * add tests for analyzing db * add account predicate * try to compress data fields as rlp too * remove compression for storage trie * add a compressing migration * more compression stats tests * fix migration import * nested encoding compression test * fix decompression, move db stats tests to rlpcompression * added malformed rlp tests, cover a few edge cases * new CompressingEncoder struct * extend migrations to state * first version working on the whole db * clean up Compressible impl * tests cleanup * add a testing migration * refactor deep compression using option, add simple compression * put tests in a module * fix compressed overlay loading * simple compression for snapshots * remove unused DecompressingDecoder * add a general compressing migration * add more common rlps to compress * use static slices for swapper * add precomputed hashes and invalid rlps * make decoder private again * cover more cases with tests * style * fix weird indentation * remove possible panic in payload_info * make prefix checking safe * fix db existence check * remove db dir from test * pass usize by value [ci skip] * Improve comment on panic removal. * add common blocks db rlps * add compression to blockchain db * add blocks db migration * fix the migrations * remove state compression * add a separate snapshot swapper * ability to use different swappers and traversal * update tests to new interface * clean up code ordering * update usage * fix compilation * remove unnecessary changes * move methods to functions to reduce interface * move test to module * update common rlps to blocks db * move tests to tests modules * remove redundant &
2016-07-27 17:11:41 +02:00
}
}
2015-12-07 16:32:06 +01:00
}
/// Data-oriented view onto rlp-slice.
2015-12-22 22:23:01 +01:00
///
2015-12-07 16:32:06 +01:00
/// This is immutable structere. No operations change it.
2015-12-22 22:23:01 +01:00
///
2015-12-07 16:32:06 +01:00
/// Should be used in places where, error handling is required,
/// eg. on input
#[derive(Debug)]
pub struct UntrustedRlp<'a> {
bytes: &'a [u8],
2015-12-22 22:23:01 +01:00
offset_cache: Cell<OffsetCache>,
count_cache: Cell<Option<usize>>,
2015-12-07 16:32:06 +01:00
}
2015-12-07 23:47:26 +01:00
impl<'a> Clone for UntrustedRlp<'a> {
fn clone(&self) -> UntrustedRlp<'a> {
UntrustedRlp {
bytes: self.bytes,
2015-12-22 22:23:01 +01:00
offset_cache: self.offset_cache.clone(),
count_cache: self.count_cache.clone(),
2015-12-07 23:47:26 +01:00
}
}
}
2016-01-15 21:03:38 +01:00
impl<'a> fmt::Display for UntrustedRlp<'a> {
fn fmt(&self, f: &mut fmt::Formatter) -> Result<(), fmt::Error> {
match self.prototype() {
Ok(Prototype::Null) => write!(f, "null"),
Ok(Prototype::Data(_)) => write!(f, "\"0x{}\"", self.data().unwrap().to_hex()),
Ok(Prototype::List(len)) => {
write!(f, "[")?;
2016-01-15 21:03:38 +01:00
for i in 0..len-1 {
write!(f, "{}, ", self.at(i).unwrap())?;
2016-01-15 21:03:38 +01:00
}
write!(f, "{}", self.at(len - 1).unwrap())?;
2016-01-15 21:03:38 +01:00
write!(f, "]")
},
Err(err) => write!(f, "{:?}", err)
}
}
}
impl<'a, 'view> UntrustedRlp<'a> where 'a: 'view {
pub fn new(bytes: &'a [u8]) -> UntrustedRlp<'a> {
2015-12-07 16:32:06 +01:00
UntrustedRlp {
bytes: bytes,
2015-12-22 22:23:01 +01:00
offset_cache: Cell::new(OffsetCache::new(usize::max_value(), 0)),
count_cache: Cell::new(None)
2015-12-07 16:32:06 +01:00
}
}
2015-12-22 22:23:01 +01:00
pub fn as_raw(&'view self) -> &'a [u8] {
2015-12-07 16:32:06 +01:00
self.bytes
}
pub fn prototype(&self) -> Result<Prototype, DecoderError> {
2015-12-07 23:47:26 +01:00
// optimize? && return appropriate errors
if self.is_data() {
Ok(Prototype::Data(self.size()))
} else if self.is_list() {
self.item_count().map(Prototype::List)
2015-12-07 23:47:26 +01:00
} else {
Ok(Prototype::Null)
}
2015-12-07 16:32:06 +01:00
}
pub fn payload_info(&self) -> Result<PayloadInfo, DecoderError> {
2015-12-07 23:47:26 +01:00
BasicDecoder::payload_info(self.bytes)
2015-12-07 16:32:06 +01:00
}
pub fn data(&'view self) -> Result<&'a [u8], DecoderError> {
let pi = BasicDecoder::payload_info(self.bytes)?;
2015-12-07 23:47:26 +01:00
Ok(&self.bytes[pi.header_len..(pi.header_len + pi.value_len)])
2015-12-07 16:32:06 +01:00
}
pub fn item_count(&self) -> Result<usize, DecoderError> {
2015-12-07 23:47:26 +01:00
match self.is_list() {
2015-12-22 22:23:01 +01:00
true => match self.count_cache.get() {
Some(c) => Ok(c),
2015-12-22 22:23:01 +01:00
None => {
let c = self.iter().count();
self.count_cache.set(Some(c));
Ok(c)
2015-12-22 22:23:01 +01:00
}
},
false => Err(DecoderError::RlpExpectedToBeList),
2015-12-07 23:47:26 +01:00
}
2015-12-07 16:32:06 +01:00
}
pub fn size(&self) -> usize {
2015-12-07 23:47:26 +01:00
match self.is_data() {
Blocks and snapshot compression (#1687) * new Compressible rlp trait * new Compressible rlp trait * make compressed rlp iterable * make compressed rlp iterable * invalid rlp slice swapper * switch compress to swapper, add reverse swapper test case * add basic account compression test * add new rlp trait * new Compressible rlp trait * make compressed rlp iterable * invalid rlp slice swapper * invalid rlp slice swapper * switch compress to swapper, add reverse swapper test case * switch compress to swapper, add reverse swapper test case * add account compress/ decompress test * make compressor cleaner, use hashmaps for swapper * improve compression tests * add a DecompressingDecoder, change Decoder to take refernce * separate rlp compression related stuff * new Compressible rlp trait * new Compressible rlp trait * new Compressible rlp trait * make compressed rlp iterable * make compressed rlp iterable * make compressed rlp iterable * invalid rlp slice swapper * invalid rlp slice swapper * invalid rlp slice swapper * switch compress to swapper, add reverse swapper test case * switch compress to swapper, add reverse swapper test case * switch compress to swapper, add reverse swapper test case * add basic account compression test * add new rlp trait * add account compress/ decompress test * make compressor cleaner, use hashmaps for swapper * improve compression tests * add a DecompressingDecoder, change Decoder to take refernce * separate rlp compression related stuff * DecompressingDecoder test * initial compressing HashDB wrapper * remove unused test * change CompressedDB to struct wrapper with overlay * simplify compressor * failed RefCell attempt * use denote to return reference * compiled compresseddb * compressdb test, add overlay emplace * fix overlay reference count handling * add immutable compresseddb, make account use hashdb * simplify using trait objects * enable hashdb for account * initial state compression attempt * wrap state db * add tests for analyzing db * add account predicate * try to compress data fields as rlp too * remove compression for storage trie * add a compressing migration * more compression stats tests * fix migration import * nested encoding compression test * fix decompression, move db stats tests to rlpcompression * added malformed rlp tests, cover a few edge cases * new CompressingEncoder struct * extend migrations to state * first version working on the whole db * clean up Compressible impl * tests cleanup * add a testing migration * refactor deep compression using option, add simple compression * put tests in a module * fix compressed overlay loading * simple compression for snapshots * remove unused DecompressingDecoder * add a general compressing migration * add more common rlps to compress * use static slices for swapper * add precomputed hashes and invalid rlps * make decoder private again * cover more cases with tests * style * fix weird indentation * remove possible panic in payload_info * make prefix checking safe * fix db existence check * remove db dir from test * pass usize by value [ci skip] * Improve comment on panic removal. * add common blocks db rlps * add compression to blockchain db * add blocks db migration * fix the migrations * remove state compression * add a separate snapshot swapper * ability to use different swappers and traversal * update tests to new interface * clean up code ordering * update usage * fix compilation * remove unnecessary changes * move methods to functions to reduce interface * move test to module * update common rlps to blocks db * move tests to tests modules * remove redundant &
2016-07-27 17:11:41 +02:00
// TODO: No panic on malformed data, but ideally would Err on no PayloadInfo.
true => BasicDecoder::payload_info(self.bytes).map(|b| b.value_len).unwrap_or(0),
2015-12-07 23:47:26 +01:00
false => 0
}
2015-12-07 16:32:06 +01:00
}
pub fn at(&'view self, index: usize) -> Result<UntrustedRlp<'a>, DecoderError> {
2015-12-07 23:47:26 +01:00
if !self.is_list() {
return Err(DecoderError::RlpExpectedToBeList);
}
2016-07-19 20:42:23 +02:00
// move to cached position if its index is less or equal to
2015-12-07 23:47:26 +01:00
// current search index, otherwise move to beginning of list
2015-12-22 22:23:01 +01:00
let c = self.offset_cache.get();
2015-12-07 23:47:26 +01:00
let (mut bytes, to_skip) = match c.index <= index {
true => (UntrustedRlp::consume(self.bytes, c.offset)?, index - c.index),
false => (self.consume_list_payload()?, index),
2015-12-07 23:47:26 +01:00
};
// skip up to x items
bytes = UntrustedRlp::consume_items(bytes, to_skip)?;
2015-12-07 23:47:26 +01:00
// update the cache
2015-12-22 22:23:01 +01:00
self.offset_cache.set(OffsetCache::new(index, self.bytes.len() - bytes.len()));
2015-12-07 23:47:26 +01:00
// construct new rlp
let found = BasicDecoder::payload_info(bytes)?;
2015-12-07 23:47:26 +01:00
Ok(UntrustedRlp::new(&bytes[0..found.header_len + found.value_len]))
2015-12-07 16:32:06 +01:00
}
pub fn is_null(&self) -> bool {
2015-12-07 23:47:26 +01:00
self.bytes.len() == 0
2015-12-07 16:32:06 +01:00
}
pub fn is_empty(&self) -> bool {
2015-12-07 23:47:26 +01:00
!self.is_null() && (self.bytes[0] == 0xc0 || self.bytes[0] == 0x80)
2015-12-07 16:32:06 +01:00
}
pub fn is_list(&self) -> bool {
2015-12-07 23:47:26 +01:00
!self.is_null() && self.bytes[0] >= 0xc0
2015-12-07 16:32:06 +01:00
}
pub fn is_data(&self) -> bool {
2015-12-07 23:47:26 +01:00
!self.is_null() && self.bytes[0] < 0xc0
2015-12-07 16:32:06 +01:00
}
pub fn is_int(&self) -> bool {
2015-12-07 23:47:26 +01:00
if self.is_null() {
return false;
}
match self.bytes[0] {
0...0x80 => true,
0x81...0xb7 => self.bytes[1] != 0,
b @ 0xb8...0xbf => self.bytes[1 + b as usize - 0xb7] != 0,
_ => false
}
}
pub fn iter(&'view self) -> UntrustedRlpIterator<'a, 'view> {
2015-12-07 23:47:26 +01:00
self.into_iter()
}
pub fn as_val<T>(&self) -> Result<T, DecoderError> where T: Decodable {
T::decode(self)
2015-12-07 23:47:26 +01:00
}
2015-12-08 14:44:22 +01:00
pub fn as_list<T>(&self) -> Result<Vec<T>, DecoderError> where T: Decodable {
self.iter().map(|rlp| rlp.as_val()).collect()
}
pub fn val_at<T>(&self, index: usize) -> Result<T, DecoderError> where T: Decodable {
self.at(index)?.as_val()
2015-12-08 14:44:22 +01:00
}
2015-12-07 23:47:26 +01:00
pub fn list_at<T>(&self, index: usize) -> Result<Vec<T>, DecoderError> where T: Decodable {
self.at(index)?.as_list()
}
pub fn decoder(&self) -> BasicDecoder {
BasicDecoder::new(self.clone())
}
2015-12-07 23:47:26 +01:00
/// consumes first found prefix
fn consume_list_payload(&self) -> Result<&'a [u8], DecoderError> {
let item = BasicDecoder::payload_info(self.bytes)?;
let bytes = UntrustedRlp::consume(self.bytes, item.header_len)?;
2015-12-07 23:47:26 +01:00
Ok(bytes)
}
/// consumes fixed number of items
fn consume_items(bytes: &'a [u8], items: usize) -> Result<&'a [u8], DecoderError> {
let mut result = bytes;
for _ in 0..items {
let i = BasicDecoder::payload_info(result)?;
result = UntrustedRlp::consume(result, (i.header_len + i.value_len))?;
2015-12-07 23:47:26 +01:00
}
Ok(result)
}
/// consumes slice prefix of length `len`
fn consume(bytes: &'a [u8], len: usize) -> Result<&'a [u8], DecoderError> {
match bytes.len() >= len {
true => Ok(&bytes[len..]),
false => Err(DecoderError::RlpIsTooShort),
}
2015-12-07 16:32:06 +01:00
}
}
2015-12-07 23:47:26 +01:00
/// Iterator over rlp-slice list elements.
pub struct UntrustedRlpIterator<'a, 'view> where 'a: 'view {
rlp: &'view UntrustedRlp<'a>,
index: usize,
}
impl<'a, 'view> IntoIterator for &'view UntrustedRlp<'a> where 'a: 'view {
type Item = UntrustedRlp<'a>;
type IntoIter = UntrustedRlpIterator<'a, 'view>;
fn into_iter(self) -> Self::IntoIter {
UntrustedRlpIterator {
rlp: self,
index: 0,
}
}
}
impl<'a, 'view> Iterator for UntrustedRlpIterator<'a, 'view> {
type Item = UntrustedRlp<'a>;
fn next(&mut self) -> Option<UntrustedRlp<'a>> {
let index = self.index;
let result = self.rlp.at(index).ok();
self.index += 1;
result
}
}
pub struct BasicDecoder<'a> {
2015-12-07 23:47:26 +01:00
rlp: UntrustedRlp<'a>
}
impl<'a> BasicDecoder<'a> {
pub fn new(rlp: UntrustedRlp<'a>) -> BasicDecoder<'a> {
BasicDecoder {
rlp: rlp
}
}
/// Return first item info.
2015-12-07 23:47:26 +01:00
fn payload_info(bytes: &[u8]) -> Result<PayloadInfo, DecoderError> {
let item = PayloadInfo::from(bytes)?;
2016-07-19 20:42:23 +02:00
match item.header_len.checked_add(item.value_len) {
Some(x) if x <= bytes.len() => Ok(item),
_ => Err(DecoderError::RlpIsTooShort),
2015-12-07 23:47:26 +01:00
}
}
pub fn decode_value<T, F>(&self, f: F) -> Result<T, DecoderError>
Blocks and snapshot compression (#1687) * new Compressible rlp trait * new Compressible rlp trait * make compressed rlp iterable * make compressed rlp iterable * invalid rlp slice swapper * switch compress to swapper, add reverse swapper test case * add basic account compression test * add new rlp trait * new Compressible rlp trait * make compressed rlp iterable * invalid rlp slice swapper * invalid rlp slice swapper * switch compress to swapper, add reverse swapper test case * switch compress to swapper, add reverse swapper test case * add account compress/ decompress test * make compressor cleaner, use hashmaps for swapper * improve compression tests * add a DecompressingDecoder, change Decoder to take refernce * separate rlp compression related stuff * new Compressible rlp trait * new Compressible rlp trait * new Compressible rlp trait * make compressed rlp iterable * make compressed rlp iterable * make compressed rlp iterable * invalid rlp slice swapper * invalid rlp slice swapper * invalid rlp slice swapper * switch compress to swapper, add reverse swapper test case * switch compress to swapper, add reverse swapper test case * switch compress to swapper, add reverse swapper test case * add basic account compression test * add new rlp trait * add account compress/ decompress test * make compressor cleaner, use hashmaps for swapper * improve compression tests * add a DecompressingDecoder, change Decoder to take refernce * separate rlp compression related stuff * DecompressingDecoder test * initial compressing HashDB wrapper * remove unused test * change CompressedDB to struct wrapper with overlay * simplify compressor * failed RefCell attempt * use denote to return reference * compiled compresseddb * compressdb test, add overlay emplace * fix overlay reference count handling * add immutable compresseddb, make account use hashdb * simplify using trait objects * enable hashdb for account * initial state compression attempt * wrap state db * add tests for analyzing db * add account predicate * try to compress data fields as rlp too * remove compression for storage trie * add a compressing migration * more compression stats tests * fix migration import * nested encoding compression test * fix decompression, move db stats tests to rlpcompression * added malformed rlp tests, cover a few edge cases * new CompressingEncoder struct * extend migrations to state * first version working on the whole db * clean up Compressible impl * tests cleanup * add a testing migration * refactor deep compression using option, add simple compression * put tests in a module * fix compressed overlay loading * simple compression for snapshots * remove unused DecompressingDecoder * add a general compressing migration * add more common rlps to compress * use static slices for swapper * add precomputed hashes and invalid rlps * make decoder private again * cover more cases with tests * style * fix weird indentation * remove possible panic in payload_info * make prefix checking safe * fix db existence check * remove db dir from test * pass usize by value [ci skip] * Improve comment on panic removal. * add common blocks db rlps * add compression to blockchain db * add blocks db migration * fix the migrations * remove state compression * add a separate snapshot swapper * ability to use different swappers and traversal * update tests to new interface * clean up code ordering * update usage * fix compilation * remove unnecessary changes * move methods to functions to reduce interface * move test to module * update common rlps to blocks db * move tests to tests modules * remove redundant &
2016-07-27 17:11:41 +02:00
where F: Fn(&[u8]) -> Result<T, DecoderError> {
2015-12-07 23:47:26 +01:00
2016-01-08 15:52:43 +01:00
let bytes = self.rlp.as_raw();
2015-12-07 23:47:26 +01:00
2016-01-19 12:14:29 +01:00
match bytes.first().cloned() {
Blocks and snapshot compression (#1687) * new Compressible rlp trait * new Compressible rlp trait * make compressed rlp iterable * make compressed rlp iterable * invalid rlp slice swapper * switch compress to swapper, add reverse swapper test case * add basic account compression test * add new rlp trait * new Compressible rlp trait * make compressed rlp iterable * invalid rlp slice swapper * invalid rlp slice swapper * switch compress to swapper, add reverse swapper test case * switch compress to swapper, add reverse swapper test case * add account compress/ decompress test * make compressor cleaner, use hashmaps for swapper * improve compression tests * add a DecompressingDecoder, change Decoder to take refernce * separate rlp compression related stuff * new Compressible rlp trait * new Compressible rlp trait * new Compressible rlp trait * make compressed rlp iterable * make compressed rlp iterable * make compressed rlp iterable * invalid rlp slice swapper * invalid rlp slice swapper * invalid rlp slice swapper * switch compress to swapper, add reverse swapper test case * switch compress to swapper, add reverse swapper test case * switch compress to swapper, add reverse swapper test case * add basic account compression test * add new rlp trait * add account compress/ decompress test * make compressor cleaner, use hashmaps for swapper * improve compression tests * add a DecompressingDecoder, change Decoder to take refernce * separate rlp compression related stuff * DecompressingDecoder test * initial compressing HashDB wrapper * remove unused test * change CompressedDB to struct wrapper with overlay * simplify compressor * failed RefCell attempt * use denote to return reference * compiled compresseddb * compressdb test, add overlay emplace * fix overlay reference count handling * add immutable compresseddb, make account use hashdb * simplify using trait objects * enable hashdb for account * initial state compression attempt * wrap state db * add tests for analyzing db * add account predicate * try to compress data fields as rlp too * remove compression for storage trie * add a compressing migration * more compression stats tests * fix migration import * nested encoding compression test * fix decompression, move db stats tests to rlpcompression * added malformed rlp tests, cover a few edge cases * new CompressingEncoder struct * extend migrations to state * first version working on the whole db * clean up Compressible impl * tests cleanup * add a testing migration * refactor deep compression using option, add simple compression * put tests in a module * fix compressed overlay loading * simple compression for snapshots * remove unused DecompressingDecoder * add a general compressing migration * add more common rlps to compress * use static slices for swapper * add precomputed hashes and invalid rlps * make decoder private again * cover more cases with tests * style * fix weird indentation * remove possible panic in payload_info * make prefix checking safe * fix db existence check * remove db dir from test * pass usize by value [ci skip] * Improve comment on panic removal. * add common blocks db rlps * add compression to blockchain db * add blocks db migration * fix the migrations * remove state compression * add a separate snapshot swapper * ability to use different swappers and traversal * update tests to new interface * clean up code ordering * update usage * fix compilation * remove unnecessary changes * move methods to functions to reduce interface * move test to module * update common rlps to blocks db * move tests to tests modules * remove redundant &
2016-07-27 17:11:41 +02:00
// RLP is too short.
2015-12-07 23:47:26 +01:00
None => Err(DecoderError::RlpIsTooShort),
Blocks and snapshot compression (#1687) * new Compressible rlp trait * new Compressible rlp trait * make compressed rlp iterable * make compressed rlp iterable * invalid rlp slice swapper * switch compress to swapper, add reverse swapper test case * add basic account compression test * add new rlp trait * new Compressible rlp trait * make compressed rlp iterable * invalid rlp slice swapper * invalid rlp slice swapper * switch compress to swapper, add reverse swapper test case * switch compress to swapper, add reverse swapper test case * add account compress/ decompress test * make compressor cleaner, use hashmaps for swapper * improve compression tests * add a DecompressingDecoder, change Decoder to take refernce * separate rlp compression related stuff * new Compressible rlp trait * new Compressible rlp trait * new Compressible rlp trait * make compressed rlp iterable * make compressed rlp iterable * make compressed rlp iterable * invalid rlp slice swapper * invalid rlp slice swapper * invalid rlp slice swapper * switch compress to swapper, add reverse swapper test case * switch compress to swapper, add reverse swapper test case * switch compress to swapper, add reverse swapper test case * add basic account compression test * add new rlp trait * add account compress/ decompress test * make compressor cleaner, use hashmaps for swapper * improve compression tests * add a DecompressingDecoder, change Decoder to take refernce * separate rlp compression related stuff * DecompressingDecoder test * initial compressing HashDB wrapper * remove unused test * change CompressedDB to struct wrapper with overlay * simplify compressor * failed RefCell attempt * use denote to return reference * compiled compresseddb * compressdb test, add overlay emplace * fix overlay reference count handling * add immutable compresseddb, make account use hashdb * simplify using trait objects * enable hashdb for account * initial state compression attempt * wrap state db * add tests for analyzing db * add account predicate * try to compress data fields as rlp too * remove compression for storage trie * add a compressing migration * more compression stats tests * fix migration import * nested encoding compression test * fix decompression, move db stats tests to rlpcompression * added malformed rlp tests, cover a few edge cases * new CompressingEncoder struct * extend migrations to state * first version working on the whole db * clean up Compressible impl * tests cleanup * add a testing migration * refactor deep compression using option, add simple compression * put tests in a module * fix compressed overlay loading * simple compression for snapshots * remove unused DecompressingDecoder * add a general compressing migration * add more common rlps to compress * use static slices for swapper * add precomputed hashes and invalid rlps * make decoder private again * cover more cases with tests * style * fix weird indentation * remove possible panic in payload_info * make prefix checking safe * fix db existence check * remove db dir from test * pass usize by value [ci skip] * Improve comment on panic removal. * add common blocks db rlps * add compression to blockchain db * add blocks db migration * fix the migrations * remove state compression * add a separate snapshot swapper * ability to use different swappers and traversal * update tests to new interface * clean up code ordering * update usage * fix compilation * remove unnecessary changes * move methods to functions to reduce interface * move test to module * update common rlps to blocks db * move tests to tests modules * remove redundant &
2016-07-27 17:11:41 +02:00
// Single byte value.
Some(l @ 0...0x7f) => Ok(f(&[l])?),
2015-12-07 23:47:26 +01:00
// 0-55 bytes
Some(l @ 0x80...0xb7) => {
2016-01-26 11:57:43 +01:00
let last_index_of = 1 + l as usize - 0x80;
2016-01-25 13:23:05 +01:00
if bytes.len() < last_index_of {
2016-01-25 14:27:11 +01:00
return Err(DecoderError::RlpInconsistentLengthAndData);
2016-01-25 13:23:05 +01:00
}
let d = &bytes[1..last_index_of];
if l == 0x81 && d[0] < 0x80 {
return Err(DecoderError::RlpInvalidIndirection);
}
Ok(f(d)?)
},
Blocks and snapshot compression (#1687) * new Compressible rlp trait * new Compressible rlp trait * make compressed rlp iterable * make compressed rlp iterable * invalid rlp slice swapper * switch compress to swapper, add reverse swapper test case * add basic account compression test * add new rlp trait * new Compressible rlp trait * make compressed rlp iterable * invalid rlp slice swapper * invalid rlp slice swapper * switch compress to swapper, add reverse swapper test case * switch compress to swapper, add reverse swapper test case * add account compress/ decompress test * make compressor cleaner, use hashmaps for swapper * improve compression tests * add a DecompressingDecoder, change Decoder to take refernce * separate rlp compression related stuff * new Compressible rlp trait * new Compressible rlp trait * new Compressible rlp trait * make compressed rlp iterable * make compressed rlp iterable * make compressed rlp iterable * invalid rlp slice swapper * invalid rlp slice swapper * invalid rlp slice swapper * switch compress to swapper, add reverse swapper test case * switch compress to swapper, add reverse swapper test case * switch compress to swapper, add reverse swapper test case * add basic account compression test * add new rlp trait * add account compress/ decompress test * make compressor cleaner, use hashmaps for swapper * improve compression tests * add a DecompressingDecoder, change Decoder to take refernce * separate rlp compression related stuff * DecompressingDecoder test * initial compressing HashDB wrapper * remove unused test * change CompressedDB to struct wrapper with overlay * simplify compressor * failed RefCell attempt * use denote to return reference * compiled compresseddb * compressdb test, add overlay emplace * fix overlay reference count handling * add immutable compresseddb, make account use hashdb * simplify using trait objects * enable hashdb for account * initial state compression attempt * wrap state db * add tests for analyzing db * add account predicate * try to compress data fields as rlp too * remove compression for storage trie * add a compressing migration * more compression stats tests * fix migration import * nested encoding compression test * fix decompression, move db stats tests to rlpcompression * added malformed rlp tests, cover a few edge cases * new CompressingEncoder struct * extend migrations to state * first version working on the whole db * clean up Compressible impl * tests cleanup * add a testing migration * refactor deep compression using option, add simple compression * put tests in a module * fix compressed overlay loading * simple compression for snapshots * remove unused DecompressingDecoder * add a general compressing migration * add more common rlps to compress * use static slices for swapper * add precomputed hashes and invalid rlps * make decoder private again * cover more cases with tests * style * fix weird indentation * remove possible panic in payload_info * make prefix checking safe * fix db existence check * remove db dir from test * pass usize by value [ci skip] * Improve comment on panic removal. * add common blocks db rlps * add compression to blockchain db * add blocks db migration * fix the migrations * remove state compression * add a separate snapshot swapper * ability to use different swappers and traversal * update tests to new interface * clean up code ordering * update usage * fix compilation * remove unnecessary changes * move methods to functions to reduce interface * move test to module * update common rlps to blocks db * move tests to tests modules * remove redundant &
2016-07-27 17:11:41 +02:00
// Longer than 55 bytes.
2015-12-07 23:47:26 +01:00
Some(l @ 0xb8...0xbf) => {
let len_of_len = l as usize - 0xb7;
let begin_of_value = 1 as usize + len_of_len;
2016-01-25 14:27:11 +01:00
if bytes.len() < begin_of_value {
return Err(DecoderError::RlpInconsistentLengthAndData);
}
let len = decode_usize(&bytes[1..begin_of_value])?;
2016-01-25 14:27:11 +01:00
let last_index_of_value = begin_of_value + len;
if bytes.len() < last_index_of_value {
return Err(DecoderError::RlpInconsistentLengthAndData);
}
Ok(f(&bytes[begin_of_value..last_index_of_value])?)
2015-12-07 23:47:26 +01:00
}
Blocks and snapshot compression (#1687) * new Compressible rlp trait * new Compressible rlp trait * make compressed rlp iterable * make compressed rlp iterable * invalid rlp slice swapper * switch compress to swapper, add reverse swapper test case * add basic account compression test * add new rlp trait * new Compressible rlp trait * make compressed rlp iterable * invalid rlp slice swapper * invalid rlp slice swapper * switch compress to swapper, add reverse swapper test case * switch compress to swapper, add reverse swapper test case * add account compress/ decompress test * make compressor cleaner, use hashmaps for swapper * improve compression tests * add a DecompressingDecoder, change Decoder to take refernce * separate rlp compression related stuff * new Compressible rlp trait * new Compressible rlp trait * new Compressible rlp trait * make compressed rlp iterable * make compressed rlp iterable * make compressed rlp iterable * invalid rlp slice swapper * invalid rlp slice swapper * invalid rlp slice swapper * switch compress to swapper, add reverse swapper test case * switch compress to swapper, add reverse swapper test case * switch compress to swapper, add reverse swapper test case * add basic account compression test * add new rlp trait * add account compress/ decompress test * make compressor cleaner, use hashmaps for swapper * improve compression tests * add a DecompressingDecoder, change Decoder to take refernce * separate rlp compression related stuff * DecompressingDecoder test * initial compressing HashDB wrapper * remove unused test * change CompressedDB to struct wrapper with overlay * simplify compressor * failed RefCell attempt * use denote to return reference * compiled compresseddb * compressdb test, add overlay emplace * fix overlay reference count handling * add immutable compresseddb, make account use hashdb * simplify using trait objects * enable hashdb for account * initial state compression attempt * wrap state db * add tests for analyzing db * add account predicate * try to compress data fields as rlp too * remove compression for storage trie * add a compressing migration * more compression stats tests * fix migration import * nested encoding compression test * fix decompression, move db stats tests to rlpcompression * added malformed rlp tests, cover a few edge cases * new CompressingEncoder struct * extend migrations to state * first version working on the whole db * clean up Compressible impl * tests cleanup * add a testing migration * refactor deep compression using option, add simple compression * put tests in a module * fix compressed overlay loading * simple compression for snapshots * remove unused DecompressingDecoder * add a general compressing migration * add more common rlps to compress * use static slices for swapper * add precomputed hashes and invalid rlps * make decoder private again * cover more cases with tests * style * fix weird indentation * remove possible panic in payload_info * make prefix checking safe * fix db existence check * remove db dir from test * pass usize by value [ci skip] * Improve comment on panic removal. * add common blocks db rlps * add compression to blockchain db * add blocks db migration * fix the migrations * remove state compression * add a separate snapshot swapper * ability to use different swappers and traversal * update tests to new interface * clean up code ordering * update usage * fix compilation * remove unnecessary changes * move methods to functions to reduce interface * move test to module * update common rlps to blocks db * move tests to tests modules * remove redundant &
2016-07-27 17:11:41 +02:00
// We are reading value, not a list!
_ => Err(DecoderError::RlpExpectedToBeData)
2015-12-07 23:47:26 +01:00
}
}
2016-01-29 13:59:29 +01:00
}
Blocks and snapshot compression (#1687) * new Compressible rlp trait * new Compressible rlp trait * make compressed rlp iterable * make compressed rlp iterable * invalid rlp slice swapper * switch compress to swapper, add reverse swapper test case * add basic account compression test * add new rlp trait * new Compressible rlp trait * make compressed rlp iterable * invalid rlp slice swapper * invalid rlp slice swapper * switch compress to swapper, add reverse swapper test case * switch compress to swapper, add reverse swapper test case * add account compress/ decompress test * make compressor cleaner, use hashmaps for swapper * improve compression tests * add a DecompressingDecoder, change Decoder to take refernce * separate rlp compression related stuff * new Compressible rlp trait * new Compressible rlp trait * new Compressible rlp trait * make compressed rlp iterable * make compressed rlp iterable * make compressed rlp iterable * invalid rlp slice swapper * invalid rlp slice swapper * invalid rlp slice swapper * switch compress to swapper, add reverse swapper test case * switch compress to swapper, add reverse swapper test case * switch compress to swapper, add reverse swapper test case * add basic account compression test * add new rlp trait * add account compress/ decompress test * make compressor cleaner, use hashmaps for swapper * improve compression tests * add a DecompressingDecoder, change Decoder to take refernce * separate rlp compression related stuff * DecompressingDecoder test * initial compressing HashDB wrapper * remove unused test * change CompressedDB to struct wrapper with overlay * simplify compressor * failed RefCell attempt * use denote to return reference * compiled compresseddb * compressdb test, add overlay emplace * fix overlay reference count handling * add immutable compresseddb, make account use hashdb * simplify using trait objects * enable hashdb for account * initial state compression attempt * wrap state db * add tests for analyzing db * add account predicate * try to compress data fields as rlp too * remove compression for storage trie * add a compressing migration * more compression stats tests * fix migration import * nested encoding compression test * fix decompression, move db stats tests to rlpcompression * added malformed rlp tests, cover a few edge cases * new CompressingEncoder struct * extend migrations to state * first version working on the whole db * clean up Compressible impl * tests cleanup * add a testing migration * refactor deep compression using option, add simple compression * put tests in a module * fix compressed overlay loading * simple compression for snapshots * remove unused DecompressingDecoder * add a general compressing migration * add more common rlps to compress * use static slices for swapper * add precomputed hashes and invalid rlps * make decoder private again * cover more cases with tests * style * fix weird indentation * remove possible panic in payload_info * make prefix checking safe * fix db existence check * remove db dir from test * pass usize by value [ci skip] * Improve comment on panic removal. * add common blocks db rlps * add compression to blockchain db * add blocks db migration * fix the migrations * remove state compression * add a separate snapshot swapper * ability to use different swappers and traversal * update tests to new interface * clean up code ordering * update usage * fix compilation * remove unnecessary changes * move methods to functions to reduce interface * move test to module * update common rlps to blocks db * move tests to tests modules * remove redundant &
2016-07-27 17:11:41 +02:00
#[cfg(test)]
mod tests {
use UntrustedRlp;
Blocks and snapshot compression (#1687) * new Compressible rlp trait * new Compressible rlp trait * make compressed rlp iterable * make compressed rlp iterable * invalid rlp slice swapper * switch compress to swapper, add reverse swapper test case * add basic account compression test * add new rlp trait * new Compressible rlp trait * make compressed rlp iterable * invalid rlp slice swapper * invalid rlp slice swapper * switch compress to swapper, add reverse swapper test case * switch compress to swapper, add reverse swapper test case * add account compress/ decompress test * make compressor cleaner, use hashmaps for swapper * improve compression tests * add a DecompressingDecoder, change Decoder to take refernce * separate rlp compression related stuff * new Compressible rlp trait * new Compressible rlp trait * new Compressible rlp trait * make compressed rlp iterable * make compressed rlp iterable * make compressed rlp iterable * invalid rlp slice swapper * invalid rlp slice swapper * invalid rlp slice swapper * switch compress to swapper, add reverse swapper test case * switch compress to swapper, add reverse swapper test case * switch compress to swapper, add reverse swapper test case * add basic account compression test * add new rlp trait * add account compress/ decompress test * make compressor cleaner, use hashmaps for swapper * improve compression tests * add a DecompressingDecoder, change Decoder to take refernce * separate rlp compression related stuff * DecompressingDecoder test * initial compressing HashDB wrapper * remove unused test * change CompressedDB to struct wrapper with overlay * simplify compressor * failed RefCell attempt * use denote to return reference * compiled compresseddb * compressdb test, add overlay emplace * fix overlay reference count handling * add immutable compresseddb, make account use hashdb * simplify using trait objects * enable hashdb for account * initial state compression attempt * wrap state db * add tests for analyzing db * add account predicate * try to compress data fields as rlp too * remove compression for storage trie * add a compressing migration * more compression stats tests * fix migration import * nested encoding compression test * fix decompression, move db stats tests to rlpcompression * added malformed rlp tests, cover a few edge cases * new CompressingEncoder struct * extend migrations to state * first version working on the whole db * clean up Compressible impl * tests cleanup * add a testing migration * refactor deep compression using option, add simple compression * put tests in a module * fix compressed overlay loading * simple compression for snapshots * remove unused DecompressingDecoder * add a general compressing migration * add more common rlps to compress * use static slices for swapper * add precomputed hashes and invalid rlps * make decoder private again * cover more cases with tests * style * fix weird indentation * remove possible panic in payload_info * make prefix checking safe * fix db existence check * remove db dir from test * pass usize by value [ci skip] * Improve comment on panic removal. * add common blocks db rlps * add compression to blockchain db * add blocks db migration * fix the migrations * remove state compression * add a separate snapshot swapper * ability to use different swappers and traversal * update tests to new interface * clean up code ordering * update usage * fix compilation * remove unnecessary changes * move methods to functions to reduce interface * move test to module * update common rlps to blocks db * move tests to tests modules * remove redundant &
2016-07-27 17:11:41 +02:00
#[test]
fn test_rlp_display() {
use rustc_serialize::hex::FromHex;
let data = "f84d0589010efbef67941f79b2a056e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421a0c5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470".from_hex().unwrap();
let rlp = UntrustedRlp::new(&data);
assert_eq!(format!("{}", rlp), "[\"0x05\", \"0x010efbef67941f79b2\", \"0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421\", \"0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470\"]");
}
2016-01-15 21:03:38 +01:00
}