Merge pull request #966 from ethcore/from-bytes-extend

Addressing binary serialization for db types
2016-04-20 18:17:00 +02:00 · 2016-04-20 18:17:00 +02:00 · e47af7f745
commit e47af7f745
parent 225a5ee825 bde0a5b811
6 changed files with 294 additions and 38 deletions
--- a/ethcore/src/blockchain/block_info.rs
+++ b/ethcore/src/blockchain/block_info.rs
@ -17,6 +17,8 @@
 use util::numbers::{U256,H256};
 use header::BlockNumber;

+use util::bytes::{FromRawBytesVariable, FromBytesError, ToBytesWithMap};
+
 /// Brief info about inserted block.
 #[derive(Clone)]
 pub struct BlockInfo {
@ -40,12 +42,55 @@ pub enum BlockLocation {
 	/// It's part of the fork which should become canon chain,
 	/// because it's total difficulty is higher than current
 	/// canon chain difficulty.
-	BranchBecomingCanonChain {
-		/// Hash of the newest common ancestor with old canon chain.
-		ancestor: H256,
-		/// Hashes of the blocks between ancestor and this block.
-		enacted: Vec<H256>,
-		/// Hashes of the blocks which were invalidated.
-		retracted: Vec<H256>,
+	BranchBecomingCanonChain(BranchBecomingCanonChainData),
+}
+
+#[derive(Clone)]
+pub struct BranchBecomingCanonChainData {
+	/// Hash of the newest common ancestor with old canon chain.
+	pub ancestor: H256,
+	/// Hashes of the blocks between ancestor and this block.
+	pub enacted: Vec<H256>,
+	/// Hashes of the blocks which were invalidated.
+	pub retracted: Vec<H256>,
+}
+
+impl FromRawBytesVariable for BranchBecomingCanonChainData {
+	fn from_bytes_variable(bytes: &[u8]) -> Result<BranchBecomingCanonChainData, FromBytesError> {
+		type Tuple = (Vec<H256>, Vec<H256>, H256);
+		let (enacted, retracted, ancestor) = try!(Tuple::from_bytes_variable(bytes));
+		Ok(BranchBecomingCanonChainData { ancestor: ancestor, enacted: enacted, retracted: retracted })
+	}
+}
+
+impl FromRawBytesVariable for BlockLocation {
+	fn from_bytes_variable(bytes: &[u8]) -> Result<BlockLocation, FromBytesError> {
+		match bytes[0] {
+			0 => Ok(BlockLocation::CanonChain),
+			1 => Ok(BlockLocation::Branch),
+			2 => Ok(BlockLocation::BranchBecomingCanonChain(
+				try!(BranchBecomingCanonChainData::from_bytes_variable(&bytes[1..bytes.len()])))),
+			_ => Err(FromBytesError::UnknownMarker)
+		}
+	}
+}
+
+impl ToBytesWithMap for BranchBecomingCanonChainData {
+	fn to_bytes_map(&self) -> Vec<u8> {
+		(&self.enacted, &self.retracted, &self.ancestor).to_bytes_map()
+	}
+}
+
+impl ToBytesWithMap for BlockLocation {
+	fn to_bytes_map(&self) -> Vec<u8> {
+		match *self {
+			BlockLocation::CanonChain => vec![0u8],
+			BlockLocation::Branch => vec![1u8],
+			BlockLocation::BranchBecomingCanonChain(ref data) => {
+				let mut bytes = (&data.enacted, &data.retracted, &data.ancestor).to_bytes_map();
+				bytes.insert(0, 2u8);
+				bytes
+			}
+		}
 	}
 }
--- a/ethcore/src/blockchain/blockchain.rs
+++ b/ethcore/src/blockchain/blockchain.rs
@ -24,7 +24,7 @@ use transaction::*;
 use views::*;
 use receipt::Receipt;
 use chainfilter::{ChainFilter, BloomIndex, FilterDataSource};
-use blockchain::block_info::{BlockInfo, BlockLocation};
+use blockchain::block_info::{BlockInfo, BlockLocation, BranchBecomingCanonChainData};
 use blockchain::best_block::BestBlock;
 use blockchain::bloom_indexer::BloomIndexer;
 use blockchain::tree_route::TreeRoute;
@ -583,11 +583,11 @@ impl BlockChain {
 					_ => {
 						let retracted = route.blocks.iter().take(route.index).cloned().collect::<Vec<H256>>();

-						BlockLocation::BranchBecomingCanonChain {
+						BlockLocation::BranchBecomingCanonChain(BranchBecomingCanonChainData {
 							ancestor: route.ancestor,
 							enacted: route.blocks.into_iter().skip(route.index).collect(),
 							retracted: retracted.into_iter().rev().collect(),
-						}
+						})
 					}
 				}
 			} else {
@ -608,11 +608,11 @@ impl BlockChain {
 			BlockLocation::CanonChain => {
 				block_hashes.insert(number, info.hash.clone());
 			},
-			BlockLocation::BranchBecomingCanonChain { ref ancestor, ref enacted, .. } => {
-				let ancestor_number = self.block_number(ancestor).unwrap();
+			BlockLocation::BranchBecomingCanonChain(ref data) => {
+				let ancestor_number = self.block_number(&data.ancestor).unwrap();
 				let start_number = ancestor_number + 1;

-				for (index, hash) in enacted.iter().cloned().enumerate() {
+				for (index, hash) in data.enacted.iter().cloned().enumerate() {
 					block_hashes.insert(start_number + index as BlockNumber, hash);
 				}

@ -697,11 +697,11 @@ impl BlockChain {
 				ChainFilter::new(self, self.bloom_indexer.index_size(), self.bloom_indexer.levels())
 					.add_bloom(&header.log_bloom(), header.number() as usize)
 			},
-			BlockLocation::BranchBecomingCanonChain { ref ancestor, ref enacted, .. } => {
-				let ancestor_number = self.block_number(ancestor).unwrap();
+			BlockLocation::BranchBecomingCanonChain(ref data) => {
+				let ancestor_number = self.block_number(&data.ancestor).unwrap();
 				let start_number = ancestor_number + 1;

-				let mut blooms: Vec<H2048> = enacted.iter()
+				let mut blooms: Vec<H2048> = data.enacted.iter()
 					.map(|hash| self.block(hash).unwrap())
 					.map(|bytes| BlockView::new(&bytes).header_view().log_bloom())
 					.collect();
--- a/ethcore/src/blockchain/import_route.rs
+++ b/ethcore/src/blockchain/import_route.rs
@ -45,11 +45,11 @@ impl From<BlockInfo> for ImportRoute {
 				enacted: vec![info.hash],
 			},
 			BlockLocation::Branch => ImportRoute::none(),
-			BlockLocation::BranchBecomingCanonChain { mut enacted, retracted, .. } => {
-				enacted.push(info.hash);
+			BlockLocation::BranchBecomingCanonChain(mut data) => {
+				data.enacted.push(info.hash);
 				ImportRoute {
-					retracted: retracted,
-					enacted: enacted,
+					retracted: data.retracted,
+					enacted: data.enacted,
 				}
 			}
 		}
@ -60,7 +60,7 @@ impl From<BlockInfo> for ImportRoute {
 mod tests {
 	use util::hash::H256;
 	use util::numbers::U256;
-	use blockchain::block_info::{BlockInfo, BlockLocation};
+	use blockchain::block_info::{BlockInfo, BlockLocation, BranchBecomingCanonChainData};
 	use blockchain::ImportRoute;

 	#[test]
@ -104,11 +104,11 @@ mod tests {
 			hash: H256::from(U256::from(2)),
 			number: 0,
 			total_difficulty: U256::from(0),
-			location: BlockLocation::BranchBecomingCanonChain {
-			ancestor: H256::from(U256::from(0)),
+			location: BlockLocation::BranchBecomingCanonChain(BranchBecomingCanonChainData {
+				ancestor: H256::from(U256::from(0)),
 				enacted: vec![H256::from(U256::from(1))],
 				retracted: vec![H256::from(U256::from(3)), H256::from(U256::from(4))],
-			}
+			})
 		};

 		assert_eq!(ImportRoute::from(info), ImportRoute {
--- a/ethcore/src/client/ids.rs
+++ b/ethcore/src/client/ids.rs
@ -18,6 +18,7 @@

 use util::hash::H256;
 use header::BlockNumber;
+use util::bytes::{FromRawBytes, FromBytesError, ToBytesWithMap, Populatable};

 /// Uniquely identifies block.
 #[derive(Debug, PartialEq, Clone, Hash, Eq)]
@ -50,3 +51,7 @@ pub struct UncleId (
 	/// Position in block.
 	pub usize
 );
+
+sized_binary_map!(TransactionId);
+sized_binary_map!(UncleId);
+sized_binary_map!(BlockId);
--- a/util/src/bytes.rs
+++ b/util/src/bytes.rs
@ -37,6 +37,8 @@ use std::slice;
 use std::ops::{Deref, DerefMut};
 use hash::FixedHash;
 use elastic_array::*;
+use std::mem;
+use std::cmp::Ordering;

 /// Vector like object
 pub trait VecLike<T> {
@ -239,6 +241,8 @@ pub enum FromBytesError {
 	NotLongEnough,
 	/// Too many bytes for the requested type
 	TooLong,
+	/// Invalid marker for (enums)
+	UnknownMarker,
 }

 /// Value that can be serialized from bytes array
@ -247,10 +251,8 @@ pub trait FromRawBytes : Sized {
 	fn from_bytes(d: &[u8]) -> Result<Self, FromBytesError>;
 }

-impl<T> FromRawBytes for T where T: Sized + FixedHash {
+impl<T> FromRawBytes for T where T: FixedHash {
 	fn from_bytes(bytes: &[u8]) -> Result<Self, FromBytesError> {
-		use std::mem;
-		use std::cmp::Ordering;
 		match bytes.len().cmp(&mem::size_of::<T>()) {
 			Ordering::Less => return Err(FromBytesError::NotLongEnough),
 			Ordering::Greater => return Err(FromBytesError::TooLong),
@ -263,18 +265,182 @@ impl<T> FromRawBytes for T where T: Sized + FixedHash {
 	}
 }

-impl FromRawBytes for String {
-	fn from_bytes(bytes: &[u8]) -> Result<String, FromBytesError> {
+#[macro_export]
+macro_rules! sized_binary_map {
+	($target_ty: ident) => {
+		impl FromRawBytes for $target_ty {
+			fn from_bytes(bytes: &[u8]) -> Result<Self, FromBytesError> {
+				match bytes.len().cmp(&::std::mem::size_of::<$target_ty>()) {
+					::std::cmp::Ordering::Less => return Err(FromBytesError::NotLongEnough),
+					::std::cmp::Ordering::Greater => return Err(FromBytesError::TooLong),
+					::std::cmp::Ordering::Equal => ()
+				};
+				let mut res: Self = unsafe { ::std::mem::uninitialized() };
+				res.copy_raw(bytes);
+				Ok(res)
+			}
+		}
+		impl ToBytesWithMap for $target_ty {
+			fn to_bytes_map(&self) -> Vec<u8> {
+				let sz = ::std::mem::size_of::<$target_ty>();
+				let mut res = Vec::<u8>::with_capacity(sz);
+
+				let ip: *const $target_ty = self;
+				let ptr: *const u8 = ip as *const _;
+				unsafe {
+					res.set_len(sz);
+					::std::ptr::copy(ptr, res.as_mut_ptr(), sz);
+				}
+				res
+			}
+		}
+	}
+}
+
+sized_binary_map!(u16);
+sized_binary_map!(u32);
+sized_binary_map!(u64);
+
+/// Value that can be serialized from variable-length byte array
+pub trait FromRawBytesVariable : Sized {
+	/// Create value from slice
+	fn from_bytes_variable(bytes: &[u8]) -> Result<Self, FromBytesError>;
+}
+
+impl<T> FromRawBytesVariable for T where T: FromRawBytes {
+	fn from_bytes_variable(bytes: &[u8]) -> Result<Self, FromBytesError> {
+		match bytes.len().cmp(&mem::size_of::<T>()) {
+			Ordering::Less => return Err(FromBytesError::NotLongEnough),
+			Ordering::Greater => return Err(FromBytesError::TooLong),
+			Ordering::Equal => ()
+		};
+
+		T::from_bytes(bytes)
+	}
+}
+
+impl FromRawBytesVariable for String {
+	fn from_bytes_variable(bytes: &[u8]) -> Result<String, FromBytesError> {
 		Ok(::std::str::from_utf8(bytes).unwrap().to_owned())
 	}
 }

-impl FromRawBytes for Vec<u8> {
-	fn from_bytes(bytes: &[u8]) -> Result<Vec<u8>, FromBytesError> {
+impl<T> FromRawBytesVariable for Vec<T> where T: FromRawBytes {
+	fn from_bytes_variable(bytes: &[u8]) -> Result<Self, FromBytesError> {
+		let size_of_t = mem::size_of::<T>();
+		let length_in_chunks = bytes.len() / size_of_t;
+
+		let mut result = Vec::with_capacity(length_in_chunks );
+		unsafe { result.set_len(length_in_chunks) };
+		for i in 0..length_in_chunks {
+			*result.get_mut(i).unwrap() = try!(T::from_bytes(
+				&bytes[size_of_t * i..size_of_t * (i+1)]))
+		}
+		Ok(result)
+	}
+}
+
+impl<V1, T2> FromRawBytes for (V1, T2) where V1: FromRawBytesVariable, T2: FromRawBytes {
+	fn from_bytes(bytes: &[u8]) -> Result<Self, FromBytesError> {
+		let header = 8usize;
+		let mut map: (u64, ) = unsafe { mem::uninitialized() };
+
+		if bytes.len() < header { return  Err(FromBytesError::NotLongEnough); }
+		map.copy_raw(&bytes[0..header]);
+
+		Ok((
+			try!(V1::from_bytes_variable(&bytes[header..header + (map.0 as usize)])),
+			try!(T2::from_bytes(&bytes[header + (map.0 as usize)..bytes.len()])),
+		))
+	}
+}
+
+impl<V1, V2, T3> FromRawBytes for (V1, V2, T3)
+	where V1: FromRawBytesVariable,
+		V2: FromRawBytesVariable,
+		T3: FromRawBytes
+{
+	fn from_bytes(bytes: &[u8]) -> Result<Self, FromBytesError> {
+		let header = 16usize;
+		let mut map: (u64, u64, ) = unsafe { mem::uninitialized() };
+
+		if bytes.len() < header { return  Err(FromBytesError::NotLongEnough); }
+		map.copy_raw(&bytes[0..header]);
+
+		let map_1 = (header, header + map.0 as usize);
+		let map_2 = (map_1.1 as usize, map_1.1 as usize + map.1 as usize);
+		Ok((
+			try!(V1::from_bytes_variable(&bytes[map_1.0..map_1.1])),
+			try!(V2::from_bytes_variable(&bytes[map_2.0..map_2.1])),
+			try!(T3::from_bytes(&bytes[map_2.1..bytes.len()])),
+		))
+	}
+}
+
+impl<'a, V1, T2> ToBytesWithMap for (&'a Vec<V1>, &'a T2) where V1: ToBytesWithMap, T2: ToBytesWithMap {
+	fn to_bytes_map(&self) -> Vec<u8> {
+		let header = 8usize;
+		let v1_size = mem::size_of::<V1>();
+		let mut result = Vec::with_capacity(header + self.0.len() * v1_size + mem::size_of::<T2>());
+		result.extend(((self.0.len() * v1_size) as u64).to_bytes_map());
+
+		for i in 0..self.0.len() {
+			result.extend(self.0[i].to_bytes_map());
+		}
+		result.extend(self.1.to_bytes_map());
+
+		result
+	}
+
+}
+
+impl<'a, V1, V2, T3> ToBytesWithMap for (&'a Vec<V1>, &'a Vec<V2>, &'a T3)
+	where V1: ToBytesWithMap,
+		V2: ToBytesWithMap,
+		T3: ToBytesWithMap
+{
+	fn to_bytes_map(&self) -> Vec<u8> {
+		let header = 16usize;
+		let v1_size = mem::size_of::<V1>();
+		let v2_size = mem::size_of::<V2>();
+		let mut result = Vec::with_capacity(
+			header +
+			self.0.len() * v1_size +
+			self.1.len() * v2_size +
+			mem::size_of::<T3>()
+		);
+		result.extend(((self.0.len() * v1_size) as u64).to_bytes_map());
+		result.extend(((self.1.len() * v2_size) as u64).to_bytes_map());
+		for i in 0..self.0.len() {
+			result.extend(self.0[i].to_bytes_map());
+		}
+		for i in 0..self.1.len() {
+			result.extend(self.1[i].to_bytes_map());
+		}
+		result.extend(self.2.to_bytes_map());
+
+		result
+	}
+}
+
+impl FromRawBytesVariable for Vec<u8> {
+	fn from_bytes_variable(bytes: &[u8]) -> Result<Vec<u8>, FromBytesError> {
 		Ok(bytes.clone().to_vec())
 	}
 }

+/// Value that serializes directly to variable-sized byte array and stores map
+pub trait ToBytesWithMap {
+	/// serialize to variable-sized byte array and store map
+	fn to_bytes_map(&self) -> Vec<u8>;
+}
+
+impl<T> ToBytesWithMap for T where T: FixedHash {
+	fn to_bytes_map(&self) -> Vec<u8> {
+		self.as_slice().to_vec()
+	}
+}
+
 #[test]
 fn fax_raw() {
 	let mut x = [255u8; 4];
@ -326,3 +492,43 @@ fn populate_big_types() {
 	h.copy_raw_from(&a);
 	assert_eq!(h, h256_from_hex("ffffffffffffffffffffffffffffffffffffffff000000000000000000000069"));
 }
+
+#[test]
+fn raw_bytes_from_tuple() {
+	let tup = (vec![1u16, 1u16, 1u16, 1u16], 10u16);
+	let bytes = vec![
+		// map
+		8u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8,
+		// four 1u16
+		1u8, 0u8,
+		1u8, 0u8,
+		1u8, 0u8,
+		1u8, 0u8,
+		// 10u16
+		10u8, 0u8];
+
+	type Tup = (Vec<u16>, u16);
+
+	let tup_from = Tup::from_bytes(&bytes).unwrap();
+	assert_eq!(tup, tup_from);
+
+	let tup_to = (&tup_from.0, &tup_from.1);
+	let bytes_to = tup_to.to_bytes_map();
+	assert_eq!(bytes_to, bytes);
+}
+
+#[test]
+fn bytes_map_from_triple() {
+	let data = (vec![2u16; 6], vec![6u32; 3], 12u64);
+	let bytes_map = (&data.0, &data.1, &data.2).to_bytes_map();
+	assert_eq!(bytes_map, vec![
+		// data map 2 x u64
+		12, 0, 0, 0, 0, 0, 0, 0,
+		12, 0, 0, 0, 0, 0, 0, 0,
+		// vec![2u16; 6]
+		2, 0, 2, 0, 2, 0, 2, 0, 2, 0, 2, 0,
+		// vec![6u32; 3]
+		6, 0, 0, 0, 6, 0, 0, 0, 6, 0, 0, 0,
+		// 12u64
+		12, 0, 0, 0, 0, 0, 0, 0]);
+}
--- a/util/src/keys/store.rs
+++ b/util/src/keys/store.rs
@ -34,11 +34,11 @@ const KEY_LENGTH_AES_USIZE: usize = KEY_LENGTH_AES as usize;
 /// Encrypted hash-map, each request should contain password
 pub trait EncryptedHashMap<Key: Hash + Eq> {
 	/// Returns existing value for the key, if any
-	fn get<Value: FromRawBytes + BytesConvertable>(&self, key: &Key, password: &str) ->  Result<Value, EncryptedHashMapError>;
+	fn get<Value: FromRawBytesVariable + BytesConvertable>(&self, key: &Key, password: &str) ->  Result<Value, EncryptedHashMapError>;
 	/// Insert new encrypted key-value and returns previous if there was any
-	fn insert<Value: FromRawBytes + BytesConvertable>(&mut self, key: Key, value: Value, password: &str) -> Option<Value>;
+	fn insert<Value: FromRawBytesVariable + BytesConvertable>(&mut self, key: Key, value: Value, password: &str) -> Option<Value>;
 	/// Removes key-value by key and returns the removed one, if any exists and password was provided
-	fn remove<Value: FromRawBytes + BytesConvertable> (&mut self, key: &Key, password: Option<&str>) -> Option<Value>;
+	fn remove<Value: FromRawBytesVariable + BytesConvertable> (&mut self, key: &Key, password: Option<&str>) -> Option<Value>;
 	/// Deletes key-value by key and returns if the key-value existed
 	fn delete(&mut self, key: &Key) -> bool {
 		self.remove::<Bytes>(key, None).is_some()
@ -306,7 +306,7 @@ fn derive_mac(derived_left_bits: &[u8], cipher_text: &[u8]) -> Bytes {
 }

 impl EncryptedHashMap<H128> for SecretStore {
-	fn get<Value: FromRawBytes + BytesConvertable>(&self, key: &H128, password: &str) -> Result<Value, EncryptedHashMapError> {
+	fn get<Value: FromRawBytesVariable + BytesConvertable>(&self, key: &H128, password: &str) -> Result<Value, EncryptedHashMapError> {
 		match self.directory.get(key) {
 			Some(key_file) => {
 				let (derived_left_bits, derived_right_bits) = match key_file.crypto.kdf {
@ -324,7 +324,7 @@ impl EncryptedHashMap<H128> for SecretStore {
 					}
 				};

-				match Value::from_bytes(&val) {
+				match Value::from_bytes_variable(&val) {
 					Ok(value) => Ok(value),
 					Err(bytes_error) => Err(EncryptedHashMapError::InvalidValueFormat(bytes_error))
 				}
@ -333,7 +333,7 @@ impl EncryptedHashMap<H128> for SecretStore {
 		}
 	}

-	fn insert<Value: FromRawBytes + BytesConvertable>(&mut self, key: H128, value: Value, password: &str) -> Option<Value> {
+	fn insert<Value: FromRawBytesVariable + BytesConvertable>(&mut self, key: H128, value: Value, password: &str) -> Option<Value> {
 		let previous = if let Ok(previous_value) = self.get(&key, password) { Some(previous_value) } else { None };

 		// crypto random initiators
@ -366,7 +366,7 @@ impl EncryptedHashMap<H128> for SecretStore {
 		previous
 	}

-	fn remove<Value: FromRawBytes + BytesConvertable>(&mut self, key: &H128, password: Option<&str>) -> Option<Value> {
+	fn remove<Value: FromRawBytesVariable + BytesConvertable>(&mut self, key: &H128, password: Option<&str>) -> Option<Value> {
 		let previous = if let Some(pass) = password {
 			if let Ok(previous_value) = self.get(&key, pass) { Some(previous_value) } else { None }
 		}