From 295efdba55f045a8ab63409c33b7885b98f1a182 Mon Sep 17 00:00:00 2001 From: debris Date: Fri, 15 Apr 2016 18:54:35 +0200 Subject: [PATCH 1/2] refactored blockchain extras keys building --- ethcore/src/blockchain/blockchain.rs | 31 +++-- ethcore/src/db.rs | 61 +++++++++ ethcore/src/extras.rs | 182 +++++++++++---------------- ethcore/src/lib.rs | 1 + 4 files changed, 150 insertions(+), 125 deletions(-) create mode 100644 ethcore/src/db.rs diff --git a/ethcore/src/blockchain/blockchain.rs b/ethcore/src/blockchain/blockchain.rs index ebbae306e..afa9042d4 100644 --- a/ethcore/src/blockchain/blockchain.rs +++ b/ethcore/src/blockchain/blockchain.rs @@ -30,6 +30,7 @@ use blockchain::bloom_indexer::BloomIndexer; use blockchain::tree_route::TreeRoute; use blockchain::update::ExtrasUpdate; use blockchain::{CacheSize, ImportRoute}; +use db::{Writable, Readable, Key}; const BLOOM_INDEX_SIZE: usize = 16; const BLOOM_LEVELS: u8 = 3; @@ -314,8 +315,8 @@ impl BlockChain { bc.blocks_db.put(&hash, genesis).unwrap(); let batch = DBTransaction::new(); - batch.put_extras(&hash, &details); - batch.put_extras(&header.number(), &hash); + batch.write(&hash, &details); + batch.write(&header.number(), &hash); batch.put(b"best", &hash).unwrap(); bc.extras_db.write(batch).unwrap(); @@ -467,7 +468,7 @@ impl BlockChain { { let mut write_details = self.block_details.write().unwrap(); for (hash, details) in update.block_details.into_iter() { - batch.put_extras(&hash, &details); + batch.write(&hash, &details); self.note_used(CacheID::Extras(ExtrasIndex::BlockDetails, hash.clone())); write_details.insert(hash, details); } @@ -476,7 +477,7 @@ impl BlockChain { { let mut write_receipts = self.block_receipts.write().unwrap(); for (hash, receipt) in &update.block_receipts { - batch.put_extras(hash, receipt); + batch.write(hash, receipt); write_receipts.remove(hash); } } @@ -484,7 +485,7 @@ impl BlockChain { { let mut write_blocks_blooms = self.blocks_blooms.write().unwrap(); for (bloom_hash, blocks_bloom) in &update.blocks_blooms { - batch.put_extras(bloom_hash, blocks_bloom); + batch.write(bloom_hash, blocks_bloom); write_blocks_blooms.remove(bloom_hash); } } @@ -508,12 +509,12 @@ impl BlockChain { } for (number, hash) in &update.block_hashes { - batch.put_extras(number, hash); + batch.write(number, hash); write_hashes.remove(number); } for (hash, tx_address) in &update.transactions_addresses { - batch.put_extras(hash, tx_address); + batch.write(hash, tx_address); write_txs.remove(hash); } @@ -747,8 +748,9 @@ impl BlockChain { } fn query_extras(&self, hash: &K, cache: &RwLock>) -> Option where - T: Clone + Decodable + ExtrasIndexable, - K: ExtrasSliceConvertable + Eq + Hash + Clone { + T: ExtrasIndexable + Clone + Decodable, + K: Key + Eq + Hash + Clone, + H256: From { { let read = cache.read().unwrap(); if let Some(v) = read.get(hash) { @@ -756,11 +758,9 @@ impl BlockChain { } } - if let Some(h) = hash.as_h256() { - self.note_used(CacheID::Extras(T::extras_index(), h.clone())); - } + self.note_used(CacheID::Extras(T::index(), H256::from(hash.clone()))); - self.extras_db.get_extras(hash).map(| t: T | { + self.extras_db.read(hash).map(|t: T| { let mut write = cache.write().unwrap(); write.insert(hash.clone(), t.clone()); t @@ -768,8 +768,7 @@ impl BlockChain { } fn query_extras_exist(&self, hash: &K, cache: &RwLock>) -> bool where - K: ExtrasSliceConvertable + Eq + Hash + Clone, - T: ExtrasIndexable { + K: Key + Eq + Hash + Clone { { let read = cache.read().unwrap(); if let Some(_) = read.get(hash) { @@ -777,7 +776,7 @@ impl BlockChain { } } - self.extras_db.extras_exists::<_, T>(hash) + self.extras_db.exists::(hash) } /// Get current cache size. diff --git a/ethcore/src/db.rs b/ethcore/src/db.rs new file mode 100644 index 000000000..3ef3763af --- /dev/null +++ b/ethcore/src/db.rs @@ -0,0 +1,61 @@ +// Copyright 2015, 2016 Ethcore (UK) Ltd. +// This file is part of Parity. + +// Parity is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Parity is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Parity. If not, see . + +//! Extras db utils. + +use util::{H264, DBTransaction, Database}; +use util::rlp::{encode, Encodable, decode, Decodable}; + +/// Should be used to get database key associated with given value. +pub trait Key { + /// Returns db key. + fn key(&self) -> H264; +} + +/// Should be used to write value into database. +pub trait Writable { + /// Writes key into database. + fn write(&self, key: &Key, value: &T) where T: Encodable; +} + +/// Should be used to read values from database. +pub trait Readable { + /// Returns value for given key. + fn read(&self, key: &Key) -> Option where T: Decodable; + /// Returns true if given value exists. + fn exists(&self, key: &Key) -> bool; +} + +impl Writable for DBTransaction { + fn write(&self, key: &Key, value: &T) where T: Encodable { + self.put(&key.key(), &encode(value)) + .expect("db put failed"); + } +} + +impl Readable for Database { + fn read(&self, key: &Key) -> Option where T: Decodable { + self.get(&key.key()) + .expect("db get failed") + .map(|v| decode(&v)) + } + + fn exists(&self, key: &Key) -> bool { + self.get(&key.key()) + .expect("db get failed") + .is_some() + } +} diff --git a/ethcore/src/extras.rs b/ethcore/src/extras.rs index a7c82c37c..49d867dcf 100644 --- a/ethcore/src/extras.rs +++ b/ethcore/src/extras.rs @@ -19,6 +19,7 @@ use util::*; use header::BlockNumber; use receipt::Receipt; +use db::Key; /// Represents index of extra data in database #[derive(Copy, Debug, Hash, Eq, PartialEq, Clone)] @@ -37,95 +38,88 @@ pub enum ExtrasIndex { BlockReceipts = 5, } -/// trait used to write Extras data to db -pub trait ExtrasWritable { - /// Write extra data to db - fn put_extras(&self, hash: &K, value: &T) where - T: ExtrasIndexable + Encodable, - K: ExtrasSliceConvertable; +fn with_index(hash: &H256, i: ExtrasIndex) -> H264 { + let mut slice = H264::from_slice(hash); + slice[32] = i as u8; + slice } -/// trait used to read Extras data from db -pub trait ExtrasReadable { - /// Read extra data from db - fn get_extras(&self, hash: &K) -> Option where - T: ExtrasIndexable + Decodable, - K: ExtrasSliceConvertable; - - /// Check if extra data exists in the db - fn extras_exists(&self, hash: &K) -> bool where - T: ExtrasIndexable, - K: ExtrasSliceConvertable; -} - -impl ExtrasWritable for DBTransaction { - fn put_extras(&self, hash: &K, value: &T) where - T: ExtrasIndexable + Encodable, - K: ExtrasSliceConvertable { - - self.put(&hash.to_extras_slice(T::extras_index()), &encode(value)).unwrap() - } -} - -impl ExtrasReadable for Database { - fn get_extras(&self, hash: &K) -> Option where - T: ExtrasIndexable + Decodable, - K: ExtrasSliceConvertable { - - self.get(&hash.to_extras_slice(T::extras_index())).unwrap() - .map(|v| decode(&v)) - } - - fn extras_exists(&self, hash: &K) -> bool where - T: ExtrasIndexable, - K: ExtrasSliceConvertable { - - self.get(&hash.to_extras_slice(T::extras_index())).unwrap().is_some() - } -} - -/// Implementations should convert arbitrary type to database key slice -pub trait ExtrasSliceConvertable { - /// Convert self, with `i` (the index), to a 264-bit extras DB key. - fn to_extras_slice(&self, i: ExtrasIndex) -> H264; - /// Interpret self as a 256-bit hash, if natively `H256`. - fn as_h256(&self) -> Option<&H256> { None } -} - -impl ExtrasSliceConvertable for H256 { - fn to_extras_slice(&self, i: ExtrasIndex) -> H264 { - let mut slice = H264::from_slice(self); - slice[32] = i as u8; - slice - } - fn as_h256(&self) -> Option<&H256> { Some(self) } -} - -impl ExtrasSliceConvertable for U256 { - fn to_extras_slice(&self, i: ExtrasIndex) -> H264 { - H256::from(self).to_extras_slice(i) - } -} - -// NICE: make less horrible. -impl ExtrasSliceConvertable for BlockNumber { - fn to_extras_slice(&self, i: ExtrasIndex) -> H264 { - U256::from(*self).to_extras_slice(i) - } -} - -/// Types implementing this trait can be indexed in extras database pub trait ExtrasIndexable { - /// Returns this data index - fn extras_index() -> ExtrasIndex; + fn index() -> ExtrasIndex; } impl ExtrasIndexable for H256 { - fn extras_index() -> ExtrasIndex { + fn index() -> ExtrasIndex { ExtrasIndex::BlockHash } } +impl ExtrasIndexable for BlockDetails { + fn index() -> ExtrasIndex { + ExtrasIndex::BlockDetails + } +} + +impl ExtrasIndexable for TransactionAddress { + fn index() -> ExtrasIndex { + ExtrasIndex::TransactionAddress + } +} + +impl ExtrasIndexable for BlockLogBlooms { + fn index() -> ExtrasIndex { + ExtrasIndex::BlockLogBlooms + } +} + +impl ExtrasIndexable for BlocksBlooms { + fn index() -> ExtrasIndex { + ExtrasIndex::BlocksBlooms + } +} + +impl ExtrasIndexable for BlockReceipts { + fn index() -> ExtrasIndex { + ExtrasIndex::BlockReceipts + } +} + +impl Key for BlockNumber { + fn key(&self) -> H264 { + with_index(&H256::from(*self), ExtrasIndex::BlockHash) + } +} + +impl Key for H256 { + fn key(&self) -> H264 { + with_index(self, ExtrasIndex::BlockDetails) + } +} + +impl Key for H256 { + fn key(&self) -> H264 { + with_index(self, ExtrasIndex::TransactionAddress) + } +} + +impl Key for H256 { + fn key(&self) -> H264 { + with_index(self, ExtrasIndex::BlockLogBlooms) + } +} + +impl Key for H256 { + fn key(&self) -> H264 { + with_index(self, ExtrasIndex::BlocksBlooms) + } +} + +impl Key for H256 { + fn key(&self) -> H264 { + with_index(self, ExtrasIndex::BlockReceipts) + } +} + /// Familial details concerning a block #[derive(Debug, Clone)] pub struct BlockDetails { @@ -139,12 +133,6 @@ pub struct BlockDetails { pub children: Vec } -impl ExtrasIndexable for BlockDetails { - fn extras_index() -> ExtrasIndex { - ExtrasIndex::BlockDetails - } -} - impl HeapSizeOf for BlockDetails { fn heap_size_of_children(&self) -> usize { self.children.heap_size_of_children() @@ -181,12 +169,6 @@ pub struct BlockLogBlooms { pub blooms: Vec } -impl ExtrasIndexable for BlockLogBlooms { - fn extras_index() -> ExtrasIndex { - ExtrasIndex::BlockLogBlooms - } -} - impl HeapSizeOf for BlockLogBlooms { fn heap_size_of_children(&self) -> usize { self.blooms.heap_size_of_children() @@ -227,12 +209,6 @@ impl BlocksBlooms { } } -impl ExtrasIndexable for BlocksBlooms { - fn extras_index() -> ExtrasIndex { - ExtrasIndex::BlocksBlooms - } -} - impl HeapSizeOf for BlocksBlooms { fn heap_size_of_children(&self) -> usize { 0 } } @@ -277,12 +253,6 @@ pub struct TransactionAddress { pub index: usize } -impl ExtrasIndexable for TransactionAddress { - fn extras_index() -> ExtrasIndex { - ExtrasIndex::TransactionAddress - } -} - impl HeapSizeOf for TransactionAddress { fn heap_size_of_children(&self) -> usize { 0 } } @@ -340,9 +310,3 @@ impl HeapSizeOf for BlockReceipts { self.receipts.heap_size_of_children() } } - -impl ExtrasIndexable for BlockReceipts { - fn extras_index() -> ExtrasIndex { - ExtrasIndex::BlockReceipts - } -} diff --git a/ethcore/src/lib.rs b/ethcore/src/lib.rs index f8fb223db..d267a08ce 100644 --- a/ethcore/src/lib.rs +++ b/ethcore/src/lib.rs @@ -104,6 +104,7 @@ pub mod views; pub mod receipt; pub mod pod_state; +mod db; mod common; mod basic_types; #[macro_use] mod evm; From 5c4086bf8ef637bd369c501f439131527c4afc45 Mon Sep 17 00:00:00 2001 From: debris Date: Fri, 15 Apr 2016 19:32:30 +0200 Subject: [PATCH 2/2] log errors when db queries fails --- ethcore/src/db.rs | 28 ++++++++++++++++++++-------- 1 file changed, 20 insertions(+), 8 deletions(-) diff --git a/ethcore/src/db.rs b/ethcore/src/db.rs index 3ef3763af..23e2d74d1 100644 --- a/ethcore/src/db.rs +++ b/ethcore/src/db.rs @@ -41,21 +41,33 @@ pub trait Readable { impl Writable for DBTransaction { fn write(&self, key: &Key, value: &T) where T: Encodable { - self.put(&key.key(), &encode(value)) - .expect("db put failed"); + let result = self.put(&key.key(), &encode(value)); + if let Err(err) = result { + panic!("db put failed, key: {:?}, err: {:?}", key.key(), err); + } } } impl Readable for Database { fn read(&self, key: &Key) -> Option where T: Decodable { - self.get(&key.key()) - .expect("db get failed") - .map(|v| decode(&v)) + let result = self.get(&key.key()); + + match result { + Ok(option) => option.map(|v| decode(&v)), + Err(err) => { + panic!("db get failed, key: {:?}, err: {:?}", key.key(), err); + } + } } fn exists(&self, key: &Key) -> bool { - self.get(&key.key()) - .expect("db get failed") - .is_some() + let result = self.get(&key.key()); + + match result { + Ok(v) => v.is_some(), + Err(err) => { + panic!("db get failed, key: {:?}, err: {:?}", key.key(), err); + } + } } }