grassrootseconomics/openethereum
5
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

more tests and fixes for blockchains bloom filters

Browse Source
This commit is contained in:
debris 2016-02-16 14:46:21 +01:00
parent 0699cdd5d0
commit d000ad2441
2 changed files with 141 additions and 44 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

183
ethcore/src/blockchain.rs
View File

@ -53,6 +53,60 @@ pub struct CacheSize {
pub blocks_blooms: usize
}
struct BloomIndexer {
index_size: usize,
levels: u8
}
impl BloomIndexer {
fn new(index_size: usize, levels: u8) -> Self {
BloomIndexer {
index_size: index_size,
levels: levels
}
}
/// Calculates bloom's position in database.
fn location(&self, bloom_index: &BloomIndex) -> BlocksBloomLocation {
use std::{mem, ptr};
let hash = unsafe {
let mut hash: H256 = mem::zeroed();
ptr::copy(&[bloom_index.index / self.index_size] as *const usize as *const u8, hash.as_mut_ptr(), 8);
hash[8] = bloom_index.level;
hash.reverse();
hash
};
BlocksBloomLocation {
hash: hash,
index: bloom_index.index % self.index_size
}
}
fn index_size(&self) -> usize {
self.index_size
}
fn levels(&self) -> u8 {
self.levels
}
}
/// Blockchain update info.
struct ExtrasUpdate {
/// Block hash.
hash: H256,
/// DB update batch.
batch: WriteBatch,
/// Inserted block familial details.
details: BlockDetails,
/// New best block (if it has changed).
new_best: Option<BestBlock>,
/// Changed blocks bloom location hashes.
bloom_hashes: HashSet<H256>
}
impl CacheSize {
/// Total amount used by the cache.
fn total(&self) -> usize { self.blocks + self.block_details + self.transaction_addresses + self.block_logs + self.blocks_blooms }
@ -175,14 +229,13 @@ pub struct BlockChain {
cache_man: RwLock<CacheManager>,
// blooms config
bloom_index_size: usize,
bloom_levels: u8
// blooms indexing
bloom_indexer: BloomIndexer
}
impl FilterDataSource for BlockChain {
fn bloom_at_index(&self, bloom_index: &BloomIndex) -> Option<H2048> {
let location = self.blocks_bloom_location(bloom_index);
let location = self.bloom_indexer.location(bloom_index);
self.blocks_blooms(&location.hash).and_then(|blooms| blooms.blooms.into_iter().nth(location.index).cloned())
}
}
@ -236,7 +289,7 @@ impl BlockProvider for BlockChain {
/// Returns numbers of blocks containing given bloom.
fn blocks_with_bloom(&self, bloom: &H2048, from_block: BlockNumber, to_block: BlockNumber) -> Vec<BlockNumber> {
let filter = ChainFilter::new(self, self.bloom_index_size, self.bloom_levels);
let filter = ChainFilter::new(self, self.bloom_indexer.index_size(), self.bloom_indexer.levels());
filter.blocks_with_bloom(bloom, from_block as usize, to_block as usize).into_iter().map(|b| b as BlockNumber).collect()
}
}
@ -298,8 +351,7 @@ impl BlockChain {
extras_db: extras_db,
blocks_db: blocks_db,
cache_man: RwLock::new(cache_man),
bloom_index_size: BLOOM_INDEX_SIZE,
bloom_levels: BLOOM_LEVELS
bloom_indexer: BloomIndexer::new(BLOOM_INDEX_SIZE, BLOOM_LEVELS)
};
// load best block
@ -464,27 +516,38 @@ impl BlockChain {
// store block in db
self.blocks_db.put(&hash, &bytes).unwrap();
let (batch, new_best, details) = self.block_to_extras_insert_batch(bytes, receipts);
let update = self.block_to_extras_update(bytes, receipts);
self.apply_update(update);
}
/// Applies extras update.
fn apply_update(&self, update: ExtrasUpdate) {
// update best block
let mut best_block = self.best_block.write().unwrap();
if let Some(b) = new_best {
if let Some(b) = update.new_best {
*best_block = b;
}
// update caches
let mut write = self.block_details.write().unwrap();
write.remove(&header.parent_hash());
write.insert(hash.clone(), details);
self.note_used(CacheID::Block(hash));
// update details cache
let mut write_details = self.block_details.write().unwrap();
write_details.remove(&update.details.parent);
write_details.insert(update.hash.clone(), update.details);
self.note_used(CacheID::Block(update.hash));
// update blocks blooms cache
let mut write_blocks_blooms = self.blocks_blooms.write().unwrap();
for bloom_hash in &update.bloom_hashes {
write_blocks_blooms.remove(bloom_hash);
}
// update extras database
self.extras_db.write(batch).unwrap();
self.extras_db.write(update.batch).unwrap();
}
/// Transforms block into WriteBatch that may be written into database
/// Additionally, if it's new best block it returns new best block object.
fn block_to_extras_insert_batch(&self, bytes: &[u8], _receipts: &[Receipt]) -> (WriteBatch, Option<BestBlock>, BlockDetails) {
//fn block_to_extras_insert_batch(&self, bytes: &[u8], _receipts: &[Receipt]) -> (WriteBatch, Option<BestBlock>, BlockDetails) {
fn block_to_extras_update(&self, bytes: &[u8], _receipts: &[Receipt]) -> ExtrasUpdate {
// create views onto rlp
let block = BlockView::new(bytes);
let header = block.header_view();
@ -522,8 +585,6 @@ impl BlockChain {
});
}
// save blooms (is it really required?). maybe store receipt whole instead?
//let blooms: Vec<H2048> = receipts.iter().map(|r| r.log_bloom.clone()).collect();
//batch.put_extras(&hash, &BlockLogBlooms {
@ -532,7 +593,13 @@ impl BlockChain {
// if it's not new best block, just return
if !is_new_best {
return (batch, None, details);
return ExtrasUpdate {
hash: hash.clone(),
batch: batch,
details: details,
new_best: None,
bloom_hashes: HashSet::new()
};
}
// if its new best block we need to make sure that all ancestors
@ -550,7 +617,7 @@ impl BlockChain {
batch.put_extras(&header.number(), &hash);
// update block blooms
modified_blooms = ChainFilter::new(self, self.bloom_index_size, self.bloom_levels)
modified_blooms = ChainFilter::new(self, self.bloom_indexer.index_size(), self.bloom_indexer.levels())
.add_bloom(&header.log_bloom(), header.number() as usize);
},
// it is a fork
@ -569,7 +636,7 @@ impl BlockChain {
.collect();
// reset blooms chain head
modified_blooms = ChainFilter::new(self, self.bloom_index_size, self.bloom_levels)
modified_blooms = ChainFilter::new(self, self.bloom_indexer.index_size(), self.bloom_indexer.levels())
.reset_chain_head(&blooms, start_number as usize, self.best_block_number() as usize);
},
// route.blocks.len() could be 0 only if inserted block is best block,
@ -577,10 +644,14 @@ impl BlockChain {
_ => { unreachable!(); }
};
let bloom_hashes = modified_blooms.iter()
.map(|(bloom_index, _)| self.bloom_indexer.location(&bloom_index).hash)
.collect();
for (hash, blocks_blooms) in modified_blooms.into_iter()
.fold(HashMap::new(), | mut acc, (bloom_index, bloom) | {
{
let location = self.blocks_bloom_location(&bloom_index);
let location = self.bloom_indexer.location(&bloom_index);
let mut blocks_blooms = acc.entry(location.hash).or_insert_with(BlocksBlooms::new);
blocks_blooms.blooms[location.index] = bloom;
}
@ -593,12 +664,18 @@ impl BlockChain {
batch.put(b"best", &hash).unwrap();
let best_block = BestBlock {
hash: hash,
hash: hash.clone(),
number: header.number(),
total_difficulty: total_difficulty
};
(batch, Some(best_block), details)
ExtrasUpdate {
hash: hash,
batch: batch,
new_best: Some(best_block),
details: details,
bloom_hashes: bloom_hashes
}
}
/// Get best block hash.
@ -621,22 +698,7 @@ impl BlockChain {
self.query_extras(hash, &self.blocks_blooms)
}
/// Calculates bloom's position in database.
fn blocks_bloom_location(&self, bloom_index: &BloomIndex) -> BlocksBloomLocation {
use std::{mem, ptr};
let hash = unsafe {
let mut hash: H256 = mem::zeroed();
ptr::copy(&[bloom_index.index / self.bloom_index_size] as *const usize as *const u8, hash.as_mut_ptr(), 8);
hash[8] = bloom_index.level;
hash
};
BlocksBloomLocation {
hash: hash,
index: bloom_index.index % self.bloom_index_size
}
}
fn query_extras<K, T>(&self, hash: &K, cache: &RwLock<HashMap<K, T>>) -> Option<T> where
T: Clone + Decodable + ExtrasIndexable,
@ -740,7 +802,7 @@ mod tests {
use std::str::FromStr;
use rustc_serialize::hex::FromHex;
use util::hash::*;
use blockchain::*;
use blockchain::{BlockProvider, BlockChain};
use tests::helpers::*;
#[test]
@ -946,15 +1008,50 @@ mod tests {
fn test_bloom_filter_simple() {
let genesis = "f901fcf901f7a00000000000000000000000000000000000000000000000000000000000000000a01dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347948888f1f195afa192cfee860698584c030f4c9db1a07dba07d6b448a186e9612e5f737d1c909dce473e53199901a302c00646d523c1a056e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421a056e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421b90100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000008302000080832fefd8808454c98c8142a059262c330941f3fe2a34d16d6e3c7b30d2ceb37c6a0e9a994c494ee1a61d2410885aa4c8bf8e56e264c0c0".from_hex().unwrap();
let b1 = "f90261f901f9a05716670833ec874362d65fea27a7cd35af5897d275b31a44944113111e4e96d2a01dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347948888f1f195afa192cfee860698584c030f4c9db1a0cb52de543653d86ccd13ba3ddf8b052525b04231c6884a4db3188a184681d878a0e78628dd45a1f8dc495594d83b76c588a3ee67463260f8b7d4a42f574aeab29aa0e9244cf7503b79c03d3a099e07a80d2dbc77bb0b502d8a89d51ac0d68dd31313b90100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000008302000001832fefd882520884562791e580a051b3ecba4e3f2b49c11d42dd0851ec514b1be3138080f72a2b6e83868275d98f8877671f479c414b47f862f86080018304cb2f94095e7baea6a6c7c4c2dfeb977efac326af552d870a801ca09e2709d7ec9bbe6b1bbbf0b2088828d14cd5e8642a1fee22dc74bfa89761a7f9a04bd8813dee4be989accdb708b1c2e325a7e9c695a8024e30e89d6c644e424747c0".from_hex().unwrap();
// bloom filter flow block 300054
let b1 = "f90261f901f9a05716670833ec874362d65fea27a7cd35af5897d275b31a44944113111e4e96d2a01dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347948888f1f195afa192cfee860698584c030f4c9db1a0cb52de543653d86ccd13ba3ddf8b052525b04231c6884a4db3188a184681d878a0e78628dd45a1f8dc495594d83b76c588a3ee67463260f8b7d4a42f574aeab29aa0e9244cf7503b79c03d3a099e07a80d2dbc77bb0b502d8a89d51ac0d68dd31313b90100000000000000000000000000000000000000000000000200000010000000000000000000000000000000000000000000000000000000000000000000000000001000000000000000000080000000000000000000000000000000000400000000000000000000000000000000000000000000000000000000000000000000008000000000000000000000000000000000000000000000000000000000000080000000000000000000000000000000000000000000000000000000000000000000000000000000000000020000000000000000000400000000000000000000000000000000000000000000000000000000000000000000000000000000000000008302000001832fefd882520884562791e580a051b3ecba4e3f2b49c11d42dd0851ec514b1be3138080f72a2b6e83868275d98f8877671f479c414b47f862f86080018304cb2f94095e7baea6a6c7c4c2dfeb977efac326af552d870a801ca09e2709d7ec9bbe6b1bbbf0b2088828d14cd5e8642a1fee22dc74bfa89761a7f9a04bd8813dee4be989accdb708b1c2e325a7e9c695a8024e30e89d6c644e424747c0".from_hex().unwrap();
let b2 = "f902ccf901f9a0437e51676ff10756fcfee5edd9159fa41dbcb1b2c592850450371cbecd54ee4fa01dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347948888f1f195afa192cfee860698584c030f4c9db1a0c70a5dc56146e5ef025e4e5726a6373c6f12fd2f6784093a19ead0a7d17fb292a040645cbce4fd399e7bb9160b4c30c40d7ee616a030d4e18ef0ed3b02bdb65911a086e608555f63628417032a011d107b36427af37d153f0da02ce3f90fdd5e8c08b90100000000000000000000000000000000000000000000000200000010000000000000000000000000000000000000000000000000000000000000000000000000001000000000000000000080000000000000000000000000000000000400000000000000000000000000000000000000000000000000000000000000000000008000000000000000000000000000000000000000000000000000000000000080000000000000000000000000000000000000000000000000000000000000000000000000000000000000020000000000000000000400000000000000000000000000000000000000000000000000000000000000000000000000000000000000008302004002832fefd882c0e384562791e880a0e3cc39ff775cc0a32f175995b92e84b729e5c9a3563ff899e3555b908bc21d75887c3cde283f4846a6f8cdf8cb01018304cb2f8080b87e6060604052606e8060106000396000f360606040526000357c010000000000000000000000000000000000000000000000000000000090048063c0406226146037576035565b005b60406004506056565b6040518082815260200191505060405180910390f35b6000600560006000508190555060059050606b565b90561ba05258615c63503c0a600d6994b12ea5750d45b3c69668e2a371b4fbfb9eeff6b8a0a11be762bc90491231274a2945be35a43f23c27775b1ff24dd521702fe15f73ec0".from_hex().unwrap();
let bloom = H2048::from_str("00000000000000000000000000000000000000000000020000001000000000000000000000000000000000000000000000000000000000000000000000000000100000000000000000008000000000000000000000000000000000040000000000000000000000000000000000000000000000000000000000000000000000800000000000000000000000000000000000000000000000000000000000008000000000000000000000000000000000000000000000000000000000000000000000000000000000000002000000000000000000040000000000000000000000000000000000000000000000000000000000000000000000000000000000000000").unwrap();
let temp = RandomTempPath::new();
let bc = BlockChain::new(&genesis, temp.as_path());
let blocks = bc.blocks_with_bloom(&bloom, 0, 50);
assert_eq!(blocks, vec![]);
bc.insert_block(&b1, &[]);
let blocks = bc.blocks_with_bloom(&bloom, 0, 2);
assert_eq!(blocks, vec![1]);
assert_eq!(blocks, vec![]);
bc.insert_block(&b2, &[]);
let blocks = bc.blocks_with_bloom(&bloom, 0, 50);
assert_eq!(blocks, vec![2]);
}
#[test]
fn test_bloom_indexer() {
use chainfilter::BloomIndex;
use blockchain::BloomIndexer;
use extras::BlocksBloomLocation;
let bi = BloomIndexer::new(16, 3);
let index = BloomIndex::new(0, 0);
assert_eq!(bi.location(&index), BlocksBloomLocation {
hash: H256::new(),
index: 0
});
let index = BloomIndex::new(1, 0);
assert_eq!(bi.location(&index), BlocksBloomLocation {
hash: H256::from_str("0000000000000000000000000000000000000000000000010000000000000000").unwrap(),
index: 0
});
let index = BloomIndex::new(0, 299_999);
assert_eq!(bi.location(&index), BlocksBloomLocation {
hash: H256::from_str("000000000000000000000000000000000000000000000000000000000000493d").unwrap(),
index: 15
});
}
}

2
ethcore/src/extras.rs
View File

@ -261,7 +261,7 @@ impl Encodable for BlocksBlooms {
}
/// Represents location of bloom in database.
#[derive(Debug)]
#[derive(Debug, PartialEq)]
pub struct BlocksBloomLocation {
/// Unique hash of BlocksBloom
pub hash: H256,