From 62455a4094414eba1c7fa3adb69436a7a8b0a6dc Mon Sep 17 00:00:00 2001 From: debris Date: Mon, 18 Apr 2016 18:15:03 +0200 Subject: [PATCH 1/3] separated from blockchain and made reusable db reader and batch writer --- ethcore/src/blockchain/blockchain.rs | 65 ++++++++-------------------- ethcore/src/{ => db}/db.rs | 0 ethcore/src/db/mod.rs | 23 ++++++++++ ethcore/src/db/reader.rs | 65 ++++++++++++++++++++++++++++ ethcore/src/db/writer.rs | 60 +++++++++++++++++++++++++ 5 files changed, 165 insertions(+), 48 deletions(-) rename ethcore/src/{ => db}/db.rs (100%) create mode 100644 ethcore/src/db/mod.rs create mode 100644 ethcore/src/db/reader.rs create mode 100644 ethcore/src/db/writer.rs diff --git a/ethcore/src/blockchain/blockchain.rs b/ethcore/src/blockchain/blockchain.rs index 60585d19f..4d1bec884 100644 --- a/ethcore/src/blockchain/blockchain.rs +++ b/ethcore/src/blockchain/blockchain.rs @@ -30,7 +30,7 @@ use blockchain::bloom_indexer::BloomIndexer; use blockchain::tree_route::TreeRoute; use blockchain::update::ExtrasUpdate; use blockchain::{CacheSize, ImportRoute}; -use db::{Writable, Readable, Key}; +use db::{Writable, Readable, Key, DatabaseReader, BatchWriter, CacheUpdatePolicy}; const BLOOM_INDEX_SIZE: usize = 16; const BLOOM_LEVELS: u8 = 3; @@ -183,7 +183,7 @@ impl BlockProvider for BlockChain { /// Returns true if the given block is known /// (though not necessarily a part of the canon chain). fn is_known(&self, hash: &H256) -> bool { - self.query_extras_exist(hash, &self.block_details) + DatabaseReader::new(&self.extras_db, &self.block_details).exists(hash) } // We do not store tracing information. @@ -466,28 +466,25 @@ impl BlockChain { batch.put(b"best", &update.info.hash).unwrap(); { - let mut write_details = self.block_details.write().unwrap(); - for (hash, details) in update.block_details.into_iter() { - batch.write(&hash, &details); - self.note_used(CacheID::Extras(ExtrasIndex::BlockDetails, hash.clone())); - write_details.insert(hash, details); + for hash in update.block_details.keys().cloned() { + self.note_used(CacheID::Extras(ExtrasIndex::BlockDetails, hash)); } + + let mut write_details = self.block_details.write().unwrap(); + BatchWriter::new(&batch, &mut write_details) + .extend(update.block_details, CacheUpdatePolicy::Overwrite); } { let mut write_receipts = self.block_receipts.write().unwrap(); - for (hash, receipt) in &update.block_receipts { - batch.write(hash, receipt); - write_receipts.remove(hash); - } + BatchWriter::new(&batch, &mut write_receipts) + .extend(update.block_receipts, CacheUpdatePolicy::Remove); } { let mut write_blocks_blooms = self.blocks_blooms.write().unwrap(); - for (bloom_hash, blocks_bloom) in &update.blocks_blooms { - batch.write(bloom_hash, blocks_bloom); - write_blocks_blooms.remove(bloom_hash); - } + BatchWriter::new(&batch, &mut write_blocks_blooms) + .extend(update.blocks_blooms, CacheUpdatePolicy::Remove); } // These cached values must be updated last and togeterh @@ -508,15 +505,11 @@ impl BlockChain { } } - for (number, hash) in &update.block_hashes { - batch.write(number, hash); - write_hashes.remove(number); - } + BatchWriter::new(&batch, &mut write_hashes) + .extend(update.block_hashes, CacheUpdatePolicy::Remove); - for (hash, tx_address) in &update.transactions_addresses { - batch.write(hash, tx_address); - write_txs.remove(hash); - } + BatchWriter::new(&batch, &mut write_txs) + .extend(update.transactions_addresses, CacheUpdatePolicy::Remove); // update extras database self.extras_db.write(batch).unwrap(); @@ -751,32 +744,8 @@ impl BlockChain { T: ExtrasIndexable + Clone + Decodable, K: Key + Eq + Hash + Clone, H256: From { - { - let read = cache.read().unwrap(); - if let Some(v) = read.get(hash) { - return Some(v.clone()); - } - } - self.note_used(CacheID::Extras(T::index(), H256::from(hash.clone()))); - - self.extras_db.read(hash).map(|t: T| { - let mut write = cache.write().unwrap(); - write.insert(hash.clone(), t.clone()); - t - }) - } - - fn query_extras_exist(&self, hash: &K, cache: &RwLock>) -> bool where - K: Key + Eq + Hash + Clone { - { - let read = cache.read().unwrap(); - if let Some(_) = read.get(hash) { - return true; - } - } - - self.extras_db.exists::(hash) + DatabaseReader::new(&self.extras_db, cache).read(hash) } /// Get current cache size. diff --git a/ethcore/src/db.rs b/ethcore/src/db/db.rs similarity index 100% rename from ethcore/src/db.rs rename to ethcore/src/db/db.rs diff --git a/ethcore/src/db/mod.rs b/ethcore/src/db/mod.rs new file mode 100644 index 000000000..199614b79 --- /dev/null +++ b/ethcore/src/db/mod.rs @@ -0,0 +1,23 @@ +// Copyright 2015, 2016 Ethcore (UK) Ltd. +// This file is part of Parity. + +// Parity is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Parity is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Parity. If not, see . + +pub mod db; +mod reader; +mod writer; + +pub use self::db::{Key, Writable, Readable}; +pub use self::reader::DatabaseReader; +pub use self::writer::{BatchWriter, CacheUpdatePolicy}; diff --git a/ethcore/src/db/reader.rs b/ethcore/src/db/reader.rs new file mode 100644 index 000000000..4e4755b11 --- /dev/null +++ b/ethcore/src/db/reader.rs @@ -0,0 +1,65 @@ +// Copyright 2015, 2016 Ethcore (UK) Ltd. +// This file is part of Parity. + +// Parity is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Parity is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Parity. If not, see . + +use std::hash::Hash; +use std::sync::RwLock; +use std::collections::HashMap; +use util::Database; +use util::rlp::Decodable; +use super::{Readable, Key}; + +pub struct DatabaseReader<'a, K, V> where K: 'a, V: 'a { + db: &'a Database, + cache: &'a RwLock>, +} + +impl<'a, K, V> DatabaseReader<'a, K, V> { + pub fn new(db: &'a Database, cache: &'a RwLock>) -> Self { + DatabaseReader { + db: db, + cache: cache, + } + } + + pub fn read(&self, key: &K) -> Option where + K: Eq + Hash + Clone + Key, + V: Clone + Decodable { + { + let read = self.cache.read().unwrap(); + if let Some(v) = read.get(key) { + return Some(v.clone()); + } + } + + self.db.read(key).map(|value: V|{ + let mut write = self.cache.write().unwrap(); + write.insert(key.clone(), value.clone()); + value + }) + } + + pub fn exists(&self, key: &K) -> bool where + K: Eq + Hash + Key { + { + let read = self.cache.read().unwrap(); + if read.get(key).is_some() { + return true; + } + } + + self.db.exists::(key) + } +} diff --git a/ethcore/src/db/writer.rs b/ethcore/src/db/writer.rs new file mode 100644 index 000000000..35658eadc --- /dev/null +++ b/ethcore/src/db/writer.rs @@ -0,0 +1,60 @@ +// Copyright 2015, 2016 Ethcore (UK) Ltd. +// This file is part of Parity. + +// Parity is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Parity is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Parity. If not, see . + +use std::collections::HashMap; +use std::hash::Hash; +use util::DBTransaction; +use util::rlp::Encodable; +use super::{Key, Writable}; + +#[derive(Clone, Copy)] +pub enum CacheUpdatePolicy { + Overwrite, + Remove, +} + +pub struct BatchWriter<'a, K, V> where K: 'a, V: 'a { + batch: &'a DBTransaction, + cache: &'a mut HashMap, +} + +impl<'a, K, V> BatchWriter<'a, K, V> { + pub fn new(batch: &'a DBTransaction, cache: &'a mut HashMap) -> Self { + BatchWriter { + batch: batch, + cache: cache, + } + } + + pub fn extend(&mut self, map: HashMap, policy: CacheUpdatePolicy) where + K: Key + Hash + Eq, + V: Encodable { + match policy { + CacheUpdatePolicy::Overwrite => { + for (key, value) in map.into_iter() { + self.batch.write(&key, &value); + self.cache.insert(key, value); + } + }, + CacheUpdatePolicy::Remove => { + for (key, value) in &map { + self.batch.write(key, value); + self.cache.remove(key); + } + }, + } + } +} From bffa1e1ec9d336f6855f7dd756e9ad4fb1ea51cb Mon Sep 17 00:00:00 2001 From: debris Date: Wed, 20 Apr 2016 15:45:42 +0200 Subject: [PATCH 2/3] simplified writing and reading from database with cache --- ethcore/src/blockchain/blockchain.rs | 22 +++----- ethcore/src/db/db.rs | 77 +++++++++++++++++++++++++++- ethcore/src/db/mod.rs | 7 +-- ethcore/src/db/reader.rs | 65 ----------------------- ethcore/src/db/writer.rs | 60 ---------------------- 5 files changed, 85 insertions(+), 146 deletions(-) delete mode 100644 ethcore/src/db/reader.rs delete mode 100644 ethcore/src/db/writer.rs diff --git a/ethcore/src/blockchain/blockchain.rs b/ethcore/src/blockchain/blockchain.rs index 4d1bec884..a35b14124 100644 --- a/ethcore/src/blockchain/blockchain.rs +++ b/ethcore/src/blockchain/blockchain.rs @@ -30,7 +30,7 @@ use blockchain::bloom_indexer::BloomIndexer; use blockchain::tree_route::TreeRoute; use blockchain::update::ExtrasUpdate; use blockchain::{CacheSize, ImportRoute}; -use db::{Writable, Readable, Key, DatabaseReader, BatchWriter, CacheUpdatePolicy}; +use db::{Writable, Readable, Key, CacheUpdatePolicy}; const BLOOM_INDEX_SIZE: usize = 16; const BLOOM_LEVELS: u8 = 3; @@ -183,7 +183,7 @@ impl BlockProvider for BlockChain { /// Returns true if the given block is known /// (though not necessarily a part of the canon chain). fn is_known(&self, hash: &H256) -> bool { - DatabaseReader::new(&self.extras_db, &self.block_details).exists(hash) + self.extras_db.exists_with_cache(&self.block_details, hash) } // We do not store tracing information. @@ -471,20 +471,17 @@ impl BlockChain { } let mut write_details = self.block_details.write().unwrap(); - BatchWriter::new(&batch, &mut write_details) - .extend(update.block_details, CacheUpdatePolicy::Overwrite); + batch.extend_with_cache(&mut write_details, update.block_details, CacheUpdatePolicy::Overwrite); } { let mut write_receipts = self.block_receipts.write().unwrap(); - BatchWriter::new(&batch, &mut write_receipts) - .extend(update.block_receipts, CacheUpdatePolicy::Remove); + batch.extend_with_cache(&mut write_receipts, update.block_receipts, CacheUpdatePolicy::Remove); } { let mut write_blocks_blooms = self.blocks_blooms.write().unwrap(); - BatchWriter::new(&batch, &mut write_blocks_blooms) - .extend(update.blocks_blooms, CacheUpdatePolicy::Remove); + batch.extend_with_cache(&mut write_blocks_blooms, update.blocks_blooms, CacheUpdatePolicy::Remove); } // These cached values must be updated last and togeterh @@ -505,11 +502,8 @@ impl BlockChain { } } - BatchWriter::new(&batch, &mut write_hashes) - .extend(update.block_hashes, CacheUpdatePolicy::Remove); - - BatchWriter::new(&batch, &mut write_txs) - .extend(update.transactions_addresses, CacheUpdatePolicy::Remove); + batch.extend_with_cache(&mut write_hashes, update.block_hashes, CacheUpdatePolicy::Remove); + batch.extend_with_cache(&mut write_txs, update.transactions_addresses, CacheUpdatePolicy::Remove); // update extras database self.extras_db.write(batch).unwrap(); @@ -745,7 +739,7 @@ impl BlockChain { K: Key + Eq + Hash + Clone, H256: From { self.note_used(CacheID::Extras(T::index(), H256::from(hash.clone()))); - DatabaseReader::new(&self.extras_db, cache).read(hash) + self.extras_db.read_with_cache(cache, hash) } /// Get current cache size. diff --git a/ethcore/src/db/db.rs b/ethcore/src/db/db.rs index 23e2d74d1..3dca6ca19 100644 --- a/ethcore/src/db/db.rs +++ b/ethcore/src/db/db.rs @@ -16,9 +16,18 @@ //! Extras db utils. +use std::hash::Hash; +use std::sync::RwLock; +use std::collections::HashMap; use util::{H264, DBTransaction, Database}; use util::rlp::{encode, Encodable, decode, Decodable}; +#[derive(Clone, Copy)] +pub enum CacheUpdatePolicy { + Overwrite, + Remove, +} + /// Should be used to get database key associated with given value. pub trait Key { /// Returns db key. @@ -27,16 +36,82 @@ pub trait Key { /// Should be used to write value into database. pub trait Writable { - /// Writes key into database. + /// Writes the value into the database. fn write(&self, key: &Key, value: &T) where T: Encodable; + + /// Writes the value into the database and updates the cache. + fn write_with_cache(&self, cache: &mut HashMap, key: K, value: T, policy: CacheUpdatePolicy) where + K: Key + Hash + Eq, + T: Encodable { + self.write(&key, &value); + match policy { + CacheUpdatePolicy::Overwrite => { + cache.insert(key, value); + }, + CacheUpdatePolicy::Remove => { + cache.remove(&key); + } + } + } + + /// Writes the values into the database and updates the cache. + fn extend_with_cache(&self, cache: &mut HashMap, values: HashMap, policy: CacheUpdatePolicy) + where K: Key + Hash + Eq, T: Encodable { + match policy { + CacheUpdatePolicy::Overwrite => { + for (key, value) in values.into_iter() { + self.write(&key, &value); + cache.insert(key, value); + } + }, + CacheUpdatePolicy::Remove => { + for (key, value) in &values { + self.write(key, value); + cache.remove(key); + } + }, + } + } } /// Should be used to read values from database. pub trait Readable { /// Returns value for given key. fn read(&self, key: &Key) -> Option where T: Decodable; + + /// Returns value for given key either in cache or in database. + fn read_with_cache(&self, cache: &RwLock>, key: &K) -> Option where + K: Key + Eq + Hash + Clone, + T: Clone + Decodable { + { + let read = cache.read().unwrap(); + if let Some(v) = read.get(key) { + return Some(v.clone()); + } + } + + self.read(key).map(|value: T|{ + let mut write = cache.write().unwrap(); + write.insert(key.clone(), value.clone()); + value + }) + } + /// Returns true if given value exists. fn exists(&self, key: &Key) -> bool; + + /// Returns true if given value exists either in cache or in database. + fn exists_with_cache(&self, cache: &RwLock>, key: &K) -> bool where + K: Eq + Hash + Key { + { + let read = cache.read().unwrap(); + if read.get(key).is_some() { + return true; + } + } + + self.exists::(key) + } } impl Writable for DBTransaction { diff --git a/ethcore/src/db/mod.rs b/ethcore/src/db/mod.rs index 199614b79..f56ff3cfc 100644 --- a/ethcore/src/db/mod.rs +++ b/ethcore/src/db/mod.rs @@ -15,9 +15,4 @@ // along with Parity. If not, see . pub mod db; -mod reader; -mod writer; - -pub use self::db::{Key, Writable, Readable}; -pub use self::reader::DatabaseReader; -pub use self::writer::{BatchWriter, CacheUpdatePolicy}; +pub use self::db::{Key, Writable, Readable, CacheUpdatePolicy}; diff --git a/ethcore/src/db/reader.rs b/ethcore/src/db/reader.rs deleted file mode 100644 index 4e4755b11..000000000 --- a/ethcore/src/db/reader.rs +++ /dev/null @@ -1,65 +0,0 @@ -// Copyright 2015, 2016 Ethcore (UK) Ltd. -// This file is part of Parity. - -// Parity is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// Parity is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. - -// You should have received a copy of the GNU General Public License -// along with Parity. If not, see . - -use std::hash::Hash; -use std::sync::RwLock; -use std::collections::HashMap; -use util::Database; -use util::rlp::Decodable; -use super::{Readable, Key}; - -pub struct DatabaseReader<'a, K, V> where K: 'a, V: 'a { - db: &'a Database, - cache: &'a RwLock>, -} - -impl<'a, K, V> DatabaseReader<'a, K, V> { - pub fn new(db: &'a Database, cache: &'a RwLock>) -> Self { - DatabaseReader { - db: db, - cache: cache, - } - } - - pub fn read(&self, key: &K) -> Option where - K: Eq + Hash + Clone + Key, - V: Clone + Decodable { - { - let read = self.cache.read().unwrap(); - if let Some(v) = read.get(key) { - return Some(v.clone()); - } - } - - self.db.read(key).map(|value: V|{ - let mut write = self.cache.write().unwrap(); - write.insert(key.clone(), value.clone()); - value - }) - } - - pub fn exists(&self, key: &K) -> bool where - K: Eq + Hash + Key { - { - let read = self.cache.read().unwrap(); - if read.get(key).is_some() { - return true; - } - } - - self.db.exists::(key) - } -} diff --git a/ethcore/src/db/writer.rs b/ethcore/src/db/writer.rs deleted file mode 100644 index 35658eadc..000000000 --- a/ethcore/src/db/writer.rs +++ /dev/null @@ -1,60 +0,0 @@ -// Copyright 2015, 2016 Ethcore (UK) Ltd. -// This file is part of Parity. - -// Parity is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// Parity is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. - -// You should have received a copy of the GNU General Public License -// along with Parity. If not, see . - -use std::collections::HashMap; -use std::hash::Hash; -use util::DBTransaction; -use util::rlp::Encodable; -use super::{Key, Writable}; - -#[derive(Clone, Copy)] -pub enum CacheUpdatePolicy { - Overwrite, - Remove, -} - -pub struct BatchWriter<'a, K, V> where K: 'a, V: 'a { - batch: &'a DBTransaction, - cache: &'a mut HashMap, -} - -impl<'a, K, V> BatchWriter<'a, K, V> { - pub fn new(batch: &'a DBTransaction, cache: &'a mut HashMap) -> Self { - BatchWriter { - batch: batch, - cache: cache, - } - } - - pub fn extend(&mut self, map: HashMap, policy: CacheUpdatePolicy) where - K: Key + Hash + Eq, - V: Encodable { - match policy { - CacheUpdatePolicy::Overwrite => { - for (key, value) in map.into_iter() { - self.batch.write(&key, &value); - self.cache.insert(key, value); - } - }, - CacheUpdatePolicy::Remove => { - for (key, value) in &map { - self.batch.write(key, value); - self.cache.remove(key); - } - }, - } - } -} From 273e4d6f32c2741f973ebf1de2dbb4410c57fca4 Mon Sep 17 00:00:00 2001 From: debris Date: Wed, 20 Apr 2016 15:53:01 +0200 Subject: [PATCH 3/3] removed db/module, a single file is enoguh --- ethcore/src/{db => }/db.rs | 0 ethcore/src/db/mod.rs | 18 ------------------ 2 files changed, 18 deletions(-) rename ethcore/src/{db => }/db.rs (100%) delete mode 100644 ethcore/src/db/mod.rs diff --git a/ethcore/src/db/db.rs b/ethcore/src/db.rs similarity index 100% rename from ethcore/src/db/db.rs rename to ethcore/src/db.rs diff --git a/ethcore/src/db/mod.rs b/ethcore/src/db/mod.rs deleted file mode 100644 index f56ff3cfc..000000000 --- a/ethcore/src/db/mod.rs +++ /dev/null @@ -1,18 +0,0 @@ -// Copyright 2015, 2016 Ethcore (UK) Ltd. -// This file is part of Parity. - -// Parity is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// Parity is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. - -// You should have received a copy of the GNU General Public License -// along with Parity. If not, see . - -pub mod db; -pub use self::db::{Key, Writable, Readable, CacheUpdatePolicy};