From e4f0c0b2158b4f7356b2ecc7b343de5999239b44 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tomasz=20Drwi=C4=99ga?= Date: Thu, 28 Jul 2016 23:46:24 +0200 Subject: [PATCH] Single DB (#1741) * Consolidation migration * Started db amalgamation * Using client constants for columns * Adding with_columns constructor * Migrating to single db * Fixing tests. * test.sh without verbose * Fixing warnings * add migration tests that catch the bug * make multiple migrations more robust * add moved v9 * Merge branch 'noop-migrations' into single-db * spurious line * clean up migrations ordering * update comment [ci skip] * Bumping default number of max_open_files & re-ordering columns. * fix merge * fix ignored analysis tests * Caching best block content * Faster best_block_header * Adding progress to v8 migration * clean up warnings * Separate hashes and bodies in the DB * Separate hashes and bodies in the DB * Fixed tests --- .travis.yml | 2 +- Cargo.lock | 4 +- ethcore/src/blockchain/best_block.rs | 5 +- ethcore/src/blockchain/blockchain.rs | 445 +++++++++++++++-------- ethcore/src/blockchain/update.rs | 4 +- ethcore/src/client/client.rs | 101 ++--- ethcore/src/client/test_client.rs | 7 +- ethcore/src/client/traits.rs | 5 +- ethcore/src/db.rs | 36 +- ethcore/src/json_tests/executive.rs | 2 +- ethcore/src/json_tests/transaction.rs | 2 +- ethcore/src/migrations/blocks/v8.rs | 7 +- ethcore/src/migrations/extras/v6.rs | 7 +- ethcore/src/migrations/mod.rs | 4 + ethcore/src/migrations/state/v7.rs | 36 +- ethcore/src/migrations/v9.rs | 82 +++++ ethcore/src/snapshot/mod.rs | 11 +- ethcore/src/tests/helpers.rs | 36 +- ethcore/src/trace/db.rs | 80 ++-- ethcore/src/trace/mod.rs | 4 +- ethcore/src/verification/verification.rs | 8 + ethcore/src/views/block.rs | 5 + ethcore/src/views/body.rs | 144 ++++++++ ethcore/src/views/mod.rs | 2 + evmbin/Cargo.lock | 25 +- parity/dir.rs | 11 +- parity/helpers.rs | 2 +- parity/migration.rs | 192 +++++++--- parity/params.rs | 16 +- test.sh | 2 +- util/src/journaldb/archivedb.rs | 166 +++++---- util/src/journaldb/earlymergedb.rs | 352 +++++++++--------- util/src/journaldb/mod.rs | 13 +- util/src/journaldb/overlayrecentdb.rs | 313 ++++++++-------- util/src/journaldb/refcounteddb.rs | 88 ++--- util/src/journaldb/traits.rs | 14 +- util/src/kvdb.rs | 153 ++++---- util/src/migration/mod.rs | 79 +++- util/src/migration/tests.rs | 20 +- util/src/overlaydb.rs | 147 +++----- util/src/rlp/commonrlps.rs | 2 +- util/src/rlp/rlpcompression.rs | 2 +- 42 files changed, 1578 insertions(+), 1058 deletions(-) create mode 100644 ethcore/src/migrations/v9.rs create mode 100644 ethcore/src/views/body.rs diff --git a/.travis.yml b/.travis.yml index bede69962..03b4edf9d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -64,7 +64,7 @@ install: ) script: - - if [ "$RUN_TESTS" = "true" ]; then ./test.sh; fi + - if [ "$RUN_TESTS" = "true" ]; then ./test.sh --verbose; fi - if [ "$RUN_COVERAGE" = "true" ]; then ./scripts/cov.sh "$KCOV_CMD"; fi after_success: | diff --git a/Cargo.lock b/Cargo.lock index c54f44295..f3d026850 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1093,7 +1093,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "rocksdb" version = "0.4.5" -source = "git+https://github.com/ethcore/rust-rocksdb#dd597245bfcb621c6ffc45478e1fda0b05d2f409" +source = "git+https://github.com/ethcore/rust-rocksdb#eadce7f74cfe92b99ce63a77af425b47857239b8" dependencies = [ "libc 0.2.12 (registry+https://github.com/rust-lang/crates.io-index)", "rocksdb-sys 0.3.0 (git+https://github.com/ethcore/rust-rocksdb)", @@ -1102,7 +1102,7 @@ dependencies = [ [[package]] name = "rocksdb-sys" version = "0.3.0" -source = "git+https://github.com/ethcore/rust-rocksdb#dd597245bfcb621c6ffc45478e1fda0b05d2f409" +source = "git+https://github.com/ethcore/rust-rocksdb#eadce7f74cfe92b99ce63a77af425b47857239b8" dependencies = [ "gcc 0.3.28 (registry+https://github.com/rust-lang/crates.io-index)", "libc 0.2.12 (registry+https://github.com/rust-lang/crates.io-index)", diff --git a/ethcore/src/blockchain/best_block.rs b/ethcore/src/blockchain/best_block.rs index 00c092713..aa1e1854e 100644 --- a/ethcore/src/blockchain/best_block.rs +++ b/ethcore/src/blockchain/best_block.rs @@ -14,6 +14,7 @@ // You should have received a copy of the GNU General Public License // along with Parity. If not, see . +use util::bytes::Bytes; use util::numbers::{U256,H256}; use header::BlockNumber; @@ -25,5 +26,7 @@ pub struct BestBlock { /// Best block number. pub number: BlockNumber, /// Best block total difficulty. - pub total_difficulty: U256 + pub total_difficulty: U256, + /// Best block uncompressed bytes + pub block: Bytes, } diff --git a/ethcore/src/blockchain/blockchain.rs b/ethcore/src/blockchain/blockchain.rs index fd6e88b7e..8b120caa2 100644 --- a/ethcore/src/blockchain/blockchain.rs +++ b/ethcore/src/blockchain/blockchain.rs @@ -31,6 +31,7 @@ use types::tree_route::TreeRoute; use blockchain::update::ExtrasUpdate; use blockchain::{CacheSize, ImportRoute, Config}; use db::{Writable, Readable, CacheUpdatePolicy}; +use client::{DB_COL_EXTRA, DB_COL_HEADERS, DB_COL_BODIES}; const LOG_BLOOMS_LEVELS: usize = 3; const LOG_BLOOMS_ELEMENTS_PER_INDEX: usize = 16; @@ -58,29 +59,37 @@ pub trait BlockProvider { /// Get the partial-header of a block. fn block_header(&self, hash: &H256) -> Option
{ - self.block(hash).map(|bytes| BlockView::new(&bytes).header()) + self.block_header_data(hash).map(|header| decode(&header)) } + /// Get the header RLP of a block. + fn block_header_data(&self, hash: &H256) -> Option; + + /// Get the block body (uncles and transactions). + fn block_body(&self, hash: &H256) -> Option; + /// Get a list of uncles for a given block. /// Returns None if block does not exist. fn uncles(&self, hash: &H256) -> Option> { - self.block(hash).map(|bytes| BlockView::new(&bytes).uncles()) + self.block_body(hash).map(|bytes| BodyView::new(&bytes).uncles()) } /// Get a list of uncle hashes for a given block. /// Returns None if block does not exist. fn uncle_hashes(&self, hash: &H256) -> Option> { - self.block(hash).map(|bytes| BlockView::new(&bytes).uncle_hashes()) + self.block_body(hash).map(|bytes| BodyView::new(&bytes).uncle_hashes()) } /// Get the number of given block's hash. fn block_number(&self, hash: &H256) -> Option { - self.block(hash).map(|bytes| BlockView::new(&bytes).header_view().number()) + self.block_details(hash).map(|details| details.number) } /// Get transaction with given transaction hash. fn transaction(&self, address: &TransactionAddress) -> Option { - self.block(&address.block_hash).and_then(|bytes| BlockView::new(&bytes).localized_transaction_at(address.index)) + self.block_body(&address.block_hash) + .and_then(|bytes| self.block_number(&address.block_hash) + .and_then(|n| BodyView::new(&bytes).localized_transaction_at(&address.block_hash, n, address.index))) } /// Get transaction receipt. @@ -91,7 +100,9 @@ pub trait BlockProvider { /// Get a list of transactions for a given block. /// Returns None if block does not exist. fn transactions(&self, hash: &H256) -> Option> { - self.block(hash).map(|bytes| BlockView::new(&bytes).localized_transactions()) + self.block_body(hash) + .and_then(|bytes| self.block_number(hash) + .map(|n| BodyView::new(&bytes).localized_transactions(hash, n))) } /// Returns reference to genesis hash. @@ -110,7 +121,8 @@ pub trait BlockProvider { #[derive(Debug, Hash, Eq, PartialEq, Clone)] enum CacheID { - Block(H256), + BlockHeader(H256), + BlockBody(H256), BlockDetails(H256), BlockHashes(BlockNumber), TransactionAddresses(H256), @@ -127,7 +139,7 @@ impl bc::group::BloomGroupDatabase for BlockChain { fn blooms_at(&self, position: &bc::group::GroupPosition) -> Option { let position = LogGroupPosition::from(position.clone()); self.note_used(CacheID::BlocksBlooms(position.clone())); - self.extras_db.read_with_cache(&self.blocks_blooms, &position).map(Into::into) + self.db.read_with_cache(DB_COL_EXTRA, &self.blocks_blooms, &position).map(Into::into) } } @@ -143,7 +155,8 @@ pub struct BlockChain { best_block: RwLock, // block cache - blocks: RwLock>, + block_headers: RwLock>, + block_bodies: RwLock>, // extra caches block_details: RwLock>, @@ -152,39 +165,96 @@ pub struct BlockChain { blocks_blooms: RwLock>, block_receipts: RwLock>, - extras_db: Database, - blocks_db: Database, + db: Arc, cache_man: RwLock, - - insert_lock: Mutex<()> } impl BlockProvider for BlockChain { /// Returns true if the given block is known /// (though not necessarily a part of the canon chain). fn is_known(&self, hash: &H256) -> bool { - self.extras_db.exists_with_cache(&self.block_details, hash) + self.db.exists_with_cache(DB_COL_EXTRA, &self.block_details, hash) } /// Get raw block data fn block(&self, hash: &H256) -> Option { + match (self.block_header_data(hash), self.block_body(hash)) { + (Some(header), Some(body)) => { + let mut block = RlpStream::new_list(3); + let body_rlp = Rlp::new(&body); + block.append_raw(&header, 1); + block.append_raw(body_rlp.at(0).as_raw(), 1); + block.append_raw(body_rlp.at(1).as_raw(), 1); + Some(block.out()) + }, + _ => None, + } + } + + /// Get block header data + fn block_header_data(&self, hash: &H256) -> Option { + // Check cache first { - let read = self.blocks.read(); + let read = self.block_headers.read(); if let Some(v) = read.get(hash) { return Some(v.clone()); } } - let opt = self.blocks_db.get(hash) + // Check if it's the best block + { + let best_block = self.best_block.read(); + if &best_block.hash == hash { + return Some(Rlp::new(&best_block.block).at(0).as_raw().to_vec()); + } + } + + // Read from DB and populate cache + let opt = self.db.get(DB_COL_HEADERS, hash) .expect("Low level database error. Some issue with disk?"); - self.note_used(CacheID::Block(hash.clone())); + self.note_used(CacheID::BlockHeader(hash.clone())); match opt { Some(b) => { let bytes: Bytes = UntrustedRlp::new(&b).decompress(RlpType::Blocks).to_vec(); - let mut write = self.blocks.write(); + let mut write = self.block_headers.write(); + write.insert(hash.clone(), bytes.clone()); + Some(bytes) + }, + None => None + } + } + + /// Get block body data + fn block_body(&self, hash: &H256) -> Option { + // Check cache first + { + let read = self.block_bodies.read(); + if let Some(v) = read.get(hash) { + return Some(v.clone()); + } + } + + // Check if it's the best block + { + let best_block = self.best_block.read(); + if &best_block.hash == hash { + return Some(Self::block_to_body(&best_block.block)); + } + } + + // Read from DB and populate cache + let opt = self.db.get(DB_COL_BODIES, hash) + .expect("Low level database error. Some issue with disk?"); + + self.note_used(CacheID::BlockBody(hash.clone())); + + match opt { + Some(b) => { + let bytes: Bytes = UntrustedRlp::new(&b).decompress(RlpType::Blocks).to_vec(); + let mut write = self.block_bodies.write(); write.insert(hash.clone(), bytes.clone()); Some(bytes) }, @@ -195,25 +265,25 @@ impl BlockProvider for BlockChain { /// Get the familial details concerning a block. fn block_details(&self, hash: &H256) -> Option { self.note_used(CacheID::BlockDetails(hash.clone())); - self.extras_db.read_with_cache(&self.block_details, hash) + self.db.read_with_cache(DB_COL_EXTRA, &self.block_details, hash) } /// Get the hash of given block's number. fn block_hash(&self, index: BlockNumber) -> Option { self.note_used(CacheID::BlockHashes(index)); - self.extras_db.read_with_cache(&self.block_hashes, &index) + self.db.read_with_cache(DB_COL_EXTRA, &self.block_hashes, &index) } /// Get the address of transaction with given hash. fn transaction_address(&self, hash: &H256) -> Option { self.note_used(CacheID::TransactionAddresses(hash.clone())); - self.extras_db.read_with_cache(&self.transaction_addresses, hash) + self.db.read_with_cache(DB_COL_EXTRA, &self.transaction_addresses, hash) } /// Get receipts of block with given hash. fn block_receipts(&self, hash: &H256) -> Option { self.note_used(CacheID::BlockReceipts(hash.clone())); - self.extras_db.read_with_cache(&self.block_receipts, hash) + self.db.read_with_cache(DB_COL_EXTRA, &self.block_receipts, hash) } /// Returns numbers of blocks containing given bloom. @@ -249,27 +319,7 @@ impl<'a> Iterator for AncestryIter<'a> { impl BlockChain { /// Create new instance of blockchain from given Genesis - pub fn new(config: Config, genesis: &[u8], path: &Path) -> BlockChain { - // open extras db - let mut extras_path = path.to_path_buf(); - extras_path.push("extras"); - let extras_db = match config.db_cache_size { - None => Database::open_default(extras_path.to_str().unwrap()).unwrap(), - Some(cache_size) => Database::open( - &DatabaseConfig::with_cache(cache_size/2), - extras_path.to_str().unwrap()).unwrap(), - }; - - // open blocks db - let mut blocks_path = path.to_path_buf(); - blocks_path.push("blocks"); - let blocks_db = match config.db_cache_size { - None => Database::open_default(blocks_path.to_str().unwrap()).unwrap(), - Some(cache_size) => Database::open( - &DatabaseConfig::with_cache(cache_size/2), - blocks_path.to_str().unwrap()).unwrap(), - }; - + pub fn new(config: Config, genesis: &[u8], db: Arc) -> BlockChain { let mut cache_man = CacheManager{cache_usage: VecDeque::new(), in_use: HashSet::new()}; (0..COLLECTION_QUEUE_SIZE).foreach(|_| cache_man.cache_usage.push_back(HashSet::new())); @@ -281,39 +331,21 @@ impl BlockChain { elements_per_index: LOG_BLOOMS_ELEMENTS_PER_INDEX, }, best_block: RwLock::new(BestBlock::default()), - blocks: RwLock::new(HashMap::new()), + block_headers: RwLock::new(HashMap::new()), + block_bodies: RwLock::new(HashMap::new()), block_details: RwLock::new(HashMap::new()), block_hashes: RwLock::new(HashMap::new()), transaction_addresses: RwLock::new(HashMap::new()), blocks_blooms: RwLock::new(HashMap::new()), block_receipts: RwLock::new(HashMap::new()), - extras_db: extras_db, - blocks_db: blocks_db, + db: db.clone(), cache_man: RwLock::new(cache_man), - insert_lock: Mutex::new(()), }; // load best block - let best_block_hash = match bc.extras_db.get(b"best").unwrap() { + let best_block_hash = match bc.db.get(DB_COL_EXTRA, b"best").unwrap() { Some(best) => { - let new_best = H256::from_slice(&best); - if !bc.blocks_db.get(&new_best).unwrap().is_some() { - warn!("Best block {} not found", new_best.hex()); - } - /* TODO: enable this once the best block issue is resolved - while !bc.blocks_db.get(&new_best).unwrap().is_some() { - match bc.rewind() { - Some(h) => { - new_best = h; - } - None => { - warn!("Can't rewind blockchain"); - break; - } - } - info!("Restored mismatched best block. Was: {}, new: {}", H256::from_slice(&best).hex(), new_best.hex()); - }*/ - new_best + H256::from_slice(&best) } None => { // best block does not exist @@ -329,25 +361,32 @@ impl BlockChain { children: vec![] }; - let block_batch = DBTransaction::new(); - block_batch.put(&hash, genesis).unwrap(); - bc.blocks_db.write(block_batch).expect("Low level database error. Some issue with disk?"); - - let batch = DBTransaction::new(); - batch.write(&hash, &details); - batch.write(&header.number(), &hash); - batch.put(b"best", &hash).unwrap(); - bc.extras_db.write(batch).unwrap(); + let batch = DBTransaction::new(&db); + batch.put(DB_COL_HEADERS, &hash, block.header_rlp().as_raw()).unwrap(); + batch.put(DB_COL_BODIES, &hash, &Self::block_to_body(&genesis)).unwrap(); + batch.write(DB_COL_EXTRA, &hash, &details); + batch.write(DB_COL_EXTRA, &header.number(), &hash); + batch.put(DB_COL_EXTRA, b"best", &hash).unwrap(); + bc.db.write(batch).expect("Low level database error. Some issue with disk?"); hash } }; { + // Fetch best block details + let best_block_number = bc.block_number(&best_block_hash).unwrap(); + let best_block_total_difficulty = bc.block_details(&best_block_hash).unwrap().total_difficulty; + let best_block_rlp = bc.block(&best_block_hash).unwrap(); + + // and write them let mut best_block = bc.best_block.write(); - best_block.number = bc.block_number(&best_block_hash).unwrap(); - best_block.total_difficulty = bc.block_details(&best_block_hash).unwrap().total_difficulty; - best_block.hash = best_block_hash; + *best_block = BestBlock { + number: best_block_number, + total_difficulty: best_block_total_difficulty, + hash: best_block_hash, + block: best_block_rlp, + }; } bc @@ -356,44 +395,52 @@ impl BlockChain { /// Returns true if the given parent block has given child /// (though not necessarily a part of the canon chain). fn is_known_child(&self, parent: &H256, hash: &H256) -> bool { - self.extras_db.read_with_cache(&self.block_details, parent).map_or(false, |d| d.children.contains(hash)) + self.db.read_with_cache(DB_COL_EXTRA, &self.block_details, parent).map_or(false, |d| d.children.contains(hash)) } /// Rewind to a previous block #[cfg(test)] fn rewind(&self) -> Option { use db::Key; - let batch = DBTransaction::new(); + let batch = self.db.transaction(); // track back to the best block we have in the blocks database - if let Some(best_block_hash) = self.extras_db.get(b"best").unwrap() { + if let Some(best_block_hash) = self.db.get(DB_COL_EXTRA, b"best").unwrap() { let best_block_hash = H256::from_slice(&best_block_hash); if best_block_hash == self.genesis_hash() { return None; } - if let Some(extras) = self.extras_db.read(&best_block_hash) as Option { + if let Some(extras) = self.db.read(DB_COL_EXTRA, &best_block_hash) as Option { type DetailsKey = Key; - batch.delete(&(DetailsKey::key(&best_block_hash))).unwrap(); + batch.delete(DB_COL_EXTRA, &(DetailsKey::key(&best_block_hash))).unwrap(); let hash = extras.parent; let range = extras.number as bc::Number .. extras.number as bc::Number; let chain = bc::group::BloomGroupChain::new(self.blooms_config, self); let changes = chain.replace(&range, vec![]); for (k, v) in changes.into_iter() { - batch.write(&LogGroupPosition::from(k), &BloomGroup::from(v)); + batch.write(DB_COL_EXTRA, &LogGroupPosition::from(k), &BloomGroup::from(v)); } - batch.put(b"best", &hash).unwrap(); + batch.put(DB_COL_EXTRA, b"best", &hash).unwrap(); + + let best_block_total_difficulty = self.block_details(&hash).unwrap().total_difficulty; + let best_block_rlp = self.block(&hash).unwrap(); + let mut best_block = self.best_block.write(); - best_block.number = extras.number - 1; - best_block.total_difficulty = self.block_details(&hash).unwrap().total_difficulty; - best_block.hash = hash; + *best_block = BestBlock { + number: extras.number - 1, + total_difficulty: best_block_total_difficulty, + hash: hash, + block: best_block_rlp, + }; // update parent extras - if let Some(mut details) = self.extras_db.read(&hash) as Option { + if let Some(mut details) = self.db.read(DB_COL_EXTRA, &hash) as Option { details.children.clear(); - batch.write(&hash, &details); + batch.write(DB_COL_EXTRA, &hash, &details); } - self.extras_db.write(batch).unwrap(); + self.db.write(batch).expect("Writing to db failed"); self.block_details.write().clear(); self.block_hashes.write().clear(); - self.blocks.write().clear(); + self.block_headers.write().clear(); + self.block_bodies.write().clear(); self.block_receipts.write().clear(); return Some(hash); } @@ -500,7 +547,7 @@ impl BlockChain { /// Inserts the block into backing cache database. /// Expects the block to be valid and already verified. /// If the block is already known, does nothing. - pub fn insert_block(&self, bytes: &[u8], receipts: Vec) -> ImportRoute { + pub fn insert_block(&self, batch: &DBTransaction, bytes: &[u8], receipts: Vec) -> ImportRoute { // create views onto rlp let block = BlockView::new(bytes); let header = block.header_view(); @@ -510,11 +557,13 @@ impl BlockChain { return ImportRoute::none(); } - let compressed = UntrustedRlp::new(bytes).compress(RlpType::Blocks).to_vec(); + let block_rlp = UntrustedRlp::new(bytes); + let compressed_header = block_rlp.at(0).unwrap().compress(RlpType::Blocks); + let compressed_body = UntrustedRlp::new(&Self::block_to_body(bytes)).compress(RlpType::Blocks); - let _lock = self.insert_lock.lock(); // store block in db - self.blocks_db.put(&hash, &compressed).unwrap(); + batch.put(DB_COL_HEADERS, &hash, &compressed_header).unwrap(); + batch.put(DB_COL_BODIES, &hash, &compressed_body).unwrap(); let info = self.block_info(bytes); @@ -527,13 +576,14 @@ impl BlockChain { ); } - self.apply_update(ExtrasUpdate { + self.apply_update(batch, ExtrasUpdate { block_hashes: self.prepare_block_hashes_update(bytes, &info), block_details: self.prepare_block_details_update(bytes, &info), block_receipts: self.prepare_block_receipts_update(receipts, &info), transactions_addresses: self.prepare_transaction_addresses_update(bytes, &info), blocks_blooms: self.prepare_block_blooms_update(bytes, &info), info: info.clone(), + block: bytes, }); ImportRoute::from(info) @@ -582,26 +632,24 @@ impl BlockChain { } /// Applies extras update. - fn apply_update(&self, update: ExtrasUpdate) { - let batch = DBTransaction::new(); - + fn apply_update(&self, batch: &DBTransaction, update: ExtrasUpdate) { { for hash in update.block_details.keys().cloned() { self.note_used(CacheID::BlockDetails(hash)); } let mut write_details = self.block_details.write(); - batch.extend_with_cache(&mut *write_details, update.block_details, CacheUpdatePolicy::Overwrite); + batch.extend_with_cache(DB_COL_EXTRA, &mut *write_details, update.block_details, CacheUpdatePolicy::Overwrite); } { let mut write_receipts = self.block_receipts.write(); - batch.extend_with_cache(&mut *write_receipts, update.block_receipts, CacheUpdatePolicy::Remove); + batch.extend_with_cache(DB_COL_EXTRA, &mut *write_receipts, update.block_receipts, CacheUpdatePolicy::Remove); } { let mut write_blocks_blooms = self.blocks_blooms.write(); - batch.extend_with_cache(&mut *write_blocks_blooms, update.blocks_blooms, CacheUpdatePolicy::Remove); + batch.extend_with_cache(DB_COL_EXTRA, &mut *write_blocks_blooms, update.blocks_blooms, CacheUpdatePolicy::Remove); } // These cached values must be updated last with all three locks taken to avoid @@ -612,11 +660,12 @@ impl BlockChain { match update.info.location { BlockLocation::Branch => (), _ => { - batch.put(b"best", &update.info.hash).unwrap(); + batch.put(DB_COL_EXTRA, b"best", &update.info.hash).unwrap(); *best_block = BestBlock { hash: update.info.hash, number: update.info.number, - total_difficulty: update.info.total_difficulty + total_difficulty: update.info.total_difficulty, + block: update.block.to_vec(), }; } } @@ -624,11 +673,8 @@ impl BlockChain { let mut write_hashes = self.block_hashes.write(); let mut write_txs = self.transaction_addresses.write(); - batch.extend_with_cache(&mut *write_hashes, update.block_hashes, CacheUpdatePolicy::Remove); - batch.extend_with_cache(&mut *write_txs, update.transactions_addresses, CacheUpdatePolicy::Remove); - - // update extras database - self.extras_db.write(batch).unwrap(); + batch.extend_with_cache(DB_COL_EXTRA, &mut *write_hashes, update.block_hashes, CacheUpdatePolicy::Remove); + batch.extend_with_cache(DB_COL_EXTRA, &mut *write_txs, update.transactions_addresses, CacheUpdatePolicy::Remove); } } @@ -681,7 +727,7 @@ impl BlockChain { block_hashes.insert(number, info.hash.clone()); }, BlockLocation::BranchBecomingCanonChain(ref data) => { - let ancestor_number = self.block_number(&data.ancestor).unwrap(); + let ancestor_number = self.block_number(&data.ancestor).expect("Block number of ancestor is always in DB"); let start_number = ancestor_number + 1; for (index, hash) in data.enacted.iter().cloned().enumerate() { @@ -775,8 +821,8 @@ impl BlockChain { let range = start_number as bc::Number..self.best_block_number() as bc::Number; let mut blooms: Vec = data.enacted.iter() - .map(|hash| self.block(hash).unwrap()) - .map(|bytes| BlockView::new(&bytes).header_view().log_bloom()) + .map(|hash| self.block_header_data(hash).unwrap()) + .map(|bytes| HeaderView::new(&bytes).log_bloom()) .map(Bloom::from) .map(Into::into) .collect(); @@ -808,10 +854,16 @@ impl BlockChain { self.best_block.read().total_difficulty } + /// Get best block header + pub fn best_block_header(&self) -> Bytes { + let block = self.best_block.read(); + BlockView::new(&block.block).header_view().rlp().as_raw().to_vec() + } + /// Get current cache size. pub fn cache_size(&self) -> CacheSize { CacheSize { - blocks: self.blocks.read().heap_size_of_children(), + blocks: self.block_headers.read().heap_size_of_children() + self.block_bodies.read().heap_size_of_children(), block_details: self.block_details.read().heap_size_of_children(), transaction_addresses: self.transaction_addresses.read().heap_size_of_children(), blocks_blooms: self.blocks_blooms.read().heap_size_of_children(), @@ -851,7 +903,8 @@ impl BlockChain { for i in 0..COLLECTION_QUEUE_SIZE { { trace!("Cache cleanup round started {}, cache_size = {}", i, self.cache_size().total()); - let mut blocks = self.blocks.write(); + let mut block_headers = self.block_headers.write(); + let mut block_bodies = self.block_bodies.write(); let mut block_details = self.block_details.write(); let mut block_hashes = self.block_hashes.write(); let mut transaction_addresses = self.transaction_addresses.write(); @@ -862,7 +915,8 @@ impl BlockChain { for id in cache_man.cache_usage.pop_back().unwrap().into_iter() { cache_man.in_use.remove(&id); match id { - CacheID::Block(h) => { blocks.remove(&h); }, + CacheID::BlockHeader(h) => { block_headers.remove(&h); }, + CacheID::BlockBody(h) => { block_bodies.remove(&h); }, CacheID::BlockDetails(h) => { block_details.remove(&h); } CacheID::BlockHashes(h) => { block_hashes.remove(&h); } CacheID::TransactionAddresses(h) => { transaction_addresses.remove(&h); } @@ -875,7 +929,8 @@ impl BlockChain { // TODO: handle block_hashes properly. block_hashes.clear(); - blocks.shrink_to_fit(); + block_headers.shrink_to_fit(); + block_bodies.shrink_to_fit(); block_details.shrink_to_fit(); block_hashes.shrink_to_fit(); transaction_addresses.shrink_to_fit(); @@ -888,20 +943,60 @@ impl BlockChain { // TODO: m_lastCollection = chrono::system_clock::now(); } + + /// Create a block body from a block. + pub fn block_to_body(block: &[u8]) -> Bytes { + let mut body = RlpStream::new_list(2); + let block_rlp = Rlp::new(block); + body.append_raw(block_rlp.at(1).as_raw(), 1); + body.append_raw(block_rlp.at(2).as_raw(), 1); + body.out() + } } #[cfg(test)] mod tests { #![cfg_attr(feature="dev", allow(similar_names))] use std::str::FromStr; + use std::sync::Arc; use rustc_serialize::hex::FromHex; + use util::{Database, DatabaseConfig}; use util::hash::*; use util::sha3::Hashable; + use receipt::Receipt; use blockchain::{BlockProvider, BlockChain, Config, ImportRoute}; use tests::helpers::*; use devtools::*; use blockchain::generator::{ChainGenerator, ChainIterator, BlockFinalizer}; use views::BlockView; + use client; + + fn new_db(path: &str) -> Arc { + Arc::new(Database::open(&DatabaseConfig::with_columns(client::DB_NO_OF_COLUMNS), path).unwrap()) + } + + #[test] + fn should_cache_best_block() { + // given + let mut canon_chain = ChainGenerator::default(); + let mut finalizer = BlockFinalizer::default(); + let genesis = canon_chain.generate(&mut finalizer).unwrap(); + let first = canon_chain.generate(&mut finalizer).unwrap(); + + let temp = RandomTempPath::new(); + let db = new_db(temp.as_str()); + let bc = BlockChain::new(Config::default(), &genesis, db.clone()); + assert_eq!(bc.best_block_number(), 0); + + // when + let batch = db.transaction(); + bc.insert_block(&batch, &first, vec![]); + // NOTE no db.write here (we want to check if best block is cached) + + // then + assert_eq!(bc.best_block_number(), 1); + assert!(bc.block(&bc.best_block_hash()).is_some(), "Best block should be queryable even without DB write."); + } #[test] fn basic_blockchain_insert() { @@ -913,16 +1008,18 @@ mod tests { let first_hash = BlockView::new(&first).header_view().sha3(); let temp = RandomTempPath::new(); - let bc = BlockChain::new(Config::default(), &genesis, temp.as_path()); + let db = new_db(temp.as_str()); + let bc = BlockChain::new(Config::default(), &genesis, db.clone()); assert_eq!(bc.genesis_hash(), genesis_hash.clone()); - assert_eq!(bc.best_block_number(), 0); assert_eq!(bc.best_block_hash(), genesis_hash.clone()); assert_eq!(bc.block_hash(0), Some(genesis_hash.clone())); assert_eq!(bc.block_hash(1), None); assert_eq!(bc.block_details(&genesis_hash).unwrap().children, vec![]); - bc.insert_block(&first, vec![]); + let batch = db.transaction(); + bc.insert_block(&batch, &first, vec![]); + db.write(batch).unwrap(); assert_eq!(bc.block_hash(0), Some(genesis_hash.clone())); assert_eq!(bc.best_block_number(), 1); @@ -941,14 +1038,17 @@ mod tests { let genesis_hash = BlockView::new(&genesis).header_view().sha3(); let temp = RandomTempPath::new(); - let bc = BlockChain::new(Config::default(), &genesis, temp.as_path()); + let db = new_db(temp.as_str()); + let bc = BlockChain::new(Config::default(), &genesis, db.clone()); let mut block_hashes = vec![genesis_hash.clone()]; + let batch = db.transaction(); for _ in 0..10 { let block = canon_chain.generate(&mut finalizer).unwrap(); block_hashes.push(BlockView::new(&block).header_view().sha3()); - bc.insert_block(&block, vec![]); + bc.insert_block(&batch, &block, vec![]); } + db.write(batch).unwrap(); block_hashes.reverse(); @@ -973,17 +1073,21 @@ mod tests { let b5a = canon_chain.generate(&mut finalizer).unwrap(); let temp = RandomTempPath::new(); - let bc = BlockChain::new(Config::default(), &genesis, temp.as_path()); - bc.insert_block(&b1a, vec![]); - bc.insert_block(&b1b, vec![]); - bc.insert_block(&b2a, vec![]); - bc.insert_block(&b2b, vec![]); - bc.insert_block(&b3a, vec![]); - bc.insert_block(&b3b, vec![]); - bc.insert_block(&b4a, vec![]); - bc.insert_block(&b4b, vec![]); - bc.insert_block(&b5a, vec![]); - bc.insert_block(&b5b, vec![]); + let db = new_db(temp.as_str()); + let bc = BlockChain::new(Config::default(), &genesis, db.clone()); + + let batch = db.transaction(); + bc.insert_block(&batch, &b1a, vec![]); + bc.insert_block(&batch, &b1b, vec![]); + bc.insert_block(&batch, &b2a, vec![]); + bc.insert_block(&batch, &b2b, vec![]); + bc.insert_block(&batch, &b3a, vec![]); + bc.insert_block(&batch, &b3b, vec![]); + bc.insert_block(&batch, &b4a, vec![]); + bc.insert_block(&batch, &b4b, vec![]); + bc.insert_block(&batch, &b5a, vec![]); + bc.insert_block(&batch, &b5b, vec![]); + db.write(batch).unwrap(); assert_eq!( [&b4b, &b3b, &b2b].iter().map(|b| BlockView::new(b).header()).collect::>(), @@ -1014,11 +1118,17 @@ mod tests { let best_block_hash = b3a_hash.clone(); let temp = RandomTempPath::new(); - let bc = BlockChain::new(Config::default(), &genesis, temp.as_path()); - let ir1 = bc.insert_block(&b1, vec![]); - let ir2 = bc.insert_block(&b2, vec![]); - let ir3b = bc.insert_block(&b3b, vec![]); - let ir3a = bc.insert_block(&b3a, vec![]); + let db = new_db(temp.as_str()); + let bc = BlockChain::new(Config::default(), &genesis, db.clone()); + + let batch = db.transaction(); + let ir1 = bc.insert_block(&batch, &b1, vec![]); + let ir2 = bc.insert_block(&batch, &b2, vec![]); + let ir3b = bc.insert_block(&batch, &b3b, vec![]); + db.write(batch).unwrap(); + let batch = db.transaction(); + let ir3a = bc.insert_block(&batch, &b3a, vec![]); + db.write(batch).unwrap(); assert_eq!(ir1, ImportRoute { enacted: vec![b1_hash], @@ -1119,14 +1229,19 @@ mod tests { let temp = RandomTempPath::new(); { - let bc = BlockChain::new(Config::default(), &genesis, temp.as_path()); + let db = new_db(temp.as_str()); + let bc = BlockChain::new(Config::default(), &genesis, db.clone()); assert_eq!(bc.best_block_hash(), genesis_hash); - bc.insert_block(&first, vec![]); + let batch = db.transaction(); + bc.insert_block(&batch, &first, vec![]); + db.write(batch).unwrap(); assert_eq!(bc.best_block_hash(), first_hash); } { - let bc = BlockChain::new(Config::default(), &genesis, temp.as_path()); + let db = new_db(temp.as_str()); + let bc = BlockChain::new(Config::default(), &genesis, db.clone()); + assert_eq!(bc.best_block_hash(), first_hash); } } @@ -1179,8 +1294,11 @@ mod tests { let b1_hash = H256::from_str("f53f268d23a71e85c7d6d83a9504298712b84c1a2ba220441c86eeda0bf0b6e3").unwrap(); let temp = RandomTempPath::new(); - let bc = BlockChain::new(Config::default(), &genesis, temp.as_path()); - bc.insert_block(&b1, vec![]); + let db = new_db(temp.as_str()); + let bc = BlockChain::new(Config::default(), &genesis, db.clone()); + let batch = db.transaction(); + bc.insert_block(&batch, &b1, vec![]); + db.write(batch).unwrap(); let transactions = bc.transactions(&b1_hash).unwrap(); assert_eq!(transactions.len(), 7); @@ -1189,6 +1307,13 @@ mod tests { } } + fn insert_block(db: &Arc, bc: &BlockChain, bytes: &[u8], receipts: Vec) -> ImportRoute { + let batch = db.transaction(); + let res = bc.insert_block(&batch, bytes, receipts); + db.write(batch).unwrap(); + res + } + #[test] fn test_bloom_filter_simple() { // TODO: From here @@ -1210,27 +1335,28 @@ mod tests { let b2a = canon_chain.with_bloom(bloom_ba.clone()).generate(&mut finalizer).unwrap(); let temp = RandomTempPath::new(); - let bc = BlockChain::new(Config::default(), &genesis, temp.as_path()); + let db = new_db(temp.as_str()); + let bc = BlockChain::new(Config::default(), &genesis, db.clone()); let blocks_b1 = bc.blocks_with_bloom(&bloom_b1, 0, 5); let blocks_b2 = bc.blocks_with_bloom(&bloom_b2, 0, 5); assert_eq!(blocks_b1, vec![]); assert_eq!(blocks_b2, vec![]); - bc.insert_block(&b1, vec![]); + insert_block(&db, &bc, &b1, vec![]); let blocks_b1 = bc.blocks_with_bloom(&bloom_b1, 0, 5); let blocks_b2 = bc.blocks_with_bloom(&bloom_b2, 0, 5); assert_eq!(blocks_b1, vec![1]); assert_eq!(blocks_b2, vec![]); - bc.insert_block(&b2, vec![]); + insert_block(&db, &bc, &b2, vec![]); let blocks_b1 = bc.blocks_with_bloom(&bloom_b1, 0, 5); let blocks_b2 = bc.blocks_with_bloom(&bloom_b2, 0, 5); assert_eq!(blocks_b1, vec![1]); assert_eq!(blocks_b2, vec![2]); // hasn't been forked yet - bc.insert_block(&b1a, vec![]); + insert_block(&db, &bc, &b1a, vec![]); let blocks_b1 = bc.blocks_with_bloom(&bloom_b1, 0, 5); let blocks_b2 = bc.blocks_with_bloom(&bloom_b2, 0, 5); let blocks_ba = bc.blocks_with_bloom(&bloom_ba, 0, 5); @@ -1239,7 +1365,7 @@ mod tests { assert_eq!(blocks_ba, vec![]); // fork has happend - bc.insert_block(&b2a, vec![]); + insert_block(&db, &bc, &b2a, vec![]); let blocks_b1 = bc.blocks_with_bloom(&bloom_b1, 0, 5); let blocks_b2 = bc.blocks_with_bloom(&bloom_b2, 0, 5); let blocks_ba = bc.blocks_with_bloom(&bloom_ba, 0, 5); @@ -1248,7 +1374,7 @@ mod tests { assert_eq!(blocks_ba, vec![1, 2]); // fork back - bc.insert_block(&b3, vec![]); + insert_block(&db, &bc, &b3, vec![]); let blocks_b1 = bc.blocks_with_bloom(&bloom_b1, 0, 5); let blocks_b2 = bc.blocks_with_bloom(&bloom_b2, 0, 5); let blocks_ba = bc.blocks_with_bloom(&bloom_ba, 0, 5); @@ -1266,21 +1392,25 @@ mod tests { let temp = RandomTempPath::new(); { - let bc = BlockChain::new(Config::default(), &genesis, temp.as_path()); + let db = new_db(temp.as_str()); + let bc = BlockChain::new(Config::default(), &genesis, db.clone()); let uncle = canon_chain.fork(1).generate(&mut finalizer.fork()).unwrap(); + let batch = db.transaction(); // create a longer fork for _ in 0..5 { let canon_block = canon_chain.generate(&mut finalizer).unwrap(); - bc.insert_block(&canon_block, vec![]); + bc.insert_block(&batch, &canon_block, vec![]); } assert_eq!(bc.best_block_number(), 5); - bc.insert_block(&uncle, vec![]); + bc.insert_block(&batch, &uncle, vec![]); + db.write(batch).unwrap(); } // re-loading the blockchain should load the correct best block. - let bc = BlockChain::new(Config::default(), &genesis, temp.as_path()); + let db = new_db(temp.as_str()); + let bc = BlockChain::new(Config::default(), &genesis, db.clone()); assert_eq!(bc.best_block_number(), 5); } @@ -1296,10 +1426,13 @@ mod tests { let second_hash = BlockView::new(&second).header_view().sha3(); let temp = RandomTempPath::new(); - let bc = BlockChain::new(Config::default(), &genesis, temp.as_path()); + let db = new_db(temp.as_str()); + let bc = BlockChain::new(Config::default(), &genesis, db.clone()); - bc.insert_block(&first, vec![]); - bc.insert_block(&second, vec![]); + let batch = db.transaction(); + bc.insert_block(&batch, &first, vec![]); + bc.insert_block(&batch, &second, vec![]); + db.write(batch).unwrap(); assert_eq!(bc.rewind(), Some(first_hash.clone())); assert!(!bc.is_known(&second_hash)); diff --git a/ethcore/src/blockchain/update.rs b/ethcore/src/blockchain/update.rs index 029d0d377..962365338 100644 --- a/ethcore/src/blockchain/update.rs +++ b/ethcore/src/blockchain/update.rs @@ -6,9 +6,11 @@ use blooms::BloomGroup; use super::extras::{BlockDetails, BlockReceipts, TransactionAddress, LogGroupPosition}; /// Block extras update info. -pub struct ExtrasUpdate { +pub struct ExtrasUpdate<'a> { /// Block info. pub info: BlockInfo, + /// Current block uncompressed rlp bytes + pub block: &'a [u8], /// Modified block hashes. pub block_hashes: HashMap, /// Modified block details. diff --git a/ethcore/src/client/client.rs b/ethcore/src/client/client.rs index bf0196cb5..aa23367e5 100644 --- a/ethcore/src/client/client.rs +++ b/ethcore/src/client/client.rs @@ -24,9 +24,9 @@ use std::time::{Instant}; use time::precise_time_ns; // util -use util::{journaldb, rlp, Bytes, Stream, View, PerfTimer, Itertools, Mutex, RwLock}; +use util::{journaldb, rlp, Bytes, View, PerfTimer, Itertools, Mutex, RwLock}; use util::journaldb::JournalDB; -use util::rlp::{RlpStream, Rlp, UntrustedRlp}; +use util::rlp::{UntrustedRlp}; use util::numbers::*; use util::panics::*; use util::io::*; @@ -34,14 +34,13 @@ use util::sha3::*; use util::kvdb::*; // other -use views::BlockView; +use views::{BlockView, HeaderView, BodyView}; use error::{ImportError, ExecutionError, ReplayError, BlockError, ImportResult}; use header::BlockNumber; use state::State; use spec::Spec; use basic_types::Seal; use engines::Engine; -use views::HeaderView; use service::ClientIoMessage; use env_info::LastHashes; use verification; @@ -123,6 +122,7 @@ pub struct Client { chain: Arc, tracedb: Arc>, engine: Arc>, + db: Arc, state_db: Mutex>, block_queue: BlockQueue, report: RwLock, @@ -141,6 +141,19 @@ pub struct Client { } const HISTORY: u64 = 1200; +// database columns +/// Column for State +pub const DB_COL_STATE: Option = Some(0); +/// Column for Block headers +pub const DB_COL_HEADERS: Option = Some(1); +/// Column for Block bodies +pub const DB_COL_BODIES: Option = Some(2); +/// Column for Extras +pub const DB_COL_EXTRA: Option = Some(3); +/// Column for Traces +pub const DB_COL_TRACE: Option = Some(4); +/// Number of columns in DB +pub const DB_NO_OF_COLUMNS: Option = Some(5); /// Append a path element to the given path and return the string. pub fn append_path

(path: P, item: &str) -> String where P: AsRef { @@ -160,36 +173,25 @@ impl Client { ) -> Result, ClientError> { let path = path.to_path_buf(); let gb = spec.genesis_block(); - let chain = Arc::new(BlockChain::new(config.blockchain, &gb, &path)); - let tracedb = Arc::new(try!(TraceDB::new(config.tracing, &path, chain.clone()))); + let mut db_config = DatabaseConfig::with_columns(DB_NO_OF_COLUMNS); + db_config.cache_size = config.db_cache_size; + db_config.compaction = config.db_compaction.compaction_profile(); - let mut state_db_config = match config.db_cache_size { - None => DatabaseConfig::default(), - Some(cache_size) => DatabaseConfig::with_cache(cache_size), - }; - - state_db_config = state_db_config.compaction(config.db_compaction.compaction_profile()); - - let mut state_db = journaldb::new( - &append_path(&path, "state"), - config.pruning, - state_db_config - ); + let db = Arc::new(Database::open(&db_config, &path.to_str().unwrap()).expect("Error opening database")); + let chain = Arc::new(BlockChain::new(config.blockchain, &gb, db.clone())); + let tracedb = Arc::new(try!(TraceDB::new(config.tracing, db.clone(), chain.clone()))); + let mut state_db = journaldb::new(db.clone(), config.pruning, DB_COL_STATE); if state_db.is_empty() && spec.ensure_db_good(state_db.as_hashdb_mut()) { - state_db.commit(0, &spec.genesis_header().hash(), None).expect("Error commiting genesis state to state DB"); + let batch = DBTransaction::new(&db); + state_db.commit(&batch, 0, &spec.genesis_header().hash(), None).expect("Error commiting genesis state to state DB"); + db.write(batch).expect("Error writing genesis state to state DB"); } if !chain.block_header(&chain.best_block_hash()).map_or(true, |h| state_db.contains(h.state_root())) { warn!("State root not found for block #{} ({})", chain.best_block_number(), chain.best_block_hash().hex()); } - /* TODO: enable this once the best block issue is resolved - while !chain.block_header(&chain.best_block_hash()).map_or(true, |h| state_db.contains(h.state_root())) { - warn!("State root not found for block #{} ({}), recovering...", chain.best_block_number(), chain.best_block_hash().hex()); - chain.rewind(); - }*/ - let engine = Arc::new(spec.engine); let block_queue = BlockQueue::new(config.queue, engine.clone(), message_channel.clone()); @@ -204,6 +206,7 @@ impl Client { chain: chain, tracedb: tracedb, engine: engine, + db: db, state_db: Mutex::new(state_db), block_queue: block_queue, report: RwLock::new(Default::default()), @@ -432,21 +435,23 @@ impl Client { //let traces = From::from(block.traces().clone().unwrap_or_else(Vec::new)); + let batch = DBTransaction::new(&self.db); // CHECK! I *think* this is fine, even if the state_root is equal to another // already-imported block of the same number. // TODO: Prove it with a test. - block.drain().commit(number, hash, ancient).expect("State DB commit failed."); + block.drain().commit(&batch, number, hash, ancient).expect("State DB commit failed."); - // And update the chain after commit to prevent race conditions - // (when something is in chain but you are not able to fetch details) - let route = self.chain.insert_block(block_data, receipts); - self.tracedb.import(TraceImportRequest { + let route = self.chain.insert_block(&batch, block_data, receipts); + self.tracedb.import(&batch, TraceImportRequest { traces: traces.into(), block_hash: hash.clone(), block_number: number, enacted: route.enacted.clone(), retracted: route.retracted.len() }); + // Final commit to the DB + self.db.write(batch).expect("State DB write failed."); + self.update_last_hashes(&parent, hash); route } @@ -674,17 +679,17 @@ impl BlockChainClient for Client { fn replay(&self, id: TransactionID, analytics: CallAnalytics) -> Result { let address = try!(self.transaction_address(id).ok_or(ReplayError::TransactionNotFound)); - let block_data = try!(self.block(BlockID::Hash(address.block_hash)).ok_or(ReplayError::StatePruned)); + let header_data = try!(self.block_header(BlockID::Hash(address.block_hash)).ok_or(ReplayError::StatePruned)); + let body_data = try!(self.block_body(BlockID::Hash(address.block_hash)).ok_or(ReplayError::StatePruned)); let mut state = try!(self.state_at_beginning(BlockID::Hash(address.block_hash)).ok_or(ReplayError::StatePruned)); - let block = BlockView::new(&block_data); - let txs = block.transactions(); + let txs = BodyView::new(&body_data).transactions(); if address.index >= txs.len() { return Err(ReplayError::TransactionNotFound); } let options = TransactOptions { tracing: analytics.transaction_tracing, vm_tracing: analytics.vm_tracing, check_nonce: false }; - let view = block.header_view(); + let view = HeaderView::new(&header_data); let last_hashes = self.build_last_hashes(view.hash()); let mut env_info = EnvInfo { number: view.number(), @@ -719,20 +724,16 @@ impl BlockChainClient for Client { } } + fn best_block_header(&self) -> Bytes { + self.chain.best_block_header() + } + fn block_header(&self, id: BlockID) -> Option { - Self::block_hash(&self.chain, id).and_then(|hash| self.chain.block(&hash).map(|bytes| BlockView::new(&bytes).rlp().at(0).as_raw().to_vec())) + Self::block_hash(&self.chain, id).and_then(|hash| self.chain.block_header_data(&hash)) } fn block_body(&self, id: BlockID) -> Option { - Self::block_hash(&self.chain, id).and_then(|hash| { - self.chain.block(&hash).map(|bytes| { - let rlp = Rlp::new(&bytes); - let mut body = RlpStream::new_list(2); - body.append_raw(rlp.at(1).as_raw(), 1); - body.append_raw(rlp.at(2).as_raw(), 1); - body.out() - }) - }) + Self::block_hash(&self.chain, id).and_then(|hash| self.chain.block_body(&hash)) } fn block(&self, id: BlockID) -> Option { @@ -789,13 +790,13 @@ impl BlockChainClient for Client { fn uncle(&self, id: UncleID) -> Option { let index = id.position; - self.block(id.block).and_then(|block| BlockView::new(&block).uncle_rlp_at(index)) + self.block_body(id.block).and_then(|body| BodyView::new(&body).uncle_rlp_at(index)) } fn transaction_receipt(&self, id: TransactionID) -> Option { - self.transaction_address(id).and_then(|address| { - let t = self.chain.block(&address.block_hash) - .and_then(|block| BlockView::new(&block).localized_transaction_at(address.index)); + self.transaction_address(id).and_then(|address| self.chain.block_number(&address.block_hash).and_then(|block_number| { + let t = self.chain.block_body(&address.block_hash) + .and_then(|block| BodyView::new(&block).localized_transaction_at(&address.block_hash, block_number, address.index)); match (t, self.chain.transaction_receipt(&address)) { (Some(tx), Some(receipt)) => { @@ -834,7 +835,7 @@ impl BlockChainClient for Client { }, _ => None } - }) + })) } fn tree_route(&self, from: &H256, to: &H256) -> Option { @@ -910,7 +911,7 @@ impl BlockChainClient for Client { blocks.into_iter() .filter_map(|number| self.chain.block_hash(number).map(|hash| (number, hash))) .filter_map(|(number, hash)| self.chain.block_receipts(&hash).map(|r| (number, hash, r.receipts))) - .filter_map(|(number, hash, receipts)| self.chain.block(&hash).map(|ref b| (number, hash, receipts, BlockView::new(b).transaction_hashes()))) + .filter_map(|(number, hash, receipts)| self.chain.block_body(&hash).map(|ref b| (number, hash, receipts, BodyView::new(b).transaction_hashes()))) .flat_map(|(number, hash, receipts, hashes)| { let mut log_index = 0; receipts.into_iter() diff --git a/ethcore/src/client/test_client.rs b/ethcore/src/client/test_client.rs index 9b13d7be3..cdffe4302 100644 --- a/ethcore/src/client/test_client.rs +++ b/ethcore/src/client/test_client.rs @@ -248,7 +248,8 @@ impl TestBlockChainClient { pub fn get_temp_journal_db() -> GuardedTempResult> { let temp = RandomTempPath::new(); - let journal_db = journaldb::new(temp.as_str(), journaldb::Algorithm::EarlyMerge, DatabaseConfig::default()); + let db = Database::open_default(temp.as_str()).unwrap(); + let journal_db = journaldb::new(Arc::new(db), journaldb::Algorithm::EarlyMerge, None); GuardedTempResult { _temp: temp, result: Some(journal_db) @@ -363,6 +364,10 @@ impl BlockChainClient for TestBlockChainClient { unimplemented!(); } + fn best_block_header(&self) -> Bytes { + self.block_header(BlockID::Hash(self.chain_info().best_block_hash)).expect("Best block always have header.") + } + fn block_header(&self, id: BlockID) -> Option { self.block_hash(id).and_then(|hash| self.blocks.read().get(&hash).map(|r| Rlp::new(r).at(0).as_raw().to_vec())) } diff --git a/ethcore/src/client/traits.rs b/ethcore/src/client/traits.rs index 7d13dd23d..348b90c90 100644 --- a/ethcore/src/client/traits.rs +++ b/ethcore/src/client/traits.rs @@ -145,10 +145,7 @@ pub trait BlockChainClient : Sync + Send { fn chain_info(&self) -> BlockChainInfo; /// Get the best block header. - fn best_block_header(&self) -> Bytes { - // TODO: lock blockchain only once - self.block_header(BlockID::Hash(self.chain_info().best_block_hash)).unwrap() - } + fn best_block_header(&self) -> Bytes; /// Returns numbers of blocks containing given bloom. fn blocks_with_bloom(&self, bloom: &H2048, from_block: BlockID, to_block: BlockID) -> Option>; diff --git a/ethcore/src/db.rs b/ethcore/src/db.rs index 57b4cfdc6..eab0e2eb5 100644 --- a/ethcore/src/db.rs +++ b/ethcore/src/db.rs @@ -62,14 +62,14 @@ pub trait Key { /// Should be used to write value into database. pub trait Writable { /// Writes the value into the database. - fn write(&self, key: &Key, value: &T) where T: Encodable, R: Deref; + fn write(&self, col: Option, key: &Key, value: &T) where T: Encodable, R: Deref; /// Writes the value into the database and updates the cache. - fn write_with_cache(&self, cache: &mut Cache, key: K, value: T, policy: CacheUpdatePolicy) where + fn write_with_cache(&self, col: Option, cache: &mut Cache, key: K, value: T, policy: CacheUpdatePolicy) where K: Key + Hash + Eq, T: Encodable, R: Deref { - self.write(&key, &value); + self.write(col, &key, &value); match policy { CacheUpdatePolicy::Overwrite => { cache.insert(key, value); @@ -81,20 +81,20 @@ pub trait Writable { } /// Writes the values into the database and updates the cache. - fn extend_with_cache(&self, cache: &mut Cache, values: HashMap, policy: CacheUpdatePolicy) where + fn extend_with_cache(&self, col: Option, cache: &mut Cache, values: HashMap, policy: CacheUpdatePolicy) where K: Key + Hash + Eq, T: Encodable, R: Deref { match policy { CacheUpdatePolicy::Overwrite => { for (key, value) in values.into_iter() { - self.write(&key, &value); + self.write(col, &key, &value); cache.insert(key, value); } }, CacheUpdatePolicy::Remove => { for (key, value) in &values { - self.write(key, value); + self.write(col, key, value); cache.remove(key); } }, @@ -105,12 +105,12 @@ pub trait Writable { /// Should be used to read values from database. pub trait Readable { /// Returns value for given key. - fn read(&self, key: &Key) -> Option where + fn read(&self, col: Option, key: &Key) -> Option where T: Decodable, R: Deref; /// Returns value for given key either in cache or in database. - fn read_with_cache(&self, cache: &RwLock, key: &K) -> Option where + fn read_with_cache(&self, col: Option, cache: &RwLock, key: &K) -> Option where K: Key + Eq + Hash + Clone, T: Clone + Decodable, C: Cache { @@ -121,7 +121,7 @@ pub trait Readable { } } - self.read(key).map(|value: T|{ + self.read(col, key).map(|value: T|{ let mut write = cache.write(); write.insert(key.clone(), value.clone()); value @@ -129,10 +129,10 @@ pub trait Readable { } /// Returns true if given value exists. - fn exists(&self, key: &Key) -> bool where R: Deref; + fn exists(&self, col: Option, key: &Key) -> bool where R: Deref; /// Returns true if given value exists either in cache or in database. - fn exists_with_cache(&self, cache: &RwLock, key: &K) -> bool where + fn exists_with_cache(&self, col: Option, cache: &RwLock, key: &K) -> bool where K: Eq + Hash + Key, R: Deref, C: Cache { @@ -143,13 +143,13 @@ pub trait Readable { } } - self.exists::(key) + self.exists::(col, key) } } impl Writable for DBTransaction { - fn write(&self, key: &Key, value: &T) where T: Encodable, R: Deref { - let result = self.put(&key.key(), &encode(value)); + fn write(&self, col: Option, key: &Key, value: &T) where T: Encodable, R: Deref { + let result = self.put(col, &key.key(), &encode(value)); if let Err(err) = result { panic!("db put failed, key: {:?}, err: {:?}", &key.key() as &[u8], err); } @@ -157,8 +157,8 @@ impl Writable for DBTransaction { } impl Readable for Database { - fn read(&self, key: &Key) -> Option where T: Decodable, R: Deref { - let result = self.get(&key.key()); + fn read(&self, col: Option, key: &Key) -> Option where T: Decodable, R: Deref { + let result = self.get(col, &key.key()); match result { Ok(option) => option.map(|v| decode(&v)), @@ -168,8 +168,8 @@ impl Readable for Database { } } - fn exists(&self, key: &Key) -> bool where R: Deref { - let result = self.get(&key.key()); + fn exists(&self, col: Option, key: &Key) -> bool where R: Deref { + let result = self.get(col, &key.key()); match result { Ok(v) => v.is_some(), diff --git a/ethcore/src/json_tests/executive.rs b/ethcore/src/json_tests/executive.rs index f1be3e4e0..adba16703 100644 --- a/ethcore/src/json_tests/executive.rs +++ b/ethcore/src/json_tests/executive.rs @@ -38,7 +38,7 @@ struct CallCreate { impl From for CallCreate { fn from(c: ethjson::vm::Call) -> Self { - let dst: Option<_> = c.destination.into(); + let dst: Option = c.destination.into(); CallCreate { data: c.data.into(), destination: dst.map(Into::into), diff --git a/ethcore/src/json_tests/transaction.rs b/ethcore/src/json_tests/transaction.rs index 673ff8650..7c9a3327e 100644 --- a/ethcore/src/json_tests/transaction.rs +++ b/ethcore/src/json_tests/transaction.rs @@ -49,7 +49,7 @@ fn do_json_test(json_data: &[u8]) -> Vec { fail_unless(t.gas_price == tx.gas_price.into()); fail_unless(t.nonce == tx.nonce.into()); fail_unless(t.value == tx.value.into()); - let to: Option<_> = tx.to.into(); + let to: Option = tx.to.into(); let to: Option

= to.map(Into::into); match t.action { Action::Call(dest) => fail_unless(Some(dest) == to), diff --git a/ethcore/src/migrations/blocks/v8.rs b/ethcore/src/migrations/blocks/v8.rs index 041ceaac8..798be0790 100644 --- a/ethcore/src/migrations/blocks/v8.rs +++ b/ethcore/src/migrations/blocks/v8.rs @@ -16,19 +16,22 @@ //! This migration compresses the state db. -use util::migration::SimpleMigration; +use util::migration::{SimpleMigration, Progress}; use util::rlp::{Compressible, UntrustedRlp, View, RlpType}; /// Compressing migration. #[derive(Default)] -pub struct V8; +pub struct V8(Progress); impl SimpleMigration for V8 { fn version(&self) -> u32 { 8 } + fn columns(&self) -> Option { None } + fn simple_migrate(&mut self, key: Vec, value: Vec) -> Option<(Vec, Vec)> { + self.0.tick(); Some((key,UntrustedRlp::new(&value).compress(RlpType::Blocks).to_vec())) } } diff --git a/ethcore/src/migrations/extras/v6.rs b/ethcore/src/migrations/extras/v6.rs index af2d0389b..9b746b9d2 100644 --- a/ethcore/src/migrations/extras/v6.rs +++ b/ethcore/src/migrations/extras/v6.rs @@ -34,9 +34,10 @@ impl ToV6 { } impl SimpleMigration for ToV6 { - fn version(&self) -> u32 { - 6 - } + + fn columns(&self) -> Option { None } + + fn version(&self) -> u32 { 6 } fn simple_migrate(&mut self, key: Vec, value: Vec) -> Option<(Vec, Vec)> { diff --git a/ethcore/src/migrations/mod.rs b/ethcore/src/migrations/mod.rs index e3e1f4031..5c0c6f420 100644 --- a/ethcore/src/migrations/mod.rs +++ b/ethcore/src/migrations/mod.rs @@ -19,3 +19,7 @@ pub mod state; pub mod blocks; pub mod extras; + +mod v9; +pub use self::v9::ToV9; +pub use self::v9::Extract; diff --git a/ethcore/src/migrations/state/v7.rs b/ethcore/src/migrations/state/v7.rs index faa289bd7..036ff707c 100644 --- a/ethcore/src/migrations/state/v7.rs +++ b/ethcore/src/migrations/state/v7.rs @@ -22,7 +22,7 @@ use std::collections::HashMap; use util::Bytes; use util::hash::{Address, FixedHash, H256}; use util::kvdb::Database; -use util::migration::{Batch, Config, Error, Migration, SimpleMigration}; +use util::migration::{Batch, Config, Error, Migration, SimpleMigration, Progress}; use util::rlp::{decode, Rlp, RlpStream, Stream, View}; use util::sha3::Hashable; @@ -63,19 +63,16 @@ fn attempt_migrate(mut key_h: H256, val: &[u8]) -> Option { /// Version for `ArchiveDB`. #[derive(Default)] -pub struct ArchiveV7(usize); +pub struct ArchiveV7(Progress); impl SimpleMigration for ArchiveV7 { - fn version(&self) -> u32 { - 7 - } + + fn columns(&self) -> Option { None } + + fn version(&self) -> u32 { 7 } fn simple_migrate(&mut self, key: Vec, value: Vec) -> Option<(Vec, Vec)> { - self.0 += 1; - if self.0 == 100_000 { - self.0 = 0; - flush!("."); - } + self.0.tick(); if key.len() != 32 { // metadata key, ignore. @@ -109,7 +106,7 @@ impl OverlayRecentV7 { // walk all journal entries in the database backwards. // find migrations for any possible inserted keys. fn walk_journal(&mut self, source: &Database) -> Result<(), Error> { - if let Some(val) = try!(source.get(V7_LATEST_ERA_KEY).map_err(Error::Custom)) { + if let Some(val) = try!(source.get(None, V7_LATEST_ERA_KEY).map_err(Error::Custom)) { let mut era = decode::(&val); loop { let mut index: usize = 0; @@ -120,7 +117,7 @@ impl OverlayRecentV7 { r.out() }; - if let Some(journal_raw) = try!(source.get(&entry_key).map_err(Error::Custom)) { + if let Some(journal_raw) = try!(source.get(None, &entry_key).map_err(Error::Custom)) { let rlp = Rlp::new(&journal_raw); // migrate all inserted keys. @@ -153,7 +150,7 @@ impl OverlayRecentV7 { // replace all possible inserted/deleted keys with their migrated counterparts // and commit the altered entries. fn migrate_journal(&self, source: &Database, mut batch: Batch, dest: &mut Database) -> Result<(), Error> { - if let Some(val) = try!(source.get(V7_LATEST_ERA_KEY).map_err(Error::Custom)) { + if let Some(val) = try!(source.get(None, V7_LATEST_ERA_KEY).map_err(Error::Custom)) { try!(batch.insert(V7_LATEST_ERA_KEY.into(), val.to_owned(), dest)); let mut era = decode::(&val); @@ -166,7 +163,7 @@ impl OverlayRecentV7 { r.out() }; - if let Some(journal_raw) = try!(source.get(&entry_key).map_err(Error::Custom)) { + if let Some(journal_raw) = try!(source.get(None, &entry_key).map_err(Error::Custom)) { let rlp = Rlp::new(&journal_raw); let id: H256 = rlp.val_at(0); let mut inserted_keys: Vec<(H256, Bytes)> = Vec::new(); @@ -221,22 +218,25 @@ impl OverlayRecentV7 { } impl Migration for OverlayRecentV7 { + + fn columns(&self) -> Option { None } + fn version(&self) -> u32 { 7 } // walk all records in the database, attempting to migrate any possible and // keeping records of those that we do. then migrate the journal using // this information. - fn migrate(&mut self, source: &Database, config: &Config, dest: &mut Database) -> Result<(), Error> { - let mut batch = Batch::new(config); + fn migrate(&mut self, source: &Database, config: &Config, dest: &mut Database, col: Option) -> Result<(), Error> { + let mut batch = Batch::new(config, col); // check version metadata. - match try!(source.get(V7_VERSION_KEY).map_err(Error::Custom)) { + match try!(source.get(None, V7_VERSION_KEY).map_err(Error::Custom)) { Some(ref version) if decode::(&*version) == DB_VERSION => {} _ => return Err(Error::MigrationImpossible), // missing or wrong version } let mut count = 0; - for (key, value) in source.iter() { + for (key, value) in source.iter(None) { count += 1; if count == 100_000 { count = 0; diff --git a/ethcore/src/migrations/v9.rs b/ethcore/src/migrations/v9.rs new file mode 100644 index 000000000..0c8e77588 --- /dev/null +++ b/ethcore/src/migrations/v9.rs @@ -0,0 +1,82 @@ +// Copyright 2015, 2016 Ethcore (UK) Ltd. +// This file is part of Parity. + +// Parity is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Parity is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Parity. If not, see . + + +//! This migration consolidates all databases into single one using Column Families. + +use util::{Rlp, RlpStream, View, Stream}; +use util::kvdb::Database; +use util::migration::{Batch, Config, Error, Migration, Progress}; + +/// Which part of block to preserve +pub enum Extract { + /// Extract block header RLP. + Header, + /// Extract block body RLP. + Body, + /// Don't change the value. + All, +} + +/// Consolidation of extras/block/state databases into single one. +pub struct ToV9 { + progress: Progress, + column: Option, + extract: Extract, +} + +impl ToV9 { + /// Creates new V9 migration and assigns all `(key,value)` pairs from `source` DB to given Column Family + pub fn new(column: Option, extract: Extract) -> Self { + ToV9 { + progress: Progress::default(), + column: column, + extract: extract, + } + } +} + +impl Migration for ToV9 { + + fn columns(&self) -> Option { Some(5) } + + fn version(&self) -> u32 { 9 } + + fn migrate(&mut self, source: &Database, config: &Config, dest: &mut Database, col: Option) -> Result<(), Error> { + let mut batch = Batch::new(config, self.column); + + for (key, value) in source.iter(col) { + self.progress.tick(); + match self.extract { + Extract::Header => { + try!(batch.insert(key.to_vec(), Rlp::new(&value).at(0).as_raw().to_vec(), dest)) + }, + Extract::Body => { + let mut body = RlpStream::new_list(2); + let block_rlp = Rlp::new(&value); + body.append_raw(block_rlp.at(1).as_raw(), 1); + body.append_raw(block_rlp.at(2).as_raw(), 1); + try!(batch.insert(key.to_vec(), body.out(), dest)) + }, + Extract::All => { + try!(batch.insert(key.to_vec(), value.to_vec(), dest)) + } + } + } + + batch.commit(dest) + } +} diff --git a/ethcore/src/snapshot/mod.rs b/ethcore/src/snapshot/mod.rs index 08ba2ca24..47d95722a 100644 --- a/ethcore/src/snapshot/mod.rs +++ b/ethcore/src/snapshot/mod.rs @@ -27,7 +27,8 @@ use error::Error; use ids::BlockID; use views::{BlockView, HeaderView}; -use util::{Bytes, Hashable, HashDB, JournalDB, snappy, TrieDB, TrieDBMut, TrieMut}; +use util::{Bytes, Hashable, HashDB, JournalDB, snappy, TrieDB, TrieDBMut, TrieMut, DBTransaction}; +use util::error::UtilError; use util::hash::{FixedHash, H256}; use util::rlp::{DecoderError, RlpStream, Stream, UntrustedRlp, View, Compressible, RlpType}; @@ -359,7 +360,9 @@ impl StateRebuilder { try!(rebuild_account_trie(db.as_hashdb_mut(), account_chunk, out_pairs_chunk)); // commit the db changes we made in this thread. - try!(db.commit(0, &H256::zero(), None)); + let batch = DBTransaction::new(&db.backing()); + try!(db.commit(&batch, 0, &H256::zero(), None)); + try!(db.backing().write(batch).map_err(UtilError::SimpleString)); Ok(()) }); @@ -388,7 +391,9 @@ impl StateRebuilder { } } - try!(self.db.commit(0, &H256::zero(), None)); + let batch = DBTransaction::new(&self.db.backing()); + try!(self.db.commit(&batch, 0, &H256::zero(), None)); + try!(self.db.backing().write(batch).map_err(|e| Error::Util(e.into()))); Ok(()) } diff --git a/ethcore/src/tests/helpers.rs b/ethcore/src/tests/helpers.rs index fe717aa09..b88c773c2 100644 --- a/ethcore/src/tests/helpers.rs +++ b/ethcore/src/tests/helpers.rs @@ -14,7 +14,7 @@ // You should have received a copy of the GNU General Public License // along with Parity. If not, see . -use client::{BlockChainClient, Client, ClientConfig}; +use client::{self, BlockChainClient, Client, ClientConfig}; use common::*; use spec::*; use block::{OpenBlock, Drain}; @@ -246,12 +246,23 @@ pub fn get_test_client_with_blocks(blocks: Vec) -> GuardedTempResult Arc { + Arc::new( + Database::open(&DatabaseConfig::with_columns(client::DB_NO_OF_COLUMNS), path) + .expect("Opening database for tests should always work.") + ) +} + pub fn generate_dummy_blockchain(block_number: u32) -> GuardedTempResult { let temp = RandomTempPath::new(); - let bc = BlockChain::new(BlockChainConfig::default(), &create_unverifiable_block(0, H256::zero()), temp.as_path()); + let db = new_db(temp.as_str()); + let bc = BlockChain::new(BlockChainConfig::default(), &create_unverifiable_block(0, H256::zero()), db.clone()); + + let batch = db.transaction(); for block_order in 1..block_number { - bc.insert_block(&create_unverifiable_block(block_order, bc.best_block_hash()), vec![]); + bc.insert_block(&batch, &create_unverifiable_block(block_order, bc.best_block_hash()), vec![]); } + db.write(batch).unwrap(); GuardedTempResult:: { _temp: temp, @@ -261,10 +272,15 @@ pub fn generate_dummy_blockchain(block_number: u32) -> GuardedTempResult GuardedTempResult { let temp = RandomTempPath::new(); - let bc = BlockChain::new(BlockChainConfig::default(), &create_unverifiable_block(0, H256::zero()), temp.as_path()); + let db = new_db(temp.as_str()); + let bc = BlockChain::new(BlockChainConfig::default(), &create_unverifiable_block(0, H256::zero()), db.clone()); + + + let batch = db.transaction(); for block_order in 1..block_number { - bc.insert_block(&create_unverifiable_block_with_extra(block_order, bc.best_block_hash(), None), vec![]); + bc.insert_block(&batch, &create_unverifiable_block_with_extra(block_order, bc.best_block_hash(), None), vec![]); } + db.write(batch).unwrap(); GuardedTempResult:: { _temp: temp, @@ -274,7 +290,8 @@ pub fn generate_dummy_blockchain_with_extra(block_number: u32) -> GuardedTempRes pub fn generate_dummy_empty_blockchain() -> GuardedTempResult { let temp = RandomTempPath::new(); - let bc = BlockChain::new(BlockChainConfig::default(), &create_unverifiable_block(0, H256::zero()), temp.as_path()); + let db = new_db(temp.as_str()); + let bc = BlockChain::new(BlockChainConfig::default(), &create_unverifiable_block(0, H256::zero()), db.clone()); GuardedTempResult:: { _temp: temp, @@ -284,7 +301,8 @@ pub fn generate_dummy_empty_blockchain() -> GuardedTempResult { pub fn get_temp_journal_db() -> GuardedTempResult> { let temp = RandomTempPath::new(); - let journal_db = journaldb::new(temp.as_str(), journaldb::Algorithm::EarlyMerge, DatabaseConfig::default()); + let journal_db = get_temp_journal_db_in(temp.as_path()); + GuardedTempResult { _temp: temp, result: Some(journal_db) @@ -294,6 +312,7 @@ pub fn get_temp_journal_db() -> GuardedTempResult> { pub fn get_temp_state() -> GuardedTempResult { let temp = RandomTempPath::new(); let journal_db = get_temp_journal_db_in(temp.as_path()); + GuardedTempResult { _temp: temp, result: Some(State::new(journal_db, U256::from(0), Default::default())), @@ -301,7 +320,8 @@ pub fn get_temp_state() -> GuardedTempResult { } pub fn get_temp_journal_db_in(path: &Path) -> Box { - journaldb::new(path.to_str().unwrap(), journaldb::Algorithm::EarlyMerge, DatabaseConfig::default()) + let db = new_db(path.to_str().expect("Only valid utf8 paths for tests.")); + journaldb::new(db.clone(), journaldb::Algorithm::EarlyMerge, None) } pub fn get_temp_state_in(path: &Path) -> State { diff --git a/ethcore/src/trace/db.rs b/ethcore/src/trace/db.rs index 8fb0f531f..c0dab5d17 100644 --- a/ethcore/src/trace/db.rs +++ b/ethcore/src/trace/db.rs @@ -18,15 +18,15 @@ use std::ops::{Deref, DerefMut}; use std::collections::HashMap; use std::sync::Arc; -use std::path::Path; use bloomchain::{Number, Config as BloomConfig}; use bloomchain::group::{BloomGroupDatabase, BloomGroupChain, GroupPosition, BloomGroup}; -use util::{H256, H264, Database, DatabaseConfig, DBTransaction, RwLock}; +use util::{H256, H264, Database, DBTransaction, RwLock}; use header::BlockNumber; use trace::{LocalizedTrace, Config, Switch, Filter, Database as TraceDatabase, ImportRequest, DatabaseExtras, Error}; use db::{Key, Writable, Readable, CacheUpdatePolicy}; use blooms; use super::flat::{FlatTrace, FlatBlockTraces, FlatTransactionTraces}; +use client::DB_COL_TRACE; const TRACE_DB_VER: &'static [u8] = b"1.0"; @@ -94,7 +94,7 @@ pub struct TraceDB where T: DatabaseExtras { traces: RwLock>, blooms: RwLock>, // db - tracesdb: Database, + tracesdb: Arc, // config, bloom_config: BloomConfig, // tracing enabled @@ -106,24 +106,15 @@ pub struct TraceDB where T: DatabaseExtras { impl BloomGroupDatabase for TraceDB where T: DatabaseExtras { fn blooms_at(&self, position: &GroupPosition) -> Option { let position = TraceGroupPosition::from(position.clone()); - self.tracesdb.read_with_cache(&self.blooms, &position).map(Into::into) + self.tracesdb.read_with_cache(DB_COL_TRACE, &self.blooms, &position).map(Into::into) } } impl TraceDB where T: DatabaseExtras { /// Creates new instance of `TraceDB`. - pub fn new(config: Config, path: &Path, extras: Arc) -> Result { - let mut tracedb_path = path.to_path_buf(); - tracedb_path.push("tracedb"); - let tracesdb = match config.db_cache_size { - None => Database::open_default(tracedb_path.to_str().unwrap()).unwrap(), - Some(db_cache) => Database::open( - &DatabaseConfig::with_cache(db_cache), - tracedb_path.to_str().unwrap()).unwrap(), - }; - + pub fn new(config: Config, tracesdb: Arc, extras: Arc) -> Result { // check if in previously tracing was enabled - let old_tracing = match tracesdb.get(b"enabled").unwrap() { + let old_tracing = match tracesdb.get(DB_COL_TRACE, b"enabled").unwrap() { Some(ref value) if value as &[u8] == &[0x1] => Switch::On, Some(ref value) if value as &[u8] == &[0x0] => Switch::Off, Some(_) => { panic!("tracesdb is corrupted") }, @@ -137,8 +128,10 @@ impl TraceDB where T: DatabaseExtras { false => [0x0] }; - tracesdb.put(b"enabled", &encoded_tracing).unwrap(); - tracesdb.put(b"version", TRACE_DB_VER).unwrap(); + let batch = DBTransaction::new(&tracesdb); + batch.put(DB_COL_TRACE, b"enabled", &encoded_tracing).unwrap(); + batch.put(DB_COL_TRACE, b"version", TRACE_DB_VER).unwrap(); + tracesdb.write(batch).unwrap(); let db = TraceDB { traces: RwLock::new(HashMap::new()), @@ -154,7 +147,7 @@ impl TraceDB where T: DatabaseExtras { /// Returns traces for block with hash. fn traces(&self, block_hash: &H256) -> Option { - self.tracesdb.read_with_cache(&self.traces, block_hash) + self.tracesdb.read_with_cache(DB_COL_TRACE, &self.traces, block_hash) } /// Returns vector of transaction traces for given block. @@ -217,20 +210,18 @@ impl TraceDatabase for TraceDB where T: DatabaseExtras { /// Traces of import request's enacted blocks are expected to be already in database /// or to be the currently inserted trace. - fn import(&self, request: ImportRequest) { + fn import(&self, batch: &DBTransaction, request: ImportRequest) { // fast return if tracing is disabled if !self.tracing_enabled() { return; } - let batch = DBTransaction::new(); - // at first, let's insert new block traces { let mut traces = self.traces.write(); // it's important to use overwrite here, // cause this value might be queried by hash later - batch.write_with_cache(traces.deref_mut(), request.block_hash, request.traces, CacheUpdatePolicy::Overwrite); + batch.write_with_cache(DB_COL_TRACE, traces.deref_mut(), request.block_hash, request.traces, CacheUpdatePolicy::Overwrite); } // now let's rebuild the blooms @@ -256,10 +247,8 @@ impl TraceDatabase for TraceDB where T: DatabaseExtras { .collect::>(); let mut blooms = self.blooms.write(); - batch.extend_with_cache(blooms.deref_mut(), blooms_to_insert, CacheUpdatePolicy::Remove); + batch.extend_with_cache(DB_COL_TRACE, blooms.deref_mut(), blooms_to_insert, CacheUpdatePolicy::Remove); } - - self.tracesdb.write(batch).unwrap(); } fn trace(&self, block_number: BlockNumber, tx_position: usize, trace_position: Vec) -> Option { @@ -362,13 +351,14 @@ impl TraceDatabase for TraceDB where T: DatabaseExtras { mod tests { use std::collections::HashMap; use std::sync::Arc; - use util::{Address, U256, H256}; + use util::{Address, U256, H256, Database, DatabaseConfig, DBTransaction}; use devtools::RandomTempPath; use header::BlockNumber; - use trace::{Config, Switch, TraceDB, Database, DatabaseExtras, ImportRequest}; + use trace::{Config, Switch, TraceDB, Database as TraceDatabase, DatabaseExtras, ImportRequest}; use trace::{Filter, LocalizedTrace, AddressesFilter}; use trace::trace::{Call, Action, Res}; use trace::flat::{FlatTrace, FlatBlockTraces, FlatTransactionTraces}; + use client::DB_NO_OF_COLUMNS; use types::executed::CallType; struct NoopExtras; @@ -408,28 +398,33 @@ mod tests { } } + fn new_db(path: &str) -> Arc { + Arc::new(Database::open(&DatabaseConfig::with_columns(DB_NO_OF_COLUMNS), path).unwrap()) + } + #[test] fn test_reopening_db_with_tracing_off() { let temp = RandomTempPath::new(); + let db = new_db(temp.as_str()); let mut config = Config::default(); // set autotracing config.enabled = Switch::Auto; { - let tracedb = TraceDB::new(config.clone(), temp.as_path(), Arc::new(NoopExtras)).unwrap(); + let tracedb = TraceDB::new(config.clone(), db.clone(), Arc::new(NoopExtras)).unwrap(); assert_eq!(tracedb.tracing_enabled(), false); } { - let tracedb = TraceDB::new(config.clone(), temp.as_path(), Arc::new(NoopExtras)).unwrap(); + let tracedb = TraceDB::new(config.clone(), db.clone(), Arc::new(NoopExtras)).unwrap(); assert_eq!(tracedb.tracing_enabled(), false); } config.enabled = Switch::Off; { - let tracedb = TraceDB::new(config.clone(), temp.as_path(), Arc::new(NoopExtras)).unwrap(); + let tracedb = TraceDB::new(config.clone(), db.clone(), Arc::new(NoopExtras)).unwrap(); assert_eq!(tracedb.tracing_enabled(), false); } } @@ -437,32 +432,33 @@ mod tests { #[test] fn test_reopening_db_with_tracing_on() { let temp = RandomTempPath::new(); + let db = new_db(temp.as_str()); let mut config = Config::default(); // set tracing on config.enabled = Switch::On; { - let tracedb = TraceDB::new(config.clone(), temp.as_path(), Arc::new(NoopExtras)).unwrap(); + let tracedb = TraceDB::new(config.clone(), db.clone(), Arc::new(NoopExtras)).unwrap(); assert_eq!(tracedb.tracing_enabled(), true); } { - let tracedb = TraceDB::new(config.clone(), temp.as_path(), Arc::new(NoopExtras)).unwrap(); + let tracedb = TraceDB::new(config.clone(), db.clone(), Arc::new(NoopExtras)).unwrap(); assert_eq!(tracedb.tracing_enabled(), true); } config.enabled = Switch::Auto; { - let tracedb = TraceDB::new(config.clone(), temp.as_path(), Arc::new(NoopExtras)).unwrap(); + let tracedb = TraceDB::new(config.clone(), db.clone(), Arc::new(NoopExtras)).unwrap(); assert_eq!(tracedb.tracing_enabled(), true); } config.enabled = Switch::Off; { - let tracedb = TraceDB::new(config.clone(), temp.as_path(), Arc::new(NoopExtras)).unwrap(); + let tracedb = TraceDB::new(config.clone(), db.clone(), Arc::new(NoopExtras)).unwrap(); assert_eq!(tracedb.tracing_enabled(), false); } } @@ -471,18 +467,19 @@ mod tests { #[should_panic] fn test_invalid_reopening_db() { let temp = RandomTempPath::new(); + let db = new_db(temp.as_str()); let mut config = Config::default(); // set tracing on config.enabled = Switch::Off; { - let tracedb = TraceDB::new(config.clone(), temp.as_path(), Arc::new(NoopExtras)).unwrap(); + let tracedb = TraceDB::new(config.clone(), db.clone(), Arc::new(NoopExtras)).unwrap(); assert_eq!(tracedb.tracing_enabled(), true); } config.enabled = Switch::On; - TraceDB::new(config.clone(), temp.as_path(), Arc::new(NoopExtras)).unwrap(); // should panic! + TraceDB::new(config.clone(), db.clone(), Arc::new(NoopExtras)).unwrap(); // should panic! } fn create_simple_import_request(block_number: BlockNumber, block_hash: H256) -> ImportRequest { @@ -531,6 +528,7 @@ mod tests { #[test] fn test_import() { let temp = RandomTempPath::new(); + let db = Arc::new(Database::open(&DatabaseConfig::with_columns(DB_NO_OF_COLUMNS), temp.as_str()).unwrap()); let mut config = Config::default(); config.enabled = Switch::On; let block_0 = H256::from(0xa1); @@ -544,11 +542,13 @@ mod tests { extras.transaction_hashes.insert(0, vec![tx_0.clone()]); extras.transaction_hashes.insert(1, vec![tx_1.clone()]); - let tracedb = TraceDB::new(config, temp.as_path(), Arc::new(extras)).unwrap(); + let tracedb = TraceDB::new(config, db.clone(), Arc::new(extras)).unwrap(); // import block 0 let request = create_simple_import_request(0, block_0.clone()); - tracedb.import(request); + let batch = DBTransaction::new(&db); + tracedb.import(&batch, request); + db.write(batch).unwrap(); let filter = Filter { range: (0..0), @@ -562,7 +562,9 @@ mod tests { // import block 1 let request = create_simple_import_request(1, block_1.clone()); - tracedb.import(request); + let batch = DBTransaction::new(&db); + tracedb.import(&batch, request); + db.write(batch).unwrap(); let filter = Filter { range: (0..1), diff --git a/ethcore/src/trace/mod.rs b/ethcore/src/trace/mod.rs index 3a3e24192..277227729 100644 --- a/ethcore/src/trace/mod.rs +++ b/ethcore/src/trace/mod.rs @@ -35,7 +35,7 @@ pub use self::executive_tracer::{ExecutiveTracer, ExecutiveVMTracer}; pub use types::trace_types::filter::{Filter, AddressesFilter}; pub use self::import::ImportRequest; pub use self::localized::LocalizedTrace; -use util::{Bytes, Address, U256, H256}; +use util::{Bytes, Address, U256, H256, DBTransaction}; use self::trace::{Call, Create}; use action_params::ActionParams; use header::BlockNumber; @@ -121,7 +121,7 @@ pub trait Database { fn tracing_enabled(&self) -> bool; /// Imports new block traces. - fn import(&self, request: ImportRequest); + fn import(&self, batch: &DBTransaction, request: ImportRequest); /// Returns localized trace at given position. fn trace(&self, block_number: BlockNumber, tx_position: usize, trace_position: Vec) -> Option; diff --git a/ethcore/src/verification/verification.rs b/ethcore/src/verification/verification.rs index f0f67c4c2..ed094a1d2 100644 --- a/ethcore/src/verification/verification.rs +++ b/ethcore/src/verification/verification.rs @@ -287,6 +287,14 @@ mod tests { self.blocks.get(hash).cloned() } + fn block_header_data(&self, hash: &H256) -> Option { + self.block(hash).map(|b| BlockView::new(&b).header_rlp().as_raw().to_vec()) + } + + fn block_body(&self, hash: &H256) -> Option { + self.block(hash).map(|b| BlockChain::block_to_body(&b)) + } + /// Get the familial details concerning a block. fn block_details(&self, hash: &H256) -> Option { self.blocks.get(hash).map(|bytes| { diff --git a/ethcore/src/views/block.rs b/ethcore/src/views/block.rs index 42fd52a20..fdcae383b 100644 --- a/ethcore/src/views/block.rs +++ b/ethcore/src/views/block.rs @@ -56,6 +56,11 @@ impl<'a> BlockView<'a> { self.rlp.val_at(0) } + /// Return header rlp. + pub fn header_rlp(&self) -> Rlp { + self.rlp.at(0) + } + /// Create new header view obto block head rlp. pub fn header_view(&self) -> HeaderView<'a> { HeaderView::new_from_rlp(self.rlp.at(0)) diff --git a/ethcore/src/views/body.rs b/ethcore/src/views/body.rs new file mode 100644 index 000000000..8f1295f31 --- /dev/null +++ b/ethcore/src/views/body.rs @@ -0,0 +1,144 @@ +// Copyright 2015, 2016 Ethcore (UK) Ltd. +// This file is part of Parity. + +// Parity is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Parity is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Parity. If not, see . + +//! View onto block body rlp. + +use util::*; +use header::*; +use transaction::*; +use super::{TransactionView, HeaderView}; + +/// View onto block rlp. +pub struct BodyView<'a> { + rlp: Rlp<'a> +} + +impl<'a> BodyView<'a> { + /// Creates new view onto block from raw bytes. + pub fn new(bytes: &'a [u8]) -> BodyView<'a> { + BodyView { + rlp: Rlp::new(bytes) + } + } + + /// Creates new view onto block from rlp. + pub fn new_from_rlp(rlp: Rlp<'a>) -> BodyView<'a> { + BodyView { + rlp: rlp + } + } + + /// Return reference to underlaying rlp. + pub fn rlp(&self) -> &Rlp<'a> { + &self.rlp + } + + /// Return List of transactions in given block. + pub fn transactions(&self) -> Vec { + self.rlp.val_at(0) + } + + /// Return List of transactions with additional localization info. + pub fn localized_transactions(&self, block_hash: &H256, block_number: BlockNumber) -> Vec { + self.transactions() + .into_iter() + .enumerate() + .map(|(i, t)| LocalizedTransaction { + signed: t, + block_hash: block_hash.clone(), + block_number: block_number, + transaction_index: i + }).collect() + } + + /// Return number of transactions in given block, without deserializing them. + pub fn transactions_count(&self) -> usize { + self.rlp.at(0).item_count() + } + + /// Return List of transactions in given block. + pub fn transaction_views(&self) -> Vec { + self.rlp.at(0).iter().map(TransactionView::new_from_rlp).collect() + } + + /// Return transaction hashes. + pub fn transaction_hashes(&self) -> Vec { + self.rlp.at(0).iter().map(|rlp| rlp.as_raw().sha3()).collect() + } + + /// Returns transaction at given index without deserializing unnecessary data. + pub fn transaction_at(&self, index: usize) -> Option { + self.rlp.at(0).iter().nth(index).map(|rlp| rlp.as_val()) + } + + /// Returns localized transaction at given index. + pub fn localized_transaction_at(&self, block_hash: &H256, block_number: BlockNumber, index: usize) -> Option { + self.transaction_at(index).map(|t| LocalizedTransaction { + signed: t, + block_hash: block_hash.clone(), + block_number: block_number, + transaction_index: index + }) + } + + /// Return list of uncles of given block. + pub fn uncles(&self) -> Vec
{ + self.rlp.val_at(1) + } + + /// Return number of uncles in given block, without deserializing them. + pub fn uncles_count(&self) -> usize { + self.rlp.at(1).item_count() + } + + /// Return List of transactions in given block. + pub fn uncle_views(&self) -> Vec { + self.rlp.at(1).iter().map(HeaderView::new_from_rlp).collect() + } + + /// Return list of uncle hashes of given block. + pub fn uncle_hashes(&self) -> Vec { + self.rlp.at(1).iter().map(|rlp| rlp.as_raw().sha3()).collect() + } + + /// Return nth uncle. + pub fn uncle_at(&self, index: usize) -> Option
{ + self.rlp.at(1).iter().nth(index).map(|rlp| rlp.as_val()) + } + + /// Return nth uncle rlp. + pub fn uncle_rlp_at(&self, index: usize) -> Option { + self.rlp.at(1).iter().nth(index).map(|rlp| rlp.as_raw().to_vec()) + } +} + +#[cfg(test)] +mod tests { + use util::*; + use super::BodyView; + use blockchain::BlockChain; + + #[test] + fn test_block_view() { + // that's rlp of block created with ethash engine. + let rlp = "f90261f901f9a0d405da4e66f1445d455195229624e133f5baafe72b5cf7b3c36c12c8146e98b7a01dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347948888f1f195afa192cfee860698584c030f4c9db1a05fb2b4bfdef7b314451cb138a534d225c922fc0e5fbe25e451142732c3e25c25a088d2ec6b9860aae1a2c3b299f72b6a5d70d7f7ba4722c78f2c49ba96273c2158a007c6fdfa8eea7e86b81f5b0fc0f78f90cc19f4aa60d323151e0cac660199e9a1b90100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000008302008003832fefba82524d84568e932a80a0a0349d8c3df71f1a48a9df7d03fd5f14aeee7d91332c009ecaff0a71ead405bd88ab4e252a7e8c2a23f862f86002018304cb2f94ec0e71ad0a90ffe1909d27dac207f7680abba42d01801ba03a347e72953c860f32b1eb2c78a680d8734b2ea08085d949d729479796f218d5a047ea6239d9e31ccac8af3366f5ca37184d26e7646e3191a3aeb81c4cf74de500c0".from_hex().unwrap(); + let body = BlockChain::block_to_body(&rlp); + let view = BodyView::new(&body); + assert_eq!(view.transactions_count(), 1); + assert_eq!(view.uncles_count(), 0); + } +} + diff --git a/ethcore/src/views/mod.rs b/ethcore/src/views/mod.rs index c0102be3d..e8267e15a 100644 --- a/ethcore/src/views/mod.rs +++ b/ethcore/src/views/mod.rs @@ -19,7 +19,9 @@ mod block; mod header; mod transaction; +mod body; pub use self::block::BlockView; pub use self::header::HeaderView; +pub use self::body::BodyView; pub use self::transaction::TransactionView; diff --git a/evmbin/Cargo.lock b/evmbin/Cargo.lock index 14c6d9bcb..f135b3b0b 100644 --- a/evmbin/Cargo.lock +++ b/evmbin/Cargo.lock @@ -159,6 +159,7 @@ name = "ethash" version = "1.3.0" dependencies = [ "log 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)", + "parking_lot 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)", "primal 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)", "sha3 0.1.0", ] @@ -250,8 +251,9 @@ dependencies = [ "log 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)", "mio 0.5.1 (git+https://github.com/ethcore/mio?branch=v0.5.x)", "nix 0.5.0 (registry+https://github.com/rust-lang/crates.io-index)", + "parking_lot 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)", "rand 0.3.14 (registry+https://github.com/rust-lang/crates.io-index)", - "rocksdb 0.4.5 (git+https://github.com/ethcore/rust-rocksdb)", + "rocksdb 0.4.5", "rust-crypto 0.2.36 (registry+https://github.com/rust-lang/crates.io-index)", "rustc-serialize 0.3.19 (registry+https://github.com/rust-lang/crates.io-index)", "rustc_version 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)", @@ -301,6 +303,7 @@ dependencies = [ "serde_codegen 0.7.9 (registry+https://github.com/rust-lang/crates.io-index)", "serde_json 0.7.1 (registry+https://github.com/rust-lang/crates.io-index)", "syntex 0.33.0 (registry+https://github.com/rust-lang/crates.io-index)", + "time 0.1.35 (registry+https://github.com/rust-lang/crates.io-index)", "tiny-keccak 1.0.5 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -622,6 +625,17 @@ name = "odds" version = "0.2.12" source = "registry+https://github.com/rust-lang/crates.io-index" +[[package]] +name = "parking_lot" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "kernel32-sys 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.12 (registry+https://github.com/rust-lang/crates.io-index)", + "smallvec 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "primal" version = "0.2.3" @@ -724,16 +738,14 @@ source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "rocksdb" version = "0.4.5" -source = "git+https://github.com/ethcore/rust-rocksdb#9be41e05923616dfa28741c58b22776d479751e6" dependencies = [ "libc 0.2.12 (registry+https://github.com/rust-lang/crates.io-index)", - "rocksdb-sys 0.3.0 (git+https://github.com/ethcore/rust-rocksdb)", + "rocksdb-sys 0.3.0", ] [[package]] name = "rocksdb-sys" version = "0.3.0" -source = "git+https://github.com/ethcore/rust-rocksdb#9be41e05923616dfa28741c58b22776d479751e6" dependencies = [ "gcc 0.3.28 (registry+https://github.com/rust-lang/crates.io-index)", "libc 0.2.12 (registry+https://github.com/rust-lang/crates.io-index)", @@ -831,6 +843,11 @@ name = "slab" version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" +[[package]] +name = "smallvec" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" + [[package]] name = "solicit" version = "0.4.4" diff --git a/parity/dir.rs b/parity/dir.rs index e1400e8e8..bb92e1277 100644 --- a/parity/dir.rs +++ b/parity/dir.rs @@ -52,13 +52,20 @@ impl Directories { Ok(()) } - /// Get the path for the databases given the root path and information on the databases. - pub fn client_path(&self, genesis_hash: H256, fork_name: Option<&String>, pruning: Algorithm) -> PathBuf { + /// Get the root path for database + pub fn db_version_path(&self, genesis_hash: H256, fork_name: Option<&String>, pruning: Algorithm) -> PathBuf { let mut dir = Path::new(&self.db).to_path_buf(); dir.push(format!("{:?}{}", H64::from(genesis_hash), fork_name.map(|f| format!("-{}", f)).unwrap_or_default())); dir.push(format!("v{}-sec-{}", LEGACY_CLIENT_DB_VER_STR, pruning.as_internal_name_str())); dir } + + /// Get the path for the databases given the genesis_hash and information on the databases. + pub fn client_path(&self, genesis_hash: H256, fork_name: Option<&String>, pruning: Algorithm) -> PathBuf { + let mut dir = self.db_version_path(genesis_hash, fork_name, pruning); + dir.push("db"); + dir + } } #[cfg(test)] diff --git a/parity/helpers.rs b/parity/helpers.rs index 76d8250e5..881dd9c8f 100644 --- a/parity/helpers.rs +++ b/parity/helpers.rs @@ -238,7 +238,7 @@ pub fn execute_upgrades( _ => {}, } - let client_path = dirs.client_path(genesis_hash, fork_name, pruning); + let client_path = dirs.db_version_path(genesis_hash, fork_name, pruning); migrate(&client_path, pruning, compaction_profile).map_err(|e| format!("{}", e)) } diff --git a/parity/migration.rs b/parity/migration.rs index 33a000c56..cbfa78ccc 100644 --- a/parity/migration.rs +++ b/parity/migration.rs @@ -20,14 +20,18 @@ use std::io::{Read, Write, Error as IoError, ErrorKind}; use std::path::{Path, PathBuf}; use std::fmt::{Display, Formatter, Error as FmtError}; use util::journaldb::Algorithm; -use util::migration::{Manager as MigrationManager, Config as MigrationConfig, Error as MigrationError}; -use util::kvdb::CompactionProfile; +use util::migration::{Manager as MigrationManager, Config as MigrationConfig, Error as MigrationError, Migration}; +use util::kvdb::{CompactionProfile, Database, DatabaseConfig}; use ethcore::migrations; +use ethcore::client; +use ethcore::migrations::Extract; /// Database is assumed to be at default version, when no version file is found. const DEFAULT_VERSION: u32 = 5; /// Current version of database models. -const CURRENT_VERSION: u32 = 8; +const CURRENT_VERSION: u32 = 9; +/// First version of the consolidated database. +const CONSOLIDATION_VERSION: u32 = 9; /// Defines how many items are migrated to the new version of database at once. const BATCH_SIZE: usize = 1024; /// Version file name. @@ -111,27 +115,13 @@ fn update_version(path: &Path) -> Result<(), Error> { Ok(()) } -/// State database path. -fn state_database_path(path: &Path) -> PathBuf { +/// Consolidated database path +fn consolidated_database_path(path: &Path) -> PathBuf { let mut state_path = path.to_owned(); - state_path.push("state"); + state_path.push("db"); state_path } -/// Blocks database path. -fn blocks_database_path(path: &Path) -> PathBuf { - let mut blocks_path = path.to_owned(); - blocks_path.push("blocks"); - blocks_path -} - -/// Extras database path. -fn extras_database_path(path: &Path) -> PathBuf { - let mut extras_path = path.to_owned(); - extras_path.push("extras"); - extras_path -} - /// Database backup fn backup_database_path(path: &Path) -> PathBuf { let mut backup_path = path.to_owned(); @@ -141,40 +131,55 @@ fn backup_database_path(path: &Path) -> PathBuf { } /// Default migration settings. -fn default_migration_settings(compaction_profile: CompactionProfile) -> MigrationConfig { +pub fn default_migration_settings(compaction_profile: &CompactionProfile) -> MigrationConfig { MigrationConfig { batch_size: BATCH_SIZE, - compaction_profile: compaction_profile, + compaction_profile: *compaction_profile, } } -/// Migrations on the blocks database. -fn blocks_database_migrations(compaction_profile: CompactionProfile) -> Result { - let mut manager = MigrationManager::new(default_migration_settings(compaction_profile)); - try!(manager.add_migration(migrations::blocks::V8::default()).map_err(|_| Error::MigrationImpossible)); +/// Migrations on the consolidated database. +fn consolidated_database_migrations(compaction_profile: &CompactionProfile) -> Result { + let manager = MigrationManager::new(default_migration_settings(compaction_profile)); Ok(manager) } -/// Migrations on the extras database. -fn extras_database_migrations(compaction_profile: CompactionProfile) -> Result { - let mut manager = MigrationManager::new(default_migration_settings(compaction_profile)); - try!(manager.add_migration(migrations::extras::ToV6).map_err(|_| Error::MigrationImpossible)); - Ok(manager) -} +/// Consolidates legacy databases into single one. +fn consolidate_database( + old_db_path: PathBuf, + new_db_path: PathBuf, + column: Option, + extract: Extract, + compaction_profile: &CompactionProfile) -> Result<(), Error> { + fn db_error(e: String) -> Error { + warn!("Cannot open Database for consolidation: {:?}", e); + Error::MigrationFailed + } -/// Migrations on the state database. -fn state_database_migrations(pruning: Algorithm, compaction_profile: CompactionProfile) -> Result { - let mut manager = MigrationManager::new(default_migration_settings(compaction_profile)); - let res = match pruning { - Algorithm::Archive => manager.add_migration(migrations::state::ArchiveV7::default()), - Algorithm::OverlayRecent => manager.add_migration(migrations::state::OverlayRecentV7::default()), - _ => return Err(Error::UnsuportedPruningMethod), + let mut migration = migrations::ToV9::new(column, extract); + let config = default_migration_settings(compaction_profile); + let mut db_config = DatabaseConfig { + max_open_files: 64, + cache_size: None, + compaction: config.compaction_profile.clone(), + columns: None, }; - try!(res.map_err(|_| Error::MigrationImpossible)); - Ok(manager) + let old_path_str = try!(old_db_path.to_str().ok_or(Error::MigrationImpossible)); + let new_path_str = try!(new_db_path.to_str().ok_or(Error::MigrationImpossible)); + + let cur_db = try!(Database::open(&db_config, old_path_str).map_err(db_error)); + // open new DB with proper number of columns + db_config.columns = migration.columns(); + let mut new_db = try!(Database::open(&db_config, new_path_str).map_err(db_error)); + + // Migrate to new database (default column only) + try!(migration.migrate(&cur_db, &config, &mut new_db, None)); + + Ok(()) } + /// Migrates database at given position with given migration rules. fn migrate_database(version: u32, db_path: PathBuf, mut migrations: MigrationManager) -> Result<(), Error> { // check if migration is needed @@ -216,17 +221,108 @@ pub fn migrate(path: &Path, pruning: Algorithm, compaction_profile: CompactionPr // migrate the databases. // main db directory may already exists, so let's check if we have blocks dir - if version < CURRENT_VERSION && exists(&blocks_database_path(path)) { - println!("Migrating database from version {} to {}", version, CURRENT_VERSION); - try!(migrate_database(version, blocks_database_path(path), try!(blocks_database_migrations(compaction_profile.clone())))); - try!(migrate_database(version, extras_database_path(path), try!(extras_database_migrations(compaction_profile.clone())))); - try!(migrate_database(version, state_database_path(path), try!(state_database_migrations(pruning, compaction_profile)))); - println!("Migration finished"); - } else if version > CURRENT_VERSION { + if version > CURRENT_VERSION { return Err(Error::FutureDBVersion); } + // We are in the latest version, yay! + if version == CURRENT_VERSION { + return Ok(()) + } + + // Perform pre-consolidation migrations + if version < CONSOLIDATION_VERSION && exists(&legacy::blocks_database_path(path)) { + println!("Migrating database from version {} to {}", version, CONSOLIDATION_VERSION); + try!(migrate_database(version, legacy::blocks_database_path(path), try!(legacy::blocks_database_migrations(&compaction_profile)))); + try!(migrate_database(version, legacy::extras_database_path(path), try!(legacy::extras_database_migrations(&compaction_profile)))); + try!(migrate_database(version, legacy::state_database_path(path), try!(legacy::state_database_migrations(pruning, &compaction_profile)))); + let db_path = consolidated_database_path(path); + // Remove the database dir (it shouldn't exist anyway, but it might when migration was interrupted) + let _ = fs::remove_dir_all(db_path.clone()); + try!(consolidate_database(legacy::blocks_database_path(path), db_path.clone(), client::DB_COL_HEADERS, Extract::Header, &compaction_profile)); + try!(consolidate_database(legacy::blocks_database_path(path), db_path.clone(), client::DB_COL_BODIES, Extract::Header, &compaction_profile)); + try!(consolidate_database(legacy::extras_database_path(path), db_path.clone(), client::DB_COL_EXTRA, Extract::All, &compaction_profile)); + try!(consolidate_database(legacy::state_database_path(path), db_path.clone(), client::DB_COL_STATE, Extract::All, &compaction_profile)); + try!(consolidate_database(legacy::trace_database_path(path), db_path.clone(), client::DB_COL_TRACE, Extract::All, &compaction_profile)); + let _ = fs::remove_dir_all(legacy::blocks_database_path(path)); + let _ = fs::remove_dir_all(legacy::extras_database_path(path)); + let _ = fs::remove_dir_all(legacy::state_database_path(path)); + let _ = fs::remove_dir_all(legacy::trace_database_path(path)); + println!("Migration finished"); + } + + // Further migrations + if version >= CONSOLIDATION_VERSION && version < CURRENT_VERSION && exists(&consolidated_database_path(path)) { + println!("Migrating database from version {} to {}", ::std::cmp::max(CONSOLIDATION_VERSION, version), CURRENT_VERSION); + try!(migrate_database(version, consolidated_database_path(path), try!(consolidated_database_migrations(&compaction_profile)))); + println!("Migration finished"); + } + // update version file. update_version(path) } +/// Old migrations utilities +mod legacy { + use super::*; + use std::path::{Path, PathBuf}; + use util::journaldb::Algorithm; + use util::migration::{Manager as MigrationManager}; + use util::kvdb::CompactionProfile; + use ethcore::migrations; + + /// Blocks database path. + pub fn blocks_database_path(path: &Path) -> PathBuf { + let mut blocks_path = path.to_owned(); + blocks_path.push("blocks"); + blocks_path + } + + /// Extras database path. + pub fn extras_database_path(path: &Path) -> PathBuf { + let mut extras_path = path.to_owned(); + extras_path.push("extras"); + extras_path + } + + /// State database path. + pub fn state_database_path(path: &Path) -> PathBuf { + let mut state_path = path.to_owned(); + state_path.push("state"); + state_path + } + + /// Trace database path. + pub fn trace_database_path(path: &Path) -> PathBuf { + let mut blocks_path = path.to_owned(); + blocks_path.push("tracedb"); + blocks_path + } + + /// Migrations on the blocks database. + pub fn blocks_database_migrations(compaction_profile: &CompactionProfile) -> Result { + let mut manager = MigrationManager::new(default_migration_settings(compaction_profile)); + try!(manager.add_migration(migrations::blocks::V8::default()).map_err(|_| Error::MigrationImpossible)); + Ok(manager) + } + + /// Migrations on the extras database. + pub fn extras_database_migrations(compaction_profile: &CompactionProfile) -> Result { + let mut manager = MigrationManager::new(default_migration_settings(compaction_profile)); + try!(manager.add_migration(migrations::extras::ToV6).map_err(|_| Error::MigrationImpossible)); + Ok(manager) + } + + /// Migrations on the state database. + pub fn state_database_migrations(pruning: Algorithm, compaction_profile: &CompactionProfile) -> Result { + let mut manager = MigrationManager::new(default_migration_settings(compaction_profile)); + let res = match pruning { + Algorithm::Archive => manager.add_migration(migrations::state::ArchiveV7::default()), + Algorithm::OverlayRecent => manager.add_migration(migrations::state::OverlayRecentV7::default()), + _ => return Err(Error::UnsuportedPruningMethod), + }; + + try!(res.map_err(|_| Error::MigrationImpossible)); + Ok(manager) + } +} diff --git a/parity/params.rs b/parity/params.rs index c48afa37d..6e105f524 100644 --- a/parity/params.rs +++ b/parity/params.rs @@ -15,9 +15,11 @@ // along with Parity. If not, see . use std::str::FromStr; +use std::sync::Arc; use std::time::Duration; -use util::{contents, DatabaseConfig, journaldb, H256, Address, U256, version_data}; +use util::{contents, Database, DatabaseConfig, journaldb, H256, Address, U256, version_data}; use util::journaldb::Algorithm; +use ethcore::client; use ethcore::spec::Spec; use ethcore::ethereum; use ethcore::miner::{GasPricer, GasPriceCalibratorOptions}; @@ -103,11 +105,15 @@ impl Pruning { algo_types.push(Algorithm::default()); algo_types.into_iter().max_by_key(|i| { - let mut client_path = dirs.client_path(genesis_hash, fork_name, *i); - client_path.push("state"); - let db = journaldb::new(client_path.to_str().unwrap(), *i, DatabaseConfig::default()); + let client_path = dirs.client_path(genesis_hash, fork_name, *i); + let config = DatabaseConfig::with_columns(client::DB_NO_OF_COLUMNS); + let db = match Database::open(&config, client_path.to_str().unwrap()) { + Ok(db) => db, + Err(_) => return 0, + }; + let db = journaldb::new(Arc::new(db), *i, client::DB_COL_STATE); trace!(target: "parity", "Looking for best DB: {} at {:?}", i, db.latest_era()); - db.latest_era() + db.latest_era().unwrap_or(0) }).unwrap() } } diff --git a/test.sh b/test.sh index af51fd3fd..18a7bb6b6 100755 --- a/test.sh +++ b/test.sh @@ -14,5 +14,5 @@ case $1 in esac . ./scripts/targets.sh -cargo test --release --verbose $FEATURES $TARGETS $1 \ +cargo test --release $FEATURES $TARGETS $1 \ diff --git a/util/src/journaldb/archivedb.rs b/util/src/journaldb/archivedb.rs index 4bf377cf7..b8c5d1664 100644 --- a/util/src/journaldb/archivedb.rs +++ b/util/src/journaldb/archivedb.rs @@ -20,9 +20,9 @@ use common::*; use rlp::*; use hashdb::*; use memorydb::*; -use super::{DB_PREFIX_LEN, LATEST_ERA_KEY, VERSION_KEY}; +use super::{DB_PREFIX_LEN, LATEST_ERA_KEY}; use super::traits::JournalDB; -use kvdb::{Database, DBTransaction, DatabaseConfig}; +use kvdb::{Database, DBTransaction}; #[cfg(test)] use std::env; @@ -30,9 +30,6 @@ use std::env; /// Would be nich to use rocksdb columns for this eventually. const AUX_FLAG: u8 = 255; -/// Database version. -const DB_VERSION : u32 = 0x103; - /// Implementation of the `HashDB` trait for a disk-backed database with a memory overlay /// and latent-removal semantics. /// @@ -44,28 +41,18 @@ pub struct ArchiveDB { overlay: MemoryDB, backing: Arc, latest_era: Option, + column: Option, } impl ArchiveDB { /// Create a new instance from file - pub fn new(path: &str, config: DatabaseConfig) -> ArchiveDB { - let backing = Database::open(&config, path).unwrap_or_else(|e| { - panic!("Error opening state db: {}", e); - }); - if !backing.is_empty() { - match backing.get(&VERSION_KEY).map(|d| d.map(|v| decode::(&v))) { - Ok(Some(DB_VERSION)) => {}, - v => panic!("Incompatible DB version, expected {}, got {:?}; to resolve, remove {} and restart.", DB_VERSION, v, path) - } - } else { - backing.put(&VERSION_KEY, &encode(&DB_VERSION)).expect("Error writing version to database"); - } - - let latest_era = backing.get(&LATEST_ERA_KEY).expect("Low-level database error.").map(|val| decode::(&val)); + pub fn new(backing: Arc, col: Option) -> ArchiveDB { + let latest_era = backing.get(col, &LATEST_ERA_KEY).expect("Low-level database error.").map(|val| decode::(&val)); ArchiveDB { overlay: MemoryDB::new(), - backing: Arc::new(backing), + backing: backing, latest_era: latest_era, + column: col, } } @@ -74,18 +61,19 @@ impl ArchiveDB { fn new_temp() -> ArchiveDB { let mut dir = env::temp_dir(); dir.push(H32::random().hex()); - Self::new(dir.to_str().unwrap(), DatabaseConfig::default()) + let backing = Arc::new(Database::open_default(dir.to_str().unwrap()).unwrap()); + Self::new(backing, None) } fn payload(&self, key: &H256) -> Option { - self.backing.get(key).expect("Low-level database error. Some issue with your hard disk?").map(|v| v.to_vec()) + self.backing.get(self.column, key).expect("Low-level database error. Some issue with your hard disk?").map(|v| v.to_vec()) } } impl HashDB for ArchiveDB { fn keys(&self) -> HashMap { let mut ret: HashMap = HashMap::new(); - for (key, _) in self.backing.iter() { + for (key, _) in self.backing.iter(self.column) { let h = H256::from_slice(key.deref()); ret.insert(h, 1); } @@ -140,7 +128,7 @@ impl HashDB for ArchiveDB { let mut db_hash = hash.to_vec(); db_hash.push(AUX_FLAG); - self.backing.get(&db_hash) + self.backing.get(self.column, &db_hash) .expect("Low-level database error. Some issue with your hard disk?") .map(|v| v.to_vec()) } @@ -156,6 +144,7 @@ impl JournalDB for ArchiveDB { overlay: self.overlay.clone(), backing: self.backing.clone(), latest_era: self.latest_era, + column: self.column.clone(), }) } @@ -167,8 +156,7 @@ impl JournalDB for ArchiveDB { self.latest_era.is_none() } - fn commit(&mut self, now: u64, _: &H256, _: Option<(u64, H256)>) -> Result { - let batch = DBTransaction::new(); + fn commit(&mut self, batch: &DBTransaction, now: u64, _id: &H256, _end: Option<(u64, H256)>) -> Result { let mut inserts = 0usize; let mut deletes = 0usize; @@ -176,7 +164,7 @@ impl JournalDB for ArchiveDB { let (key, (value, rc)) = i; if rc > 0 { assert!(rc == 1); - batch.put(&key, &value).expect("Low-level database error. Some issue with your hard disk?"); + batch.put(self.column, &key, &value).expect("Low-level database error. Some issue with your hard disk?"); inserts += 1; } if rc < 0 { @@ -187,24 +175,27 @@ impl JournalDB for ArchiveDB { for (mut key, value) in self.overlay.drain_aux().into_iter() { key.push(AUX_FLAG); - batch.put(&key, &value).expect("Low-level database error. Some issue with your hard disk?"); + batch.put(self.column, &key, &value).expect("Low-level database error. Some issue with your hard disk?"); } if self.latest_era.map_or(true, |e| now > e) { - try!(batch.put(&LATEST_ERA_KEY, &encode(&now))); + try!(batch.put(self.column, &LATEST_ERA_KEY, &encode(&now))); self.latest_era = Some(now); } - try!(self.backing.write(batch)); Ok((inserts + deletes) as u32) } fn latest_era(&self) -> Option { self.latest_era } fn state(&self, id: &H256) -> Option { - self.backing.get_by_prefix(&id[0..DB_PREFIX_LEN]).map(|b| b.to_vec()) + self.backing.get_by_prefix(self.column, &id[0..DB_PREFIX_LEN]).map(|b| b.to_vec()) } fn is_pruned(&self) -> bool { false } + + fn backing(&self) -> &Arc { + &self.backing + } } #[cfg(test)] @@ -216,7 +207,7 @@ mod tests { use super::*; use hashdb::*; use journaldb::traits::JournalDB; - use kvdb::DatabaseConfig; + use kvdb::Database; #[test] fn insert_same_in_fork() { @@ -224,18 +215,18 @@ mod tests { let mut jdb = ArchiveDB::new_temp(); let x = jdb.insert(b"X"); - jdb.commit(1, &b"1".sha3(), None).unwrap(); - jdb.commit(2, &b"2".sha3(), None).unwrap(); - jdb.commit(3, &b"1002a".sha3(), Some((1, b"1".sha3()))).unwrap(); - jdb.commit(4, &b"1003a".sha3(), Some((2, b"2".sha3()))).unwrap(); + jdb.commit_batch(1, &b"1".sha3(), None).unwrap(); + jdb.commit_batch(2, &b"2".sha3(), None).unwrap(); + jdb.commit_batch(3, &b"1002a".sha3(), Some((1, b"1".sha3()))).unwrap(); + jdb.commit_batch(4, &b"1003a".sha3(), Some((2, b"2".sha3()))).unwrap(); jdb.remove(&x); - jdb.commit(3, &b"1002b".sha3(), Some((1, b"1".sha3()))).unwrap(); + jdb.commit_batch(3, &b"1002b".sha3(), Some((1, b"1".sha3()))).unwrap(); let x = jdb.insert(b"X"); - jdb.commit(4, &b"1003b".sha3(), Some((2, b"2".sha3()))).unwrap(); + jdb.commit_batch(4, &b"1003b".sha3(), Some((2, b"2".sha3()))).unwrap(); - jdb.commit(5, &b"1004a".sha3(), Some((3, b"1002a".sha3()))).unwrap(); - jdb.commit(6, &b"1005a".sha3(), Some((4, b"1003a".sha3()))).unwrap(); + jdb.commit_batch(5, &b"1004a".sha3(), Some((3, b"1002a".sha3()))).unwrap(); + jdb.commit_batch(6, &b"1005a".sha3(), Some((4, b"1003a".sha3()))).unwrap(); assert!(jdb.contains(&x)); } @@ -245,16 +236,16 @@ mod tests { // history is 3 let mut jdb = ArchiveDB::new_temp(); let h = jdb.insert(b"foo"); - jdb.commit(0, &b"0".sha3(), None).unwrap(); + jdb.commit_batch(0, &b"0".sha3(), None).unwrap(); assert!(jdb.contains(&h)); jdb.remove(&h); - jdb.commit(1, &b"1".sha3(), None).unwrap(); + jdb.commit_batch(1, &b"1".sha3(), None).unwrap(); assert!(jdb.contains(&h)); - jdb.commit(2, &b"2".sha3(), None).unwrap(); + jdb.commit_batch(2, &b"2".sha3(), None).unwrap(); assert!(jdb.contains(&h)); - jdb.commit(3, &b"3".sha3(), Some((0, b"0".sha3()))).unwrap(); + jdb.commit_batch(3, &b"3".sha3(), Some((0, b"0".sha3()))).unwrap(); assert!(jdb.contains(&h)); - jdb.commit(4, &b"4".sha3(), Some((1, b"1".sha3()))).unwrap(); + jdb.commit_batch(4, &b"4".sha3(), Some((1, b"1".sha3()))).unwrap(); } #[test] @@ -264,29 +255,29 @@ mod tests { let foo = jdb.insert(b"foo"); let bar = jdb.insert(b"bar"); - jdb.commit(0, &b"0".sha3(), None).unwrap(); + jdb.commit_batch(0, &b"0".sha3(), None).unwrap(); assert!(jdb.contains(&foo)); assert!(jdb.contains(&bar)); jdb.remove(&foo); jdb.remove(&bar); let baz = jdb.insert(b"baz"); - jdb.commit(1, &b"1".sha3(), Some((0, b"0".sha3()))).unwrap(); + jdb.commit_batch(1, &b"1".sha3(), Some((0, b"0".sha3()))).unwrap(); assert!(jdb.contains(&foo)); assert!(jdb.contains(&bar)); assert!(jdb.contains(&baz)); let foo = jdb.insert(b"foo"); jdb.remove(&baz); - jdb.commit(2, &b"2".sha3(), Some((1, b"1".sha3()))).unwrap(); + jdb.commit_batch(2, &b"2".sha3(), Some((1, b"1".sha3()))).unwrap(); assert!(jdb.contains(&foo)); assert!(jdb.contains(&baz)); jdb.remove(&foo); - jdb.commit(3, &b"3".sha3(), Some((2, b"2".sha3()))).unwrap(); + jdb.commit_batch(3, &b"3".sha3(), Some((2, b"2".sha3()))).unwrap(); assert!(jdb.contains(&foo)); - jdb.commit(4, &b"4".sha3(), Some((3, b"3".sha3()))).unwrap(); + jdb.commit_batch(4, &b"4".sha3(), Some((3, b"3".sha3()))).unwrap(); } #[test] @@ -296,22 +287,22 @@ mod tests { let foo = jdb.insert(b"foo"); let bar = jdb.insert(b"bar"); - jdb.commit(0, &b"0".sha3(), None).unwrap(); + jdb.commit_batch(0, &b"0".sha3(), None).unwrap(); assert!(jdb.contains(&foo)); assert!(jdb.contains(&bar)); jdb.remove(&foo); let baz = jdb.insert(b"baz"); - jdb.commit(1, &b"1a".sha3(), Some((0, b"0".sha3()))).unwrap(); + jdb.commit_batch(1, &b"1a".sha3(), Some((0, b"0".sha3()))).unwrap(); jdb.remove(&bar); - jdb.commit(1, &b"1b".sha3(), Some((0, b"0".sha3()))).unwrap(); + jdb.commit_batch(1, &b"1b".sha3(), Some((0, b"0".sha3()))).unwrap(); assert!(jdb.contains(&foo)); assert!(jdb.contains(&bar)); assert!(jdb.contains(&baz)); - jdb.commit(2, &b"2b".sha3(), Some((1, b"1b".sha3()))).unwrap(); + jdb.commit_batch(2, &b"2b".sha3(), Some((1, b"1b".sha3()))).unwrap(); assert!(jdb.contains(&foo)); } @@ -321,16 +312,16 @@ mod tests { let mut jdb = ArchiveDB::new_temp(); let foo = jdb.insert(b"foo"); - jdb.commit(0, &b"0".sha3(), None).unwrap(); + jdb.commit_batch(0, &b"0".sha3(), None).unwrap(); assert!(jdb.contains(&foo)); jdb.remove(&foo); - jdb.commit(1, &b"1".sha3(), Some((0, b"0".sha3()))).unwrap(); + jdb.commit_batch(1, &b"1".sha3(), Some((0, b"0".sha3()))).unwrap(); jdb.insert(b"foo"); assert!(jdb.contains(&foo)); - jdb.commit(2, &b"2".sha3(), Some((1, b"1".sha3()))).unwrap(); + jdb.commit_batch(2, &b"2".sha3(), Some((1, b"1".sha3()))).unwrap(); assert!(jdb.contains(&foo)); - jdb.commit(3, &b"2".sha3(), Some((0, b"2".sha3()))).unwrap(); + jdb.commit_batch(3, &b"2".sha3(), Some((0, b"2".sha3()))).unwrap(); assert!(jdb.contains(&foo)); } @@ -338,19 +329,24 @@ mod tests { fn fork_same_key() { // history is 1 let mut jdb = ArchiveDB::new_temp(); - jdb.commit(0, &b"0".sha3(), None).unwrap(); + jdb.commit_batch(0, &b"0".sha3(), None).unwrap(); let foo = jdb.insert(b"foo"); - jdb.commit(1, &b"1a".sha3(), Some((0, b"0".sha3()))).unwrap(); + jdb.commit_batch(1, &b"1a".sha3(), Some((0, b"0".sha3()))).unwrap(); jdb.insert(b"foo"); - jdb.commit(1, &b"1b".sha3(), Some((0, b"0".sha3()))).unwrap(); + jdb.commit_batch(1, &b"1b".sha3(), Some((0, b"0".sha3()))).unwrap(); assert!(jdb.contains(&foo)); - jdb.commit(2, &b"2a".sha3(), Some((1, b"1a".sha3()))).unwrap(); + jdb.commit_batch(2, &b"2a".sha3(), Some((1, b"1a".sha3()))).unwrap(); assert!(jdb.contains(&foo)); } + fn new_db(dir: &Path) -> ArchiveDB { + let db = Database::open_default(dir.to_str().unwrap()).unwrap(); + ArchiveDB::new(Arc::new(db), None) + } + #[test] fn reopen() { let mut dir = ::std::env::temp_dir(); @@ -358,25 +354,25 @@ mod tests { let bar = H256::random(); let foo = { - let mut jdb = ArchiveDB::new(dir.to_str().unwrap(), DatabaseConfig::default()); + let mut jdb = new_db(&dir); // history is 1 let foo = jdb.insert(b"foo"); jdb.emplace(bar.clone(), b"bar".to_vec()); - jdb.commit(0, &b"0".sha3(), None).unwrap(); + jdb.commit_batch(0, &b"0".sha3(), None).unwrap(); foo }; { - let mut jdb = ArchiveDB::new(dir.to_str().unwrap(), DatabaseConfig::default()); + let mut jdb = new_db(&dir); jdb.remove(&foo); - jdb.commit(1, &b"1".sha3(), Some((0, b"0".sha3()))).unwrap(); + jdb.commit_batch(1, &b"1".sha3(), Some((0, b"0".sha3()))).unwrap(); } { - let mut jdb = ArchiveDB::new(dir.to_str().unwrap(), DatabaseConfig::default()); + let mut jdb = new_db(&dir); assert!(jdb.contains(&foo)); assert!(jdb.contains(&bar)); - jdb.commit(2, &b"2".sha3(), Some((1, b"1".sha3()))).unwrap(); + jdb.commit_batch(2, &b"2".sha3(), Some((1, b"1".sha3()))).unwrap(); } } @@ -386,27 +382,27 @@ mod tests { dir.push(H32::random().hex()); let foo = { - let mut jdb = ArchiveDB::new(dir.to_str().unwrap(), DatabaseConfig::default()); + let mut jdb = new_db(&dir); // history is 1 let foo = jdb.insert(b"foo"); - jdb.commit(0, &b"0".sha3(), None).unwrap(); - jdb.commit(1, &b"1".sha3(), Some((0, b"0".sha3()))).unwrap(); + jdb.commit_batch(0, &b"0".sha3(), None).unwrap(); + jdb.commit_batch(1, &b"1".sha3(), Some((0, b"0".sha3()))).unwrap(); // foo is ancient history. jdb.insert(b"foo"); - jdb.commit(2, &b"2".sha3(), Some((1, b"1".sha3()))).unwrap(); + jdb.commit_batch(2, &b"2".sha3(), Some((1, b"1".sha3()))).unwrap(); foo }; { - let mut jdb = ArchiveDB::new(dir.to_str().unwrap(), DatabaseConfig::default()); + let mut jdb = new_db(&dir); jdb.remove(&foo); - jdb.commit(3, &b"3".sha3(), Some((2, b"2".sha3()))).unwrap(); + jdb.commit_batch(3, &b"3".sha3(), Some((2, b"2".sha3()))).unwrap(); assert!(jdb.contains(&foo)); jdb.remove(&foo); - jdb.commit(4, &b"4".sha3(), Some((3, b"3".sha3()))).unwrap(); - jdb.commit(5, &b"5".sha3(), Some((4, b"4".sha3()))).unwrap(); + jdb.commit_batch(4, &b"4".sha3(), Some((3, b"3".sha3()))).unwrap(); + jdb.commit_batch(5, &b"5".sha3(), Some((4, b"4".sha3()))).unwrap(); } } @@ -415,23 +411,23 @@ mod tests { let mut dir = ::std::env::temp_dir(); dir.push(H32::random().hex()); let (foo, _, _) = { - let mut jdb = ArchiveDB::new(dir.to_str().unwrap(), DatabaseConfig::default()); + let mut jdb = new_db(&dir); // history is 1 let foo = jdb.insert(b"foo"); let bar = jdb.insert(b"bar"); - jdb.commit(0, &b"0".sha3(), None).unwrap(); + jdb.commit_batch(0, &b"0".sha3(), None).unwrap(); jdb.remove(&foo); let baz = jdb.insert(b"baz"); - jdb.commit(1, &b"1a".sha3(), Some((0, b"0".sha3()))).unwrap(); + jdb.commit_batch(1, &b"1a".sha3(), Some((0, b"0".sha3()))).unwrap(); jdb.remove(&bar); - jdb.commit(1, &b"1b".sha3(), Some((0, b"0".sha3()))).unwrap(); + jdb.commit_batch(1, &b"1b".sha3(), Some((0, b"0".sha3()))).unwrap(); (foo, bar, baz) }; { - let mut jdb = ArchiveDB::new(dir.to_str().unwrap(), DatabaseConfig::default()); - jdb.commit(2, &b"2b".sha3(), Some((1, b"1b".sha3()))).unwrap(); + let mut jdb = new_db(&dir); + jdb.commit_batch(2, &b"2b".sha3(), Some((1, b"1b".sha3()))).unwrap(); assert!(jdb.contains(&foo)); } } @@ -441,14 +437,14 @@ mod tests { let temp = ::devtools::RandomTempPath::new(); let key = { - let mut jdb = ArchiveDB::new(temp.as_str(), DatabaseConfig::default()); + let mut jdb = new_db(temp.as_path().as_path()); let key = jdb.insert(b"foo"); - jdb.commit(0, &b"0".sha3(), None).unwrap(); + jdb.commit_batch(0, &b"0".sha3(), None).unwrap(); key }; { - let jdb = ArchiveDB::new(temp.as_str(), DatabaseConfig::default()); + let jdb = new_db(temp.as_path().as_path()); let state = jdb.state(&key); assert!(state.is_some()); } diff --git a/util/src/journaldb/earlymergedb.rs b/util/src/journaldb/earlymergedb.rs index 2fe4ca10d..908c5cadf 100644 --- a/util/src/journaldb/earlymergedb.rs +++ b/util/src/journaldb/earlymergedb.rs @@ -20,9 +20,9 @@ use common::*; use rlp::*; use hashdb::*; use memorydb::*; -use super::{DB_PREFIX_LEN, LATEST_ERA_KEY, VERSION_KEY}; +use super::{DB_PREFIX_LEN, LATEST_ERA_KEY}; use super::traits::JournalDB; -use kvdb::{Database, DBTransaction, DatabaseConfig}; +use kvdb::{Database, DBTransaction}; #[cfg(test)] use std::env; @@ -66,33 +66,22 @@ pub struct EarlyMergeDB { backing: Arc, refs: Option>>>, latest_era: Option, + column: Option, } -const DB_VERSION : u32 = 0x003; const PADDING : [u8; 10] = [ 0u8; 10 ]; impl EarlyMergeDB { /// Create a new instance from file - pub fn new(path: &str, config: DatabaseConfig) -> EarlyMergeDB { - let backing = Database::open(&config, path).unwrap_or_else(|e| { - panic!("Error opening state db: {}", e); - }); - if !backing.is_empty() { - match backing.get(&VERSION_KEY).map(|d| d.map(|v| decode::(&v))) { - Ok(Some(DB_VERSION)) => {}, - v => panic!("Incompatible DB version, expected {}, got {:?}; to resolve, remove {} and restart.", DB_VERSION, v, path) - } - } else { - backing.put(&VERSION_KEY, &encode(&DB_VERSION)).expect("Error writing version to database"); - } - - let (latest_era, refs) = EarlyMergeDB::read_refs(&backing); + pub fn new(backing: Arc, col: Option) -> EarlyMergeDB { + let (latest_era, refs) = EarlyMergeDB::read_refs(&backing, col); let refs = Some(Arc::new(RwLock::new(refs))); EarlyMergeDB { overlay: MemoryDB::new(), - backing: Arc::new(backing), + backing: backing, refs: refs, latest_era: latest_era, + column: col, } } @@ -101,7 +90,8 @@ impl EarlyMergeDB { fn new_temp() -> EarlyMergeDB { let mut dir = env::temp_dir(); dir.push(H32::random().hex()); - Self::new(dir.to_str().unwrap(), DatabaseConfig::default()) + let backing = Arc::new(Database::open_default(dir.to_str().unwrap()).unwrap()); + Self::new(backing, None) } fn morph_key(key: &H256, index: u8) -> Bytes { @@ -111,13 +101,13 @@ impl EarlyMergeDB { } // The next three are valid only as long as there is an insert operation of `key` in the journal. - fn set_already_in(batch: &DBTransaction, key: &H256) { batch.put(&Self::morph_key(key, 0), &[1u8]).expect("Low-level database error. Some issue with your hard disk?"); } - fn reset_already_in(batch: &DBTransaction, key: &H256) { batch.delete(&Self::morph_key(key, 0)).expect("Low-level database error. Some issue with your hard disk?"); } - fn is_already_in(backing: &Database, key: &H256) -> bool { - backing.get(&Self::morph_key(key, 0)).expect("Low-level database error. Some issue with your hard disk?").is_some() + fn set_already_in(batch: &DBTransaction, col: Option, key: &H256) { batch.put(col, &Self::morph_key(key, 0), &[1u8]).expect("Low-level database error. Some issue with your hard disk?"); } + fn reset_already_in(batch: &DBTransaction, col: Option, key: &H256) { batch.delete(col, &Self::morph_key(key, 0)).expect("Low-level database error. Some issue with your hard disk?"); } + fn is_already_in(backing: &Database, col: Option, key: &H256) -> bool { + backing.get(col, &Self::morph_key(key, 0)).expect("Low-level database error. Some issue with your hard disk?").is_some() } - fn insert_keys(inserts: &[(H256, Bytes)], backing: &Database, refs: &mut HashMap, batch: &DBTransaction, trace: bool) { + fn insert_keys(inserts: &[(H256, Bytes)], backing: &Database, col: Option, refs: &mut HashMap, batch: &DBTransaction, trace: bool) { for &(ref h, ref d) in inserts { if let Some(c) = refs.get_mut(h) { // already counting. increment. @@ -129,9 +119,9 @@ impl EarlyMergeDB { } // this is the first entry for this node in the journal. - if backing.get(h).expect("Low-level database error. Some issue with your hard disk?").is_some() { + if backing.get(col, h).expect("Low-level database error. Some issue with your hard disk?").is_some() { // already in the backing DB. start counting, and remember it was already in. - Self::set_already_in(batch, h); + Self::set_already_in(batch, col, h); refs.insert(h.clone(), RefInfo{queue_refs: 1, in_archive: true}); if trace { trace!(target: "jdb.fine", " insert({}): New to queue, in DB: Recording and inserting into queue", h); @@ -141,8 +131,8 @@ impl EarlyMergeDB { // Gets removed when a key leaves the journal, so should never be set when we're placing a new key. //Self::reset_already_in(&h); - assert!(!Self::is_already_in(backing, &h)); - batch.put(h, d).expect("Low-level database error. Some issue with your hard disk?"); + assert!(!Self::is_already_in(backing, col, &h)); + batch.put(col, h, d).expect("Low-level database error. Some issue with your hard disk?"); refs.insert(h.clone(), RefInfo{queue_refs: 1, in_archive: false}); if trace { trace!(target: "jdb.fine", " insert({}): New to queue, not in DB: Inserting into queue and DB", h); @@ -150,7 +140,7 @@ impl EarlyMergeDB { } } - fn replay_keys(inserts: &[H256], backing: &Database, refs: &mut HashMap) { + fn replay_keys(inserts: &[H256], backing: &Database, col: Option, refs: &mut HashMap) { trace!(target: "jdb.fine", "replay_keys: inserts={:?}, refs={:?}", inserts, refs); for h in inserts { if let Some(c) = refs.get_mut(h) { @@ -161,12 +151,12 @@ impl EarlyMergeDB { // this is the first entry for this node in the journal. // it is initialised to 1 if it was already in. - refs.insert(h.clone(), RefInfo{queue_refs: 1, in_archive: Self::is_already_in(backing, h)}); + refs.insert(h.clone(), RefInfo{queue_refs: 1, in_archive: Self::is_already_in(backing, col, h)}); } trace!(target: "jdb.fine", "replay_keys: (end) refs={:?}", refs); } - fn remove_keys(deletes: &[H256], refs: &mut HashMap, batch: &DBTransaction, from: RemoveFrom, trace: bool) { + fn remove_keys(deletes: &[H256], refs: &mut HashMap, batch: &DBTransaction, col: Option, from: RemoveFrom, trace: bool) { // with a remove on {queue_refs: 1, in_archive: true}, we have two options: // - convert to {queue_refs: 1, in_archive: false} (i.e. remove it from the conceptual archive) // - convert to {queue_refs: 0, in_archive: true} (i.e. remove it from the conceptual queue) @@ -178,7 +168,7 @@ impl EarlyMergeDB { if let Some(c) = refs.get_mut(h) { if c.in_archive && from == RemoveFrom::Archive { c.in_archive = false; - Self::reset_already_in(batch, h); + Self::reset_already_in(batch, col, h); if trace { trace!(target: "jdb.fine", " remove({}): In archive, 1 in queue: Reducing to queue only and recording", h); } @@ -196,14 +186,14 @@ impl EarlyMergeDB { match n { Some(RefInfo{queue_refs: 1, in_archive: true}) => { refs.remove(h); - Self::reset_already_in(batch, h); + Self::reset_already_in(batch, col, h); if trace { trace!(target: "jdb.fine", " remove({}): In archive, 1 in queue: Removing from queue and leaving in archive", h); } } Some(RefInfo{queue_refs: 1, in_archive: false}) => { refs.remove(h); - batch.delete(h).expect("Low-level database error. Some issue with your hard disk?"); + batch.delete(col, h).expect("Low-level database error. Some issue with your hard disk?"); if trace { trace!(target: "jdb.fine", " remove({}): Not in archive, only 1 ref in queue: Removing from queue and DB", h); } @@ -211,7 +201,7 @@ impl EarlyMergeDB { None => { // Gets removed when moving from 1 to 0 additional refs. Should never be here at 0 additional refs. //assert!(!Self::is_already_in(db, &h)); - batch.delete(h).expect("Low-level database error. Some issue with your hard disk?"); + batch.delete(col, h).expect("Low-level database error. Some issue with your hard disk?"); if trace { trace!(target: "jdb.fine", " remove({}): Not in queue - MUST BE IN ARCHIVE: Removing from DB", h); } @@ -223,7 +213,7 @@ impl EarlyMergeDB { #[cfg(test)] fn can_reconstruct_refs(&self) -> bool { - let (latest_era, reconstructed) = Self::read_refs(&self.backing); + let (latest_era, reconstructed) = Self::read_refs(&self.backing, self.column); let refs = self.refs.as_ref().unwrap().write(); if *refs != reconstructed || latest_era != self.latest_era { let clean_refs = refs.iter().filter_map(|(k, v)| if reconstructed.get(k) == Some(v) {None} else {Some((k.clone(), v.clone()))}).collect::>(); @@ -236,18 +226,18 @@ impl EarlyMergeDB { } fn payload(&self, key: &H256) -> Option { - self.backing.get(key).expect("Low-level database error. Some issue with your hard disk?").map(|v| v.to_vec()) + self.backing.get(self.column, key).expect("Low-level database error. Some issue with your hard disk?").map(|v| v.to_vec()) } - fn read_refs(db: &Database) -> (Option, HashMap) { + fn read_refs(db: &Database, col: Option) -> (Option, HashMap) { let mut refs = HashMap::new(); let mut latest_era = None; - if let Some(val) = db.get(&LATEST_ERA_KEY).expect("Low-level database error.") { + if let Some(val) = db.get(col, &LATEST_ERA_KEY).expect("Low-level database error.") { let mut era = decode::(&val); latest_era = Some(era); loop { let mut index = 0usize; - while let Some(rlp_data) = db.get({ + while let Some(rlp_data) = db.get(col, { let mut r = RlpStream::new_list(3); r.append(&era); r.append(&index); @@ -256,7 +246,7 @@ impl EarlyMergeDB { }).expect("Low-level database error.") { let rlp = Rlp::new(&rlp_data); let inserts: Vec = rlp.val_at(1); - Self::replay_keys(&inserts, db, &mut refs); + Self::replay_keys(&inserts, db, col, &mut refs); index += 1; }; if index == 0 || era == 0 { @@ -267,12 +257,12 @@ impl EarlyMergeDB { } (latest_era, refs) } - } +} impl HashDB for EarlyMergeDB { fn keys(&self) -> HashMap { let mut ret: HashMap = HashMap::new(); - for (key, _) in self.backing.iter() { + for (key, _) in self.backing.iter(self.column) { let h = H256::from_slice(key.deref()); ret.insert(h, 1); } @@ -321,11 +311,16 @@ impl JournalDB for EarlyMergeDB { backing: self.backing.clone(), refs: self.refs.clone(), latest_era: self.latest_era.clone(), + column: self.column.clone(), }) } fn is_empty(&self) -> bool { - self.backing.get(&LATEST_ERA_KEY).expect("Low level database error").is_none() + self.backing.get(self.column, &LATEST_ERA_KEY).expect("Low level database error").is_none() + } + + fn backing(&self) -> &Arc { + &self.backing } fn latest_era(&self) -> Option { self.latest_era } @@ -338,11 +333,11 @@ impl JournalDB for EarlyMergeDB { } fn state(&self, id: &H256) -> Option { - self.backing.get_by_prefix(&id[0..DB_PREFIX_LEN]).map(|b| b.to_vec()) + self.backing.get_by_prefix(self.column, &id[0..DB_PREFIX_LEN]).map(|b| b.to_vec()) } #[cfg_attr(feature="dev", allow(cyclomatic_complexity))] - fn commit(&mut self, now: u64, id: &H256, end: Option<(u64, H256)>) -> Result { + fn commit(&mut self, batch: &DBTransaction, now: u64, id: &H256, end: Option<(u64, H256)>) -> Result { // journal format: // [era, 0] => [ id, [insert_0, ...], [remove_0, ...] ] // [era, 1] => [ id, [insert_0, ...], [remove_0, ...] ] @@ -389,13 +384,12 @@ impl JournalDB for EarlyMergeDB { // record new commit's details. let mut refs = self.refs.as_ref().unwrap().write(); - let batch = DBTransaction::new(); let trace = false; { let mut index = 0usize; let mut last; - while try!(self.backing.get({ + while try!(self.backing.get(self.column, { let mut r = RlpStream::new_list(3); r.append(&now); r.append(&index); @@ -436,15 +430,15 @@ impl JournalDB for EarlyMergeDB { r.begin_list(inserts.len()); inserts.iter().foreach(|&(k, _)| {r.append(&k);}); r.append(&removes); - Self::insert_keys(&inserts, &self.backing, &mut refs, &batch, trace); + Self::insert_keys(&inserts, &self.backing, self.column, &mut refs, &batch, trace); if trace { let ins = inserts.iter().map(|&(k, _)| k).collect::>(); trace!(target: "jdb.ops", " Inserts: {:?}", ins); trace!(target: "jdb.ops", " Deletes: {:?}", removes); } - try!(batch.put(&last, r.as_raw())); + try!(batch.put(self.column, &last, r.as_raw())); if self.latest_era.map_or(true, |e| now > e) { - try!(batch.put(&LATEST_ERA_KEY, &encode(&now))); + try!(batch.put(self.column, &LATEST_ERA_KEY, &encode(&now))); self.latest_era = Some(now); } } @@ -453,7 +447,7 @@ impl JournalDB for EarlyMergeDB { if let Some((end_era, canon_id)) = end { let mut index = 0usize; let mut last; - while let Some(rlp_data) = try!(self.backing.get({ + while let Some(rlp_data) = try!(self.backing.get(self.column, { let mut r = RlpStream::new_list(3); r.append(&end_era); r.append(&index); @@ -470,7 +464,7 @@ impl JournalDB for EarlyMergeDB { if trace { trace!(target: "jdb.ops", " Expunging: {:?}", deletes); } - Self::remove_keys(&deletes, &mut refs, &batch, RemoveFrom::Archive, trace); + Self::remove_keys(&deletes, &mut refs, &batch, self.column, RemoveFrom::Archive, trace); if trace { trace!(target: "jdb.ops", " Finalising: {:?}", inserts); @@ -488,7 +482,7 @@ impl JournalDB for EarlyMergeDB { } Some( RefInfo{queue_refs: x, in_archive: false} ) => { // must set already in; , - Self::set_already_in(&batch, k); + Self::set_already_in(&batch, self.column, k); refs.insert(k.clone(), RefInfo{ queue_refs: x - 1, in_archive: true }); } Some( RefInfo{in_archive: true, ..} ) => { @@ -502,10 +496,10 @@ impl JournalDB for EarlyMergeDB { if trace { trace!(target: "jdb.ops", " Reverting: {:?}", inserts); } - Self::remove_keys(&inserts, &mut refs, &batch, RemoveFrom::Queue, trace); + Self::remove_keys(&inserts, &mut refs, &batch, self.column, RemoveFrom::Queue, trace); } - try!(batch.delete(&last)); + try!(batch.delete(self.column, &last)); index += 1; } if trace { @@ -513,10 +507,6 @@ impl JournalDB for EarlyMergeDB { } } - try!(self.backing.write(batch)); - - // Comment out for now. TODO: automatically enable in tests. - if trace { trace!(target: "jdb", "OK: {:?}", refs.clone()); } @@ -535,7 +525,7 @@ mod tests { use super::super::traits::JournalDB; use hashdb::*; use log::init_log; - use kvdb::DatabaseConfig; + use kvdb::{Database, DatabaseConfig}; #[test] fn insert_same_in_fork() { @@ -543,25 +533,25 @@ mod tests { let mut jdb = EarlyMergeDB::new_temp(); let x = jdb.insert(b"X"); - jdb.commit(1, &b"1".sha3(), None).unwrap(); + jdb.commit_batch(1, &b"1".sha3(), None).unwrap(); assert!(jdb.can_reconstruct_refs()); - jdb.commit(2, &b"2".sha3(), None).unwrap(); + jdb.commit_batch(2, &b"2".sha3(), None).unwrap(); assert!(jdb.can_reconstruct_refs()); - jdb.commit(3, &b"1002a".sha3(), Some((1, b"1".sha3()))).unwrap(); + jdb.commit_batch(3, &b"1002a".sha3(), Some((1, b"1".sha3()))).unwrap(); assert!(jdb.can_reconstruct_refs()); - jdb.commit(4, &b"1003a".sha3(), Some((2, b"2".sha3()))).unwrap(); + jdb.commit_batch(4, &b"1003a".sha3(), Some((2, b"2".sha3()))).unwrap(); assert!(jdb.can_reconstruct_refs()); jdb.remove(&x); - jdb.commit(3, &b"1002b".sha3(), Some((1, b"1".sha3()))).unwrap(); + jdb.commit_batch(3, &b"1002b".sha3(), Some((1, b"1".sha3()))).unwrap(); assert!(jdb.can_reconstruct_refs()); let x = jdb.insert(b"X"); - jdb.commit(4, &b"1003b".sha3(), Some((2, b"2".sha3()))).unwrap(); + jdb.commit_batch(4, &b"1003b".sha3(), Some((2, b"2".sha3()))).unwrap(); assert!(jdb.can_reconstruct_refs()); - jdb.commit(5, &b"1004a".sha3(), Some((3, b"1002a".sha3()))).unwrap(); + jdb.commit_batch(5, &b"1004a".sha3(), Some((3, b"1002a".sha3()))).unwrap(); assert!(jdb.can_reconstruct_refs()); - jdb.commit(6, &b"1005a".sha3(), Some((4, b"1003a".sha3()))).unwrap(); + jdb.commit_batch(6, &b"1005a".sha3(), Some((4, b"1003a".sha3()))).unwrap(); assert!(jdb.can_reconstruct_refs()); assert!(jdb.contains(&x)); @@ -571,17 +561,17 @@ mod tests { fn insert_older_era() { let mut jdb = EarlyMergeDB::new_temp(); let foo = jdb.insert(b"foo"); - jdb.commit(0, &b"0a".sha3(), None).unwrap(); + jdb.commit_batch(0, &b"0a".sha3(), None).unwrap(); assert!(jdb.can_reconstruct_refs()); let bar = jdb.insert(b"bar"); - jdb.commit(1, &b"1".sha3(), Some((0, b"0a".sha3()))).unwrap(); + jdb.commit_batch(1, &b"1".sha3(), Some((0, b"0a".sha3()))).unwrap(); assert!(jdb.can_reconstruct_refs()); jdb.remove(&bar); - jdb.commit(0, &b"0b".sha3(), None).unwrap(); + jdb.commit_batch(0, &b"0b".sha3(), None).unwrap(); assert!(jdb.can_reconstruct_refs()); - jdb.commit(2, &b"2".sha3(), Some((1, b"1".sha3()))).unwrap(); + jdb.commit_batch(2, &b"2".sha3(), Some((1, b"1".sha3()))).unwrap(); assert!(jdb.contains(&foo)); assert!(jdb.contains(&bar)); @@ -592,20 +582,20 @@ mod tests { // history is 3 let mut jdb = EarlyMergeDB::new_temp(); let h = jdb.insert(b"foo"); - jdb.commit(0, &b"0".sha3(), None).unwrap(); + jdb.commit_batch(0, &b"0".sha3(), None).unwrap(); assert!(jdb.can_reconstruct_refs()); assert!(jdb.contains(&h)); jdb.remove(&h); - jdb.commit(1, &b"1".sha3(), None).unwrap(); + jdb.commit_batch(1, &b"1".sha3(), None).unwrap(); assert!(jdb.can_reconstruct_refs()); assert!(jdb.contains(&h)); - jdb.commit(2, &b"2".sha3(), None).unwrap(); + jdb.commit_batch(2, &b"2".sha3(), None).unwrap(); assert!(jdb.can_reconstruct_refs()); assert!(jdb.contains(&h)); - jdb.commit(3, &b"3".sha3(), Some((0, b"0".sha3()))).unwrap(); + jdb.commit_batch(3, &b"3".sha3(), Some((0, b"0".sha3()))).unwrap(); assert!(jdb.can_reconstruct_refs()); assert!(jdb.contains(&h)); - jdb.commit(4, &b"4".sha3(), Some((1, b"1".sha3()))).unwrap(); + jdb.commit_batch(4, &b"4".sha3(), Some((1, b"1".sha3()))).unwrap(); assert!(jdb.can_reconstruct_refs()); assert!(!jdb.contains(&h)); } @@ -617,7 +607,7 @@ mod tests { let foo = jdb.insert(b"foo"); let bar = jdb.insert(b"bar"); - jdb.commit(0, &b"0".sha3(), None).unwrap(); + jdb.commit_batch(0, &b"0".sha3(), None).unwrap(); assert!(jdb.can_reconstruct_refs()); assert!(jdb.contains(&foo)); assert!(jdb.contains(&bar)); @@ -625,7 +615,7 @@ mod tests { jdb.remove(&foo); jdb.remove(&bar); let baz = jdb.insert(b"baz"); - jdb.commit(1, &b"1".sha3(), Some((0, b"0".sha3()))).unwrap(); + jdb.commit_batch(1, &b"1".sha3(), Some((0, b"0".sha3()))).unwrap(); assert!(jdb.can_reconstruct_refs()); assert!(jdb.contains(&foo)); assert!(jdb.contains(&bar)); @@ -633,20 +623,20 @@ mod tests { let foo = jdb.insert(b"foo"); jdb.remove(&baz); - jdb.commit(2, &b"2".sha3(), Some((1, b"1".sha3()))).unwrap(); + jdb.commit_batch(2, &b"2".sha3(), Some((1, b"1".sha3()))).unwrap(); assert!(jdb.can_reconstruct_refs()); assert!(jdb.contains(&foo)); assert!(!jdb.contains(&bar)); assert!(jdb.contains(&baz)); jdb.remove(&foo); - jdb.commit(3, &b"3".sha3(), Some((2, b"2".sha3()))).unwrap(); + jdb.commit_batch(3, &b"3".sha3(), Some((2, b"2".sha3()))).unwrap(); assert!(jdb.can_reconstruct_refs()); assert!(jdb.contains(&foo)); assert!(!jdb.contains(&bar)); assert!(!jdb.contains(&baz)); - jdb.commit(4, &b"4".sha3(), Some((3, b"3".sha3()))).unwrap(); + jdb.commit_batch(4, &b"4".sha3(), Some((3, b"3".sha3()))).unwrap(); assert!(jdb.can_reconstruct_refs()); assert!(!jdb.contains(&foo)); assert!(!jdb.contains(&bar)); @@ -660,25 +650,25 @@ mod tests { let foo = jdb.insert(b"foo"); let bar = jdb.insert(b"bar"); - jdb.commit(0, &b"0".sha3(), None).unwrap(); + jdb.commit_batch(0, &b"0".sha3(), None).unwrap(); assert!(jdb.can_reconstruct_refs()); assert!(jdb.contains(&foo)); assert!(jdb.contains(&bar)); jdb.remove(&foo); let baz = jdb.insert(b"baz"); - jdb.commit(1, &b"1a".sha3(), Some((0, b"0".sha3()))).unwrap(); + jdb.commit_batch(1, &b"1a".sha3(), Some((0, b"0".sha3()))).unwrap(); assert!(jdb.can_reconstruct_refs()); jdb.remove(&bar); - jdb.commit(1, &b"1b".sha3(), Some((0, b"0".sha3()))).unwrap(); + jdb.commit_batch(1, &b"1b".sha3(), Some((0, b"0".sha3()))).unwrap(); assert!(jdb.can_reconstruct_refs()); assert!(jdb.contains(&foo)); assert!(jdb.contains(&bar)); assert!(jdb.contains(&baz)); - jdb.commit(2, &b"2b".sha3(), Some((1, b"1b".sha3()))).unwrap(); + jdb.commit_batch(2, &b"2b".sha3(), Some((1, b"1b".sha3()))).unwrap(); assert!(jdb.can_reconstruct_refs()); assert!(jdb.contains(&foo)); assert!(!jdb.contains(&baz)); @@ -691,115 +681,113 @@ mod tests { let mut jdb = EarlyMergeDB::new_temp(); let foo = jdb.insert(b"foo"); - jdb.commit(0, &b"0".sha3(), None).unwrap(); + jdb.commit_batch(0, &b"0".sha3(), None).unwrap(); assert!(jdb.can_reconstruct_refs()); assert!(jdb.contains(&foo)); jdb.remove(&foo); - jdb.commit(1, &b"1".sha3(), Some((0, b"0".sha3()))).unwrap(); + jdb.commit_batch(1, &b"1".sha3(), Some((0, b"0".sha3()))).unwrap(); assert!(jdb.can_reconstruct_refs()); jdb.insert(b"foo"); assert!(jdb.contains(&foo)); - jdb.commit(2, &b"2".sha3(), Some((1, b"1".sha3()))).unwrap(); + jdb.commit_batch(2, &b"2".sha3(), Some((1, b"1".sha3()))).unwrap(); assert!(jdb.can_reconstruct_refs()); assert!(jdb.contains(&foo)); - jdb.commit(3, &b"2".sha3(), Some((0, b"2".sha3()))).unwrap(); + jdb.commit_batch(3, &b"2".sha3(), Some((0, b"2".sha3()))).unwrap(); assert!(jdb.can_reconstruct_refs()); assert!(jdb.contains(&foo)); } #[test] fn fork_same_key_one() { - let mut dir = ::std::env::temp_dir(); - dir.push(H32::random().hex()); - let mut jdb = EarlyMergeDB::new(dir.to_str().unwrap(), DatabaseConfig::default()); - jdb.commit(0, &b"0".sha3(), None).unwrap(); + let mut jdb = EarlyMergeDB::new_temp(); + jdb.commit_batch(0, &b"0".sha3(), None).unwrap(); assert!(jdb.can_reconstruct_refs()); let foo = jdb.insert(b"foo"); - jdb.commit(1, &b"1a".sha3(), Some((0, b"0".sha3()))).unwrap(); + jdb.commit_batch(1, &b"1a".sha3(), Some((0, b"0".sha3()))).unwrap(); assert!(jdb.can_reconstruct_refs()); jdb.insert(b"foo"); - jdb.commit(1, &b"1b".sha3(), Some((0, b"0".sha3()))).unwrap(); + jdb.commit_batch(1, &b"1b".sha3(), Some((0, b"0".sha3()))).unwrap(); assert!(jdb.can_reconstruct_refs()); jdb.insert(b"foo"); - jdb.commit(1, &b"1c".sha3(), Some((0, b"0".sha3()))).unwrap(); + jdb.commit_batch(1, &b"1c".sha3(), Some((0, b"0".sha3()))).unwrap(); assert!(jdb.can_reconstruct_refs()); assert!(jdb.contains(&foo)); - jdb.commit(2, &b"2a".sha3(), Some((1, b"1a".sha3()))).unwrap(); + jdb.commit_batch(2, &b"2a".sha3(), Some((1, b"1a".sha3()))).unwrap(); assert!(jdb.can_reconstruct_refs()); assert!(jdb.contains(&foo)); } #[test] fn fork_same_key_other() { - let mut dir = ::std::env::temp_dir(); - dir.push(H32::random().hex()); - - let mut jdb = EarlyMergeDB::new(dir.to_str().unwrap(), DatabaseConfig::default()); - jdb.commit(0, &b"0".sha3(), None).unwrap(); + let mut jdb = EarlyMergeDB::new_temp(); + jdb.commit_batch(0, &b"0".sha3(), None).unwrap(); assert!(jdb.can_reconstruct_refs()); let foo = jdb.insert(b"foo"); - jdb.commit(1, &b"1a".sha3(), Some((0, b"0".sha3()))).unwrap(); + jdb.commit_batch(1, &b"1a".sha3(), Some((0, b"0".sha3()))).unwrap(); assert!(jdb.can_reconstruct_refs()); jdb.insert(b"foo"); - jdb.commit(1, &b"1b".sha3(), Some((0, b"0".sha3()))).unwrap(); + jdb.commit_batch(1, &b"1b".sha3(), Some((0, b"0".sha3()))).unwrap(); assert!(jdb.can_reconstruct_refs()); jdb.insert(b"foo"); - jdb.commit(1, &b"1c".sha3(), Some((0, b"0".sha3()))).unwrap(); + jdb.commit_batch(1, &b"1c".sha3(), Some((0, b"0".sha3()))).unwrap(); assert!(jdb.can_reconstruct_refs()); assert!(jdb.contains(&foo)); - jdb.commit(2, &b"2b".sha3(), Some((1, b"1b".sha3()))).unwrap(); + jdb.commit_batch(2, &b"2b".sha3(), Some((1, b"1b".sha3()))).unwrap(); assert!(jdb.can_reconstruct_refs()); assert!(jdb.contains(&foo)); } #[test] fn fork_ins_del_ins() { - let mut dir = ::std::env::temp_dir(); - dir.push(H32::random().hex()); - - let mut jdb = EarlyMergeDB::new(dir.to_str().unwrap(), DatabaseConfig::default()); - jdb.commit(0, &b"0".sha3(), None).unwrap(); + let mut jdb = EarlyMergeDB::new_temp(); + jdb.commit_batch(0, &b"0".sha3(), None).unwrap(); assert!(jdb.can_reconstruct_refs()); let foo = jdb.insert(b"foo"); - jdb.commit(1, &b"1".sha3(), None).unwrap(); + jdb.commit_batch(1, &b"1".sha3(), None).unwrap(); assert!(jdb.can_reconstruct_refs()); jdb.remove(&foo); - jdb.commit(2, &b"2a".sha3(), Some((0, b"0".sha3()))).unwrap(); + jdb.commit_batch(2, &b"2a".sha3(), Some((0, b"0".sha3()))).unwrap(); assert!(jdb.can_reconstruct_refs()); jdb.remove(&foo); - jdb.commit(2, &b"2b".sha3(), Some((0, b"0".sha3()))).unwrap(); + jdb.commit_batch(2, &b"2b".sha3(), Some((0, b"0".sha3()))).unwrap(); assert!(jdb.can_reconstruct_refs()); jdb.insert(b"foo"); - jdb.commit(3, &b"3a".sha3(), Some((1, b"1".sha3()))).unwrap(); + jdb.commit_batch(3, &b"3a".sha3(), Some((1, b"1".sha3()))).unwrap(); assert!(jdb.can_reconstruct_refs()); jdb.insert(b"foo"); - jdb.commit(3, &b"3b".sha3(), Some((1, b"1".sha3()))).unwrap(); + jdb.commit_batch(3, &b"3b".sha3(), Some((1, b"1".sha3()))).unwrap(); assert!(jdb.can_reconstruct_refs()); - jdb.commit(4, &b"4a".sha3(), Some((2, b"2a".sha3()))).unwrap(); + jdb.commit_batch(4, &b"4a".sha3(), Some((2, b"2a".sha3()))).unwrap(); assert!(jdb.can_reconstruct_refs()); - jdb.commit(5, &b"5a".sha3(), Some((3, b"3a".sha3()))).unwrap(); + jdb.commit_batch(5, &b"5a".sha3(), Some((3, b"3a".sha3()))).unwrap(); assert!(jdb.can_reconstruct_refs()); } + fn new_db(path: &Path) -> EarlyMergeDB { + let config = DatabaseConfig::with_columns(Some(1)); + let backing = Arc::new(Database::open(&config, path.to_str().unwrap()).unwrap()); + EarlyMergeDB::new(backing, Some(0)) + } + #[test] fn reopen() { let mut dir = ::std::env::temp_dir(); @@ -807,27 +795,27 @@ mod tests { let bar = H256::random(); let foo = { - let mut jdb = EarlyMergeDB::new(dir.to_str().unwrap(), DatabaseConfig::default()); + let mut jdb = new_db(&dir); // history is 1 let foo = jdb.insert(b"foo"); jdb.emplace(bar.clone(), b"bar".to_vec()); - jdb.commit(0, &b"0".sha3(), None).unwrap(); + jdb.commit_batch(0, &b"0".sha3(), None).unwrap(); assert!(jdb.can_reconstruct_refs()); foo }; { - let mut jdb = EarlyMergeDB::new(dir.to_str().unwrap(), DatabaseConfig::default()); + let mut jdb = new_db(&dir); jdb.remove(&foo); - jdb.commit(1, &b"1".sha3(), Some((0, b"0".sha3()))).unwrap(); + jdb.commit_batch(1, &b"1".sha3(), Some((0, b"0".sha3()))).unwrap(); assert!(jdb.can_reconstruct_refs()); } { - let mut jdb = EarlyMergeDB::new(dir.to_str().unwrap(), DatabaseConfig::default()); + let mut jdb = new_db(&dir); assert!(jdb.contains(&foo)); assert!(jdb.contains(&bar)); - jdb.commit(2, &b"2".sha3(), Some((1, b"1".sha3()))).unwrap(); + jdb.commit_batch(2, &b"2".sha3(), Some((1, b"1".sha3()))).unwrap(); assert!(jdb.can_reconstruct_refs()); assert!(!jdb.contains(&foo)); } @@ -836,145 +824,136 @@ mod tests { #[test] fn insert_delete_insert_delete_insert_expunge() { init_log(); - let mut dir = ::std::env::temp_dir(); - dir.push(H32::random().hex()); - let mut jdb = EarlyMergeDB::new(dir.to_str().unwrap(), DatabaseConfig::default()); + let mut jdb = EarlyMergeDB::new_temp(); // history is 4 let foo = jdb.insert(b"foo"); - jdb.commit(0, &b"0".sha3(), None).unwrap(); + jdb.commit_batch(0, &b"0".sha3(), None).unwrap(); assert!(jdb.can_reconstruct_refs()); jdb.remove(&foo); - jdb.commit(1, &b"1".sha3(), None).unwrap(); + jdb.commit_batch(1, &b"1".sha3(), None).unwrap(); assert!(jdb.can_reconstruct_refs()); jdb.insert(b"foo"); - jdb.commit(2, &b"2".sha3(), None).unwrap(); + jdb.commit_batch(2, &b"2".sha3(), None).unwrap(); assert!(jdb.can_reconstruct_refs()); jdb.remove(&foo); - jdb.commit(3, &b"3".sha3(), None).unwrap(); + jdb.commit_batch(3, &b"3".sha3(), None).unwrap(); assert!(jdb.can_reconstruct_refs()); jdb.insert(b"foo"); - jdb.commit(4, &b"4".sha3(), Some((0, b"0".sha3()))).unwrap(); + jdb.commit_batch(4, &b"4".sha3(), Some((0, b"0".sha3()))).unwrap(); assert!(jdb.can_reconstruct_refs()); // expunge foo - jdb.commit(5, &b"5".sha3(), Some((1, b"1".sha3()))).unwrap(); + jdb.commit_batch(5, &b"5".sha3(), Some((1, b"1".sha3()))).unwrap(); assert!(jdb.can_reconstruct_refs()); } #[test] fn forked_insert_delete_insert_delete_insert_expunge() { init_log(); - let mut dir = ::std::env::temp_dir(); - dir.push(H32::random().hex()); - - let mut jdb = EarlyMergeDB::new(dir.to_str().unwrap(), DatabaseConfig::default()); + let mut jdb = EarlyMergeDB::new_temp(); // history is 4 let foo = jdb.insert(b"foo"); - jdb.commit(0, &b"0".sha3(), None).unwrap(); + jdb.commit_batch(0, &b"0".sha3(), None).unwrap(); assert!(jdb.can_reconstruct_refs()); jdb.remove(&foo); - jdb.commit(1, &b"1a".sha3(), None).unwrap(); + jdb.commit_batch(1, &b"1a".sha3(), None).unwrap(); assert!(jdb.can_reconstruct_refs()); jdb.remove(&foo); - jdb.commit(1, &b"1b".sha3(), None).unwrap(); + jdb.commit_batch(1, &b"1b".sha3(), None).unwrap(); assert!(jdb.can_reconstruct_refs()); jdb.insert(b"foo"); - jdb.commit(2, &b"2a".sha3(), None).unwrap(); + jdb.commit_batch(2, &b"2a".sha3(), None).unwrap(); assert!(jdb.can_reconstruct_refs()); jdb.insert(b"foo"); - jdb.commit(2, &b"2b".sha3(), None).unwrap(); + jdb.commit_batch(2, &b"2b".sha3(), None).unwrap(); assert!(jdb.can_reconstruct_refs()); jdb.remove(&foo); - jdb.commit(3, &b"3a".sha3(), None).unwrap(); + jdb.commit_batch(3, &b"3a".sha3(), None).unwrap(); assert!(jdb.can_reconstruct_refs()); jdb.remove(&foo); - jdb.commit(3, &b"3b".sha3(), None).unwrap(); + jdb.commit_batch(3, &b"3b".sha3(), None).unwrap(); assert!(jdb.can_reconstruct_refs()); jdb.insert(b"foo"); - jdb.commit(4, &b"4a".sha3(), Some((0, b"0".sha3()))).unwrap(); + jdb.commit_batch(4, &b"4a".sha3(), Some((0, b"0".sha3()))).unwrap(); assert!(jdb.can_reconstruct_refs()); jdb.insert(b"foo"); - jdb.commit(4, &b"4b".sha3(), Some((0, b"0".sha3()))).unwrap(); + jdb.commit_batch(4, &b"4b".sha3(), Some((0, b"0".sha3()))).unwrap(); assert!(jdb.can_reconstruct_refs()); // expunge foo - jdb.commit(5, &b"5".sha3(), Some((1, b"1a".sha3()))).unwrap(); + jdb.commit_batch(5, &b"5".sha3(), Some((1, b"1a".sha3()))).unwrap(); assert!(jdb.can_reconstruct_refs()); } #[test] fn broken_assert() { - let mut dir = ::std::env::temp_dir(); - dir.push(H32::random().hex()); + let mut jdb = EarlyMergeDB::new_temp(); - let mut jdb = EarlyMergeDB::new(dir.to_str().unwrap(), DatabaseConfig::default()); // history is 1 let foo = jdb.insert(b"foo"); - jdb.commit(1, &b"1".sha3(), Some((0, b"0".sha3()))).unwrap(); + jdb.commit_batch(1, &b"1".sha3(), Some((0, b"0".sha3()))).unwrap(); assert!(jdb.can_reconstruct_refs()); // foo is ancient history. jdb.remove(&foo); - jdb.commit(2, &b"2".sha3(), Some((1, b"1".sha3()))).unwrap(); + jdb.commit_batch(2, &b"2".sha3(), Some((1, b"1".sha3()))).unwrap(); assert!(jdb.can_reconstruct_refs()); jdb.insert(b"foo"); - jdb.commit(3, &b"3".sha3(), Some((2, b"2".sha3()))).unwrap(); // BROKEN + jdb.commit_batch(3, &b"3".sha3(), Some((2, b"2".sha3()))).unwrap(); // BROKEN assert!(jdb.can_reconstruct_refs()); assert!(jdb.contains(&foo)); jdb.remove(&foo); - jdb.commit(4, &b"4".sha3(), Some((3, b"3".sha3()))).unwrap(); + jdb.commit_batch(4, &b"4".sha3(), Some((3, b"3".sha3()))).unwrap(); assert!(jdb.can_reconstruct_refs()); - jdb.commit(5, &b"5".sha3(), Some((4, b"4".sha3()))).unwrap(); + jdb.commit_batch(5, &b"5".sha3(), Some((4, b"4".sha3()))).unwrap(); assert!(jdb.can_reconstruct_refs()); assert!(!jdb.contains(&foo)); } #[test] fn reopen_test() { - let mut dir = ::std::env::temp_dir(); - dir.push(H32::random().hex()); + let mut jdb = EarlyMergeDB::new_temp(); - let mut jdb = EarlyMergeDB::new(dir.to_str().unwrap(), DatabaseConfig::default()); // history is 4 let foo = jdb.insert(b"foo"); - jdb.commit(0, &b"0".sha3(), None).unwrap(); + jdb.commit_batch(0, &b"0".sha3(), None).unwrap(); assert!(jdb.can_reconstruct_refs()); - jdb.commit(1, &b"1".sha3(), None).unwrap(); + jdb.commit_batch(1, &b"1".sha3(), None).unwrap(); assert!(jdb.can_reconstruct_refs()); - jdb.commit(2, &b"2".sha3(), None).unwrap(); + jdb.commit_batch(2, &b"2".sha3(), None).unwrap(); assert!(jdb.can_reconstruct_refs()); - jdb.commit(3, &b"3".sha3(), None).unwrap(); + jdb.commit_batch(3, &b"3".sha3(), None).unwrap(); assert!(jdb.can_reconstruct_refs()); - jdb.commit(4, &b"4".sha3(), Some((0, b"0".sha3()))).unwrap(); + jdb.commit_batch(4, &b"4".sha3(), Some((0, b"0".sha3()))).unwrap(); assert!(jdb.can_reconstruct_refs()); // foo is ancient history. jdb.insert(b"foo"); let bar = jdb.insert(b"bar"); - jdb.commit(5, &b"5".sha3(), Some((1, b"1".sha3()))).unwrap(); + jdb.commit_batch(5, &b"5".sha3(), Some((1, b"1".sha3()))).unwrap(); assert!(jdb.can_reconstruct_refs()); jdb.remove(&foo); jdb.remove(&bar); - jdb.commit(6, &b"6".sha3(), Some((2, b"2".sha3()))).unwrap(); + jdb.commit_batch(6, &b"6".sha3(), Some((2, b"2".sha3()))).unwrap(); assert!(jdb.can_reconstruct_refs()); jdb.insert(b"foo"); jdb.insert(b"bar"); - jdb.commit(7, &b"7".sha3(), Some((3, b"3".sha3()))).unwrap(); + jdb.commit_batch(7, &b"7".sha3(), Some((3, b"3".sha3()))).unwrap(); assert!(jdb.can_reconstruct_refs()); } @@ -988,45 +967,48 @@ mod tests { let foo = b"foo".sha3(); { - let mut jdb = EarlyMergeDB::new(dir.to_str().unwrap(), DatabaseConfig::default()); + let mut jdb = new_db(&dir); // history is 1 jdb.insert(b"foo"); - jdb.commit(0, &b"0".sha3(), None).unwrap(); + jdb.commit_batch(0, &b"0".sha3(), None).unwrap(); assert!(jdb.can_reconstruct_refs()); - jdb.commit(1, &b"1".sha3(), None).unwrap(); + jdb.commit_batch(1, &b"1".sha3(), None).unwrap(); assert!(jdb.can_reconstruct_refs()); // foo is ancient history. jdb.remove(&foo); - jdb.commit(2, &b"2".sha3(), Some((0, b"0".sha3()))).unwrap(); + jdb.commit_batch(2, &b"2".sha3(), Some((0, b"0".sha3()))).unwrap(); assert!(jdb.can_reconstruct_refs()); assert!(jdb.contains(&foo)); jdb.insert(b"foo"); - jdb.commit(3, &b"3".sha3(), Some((1, b"1".sha3()))).unwrap(); + jdb.commit_batch(3, &b"3".sha3(), Some((1, b"1".sha3()))).unwrap(); assert!(jdb.can_reconstruct_refs()); assert!(jdb.contains(&foo)); // incantation to reopen the db - }; { let mut jdb = EarlyMergeDB::new(dir.to_str().unwrap(), DatabaseConfig::default()); + }; { + let mut jdb = new_db(&dir); jdb.remove(&foo); - jdb.commit(4, &b"4".sha3(), Some((2, b"2".sha3()))).unwrap(); + jdb.commit_batch(4, &b"4".sha3(), Some((2, b"2".sha3()))).unwrap(); assert!(jdb.can_reconstruct_refs()); assert!(jdb.contains(&foo)); // incantation to reopen the db - }; { let mut jdb = EarlyMergeDB::new(dir.to_str().unwrap(), DatabaseConfig::default()); + }; { + let mut jdb = new_db(&dir); - jdb.commit(5, &b"5".sha3(), Some((3, b"3".sha3()))).unwrap(); + jdb.commit_batch(5, &b"5".sha3(), Some((3, b"3".sha3()))).unwrap(); assert!(jdb.can_reconstruct_refs()); assert!(jdb.contains(&foo)); // incantation to reopen the db - }; { let mut jdb = EarlyMergeDB::new(dir.to_str().unwrap(), DatabaseConfig::default()); + }; { + let mut jdb = new_db(&dir); - jdb.commit(6, &b"6".sha3(), Some((4, b"4".sha3()))).unwrap(); + jdb.commit_batch(6, &b"6".sha3(), Some((4, b"4".sha3()))).unwrap(); assert!(jdb.can_reconstruct_refs()); assert!(!jdb.contains(&foo)); } @@ -1037,26 +1019,26 @@ mod tests { let mut dir = ::std::env::temp_dir(); dir.push(H32::random().hex()); let (foo, bar, baz) = { - let mut jdb = EarlyMergeDB::new(dir.to_str().unwrap(), DatabaseConfig::default()); + let mut jdb = new_db(&dir); // history is 1 let foo = jdb.insert(b"foo"); let bar = jdb.insert(b"bar"); - jdb.commit(0, &b"0".sha3(), None).unwrap(); + jdb.commit_batch(0, &b"0".sha3(), None).unwrap(); assert!(jdb.can_reconstruct_refs()); jdb.remove(&foo); let baz = jdb.insert(b"baz"); - jdb.commit(1, &b"1a".sha3(), Some((0, b"0".sha3()))).unwrap(); + jdb.commit_batch(1, &b"1a".sha3(), Some((0, b"0".sha3()))).unwrap(); assert!(jdb.can_reconstruct_refs()); jdb.remove(&bar); - jdb.commit(1, &b"1b".sha3(), Some((0, b"0".sha3()))).unwrap(); + jdb.commit_batch(1, &b"1b".sha3(), Some((0, b"0".sha3()))).unwrap(); assert!(jdb.can_reconstruct_refs()); (foo, bar, baz) }; { - let mut jdb = EarlyMergeDB::new(dir.to_str().unwrap(), DatabaseConfig::default()); - jdb.commit(2, &b"2b".sha3(), Some((1, b"1b".sha3()))).unwrap(); + let mut jdb = new_db(&dir); + jdb.commit_batch(2, &b"2b".sha3(), Some((1, b"1b".sha3()))).unwrap(); assert!(jdb.can_reconstruct_refs()); assert!(jdb.contains(&foo)); assert!(!jdb.contains(&baz)); diff --git a/util/src/journaldb/mod.rs b/util/src/journaldb/mod.rs index 567f6100d..bb79ed9a8 100644 --- a/util/src/journaldb/mod.rs +++ b/util/src/journaldb/mod.rs @@ -17,7 +17,7 @@ //! `JournalDB` interface and implementation. use common::*; -use kvdb::DatabaseConfig; +use kvdb::Database; /// Export the journaldb module. pub mod traits; @@ -116,19 +116,18 @@ impl fmt::Display for Algorithm { } /// Create a new `JournalDB` trait object. -pub fn new(path: &str, algorithm: Algorithm, config: DatabaseConfig) -> Box { +pub fn new(backing: Arc, algorithm: Algorithm, col: Option) -> Box { match algorithm { - Algorithm::Archive => Box::new(archivedb::ArchiveDB::new(path, config)), - Algorithm::EarlyMerge => Box::new(earlymergedb::EarlyMergeDB::new(path, config)), - Algorithm::OverlayRecent => Box::new(overlayrecentdb::OverlayRecentDB::new(path, config)), - Algorithm::RefCounted => Box::new(refcounteddb::RefCountedDB::new(path, config)), + Algorithm::Archive => Box::new(archivedb::ArchiveDB::new(backing, col)), + Algorithm::EarlyMerge => Box::new(earlymergedb::EarlyMergeDB::new(backing, col)), + Algorithm::OverlayRecent => Box::new(overlayrecentdb::OverlayRecentDB::new(backing, col)), + Algorithm::RefCounted => Box::new(refcounteddb::RefCountedDB::new(backing, col)), } } // all keys must be at least 12 bytes const DB_PREFIX_LEN : usize = 12; const LATEST_ERA_KEY : [u8; DB_PREFIX_LEN] = [ b'l', b'a', b's', b't', 0, 0, 0, 0, 0, 0, 0, 0 ]; -const VERSION_KEY : [u8; DB_PREFIX_LEN] = [ b'j', b'v', b'e', b'r', 0, 0, 0, 0, 0, 0, 0, 0 ]; #[cfg(test)] mod tests { diff --git a/util/src/journaldb/overlayrecentdb.rs b/util/src/journaldb/overlayrecentdb.rs index 082692403..186fd4040 100644 --- a/util/src/journaldb/overlayrecentdb.rs +++ b/util/src/journaldb/overlayrecentdb.rs @@ -20,8 +20,8 @@ use common::*; use rlp::*; use hashdb::*; use memorydb::*; -use super::{DB_PREFIX_LEN, LATEST_ERA_KEY, VERSION_KEY}; -use kvdb::{Database, DBTransaction, DatabaseConfig}; +use super::{DB_PREFIX_LEN, LATEST_ERA_KEY}; +use kvdb::{Database, DBTransaction}; #[cfg(test)] use std::env; use super::JournalDB; @@ -61,6 +61,7 @@ pub struct OverlayRecentDB { transaction_overlay: MemoryDB, backing: Arc, journal_overlay: Arc>, + column: Option, } #[derive(PartialEq)] @@ -89,38 +90,22 @@ impl Clone for OverlayRecentDB { transaction_overlay: self.transaction_overlay.clone(), backing: self.backing.clone(), journal_overlay: self.journal_overlay.clone(), + column: self.column.clone(), } } } -const DB_VERSION : u32 = 0x203; const PADDING : [u8; 10] = [ 0u8; 10 ]; impl OverlayRecentDB { - /// Create a new instance from file - pub fn new(path: &str, config: DatabaseConfig) -> OverlayRecentDB { - Self::from_prefs(path, config) - } - - /// Create a new instance from file - pub fn from_prefs(path: &str, config: DatabaseConfig) -> OverlayRecentDB { - let backing = Database::open(&config, path).unwrap_or_else(|e| { - panic!("Error opening state db: {}", e); - }); - if !backing.is_empty() { - match backing.get(&VERSION_KEY).map(|d| d.map(|v| decode::(&v))) { - Ok(Some(DB_VERSION)) => {} - v => panic!("Incompatible DB version, expected {}, got {:?}; to resolve, remove {} and restart.", DB_VERSION, v, path) - } - } else { - backing.put(&VERSION_KEY, &encode(&DB_VERSION)).expect("Error writing version to database"); - } - - let journal_overlay = Arc::new(RwLock::new(OverlayRecentDB::read_overlay(&backing))); + /// Create a new instance. + pub fn new(backing: Arc, col: Option) -> OverlayRecentDB { + let journal_overlay = Arc::new(RwLock::new(OverlayRecentDB::read_overlay(&backing, col))); OverlayRecentDB { transaction_overlay: MemoryDB::new(), - backing: Arc::new(backing), + backing: backing, journal_overlay: journal_overlay, + column: col, } } @@ -129,31 +114,32 @@ impl OverlayRecentDB { pub fn new_temp() -> OverlayRecentDB { let mut dir = env::temp_dir(); dir.push(H32::random().hex()); - Self::new(dir.to_str().unwrap(), DatabaseConfig::default()) + let backing = Arc::new(Database::open_default(dir.to_str().unwrap()).unwrap()); + Self::new(backing, None) } #[cfg(test)] fn can_reconstruct_refs(&self) -> bool { - let reconstructed = Self::read_overlay(&self.backing); + let reconstructed = Self::read_overlay(&self.backing, self.column); let journal_overlay = self.journal_overlay.read(); *journal_overlay == reconstructed } fn payload(&self, key: &H256) -> Option { - self.backing.get(key).expect("Low-level database error. Some issue with your hard disk?").map(|v| v.to_vec()) + self.backing.get(self.column, key).expect("Low-level database error. Some issue with your hard disk?").map(|v| v.to_vec()) } - fn read_overlay(db: &Database) -> JournalOverlay { + fn read_overlay(db: &Database, col: Option) -> JournalOverlay { let mut journal = HashMap::new(); let mut overlay = MemoryDB::new(); let mut count = 0; let mut latest_era = None; - if let Some(val) = db.get(&LATEST_ERA_KEY).expect("Low-level database error.") { + if let Some(val) = db.get(col, &LATEST_ERA_KEY).expect("Low-level database error.") { let mut era = decode::(&val); latest_era = Some(era); loop { let mut index = 0usize; - while let Some(rlp_data) = db.get({ + while let Some(rlp_data) = db.get(col, { let mut r = RlpStream::new_list(3); r.append(&era); r.append(&index); @@ -212,21 +198,24 @@ impl JournalDB for OverlayRecentDB { } fn is_empty(&self) -> bool { - self.backing.get(&LATEST_ERA_KEY).expect("Low level database error").is_none() + self.backing.get(self.column, &LATEST_ERA_KEY).expect("Low level database error").is_none() + } + + fn backing(&self) -> &Arc { + &self.backing } fn latest_era(&self) -> Option { self.journal_overlay.read().latest_era } fn state(&self, key: &H256) -> Option { let v = self.journal_overlay.read().backing_overlay.get(&OverlayRecentDB::to_short_key(key)).map(|v| v.to_vec()); - v.or_else(|| self.backing.get_by_prefix(&key[0..DB_PREFIX_LEN]).map(|b| b.to_vec())) + v.or_else(|| self.backing.get_by_prefix(self.column, &key[0..DB_PREFIX_LEN]).map(|b| b.to_vec())) } - fn commit(&mut self, now: u64, id: &H256, end: Option<(u64, H256)>) -> Result { + fn commit(&mut self, batch: &DBTransaction, now: u64, id: &H256, end: Option<(u64, H256)>) -> Result { // record new commit's details. trace!("commit: #{} ({}), end era: {:?}", now, id, end); let mut journal_overlay = self.journal_overlay.write(); - let batch = DBTransaction::new(); { let mut r = RlpStream::new_list(3); let mut tx = self.transaction_overlay.drain(); @@ -249,9 +238,9 @@ impl JournalDB for OverlayRecentDB { k.append(&now); k.append(&index); k.append(&&PADDING[..]); - try!(batch.put(&k.drain(), r.as_raw())); + try!(batch.put(self.column, &k.drain(), r.as_raw())); if journal_overlay.latest_era.map_or(true, |e| now > e) { - try!(batch.put(&LATEST_ERA_KEY, &encode(&now))); + try!(batch.put(self.column, &LATEST_ERA_KEY, &encode(&now))); journal_overlay.latest_era = Some(now); } journal_overlay.journal.entry(now).or_insert_with(Vec::new).push(JournalEntry { id: id.clone(), insertions: inserted_keys, deletions: removed_keys }); @@ -271,7 +260,7 @@ impl JournalDB for OverlayRecentDB { r.append(&end_era); r.append(&index); r.append(&&PADDING[..]); - try!(batch.delete(&r.drain())); + try!(batch.delete(self.column, &r.drain())); trace!("commit: Delete journal for time #{}.{}: {}, (canon was {}): +{} -{} entries", end_era, index, journal.id, canon_id, journal.insertions.len(), journal.deletions.len()); { if canon_id == journal.id { @@ -290,7 +279,7 @@ impl JournalDB for OverlayRecentDB { } // apply canon inserts first for (k, v) in canon_insertions { - try!(batch.put(&k, &v)); + try!(batch.put(self.column, &k, &v)); } // update the overlay for k in overlay_deletions { @@ -299,13 +288,12 @@ impl JournalDB for OverlayRecentDB { // apply canon deletions for k in canon_deletions { if !journal_overlay.backing_overlay.contains(&OverlayRecentDB::to_short_key(&k)) { - try!(batch.delete(&k)); + try!(batch.delete(self.column, &k)); } } } journal_overlay.journal.remove(&end_era); } - try!(self.backing.write(batch)); Ok(0) } @@ -314,7 +302,7 @@ impl JournalDB for OverlayRecentDB { impl HashDB for OverlayRecentDB { fn keys(&self) -> HashMap { let mut ret: HashMap = HashMap::new(); - for (key, _) in self.backing.iter() { + for (key, _) in self.backing.iter(self.column) { let h = H256::from_slice(key.deref()); ret.insert(h, 1); } @@ -374,7 +362,12 @@ mod tests { use hashdb::*; use log::init_log; use journaldb::JournalDB; - use kvdb::DatabaseConfig; + use kvdb::Database; + + fn new_db(path: &Path) -> OverlayRecentDB { + let backing = Arc::new(Database::open_default(path.to_str().unwrap()).unwrap()); + OverlayRecentDB::new(backing, None) + } #[test] fn insert_same_in_fork() { @@ -382,25 +375,25 @@ mod tests { let mut jdb = OverlayRecentDB::new_temp(); let x = jdb.insert(b"X"); - jdb.commit(1, &b"1".sha3(), None).unwrap(); + jdb.commit_batch(1, &b"1".sha3(), None).unwrap(); assert!(jdb.can_reconstruct_refs()); - jdb.commit(2, &b"2".sha3(), None).unwrap(); + jdb.commit_batch(2, &b"2".sha3(), None).unwrap(); assert!(jdb.can_reconstruct_refs()); - jdb.commit(3, &b"1002a".sha3(), Some((1, b"1".sha3()))).unwrap(); + jdb.commit_batch(3, &b"1002a".sha3(), Some((1, b"1".sha3()))).unwrap(); assert!(jdb.can_reconstruct_refs()); - jdb.commit(4, &b"1003a".sha3(), Some((2, b"2".sha3()))).unwrap(); + jdb.commit_batch(4, &b"1003a".sha3(), Some((2, b"2".sha3()))).unwrap(); assert!(jdb.can_reconstruct_refs()); jdb.remove(&x); - jdb.commit(3, &b"1002b".sha3(), Some((1, b"1".sha3()))).unwrap(); + jdb.commit_batch(3, &b"1002b".sha3(), Some((1, b"1".sha3()))).unwrap(); assert!(jdb.can_reconstruct_refs()); let x = jdb.insert(b"X"); - jdb.commit(4, &b"1003b".sha3(), Some((2, b"2".sha3()))).unwrap(); + jdb.commit_batch(4, &b"1003b".sha3(), Some((2, b"2".sha3()))).unwrap(); assert!(jdb.can_reconstruct_refs()); - jdb.commit(5, &b"1004a".sha3(), Some((3, b"1002a".sha3()))).unwrap(); + jdb.commit_batch(5, &b"1004a".sha3(), Some((3, b"1002a".sha3()))).unwrap(); assert!(jdb.can_reconstruct_refs()); - jdb.commit(6, &b"1005a".sha3(), Some((4, b"1003a".sha3()))).unwrap(); + jdb.commit_batch(6, &b"1005a".sha3(), Some((4, b"1003a".sha3()))).unwrap(); assert!(jdb.can_reconstruct_refs()); assert!(jdb.contains(&x)); @@ -411,20 +404,20 @@ mod tests { // history is 3 let mut jdb = OverlayRecentDB::new_temp(); let h = jdb.insert(b"foo"); - jdb.commit(0, &b"0".sha3(), None).unwrap(); + jdb.commit_batch(0, &b"0".sha3(), None).unwrap(); assert!(jdb.can_reconstruct_refs()); assert!(jdb.contains(&h)); jdb.remove(&h); - jdb.commit(1, &b"1".sha3(), None).unwrap(); + jdb.commit_batch(1, &b"1".sha3(), None).unwrap(); assert!(jdb.can_reconstruct_refs()); assert!(jdb.contains(&h)); - jdb.commit(2, &b"2".sha3(), None).unwrap(); + jdb.commit_batch(2, &b"2".sha3(), None).unwrap(); assert!(jdb.can_reconstruct_refs()); assert!(jdb.contains(&h)); - jdb.commit(3, &b"3".sha3(), Some((0, b"0".sha3()))).unwrap(); + jdb.commit_batch(3, &b"3".sha3(), Some((0, b"0".sha3()))).unwrap(); assert!(jdb.can_reconstruct_refs()); assert!(jdb.contains(&h)); - jdb.commit(4, &b"4".sha3(), Some((1, b"1".sha3()))).unwrap(); + jdb.commit_batch(4, &b"4".sha3(), Some((1, b"1".sha3()))).unwrap(); assert!(jdb.can_reconstruct_refs()); assert!(!jdb.contains(&h)); } @@ -436,7 +429,7 @@ mod tests { let foo = jdb.insert(b"foo"); let bar = jdb.insert(b"bar"); - jdb.commit(0, &b"0".sha3(), None).unwrap(); + jdb.commit_batch(0, &b"0".sha3(), None).unwrap(); assert!(jdb.can_reconstruct_refs()); assert!(jdb.contains(&foo)); assert!(jdb.contains(&bar)); @@ -444,7 +437,7 @@ mod tests { jdb.remove(&foo); jdb.remove(&bar); let baz = jdb.insert(b"baz"); - jdb.commit(1, &b"1".sha3(), Some((0, b"0".sha3()))).unwrap(); + jdb.commit_batch(1, &b"1".sha3(), Some((0, b"0".sha3()))).unwrap(); assert!(jdb.can_reconstruct_refs()); assert!(jdb.contains(&foo)); assert!(jdb.contains(&bar)); @@ -452,20 +445,20 @@ mod tests { let foo = jdb.insert(b"foo"); jdb.remove(&baz); - jdb.commit(2, &b"2".sha3(), Some((1, b"1".sha3()))).unwrap(); + jdb.commit_batch(2, &b"2".sha3(), Some((1, b"1".sha3()))).unwrap(); assert!(jdb.can_reconstruct_refs()); assert!(jdb.contains(&foo)); assert!(!jdb.contains(&bar)); assert!(jdb.contains(&baz)); jdb.remove(&foo); - jdb.commit(3, &b"3".sha3(), Some((2, b"2".sha3()))).unwrap(); + jdb.commit_batch(3, &b"3".sha3(), Some((2, b"2".sha3()))).unwrap(); assert!(jdb.can_reconstruct_refs()); assert!(jdb.contains(&foo)); assert!(!jdb.contains(&bar)); assert!(!jdb.contains(&baz)); - jdb.commit(4, &b"4".sha3(), Some((3, b"3".sha3()))).unwrap(); + jdb.commit_batch(4, &b"4".sha3(), Some((3, b"3".sha3()))).unwrap(); assert!(jdb.can_reconstruct_refs()); assert!(!jdb.contains(&foo)); assert!(!jdb.contains(&bar)); @@ -479,25 +472,25 @@ mod tests { let foo = jdb.insert(b"foo"); let bar = jdb.insert(b"bar"); - jdb.commit(0, &b"0".sha3(), None).unwrap(); + jdb.commit_batch(0, &b"0".sha3(), None).unwrap(); assert!(jdb.can_reconstruct_refs()); assert!(jdb.contains(&foo)); assert!(jdb.contains(&bar)); jdb.remove(&foo); let baz = jdb.insert(b"baz"); - jdb.commit(1, &b"1a".sha3(), Some((0, b"0".sha3()))).unwrap(); + jdb.commit_batch(1, &b"1a".sha3(), Some((0, b"0".sha3()))).unwrap(); assert!(jdb.can_reconstruct_refs()); jdb.remove(&bar); - jdb.commit(1, &b"1b".sha3(), Some((0, b"0".sha3()))).unwrap(); + jdb.commit_batch(1, &b"1b".sha3(), Some((0, b"0".sha3()))).unwrap(); assert!(jdb.can_reconstruct_refs()); assert!(jdb.contains(&foo)); assert!(jdb.contains(&bar)); assert!(jdb.contains(&baz)); - jdb.commit(2, &b"2b".sha3(), Some((1, b"1b".sha3()))).unwrap(); + jdb.commit_batch(2, &b"2b".sha3(), Some((1, b"1b".sha3()))).unwrap(); assert!(jdb.can_reconstruct_refs()); assert!(jdb.contains(&foo)); assert!(!jdb.contains(&baz)); @@ -510,112 +503,105 @@ mod tests { let mut jdb = OverlayRecentDB::new_temp(); let foo = jdb.insert(b"foo"); - jdb.commit(0, &b"0".sha3(), None).unwrap(); + jdb.commit_batch(0, &b"0".sha3(), None).unwrap(); assert!(jdb.can_reconstruct_refs()); assert!(jdb.contains(&foo)); jdb.remove(&foo); - jdb.commit(1, &b"1".sha3(), Some((0, b"0".sha3()))).unwrap(); + jdb.commit_batch(1, &b"1".sha3(), Some((0, b"0".sha3()))).unwrap(); assert!(jdb.can_reconstruct_refs()); jdb.insert(b"foo"); assert!(jdb.contains(&foo)); - jdb.commit(2, &b"2".sha3(), Some((1, b"1".sha3()))).unwrap(); + jdb.commit_batch(2, &b"2".sha3(), Some((1, b"1".sha3()))).unwrap(); assert!(jdb.can_reconstruct_refs()); assert!(jdb.contains(&foo)); - jdb.commit(3, &b"2".sha3(), Some((0, b"2".sha3()))).unwrap(); + jdb.commit_batch(3, &b"2".sha3(), Some((0, b"2".sha3()))).unwrap(); assert!(jdb.can_reconstruct_refs()); assert!(jdb.contains(&foo)); } #[test] fn fork_same_key_one() { - let mut dir = ::std::env::temp_dir(); - dir.push(H32::random().hex()); - - let mut jdb = OverlayRecentDB::new(dir.to_str().unwrap(), DatabaseConfig::default()); - jdb.commit(0, &b"0".sha3(), None).unwrap(); + let mut jdb = OverlayRecentDB::new_temp(); + jdb.commit_batch(0, &b"0".sha3(), None).unwrap(); assert!(jdb.can_reconstruct_refs()); let foo = jdb.insert(b"foo"); - jdb.commit(1, &b"1a".sha3(), Some((0, b"0".sha3()))).unwrap(); + jdb.commit_batch(1, &b"1a".sha3(), Some((0, b"0".sha3()))).unwrap(); assert!(jdb.can_reconstruct_refs()); jdb.insert(b"foo"); - jdb.commit(1, &b"1b".sha3(), Some((0, b"0".sha3()))).unwrap(); + jdb.commit_batch(1, &b"1b".sha3(), Some((0, b"0".sha3()))).unwrap(); assert!(jdb.can_reconstruct_refs()); jdb.insert(b"foo"); - jdb.commit(1, &b"1c".sha3(), Some((0, b"0".sha3()))).unwrap(); + jdb.commit_batch(1, &b"1c".sha3(), Some((0, b"0".sha3()))).unwrap(); assert!(jdb.can_reconstruct_refs()); assert!(jdb.contains(&foo)); - jdb.commit(2, &b"2a".sha3(), Some((1, b"1a".sha3()))).unwrap(); + jdb.commit_batch(2, &b"2a".sha3(), Some((1, b"1a".sha3()))).unwrap(); assert!(jdb.can_reconstruct_refs()); assert!(jdb.contains(&foo)); } #[test] fn fork_same_key_other() { - let mut dir = ::std::env::temp_dir(); - dir.push(H32::random().hex()); + let mut jdb = OverlayRecentDB::new_temp(); - let mut jdb = OverlayRecentDB::new(dir.to_str().unwrap(), DatabaseConfig::default()); - jdb.commit(0, &b"0".sha3(), None).unwrap(); + jdb.commit_batch(0, &b"0".sha3(), None).unwrap(); assert!(jdb.can_reconstruct_refs()); let foo = jdb.insert(b"foo"); - jdb.commit(1, &b"1a".sha3(), Some((0, b"0".sha3()))).unwrap(); + jdb.commit_batch(1, &b"1a".sha3(), Some((0, b"0".sha3()))).unwrap(); assert!(jdb.can_reconstruct_refs()); jdb.insert(b"foo"); - jdb.commit(1, &b"1b".sha3(), Some((0, b"0".sha3()))).unwrap(); + jdb.commit_batch(1, &b"1b".sha3(), Some((0, b"0".sha3()))).unwrap(); assert!(jdb.can_reconstruct_refs()); jdb.insert(b"foo"); - jdb.commit(1, &b"1c".sha3(), Some((0, b"0".sha3()))).unwrap(); + jdb.commit_batch(1, &b"1c".sha3(), Some((0, b"0".sha3()))).unwrap(); assert!(jdb.can_reconstruct_refs()); assert!(jdb.contains(&foo)); - jdb.commit(2, &b"2b".sha3(), Some((1, b"1b".sha3()))).unwrap(); + jdb.commit_batch(2, &b"2b".sha3(), Some((1, b"1b".sha3()))).unwrap(); assert!(jdb.can_reconstruct_refs()); assert!(jdb.contains(&foo)); } #[test] fn fork_ins_del_ins() { - let mut dir = ::std::env::temp_dir(); - dir.push(H32::random().hex()); + let mut jdb = OverlayRecentDB::new_temp(); - let mut jdb = OverlayRecentDB::new(dir.to_str().unwrap(), DatabaseConfig::default()); - jdb.commit(0, &b"0".sha3(), None).unwrap(); + jdb.commit_batch(0, &b"0".sha3(), None).unwrap(); assert!(jdb.can_reconstruct_refs()); let foo = jdb.insert(b"foo"); - jdb.commit(1, &b"1".sha3(), None).unwrap(); + jdb.commit_batch(1, &b"1".sha3(), None).unwrap(); assert!(jdb.can_reconstruct_refs()); jdb.remove(&foo); - jdb.commit(2, &b"2a".sha3(), Some((0, b"0".sha3()))).unwrap(); + jdb.commit_batch(2, &b"2a".sha3(), Some((0, b"0".sha3()))).unwrap(); assert!(jdb.can_reconstruct_refs()); jdb.remove(&foo); - jdb.commit(2, &b"2b".sha3(), Some((0, b"0".sha3()))).unwrap(); + jdb.commit_batch(2, &b"2b".sha3(), Some((0, b"0".sha3()))).unwrap(); assert!(jdb.can_reconstruct_refs()); jdb.insert(b"foo"); - jdb.commit(3, &b"3a".sha3(), Some((1, b"1".sha3()))).unwrap(); + jdb.commit_batch(3, &b"3a".sha3(), Some((1, b"1".sha3()))).unwrap(); assert!(jdb.can_reconstruct_refs()); jdb.insert(b"foo"); - jdb.commit(3, &b"3b".sha3(), Some((1, b"1".sha3()))).unwrap(); + jdb.commit_batch(3, &b"3b".sha3(), Some((1, b"1".sha3()))).unwrap(); assert!(jdb.can_reconstruct_refs()); - jdb.commit(4, &b"4a".sha3(), Some((2, b"2a".sha3()))).unwrap(); + jdb.commit_batch(4, &b"4a".sha3(), Some((2, b"2a".sha3()))).unwrap(); assert!(jdb.can_reconstruct_refs()); - jdb.commit(5, &b"5a".sha3(), Some((3, b"3a".sha3()))).unwrap(); + jdb.commit_batch(5, &b"5a".sha3(), Some((3, b"3a".sha3()))).unwrap(); assert!(jdb.can_reconstruct_refs()); } @@ -626,27 +612,27 @@ mod tests { let bar = H256::random(); let foo = { - let mut jdb = OverlayRecentDB::new(dir.to_str().unwrap(), DatabaseConfig::default()); + let mut jdb = new_db(&dir); // history is 1 let foo = jdb.insert(b"foo"); jdb.emplace(bar.clone(), b"bar".to_vec()); - jdb.commit(0, &b"0".sha3(), None).unwrap(); + jdb.commit_batch(0, &b"0".sha3(), None).unwrap(); assert!(jdb.can_reconstruct_refs()); foo }; { - let mut jdb = OverlayRecentDB::new(dir.to_str().unwrap(), DatabaseConfig::default()); + let mut jdb = new_db(&dir); jdb.remove(&foo); - jdb.commit(1, &b"1".sha3(), Some((0, b"0".sha3()))).unwrap(); + jdb.commit_batch(1, &b"1".sha3(), Some((0, b"0".sha3()))).unwrap(); assert!(jdb.can_reconstruct_refs()); } { - let mut jdb = OverlayRecentDB::new(dir.to_str().unwrap(), DatabaseConfig::default()); + let mut jdb = new_db(&dir); assert!(jdb.contains(&foo)); assert!(jdb.contains(&bar)); - jdb.commit(2, &b"2".sha3(), Some((1, b"1".sha3()))).unwrap(); + jdb.commit_batch(2, &b"2".sha3(), Some((1, b"1".sha3()))).unwrap(); assert!(jdb.can_reconstruct_refs()); assert!(!jdb.contains(&foo)); } @@ -655,145 +641,133 @@ mod tests { #[test] fn insert_delete_insert_delete_insert_expunge() { init_log(); - let mut dir = ::std::env::temp_dir(); - dir.push(H32::random().hex()); - - let mut jdb = OverlayRecentDB::new(dir.to_str().unwrap(), DatabaseConfig::default()); + let mut jdb = OverlayRecentDB::new_temp(); // history is 4 let foo = jdb.insert(b"foo"); - jdb.commit(0, &b"0".sha3(), None).unwrap(); + jdb.commit_batch(0, &b"0".sha3(), None).unwrap(); assert!(jdb.can_reconstruct_refs()); jdb.remove(&foo); - jdb.commit(1, &b"1".sha3(), None).unwrap(); + jdb.commit_batch(1, &b"1".sha3(), None).unwrap(); assert!(jdb.can_reconstruct_refs()); jdb.insert(b"foo"); - jdb.commit(2, &b"2".sha3(), None).unwrap(); + jdb.commit_batch(2, &b"2".sha3(), None).unwrap(); assert!(jdb.can_reconstruct_refs()); jdb.remove(&foo); - jdb.commit(3, &b"3".sha3(), None).unwrap(); + jdb.commit_batch(3, &b"3".sha3(), None).unwrap(); assert!(jdb.can_reconstruct_refs()); jdb.insert(b"foo"); - jdb.commit(4, &b"4".sha3(), Some((0, b"0".sha3()))).unwrap(); + jdb.commit_batch(4, &b"4".sha3(), Some((0, b"0".sha3()))).unwrap(); assert!(jdb.can_reconstruct_refs()); // expunge foo - jdb.commit(5, &b"5".sha3(), Some((1, b"1".sha3()))).unwrap(); + jdb.commit_batch(5, &b"5".sha3(), Some((1, b"1".sha3()))).unwrap(); assert!(jdb.can_reconstruct_refs()); } #[test] fn forked_insert_delete_insert_delete_insert_expunge() { init_log(); - let mut dir = ::std::env::temp_dir(); - dir.push(H32::random().hex()); - - let mut jdb = OverlayRecentDB::new(dir.to_str().unwrap(), DatabaseConfig::default()); + let mut jdb = OverlayRecentDB::new_temp(); // history is 4 let foo = jdb.insert(b"foo"); - jdb.commit(0, &b"0".sha3(), None).unwrap(); + jdb.commit_batch(0, &b"0".sha3(), None).unwrap(); assert!(jdb.can_reconstruct_refs()); jdb.remove(&foo); - jdb.commit(1, &b"1a".sha3(), None).unwrap(); + jdb.commit_batch(1, &b"1a".sha3(), None).unwrap(); assert!(jdb.can_reconstruct_refs()); jdb.remove(&foo); - jdb.commit(1, &b"1b".sha3(), None).unwrap(); + jdb.commit_batch(1, &b"1b".sha3(), None).unwrap(); assert!(jdb.can_reconstruct_refs()); jdb.insert(b"foo"); - jdb.commit(2, &b"2a".sha3(), None).unwrap(); + jdb.commit_batch(2, &b"2a".sha3(), None).unwrap(); assert!(jdb.can_reconstruct_refs()); jdb.insert(b"foo"); - jdb.commit(2, &b"2b".sha3(), None).unwrap(); + jdb.commit_batch(2, &b"2b".sha3(), None).unwrap(); assert!(jdb.can_reconstruct_refs()); jdb.remove(&foo); - jdb.commit(3, &b"3a".sha3(), None).unwrap(); + jdb.commit_batch(3, &b"3a".sha3(), None).unwrap(); assert!(jdb.can_reconstruct_refs()); jdb.remove(&foo); - jdb.commit(3, &b"3b".sha3(), None).unwrap(); + jdb.commit_batch(3, &b"3b".sha3(), None).unwrap(); assert!(jdb.can_reconstruct_refs()); jdb.insert(b"foo"); - jdb.commit(4, &b"4a".sha3(), Some((0, b"0".sha3()))).unwrap(); + jdb.commit_batch(4, &b"4a".sha3(), Some((0, b"0".sha3()))).unwrap(); assert!(jdb.can_reconstruct_refs()); jdb.insert(b"foo"); - jdb.commit(4, &b"4b".sha3(), Some((0, b"0".sha3()))).unwrap(); + jdb.commit_batch(4, &b"4b".sha3(), Some((0, b"0".sha3()))).unwrap(); assert!(jdb.can_reconstruct_refs()); // expunge foo - jdb.commit(5, &b"5".sha3(), Some((1, b"1a".sha3()))).unwrap(); + jdb.commit_batch(5, &b"5".sha3(), Some((1, b"1a".sha3()))).unwrap(); assert!(jdb.can_reconstruct_refs()); } #[test] fn broken_assert() { - let mut dir = ::std::env::temp_dir(); - dir.push(H32::random().hex()); + let mut jdb = OverlayRecentDB::new_temp(); - let mut jdb = OverlayRecentDB::new(dir.to_str().unwrap(), DatabaseConfig::default()); - // history is 1 let foo = jdb.insert(b"foo"); - jdb.commit(1, &b"1".sha3(), Some((0, b"0".sha3()))).unwrap(); + jdb.commit_batch(1, &b"1".sha3(), Some((0, b"0".sha3()))).unwrap(); assert!(jdb.can_reconstruct_refs()); // foo is ancient history. jdb.remove(&foo); - jdb.commit(2, &b"2".sha3(), Some((1, b"1".sha3()))).unwrap(); + jdb.commit_batch(2, &b"2".sha3(), Some((1, b"1".sha3()))).unwrap(); assert!(jdb.can_reconstruct_refs()); jdb.insert(b"foo"); - jdb.commit(3, &b"3".sha3(), Some((2, b"2".sha3()))).unwrap(); // BROKEN + jdb.commit_batch(3, &b"3".sha3(), Some((2, b"2".sha3()))).unwrap(); // BROKEN assert!(jdb.can_reconstruct_refs()); assert!(jdb.contains(&foo)); jdb.remove(&foo); - jdb.commit(4, &b"4".sha3(), Some((3, b"3".sha3()))).unwrap(); + jdb.commit_batch(4, &b"4".sha3(), Some((3, b"3".sha3()))).unwrap(); assert!(jdb.can_reconstruct_refs()); - jdb.commit(5, &b"5".sha3(), Some((4, b"4".sha3()))).unwrap(); + jdb.commit_batch(5, &b"5".sha3(), Some((4, b"4".sha3()))).unwrap(); assert!(jdb.can_reconstruct_refs()); assert!(!jdb.contains(&foo)); } #[test] fn reopen_test() { - let mut dir = ::std::env::temp_dir(); - dir.push(H32::random().hex()); - - let mut jdb = OverlayRecentDB::new(dir.to_str().unwrap(), DatabaseConfig::default()); + let mut jdb = OverlayRecentDB::new_temp(); // history is 4 let foo = jdb.insert(b"foo"); - jdb.commit(0, &b"0".sha3(), None).unwrap(); + jdb.commit_batch(0, &b"0".sha3(), None).unwrap(); assert!(jdb.can_reconstruct_refs()); - jdb.commit(1, &b"1".sha3(), None).unwrap(); + jdb.commit_batch(1, &b"1".sha3(), None).unwrap(); assert!(jdb.can_reconstruct_refs()); - jdb.commit(2, &b"2".sha3(), None).unwrap(); + jdb.commit_batch(2, &b"2".sha3(), None).unwrap(); assert!(jdb.can_reconstruct_refs()); - jdb.commit(3, &b"3".sha3(), None).unwrap(); + jdb.commit_batch(3, &b"3".sha3(), None).unwrap(); assert!(jdb.can_reconstruct_refs()); - jdb.commit(4, &b"4".sha3(), Some((0, b"0".sha3()))).unwrap(); + jdb.commit_batch(4, &b"4".sha3(), Some((0, b"0".sha3()))).unwrap(); assert!(jdb.can_reconstruct_refs()); // foo is ancient history. jdb.insert(b"foo"); let bar = jdb.insert(b"bar"); - jdb.commit(5, &b"5".sha3(), Some((1, b"1".sha3()))).unwrap(); + jdb.commit_batch(5, &b"5".sha3(), Some((1, b"1".sha3()))).unwrap(); assert!(jdb.can_reconstruct_refs()); jdb.remove(&foo); jdb.remove(&bar); - jdb.commit(6, &b"6".sha3(), Some((2, b"2".sha3()))).unwrap(); + jdb.commit_batch(6, &b"6".sha3(), Some((2, b"2".sha3()))).unwrap(); assert!(jdb.can_reconstruct_refs()); jdb.insert(b"foo"); jdb.insert(b"bar"); - jdb.commit(7, &b"7".sha3(), Some((3, b"3".sha3()))).unwrap(); + jdb.commit_batch(7, &b"7".sha3(), Some((3, b"3".sha3()))).unwrap(); assert!(jdb.can_reconstruct_refs()); } @@ -807,45 +781,48 @@ mod tests { let foo = b"foo".sha3(); { - let mut jdb = OverlayRecentDB::new(dir.to_str().unwrap(), DatabaseConfig::default()); + let mut jdb = new_db(&dir); // history is 1 jdb.insert(b"foo"); - jdb.commit(0, &b"0".sha3(), None).unwrap(); + jdb.commit_batch(0, &b"0".sha3(), None).unwrap(); assert!(jdb.can_reconstruct_refs()); - jdb.commit(1, &b"1".sha3(), None).unwrap(); + jdb.commit_batch(1, &b"1".sha3(), None).unwrap(); assert!(jdb.can_reconstruct_refs()); // foo is ancient history. jdb.remove(&foo); - jdb.commit(2, &b"2".sha3(), Some((0, b"0".sha3()))).unwrap(); + jdb.commit_batch(2, &b"2".sha3(), Some((0, b"0".sha3()))).unwrap(); assert!(jdb.can_reconstruct_refs()); assert!(jdb.contains(&foo)); jdb.insert(b"foo"); - jdb.commit(3, &b"3".sha3(), Some((1, b"1".sha3()))).unwrap(); + jdb.commit_batch(3, &b"3".sha3(), Some((1, b"1".sha3()))).unwrap(); assert!(jdb.can_reconstruct_refs()); assert!(jdb.contains(&foo)); // incantation to reopen the db - }; { let mut jdb = OverlayRecentDB::new(dir.to_str().unwrap(), DatabaseConfig::default()); + }; { + let mut jdb = new_db(&dir); jdb.remove(&foo); - jdb.commit(4, &b"4".sha3(), Some((2, b"2".sha3()))).unwrap(); + jdb.commit_batch(4, &b"4".sha3(), Some((2, b"2".sha3()))).unwrap(); assert!(jdb.can_reconstruct_refs()); assert!(jdb.contains(&foo)); // incantation to reopen the db - }; { let mut jdb = OverlayRecentDB::new(dir.to_str().unwrap(), DatabaseConfig::default()); + }; { + let mut jdb = new_db(&dir); - jdb.commit(5, &b"5".sha3(), Some((3, b"3".sha3()))).unwrap(); + jdb.commit_batch(5, &b"5".sha3(), Some((3, b"3".sha3()))).unwrap(); assert!(jdb.can_reconstruct_refs()); assert!(jdb.contains(&foo)); // incantation to reopen the db - }; { let mut jdb = OverlayRecentDB::new(dir.to_str().unwrap(), DatabaseConfig::default()); + }; { + let mut jdb = new_db(&dir); - jdb.commit(6, &b"6".sha3(), Some((4, b"4".sha3()))).unwrap(); + jdb.commit_batch(6, &b"6".sha3(), Some((4, b"4".sha3()))).unwrap(); assert!(jdb.can_reconstruct_refs()); assert!(!jdb.contains(&foo)); } @@ -856,26 +833,26 @@ mod tests { let mut dir = ::std::env::temp_dir(); dir.push(H32::random().hex()); let (foo, bar, baz) = { - let mut jdb = OverlayRecentDB::new(dir.to_str().unwrap(), DatabaseConfig::default()); + let mut jdb = new_db(&dir); // history is 1 let foo = jdb.insert(b"foo"); let bar = jdb.insert(b"bar"); - jdb.commit(0, &b"0".sha3(), None).unwrap(); + jdb.commit_batch(0, &b"0".sha3(), None).unwrap(); assert!(jdb.can_reconstruct_refs()); jdb.remove(&foo); let baz = jdb.insert(b"baz"); - jdb.commit(1, &b"1a".sha3(), Some((0, b"0".sha3()))).unwrap(); + jdb.commit_batch(1, &b"1a".sha3(), Some((0, b"0".sha3()))).unwrap(); assert!(jdb.can_reconstruct_refs()); jdb.remove(&bar); - jdb.commit(1, &b"1b".sha3(), Some((0, b"0".sha3()))).unwrap(); + jdb.commit_batch(1, &b"1b".sha3(), Some((0, b"0".sha3()))).unwrap(); assert!(jdb.can_reconstruct_refs()); (foo, bar, baz) }; { - let mut jdb = OverlayRecentDB::new(dir.to_str().unwrap(), DatabaseConfig::default()); - jdb.commit(2, &b"2b".sha3(), Some((1, b"1b".sha3()))).unwrap(); + let mut jdb = new_db(&dir); + jdb.commit_batch(2, &b"2b".sha3(), Some((1, b"1b".sha3()))).unwrap(); assert!(jdb.can_reconstruct_refs()); assert!(jdb.contains(&foo)); assert!(!jdb.contains(&baz)); @@ -887,17 +864,17 @@ mod tests { fn insert_older_era() { let mut jdb = OverlayRecentDB::new_temp(); let foo = jdb.insert(b"foo"); - jdb.commit(0, &b"0a".sha3(), None).unwrap(); + jdb.commit_batch(0, &b"0a".sha3(), None).unwrap(); assert!(jdb.can_reconstruct_refs()); let bar = jdb.insert(b"bar"); - jdb.commit(1, &b"1".sha3(), Some((0, b"0a".sha3()))).unwrap(); + jdb.commit_batch(1, &b"1".sha3(), Some((0, b"0a".sha3()))).unwrap(); assert!(jdb.can_reconstruct_refs()); jdb.remove(&bar); - jdb.commit(0, &b"0b".sha3(), None).unwrap(); + jdb.commit_batch(0, &b"0b".sha3(), None).unwrap(); assert!(jdb.can_reconstruct_refs()); - jdb.commit(2, &b"2".sha3(), Some((1, b"1".sha3()))).unwrap(); + jdb.commit_batch(2, &b"2".sha3(), Some((1, b"1".sha3()))).unwrap(); assert!(jdb.contains(&foo)); assert!(jdb.contains(&bar)); diff --git a/util/src/journaldb/refcounteddb.rs b/util/src/journaldb/refcounteddb.rs index b50fc2a72..2fcbd4851 100644 --- a/util/src/journaldb/refcounteddb.rs +++ b/util/src/journaldb/refcounteddb.rs @@ -20,9 +20,9 @@ use common::*; use rlp::*; use hashdb::*; use overlaydb::*; -use super::{DB_PREFIX_LEN, LATEST_ERA_KEY, VERSION_KEY}; +use super::{DB_PREFIX_LEN, LATEST_ERA_KEY}; use super::traits::JournalDB; -use kvdb::{Database, DBTransaction, DatabaseConfig}; +use kvdb::{Database, DBTransaction}; #[cfg(test)] use std::env; @@ -39,35 +39,23 @@ pub struct RefCountedDB { latest_era: Option, inserts: Vec, removes: Vec, + column: Option, } -const DB_VERSION : u32 = 0x200; const PADDING : [u8; 10] = [ 0u8; 10 ]; impl RefCountedDB { /// Create a new instance given a `backing` database. - pub fn new(path: &str, config: DatabaseConfig) -> RefCountedDB { - let backing = Database::open(&config, path).unwrap_or_else(|e| { - panic!("Error opening state db: {}", e); - }); - if !backing.is_empty() { - match backing.get(&VERSION_KEY).map(|d| d.map(|v| decode::(&v))) { - Ok(Some(DB_VERSION)) => {}, - v => panic!("Incompatible DB version, expected {}, got {:?}; to resolve, remove {} and restart.", DB_VERSION, v, path) - } - } else { - backing.put(&VERSION_KEY, &encode(&DB_VERSION)).expect("Error writing version to database"); - } - - let backing = Arc::new(backing); - let latest_era = backing.get(&LATEST_ERA_KEY).expect("Low-level database error.").map(|val| decode::(&val)); + pub fn new(backing: Arc, col: Option) -> RefCountedDB { + let latest_era = backing.get(col, &LATEST_ERA_KEY).expect("Low-level database error.").map(|val| decode::(&val)); RefCountedDB { - forward: OverlayDB::new_with_arc(backing.clone()), + forward: OverlayDB::new(backing.clone(), col), backing: backing, inserts: vec![], removes: vec![], latest_era: latest_era, + column: col, } } @@ -76,7 +64,8 @@ impl RefCountedDB { fn new_temp() -> RefCountedDB { let mut dir = env::temp_dir(); dir.push(H32::random().hex()); - Self::new(dir.to_str().unwrap(), DatabaseConfig::default()) + let backing = Arc::new(Database::open_default(dir.to_str().unwrap()).unwrap()); + Self::new(backing, None) } } @@ -97,6 +86,7 @@ impl JournalDB for RefCountedDB { latest_era: self.latest_era, inserts: self.inserts.clone(), removes: self.removes.clone(), + column: self.column.clone(), }) } @@ -108,13 +98,17 @@ impl JournalDB for RefCountedDB { self.latest_era.is_none() } + fn backing(&self) -> &Arc { + &self.backing + } + fn latest_era(&self) -> Option { self.latest_era } fn state(&self, id: &H256) -> Option { - self.backing.get_by_prefix(&id[0..DB_PREFIX_LEN]).map(|b| b.to_vec()) + self.backing.get_by_prefix(self.column, &id[0..DB_PREFIX_LEN]).map(|b| b.to_vec()) } - fn commit(&mut self, now: u64, id: &H256, end: Option<(u64, H256)>) -> Result { + fn commit(&mut self, batch: &DBTransaction, now: u64, id: &H256, end: Option<(u64, H256)>) -> Result { // journal format: // [era, 0] => [ id, [insert_0, ...], [remove_0, ...] ] // [era, 1] => [ id, [insert_0, ...], [remove_0, ...] ] @@ -128,12 +122,11 @@ impl JournalDB for RefCountedDB { // of its inserts otherwise. // record new commit's details. - let batch = DBTransaction::new(); { let mut index = 0usize; let mut last; - while try!(self.backing.get({ + while try!(self.backing.get(self.column, { let mut r = RlpStream::new_list(3); r.append(&now); r.append(&index); @@ -148,7 +141,7 @@ impl JournalDB for RefCountedDB { r.append(id); r.append(&self.inserts); r.append(&self.removes); - try!(batch.put(&last, r.as_raw())); + try!(batch.put(self.column, &last, r.as_raw())); trace!(target: "rcdb", "new journal for time #{}.{} => {}: inserts={:?}, removes={:?}", now, index, id, self.inserts, self.removes); @@ -156,7 +149,7 @@ impl JournalDB for RefCountedDB { self.removes.clear(); if self.latest_era.map_or(true, |e| now > e) { - try!(batch.put(&LATEST_ERA_KEY, &encode(&now))); + try!(batch.put(self.column, &LATEST_ERA_KEY, &encode(&now))); self.latest_era = Some(now); } } @@ -167,7 +160,7 @@ impl JournalDB for RefCountedDB { let mut last; while let Some(rlp_data) = { // trace!(target: "rcdb", "checking for journal #{}.{}", end_era, index); - try!(self.backing.get({ + try!(self.backing.get(self.column, { let mut r = RlpStream::new_list(3); r.append(&end_era); r.append(&index); @@ -183,13 +176,12 @@ impl JournalDB for RefCountedDB { for i in &to_remove { self.forward.remove(i); } - try!(batch.delete(&last)); + try!(batch.delete(self.column, &last)); index += 1; } } let r = try!(self.forward.commit_to_batch(&batch)); - try!(self.backing.write(batch)); Ok(r) } } @@ -209,16 +201,16 @@ mod tests { // history is 3 let mut jdb = RefCountedDB::new_temp(); let h = jdb.insert(b"foo"); - jdb.commit(0, &b"0".sha3(), None).unwrap(); + jdb.commit_batch(0, &b"0".sha3(), None).unwrap(); assert!(jdb.contains(&h)); jdb.remove(&h); - jdb.commit(1, &b"1".sha3(), None).unwrap(); + jdb.commit_batch(1, &b"1".sha3(), None).unwrap(); assert!(jdb.contains(&h)); - jdb.commit(2, &b"2".sha3(), None).unwrap(); + jdb.commit_batch(2, &b"2".sha3(), None).unwrap(); assert!(jdb.contains(&h)); - jdb.commit(3, &b"3".sha3(), Some((0, b"0".sha3()))).unwrap(); + jdb.commit_batch(3, &b"3".sha3(), Some((0, b"0".sha3()))).unwrap(); assert!(jdb.contains(&h)); - jdb.commit(4, &b"4".sha3(), Some((1, b"1".sha3()))).unwrap(); + jdb.commit_batch(4, &b"4".sha3(), Some((1, b"1".sha3()))).unwrap(); assert!(!jdb.contains(&h)); } @@ -228,16 +220,16 @@ mod tests { let mut jdb = RefCountedDB::new_temp(); assert_eq!(jdb.latest_era(), None); let h = jdb.insert(b"foo"); - jdb.commit(0, &b"0".sha3(), None).unwrap(); + jdb.commit_batch(0, &b"0".sha3(), None).unwrap(); assert_eq!(jdb.latest_era(), Some(0)); jdb.remove(&h); - jdb.commit(1, &b"1".sha3(), None).unwrap(); + jdb.commit_batch(1, &b"1".sha3(), None).unwrap(); assert_eq!(jdb.latest_era(), Some(1)); - jdb.commit(2, &b"2".sha3(), None).unwrap(); + jdb.commit_batch(2, &b"2".sha3(), None).unwrap(); assert_eq!(jdb.latest_era(), Some(2)); - jdb.commit(3, &b"3".sha3(), Some((0, b"0".sha3()))).unwrap(); + jdb.commit_batch(3, &b"3".sha3(), Some((0, b"0".sha3()))).unwrap(); assert_eq!(jdb.latest_era(), Some(3)); - jdb.commit(4, &b"4".sha3(), Some((1, b"1".sha3()))).unwrap(); + jdb.commit_batch(4, &b"4".sha3(), Some((1, b"1".sha3()))).unwrap(); assert_eq!(jdb.latest_era(), Some(4)); } @@ -248,32 +240,32 @@ mod tests { let foo = jdb.insert(b"foo"); let bar = jdb.insert(b"bar"); - jdb.commit(0, &b"0".sha3(), None).unwrap(); + jdb.commit_batch(0, &b"0".sha3(), None).unwrap(); assert!(jdb.contains(&foo)); assert!(jdb.contains(&bar)); jdb.remove(&foo); jdb.remove(&bar); let baz = jdb.insert(b"baz"); - jdb.commit(1, &b"1".sha3(), Some((0, b"0".sha3()))).unwrap(); + jdb.commit_batch(1, &b"1".sha3(), Some((0, b"0".sha3()))).unwrap(); assert!(jdb.contains(&foo)); assert!(jdb.contains(&bar)); assert!(jdb.contains(&baz)); let foo = jdb.insert(b"foo"); jdb.remove(&baz); - jdb.commit(2, &b"2".sha3(), Some((1, b"1".sha3()))).unwrap(); + jdb.commit_batch(2, &b"2".sha3(), Some((1, b"1".sha3()))).unwrap(); assert!(jdb.contains(&foo)); assert!(!jdb.contains(&bar)); assert!(jdb.contains(&baz)); jdb.remove(&foo); - jdb.commit(3, &b"3".sha3(), Some((2, b"2".sha3()))).unwrap(); + jdb.commit_batch(3, &b"3".sha3(), Some((2, b"2".sha3()))).unwrap(); assert!(jdb.contains(&foo)); assert!(!jdb.contains(&bar)); assert!(!jdb.contains(&baz)); - jdb.commit(4, &b"4".sha3(), Some((3, b"3".sha3()))).unwrap(); + jdb.commit_batch(4, &b"4".sha3(), Some((3, b"3".sha3()))).unwrap(); assert!(!jdb.contains(&foo)); assert!(!jdb.contains(&bar)); assert!(!jdb.contains(&baz)); @@ -286,22 +278,22 @@ mod tests { let foo = jdb.insert(b"foo"); let bar = jdb.insert(b"bar"); - jdb.commit(0, &b"0".sha3(), None).unwrap(); + jdb.commit_batch(0, &b"0".sha3(), None).unwrap(); assert!(jdb.contains(&foo)); assert!(jdb.contains(&bar)); jdb.remove(&foo); let baz = jdb.insert(b"baz"); - jdb.commit(1, &b"1a".sha3(), Some((0, b"0".sha3()))).unwrap(); + jdb.commit_batch(1, &b"1a".sha3(), Some((0, b"0".sha3()))).unwrap(); jdb.remove(&bar); - jdb.commit(1, &b"1b".sha3(), Some((0, b"0".sha3()))).unwrap(); + jdb.commit_batch(1, &b"1b".sha3(), Some((0, b"0".sha3()))).unwrap(); assert!(jdb.contains(&foo)); assert!(jdb.contains(&bar)); assert!(jdb.contains(&baz)); - jdb.commit(2, &b"2b".sha3(), Some((1, b"1b".sha3()))).unwrap(); + jdb.commit_batch(2, &b"2b".sha3(), Some((1, b"1b".sha3()))).unwrap(); assert!(jdb.contains(&foo)); assert!(!jdb.contains(&baz)); assert!(!jdb.contains(&bar)); diff --git a/util/src/journaldb/traits.rs b/util/src/journaldb/traits.rs index 53fd17a62..8fd597349 100644 --- a/util/src/journaldb/traits.rs +++ b/util/src/journaldb/traits.rs @@ -18,6 +18,7 @@ use common::*; use hashdb::*; +use kvdb::{Database, DBTransaction}; /// A `HashDB` which can manage a short-term journal potentially containing many forks of mutually /// exclusive actions. @@ -36,11 +37,22 @@ pub trait JournalDB : HashDB + Send + Sync { /// Commit all recent insert operations and canonical historical commits' removals from the /// old era to the backing database, reverting any non-canonical historical commit's inserts. - fn commit(&mut self, now: u64, id: &H256, end: Option<(u64, H256)>) -> Result; + fn commit(&mut self, batch: &DBTransaction, now: u64, id: &H256, end: Option<(u64, H256)>) -> Result; /// State data query fn state(&self, _id: &H256) -> Option; /// Whether this database is pruned. fn is_pruned(&self) -> bool { true } + + /// Get backing database. + fn backing(&self) -> &Arc; + + #[cfg(test)] + /// Commit all changes in a single batch + fn commit_batch(&mut self, now: u64, id: &H256, end: Option<(u64, H256)>) -> Result { + let batch = self.backing().transaction(); + let res = try!(self.commit(&batch, now, id, end)); + self.backing().write(batch).map(|_| res).map_err(Into::into) + } } diff --git a/util/src/kvdb.rs b/util/src/kvdb.rs index f3dd5fa12..744966de4 100644 --- a/util/src/kvdb.rs +++ b/util/src/kvdb.rs @@ -18,7 +18,7 @@ use std::default::Default; use rocksdb::{DB, Writable, WriteBatch, WriteOptions, IteratorMode, DBVector, DBIterator, - Options, DBCompactionStyle, BlockBasedOptions, Direction, Cache}; + Options, DBCompactionStyle, BlockBasedOptions, Direction, Cache, Column}; const DB_BACKGROUND_FLUSHES: i32 = 2; const DB_BACKGROUND_COMPACTIONS: i32 = 2; @@ -26,33 +26,31 @@ const DB_BACKGROUND_COMPACTIONS: i32 = 2; /// Write transaction. Batches a sequence of put/delete operations for efficiency. pub struct DBTransaction { batch: WriteBatch, -} - -impl Default for DBTransaction { - fn default() -> Self { - DBTransaction::new() - } + cfs: Vec, } impl DBTransaction { /// Create new transaction. - pub fn new() -> DBTransaction { - DBTransaction { batch: WriteBatch::new() } + pub fn new(db: &Database) -> DBTransaction { + DBTransaction { + batch: WriteBatch::new(), + cfs: db.cfs.clone(), + } } /// Insert a key-value pair in the transaction. Any existing value value will be overwritten upon write. - pub fn put(&self, key: &[u8], value: &[u8]) -> Result<(), String> { - self.batch.put(key, value) + pub fn put(&self, col: Option, key: &[u8], value: &[u8]) -> Result<(), String> { + col.map_or_else(|| self.batch.put(key, value), |c| self.batch.put_cf(self.cfs[c as usize], key, value)) } /// Delete value by key. - pub fn delete(&self, key: &[u8]) -> Result<(), String> { - self.batch.delete(key) + pub fn delete(&self, col: Option, key: &[u8]) -> Result<(), String> { + col.map_or_else(|| self.batch.delete(key), |c| self.batch.delete_cf(self.cfs[c as usize], key)) } } /// Compaction profile for the database settings -#[derive(Clone)] +#[derive(Clone, Copy)] pub struct CompactionProfile { /// L0-L1 target file size pub initial_file_size: u64, @@ -85,6 +83,7 @@ impl CompactionProfile { } /// Database configuration +#[derive(Clone, Copy)] pub struct DatabaseConfig { /// Max number of open files. pub max_open_files: i32, @@ -92,22 +91,16 @@ pub struct DatabaseConfig { pub cache_size: Option, /// Compaction profile pub compaction: CompactionProfile, + /// Set number of columns + pub columns: Option, } impl DatabaseConfig { - /// Database with default settings and specified cache size - pub fn with_cache(cache_size: usize) -> DatabaseConfig { - DatabaseConfig { - cache_size: Some(cache_size), - max_open_files: 256, - compaction: CompactionProfile::default(), - } - } - - /// Modify the compaction profile - pub fn compaction(mut self, profile: CompactionProfile) -> Self { - self.compaction = profile; - self + /// Create new `DatabaseConfig` with default parameters and specified set of columns. + pub fn with_columns(columns: Option) -> Self { + let mut config = Self::default(); + config.columns = columns; + config } } @@ -115,8 +108,9 @@ impl Default for DatabaseConfig { fn default() -> DatabaseConfig { DatabaseConfig { cache_size: None, - max_open_files: 256, + max_open_files: 1024, compaction: CompactionProfile::default(), + columns: None, } } } @@ -138,6 +132,7 @@ impl<'a> Iterator for DatabaseIterator { pub struct Database { db: DB, write_opts: WriteOptions, + cfs: Vec, } impl Database { @@ -171,10 +166,35 @@ impl Database { opts.set_block_based_table_factory(&block_opts); } - let write_opts = WriteOptions::new(); - //write_opts.disable_wal(true); // TODO: make sure this is safe + let mut write_opts = WriteOptions::new(); + write_opts.disable_wal(true); // TODO: make sure this is safe - let db = match DB::open(&opts, path) { + let mut cfs: Vec = Vec::new(); + let db = match config.columns { + Some(columns) => { + let cfnames: Vec<_> = (0..columns).map(|c| format!("col{}", c)).collect(); + let cfnames: Vec<&str> = cfnames.iter().map(|n| n as &str).collect(); + match DB::open_cf(&opts, path, &cfnames) { + Ok(db) => { + cfs = cfnames.iter().map(|n| db.cf_handle(n).unwrap()).collect(); + assert!(cfs.len() == columns as usize); + Ok(db) + } + Err(_) => { + // retry and create CFs + match DB::open_cf(&opts, path, &[]) { + Ok(mut db) => { + cfs = cfnames.iter().map(|n| db.create_cf(n, &opts).unwrap()).collect(); + Ok(db) + }, + err @ Err(_) => err, + } + } + } + }, + None => DB::open(&opts, path) + }; + let db = match db { Ok(db) => db, Err(ref s) if s.starts_with("Corruption:") => { info!("{}", s); @@ -184,17 +204,12 @@ impl Database { }, Err(s) => { return Err(s); } }; - Ok(Database { db: db, write_opts: write_opts, }) + Ok(Database { db: db, write_opts: write_opts, cfs: cfs }) } - /// Insert a key-value pair in the transaction. Any existing value value will be overwritten. - pub fn put(&self, key: &[u8], value: &[u8]) -> Result<(), String> { - self.db.put_opt(key, value, &self.write_opts) - } - - /// Delete value by key. - pub fn delete(&self, key: &[u8]) -> Result<(), String> { - self.db.delete_opt(key, &self.write_opts) + /// Creates new transaction for this database. + pub fn transaction(&self) -> DBTransaction { + DBTransaction::new(self) } /// Commit transaction to database. @@ -203,13 +218,14 @@ impl Database { } /// Get value by key. - pub fn get(&self, key: &[u8]) -> Result, String> { - self.db.get(key) + pub fn get(&self, col: Option, key: &[u8]) -> Result, String> { + col.map_or_else(|| self.db.get(key), |c| self.db.get_cf(self.cfs[c as usize], key)) } /// Get value by partial key. Prefix size should match configured prefix size. - pub fn get_by_prefix(&self, prefix: &[u8]) -> Option> { - let mut iter = self.db.iterator(IteratorMode::From(prefix, Direction::Forward)); + pub fn get_by_prefix(&self, col: Option, prefix: &[u8]) -> Option> { + let mut iter = col.map_or_else(|| self.db.iterator(IteratorMode::From(prefix, Direction::Forward)), + |c| self.db.iterator_cf(self.cfs[c as usize], IteratorMode::From(prefix, Direction::Forward)).unwrap()); match iter.next() { // TODO: use prefix_same_as_start read option (not availabele in C API currently) Some((k, v)) => if k[0 .. prefix.len()] == prefix[..] { Some(v) } else { None }, @@ -218,13 +234,14 @@ impl Database { } /// Check if there is anything in the database. - pub fn is_empty(&self) -> bool { - self.db.iterator(IteratorMode::Start).next().is_none() + pub fn is_empty(&self, col: Option) -> bool { + self.iter(col).next().is_none() } - /// Check if there is anything in the database. - pub fn iter(&self) -> DatabaseIterator { - DatabaseIterator { iter: self.db.iterator(IteratorMode::Start) } + /// Get database iterator. + pub fn iter(&self, col: Option) -> DatabaseIterator { + col.map_or_else(|| DatabaseIterator { iter: self.db.iterator(IteratorMode::Start) }, + |c| DatabaseIterator { iter: self.db.iterator_cf(self.cfs[c as usize], IteratorMode::Start).unwrap() }) } } @@ -243,38 +260,46 @@ mod tests { let key2 = H256::from_str("03c69be41d0b7e40352fc85be1cd65eb03d40ef8427a0ca4596b1ead9a00e9fc").unwrap(); let key3 = H256::from_str("01c69be41d0b7e40352fc85be1cd65eb03d40ef8427a0ca4596b1ead9a00e9fc").unwrap(); - db.put(&key1, b"cat").unwrap(); - db.put(&key2, b"dog").unwrap(); + let batch = db.transaction(); + batch.put(None, &key1, b"cat").unwrap(); + batch.put(None, &key2, b"dog").unwrap(); + db.write(batch).unwrap(); - assert_eq!(db.get(&key1).unwrap().unwrap().deref(), b"cat"); + assert_eq!(db.get(None, &key1).unwrap().unwrap().deref(), b"cat"); - let contents: Vec<_> = db.iter().collect(); + let contents: Vec<_> = db.iter(None).collect(); assert_eq!(contents.len(), 2); assert_eq!(&*contents[0].0, key1.deref()); assert_eq!(&*contents[0].1, b"cat"); assert_eq!(&*contents[1].0, key2.deref()); assert_eq!(&*contents[1].1, b"dog"); - db.delete(&key1).unwrap(); - assert!(db.get(&key1).unwrap().is_none()); - db.put(&key1, b"cat").unwrap(); + let batch = db.transaction(); + batch.delete(None, &key1).unwrap(); + db.write(batch).unwrap(); - let transaction = DBTransaction::new(); - transaction.put(&key3, b"elephant").unwrap(); - transaction.delete(&key1).unwrap(); + assert!(db.get(None, &key1).unwrap().is_none()); + + let batch = db.transaction(); + batch.put(None, &key1, b"cat").unwrap(); + db.write(batch).unwrap(); + + let transaction = db.transaction(); + transaction.put(None, &key3, b"elephant").unwrap(); + transaction.delete(None, &key1).unwrap(); db.write(transaction).unwrap(); - assert!(db.get(&key1).unwrap().is_none()); - assert_eq!(db.get(&key3).unwrap().unwrap().deref(), b"elephant"); + assert!(db.get(None, &key1).unwrap().is_none()); + assert_eq!(db.get(None, &key3).unwrap().unwrap().deref(), b"elephant"); - assert_eq!(db.get_by_prefix(&key3).unwrap().deref(), b"elephant"); - assert_eq!(db.get_by_prefix(&key2).unwrap().deref(), b"dog"); + assert_eq!(db.get_by_prefix(None, &key3).unwrap().deref(), b"elephant"); + assert_eq!(db.get_by_prefix(None, &key2).unwrap().deref(), b"dog"); } #[test] fn kvdb() { let path = RandomTempPath::create_dir(); let smoke = Database::open_default(path.as_path().to_str().unwrap()).unwrap(); - assert!(smoke.is_empty()); + assert!(smoke.is_empty(None)); test_db(&DatabaseConfig::default()); } } diff --git a/util/src/migration/mod.rs b/util/src/migration/mod.rs index de62c0519..5a4ba2ecc 100644 --- a/util/src/migration/mod.rs +++ b/util/src/migration/mod.rs @@ -46,14 +46,16 @@ impl Default for Config { pub struct Batch { inner: BTreeMap, Vec>, batch_size: usize, + column: Option, } impl Batch { /// Make a new batch with the given config. - pub fn new(config: &Config) -> Self { + pub fn new(config: &Config, col: Option) -> Self { Batch { inner: BTreeMap::new(), batch_size: config.batch_size, + column: col, } } @@ -70,10 +72,10 @@ impl Batch { pub fn commit(&mut self, dest: &mut Database) -> Result<(), Error> { if self.inner.is_empty() { return Ok(()) } - let transaction = DBTransaction::new(); + let transaction = DBTransaction::new(dest); for keypair in &self.inner { - try!(transaction.put(&keypair.0, &keypair.1).map_err(Error::Custom)); + try!(transaction.put(self.column, &keypair.0, &keypair.1).map_err(Error::Custom)); } self.inner.clear(); @@ -102,14 +104,18 @@ impl From<::std::io::Error> for Error { /// A generalized migration from the given db to a destination db. pub trait Migration: 'static { + /// Number of columns in database after the migration. + fn columns(&self) -> Option; /// Version of the database after the migration. fn version(&self) -> u32; /// Migrate a source to a destination. - fn migrate(&mut self, source: &Database, config: &Config, destination: &mut Database) -> Result<(), Error>; + fn migrate(&mut self, source: &Database, config: &Config, destination: &mut Database, col: Option) -> Result<(), Error>; } /// A simple migration over key-value pairs. pub trait SimpleMigration: 'static { + /// Number of columns in database after the migration. + fn columns(&self) -> Option; /// Version of database after the migration. fn version(&self) -> u32; /// Should migrate existing object to new database. @@ -118,12 +124,14 @@ pub trait SimpleMigration: 'static { } impl Migration for T { + fn columns(&self) -> Option { SimpleMigration::columns(self) } + fn version(&self) -> u32 { SimpleMigration::version(self) } - fn migrate(&mut self, source: &Database, config: &Config, dest: &mut Database) -> Result<(), Error> { - let mut batch = Batch::new(config); + fn migrate(&mut self, source: &Database, config: &Config, dest: &mut Database, col: Option) -> Result<(), Error> { + let mut batch = Batch::new(config, col); - for (key, value) in source.iter() { + for (key, value) in source.iter(col) { if let Some((key, value)) = self.simple_migrate(key.to_vec(), value.to_vec()) { try!(batch.insert(key, value, dest)); } @@ -197,12 +205,14 @@ impl Manager { /// and producing a path where the final migration lives. pub fn execute(&mut self, old_path: &Path, version: u32) -> Result { let config = self.config.clone(); - let mut migrations = self.migrations_from(version); + let columns = self.no_of_columns_at(version); + let migrations = self.migrations_from(version); if migrations.is_empty() { return Err(Error::MigrationImpossible) }; - let db_config = DatabaseConfig { + let mut db_config = DatabaseConfig { max_open_files: 64, cache_size: None, - compaction: config.compaction_profile.clone(), + compaction: config.compaction_profile, + columns: columns, }; let db_root = database_path(old_path); @@ -212,14 +222,28 @@ impl Manager { // start with the old db. let old_path_str = try!(old_path.to_str().ok_or(Error::MigrationImpossible)); let mut cur_db = try!(Database::open(&db_config, old_path_str).map_err(Error::Custom)); + for migration in migrations { + // Change number of columns in new db + let current_columns = db_config.columns; + db_config.columns = migration.columns(); + // open the target temporary database. temp_path = temp_idx.path(&db_root); let temp_path_str = try!(temp_path.to_str().ok_or(Error::MigrationImpossible)); let mut new_db = try!(Database::open(&db_config, temp_path_str).map_err(Error::Custom)); // perform the migration from cur_db to new_db. - try!(migration.migrate(&cur_db, &config, &mut new_db)); + match current_columns { + // migrate only default column + None => try!(migration.migrate(&cur_db, &config, &mut new_db, None)), + Some(v) => { + // Migrate all columns in previous DB + for col in 0..v { + try!(migration.migrate(&cur_db, &config, &mut new_db, Some(col))) + } + } + } // next iteration, we will migrate from this db into the other temp. cur_db = new_db; temp_idx.swap(); @@ -242,5 +266,38 @@ impl Manager { fn migrations_from(&mut self, version: u32) -> Vec<&mut Box> { self.migrations.iter_mut().filter(|m| m.version() > version).collect() } + + fn no_of_columns_at(&self, version: u32) -> Option { + let migration = self.migrations.iter().find(|m| m.version() == version); + match migration { + Some(m) => m.columns(), + None => None + } + } } +/// Prints a dot every `max` ticks +pub struct Progress { + current: usize, + max: usize, +} + +impl Default for Progress { + fn default() -> Self { + Progress { + current: 0, + max: 100_000, + } + } +} + +impl Progress { + /// Tick progress meter. + pub fn tick(&mut self) { + self.current += 1; + if self.current == self.max { + self.current = 0; + flush!("."); + } + } +} diff --git a/util/src/migration/tests.rs b/util/src/migration/tests.rs index 584e19f99..8eec87c21 100644 --- a/util/src/migration/tests.rs +++ b/util/src/migration/tests.rs @@ -20,7 +20,7 @@ use common::*; use migration::{Config, SimpleMigration, Manager}; -use kvdb::{Database, DBTransaction}; +use kvdb::Database; use devtools::RandomTempPath; use std::path::PathBuf; @@ -35,9 +35,9 @@ fn db_path(path: &Path) -> PathBuf { fn make_db(path: &Path, pairs: BTreeMap, Vec>) { let db = Database::open_default(path.to_str().unwrap()).expect("failed to open temp database"); { - let transaction = DBTransaction::new(); + let transaction = db.transaction(); for (k, v) in pairs { - transaction.put(&k, &v).expect("failed to add pair to transaction"); + transaction.put(None, &k, &v).expect("failed to add pair to transaction"); } db.write(transaction).expect("failed to write db transaction"); @@ -49,7 +49,7 @@ fn verify_migration(path: &Path, pairs: BTreeMap, Vec>) { let db = Database::open_default(path.to_str().unwrap()).unwrap(); for (k, v) in pairs { - let x = db.get(&k).unwrap().unwrap(); + let x = db.get(None, &k).unwrap().unwrap(); assert_eq!(&x[..], &v[..]); } @@ -58,9 +58,9 @@ fn verify_migration(path: &Path, pairs: BTreeMap, Vec>) { struct Migration0; impl SimpleMigration for Migration0 { - fn version(&self) -> u32 { - 1 - } + fn columns(&self) -> Option { None } + + fn version(&self) -> u32 { 1 } fn simple_migrate(&mut self, key: Vec, value: Vec) -> Option<(Vec, Vec)> { let mut key = key; @@ -74,9 +74,9 @@ impl SimpleMigration for Migration0 { struct Migration1; impl SimpleMigration for Migration1 { - fn version(&self) -> u32 { - 2 - } + fn columns(&self) -> Option { None } + + fn version(&self) -> u32 { 2 } fn simple_migrate(&mut self, key: Vec, _value: Vec) -> Option<(Vec, Vec)> { Some((key, vec![])) diff --git a/util/src/overlaydb.rs b/util/src/overlaydb.rs index fcf08795b..7c2ab3b71 100644 --- a/util/src/overlaydb.rs +++ b/util/src/overlaydb.rs @@ -24,7 +24,6 @@ use hashdb::*; use memorydb::*; use std::ops::*; use std::sync::*; -use std::env; use std::collections::HashMap; use kvdb::{Database, DBTransaction}; @@ -40,22 +39,29 @@ use kvdb::{Database, DBTransaction}; pub struct OverlayDB { overlay: MemoryDB, backing: Arc, + column: Option, } impl OverlayDB { /// Create a new instance of OverlayDB given a `backing` database. - pub fn new(backing: Database) -> OverlayDB { Self::new_with_arc(Arc::new(backing)) } - - /// Create a new instance of OverlayDB given a `backing` database. - pub fn new_with_arc(backing: Arc) -> OverlayDB { - OverlayDB{ overlay: MemoryDB::new(), backing: backing } + pub fn new(backing: Arc, col: Option) -> OverlayDB { + OverlayDB{ overlay: MemoryDB::new(), backing: backing, column: col } } /// Create a new instance of OverlayDB with an anonymous temporary database. + #[cfg(test)] pub fn new_temp() -> OverlayDB { - let mut dir = env::temp_dir(); + let mut dir = ::std::env::temp_dir(); dir.push(H32::random().hex()); - Self::new(Database::open_default(dir.to_str().unwrap()).unwrap()) + Self::new(Arc::new(Database::open_default(dir.to_str().unwrap()).unwrap()), None) + } + + /// Commit all operations in a single batch. + #[cfg(test)] + pub fn commit(&mut self) -> Result { + let batch = self.backing.transaction(); + let res = try!(self.commit_to_batch(&batch)); + self.backing.write(batch).map(|_| res).map_err(|e| e.into()) } /// Commit all operations to given batch. @@ -88,83 +94,8 @@ impl OverlayDB { Ok(ret) } - /// Commit all memory operations to the backing database. - /// - /// Returns either an error or the number of items changed in the backing database. - /// - /// Will return an error if the number of `remove()`s ever exceeds the number of - /// `insert()`s for any key. This will leave the database in an undeterminate - /// state. Don't ever let it happen. - /// - /// # Example - /// ``` - /// extern crate ethcore_util; - /// use ethcore_util::hashdb::*; - /// use ethcore_util::overlaydb::*; - /// fn main() { - /// let mut m = OverlayDB::new_temp(); - /// let key = m.insert(b"foo"); // insert item. - /// assert!(m.contains(&key)); // key exists (in memory). - /// assert_eq!(m.commit().unwrap(), 1); // 1 item changed. - /// assert!(m.contains(&key)); // key still exists (in backing). - /// m.remove(&key); // delete item. - /// assert!(!m.contains(&key)); // key "doesn't exist" (though still does in backing). - /// m.remove(&key); // oh dear... more removes than inserts for the key... - /// //m.commit().unwrap(); // this commit/unwrap would cause a panic. - /// m.revert(); // revert both removes. - /// assert!(m.contains(&key)); // key now still exists. - /// } - /// ``` - pub fn commit(&mut self) -> Result { - let mut ret = 0u32; - let mut deletes = 0usize; - for i in self.overlay.drain().into_iter() { - let (key, (value, rc)) = i; - if rc != 0 { - match self.payload(&key) { - Some(x) => { - let (back_value, back_rc) = x; - let total_rc: i32 = back_rc as i32 + rc; - if total_rc < 0 { - return Err(From::from(BaseDataError::NegativelyReferencedHash(key))); - } - deletes += if self.put_payload(&key, (back_value, total_rc as u32)) {1} else {0}; - } - None => { - if rc < 0 { - return Err(From::from(BaseDataError::NegativelyReferencedHash(key))); - } - self.put_payload(&key, (value, rc as u32)); - } - }; - ret += 1; - } - } - trace!("OverlayDB::commit() deleted {} nodes", deletes); - Ok(ret) - } - /// Revert all operations on this object (i.e. `insert()`s and `remove()`s) since the /// last `commit()`. - /// - /// # Example - /// ``` - /// extern crate ethcore_util; - /// use ethcore_util::hashdb::*; - /// use ethcore_util::overlaydb::*; - /// fn main() { - /// let mut m = OverlayDB::new_temp(); - /// let foo = m.insert(b"foo"); // insert foo. - /// m.commit().unwrap(); // commit - new operations begin here... - /// let bar = m.insert(b"bar"); // insert bar. - /// m.remove(&foo); // remove foo. - /// assert!(!m.contains(&foo)); // foo is gone. - /// assert!(m.contains(&bar)); // bar is here. - /// m.revert(); // revert the last two operations. - /// assert!(m.contains(&foo)); // foo is here. - /// assert!(!m.contains(&bar)); // bar is gone. - /// } - /// ``` pub fn revert(&mut self) { self.overlay.clear(); } /// Get the number of references that would be committed. @@ -172,7 +103,7 @@ impl OverlayDB { /// Get the refs and value of the given key. fn payload(&self, key: &H256) -> Option<(Bytes, u32)> { - self.backing.get(key) + self.backing.get(self.column, key) .expect("Low-level database error. Some issue with your hard disk?") .map(|d| { let r = Rlp::new(&d); @@ -186,24 +117,10 @@ impl OverlayDB { let mut s = RlpStream::new_list(2); s.append(&payload.1); s.append(&payload.0); - batch.put(key, s.as_raw()).expect("Low-level database error. Some issue with your hard disk?"); + batch.put(self.column, key, s.as_raw()).expect("Low-level database error. Some issue with your hard disk?"); false } else { - batch.delete(key).expect("Low-level database error. Some issue with your hard disk?"); - true - } - } - - /// Put the refs and value of the given key, possibly deleting it from the db. - fn put_payload(&self, key: &H256, payload: (Bytes, u32)) -> bool { - if payload.1 > 0 { - let mut s = RlpStream::new_list(2); - s.append(&payload.1); - s.append(&payload.0); - self.backing.put(key, s.as_raw()).expect("Low-level database error. Some issue with your hard disk?"); - false - } else { - self.backing.delete(key).expect("Low-level database error. Some issue with your hard disk?"); + batch.delete(self.column, key).expect("Low-level database error. Some issue with your hard disk?"); true } } @@ -212,7 +129,7 @@ impl OverlayDB { impl HashDB for OverlayDB { fn keys(&self) -> HashMap { let mut ret: HashMap = HashMap::new(); - for (key, _) in self.backing.iter() { + for (key, _) in self.backing.iter(self.column) { let h = H256::from_slice(key.deref()); let r = self.payload(&h).unwrap().1; ret.insert(h, r as i32); @@ -274,6 +191,22 @@ impl HashDB for OverlayDB { fn remove(&mut self, key: &H256) { self.overlay.remove(key); } } +#[test] +fn overlaydb_revert() { + let mut m = OverlayDB::new_temp(); + let foo = m.insert(b"foo"); // insert foo. + let batch = m.backing.transaction(); + m.commit_to_batch(&batch).unwrap(); // commit - new operations begin here... + m.backing.write(batch).unwrap(); + let bar = m.insert(b"bar"); // insert bar. + m.remove(&foo); // remove foo. + assert!(!m.contains(&foo)); // foo is gone. + assert!(m.contains(&bar)); // bar is here. + m.revert(); // revert the last two operations. + assert!(m.contains(&foo)); // foo is here. + assert!(!m.contains(&bar)); // bar is gone. +} + #[test] fn overlaydb_overlay_insert_and_remove() { let mut trie = OverlayDB::new_temp(); @@ -366,14 +299,18 @@ fn overlaydb_complex() { fn playpen() { use std::fs; { - let db: Database = Database::open_default("/tmp/test").unwrap(); - db.put(b"test", b"test2").unwrap(); - match db.get(b"test") { + let db = Database::open_default("/tmp/test").unwrap(); + let batch = db.transaction(); + batch.put(None, b"test", b"test2").unwrap(); + db.write(batch).unwrap(); + match db.get(None, b"test") { Ok(Some(value)) => println!("Got value {:?}", value.deref()), Ok(None) => println!("No value for that key"), Err(..) => println!("Gah"), } - db.delete(b"test").unwrap(); + let batch = db.transaction(); + batch.delete(None, b"test").unwrap(); + db.write(batch).unwrap(); } fs::remove_dir_all("/tmp/test").unwrap(); } diff --git a/util/src/rlp/commonrlps.rs b/util/src/rlp/commonrlps.rs index f475d2137..670657224 100644 --- a/util/src/rlp/commonrlps.rs +++ b/util/src/rlp/commonrlps.rs @@ -59,7 +59,7 @@ mod tests { use kvdb::*; let path = "db path".to_string(); - let values: Vec<_> = Database::open_default(&path).unwrap().iter().map(|(_, v)| v).collect(); + let values: Vec<_> = Database::open_default(&path).unwrap().iter(Some(2)).map(|(_, v)| v).collect(); let mut rlp_counts: HashMap<_, u32> = HashMap::new(); let mut rlp_sizes: HashMap<_, u32> = HashMap::new(); diff --git a/util/src/rlp/rlpcompression.rs b/util/src/rlp/rlpcompression.rs index 6d34cdfd5..a9e74addc 100644 --- a/util/src/rlp/rlpcompression.rs +++ b/util/src/rlp/rlpcompression.rs @@ -228,7 +228,7 @@ mod tests { fn test_compression() { use kvdb::*; let path = "db to test".to_string(); - let values: Vec<_> = Database::open_default(&path).unwrap().iter().map(|(_, v)| v).collect(); + let values: Vec<_> = Database::open_default(&path).unwrap().iter(Some(2)).map(|(_, v)| v).collect(); let mut decomp_size = 0; let mut comp_size = 0;