From 56b6adec689561139d7e9b2d7a42c12217b2850c Mon Sep 17 00:00:00 2001 From: efyang Date: Sun, 22 Oct 2017 20:58:06 -0500 Subject: [PATCH] Iterate over both buffered and unbuffered database entries --- Cargo.lock | 7 +++++++ util/kvdb-rocksdb/Cargo.toml | 1 + util/kvdb-rocksdb/src/lib.rs | 20 +++++++++++++++----- 3 files changed, 23 insertions(+), 5 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 935389755..5e5ea3ae3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1218,6 +1218,11 @@ name = "integer-encoding" version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" +[[package]] +name = "interleaved-ordered" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + [[package]] name = "iovec" version = "0.1.0" @@ -1402,6 +1407,7 @@ version = "0.1.0" dependencies = [ "elastic-array 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", "ethcore-bigint 0.1.3", + "interleaved-ordered 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", "kvdb 0.1.0", "log 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)", "parking_lot 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", @@ -3609,6 +3615,7 @@ dependencies = [ "checksum idna 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)" = "014b298351066f1512874135335d62a789ffe78a9974f94b43ed5621951eaf7d" "checksum igd 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)" = "356a0dc23a4fa0f8ce4777258085d00a01ea4923b2efd93538fc44bf5e1bda76" "checksum integer-encoding 1.0.3 (registry+https://github.com/rust-lang/crates.io-index)" = "a053c9c7dcb7db1f2aa012c37dc176c62e4cdf14898dee0eecc606de835b8acb" +"checksum interleaved-ordered 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "6e385d0f35662722ffac6301494e37d201e884bd27d263cfbcc058febf994d16" "checksum iovec 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "29d062ee61fccdf25be172e70f34c9f6efc597e1fb8f6526e8437b2046ab26be" "checksum ipnetwork 0.12.7 (registry+https://github.com/rust-lang/crates.io-index)" = "2134e210e2a024b5684f90e1556d5f71a1ce7f8b12e9ac9924c67fb36f63b336" "checksum isatty 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)" = "fa500db770a99afe2a0f2229be2a3d09c7ed9d7e4e8440bf71253141994e240f" diff --git a/util/kvdb-rocksdb/Cargo.toml b/util/kvdb-rocksdb/Cargo.toml index e055c853e..f2eb569dc 100644 --- a/util/kvdb-rocksdb/Cargo.toml +++ b/util/kvdb-rocksdb/Cargo.toml @@ -12,6 +12,7 @@ parking_lot = "0.4" regex = "0.2" rlp = { path = "../rlp" } rocksdb = { git = "https://github.com/paritytech/rust-rocksdb" } +interleaved-ordered = "0.1.0" [dev-dependencies] tempdir = "0.3" diff --git a/util/kvdb-rocksdb/src/lib.rs b/util/kvdb-rocksdb/src/lib.rs index d08c8ec85..69b8164d6 100644 --- a/util/kvdb-rocksdb/src/lib.rs +++ b/util/kvdb-rocksdb/src/lib.rs @@ -21,6 +21,7 @@ extern crate elastic_array; extern crate parking_lot; extern crate regex; extern crate rocksdb; +extern crate interleaved_ordered; extern crate ethcore_bigint as bigint; extern crate kvdb; @@ -36,6 +37,7 @@ use rocksdb::{ DB, Writable, WriteBatch, WriteOptions, IteratorMode, DBIterator, Options, DBCompactionStyle, BlockBasedOptions, Direction, Cache, Column, ReadOptions }; +use interleaved_ordered::{interleave_ordered, InterleaveOrdered}; use elastic_array::ElasticArray32; use rlp::{UntrustedRlp, RlpType, Compressible}; @@ -197,14 +199,14 @@ impl Default for DatabaseConfig { // inner DB (to prevent closing via restoration) may be re-evaluated in the future. // pub struct DatabaseIterator<'a> { - iter: DBIterator, + iter: InterleaveOrdered<::std::vec::IntoIter<(Box<[u8]>, Box<[u8]>)>, DBIterator>, _marker: PhantomData<&'a Database>, } impl<'a> Iterator for DatabaseIterator<'a> { type Item = (Box<[u8]>, Box<[u8]>); - fn next(&mut self) -> Option { + fn next(&mut self) -> Option { self.iter.next() } } @@ -510,9 +512,17 @@ impl Database { /// Get database iterator for flushed data. pub fn iter(&self, col: Option) -> Option { - //TODO: iterate over overlay match *self.db.read() { Some(DBAndColumns { ref db, ref cfs }) => { + let overlay = &self.overlay.read()[Self::to_overlay_column(col)]; + let overlay_data = overlay.iter() + .filter_map(|(k, v)| match v { + &KeyState::Insert(ref value) | + &KeyState::InsertCompressed(ref value) => + Some((k.clone().into_vec().into_boxed_slice(), value.clone().into_vec().into_boxed_slice())), + &KeyState::Delete => None, + }).collect::>(); + let iter = col.map_or_else( || db.iterator_opt(IteratorMode::Start, &self.read_opts), |c| db.iterator_cf_opt(cfs[c as usize], IteratorMode::Start, &self.read_opts) @@ -520,7 +530,7 @@ impl Database { ); Some(DatabaseIterator { - iter: iter, + iter: interleave_ordered(overlay_data, iter), _marker: PhantomData, }) }, @@ -536,7 +546,7 @@ impl Database { .expect("iterator params are valid; qed")); Some(DatabaseIterator { - iter: iter, + iter: interleave_ordered(Vec::new(), iter), _marker: PhantomData, }) },