Merge pull request #7048 from paritytech/efyang

reopened 6860 - iterate over both buffered and unbuffered database entries
This commit is contained in:
Marek Kotewicz 2017-11-16 17:49:02 +01:00 committed by GitHub
commit 3c82a0d162
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 24 additions and 5 deletions

7
Cargo.lock generated
View File

@ -1173,6 +1173,11 @@ name = "integer-encoding"
version = "1.0.3" version = "1.0.3"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
[[package]]
name = "interleaved-ordered"
version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
[[package]] [[package]]
name = "iovec" name = "iovec"
version = "0.1.0" version = "0.1.0"
@ -1358,6 +1363,7 @@ version = "0.1.0"
dependencies = [ dependencies = [
"elastic-array 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", "elastic-array 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)",
"ethcore-bigint 0.2.1", "ethcore-bigint 0.2.1",
"interleaved-ordered 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
"kvdb 0.1.0", "kvdb 0.1.0",
"log 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)",
"parking_lot 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", "parking_lot 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)",
@ -3576,6 +3582,7 @@ dependencies = [
"checksum idna 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)" = "014b298351066f1512874135335d62a789ffe78a9974f94b43ed5621951eaf7d" "checksum idna 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)" = "014b298351066f1512874135335d62a789ffe78a9974f94b43ed5621951eaf7d"
"checksum igd 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)" = "356a0dc23a4fa0f8ce4777258085d00a01ea4923b2efd93538fc44bf5e1bda76" "checksum igd 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)" = "356a0dc23a4fa0f8ce4777258085d00a01ea4923b2efd93538fc44bf5e1bda76"
"checksum integer-encoding 1.0.3 (registry+https://github.com/rust-lang/crates.io-index)" = "a053c9c7dcb7db1f2aa012c37dc176c62e4cdf14898dee0eecc606de835b8acb" "checksum integer-encoding 1.0.3 (registry+https://github.com/rust-lang/crates.io-index)" = "a053c9c7dcb7db1f2aa012c37dc176c62e4cdf14898dee0eecc606de835b8acb"
"checksum interleaved-ordered 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "141340095b15ed7491bd3d4ced9d20cebfb826174b6bb03386381f62b01e3d77"
"checksum iovec 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "29d062ee61fccdf25be172e70f34c9f6efc597e1fb8f6526e8437b2046ab26be" "checksum iovec 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "29d062ee61fccdf25be172e70f34c9f6efc597e1fb8f6526e8437b2046ab26be"
"checksum ipnetwork 0.12.7 (registry+https://github.com/rust-lang/crates.io-index)" = "2134e210e2a024b5684f90e1556d5f71a1ce7f8b12e9ac9924c67fb36f63b336" "checksum ipnetwork 0.12.7 (registry+https://github.com/rust-lang/crates.io-index)" = "2134e210e2a024b5684f90e1556d5f71a1ce7f8b12e9ac9924c67fb36f63b336"
"checksum isatty 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)" = "fa500db770a99afe2a0f2229be2a3d09c7ed9d7e4e8440bf71253141994e240f" "checksum isatty 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)" = "fa500db770a99afe2a0f2229be2a3d09c7ed9d7e4e8440bf71253141994e240f"

View File

@ -12,6 +12,7 @@ parking_lot = "0.4"
regex = "0.2" regex = "0.2"
rlp = { path = "../rlp" } rlp = { path = "../rlp" }
rocksdb = { git = "https://github.com/paritytech/rust-rocksdb" } rocksdb = { git = "https://github.com/paritytech/rust-rocksdb" }
interleaved-ordered = "0.1.0"
[dev-dependencies] [dev-dependencies]
tempdir = "0.3" tempdir = "0.3"

View File

@ -21,6 +21,7 @@ extern crate elastic_array;
extern crate parking_lot; extern crate parking_lot;
extern crate regex; extern crate regex;
extern crate rocksdb; extern crate rocksdb;
extern crate interleaved_ordered;
extern crate ethcore_bigint as bigint; extern crate ethcore_bigint as bigint;
extern crate kvdb; extern crate kvdb;
@ -36,6 +37,7 @@ use rocksdb::{
DB, Writable, WriteBatch, WriteOptions, IteratorMode, DBIterator, DB, Writable, WriteBatch, WriteOptions, IteratorMode, DBIterator,
Options, DBCompactionStyle, BlockBasedOptions, Direction, Cache, Column, ReadOptions Options, DBCompactionStyle, BlockBasedOptions, Direction, Cache, Column, ReadOptions
}; };
use interleaved_ordered::{interleave_ordered, InterleaveOrdered};
use elastic_array::ElasticArray32; use elastic_array::ElasticArray32;
use rlp::{UntrustedRlp, RlpType, Compressible}; use rlp::{UntrustedRlp, RlpType, Compressible};
@ -197,14 +199,14 @@ impl Default for DatabaseConfig {
// inner DB (to prevent closing via restoration) may be re-evaluated in the future. // inner DB (to prevent closing via restoration) may be re-evaluated in the future.
// //
pub struct DatabaseIterator<'a> { pub struct DatabaseIterator<'a> {
iter: DBIterator, iter: InterleaveOrdered<::std::vec::IntoIter<(Box<[u8]>, Box<[u8]>)>, DBIterator>,
_marker: PhantomData<&'a Database>, _marker: PhantomData<&'a Database>,
} }
impl<'a> Iterator for DatabaseIterator<'a> { impl<'a> Iterator for DatabaseIterator<'a> {
type Item = (Box<[u8]>, Box<[u8]>); type Item = (Box<[u8]>, Box<[u8]>);
fn next(&mut self) -> Option<Self::Item> { fn next(&mut self) -> Option<Self::Item> {
self.iter.next() self.iter.next()
} }
} }
@ -510,9 +512,18 @@ impl Database {
/// Get database iterator for flushed data. /// Get database iterator for flushed data.
pub fn iter(&self, col: Option<u32>) -> Option<DatabaseIterator> { pub fn iter(&self, col: Option<u32>) -> Option<DatabaseIterator> {
//TODO: iterate over overlay
match *self.db.read() { match *self.db.read() {
Some(DBAndColumns { ref db, ref cfs }) => { Some(DBAndColumns { ref db, ref cfs }) => {
let overlay = &self.overlay.read()[Self::to_overlay_column(col)];
let mut overlay_data = overlay.iter()
.filter_map(|(k, v)| match *v {
KeyState::Insert(ref value) |
KeyState::InsertCompressed(ref value) =>
Some((k.clone().into_vec().into_boxed_slice(), value.clone().into_vec().into_boxed_slice())),
KeyState::Delete => None,
}).collect::<Vec<_>>();
overlay_data.sort();
let iter = col.map_or_else( let iter = col.map_or_else(
|| db.iterator_opt(IteratorMode::Start, &self.read_opts), || db.iterator_opt(IteratorMode::Start, &self.read_opts),
|c| db.iterator_cf_opt(cfs[c as usize], IteratorMode::Start, &self.read_opts) |c| db.iterator_cf_opt(cfs[c as usize], IteratorMode::Start, &self.read_opts)
@ -520,7 +531,7 @@ impl Database {
); );
Some(DatabaseIterator { Some(DatabaseIterator {
iter: iter, iter: interleave_ordered(overlay_data, iter),
_marker: PhantomData, _marker: PhantomData,
}) })
}, },
@ -536,7 +547,7 @@ impl Database {
.expect("iterator params are valid; qed")); .expect("iterator params are valid; qed"));
Some(DatabaseIterator { Some(DatabaseIterator {
iter: iter, iter: interleave_ordered(Vec::new(), iter),
_marker: PhantomData, _marker: PhantomData,
}) })
}, },