Merge pull request #7048 from paritytech/efyang

reopened 6860 - iterate over both buffered and unbuffered database entries

Cargo.lock

@@ -1173,6 +1173,11 @@ name = "integer-encoding"
 version = "1.0.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 
+[[package]]
+name = "interleaved-ordered"
+version = "0.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+
 [[package]]
 name = "iovec"
 version = "0.1.0"
@@ -1358,6 +1363,7 @@ version = "0.1.0"
 dependencies = [
  "elastic-array 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "ethcore-bigint 0.2.1",
+ "interleaved-ordered 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
  "kvdb 0.1.0",
  "log 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)",
  "parking_lot 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -3576,6 +3582,7 @@ dependencies = [
 "checksum idna 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)" = "014b298351066f1512874135335d62a789ffe78a9974f94b43ed5621951eaf7d"
 "checksum igd 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)" = "356a0dc23a4fa0f8ce4777258085d00a01ea4923b2efd93538fc44bf5e1bda76"
 "checksum integer-encoding 1.0.3 (registry+https://github.com/rust-lang/crates.io-index)" = "a053c9c7dcb7db1f2aa012c37dc176c62e4cdf14898dee0eecc606de835b8acb"
+"checksum interleaved-ordered 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "141340095b15ed7491bd3d4ced9d20cebfb826174b6bb03386381f62b01e3d77"
 "checksum iovec 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "29d062ee61fccdf25be172e70f34c9f6efc597e1fb8f6526e8437b2046ab26be"
 "checksum ipnetwork 0.12.7 (registry+https://github.com/rust-lang/crates.io-index)" = "2134e210e2a024b5684f90e1556d5f71a1ce7f8b12e9ac9924c67fb36f63b336"
 "checksum isatty 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)" = "fa500db770a99afe2a0f2229be2a3d09c7ed9d7e4e8440bf71253141994e240f"

util/kvdb-rocksdb/Cargo.toml

@@ -12,6 +12,7 @@ parking_lot = "0.4"
 regex = "0.2"
 rlp = { path = "../rlp" }
 rocksdb = { git = "https://github.com/paritytech/rust-rocksdb" }
+interleaved-ordered = "0.1.0"
 
 [dev-dependencies]
 tempdir = "0.3"

util/kvdb-rocksdb/src/lib.rs

@@ -21,6 +21,7 @@ extern crate elastic_array;
 extern crate parking_lot;
 extern crate regex;
 extern crate rocksdb;
+extern crate interleaved_ordered;
 
 extern crate ethcore_bigint as bigint;
 extern crate kvdb;
@@ -36,6 +37,7 @@ use rocksdb::{
 	DB, Writable, WriteBatch, WriteOptions, IteratorMode, DBIterator,
 	Options, DBCompactionStyle, BlockBasedOptions, Direction, Cache, Column, ReadOptions
 };
+use interleaved_ordered::{interleave_ordered, InterleaveOrdered};
 
 use elastic_array::ElasticArray32;
 use rlp::{UntrustedRlp, RlpType, Compressible};
@@ -197,14 +199,14 @@ impl Default for DatabaseConfig {
 // inner DB (to prevent closing via restoration) may be re-evaluated in the future.
 //
 pub struct DatabaseIterator<'a> {
-	iter: DBIterator,
+	iter: InterleaveOrdered<::std::vec::IntoIter<(Box<[u8]>, Box<[u8]>)>, DBIterator>,
 	_marker: PhantomData<&'a Database>,
 }
 
 impl<'a> Iterator for DatabaseIterator<'a> {
 	type Item = (Box<[u8]>, Box<[u8]>);
 
-    fn next(&mut self) -> Option<Self::Item> {
+	fn next(&mut self) -> Option<Self::Item> {
 		self.iter.next()
 	}
 }
@@ -510,9 +512,18 @@ impl Database {
 
 	/// Get database iterator for flushed data.
 	pub fn iter(&self, col: Option<u32>) -> Option<DatabaseIterator> {
-		//TODO: iterate over overlay
 		match *self.db.read() {
 			Some(DBAndColumns { ref db, ref cfs }) => {
+				let overlay = &self.overlay.read()[Self::to_overlay_column(col)];
+				let mut overlay_data = overlay.iter()
+					.filter_map(|(k, v)| match *v {
+						KeyState::Insert(ref value) |
+						KeyState::InsertCompressed(ref value) =>
+							Some((k.clone().into_vec().into_boxed_slice(), value.clone().into_vec().into_boxed_slice())),
+						KeyState::Delete => None,
+					}).collect::<Vec<_>>();
+				overlay_data.sort();
+
 				let iter = col.map_or_else(
 					|| db.iterator_opt(IteratorMode::Start, &self.read_opts),
 					|c| db.iterator_cf_opt(cfs[c as usize], IteratorMode::Start, &self.read_opts)
@@ -520,7 +531,7 @@ impl Database {
 				);
 
 				Some(DatabaseIterator {
-					iter: iter,
+					iter: interleave_ordered(overlay_data, iter),
 					_marker: PhantomData,
 				})
 			},
@@ -536,7 +547,7 @@ impl Database {
 						.expect("iterator params are valid; qed"));
 
 				Some(DatabaseIterator {
-					iter: iter,
+					iter: interleave_ordered(Vec::new(), iter),
 					_marker: PhantomData,
 				})
 			},