Snapshot optimizations (#1991)

* apply RLP compression to abridged blocks

* add memorydb consolidate

* code hash optimization

* add warning to snapshot restoration CLI

util/src/journaldb/archivedb.rs

@@ -228,6 +228,10 @@ impl JournalDB for ArchiveDB {
 	fn backing(&self) -> &Arc<Database> {
 		&self.backing
 	}
+
+	fn consolidate(&mut self, with: MemoryDB) {
+		self.overlay.consolidate(with);
+	}
 }
 
 #[cfg(test)]

util/src/journaldb/earlymergedb.rs

@@ -539,6 +539,10 @@ impl JournalDB for EarlyMergeDB {
 
 		Ok(ops)
 	}
+
+	fn consolidate(&mut self, with: MemoryDB) {
+		self.overlay.consolidate(with);
+	}
 }
 
 #[cfg(test)]

util/src/journaldb/overlayrecentdb.rs

@@ -339,6 +339,10 @@ impl JournalDB for OverlayRecentDB {
 
 		Ok(ops)
 	}
+
+	fn consolidate(&mut self, with: MemoryDB) {
+		self.transaction_overlay.consolidate(with);
+	}
 }
 
 impl HashDB for OverlayRecentDB {

util/src/journaldb/refcounteddb.rs

@@ -20,6 +20,7 @@ use common::*;
 use rlp::*;
 use hashdb::*;
 use overlaydb::OverlayDB;
+use memorydb::MemoryDB;
 use super::{DB_PREFIX_LEN, LATEST_ERA_KEY};
 use super::traits::JournalDB;
 use kvdb::{Database, DBTransaction};
@@ -192,6 +193,18 @@ impl JournalDB for RefCountedDB {
 		}
 		self.forward.commit_to_batch(batch)
 	}
+
+	fn consolidate(&mut self, mut with: MemoryDB) {
+		for (key, (value, rc)) in with.drain() {
+			for _ in 0..rc {
+				self.emplace(key.clone(), value.clone());
+			}
+
+			for _ in rc..0 {
+				self.remove(&key);
+			}
+		}
+	}
 }
 
 #[cfg(test)]

util/src/journaldb/traits.rs

@@ -61,6 +61,9 @@ pub trait JournalDB: HashDB {
 	/// to the backing strage
 	fn flush(&self) {}
 
+	/// Consolidate all the insertions and deletions in the given memory overlay.
+	fn consolidate(&mut self, overlay: ::memorydb::MemoryDB);
+
 	/// Commit all changes in a single batch
 	#[cfg(test)]
 	fn commit_batch(&mut self, now: u64, id: &H256, end: Option<(u64, H256)>) -> Result<u32, UtilError> {

util/src/memorydb.rs

@@ -174,6 +174,24 @@ impl MemoryDB {
 			}
 		}
 	}
+
+	/// Consolidate all the entries of `other` into `self`.
+	pub fn consolidate(&mut self, mut other: Self) {
+		for (key, (value, rc)) in other.drain() {
+			match self.data.entry(key) {
+				Entry::Occupied(mut entry) => {
+					if entry.get().1 < 0 {
+						entry.get_mut().0 = value;
+					}
+
+					entry.get_mut().1 += rc;
+				}
+				Entry::Vacant(entry) => {
+					entry.insert((value, rc));
+				}
+			}
+		}
+	}
 }
 
 static NULL_RLP_STATIC: [u8; 1] = [0x80; 1];
@@ -310,3 +328,21 @@ fn memorydb_remove_and_purge() {
 	m.remove_and_purge(&hello_key);
 	assert_eq!(m.raw(&hello_key), None);
 }
+
+#[test]
+fn consolidate() {
+	let mut main = MemoryDB::new();
+	let mut other = MemoryDB::new();
+	let remove_key = other.insert(b"doggo");
+	main.remove(&remove_key);
+
+	let insert_key = other.insert(b"arf");
+	main.emplace(insert_key, b"arf".to_vec());
+
+	main.consolidate(other);
+
+	let overlay = main.drain();
+
+	assert_eq!(overlay.get(&remove_key).unwrap(), &(b"doggo".to_vec(), 0));
+	assert_eq!(overlay.get(&insert_key).unwrap(), &(b"arf".to_vec(), 2));
+}