Merge branch 'master' into clippy

Conflicts:
	util/src/lib.rs

util/src/error.rs

@@ -22,6 +22,7 @@ pub enum UtilError {
 	BaseData(BaseDataError),
 	Network(NetworkError),
 	Decoder(DecoderError),
+	SimpleString(String),
 	BadSize,
 }
 
@@ -73,6 +74,12 @@ impl From<::rlp::DecoderError> for UtilError {
 	}
 }
 
+impl From<String> for UtilError {
+	fn from(err: String) -> UtilError {
+		UtilError::SimpleString(err)
+	}
+}
+
 // TODO: uncomment below once https://github.com/rust-lang/rust/issues/27336 sorted.
 /*#![feature(concat_idents)]
 macro_rules! assimilate {

util/src/journaldb.rs

@@ -0,0 +1,214 @@
+//! Disk-backed HashDB implementation.
+
+use std::env;
+use common::*;
+use rlp::*;
+use hashdb::*;
+use overlaydb::*;
+use rocksdb::{DB, Writable};
+
+#[derive(Clone)]
+/// Implementation of the HashDB trait for a disk-backed database with a memory overlay
+/// and latent-removal semantics.
+///
+/// Like OverlayDB, there is a memory overlay; `commit()` must be called in order to 
+/// write operations out to disk. Unlike OverlayDB, `remove()` operations do not take effect
+/// immediately. Rather some age (based on a linear but arbitrary metric) must pass before
+/// the removals actually take effect.
+pub struct JournalDB {
+	forward: OverlayDB,
+	backing: Arc<DB>,
+	inserts: Vec<H256>,
+	removes: Vec<H256>,
+}
+
+impl JournalDB {
+	/// Create a new instance given a `backing` database.
+	pub fn new(backing: DB) -> JournalDB {
+		let db = Arc::new(backing);
+		JournalDB {
+			forward: OverlayDB::new_with_arc(db.clone()),
+			backing: db,
+			inserts: vec![],
+			removes: vec![],
+		}
+	}
+
+	/// Create a new instance with an anonymous temporary database.
+	pub fn new_temp() -> JournalDB {
+		let mut dir = env::temp_dir();
+		dir.push(H32::random().hex());
+		Self::new(DB::open_default(dir.to_str().unwrap()).unwrap())
+	}
+
+	/// Get a clone of the overlay db portion of this.
+	pub fn to_overlaydb(&self) -> OverlayDB { self.forward.clone() }
+
+	/// Commit all recent insert operations and historical removals from the old era
+	/// to the backing database.
+	pub fn commit(&mut self, now: u64, id: &H256, end: Option<(u64, H256)>) -> Result<u32, UtilError> {
+		// journal format: 
+		// [era, 0] => [ id, [insert_0, ...], [remove_0, ...] ]
+		// [era, 1] => [ id, [insert_0, ...], [remove_0, ...] ]
+		// [era, n] => [ ... ]
+
+		// TODO: store last_era, reclaim_period.
+
+		// when we make a new commit, we journal the inserts and removes.
+		// for each end_era that we journaled that we are no passing by, 
+		// we remove all of its removes assuming it is canonical and all
+		// of its inserts otherwise.
+
+		// record new commit's details.
+		{
+			let mut index = 0usize;
+			let mut last;
+
+			while try!(self.backing.get({
+				let mut r = RlpStream::new_list(2);
+				r.append(&now);
+				r.append(&index);
+				last = r.drain();
+				&last
+			})).is_some() {
+				index += 1;
+			}
+
+			let mut r = RlpStream::new_list(3);
+			r.append(id);
+			r.append(&self.inserts);
+			r.append(&self.removes);
+			try!(self.backing.put(&last, r.as_raw()));
+			self.inserts.clear();
+			self.removes.clear();
+		}
+
+		// apply old commits' details
+		if let Some((end_era, canon_id)) = end {
+			let mut index = 0usize;
+			let mut last;
+			while let Some(rlp_data) = try!(self.backing.get({
+				let mut r = RlpStream::new_list(2);
+				r.append(&end_era);
+				r.append(&index);
+				last = r.drain();
+				&last
+			})) {
+				let rlp = Rlp::new(&rlp_data);
+				let to_remove: Vec<H256> = rlp.val_at(if canon_id == rlp.val_at(0) {2} else {1});
+				for i in to_remove.iter() {
+					self.forward.remove(i);
+				}
+				try!(self.backing.delete(&last));
+				trace!("JournalDB: delete journal for time #{}.{}, (canon was {}): {} entries", end_era, index, canon_id, to_remove.len());
+				index += 1;
+			}
+		}
+
+		self.forward.commit()
+	}
+
+	/// Revert all operations on this object (i.e. `insert()`s and `removes()`s) since the
+	/// last `commit()`.
+	pub fn revert(&mut self) { self.forward.revert(); self.removes.clear(); }
+}
+
+impl HashDB for JournalDB {
+	fn keys(&self) -> HashMap<H256, i32> { self.forward.keys() }
+	fn lookup(&self, key: &H256) -> Option<&[u8]> { self.forward.lookup(key) }
+	fn exists(&self, key: &H256) -> bool { self.forward.exists(key) }
+	fn insert(&mut self, value: &[u8]) -> H256 { let r = self.forward.insert(value); self.inserts.push(r.clone()); r }
+	fn emplace(&mut self, key: H256, value: Bytes) { self.inserts.push(key.clone()); self.forward.emplace(key, value); }
+	fn kill(&mut self, key: &H256) { self.removes.push(key.clone()); }
+}
+
+#[cfg(test)]
+mod tests {
+	use common::*;
+	use super::*;
+	use hashdb::*;
+
+	#[test]
+	fn long_history() {
+		// history is 3
+		let mut jdb = JournalDB::new_temp();
+		let h = jdb.insert(b"foo");
+		jdb.commit(0, &b"0".sha3(), None).unwrap();
+		assert!(jdb.exists(&h));
+		jdb.remove(&h);
+		jdb.commit(1, &b"1".sha3(), None).unwrap();
+		assert!(jdb.exists(&h));
+		jdb.commit(2, &b"2".sha3(), None).unwrap();
+		assert!(jdb.exists(&h));
+		jdb.commit(3, &b"3".sha3(), Some((0, b"0".sha3()))).unwrap();
+		assert!(jdb.exists(&h));
+		jdb.commit(4, &b"4".sha3(), Some((1, b"1".sha3()))).unwrap();
+		assert!(!jdb.exists(&h));
+	}
+
+	#[test]
+	fn complex() {
+		// history is 1
+		let mut jdb = JournalDB::new_temp();
+
+		let foo = jdb.insert(b"foo");
+		let bar = jdb.insert(b"bar");
+		jdb.commit(0, &b"0".sha3(), None).unwrap();
+		assert!(jdb.exists(&foo));
+		assert!(jdb.exists(&bar));
+
+		jdb.remove(&foo);
+		jdb.remove(&bar);
+		let baz = jdb.insert(b"baz");
+		jdb.commit(1, &b"1".sha3(), Some((0, b"0".sha3()))).unwrap();
+		assert!(jdb.exists(&foo));
+		assert!(jdb.exists(&bar));
+		assert!(jdb.exists(&baz));
+
+		let foo = jdb.insert(b"foo");
+		jdb.remove(&baz);
+		jdb.commit(2, &b"2".sha3(), Some((1, b"1".sha3()))).unwrap();
+		assert!(jdb.exists(&foo));
+		assert!(!jdb.exists(&bar));
+		assert!(jdb.exists(&baz));
+
+		jdb.remove(&foo);
+		jdb.commit(3, &b"3".sha3(), Some((2, b"2".sha3()))).unwrap();
+		assert!(jdb.exists(&foo));
+		assert!(!jdb.exists(&bar));
+		assert!(!jdb.exists(&baz));
+
+		jdb.commit(4, &b"4".sha3(), Some((3, b"3".sha3()))).unwrap();
+		assert!(!jdb.exists(&foo));
+		assert!(!jdb.exists(&bar));
+		assert!(!jdb.exists(&baz));
+	}
+
+	#[test]
+	fn fork() {
+		// history is 1
+		let mut jdb = JournalDB::new_temp();
+
+		let foo = jdb.insert(b"foo");
+		let bar = jdb.insert(b"bar");
+		jdb.commit(0, &b"0".sha3(), None).unwrap();
+		assert!(jdb.exists(&foo));
+		assert!(jdb.exists(&bar));
+
+		jdb.remove(&foo);
+		let baz = jdb.insert(b"baz");
+		jdb.commit(1, &b"1a".sha3(), Some((0, b"0".sha3()))).unwrap();
+
+		jdb.remove(&bar);
+		jdb.commit(1, &b"1b".sha3(), Some((0, b"0".sha3()))).unwrap();
+
+		assert!(jdb.exists(&foo));
+		assert!(jdb.exists(&bar));
+		assert!(jdb.exists(&baz));
+
+		jdb.commit(2, &b"2b".sha3(), Some((1, b"1b".sha3()))).unwrap();
+		assert!(jdb.exists(&foo));
+		assert!(!jdb.exists(&baz));
+		assert!(!jdb.exists(&bar));
+	}
+}

util/src/lib.rs

@@ -1,7 +1,6 @@
 #![feature(op_assign_traits)]
 #![feature(augmented_assignments)]
 #![feature(associated_consts)]
-#![feature(wrapping)]
 #![allow(needless_range_loop, match_bool)]
 //! Ethcore-util library
 //!
@@ -70,6 +69,7 @@ pub mod sha3;
 pub mod hashdb;
 pub mod memorydb;
 pub mod overlaydb;
+pub mod journaldb;
 pub mod math;
 pub mod chainfilter;
 pub mod crypto;
@@ -89,6 +89,7 @@ pub use rlp::*;
 pub use hashdb::*;
 pub use memorydb::*;
 pub use overlaydb::*;
+pub use journaldb::*;
 pub use math::*;
 pub use chainfilter::*;
 pub use crypto::*;

util/src/overlaydb.rs

@@ -15,11 +15,11 @@ use rocksdb::{DB, Writable, IteratorMode};
 #[derive(Clone)]
 /// Implementation of the HashDB trait for a disk-backed database with a memory overlay.
 ///
-/// The operations `insert()` and `kill()` take place on the memory overlay; batches of
+/// The operations `insert()` and `remove()` take place on the memory overlay; batches of
 /// such operations may be flushed to the disk-backed DB with `commit()` or discarded with
 /// `revert()`.
 ///
-/// `lookup()` and `exists()` maintain normal behaviour - all `insert()` and `kill()` 
+/// `lookup()` and `contains()` maintain normal behaviour - all `insert()` and `remove()` 
 /// queries have an immediate effect in terms of these functions.
 pub struct OverlayDB {
 	overlay: MemoryDB,
@@ -28,8 +28,11 @@ pub struct OverlayDB {
 
 impl OverlayDB {
 	/// Create a new instance of OverlayDB given a `backing` database.
-	pub fn new(backing: DB) -> OverlayDB {
-		OverlayDB{ overlay: MemoryDB::new(), backing: Arc::new(backing) }
+	pub fn new(backing: DB) -> OverlayDB { Self::new_with_arc(Arc::new(backing)) }
+
+	/// Create a new instance of OverlayDB given a `backing` database.
+	pub fn new_with_arc(backing: Arc<DB>) -> OverlayDB {
+		OverlayDB{ overlay: MemoryDB::new(), backing: backing }
 	}
 
 	/// Create a new instance of OverlayDB with an anonymous temporary database.
@@ -68,11 +71,10 @@ impl OverlayDB {
 	/// ```
 	pub fn commit(&mut self) -> Result<u32, UtilError> {
 		let mut ret = 0u32;
+		let mut deletes = 0usize;
 		for i in self.overlay.drain().into_iter() {
 			let (key, (value, rc)) = i;
-			// until we figure out state trie pruning, only commit stuff when it has a strictly positive delkta of RCs - 
-			// this prevents RCs being reduced to 0 where the DB would pretent that the node had been removed.
-			if rc > 0 {
+			if rc != 0 {
 				match self.payload(&key) {
 					Some(x) => {
 						let (back_value, back_rc) = x;
@@ -80,7 +82,7 @@ impl OverlayDB {
 						if total_rc < 0 {
 							return Err(From::from(BaseDataError::NegativelyReferencedHash));
 						}
-						self.put_payload(&key, (back_value, total_rc as u32));
+						deletes += if self.put_payload(&key, (back_value, total_rc as u32)) {1} else {0};
 					}
 					None => {
 						if rc < 0 {
@@ -92,6 +94,7 @@ impl OverlayDB {
 				ret += 1;
 			}
 		}
+		trace!("OverlayDB::commit() deleted {} nodes", deletes);
 		Ok(ret)
 	}
 
@@ -129,11 +132,18 @@ impl OverlayDB {
 	}
 
 	/// Get the refs and value of the given key.
-	fn put_payload(&self, key: &H256, payload: (Bytes, u32)) {
-		let mut s = RlpStream::new_list(2);
-		s.append(&payload.1);
-		s.append(&payload.0);
-		self.backing.put(&key.bytes(), &s.out()).expect("Low-level database error. Some issue with your hard disk?");
+	fn put_payload(&self, key: &H256, payload: (Bytes, u32)) -> bool {
+		if payload.1 > 0 {
+			let mut s = RlpStream::new_list(2);
+			s.append(&payload.1);
+			s.append(&payload.0);
+			self.backing.put(&key.bytes(), s.as_raw()).expect("Low-level database error. Some issue with your hard disk?");
+			false
+		} else {
+			self.backing.delete(&key.bytes()).expect("Low-level database error. Some issue with your hard disk?");
+			true
+		}
+
 	}
 }
 

util/src/rlp/rlpstream.rs

@@ -142,6 +142,14 @@ impl RlpStream {
 			self.note_appended(1);
 		}
 	}
+
+	/// Drain the object and return the underlying ElasticArray.
+	pub fn drain(self) -> ElasticArray1024<u8> {
+		match self.is_finished() {
+			true => self.encoder.bytes,
+			false => panic!()
+		}
+	}
 }
 
 struct BasicEncoder {

util/src/uint.rs

@@ -23,7 +23,6 @@
 
 use standard::*;
 use from_json::*;
-use std::num::wrapping::OverflowingOps;
 
 macro_rules! impl_map_from {
 	($thing:ident, $from:ty, $to:ty) => {
@@ -97,6 +96,21 @@ pub trait Uint: Sized + Default + FromStr + From<u64> + FromJson + fmt::Debug +
 	fn pow(self, other: Self) -> Self;
 	/// Return wrapped eponentation `self**other` and flag if there was an overflow
 	fn overflowing_pow(self, other: Self) -> (Self, bool);
+
+
+	fn overflowing_add(self, other: Self) -> (Self, bool);
+
+	fn overflowing_sub(self, other: Self) -> (Self, bool);
+
+	fn overflowing_mul(self, other: Self) -> (Self, bool);
+	
+	fn overflowing_div(self, other: Self) -> (Self, bool);
+
+	fn overflowing_rem(self, other: Self) -> (Self, bool);
+
+	fn overflowing_neg(self) -> (Self, bool);
+	
+	fn overflowing_shl(self, shift: u32) -> (Self, bool);
 }
 
 macro_rules! construct_uint {
@@ -259,6 +273,98 @@ macro_rules! construct_uint {
 				let res = overflowing!(x.overflowing_mul(y), overflow);
 				(res, overflow)
 			}
+
+			fn overflowing_add(self, other: $name) -> ($name, bool) {
+				let $name(ref me) = self;
+				let $name(ref you) = other;
+				let mut ret = [0u64; $n_words];
+				let mut carry = [0u64; $n_words];
+				let mut b_carry = false;
+				let mut overflow = false;
+
+				for i in 0..$n_words {
+					ret[i] = me[i].wrapping_add(you[i]);
+
+					if ret[i] < me[i] {
+						if i < $n_words - 1 {
+							carry[i + 1] = 1;
+							b_carry = true;
+						} else {
+							overflow = true;
+						}
+					}
+				}
+				if b_carry {
+					let ret = overflowing!($name(ret).overflowing_add($name(carry)), overflow);
+					(ret, overflow)
+				} else { 
+					($name(ret), overflow)
+				}
+			}
+
+			fn overflowing_sub(self, other: $name) -> ($name, bool) {
+				let res = overflowing!((!other).overflowing_add(From::from(1u64)));
+				let res = overflowing!(self.overflowing_add(res));
+				(res, self < other)
+			}
+
+			fn overflowing_mul(self, other: $name) -> ($name, bool) {
+				let mut res = $name::from(0u64);
+				let mut overflow = false;
+				// TODO: be more efficient about this
+				for i in 0..(2 * $n_words) {
+					let v = overflowing!(self.overflowing_mul_u32((other >> (32 * i)).low_u32()), overflow);
+					let res2 = overflowing!(v.overflowing_shl(32 * i as u32), overflow);
+					res = overflowing!(res.overflowing_add(res2), overflow);
+				}
+				(res, overflow)
+			}
+
+			fn overflowing_div(self, other: $name) -> ($name, bool) {
+				(self / other, false)
+			}
+
+			fn overflowing_rem(self, other: $name) -> ($name, bool) {
+				(self % other, false)
+			}
+
+			fn overflowing_neg(self) -> ($name, bool) {
+				(!self, true)
+			}
+
+			fn overflowing_shl(self, shift32: u32) -> ($name, bool) {
+				let $name(ref original) = self;
+				let mut ret = [0u64; $n_words];
+				let shift = shift32 as usize;
+				let word_shift = shift / 64;
+				let bit_shift = shift % 64;
+				for i in 0..$n_words {
+					// Shift
+					if i + word_shift < $n_words {
+						ret[i + word_shift] += original[i] << bit_shift;
+					}
+					// Carry
+					if bit_shift > 0 && i + word_shift + 1 < $n_words {
+						ret[i + word_shift + 1] += original[i] >> (64 - bit_shift);
+					}
+				}
+				// Detecting overflow
+				let last = $n_words - word_shift - if bit_shift > 0 { 1 } else { 0 };
+				let overflow = if bit_shift > 0 {
+					(original[last] >> (64 - bit_shift)) > 0
+				} else if word_shift > 0 {
+					original[last] > 0
+				} else {
+					false
+				};
+
+				for i in last+1..$n_words-1 {
+					if original[i] > 0 {
+						return ($name(ret), true);
+					}
+				}
+				($name(ret), overflow)
+			}
 		}
 
 		impl $name {
@@ -390,105 +496,6 @@ macro_rules! construct_uint {
 			}
 		}
 
-		impl OverflowingOps for $name {
-			fn overflowing_add(self, other: $name) -> ($name, bool) {
-				let $name(ref me) = self;
-				let $name(ref you) = other;
-				let mut ret = [0u64; $n_words];
-				let mut carry = [0u64; $n_words];
-				let mut b_carry = false;
-				let mut overflow = false;
-
-				for i in 0..$n_words {
-					ret[i] = me[i].wrapping_add(you[i]);
-
-					if ret[i] < me[i] {
-						if i < $n_words - 1 {
-							carry[i + 1] = 1;
-							b_carry = true;
-						} else {
-							overflow = true;
-						}
-					}
-				}
-				if b_carry {
-					let ret = overflowing!($name(ret).overflowing_add($name(carry)), overflow);
-					(ret, overflow)
-				} else { 
-					($name(ret), overflow)
-				}
-			}
-
-			fn overflowing_sub(self, other: $name) -> ($name, bool) {
-				let res = overflowing!((!other).overflowing_add(From::from(1u64)));
-				let res = overflowing!(self.overflowing_add(res));
-				(res, self < other)
-			}
-
-			fn overflowing_mul(self, other: $name) -> ($name, bool) {
-				let mut res = $name::from(0u64);
-				let mut overflow = false;
-				// TODO: be more efficient about this
-				for i in 0..(2 * $n_words) {
-					let v = overflowing!(self.overflowing_mul_u32((other >> (32 * i)).low_u32()), overflow);
-					let res2 = overflowing!(v.overflowing_shl(32 * i as u32), overflow);
-					res = overflowing!(res.overflowing_add(res2), overflow);
-				}
-				(res, overflow)
-			}
-
-			fn overflowing_div(self, other: $name) -> ($name, bool) {
-				(self / other, false)
-			}
-
-			fn overflowing_rem(self, other: $name) -> ($name, bool) {
-				(self % other, false)
-			}
-
-			fn overflowing_neg(self) -> ($name, bool) {
-				(!self, true)
-			}
-
-			fn overflowing_shl(self, shift32: u32) -> ($name, bool) {
-				let $name(ref original) = self;
-				let mut ret = [0u64; $n_words];
-				let shift = shift32 as usize;
-				let word_shift = shift / 64;
-				let bit_shift = shift % 64;
-				for i in 0..$n_words {
-					// Shift
-					if i + word_shift < $n_words {
-						ret[i + word_shift] += original[i] << bit_shift;
-					}
-					// Carry
-					if bit_shift > 0 && i + word_shift + 1 < $n_words {
-						ret[i + word_shift + 1] += original[i] >> (64 - bit_shift);
-					}
-				}
-				// Detecting overflow
-				let last = $n_words - word_shift - if bit_shift > 0 { 1 } else { 0 };
-				let overflow = if bit_shift > 0 {
-					(original[last] >> (64 - bit_shift)) > 0
-				} else if word_shift > 0 {
-					original[last] > 0
-				} else {
-					false
-				};
-
-				for i in last+1..$n_words-1 {
-					if original[i] > 0 {
-						return ($name(ret), true);
-					}
-				}
-				($name(ret), overflow)
-			}
-
-			fn overflowing_shr(self, _shift32: u32) -> ($name, bool) {
-				// TODO [todr] not used for now
-				unimplemented!();
-			}
-		}
-
 		impl Add<$name> for $name {
 			type Output = $name;
 
@@ -915,7 +922,6 @@ pub const BAD_U256: U256 = U256([0xffffffffffffffffu64; 4]);
 mod tests {
 	use uint::{Uint, U128, U256, U512};
 	use std::str::FromStr;
-	use std::num::wrapping::OverflowingOps;
 
 	#[test]
 	pub fn assign_ops() {
@@ -1297,28 +1303,6 @@ mod tests {
 		);
 	}
 
-	#[ignore]
-	#[test]
-	pub fn uint256_shr_overflow() {
-		assert_eq!(
-			U256::from_str("ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff").unwrap()
-			.overflowing_shr(4),
-			(U256::from_str("0fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff").unwrap(), true)
-		);
-	}
-
-	#[ignore]
-	#[test]
-	pub fn uint256_shr_overflow2() {
-		assert_eq!(
-			U256::from_str("fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff0").unwrap()
-			.overflowing_shr(4),
-			(U256::from_str("0fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff").unwrap(), false)
-		);
-	}
-
-
-
 	#[test]
 	pub fn uint256_mul() {
 		assert_eq!(