Merge remote-tracking branch 'parity/master'

util/Cargo.toml

@@ -23,7 +23,7 @@ rlp = { path = "rlp" }
 heapsize = { version = "0.3", features = ["unstable"] }
 itertools = "0.4"
 sha3 = { path = "sha3" }
-clippy = { version = "0.0.90", optional = true}
+clippy = { version = "0.0.96", optional = true}
 ethcore-devtools = { path = "../devtools" }
 libc = "0.2.7"
 vergen = "0.1"
@@ -36,6 +36,7 @@ ansi_term = "0.7"
 tiny-keccak= "1.0"
 ethcore-bloom-journal = { path = "bloom" }
 regex = "0.1"
+lru-cache = "0.1.0"
 
 [features]
 default = []

util/network/Cargo.toml

@@ -14,7 +14,7 @@ time = "0.1.34"
 tiny-keccak = "1.0"
 rust-crypto = "0.2.34"
 slab = "0.2"
-clippy = { version = "0.0.90", optional = true}
+clippy = { version = "0.0.96", optional = true}
 igd = "0.5.0"
 libc = "0.2.7"
 parking_lot = "0.3"

util/network/src/session.rs

@@ -395,7 +395,7 @@ impl Session {
 			PACKET_PEERS => Ok(SessionData::None),
 			PACKET_USER ... PACKET_LAST => {
 				let mut i = 0usize;
-				while packet_id > self.info.capabilities[i].id_offset + self.info.capabilities[i].packet_count {
+				while packet_id >= self.info.capabilities[i].id_offset + self.info.capabilities[i].packet_count {
 					i += 1;
 					if i == self.info.capabilities.len() {
 						debug!(target: "network", "Unknown packet: {:?}", packet_id);
@@ -406,6 +406,7 @@ impl Session {
 				// map to protocol
 				let protocol = self.info.capabilities[i].protocol;
 				let pid = packet_id - self.info.capabilities[i].id_offset;
+				trace!(target: "network", "Packet {} mapped to {:?}:{}, i={}, capabilities={:?}", packet_id, protocol, pid, i, self.info.capabilities);
 				Ok(SessionData::Packet { data: packet.data, protocol: protocol, packet_id: pid } )
 			},
 			_ => {

util/src/cache.rs

@@ -0,0 +1,79 @@
+// Copyright 2015, 2016 Ethcore (UK) Ltd.
+// This file is part of Parity.
+
+// Parity is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Parity is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Parity.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Lru-cache related utilities as quick-and-dirty wrappers around the lru-cache
+//! crate.
+// TODO: push changes upstream in a clean way.
+
+use heapsize::HeapSizeOf;
+use lru_cache::LruCache;
+
+use std::hash::Hash;
+
+const INITIAL_CAPACITY: usize = 4;
+
+/// An LRU-cache which operates on memory used.
+pub struct MemoryLruCache<K: Eq + Hash, V: HeapSizeOf> {
+	inner: LruCache<K, V>,
+	cur_size: usize,
+	max_size: usize,
+}
+
+impl<K: Eq + Hash, V: HeapSizeOf> MemoryLruCache<K, V> {
+	/// Create a new cache with a maximum size in bytes.
+	pub fn new(max_size: usize) -> Self {
+		MemoryLruCache {
+			inner: LruCache::new(INITIAL_CAPACITY),
+			max_size: max_size,
+			cur_size: 0,
+		}
+	}
+
+	/// Insert an item.
+	pub fn insert(&mut self, key: K, val: V) {
+		let cap = self.inner.capacity();
+
+		// grow the cache as necessary; it operates on amount of items
+		// but we're working based on memory usage.
+		if self.inner.len() == cap && self.cur_size < self.max_size {
+			self.inner.set_capacity(cap * 2);
+		}
+
+		// account for any element displaced from the cache.
+		if let Some(lru) = self.inner.insert(key, val) {
+			self.cur_size -= lru.heap_size_of_children();
+		}
+
+		// remove elements until we are below the memory target.
+		while self.cur_size > self.max_size {
+			match self.inner.remove_lru() {
+				Some((_, v)) => self.cur_size -= v.heap_size_of_children(),
+				_ => break,
+			}
+		}
+	}
+
+	/// Get a reference to an item in the cache. It is a logic error for its
+	/// heap size to be altered while borrowed.
+	pub fn get_mut(&mut self, key: &K) -> Option<&mut V> {
+		self.inner.get_mut(key)
+	}
+
+	/// Currently-used size of values in bytes.
+	pub fn current_size(&self) -> usize {
+		self.cur_size
+	}
+}

util/src/journaldb/archivedb.rs

@@ -78,7 +78,7 @@ impl HashDB for ArchiveDB {
 			ret.insert(h, 1);
 		}
 
-		for (key, refs) in self.overlay.keys().into_iter() {
+		for (key, refs) in self.overlay.keys() {
 			let refs = *ret.get(&key).unwrap_or(&0) + refs;
 			ret.insert(key, refs);
 		}
@@ -152,7 +152,7 @@ impl JournalDB for ArchiveDB {
 		let mut inserts = 0usize;
 		let mut deletes = 0usize;
 
-		for i in self.overlay.drain().into_iter() {
+		for i in self.overlay.drain() {
 			let (key, (value, rc)) = i;
 			if rc > 0 {
 				batch.put(self.column, &key, &value);
@@ -164,7 +164,7 @@ impl JournalDB for ArchiveDB {
 			}
 		}
 
-		for (mut key, value) in self.overlay.drain_aux().into_iter() {
+		for (mut key, value) in self.overlay.drain_aux() {
 			key.push(AUX_FLAG);
 			batch.put(self.column, &key, &value);
 		}
@@ -185,7 +185,7 @@ impl JournalDB for ArchiveDB {
 		let mut inserts = 0usize;
 		let mut deletes = 0usize;
 
-		for i in self.overlay.drain().into_iter() {
+		for i in self.overlay.drain() {
 			let (key, (value, rc)) = i;
 			if rc > 0 {
 				if try!(self.backing.get(self.column, &key)).is_some() {
@@ -204,7 +204,7 @@ impl JournalDB for ArchiveDB {
 			}
 		}
 
-		for (mut key, value) in self.overlay.drain_aux().into_iter() {
+		for (mut key, value) in self.overlay.drain_aux() {
 			key.push(AUX_FLAG);
 			batch.put(self.column, &key, &value);
 		}

util/src/journaldb/earlymergedb.rs

@@ -63,9 +63,11 @@ enum RemoveFrom {
 /// the removals actually take effect.
 ///
 /// journal format:
+/// ```
 /// [era, 0] => [ id, [insert_0, ...], [remove_0, ...] ]
 /// [era, 1] => [ id, [insert_0, ...], [remove_0, ...] ]
 /// [era, n] => [ ... ]
+/// ```
 ///
 /// When we make a new commit, we make a journal of all blocks in the recent history and record
 /// all keys that were inserted and deleted. The journal is ordered by era; multiple commits can
@@ -80,6 +82,7 @@ enum RemoveFrom {
 /// which includes an original key, if any.
 ///
 /// The semantics of the `counter` are:
+/// ```
 /// insert key k:
 ///   counter already contains k: count += 1
 ///   counter doesn't contain k:
@@ -91,9 +94,11 @@ enum RemoveFrom {
 ///     count == 1: remove counter
 ///     count == 0: remove key from backing db
 ///   counter doesn't contain k: remove key from backing db
+/// ```
 ///
 /// Practically, this means that for each commit block turning from recent to ancient we do the
 /// following:
+/// ```
 /// is_canonical:
 ///   inserts: Ignored (left alone in the backing database).
 ///   deletes: Enacted; however, recent history queue is checked for ongoing references. This is
@@ -102,8 +107,9 @@ enum RemoveFrom {
 ///   inserts: Reverted; however, recent history queue is checked for ongoing references. This is
 ///            reduced as a preference to deletion from the backing database.
 ///   deletes: Ignored (they were never inserted).
+/// ```
 ///
-/// TODO: store_reclaim_period
+/// TODO: `store_reclaim_period`
 pub struct EarlyMergeDB {
 	overlay: MemoryDB,
 	backing: Arc<Database>,
@@ -310,7 +316,7 @@ impl HashDB for EarlyMergeDB {
 			ret.insert(h, 1);
 		}
 
-		for (key, refs) in self.overlay.keys().into_iter() {
+		for (key, refs) in self.overlay.keys() {
 			let refs = *ret.get(&key).unwrap_or(&0) + refs;
 			ret.insert(key, refs);
 		}

util/src/journaldb/overlayrecentdb.rs

@@ -379,7 +379,7 @@ impl HashDB for OverlayRecentDB {
 			ret.insert(h, 1);
 		}
 
-		for (key, refs) in self.transaction_overlay.keys().into_iter() {
+		for (key, refs) in self.transaction_overlay.keys() {
 			let refs = *ret.get(&key).unwrap_or(&0) + refs;
 			ret.insert(key, refs);
 		}

util/src/journaldb/refcounteddb.rs

@@ -36,12 +36,14 @@ use std::env;
 /// the removals actually take effect.
 ///
 /// journal format:
+/// ```
 /// [era, 0] => [ id, [insert_0, ...], [remove_0, ...] ]
 /// [era, 1] => [ id, [insert_0, ...], [remove_0, ...] ]
 /// [era, n] => [ ... ]
+/// ```
 ///
 /// when we make a new commit, we journal the inserts and removes.
-/// for each end_era that we journaled that we are no passing by,
+/// for each `end_era` that we journaled that we are no passing by,
 /// we remove all of its removes assuming it is canonical and all
 /// of its inserts otherwise.
 // TODO: store last_era, reclaim_period.

util/src/lib.rs

@@ -105,6 +105,7 @@ extern crate ansi_term;
 extern crate tiny_keccak;
 extern crate rlp;
 extern crate regex;
+extern crate lru_cache;
 
 #[macro_use]
 extern crate heapsize;
@@ -143,6 +144,7 @@ pub mod semantic_version;
 pub mod log;
 pub mod path;
 pub mod snappy;
+pub mod cache;
 mod timer;
 
 pub use common::*;

util/src/migration/mod.rs

@@ -231,7 +231,7 @@ impl Manager {
 		trace!(target: "migration", "Total migrations to execute for version {}: {}", version, migrations.len());
 		if migrations.is_empty() { return Err(Error::MigrationImpossible) };
 
-		let columns = migrations.iter().nth(0).and_then(|m| m.pre_columns());
+		let columns = migrations.get(0).and_then(|m| m.pre_columns());
 
 		trace!(target: "migration", "Expecting database to contain {:?} columns", columns);
 		let mut db_config = DatabaseConfig {

util/src/overlaydb.rs

@@ -66,7 +66,7 @@ impl OverlayDB {
 	pub fn commit_to_batch(&mut self, batch: &mut DBTransaction) -> Result<u32, UtilError> {
 		let mut ret = 0u32;
 		let mut deletes = 0usize;
-		for i in self.overlay.drain().into_iter() {
+		for i in self.overlay.drain() {
 			let (key, (value, rc)) = i;
 			if rc != 0 {
 				match self.payload(&key) {
@@ -133,7 +133,7 @@ impl HashDB for OverlayDB {
 			ret.insert(h, r as i32);
 		}
 
-		for (key, refs) in self.overlay.keys().into_iter() {
+		for (key, refs) in self.overlay.keys() {
 			let refs = *ret.get(&key).unwrap_or(&0) + refs;
 			ret.insert(key, refs);
 		}

util/src/trie/journal.rs

@@ -84,7 +84,7 @@ impl Journal {
 	pub fn apply(self, db: &mut HashDB) -> Score {
 		trace!("applying {:?} changes", self.0.len());
 		let mut ret = Score{inserts: 0, removes: 0};
-		for d in self.0.into_iter() {
+		for d in self.0 {
 			match d {
 				Operation::Delete(h) => {
 					trace!("TrieDBMut::apply --- {:?}", &h);

util/src/trie/triedb.rs

@@ -87,7 +87,7 @@ impl<'db> TrieDB<'db> {
 	/// Convert a vector of hashes to a hashmap of hash to occurrences.
 	pub fn to_map(hashes: Vec<H256>) -> HashMap<H256, u32> {
 		let mut r: HashMap<H256, u32> = HashMap::new();
-		for h in hashes.into_iter() {
+		for h in hashes {
 			*r.entry(h).or_insert(0) += 1;
 		}
 		r
@@ -97,7 +97,7 @@ impl<'db> TrieDB<'db> {
 	/// trie.
 	pub fn db_items_remaining(&self) -> super::Result<HashMap<H256, i32>> {
 		let mut ret = self.db.keys();
-		for (k, v) in Self::to_map(try!(self.keys())).into_iter() {
+		for (k, v) in Self::to_map(try!(self.keys())) {
 			let keycount = *ret.get(&k).unwrap_or(&0);
 			match keycount <= v as i32 {
 				true => ret.remove(&k),