Merge remote-tracking branch 'parity/master'

util/Cargo.toml

@@ -18,7 +18,7 @@ rocksdb = { git = "https://github.com/ethcore/rust-rocksdb" }
 lazy_static = "0.2"
 eth-secp256k1 = { git = "https://github.com/ethcore/rust-secp256k1" }
 rust-crypto = "0.2.34"
-elastic-array = "0.5"
+elastic-array = { git = "https://github.com/ethcore/elastic-array" }
 rlp = { path = "rlp" }
 heapsize = { version = "0.3", features = ["unstable"] }
 itertools = "0.4"
@@ -29,11 +29,13 @@ libc = "0.2.7"
 vergen = "0.1"
 target_info = "0.1"
 ethcore-bigint = { path = "bigint" }
-parking_lot = "0.2.6"
+parking_lot = "0.3"
 using_queue = { path = "using_queue" }
 table = { path = "table" }
 ansi_term = "0.7"
 tiny-keccak= "1.0"
+ethcore-bloom-journal = { path = "bloom" }
+regex = "0.1"
 
 [features]
 default = []

util/benches/trie.rs

@@ -21,7 +21,7 @@ extern crate ethcore_util;
 #[macro_use]
 extern crate log;
 
-use test::Bencher;
+use test::{Bencher, black_box};
 use ethcore_util::hash::*;
 use ethcore_util::bytes::*;
 use ethcore_util::trie::*;
@@ -78,6 +78,32 @@ fn trie_insertions_32_mir_1k(b: &mut Bencher) {
 	});
 //	println!("hash_count: {}", hash_count);
 }
+#[bench]
+fn trie_iter(b: &mut Bencher) {
+	let st = StandardMap {
+		alphabet: Alphabet::All,
+		min_key: 32,
+		journal_key: 0,
+		value_mode: ValueMode::Mirror,
+		count: 1000,
+	};
+	let d = st.make();
+	let mut memdb = MemoryDB::new();
+	let mut root = H256::new();
+	{
+		let mut t = TrieDBMut::new(&mut memdb, &mut root);
+		for i in d.iter() {
+			t.insert(&i.0, &i.1).unwrap();
+		}
+	}
+
+	b.iter(&mut ||{
+		let t = TrieDB::new(&memdb, &root).unwrap();
+		for n in t.iter().unwrap() {
+			black_box(n).unwrap();
+		}
+	});
+}
 
 #[bench]
 fn triehash_insertions_32_mir_1k(b: &mut Bencher) {

util/bigint/Cargo.toml

@@ -4,7 +4,7 @@ homepage = "http://ethcore.io"
 repository = "https://github.com/ethcore/parity"
 license = "GPL-3.0"
 name = "ethcore-bigint"
-version = "0.1.0"
+version = "0.1.1"
 authors = ["Ethcore <admin@ethcore.io>"]
 build = "build.rs"
 

util/bigint/src/hash.rs

@@ -64,11 +64,11 @@ pub fn clean_0x(s: &str) -> &str {
 
 macro_rules! impl_hash {
 	($from: ident, $size: expr) => {
-		#[derive(Eq)]
 		#[repr(C)]
 		/// Unformatted binary data of fixed length.
 		pub struct $from (pub [u8; $size]);
 
+
 		impl From<[u8; $size]> for $from {
 			fn from(bytes: [u8; $size]) -> Self {
 				$from(bytes)
@@ -210,6 +210,8 @@ macro_rules! impl_hash {
 			}
 		}
 
+		impl Eq for $from {}
+
 		impl PartialEq for $from {
 			fn eq(&self, other: &Self) -> bool {
 				for i in 0..$size {

util/bigint/src/lib.rs

@@ -24,3 +24,16 @@ extern crate rustc_serialize;
 
 pub mod uint;
 pub mod hash;
+
+/// A prelude module for re-exporting all the types defined in this crate.
+///
+/// ```rust
+/// use ethcore_bigint::prelude::*;
+///
+/// let x: U256 = U256::zero();
+/// let y = x + 1.into();
+/// ```
+pub mod prelude {
+	pub use ::uint::*;
+	pub use ::hash::*;
+}

util/bloom/Cargo.toml

@@ -0,0 +1,12 @@
+[project]
+name = "ethcore-bloom-journal"
+version = "0.1.0"
+authors = ["Ethcore<admin@ethcore.io>"]
+description = "Journaling bloom filter"
+license = "GPL3"
+
+[lib]
+path = "src/lib.rs"
+
+[dependencies]
+siphasher = "0.1.1"

util/bloom/src/lib.rs

@@ -0,0 +1,251 @@
+// Copyright 2015, 2016 Ethcore (UK) Ltd.
+// This file is part of Parity.
+
+// Parity is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Parity is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Parity.  If not, see <http://www.gnu.org/licenses/>.
+
+
+extern crate siphasher;
+
+use std::cmp;
+use std::mem;
+use std::f64;
+use std::hash::{Hash, Hasher};
+use std::collections::HashSet;
+use siphasher::sip::SipHasher;
+
+// TODO [ToDr] Both hashers are exactly the same - no point to keep two.
+const NUMBER_OF_HASHERS: usize = 2;
+
+/// BitVec structure with journalling
+/// Every time any of the blocks is getting set it's index is tracked
+/// and can be then drained by `drain` method
+struct BitVecJournal {
+    elems: Vec<u64>,
+    journal: HashSet<usize>,
+}
+
+impl BitVecJournal {
+	pub fn new(size: usize) -> BitVecJournal {
+		let extra = if size % 8 > 0  { 1 } else { 0 };
+		BitVecJournal {
+			elems: vec![0u64; size / 8 + extra],
+			journal: HashSet::new(),
+		}
+	}
+
+	pub fn from_parts(parts: &[u64]) -> BitVecJournal {
+		BitVecJournal {
+			elems: parts.to_vec(),
+			journal: HashSet::new(),
+		}
+	}
+
+	pub fn set(&mut self, index: usize) {
+		let e_index = index / 64;
+		let bit_index = index % 64;
+		let val = self.elems.get_mut(e_index).unwrap();
+		*val |= 1u64 << bit_index;
+		self.journal.insert(e_index);
+	}
+
+	pub fn get(&self, index: usize) -> bool {
+		let e_index = index / 64;
+		let bit_index = index % 64;
+		self.elems[e_index] & (1 << bit_index) != 0
+	}
+
+	pub fn drain(&mut self) -> Vec<(usize, u64)> {
+		let journal = mem::replace(&mut self.journal, HashSet::new()).into_iter();
+		journal.map(|idx| (idx, self.elems[idx])).collect::<Vec<(usize, u64)>>()
+	}
+
+	pub fn saturation(&self) -> f64 {
+		self.elems.iter().fold(0u64, |acc, e| acc + e.count_ones() as u64) as f64 / (self.elems.len() * 64) as f64
+	}
+}
+
+/// Bloom filter structure
+pub struct Bloom {
+	bitmap: BitVecJournal,
+	bitmap_bits: u64,
+	k_num: u32,
+	// TODO [ToDr] Both hashers are exactly the same - no point to keep two.
+	sips: [SipHasher; NUMBER_OF_HASHERS],
+}
+
+impl Bloom {
+	/// Create a new bloom filter structure.
+	/// bitmap_size is the size in bytes (not bits) that will be allocated in memory
+	/// items_count is an estimation of the maximum number of items to store.
+	pub fn new(bitmap_size: usize, items_count: usize) -> Bloom {
+		assert!(bitmap_size > 0 && items_count > 0);
+		let bitmap_bits = (bitmap_size as u64) * 8u64;
+		let k_num = Bloom::optimal_k_num(bitmap_bits, items_count);
+		let bitmap = BitVecJournal::new(bitmap_bits as usize);
+		let sips = [SipHasher::new(), SipHasher::new()];
+		Bloom {
+			bitmap: bitmap,
+			bitmap_bits: bitmap_bits,
+			k_num: k_num,
+			sips: sips,
+		}
+	}
+
+	/// Initializes bloom filter from saved state
+	pub fn from_parts(parts: &[u64], k_num: u32) -> Bloom {
+		let bitmap_size = parts.len() * 8;
+		let bitmap_bits = (bitmap_size as u64) * 8u64;
+		let bitmap = BitVecJournal::from_parts(parts);
+		let sips = [SipHasher::new(), SipHasher::new()];
+		Bloom {
+			bitmap: bitmap,
+			bitmap_bits: bitmap_bits,
+			k_num: k_num,
+			sips: sips,
+		}
+	}
+
+	/// Create a new bloom filter structure.
+	/// items_count is an estimation of the maximum number of items to store.
+	/// fp_p is the wanted rate of false positives, in ]0.0, 1.0[
+	pub fn new_for_fp_rate(items_count: usize, fp_p: f64) -> Bloom {
+		let bitmap_size = Bloom::compute_bitmap_size(items_count, fp_p);
+		Bloom::new(bitmap_size, items_count)
+	}
+
+	/// Compute a recommended bitmap size for items_count items
+	/// and a fp_p rate of false positives.
+	/// fp_p obviously has to be within the ]0.0, 1.0[ range.
+	pub fn compute_bitmap_size(items_count: usize, fp_p: f64) -> usize {
+		assert!(items_count > 0);
+		assert!(fp_p > 0.0 && fp_p < 1.0);
+		let log2 = f64::consts::LN_2;
+		let log2_2 = log2 * log2;
+		((items_count as f64) * f64::ln(fp_p) / (-8.0 * log2_2)).ceil() as usize
+	}
+
+	/// Records the presence of an item.
+	pub fn set<T>(&mut self, item: T)
+		where T: Hash
+	{
+		let mut hashes = [0u64, 0u64];
+		for k_i in 0..self.k_num {
+			let bit_offset = (self.bloom_hash(&mut hashes, &item, k_i) % self.bitmap_bits) as usize;
+			self.bitmap.set(bit_offset);
+		}
+	}
+
+	/// Check if an item is present in the set.
+	/// There can be false positives, but no false negatives.
+	pub fn check<T>(&self, item: T) -> bool
+		where T: Hash
+	{
+		let mut hashes = [0u64, 0u64];
+		for k_i in 0..self.k_num {
+			let bit_offset = (self.bloom_hash(&mut hashes, &item, k_i) % self.bitmap_bits) as usize;
+			if !self.bitmap.get(bit_offset) {
+				return false;
+			}
+		}
+		true
+	}
+
+	/// Return the number of bits in the filter
+	pub fn number_of_bits(&self) -> u64 {
+		self.bitmap_bits
+	}
+
+	/// Return the number of hash functions used for `check` and `set`
+	pub fn number_of_hash_functions(&self) -> u32 {
+		self.k_num
+	}
+
+	fn optimal_k_num(bitmap_bits: u64, items_count: usize) -> u32 {
+		let m = bitmap_bits as f64;
+		let n = items_count as f64;
+		let k_num = (m / n * f64::ln(2.0f64)).ceil() as u32;
+		cmp::max(k_num, 1)
+	}
+
+	fn bloom_hash<T>(&self, hashes: &mut [u64; NUMBER_OF_HASHERS], item: &T, k_i: u32) -> u64
+		where T: Hash
+	{
+		if k_i < NUMBER_OF_HASHERS as u32 {
+			let mut sip = self.sips[k_i as usize].clone();
+			item.hash(&mut sip);
+			let hash = sip.finish();
+			hashes[k_i as usize] = hash;
+			hash
+		} else {
+			hashes[0].wrapping_add((k_i as u64).wrapping_mul(hashes[1]) % 0xffffffffffffffc5)
+		}
+	}
+
+	/// Drains the bloom journal returning the updated bloom part
+	pub fn drain_journal(&mut self) -> BloomJournal {
+		BloomJournal {
+			entries: self.bitmap.drain(),
+			hash_functions: self.k_num,
+		}
+	}
+
+	/// Returns the ratio of set bits in the bloom filter to the total bits
+	pub fn saturation(&self) -> f64 {
+		self.bitmap.saturation()
+	}
+}
+
+/// Bloom journal
+/// Returns the tuple of (bloom part index, bloom part value) where each one is representing
+/// an index of bloom parts that was updated since the last drain
+pub struct BloomJournal {
+    pub hash_functions: u32,
+    pub entries: Vec<(usize, u64)>,
+}
+
+
+#[cfg(test)]
+mod tests {
+	use super::Bloom;
+
+	#[test]
+	fn get_set() {
+		let mut bloom = Bloom::new(10, 80);
+		let key = vec![115u8, 99];
+		assert!(!bloom.check(&key));
+		bloom.set(&key);
+		assert!(bloom.check(&key));
+	}
+
+	#[test]
+	fn journalling() {
+		let initial = vec![0u64; 8];
+		let mut bloom = Bloom::from_parts(&initial, 3);
+		bloom.set(&vec![5u8, 4]);
+		let drain = bloom.drain_journal();
+
+		assert_eq!(2, drain.entries.len())
+	}
+
+	#[test]
+	fn saturation() {
+		let initial = vec![0u64; 8];
+		let mut bloom = Bloom::from_parts(&initial, 3);
+		bloom.set(&vec![5u8, 4]);
+
+		let full = bloom.saturation();
+		// 2/8/64 = 0.00390625
+		assert!(full >= 0.0039f64 && full <= 0.004f64);
+	}
+}

util/fetch/Cargo.toml

@@ -0,0 +1,18 @@
+[package]
+description = "HTTP/HTTPS fetching library"
+homepage = "http://ethcore.io"
+license = "GPL-3.0"
+name = "fetch"
+version = "0.1.0"
+authors = ["Ethcore <admin@ethcore.io>"]
+
+[dependencies]
+log = "0.3"
+rand = "0.3"
+hyper = { default-features = false, git = "https://github.com/ethcore/hyper" }
+https-fetch = { path = "../https-fetch" }
+clippy = { version = "0.0.90", optional = true}
+
+[features]
+default = []
+dev = ["clippy"]

util/fetch/src/client.rs

@@ -0,0 +1,146 @@
+// Copyright 2015, 2016 Ethcore (UK) Ltd.
+// This file is part of Parity.
+
+// Parity is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Parity is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Parity.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Fetching
+
+use std::{env, io};
+use std::sync::{mpsc, Arc};
+use std::sync::atomic::AtomicBool;
+use std::path::PathBuf;
+
+use hyper;
+use https_fetch as https;
+
+use fetch_file::{FetchHandler, Error as HttpFetchError};
+
+pub type FetchResult = Result<PathBuf, FetchError>;
+
+#[derive(Debug)]
+pub enum FetchError {
+	InvalidUrl,
+	Http(HttpFetchError),
+	Https(https::FetchError),
+	Io(io::Error),
+	Other(String),
+}
+
+impl From<HttpFetchError> for FetchError {
+	fn from(e: HttpFetchError) -> Self {
+		FetchError::Http(e)
+	}
+}
+
+impl From<io::Error> for FetchError {
+	fn from(e: io::Error) -> Self {
+		FetchError::Io(e)
+	}
+}
+
+pub trait Fetch: Default + Send {
+	/// Fetch URL and get the result in callback.
+	fn request_async(&mut self, url: &str, abort: Arc<AtomicBool>, on_done: Box<Fn(FetchResult) + Send>) -> Result<(), FetchError>;
+
+	/// Fetch URL and get a result Receiver. You will be notified when receiver is ready by `on_done` callback.
+	fn request(&mut self, url: &str, abort: Arc<AtomicBool>, on_done: Box<Fn() + Send>) -> Result<mpsc::Receiver<FetchResult>, FetchError> {
+		let (tx, rx) = mpsc::channel();
+		try!(self.request_async(url, abort, Box::new(move |result| {
+			let res = tx.send(result);
+			if let Err(_) = res {
+				warn!("Fetch finished, but no one was listening");
+			}
+			on_done();
+		})));
+		Ok(rx)
+	}
+
+	/// Closes this client
+	fn close(self) {}
+
+	/// Returns a random filename
+	fn random_filename() -> String {
+		use ::rand::Rng;
+		let mut rng = ::rand::OsRng::new().unwrap();
+		rng.gen_ascii_chars().take(12).collect()
+	}
+}
+
+pub struct Client {
+	http_client: hyper::Client<FetchHandler>,
+	https_client: https::Client,
+	limit: Option<usize>,
+}
+
+impl Default for Client {
+	fn default() -> Self {
+		// Max 15MB will be downloaded.
+		Client::with_limit(Some(15*1024*1024))
+	}
+}
+
+impl Client {
+	fn with_limit(limit: Option<usize>) -> Self {
+		Client {
+			http_client: hyper::Client::new().expect("Unable to initialize http client."),
+			https_client: https::Client::with_limit(limit).expect("Unable to initialize https client."),
+			limit: limit,
+		}
+	}
+
+	fn convert_url(url: hyper::Url) -> Result<https::Url, FetchError> {
+		let host = format!("{}", try!(url.host().ok_or(FetchError::InvalidUrl)));
+		let port = try!(url.port_or_known_default().ok_or(FetchError::InvalidUrl));
+		https::Url::new(&host, port, url.path()).map_err(|_| FetchError::InvalidUrl)
+	}
+
+	fn temp_path() -> PathBuf {
+		let mut dir = env::temp_dir();
+		dir.push(Self::random_filename());
+		dir
+	}
+}
+
+impl Fetch for Client {
+	fn close(self) {
+		self.http_client.close();
+		self.https_client.close();
+	}
+
+	fn request_async(&mut self, url: &str, abort: Arc<AtomicBool>, on_done: Box<Fn(FetchResult) + Send>) -> Result<(), FetchError> {
+		let is_https = url.starts_with("https://");
+		let url = try!(url.parse().map_err(|_| FetchError::InvalidUrl));
+		let temp_path = Self::temp_path();
+
+		trace!(target: "fetch", "Fetching from: {:?}", url);
+
+		if is_https {
+			let url = try!(Self::convert_url(url));
+			try!(self.https_client.fetch_to_file(
+				url,
+				temp_path.clone(),
+				abort,
+				move |result| on_done(result.map(|_| temp_path).map_err(FetchError::Https)),
+			).map_err(|e| FetchError::Other(format!("{:?}", e))));
+		} else {
+			try!(self.http_client.request(
+				url,
+				FetchHandler::new(temp_path, abort, Box::new(move |result| on_done(result)), self.limit.map(|v| v as u64).clone()),
+			).map_err(|e| FetchError::Other(format!("{:?}", e))));
+		}
+
+		Ok(())
+	}
+}
+

util/fetch/src/fetch_file.rs

@@ -0,0 +1,176 @@
+// Copyright 2015, 2016 Ethcore (UK) Ltd.
+// This file is part of Parity.
+
+// Parity is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Parity is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Parity.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Hyper Client Handler to Fetch File
+
+use std::{io, fs, fmt};
+use std::path::PathBuf;
+use std::sync::Arc;
+use std::sync::atomic::{AtomicBool, Ordering};
+use std::time::Duration;
+
+use hyper::status::StatusCode;
+use hyper::client::{Request, Response, DefaultTransport as HttpStream};
+use hyper::header::Connection;
+use hyper::{self, Decoder, Encoder, Next};
+
+use super::FetchError;
+
+#[derive(Debug)]
+pub enum Error {
+	Aborted,
+	NotStarted,
+	SizeLimit,
+	UnexpectedStatus(StatusCode),
+	IoError(io::Error),
+	HyperError(hyper::Error),
+}
+
+pub type FetchResult = Result<PathBuf, FetchError>;
+pub type OnDone = Box<Fn(FetchResult) + Send>;
+
+pub struct FetchHandler {
+	path: PathBuf,
+	abort: Arc<AtomicBool>,
+	file: Option<fs::File>,
+	result: Option<FetchResult>,
+	on_done: Option<OnDone>,
+	size_limit: Option<u64>,
+}
+
+impl fmt::Debug for FetchHandler {
+	fn fmt(&self, f: &mut fmt::Formatter) -> Result<(), fmt::Error> {
+		write!(f, "Fetch {{ path: {:?}, file: {:?}, result: {:?} }}", self.path, self.file, self.result)
+	}
+}
+
+impl Drop for FetchHandler {
+    fn drop(&mut self) {
+		let res = self.result.take().unwrap_or(Err(Error::NotStarted.into()));
+		// Remove file if there was an error
+		if res.is_err() || self.is_aborted() {
+			if let Some(file) = self.file.take() {
+				drop(file);
+				// Remove file
+				let _ = fs::remove_file(&self.path);
+			}
+		}
+		// send result
+		if let Some(f) = self.on_done.take() {
+			f(res);
+		}
+    }
+}
+
+impl FetchHandler {
+	pub fn new(path: PathBuf, abort: Arc<AtomicBool>, on_done: OnDone, size_limit: Option<u64>) -> Self {
+		FetchHandler {
+			path: path,
+			abort: abort,
+			file: None,
+			result: None,
+			on_done: Some(on_done),
+			size_limit: size_limit,
+		}
+	}
+
+	fn is_aborted(&self) -> bool {
+		self.abort.load(Ordering::SeqCst)
+	}
+
+	fn mark_aborted(&mut self) -> Next {
+		self.result = Some(Err(Error::Aborted.into()));
+		Next::end()
+	}
+}
+
+impl hyper::client::Handler<HttpStream> for FetchHandler {
+    fn on_request(&mut self, req: &mut Request) -> Next {
+		if self.is_aborted() {
+			return self.mark_aborted();
+		}
+        req.headers_mut().set(Connection::close());
+        read()
+    }
+
+    fn on_request_writable(&mut self, _encoder: &mut Encoder<HttpStream>) -> Next {
+		if self.is_aborted() {
+			return self.mark_aborted();
+		}
+        read()
+    }
+
+    fn on_response(&mut self, res: Response) -> Next {
+		if self.is_aborted() {
+			return self.mark_aborted();
+		}
+		if *res.status() != StatusCode::Ok {
+			self.result = Some(Err(Error::UnexpectedStatus(*res.status()).into()));
+			return Next::end();
+		}
+
+		// Open file to write
+		match fs::File::create(&self.path) {
+			Ok(file) => {
+				self.file = Some(file);
+				self.result = Some(Ok(self.path.clone()));
+				read()
+			},
+			Err(err) => {
+				self.result = Some(Err(Error::IoError(err).into()));
+				Next::end()
+			},
+		}
+    }
+
+    fn on_response_readable(&mut self, decoder: &mut Decoder<HttpStream>) -> Next {
+		if self.is_aborted() {
+			return self.mark_aborted();
+		}
+        match io::copy(decoder, self.file.as_mut().expect("File is there because on_response has created it.")) {
+            Ok(0) => Next::end(),
+            Ok(bytes_read) => match self.size_limit {
+				None => read(),
+				// Check limit
+				Some(limit) if limit > bytes_read => {
+					self.size_limit = Some(limit - bytes_read);
+					read()
+				},
+				// Size limit reached
+				_ => {
+					self.result = Some(Err(Error::SizeLimit.into()));
+					Next::end()
+				},
+			},
+            Err(e) => match e.kind() {
+                io::ErrorKind::WouldBlock => Next::read(),
+                _ => {
+					self.result = Some(Err(Error::IoError(e).into()));
+                    Next::end()
+                }
+            }
+        }
+    }
+
+    fn on_error(&mut self, err: hyper::Error) -> Next {
+		self.result = Some(Err(Error::HyperError(err).into()));
+        Next::remove()
+    }
+}
+
+fn read() -> Next {
+    Next::read().timeout(Duration::from_secs(15))
+}

util/fetch/src/lib.rs

@@ -0,0 +1,29 @@
+// Copyright 2015, 2016 Ethcore (UK) Ltd.
+// This file is part of Parity.
+
+// Parity is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Parity is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Parity.  If not, see <http://www.gnu.org/licenses/>.
+
+//! A service to fetch any HTTP / HTTPS content.
+
+#[macro_use]
+extern crate log;
+extern crate hyper;
+extern crate https_fetch;
+extern crate rand;
+
+
+pub mod client;
+pub mod fetch_file;
+
+pub use self::client::{Client, Fetch, FetchError, FetchResult};

util/https-fetch/src/client.rs

@@ -31,6 +31,7 @@ use url::Url;
 pub enum FetchError {
 	InvalidAddress,
 	ReadingCaCertificates,
+	UnexpectedStatus(String),
 	CaCertificates(io::Error),
 	Io(io::Error),
 	Notify(mio::NotifyError<ClientMessage>),
@@ -78,6 +79,10 @@ impl Drop for Client {
 
 impl Client {
 	pub fn new() -> Result<Self, FetchError> {
+		Self::with_limit(None)
+	}
+
+	pub fn with_limit(size_limit: Option<usize>) -> Result<Self, FetchError> {
 		let mut event_loop = try!(mio::EventLoop::new());
 		let channel = event_loop.channel();
 
@@ -85,6 +90,7 @@ impl Client {
 			let mut client = ClientLoop {
 				next_token: 0,
 				sessions: HashMap::new(),
+				size_limit: size_limit,
 			};
 			event_loop.run(&mut client).unwrap();
 		});
@@ -128,6 +134,7 @@ impl Client {
 pub struct ClientLoop {
 	next_token: usize,
 	sessions: HashMap<usize, TlsClient>,
+	size_limit: Option<usize>,
 }
 
 impl mio::Handler for ClientLoop {
@@ -154,12 +161,15 @@ impl mio::Handler for ClientLoop {
 				let token = self.next_token;
 				self.next_token += 1;
 
-				if let Ok(mut tlsclient) = TlsClient::new(mio::Token(token), &url, writer, abort, callback) {
+				if let Ok(mut tlsclient) = TlsClient::new(mio::Token(token), &url, writer, abort, callback, self.size_limit.clone()) {
 					let httpreq = format!(
-						"GET {} HTTP/1.1\r\nHost: {}\r\nConnection: close\r\nAccept-Encoding: identity\r\n\r\n",
+						"GET {} HTTP/1.1\r\nHost: {}\r\nConnection: close\r\nUser-Agent: {}/{}\r\nAccept-Encoding: identity\r\n\r\n",
 						url.path(),
-						url.hostname()
+						url.hostname(),
+						env!("CARGO_PKG_NAME"),
+						env!("CARGO_PKG_VERSION")
 					);
+					debug!("Requesting content: {}", httpreq);
 					let _ = tlsclient.write(httpreq.as_bytes());
 					tlsclient.register(event_loop);
 

util/https-fetch/src/http.rs

@@ -35,26 +35,31 @@ pub struct HttpProcessor {
 	status: Option<String>,
 	headers: Vec<String>,
 	body_writer: io::BufWriter<Box<io::Write>>,
+	size_limit: Option<usize>,
 }
 
 const BREAK_LEN: usize = 2;
 
 impl HttpProcessor {
-	pub fn new(body_writer: Box<io::Write>) -> Self {
+	pub fn new(body_writer: Box<io::Write>, size_limit: Option<usize>) -> Self {
 		HttpProcessor {
 			state: State::WaitingForStatus,
 			buffer: Cursor::new(Vec::new()),
 			status: None,
 			headers: Vec::new(),
-			body_writer: io::BufWriter::new(body_writer)
+			body_writer: io::BufWriter::new(body_writer),
+			size_limit: size_limit,
 		}
 	}
 
-	#[cfg(test)]
 	pub fn status(&self) -> Option<&String> {
 		self.status.as_ref()
 	}
 
+	pub fn status_is_ok(&self) -> bool {
+		self.status == Some("HTTP/1.1 200 OK".into())
+	}
+
 	#[cfg(test)]
 	pub fn headers(&self) -> &[String] {
 		&self.headers
@@ -140,9 +145,18 @@ impl HttpProcessor {
 				},
 				State::WritingBody => {
 					let len = self.buffer.get_ref().len();
+					match self.size_limit {
+						None => {},
+						Some(limit) if limit > len => {},
+						_ => {
+							warn!("Finishing file fetching because limit was reached.");
+							self.set_state(State::Finished);
+							continue;
+						}
+					}
 					try!(self.body_writer.write_all(self.buffer.get_ref()));
 					self.buffer_consume(len);
-					return Ok(());
+					return self.body_writer.flush();
 				},
 				State::WaitingForChunk => {
 					match self.find_break_index() {
@@ -167,7 +181,22 @@ impl HttpProcessor {
 				},
 				// Buffers the data until we have a full chunk
 				State::WritingChunk(left) if self.buffer.get_ref().len() >= left => {
-					try!(self.body_writer.write_all(&self.buffer.get_ref()[0..left]));
+					match self.size_limit {
+						None => {},
+						Some(limit) if limit > left => {
+							self.size_limit = Some(limit - left);
+						},
+						_ => {
+							warn!("Finishing file fetching because limit was reached.");
+							self.set_state(State::Finished);
+							continue;
+						}
+					}
+					{
+						let chunk = &self.buffer.get_ref()[0..left];
+						trace!("Writing chunk: {:?}", String::from_utf8_lossy(chunk));
+						try!(self.body_writer.write_all(chunk));
+					}
 					self.buffer_consume(left + BREAK_LEN);
 
 					self.set_state(State::WaitingForChunk);
@@ -178,7 +207,7 @@ impl HttpProcessor {
 				State::Finished => {
 					let len = self.buffer.get_ref().len();
 					self.buffer_consume(len);
-					return Ok(());
+					return self.body_writer.flush();
 				},
 			}
 		}
@@ -230,7 +259,7 @@ mod tests {
 	#[test]
 	fn should_be_able_to_process_status_line() {
 		// given
-		let mut http = HttpProcessor::new(Box::new(Cursor::new(Vec::new())));
+		let mut http = HttpProcessor::new(Box::new(Cursor::new(Vec::new())), None);
 
 		// when
 		let out =
@@ -249,7 +278,7 @@ mod tests {
 	#[test]
 	fn should_be_able_to_process_headers() {
 		// given
-		let mut http = HttpProcessor::new(Box::new(Cursor::new(Vec::new())));
+		let mut http = HttpProcessor::new(Box::new(Cursor::new(Vec::new())), None);
 
 		// when
 		let out =
@@ -274,7 +303,7 @@ mod tests {
 	fn should_be_able_to_consume_body() {
 		// given
 		let (writer, data) = Writer::new();
-		let mut http = HttpProcessor::new(Box::new(writer));
+		let mut http = HttpProcessor::new(Box::new(writer), None);
 
 		// when
 		let out =
@@ -301,7 +330,7 @@ mod tests {
 	fn should_correctly_handle_chunked_content() {
 		// given
 		let (writer, data) = Writer::new();
-		let mut http = HttpProcessor::new(Box::new(writer));
+		let mut http = HttpProcessor::new(Box::new(writer), None);
 
 		// when
 		let out =
@@ -331,4 +360,40 @@ mod tests {
 		assert_eq!(data.borrow().get_ref()[..], b"Parity in\r\n\r\nchunks."[..]);
 		assert_eq!(http.state(), State::Finished);
 	}
+
+	#[test]
+	fn should_stop_fetching_when_limit_is_reached() {
+		// given
+		let (writer, data) = Writer::new();
+		let mut http = HttpProcessor::new(Box::new(writer), Some(5));
+
+		// when
+		let out =
+			"\
+				HTTP/1.1 200 OK\r\n\
+				Host: 127.0.0.1:8080\r\n\
+				Transfer-Encoding: chunked\r\n\
+				Connection: close\r\n\
+				\r\n\
+				4\r\n\
+				Pari\r\n\
+				3\r\n\
+				ty \r\n\
+				D\r\n\
+				in\r\n\
+				\r\n\
+				chunks.\r\n\
+				0\r\n\
+				\r\n\
+			";
+		http.write_all(out.as_bytes()).unwrap();
+		http.flush().unwrap();
+
+		// then
+		assert_eq!(http.status().unwrap(), "HTTP/1.1 200 OK");
+		assert_eq!(http.headers().len(), 3);
+		assert_eq!(data.borrow().get_ref()[..], b"Pari"[..]);
+		assert_eq!(http.state(), State::Finished);
+	}
+
 }

util/https-fetch/src/tlsclient.rs

@@ -87,6 +87,7 @@ impl TlsClient {
 		writer: Box<io::Write + Send>,
 		abort: Arc<AtomicBool>,
 		mut callback: Box<FnMut(FetchResult) + Send>,
+		size_limit: Option<usize>,
 		) -> Result<Self, FetchError> {
 			let res = TlsClient::make_config().and_then(|cfg| {
 				TcpStream::connect(url.address()).map(|sock| {
@@ -98,7 +99,7 @@ impl TlsClient {
 				Ok((cfg, sock)) => Ok(TlsClient {
 					abort: abort,
 					token: token,
-					writer: HttpProcessor::new(writer),
+					writer: HttpProcessor::new(writer, size_limit),
 					socket: sock,
 					closing: false,
 					error: None,
@@ -136,8 +137,11 @@ impl TlsClient {
 			trace!("Connection closed");
 			let callback = &mut self.callback;
 			callback(match self.error.take() {
+				None if self.writer.status_is_ok() => Ok(()),
 				Some(err) => Err(err.into()),
-				None => Ok(()),
+				None => {
+					Err(FetchError::UnexpectedStatus(format!("{:?}", self.writer.status())))
+				},
 			});
 
 			return true;

util/io/Cargo.toml

@@ -9,7 +9,7 @@ authors = ["Ethcore <admin@ethcore.io>"]
 [dependencies]
 mio = { git = "https://github.com/ethcore/mio", branch = "v0.5.x" }
 crossbeam = "0.2"
-parking_lot = "0.2.6"
+parking_lot = "0.3"
 log = "0.3"
 slab = "0.2"
 

util/io/src/lib.rs

@@ -68,6 +68,8 @@ mod panics;
 use mio::{EventLoop, Token};
 use std::fmt;
 
+pub use worker::LOCAL_STACK_SIZE;
+
 #[derive(Debug)]
 /// IO Error
 pub enum IoError {

util/io/src/service.rs

@@ -14,7 +14,7 @@
 // You should have received a copy of the GNU General Public License
 // along with Parity.  If not, see <http://www.gnu.org/licenses/>.
 
-use std::sync::Arc;
+use std::sync::{Arc, Weak};
 use std::thread::{self, JoinHandle};
 use std::collections::HashMap;
 use mio::*;
@@ -76,12 +76,12 @@ pub enum IoMessage<Message> where Message: Send + Clone + Sized {
 }
 
 /// IO access point. This is passed to all IO handlers and provides an interface to the IO subsystem.
-pub struct IoContext<Message> where Message: Send + Clone + 'static {
+pub struct IoContext<Message> where Message: Send + Clone + Sync + 'static {
 	channel: IoChannel<Message>,
 	handler: HandlerId,
 }
 
-impl<Message> IoContext<Message> where Message: Send + Clone + 'static {
+impl<Message> IoContext<Message> where Message: Send + Clone + Sync + 'static {
 	/// Create a new IO access point. Takes references to all the data that can be updated within the IO handler.
 	pub fn new(channel: IoChannel<Message>, handler: HandlerId) -> IoContext<Message> {
 		IoContext {
@@ -179,7 +179,7 @@ struct UserTimer {
 /// Root IO handler. Manages user handlers, messages and IO timers.
 pub struct IoManager<Message> where Message: Send + Sync {
 	timers: Arc<RwLock<HashMap<HandlerId, UserTimer>>>,
-	handlers: Slab<Arc<IoHandler<Message>>, HandlerId>,
+	handlers: Arc<RwLock<Slab<Arc<IoHandler<Message>>, HandlerId>>>,
 	workers: Vec<Worker>,
 	worker_channel: chase_lev::Worker<Work<Message>>,
 	work_ready: Arc<SCondvar>,
@@ -187,7 +187,11 @@ pub struct IoManager<Message> where Message: Send + Sync {
 
 impl<Message> IoManager<Message> where Message: Send + Sync + Clone + 'static {
 	/// Creates a new instance and registers it with the event loop.
-	pub fn start(panic_handler: Arc<PanicHandler>, event_loop: &mut EventLoop<IoManager<Message>>) -> Result<(), IoError> {
+	pub fn start(
+		panic_handler: Arc<PanicHandler>, 
+		event_loop: &mut EventLoop<IoManager<Message>>,
+		handlers: Arc<RwLock<Slab<Arc<IoHandler<Message>>, HandlerId>>>
+	) -> Result<(), IoError> {
 		let (worker, stealer) = chase_lev::deque();
 		let num_workers = 4;
 		let work_ready_mutex =  Arc::new(SMutex::new(()));
@@ -196,7 +200,7 @@ impl<Message> IoManager<Message> where Message: Send + Sync + Clone + 'static {
 			Worker::new(
 				i,
 				stealer.clone(),
-				IoChannel::new(event_loop.channel()),
+				IoChannel::new(event_loop.channel(), Arc::downgrade(&handlers)),
 				work_ready.clone(),
 				work_ready_mutex.clone(),
 				panic_handler.clone(),
@@ -205,7 +209,7 @@ impl<Message> IoManager<Message> where Message: Send + Sync + Clone + 'static {
 
 		let mut io = IoManager {
 			timers: Arc::new(RwLock::new(HashMap::new())),
-			handlers: Slab::new(MAX_HANDLERS),
+			handlers: handlers,
 			worker_channel: worker,
 			workers: workers,
 			work_ready: work_ready,
@@ -222,7 +226,7 @@ impl<Message> Handler for IoManager<Message> where Message: Send + Clone + Sync
 	fn ready(&mut self, _event_loop: &mut EventLoop<Self>, token: Token, events: EventSet) {
 		let handler_index  = token.as_usize() / TOKENS_PER_HANDLER;
 		let token_id  = token.as_usize() % TOKENS_PER_HANDLER;
-		if let Some(handler) = self.handlers.get(handler_index) {
+		if let Some(handler) = self.handlers.read().get(handler_index) {
 			if events.is_hup() {
 				self.worker_channel.push(Work { work_type: WorkType::Hup, token: token_id, handler: handler.clone(), handler_id: handler_index });
 			}
@@ -241,9 +245,8 @@ impl<Message> Handler for IoManager<Message> where Message: Send + Clone + Sync
 	fn timeout(&mut self, event_loop: &mut EventLoop<Self>, token: Token) {
 		let handler_index  = token.as_usize()  / TOKENS_PER_HANDLER;
 		let token_id  = token.as_usize()  % TOKENS_PER_HANDLER;
-		if let Some(handler) = self.handlers.get(handler_index) {
-			let option = self.timers.read().get(&token.as_usize()).cloned(); 
-			if let Some(timer) = option {
+		if let Some(handler) = self.handlers.read().get(handler_index) {
+			if let Some(timer) = self.timers.read().get(&token.as_usize()) {
 				if timer.once {
 					self.timers.write().remove(&token_id);
 					event_loop.clear_timeout(timer.timeout);
@@ -263,12 +266,12 @@ impl<Message> Handler for IoManager<Message> where Message: Send + Clone + Sync
 				event_loop.shutdown();
 			},
 			IoMessage::AddHandler { handler } => {
-				let handler_id = self.handlers.insert(handler.clone()).unwrap_or_else(|_| panic!("Too many handlers registered"));
-				handler.initialize(&IoContext::new(IoChannel::new(event_loop.channel()), handler_id));
+				let handler_id = self.handlers.write().insert(handler.clone()).unwrap_or_else(|_| panic!("Too many handlers registered"));
+				handler.initialize(&IoContext::new(IoChannel::new(event_loop.channel(), Arc::downgrade(&self.handlers)), handler_id));
 			},
 			IoMessage::RemoveHandler { handler_id } => {
 				// TODO: flush event loop
-				self.handlers.remove(handler_id);
+				self.handlers.write().remove(handler_id);
 				// unregister timers
 				let mut timers = self.timers.write();
 				let to_remove: Vec<_> = timers.keys().cloned().filter(|timer_id| timer_id / TOKENS_PER_HANDLER == handler_id).collect();
@@ -289,12 +292,12 @@ impl<Message> Handler for IoManager<Message> where Message: Send + Clone + Sync
 				}
 			},
 			IoMessage::RegisterStream { handler_id, token } => {
-				if let Some(handler) = self.handlers.get(handler_id) {
+				if let Some(handler) = self.handlers.read().get(handler_id) {
 					handler.register_stream(token, Token(token + handler_id * TOKENS_PER_HANDLER), event_loop);
 				}
 			},
 			IoMessage::DeregisterStream { handler_id, token } => {
-				if let Some(handler) = self.handlers.get(handler_id) {
+				if let Some(handler) = self.handlers.read().get(handler_id) {
 					handler.deregister_stream(token, event_loop);
 					// unregister a timer associated with the token (if any)
 					let timer_id = token + handler_id * TOKENS_PER_HANDLER;
@@ -304,14 +307,14 @@ impl<Message> Handler for IoManager<Message> where Message: Send + Clone + Sync
 				}
 			},
 			IoMessage::UpdateStreamRegistration { handler_id, token } => {
-				if let Some(handler) = self.handlers.get(handler_id) {
+				if let Some(handler) = self.handlers.read().get(handler_id) {
 					handler.update_stream(token, Token(token + handler_id * TOKENS_PER_HANDLER), event_loop);
 				}
 			},
 			IoMessage::UserMessage(data) => {
 				//TODO: better way to iterate the slab
 				for id in 0 .. MAX_HANDLERS {
-					if let Some(h) = self.handlers.get(id) {
+					if let Some(h) = self.handlers.read().get(id) {
 						let handler = h.clone();
 						self.worker_channel.push(Work { work_type: WorkType::Message(data.clone()), token: 0, handler: handler, handler_id: id });
 					}
@@ -325,19 +328,21 @@ impl<Message> Handler for IoManager<Message> where Message: Send + Clone + Sync
 /// Allows sending messages into the event loop. All the IO handlers will get the message
 /// in the `message` callback.
 pub struct IoChannel<Message> where Message: Send + Clone{
-	channel: Option<Sender<IoMessage<Message>>>
+	channel: Option<Sender<IoMessage<Message>>>,
+	handlers: Weak<RwLock<Slab<Arc<IoHandler<Message>>, HandlerId>>>,
 }
 
-impl<Message> Clone for IoChannel<Message> where Message: Send + Clone {
+impl<Message> Clone for IoChannel<Message> where Message: Send + Clone + Sync + 'static {
 	fn clone(&self) -> IoChannel<Message> {
 		IoChannel {
-			channel: self.channel.clone()
+			channel: self.channel.clone(),
+			handlers: self.handlers.clone(),
 		}
 	}
 }
 
-impl<Message> IoChannel<Message> where Message: Send + Clone {
-	/// Send a msessage through the channel
+impl<Message> IoChannel<Message> where Message: Send + Clone + Sync + 'static {
+	/// Send a message through the channel
 	pub fn send(&self, message: Message) -> Result<(), IoError> {
 		if let Some(ref channel) = self.channel {
 			try!(channel.send(IoMessage::UserMessage(message)));
@@ -345,6 +350,19 @@ impl<Message> IoChannel<Message> where Message: Send + Clone {
 		Ok(())
 	}
 
+	/// Send a message through the channel and handle it synchronously
+	pub fn send_sync(&self, message: Message) -> Result<(), IoError> {
+		if let Some(handlers) = self.handlers.upgrade() {
+			for id in 0 .. MAX_HANDLERS {
+				if let Some(h) = handlers.read().get(id) {
+					let handler = h.clone();
+					handler.message(&IoContext::new(self.clone(), id), &message);
+				}
+			}
+		}
+		Ok(())
+	}
+
 	/// Send low level io message
 	pub fn send_io(&self, message: IoMessage<Message>) -> Result<(), IoError> {
 		if let Some(ref channel) = self.channel {
@@ -354,11 +372,17 @@ impl<Message> IoChannel<Message> where Message: Send + Clone {
 	}
 	/// Create a new channel to connected to event loop.
 	pub fn disconnected() -> IoChannel<Message> {
-		IoChannel { channel: None }
+		IoChannel {
+			channel: None,
+			handlers: Weak::default(),
+		}
 	}
 
-	fn new(channel: Sender<IoMessage<Message>>) -> IoChannel<Message> {
-		IoChannel { channel: Some(channel) }
+	fn new(channel: Sender<IoMessage<Message>>, handlers: Weak<RwLock<Slab<Arc<IoHandler<Message>>, HandlerId>>>) -> IoChannel<Message> {
+		IoChannel {
+			channel: Some(channel),
+			handlers: handlers,
+		}
 	}
 }
 
@@ -368,6 +392,7 @@ pub struct IoService<Message> where Message: Send + Sync + Clone + 'static {
 	panic_handler: Arc<PanicHandler>,
 	thread: Option<JoinHandle<()>>,
 	host_channel: Sender<IoMessage<Message>>,
+	handlers: Arc<RwLock<Slab<Arc<IoHandler<Message>>, HandlerId>>>,
 }
 
 impl<Message> MayPanic for IoService<Message> where Message: Send + Sync + Clone + 'static {
@@ -385,16 +410,19 @@ impl<Message> IoService<Message> where Message: Send + Sync + Clone + 'static {
 		let mut event_loop = EventLoop::configured(config).expect("Error creating event loop");
 		let channel = event_loop.channel();
 		let panic = panic_handler.clone();
+		let handlers = Arc::new(RwLock::new(Slab::new(MAX_HANDLERS)));
+		let h = handlers.clone();
 		let thread = thread::spawn(move || {
 			let p = panic.clone();
 			panic.catch_panic(move || {
-				IoManager::<Message>::start(p, &mut event_loop).unwrap();
+				IoManager::<Message>::start(p, &mut event_loop, h).unwrap();
 			}).unwrap()
 		});
 		Ok(IoService {
 			panic_handler: panic_handler,
 			thread: Some(thread),
-			host_channel: channel
+			host_channel: channel,
+			handlers: handlers,
 		})
 	}
 
@@ -414,7 +442,7 @@ impl<Message> IoService<Message> where Message: Send + Sync + Clone + 'static {
 
 	/// Create a new message channel
 	pub fn channel(&self) -> IoChannel<Message> {
-		IoChannel { channel: Some(self.host_channel.clone()) }
+		IoChannel::new(self.host_channel.clone(), Arc::downgrade(&self.handlers))
 	}
 }
 

util/io/src/worker.rs

@@ -22,9 +22,19 @@ use crossbeam::sync::chase_lev;
 use service::{HandlerId, IoChannel, IoContext};
 use IoHandler;
 use panics::*;
+use std::cell::Cell;
 
 use std::sync::{Condvar as SCondvar, Mutex as SMutex};
 
+const STACK_SIZE: usize = 16*1024*1024;
+
+thread_local! {
+	/// Stack size
+	/// Should be modified if it is changed in Rust since it is no way
+	/// to know or get it
+	pub static LOCAL_STACK_SIZE: Cell<usize> = Cell::new(::std::env::var("RUST_MIN_STACK").ok().and_then(|s| s.parse().ok()).unwrap_or(2 * 1024 * 1024));
+}
+
 pub enum WorkType<Message> {
 	Readable,
 	Writable,
@@ -66,8 +76,9 @@ impl Worker {
 			deleting: deleting.clone(),
 			wait_mutex: wait_mutex.clone(),
 		};
-		worker.thread = Some(thread::Builder::new().name(format!("IO Worker #{}", index)).spawn(
+		worker.thread = Some(thread::Builder::new().stack_size(STACK_SIZE).name(format!("IO Worker #{}", index)).spawn(
 			move || {
+				LOCAL_STACK_SIZE.with(|val| val.set(STACK_SIZE));
 				panic_handler.catch_panic(move || {
 					Worker::work_loop(stealer, channel.clone(), wait, wait_mutex.clone(), deleting)
 				}).unwrap()
@@ -90,11 +101,11 @@ impl Worker {
 				let _ = wait.wait(lock);
 			}
 
-			if deleting.load(AtomicOrdering::Acquire) {
-				return;
-			}
-			while let chase_lev::Steal::Data(work) = stealer.steal() {
-				Worker::do_work(work, channel.clone());
+			while !deleting.load(AtomicOrdering::Acquire) {
+				match stealer.steal() {
+					chase_lev::Steal::Data(work) => Worker::do_work(work, channel.clone()),
+					_ => break,
+				}
 			}
 		}
 	}

util/network/Cargo.toml

@@ -17,7 +17,7 @@ slab = "0.2"
 clippy = { version = "0.0.90", optional = true}
 igd = "0.5.0"
 libc = "0.2.7"
-parking_lot = "0.2.6"
+parking_lot = "0.3"
 ansi_term = "0.7"
 rustc-serialize = "0.3"
 ethcore-io = { path = "../io" }

util/network/src/connection.rs

@@ -104,12 +104,13 @@ impl<Socket: GenericSocket> GenericConnection<Socket> {
 	}
 
 	/// Add a packet to send queue.
-	pub fn send<Message>(&mut self, io: &IoContext<Message>, data: Bytes) where Message: Send + Clone {
+	pub fn send<Message>(&mut self, io: &IoContext<Message>, data: Bytes) where Message: Send + Clone + Sync + 'static {
 		if !data.is_empty() {
+			trace!(target:"network", "{}: Sending {} bytes", self.token, data.len());
 			self.send_queue.push_back(Cursor::new(data));
-		}
-		if !self.interest.is_writable() {
-			self.interest.insert(EventSet::writable());
+			if !self.interest.is_writable() {
+				self.interest.insert(EventSet::writable());
+			}
 			io.update_registration(self.token).ok();
 		}
 	}
@@ -120,7 +121,7 @@ impl<Socket: GenericSocket> GenericConnection<Socket> {
 	}
 
 	/// Writable IO handler. Called when the socket is ready to send.
-	pub fn writable<Message>(&mut self, io: &IoContext<Message>) -> Result<WriteStatus, NetworkError> where Message: Send + Clone {
+	pub fn writable<Message>(&mut self, io: &IoContext<Message>) -> Result<WriteStatus, NetworkError> where Message: Send + Clone + Sync + 'static {
 		if self.send_queue.is_empty() {
 			return Ok(WriteStatus::Complete)
 		}
@@ -191,6 +192,11 @@ impl Connection {
 		self.socket.peer_addr().map(|a| a.to_string()).unwrap_or_else(|_| "Unknown".to_owned())
 	}
 
+	/// Get local peer address string
+	pub fn local_addr_str(&self) -> String {
+		self.socket.local_addr().map(|a| a.to_string()).unwrap_or_else(|_| "Unknown".to_owned())
+	}
+
 	/// Clone this connection. Clears the receiving buffer of the returned connection.
 	pub fn try_clone(&self) -> io::Result<Self> {
 		Ok(Connection {
@@ -340,7 +346,7 @@ impl EncryptedConnection {
 	}
 
 	/// Send a packet
-	pub fn send_packet<Message>(&mut self, io: &IoContext<Message>, payload: &[u8]) -> Result<(), NetworkError> where Message: Send + Clone {
+	pub fn send_packet<Message>(&mut self, io: &IoContext<Message>, payload: &[u8]) -> Result<(), NetworkError> where Message: Send + Clone + Sync + 'static {
 		let mut header = RlpStream::new();
 		let len = payload.len() as usize;
 		header.append_raw(&[(len >> 16) as u8, (len >> 8) as u8, len as u8], 1);
@@ -435,7 +441,7 @@ impl EncryptedConnection {
 	}
 
 	/// Readable IO handler. Tracker receive status and returns decoded packet if avaialable.
-	pub fn readable<Message>(&mut self, io: &IoContext<Message>) -> Result<Option<Packet>, NetworkError> where Message: Send + Clone{
+	pub fn readable<Message>(&mut self, io: &IoContext<Message>) -> Result<Option<Packet>, NetworkError> where Message: Send + Clone + Sync + 'static {
 		try!(io.clear_timer(self.connection.token));
 		if let EncryptedConnectionState::Header = self.read_state {
 			if let Some(data) = try!(self.connection.readable()) {
@@ -458,7 +464,7 @@ impl EncryptedConnection {
 	}
 
 	/// Writable IO handler. Processes send queeue.
-	pub fn writable<Message>(&mut self, io: &IoContext<Message>) -> Result<(), NetworkError> where Message: Send + Clone {
+	pub fn writable<Message>(&mut self, io: &IoContext<Message>) -> Result<(), NetworkError> where Message: Send + Clone + Sync + 'static {
 		try!(self.connection.writable(io));
 		Ok(())
 	}

util/network/src/discovery.rs

@@ -29,6 +29,7 @@ use node_table::*;
 use error::NetworkError;
 use io::{StreamToken, IoContext};
 use ethkey::{Secret, KeyPair, sign, recover};
+use AllowIP;
 
 use PROTOCOL_VERSION;
 
@@ -95,6 +96,7 @@ pub struct Discovery {
 	send_queue: VecDeque<Datagramm>,
 	check_timestamps: bool,
 	adding_nodes: Vec<NodeEntry>,
+	allow_ips: AllowIP,
 }
 
 pub struct TableUpdates {
@@ -103,7 +105,7 @@ pub struct TableUpdates {
 }
 
 impl Discovery {
-	pub fn new(key: &KeyPair, listen: SocketAddr, public: NodeEndpoint, token: StreamToken) -> Discovery {
+	pub fn new(key: &KeyPair, listen: SocketAddr, public: NodeEndpoint, token: StreamToken, allow_ips: AllowIP) -> Discovery {
 		let socket = UdpSocket::bound(&listen).expect("Error binding UDP socket");
 		Discovery {
 			id: key.public().clone(),
@@ -118,14 +120,17 @@ impl Discovery {
 			send_queue: VecDeque::new(),
 			check_timestamps: true,
 			adding_nodes: Vec::new(),
+			allow_ips: allow_ips,
 		}
 	}
 
 	/// Add a new node to discovery table. Pings the node.
 	pub fn add_node(&mut self, e: NodeEntry) {
-		let endpoint = e.endpoint.clone();
-		self.update_node(e);
-		self.ping(&endpoint);
+		if e.endpoint.is_allowed(self.allow_ips) {
+			let endpoint = e.endpoint.clone();
+			self.update_node(e);
+			self.ping(&endpoint);
+		}
 	}
 
 	/// Add a list of nodes. Pings a few nodes each round
@@ -137,7 +142,9 @@ impl Discovery {
 	/// Add a list of known nodes to the table.
 	pub fn init_node_list(&mut self, mut nodes: Vec<NodeEntry>) {
 		for n in nodes.drain(..) {
-			self.update_node(n);
+			if n.endpoint.is_allowed(self.allow_ips) {
+				self.update_node(n);
+			}
 		}
 	}
 
@@ -394,10 +401,11 @@ impl Discovery {
 		try!(self.check_timestamp(timestamp));
 		let mut added_map = HashMap::new();
 		let entry = NodeEntry { id: node.clone(), endpoint: source.clone() };
-		if !entry.endpoint.is_valid() || !entry.endpoint.is_global() {
+		if !entry.endpoint.is_valid() {
 			debug!(target: "discovery", "Got bad address: {:?}", entry);
-		}
-		else {
+		} else if !entry.endpoint.is_allowed(self.allow_ips) {
+			debug!(target: "discovery", "Address not allowed: {:?}", entry);
+		} else {
 			self.update_node(entry.clone());
 			added_map.insert(node.clone(), entry);
 		}
@@ -470,6 +478,10 @@ impl Discovery {
 				debug!(target: "discovery", "Bad address: {:?}", endpoint);
 				continue;
 			}
+			if !endpoint.is_allowed(self.allow_ips) {
+				debug!(target: "discovery", "Address not allowed: {:?}", endpoint);
+				continue;
+			}
 			let node_id: NodeId = try!(r.val_at(3));
 			if node_id == self.id {
 				continue;
@@ -539,6 +551,7 @@ mod tests {
 	use std::str::FromStr;
 	use rustc_serialize::hex::FromHex;
 	use ethkey::{Random, Generator};
+	use AllowIP;
 
 	#[test]
 	fn find_node() {
@@ -563,8 +576,8 @@ mod tests {
 		let key2 = Random.generate().unwrap();
 		let ep1 = NodeEndpoint { address: SocketAddr::from_str("127.0.0.1:40444").unwrap(), udp_port: 40444 };
 		let ep2 = NodeEndpoint { address: SocketAddr::from_str("127.0.0.1:40445").unwrap(), udp_port: 40445 };
-		let mut discovery1 = Discovery::new(&key1, ep1.address.clone(), ep1.clone(), 0);
-		let mut discovery2 = Discovery::new(&key2, ep2.address.clone(), ep2.clone(), 0);
+		let mut discovery1 = Discovery::new(&key1, ep1.address.clone(), ep1.clone(), 0, AllowIP::All);
+		let mut discovery2 = Discovery::new(&key2, ep2.address.clone(), ep2.clone(), 0, AllowIP::All);
 
 		let node1 = Node::from_str("enode://a979fb575495b8d6db44f750317d0f4622bf4c2aa3365d6af7c284339968eef29b69ad0dce72a4d8db5ebb4968de0e3bec910127f134779fbcb0cb6d3331163c@127.0.0.1:7770").unwrap();
 		let node2 = Node::from_str("enode://b979fb575495b8d6db44f750317d0f4622bf4c2aa3365d6af7c284339968eef29b69ad0dce72a4d8db5ebb4968de0e3bec910127f134779fbcb0cb6d3331163c@127.0.0.1:7771").unwrap();
@@ -596,7 +609,7 @@ mod tests {
 	fn removes_expired() {
 		let key = Random.generate().unwrap();
 		let ep = NodeEndpoint { address: SocketAddr::from_str("127.0.0.1:40446").unwrap(), udp_port: 40447 };
-		let mut discovery = Discovery::new(&key, ep.address.clone(), ep.clone(), 0);
+		let mut discovery = Discovery::new(&key, ep.address.clone(), ep.clone(), 0, AllowIP::All);
 		for _ in 0..1200 {
 			discovery.add_node(NodeEntry { id: NodeId::random(), endpoint: ep.clone() });
 		}
@@ -624,7 +637,7 @@ mod tests {
 	fn packets() {
 		let key = Random.generate().unwrap();
 		let ep = NodeEndpoint { address: SocketAddr::from_str("127.0.0.1:40447").unwrap(), udp_port: 40447 };
-		let mut discovery = Discovery::new(&key, ep.address.clone(), ep.clone(), 0);
+		let mut discovery = Discovery::new(&key, ep.address.clone(), ep.clone(), 0, AllowIP::All);
 		discovery.check_timestamps = false;
 		let from = SocketAddr::from_str("99.99.99.99:40445").unwrap();
 

util/network/src/handshake.rs

@@ -106,7 +106,7 @@ impl Handshake {
 	}
 
 	/// Start a handhsake
-	pub fn start<Message>(&mut self, io: &IoContext<Message>, host: &HostInfo, originated: bool) -> Result<(), NetworkError> where Message: Send + Clone{
+	pub fn start<Message>(&mut self, io: &IoContext<Message>, host: &HostInfo, originated: bool) -> Result<(), NetworkError> where Message: Send + Clone+ Sync + 'static {
 		self.originated = originated;
 		io.register_timer(self.connection.token, HANDSHAKE_TIMEOUT).ok();
 		if originated {
@@ -125,7 +125,7 @@ impl Handshake {
 	}
 
 	/// Readable IO handler. Drives the state change.
-	pub fn readable<Message>(&mut self, io: &IoContext<Message>, host: &HostInfo) -> Result<(), NetworkError> where Message: Send + Clone {
+	pub fn readable<Message>(&mut self, io: &IoContext<Message>, host: &HostInfo) -> Result<(), NetworkError> where Message: Send + Clone + Sync + 'static {
 		if !self.expired() {
 			while let Some(data) = try!(self.connection.readable()) {
 				match self.state {
@@ -154,7 +154,7 @@ impl Handshake {
 	}
 
 	/// Writabe IO handler.
-	pub fn writable<Message>(&mut self, io: &IoContext<Message>) -> Result<(), NetworkError> where Message: Send + Clone {
+	pub fn writable<Message>(&mut self, io: &IoContext<Message>) -> Result<(), NetworkError> where Message: Send + Clone + Sync + 'static {
 		if !self.expired() {
 			try!(self.connection.writable(io));
 		}
@@ -172,7 +172,7 @@ impl Handshake {
 	}
 
 	/// Parse, validate and confirm auth message
-	fn read_auth<Message>(&mut self, io: &IoContext<Message>, secret: &Secret, data: &[u8]) -> Result<(), NetworkError> where Message: Send + Clone {
+	fn read_auth<Message>(&mut self, io: &IoContext<Message>, secret: &Secret, data: &[u8]) -> Result<(), NetworkError> where Message: Send + Clone + Sync + 'static {
 		trace!(target: "network", "Received handshake auth from {:?}", self.connection.remote_addr_str());
 		if data.len() != V4_AUTH_PACKET_SIZE {
 			debug!(target: "network", "Wrong auth packet size");
@@ -203,7 +203,7 @@ impl Handshake {
 		Ok(())
 	}
 
-	fn read_auth_eip8<Message>(&mut self, io: &IoContext<Message>, secret: &Secret, data: &[u8]) -> Result<(), NetworkError> where Message: Send + Clone {
+	fn read_auth_eip8<Message>(&mut self, io: &IoContext<Message>, secret: &Secret, data: &[u8]) -> Result<(), NetworkError> where Message: Send + Clone + Sync + 'static {
 		trace!(target: "network", "Received EIP8 handshake auth from {:?}", self.connection.remote_addr_str());
 		self.auth_cipher.extend_from_slice(data);
 		let auth = try!(ecies::decrypt(secret, &self.auth_cipher[0..2], &self.auth_cipher[2..]));
@@ -259,7 +259,7 @@ impl Handshake {
 	}
 
 	/// Sends auth message
-	fn write_auth<Message>(&mut self, io: &IoContext<Message>, secret: &Secret, public: &Public) -> Result<(), NetworkError> where Message: Send + Clone {
+	fn write_auth<Message>(&mut self, io: &IoContext<Message>, secret: &Secret, public: &Public) -> Result<(), NetworkError> where Message: Send + Clone + Sync + 'static {
 		trace!(target: "network", "Sending handshake auth to {:?}", self.connection.remote_addr_str());
 		let mut data = [0u8; /*Signature::SIZE*/ 65 + /*H256::SIZE*/ 32 + /*Public::SIZE*/ 64 + /*H256::SIZE*/ 32 + 1]; //TODO: use associated constants
 		let len = data.len();
@@ -286,7 +286,7 @@ impl Handshake {
 	}
 
 	/// Sends ack message
-	fn write_ack<Message>(&mut self, io: &IoContext<Message>) -> Result<(), NetworkError> where Message: Send + Clone {
+	fn write_ack<Message>(&mut self, io: &IoContext<Message>) -> Result<(), NetworkError> where Message: Send + Clone + Sync + 'static {
 		trace!(target: "network", "Sending handshake ack to {:?}", self.connection.remote_addr_str());
 		let mut data = [0u8; 1 + /*Public::SIZE*/ 64 + /*H256::SIZE*/ 32]; //TODO: use associated constants
 		let len = data.len();
@@ -305,7 +305,7 @@ impl Handshake {
 	}
 
 	/// Sends EIP8 ack message
-	fn write_ack_eip8<Message>(&mut self, io: &IoContext<Message>) -> Result<(), NetworkError> where Message: Send + Clone {
+	fn write_ack_eip8<Message>(&mut self, io: &IoContext<Message>) -> Result<(), NetworkError> where Message: Send + Clone + Sync + 'static {
 		trace!(target: "network", "Sending EIP8 handshake ack to {:?}", self.connection.remote_addr_str());
 		let mut rlp = RlpStream::new_list(3);
 		rlp.append(self.ecdhe.public());

util/network/src/host.rs

@@ -31,10 +31,10 @@ use util::hash::*;
 use util::Hashable;
 use util::version;
 use rlp::*;
-use session::{Session, SessionData};
+use session::{Session, SessionInfo, SessionData};
 use error::*;
 use io::*;
-use {NetworkProtocolHandler, NonReservedPeerMode, PROTOCOL_VERSION};
+use {NetworkProtocolHandler, NonReservedPeerMode, AllowIP, PROTOCOL_VERSION};
 use node_table::*;
 use stats::NetworkStats;
 use discovery::{Discovery, TableUpdates, NodeEntry};
@@ -45,9 +45,25 @@ use parking_lot::{Mutex, RwLock};
 type Slab<T> = ::slab::Slab<T, usize>;
 
 const MAX_SESSIONS: usize = 1024 + MAX_HANDSHAKES;
-const MAX_HANDSHAKES: usize = 80;
-const MAX_HANDSHAKES_PER_ROUND: usize = 32;
+const MAX_HANDSHAKES: usize = 1024;
+
+// Tokens
+const TCP_ACCEPT: usize = SYS_TIMER + 1;
+const IDLE: usize = SYS_TIMER + 2;
+const DISCOVERY: usize = SYS_TIMER + 3;
+const DISCOVERY_REFRESH: usize = SYS_TIMER + 4;
+const DISCOVERY_ROUND: usize = SYS_TIMER + 5;
+const NODE_TABLE: usize = SYS_TIMER + 6;
+const FIRST_SESSION: usize = 0;
+const LAST_SESSION: usize = FIRST_SESSION + MAX_SESSIONS - 1;
+const USER_TIMER: usize = LAST_SESSION + 256;
+const SYS_TIMER: usize = LAST_SESSION + 1;
+
+// Timeouts
 const MAINTENANCE_TIMEOUT: u64 = 1000;
+const DISCOVERY_REFRESH_TIMEOUT: u64 = 7200;
+const DISCOVERY_ROUND_TIMEOUT: u64 = 300;
+const NODE_TABLE_TIMEOUT: u64 = 300_000;
 
 #[derive(Debug, PartialEq, Clone)]
 /// Network service configuration
@@ -72,12 +88,18 @@ pub struct NetworkConfiguration {
 	pub use_secret: Option<Secret>,
 	/// Minimum number of connected peers to maintain
 	pub min_peers: u32,
-	/// Maximum allowd number of peers
+	/// Maximum allowed number of peers
 	pub max_peers: u32,
+	/// Maximum handshakes
+	pub max_handshakes: u32,
+	/// Reserved protocols. Peers with <key> protocol get additional <value> connection slots.
+	pub reserved_protocols: HashMap<ProtocolId, u32>,
 	/// List of reserved node addresses.
 	pub reserved_nodes: Vec<String>,
 	/// The non-reserved peer mode.
 	pub non_reserved_mode: NonReservedPeerMode,
+	/// IP filter
+	pub allow_ips: AllowIP,
 }
 
 impl Default for NetworkConfiguration {
@@ -101,6 +123,9 @@ impl NetworkConfiguration {
 			use_secret: None,
 			min_peers: 25,
 			max_peers: 50,
+			max_handshakes: 64,
+			reserved_protocols: HashMap::new(),
+			allow_ips: AllowIP::All,
 			reserved_nodes: Vec::new(),
 			non_reserved_mode: NonReservedPeerMode::Accept,
 		}
@@ -122,22 +147,10 @@ impl NetworkConfiguration {
 	}
 }
 
-// Tokens
-const TCP_ACCEPT: usize = SYS_TIMER + 1;
-const IDLE: usize = SYS_TIMER + 2;
-const DISCOVERY: usize = SYS_TIMER + 3;
-const DISCOVERY_REFRESH: usize = SYS_TIMER + 4;
-const DISCOVERY_ROUND: usize = SYS_TIMER + 5;
-const NODE_TABLE: usize = SYS_TIMER + 6;
-const FIRST_SESSION: usize = 0;
-const LAST_SESSION: usize = FIRST_SESSION + MAX_SESSIONS - 1;
-const USER_TIMER: usize = LAST_SESSION + 256;
-const SYS_TIMER: usize = LAST_SESSION + 1;
-
 /// Protocol handler level packet id
 pub type PacketId = u8;
 /// Protocol / handler id
-pub type ProtocolId = &'static str;
+pub type ProtocolId = [u8; 3];
 
 /// Messages used to communitate with the event loop from other threads.
 #[derive(Clone)]
@@ -150,6 +163,8 @@ pub enum NetworkIoMessage {
 		protocol: ProtocolId,
 		/// Supported protocol versions.
 		versions: Vec<u8>,
+		/// Number of packet IDs reserved by the protocol.
+		packet_count: u8,
 	},
 	/// Register a new protocol timer
 	AddTimer {
@@ -185,7 +200,7 @@ pub struct CapabilityInfo {
 impl Encodable for CapabilityInfo {
 	fn rlp_append(&self, s: &mut RlpStream) {
 		s.begin_list(2);
-		s.append(&self.protocol);
+		s.append(&&self.protocol[..]);
 		s.append(&self.version);
 	}
 }
@@ -226,9 +241,14 @@ impl<'s> NetworkContext<'s> {
 
 	/// Send a packet over the network to another peer.
 	pub fn send(&self, peer: PeerId, packet_id: PacketId, data: Vec<u8>) -> Result<(), NetworkError> {
+		self.send_protocol(self.protocol, peer, packet_id, data)
+	}
+
+	/// Send a packet over the network to another peer using specified protocol.
+	pub fn send_protocol(&self, protocol: ProtocolId, peer: PeerId, packet_id: PacketId, data: Vec<u8>) -> Result<(), NetworkError> {
 		let session = self.resolve_session(peer);
 		if let Some(session) = session {
-			try!(session.lock().send_packet(self.io, self.protocol, packet_id as u8, &data));
+			try!(session.lock().send_packet(self.io, protocol, packet_id as u8, &data));
 		} else  {
 			trace!(target: "network", "Send: Peer no longer exist")
 		}
@@ -246,9 +266,8 @@ impl<'s> NetworkContext<'s> {
 		self.io.channel()
 	}
 
-	/// Disable current protocol capability for given peer. If no capabilities left peer gets disconnected.
+	/// Disconnect a peer and prevent it from connecting again.
 	pub fn disable_peer(&self, peer: PeerId) {
-		//TODO: remove capability, disconnect if no capabilities left
 		self.io.message(NetworkIoMessage::DisablePeer(peer))
 			.unwrap_or_else(|e| warn!("Error sending network IO message: {:?}", e));
 	}
@@ -275,19 +294,23 @@ impl<'s> NetworkContext<'s> {
 	}
 
 	/// Returns peer identification string
-	pub fn peer_info(&self, peer: PeerId) -> String {
-		let session = self.resolve_session(peer);
-		if let Some(session) = session {
-			return session.lock().info.client_version.clone()
-		}
-		"unknown".to_owned()
+	pub fn peer_client_version(&self, peer: PeerId) -> String {
+		self.resolve_session(peer).map_or("unknown".to_owned(), |s| s.lock().info.client_version.clone())
+	}
+
+	/// Returns information on p2p session
+	pub fn session_info(&self, peer: PeerId) -> Option<SessionInfo> {
+		self.resolve_session(peer).map(|s| s.lock().info.clone())
 	}
 
 	/// Returns max version for a given protocol.
-	pub fn protocol_version(&self, peer: PeerId, protocol: &str) -> Option<u8> {
+	pub fn protocol_version(&self, protocol: ProtocolId, peer: PeerId) -> Option<u8> {
 		let session = self.resolve_session(peer);
 		session.and_then(|s| s.lock().capability_version(protocol))
 	}
+
+	/// Returns this object's subprotocol name.
+	pub fn subprotocol_name(&self) -> ProtocolId { self.protocol }
 }
 
 /// Shared host information
@@ -354,7 +377,7 @@ pub struct Host {
 
 impl Host {
 	/// Create a new instance
-	pub fn new(config: NetworkConfiguration, stats: Arc<NetworkStats>) -> Result<Host, NetworkError> {
+	pub fn new(mut config: NetworkConfiguration, stats: Arc<NetworkStats>) -> Result<Host, NetworkError> {
 		trace!(target: "host", "Creating new Host object");
 
 		let mut listen_address = match config.listen_address {
@@ -384,6 +407,7 @@ impl Host {
 
 		let boot_nodes = config.boot_nodes.clone();
 		let reserved_nodes = config.reserved_nodes.clone();
+		config.max_handshakes = min(config.max_handshakes, MAX_HANDSHAKES as u32);
 
 		let mut host = Host {
 			info: RwLock::new(HostInfo {
@@ -522,6 +546,7 @@ impl Host {
 		}
 		let local_endpoint = self.info.read().local_endpoint.clone();
 		let public_address = self.info.read().config.public_address.clone();
+		let allow_ips = self.info.read().config.allow_ips;
 		let public_endpoint = match public_address {
 			None => {
 				let public_address = select_public_address(local_endpoint.address.port());
@@ -553,7 +578,7 @@ impl Host {
 			if info.config.discovery_enabled && info.config.non_reserved_mode == NonReservedPeerMode::Accept {
 				let mut udp_addr = local_endpoint.address.clone();
 				udp_addr.set_port(local_endpoint.udp_port);
-				Some(Discovery::new(&info.keys, udp_addr, public_endpoint, DISCOVERY))
+				Some(Discovery::new(&info.keys, udp_addr, public_endpoint, DISCOVERY, allow_ips))
 			} else { None }
 		};
 
@@ -561,11 +586,11 @@ impl Host {
 			discovery.init_node_list(self.nodes.read().unordered_entries());
 			discovery.add_node_list(self.nodes.read().unordered_entries());
 			*self.discovery.lock() = Some(discovery);
-			io.register_stream(DISCOVERY).expect("Error registering UDP listener");
-			io.register_timer(DISCOVERY_REFRESH, 7200).expect("Error registering discovery timer");
-			io.register_timer(DISCOVERY_ROUND, 300).expect("Error registering discovery timer");
+			try!(io.register_stream(DISCOVERY));
+			try!(io.register_timer(DISCOVERY_REFRESH, DISCOVERY_REFRESH_TIMEOUT));
+			try!(io.register_timer(DISCOVERY_ROUND, DISCOVERY_ROUND_TIMEOUT));
 		}
-		try!(io.register_timer(NODE_TABLE, 300_000));
+		try!(io.register_timer(NODE_TABLE, NODE_TABLE_TIMEOUT));
 		try!(io.register_stream(TCP_ACCEPT));
 		Ok(())
 	}
@@ -588,7 +613,8 @@ impl Host {
 	}
 
 	fn handshake_count(&self) -> usize {
-		self.sessions.read().count() - self.session_count()
+		// session_count < total_count is possible because of the data race.
+		self.sessions.read().count().saturating_sub(self.session_count())
 	}
 
 	fn keep_alive(&self, io: &IoContext<NetworkIoMessage>) {
@@ -607,14 +633,14 @@ impl Host {
 	}
 
 	fn connect_peers(&self, io: &IoContext<NetworkIoMessage>) {
-		let (min_peers, mut pin) = {
+		let (min_peers, mut pin, max_handshakes, allow_ips) = {
 			let info = self.info.read();
 			if info.capabilities.is_empty() {
 				return;
 			}
 			let config = &info.config;
 
-			(config.min_peers, config.non_reserved_mode == NonReservedPeerMode::Deny)
+			(config.min_peers, config.non_reserved_mode == NonReservedPeerMode::Deny, config.max_handshakes as usize, config.allow_ips)
 		};
 
 		let session_count = self.session_count();
@@ -631,22 +657,22 @@ impl Host {
 
 		let handshake_count = self.handshake_count();
 		// allow 16 slots for incoming connections
-		let handshake_limit = MAX_HANDSHAKES - 16;
-		if handshake_count >= handshake_limit {
+		if handshake_count >= max_handshakes {
 			return;
 		}
 
 		// iterate over all nodes, reserved ones coming first.
 		// if we are pinned to only reserved nodes, ignore all others.
 		let nodes = reserved_nodes.iter().cloned().chain(if !pin {
-			self.nodes.read().nodes()
+			self.nodes.read().nodes(allow_ips)
 		} else {
 			Vec::new()
 		});
 
+		let max_handshakes_per_round = max_handshakes / 2;
 		let mut started: usize = 0;
 		for id in nodes.filter(|ref id| !self.have_session(id) && !self.connecting_to(id))
-			.take(min(MAX_HANDSHAKES_PER_ROUND, handshake_limit - handshake_count)) {
+			.take(min(max_handshakes_per_round, max_handshakes - handshake_count)) {
 			self.connect_peer(&id, io);
 			started += 1;
 		}
@@ -779,7 +805,14 @@ impl Host {
 							let session_count = self.session_count();
 							let (max_peers, reserved_only) = {
 								let info = self.info.read();
-								(info.config.max_peers, info.config.non_reserved_mode == NonReservedPeerMode::Deny)
+								let mut max_peers = info.config.max_peers;
+								for cap in s.info.capabilities.iter() {
+									if let Some(num) = info.config.reserved_protocols.get(&cap.protocol) {
+										max_peers += *num;
+										break;
+									}
+								}
+								(max_peers, info.config.non_reserved_mode == NonReservedPeerMode::Deny)
 							};
 
 							if session_count >= max_peers as usize || reserved_only {
@@ -801,8 +834,8 @@ impl Host {
 							}
 						}
 						for (p, _) in self.handlers.read().iter() {
-							if s.have_capability(p)  {
-								ready_data.push(p);
+							if s.have_capability(*p) {
+								ready_data.push(*p);
 							}
 						}
 					},
@@ -811,7 +844,7 @@ impl Host {
 						protocol,
 						packet_id,
 					}) => {
-						match self.handlers.read().get(protocol) {
+						match self.handlers.read().get(&protocol) {
 							None => { warn!(target: "network", "No handler found for protocol: {:?}", protocol) },
 							Some(_) => packet_data.push((protocol, packet_id, data)),
 						}
@@ -826,13 +859,13 @@ impl Host {
 		}
 		let handlers = self.handlers.read();
 		for p in ready_data {
-			let h = handlers.get(p).unwrap().clone();
+			let h = handlers.get(&p).unwrap().clone();
 			self.stats.inc_sessions();
 			let reserved = self.reserved_nodes.read();
 			h.connected(&NetworkContext::new(io, p, session.clone(), self.sessions.clone(), &reserved), &token);
 		}
 		for (p, packet_id, data) in packet_data {
-			let h = handlers.get(p).unwrap().clone();
+			let h = handlers.get(&p).unwrap().clone();
 			let reserved = self.reserved_nodes.read();
 			h.read(&NetworkContext::new(io, p, session.clone(), self.sessions.clone(), &reserved), &token, packet_id, &data[1..]);
 		}
@@ -857,8 +890,8 @@ impl Host {
 					if s.is_ready() {
 						self.num_sessions.fetch_sub(1, AtomicOrdering::SeqCst);
 						for (p, _) in self.handlers.read().iter() {
-							if s.have_capability(p)  {
-								to_disconnect.push(p);
+							if s.have_capability(*p)  {
+								to_disconnect.push(*p);
 							}
 						}
 					}
@@ -874,7 +907,7 @@ impl Host {
 			}
 		}
 		for p in to_disconnect {
-			let h = self.handlers.read().get(p).unwrap().clone();
+			let h = self.handlers.read().get(&p).unwrap().clone();
 			let reserved = self.reserved_nodes.read();
 			h.disconnected(&NetworkContext::new(io, p, expired_session.clone(), self.sessions.clone(), &reserved), &token);
 		}
@@ -883,7 +916,7 @@ impl Host {
 		}
 	}
 
-	fn update_nodes(&self, io: &IoContext<NetworkIoMessage>, node_changes: TableUpdates) {
+	fn update_nodes(&self, _io: &IoContext<NetworkIoMessage>, node_changes: TableUpdates) {
 		let mut to_remove: Vec<PeerId> = Vec::new();
 		{
 			let sessions = self.sessions.write();
@@ -898,7 +931,6 @@ impl Host {
 		}
 		for i in to_remove {
 			trace!(target: "network", "Removed from node table: {}", i);
-			self.kill_connection(i, io, false);
 		}
 		self.nodes.write().update(node_changes, &*self.reserved_nodes.read());
 	}
@@ -909,6 +941,13 @@ impl Host {
 		let context = NetworkContext::new(io, protocol, None, self.sessions.clone(), &reserved);
 		action(&context);
 	}
+
+	pub fn with_context_eval<F, T>(&self, protocol: ProtocolId, io: &IoContext<NetworkIoMessage>, action: F) -> T where F: Fn(&NetworkContext) -> T {
+		let reserved = { self.reserved_nodes.read() };
+
+		let context = NetworkContext::new(io, protocol, None, self.sessions.clone(), &reserved);
+		action(&context)
+	}
 }
 
 impl IoHandler<NetworkIoMessage> for Host {
@@ -978,9 +1017,10 @@ impl IoHandler<NetworkIoMessage> for Host {
 			NODE_TABLE => {
 				trace!(target: "network", "Refreshing node table");
 				self.nodes.write().clear_useless();
+				self.nodes.write().save();
 			},
 			_ => match self.timers.read().get(&token).cloned() {
-				Some(timer) => match self.handlers.read().get(timer.protocol).cloned() {
+				Some(timer) => match self.handlers.read().get(&timer.protocol).cloned() {
 					None => { warn!(target: "network", "No handler found for protocol: {:?}", timer.protocol) },
 					Some(h) => {
 						let reserved = self.reserved_nodes.read();
@@ -1000,15 +1040,16 @@ impl IoHandler<NetworkIoMessage> for Host {
 			NetworkIoMessage::AddHandler {
 				ref handler,
 				ref protocol,
-				ref versions
+				ref versions,
+				ref packet_count,
 			} => {
 				let h = handler.clone();
 				let reserved = self.reserved_nodes.read();
-				h.initialize(&NetworkContext::new(io, protocol, None, self.sessions.clone(), &reserved));
-				self.handlers.write().insert(protocol, h);
+				h.initialize(&NetworkContext::new(io, *protocol, None, self.sessions.clone(), &reserved));
+				self.handlers.write().insert(*protocol, h);
 				let mut info = self.info.write();
 				for v in versions {
-					info.capabilities.push(CapabilityInfo { protocol: protocol, version: *v, packet_count:0 });
+					info.capabilities.push(CapabilityInfo { protocol: *protocol, version: *v, packet_count: *packet_count });
 				}
 			},
 			NetworkIoMessage::AddTimer {
@@ -1023,7 +1064,7 @@ impl IoHandler<NetworkIoMessage> for Host {
 					*counter += 1;
 					handler_token
 				};
-				self.timers.write().insert(handler_token, ProtocolTimer { protocol: protocol, token: *token });
+				self.timers.write().insert(handler_token, ProtocolTimer { protocol: *protocol, token: *token });
 				io.register_timer(handler_token, *delay).unwrap_or_else(|e| debug!("Error registering timer {}: {:?}", token, e));
 			},
 			NetworkIoMessage::Disconnect(ref peer) => {

util/network/src/ip_utils.rs

@@ -56,6 +56,22 @@ impl SocketAddrExt for Ipv6Addr {
 	}
 }
 
+impl SocketAddrExt for IpAddr {
+	fn is_unspecified_s(&self) -> bool {
+		match *self {
+			IpAddr::V4(ref ip) => ip.is_unspecified_s(),
+			IpAddr::V6(ref ip) => ip.is_unspecified_s(),
+		}
+	}
+
+	fn is_global_s(&self) -> bool {
+		match *self {
+			IpAddr::V4(ref ip) => ip.is_global_s(),
+			IpAddr::V6(ref ip) => ip.is_global_s(),
+		}
+	}
+}
+
 #[cfg(not(windows))]
 mod getinterfaces {
 	use std::{mem, io, ptr};

util/network/src/lib.rs

@@ -45,7 +45,7 @@
 //!
 //! fn main () {
 //! 	let mut service = NetworkService::new(NetworkConfiguration::new_local()).expect("Error creating network service");
-//! 	service.register_protocol(Arc::new(MyHandler), "myproto", &[1u8]);
+//! 	service.register_protocol(Arc::new(MyHandler), *b"myp", 1, &[1u8]);
 //! 	service.start().expect("Error starting service");
 //!
 //! 	// Wait for quit condition
@@ -91,14 +91,11 @@ mod ip_utils;
 #[cfg(test)]
 mod tests;
 
-pub use host::PeerId;
-pub use host::PacketId;
-pub use host::NetworkContext;
+pub use host::{PeerId, PacketId, ProtocolId, NetworkContext, NetworkIoMessage, NetworkConfiguration};
 pub use service::NetworkService;
-pub use host::NetworkIoMessage;
 pub use error::NetworkError;
-pub use host::NetworkConfiguration;
 pub use stats::NetworkStats;
+pub use session::SessionInfo;
 
 use io::TimerToken;
 pub use node_table::is_valid_node_url;
@@ -140,3 +137,15 @@ impl NonReservedPeerMode {
 		}
 	}
 }
+
+/// IP fiter
+#[derive(Clone, Debug, PartialEq, Eq, Copy)]
+pub enum AllowIP {
+	/// Connect to any address
+	All,
+	/// Connect to private network only
+	Private,
+	/// Connect to public network only
+	Public,
+}
+

util/network/src/node_table.rs

@@ -30,6 +30,7 @@ use util::UtilError;
 use rlp::*;
 use time::Tm;
 use error::NetworkError;
+use AllowIP;
 use discovery::{TableUpdates, NodeEntry};
 use ip_utils::*;
 pub use rustc_serialize::json::Json;
@@ -53,9 +54,15 @@ impl NodeEndpoint {
 			SocketAddr::V6(a) => SocketAddr::V6(SocketAddrV6::new(a.ip().clone(), self.udp_port, a.flowinfo(), a.scope_id())),
 		}
 	}
-}
 
-impl NodeEndpoint {
+	pub fn is_allowed(&self, filter: AllowIP) -> bool {
+		match filter {
+			AllowIP::All => true,
+			AllowIP::Private => !self.address.ip().is_global_s(),
+			AllowIP::Public => self.address.ip().is_global_s(),
+		}
+	}
+
 	pub fn from_rlp(rlp: &UntrustedRlp) -> Result<Self, DecoderError> {
 		let tcp_port = try!(rlp.val_at::<u16>(2));
 		let udp_port = try!(rlp.val_at::<u16>(1));
@@ -98,13 +105,6 @@ impl NodeEndpoint {
 			SocketAddr::V6(a) => !a.ip().is_unspecified_s()
 		}
 	}
-
-	pub fn is_global(&self) -> bool {
-		match self.address {
-			SocketAddr::V4(a) => a.ip().is_global_s(),
-			SocketAddr::V6(a) => a.ip().is_global_s()
-		}
-	}
 }
 
 impl FromStr for NodeEndpoint {
@@ -219,8 +219,8 @@ impl NodeTable {
 	}
 
 	/// Returns node ids sorted by number of failures
-	pub fn nodes(&self) -> Vec<NodeId> {
-		let mut refs: Vec<&Node> = self.nodes.values().filter(|n| !self.useless_nodes.contains(&n.id)).collect();
+	pub fn nodes(&self, filter: AllowIP) -> Vec<NodeId> {
+		let mut refs: Vec<&Node> = self.nodes.values().filter(|n| !self.useless_nodes.contains(&n.id) && n.endpoint.is_allowed(filter)).collect();
 		refs.sort_by(|a, b| a.failures.cmp(&b.failures));
 		refs.iter().map(|n| n.id.clone()).collect()
 	}
@@ -266,7 +266,8 @@ impl NodeTable {
 		self.useless_nodes.clear();
 	}
 
-	fn save(&self) {
+	/// Save the nodes.json file.
+	pub fn save(&self) {
 		if let Some(ref path) = self.path {
 			let mut path_buf = PathBuf::from(path);
 			if let Err(e) = fs::create_dir_all(path_buf.as_path()) {
@@ -277,7 +278,7 @@ impl NodeTable {
 			let mut json = String::new();
 			json.push_str("{\n");
 			json.push_str("\"nodes\": [\n");
-			let node_ids = self.nodes();
+			let node_ids = self.nodes(AllowIP::All);
 			for i in 0 .. node_ids.len() {
 				let node = self.nodes.get(&node_ids[i]).unwrap();
 				json.push_str(&format!("\t{{ \"url\": \"{}\", \"failures\": {} }}{}\n", node, node.failures, if i == node_ids.len() - 1 {""} else {","}))
@@ -292,7 +293,7 @@ impl NodeTable {
 				}
 			};
 			if let Err(e) = file.write(&json.into_bytes()) {
-					warn!("Error writing node table file: {:?}", e);
+				warn!("Error writing node table file: {:?}", e);
 			}
 		}
 	}
@@ -360,6 +361,7 @@ mod tests {
 	use std::net::*;
 	use util::hash::*;
 	use devtools::*;
+	use AllowIP;
 
 	#[test]
 	fn endpoint_parse() {
@@ -405,7 +407,7 @@ mod tests {
 		table.note_failure(&id1);
 		table.note_failure(&id2);
 
-		let r = table.nodes();
+		let r = table.nodes(AllowIP::All);
 		assert_eq!(r[0][..], id3[..]);
 		assert_eq!(r[1][..], id2[..]);
 		assert_eq!(r[2][..], id1[..]);
@@ -427,7 +429,7 @@ mod tests {
 
 		{
 			let table = NodeTable::new(Some(temp_path.as_path().to_str().unwrap().to_owned()));
-			let r = table.nodes();
+			let r = table.nodes(AllowIP::All);
 			assert_eq!(r[0][..], id1[..]);
 			assert_eq!(r[1][..], id2[..]);
 		}

util/network/src/service.rs

@@ -73,11 +73,12 @@ impl NetworkService {
 	}
 
 	/// Regiter a new protocol handler with the event loop.
-	pub fn register_protocol(&self, handler: Arc<NetworkProtocolHandler + Send + Sync>, protocol: ProtocolId, versions: &[u8]) -> Result<(), NetworkError> {
+	pub fn register_protocol(&self, handler: Arc<NetworkProtocolHandler + Send + Sync>, protocol: ProtocolId, packet_count: u8, versions: &[u8]) -> Result<(), NetworkError> {
 		try!(self.io_service.send_message(NetworkIoMessage::AddHandler {
 			handler: handler,
 			protocol: protocol,
 			versions: versions.to_vec(),
+			packet_count: packet_count,
 		}));
 		Ok(())
 	}
@@ -178,6 +179,13 @@ impl NetworkService {
 			host.with_context(protocol, &io, action);
 		};
 	}
+
+	/// Evaluates function in the network context
+	pub fn with_context_eval<F, T>(&self, protocol: ProtocolId, action: F) -> Option<T> where F: Fn(&NetworkContext) -> T {
+		let io = IoContext::new(self.io_service.channel(), 0);
+		let host = self.host.read();
+		host.as_ref().map(|ref host| host.with_context_eval(protocol, &io, action))
+	}
 }
 
 impl MayPanic for NetworkService {

util/network/src/session.rs

@@ -14,8 +14,9 @@
 // You should have received a copy of the GNU General Public License
 // along with Parity.  If not, see <http://www.gnu.org/licenses/>.
 
+use std::{str, io};
 use std::net::SocketAddr;
-use std::io;
+use std::cmp::Ordering;
 use std::sync::*;
 use mio::*;
 use mio::tcp::*;
@@ -30,7 +31,7 @@ use node_table::NodeId;
 use stats::NetworkStats;
 use time;
 
-const PING_TIMEOUT_SEC: u64 = 30;
+const PING_TIMEOUT_SEC: u64 = 65;
 const PING_INTERVAL_SEC: u64 = 30;
 
 /// Peer session over encrypted connection.
@@ -63,7 +64,7 @@ pub enum SessionData {
 		/// Packet data
 		data: Vec<u8>,
 		/// Packet protocol ID
-		protocol: &'static str,
+		protocol: [u8; 3],
 		/// Zero based packet ID
 		packet_id: u8,
 	},
@@ -72,6 +73,7 @@ pub enum SessionData {
 }
 
 /// Shared session information
+#[derive(Debug, Clone)]
 pub struct SessionInfo {
 	/// Peer public key
 	pub id: Option<NodeId>,
@@ -79,38 +81,73 @@ pub struct SessionInfo {
 	pub client_version: String,
 	/// Peer RLPx protocol version
 	pub protocol_version: u32,
+	/// Session protocol capabilities
+	pub capabilities: Vec<SessionCapabilityInfo>,
 	/// Peer protocol capabilities
-	capabilities: Vec<SessionCapabilityInfo>,
+	pub peer_capabilities: Vec<PeerCapabilityInfo>,
 	/// Peer ping delay in milliseconds
 	pub ping_ms: Option<u64>,
 	/// True if this session was originated by us.
 	pub originated: bool,
+	/// Remote endpoint address of the session
+	pub remote_address: String,
+	/// Local endpoint address of the session
+	pub local_address: String,
 }
 
-#[derive(Debug, PartialEq, Eq)]
+#[derive(Debug, Clone, PartialEq, Eq)]
 pub struct PeerCapabilityInfo {
-	pub protocol: String,
+	pub protocol: ProtocolId,
 	pub version: u8,
 }
 
 impl Decodable for PeerCapabilityInfo {
 	fn decode<D>(decoder: &D) -> Result<Self, DecoderError> where D: Decoder {
 		let c = decoder.as_rlp();
+		let p: Vec<u8> = try!(c.val_at(0));
+		if p.len() != 3 {
+			return Err(DecoderError::Custom("Invalid subprotocol string length. Should be 3"));
+		}
+		let mut p2: ProtocolId = [0u8; 3];
+		p2.clone_from_slice(&p);
 		Ok(PeerCapabilityInfo {
-			protocol: try!(c.val_at(0)),
+			protocol: p2,
 			version: try!(c.val_at(1))
 		})
 	}
 }
 
-#[derive(Debug)]
-struct SessionCapabilityInfo {
-	pub protocol: &'static str,
+impl ToString for PeerCapabilityInfo {
+	fn to_string(&self) -> String {
+		format!("{}/{}", str::from_utf8(&self.protocol[..]).unwrap_or("???"), self.version)
+	}
+}
+
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct SessionCapabilityInfo {
+	pub protocol: [u8; 3],
 	pub version: u8,
 	pub packet_count: u8,
 	pub id_offset: u8,
 }
 
+impl PartialOrd for SessionCapabilityInfo {
+	fn partial_cmp(&self, other: &SessionCapabilityInfo) -> Option<Ordering> {
+		Some(self.cmp(other))
+	}
+}
+
+impl Ord for SessionCapabilityInfo {
+	fn cmp(&self, b: &SessionCapabilityInfo) -> Ordering {
+		// By protocol id first
+		if self.protocol != b.protocol {
+			return self.protocol.cmp(&b.protocol);
+		}
+		// By version
+		self.version.cmp(&b.version)
+	}
+}
+
 const PACKET_HELLO: u8 = 0x80;
 const PACKET_DISCONNECT: u8 = 0x01;
 const PACKET_PING: u8 = 0x02;
@@ -125,9 +162,10 @@ impl Session {
 	/// and leaves the handhsake in limbo to be deregistered from the event loop.
 	pub fn new<Message>(io: &IoContext<Message>, socket: TcpStream, token: StreamToken, id: Option<&NodeId>,
 		nonce: &H256, stats: Arc<NetworkStats>, host: &HostInfo) -> Result<Session, NetworkError>
-		where Message: Send + Clone {
+		where Message: Send + Clone + Sync + 'static {
 		let originated = id.is_some();
 		let mut handshake = Handshake::new(token, id, socket, nonce, stats).expect("Can't create handshake");
+		let local_addr = handshake.connection.local_addr_str();
 		try!(handshake.start(io, host, originated));
 		Ok(Session {
 			state: State::Handshake(handshake),
@@ -137,8 +175,11 @@ impl Session {
 				client_version: String::new(),
 				protocol_version: 0,
 				capabilities: Vec::new(),
+				peer_capabilities: Vec::new(),
 				ping_ms: None,
 				originated: originated,
+				remote_address: "Handshake".to_owned(),
+				local_address: local_addr,
 			},
 			ping_time_ns: 0,
 			pong_time_ns: None,
@@ -149,6 +190,7 @@ impl Session {
 	fn complete_handshake<Message>(&mut self, io: &IoContext<Message>, host: &HostInfo) -> Result<(), NetworkError> where Message: Send + Sync + Clone {
 		let connection = if let State::Handshake(ref mut h) = self.state {
 			self.info.id = Some(h.id.clone());
+			self.info.remote_address = h.connection.remote_addr_str();
 			try!(EncryptedConnection::new(h))
 		} else {
 			panic!("Unexpected state");
@@ -239,12 +281,12 @@ impl Session {
 	}
 
 	/// Checks if peer supports given capability
-	pub fn have_capability(&self, protocol: &str) -> bool {
+	pub fn have_capability(&self, protocol: [u8; 3]) -> bool {
 		self.info.capabilities.iter().any(|c| c.protocol == protocol)
 	}
 
 	/// Checks if peer supports given capability
-	pub fn capability_version(&self, protocol: &str) -> Option<u8> {
+	pub fn capability_version(&self, protocol: [u8; 3]) -> Option<u8> {
 		self.info.capabilities.iter().filter_map(|c| if c.protocol == protocol { Some(c.version) } else { None }).max()
 	}
 
@@ -270,10 +312,10 @@ impl Session {
 	}
 
 	/// Send a protocol packet to peer.
-	pub fn send_packet<Message>(&mut self, io: &IoContext<Message>, protocol: &str, packet_id: u8, data: &[u8]) -> Result<(), NetworkError>
+	pub fn send_packet<Message>(&mut self, io: &IoContext<Message>, protocol: [u8; 3], packet_id: u8, data: &[u8]) -> Result<(), NetworkError>
         where Message: Send + Sync + Clone {
 		if self.info.capabilities.is_empty() || !self.had_hello {
-			debug!(target: "network", "Sending to unconfirmed session {}, protocol: {}, packet: {}", self.token(), protocol, packet_id);
+			debug!(target: "network", "Sending to unconfirmed session {}, protocol: {}, packet: {}", self.token(), str::from_utf8(&protocol[..]).unwrap_or("??"), packet_id);
 			return Err(From::from(NetworkError::BadProtocol));
 		}
 		if self.expired() {
@@ -353,7 +395,7 @@ impl Session {
 			PACKET_PEERS => Ok(SessionData::None),
 			PACKET_USER ... PACKET_LAST => {
 				let mut i = 0usize;
-				while packet_id < self.info.capabilities[i].id_offset {
+				while packet_id > self.info.capabilities[i].id_offset + self.info.capabilities[i].packet_count {
 					i += 1;
 					if i == self.info.capabilities.len() {
 						debug!(target: "network", "Unknown packet: {:?}", packet_id);
@@ -417,6 +459,9 @@ impl Session {
 			}
 		}
 
+		// Sort capabilities alphabeticaly.
+		caps.sort();
+
 		i = 0;
 		let mut offset: u8 = PACKET_USER;
 		while i < caps.len() {
@@ -424,9 +469,11 @@ impl Session {
 			offset += caps[i].packet_count;
 			i += 1;
 		}
-		trace!(target: "network", "Hello: {} v{} {} {:?}", client_version, protocol, id, caps);
+		debug!(target: "network", "Hello: {} v{} {} {:?}", client_version, protocol, id, caps);
+		self.info.protocol_version = protocol;
 		self.info.client_version = client_version;
 		self.info.capabilities = caps;
+		self.info.peer_capabilities = peer_caps; 
 		if self.info.capabilities.is_empty() {
 			trace!(target: "network", "No common capabilities with peer.");
 			return Err(From::from(self.disconnect(io, DisconnectReason::UselessPeer)));

util/network/src/tests.rs

@@ -41,7 +41,7 @@ impl TestProtocol {
 	/// Creates and register protocol with the network service
 	pub fn register(service: &mut NetworkService, drop_session: bool) -> Arc<TestProtocol> {
 		let handler = Arc::new(TestProtocol::new(drop_session));
-		service.register_protocol(handler.clone(), "test", &[42u8, 43u8]).expect("Error registering test protocol handler");
+		service.register_protocol(handler.clone(), *b"tst", 1, &[42u8, 43u8]).expect("Error registering test protocol handler");
 		handler
 	}
 
@@ -69,7 +69,7 @@ impl NetworkProtocolHandler for TestProtocol {
 	}
 
 	fn connected(&self, io: &NetworkContext, peer: &PeerId) {
-		assert!(io.peer_info(*peer).contains("Parity"));
+		assert!(io.peer_client_version(*peer).contains("Parity"));
 		if self.drop_session {
 			io.disconnect_peer(*peer)
 		} else {
@@ -93,7 +93,7 @@ impl NetworkProtocolHandler for TestProtocol {
 fn net_service() {
 	let service = NetworkService::new(NetworkConfiguration::new_local()).expect("Error creating network service");
 	service.start().unwrap();
-	service.register_protocol(Arc::new(TestProtocol::new(false)), "myproto", &[1u8]).unwrap();
+	service.register_protocol(Arc::new(TestProtocol::new(false)), *b"myp", 1, &[1u8]).unwrap();
 }
 
 #[test]

util/rlp/Cargo.toml

@@ -6,7 +6,7 @@ version = "0.1.0"
 authors = ["Ethcore <admin@ethcore.io>"]
 
 [dependencies]
-elastic-array = "0.5"
+elastic-array = { git = "https://github.com/ethcore/elastic-array" }
 ethcore-bigint = { path = "../bigint" }
 lazy_static = "0.2"
-rustc-serialize = "0.3"
+rustc-serialize = "0.3"

util/src/common.rs

@@ -95,8 +95,8 @@ macro_rules! flushln {
 
 #[doc(hidden)]
 pub fn flush(s: String) {
-	::std::io::stdout().write(s.as_bytes()).unwrap();
-	::std::io::stdout().flush().unwrap();
+	let _ = ::std::io::stdout().write(s.as_bytes());
+	let _ = ::std::io::stdout().flush();
 }
 
 #[test]

util/src/hashdb.rs

@@ -16,8 +16,11 @@
 
 //! Database of byte-slices keyed to their Keccak hash.
 use hash::*;
-use bytes::*;
 use std::collections::HashMap;
+use elastic_array::ElasticArray256;
+
+/// `HashDB` value type.
+pub type DBValue = ElasticArray256<u8>;
 
 /// Trait modelling datastore keyed by a 32-byte Keccak hash.
 pub trait HashDB: AsHashDB + Send + Sync {
@@ -39,7 +42,7 @@ pub trait HashDB: AsHashDB + Send + Sync {
 	///   assert_eq!(m.get(&hash).unwrap(), hello_bytes);
 	/// }
 	/// ```
-	fn get(&self, key: &H256) -> Option<&[u8]>;
+	fn get(&self, key: &H256) -> Option<DBValue>;
 
 	/// Check for the existance of a hash-key.
 	///
@@ -80,7 +83,7 @@ pub trait HashDB: AsHashDB + Send + Sync {
 	fn insert(&mut self, value: &[u8]) -> H256;
 
 	/// Like `insert()` , except you provide the key and the data is all moved.
-	fn emplace(&mut self, key: H256, value: Bytes);
+	fn emplace(&mut self, key: H256, value: DBValue);
 
 	/// Remove a datum previously inserted. Insertions can be "owed" such that the same number of `insert()`s may
 	/// happen without the data being eventually being inserted into the DB.
@@ -111,7 +114,7 @@ pub trait HashDB: AsHashDB + Send + Sync {
 	}
 
 	/// Get auxiliary data from hashdb.
-	fn get_aux(&self, _hash: &[u8]) -> Option<Vec<u8>> {
+	fn get_aux(&self, _hash: &[u8]) -> Option<DBValue> {
 		unimplemented!();
 	}
 
@@ -136,4 +139,4 @@ impl<T: HashDB> AsHashDB for T {
 	fn as_hashdb_mut(&mut self) -> &mut HashDB {
 		self
 	}
-}
+}

util/src/journaldb/archivedb.rs

@@ -35,8 +35,8 @@ const AUX_FLAG: u8 = 255;
 ///
 /// Like `OverlayDB`, there is a memory overlay; `commit()` must be called in order to
 /// write operations out to disk. Unlike `OverlayDB`, `remove()` operations do not take effect
-/// immediately. Rather some age (based on a linear but arbitrary metric) must pass before
-/// the removals actually take effect.
+/// immediately. As this is an "archive" database, nothing is ever removed. This means
+/// that the states of any block the node has ever processed will be accessible.
 pub struct ArchiveDB {
 	overlay: MemoryDB,
 	backing: Arc<Database>,
@@ -65,8 +65,8 @@ impl ArchiveDB {
 		Self::new(backing, None)
 	}
 
-	fn payload(&self, key: &H256) -> Option<Bytes> {
-		self.backing.get(self.column, key).expect("Low-level database error. Some issue with your hard disk?").map(|v| v.to_vec())
+	fn payload(&self, key: &H256) -> Option<DBValue> {
+		self.backing.get(self.column, key).expect("Low-level database error. Some issue with your hard disk?")
 	}
 }
 
@@ -85,19 +85,12 @@ impl HashDB for ArchiveDB {
 		ret
 	}
 
-	fn get(&self, key: &H256) -> Option<&[u8]> {
+	fn get(&self, key: &H256) -> Option<DBValue> {
 		let k = self.overlay.raw(key);
-		match k {
-			Some((d, rc)) if rc > 0 => Some(d),
-			_ => {
-				if let Some(x) = self.payload(key) {
-					Some(self.overlay.denote(key, x).0)
-				}
-				else {
-					None
-				}
-			}
+		if let Some((d, rc)) = k {
+			if rc > 0 { return Some(d); }
 		}
+		self.payload(key)
 	}
 
 	fn contains(&self, key: &H256) -> bool {
@@ -108,7 +101,7 @@ impl HashDB for ArchiveDB {
 		self.overlay.insert(value)
 	}
 
-	fn emplace(&mut self, key: H256, value: Bytes) {
+	fn emplace(&mut self, key: H256, value: DBValue) {
 		self.overlay.emplace(key, value);
 	}
 
@@ -120,7 +113,7 @@ impl HashDB for ArchiveDB {
 		self.overlay.insert_aux(hash, value);
 	}
 
-	fn get_aux(&self, hash: &[u8]) -> Option<Vec<u8>> {
+	fn get_aux(&self, hash: &[u8]) -> Option<DBValue> {
 		if let Some(res) = self.overlay.get_aux(hash) {
 			return Some(res)
 		}
@@ -130,7 +123,6 @@ impl HashDB for ArchiveDB {
 
 		self.backing.get(self.column, &db_hash)
 			.expect("Low-level database error. Some issue with your hard disk?")
-			.map(|v| v.to_vec())
 	}
 
 	fn remove_aux(&mut self, hash: &[u8]) {
@@ -156,7 +148,7 @@ impl JournalDB for ArchiveDB {
 		self.latest_era.is_none()
 	}
 
-	fn commit(&mut self, batch: &mut DBTransaction, now: u64, _id: &H256, _end: Option<(u64, H256)>) -> Result<u32, UtilError> {
+	fn journal_under(&mut self, batch: &mut DBTransaction, now: u64, _id: &H256) -> Result<u32, UtilError> {
 		let mut inserts = 0usize;
 		let mut deletes = 0usize;
 
@@ -184,6 +176,11 @@ impl JournalDB for ArchiveDB {
 		Ok((inserts + deletes) as u32)
 	}
 
+	fn mark_canonical(&mut self, _batch: &mut DBTransaction, _end_era: u64, _canon_id: &H256) -> Result<u32, UtilError> {
+		// keep everything! it's an archive, after all.
+		Ok(0)
+	}
+
 	fn inject(&mut self, batch: &mut DBTransaction) -> Result<u32, UtilError> {
 		let mut inserts = 0usize;
 		let mut deletes = 0usize;
@@ -391,7 +388,7 @@ mod tests {
 			let mut jdb = new_db(&dir);
 			// history is 1
 			let foo = jdb.insert(b"foo");
-			jdb.emplace(bar.clone(), b"bar".to_vec());
+			jdb.emplace(bar.clone(), DBValue::from_slice(b"bar"));
 			jdb.commit_batch(0, &b"0".sha3(), None).unwrap();
 			foo
 		};
@@ -492,7 +489,7 @@ mod tests {
 		let key = jdb.insert(b"dog");
 		jdb.inject_batch().unwrap();
 
-		assert_eq!(jdb.get(&key).unwrap(), b"dog");
+		assert_eq!(jdb.get(&key).unwrap(), DBValue::from_slice(b"dog"));
 		jdb.remove(&key);
 		jdb.inject_batch().unwrap();
 

util/src/journaldb/earlymergedb.rs

@@ -61,6 +61,49 @@ enum RemoveFrom {
 /// write operations out to disk. Unlike `OverlayDB`, `remove()` operations do not take effect
 /// immediately. Rather some age (based on a linear but arbitrary metric) must pass before
 /// the removals actually take effect.
+///
+/// journal format:
+/// [era, 0] => [ id, [insert_0, ...], [remove_0, ...] ]
+/// [era, 1] => [ id, [insert_0, ...], [remove_0, ...] ]
+/// [era, n] => [ ... ]
+///
+/// When we make a new commit, we make a journal of all blocks in the recent history and record
+/// all keys that were inserted and deleted. The journal is ordered by era; multiple commits can
+/// share the same era. This forms a data structure similar to a queue but whose items are tuples.
+/// By the time comes to remove a tuple from the queue (i.e. then the era passes from recent history
+/// into ancient history) then only one commit from the tuple is considered canonical. This commit
+/// is kept in the main backing database, whereas any others from the same era are reverted.
+///
+/// It is possible that a key, properly available in the backing database be deleted and re-inserted
+/// in the recent history queue, yet have both operations in commits that are eventually non-canonical.
+/// To avoid the original, and still required, key from being deleted, we maintain a reference count
+/// which includes an original key, if any.
+///
+/// The semantics of the `counter` are:
+/// insert key k:
+///   counter already contains k: count += 1
+///   counter doesn't contain k:
+///     backing db contains k: count = 1
+///     backing db doesn't contain k: insert into backing db, count = 0
+/// delete key k:
+///   counter contains k (count is asserted to be non-zero):
+///     count > 1: counter -= 1
+///     count == 1: remove counter
+///     count == 0: remove key from backing db
+///   counter doesn't contain k: remove key from backing db
+///
+/// Practically, this means that for each commit block turning from recent to ancient we do the
+/// following:
+/// is_canonical:
+///   inserts: Ignored (left alone in the backing database).
+///   deletes: Enacted; however, recent history queue is checked for ongoing references. This is
+///            reduced as a preference to deletion from the backing database.
+/// !is_canonical:
+///   inserts: Reverted; however, recent history queue is checked for ongoing references. This is
+///            reduced as a preference to deletion from the backing database.
+///   deletes: Ignored (they were never inserted).
+///
+/// TODO: store_reclaim_period
 pub struct EarlyMergeDB {
 	overlay: MemoryDB,
 	backing: Arc<Database>,
@@ -107,7 +150,7 @@ impl EarlyMergeDB {
 		backing.get(col, &Self::morph_key(key, 0)).expect("Low-level database error. Some issue with your hard disk?").is_some()
 	}
 
-	fn insert_keys(inserts: &[(H256, Bytes)], backing: &Database, col: Option<u32>, refs: &mut HashMap<H256, RefInfo>, batch: &mut DBTransaction, trace: bool) {
+	fn insert_keys(inserts: &[(H256, DBValue)], backing: &Database, col: Option<u32>, refs: &mut HashMap<H256, RefInfo>, batch: &mut DBTransaction, trace: bool) {
 		for &(ref h, ref d) in inserts {
 			if let Some(c) = refs.get_mut(h) {
 				// already counting. increment.
@@ -225,8 +268,8 @@ impl EarlyMergeDB {
 		}
 	}
 
-	fn payload(&self, key: &H256) -> Option<Bytes> {
-		self.backing.get(self.column, key).expect("Low-level database error. Some issue with your hard disk?").map(|v| v.to_vec())
+	fn payload(&self, key: &H256) -> Option<DBValue> {
+		self.backing.get(self.column, key).expect("Low-level database error. Some issue with your hard disk?")
 	}
 
 	fn read_refs(db: &Database, col: Option<u32>) -> (Option<u64>, HashMap<H256, RefInfo>) {
@@ -274,19 +317,12 @@ impl HashDB for EarlyMergeDB {
 		ret
 	}
 
-	fn get(&self, key: &H256) -> Option<&[u8]> {
+	fn get(&self, key: &H256) -> Option<DBValue> {
 		let k = self.overlay.raw(key);
-		match k {
-			Some((d, rc)) if rc > 0 => Some(d),
-			_ => {
-				if let Some(x) = self.payload(key) {
-					Some(self.overlay.denote(key, x).0)
-				}
-				else {
-					None
-				}
-			}
+		if let Some((d, rc)) = k {
+			if rc > 0 { return Some(d) }
 		}
+		self.payload(key)
 	}
 
 	fn contains(&self, key: &H256) -> bool {
@@ -296,7 +332,7 @@ impl HashDB for EarlyMergeDB {
 	fn insert(&mut self, value: &[u8]) -> H256 {
 		self.overlay.insert(value)
 	}
-	fn emplace(&mut self, key: H256, value: Bytes) {
+	fn emplace(&mut self, key: H256, value: DBValue) {
 		self.overlay.emplace(key, value);
 	}
 	fn remove(&mut self, key: &H256) {
@@ -336,55 +372,15 @@ impl JournalDB for EarlyMergeDB {
 		self.backing.get_by_prefix(self.column, &id[0..DB_PREFIX_LEN]).map(|b| b.to_vec())
 	}
 
-	#[cfg_attr(feature="dev", allow(cyclomatic_complexity))]
-	fn commit(&mut self, batch: &mut DBTransaction, now: u64, id: &H256, end: Option<(u64, H256)>) -> Result<u32, UtilError> {
-		// journal format:
-		// [era, 0] => [ id, [insert_0, ...], [remove_0, ...] ]
-		// [era, 1] => [ id, [insert_0, ...], [remove_0, ...] ]
-		// [era, n] => [ ... ]
-
-		// TODO: store reclaim_period.
-
-		// When we make a new commit, we make a journal of all blocks in the recent history and record
-		// all keys that were inserted and deleted. The journal is ordered by era; multiple commits can
-		// share the same era. This forms a data structure similar to a queue but whose items are tuples.
-		// By the time comes to remove a tuple from the queue (i.e. then the era passes from recent history
-		// into ancient history) then only one commit from the tuple is considered canonical. This commit
-		// is kept in the main backing database, whereas any others from the same era are reverted.
-		//
-		// It is possible that a key, properly available in the backing database be deleted and re-inserted
-		// in the recent history queue, yet have both operations in commits that are eventually non-canonical.
-		// To avoid the original, and still required, key from being deleted, we maintain a reference count
-		// which includes an original key, if any.
-		//
-		// The semantics of the `counter` are:
-		// insert key k:
-		//   counter already contains k: count += 1
-		//   counter doesn't contain k:
-		//     backing db contains k: count = 1
-		//     backing db doesn't contain k: insert into backing db, count = 0
-		// delete key k:
-		//   counter contains k (count is asserted to be non-zero):
-		//     count > 1: counter -= 1
-		//     count == 1: remove counter
-		//     count == 0: remove key from backing db
-		//   counter doesn't contain k: remove key from backing db
-		//
-		// Practically, this means that for each commit block turning from recent to ancient we do the
-		// following:
-		// is_canonical:
-		//   inserts: Ignored (left alone in the backing database).
-		//   deletes: Enacted; however, recent history queue is checked for ongoing references. This is
-		//            reduced as a preference to deletion from the backing database.
-		// !is_canonical:
-		//   inserts: Reverted; however, recent history queue is checked for ongoing references. This is
-		//            reduced as a preference to deletion from the backing database.
-		//   deletes: Ignored (they were never inserted).
-		//
+	fn journal_under(&mut self, batch: &mut DBTransaction, now: u64, id: &H256) -> Result<u32, UtilError> {
+		let trace = false;
 
 		// record new commit's details.
-		let mut refs = self.refs.as_ref().unwrap().write();
-		let trace = false;
+		let mut refs = match self.refs.as_ref() {
+			Some(refs) => refs.write(),
+			None => return Ok(0),
+		};
+
 		{
 			let mut index = 0usize;
 			let mut last;
@@ -403,14 +399,14 @@ impl JournalDB for EarlyMergeDB {
 			let drained = self.overlay.drain();
 
 			if trace {
-				trace!(target: "jdb", "commit: #{} ({}), end era: {:?}", now, id, end);
+				trace!(target: "jdb", "commit: #{} ({})", now, id);
 			}
 
 			let removes: Vec<H256> = drained
 				.iter()
 				.filter_map(|(k, &(_, c))| if c < 0 {Some(k.clone())} else {None})
 				.collect();
-			let inserts: Vec<(H256, Bytes)> = drained
+			let inserts: Vec<(H256, _)> = drained
 				.into_iter()
 				.filter_map(|(k, (v, r))| if r > 0 { assert!(r == 1); Some((k, v)) } else { assert!(r >= -1); None })
 				.collect();
@@ -431,85 +427,86 @@ impl JournalDB for EarlyMergeDB {
 			inserts.iter().foreach(|&(k, _)| {r.append(&k);});
 			r.append(&removes);
 			Self::insert_keys(&inserts, &self.backing, self.column, &mut refs, batch, trace);
+
+			let ins = inserts.iter().map(|&(k, _)| k).collect::<Vec<_>>();
+
 			if trace {
-				let ins = inserts.iter().map(|&(k, _)| k).collect::<Vec<_>>();
-				trace!(target: "jdb.ops", "  Inserts: {:?}", ins);
 				trace!(target: "jdb.ops", "  Deletes: {:?}", removes);
+				trace!(target: "jdb.ops", "  Inserts: {:?}", ins);
 			}
+
 			batch.put(self.column, &last, r.as_raw());
 			if self.latest_era.map_or(true, |e| now > e) {
 				batch.put(self.column, &LATEST_ERA_KEY, &encode(&now));
 				self.latest_era = Some(now);
 			}
+
+			Ok((ins.len() + removes.len()) as u32)
 		}
+	}
+
+	#[cfg_attr(feature="dev", allow(cyclomatic_complexity))]
+	fn mark_canonical(&mut self, batch: &mut DBTransaction, end_era: u64, canon_id: &H256) -> Result<u32, UtilError> {
+		let trace = false;
+
+		let mut refs = self.refs.as_ref().unwrap().write();
 
 		// apply old commits' details
-		if let Some((end_era, canon_id)) = end {
-			let mut index = 0usize;
-			let mut last;
-			while let Some(rlp_data) = try!(self.backing.get(self.column, {
-				let mut r = RlpStream::new_list(3);
-				r.append(&end_era);
-				r.append(&index);
-				r.append(&&PADDING[..]);
-				last = r.drain();
-				&last
-			})) {
-				let rlp = Rlp::new(&rlp_data);
-				let inserts: Vec<H256> = rlp.val_at(1);
+		let mut index = 0usize;
+		let mut last;
 
-				if canon_id == rlp.val_at(0) {
-					// Collect keys to be removed. Canon block - remove the (enacted) deletes.
-					let deletes: Vec<H256> = rlp.val_at(2);
-					if trace {
-						trace!(target: "jdb.ops", "  Expunging: {:?}", deletes);
-					}
-					Self::remove_keys(&deletes, &mut refs, batch, self.column, RemoveFrom::Archive, trace);
+		while let Some(rlp_data) = try!(self.backing.get(self.column, {
+			let mut r = RlpStream::new_list(3);
+			r.append(&end_era);
+			r.append(&index);
+			r.append(&&PADDING[..]);
+			last = r.drain();
+			&last
+		})) {
+			let rlp = Rlp::new(&rlp_data);
+			let inserts: Vec<H256> = rlp.val_at(1);
 
-					if trace {
-						trace!(target: "jdb.ops", "  Finalising: {:?}", inserts);
-					}
-					for k in &inserts {
-						match refs.get(k).cloned() {
-							None => {
-								// [in archive] -> SHIFT remove -> SHIFT insert None->Some{queue_refs: 1, in_archive: true} -> TAKE remove Some{queue_refs: 1, in_archive: true}->None -> TAKE insert
-								// already expunged from the queue (which is allowed since the key is in the archive).
-								// leave well alone.
-							}
-							Some( RefInfo{queue_refs: 1, in_archive: false} ) => {
-								// just delete the refs entry.
-								refs.remove(k);
-							}
-							Some( RefInfo{queue_refs: x, in_archive: false} ) => {
-								// must set already in; ,
-								Self::set_already_in(batch, self.column, k);
-								refs.insert(k.clone(), RefInfo{ queue_refs: x - 1, in_archive: true });
-							}
-							Some( RefInfo{in_archive: true, ..} ) => {
-								// Invalid! Reinserted the same key twice.
-								warn!("Key {} inserted twice into same fork.", k);
-							}
+			if canon_id == &rlp.val_at::<H256>(0) {
+				// Collect keys to be removed. Canon block - remove the (enacted) deletes.
+				let deletes: Vec<H256> = rlp.val_at(2);
+				trace!(target: "jdb.ops", "  Expunging: {:?}", deletes);
+				Self::remove_keys(&deletes, &mut refs, batch, self.column, RemoveFrom::Archive, trace);
+
+					trace!(target: "jdb.ops", "  Finalising: {:?}", inserts);
+				for k in &inserts {
+					match refs.get(k).cloned() {
+						None => {
+							// [in archive] -> SHIFT remove -> SHIFT insert None->Some{queue_refs: 1, in_archive: true} -> TAKE remove Some{queue_refs: 1, in_archive: true}->None -> TAKE insert
+							// already expunged from the queue (which is allowed since the key is in the archive).
+							// leave well alone.
+						}
+						Some( RefInfo{queue_refs: 1, in_archive: false} ) => {
+							// just delete the refs entry.
+							refs.remove(k);
+						}
+						Some( RefInfo{queue_refs: x, in_archive: false} ) => {
+							// must set already in; ,
+							Self::set_already_in(batch, self.column, k);
+							refs.insert(k.clone(), RefInfo{ queue_refs: x - 1, in_archive: true });
+						}
+						Some( RefInfo{in_archive: true, ..} ) => {
+							// Invalid! Reinserted the same key twice.
+							warn!("Key {} inserted twice into same fork.", k);
 						}
 					}
-				} else {
-					// Collect keys to be removed. Non-canon block - remove the (reverted) inserts.
-					if trace {
-						trace!(target: "jdb.ops", "  Reverting: {:?}", inserts);
-					}
-					Self::remove_keys(&inserts, &mut refs, batch, self.column, RemoveFrom::Queue, trace);
 				}
+			} else {
+				// Collect keys to be removed. Non-canon block - remove the (reverted) inserts.
+				trace!(target: "jdb.ops", "  Reverting: {:?}", inserts);
+				Self::remove_keys(&inserts, &mut refs, batch, self.column, RemoveFrom::Queue, trace);
+			}
 
-				batch.delete(self.column, &last);
-				index += 1;
-			}
-			if trace {
-				trace!(target: "jdb", "EarlyMergeDB: delete journal for time #{}.{}, (canon was {})", end_era, index, canon_id);
-			}
+			batch.delete(self.column, &last);
+			index += 1;
 		}
 
-		if trace {
-			trace!(target: "jdb", "OK: {:?}", refs.clone());
-		}
+		trace!(target: "jdb", "EarlyMergeDB: delete journal for time #{}.{}, (canon was {})", end_era, index, canon_id);
+		trace!(target: "jdb", "OK: {:?}", refs.clone());
 
 		Ok(0)
 	}
@@ -828,7 +825,7 @@ mod tests {
 			let mut jdb = new_db(&dir);
 			// history is 1
 			let foo = jdb.insert(b"foo");
-			jdb.emplace(bar.clone(), b"bar".to_vec());
+			jdb.emplace(bar.clone(), DBValue::from_slice(b"bar"));
 			jdb.commit_batch(0, &b"0".sha3(), None).unwrap();
 			assert!(jdb.can_reconstruct_refs());
 			foo
@@ -1084,7 +1081,7 @@ mod tests {
 		let key = jdb.insert(b"dog");
 		jdb.inject_batch().unwrap();
 
-		assert_eq!(jdb.get(&key).unwrap(), b"dog");
+		assert_eq!(jdb.get(&key).unwrap(), DBValue::from_slice(b"dog"));
 		jdb.remove(&key);
 		jdb.inject_batch().unwrap();
 

util/src/journaldb/overlayrecentdb.rs

@@ -67,9 +67,10 @@ pub struct OverlayRecentDB {
 #[derive(PartialEq)]
 struct JournalOverlay {
 	backing_overlay: MemoryDB, // Nodes added in the history period
-	pending_overlay: H256FastMap<Bytes>, // Nodes being transfered from backing_overlay to backing db
+	pending_overlay: H256FastMap<DBValue>, // Nodes being transfered from backing_overlay to backing db
 	journal: HashMap<u64, Vec<JournalEntry>>,
 	latest_era: Option<u64>,
+	earliest_era: Option<u64>,
 }
 
 #[derive(PartialEq)]
@@ -123,10 +124,13 @@ impl OverlayRecentDB {
 	fn can_reconstruct_refs(&self) -> bool {
 		let reconstructed = Self::read_overlay(&self.backing, self.column);
 		let journal_overlay = self.journal_overlay.read();
-		*journal_overlay == reconstructed
+		journal_overlay.backing_overlay == reconstructed.backing_overlay &&
+		journal_overlay.pending_overlay == reconstructed.pending_overlay &&
+		journal_overlay.journal == reconstructed.journal &&
+		journal_overlay.latest_era == reconstructed.latest_era
 	}
 
-	fn payload(&self, key: &H256) -> Option<Bytes> {
+	fn payload(&self, key: &H256) -> Option<DBValue> {
 		self.backing.get(self.column, key).expect("Low-level database error. Some issue with your hard disk?")
 	}
 
@@ -135,6 +139,7 @@ impl OverlayRecentDB {
 		let mut overlay = MemoryDB::new();
 		let mut count = 0;
 		let mut latest_era = None;
+		let mut earliest_era = None;
 		if let Some(val) = db.get(col, &LATEST_ERA_KEY).expect("Low-level database error.") {
 			let mut era = decode::<u64>(&val);
 			latest_era = Some(era);
@@ -155,8 +160,8 @@ impl OverlayRecentDB {
 					let mut inserted_keys = Vec::new();
 					for r in insertions.iter() {
 						let k: H256 = r.val_at(0);
-						let v: Bytes = r.val_at(1);
-						overlay.emplace(to_short_key(&k), v);
+						let v = r.at(1).data();
+						overlay.emplace(to_short_key(&k), DBValue::from_slice(v));
 						inserted_keys.push(k);
 						count += 1;
 					}
@@ -166,6 +171,7 @@ impl OverlayRecentDB {
 						deletions: deletions,
 					});
 					index += 1;
+					earliest_era = Some(era);
 				};
 				if index == 0 || era == 0 {
 					break;
@@ -178,9 +184,12 @@ impl OverlayRecentDB {
 			backing_overlay: overlay,
 			pending_overlay: HashMap::default(),
 			journal: journal,
-			latest_era: latest_era }
+			latest_era: latest_era,
+			earliest_era: earliest_era,
+		}
 	}
 
+
 }
 
 #[inline]
@@ -214,100 +223,117 @@ impl JournalDB for OverlayRecentDB {
 
 	fn latest_era(&self) -> Option<u64> { self.journal_overlay.read().latest_era }
 
+	fn earliest_era(&self) -> Option<u64> { self.journal_overlay.read().earliest_era }
+
 	fn state(&self, key: &H256) -> Option<Bytes> {
 		let journal_overlay = self.journal_overlay.read();
 		let key = to_short_key(key);
 		journal_overlay.backing_overlay.get(&key).map(|v| v.to_vec())
-		.or_else(|| journal_overlay.pending_overlay.get(&key).cloned())
+		.or_else(|| journal_overlay.pending_overlay.get(&key).map(|d| d.clone().to_vec()))
 		.or_else(|| self.backing.get_by_prefix(self.column, &key[0..DB_PREFIX_LEN]).map(|b| b.to_vec()))
 	}
 
-	fn commit(&mut self, batch: &mut DBTransaction, now: u64, id: &H256, end: Option<(u64, H256)>) -> Result<u32, UtilError> {
-		// record new commit's details.
-		trace!("commit: #{} ({}), end era: {:?}", now, id, end);
+	fn journal_under(&mut self, batch: &mut DBTransaction, now: u64, id: &H256) -> Result<u32, UtilError> {
+		trace!(target: "journaldb", "entry: #{} ({})", now, id);
+
 		let mut journal_overlay = self.journal_overlay.write();
+
 		// flush previous changes
 		journal_overlay.pending_overlay.clear();
-		{
-			let mut r = RlpStream::new_list(3);
-			let mut tx = self.transaction_overlay.drain();
-			let inserted_keys: Vec<_> = tx.iter().filter_map(|(k, &(_, c))| if c > 0 { Some(k.clone()) } else { None }).collect();
-			let removed_keys: Vec<_> = tx.iter().filter_map(|(k, &(_, c))| if c < 0 { Some(k.clone()) } else { None }).collect();
-			// Increase counter for each inserted key no matter if the block is canonical or not.
-			let insertions = tx.drain().filter_map(|(k, (v, c))| if c > 0 { Some((k, v)) } else { None });
-			r.append(id);
-			r.begin_list(inserted_keys.len());
-			for (k, v) in insertions {
-				r.begin_list(2);
-				r.append(&k);
-				r.append(&v);
-				journal_overlay.backing_overlay.emplace(to_short_key(&k), v);
-			}
-			r.append(&removed_keys);
 
-			let mut k = RlpStream::new_list(3);
-			let index = journal_overlay.journal.get(&now).map_or(0, |j| j.len());
-			k.append(&now);
-			k.append(&index);
-			k.append(&&PADDING[..]);
-			batch.put_vec(self.column, &k.drain(), r.out());
-			if journal_overlay.latest_era.map_or(true, |e| now > e) {
-				batch.put_vec(self.column, &LATEST_ERA_KEY, encode(&now).to_vec());
-				journal_overlay.latest_era = Some(now);
-			}
-			journal_overlay.journal.entry(now).or_insert_with(Vec::new).push(JournalEntry { id: id.clone(), insertions: inserted_keys, deletions: removed_keys });
+		let mut r = RlpStream::new_list(3);
+		let mut tx = self.transaction_overlay.drain();
+		let inserted_keys: Vec<_> = tx.iter().filter_map(|(k, &(_, c))| if c > 0 { Some(k.clone()) } else { None }).collect();
+		let removed_keys: Vec<_> = tx.iter().filter_map(|(k, &(_, c))| if c < 0 { Some(k.clone()) } else { None }).collect();
+		let ops = inserted_keys.len() + removed_keys.len();
+
+		// Increase counter for each inserted key no matter if the block is canonical or not.
+		let insertions = tx.drain().filter_map(|(k, (v, c))| if c > 0 { Some((k, v)) } else { None });
+
+		r.append(id);
+		r.begin_list(inserted_keys.len());
+		for (k, v) in insertions {
+			r.begin_list(2);
+			r.append(&k);
+			r.append(&&*v);
+			journal_overlay.backing_overlay.emplace(to_short_key(&k), v);
+		}
+		r.append(&removed_keys);
+
+		let mut k = RlpStream::new_list(3);
+		let index = journal_overlay.journal.get(&now).map_or(0, |j| j.len());
+		k.append(&now);
+		k.append(&index);
+		k.append(&&PADDING[..]);
+		batch.put_vec(self.column, &k.drain(), r.out());
+		if journal_overlay.latest_era.map_or(true, |e| now > e) {
+			batch.put_vec(self.column, &LATEST_ERA_KEY, encode(&now).to_vec());
+			journal_overlay.latest_era = Some(now);
 		}
 
+		journal_overlay.journal.entry(now).or_insert_with(Vec::new).push(JournalEntry { id: id.clone(), insertions: inserted_keys, deletions: removed_keys });
+		Ok(ops as u32)
+	}
+
+	fn mark_canonical(&mut self, batch: &mut DBTransaction, end_era: u64, canon_id: &H256) -> Result<u32, UtilError> {
+		trace!(target: "journaldb", "canonical: #{} ({})", end_era, canon_id);
+
+		let mut journal_overlay = self.journal_overlay.write();
 		let journal_overlay = &mut *journal_overlay;
+
+		let mut ops = 0;
 		// apply old commits' details
-		if let Some((end_era, canon_id)) = end {
-			if let Some(ref mut records) = journal_overlay.journal.get_mut(&end_era) {
-				let mut canon_insertions: Vec<(H256, Bytes)> = Vec::new();
-				let mut canon_deletions: Vec<H256> = Vec::new();
-				let mut overlay_deletions: Vec<H256> = Vec::new();
-				let mut index = 0usize;
-				for mut journal in records.drain(..) {
-					//delete the record from the db
-					let mut r = RlpStream::new_list(3);
-					r.append(&end_era);
-					r.append(&index);
-					r.append(&&PADDING[..]);
-					batch.delete(self.column, &r.drain());
-					trace!("commit: Delete journal for time #{}.{}: {}, (canon was {}): +{} -{} entries", end_era, index, journal.id, canon_id, journal.insertions.len(), journal.deletions.len());
-					{
-						if canon_id == journal.id {
-							for h in &journal.insertions {
-								if let Some((d, rc)) = journal_overlay.backing_overlay.raw(&to_short_key(h)) {
-									if rc > 0 {
-										canon_insertions.push((h.clone(), d.to_owned())); //TODO: optimize this to avoid data copy
-									}
+		if let Some(ref mut records) = journal_overlay.journal.get_mut(&end_era) {
+			let mut canon_insertions: Vec<(H256, DBValue)> = Vec::new();
+			let mut canon_deletions: Vec<H256> = Vec::new();
+			let mut overlay_deletions: Vec<H256> = Vec::new();
+			let mut index = 0usize;
+			for mut journal in records.drain(..) {
+				//delete the record from the db
+				let mut r = RlpStream::new_list(3);
+				r.append(&end_era);
+				r.append(&index);
+				r.append(&&PADDING[..]);
+				batch.delete(self.column, &r.drain());
+				trace!(target: "journaldb", "Delete journal for time #{}.{}: {}, (canon was {}): +{} -{} entries", end_era, index, journal.id, canon_id, journal.insertions.len(), journal.deletions.len());
+				{
+					if *canon_id == journal.id {
+						for h in &journal.insertions {
+							if let Some((d, rc)) = journal_overlay.backing_overlay.raw(&to_short_key(h)) {
+								if rc > 0 {
+									canon_insertions.push((h.clone(), d)); //TODO: optimize this to avoid data copy
 								}
 							}
-							canon_deletions = journal.deletions;
 						}
-						overlay_deletions.append(&mut journal.insertions);
+						canon_deletions = journal.deletions;
 					}
-					index += 1;
+					overlay_deletions.append(&mut journal.insertions);
 				}
-				// apply canon inserts first
-				for (k, v) in canon_insertions {
-					batch.put(self.column, &k, &v);
-					journal_overlay.pending_overlay.insert(to_short_key(&k), v);
-				}
-				// update the overlay
-				for k in overlay_deletions {
-					journal_overlay.backing_overlay.remove_and_purge(&to_short_key(&k));
-				}
-				// apply canon deletions
-				for k in canon_deletions {
-					if !journal_overlay.backing_overlay.contains(&to_short_key(&k)) {
-						batch.delete(self.column, &k);
-					}
+				index += 1;
+			}
+
+			ops += canon_insertions.len();
+			ops += canon_deletions.len();
+
+			// apply canon inserts first
+			for (k, v) in canon_insertions {
+				batch.put(self.column, &k, &v);
+				journal_overlay.pending_overlay.insert(to_short_key(&k), v);
+			}
+			// update the overlay
+			for k in overlay_deletions {
+				journal_overlay.backing_overlay.remove_and_purge(&to_short_key(&k));
+			}
+			// apply canon deletions
+			for k in canon_deletions {
+				if !journal_overlay.backing_overlay.contains(&to_short_key(&k)) {
+					batch.delete(self.column, &k);
 				}
 			}
-			journal_overlay.journal.remove(&end_era);
 		}
-		Ok(0)
+		journal_overlay.journal.remove(&end_era);
+
+		Ok(ops as u32)
 	}
 
 	fn flush(&self) {
@@ -322,13 +348,13 @@ impl JournalDB for OverlayRecentDB {
 			match rc {
 				0 => {}
 				1 => {
-					if try!(self.backing.get(self.column, &key)).is_some() {
+					if cfg!(debug_assertions) && try!(self.backing.get(self.column, &key)).is_some() {
 						return Err(BaseDataError::AlreadyExists(key).into());
 					}
 					batch.put(self.column, &key, &value)
 				}
 				-1 => {
-					if try!(self.backing.get(self.column, &key)).is_none() {
+					if cfg!(debug_assertions) && try!(self.backing.get(self.column, &key)).is_none() {
 						return Err(BaseDataError::NegativelyReferencedHash(key).into());
 					}
 					batch.delete(self.column, &key)
@@ -360,32 +386,18 @@ impl HashDB for OverlayRecentDB {
 		ret
 	}
 
-	fn get(&self, key: &H256) -> Option<&[u8]> {
+	fn get(&self, key: &H256) -> Option<DBValue> {
 		let k = self.transaction_overlay.raw(key);
-		match k {
-			Some((d, rc)) if rc > 0 => Some(d),
-			_ => {
-				let v = {
-					let journal_overlay = self.journal_overlay.read();
-					let key = to_short_key(key);
-					journal_overlay.backing_overlay.get(&key).map(|v| v.to_vec())
-						.or_else(|| journal_overlay.pending_overlay.get(&key).cloned())
-				};
-				match v {
-					Some(x) => {
-						Some(self.transaction_overlay.denote(key, x).0)
-					}
-					_ => {
-						if let Some(x) = self.payload(key) {
-							Some(self.transaction_overlay.denote(key, x).0)
-						}
-						else {
-							None
-						}
-					}
-				}
-			}
+		if let Some((d, rc)) = k {
+			if rc > 0 { return Some(d) }
 		}
+		let v = {
+			let journal_overlay = self.journal_overlay.read();
+			let key = to_short_key(key);
+			journal_overlay.backing_overlay.get(&key)
+				.or_else(|| journal_overlay.pending_overlay.get(&key).cloned())
+		};
+		v.or_else(|| self.payload(key))
 	}
 
 	fn contains(&self, key: &H256) -> bool {
@@ -395,7 +407,7 @@ impl HashDB for OverlayRecentDB {
 	fn insert(&mut self, value: &[u8]) -> H256 {
 		self.transaction_overlay.insert(value)
 	}
-	fn emplace(&mut self, key: H256, value: Bytes) {
+	fn emplace(&mut self, key: H256, value: DBValue) {
 		self.transaction_overlay.emplace(key, value);
 	}
 	fn remove(&mut self, key: &H256) {
@@ -666,7 +678,7 @@ mod tests {
 			let mut jdb = new_db(&dir);
 			// history is 1
 			let foo = jdb.insert(b"foo");
-			jdb.emplace(bar.clone(), b"bar".to_vec());
+			jdb.emplace(bar.clone(), DBValue::from_slice(b"bar"));
 			jdb.commit_batch(0, &b"0".sha3(), None).unwrap();
 			assert!(jdb.can_reconstruct_refs());
 			foo
@@ -939,7 +951,7 @@ mod tests {
 		let key = jdb.insert(b"dog");
 		jdb.inject_batch().unwrap();
 
-		assert_eq!(jdb.get(&key).unwrap(), b"dog");
+		assert_eq!(jdb.get(&key).unwrap(), DBValue::from_slice(b"dog"));
 		jdb.remove(&key);
 		jdb.inject_batch().unwrap();
 

util/src/journaldb/refcounteddb.rs

@@ -34,6 +34,17 @@ use std::env;
 /// write operations out to disk. Unlike `OverlayDB`, `remove()` operations do not take effect
 /// immediately. Rather some age (based on a linear but arbitrary metric) must pass before
 /// the removals actually take effect.
+///
+/// journal format:
+/// [era, 0] => [ id, [insert_0, ...], [remove_0, ...] ]
+/// [era, 1] => [ id, [insert_0, ...], [remove_0, ...] ]
+/// [era, n] => [ ... ]
+///
+/// when we make a new commit, we journal the inserts and removes.
+/// for each end_era that we journaled that we are no passing by,
+/// we remove all of its removes assuming it is canonical and all
+/// of its inserts otherwise.
+// TODO: store last_era, reclaim_period.
 pub struct RefCountedDB {
 	forward: OverlayDB,
 	backing: Arc<Database>,
@@ -72,10 +83,10 @@ impl RefCountedDB {
 
 impl HashDB for RefCountedDB {
 	fn keys(&self) -> HashMap<H256, i32> { self.forward.keys() }
-	fn get(&self, key: &H256) -> Option<&[u8]> { self.forward.get(key) }
+	fn get(&self, key: &H256) -> Option<DBValue> { self.forward.get(key) }
 	fn contains(&self, key: &H256) -> bool { self.forward.contains(key) }
 	fn insert(&mut self, value: &[u8]) -> H256 { let r = self.forward.insert(value); self.inserts.push(r.clone()); r }
-	fn emplace(&mut self, key: H256, value: Bytes) { self.inserts.push(key.clone()); self.forward.emplace(key, value); }
+	fn emplace(&mut self, key: H256, value: DBValue) { self.inserts.push(key.clone()); self.forward.emplace(key, value); }
 	fn remove(&mut self, key: &H256) { self.removes.push(key.clone()); }
 }
 
@@ -109,77 +120,66 @@ impl JournalDB for RefCountedDB {
 		self.backing.get_by_prefix(self.column, &id[0..DB_PREFIX_LEN]).map(|b| b.to_vec())
 	}
 
-	fn commit(&mut self, batch: &mut DBTransaction, now: u64, id: &H256, end: Option<(u64, H256)>) -> Result<u32, UtilError> {
-		// journal format:
-		// [era, 0] => [ id, [insert_0, ...], [remove_0, ...] ]
-		// [era, 1] => [ id, [insert_0, ...], [remove_0, ...] ]
-		// [era, n] => [ ... ]
-
-		// TODO: store last_era, reclaim_period.
-
-		// when we make a new commit, we journal the inserts and removes.
-		// for each end_era that we journaled that we are no passing by,
-		// we remove all of its removes assuming it is canonical and all
-		// of its inserts otherwise.
-
+	fn journal_under(&mut self, batch: &mut DBTransaction, now: u64, id: &H256) -> Result<u32, UtilError> {
 		// record new commit's details.
-		{
-			let mut index = 0usize;
-			let mut last;
+		let mut index = 0usize;
+		let mut last;
 
-			while try!(self.backing.get(self.column, {
+		while try!(self.backing.get(self.column, {
+			let mut r = RlpStream::new_list(3);
+			r.append(&now);
+			r.append(&index);
+			r.append(&&PADDING[..]);
+			last = r.drain();
+			&last
+		})).is_some() {
+			index += 1;
+		}
+
+		let mut r = RlpStream::new_list(3);
+		r.append(id);
+		r.append(&self.inserts);
+		r.append(&self.removes);
+		batch.put(self.column, &last, r.as_raw());
+
+		let ops = self.inserts.len() + self.removes.len();
+
+		trace!(target: "rcdb", "new journal for time #{}.{} => {}: inserts={:?}, removes={:?}", now, index, id, self.inserts, self.removes);
+
+		self.inserts.clear();
+		self.removes.clear();
+
+		if self.latest_era.map_or(true, |e| now > e) {
+			batch.put(self.column, &LATEST_ERA_KEY, &encode(&now));
+			self.latest_era = Some(now);
+		}
+
+		Ok(ops as u32)
+	}
+
+	fn mark_canonical(&mut self, batch: &mut DBTransaction, end_era: u64, canon_id: &H256) -> Result<u32, UtilError> {
+		// apply old commits' details
+		let mut index = 0usize;
+		let mut last;
+		while let Some(rlp_data) = {
+			try!(self.backing.get(self.column, {
 				let mut r = RlpStream::new_list(3);
-				r.append(&now);
+				r.append(&end_era);
 				r.append(&index);
 				r.append(&&PADDING[..]);
 				last = r.drain();
 				&last
-			})).is_some() {
-				index += 1;
-			}
-
-			let mut r = RlpStream::new_list(3);
-			r.append(id);
-			r.append(&self.inserts);
-			r.append(&self.removes);
-			batch.put(self.column, &last, r.as_raw());
-
-			trace!(target: "rcdb", "new journal for time #{}.{} => {}: inserts={:?}, removes={:?}", now, index, id, self.inserts, self.removes);
-
-			self.inserts.clear();
-			self.removes.clear();
-
-			if self.latest_era.map_or(true, |e| now > e) {
-				batch.put(self.column, &LATEST_ERA_KEY, &encode(&now));
-				self.latest_era = Some(now);
-			}
-		}
-
-		// apply old commits' details
-		if let Some((end_era, canon_id)) = end {
-			let mut index = 0usize;
-			let mut last;
-			while let Some(rlp_data) = {
-//				trace!(target: "rcdb", "checking for journal #{}.{}", end_era, index);
-				try!(self.backing.get(self.column, {
-					let mut r = RlpStream::new_list(3);
-					r.append(&end_era);
-					r.append(&index);
-					r.append(&&PADDING[..]);
-					last = r.drain();
-					&last
-				}))
-			} {
-				let rlp = Rlp::new(&rlp_data);
-				let our_id: H256 = rlp.val_at(0);
-				let to_remove: Vec<H256> = rlp.val_at(if canon_id == our_id {2} else {1});
-				trace!(target: "rcdb", "delete journal for time #{}.{}=>{}, (canon was {}): deleting {:?}", end_era, index, our_id, canon_id, to_remove);
-				for i in &to_remove {
-					self.forward.remove(i);
-				}
-				batch.delete(self.column, &last);
-				index += 1;
+			}))
+		} {
+			let rlp = Rlp::new(&rlp_data);
+			let our_id: H256 = rlp.val_at(0);
+			let to_remove: Vec<H256> = rlp.val_at(if *canon_id == our_id {2} else {1});
+			trace!(target: "rcdb", "delete journal for time #{}.{}=>{}, (canon was {}): deleting {:?}", end_era, index, our_id, canon_id, to_remove);
+			for i in &to_remove {
+				self.forward.remove(i);
 			}
+			batch.delete(self.column, &last);
+			index += 1;
 		}
 
 		let r = try!(self.forward.commit_to_batch(batch));
@@ -326,7 +326,7 @@ mod tests {
 		let key = jdb.insert(b"dog");
 		jdb.inject_batch().unwrap();
 
-		assert_eq!(jdb.get(&key).unwrap(), b"dog");
+		assert_eq!(jdb.get(&key).unwrap(), DBValue::from_slice(b"dog"));
 		jdb.remove(&key);
 		jdb.inject_batch().unwrap();
 

util/src/journaldb/traits.rs

@@ -32,12 +32,18 @@ pub trait JournalDB: HashDB {
 	/// Check if this database has any commits
 	fn is_empty(&self) -> bool;
 
+	/// Get the earliest era in the DB. None if there isn't yet any data in there.
+	fn earliest_era(&self) -> Option<u64> { None }
+
 	/// Get the latest era in the DB. None if there isn't yet any data in there.
 	fn latest_era(&self) -> Option<u64>;
 
-	/// Commit all recent insert operations and canonical historical commits' removals from the
-	/// old era to the backing database, reverting any non-canonical historical commit's inserts.
-	fn commit(&mut self, batch: &mut DBTransaction, now: u64, id: &H256, end: Option<(u64, H256)>) -> Result<u32, UtilError>;
+	/// Journal recent database operations as being associated with a given era and id.
+	// TODO: give the overlay to this function so journaldbs don't manage the overlays themeselves.
+	fn journal_under(&mut self, batch: &mut DBTransaction, now: u64, id: &H256) -> Result<u32, UtilError>;
+
+	/// Mark a given block as canonical, indicating that competing blocks' states may be pruned out.
+	fn mark_canonical(&mut self, batch: &mut DBTransaction, era: u64, id: &H256) -> Result<u32, UtilError>;
 
 	/// Commit all queued insert and delete operations without affecting any journalling -- this requires that all insertions
 	/// and deletions are indeed canonical and will likely lead to an invalid database if that assumption is violated.
@@ -68,8 +74,13 @@ pub trait JournalDB: HashDB {
 	#[cfg(test)]
 	fn commit_batch(&mut self, now: u64, id: &H256, end: Option<(u64, H256)>) -> Result<u32, UtilError> {
 		let mut batch = self.backing().transaction();
-		let res = try!(self.commit(&mut batch, now, id, end));
-		let result = self.backing().write(batch).map(|_| res).map_err(Into::into);
+		let mut ops = try!(self.journal_under(&mut batch, now, id));
+
+		if let Some((end_era, canon_id)) = end {
+			ops += try!(self.mark_canonical(&mut batch, end_era, &canon_id));
+		}
+
+		let result = self.backing().write(batch).map(|_| ops).map_err(Into::into);
 		self.flush();
 		result
 	}

util/src/kvdb.rs

@@ -21,9 +21,16 @@ use common::*;
 use elastic_array::*;
 use std::default::Default;
 use std::path::PathBuf;
+use hashdb::DBValue;
 use rlp::{UntrustedRlp, RlpType, View, Compressible};
 use rocksdb::{DB, Writable, WriteBatch, WriteOptions, IteratorMode, DBIterator,
-	Options, DBCompactionStyle, BlockBasedOptions, Direction, Cache, Column};
+	Options, DBCompactionStyle, BlockBasedOptions, Direction, Cache, Column, ReadOptions};
+#[cfg(target_os = "linux")]
+use regex::Regex;
+#[cfg(target_os = "linux")]
+use std::process::Command;
+#[cfg(target_os = "linux")]
+use std::fs::File;
 
 const DB_BACKGROUND_FLUSHES: i32 = 2;
 const DB_BACKGROUND_COMPACTIONS: i32 = 2;
@@ -37,12 +44,12 @@ enum DBOp {
 	Insert {
 		col: Option<u32>,
 		key: ElasticArray32<u8>,
-		value: Bytes,
+		value: DBValue,
 	},
 	InsertCompressed {
 		col: Option<u32>,
 		key: ElasticArray32<u8>,
-		value: Bytes,
+		value: DBValue,
 	},
 	Delete {
 		col: Option<u32>,
@@ -65,7 +72,7 @@ impl DBTransaction {
 		self.ops.push(DBOp::Insert {
 			col: col,
 			key: ekey,
-			value: value.to_vec(),
+			value: DBValue::from_slice(value),
 		});
 	}
 
@@ -76,7 +83,7 @@ impl DBTransaction {
 		self.ops.push(DBOp::Insert {
 			col: col,
 			key: ekey,
-			value: value,
+			value: DBValue::from_vec(value),
 		});
 	}
 
@@ -88,7 +95,7 @@ impl DBTransaction {
 		self.ops.push(DBOp::InsertCompressed {
 			col: col,
 			key: ekey,
-			value: value,
+			value: DBValue::from_vec(value),
 		});
 	}
 
@@ -104,13 +111,13 @@ impl DBTransaction {
 }
 
 enum KeyState {
-	Insert(Bytes),
-	InsertCompressed(Bytes),
+	Insert(DBValue),
+	InsertCompressed(DBValue),
 	Delete,
 }
 
 /// Compaction profile for the database settings
-#[derive(Clone, Copy)]
+#[derive(Clone, Copy, PartialEq, Debug)]
 pub struct CompactionProfile {
 	/// L0-L1 target file size
 	pub initial_file_size: u64,
@@ -123,16 +130,73 @@ pub struct CompactionProfile {
 impl Default for CompactionProfile {
 	/// Default profile suitable for most storage
 	fn default() -> CompactionProfile {
+		CompactionProfile::ssd()
+	}
+}
+
+/// Given output of df command return Linux rotational flag file path.
+#[cfg(target_os = "linux")]
+pub fn rotational_from_df_output(df_out: Vec<u8>) -> Option<PathBuf> {
+	str::from_utf8(df_out.as_slice())
+		.ok()
+		// Get the drive name.
+		.and_then(|df_str| Regex::new(r"/dev/(sd[:alpha:]{1,2})")
+			.ok()
+			.and_then(|re| re.captures(df_str))
+			.and_then(|captures| captures.at(1)))
+		// Generate path e.g. /sys/block/sda/queue/rotational
+		.map(|drive_path| {
+			let mut p = PathBuf::from("/sys/block");
+			p.push(drive_path);
+			p.push("queue/rotational");
+			p
+		})
+}
+
+impl CompactionProfile {
+	/// Attempt to determine the best profile automatically, only Linux for now.
+	#[cfg(target_os = "linux")]
+	pub fn auto(db_path: &Path) -> CompactionProfile {
+		let hdd_check_file = db_path
+			.to_str()
+			.and_then(|path_str| Command::new("df").arg(path_str).output().ok())
+			.and_then(|df_res| match df_res.status.success() {
+				true => Some(df_res.stdout),
+				false => None,
+			})
+			.and_then(rotational_from_df_output);
+		// Read out the file and match compaction profile.
+		if let Some(hdd_check) = hdd_check_file {
+			if let Ok(mut file) = File::open(hdd_check.as_path()) {
+				let mut buffer = [0; 1];
+				if file.read_exact(&mut buffer).is_ok() {
+					// 0 means not rotational.
+					if buffer == [48] { return Self::ssd(); }
+					// 1 means rotational.
+					if buffer == [49] { return Self::hdd(); }
+				}
+			}
+		}
+		// Fallback if drive type was not determined.
+		Self::default()
+	}
+
+	/// Just default for other platforms.
+	#[cfg(not(target_os = "linux"))]
+	pub fn auto(_db_path: &Path) -> CompactionProfile {
+		Self::default()
+	}
+
+	/// Default profile suitable for SSD storage
+	pub fn ssd() -> CompactionProfile {
 		CompactionProfile {
 			initial_file_size: 32 * 1024 * 1024,
 			file_size_multiplier: 2,
 			write_rate_limit: None,
 		}
 	}
-}
 
-impl CompactionProfile {
-	/// Slow hdd compaction profile
+	/// Slow HDD compaction profile
 	pub fn hdd() -> CompactionProfile {
 		CompactionProfile {
 			initial_file_size: 192 * 1024 * 1024,
@@ -143,12 +207,12 @@ impl CompactionProfile {
 }
 
 /// Database configuration
-#[derive(Clone, Copy)]
+#[derive(Clone)]
 pub struct DatabaseConfig {
 	/// Max number of open files.
 	pub max_open_files: i32,
-	/// Cache-size
-	pub cache_size: Option<usize>,
+	/// Cache sizes (in MiB) for specific columns.
+	pub cache_sizes: HashMap<Option<u32>, usize>,
 	/// Compaction profile
 	pub compaction: CompactionProfile,
 	/// Set number of columns
@@ -159,17 +223,23 @@ pub struct DatabaseConfig {
 
 impl DatabaseConfig {
 	/// Create new `DatabaseConfig` with default parameters and specified set of columns.
+	/// Note that cache sizes must be explicitly set.
 	pub fn with_columns(columns: Option<u32>) -> Self {
 		let mut config = Self::default();
 		config.columns = columns;
 		config
 	}
+
+	/// Set the column cache size in MiB.
+	pub fn set_cache(&mut self, col: Option<u32>, size: usize) {
+		self.cache_sizes.insert(col, size);
+	}
 }
 
 impl Default for DatabaseConfig {
 	fn default() -> DatabaseConfig {
 		DatabaseConfig {
-			cache_size: None,
+			cache_sizes: HashMap::new(),
 			max_open_files: 512,
 			compaction: CompactionProfile::default(),
 			columns: None,
@@ -201,8 +271,15 @@ pub struct Database {
 	db: RwLock<Option<DBAndColumns>>,
 	config: DatabaseConfig,
 	write_opts: WriteOptions,
-	overlay: RwLock<Vec<HashMap<ElasticArray32<u8>, KeyState>>>,
+	read_opts: ReadOptions,
 	path: String,
+	// Dirty values added with `write_buffered`. Cleaned on `flush`.
+	overlay: RwLock<Vec<HashMap<ElasticArray32<u8>, KeyState>>>,
+	// Values currently being flushed. Cleared when `flush` completes.
+	flushing: RwLock<Vec<HashMap<ElasticArray32<u8>, KeyState>>>,
+	// Prevents concurrent flushes.
+	// Value indicates if a flush is in progress.
+	flushing_lock: Mutex<bool>,
 }
 
 impl Database {
@@ -213,11 +290,15 @@ impl Database {
 
 	/// Open database file. Creates if it does not exist.
 	pub fn open(config: &DatabaseConfig, path: &str) -> Result<Database, String> {
+		// default cache size for columns not specified.
+		const DEFAULT_CACHE: usize = 2;
+
 		let mut opts = Options::new();
 		if let Some(rate_limit) = config.compaction.write_rate_limit {
 			try!(opts.set_parsed_options(&format!("rate_limiter_bytes_per_sec={}", rate_limit)));
 		}
 		try!(opts.set_parsed_options(&format!("max_total_wal_size={}", 64 * 1024 * 1024)));
+		try!(opts.set_parsed_options("verify_checksums_in_compaction=0"));
 		opts.set_max_open_files(config.max_open_files);
 		opts.create_if_missing(true);
 		opts.set_use_fsync(false);
@@ -232,17 +313,22 @@ impl Database {
 
 		let mut cf_options = Vec::with_capacity(config.columns.unwrap_or(0) as usize);
 
-		for _ in 0 .. config.columns.unwrap_or(0) {
+		for col in 0 .. config.columns.unwrap_or(0) {
 			let mut opts = Options::new();
 			opts.set_compaction_style(DBCompactionStyle::DBUniversalCompaction);
 			opts.set_target_file_size_base(config.compaction.initial_file_size);
 			opts.set_target_file_size_multiplier(config.compaction.file_size_multiplier);
-			if let Some(cache_size) = config.cache_size {
+
+			let col_opt = config.columns.map(|_| col);
+
+			{
+				let cache_size = config.cache_sizes.get(&col_opt).cloned().unwrap_or(DEFAULT_CACHE);
 				let mut block_opts = BlockBasedOptions::new();
-				// all goes to read cache
+				// all goes to read cache.
 				block_opts.set_cache(Cache::new(cache_size * 1024 * 1024));
 				opts.set_block_based_table_factory(&block_opts);
 			}
+
 			cf_options.push(opts);
 		}
 
@@ -250,6 +336,8 @@ impl Database {
 		if !config.wal {
 			write_opts.disable_wal(true);
 		}
+		let mut read_opts = ReadOptions::new();
+		read_opts.set_verify_checksums(false);
 
 		let mut cfs: Vec<Column> = Vec::new();
 		let db = match config.columns {
@@ -258,7 +346,8 @@ impl Database {
 				let cfnames: Vec<&str> = cfnames.iter().map(|n| n as &str).collect();
 				match DB::open_cf(&opts, path, &cfnames, &cf_options) {
 					Ok(db) => {
-						cfs = cfnames.iter().map(|n| db.cf_handle(n).unwrap()).collect();
+						cfs = cfnames.iter().map(|n| db.cf_handle(n)
+							.expect("rocksdb opens a cf_handle for each cfname; qed")).collect();
 						assert!(cfs.len() == columns as usize);
 						Ok(db)
 					}
@@ -266,7 +355,7 @@ impl Database {
 						// retry and create CFs
 						match DB::open_cf(&opts, path, &[], &[]) {
 							Ok(mut db) => {
-								cfs = cfnames.iter().enumerate().map(|(i, n)| db.create_cf(n, &cf_options[i]).unwrap()).collect();
+								cfs = try!(cfnames.iter().enumerate().map(|(i, n)| db.create_cf(n, &cf_options[i])).collect());
 								Ok(db)
 							},
 							err @ Err(_) => err,
@@ -292,7 +381,10 @@ impl Database {
 			config: config.clone(),
 			write_opts: write_opts,
 			overlay: RwLock::new((0..(num_cols + 1)).map(|_| HashMap::new()).collect()),
+			flushing: RwLock::new((0..(num_cols + 1)).map(|_| HashMap::new()).collect()),
+			flushing_lock: Mutex::new((false)),
 			path: path.to_owned(),
+			read_opts: read_opts,
 		})
 	}
 
@@ -328,48 +420,66 @@ impl Database {
 		};
 	}
 
-	/// Commit buffered changes to database.
-	pub fn flush(&self) -> Result<(), String> {
+	/// Commit buffered changes to database. Must be called under `flush_lock`
+	fn write_flushing_with_lock(&self, _lock: &mut MutexGuard<bool>) -> Result<(), String> {
 		match *self.db.read() {
 			Some(DBAndColumns { ref db, ref cfs }) => {
 				let batch = WriteBatch::new();
-				let mut overlay = self.overlay.write();
-
-				for (c, column) in overlay.iter_mut().enumerate() {
-					let column_data = mem::replace(column, HashMap::new());
-					for (key, state) in column_data.into_iter() {
-						match state {
-							KeyState::Delete => {
-								if c > 0 {
-									try!(batch.delete_cf(cfs[c - 1], &key));
-								} else {
-									try!(batch.delete(&key));
-								}
-							},
-							KeyState::Insert(value) => {
-								if c > 0 {
-									try!(batch.put_cf(cfs[c - 1], &key, &value));
-								} else {
-									try!(batch.put(&key, &value));
-								}
-							},
-							KeyState::InsertCompressed(value) => {
-								let compressed = UntrustedRlp::new(&value).compress(RlpType::Blocks);
-								if c > 0 {
-									try!(batch.put_cf(cfs[c - 1], &key, &compressed));
-								} else {
-									try!(batch.put(&key, &value));
+				mem::swap(&mut *self.overlay.write(), &mut *self.flushing.write());
+				{
+					for (c, column) in self.flushing.read().iter().enumerate() {
+						for (ref key, ref state) in column.iter() {
+							match **state {
+								KeyState::Delete => {
+									if c > 0 {
+										try!(batch.delete_cf(cfs[c - 1], &key));
+									} else {
+										try!(batch.delete(&key));
+									}
+								},
+								KeyState::Insert(ref value) => {
+									if c > 0 {
+										try!(batch.put_cf(cfs[c - 1], &key, value));
+									} else {
+										try!(batch.put(&key, &value));
+									}
+								},
+								KeyState::InsertCompressed(ref value) => {
+									let compressed = UntrustedRlp::new(&value).compress(RlpType::Blocks);
+									if c > 0 {
+										try!(batch.put_cf(cfs[c - 1], &key, &compressed));
+									} else {
+										try!(batch.put(&key, &value));
+									}
 								}
 							}
 						}
 					}
 				}
-				db.write_opt(batch, &self.write_opts)
+				try!(db.write_opt(batch, &self.write_opts));
+				for column in self.flushing.write().iter_mut() {
+					column.clear();
+				}
+				Ok(())
 			},
 			None => Err("Database is closed".to_owned())
 		}
 	}
 
+	/// Commit buffered changes to database.
+	pub fn flush(&self) -> Result<(), String> {
+		let mut lock = self.flushing_lock.lock();
+		// If RocksDB batch allocation fails the thread gets terminated and the lock is released.
+		// The value inside the lock is used to detect that.
+		if *lock {
+			// This can only happen if another flushing thread is terminated unexpectedly.
+			return Err("Database write failure. Running low on memory perhaps?".to_owned());
+		}
+		*lock = true;
+		let result = self.write_flushing_with_lock(&mut lock);
+		*lock = false;
+		result
+	}
 
 	/// Commit transaction to database.
 	pub fn write(&self, tr: DBTransaction) -> Result<(), String> {
@@ -398,7 +508,7 @@ impl Database {
 	}
 
 	/// Get value by key.
-	pub fn get(&self, col: Option<u32>, key: &[u8]) -> Result<Option<Bytes>, String> {
+	pub fn get(&self, col: Option<u32>, key: &[u8]) -> Result<Option<DBValue>, String> {
 		match *self.db.read() {
 			Some(DBAndColumns { ref db, ref cfs }) => {
 				let overlay = &self.overlay.read()[Self::to_overlay_column(col)];
@@ -406,9 +516,16 @@ impl Database {
 					Some(&KeyState::Insert(ref value)) | Some(&KeyState::InsertCompressed(ref value)) => Ok(Some(value.clone())),
 					Some(&KeyState::Delete) => Ok(None),
 					None => {
-						col.map_or_else(
-							|| db.get(key).map(|r| r.map(|v| v.to_vec())),
-							|c| db.get_cf(cfs[c as usize], key).map(|r| r.map(|v| v.to_vec())))
+						let flushing = &self.flushing.read()[Self::to_overlay_column(col)];
+						match flushing.get(key) {
+							Some(&KeyState::Insert(ref value)) | Some(&KeyState::InsertCompressed(ref value)) => Ok(Some(value.clone())),
+							Some(&KeyState::Delete) => Ok(None),
+							None => {
+								col.map_or_else(
+									|| db.get_opt(key, &self.read_opts).map(|r| r.map(|v| DBValue::from_slice(&v))),
+									|c| db.get_cf_opt(cfs[c as usize], key, &self.read_opts).map(|r| r.map(|v| DBValue::from_slice(&v))))
+							},
+						}
 					},
 				}
 			},
@@ -421,8 +538,9 @@ impl Database {
 	pub fn get_by_prefix(&self, col: Option<u32>, prefix: &[u8]) -> Option<Box<[u8]>> {
 		match *self.db.read() {
 			Some(DBAndColumns { ref db, ref cfs }) => {
-				let mut iter = col.map_or_else(|| db.iterator(IteratorMode::From(prefix, Direction::Forward)),
-					|c| db.iterator_cf(cfs[c as usize], IteratorMode::From(prefix, Direction::Forward)).unwrap());
+				let mut iter = col.map_or_else(|| db.iterator_opt(IteratorMode::From(prefix, Direction::Forward), &self.read_opts),
+					|c| db.iterator_cf_opt(cfs[c as usize], IteratorMode::From(prefix, Direction::Forward), &self.read_opts)
+						.expect("iterator params are valid; qed"));
 				match iter.next() {
 					// TODO: use prefix_same_as_start read option (not availabele in C API currently)
 					Some((k, v)) => if k[0 .. prefix.len()] == prefix[..] { Some(v) } else { None },
@@ -438,8 +556,9 @@ impl Database {
 		//TODO: iterate over overlay
 		match *self.db.read() {
 			Some(DBAndColumns { ref db, ref cfs }) => {
-				col.map_or_else(|| DatabaseIterator { iter: db.iterator(IteratorMode::Start) },
-					|c| DatabaseIterator { iter: db.iterator_cf(cfs[c as usize], IteratorMode::Start).unwrap() })
+				col.map_or_else(|| DatabaseIterator { iter: db.iterator_opt(IteratorMode::Start, &self.read_opts) },
+					|c| DatabaseIterator { iter: db.iterator_cf_opt(cfs[c as usize], IteratorMode::Start, &self.read_opts)
+						.expect("iterator params are valid; qed") })
 			},
 			None => panic!("Not supported yet") //TODO: return an empty iterator or change return type
 		}
@@ -449,6 +568,7 @@ impl Database {
 	fn close(&self) {
 		*self.db.write() = None;
 		self.overlay.write().clear();
+		self.flushing.write().clear();
 	}
 
 	/// Restore the database from a copy at given path.
@@ -488,10 +608,18 @@ impl Database {
 		let db = try!(Self::open(&self.config, &self.path));
 		*self.db.write() = mem::replace(&mut *db.db.write(), None);
 		*self.overlay.write() = mem::replace(&mut *db.overlay.write(), Vec::new());
+		*self.flushing.write() = mem::replace(&mut *db.flushing.write(), Vec::new());
 		Ok(())
 	}
 }
 
+impl Drop for Database {
+	fn drop(&mut self) {
+		// write all buffered changes if we can.
+		let _ = self.flush();
+	}
+}
+
 #[cfg(test)]
 mod tests {
 	use hash::*;
@@ -558,4 +686,14 @@ mod tests {
 		let _ = Database::open_default(path.as_path().to_str().unwrap()).unwrap();
 		test_db(&DatabaseConfig::default());
 	}
+
+	#[test]
+	#[cfg(target_os = "linux")]
+	fn df_to_rotational() {
+		use std::path::PathBuf;
+		// Example df output.
+		let example_df = vec![70, 105, 108, 101, 115, 121, 115, 116, 101, 109, 32, 32, 32, 32, 32, 49, 75, 45, 98, 108, 111, 99, 107, 115, 32, 32, 32, 32, 32, 85, 115, 101, 100, 32, 65, 118, 97, 105, 108, 97, 98, 108, 101, 32, 85, 115, 101, 37, 32, 77, 111, 117, 110, 116, 101, 100, 32, 111, 110, 10, 47, 100, 101, 118, 47, 115, 100, 97, 49, 32, 32, 32, 32, 32, 32, 32, 54, 49, 52, 48, 57, 51, 48, 48, 32, 51, 56, 56, 50, 50, 50, 51, 54, 32, 32, 49, 57, 52, 52, 52, 54, 49, 54, 32, 32, 54, 55, 37, 32, 47, 10];
+		let expected_output = Some(PathBuf::from("/sys/block/sda/queue/rotational"));
+		assert_eq!(rotational_from_df_output(example_df), expected_output);
+	}
 }

util/src/lib.rs

@@ -104,6 +104,7 @@ extern crate parking_lot;
 extern crate ansi_term;
 extern crate tiny_keccak;
 extern crate rlp;
+extern crate regex;
 
 #[macro_use]
 extern crate heapsize;
@@ -147,7 +148,7 @@ mod timer;
 pub use common::*;
 pub use misc::*;
 pub use hashdb::*;
-pub use memorydb::*;
+pub use memorydb::MemoryDB;
 pub use overlaydb::*;
 pub use journaldb::JournalDB;
 pub use triehash::*;

util/src/memorydb.rs

@@ -24,8 +24,6 @@ use hashdb::*;
 use heapsize::*;
 use std::mem;
 use std::collections::HashMap;
-
-const STATIC_NULL_RLP: (&'static [u8], i32) = (&[0x80; 1], 1);
 use std::collections::hash_map::Entry;
 
 /// Reference-counted memory-based `HashDB` implementation.
@@ -73,8 +71,8 @@ use std::collections::hash_map::Entry;
 /// ```
 #[derive(Default, Clone, PartialEq)]
 pub struct MemoryDB {
-	data: H256FastMap<(Bytes, i32)>,
-	aux: HashMap<Bytes, Bytes>,
+	data: H256FastMap<(DBValue, i32)>,
+	aux: HashMap<Bytes, DBValue>,
 }
 
 impl MemoryDB {
@@ -116,12 +114,12 @@ impl MemoryDB {
 	}
 
 	/// Return the internal map of hashes to data, clearing the current state.
-	pub fn drain(&mut self) -> H256FastMap<(Bytes, i32)> {
+	pub fn drain(&mut self) -> H256FastMap<(DBValue, i32)> {
 		mem::replace(&mut self.data, H256FastMap::default())
 	}
 
 	/// Return the internal map of auxiliary data, clearing the current state.
-	pub fn drain_aux(&mut self) -> HashMap<Bytes, Bytes> {
+	pub fn drain_aux(&mut self) -> HashMap<Bytes, DBValue> {
 		mem::replace(&mut self.aux, HashMap::new())
 	}
 
@@ -130,25 +128,11 @@ impl MemoryDB {
 	///
 	/// Even when Some is returned, the data is only guaranteed to be useful
 	/// when the refs > 0.
-	pub fn raw(&self, key: &H256) -> Option<(&[u8], i32)> {
+	pub fn raw(&self, key: &H256) -> Option<(DBValue, i32)> {
 		if key == &SHA3_NULL_RLP {
-			return Some(STATIC_NULL_RLP.clone());
+			return Some((DBValue::from_slice(&NULL_RLP_STATIC), 1));
 		}
-		self.data.get(key).map(|&(ref val, rc)| (&val[..], rc))
-	}
-
-	/// Denote than an existing value has the given key. Used when a key gets removed without
-	/// a prior insert and thus has a negative reference with no value.
-	///
-	/// May safely be called even if the key's value is known, in which case it will be a no-op.
-	pub fn denote(&self, key: &H256, value: Bytes) -> (&[u8], i32) {
-		if self.raw(key) == None {
-			unsafe {
-				let p = &self.data as *const H256FastMap<(Bytes, i32)> as *mut H256FastMap<(Bytes, i32)>;
-				(*p).insert(key.clone(), (value, 0));
-			}
-		}
-		self.raw(key).unwrap()
+		self.data.get(key).cloned()
 	}
 
 	/// Returns the size of allocated heap memory
@@ -170,7 +154,7 @@ impl MemoryDB {
 					entry.get_mut().1 -= 1;
 				},
 			Entry::Vacant(entry) => {
-				entry.insert((Bytes::new(), -1));
+				entry.insert((DBValue::new(), -1));
 			}
 		}
 	}
@@ -197,13 +181,13 @@ impl MemoryDB {
 static NULL_RLP_STATIC: [u8; 1] = [0x80; 1];
 
 impl HashDB for MemoryDB {
-	fn get(&self, key: &H256) -> Option<&[u8]> {
+	fn get(&self, key: &H256) -> Option<DBValue> {
 		if key == &SHA3_NULL_RLP {
-			return Some(&NULL_RLP_STATIC);
+			return Some(DBValue::from_slice(&NULL_RLP_STATIC));
 		}
 
 		match self.data.get(key) {
-			Some(&(ref d, rc)) if rc > 0 => Some(d),
+			Some(&(ref d, rc)) if rc > 0 => Some(d.clone()),
 			_ => None
 		}
 	}
@@ -230,20 +214,20 @@ impl HashDB for MemoryDB {
 		let key = value.sha3();
 		if match self.data.get_mut(&key) {
 			Some(&mut (ref mut old_value, ref mut rc @ -0x80000000i32 ... 0)) => {
-				*old_value = value.into();
+				*old_value = DBValue::from_slice(value);
 				*rc += 1;
 				false
 			},
 			Some(&mut (_, ref mut x)) => { *x += 1; false } ,
 			None => true,
 		}{	// ... None falls through into...
-			self.data.insert(key.clone(), (value.into(), 1));
+			self.data.insert(key.clone(), (DBValue::from_slice(value), 1));
 		}
 		key
 	}
 
-	fn emplace(&mut self, key: H256, value: Bytes) {
-		if value == &NULL_RLP {
+	fn emplace(&mut self, key: H256, value: DBValue) {
+		if &*value == &NULL_RLP {
 			return;
 		}
 
@@ -269,15 +253,15 @@ impl HashDB for MemoryDB {
 			Some(&mut (_, ref mut x)) => { *x -= 1; false }
 			None => true
 		}{	// ... None falls through into...
-			self.data.insert(key.clone(), (Bytes::new(), -1));
+			self.data.insert(key.clone(), (DBValue::new(), -1));
 		}
 	}
 
 	fn insert_aux(&mut self, hash: Vec<u8>, value: Vec<u8>) {
-		self.aux.insert(hash, value);
+		self.aux.insert(hash, DBValue::from_vec(value));
 	}
 
-	fn get_aux(&self, hash: &[u8]) -> Option<Vec<u8>> {
+	fn get_aux(&self, hash: &[u8]) -> Option<DBValue> {
 		self.aux.get(hash).cloned()
 	}
 
@@ -286,24 +270,6 @@ impl HashDB for MemoryDB {
 	}
 }
 
-#[test]
-fn memorydb_denote() {
-	let mut m = MemoryDB::new();
-	let hello_bytes = b"Hello world!";
-	let hash = m.insert(hello_bytes);
-	assert_eq!(m.get(&hash).unwrap(), b"Hello world!");
-
-	for _ in 0..1000 {
-		let r = H256::random();
-		let k = r.sha3();
-		let (v, rc) = m.denote(&k, r.to_vec());
-		assert_eq!(v, &*r);
-		assert_eq!(rc, 0);
-	}
-
-	assert_eq!(m.get(&hash).unwrap(), b"Hello world!");
-}
-
 #[test]
 fn memorydb_remove_and_purge() {
 	let hello_bytes = b"Hello world!";
@@ -337,12 +303,12 @@ fn consolidate() {
 	main.remove(&remove_key);
 
 	let insert_key = other.insert(b"arf");
-	main.emplace(insert_key, b"arf".to_vec());
+	main.emplace(insert_key, DBValue::from_slice(b"arf"));
 
 	main.consolidate(other);
 
 	let overlay = main.drain();
 
-	assert_eq!(overlay.get(&remove_key).unwrap(), &(b"doggo".to_vec(), 0));
-	assert_eq!(overlay.get(&insert_key).unwrap(), &(b"arf".to_vec(), 2));
-}
+	assert_eq!(overlay.get(&remove_key).unwrap(), &(DBValue::from_slice(b"doggo"), 0));
+	assert_eq!(overlay.get(&insert_key).unwrap(), &(DBValue::from_slice(b"arf"), 2));
+}

util/src/migration/mod.rs

@@ -20,7 +20,9 @@ mod tests;
 
 use std::collections::BTreeMap;
 use std::fs;
+use std::fmt;
 use std::path::{Path, PathBuf};
+use std::sync::Arc;
 
 use ::kvdb::{CompactionProfile, Database, DatabaseConfig, DBTransaction};
 
@@ -96,20 +98,39 @@ pub enum Error {
 	Custom(String),
 }
 
+impl fmt::Display for Error {
+	fn fmt(&self, f: &mut fmt::Formatter) -> Result<(), fmt::Error> {
+		match *self {
+			Error::CannotAddMigration => write!(f, "Cannot add migration"),
+			Error::MigrationImpossible => write!(f, "Migration impossible"),
+			Error::Io(ref err) => write!(f, "{}", err),
+			Error::Custom(ref err) => write!(f, "{}", err),
+		}
+	}
+}
+
 impl From<::std::io::Error> for Error {
 	fn from(e: ::std::io::Error) -> Self {
 		Error::Io(e)
 	}
 }
 
+impl From<String> for Error {
+	fn from(e: String) -> Self {
+		Error::Custom(e)
+	}
+}
+
 /// A generalized migration from the given db to a destination db.
 pub trait Migration: 'static {
+	/// Number of columns in the database before the migration.
+	fn pre_columns(&self) -> Option<u32> { self.columns() }
 	/// Number of columns in database after the migration.
 	fn columns(&self) -> Option<u32>;
 	/// Version of the database after the migration.
 	fn version(&self) -> u32;
 	/// Migrate a source to a destination.
-	fn migrate(&mut self, source: &Database, config: &Config, destination: &mut Database, col: Option<u32>) -> Result<(), Error>;
+	fn migrate(&mut self, source: Arc<Database>, config: &Config, destination: &mut Database, col: Option<u32>) -> Result<(), Error>;
 }
 
 /// A simple migration over key-value pairs.
@@ -128,7 +149,7 @@ impl<T: SimpleMigration> Migration for T {
 
 	fn version(&self) -> u32 { SimpleMigration::version(self) }
 
-	fn migrate(&mut self, source: &Database, config: &Config, dest: &mut Database, col: Option<u32>) -> Result<(), Error> {
+	fn migrate(&mut self, source: Arc<Database>, config: &Config, dest: &mut Database, col: Option<u32>) -> Result<(), Error> {
 		let mut batch = Batch::new(config, col);
 
 		for (key, value) in source.iter(col) {
@@ -195,6 +216,7 @@ impl Manager {
 			Some(last) => migration.version() > last.version(),
 			None => true,
 		};
+
 		match is_new {
 			true => Ok(self.migrations.push(Box::new(migration))),
 			false => Err(Error::CannotAddMigration),
@@ -205,12 +227,16 @@ impl Manager {
 	/// and producing a path where the final migration lives.
 	pub fn execute(&mut self, old_path: &Path, version: u32) -> Result<PathBuf, Error> {
 		let config = self.config.clone();
-		let columns = self.no_of_columns_at(version);
 		let migrations = self.migrations_from(version);
+		trace!(target: "migration", "Total migrations to execute for version {}: {}", version, migrations.len());
 		if migrations.is_empty() { return Err(Error::MigrationImpossible) };
+
+		let columns = migrations.iter().nth(0).and_then(|m| m.pre_columns());
+
+		trace!(target: "migration", "Expecting database to contain {:?} columns", columns);
 		let mut db_config = DatabaseConfig {
 			max_open_files: 64,
-			cache_size: None,
+			cache_sizes: Default::default(),
 			compaction: config.compaction_profile,
 			columns: columns,
 			wal: true,
@@ -222,7 +248,7 @@ impl Manager {
 
 		// start with the old db.
 		let old_path_str = try!(old_path.to_str().ok_or(Error::MigrationImpossible));
-		let mut cur_db = try!(Database::open(&db_config, old_path_str).map_err(Error::Custom));
+		let mut cur_db = Arc::new(try!(Database::open(&db_config, old_path_str).map_err(Error::Custom)));
 
 		for migration in migrations {
 			// Change number of columns in new db
@@ -237,16 +263,16 @@ impl Manager {
 			// perform the migration from cur_db to new_db.
 			match current_columns {
 				// migrate only default column
-				None => try!(migration.migrate(&cur_db, &config, &mut new_db, None)),
+				None => try!(migration.migrate(cur_db.clone(), &config, &mut new_db, None)),
 				Some(v) => {
 					// Migrate all columns in previous DB
 					for col in 0..v {
-						try!(migration.migrate(&cur_db, &config, &mut new_db, Some(col)))
+						try!(migration.migrate(cur_db.clone(), &config, &mut new_db, Some(col)))
 					}
 				}
 			}
 			// next iteration, we will migrate from this db into the other temp.
-			cur_db = new_db;
+			cur_db = Arc::new(new_db);
 			temp_idx.swap();
 
 			// remove the other temporary migration database.
@@ -267,14 +293,6 @@ impl Manager {
 	fn migrations_from(&mut self, version: u32) -> Vec<&mut Box<Migration>> {
 		self.migrations.iter_mut().filter(|m| m.version() > version).collect()
 	}
-
-	fn no_of_columns_at(&self, version: u32) -> Option<u32> {
-		let migration = self.migrations.iter().find(|m| m.version() == version);
-		match migration {
-			Some(m) => m.columns(),
-			None => None
-		}
-	}
 }
 
 /// Prints a dot every `max` ticks

util/src/migration/tests.rs

@@ -19,7 +19,7 @@
 //! are performed in temp sub-directories.
 
 use common::*;
-use migration::{Config, SimpleMigration, Manager};
+use migration::{Batch, Config, Error, SimpleMigration, Migration, Manager};
 use kvdb::Database;
 
 use devtools::RandomTempPath;
@@ -62,11 +62,10 @@ impl SimpleMigration for Migration0 {
 
 	fn version(&self) -> u32 { 1 }
 
-	fn simple_migrate(&mut self, key: Vec<u8>, value: Vec<u8>) -> Option<(Vec<u8>, Vec<u8>)> {
-		let mut key = key;
+	fn simple_migrate(&mut self, mut key: Vec<u8>, mut value: Vec<u8>) -> Option<(Vec<u8>, Vec<u8>)> {
 		key.push(0x11);
-		let mut value = value;
 		value.push(0x22);
+
 		Some((key, value))
 	}
 }
@@ -83,6 +82,31 @@ impl SimpleMigration for Migration1 {
 	}
 }
 
+struct AddsColumn;
+
+impl Migration for AddsColumn {
+	fn pre_columns(&self) -> Option<u32> { None }
+
+	fn columns(&self) -> Option<u32> { Some(1) }
+
+	fn version(&self) -> u32 { 1 }
+
+	fn migrate(&mut self, source: Arc<Database>, config: &Config, dest: &mut Database, col: Option<u32>) -> Result<(), Error> {
+		let mut batch = Batch::new(config, col);
+
+		for (key, value) in source.iter(col) {
+			try!(batch.insert(key.to_vec(), value.to_vec(), dest));
+		}
+
+
+		if col == Some(1) {
+			try!(batch.insert(vec![1, 2, 3], vec![4, 5, 6], dest));
+		}
+
+		batch.commit(dest)
+	}
+}
+
 #[test]
 fn one_simple_migration() {
 	let dir = RandomTempPath::create_dir();
@@ -189,3 +213,16 @@ fn is_migration_needed() {
 	assert!(manager.is_needed(1));
 	assert!(!manager.is_needed(2));
 }
+
+#[test]
+fn pre_columns() {
+	let mut manager = Manager::new(Config::default());
+	manager.add_migration(AddsColumn).unwrap();
+
+	let dir = RandomTempPath::create_dir();
+	let db_path = db_path(dir.as_path());
+
+	// this shouldn't fail to open the database even though it's one column
+	// short of the one before it.
+	manager.execute(&db_path, 0).unwrap();
+}

util/src/misc.rs

@@ -23,7 +23,7 @@ use target_info::Target;
 include!(concat!(env!("OUT_DIR"), "/version.rs"));
 include!(concat!(env!("OUT_DIR"), "/rustc_version.rs"));
 
-#[derive(PartialEq,Eq,Clone,Copy)]
+#[derive(PartialEq, Eq, Clone, Copy, Debug)]
 /// Boolean type for clean/dirty status.
 pub enum Filth {
 	/// Data has not been changed.

util/src/nibbleslice.rs

@@ -17,7 +17,7 @@
 //! Nibble-orientated view onto byte-slice, allowing nibble-precision offsets.
 use std::cmp::*;
 use std::fmt;
-use bytes::*;
+use elastic_array::ElasticArray36;
 
 /// Nibble-orientated view onto byte-slice, allowing nibble-precision offsets.
 ///
@@ -149,9 +149,9 @@ impl<'a, 'view> NibbleSlice<'a> where 'a: 'view {
 	}
 
 	/// Encode while nibble slice in prefixed hex notation, noting whether it `is_leaf`.
-	pub fn encoded(&self, is_leaf: bool) -> Bytes {
+	pub fn encoded(&self, is_leaf: bool) -> ElasticArray36<u8> {
 		let l = self.len();
-		let mut r = Bytes::with_capacity(l / 2 + 1);
+		let mut r = ElasticArray36::new();
 		let mut i = l % 2;
 		r.push(if i == 1 {0x10 + self.at(0)} else {0} + if is_leaf {0x20} else {0});
 		while i < l {
@@ -163,9 +163,9 @@ impl<'a, 'view> NibbleSlice<'a> where 'a: 'view {
 
 	/// Encode only the leftmost `n` bytes of the nibble slice in prefixed hex notation,
 	/// noting whether it `is_leaf`.
-	pub fn encoded_leftmost(&self, n: usize, is_leaf: bool) -> Bytes {
+	pub fn encoded_leftmost(&self, n: usize, is_leaf: bool) -> ElasticArray36<u8> {
 		let l = min(self.len(), n);
-		let mut r = Bytes::with_capacity(l / 2 + 1);
+		let mut r = ElasticArray36::new();
 		let mut i = l % 2;
 		r.push(if i == 1 {0x10 + self.at(0)} else {0} + if is_leaf {0x20} else {0});
 		while i < l {
@@ -212,6 +212,7 @@ impl<'a> fmt::Debug for NibbleSlice<'a> {
 #[cfg(test)]
 mod tests {
 	use super::NibbleSlice;
+	use elastic_array::ElasticArray36;
 	static D: &'static [u8;3] = &[0x01u8, 0x23, 0x45];
 
 	#[test]
@@ -254,10 +255,10 @@ mod tests {
 	#[test]
 	fn encoded() {
 		let n = NibbleSlice::new(D);
-		assert_eq!(n.encoded(false), &[0x00, 0x01, 0x23, 0x45]);
-		assert_eq!(n.encoded(true), &[0x20, 0x01, 0x23, 0x45]);
-		assert_eq!(n.mid(1).encoded(false), &[0x11, 0x23, 0x45]);
-		assert_eq!(n.mid(1).encoded(true), &[0x31, 0x23, 0x45]);
+		assert_eq!(n.encoded(false), ElasticArray36::from_slice(&[0x00, 0x01, 0x23, 0x45]));
+		assert_eq!(n.encoded(true), ElasticArray36::from_slice(&[0x20, 0x01, 0x23, 0x45]));
+		assert_eq!(n.mid(1).encoded(false), ElasticArray36::from_slice(&[0x11, 0x23, 0x45]));
+		assert_eq!(n.mid(1).encoded(true), ElasticArray36::from_slice(&[0x31, 0x23, 0x45]));
 	}
 
 	#[test]

util/src/overlaydb.rs

@@ -18,7 +18,6 @@
 
 use error::*;
 use hash::*;
-use bytes::*;
 use rlp::*;
 use hashdb::*;
 use memorydb::*;
@@ -101,21 +100,21 @@ impl OverlayDB {
 	pub fn commit_refs(&self, key: &H256) -> i32 { self.overlay.raw(key).map_or(0, |(_, refs)| refs) }
 
 	/// Get the refs and value of the given key.
-	fn payload(&self, key: &H256) -> Option<(Bytes, u32)> {
+	fn payload(&self, key: &H256) -> Option<(DBValue, u32)> {
 		self.backing.get(self.column, key)
 			.expect("Low-level database error. Some issue with your hard disk?")
 			.map(|d| {
 				let r = Rlp::new(&d);
-				(r.at(1).as_val(), r.at(0).as_val())
+				(DBValue::from_slice(r.at(1).data()), r.at(0).as_val())
 			})
 	}
 
 	/// Put the refs and value of the given key, possibly deleting it from the db.
-	fn put_payload_in_batch(&self, batch: &mut DBTransaction, key: &H256, payload: (Bytes, u32)) -> bool {
+	fn put_payload_in_batch(&self, batch: &mut DBTransaction, key: &H256, payload: (DBValue, u32)) -> bool {
 		if payload.1 > 0 {
 			let mut s = RlpStream::new_list(2);
 			s.append(&payload.1);
-			s.append(&payload.0);
+			s.append(&&*payload.0);
 			batch.put(self.column, key, s.as_raw());
 			false
 		} else {
@@ -140,29 +139,31 @@ impl HashDB for OverlayDB {
 		}
 		ret
 	}
-	fn get(&self, key: &H256) -> Option<&[u8]> {
+	fn get(&self, key: &H256) -> Option<DBValue> {
 		// return ok if positive; if negative, check backing - might be enough references there to make
 		// it positive again.
 		let k = self.overlay.raw(key);
-		match k {
-			Some((d, rc)) if rc > 0 => Some(d),
-			_ => {
-				let memrc = k.map_or(0, |(_, rc)| rc);
-				match self.payload(key) {
-					Some(x) => {
-						let (d, rc) = x;
-						if rc as i32 + memrc > 0 {
-							Some(self.overlay.denote(key, d).0)
-						}
-						else {
-							None
-						}
-					}
-					// Replace above match arm with this once https://github.com/rust-lang/rust/issues/15287 is done.
-					//Some((d, rc)) if rc + memrc > 0 => Some(d),
-					_ => None,
+		let memrc = {
+			if let Some((d, rc)) = k {
+				if rc > 0 { return Some(d); }
+				rc
+			} else {
+				0
+			}
+		};
+		match self.payload(key) {
+			Some(x) => {
+				let (d, rc) = x;
+				if rc as i32 + memrc > 0 {
+					Some(d)
+				}
+				else {
+					None
 				}
 			}
+			// Replace above match arm with this once https://github.com/rust-lang/rust/issues/15287 is done.
+			//Some((d, rc)) if rc + memrc > 0 => Some(d),
+			_ => None,
 		}
 	}
 	fn contains(&self, key: &H256) -> bool {
@@ -186,7 +187,7 @@ impl HashDB for OverlayDB {
 		}
 	}
 	fn insert(&mut self, value: &[u8]) -> H256 { self.overlay.insert(value) }
-	fn emplace(&mut self, key: H256, value: Bytes) { self.overlay.emplace(key, value); }
+	fn emplace(&mut self, key: H256, value: DBValue) { self.overlay.emplace(key, value); }
 	fn remove(&mut self, key: &H256) { self.overlay.remove(key); }
 }
 
@@ -211,7 +212,7 @@ fn overlaydb_revert() {
 fn overlaydb_overlay_insert_and_remove() {
 	let mut trie = OverlayDB::new_temp();
 	let h = trie.insert(b"hello world");
-	assert_eq!(trie.get(&h).unwrap(), b"hello world");
+	assert_eq!(trie.get(&h).unwrap(), DBValue::from_slice(b"hello world"));
 	trie.remove(&h);
 	assert_eq!(trie.get(&h), None);
 }
@@ -220,11 +221,11 @@ fn overlaydb_overlay_insert_and_remove() {
 fn overlaydb_backing_insert_revert() {
 	let mut trie = OverlayDB::new_temp();
 	let h = trie.insert(b"hello world");
-	assert_eq!(trie.get(&h).unwrap(), b"hello world");
+	assert_eq!(trie.get(&h).unwrap(), DBValue::from_slice(b"hello world"));
 	trie.commit().unwrap();
-	assert_eq!(trie.get(&h).unwrap(), b"hello world");
+	assert_eq!(trie.get(&h).unwrap(), DBValue::from_slice(b"hello world"));
 	trie.revert();
-	assert_eq!(trie.get(&h).unwrap(), b"hello world");
+	assert_eq!(trie.get(&h).unwrap(), DBValue::from_slice(b"hello world"));
 }
 
 #[test]
@@ -248,7 +249,7 @@ fn overlaydb_backing_remove_revert() {
 	trie.remove(&h);
 	assert_eq!(trie.get(&h), None);
 	trie.revert();
-	assert_eq!(trie.get(&h).unwrap(), b"hello world");
+	assert_eq!(trie.get(&h).unwrap(), DBValue::from_slice(b"hello world"));
 }
 
 #[test]
@@ -266,29 +267,29 @@ fn overlaydb_negative() {
 fn overlaydb_complex() {
 	let mut trie = OverlayDB::new_temp();
 	let hfoo = trie.insert(b"foo");
-	assert_eq!(trie.get(&hfoo).unwrap(), b"foo");
+	assert_eq!(trie.get(&hfoo).unwrap(), DBValue::from_slice(b"foo"));
 	let hbar = trie.insert(b"bar");
-	assert_eq!(trie.get(&hbar).unwrap(), b"bar");
+	assert_eq!(trie.get(&hbar).unwrap(), DBValue::from_slice(b"bar"));
 	trie.commit().unwrap();
-	assert_eq!(trie.get(&hfoo).unwrap(), b"foo");
-	assert_eq!(trie.get(&hbar).unwrap(), b"bar");
+	assert_eq!(trie.get(&hfoo).unwrap(), DBValue::from_slice(b"foo"));
+	assert_eq!(trie.get(&hbar).unwrap(), DBValue::from_slice(b"bar"));
 	trie.insert(b"foo");	// two refs
-	assert_eq!(trie.get(&hfoo).unwrap(), b"foo");
+	assert_eq!(trie.get(&hfoo).unwrap(), DBValue::from_slice(b"foo"));
 	trie.commit().unwrap();
-	assert_eq!(trie.get(&hfoo).unwrap(), b"foo");
-	assert_eq!(trie.get(&hbar).unwrap(), b"bar");
+	assert_eq!(trie.get(&hfoo).unwrap(), DBValue::from_slice(b"foo"));
+	assert_eq!(trie.get(&hbar).unwrap(), DBValue::from_slice(b"bar"));
 	trie.remove(&hbar);		// zero refs - delete
 	assert_eq!(trie.get(&hbar), None);
 	trie.remove(&hfoo);		// one ref - keep
-	assert_eq!(trie.get(&hfoo).unwrap(), b"foo");
+	assert_eq!(trie.get(&hfoo).unwrap(), DBValue::from_slice(b"foo"));
 	trie.commit().unwrap();
-	assert_eq!(trie.get(&hfoo).unwrap(), b"foo");
+	assert_eq!(trie.get(&hfoo).unwrap(), DBValue::from_slice(b"foo"));
 	trie.remove(&hfoo);		// zero ref - would delete, but...
 	assert_eq!(trie.get(&hfoo), None);
 	trie.insert(b"foo");	// one ref - keep after all.
-	assert_eq!(trie.get(&hfoo).unwrap(), b"foo");
+	assert_eq!(trie.get(&hfoo).unwrap(), DBValue::from_slice(b"foo"));
 	trie.commit().unwrap();
-	assert_eq!(trie.get(&hfoo).unwrap(), b"foo");
+	assert_eq!(trie.get(&hfoo).unwrap(), DBValue::from_slice(b"foo"));
 	trie.remove(&hfoo);		// zero ref - delete
 	assert_eq!(trie.get(&hfoo), None);
 	trie.commit().unwrap();	//

util/src/path.rs

@@ -62,6 +62,13 @@ pub mod ethereum {
 	/// Default path for ethereum installation on Mac Os
 	pub fn default() -> PathBuf { super::config_path("Ethereum") }
 
+	/// Default path for ethereum installation (testnet)
+	pub fn test() -> PathBuf {
+		let mut path = default();
+		path.push("testnet");
+		path
+	}
+
 	/// Get the specific folder inside default ethereum installation
 	pub fn with_default(s: &str) -> PathBuf {
 		let mut path = default();

util/src/trie/fatdb.rs

@@ -16,7 +16,7 @@
 
 use hash::H256;
 use sha3::Hashable;
-use hashdb::HashDB;
+use hashdb::{HashDB, DBValue};
 use super::{TrieDB, Trie, TrieDBIterator, TrieItem, Recorder};
 
 /// A `Trie` implementation which hashes keys and uses a generic `HashDB` backing database.
@@ -58,7 +58,7 @@ impl<'db> Trie for FatDB<'db> {
 		self.raw.contains(&key.sha3())
 	}
 
-	fn get_recorded<'a, 'b, R: 'b>(&'a self, key: &'b [u8], rec: &'b mut R) -> super::Result<Option<&'a [u8]>>
+	fn get_recorded<'a, 'b, R: 'b>(&'a self, key: &'b [u8], rec: &'b mut R) -> super::Result<Option<DBValue>>
 		where 'a: 'b, R: Recorder
 	{
 		self.raw.get_recorded(&key.sha3(), rec)
@@ -88,7 +88,7 @@ impl<'db> Iterator for FatDBIterator<'db> {
 		self.trie_iterator.next()
 			.map(|res|
 				res.map(|(hash, value)| {
-					(self.trie.db().get_aux(&hash).expect("Missing fatdb hash"), value)
+					(self.trie.db().get_aux(&hash).expect("Missing fatdb hash").to_vec(), value)
 				})
 			)
 	}
@@ -106,6 +106,6 @@ fn fatdb_to_trie() {
 		t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]).unwrap();
 	}
 	let t = FatDB::new(&memdb, &root).unwrap();
-	assert_eq!(t.get(&[0x01u8, 0x23]).unwrap().unwrap(), &[0x01u8, 0x23]);
-	assert_eq!(t.iter().unwrap().map(Result::unwrap).collect::<Vec<_>>(), vec![(vec![0x01u8, 0x23], &[0x01u8, 0x23] as &[u8])]);
+	assert_eq!(t.get(&[0x01u8, 0x23]).unwrap().unwrap(), DBValue::from_slice(&[0x01u8, 0x23]));
+	assert_eq!(t.iter().unwrap().map(Result::unwrap).collect::<Vec<_>>(), vec![(vec![0x01u8, 0x23], DBValue::from_slice(&[0x01u8, 0x23] as &[u8]))]);
 }

util/src/trie/fatdbmut.rs

@@ -16,7 +16,7 @@
 
 use hash::H256;
 use sha3::Hashable;
-use hashdb::HashDB;
+use hashdb::{HashDB, DBValue};
 use super::{TrieDBMut, TrieMut};
 
 /// A mutable `Trie` implementation which hashes keys and uses a generic `HashDB` backing database.
@@ -66,7 +66,7 @@ impl<'db> TrieMut for FatDBMut<'db> {
 		self.raw.contains(&key.sha3())
 	}
 
-	fn get<'a, 'key>(&'a self, key: &'key [u8]) -> super::Result<Option<&'a [u8]>>
+	fn get<'a, 'key>(&'a self, key: &'key [u8]) -> super::Result<Option<DBValue>>
 		where 'a: 'key
 	{
 		self.raw.get(&key.sha3())
@@ -98,5 +98,5 @@ fn fatdb_to_trie() {
 		t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]).unwrap();
 	}
 	let t = TrieDB::new(&memdb, &root).unwrap();
-	assert_eq!(t.get(&(&[0x01u8, 0x23]).sha3()).unwrap().unwrap(), &[0x01u8, 0x23]);
+	assert_eq!(t.get(&(&[0x01u8, 0x23]).sha3()).unwrap().unwrap(), DBValue::from_slice(&[0x01u8, 0x23]));
 }

util/src/trie/journal.rs

@@ -24,7 +24,7 @@ use hashdb::*;
 /// Type of operation for the backing database - either a new node or a node deletion.
 #[derive(Debug)]
 enum Operation {
-	New(H256, Bytes),
+	New(H256, DBValue),
 	Delete(H256),
 }
 
@@ -52,16 +52,16 @@ impl Journal {
 
 	/// Given the RLP that encodes a node, append a reference to that node `out` and leave `journal`
 	/// such that the reference is valid, once applied.
-	pub fn new_node(&mut self, rlp: Bytes, out: &mut RlpStream) {
+	pub fn new_node(&mut self, rlp: DBValue, out: &mut RlpStream) {
 		if rlp.len() >= 32 {
 			let rlp_sha3 = rlp.sha3();
 
-			trace!("new_node: reference node {:?} => {:?}", rlp_sha3, rlp.pretty());
+			trace!("new_node: reference node {:?} => {:?}", rlp_sha3, &*rlp);
 			out.append(&rlp_sha3);
 			self.0.push(Operation::New(rlp_sha3, rlp));
 		}
 		else {
-			trace!("new_node: inline node {:?}", rlp.pretty());
+			trace!("new_node: inline node {:?}", &*rlp);
 			out.append_raw(&rlp, 1);
 		}
 	}

util/src/trie/mod.rs

@@ -18,7 +18,7 @@
 
 use std::fmt;
 use hash::H256;
-use hashdb::HashDB;
+use hashdb::{HashDB, DBValue};
 
 /// Export the standardmap module.
 pub mod standardmap;
@@ -76,7 +76,7 @@ impl fmt::Display for TrieError {
 pub type Result<T> = ::std::result::Result<T, Box<TrieError>>;
 
 /// Trie-Item type.
-pub type TrieItem<'a> = Result<(Vec<u8>, &'a [u8])>;
+pub type TrieItem<'a> = Result<(Vec<u8>, DBValue)>;
 
 /// A key-value datastore implemented as a database-backed modified Merkle tree.
 pub trait Trie {
@@ -92,13 +92,13 @@ pub trait Trie {
 	}
 
 	/// What is the value of the given key in this trie?
-	fn get<'a, 'key>(&'a self, key: &'key [u8]) -> Result<Option<&'a [u8]>> where 'a: 'key {
+	fn get<'a, 'key>(&'a self, key: &'key [u8]) -> Result<Option<DBValue>> where 'a: 'key {
 		self.get_recorded(key, &mut recorder::NoOp)
 	}
 
 	/// Query the value of the given key in this trie while recording visited nodes
 	/// to the given recorder. If the query fails, the nodes passed to the recorder are unspecified.
-	fn get_recorded<'a, 'b, R: 'b>(&'a self, key: &'b [u8], rec: &'b mut R) -> Result<Option<&'a [u8]>>
+	fn get_recorded<'a, 'b, R: 'b>(&'a self, key: &'b [u8], rec: &'b mut R) -> Result<Option<DBValue>>
 		where 'a: 'b, R: Recorder;
 
 	/// Returns an iterator over elements of trie.
@@ -119,7 +119,7 @@ pub trait TrieMut {
 	}
 
 	/// What is the value of the given key in this trie?
-	fn get<'a, 'key>(&'a self, key: &'key [u8]) -> Result<Option<&'a [u8]>> where 'a: 'key;
+	fn get<'a, 'key>(&'a self, key: &'key [u8]) -> Result<Option<DBValue>> where 'a: 'key;
 
 	/// Insert a `key`/`value` pair into the trie. An `empty` value is equivalent to removing
 	/// `key` from the trie.
@@ -188,7 +188,7 @@ impl<'db> Trie for TrieKinds<'db> {
 		wrapper!(self, contains, key)
 	}
 
-	fn get_recorded<'a, 'b, R: 'b>(&'a self, key: &'b [u8], r: &'b mut R) -> Result<Option<&'a [u8]>>
+	fn get_recorded<'a, 'b, R: 'b>(&'a self, key: &'b [u8], r: &'b mut R) -> Result<Option<DBValue>>
 		where 'a: 'b, R: Recorder {
 		wrapper!(self, get_recorded, key, r)
 	}
@@ -233,4 +233,7 @@ impl TrieFactory {
 			TrieSpec::Fat => Ok(Box::new(try!(FatDBMut::from_existing(db, root)))),
 		}
 	}
+
+	/// Returns true iff the trie DB is a fat DB (allows enumeration of keys).
+	pub fn is_fat(&self) -> bool { self.spec == TrieSpec::Fat } 
 }

util/src/trie/node.rs

@@ -14,27 +14,51 @@
 // You should have received a copy of the GNU General Public License
 // along with Parity.  If not, see <http://www.gnu.org/licenses/>.
 
+use elastic_array::ElasticArray36;
 use nibbleslice::*;
 use bytes::*;
 use rlp::*;
 use super::journal::*;
+use hashdb::DBValue;
+
+/// Partial node key type.
+pub type NodeKey = ElasticArray36<u8>;
 
 /// Type of node in the trie and essential information thereof.
-#[derive(Clone, Eq, PartialEq, Debug)]
-pub enum Node<'a> {
+#[derive(Eq, PartialEq, Debug)]
+pub enum Node {
 	/// Null trie node; could be an empty root or an empty branch entry.
 	Empty,
 	/// Leaf node; has key slice and value. Value may not be empty.
-	Leaf(NibbleSlice<'a>, &'a [u8]),
+	Leaf(NodeKey, DBValue),
 	/// Extension node; has key slice and node data. Data may not be null.
-	Extension(NibbleSlice<'a>, &'a [u8]),
+	Extension(NodeKey, DBValue),
 	/// Branch node; has array of 16 child nodes (each possibly null) and an optional immediate node data.
-	Branch([&'a [u8]; 16], Option<&'a [u8]>)
+	Branch([NodeKey; 16], Option<DBValue>)
 }
 
-impl<'a> Node<'a> {
+impl Clone for Node {
+	fn clone(&self) -> Node {
+		match *self {
+			Node::Empty => Node::Empty,
+			Node::Leaf(ref k, ref v) => Node::Leaf(k.clone(), v.clone()),
+			Node::Extension(ref k, ref v) => Node::Extension(k.clone(), v.clone()),
+			Node::Branch(ref k, ref v) => {
+				let mut branch = [NodeKey::new(), NodeKey::new(), NodeKey::new(), NodeKey::new(), NodeKey::new(),
+					NodeKey::new(), NodeKey::new(), NodeKey::new(), NodeKey::new(), NodeKey::new(), NodeKey::new(),
+					NodeKey::new(), NodeKey::new(), NodeKey::new(), NodeKey::new(), NodeKey::new()];
+				for i in 0 .. 16 {
+					branch[i] = k[i].clone();
+				}
+				Node::Branch(branch, v.clone())
+			}
+		}
+	}
+}
+
+impl Node {
 	/// Decode the `node_rlp` and return the Node.
-	pub fn decoded(node_rlp: &'a [u8]) -> Node<'a> {
+	pub fn decoded(node_rlp: &[u8]) -> Node {
 		let r = Rlp::new(node_rlp);
 		match r.prototype() {
 			// either leaf or extension - decode first item with NibbleSlice::???
@@ -43,16 +67,18 @@ impl<'a> Node<'a> {
 			// if extension, second item is a node (either SHA3 to be looked up and
 			// fed back into this function or inline RLP which can be fed back into this function).
 			Prototype::List(2) => match NibbleSlice::from_encoded(r.at(0).data()) {
-				(slice, true) => Node::Leaf(slice, r.at(1).data()),
-				(slice, false) => Node::Extension(slice, r.at(1).as_raw()),
+				(slice, true) => Node::Leaf(slice.encoded(true), DBValue::from_slice(r.at(1).data())),
+				(slice, false) => Node::Extension(slice.encoded(false), DBValue::from_slice(r.at(1).as_raw())),
 			},
 			// branch - first 16 are nodes, 17th is a value (or empty).
 			Prototype::List(17) => {
-				let mut nodes: [&'a [u8]; 16] = [&[], &[], &[], &[], &[], &[], &[], &[], &[], &[], &[], &[], &[], &[], &[], &[]];
+				let mut nodes: [NodeKey; 16] = [NodeKey::new(), NodeKey::new(), NodeKey::new(), NodeKey::new(), NodeKey::new(),
+					NodeKey::new(), NodeKey::new(), NodeKey::new(), NodeKey::new(), NodeKey::new(), NodeKey::new(),
+					NodeKey::new(), NodeKey::new(), NodeKey::new(), NodeKey::new(), NodeKey::new()];
 				for i in 0..16 {
-					nodes[i] = r.at(i).as_raw();
+					nodes[i] = NodeKey::from_slice(r.at(i).as_raw());
 				}
-				Node::Branch(nodes, if r.at(16).is_empty() { None } else { Some(r.at(16).data()) })
+				Node::Branch(nodes, if r.at(16).is_empty() { None } else { Some(DBValue::from_slice(r.at(16).data())) })
 			},
 			// an empty branch index.
 			Prototype::Data(0) => Node::Empty,
@@ -69,23 +95,23 @@ impl<'a> Node<'a> {
 		match *self {
 			Node::Leaf(ref slice, ref value) => {
 				let mut stream = RlpStream::new_list(2);
-				stream.append(&slice.encoded(true));
-				stream.append(value);
+				stream.append(&&**slice);
+				stream.append(&&**value);
 				stream.out()
 			},
-			Node::Extension(ref slice, raw_rlp) => {
+			Node::Extension(ref slice, ref raw_rlp) => {
 				let mut stream = RlpStream::new_list(2);
-				stream.append(&slice.encoded(false));
-				stream.append_raw(raw_rlp, 1);
+				stream.append(&&**slice);
+				stream.append_raw(&&*raw_rlp, 1);
 				stream.out()
 			},
 			Node::Branch(ref nodes, ref value) => {
 				let mut stream = RlpStream::new_list(17);
 				for i in 0..16 {
-					stream.append_raw(nodes[i], 1);
+					stream.append_raw(&*nodes[i], 1);
 				}
 				match *value {
-					Some(n) => { stream.append(&n); },
+					Some(ref n) => { stream.append(&&**n); },
 					None => { stream.append_empty_data(); },
 				}
 				stream.out()
@@ -100,26 +126,26 @@ impl<'a> Node<'a> {
 
 	/// Encode the node, adding it to `journal` if necessary and return the RLP valid for
 	/// insertion into a parent node.
-	pub fn encoded_and_added(&self, journal: &mut Journal) -> Bytes {
+	pub fn encoded_and_added(&self, journal: &mut Journal) -> DBValue {
 		let mut stream = RlpStream::new();
 		match *self {
 			Node::Leaf(ref slice, ref value) => {
 				stream.begin_list(2);
-				stream.append(&slice.encoded(true));
-				stream.append(value);
+				stream.append(&&**slice);
+				stream.append(&&**value);
 			},
-			Node::Extension(ref slice, raw_rlp) => {
+			Node::Extension(ref slice, ref raw_rlp) => {
 				stream.begin_list(2);
-				stream.append(&slice.encoded(false));
-				stream.append_raw(raw_rlp, 1);
+				stream.append(&&**slice);
+				stream.append_raw(&&**raw_rlp, 1);
 			},
 			Node::Branch(ref nodes, ref value) => {
 				stream.begin_list(17);
 				for i in 0..16 {
-					stream.append_raw(nodes[i], 1);
+					stream.append_raw(&*nodes[i], 1);
 				}
 				match *value {
-					Some(n) => { stream.append(&n); },
+					Some(ref n) => { stream.append(&&**n); },
 					None => { stream.append_empty_data(); },
 				}
 			},
@@ -127,13 +153,13 @@ impl<'a> Node<'a> {
 				stream.append_empty_data();
 			}
 		}
-		let node = stream.out();
+		let node = DBValue::from_slice(stream.as_raw());
 		match node.len() {
 			0 ... 31 => node,
 			_ => {
 				let mut stream = RlpStream::new();
 				journal.new_node(node, &mut stream);
-				stream.out()
+				DBValue::from_slice(stream.as_raw())
 			}
 		}
 	}

util/src/trie/sectriedb.rs

@@ -16,7 +16,7 @@
 
 use hash::H256;
 use sha3::Hashable;
-use hashdb::HashDB;
+use hashdb::{HashDB, DBValue};
 use super::triedb::TrieDB;
 use super::{Trie, TrieItem, Recorder};
 
@@ -59,7 +59,7 @@ impl<'db> Trie for SecTrieDB<'db> {
 		self.raw.contains(&key.sha3())
 	}
 
-	fn get_recorded<'a, 'b, R: 'b>(&'a self, key: &'b [u8], rec: &'b mut R) -> super::Result<Option<&'a [u8]>>
+	fn get_recorded<'a, 'b, R: 'b>(&'a self, key: &'b [u8], rec: &'b mut R) -> super::Result<Option<DBValue>>
 		where 'a: 'b, R: Recorder
 	{
 		self.raw.get_recorded(&key.sha3(), rec)
@@ -79,5 +79,5 @@ fn trie_to_sectrie() {
 		t.insert(&(&[0x01u8, 0x23]).sha3(), &[0x01u8, 0x23]).unwrap();
 	}
 	let t = SecTrieDB::new(&memdb, &root).unwrap();
-	assert_eq!(t.get(&[0x01u8, 0x23]).unwrap().unwrap(), &[0x01u8, 0x23]);
+	assert_eq!(t.get(&[0x01u8, 0x23]).unwrap().unwrap(), DBValue::from_slice(&[0x01u8, 0x23]));
 }

util/src/trie/sectriedbmut.rs

@@ -16,7 +16,7 @@
 
 use hash::H256;
 use sha3::Hashable;
-use hashdb::HashDB;
+use hashdb::{HashDB, DBValue};
 use super::triedbmut::TrieDBMut;
 use super::TrieMut;
 
@@ -62,7 +62,7 @@ impl<'db> TrieMut for SecTrieDBMut<'db> {
 		self.raw.contains(&key.sha3())
 	}
 
-	fn get<'a, 'key>(&'a self, key: &'key [u8]) -> super::Result<Option<&'a [u8]>>
+	fn get<'a, 'key>(&'a self, key: &'key [u8]) -> super::Result<Option<DBValue>>
 		where 'a: 'key
 	{
 		self.raw.get(&key.sha3())
@@ -90,5 +90,5 @@ fn sectrie_to_trie() {
 		t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]).unwrap();
 	}
 	let t = TrieDB::new(&memdb, &root).unwrap();
-	assert_eq!(t.get(&(&[0x01u8, 0x23]).sha3()).unwrap().unwrap(), &[0x01u8, 0x23]);
+	assert_eq!(t.get(&(&[0x01u8, 0x23]).sha3()).unwrap().unwrap(), DBValue::from_slice(&[0x01u8, 0x23]));
 }

util/src/trie/triedb.rs

@@ -44,7 +44,7 @@ use super::{Trie, TrieItem, TrieError};
 ///   TrieDBMut::new(&mut memdb, &mut root).insert(b"foo", b"bar").unwrap();
 ///   let t = TrieDB::new(&memdb, &root).unwrap();
 ///   assert!(t.contains(b"foo").unwrap());
-///   assert_eq!(t.get(b"foo").unwrap().unwrap(), b"bar");
+///   assert_eq!(t.get(b"foo").unwrap().unwrap(), DBValue::from_slice(b"bar"));
 ///   assert!(t.db_items_remaining().unwrap().is_empty());
 /// }
 /// ```
@@ -119,8 +119,8 @@ impl<'db> TrieDB<'db> {
 		};
 
 		match node {
-			Node::Extension(_, payload) => try!(handle_payload(payload)),
-			Node::Branch(payloads, _) => for payload in &payloads { try!(handle_payload(payload)) },
+			Node::Extension(_, ref payload) => try!(handle_payload(payload)),
+			Node::Branch(ref payloads, _) => for payload in payloads { try!(handle_payload(payload)) },
 			_ => {},
 		}
 
@@ -129,18 +129,18 @@ impl<'db> TrieDB<'db> {
 
 	/// Get the root node's RLP.
 	fn root_node<R: Recorder>(&self, r: &mut R) -> super::Result<Node> {
-		self.root_data(r).map(Node::decoded)
+		self.root_data(r).map(|d| Node::decoded(&d))
 	}
 
 	/// Get the data of the root node.
-	fn root_data<'a, R: 'a + Recorder>(&self, r: &'a mut R) -> super::Result<&[u8]> {
+	fn root_data<'a, R: 'a + Recorder>(&self, r: &'a mut R) -> super::Result<DBValue> {
 		self.db.get(self.root).ok_or_else(|| Box::new(TrieError::InvalidStateRoot(*self.root)))
-			.map(|node| { r.record(self.root, node, 0); node })
+			.map(|node| { r.record(self.root, &*node, 0); node })
 	}
 
 	/// Get the root node as a `Node`.
 	fn get_node<'a, R: 'a + Recorder>(&'db self, node: &'db [u8], r: &'a mut R, depth: u32) -> super::Result<Node> {
-		self.get_raw_or_lookup(node, r, depth).map(Node::decoded)
+		self.get_raw_or_lookup(node, r, depth).map(|n| Node::decoded(&n))
 	}
 
 	/// Indentation helper for `formal_all`.
@@ -155,20 +155,20 @@ impl<'db> TrieDB<'db> {
 	fn fmt_all(&self, node: Node, f: &mut fmt::Formatter, deepness: usize) -> fmt::Result {
 		match node {
 			Node::Leaf(slice, value) => try!(writeln!(f, "'{:?}: {:?}.", slice, value.pretty())),
-			Node::Extension(ref slice, item) => {
+			Node::Extension(ref slice, ref item) => {
 				try!(write!(f, "'{:?} ", slice));
-				if let Ok(node) = self.get_node(item, &mut NoOp, 0) {
+				if let Ok(node) = self.get_node(&*item, &mut NoOp, 0) {
 					try!(self.fmt_all(node, f, deepness));
 				}
 			},
 			Node::Branch(ref nodes, ref value) => {
 				try!(writeln!(f, ""));
-				if let Some(v) = *value {
+				if let Some(ref v) = *value {
 					try!(self.fmt_indent(f, deepness + 1));
 					try!(writeln!(f, "=: {:?}", v.pretty()))
 				}
 				for i in 0..16 {
-					match self.get_node(nodes[i], &mut NoOp, 0) {
+					match self.get_node(&*nodes[i], &mut NoOp, 0) {
 						Ok(Node::Empty) => {},
 						Ok(n) => {
 							try!(self.fmt_indent(f, deepness + 1));
@@ -190,11 +190,11 @@ impl<'db> TrieDB<'db> {
 	}
 
 	/// Return optional data for a key given as a `NibbleSlice`. Returns `None` if no data exists.
-	fn do_lookup<'key, R: 'key>(&'db self, key: &NibbleSlice<'key>, r: &'key mut R) -> super::Result<Option<&'db [u8]>>
+	fn do_lookup<'key, R: 'key>(&'db self, key: &NibbleSlice<'key>, r: &'key mut R) -> super::Result<Option<DBValue>>
 		where 'db: 'key, R: Recorder
 	{
 		let root_rlp = try!(self.root_data(r));
-		self.get_from_node(root_rlp, key, r, 1)
+		self.get_from_node(&root_rlp, key, r, 1)
 	}
 
 	/// Recursible function to retrieve the value given a `node` and a partial `key`. `None` if no
@@ -207,18 +207,23 @@ impl<'db> TrieDB<'db> {
 		key: &NibbleSlice<'key>,
 		r: &'key mut R,
 		d: u32
-	) -> super::Result<Option<&'db [u8]>> where 'db: 'key, R: Recorder {
+	) -> super::Result<Option<DBValue>> where 'db: 'key, R: Recorder {
 		match Node::decoded(node) {
-			Node::Leaf(ref slice, value) if key == slice => Ok(Some(value)),
-			Node::Extension(ref slice, item) if key.starts_with(slice) => {
-				let data = try!(self.get_raw_or_lookup(item, r, d));
-				self.get_from_node(data, &key.mid(slice.len()), r, d + 1)
+			Node::Leaf(ref slice, ref value) if NibbleSlice::from_encoded(slice).0 == *key => Ok(Some(value.clone())),
+			Node::Extension(ref slice, ref item) => {
+				let slice = &NibbleSlice::from_encoded(slice).0;
+				if key.starts_with(slice) {
+					let data = try!(self.get_raw_or_lookup(&*item, r, d));
+					self.get_from_node(&data, &key.mid(slice.len()), r, d + 1)
+				} else {
+					Ok(None)
+				}
 			},
-			Node::Branch(ref nodes, value) => match key.is_empty() {
-				true => Ok(value),
+			Node::Branch(ref nodes, ref value) => match key.is_empty() {
+				true => Ok(value.clone()),
 				false => {
-					let node = try!(self.get_raw_or_lookup(nodes[key.at(0) as usize], r, d));
-					self.get_from_node(node, &key.mid(1), r, d + 1)
+					let node = try!(self.get_raw_or_lookup(&*nodes[key.at(0) as usize], r, d));
+					self.get_from_node(&node, &key.mid(1), r, d + 1)
 				}
 			},
 			_ => Ok(None)
@@ -228,16 +233,16 @@ impl<'db> TrieDB<'db> {
 	/// Given some node-describing data `node`, return the actual node RLP.
 	/// This could be a simple identity operation in the case that the node is sufficiently small, but
 	/// may require a database lookup.
-	fn get_raw_or_lookup<R: Recorder>(&'db self, node: &'db [u8], rec: &mut R, d: u32) -> super::Result<&'db [u8]> {
+	fn get_raw_or_lookup<R: Recorder>(&'db self, node: &'db [u8], rec: &mut R, d: u32) -> super::Result<DBValue> {
 		// check if its sha3 + len
 		let r = Rlp::new(node);
 		match r.is_data() && r.size() == 32 {
 			true => {
 				let key = r.as_val::<H256>();
 				self.db.get(&key).ok_or_else(|| Box::new(TrieError::IncompleteDatabase(key)))
-					.map(|raw| { rec.record(&key, raw, d); raw })
+					.map(|raw| { rec.record(&key, &raw, d); raw })
 			}
-			false => Ok(node)
+			false => Ok(DBValue::from_slice(node))
 		}
 	}
 }
@@ -251,12 +256,12 @@ enum Status {
 }
 
 #[derive(Clone, Eq, PartialEq)]
-struct Crumb<'a> {
-	node: Node<'a>,
+struct Crumb {
+	node: Node,
 	status: Status,
 }
 
-impl<'a> Crumb<'a> {
+impl Crumb {
 	/// Move on to next status in the node's sequence.
 	fn increment(&mut self) {
 		self.status = match (&self.status, &self.node) {
@@ -273,7 +278,7 @@ impl<'a> Crumb<'a> {
 #[derive(Clone)]
 pub struct TrieDBIterator<'a> {
 	db: &'a TrieDB<'a>,
-	trail: Vec<Crumb<'a>>,
+	trail: Vec<Crumb>,
 	key_nibbles: Bytes,
 }
 
@@ -286,32 +291,24 @@ impl<'a> TrieDBIterator<'a> {
 			key_nibbles: Vec::new(),
 		};
 
-		try!(db.root_data(&mut NoOp).and_then(|root| r.descend(root)));
+		try!(db.root_data(&mut NoOp).and_then(|root| r.descend(&root)));
 		Ok(r)
 	}
 
 	/// Descend into a payload.
-	fn descend(&mut self, d: &'a [u8]) -> super::Result<()> {
+	fn descend(&mut self, d: &[u8]) -> super::Result<()> {
 		self.trail.push(Crumb {
 			status: Status::Entering,
 			node: try!(self.db.get_node(d, &mut NoOp, 0)),
 		});
-		match self.trail.last().unwrap().node {
-			Node::Leaf(n, _) | Node::Extension(n, _) => { self.key_nibbles.extend(n.iter()); },
+		match self.trail.last().expect("just pushed item; qed").node {
+			Node::Leaf(ref n, _) | Node::Extension(ref n, _) => { self.key_nibbles.extend(NibbleSlice::from_encoded(n).0.iter()); },
 			_ => {}
 		}
 
 		Ok(())
 	}
 
-	/// Descend into a payload and get the next item.
-	fn descend_next(&mut self, d: &'a [u8]) -> Option<TrieItem<'a>> {
-		match self.descend(d) {
-			Ok(()) => self.next(),
-			Err(e) => Some(Err(e)),
-		}
-	}
-
 	/// The present key.
 	fn key(&self) -> Bytes {
 		// collapse the key_nibbles down to bytes.
@@ -323,38 +320,53 @@ impl<'a> Iterator for TrieDBIterator<'a> {
 	type Item = TrieItem<'a>;
 
 	fn next(&mut self) -> Option<Self::Item> {
-		let b = match self.trail.last_mut() {
-			Some(mut b) => { b.increment(); b.clone() },
-			None => return None,
-		};
-		match (b.status, b.node) {
-			(Status::Exiting, n) => {
-				match n {
-					Node::Leaf(n, _) | Node::Extension(n, _) => {
-						let l = self.key_nibbles.len();
-						self.key_nibbles.truncate(l - n.len());
-					},
-					Node::Branch(_, _) => { self.key_nibbles.pop(); },
-					_ => {}
-				}
-				self.trail.pop();
-				self.next()
-			},
-			(Status::At, Node::Leaf(_, v)) | (Status::At, Node::Branch(_, Some(v))) => Some(Ok((self.key(), v))),
-			(Status::At, Node::Extension(_, d)) => self.descend_next(d),
-			(Status::At, Node::Branch(_, _)) => self.next(),
-			(Status::AtChild(i), Node::Branch(children, _)) if children[i].len() > 0 => {
-				match i {
-					0 => self.key_nibbles.push(0),
-					i => *self.key_nibbles.last_mut().unwrap() = i as u8,
-				}
-				self.descend_next(children[i])
-			},
-			(Status::AtChild(i), Node::Branch(_, _)) => {
-				if i == 0 { self.key_nibbles.push(0); }
-				self.next()
-			},
-			_ => panic!() // Should never see Entering or AtChild without a Branch here.
+		loop {
+			let b = match self.trail.last_mut() {
+				Some(mut b) => { b.increment(); b.clone() },
+				None => return None,
+			};
+			match (b.status, b.node) {
+				(Status::Exiting, n) => {
+					match n {
+						Node::Leaf(n, _) | Node::Extension(n, _) => {
+							let l = self.key_nibbles.len();
+							self.key_nibbles.truncate(l - NibbleSlice::from_encoded(&*n).0.len());
+						},
+						Node::Branch(_, _) => { self.key_nibbles.pop(); },
+						_ => {}
+					}
+					self.trail.pop();
+					// continue
+				},
+				(Status::At, Node::Leaf(_, v)) | (Status::At, Node::Branch(_, Some(v))) => {
+					return Some(Ok((self.key(), v)));
+				},
+				(Status::At, Node::Extension(_, d)) => {
+					if let Err(e) = self.descend(&*d) {
+						return Some(Err(e));
+					}
+					// continue
+				},
+				(Status::At, Node::Branch(_, _)) => {},
+				(Status::AtChild(i), Node::Branch(ref children, _)) if children[i].len() > 0 => {
+					match i {
+						0 => self.key_nibbles.push(0),
+						i => *self.key_nibbles.last_mut()
+							.expect("pushed as 0; moves sequentially; removed afterwards; qed") = i as u8,
+					}
+					if let Err(e) = self.descend(&*children[i]) {
+						return Some(Err(e));
+					}
+					// continue
+				},
+				(Status::AtChild(i), Node::Branch(_, _)) => {
+					if i == 0 {
+						self.key_nibbles.push(0);
+					}
+					// continue
+				},
+				_ => panic!() // Should never see Entering or AtChild without a Branch here.
+			}
 		}
 	}
 }
@@ -366,7 +378,7 @@ impl<'db> Trie for TrieDB<'db> {
 
 	fn root(&self) -> &H256 { self.root }
 
-	fn get_recorded<'a, 'b, R: 'b>(&'a self, key: &'b [u8], rec: &'b mut R) -> super::Result<Option<&'a [u8]>>
+	fn get_recorded<'a, 'b, R: 'b>(&'a self, key: &'b [u8], rec: &'b mut R) -> super::Result<Option<DBValue>>
 		where 'a: 'b, R: Recorder
 	{
 		self.do_lookup(&NibbleSlice::new(key), rec)
@@ -377,7 +389,7 @@ impl<'db> fmt::Debug for TrieDB<'db> {
 	fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
 		try!(writeln!(f, "c={:?} [", self.hash_count));
 		let root_rlp = self.db.get(self.root).expect("Trie root not found!");
-		try!(self.fmt_all(Node::decoded(root_rlp), f, 0));
+		try!(self.fmt_all(Node::decoded(&root_rlp), f, 0));
 		writeln!(f, "]")
 	}
 }
@@ -388,7 +400,7 @@ fn iterator() {
 	use super::TrieMut;
 	use super::triedbmut::*;
 
-	let d = vec![ &b"A"[..], &b"AA"[..], &b"AB"[..], &b"B"[..] ];
+	let d = vec![ DBValue::from_slice(b"A"), DBValue::from_slice(b"AA"), DBValue::from_slice(b"AB"), DBValue::from_slice(b"B") ];
 
 	let mut memdb = MemoryDB::new();
 	let mut root = H256::new();
@@ -400,6 +412,6 @@ fn iterator() {
 	}
 
 	let t = TrieDB::new(&memdb, &root).unwrap();
-	assert_eq!(d.iter().map(|i|i.to_vec()).collect::<Vec<_>>(), t.iter().unwrap().map(|x| x.unwrap().0).collect::<Vec<_>>());
+	assert_eq!(d.iter().map(|i| i.clone().to_vec()).collect::<Vec<_>>(), t.iter().unwrap().map(|x| x.unwrap().0).collect::<Vec<_>>());
 	assert_eq!(d, t.iter().unwrap().map(|x| x.unwrap().1).collect::<Vec<_>>());
 }

util/src/trie/triedbmut.rs

@@ -18,12 +18,14 @@
 
 use super::{TrieError, TrieMut};
 use super::node::Node as RlpNode;
+use super::node::NodeKey;
 
-use ::{Bytes, HashDB, H256};
+use ::{HashDB, H256};
 use ::bytes::ToPretty;
 use ::nibbleslice::NibbleSlice;
 use ::rlp::{Rlp, RlpStream, View, Stream};
 use ::sha3::SHA3_NULL_RLP;
+use hashdb::DBValue;
 
 use elastic_array::ElasticArray1024;
 
@@ -72,14 +74,14 @@ enum Node {
 	/// A leaf node contains the end of a key and a value.
 	/// This key is encoded from a `NibbleSlice`, meaning it contains
 	/// a flag indicating it is a leaf.
-	Leaf(Bytes, Bytes),
+	Leaf(NodeKey, DBValue),
 	/// An extension contains a shared portion of a key and a child node.
 	/// The shared portion is encoded from a `NibbleSlice` meaning it contains
 	/// a flag indicating it is an extension.
 	/// The child node is always a branch.
-	Extension(Bytes, NodeHandle),
+	Extension(NodeKey, NodeHandle),
 	/// A branch has up to 16 children and an optional value.
-	Branch(Box<[Option<NodeHandle>; 16]>, Option<Bytes>)
+	Branch(Box<[Option<NodeHandle>; 16]>, Option<DBValue>)
 }
 
 impl Node {
@@ -98,21 +100,18 @@ impl Node {
 	fn from_rlp(rlp: &[u8], db: &HashDB, storage: &mut NodeStorage) -> Self {
 		match RlpNode::decoded(rlp) {
 			RlpNode::Empty => Node::Empty,
-			RlpNode::Leaf(k, v) => Node::Leaf(k.encoded(true), v.to_owned()),
-			RlpNode::Extension(partial, cb) => {
-				let key = partial.encoded(false);
-
-				Node::Extension(key, Self::inline_or_hash(cb, db, storage))
+			RlpNode::Leaf(k, v) => Node::Leaf(k, v),
+			RlpNode::Extension(key, cb) => {
+				Node::Extension(key, Self::inline_or_hash(&*cb, db, storage))
 			}
-			RlpNode::Branch(children_rlp, v) => {
-				let val = v.map(|x| x.to_owned());
+			RlpNode::Branch(children_rlp, val) => {
 				let mut children = empty_children();
 
 				for i in 0..16 {
-					let raw = children_rlp[i];
-					let child_rlp = Rlp::new(raw);
+					let raw = &children_rlp[i];
+					let child_rlp = Rlp::new(&*raw);
 					if !child_rlp.is_empty()  {
-						children[i] = Some(Self::inline_or_hash(raw, db, storage));
+						children[i] = Some(Self::inline_or_hash(&*raw, db, storage));
 					}
 				}
 
@@ -134,13 +133,13 @@ impl Node {
 			}
 			Node::Leaf(partial, value) => {
 				let mut stream = RlpStream::new_list(2);
-				stream.append(&partial);
-				stream.append(&value);
+				stream.append(&&*partial);
+				stream.append(&&*value);
 				stream.drain()
 			}
 			Node::Extension(partial, child) => {
 				let mut stream = RlpStream::new_list(2);
-				stream.append(&partial);
+				stream.append(&&*partial);
 				child_cb(child, &mut stream);
 				stream.drain()
 			}
@@ -154,7 +153,7 @@ impl Node {
 					}
 				}
 				if let Some(value) = value {
-					stream.append(&value);
+					stream.append(&&*value);
 				} else {
 					stream.append_empty_data();
 				}
@@ -276,7 +275,7 @@ impl<'a> Index<&'a StorageHandle> for NodeStorage {
 ///   assert_eq!(*t.root(), ::util::sha3::SHA3_NULL_RLP);
 ///   t.insert(b"foo", b"bar").unwrap();
 ///   assert!(t.contains(b"foo").unwrap());
-///   assert_eq!(t.get(b"foo").unwrap().unwrap(), b"bar");
+///   assert_eq!(t.get(b"foo").unwrap().unwrap(), DBValue::from_slice(b"bar"));
 ///   t.remove(b"foo").unwrap();
 ///   assert!(!t.contains(b"foo").unwrap());
 /// }
@@ -338,7 +337,7 @@ impl<'a> TrieDBMut<'a> {
 	// cache a node by hash
 	fn cache(&mut self, hash: H256) -> super::Result<StorageHandle> {
 		let node_rlp = try!(self.db.get(&hash).ok_or_else(|| Box::new(TrieError::IncompleteDatabase(hash))));
-		let node = Node::from_rlp(node_rlp, &*self.db, &mut self.storage);
+		let node = Node::from_rlp(&node_rlp, &*self.db, &mut self.storage);
 		Ok(self.storage.alloc(Stored::Cached(node, hash)))
 	}
 
@@ -367,7 +366,7 @@ impl<'a> TrieDBMut<'a> {
 	}
 
 	// walk the trie, attempting to find the key's node.
-	fn lookup<'x, 'key>(&'x self, partial: NibbleSlice<'key>, handle: &NodeHandle) -> super::Result<Option<&'x [u8]>>
+	fn lookup<'x, 'key>(&'x self, partial: NibbleSlice<'key>, handle: &NodeHandle) -> super::Result<Option<DBValue>>
 		where 'x: 'key
 	{
 		match *handle {
@@ -376,7 +375,7 @@ impl<'a> TrieDBMut<'a> {
 				Node::Empty => Ok(None),
 				Node::Leaf(ref key, ref value) => {
 					if NibbleSlice::from_encoded(key).0 == partial {
-						Ok(Some(value))
+						Ok(Some(DBValue::from_slice(value)))
 					} else {
 						Ok(None)
 					}
@@ -391,7 +390,7 @@ impl<'a> TrieDBMut<'a> {
 				}
 				Node::Branch(ref children, ref value) => {
 					if partial.is_empty() {
-						Ok(value.as_ref().map(|v| &v[..]))
+						Ok(value.as_ref().map(|v| DBValue::from_slice(v)))
 					} else {
 						let idx = partial.at(0);
 						match children[idx as usize].as_ref() {
@@ -405,28 +404,33 @@ impl<'a> TrieDBMut<'a> {
 	}
 
 	/// Return optional data for a key given as a `NibbleSlice`. Returns `None` if no data exists.
-	fn do_db_lookup<'x, 'key>(&'x self, hash: &H256, key: NibbleSlice<'key>) -> super::Result<Option<&'x [u8]>>
+	fn do_db_lookup<'x, 'key>(&'x self, hash: &H256, key: NibbleSlice<'key>) -> super::Result<Option<DBValue>>
 		where 'x: 'key
 	{
 		self.db.get(hash).ok_or_else(|| Box::new(TrieError::IncompleteDatabase(*hash)))
-			.and_then(|node_rlp| self.get_from_db_node(node_rlp, key))
+			.and_then(|node_rlp| self.get_from_db_node(&node_rlp, key))
 	}
 
 	/// Recursible function to retrieve the value given a `node` and a partial `key`. `None` if no
 	/// value exists for the key.
 	///
 	/// Note: Not a public API; use Trie trait functions.
-	fn get_from_db_node<'x, 'key>(&'x self, node: &'x [u8], key: NibbleSlice<'key>) -> super::Result<Option<&'x [u8]>>
+	fn get_from_db_node<'x, 'key>(&'x self, node: &'x [u8], key: NibbleSlice<'key>) -> super::Result<Option<DBValue>>
 		where 'x: 'key
 	{
 		match RlpNode::decoded(node) {
-			RlpNode::Leaf(ref slice, value) if &key == slice => Ok(Some(value)),
-			RlpNode::Extension(ref slice, item) if key.starts_with(slice) => {
-				self.get_from_db_node(try!(self.get_raw_or_lookup(item)), key.mid(slice.len()))
+			RlpNode::Leaf(ref slice, ref value) if NibbleSlice::from_encoded(slice).0 == key => Ok(Some(value.clone())),
+			RlpNode::Extension(ref slice, ref item) => {
+				let slice = &NibbleSlice::from_encoded(slice).0;
+				if key.starts_with(slice) {
+					self.get_from_db_node(&try!(self.get_raw_or_lookup(&*item)), key.mid(slice.len()))
+				} else {
+					Ok(None)
+				}
 			},
-			RlpNode::Branch(ref nodes, value) => match key.is_empty() {
-				true => Ok(value),
-				false => self.get_from_db_node(try!(self.get_raw_or_lookup(nodes[key.at(0) as usize])), key.mid(1))
+			RlpNode::Branch(ref nodes, ref value) => match key.is_empty() {
+				true => Ok(value.clone()),
+				false => self.get_from_db_node(&try!(self.get_raw_or_lookup(&*nodes[key.at(0) as usize])), key.mid(1))
 			},
 			_ => Ok(None),
 		}
@@ -435,7 +439,7 @@ impl<'a> TrieDBMut<'a> {
 	/// Given some node-describing data `node`, return the actual node RLP.
 	/// This could be a simple identity operation in the case that the node is sufficiently small, but
 	/// may require a database lookup.
-	fn get_raw_or_lookup<'x>(&'x self, node: &'x [u8]) -> super::Result<&'x [u8]> {
+	fn get_raw_or_lookup<'x>(&'x self, node: &'x [u8]) -> super::Result<DBValue> {
 		// check if its sha3 + len
 		let r = Rlp::new(node);
 		match r.is_data() && r.size() == 32 {
@@ -443,12 +447,12 @@ impl<'a> TrieDBMut<'a> {
 				let key = r.as_val::<H256>();
 				self.db.get(&key).ok_or_else(|| Box::new(TrieError::IncompleteDatabase(key)))
 			}
-			false => Ok(node)
+			false => Ok(DBValue::from_slice(node))
 		}
 	}
 
 	/// insert a key, value pair into the trie, creating new nodes if necessary.
-	fn insert_at(&mut self, handle: NodeHandle, partial: NibbleSlice, value: Bytes) -> super::Result<(StorageHandle, bool)> {
+	fn insert_at(&mut self, handle: NodeHandle, partial: NibbleSlice, value: DBValue) -> super::Result<(StorageHandle, bool)> {
 		let h = match handle {
 			NodeHandle::InMemory(h) => h,
 			NodeHandle::Hash(h) => try!(self.cache(h)),
@@ -463,7 +467,7 @@ impl<'a> TrieDBMut<'a> {
 
 	/// the insertion inspector.
 	#[cfg_attr(feature = "dev", allow(cyclomatic_complexity))]
-	fn insert_inspector(&mut self, node: Node, partial: NibbleSlice, value: Bytes) -> super::Result<InsertAction> {
+	fn insert_inspector(&mut self, node: Node, partial: NibbleSlice, value: DBValue) -> super::Result<InsertAction> {
 		trace!(target: "trie", "augmented (partial: {:?}, value: {:?})", partial, value.pretty());
 
 		Ok(match node {
@@ -744,7 +748,8 @@ impl<'a> TrieDBMut<'a> {
 					(UsedIndex::One(a), None) => {
 						// only one onward node. make an extension.
 						let new_partial = NibbleSlice::new_offset(&[a], 1).encoded(false);
-						let new_node = Node::Extension(new_partial, children[a as usize].take().unwrap());
+						let child = children[a as usize].take().expect("used_index only set if occupied; qed");
+						let new_node = Node::Extension(new_partial, child);
 						self.fix(new_node)
 					}
 					(UsedIndex::None, Some(value)) => {
@@ -897,7 +902,7 @@ impl<'a> TrieMut for TrieDBMut<'a> {
 		}
 	}
 
-	fn get<'x, 'key>(&'x self, key: &'key [u8]) -> super::Result<Option<&'x [u8]>> where 'x: 'key {
+	fn get<'x, 'key>(&'x self, key: &'key [u8]) -> super::Result<Option<DBValue>> where 'x: 'key {
 		self.lookup(NibbleSlice::new(key), &self.root_handle)
 	}
 
@@ -910,7 +915,7 @@ impl<'a> TrieMut for TrieDBMut<'a> {
 		trace!(target: "trie", "insert: key={:?}, value={:?}", key.pretty(), value.pretty());
 
 		let root_handle = self.root_handle();
-		let (new_handle, changed) = try!(self.insert_at(root_handle, NibbleSlice::new(key), value.to_owned()));
+		let (new_handle, changed) = try!(self.insert_at(root_handle, NibbleSlice::new(key), DBValue::from_slice(value)));
 
 		trace!(target: "trie", "insert: altered trie={}", changed);
 		self.root_handle = NodeHandle::InMemory(new_handle);
@@ -1179,9 +1184,9 @@ mod tests {
 		let mut root = H256::new();
 		let mut t = TrieDBMut::new(&mut memdb, &mut root);
 		t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]).unwrap();
-		assert_eq!(t.get(&[0x1, 0x23]).unwrap().unwrap(), &[0x1u8, 0x23]);
+		assert_eq!(t.get(&[0x1, 0x23]).unwrap().unwrap(), DBValue::from_slice(&[0x1u8, 0x23]));
 		t.commit();
-		assert_eq!(t.get(&[0x1, 0x23]).unwrap().unwrap(), &[0x1u8, 0x23]);
+		assert_eq!(t.get(&[0x1, 0x23]).unwrap().unwrap(), DBValue::from_slice(&[0x1u8, 0x23]));
 	}
 
 	#[test]
@@ -1192,14 +1197,14 @@ mod tests {
 		t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]).unwrap();
 		t.insert(&[0xf1u8, 0x23], &[0xf1u8, 0x23]).unwrap();
 		t.insert(&[0x81u8, 0x23], &[0x81u8, 0x23]).unwrap();
-		assert_eq!(t.get(&[0x01, 0x23]).unwrap().unwrap(), &[0x01u8, 0x23]);
-		assert_eq!(t.get(&[0xf1, 0x23]).unwrap().unwrap(), &[0xf1u8, 0x23]);
-		assert_eq!(t.get(&[0x81, 0x23]).unwrap().unwrap(), &[0x81u8, 0x23]);
+		assert_eq!(t.get(&[0x01, 0x23]).unwrap().unwrap(), DBValue::from_slice(&[0x01u8, 0x23]));
+		assert_eq!(t.get(&[0xf1, 0x23]).unwrap().unwrap(), DBValue::from_slice(&[0xf1u8, 0x23]));
+		assert_eq!(t.get(&[0x81, 0x23]).unwrap().unwrap(), DBValue::from_slice(&[0x81u8, 0x23]));
 		assert_eq!(t.get(&[0x82, 0x23]), Ok(None));
 		t.commit();
-		assert_eq!(t.get(&[0x01, 0x23]).unwrap().unwrap(), &[0x01u8, 0x23]);
-		assert_eq!(t.get(&[0xf1, 0x23]).unwrap().unwrap(), &[0xf1u8, 0x23]);
-		assert_eq!(t.get(&[0x81, 0x23]).unwrap().unwrap(), &[0x81u8, 0x23]);
+		assert_eq!(t.get(&[0x01, 0x23]).unwrap().unwrap(), DBValue::from_slice(&[0x01u8, 0x23]));
+		assert_eq!(t.get(&[0xf1, 0x23]).unwrap().unwrap(), DBValue::from_slice(&[0xf1u8, 0x23]));
+		assert_eq!(t.get(&[0x81, 0x23]).unwrap().unwrap(), DBValue::from_slice(&[0x81u8, 0x23]));
 		assert_eq!(t.get(&[0x82, 0x23]), Ok(None));
 	}