grassrootseconomics/openethereum
8
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Merge branch 'master' of github.com:ethcore/parity into move_hash

Browse Source
This commit is contained in:
debris
2016-07-30 14:14:25 +02:00
parent 4dc1d42a93 85c471b905
commit e9b3740ea8
154 changed files with 6895 additions and 3541 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
util/bigint/src/hash.rs
View File

@@ -62,7 +62,7 @@ pub trait FixedHash: Sized + FromStr + Default + DerefMut<Target = [u8]> {
/// Return `s` without the `0x` at the beginning of it, if any.
pub fn clean_0x(s: &str) -> &str {
if s.len() >= 2 && &s[0..2] == "0x" {
if s.starts_with("0x") {
&s[2..]
} else {
s
@@ -436,13 +436,13 @@ macro_rules! impl_hash {
}
}
impl<'a> From<&'a str> for $from {
fn from(s: &'a str) -> $from {
use std::str::FromStr;
impl From<&'static str> for $from {
fn from(s: &'static str) -> $from {
let s = clean_0x(s);
if s.len() % 2 == 1 {
$from::from_str(&("0".to_owned() + &(clean_0x(s).to_owned()))[..]).unwrap_or_else(|_| $from::new())
$from::from_str(&("0".to_owned() + s)).unwrap()
} else {
$from::from_str(clean_0x(s)).unwrap_or_else(|_| $from::new())
$from::from_str(s).unwrap()
}
}
}
@@ -620,8 +620,6 @@ mod tests {
assert_eq!(H64::from(0x1234567890abcdef), H64::from("0x1234567890abcdef"));
assert_eq!(H64::from(0x1234567890abcdef), H64::from("1234567890abcdef"));
assert_eq!(H64::from(0x234567890abcdef), H64::from("0x234567890abcdef"));
// too short.
assert_eq!(H64::from(0), H64::from("0x34567890abcdef"));
}
#[test]

3
util/bigint/src/uint.rs
View File

@@ -565,7 +565,7 @@ macro_rules! construct_uint {
impl Uint for $name {
fn from_dec_str(value: &str) -> Result<Self, FromDecStrErr> {
if value.bytes().any(|b| b < 48 && b > 57) {
if !value.bytes().all(|b| b >= 48 && b <= 57) {
return Err(FromDecStrErr::InvalidCharacter)
}
@@ -1788,6 +1788,7 @@ mod tests {
assert_eq!(U256::from_dec_str("10").unwrap(), U256::from(10u64));
assert_eq!(U256::from_dec_str("1024").unwrap(), U256::from(1024u64));
assert_eq!(U256::from_dec_str("115792089237316195423570985008687907853269984665640564039457584007913129639936"), Err(FromDecStrErr::InvalidLength));
assert_eq!(U256::from_dec_str("0x11"), Err(FromDecStrErr::InvalidCharacter));
}
#[test]

9
util/fdlimit/src/lib.rs
View File

@@ -1,18 +1,19 @@
// Copyright 2015, 2016 Ethcore (UK) Ltd.
// This file is part of Parity.
//
//
// Parity is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
//
// Parity is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
//
// You should have received a copy of the GNU General Public License
// along with Parity. If not, see <http://www.gnu.org/licenses/>.extern crate libc;
extern crate libc;
pub mod raise_fd_limit;
mod raise_fd_limit;
pub use raise_fd_limit::raise_fd_limit;

106
util/fdlimit/src/raise_fd_limit.rs
View File

@@ -15,70 +15,74 @@
///
#[cfg(any(target_os = "macos", target_os = "ios"))]
#[allow(non_camel_case_types)]
pub unsafe fn raise_fd_limit() {
use libc;
use std::cmp;
use std::io;
use std::mem::size_of_val;
use std::ptr::null_mut;
pub fn raise_fd_limit() {
use libc;
use std::cmp;
use std::io;
use std::mem::size_of_val;
use std::ptr::null_mut;
static CTL_KERN: libc::c_int = 1;
static KERN_MAXFILESPERPROC: libc::c_int = 29;
unsafe {
static CTL_KERN: libc::c_int = 1;
static KERN_MAXFILESPERPROC: libc::c_int = 29;
// The strategy here is to fetch the current resource limits, read the
// kern.maxfilesperproc sysctl value, and bump the soft resource limit for
// maxfiles up to the sysctl value.
// The strategy here is to fetch the current resource limits, read the
// kern.maxfilesperproc sysctl value, and bump the soft resource limit for
// maxfiles up to the sysctl value.
// Fetch the kern.maxfilesperproc value
let mut mib: [libc::c_int; 2] = [CTL_KERN, KERN_MAXFILESPERPROC];
let mut maxfiles: libc::c_int = 0;
let mut size: libc::size_t = size_of_val(&maxfiles) as libc::size_t;
if libc::sysctl(&mut mib[0], 2, &mut maxfiles as *mut _ as *mut _, &mut size,
null_mut(), 0) != 0 {
let err = io::Error::last_os_error();
panic!("raise_fd_limit: error calling sysctl: {}", err);
}
// Fetch the kern.maxfilesperproc value
let mut mib: [libc::c_int; 2] = [CTL_KERN, KERN_MAXFILESPERPROC];
let mut maxfiles: libc::c_int = 0;
let mut size: libc::size_t = size_of_val(&maxfiles) as libc::size_t;
if libc::sysctl(&mut mib[0], 2, &mut maxfiles as *mut _ as *mut _, &mut size,
null_mut(), 0) != 0 {
let err = io::Error::last_os_error();
panic!("raise_fd_limit: error calling sysctl: {}", err);
}
// Fetch the current resource limits
let mut rlim = libc::rlimit{rlim_cur: 0, rlim_max: 0};
if libc::getrlimit(libc::RLIMIT_NOFILE, &mut rlim) != 0 {
let err = io::Error::last_os_error();
panic!("raise_fd_limit: error calling getrlimit: {}", err);
}
// Fetch the current resource limits
let mut rlim = libc::rlimit{rlim_cur: 0, rlim_max: 0};
if libc::getrlimit(libc::RLIMIT_NOFILE, &mut rlim) != 0 {
let err = io::Error::last_os_error();
panic!("raise_fd_limit: error calling getrlimit: {}", err);
}
// Bump the soft limit to the smaller of kern.maxfilesperproc and the hard
// limit
rlim.rlim_cur = cmp::min(maxfiles as libc::rlim_t, rlim.rlim_max);
// Bump the soft limit to the smaller of kern.maxfilesperproc and the hard
// limit
rlim.rlim_cur = cmp::min(maxfiles as libc::rlim_t, rlim.rlim_max);
// Set our newly-increased resource limit
if libc::setrlimit(libc::RLIMIT_NOFILE, &rlim) != 0 {
let err = io::Error::last_os_error();
panic!("raise_fd_limit: error calling setrlimit: {}", err);
}
// Set our newly-increased resource limit
if libc::setrlimit(libc::RLIMIT_NOFILE, &rlim) != 0 {
let err = io::Error::last_os_error();
panic!("raise_fd_limit: error calling setrlimit: {}", err);
}
}
}
#[cfg(any(target_os = "linux"))]
#[allow(non_camel_case_types)]
pub unsafe fn raise_fd_limit() {
use libc;
use std::io;
pub fn raise_fd_limit() {
use libc;
use std::io;
// Fetch the current resource limits
let mut rlim = libc::rlimit{rlim_cur: 0, rlim_max: 0};
if libc::getrlimit(libc::RLIMIT_NOFILE, &mut rlim) != 0 {
let err = io::Error::last_os_error();
panic!("raise_fd_limit: error calling getrlimit: {}", err);
}
unsafe {
// Fetch the current resource limits
let mut rlim = libc::rlimit{rlim_cur: 0, rlim_max: 0};
if libc::getrlimit(libc::RLIMIT_NOFILE, &mut rlim) != 0 {
let err = io::Error::last_os_error();
panic!("raise_fd_limit: error calling getrlimit: {}", err);
}
// Set soft limit to hard imit
rlim.rlim_cur = rlim.rlim_max;
// Set soft limit to hard imit
rlim.rlim_cur = rlim.rlim_max;
// Set our newly-increased resource limit
if libc::setrlimit(libc::RLIMIT_NOFILE, &rlim) != 0 {
let err = io::Error::last_os_error();
panic!("raise_fd_limit: error calling setrlimit: {}", err);
}
// Set our newly-increased resource limit
if libc::setrlimit(libc::RLIMIT_NOFILE, &rlim) != 0 {
let err = io::Error::last_os_error();
panic!("raise_fd_limit: error calling setrlimit: {}", err);
}
}
}
#[cfg(not(any(target_os = "macos", target_os = "ios", target_os = "linux")))]
pub unsafe fn raise_fd_limit() {}
pub fn raise_fd_limit() {}

2
util/src/bytes.rs
View File

@@ -195,7 +195,7 @@ pub trait Populatable {
/// If `d` is smaller, will leave some bytes untouched.
fn copy_raw(&mut self, d: &[u8]) {
use std::io::Write;
self.as_slice_mut().write(&d).unwrap();
self.as_slice_mut().write(d).unwrap();
}
/// Copies the raw representation of an object `d` to `self`, overwriting as necessary.

2
util/src/crypto.rs
View File

@@ -275,7 +275,7 @@ pub mod ecdh {
let publ = try!(key::PublicKey::from_slice(context, &pdata));
// no way to create SecretKey from raw byte array.
let sec: &key::SecretKey = unsafe { ::std::mem::transmute(secret) };
let shared = ecdh::SharedSecret::new_raw(context, &publ, &sec);
let shared = ecdh::SharedSecret::new_raw(context, &publ, sec);
let mut s = crypto::Secret::new();
s.copy_from_slice(&shared[0..32]);

2
util/src/error.rs
View File

@@ -82,7 +82,7 @@ impl fmt::Display for UtilError {
UtilError::BaseData(ref err) => f.write_fmt(format_args!("{}", err)),
UtilError::Network(ref err) => f.write_fmt(format_args!("{}", err)),
UtilError::Decoder(ref err) => f.write_fmt(format_args!("{}", err)),
UtilError::SimpleString(ref msg) => f.write_str(&msg),
UtilError::SimpleString(ref msg) => f.write_str(msg),
UtilError::BadSize => f.write_str("Bad input size."),
UtilError::Snappy(ref err) => f.write_fmt(format_args!("{}", err)),
}

9
util/src/io/service.rs
View File

@@ -25,7 +25,8 @@ use io::{IoError, IoHandler};
use io::worker::{Worker, Work, WorkType};
use panics::*;
use parking_lot::{Condvar, RwLock, Mutex};
use parking_lot::{RwLock};
use std::sync::{Condvar as SCondvar, Mutex as SMutex};
/// Timer ID
pub type TimerToken = usize;
@@ -169,7 +170,7 @@ pub struct IoManager<Message> where Message: Send + Sync {
handlers: Slab<Arc<IoHandler<Message>>, HandlerId>,
workers: Vec<Worker>,
worker_channel: chase_lev::Worker<Work<Message>>,
work_ready: Arc<Condvar>,
work_ready: Arc<SCondvar>,
}
impl<Message> IoManager<Message> where Message: Send + Sync + Clone + 'static {
@@ -177,8 +178,8 @@ impl<Message> IoManager<Message> where Message: Send + Sync + Clone + 'static {
pub fn start(panic_handler: Arc<PanicHandler>, event_loop: &mut EventLoop<IoManager<Message>>) -> Result<(), UtilError> {
let (worker, stealer) = chase_lev::deque();
let num_workers = 4;
let work_ready_mutex = Arc::new(Mutex::new(()));
let work_ready = Arc::new(Condvar::new());
let work_ready_mutex = Arc::new(SMutex::new(()));
let work_ready = Arc::new(SCondvar::new());
let workers = (0..num_workers).map(|i|
Worker::new(
i,

20
util/src/io/worker.rs
View File

@@ -23,7 +23,7 @@ use io::service::{HandlerId, IoChannel, IoContext};
use io::{IoHandler};
use panics::*;
use parking_lot::{Condvar, Mutex};
use std::sync::{Condvar as SCondvar, Mutex as SMutex};
pub enum WorkType<Message> {
Readable,
@@ -44,9 +44,9 @@ pub struct Work<Message> {
/// Sorts them ready for blockchain insertion.
pub struct Worker {
thread: Option<JoinHandle<()>>,
wait: Arc<Condvar>,
wait: Arc<SCondvar>,
deleting: Arc<AtomicBool>,
wait_mutex: Arc<Mutex<()>>,
wait_mutex: Arc<SMutex<()>>,
}
impl Worker {
@@ -54,8 +54,8 @@ impl Worker {
pub fn new<Message>(index: usize,
stealer: chase_lev::Stealer<Work<Message>>,
channel: IoChannel<Message>,
wait: Arc<Condvar>,
wait_mutex: Arc<Mutex<()>>,
wait: Arc<SCondvar>,
wait_mutex: Arc<SMutex<()>>,
panic_handler: Arc<PanicHandler>
) -> Worker
where Message: Send + Sync + Clone + 'static {
@@ -77,17 +77,17 @@ impl Worker {
}
fn work_loop<Message>(stealer: chase_lev::Stealer<Work<Message>>,
channel: IoChannel<Message>, wait: Arc<Condvar>,
wait_mutex: Arc<Mutex<()>>,
channel: IoChannel<Message>, wait: Arc<SCondvar>,
wait_mutex: Arc<SMutex<()>>,
deleting: Arc<AtomicBool>)
where Message: Send + Sync + Clone + 'static {
loop {
{
let mut lock = wait_mutex.lock();
let lock = wait_mutex.lock().unwrap();
if deleting.load(AtomicOrdering::Acquire) {
return;
}
wait.wait(&mut lock);
let _ = wait.wait(lock);
}
if deleting.load(AtomicOrdering::Acquire) {
@@ -123,7 +123,7 @@ impl Worker {
impl Drop for Worker {
fn drop(&mut self) {
trace!(target: "shutdown", "[IoWorker] Closing...");
let _ = self.wait_mutex.lock();
let _ = self.wait_mutex.lock().unwrap();
self.deleting.store(true, AtomicOrdering::Release);
self.wait.notify_all();
let thread = mem::replace(&mut self.thread, None).unwrap();

166
util/src/journaldb/archivedb.rs
View File

@@ -20,9 +20,9 @@ use common::*;
use rlp::*;
use hashdb::*;
use memorydb::*;
use super::{DB_PREFIX_LEN, LATEST_ERA_KEY, VERSION_KEY};
use super::{DB_PREFIX_LEN, LATEST_ERA_KEY};
use super::traits::JournalDB;
use kvdb::{Database, DBTransaction, DatabaseConfig};
use kvdb::{Database, DBTransaction};
#[cfg(test)]
use std::env;
@@ -30,9 +30,6 @@ use std::env;
/// Would be nich to use rocksdb columns for this eventually.
const AUX_FLAG: u8 = 255;
/// Database version.
const DB_VERSION : u32 = 0x103;
/// Implementation of the `HashDB` trait for a disk-backed database with a memory overlay
/// and latent-removal semantics.
///
@@ -44,28 +41,18 @@ pub struct ArchiveDB {
overlay: MemoryDB,
backing: Arc<Database>,
latest_era: Option<u64>,
column: Option<u32>,
}
impl ArchiveDB {
/// Create a new instance from file
pub fn new(path: &str, config: DatabaseConfig) -> ArchiveDB {
let backing = Database::open(&config, path).unwrap_or_else(|e| {
panic!("Error opening state db: {}", e);
});
if !backing.is_empty() {
match backing.get(&VERSION_KEY).map(|d| d.map(|v| decode::<u32>(&v))) {
Ok(Some(DB_VERSION)) => {},
v => panic!("Incompatible DB version, expected {}, got {:?}; to resolve, remove {} and restart.", DB_VERSION, v, path)
}
} else {
backing.put(&VERSION_KEY, &encode(&DB_VERSION)).expect("Error writing version to database");
}
let latest_era = backing.get(&LATEST_ERA_KEY).expect("Low-level database error.").map(|val| decode::<u64>(&val));
pub fn new(backing: Arc<Database>, col: Option<u32>) -> ArchiveDB {
let latest_era = backing.get(col, &LATEST_ERA_KEY).expect("Low-level database error.").map(|val| decode::<u64>(&val));
ArchiveDB {
overlay: MemoryDB::new(),
backing: Arc::new(backing),
backing: backing,
latest_era: latest_era,
column: col,
}
}
@@ -74,18 +61,19 @@ impl ArchiveDB {
fn new_temp() -> ArchiveDB {
let mut dir = env::temp_dir();
dir.push(H32::random().hex());
Self::new(dir.to_str().unwrap(), DatabaseConfig::default())
let backing = Arc::new(Database::open_default(dir.to_str().unwrap()).unwrap());
Self::new(backing, None)
}
fn payload(&self, key: &H256) -> Option<Bytes> {
self.backing.get(key).expect("Low-level database error. Some issue with your hard disk?").map(|v| v.to_vec())
self.backing.get(self.column, key).expect("Low-level database error. Some issue with your hard disk?").map(|v| v.to_vec())
}
}
impl HashDB for ArchiveDB {
fn keys(&self) -> HashMap<H256, i32> {
let mut ret: HashMap<H256, i32> = HashMap::new();
for (key, _) in self.backing.iter() {
for (key, _) in self.backing.iter(self.column) {
let h = H256::from_slice(key.deref());
ret.insert(h, 1);
}
@@ -140,7 +128,7 @@ impl HashDB for ArchiveDB {
let mut db_hash = hash.to_vec();
db_hash.push(AUX_FLAG);
self.backing.get(&db_hash)
self.backing.get(self.column, &db_hash)
.expect("Low-level database error. Some issue with your hard disk?")
.map(|v| v.to_vec())
}
@@ -156,6 +144,7 @@ impl JournalDB for ArchiveDB {
overlay: self.overlay.clone(),
backing: self.backing.clone(),
latest_era: self.latest_era,
column: self.column.clone(),
})
}
@@ -167,8 +156,7 @@ impl JournalDB for ArchiveDB {
self.latest_era.is_none()
}
fn commit(&mut self, now: u64, _: &H256, _: Option<(u64, H256)>) -> Result<u32, UtilError> {
let batch = DBTransaction::new();
fn commit(&mut self, batch: &DBTransaction, now: u64, _id: &H256, _end: Option<(u64, H256)>) -> Result<u32, UtilError> {
let mut inserts = 0usize;
let mut deletes = 0usize;
@@ -176,7 +164,7 @@ impl JournalDB for ArchiveDB {
let (key, (value, rc)) = i;
if rc > 0 {
assert!(rc == 1);
batch.put(&key, &value).expect("Low-level database error. Some issue with your hard disk?");
batch.put(self.column, &key, &value).expect("Low-level database error. Some issue with your hard disk?");
inserts += 1;
}
if rc < 0 {
@@ -187,24 +175,27 @@ impl JournalDB for ArchiveDB {
for (mut key, value) in self.overlay.drain_aux().into_iter() {
key.push(AUX_FLAG);
batch.put(&key, &value).expect("Low-level database error. Some issue with your hard disk?");
batch.put(self.column, &key, &value).expect("Low-level database error. Some issue with your hard disk?");
}
if self.latest_era.map_or(true, |e| now > e) {
try!(batch.put(&LATEST_ERA_KEY, &encode(&now)));
try!(batch.put(self.column, &LATEST_ERA_KEY, &encode(&now)));
self.latest_era = Some(now);
}
try!(self.backing.write(batch));
Ok((inserts + deletes) as u32)
}
fn latest_era(&self) -> Option<u64> { self.latest_era }
fn state(&self, id: &H256) -> Option<Bytes> {
self.backing.get_by_prefix(&id[0..DB_PREFIX_LEN]).map(|b| b.to_vec())
self.backing.get_by_prefix(self.column, &id[0..DB_PREFIX_LEN]).map(|b| b.to_vec())
}
fn is_pruned(&self) -> bool { false }
fn backing(&self) -> &Arc<Database> {
&self.backing
}
}
#[cfg(test)]
@@ -216,7 +207,7 @@ mod tests {
use super::*;
use hashdb::*;
use journaldb::traits::JournalDB;
use kvdb::DatabaseConfig;
use kvdb::Database;
#[test]
fn insert_same_in_fork() {
@@ -224,18 +215,18 @@ mod tests {
let mut jdb = ArchiveDB::new_temp();
let x = jdb.insert(b"X");
jdb.commit(1, &b"1".sha3(), None).unwrap();
jdb.commit(2, &b"2".sha3(), None).unwrap();
jdb.commit(3, &b"1002a".sha3(), Some((1, b"1".sha3()))).unwrap();
jdb.commit(4, &b"1003a".sha3(), Some((2, b"2".sha3()))).unwrap();
jdb.commit_batch(1, &b"1".sha3(), None).unwrap();
jdb.commit_batch(2, &b"2".sha3(), None).unwrap();
jdb.commit_batch(3, &b"1002a".sha3(), Some((1, b"1".sha3()))).unwrap();
jdb.commit_batch(4, &b"1003a".sha3(), Some((2, b"2".sha3()))).unwrap();
jdb.remove(&x);
jdb.commit(3, &b"1002b".sha3(), Some((1, b"1".sha3()))).unwrap();
jdb.commit_batch(3, &b"1002b".sha3(), Some((1, b"1".sha3()))).unwrap();
let x = jdb.insert(b"X");
jdb.commit(4, &b"1003b".sha3(), Some((2, b"2".sha3()))).unwrap();
jdb.commit_batch(4, &b"1003b".sha3(), Some((2, b"2".sha3()))).unwrap();
jdb.commit(5, &b"1004a".sha3(), Some((3, b"1002a".sha3()))).unwrap();
jdb.commit(6, &b"1005a".sha3(), Some((4, b"1003a".sha3()))).unwrap();
jdb.commit_batch(5, &b"1004a".sha3(), Some((3, b"1002a".sha3()))).unwrap();
jdb.commit_batch(6, &b"1005a".sha3(), Some((4, b"1003a".sha3()))).unwrap();
assert!(jdb.contains(&x));
}
@@ -245,16 +236,16 @@ mod tests {
// history is 3
let mut jdb = ArchiveDB::new_temp();
let h = jdb.insert(b"foo");
jdb.commit(0, &b"0".sha3(), None).unwrap();
jdb.commit_batch(0, &b"0".sha3(), None).unwrap();
assert!(jdb.contains(&h));
jdb.remove(&h);
jdb.commit(1, &b"1".sha3(), None).unwrap();
jdb.commit_batch(1, &b"1".sha3(), None).unwrap();
assert!(jdb.contains(&h));
jdb.commit(2, &b"2".sha3(), None).unwrap();
jdb.commit_batch(2, &b"2".sha3(), None).unwrap();
assert!(jdb.contains(&h));
jdb.commit(3, &b"3".sha3(), Some((0, b"0".sha3()))).unwrap();
jdb.commit_batch(3, &b"3".sha3(), Some((0, b"0".sha3()))).unwrap();
assert!(jdb.contains(&h));
jdb.commit(4, &b"4".sha3(), Some((1, b"1".sha3()))).unwrap();
jdb.commit_batch(4, &b"4".sha3(), Some((1, b"1".sha3()))).unwrap();
}
#[test]
@@ -264,29 +255,29 @@ mod tests {
let foo = jdb.insert(b"foo");
let bar = jdb.insert(b"bar");
jdb.commit(0, &b"0".sha3(), None).unwrap();
jdb.commit_batch(0, &b"0".sha3(), None).unwrap();
assert!(jdb.contains(&foo));
assert!(jdb.contains(&bar));
jdb.remove(&foo);
jdb.remove(&bar);
let baz = jdb.insert(b"baz");
jdb.commit(1, &b"1".sha3(), Some((0, b"0".sha3()))).unwrap();
jdb.commit_batch(1, &b"1".sha3(), Some((0, b"0".sha3()))).unwrap();
assert!(jdb.contains(&foo));
assert!(jdb.contains(&bar));
assert!(jdb.contains(&baz));
let foo = jdb.insert(b"foo");
jdb.remove(&baz);
jdb.commit(2, &b"2".sha3(), Some((1, b"1".sha3()))).unwrap();
jdb.commit_batch(2, &b"2".sha3(), Some((1, b"1".sha3()))).unwrap();
assert!(jdb.contains(&foo));
assert!(jdb.contains(&baz));
jdb.remove(&foo);
jdb.commit(3, &b"3".sha3(), Some((2, b"2".sha3()))).unwrap();
jdb.commit_batch(3, &b"3".sha3(), Some((2, b"2".sha3()))).unwrap();
assert!(jdb.contains(&foo));
jdb.commit(4, &b"4".sha3(), Some((3, b"3".sha3()))).unwrap();
jdb.commit_batch(4, &b"4".sha3(), Some((3, b"3".sha3()))).unwrap();
}
#[test]
@@ -296,22 +287,22 @@ mod tests {
let foo = jdb.insert(b"foo");
let bar = jdb.insert(b"bar");
jdb.commit(0, &b"0".sha3(), None).unwrap();
jdb.commit_batch(0, &b"0".sha3(), None).unwrap();
assert!(jdb.contains(&foo));
assert!(jdb.contains(&bar));
jdb.remove(&foo);
let baz = jdb.insert(b"baz");
jdb.commit(1, &b"1a".sha3(), Some((0, b"0".sha3()))).unwrap();
jdb.commit_batch(1, &b"1a".sha3(), Some((0, b"0".sha3()))).unwrap();
jdb.remove(&bar);
jdb.commit(1, &b"1b".sha3(), Some((0, b"0".sha3()))).unwrap();
jdb.commit_batch(1, &b"1b".sha3(), Some((0, b"0".sha3()))).unwrap();
assert!(jdb.contains(&foo));
assert!(jdb.contains(&bar));
assert!(jdb.contains(&baz));
jdb.commit(2, &b"2b".sha3(), Some((1, b"1b".sha3()))).unwrap();
jdb.commit_batch(2, &b"2b".sha3(), Some((1, b"1b".sha3()))).unwrap();
assert!(jdb.contains(&foo));
}
@@ -321,16 +312,16 @@ mod tests {
let mut jdb = ArchiveDB::new_temp();
let foo = jdb.insert(b"foo");
jdb.commit(0, &b"0".sha3(), None).unwrap();
jdb.commit_batch(0, &b"0".sha3(), None).unwrap();
assert!(jdb.contains(&foo));
jdb.remove(&foo);
jdb.commit(1, &b"1".sha3(), Some((0, b"0".sha3()))).unwrap();
jdb.commit_batch(1, &b"1".sha3(), Some((0, b"0".sha3()))).unwrap();
jdb.insert(b"foo");
assert!(jdb.contains(&foo));
jdb.commit(2, &b"2".sha3(), Some((1, b"1".sha3()))).unwrap();
jdb.commit_batch(2, &b"2".sha3(), Some((1, b"1".sha3()))).unwrap();
assert!(jdb.contains(&foo));
jdb.commit(3, &b"2".sha3(), Some((0, b"2".sha3()))).unwrap();
jdb.commit_batch(3, &b"2".sha3(), Some((0, b"2".sha3()))).unwrap();
assert!(jdb.contains(&foo));
}
@@ -338,19 +329,24 @@ mod tests {
fn fork_same_key() {
// history is 1
let mut jdb = ArchiveDB::new_temp();
jdb.commit(0, &b"0".sha3(), None).unwrap();
jdb.commit_batch(0, &b"0".sha3(), None).unwrap();
let foo = jdb.insert(b"foo");
jdb.commit(1, &b"1a".sha3(), Some((0, b"0".sha3()))).unwrap();
jdb.commit_batch(1, &b"1a".sha3(), Some((0, b"0".sha3()))).unwrap();
jdb.insert(b"foo");
jdb.commit(1, &b"1b".sha3(), Some((0, b"0".sha3()))).unwrap();
jdb.commit_batch(1, &b"1b".sha3(), Some((0, b"0".sha3()))).unwrap();
assert!(jdb.contains(&foo));
jdb.commit(2, &b"2a".sha3(), Some((1, b"1a".sha3()))).unwrap();
jdb.commit_batch(2, &b"2a".sha3(), Some((1, b"1a".sha3()))).unwrap();
assert!(jdb.contains(&foo));
}
fn new_db(dir: &Path) -> ArchiveDB {
let db = Database::open_default(dir.to_str().unwrap()).unwrap();
ArchiveDB::new(Arc::new(db), None)
}
#[test]
fn reopen() {
let mut dir = ::std::env::temp_dir();
@@ -358,25 +354,25 @@ mod tests {
let bar = H256::random();
let foo = {
let mut jdb = ArchiveDB::new(dir.to_str().unwrap(), DatabaseConfig::default());
let mut jdb = new_db(&dir);
// history is 1
let foo = jdb.insert(b"foo");
jdb.emplace(bar.clone(), b"bar".to_vec());
jdb.commit(0, &b"0".sha3(), None).unwrap();
jdb.commit_batch(0, &b"0".sha3(), None).unwrap();
foo
};
{
let mut jdb = ArchiveDB::new(dir.to_str().unwrap(), DatabaseConfig::default());
let mut jdb = new_db(&dir);
jdb.remove(&foo);
jdb.commit(1, &b"1".sha3(), Some((0, b"0".sha3()))).unwrap();
jdb.commit_batch(1, &b"1".sha3(), Some((0, b"0".sha3()))).unwrap();
}
{
let mut jdb = ArchiveDB::new(dir.to_str().unwrap(), DatabaseConfig::default());
let mut jdb = new_db(&dir);
assert!(jdb.contains(&foo));
assert!(jdb.contains(&bar));
jdb.commit(2, &b"2".sha3(), Some((1, b"1".sha3()))).unwrap();
jdb.commit_batch(2, &b"2".sha3(), Some((1, b"1".sha3()))).unwrap();
}
}
@@ -386,27 +382,27 @@ mod tests {
dir.push(H32::random().hex());
let foo = {
let mut jdb = ArchiveDB::new(dir.to_str().unwrap(), DatabaseConfig::default());
let mut jdb = new_db(&dir);
// history is 1
let foo = jdb.insert(b"foo");
jdb.commit(0, &b"0".sha3(), None).unwrap();
jdb.commit(1, &b"1".sha3(), Some((0, b"0".sha3()))).unwrap();
jdb.commit_batch(0, &b"0".sha3(), None).unwrap();
jdb.commit_batch(1, &b"1".sha3(), Some((0, b"0".sha3()))).unwrap();
// foo is ancient history.
jdb.insert(b"foo");
jdb.commit(2, &b"2".sha3(), Some((1, b"1".sha3()))).unwrap();
jdb.commit_batch(2, &b"2".sha3(), Some((1, b"1".sha3()))).unwrap();
foo
};
{
let mut jdb = ArchiveDB::new(dir.to_str().unwrap(), DatabaseConfig::default());
let mut jdb = new_db(&dir);
jdb.remove(&foo);
jdb.commit(3, &b"3".sha3(), Some((2, b"2".sha3()))).unwrap();
jdb.commit_batch(3, &b"3".sha3(), Some((2, b"2".sha3()))).unwrap();
assert!(jdb.contains(&foo));
jdb.remove(&foo);
jdb.commit(4, &b"4".sha3(), Some((3, b"3".sha3()))).unwrap();
jdb.commit(5, &b"5".sha3(), Some((4, b"4".sha3()))).unwrap();
jdb.commit_batch(4, &b"4".sha3(), Some((3, b"3".sha3()))).unwrap();
jdb.commit_batch(5, &b"5".sha3(), Some((4, b"4".sha3()))).unwrap();
}
}
@@ -415,23 +411,23 @@ mod tests {
let mut dir = ::std::env::temp_dir();
dir.push(H32::random().hex());
let (foo, _, _) = {
let mut jdb = ArchiveDB::new(dir.to_str().unwrap(), DatabaseConfig::default());
let mut jdb = new_db(&dir);
// history is 1
let foo = jdb.insert(b"foo");
let bar = jdb.insert(b"bar");
jdb.commit(0, &b"0".sha3(), None).unwrap();
jdb.commit_batch(0, &b"0".sha3(), None).unwrap();
jdb.remove(&foo);
let baz = jdb.insert(b"baz");
jdb.commit(1, &b"1a".sha3(), Some((0, b"0".sha3()))).unwrap();
jdb.commit_batch(1, &b"1a".sha3(), Some((0, b"0".sha3()))).unwrap();
jdb.remove(&bar);
jdb.commit(1, &b"1b".sha3(), Some((0, b"0".sha3()))).unwrap();
jdb.commit_batch(1, &b"1b".sha3(), Some((0, b"0".sha3()))).unwrap();
(foo, bar, baz)
};
{
let mut jdb = ArchiveDB::new(dir.to_str().unwrap(), DatabaseConfig::default());
jdb.commit(2, &b"2b".sha3(), Some((1, b"1b".sha3()))).unwrap();
let mut jdb = new_db(&dir);
jdb.commit_batch(2, &b"2b".sha3(), Some((1, b"1b".sha3()))).unwrap();
assert!(jdb.contains(&foo));
}
}
@@ -441,14 +437,14 @@ mod tests {
let temp = ::devtools::RandomTempPath::new();
let key = {
let mut jdb = ArchiveDB::new(temp.as_str(), DatabaseConfig::default());
let mut jdb = new_db(temp.as_path().as_path());
let key = jdb.insert(b"foo");
jdb.commit(0, &b"0".sha3(), None).unwrap();
jdb.commit_batch(0, &b"0".sha3(), None).unwrap();
key
};
{
let jdb = ArchiveDB::new(temp.as_str(), DatabaseConfig::default());
let jdb = new_db(temp.as_path().as_path());
let state = jdb.state(&key);
assert!(state.is_some());
}

352
util/src/journaldb/earlymergedb.rs
View File

@@ -20,9 +20,9 @@ use common::*;
use rlp::*;
use hashdb::*;
use memorydb::*;
use super::{DB_PREFIX_LEN, LATEST_ERA_KEY, VERSION_KEY};
use super::{DB_PREFIX_LEN, LATEST_ERA_KEY};
use super::traits::JournalDB;
use kvdb::{Database, DBTransaction, DatabaseConfig};
use kvdb::{Database, DBTransaction};
#[cfg(test)]
use std::env;
@@ -66,33 +66,22 @@ pub struct EarlyMergeDB {
backing: Arc<Database>,
refs: Option<Arc<RwLock<HashMap<H256, RefInfo>>>>,
latest_era: Option<u64>,
column: Option<u32>,
}
const DB_VERSION : u32 = 0x003;
const PADDING : [u8; 10] = [ 0u8; 10 ];
impl EarlyMergeDB {
/// Create a new instance from file
pub fn new(path: &str, config: DatabaseConfig) -> EarlyMergeDB {
let backing = Database::open(&config, path).unwrap_or_else(|e| {
panic!("Error opening state db: {}", e);
});
if !backing.is_empty() {
match backing.get(&VERSION_KEY).map(|d| d.map(|v| decode::<u32>(&v))) {
Ok(Some(DB_VERSION)) => {},
v => panic!("Incompatible DB version, expected {}, got {:?}; to resolve, remove {} and restart.", DB_VERSION, v, path)
}
} else {
backing.put(&VERSION_KEY, &encode(&DB_VERSION)).expect("Error writing version to database");
}
let (latest_era, refs) = EarlyMergeDB::read_refs(&backing);
pub fn new(backing: Arc<Database>, col: Option<u32>) -> EarlyMergeDB {
let (latest_era, refs) = EarlyMergeDB::read_refs(&backing, col);
let refs = Some(Arc::new(RwLock::new(refs)));
EarlyMergeDB {
overlay: MemoryDB::new(),
backing: Arc::new(backing),
backing: backing,
refs: refs,
latest_era: latest_era,
column: col,
}
}
@@ -101,7 +90,8 @@ impl EarlyMergeDB {
fn new_temp() -> EarlyMergeDB {
let mut dir = env::temp_dir();
dir.push(H32::random().hex());
Self::new(dir.to_str().unwrap(), DatabaseConfig::default())
let backing = Arc::new(Database::open_default(dir.to_str().unwrap()).unwrap());
Self::new(backing, None)
}
fn morph_key(key: &H256, index: u8) -> Bytes {
@@ -111,13 +101,13 @@ impl EarlyMergeDB {
}
// The next three are valid only as long as there is an insert operation of `key` in the journal.
fn set_already_in(batch: &DBTransaction, key: &H256) { batch.put(&Self::morph_key(key, 0), &[1u8]).expect("Low-level database error. Some issue with your hard disk?"); }
fn reset_already_in(batch: &DBTransaction, key: &H256) { batch.delete(&Self::morph_key(key, 0)).expect("Low-level database error. Some issue with your hard disk?"); }
fn is_already_in(backing: &Database, key: &H256) -> bool {
backing.get(&Self::morph_key(key, 0)).expect("Low-level database error. Some issue with your hard disk?").is_some()
fn set_already_in(batch: &DBTransaction, col: Option<u32>, key: &H256) { batch.put(col, &Self::morph_key(key, 0), &[1u8]).expect("Low-level database error. Some issue with your hard disk?"); }
fn reset_already_in(batch: &DBTransaction, col: Option<u32>, key: &H256) { batch.delete(col, &Self::morph_key(key, 0)).expect("Low-level database error. Some issue with your hard disk?"); }
fn is_already_in(backing: &Database, col: Option<u32>, key: &H256) -> bool {
backing.get(col, &Self::morph_key(key, 0)).expect("Low-level database error. Some issue with your hard disk?").is_some()
}
fn insert_keys(inserts: &[(H256, Bytes)], backing: &Database, refs: &mut HashMap<H256, RefInfo>, batch: &DBTransaction, trace: bool) {
fn insert_keys(inserts: &[(H256, Bytes)], backing: &Database, col: Option<u32>, refs: &mut HashMap<H256, RefInfo>, batch: &DBTransaction, trace: bool) {
for &(ref h, ref d) in inserts {
if let Some(c) = refs.get_mut(h) {
// already counting. increment.
@@ -129,9 +119,9 @@ impl EarlyMergeDB {
}
// this is the first entry for this node in the journal.
if backing.get(h).expect("Low-level database error. Some issue with your hard disk?").is_some() {
if backing.get(col, h).expect("Low-level database error. Some issue with your hard disk?").is_some() {
// already in the backing DB. start counting, and remember it was already in.
Self::set_already_in(batch, &h);
Self::set_already_in(batch, col, h);
refs.insert(h.clone(), RefInfo{queue_refs: 1, in_archive: true});
if trace {
trace!(target: "jdb.fine", " insert({}): New to queue, in DB: Recording and inserting into queue", h);
@@ -141,8 +131,8 @@ impl EarlyMergeDB {
// Gets removed when a key leaves the journal, so should never be set when we're placing a new key.
//Self::reset_already_in(&h);
assert!(!Self::is_already_in(backing, &h));
batch.put(h, d).expect("Low-level database error. Some issue with your hard disk?");
assert!(!Self::is_already_in(backing, col, &h));
batch.put(col, h, d).expect("Low-level database error. Some issue with your hard disk?");
refs.insert(h.clone(), RefInfo{queue_refs: 1, in_archive: false});
if trace {
trace!(target: "jdb.fine", " insert({}): New to queue, not in DB: Inserting into queue and DB", h);
@@ -150,7 +140,7 @@ impl EarlyMergeDB {
}
}
fn replay_keys(inserts: &[H256], backing: &Database, refs: &mut HashMap<H256, RefInfo>) {
fn replay_keys(inserts: &[H256], backing: &Database, col: Option<u32>, refs: &mut HashMap<H256, RefInfo>) {
trace!(target: "jdb.fine", "replay_keys: inserts={:?}, refs={:?}", inserts, refs);
for h in inserts {
if let Some(c) = refs.get_mut(h) {
@@ -161,12 +151,12 @@ impl EarlyMergeDB {
// this is the first entry for this node in the journal.
// it is initialised to 1 if it was already in.
refs.insert(h.clone(), RefInfo{queue_refs: 1, in_archive: Self::is_already_in(backing, h)});
refs.insert(h.clone(), RefInfo{queue_refs: 1, in_archive: Self::is_already_in(backing, col, h)});
}
trace!(target: "jdb.fine", "replay_keys: (end) refs={:?}", refs);
}
fn remove_keys(deletes: &[H256], refs: &mut HashMap<H256, RefInfo>, batch: &DBTransaction, from: RemoveFrom, trace: bool) {
fn remove_keys(deletes: &[H256], refs: &mut HashMap<H256, RefInfo>, batch: &DBTransaction, col: Option<u32>, from: RemoveFrom, trace: bool) {
// with a remove on {queue_refs: 1, in_archive: true}, we have two options:
// - convert to {queue_refs: 1, in_archive: false} (i.e. remove it from the conceptual archive)
// - convert to {queue_refs: 0, in_archive: true} (i.e. remove it from the conceptual queue)
@@ -178,7 +168,7 @@ impl EarlyMergeDB {
if let Some(c) = refs.get_mut(h) {
if c.in_archive && from == RemoveFrom::Archive {
c.in_archive = false;
Self::reset_already_in(batch, h);
Self::reset_already_in(batch, col, h);
if trace {
trace!(target: "jdb.fine", " remove({}): In archive, 1 in queue: Reducing to queue only and recording", h);
}
@@ -196,14 +186,14 @@ impl EarlyMergeDB {
match n {
Some(RefInfo{queue_refs: 1, in_archive: true}) => {
refs.remove(h);
Self::reset_already_in(batch, h);
Self::reset_already_in(batch, col, h);
if trace {
trace!(target: "jdb.fine", " remove({}): In archive, 1 in queue: Removing from queue and leaving in archive", h);
}
}
Some(RefInfo{queue_refs: 1, in_archive: false}) => {
refs.remove(h);
batch.delete(h).expect("Low-level database error. Some issue with your hard disk?");
batch.delete(col, h).expect("Low-level database error. Some issue with your hard disk?");
if trace {
trace!(target: "jdb.fine", " remove({}): Not in archive, only 1 ref in queue: Removing from queue and DB", h);
}
@@ -211,7 +201,7 @@ impl EarlyMergeDB {
None => {
// Gets removed when moving from 1 to 0 additional refs. Should never be here at 0 additional refs.
//assert!(!Self::is_already_in(db, &h));
batch.delete(h).expect("Low-level database error. Some issue with your hard disk?");
batch.delete(col, h).expect("Low-level database error. Some issue with your hard disk?");
if trace {
trace!(target: "jdb.fine", " remove({}): Not in queue - MUST BE IN ARCHIVE: Removing from DB", h);
}
@@ -223,7 +213,7 @@ impl EarlyMergeDB {
#[cfg(test)]
fn can_reconstruct_refs(&self) -> bool {
let (latest_era, reconstructed) = Self::read_refs(&self.backing);
let (latest_era, reconstructed) = Self::read_refs(&self.backing, self.column);
let refs = self.refs.as_ref().unwrap().write();
if *refs != reconstructed || latest_era != self.latest_era {
let clean_refs = refs.iter().filter_map(|(k, v)| if reconstructed.get(k) == Some(v) {None} else {Some((k.clone(), v.clone()))}).collect::<HashMap<_, _>>();
@@ -236,18 +226,18 @@ impl EarlyMergeDB {
}
fn payload(&self, key: &H256) -> Option<Bytes> {
self.backing.get(key).expect("Low-level database error. Some issue with your hard disk?").map(|v| v.to_vec())
self.backing.get(self.column, key).expect("Low-level database error. Some issue with your hard disk?").map(|v| v.to_vec())
}
fn read_refs(db: &Database) -> (Option<u64>, HashMap<H256, RefInfo>) {
fn read_refs(db: &Database, col: Option<u32>) -> (Option<u64>, HashMap<H256, RefInfo>) {
let mut refs = HashMap::new();
let mut latest_era = None;
if let Some(val) = db.get(&LATEST_ERA_KEY).expect("Low-level database error.") {
if let Some(val) = db.get(col, &LATEST_ERA_KEY).expect("Low-level database error.") {
let mut era = decode::<u64>(&val);
latest_era = Some(era);
loop {
let mut index = 0usize;
while let Some(rlp_data) = db.get({
while let Some(rlp_data) = db.get(col, {
let mut r = RlpStream::new_list(3);
r.append(&era);
r.append(&index);
@@ -256,7 +246,7 @@ impl EarlyMergeDB {
}).expect("Low-level database error.") {
let rlp = Rlp::new(&rlp_data);
let inserts: Vec<H256> = rlp.val_at(1);
Self::replay_keys(&inserts, db, &mut refs);
Self::replay_keys(&inserts, db, col, &mut refs);
index += 1;
};
if index == 0 || era == 0 {
@@ -267,12 +257,12 @@ impl EarlyMergeDB {
}
(latest_era, refs)
}
}
}
impl HashDB for EarlyMergeDB {
fn keys(&self) -> HashMap<H256, i32> {
let mut ret: HashMap<H256, i32> = HashMap::new();
for (key, _) in self.backing.iter() {
for (key, _) in self.backing.iter(self.column) {
let h = H256::from_slice(key.deref());
ret.insert(h, 1);
}
@@ -321,11 +311,16 @@ impl JournalDB for EarlyMergeDB {
backing: self.backing.clone(),
refs: self.refs.clone(),
latest_era: self.latest_era.clone(),
column: self.column.clone(),
})
}
fn is_empty(&self) -> bool {
self.backing.get(&LATEST_ERA_KEY).expect("Low level database error").is_none()
self.backing.get(self.column, &LATEST_ERA_KEY).expect("Low level database error").is_none()
}
fn backing(&self) -> &Arc<Database> {
&self.backing
}
fn latest_era(&self) -> Option<u64> { self.latest_era }
@@ -338,11 +333,11 @@ impl JournalDB for EarlyMergeDB {
}
fn state(&self, id: &H256) -> Option<Bytes> {
self.backing.get_by_prefix(&id[0..DB_PREFIX_LEN]).map(|b| b.to_vec())
self.backing.get_by_prefix(self.column, &id[0..DB_PREFIX_LEN]).map(|b| b.to_vec())
}
#[cfg_attr(feature="dev", allow(cyclomatic_complexity))]
fn commit(&mut self, now: u64, id: &H256, end: Option<(u64, H256)>) -> Result<u32, UtilError> {
fn commit(&mut self, batch: &DBTransaction, now: u64, id: &H256, end: Option<(u64, H256)>) -> Result<u32, UtilError> {
// journal format:
// [era, 0] => [ id, [insert_0, ...], [remove_0, ...] ]
// [era, 1] => [ id, [insert_0, ...], [remove_0, ...] ]
@@ -389,13 +384,12 @@ impl JournalDB for EarlyMergeDB {
// record new commit's details.
let mut refs = self.refs.as_ref().unwrap().write();
let batch = DBTransaction::new();
let trace = false;
{
let mut index = 0usize;
let mut last;
while try!(self.backing.get({
while try!(self.backing.get(self.column, {
let mut r = RlpStream::new_list(3);
r.append(&now);
r.append(&index);
@@ -436,15 +430,15 @@ impl JournalDB for EarlyMergeDB {
r.begin_list(inserts.len());
inserts.iter().foreach(|&(k, _)| {r.append(&k);});
r.append(&removes);
Self::insert_keys(&inserts, &self.backing, &mut refs, &batch, trace);
Self::insert_keys(&inserts, &self.backing, self.column, &mut refs, &batch, trace);
if trace {
let ins = inserts.iter().map(|&(k, _)| k).collect::<Vec<_>>();
trace!(target: "jdb.ops", " Inserts: {:?}", ins);
trace!(target: "jdb.ops", " Deletes: {:?}", removes);
}
try!(batch.put(&last, r.as_raw()));
try!(batch.put(self.column, &last, r.as_raw()));
if self.latest_era.map_or(true, |e| now > e) {
try!(batch.put(&LATEST_ERA_KEY, &encode(&now)));
try!(batch.put(self.column, &LATEST_ERA_KEY, &encode(&now)));
self.latest_era = Some(now);
}
}
@@ -453,7 +447,7 @@ impl JournalDB for EarlyMergeDB {
if let Some((end_era, canon_id)) = end {
let mut index = 0usize;
let mut last;
while let Some(rlp_data) = try!(self.backing.get({
while let Some(rlp_data) = try!(self.backing.get(self.column, {
let mut r = RlpStream::new_list(3);
r.append(&end_era);
r.append(&index);
@@ -470,7 +464,7 @@ impl JournalDB for EarlyMergeDB {
if trace {
trace!(target: "jdb.ops", " Expunging: {:?}", deletes);
}
Self::remove_keys(&deletes, &mut refs, &batch, RemoveFrom::Archive, trace);
Self::remove_keys(&deletes, &mut refs, &batch, self.column, RemoveFrom::Archive, trace);
if trace {
trace!(target: "jdb.ops", " Finalising: {:?}", inserts);
@@ -488,7 +482,7 @@ impl JournalDB for EarlyMergeDB {
}
Some( RefInfo{queue_refs: x, in_archive: false} ) => {
// must set already in; ,
Self::set_already_in(&batch, k);
Self::set_already_in(&batch, self.column, k);
refs.insert(k.clone(), RefInfo{ queue_refs: x - 1, in_archive: true });
}
Some( RefInfo{in_archive: true, ..} ) => {
@@ -502,10 +496,10 @@ impl JournalDB for EarlyMergeDB {
if trace {
trace!(target: "jdb.ops", " Reverting: {:?}", inserts);
}
Self::remove_keys(&inserts, &mut refs, &batch, RemoveFrom::Queue, trace);
Self::remove_keys(&inserts, &mut refs, &batch, self.column, RemoveFrom::Queue, trace);
}
try!(batch.delete(&last));
try!(batch.delete(self.column, &last));
index += 1;
}
if trace {
@@ -513,10 +507,6 @@ impl JournalDB for EarlyMergeDB {
}
}
try!(self.backing.write(batch));
// Comment out for now. TODO: automatically enable in tests.
if trace {
trace!(target: "jdb", "OK: {:?}", refs.clone());
}
@@ -535,7 +525,7 @@ mod tests {
use super::super::traits::JournalDB;
use hashdb::*;
use log::init_log;
use kvdb::DatabaseConfig;
use kvdb::{Database, DatabaseConfig};
#[test]
fn insert_same_in_fork() {
@@ -543,25 +533,25 @@ mod tests {
let mut jdb = EarlyMergeDB::new_temp();
let x = jdb.insert(b"X");
jdb.commit(1, &b"1".sha3(), None).unwrap();
jdb.commit_batch(1, &b"1".sha3(), None).unwrap();
assert!(jdb.can_reconstruct_refs());
jdb.commit(2, &b"2".sha3(), None).unwrap();
jdb.commit_batch(2, &b"2".sha3(), None).unwrap();
assert!(jdb.can_reconstruct_refs());
jdb.commit(3, &b"1002a".sha3(), Some((1, b"1".sha3()))).unwrap();
jdb.commit_batch(3, &b"1002a".sha3(), Some((1, b"1".sha3()))).unwrap();
assert!(jdb.can_reconstruct_refs());
jdb.commit(4, &b"1003a".sha3(), Some((2, b"2".sha3()))).unwrap();
jdb.commit_batch(4, &b"1003a".sha3(), Some((2, b"2".sha3()))).unwrap();
assert!(jdb.can_reconstruct_refs());
jdb.remove(&x);
jdb.commit(3, &b"1002b".sha3(), Some((1, b"1".sha3()))).unwrap();
jdb.commit_batch(3, &b"1002b".sha3(), Some((1, b"1".sha3()))).unwrap();
assert!(jdb.can_reconstruct_refs());
let x = jdb.insert(b"X");
jdb.commit(4, &b"1003b".sha3(), Some((2, b"2".sha3()))).unwrap();
jdb.commit_batch(4, &b"1003b".sha3(), Some((2, b"2".sha3()))).unwrap();
assert!(jdb.can_reconstruct_refs());
jdb.commit(5, &b"1004a".sha3(), Some((3, b"1002a".sha3()))).unwrap();
jdb.commit_batch(5, &b"1004a".sha3(), Some((3, b"1002a".sha3()))).unwrap();
assert!(jdb.can_reconstruct_refs());
jdb.commit(6, &b"1005a".sha3(), Some((4, b"1003a".sha3()))).unwrap();
jdb.commit_batch(6, &b"1005a".sha3(), Some((4, b"1003a".sha3()))).unwrap();
assert!(jdb.can_reconstruct_refs());
assert!(jdb.contains(&x));
@@ -571,17 +561,17 @@ mod tests {
fn insert_older_era() {
let mut jdb = EarlyMergeDB::new_temp();
let foo = jdb.insert(b"foo");
jdb.commit(0, &b"0a".sha3(), None).unwrap();
jdb.commit_batch(0, &b"0a".sha3(), None).unwrap();
assert!(jdb.can_reconstruct_refs());
let bar = jdb.insert(b"bar");
jdb.commit(1, &b"1".sha3(), Some((0, b"0a".sha3()))).unwrap();
jdb.commit_batch(1, &b"1".sha3(), Some((0, b"0a".sha3()))).unwrap();
assert!(jdb.can_reconstruct_refs());
jdb.remove(&bar);
jdb.commit(0, &b"0b".sha3(), None).unwrap();
jdb.commit_batch(0, &b"0b".sha3(), None).unwrap();
assert!(jdb.can_reconstruct_refs());
jdb.commit(2, &b"2".sha3(), Some((1, b"1".sha3()))).unwrap();
jdb.commit_batch(2, &b"2".sha3(), Some((1, b"1".sha3()))).unwrap();
assert!(jdb.contains(&foo));
assert!(jdb.contains(&bar));
@@ -592,20 +582,20 @@ mod tests {
// history is 3
let mut jdb = EarlyMergeDB::new_temp();
let h = jdb.insert(b"foo");
jdb.commit(0, &b"0".sha3(), None).unwrap();
jdb.commit_batch(0, &b"0".sha3(), None).unwrap();
assert!(jdb.can_reconstruct_refs());
assert!(jdb.contains(&h));
jdb.remove(&h);
jdb.commit(1, &b"1".sha3(), None).unwrap();
jdb.commit_batch(1, &b"1".sha3(), None).unwrap();
assert!(jdb.can_reconstruct_refs());
assert!(jdb.contains(&h));
jdb.commit(2, &b"2".sha3(), None).unwrap();
jdb.commit_batch(2, &b"2".sha3(), None).unwrap();
assert!(jdb.can_reconstruct_refs());
assert!(jdb.contains(&h));
jdb.commit(3, &b"3".sha3(), Some((0, b"0".sha3()))).unwrap();
jdb.commit_batch(3, &b"3".sha3(), Some((0, b"0".sha3()))).unwrap();
assert!(jdb.can_reconstruct_refs());
assert!(jdb.contains(&h));
jdb.commit(4, &b"4".sha3(), Some((1, b"1".sha3()))).unwrap();
jdb.commit_batch(4, &b"4".sha3(), Some((1, b"1".sha3()))).unwrap();
assert!(jdb.can_reconstruct_refs());
assert!(!jdb.contains(&h));
}
@@ -617,7 +607,7 @@ mod tests {
let foo = jdb.insert(b"foo");
let bar = jdb.insert(b"bar");
jdb.commit(0, &b"0".sha3(), None).unwrap();
jdb.commit_batch(0, &b"0".sha3(), None).unwrap();
assert!(jdb.can_reconstruct_refs());
assert!(jdb.contains(&foo));
assert!(jdb.contains(&bar));
@@ -625,7 +615,7 @@ mod tests {
jdb.remove(&foo);
jdb.remove(&bar);
let baz = jdb.insert(b"baz");
jdb.commit(1, &b"1".sha3(), Some((0, b"0".sha3()))).unwrap();
jdb.commit_batch(1, &b"1".sha3(), Some((0, b"0".sha3()))).unwrap();
assert!(jdb.can_reconstruct_refs());
assert!(jdb.contains(&foo));
assert!(jdb.contains(&bar));
@@ -633,20 +623,20 @@ mod tests {
let foo = jdb.insert(b"foo");
jdb.remove(&baz);
jdb.commit(2, &b"2".sha3(), Some((1, b"1".sha3()))).unwrap();
jdb.commit_batch(2, &b"2".sha3(), Some((1, b"1".sha3()))).unwrap();
assert!(jdb.can_reconstruct_refs());
assert!(jdb.contains(&foo));
assert!(!jdb.contains(&bar));
assert!(jdb.contains(&baz));
jdb.remove(&foo);
jdb.commit(3, &b"3".sha3(), Some((2, b"2".sha3()))).unwrap();
jdb.commit_batch(3, &b"3".sha3(), Some((2, b"2".sha3()))).unwrap();
assert!(jdb.can_reconstruct_refs());
assert!(jdb.contains(&foo));
assert!(!jdb.contains(&bar));
assert!(!jdb.contains(&baz));
jdb.commit(4, &b"4".sha3(), Some((3, b"3".sha3()))).unwrap();
jdb.commit_batch(4, &b"4".sha3(), Some((3, b"3".sha3()))).unwrap();
assert!(jdb.can_reconstruct_refs());
assert!(!jdb.contains(&foo));
assert!(!jdb.contains(&bar));
@@ -660,25 +650,25 @@ mod tests {
let foo = jdb.insert(b"foo");
let bar = jdb.insert(b"bar");
jdb.commit(0, &b"0".sha3(), None).unwrap();
jdb.commit_batch(0, &b"0".sha3(), None).unwrap();
assert!(jdb.can_reconstruct_refs());
assert!(jdb.contains(&foo));
assert!(jdb.contains(&bar));
jdb.remove(&foo);
let baz = jdb.insert(b"baz");
jdb.commit(1, &b"1a".sha3(), Some((0, b"0".sha3()))).unwrap();
jdb.commit_batch(1, &b"1a".sha3(), Some((0, b"0".sha3()))).unwrap();
assert!(jdb.can_reconstruct_refs());
jdb.remove(&bar);
jdb.commit(1, &b"1b".sha3(), Some((0, b"0".sha3()))).unwrap();
jdb.commit_batch(1, &b"1b".sha3(), Some((0, b"0".sha3()))).unwrap();
assert!(jdb.can_reconstruct_refs());
assert!(jdb.contains(&foo));
assert!(jdb.contains(&bar));
assert!(jdb.contains(&baz));
jdb.commit(2, &b"2b".sha3(), Some((1, b"1b".sha3()))).unwrap();
jdb.commit_batch(2, &b"2b".sha3(), Some((1, b"1b".sha3()))).unwrap();
assert!(jdb.can_reconstruct_refs());
assert!(jdb.contains(&foo));
assert!(!jdb.contains(&baz));
@@ -691,115 +681,113 @@ mod tests {
let mut jdb = EarlyMergeDB::new_temp();
let foo = jdb.insert(b"foo");
jdb.commit(0, &b"0".sha3(), None).unwrap();
jdb.commit_batch(0, &b"0".sha3(), None).unwrap();
assert!(jdb.can_reconstruct_refs());
assert!(jdb.contains(&foo));
jdb.remove(&foo);
jdb.commit(1, &b"1".sha3(), Some((0, b"0".sha3()))).unwrap();
jdb.commit_batch(1, &b"1".sha3(), Some((0, b"0".sha3()))).unwrap();
assert!(jdb.can_reconstruct_refs());
jdb.insert(b"foo");
assert!(jdb.contains(&foo));
jdb.commit(2, &b"2".sha3(), Some((1, b"1".sha3()))).unwrap();
jdb.commit_batch(2, &b"2".sha3(), Some((1, b"1".sha3()))).unwrap();
assert!(jdb.can_reconstruct_refs());
assert!(jdb.contains(&foo));
jdb.commit(3, &b"2".sha3(), Some((0, b"2".sha3()))).unwrap();
jdb.commit_batch(3, &b"2".sha3(), Some((0, b"2".sha3()))).unwrap();
assert!(jdb.can_reconstruct_refs());
assert!(jdb.contains(&foo));
}
#[test]
fn fork_same_key_one() {
let mut dir = ::std::env::temp_dir();
dir.push(H32::random().hex());
let mut jdb = EarlyMergeDB::new(dir.to_str().unwrap(), DatabaseConfig::default());
jdb.commit(0, &b"0".sha3(), None).unwrap();
let mut jdb = EarlyMergeDB::new_temp();
jdb.commit_batch(0, &b"0".sha3(), None).unwrap();
assert!(jdb.can_reconstruct_refs());
let foo = jdb.insert(b"foo");
jdb.commit(1, &b"1a".sha3(), Some((0, b"0".sha3()))).unwrap();
jdb.commit_batch(1, &b"1a".sha3(), Some((0, b"0".sha3()))).unwrap();
assert!(jdb.can_reconstruct_refs());
jdb.insert(b"foo");
jdb.commit(1, &b"1b".sha3(), Some((0, b"0".sha3()))).unwrap();
jdb.commit_batch(1, &b"1b".sha3(), Some((0, b"0".sha3()))).unwrap();
assert!(jdb.can_reconstruct_refs());
jdb.insert(b"foo");
jdb.commit(1, &b"1c".sha3(), Some((0, b"0".sha3()))).unwrap();
jdb.commit_batch(1, &b"1c".sha3(), Some((0, b"0".sha3()))).unwrap();
assert!(jdb.can_reconstruct_refs());
assert!(jdb.contains(&foo));
jdb.commit(2, &b"2a".sha3(), Some((1, b"1a".sha3()))).unwrap();
jdb.commit_batch(2, &b"2a".sha3(), Some((1, b"1a".sha3()))).unwrap();
assert!(jdb.can_reconstruct_refs());
assert!(jdb.contains(&foo));
}
#[test]
fn fork_same_key_other() {
let mut dir = ::std::env::temp_dir();
dir.push(H32::random().hex());
let mut jdb = EarlyMergeDB::new(dir.to_str().unwrap(), DatabaseConfig::default());
jdb.commit(0, &b"0".sha3(), None).unwrap();
let mut jdb = EarlyMergeDB::new_temp();
jdb.commit_batch(0, &b"0".sha3(), None).unwrap();
assert!(jdb.can_reconstruct_refs());
let foo = jdb.insert(b"foo");
jdb.commit(1, &b"1a".sha3(), Some((0, b"0".sha3()))).unwrap();
jdb.commit_batch(1, &b"1a".sha3(), Some((0, b"0".sha3()))).unwrap();
assert!(jdb.can_reconstruct_refs());
jdb.insert(b"foo");
jdb.commit(1, &b"1b".sha3(), Some((0, b"0".sha3()))).unwrap();
jdb.commit_batch(1, &b"1b".sha3(), Some((0, b"0".sha3()))).unwrap();
assert!(jdb.can_reconstruct_refs());
jdb.insert(b"foo");
jdb.commit(1, &b"1c".sha3(), Some((0, b"0".sha3()))).unwrap();
jdb.commit_batch(1, &b"1c".sha3(), Some((0, b"0".sha3()))).unwrap();
assert!(jdb.can_reconstruct_refs());
assert!(jdb.contains(&foo));
jdb.commit(2, &b"2b".sha3(), Some((1, b"1b".sha3()))).unwrap();
jdb.commit_batch(2, &b"2b".sha3(), Some((1, b"1b".sha3()))).unwrap();
assert!(jdb.can_reconstruct_refs());
assert!(jdb.contains(&foo));
}
#[test]
fn fork_ins_del_ins() {
let mut dir = ::std::env::temp_dir();
dir.push(H32::random().hex());
let mut jdb = EarlyMergeDB::new(dir.to_str().unwrap(), DatabaseConfig::default());
jdb.commit(0, &b"0".sha3(), None).unwrap();
let mut jdb = EarlyMergeDB::new_temp();
jdb.commit_batch(0, &b"0".sha3(), None).unwrap();
assert!(jdb.can_reconstruct_refs());
let foo = jdb.insert(b"foo");
jdb.commit(1, &b"1".sha3(), None).unwrap();
jdb.commit_batch(1, &b"1".sha3(), None).unwrap();
assert!(jdb.can_reconstruct_refs());
jdb.remove(&foo);
jdb.commit(2, &b"2a".sha3(), Some((0, b"0".sha3()))).unwrap();
jdb.commit_batch(2, &b"2a".sha3(), Some((0, b"0".sha3()))).unwrap();
assert!(jdb.can_reconstruct_refs());
jdb.remove(&foo);
jdb.commit(2, &b"2b".sha3(), Some((0, b"0".sha3()))).unwrap();
jdb.commit_batch(2, &b"2b".sha3(), Some((0, b"0".sha3()))).unwrap();
assert!(jdb.can_reconstruct_refs());
jdb.insert(b"foo");
jdb.commit(3, &b"3a".sha3(), Some((1, b"1".sha3()))).unwrap();
jdb.commit_batch(3, &b"3a".sha3(), Some((1, b"1".sha3()))).unwrap();
assert!(jdb.can_reconstruct_refs());
jdb.insert(b"foo");
jdb.commit(3, &b"3b".sha3(), Some((1, b"1".sha3()))).unwrap();
jdb.commit_batch(3, &b"3b".sha3(), Some((1, b"1".sha3()))).unwrap();
assert!(jdb.can_reconstruct_refs());
jdb.commit(4, &b"4a".sha3(), Some((2, b"2a".sha3()))).unwrap();
jdb.commit_batch(4, &b"4a".sha3(), Some((2, b"2a".sha3()))).unwrap();
assert!(jdb.can_reconstruct_refs());
jdb.commit(5, &b"5a".sha3(), Some((3, b"3a".sha3()))).unwrap();
jdb.commit_batch(5, &b"5a".sha3(), Some((3, b"3a".sha3()))).unwrap();
assert!(jdb.can_reconstruct_refs());
}
fn new_db(path: &Path) -> EarlyMergeDB {
let config = DatabaseConfig::with_columns(Some(1));
let backing = Arc::new(Database::open(&config, path.to_str().unwrap()).unwrap());
EarlyMergeDB::new(backing, Some(0))
}
#[test]
fn reopen() {
let mut dir = ::std::env::temp_dir();
@@ -807,27 +795,27 @@ mod tests {
let bar = H256::random();
let foo = {
let mut jdb = EarlyMergeDB::new(dir.to_str().unwrap(), DatabaseConfig::default());
let mut jdb = new_db(&dir);
// history is 1
let foo = jdb.insert(b"foo");
jdb.emplace(bar.clone(), b"bar".to_vec());
jdb.commit(0, &b"0".sha3(), None).unwrap();
jdb.commit_batch(0, &b"0".sha3(), None).unwrap();
assert!(jdb.can_reconstruct_refs());
foo
};
{
let mut jdb = EarlyMergeDB::new(dir.to_str().unwrap(), DatabaseConfig::default());
let mut jdb = new_db(&dir);
jdb.remove(&foo);
jdb.commit(1, &b"1".sha3(), Some((0, b"0".sha3()))).unwrap();
jdb.commit_batch(1, &b"1".sha3(), Some((0, b"0".sha3()))).unwrap();
assert!(jdb.can_reconstruct_refs());
}
{
let mut jdb = EarlyMergeDB::new(dir.to_str().unwrap(), DatabaseConfig::default());
let mut jdb = new_db(&dir);
assert!(jdb.contains(&foo));
assert!(jdb.contains(&bar));
jdb.commit(2, &b"2".sha3(), Some((1, b"1".sha3()))).unwrap();
jdb.commit_batch(2, &b"2".sha3(), Some((1, b"1".sha3()))).unwrap();
assert!(jdb.can_reconstruct_refs());
assert!(!jdb.contains(&foo));
}
@@ -836,145 +824,136 @@ mod tests {
#[test]
fn insert_delete_insert_delete_insert_expunge() {
init_log();
let mut dir = ::std::env::temp_dir();
dir.push(H32::random().hex());
let mut jdb = EarlyMergeDB::new(dir.to_str().unwrap(), DatabaseConfig::default());
let mut jdb = EarlyMergeDB::new_temp();
// history is 4
let foo = jdb.insert(b"foo");
jdb.commit(0, &b"0".sha3(), None).unwrap();
jdb.commit_batch(0, &b"0".sha3(), None).unwrap();
assert!(jdb.can_reconstruct_refs());
jdb.remove(&foo);
jdb.commit(1, &b"1".sha3(), None).unwrap();
jdb.commit_batch(1, &b"1".sha3(), None).unwrap();
assert!(jdb.can_reconstruct_refs());
jdb.insert(b"foo");
jdb.commit(2, &b"2".sha3(), None).unwrap();
jdb.commit_batch(2, &b"2".sha3(), None).unwrap();
assert!(jdb.can_reconstruct_refs());
jdb.remove(&foo);
jdb.commit(3, &b"3".sha3(), None).unwrap();
jdb.commit_batch(3, &b"3".sha3(), None).unwrap();
assert!(jdb.can_reconstruct_refs());
jdb.insert(b"foo");
jdb.commit(4, &b"4".sha3(), Some((0, b"0".sha3()))).unwrap();
jdb.commit_batch(4, &b"4".sha3(), Some((0, b"0".sha3()))).unwrap();
assert!(jdb.can_reconstruct_refs());
// expunge foo
jdb.commit(5, &b"5".sha3(), Some((1, b"1".sha3()))).unwrap();
jdb.commit_batch(5, &b"5".sha3(), Some((1, b"1".sha3()))).unwrap();
assert!(jdb.can_reconstruct_refs());
}
#[test]
fn forked_insert_delete_insert_delete_insert_expunge() {
init_log();
let mut dir = ::std::env::temp_dir();
dir.push(H32::random().hex());
let mut jdb = EarlyMergeDB::new(dir.to_str().unwrap(), DatabaseConfig::default());
let mut jdb = EarlyMergeDB::new_temp();
// history is 4
let foo = jdb.insert(b"foo");
jdb.commit(0, &b"0".sha3(), None).unwrap();
jdb.commit_batch(0, &b"0".sha3(), None).unwrap();
assert!(jdb.can_reconstruct_refs());
jdb.remove(&foo);
jdb.commit(1, &b"1a".sha3(), None).unwrap();
jdb.commit_batch(1, &b"1a".sha3(), None).unwrap();
assert!(jdb.can_reconstruct_refs());
jdb.remove(&foo);
jdb.commit(1, &b"1b".sha3(), None).unwrap();
jdb.commit_batch(1, &b"1b".sha3(), None).unwrap();
assert!(jdb.can_reconstruct_refs());
jdb.insert(b"foo");
jdb.commit(2, &b"2a".sha3(), None).unwrap();
jdb.commit_batch(2, &b"2a".sha3(), None).unwrap();
assert!(jdb.can_reconstruct_refs());
jdb.insert(b"foo");
jdb.commit(2, &b"2b".sha3(), None).unwrap();
jdb.commit_batch(2, &b"2b".sha3(), None).unwrap();
assert!(jdb.can_reconstruct_refs());
jdb.remove(&foo);
jdb.commit(3, &b"3a".sha3(), None).unwrap();
jdb.commit_batch(3, &b"3a".sha3(), None).unwrap();
assert!(jdb.can_reconstruct_refs());
jdb.remove(&foo);
jdb.commit(3, &b"3b".sha3(), None).unwrap();
jdb.commit_batch(3, &b"3b".sha3(), None).unwrap();
assert!(jdb.can_reconstruct_refs());
jdb.insert(b"foo");
jdb.commit(4, &b"4a".sha3(), Some((0, b"0".sha3()))).unwrap();
jdb.commit_batch(4, &b"4a".sha3(), Some((0, b"0".sha3()))).unwrap();
assert!(jdb.can_reconstruct_refs());
jdb.insert(b"foo");
jdb.commit(4, &b"4b".sha3(), Some((0, b"0".sha3()))).unwrap();
jdb.commit_batch(4, &b"4b".sha3(), Some((0, b"0".sha3()))).unwrap();
assert!(jdb.can_reconstruct_refs());
// expunge foo
jdb.commit(5, &b"5".sha3(), Some((1, b"1a".sha3()))).unwrap();
jdb.commit_batch(5, &b"5".sha3(), Some((1, b"1a".sha3()))).unwrap();
assert!(jdb.can_reconstruct_refs());
}
#[test]
fn broken_assert() {
let mut dir = ::std::env::temp_dir();
dir.push(H32::random().hex());
let mut jdb = EarlyMergeDB::new_temp();
let mut jdb = EarlyMergeDB::new(dir.to_str().unwrap(), DatabaseConfig::default());
// history is 1
let foo = jdb.insert(b"foo");
jdb.commit(1, &b"1".sha3(), Some((0, b"0".sha3()))).unwrap();
jdb.commit_batch(1, &b"1".sha3(), Some((0, b"0".sha3()))).unwrap();
assert!(jdb.can_reconstruct_refs());
// foo is ancient history.
jdb.remove(&foo);
jdb.commit(2, &b"2".sha3(), Some((1, b"1".sha3()))).unwrap();
jdb.commit_batch(2, &b"2".sha3(), Some((1, b"1".sha3()))).unwrap();
assert!(jdb.can_reconstruct_refs());
jdb.insert(b"foo");
jdb.commit(3, &b"3".sha3(), Some((2, b"2".sha3()))).unwrap(); // BROKEN
jdb.commit_batch(3, &b"3".sha3(), Some((2, b"2".sha3()))).unwrap(); // BROKEN
assert!(jdb.can_reconstruct_refs());
assert!(jdb.contains(&foo));
jdb.remove(&foo);
jdb.commit(4, &b"4".sha3(), Some((3, b"3".sha3()))).unwrap();
jdb.commit_batch(4, &b"4".sha3(), Some((3, b"3".sha3()))).unwrap();
assert!(jdb.can_reconstruct_refs());
jdb.commit(5, &b"5".sha3(), Some((4, b"4".sha3()))).unwrap();
jdb.commit_batch(5, &b"5".sha3(), Some((4, b"4".sha3()))).unwrap();
assert!(jdb.can_reconstruct_refs());
assert!(!jdb.contains(&foo));
}
#[test]
fn reopen_test() {
let mut dir = ::std::env::temp_dir();
dir.push(H32::random().hex());
let mut jdb = EarlyMergeDB::new_temp();
let mut jdb = EarlyMergeDB::new(dir.to_str().unwrap(), DatabaseConfig::default());
// history is 4
let foo = jdb.insert(b"foo");
jdb.commit(0, &b"0".sha3(), None).unwrap();
jdb.commit_batch(0, &b"0".sha3(), None).unwrap();
assert!(jdb.can_reconstruct_refs());
jdb.commit(1, &b"1".sha3(), None).unwrap();
jdb.commit_batch(1, &b"1".sha3(), None).unwrap();
assert!(jdb.can_reconstruct_refs());
jdb.commit(2, &b"2".sha3(), None).unwrap();
jdb.commit_batch(2, &b"2".sha3(), None).unwrap();
assert!(jdb.can_reconstruct_refs());
jdb.commit(3, &b"3".sha3(), None).unwrap();
jdb.commit_batch(3, &b"3".sha3(), None).unwrap();
assert!(jdb.can_reconstruct_refs());
jdb.commit(4, &b"4".sha3(), Some((0, b"0".sha3()))).unwrap();
jdb.commit_batch(4, &b"4".sha3(), Some((0, b"0".sha3()))).unwrap();
assert!(jdb.can_reconstruct_refs());
// foo is ancient history.
jdb.insert(b"foo");
let bar = jdb.insert(b"bar");
jdb.commit(5, &b"5".sha3(), Some((1, b"1".sha3()))).unwrap();
jdb.commit_batch(5, &b"5".sha3(), Some((1, b"1".sha3()))).unwrap();
assert!(jdb.can_reconstruct_refs());
jdb.remove(&foo);
jdb.remove(&bar);
jdb.commit(6, &b"6".sha3(), Some((2, b"2".sha3()))).unwrap();
jdb.commit_batch(6, &b"6".sha3(), Some((2, b"2".sha3()))).unwrap();
assert!(jdb.can_reconstruct_refs());
jdb.insert(b"foo");
jdb.insert(b"bar");
jdb.commit(7, &b"7".sha3(), Some((3, b"3".sha3()))).unwrap();
jdb.commit_batch(7, &b"7".sha3(), Some((3, b"3".sha3()))).unwrap();
assert!(jdb.can_reconstruct_refs());
}
@@ -988,45 +967,48 @@ mod tests {
let foo = b"foo".sha3();
{
let mut jdb = EarlyMergeDB::new(dir.to_str().unwrap(), DatabaseConfig::default());
let mut jdb = new_db(&dir);
// history is 1
jdb.insert(b"foo");
jdb.commit(0, &b"0".sha3(), None).unwrap();
jdb.commit_batch(0, &b"0".sha3(), None).unwrap();
assert!(jdb.can_reconstruct_refs());
jdb.commit(1, &b"1".sha3(), None).unwrap();
jdb.commit_batch(1, &b"1".sha3(), None).unwrap();
assert!(jdb.can_reconstruct_refs());
// foo is ancient history.
jdb.remove(&foo);
jdb.commit(2, &b"2".sha3(), Some((0, b"0".sha3()))).unwrap();
jdb.commit_batch(2, &b"2".sha3(), Some((0, b"0".sha3()))).unwrap();
assert!(jdb.can_reconstruct_refs());
assert!(jdb.contains(&foo));
jdb.insert(b"foo");
jdb.commit(3, &b"3".sha3(), Some((1, b"1".sha3()))).unwrap();
jdb.commit_batch(3, &b"3".sha3(), Some((1, b"1".sha3()))).unwrap();
assert!(jdb.can_reconstruct_refs());
assert!(jdb.contains(&foo));
// incantation to reopen the db
}; { let mut jdb = EarlyMergeDB::new(dir.to_str().unwrap(), DatabaseConfig::default());
}; {
let mut jdb = new_db(&dir);
jdb.remove(&foo);
jdb.commit(4, &b"4".sha3(), Some((2, b"2".sha3()))).unwrap();
jdb.commit_batch(4, &b"4".sha3(), Some((2, b"2".sha3()))).unwrap();
assert!(jdb.can_reconstruct_refs());
assert!(jdb.contains(&foo));
// incantation to reopen the db
}; { let mut jdb = EarlyMergeDB::new(dir.to_str().unwrap(), DatabaseConfig::default());
}; {
let mut jdb = new_db(&dir);
jdb.commit(5, &b"5".sha3(), Some((3, b"3".sha3()))).unwrap();
jdb.commit_batch(5, &b"5".sha3(), Some((3, b"3".sha3()))).unwrap();
assert!(jdb.can_reconstruct_refs());
assert!(jdb.contains(&foo));
// incantation to reopen the db
}; { let mut jdb = EarlyMergeDB::new(dir.to_str().unwrap(), DatabaseConfig::default());
}; {
let mut jdb = new_db(&dir);
jdb.commit(6, &b"6".sha3(), Some((4, b"4".sha3()))).unwrap();
jdb.commit_batch(6, &b"6".sha3(), Some((4, b"4".sha3()))).unwrap();
assert!(jdb.can_reconstruct_refs());
assert!(!jdb.contains(&foo));
}
@@ -1037,26 +1019,26 @@ mod tests {
let mut dir = ::std::env::temp_dir();
dir.push(H32::random().hex());
let (foo, bar, baz) = {
let mut jdb = EarlyMergeDB::new(dir.to_str().unwrap(), DatabaseConfig::default());
let mut jdb = new_db(&dir);
// history is 1
let foo = jdb.insert(b"foo");
let bar = jdb.insert(b"bar");
jdb.commit(0, &b"0".sha3(), None).unwrap();
jdb.commit_batch(0, &b"0".sha3(), None).unwrap();
assert!(jdb.can_reconstruct_refs());
jdb.remove(&foo);
let baz = jdb.insert(b"baz");
jdb.commit(1, &b"1a".sha3(), Some((0, b"0".sha3()))).unwrap();
jdb.commit_batch(1, &b"1a".sha3(), Some((0, b"0".sha3()))).unwrap();
assert!(jdb.can_reconstruct_refs());
jdb.remove(&bar);
jdb.commit(1, &b"1b".sha3(), Some((0, b"0".sha3()))).unwrap();
jdb.commit_batch(1, &b"1b".sha3(), Some((0, b"0".sha3()))).unwrap();
assert!(jdb.can_reconstruct_refs());
(foo, bar, baz)
};
{
let mut jdb = EarlyMergeDB::new(dir.to_str().unwrap(), DatabaseConfig::default());
jdb.commit(2, &b"2b".sha3(), Some((1, b"1b".sha3()))).unwrap();
let mut jdb = new_db(&dir);
jdb.commit_batch(2, &b"2b".sha3(), Some((1, b"1b".sha3()))).unwrap();
assert!(jdb.can_reconstruct_refs());
assert!(jdb.contains(&foo));
assert!(!jdb.contains(&baz));

128
util/src/journaldb/mod.rs
View File

@@ -17,7 +17,7 @@
//! `JournalDB` interface and implementation.
use common::*;
use kvdb::DatabaseConfig;
use kvdb::Database;
/// Export the journaldb module.
pub mod traits;
@@ -30,7 +30,7 @@ mod refcounteddb;
pub use self::traits::JournalDB;
/// A journal database algorithm.
#[derive(Debug, Clone, Copy)]
#[derive(Debug, PartialEq, Clone, Copy)]
pub enum Algorithm {
/// Keep all keys forever.
Archive,
@@ -60,28 +60,128 @@ impl Default for Algorithm {
fn default() -> Algorithm { Algorithm::OverlayRecent }
}
impl FromStr for Algorithm {
type Err = String;
fn from_str(s: &str) -> Result<Self, Self::Err> {
match s {
"archive" => Ok(Algorithm::Archive),
"light" => Ok(Algorithm::EarlyMerge),
"fast" => Ok(Algorithm::OverlayRecent),
"basic" => Ok(Algorithm::RefCounted),
e => Err(format!("Invalid algorithm: {}", e)),
}
}
}
impl Algorithm {
/// Returns static str describing journal database algorithm.
pub fn as_str(&self) -> &'static str {
match *self {
Algorithm::Archive => "archive",
Algorithm::EarlyMerge => "light",
Algorithm::OverlayRecent => "fast",
Algorithm::RefCounted => "basic",
}
}
/// Returns static str describing journal database algorithm.
pub fn as_internal_name_str(&self) -> &'static str {
match *self {
Algorithm::Archive => "archive",
Algorithm::EarlyMerge => "earlymerge",
Algorithm::OverlayRecent => "overlayrecent",
Algorithm::RefCounted => "refcounted",
}
}
/// Returns true if pruning strategy is stable
pub fn is_stable(&self) -> bool {
match *self {
Algorithm::Archive | Algorithm::OverlayRecent => true,
_ => false,
}
}
/// Returns all algorithm types.
pub fn all_types() -> Vec<Algorithm> {
vec![Algorithm::Archive, Algorithm::EarlyMerge, Algorithm::OverlayRecent, Algorithm::RefCounted]
}
}
impl fmt::Display for Algorithm {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
write!(f, "{}", match self {
&Algorithm::Archive => "archive",
&Algorithm::EarlyMerge => "earlymerge",
&Algorithm::OverlayRecent => "overlayrecent",
&Algorithm::RefCounted => "refcounted",
})
write!(f, "{}", self.as_str())
}
}
/// Create a new `JournalDB` trait object.
pub fn new(path: &str, algorithm: Algorithm, config: DatabaseConfig) -> Box<JournalDB> {
pub fn new(backing: Arc<Database>, algorithm: Algorithm, col: Option<u32>) -> Box<JournalDB> {
match algorithm {
Algorithm::Archive => Box::new(archivedb::ArchiveDB::new(path, config)),
Algorithm::EarlyMerge => Box::new(earlymergedb::EarlyMergeDB::new(path, config)),
Algorithm::OverlayRecent => Box::new(overlayrecentdb::OverlayRecentDB::new(path, config)),
Algorithm::RefCounted => Box::new(refcounteddb::RefCountedDB::new(path, config)),
Algorithm::Archive => Box::new(archivedb::ArchiveDB::new(backing, col)),
Algorithm::EarlyMerge => Box::new(earlymergedb::EarlyMergeDB::new(backing, col)),
Algorithm::OverlayRecent => Box::new(overlayrecentdb::OverlayRecentDB::new(backing, col)),
Algorithm::RefCounted => Box::new(refcounteddb::RefCountedDB::new(backing, col)),
}
}
// all keys must be at least 12 bytes
const DB_PREFIX_LEN : usize = 12;
const LATEST_ERA_KEY : [u8; DB_PREFIX_LEN] = [ b'l', b'a', b's', b't', 0, 0, 0, 0, 0, 0, 0, 0 ];
const VERSION_KEY : [u8; DB_PREFIX_LEN] = [ b'j', b'v', b'e', b'r', 0, 0, 0, 0, 0, 0, 0, 0 ];
#[cfg(test)]
mod tests {
use super::Algorithm;
#[test]
fn test_journal_algorithm_parsing() {
assert_eq!(Algorithm::Archive, "archive".parse().unwrap());
assert_eq!(Algorithm::EarlyMerge, "light".parse().unwrap());
assert_eq!(Algorithm::OverlayRecent, "fast".parse().unwrap());
assert_eq!(Algorithm::RefCounted, "basic".parse().unwrap());
}
#[test]
fn test_journal_algorithm_printing() {
assert_eq!(Algorithm::Archive.to_string(), "archive".to_owned());
assert_eq!(Algorithm::EarlyMerge.to_string(), "light".to_owned());
assert_eq!(Algorithm::OverlayRecent.to_string(), "fast".to_owned());
assert_eq!(Algorithm::RefCounted.to_string(), "basic".to_owned());
}
#[test]
fn test_journal_algorithm_is_stable() {
assert!(Algorithm::Archive.is_stable());
assert!(Algorithm::OverlayRecent.is_stable());
assert!(!Algorithm::EarlyMerge.is_stable());
assert!(!Algorithm::RefCounted.is_stable());
}
#[test]
fn test_journal_algorithm_default() {
assert_eq!(Algorithm::default(), Algorithm::OverlayRecent);
}
#[test]
fn test_journal_algorithm_all_types() {
// compiling should fail if some cases are not covered
let mut archive = 0;
let mut earlymerge = 0;
let mut overlayrecent = 0;
let mut refcounted = 0;
for a in &Algorithm::all_types() {
match *a {
Algorithm::Archive => archive += 1,
Algorithm::EarlyMerge => earlymerge += 1,
Algorithm::OverlayRecent => overlayrecent += 1,
Algorithm::RefCounted => refcounted += 1,
}
}
assert_eq!(archive, 1);
assert_eq!(earlymerge, 1);
assert_eq!(overlayrecent, 1);
assert_eq!(refcounted, 1);
}
}

315
util/src/journaldb/overlayrecentdb.rs
View File

@@ -20,8 +20,8 @@ use common::*;
use rlp::*;
use hashdb::*;
use memorydb::*;
use super::{DB_PREFIX_LEN, LATEST_ERA_KEY, VERSION_KEY};
use kvdb::{Database, DBTransaction, DatabaseConfig};
use super::{DB_PREFIX_LEN, LATEST_ERA_KEY};
use kvdb::{Database, DBTransaction};
#[cfg(test)]
use std::env;
use super::JournalDB;
@@ -61,6 +61,7 @@ pub struct OverlayRecentDB {
transaction_overlay: MemoryDB,
backing: Arc<Database>,
journal_overlay: Arc<RwLock<JournalOverlay>>,
column: Option<u32>,
}
#[derive(PartialEq)]
@@ -89,38 +90,22 @@ impl Clone for OverlayRecentDB {
transaction_overlay: self.transaction_overlay.clone(),
backing: self.backing.clone(),
journal_overlay: self.journal_overlay.clone(),
column: self.column.clone(),
}
}
}
const DB_VERSION : u32 = 0x203;
const PADDING : [u8; 10] = [ 0u8; 10 ];
impl OverlayRecentDB {
/// Create a new instance from file
pub fn new(path: &str, config: DatabaseConfig) -> OverlayRecentDB {
Self::from_prefs(path, config)
}
/// Create a new instance from file
pub fn from_prefs(path: &str, config: DatabaseConfig) -> OverlayRecentDB {
let backing = Database::open(&config, path).unwrap_or_else(|e| {
panic!("Error opening state db: {}", e);
});
if !backing.is_empty() {
match backing.get(&VERSION_KEY).map(|d| d.map(|v| decode::<u32>(&v))) {
Ok(Some(DB_VERSION)) => {}
v => panic!("Incompatible DB version, expected {}, got {:?}; to resolve, remove {} and restart.", DB_VERSION, v, path)
}
} else {
backing.put(&VERSION_KEY, &encode(&DB_VERSION)).expect("Error writing version to database");
}
let journal_overlay = Arc::new(RwLock::new(OverlayRecentDB::read_overlay(&backing)));
/// Create a new instance.
pub fn new(backing: Arc<Database>, col: Option<u32>) -> OverlayRecentDB {
let journal_overlay = Arc::new(RwLock::new(OverlayRecentDB::read_overlay(&backing, col)));
OverlayRecentDB {
transaction_overlay: MemoryDB::new(),
backing: Arc::new(backing),
backing: backing,
journal_overlay: journal_overlay,
column: col,
}
}
@@ -129,31 +114,32 @@ impl OverlayRecentDB {
pub fn new_temp() -> OverlayRecentDB {
let mut dir = env::temp_dir();
dir.push(H32::random().hex());
Self::new(dir.to_str().unwrap(), DatabaseConfig::default())
let backing = Arc::new(Database::open_default(dir.to_str().unwrap()).unwrap());
Self::new(backing, None)
}
#[cfg(test)]
fn can_reconstruct_refs(&self) -> bool {
let reconstructed = Self::read_overlay(&self.backing);
let reconstructed = Self::read_overlay(&self.backing, self.column);
let journal_overlay = self.journal_overlay.read();
*journal_overlay == reconstructed
}
fn payload(&self, key: &H256) -> Option<Bytes> {
self.backing.get(key).expect("Low-level database error. Some issue with your hard disk?").map(|v| v.to_vec())
self.backing.get(self.column, key).expect("Low-level database error. Some issue with your hard disk?").map(|v| v.to_vec())
}
fn read_overlay(db: &Database) -> JournalOverlay {
fn read_overlay(db: &Database, col: Option<u32>) -> JournalOverlay {
let mut journal = HashMap::new();
let mut overlay = MemoryDB::new();
let mut count = 0;
let mut latest_era = None;
if let Some(val) = db.get(&LATEST_ERA_KEY).expect("Low-level database error.") {
if let Some(val) = db.get(col, &LATEST_ERA_KEY).expect("Low-level database error.") {
let mut era = decode::<u64>(&val);
latest_era = Some(era);
loop {
let mut index = 0usize;
while let Some(rlp_data) = db.get({
while let Some(rlp_data) = db.get(col, {
let mut r = RlpStream::new_list(3);
r.append(&era);
r.append(&index);
@@ -193,7 +179,7 @@ impl OverlayRecentDB {
#[inline]
fn to_short_key(key: &H256) -> H256 {
let mut k = H256::new();
&mut k[0..DB_PREFIX_LEN].copy_from_slice(&key[0..DB_PREFIX_LEN]);
k[0..DB_PREFIX_LEN].copy_from_slice(&key[0..DB_PREFIX_LEN]);
k
}
}
@@ -212,21 +198,24 @@ impl JournalDB for OverlayRecentDB {
}
fn is_empty(&self) -> bool {
self.backing.get(&LATEST_ERA_KEY).expect("Low level database error").is_none()
self.backing.get(self.column, &LATEST_ERA_KEY).expect("Low level database error").is_none()
}
fn backing(&self) -> &Arc<Database> {
&self.backing
}
fn latest_era(&self) -> Option<u64> { self.journal_overlay.read().latest_era }
fn state(&self, key: &H256) -> Option<Bytes> {
let v = self.journal_overlay.read().backing_overlay.get(&OverlayRecentDB::to_short_key(key)).map(|v| v.to_vec());
v.or_else(|| self.backing.get_by_prefix(&key[0..DB_PREFIX_LEN]).map(|b| b.to_vec()))
v.or_else(|| self.backing.get_by_prefix(self.column, &key[0..DB_PREFIX_LEN]).map(|b| b.to_vec()))
}
fn commit(&mut self, now: u64, id: &H256, end: Option<(u64, H256)>) -> Result<u32, UtilError> {
fn commit(&mut self, batch: &DBTransaction, now: u64, id: &H256, end: Option<(u64, H256)>) -> Result<u32, UtilError> {
// record new commit's details.
trace!("commit: #{} ({}), end era: {:?}", now, id, end);
let mut journal_overlay = self.journal_overlay.write();
let batch = DBTransaction::new();
{
let mut r = RlpStream::new_list(3);
let mut tx = self.transaction_overlay.drain();
@@ -249,9 +238,9 @@ impl JournalDB for OverlayRecentDB {
k.append(&now);
k.append(&index);
k.append(&&PADDING[..]);
try!(batch.put(&k.drain(), r.as_raw()));
try!(batch.put(self.column, &k.drain(), r.as_raw()));
if journal_overlay.latest_era.map_or(true, |e| now > e) {
try!(batch.put(&LATEST_ERA_KEY, &encode(&now)));
try!(batch.put(self.column, &LATEST_ERA_KEY, &encode(&now)));
journal_overlay.latest_era = Some(now);
}
journal_overlay.journal.entry(now).or_insert_with(Vec::new).push(JournalEntry { id: id.clone(), insertions: inserted_keys, deletions: removed_keys });
@@ -271,7 +260,7 @@ impl JournalDB for OverlayRecentDB {
r.append(&end_era);
r.append(&index);
r.append(&&PADDING[..]);
try!(batch.delete(&r.drain()));
try!(batch.delete(self.column, &r.drain()));
trace!("commit: Delete journal for time #{}.{}: {}, (canon was {}): +{} -{} entries", end_era, index, journal.id, canon_id, journal.insertions.len(), journal.deletions.len());
{
if canon_id == journal.id {
@@ -290,7 +279,7 @@ impl JournalDB for OverlayRecentDB {
}
// apply canon inserts first
for (k, v) in canon_insertions {
try!(batch.put(&k, &v));
try!(batch.put(self.column, &k, &v));
}
// update the overlay
for k in overlay_deletions {
@@ -299,13 +288,12 @@ impl JournalDB for OverlayRecentDB {
// apply canon deletions
for k in canon_deletions {
if !journal_overlay.backing_overlay.contains(&OverlayRecentDB::to_short_key(&k)) {
try!(batch.delete(&k));
try!(batch.delete(self.column, &k));
}
}
}
journal_overlay.journal.remove(&end_era);
}
try!(self.backing.write(batch));
Ok(0)
}
@@ -314,7 +302,7 @@ impl JournalDB for OverlayRecentDB {
impl HashDB for OverlayRecentDB {
fn keys(&self) -> HashMap<H256, i32> {
let mut ret: HashMap<H256, i32> = HashMap::new();
for (key, _) in self.backing.iter() {
for (key, _) in self.backing.iter(self.column) {
let h = H256::from_slice(key.deref());
ret.insert(h, 1);
}
@@ -374,7 +362,12 @@ mod tests {
use hashdb::*;
use log::init_log;
use journaldb::JournalDB;
use kvdb::DatabaseConfig;
use kvdb::Database;
fn new_db(path: &Path) -> OverlayRecentDB {
let backing = Arc::new(Database::open_default(path.to_str().unwrap()).unwrap());
OverlayRecentDB::new(backing, None)
}
#[test]
fn insert_same_in_fork() {
@@ -382,25 +375,25 @@ mod tests {
let mut jdb = OverlayRecentDB::new_temp();
let x = jdb.insert(b"X");
jdb.commit(1, &b"1".sha3(), None).unwrap();
jdb.commit_batch(1, &b"1".sha3(), None).unwrap();
assert!(jdb.can_reconstruct_refs());
jdb.commit(2, &b"2".sha3(), None).unwrap();
jdb.commit_batch(2, &b"2".sha3(), None).unwrap();
assert!(jdb.can_reconstruct_refs());
jdb.commit(3, &b"1002a".sha3(), Some((1, b"1".sha3()))).unwrap();
jdb.commit_batch(3, &b"1002a".sha3(), Some((1, b"1".sha3()))).unwrap();
assert!(jdb.can_reconstruct_refs());
jdb.commit(4, &b"1003a".sha3(), Some((2, b"2".sha3()))).unwrap();
jdb.commit_batch(4, &b"1003a".sha3(), Some((2, b"2".sha3()))).unwrap();
assert!(jdb.can_reconstruct_refs());
jdb.remove(&x);
jdb.commit(3, &b"1002b".sha3(), Some((1, b"1".sha3()))).unwrap();
jdb.commit_batch(3, &b"1002b".sha3(), Some((1, b"1".sha3()))).unwrap();
assert!(jdb.can_reconstruct_refs());
let x = jdb.insert(b"X");
jdb.commit(4, &b"1003b".sha3(), Some((2, b"2".sha3()))).unwrap();
jdb.commit_batch(4, &b"1003b".sha3(), Some((2, b"2".sha3()))).unwrap();
assert!(jdb.can_reconstruct_refs());
jdb.commit(5, &b"1004a".sha3(), Some((3, b"1002a".sha3()))).unwrap();
jdb.commit_batch(5, &b"1004a".sha3(), Some((3, b"1002a".sha3()))).unwrap();
assert!(jdb.can_reconstruct_refs());
jdb.commit(6, &b"1005a".sha3(), Some((4, b"1003a".sha3()))).unwrap();
jdb.commit_batch(6, &b"1005a".sha3(), Some((4, b"1003a".sha3()))).unwrap();
assert!(jdb.can_reconstruct_refs());
assert!(jdb.contains(&x));
@@ -411,20 +404,20 @@ mod tests {
// history is 3
let mut jdb = OverlayRecentDB::new_temp();
let h = jdb.insert(b"foo");
jdb.commit(0, &b"0".sha3(), None).unwrap();
jdb.commit_batch(0, &b"0".sha3(), None).unwrap();
assert!(jdb.can_reconstruct_refs());
assert!(jdb.contains(&h));
jdb.remove(&h);
jdb.commit(1, &b"1".sha3(), None).unwrap();
jdb.commit_batch(1, &b"1".sha3(), None).unwrap();
assert!(jdb.can_reconstruct_refs());
assert!(jdb.contains(&h));
jdb.commit(2, &b"2".sha3(), None).unwrap();
jdb.commit_batch(2, &b"2".sha3(), None).unwrap();
assert!(jdb.can_reconstruct_refs());
assert!(jdb.contains(&h));
jdb.commit(3, &b"3".sha3(), Some((0, b"0".sha3()))).unwrap();
jdb.commit_batch(3, &b"3".sha3(), Some((0, b"0".sha3()))).unwrap();
assert!(jdb.can_reconstruct_refs());
assert!(jdb.contains(&h));
jdb.commit(4, &b"4".sha3(), Some((1, b"1".sha3()))).unwrap();
jdb.commit_batch(4, &b"4".sha3(), Some((1, b"1".sha3()))).unwrap();
assert!(jdb.can_reconstruct_refs());
assert!(!jdb.contains(&h));
}
@@ -436,7 +429,7 @@ mod tests {
let foo = jdb.insert(b"foo");
let bar = jdb.insert(b"bar");
jdb.commit(0, &b"0".sha3(), None).unwrap();
jdb.commit_batch(0, &b"0".sha3(), None).unwrap();
assert!(jdb.can_reconstruct_refs());
assert!(jdb.contains(&foo));
assert!(jdb.contains(&bar));
@@ -444,7 +437,7 @@ mod tests {
jdb.remove(&foo);
jdb.remove(&bar);
let baz = jdb.insert(b"baz");
jdb.commit(1, &b"1".sha3(), Some((0, b"0".sha3()))).unwrap();
jdb.commit_batch(1, &b"1".sha3(), Some((0, b"0".sha3()))).unwrap();
assert!(jdb.can_reconstruct_refs());
assert!(jdb.contains(&foo));
assert!(jdb.contains(&bar));
@@ -452,20 +445,20 @@ mod tests {
let foo = jdb.insert(b"foo");
jdb.remove(&baz);
jdb.commit(2, &b"2".sha3(), Some((1, b"1".sha3()))).unwrap();
jdb.commit_batch(2, &b"2".sha3(), Some((1, b"1".sha3()))).unwrap();
assert!(jdb.can_reconstruct_refs());
assert!(jdb.contains(&foo));
assert!(!jdb.contains(&bar));
assert!(jdb.contains(&baz));
jdb.remove(&foo);
jdb.commit(3, &b"3".sha3(), Some((2, b"2".sha3()))).unwrap();
jdb.commit_batch(3, &b"3".sha3(), Some((2, b"2".sha3()))).unwrap();
assert!(jdb.can_reconstruct_refs());
assert!(jdb.contains(&foo));
assert!(!jdb.contains(&bar));
assert!(!jdb.contains(&baz));
jdb.commit(4, &b"4".sha3(), Some((3, b"3".sha3()))).unwrap();
jdb.commit_batch(4, &b"4".sha3(), Some((3, b"3".sha3()))).unwrap();
assert!(jdb.can_reconstruct_refs());
assert!(!jdb.contains(&foo));
assert!(!jdb.contains(&bar));
@@ -479,25 +472,25 @@ mod tests {
let foo = jdb.insert(b"foo");
let bar = jdb.insert(b"bar");
jdb.commit(0, &b"0".sha3(), None).unwrap();
jdb.commit_batch(0, &b"0".sha3(), None).unwrap();
assert!(jdb.can_reconstruct_refs());
assert!(jdb.contains(&foo));
assert!(jdb.contains(&bar));
jdb.remove(&foo);
let baz = jdb.insert(b"baz");
jdb.commit(1, &b"1a".sha3(), Some((0, b"0".sha3()))).unwrap();
jdb.commit_batch(1, &b"1a".sha3(), Some((0, b"0".sha3()))).unwrap();
assert!(jdb.can_reconstruct_refs());
jdb.remove(&bar);
jdb.commit(1, &b"1b".sha3(), Some((0, b"0".sha3()))).unwrap();
jdb.commit_batch(1, &b"1b".sha3(), Some((0, b"0".sha3()))).unwrap();
assert!(jdb.can_reconstruct_refs());
assert!(jdb.contains(&foo));
assert!(jdb.contains(&bar));
assert!(jdb.contains(&baz));
jdb.commit(2, &b"2b".sha3(), Some((1, b"1b".sha3()))).unwrap();
jdb.commit_batch(2, &b"2b".sha3(), Some((1, b"1b".sha3()))).unwrap();
assert!(jdb.can_reconstruct_refs());
assert!(jdb.contains(&foo));
assert!(!jdb.contains(&baz));
@@ -510,112 +503,105 @@ mod tests {
let mut jdb = OverlayRecentDB::new_temp();
let foo = jdb.insert(b"foo");
jdb.commit(0, &b"0".sha3(), None).unwrap();
jdb.commit_batch(0, &b"0".sha3(), None).unwrap();
assert!(jdb.can_reconstruct_refs());
assert!(jdb.contains(&foo));
jdb.remove(&foo);
jdb.commit(1, &b"1".sha3(), Some((0, b"0".sha3()))).unwrap();
jdb.commit_batch(1, &b"1".sha3(), Some((0, b"0".sha3()))).unwrap();
assert!(jdb.can_reconstruct_refs());
jdb.insert(b"foo");
assert!(jdb.contains(&foo));
jdb.commit(2, &b"2".sha3(), Some((1, b"1".sha3()))).unwrap();
jdb.commit_batch(2, &b"2".sha3(), Some((1, b"1".sha3()))).unwrap();
assert!(jdb.can_reconstruct_refs());
assert!(jdb.contains(&foo));
jdb.commit(3, &b"2".sha3(), Some((0, b"2".sha3()))).unwrap();
jdb.commit_batch(3, &b"2".sha3(), Some((0, b"2".sha3()))).unwrap();
assert!(jdb.can_reconstruct_refs());
assert!(jdb.contains(&foo));
}
#[test]
fn fork_same_key_one() {
let mut dir = ::std::env::temp_dir();
dir.push(H32::random().hex());
let mut jdb = OverlayRecentDB::new(dir.to_str().unwrap(), DatabaseConfig::default());
jdb.commit(0, &b"0".sha3(), None).unwrap();
let mut jdb = OverlayRecentDB::new_temp();
jdb.commit_batch(0, &b"0".sha3(), None).unwrap();
assert!(jdb.can_reconstruct_refs());
let foo = jdb.insert(b"foo");
jdb.commit(1, &b"1a".sha3(), Some((0, b"0".sha3()))).unwrap();
jdb.commit_batch(1, &b"1a".sha3(), Some((0, b"0".sha3()))).unwrap();
assert!(jdb.can_reconstruct_refs());
jdb.insert(b"foo");
jdb.commit(1, &b"1b".sha3(), Some((0, b"0".sha3()))).unwrap();
jdb.commit_batch(1, &b"1b".sha3(), Some((0, b"0".sha3()))).unwrap();
assert!(jdb.can_reconstruct_refs());
jdb.insert(b"foo");
jdb.commit(1, &b"1c".sha3(), Some((0, b"0".sha3()))).unwrap();
jdb.commit_batch(1, &b"1c".sha3(), Some((0, b"0".sha3()))).unwrap();
assert!(jdb.can_reconstruct_refs());
assert!(jdb.contains(&foo));
jdb.commit(2, &b"2a".sha3(), Some((1, b"1a".sha3()))).unwrap();
jdb.commit_batch(2, &b"2a".sha3(), Some((1, b"1a".sha3()))).unwrap();
assert!(jdb.can_reconstruct_refs());
assert!(jdb.contains(&foo));
}
#[test]
fn fork_same_key_other() {
let mut dir = ::std::env::temp_dir();
dir.push(H32::random().hex());
let mut jdb = OverlayRecentDB::new_temp();
let mut jdb = OverlayRecentDB::new(dir.to_str().unwrap(), DatabaseConfig::default());
jdb.commit(0, &b"0".sha3(), None).unwrap();
jdb.commit_batch(0, &b"0".sha3(), None).unwrap();
assert!(jdb.can_reconstruct_refs());
let foo = jdb.insert(b"foo");
jdb.commit(1, &b"1a".sha3(), Some((0, b"0".sha3()))).unwrap();
jdb.commit_batch(1, &b"1a".sha3(), Some((0, b"0".sha3()))).unwrap();
assert!(jdb.can_reconstruct_refs());
jdb.insert(b"foo");
jdb.commit(1, &b"1b".sha3(), Some((0, b"0".sha3()))).unwrap();
jdb.commit_batch(1, &b"1b".sha3(), Some((0, b"0".sha3()))).unwrap();
assert!(jdb.can_reconstruct_refs());
jdb.insert(b"foo");
jdb.commit(1, &b"1c".sha3(), Some((0, b"0".sha3()))).unwrap();
jdb.commit_batch(1, &b"1c".sha3(), Some((0, b"0".sha3()))).unwrap();
assert!(jdb.can_reconstruct_refs());
assert!(jdb.contains(&foo));
jdb.commit(2, &b"2b".sha3(), Some((1, b"1b".sha3()))).unwrap();
jdb.commit_batch(2, &b"2b".sha3(), Some((1, b"1b".sha3()))).unwrap();
assert!(jdb.can_reconstruct_refs());
assert!(jdb.contains(&foo));
}
#[test]
fn fork_ins_del_ins() {
let mut dir = ::std::env::temp_dir();
dir.push(H32::random().hex());
let mut jdb = OverlayRecentDB::new_temp();
let mut jdb = OverlayRecentDB::new(dir.to_str().unwrap(), DatabaseConfig::default());
jdb.commit(0, &b"0".sha3(), None).unwrap();
jdb.commit_batch(0, &b"0".sha3(), None).unwrap();
assert!(jdb.can_reconstruct_refs());
let foo = jdb.insert(b"foo");
jdb.commit(1, &b"1".sha3(), None).unwrap();
jdb.commit_batch(1, &b"1".sha3(), None).unwrap();
assert!(jdb.can_reconstruct_refs());
jdb.remove(&foo);
jdb.commit(2, &b"2a".sha3(), Some((0, b"0".sha3()))).unwrap();
jdb.commit_batch(2, &b"2a".sha3(), Some((0, b"0".sha3()))).unwrap();
assert!(jdb.can_reconstruct_refs());
jdb.remove(&foo);
jdb.commit(2, &b"2b".sha3(), Some((0, b"0".sha3()))).unwrap();
jdb.commit_batch(2, &b"2b".sha3(), Some((0, b"0".sha3()))).unwrap();
assert!(jdb.can_reconstruct_refs());
jdb.insert(b"foo");
jdb.commit(3, &b"3a".sha3(), Some((1, b"1".sha3()))).unwrap();
jdb.commit_batch(3, &b"3a".sha3(), Some((1, b"1".sha3()))).unwrap();
assert!(jdb.can_reconstruct_refs());
jdb.insert(b"foo");
jdb.commit(3, &b"3b".sha3(), Some((1, b"1".sha3()))).unwrap();
jdb.commit_batch(3, &b"3b".sha3(), Some((1, b"1".sha3()))).unwrap();
assert!(jdb.can_reconstruct_refs());
jdb.commit(4, &b"4a".sha3(), Some((2, b"2a".sha3()))).unwrap();
jdb.commit_batch(4, &b"4a".sha3(), Some((2, b"2a".sha3()))).unwrap();
assert!(jdb.can_reconstruct_refs());
jdb.commit(5, &b"5a".sha3(), Some((3, b"3a".sha3()))).unwrap();
jdb.commit_batch(5, &b"5a".sha3(), Some((3, b"3a".sha3()))).unwrap();
assert!(jdb.can_reconstruct_refs());
}
@@ -626,27 +612,27 @@ mod tests {
let bar = H256::random();
let foo = {
let mut jdb = OverlayRecentDB::new(dir.to_str().unwrap(), DatabaseConfig::default());
let mut jdb = new_db(&dir);
// history is 1
let foo = jdb.insert(b"foo");
jdb.emplace(bar.clone(), b"bar".to_vec());
jdb.commit(0, &b"0".sha3(), None).unwrap();
jdb.commit_batch(0, &b"0".sha3(), None).unwrap();
assert!(jdb.can_reconstruct_refs());
foo
};
{
let mut jdb = OverlayRecentDB::new(dir.to_str().unwrap(), DatabaseConfig::default());
let mut jdb = new_db(&dir);
jdb.remove(&foo);
jdb.commit(1, &b"1".sha3(), Some((0, b"0".sha3()))).unwrap();
jdb.commit_batch(1, &b"1".sha3(), Some((0, b"0".sha3()))).unwrap();
assert!(jdb.can_reconstruct_refs());
}
{
let mut jdb = OverlayRecentDB::new(dir.to_str().unwrap(), DatabaseConfig::default());
let mut jdb = new_db(&dir);
assert!(jdb.contains(&foo));
assert!(jdb.contains(&bar));
jdb.commit(2, &b"2".sha3(), Some((1, b"1".sha3()))).unwrap();
jdb.commit_batch(2, &b"2".sha3(), Some((1, b"1".sha3()))).unwrap();
assert!(jdb.can_reconstruct_refs());
assert!(!jdb.contains(&foo));
}
@@ -655,145 +641,133 @@ mod tests {
#[test]
fn insert_delete_insert_delete_insert_expunge() {
init_log();
let mut dir = ::std::env::temp_dir();
dir.push(H32::random().hex());
let mut jdb = OverlayRecentDB::new(dir.to_str().unwrap(), DatabaseConfig::default());
let mut jdb = OverlayRecentDB::new_temp();
// history is 4
let foo = jdb.insert(b"foo");
jdb.commit(0, &b"0".sha3(), None).unwrap();
jdb.commit_batch(0, &b"0".sha3(), None).unwrap();
assert!(jdb.can_reconstruct_refs());
jdb.remove(&foo);
jdb.commit(1, &b"1".sha3(), None).unwrap();
jdb.commit_batch(1, &b"1".sha3(), None).unwrap();
assert!(jdb.can_reconstruct_refs());
jdb.insert(b"foo");
jdb.commit(2, &b"2".sha3(), None).unwrap();
jdb.commit_batch(2, &b"2".sha3(), None).unwrap();
assert!(jdb.can_reconstruct_refs());
jdb.remove(&foo);
jdb.commit(3, &b"3".sha3(), None).unwrap();
jdb.commit_batch(3, &b"3".sha3(), None).unwrap();
assert!(jdb.can_reconstruct_refs());
jdb.insert(b"foo");
jdb.commit(4, &b"4".sha3(), Some((0, b"0".sha3()))).unwrap();
jdb.commit_batch(4, &b"4".sha3(), Some((0, b"0".sha3()))).unwrap();
assert!(jdb.can_reconstruct_refs());
// expunge foo
jdb.commit(5, &b"5".sha3(), Some((1, b"1".sha3()))).unwrap();
jdb.commit_batch(5, &b"5".sha3(), Some((1, b"1".sha3()))).unwrap();
assert!(jdb.can_reconstruct_refs());
}
#[test]
fn forked_insert_delete_insert_delete_insert_expunge() {
init_log();
let mut dir = ::std::env::temp_dir();
dir.push(H32::random().hex());
let mut jdb = OverlayRecentDB::new(dir.to_str().unwrap(), DatabaseConfig::default());
let mut jdb = OverlayRecentDB::new_temp();
// history is 4
let foo = jdb.insert(b"foo");
jdb.commit(0, &b"0".sha3(), None).unwrap();
jdb.commit_batch(0, &b"0".sha3(), None).unwrap();
assert!(jdb.can_reconstruct_refs());
jdb.remove(&foo);
jdb.commit(1, &b"1a".sha3(), None).unwrap();
jdb.commit_batch(1, &b"1a".sha3(), None).unwrap();
assert!(jdb.can_reconstruct_refs());
jdb.remove(&foo);
jdb.commit(1, &b"1b".sha3(), None).unwrap();
jdb.commit_batch(1, &b"1b".sha3(), None).unwrap();
assert!(jdb.can_reconstruct_refs());
jdb.insert(b"foo");
jdb.commit(2, &b"2a".sha3(), None).unwrap();
jdb.commit_batch(2, &b"2a".sha3(), None).unwrap();
assert!(jdb.can_reconstruct_refs());
jdb.insert(b"foo");
jdb.commit(2, &b"2b".sha3(), None).unwrap();
jdb.commit_batch(2, &b"2b".sha3(), None).unwrap();
assert!(jdb.can_reconstruct_refs());
jdb.remove(&foo);
jdb.commit(3, &b"3a".sha3(), None).unwrap();
jdb.commit_batch(3, &b"3a".sha3(), None).unwrap();
assert!(jdb.can_reconstruct_refs());
jdb.remove(&foo);
jdb.commit(3, &b"3b".sha3(), None).unwrap();
jdb.commit_batch(3, &b"3b".sha3(), None).unwrap();
assert!(jdb.can_reconstruct_refs());
jdb.insert(b"foo");
jdb.commit(4, &b"4a".sha3(), Some((0, b"0".sha3()))).unwrap();
jdb.commit_batch(4, &b"4a".sha3(), Some((0, b"0".sha3()))).unwrap();
assert!(jdb.can_reconstruct_refs());
jdb.insert(b"foo");
jdb.commit(4, &b"4b".sha3(), Some((0, b"0".sha3()))).unwrap();
jdb.commit_batch(4, &b"4b".sha3(), Some((0, b"0".sha3()))).unwrap();
assert!(jdb.can_reconstruct_refs());
// expunge foo
jdb.commit(5, &b"5".sha3(), Some((1, b"1a".sha3()))).unwrap();
jdb.commit_batch(5, &b"5".sha3(), Some((1, b"1a".sha3()))).unwrap();
assert!(jdb.can_reconstruct_refs());
}
#[test]
fn broken_assert() {
let mut dir = ::std::env::temp_dir();
dir.push(H32::random().hex());
let mut jdb = OverlayRecentDB::new_temp();
let mut jdb = OverlayRecentDB::new(dir.to_str().unwrap(), DatabaseConfig::default());
// history is 1
let foo = jdb.insert(b"foo");
jdb.commit(1, &b"1".sha3(), Some((0, b"0".sha3()))).unwrap();
jdb.commit_batch(1, &b"1".sha3(), Some((0, b"0".sha3()))).unwrap();
assert!(jdb.can_reconstruct_refs());
// foo is ancient history.
jdb.remove(&foo);
jdb.commit(2, &b"2".sha3(), Some((1, b"1".sha3()))).unwrap();
jdb.commit_batch(2, &b"2".sha3(), Some((1, b"1".sha3()))).unwrap();
assert!(jdb.can_reconstruct_refs());
jdb.insert(b"foo");
jdb.commit(3, &b"3".sha3(), Some((2, b"2".sha3()))).unwrap(); // BROKEN
jdb.commit_batch(3, &b"3".sha3(), Some((2, b"2".sha3()))).unwrap(); // BROKEN
assert!(jdb.can_reconstruct_refs());
assert!(jdb.contains(&foo));
jdb.remove(&foo);
jdb.commit(4, &b"4".sha3(), Some((3, b"3".sha3()))).unwrap();
jdb.commit_batch(4, &b"4".sha3(), Some((3, b"3".sha3()))).unwrap();
assert!(jdb.can_reconstruct_refs());
jdb.commit(5, &b"5".sha3(), Some((4, b"4".sha3()))).unwrap();
jdb.commit_batch(5, &b"5".sha3(), Some((4, b"4".sha3()))).unwrap();
assert!(jdb.can_reconstruct_refs());
assert!(!jdb.contains(&foo));
}
#[test]
fn reopen_test() {
let mut dir = ::std::env::temp_dir();
dir.push(H32::random().hex());
let mut jdb = OverlayRecentDB::new(dir.to_str().unwrap(), DatabaseConfig::default());
let mut jdb = OverlayRecentDB::new_temp();
// history is 4
let foo = jdb.insert(b"foo");
jdb.commit(0, &b"0".sha3(), None).unwrap();
jdb.commit_batch(0, &b"0".sha3(), None).unwrap();
assert!(jdb.can_reconstruct_refs());
jdb.commit(1, &b"1".sha3(), None).unwrap();
jdb.commit_batch(1, &b"1".sha3(), None).unwrap();
assert!(jdb.can_reconstruct_refs());
jdb.commit(2, &b"2".sha3(), None).unwrap();
jdb.commit_batch(2, &b"2".sha3(), None).unwrap();
assert!(jdb.can_reconstruct_refs());
jdb.commit(3, &b"3".sha3(), None).unwrap();
jdb.commit_batch(3, &b"3".sha3(), None).unwrap();
assert!(jdb.can_reconstruct_refs());
jdb.commit(4, &b"4".sha3(), Some((0, b"0".sha3()))).unwrap();
jdb.commit_batch(4, &b"4".sha3(), Some((0, b"0".sha3()))).unwrap();
assert!(jdb.can_reconstruct_refs());
// foo is ancient history.
jdb.insert(b"foo");
let bar = jdb.insert(b"bar");
jdb.commit(5, &b"5".sha3(), Some((1, b"1".sha3()))).unwrap();
jdb.commit_batch(5, &b"5".sha3(), Some((1, b"1".sha3()))).unwrap();
assert!(jdb.can_reconstruct_refs());
jdb.remove(&foo);
jdb.remove(&bar);
jdb.commit(6, &b"6".sha3(), Some((2, b"2".sha3()))).unwrap();
jdb.commit_batch(6, &b"6".sha3(), Some((2, b"2".sha3()))).unwrap();
assert!(jdb.can_reconstruct_refs());
jdb.insert(b"foo");
jdb.insert(b"bar");
jdb.commit(7, &b"7".sha3(), Some((3, b"3".sha3()))).unwrap();
jdb.commit_batch(7, &b"7".sha3(), Some((3, b"3".sha3()))).unwrap();
assert!(jdb.can_reconstruct_refs());
}
@@ -807,45 +781,48 @@ mod tests {
let foo = b"foo".sha3();
{
let mut jdb = OverlayRecentDB::new(dir.to_str().unwrap(), DatabaseConfig::default());
let mut jdb = new_db(&dir);
// history is 1
jdb.insert(b"foo");
jdb.commit(0, &b"0".sha3(), None).unwrap();
jdb.commit_batch(0, &b"0".sha3(), None).unwrap();
assert!(jdb.can_reconstruct_refs());
jdb.commit(1, &b"1".sha3(), None).unwrap();
jdb.commit_batch(1, &b"1".sha3(), None).unwrap();
assert!(jdb.can_reconstruct_refs());
// foo is ancient history.
jdb.remove(&foo);
jdb.commit(2, &b"2".sha3(), Some((0, b"0".sha3()))).unwrap();
jdb.commit_batch(2, &b"2".sha3(), Some((0, b"0".sha3()))).unwrap();
assert!(jdb.can_reconstruct_refs());
assert!(jdb.contains(&foo));
jdb.insert(b"foo");
jdb.commit(3, &b"3".sha3(), Some((1, b"1".sha3()))).unwrap();
jdb.commit_batch(3, &b"3".sha3(), Some((1, b"1".sha3()))).unwrap();
assert!(jdb.can_reconstruct_refs());
assert!(jdb.contains(&foo));
// incantation to reopen the db
}; { let mut jdb = OverlayRecentDB::new(dir.to_str().unwrap(), DatabaseConfig::default());
}; {
let mut jdb = new_db(&dir);
jdb.remove(&foo);
jdb.commit(4, &b"4".sha3(), Some((2, b"2".sha3()))).unwrap();
jdb.commit_batch(4, &b"4".sha3(), Some((2, b"2".sha3()))).unwrap();
assert!(jdb.can_reconstruct_refs());
assert!(jdb.contains(&foo));
// incantation to reopen the db
}; { let mut jdb = OverlayRecentDB::new(dir.to_str().unwrap(), DatabaseConfig::default());
}; {
let mut jdb = new_db(&dir);
jdb.commit(5, &b"5".sha3(), Some((3, b"3".sha3()))).unwrap();
jdb.commit_batch(5, &b"5".sha3(), Some((3, b"3".sha3()))).unwrap();
assert!(jdb.can_reconstruct_refs());
assert!(jdb.contains(&foo));
// incantation to reopen the db
}; { let mut jdb = OverlayRecentDB::new(dir.to_str().unwrap(), DatabaseConfig::default());
}; {
let mut jdb = new_db(&dir);
jdb.commit(6, &b"6".sha3(), Some((4, b"4".sha3()))).unwrap();
jdb.commit_batch(6, &b"6".sha3(), Some((4, b"4".sha3()))).unwrap();
assert!(jdb.can_reconstruct_refs());
assert!(!jdb.contains(&foo));
}
@@ -856,26 +833,26 @@ mod tests {
let mut dir = ::std::env::temp_dir();
dir.push(H32::random().hex());
let (foo, bar, baz) = {
let mut jdb = OverlayRecentDB::new(dir.to_str().unwrap(), DatabaseConfig::default());
let mut jdb = new_db(&dir);
// history is 1
let foo = jdb.insert(b"foo");
let bar = jdb.insert(b"bar");
jdb.commit(0, &b"0".sha3(), None).unwrap();
jdb.commit_batch(0, &b"0".sha3(), None).unwrap();
assert!(jdb.can_reconstruct_refs());
jdb.remove(&foo);
let baz = jdb.insert(b"baz");
jdb.commit(1, &b"1a".sha3(), Some((0, b"0".sha3()))).unwrap();
jdb.commit_batch(1, &b"1a".sha3(), Some((0, b"0".sha3()))).unwrap();
assert!(jdb.can_reconstruct_refs());
jdb.remove(&bar);
jdb.commit(1, &b"1b".sha3(), Some((0, b"0".sha3()))).unwrap();
jdb.commit_batch(1, &b"1b".sha3(), Some((0, b"0".sha3()))).unwrap();
assert!(jdb.can_reconstruct_refs());
(foo, bar, baz)
};
{
let mut jdb = OverlayRecentDB::new(dir.to_str().unwrap(), DatabaseConfig::default());
jdb.commit(2, &b"2b".sha3(), Some((1, b"1b".sha3()))).unwrap();
let mut jdb = new_db(&dir);
jdb.commit_batch(2, &b"2b".sha3(), Some((1, b"1b".sha3()))).unwrap();
assert!(jdb.can_reconstruct_refs());
assert!(jdb.contains(&foo));
assert!(!jdb.contains(&baz));
@@ -887,17 +864,17 @@ mod tests {
fn insert_older_era() {
let mut jdb = OverlayRecentDB::new_temp();
let foo = jdb.insert(b"foo");
jdb.commit(0, &b"0a".sha3(), None).unwrap();
jdb.commit_batch(0, &b"0a".sha3(), None).unwrap();
assert!(jdb.can_reconstruct_refs());
let bar = jdb.insert(b"bar");
jdb.commit(1, &b"1".sha3(), Some((0, b"0a".sha3()))).unwrap();
jdb.commit_batch(1, &b"1".sha3(), Some((0, b"0a".sha3()))).unwrap();
assert!(jdb.can_reconstruct_refs());
jdb.remove(&bar);
jdb.commit(0, &b"0b".sha3(), None).unwrap();
jdb.commit_batch(0, &b"0b".sha3(), None).unwrap();
assert!(jdb.can_reconstruct_refs());
jdb.commit(2, &b"2".sha3(), Some((1, b"1".sha3()))).unwrap();
jdb.commit_batch(2, &b"2".sha3(), Some((1, b"1".sha3()))).unwrap();
assert!(jdb.contains(&foo));
assert!(jdb.contains(&bar));

88
util/src/journaldb/refcounteddb.rs
View File

@@ -20,9 +20,9 @@ use common::*;
use rlp::*;
use hashdb::*;
use overlaydb::*;
use super::{DB_PREFIX_LEN, LATEST_ERA_KEY, VERSION_KEY};
use super::{DB_PREFIX_LEN, LATEST_ERA_KEY};
use super::traits::JournalDB;
use kvdb::{Database, DBTransaction, DatabaseConfig};
use kvdb::{Database, DBTransaction};
#[cfg(test)]
use std::env;
@@ -39,35 +39,23 @@ pub struct RefCountedDB {
latest_era: Option<u64>,
inserts: Vec<H256>,
removes: Vec<H256>,
column: Option<u32>,
}
const DB_VERSION : u32 = 0x200;
const PADDING : [u8; 10] = [ 0u8; 10 ];
impl RefCountedDB {
/// Create a new instance given a `backing` database.
pub fn new(path: &str, config: DatabaseConfig) -> RefCountedDB {
let backing = Database::open(&config, path).unwrap_or_else(|e| {
panic!("Error opening state db: {}", e);
});
if !backing.is_empty() {
match backing.get(&VERSION_KEY).map(|d| d.map(|v| decode::<u32>(&v))) {
Ok(Some(DB_VERSION)) => {},
v => panic!("Incompatible DB version, expected {}, got {:?}; to resolve, remove {} and restart.", DB_VERSION, v, path)
}
} else {
backing.put(&VERSION_KEY, &encode(&DB_VERSION)).expect("Error writing version to database");
}
let backing = Arc::new(backing);
let latest_era = backing.get(&LATEST_ERA_KEY).expect("Low-level database error.").map(|val| decode::<u64>(&val));
pub fn new(backing: Arc<Database>, col: Option<u32>) -> RefCountedDB {
let latest_era = backing.get(col, &LATEST_ERA_KEY).expect("Low-level database error.").map(|val| decode::<u64>(&val));
RefCountedDB {
forward: OverlayDB::new_with_arc(backing.clone()),
forward: OverlayDB::new(backing.clone(), col),
backing: backing,
inserts: vec![],
removes: vec![],
latest_era: latest_era,
column: col,
}
}
@@ -76,7 +64,8 @@ impl RefCountedDB {
fn new_temp() -> RefCountedDB {
let mut dir = env::temp_dir();
dir.push(H32::random().hex());
Self::new(dir.to_str().unwrap(), DatabaseConfig::default())
let backing = Arc::new(Database::open_default(dir.to_str().unwrap()).unwrap());
Self::new(backing, None)
}
}
@@ -97,6 +86,7 @@ impl JournalDB for RefCountedDB {
latest_era: self.latest_era,
inserts: self.inserts.clone(),
removes: self.removes.clone(),
column: self.column.clone(),
})
}
@@ -108,13 +98,17 @@ impl JournalDB for RefCountedDB {
self.latest_era.is_none()
}
fn backing(&self) -> &Arc<Database> {
&self.backing
}
fn latest_era(&self) -> Option<u64> { self.latest_era }
fn state(&self, id: &H256) -> Option<Bytes> {
self.backing.get_by_prefix(&id[0..DB_PREFIX_LEN]).map(|b| b.to_vec())
self.backing.get_by_prefix(self.column, &id[0..DB_PREFIX_LEN]).map(|b| b.to_vec())
}
fn commit(&mut self, now: u64, id: &H256, end: Option<(u64, H256)>) -> Result<u32, UtilError> {
fn commit(&mut self, batch: &DBTransaction, now: u64, id: &H256, end: Option<(u64, H256)>) -> Result<u32, UtilError> {
// journal format:
// [era, 0] => [ id, [insert_0, ...], [remove_0, ...] ]
// [era, 1] => [ id, [insert_0, ...], [remove_0, ...] ]
@@ -128,12 +122,11 @@ impl JournalDB for RefCountedDB {
// of its inserts otherwise.
// record new commit's details.
let batch = DBTransaction::new();
{
let mut index = 0usize;
let mut last;
while try!(self.backing.get({
while try!(self.backing.get(self.column, {
let mut r = RlpStream::new_list(3);
r.append(&now);
r.append(&index);
@@ -148,7 +141,7 @@ impl JournalDB for RefCountedDB {
r.append(id);
r.append(&self.inserts);
r.append(&self.removes);
try!(batch.put(&last, r.as_raw()));
try!(batch.put(self.column, &last, r.as_raw()));
trace!(target: "rcdb", "new journal for time #{}.{} => {}: inserts={:?}, removes={:?}", now, index, id, self.inserts, self.removes);
@@ -156,7 +149,7 @@ impl JournalDB for RefCountedDB {
self.removes.clear();
if self.latest_era.map_or(true, |e| now > e) {
try!(batch.put(&LATEST_ERA_KEY, &encode(&now)));
try!(batch.put(self.column, &LATEST_ERA_KEY, &encode(&now)));
self.latest_era = Some(now);
}
}
@@ -167,7 +160,7 @@ impl JournalDB for RefCountedDB {
let mut last;
while let Some(rlp_data) = {
// trace!(target: "rcdb", "checking for journal #{}.{}", end_era, index);
try!(self.backing.get({
try!(self.backing.get(self.column, {
let mut r = RlpStream::new_list(3);
r.append(&end_era);
r.append(&index);
@@ -183,13 +176,12 @@ impl JournalDB for RefCountedDB {
for i in &to_remove {
self.forward.remove(i);
}
try!(batch.delete(&last));
try!(batch.delete(self.column, &last));
index += 1;
}
}
let r = try!(self.forward.commit_to_batch(&batch));
try!(self.backing.write(batch));
Ok(r)
}
}
@@ -209,16 +201,16 @@ mod tests {
// history is 3
let mut jdb = RefCountedDB::new_temp();
let h = jdb.insert(b"foo");
jdb.commit(0, &b"0".sha3(), None).unwrap();
jdb.commit_batch(0, &b"0".sha3(), None).unwrap();
assert!(jdb.contains(&h));
jdb.remove(&h);
jdb.commit(1, &b"1".sha3(), None).unwrap();
jdb.commit_batch(1, &b"1".sha3(), None).unwrap();
assert!(jdb.contains(&h));
jdb.commit(2, &b"2".sha3(), None).unwrap();
jdb.commit_batch(2, &b"2".sha3(), None).unwrap();
assert!(jdb.contains(&h));
jdb.commit(3, &b"3".sha3(), Some((0, b"0".sha3()))).unwrap();
jdb.commit_batch(3, &b"3".sha3(), Some((0, b"0".sha3()))).unwrap();
assert!(jdb.contains(&h));
jdb.commit(4, &b"4".sha3(), Some((1, b"1".sha3()))).unwrap();
jdb.commit_batch(4, &b"4".sha3(), Some((1, b"1".sha3()))).unwrap();
assert!(!jdb.contains(&h));
}
@@ -228,16 +220,16 @@ mod tests {
let mut jdb = RefCountedDB::new_temp();
assert_eq!(jdb.latest_era(), None);
let h = jdb.insert(b"foo");
jdb.commit(0, &b"0".sha3(), None).unwrap();
jdb.commit_batch(0, &b"0".sha3(), None).unwrap();
assert_eq!(jdb.latest_era(), Some(0));
jdb.remove(&h);
jdb.commit(1, &b"1".sha3(), None).unwrap();
jdb.commit_batch(1, &b"1".sha3(), None).unwrap();
assert_eq!(jdb.latest_era(), Some(1));
jdb.commit(2, &b"2".sha3(), None).unwrap();
jdb.commit_batch(2, &b"2".sha3(), None).unwrap();
assert_eq!(jdb.latest_era(), Some(2));
jdb.commit(3, &b"3".sha3(), Some((0, b"0".sha3()))).unwrap();
jdb.commit_batch(3, &b"3".sha3(), Some((0, b"0".sha3()))).unwrap();
assert_eq!(jdb.latest_era(), Some(3));
jdb.commit(4, &b"4".sha3(), Some((1, b"1".sha3()))).unwrap();
jdb.commit_batch(4, &b"4".sha3(), Some((1, b"1".sha3()))).unwrap();
assert_eq!(jdb.latest_era(), Some(4));
}
@@ -248,32 +240,32 @@ mod tests {
let foo = jdb.insert(b"foo");
let bar = jdb.insert(b"bar");
jdb.commit(0, &b"0".sha3(), None).unwrap();
jdb.commit_batch(0, &b"0".sha3(), None).unwrap();
assert!(jdb.contains(&foo));
assert!(jdb.contains(&bar));
jdb.remove(&foo);
jdb.remove(&bar);
let baz = jdb.insert(b"baz");
jdb.commit(1, &b"1".sha3(), Some((0, b"0".sha3()))).unwrap();
jdb.commit_batch(1, &b"1".sha3(), Some((0, b"0".sha3()))).unwrap();
assert!(jdb.contains(&foo));
assert!(jdb.contains(&bar));
assert!(jdb.contains(&baz));
let foo = jdb.insert(b"foo");
jdb.remove(&baz);
jdb.commit(2, &b"2".sha3(), Some((1, b"1".sha3()))).unwrap();
jdb.commit_batch(2, &b"2".sha3(), Some((1, b"1".sha3()))).unwrap();
assert!(jdb.contains(&foo));
assert!(!jdb.contains(&bar));
assert!(jdb.contains(&baz));
jdb.remove(&foo);
jdb.commit(3, &b"3".sha3(), Some((2, b"2".sha3()))).unwrap();
jdb.commit_batch(3, &b"3".sha3(), Some((2, b"2".sha3()))).unwrap();
assert!(jdb.contains(&foo));
assert!(!jdb.contains(&bar));
assert!(!jdb.contains(&baz));
jdb.commit(4, &b"4".sha3(), Some((3, b"3".sha3()))).unwrap();
jdb.commit_batch(4, &b"4".sha3(), Some((3, b"3".sha3()))).unwrap();
assert!(!jdb.contains(&foo));
assert!(!jdb.contains(&bar));
assert!(!jdb.contains(&baz));
@@ -286,22 +278,22 @@ mod tests {
let foo = jdb.insert(b"foo");
let bar = jdb.insert(b"bar");
jdb.commit(0, &b"0".sha3(), None).unwrap();
jdb.commit_batch(0, &b"0".sha3(), None).unwrap();
assert!(jdb.contains(&foo));
assert!(jdb.contains(&bar));
jdb.remove(&foo);
let baz = jdb.insert(b"baz");
jdb.commit(1, &b"1a".sha3(), Some((0, b"0".sha3()))).unwrap();
jdb.commit_batch(1, &b"1a".sha3(), Some((0, b"0".sha3()))).unwrap();
jdb.remove(&bar);
jdb.commit(1, &b"1b".sha3(), Some((0, b"0".sha3()))).unwrap();
jdb.commit_batch(1, &b"1b".sha3(), Some((0, b"0".sha3()))).unwrap();
assert!(jdb.contains(&foo));
assert!(jdb.contains(&bar));
assert!(jdb.contains(&baz));
jdb.commit(2, &b"2b".sha3(), Some((1, b"1b".sha3()))).unwrap();
jdb.commit_batch(2, &b"2b".sha3(), Some((1, b"1b".sha3()))).unwrap();
assert!(jdb.contains(&foo));
assert!(!jdb.contains(&baz));
assert!(!jdb.contains(&bar));

14
util/src/journaldb/traits.rs
View File

@@ -18,6 +18,7 @@
use common::*;
use hashdb::*;
use kvdb::{Database, DBTransaction};
/// A `HashDB` which can manage a short-term journal potentially containing many forks of mutually
/// exclusive actions.
@@ -36,11 +37,22 @@ pub trait JournalDB : HashDB + Send + Sync {
/// Commit all recent insert operations and canonical historical commits' removals from the
/// old era to the backing database, reverting any non-canonical historical commit's inserts.
fn commit(&mut self, now: u64, id: &H256, end: Option<(u64, H256)>) -> Result<u32, UtilError>;
fn commit(&mut self, batch: &DBTransaction, now: u64, id: &H256, end: Option<(u64, H256)>) -> Result<u32, UtilError>;
/// State data query
fn state(&self, _id: &H256) -> Option<Bytes>;
/// Whether this database is pruned.
fn is_pruned(&self) -> bool { true }
/// Get backing database.
fn backing(&self) -> &Arc<Database>;
#[cfg(test)]
/// Commit all changes in a single batch
fn commit_batch(&mut self, now: u64, id: &H256, end: Option<(u64, H256)>) -> Result<u32, UtilError> {
let batch = self.backing().transaction();
let res = try!(self.commit(&batch, now, id, end));
self.backing().write(batch).map(|_| res).map_err(Into::into)
}
}

164
util/src/kvdb.rs
View File

@@ -18,7 +18,7 @@
use std::default::Default;
use rocksdb::{DB, Writable, WriteBatch, WriteOptions, IteratorMode, DBVector, DBIterator,
Options, DBCompactionStyle, BlockBasedOptions, Direction, Cache};
Options, DBCompactionStyle, BlockBasedOptions, Direction, Cache, Column};
const DB_BACKGROUND_FLUSHES: i32 = 2;
const DB_BACKGROUND_COMPACTIONS: i32 = 2;
@@ -26,32 +26,31 @@ const DB_BACKGROUND_COMPACTIONS: i32 = 2;
/// Write transaction. Batches a sequence of put/delete operations for efficiency.
pub struct DBTransaction {
batch: WriteBatch,
}
impl Default for DBTransaction {
fn default() -> Self {
DBTransaction::new()
}
cfs: Vec<Column>,
}
impl DBTransaction {
/// Create new transaction.
pub fn new() -> DBTransaction {
DBTransaction { batch: WriteBatch::new() }
pub fn new(db: &Database) -> DBTransaction {
DBTransaction {
batch: WriteBatch::new(),
cfs: db.cfs.clone(),
}
}
/// Insert a key-value pair in the transaction. Any existing value value will be overwritten upon write.
pub fn put(&self, key: &[u8], value: &[u8]) -> Result<(), String> {
self.batch.put(key, value)
pub fn put(&self, col: Option<u32>, key: &[u8], value: &[u8]) -> Result<(), String> {
col.map_or_else(|| self.batch.put(key, value), |c| self.batch.put_cf(self.cfs[c as usize], key, value))
}
/// Delete value by key.
pub fn delete(&self, key: &[u8]) -> Result<(), String> {
self.batch.delete(key)
pub fn delete(&self, col: Option<u32>, key: &[u8]) -> Result<(), String> {
col.map_or_else(|| self.batch.delete(key), |c| self.batch.delete_cf(self.cfs[c as usize], key))
}
}
/// Compaction profile for the database settings
#[derive(Clone, Copy)]
pub struct CompactionProfile {
/// L0-L1 target file size
pub initial_file_size: u64,
@@ -61,16 +60,18 @@ pub struct CompactionProfile {
pub write_rate_limit: Option<u64>,
}
impl CompactionProfile {
impl Default for CompactionProfile {
/// Default profile suitable for most storage
pub fn default() -> CompactionProfile {
fn default() -> CompactionProfile {
CompactionProfile {
initial_file_size: 32 * 1024 * 1024,
file_size_multiplier: 2,
write_rate_limit: None,
}
}
}
impl CompactionProfile {
/// Slow hdd compaction profile
pub fn hdd() -> CompactionProfile {
CompactionProfile {
@@ -82,6 +83,7 @@ impl CompactionProfile {
}
/// Database configuration
#[derive(Clone, Copy)]
pub struct DatabaseConfig {
/// Max number of open files.
pub max_open_files: i32,
@@ -89,22 +91,18 @@ pub struct DatabaseConfig {
pub cache_size: Option<usize>,
/// Compaction profile
pub compaction: CompactionProfile,
/// Set number of columns
pub columns: Option<u32>,
/// Should we keep WAL enabled?
pub wal: bool,
}
impl DatabaseConfig {
/// Database with default settings and specified cache size
pub fn with_cache(cache_size: usize) -> DatabaseConfig {
DatabaseConfig {
cache_size: Some(cache_size),
max_open_files: 256,
compaction: CompactionProfile::default(),
}
}
/// Modify the compaction profile
pub fn compaction(mut self, profile: CompactionProfile) -> Self {
self.compaction = profile;
self
/// Create new `DatabaseConfig` with default parameters and specified set of columns.
pub fn with_columns(columns: Option<u32>) -> Self {
let mut config = Self::default();
config.columns = columns;
config
}
}
@@ -112,8 +110,10 @@ impl Default for DatabaseConfig {
fn default() -> DatabaseConfig {
DatabaseConfig {
cache_size: None,
max_open_files: 256,
max_open_files: 1024,
compaction: CompactionProfile::default(),
columns: None,
wal: true,
}
}
}
@@ -135,6 +135,7 @@ impl<'a> Iterator for DatabaseIterator {
pub struct Database {
db: DB,
write_opts: WriteOptions,
cfs: Vec<Column>,
}
impl Database {
@@ -168,10 +169,37 @@ impl Database {
opts.set_block_based_table_factory(&block_opts);
}
let write_opts = WriteOptions::new();
//write_opts.disable_wal(true); // TODO: make sure this is safe
let mut write_opts = WriteOptions::new();
if !config.wal {
write_opts.disable_wal(true);
}
let db = match DB::open(&opts, path) {
let mut cfs: Vec<Column> = Vec::new();
let db = match config.columns {
Some(columns) => {
let cfnames: Vec<_> = (0..columns).map(|c| format!("col{}", c)).collect();
let cfnames: Vec<&str> = cfnames.iter().map(|n| n as &str).collect();
match DB::open_cf(&opts, path, &cfnames) {
Ok(db) => {
cfs = cfnames.iter().map(|n| db.cf_handle(n).unwrap()).collect();
assert!(cfs.len() == columns as usize);
Ok(db)
}
Err(_) => {
// retry and create CFs
match DB::open_cf(&opts, path, &[]) {
Ok(mut db) => {
cfs = cfnames.iter().map(|n| db.create_cf(n, &opts).unwrap()).collect();
Ok(db)
},
err @ Err(_) => err,
}
}
}
},
None => DB::open(&opts, path)
};
let db = match db {
Ok(db) => db,
Err(ref s) if s.starts_with("Corruption:") => {
info!("{}", s);
@@ -181,17 +209,12 @@ impl Database {
},
Err(s) => { return Err(s); }
};
Ok(Database { db: db, write_opts: write_opts, })
Ok(Database { db: db, write_opts: write_opts, cfs: cfs })
}
/// Insert a key-value pair in the transaction. Any existing value value will be overwritten.
pub fn put(&self, key: &[u8], value: &[u8]) -> Result<(), String> {
self.db.put_opt(key, value, &self.write_opts)
}
/// Delete value by key.
pub fn delete(&self, key: &[u8]) -> Result<(), String> {
self.db.delete_opt(key, &self.write_opts)
/// Creates new transaction for this database.
pub fn transaction(&self) -> DBTransaction {
DBTransaction::new(self)
}
/// Commit transaction to database.
@@ -200,13 +223,14 @@ impl Database {
}
/// Get value by key.
pub fn get(&self, key: &[u8]) -> Result<Option<DBVector>, String> {
self.db.get(key)
pub fn get(&self, col: Option<u32>, key: &[u8]) -> Result<Option<DBVector>, String> {
col.map_or_else(|| self.db.get(key), |c| self.db.get_cf(self.cfs[c as usize], key))
}
/// Get value by partial key. Prefix size should match configured prefix size.
pub fn get_by_prefix(&self, prefix: &[u8]) -> Option<Box<[u8]>> {
let mut iter = self.db.iterator(IteratorMode::From(prefix, Direction::Forward));
pub fn get_by_prefix(&self, col: Option<u32>, prefix: &[u8]) -> Option<Box<[u8]>> {
let mut iter = col.map_or_else(|| self.db.iterator(IteratorMode::From(prefix, Direction::Forward)),
|c| self.db.iterator_cf(self.cfs[c as usize], IteratorMode::From(prefix, Direction::Forward)).unwrap());
match iter.next() {
// TODO: use prefix_same_as_start read option (not availabele in C API currently)
Some((k, v)) => if k[0 .. prefix.len()] == prefix[..] { Some(v) } else { None },
@@ -215,13 +239,14 @@ impl Database {
}
/// Check if there is anything in the database.
pub fn is_empty(&self) -> bool {
self.db.iterator(IteratorMode::Start).next().is_none()
pub fn is_empty(&self, col: Option<u32>) -> bool {
self.iter(col).next().is_none()
}
/// Check if there is anything in the database.
pub fn iter(&self) -> DatabaseIterator {
DatabaseIterator { iter: self.db.iterator(IteratorMode::Start) }
/// Get database iterator.
pub fn iter(&self, col: Option<u32>) -> DatabaseIterator {
col.map_or_else(|| DatabaseIterator { iter: self.db.iterator(IteratorMode::Start) },
|c| DatabaseIterator { iter: self.db.iterator_cf(self.cfs[c as usize], IteratorMode::Start).unwrap() })
}
}
@@ -240,39 +265,46 @@ mod tests {
let key2 = H256::from_str("03c69be41d0b7e40352fc85be1cd65eb03d40ef8427a0ca4596b1ead9a00e9fc").unwrap();
let key3 = H256::from_str("01c69be41d0b7e40352fc85be1cd65eb03d40ef8427a0ca4596b1ead9a00e9fc").unwrap();
db.put(&key1, b"cat").unwrap();
db.put(&key2, b"dog").unwrap();
let batch = db.transaction();
batch.put(None, &key1, b"cat").unwrap();
batch.put(None, &key2, b"dog").unwrap();
db.write(batch).unwrap();
assert_eq!(db.get(&key1).unwrap().unwrap().deref(), b"cat");
assert_eq!(db.get(None, &key1).unwrap().unwrap().deref(), b"cat");
let contents: Vec<_> = db.iter().collect();
let contents: Vec<_> = db.iter(None).collect();
assert_eq!(contents.len(), 2);
assert_eq!(&*contents[0].0, key1.deref());
assert_eq!(&*contents[0].1, b"cat");
assert_eq!(&*contents[1].0, key2.deref());
assert_eq!(&*contents[1].1, b"dog");
db.delete(&key1).unwrap();
assert!(db.get(&key1).unwrap().is_none());
db.put(&key1, b"cat").unwrap();
let batch = db.transaction();
batch.delete(None, &key1).unwrap();
db.write(batch).unwrap();
let transaction = DBTransaction::new();
transaction.put(&key3, b"elephant").unwrap();
transaction.delete(&key1).unwrap();
assert!(db.get(None, &key1).unwrap().is_none());
let batch = db.transaction();
batch.put(None, &key1, b"cat").unwrap();
db.write(batch).unwrap();
let transaction = db.transaction();
transaction.put(None, &key3, b"elephant").unwrap();
transaction.delete(None, &key1).unwrap();
db.write(transaction).unwrap();
assert!(db.get(&key1).unwrap().is_none());
assert_eq!(db.get(&key3).unwrap().unwrap().deref(), b"elephant");
assert!(db.get(None, &key1).unwrap().is_none());
assert_eq!(db.get(None, &key3).unwrap().unwrap().deref(), b"elephant");
assert_eq!(db.get_by_prefix(&key3).unwrap().deref(), b"elephant");
assert_eq!(db.get_by_prefix(&key2).unwrap().deref(), b"dog");
assert_eq!(db.get_by_prefix(None, &key3).unwrap().deref(), b"elephant");
assert_eq!(db.get_by_prefix(None, &key2).unwrap().deref(), b"dog");
}
#[test]
fn kvdb() {
let path = RandomTempPath::create_dir();
let smoke = Database::open_default(path.as_path().to_str().unwrap()).unwrap();
assert!(smoke.is_empty());
assert!(smoke.is_empty(None));
test_db(&DatabaseConfig::default());
}
}

8
util/src/lib.rs
View File

@@ -19,20 +19,16 @@
#![cfg_attr(feature="dev", plugin(clippy))]
// Clippy settings
// TODO [todr] not really sure
// Most of the time much more readable
#![cfg_attr(feature="dev", allow(needless_range_loop))]
// Shorter than if-else
#![cfg_attr(feature="dev", allow(match_bool))]
// We use that to be more explicit about handled cases
#![cfg_attr(feature="dev", allow(match_same_arms))]
// Keeps consistency (all lines with `.clone()`) and helpful when changing ref to non-ref.
// Keeps consistency (all lines with `.clone()`).
#![cfg_attr(feature="dev", allow(clone_on_copy))]
// In most cases it expresses function flow better
#![cfg_attr(feature="dev", allow(if_not_else))]
// TODO [todr] a lot of warnings to be fixed
#![cfg_attr(feature="dev", allow(needless_borrow))]
#![cfg_attr(feature="dev", allow(assign_op_pattern))]
#![cfg_attr(feature="dev", allow(unnecessary_operation))]
//! Ethcore-util library

98
util/src/migration/mod.rs
View File

@@ -29,12 +29,15 @@ use ::kvdb::{CompactionProfile, Database, DatabaseConfig, DBTransaction};
pub struct Config {
/// Defines how many elements should be migrated at once.
pub batch_size: usize,
/// Database compaction profile.
pub compaction_profile: CompactionProfile,
}
impl Default for Config {
fn default() -> Self {
Config {
batch_size: 1024,
compaction_profile: Default::default(),
}
}
}
@@ -43,14 +46,16 @@ impl Default for Config {
pub struct Batch {
inner: BTreeMap<Vec<u8>, Vec<u8>>,
batch_size: usize,
column: Option<u32>,
}
impl Batch {
/// Make a new batch with the given config.
pub fn new(config: &Config) -> Self {
pub fn new(config: &Config, col: Option<u32>) -> Self {
Batch {
inner: BTreeMap::new(),
batch_size: config.batch_size,
column: col,
}
}
@@ -67,10 +72,10 @@ impl Batch {
pub fn commit(&mut self, dest: &mut Database) -> Result<(), Error> {
if self.inner.is_empty() { return Ok(()) }
let transaction = DBTransaction::new();
let transaction = DBTransaction::new(dest);
for keypair in &self.inner {
try!(transaction.put(&keypair.0, &keypair.1).map_err(Error::Custom));
try!(transaction.put(self.column, &keypair.0, &keypair.1).map_err(Error::Custom));
}
self.inner.clear();
@@ -99,14 +104,18 @@ impl From<::std::io::Error> for Error {
/// A generalized migration from the given db to a destination db.
pub trait Migration: 'static {
/// Number of columns in database after the migration.
fn columns(&self) -> Option<u32>;
/// Version of the database after the migration.
fn version(&self) -> u32;
/// Migrate a source to a destination.
fn migrate(&mut self, source: &Database, config: &Config, destination: &mut Database) -> Result<(), Error>;
fn migrate(&mut self, source: &Database, config: &Config, destination: &mut Database, col: Option<u32>) -> Result<(), Error>;
}
/// A simple migration over key-value pairs.
pub trait SimpleMigration: 'static {
/// Number of columns in database after the migration.
fn columns(&self) -> Option<u32>;
/// Version of database after the migration.
fn version(&self) -> u32;
/// Should migrate existing object to new database.
@@ -115,12 +124,14 @@ pub trait SimpleMigration: 'static {
}
impl<T: SimpleMigration> Migration for T {
fn columns(&self) -> Option<u32> { SimpleMigration::columns(self) }
fn version(&self) -> u32 { SimpleMigration::version(self) }
fn migrate(&mut self, source: &Database, config: &Config, dest: &mut Database) -> Result<(), Error> {
let mut batch = Batch::new(config);
fn migrate(&mut self, source: &Database, config: &Config, dest: &mut Database, col: Option<u32>) -> Result<(), Error> {
let mut batch = Batch::new(config, col);
for (key, value) in source.iter() {
for (key, value) in source.iter(col) {
if let Some((key, value)) = self.simple_migrate(key.to_vec(), value.to_vec()) {
try!(batch.insert(key, value, dest));
}
@@ -180,12 +191,11 @@ impl Manager {
/// Adds new migration rules.
pub fn add_migration<T>(&mut self, migration: T) -> Result<(), Error> where T: Migration {
let version_match = match self.migrations.last() {
Some(last) => last.version() + 1 == migration.version(),
let is_new = match self.migrations.last() {
Some(last) => migration.version() > last.version(),
None => true,
};
match version_match {
match is_new {
true => Ok(self.migrations.push(Box::new(migration))),
false => Err(Error::CannotAddMigration),
}
@@ -195,11 +205,15 @@ impl Manager {
/// and producing a path where the final migration lives.
pub fn execute(&mut self, old_path: &Path, version: u32) -> Result<PathBuf, Error> {
let config = self.config.clone();
let migrations = try!(self.migrations_from(version).ok_or(Error::MigrationImpossible));
let db_config = DatabaseConfig {
let columns = self.no_of_columns_at(version);
let migrations = self.migrations_from(version);
if migrations.is_empty() { return Err(Error::MigrationImpossible) };
let mut db_config = DatabaseConfig {
max_open_files: 64,
cache_size: None,
compaction: CompactionProfile::default(),
compaction: config.compaction_profile,
columns: columns,
wal: true,
};
let db_root = database_path(old_path);
@@ -209,14 +223,28 @@ impl Manager {
// start with the old db.
let old_path_str = try!(old_path.to_str().ok_or(Error::MigrationImpossible));
let mut cur_db = try!(Database::open(&db_config, old_path_str).map_err(Error::Custom));
for migration in migrations {
// Change number of columns in new db
let current_columns = db_config.columns;
db_config.columns = migration.columns();
// open the target temporary database.
temp_path = temp_idx.path(&db_root);
let temp_path_str = try!(temp_path.to_str().ok_or(Error::MigrationImpossible));
let mut new_db = try!(Database::open(&db_config, temp_path_str).map_err(Error::Custom));
// perform the migration from cur_db to new_db.
try!(migration.migrate(&cur_db, &config, &mut new_db));
match current_columns {
// migrate only default column
None => try!(migration.migrate(&cur_db, &config, &mut new_db, None)),
Some(v) => {
// Migrate all columns in previous DB
for col in 0..v {
try!(migration.migrate(&cur_db, &config, &mut new_db, Some(col)))
}
}
}
// next iteration, we will migrate from this db into the other temp.
cur_db = new_db;
temp_idx.swap();
@@ -235,10 +263,42 @@ impl Manager {
}
}
fn migrations_from(&mut self, version: u32) -> Option<&mut [Box<Migration>]> {
// index of the first required migration
let position = self.migrations.iter().position(|m| m.version() == version + 1);
position.map(move |p| &mut self.migrations[p..])
/// Find all needed migrations.
fn migrations_from(&mut self, version: u32) -> Vec<&mut Box<Migration>> {
self.migrations.iter_mut().filter(|m| m.version() > version).collect()
}
fn no_of_columns_at(&self, version: u32) -> Option<u32> {
let migration = self.migrations.iter().find(|m| m.version() == version);
match migration {
Some(m) => m.columns(),
None => None
}
}
}
/// Prints a dot every `max` ticks
pub struct Progress {
current: usize,
max: usize,
}
impl Default for Progress {
fn default() -> Self {
Progress {
current: 0,
max: 100_000,
}
}
}
impl Progress {
/// Tick progress meter.
pub fn tick(&mut self) {
self.current += 1;
if self.current == self.max {
self.current = 0;
flush!(".");
}
}
}

60
util/src/migration/tests.rs
View File

@@ -20,7 +20,7 @@
use common::*;
use migration::{Config, SimpleMigration, Manager};
use kvdb::{Database, DBTransaction};
use kvdb::Database;
use devtools::RandomTempPath;
use std::path::PathBuf;
@@ -35,9 +35,9 @@ fn db_path(path: &Path) -> PathBuf {
fn make_db(path: &Path, pairs: BTreeMap<Vec<u8>, Vec<u8>>) {
let db = Database::open_default(path.to_str().unwrap()).expect("failed to open temp database");
{
let transaction = DBTransaction::new();
let transaction = db.transaction();
for (k, v) in pairs {
transaction.put(&k, &v).expect("failed to add pair to transaction");
transaction.put(None, &k, &v).expect("failed to add pair to transaction");
}
db.write(transaction).expect("failed to write db transaction");
@@ -49,7 +49,7 @@ fn verify_migration(path: &Path, pairs: BTreeMap<Vec<u8>, Vec<u8>>) {
let db = Database::open_default(path.to_str().unwrap()).unwrap();
for (k, v) in pairs {
let x = db.get(&k).unwrap().unwrap();
let x = db.get(None, &k).unwrap().unwrap();
assert_eq!(&x[..], &v[..]);
}
@@ -58,9 +58,9 @@ fn verify_migration(path: &Path, pairs: BTreeMap<Vec<u8>, Vec<u8>>) {
struct Migration0;
impl SimpleMigration for Migration0 {
fn version(&self) -> u32 {
1
}
fn columns(&self) -> Option<u32> { None }
fn version(&self) -> u32 { 1 }
fn simple_migrate(&mut self, key: Vec<u8>, value: Vec<u8>) -> Option<(Vec<u8>, Vec<u8>)> {
let mut key = key;
@@ -74,9 +74,9 @@ impl SimpleMigration for Migration0 {
struct Migration1;
impl SimpleMigration for Migration1 {
fn version(&self) -> u32 {
2
}
fn columns(&self) -> Option<u32> { None }
fn version(&self) -> u32 { 2 }
fn simple_migrate(&mut self, key: Vec<u8>, _value: Vec<u8>) -> Option<(Vec<u8>, Vec<u8>)> {
Some((key, vec![]))
@@ -109,6 +109,18 @@ fn no_migration_needed() {
manager.execute(&db_path, 1).unwrap();
}
#[test]
#[should_panic]
fn wrong_adding_order() {
let dir = RandomTempPath::create_dir();
let db_path = db_path(dir.as_path());
let mut manager = Manager::new(Config::default());
make_db(&db_path, map![vec![] => vec![], vec![1] => vec![1]]);
manager.add_migration(Migration1).unwrap();
manager.add_migration(Migration0).unwrap();
}
#[test]
fn multiple_migrations() {
let dir = RandomTempPath::create_dir();
@@ -139,6 +151,34 @@ fn second_migration() {
verify_migration(&end_path, expected);
}
#[test]
fn first_and_noop_migration() {
let dir = RandomTempPath::create_dir();
let db_path = db_path(dir.as_path());
let mut manager = Manager::new(Config::default());
make_db(&db_path, map![vec![] => vec![], vec![1] => vec![1]]);
let expected = map![vec![0x11] => vec![0x22], vec![1, 0x11] => vec![1, 0x22]];
manager.add_migration(Migration0).unwrap();
let end_path = manager.execute(&db_path, 0).unwrap();
verify_migration(&end_path, expected);
}
#[test]
fn noop_and_second_migration() {
let dir = RandomTempPath::create_dir();
let db_path = db_path(dir.as_path());
let mut manager = Manager::new(Config::default());
make_db(&db_path, map![vec![] => vec![], vec![1] => vec![1]]);
let expected = map![vec![] => vec![], vec![1] => vec![]];
manager.add_migration(Migration1).unwrap();
let end_path = manager.execute(&db_path, 0).unwrap();
verify_migration(&end_path, expected);
}
#[test]
fn is_migration_needed() {
let mut manager = Manager::new(Config::default());

4
util/src/network/connection.rs
View File

@@ -77,7 +77,7 @@ impl<Socket: GenericSocket> GenericConnection<Socket> {
/// Readable IO handler. Called when there is some data to be read.
pub fn readable(&mut self) -> io::Result<Option<Bytes>> {
if self.rec_size == 0 || self.rec_buf.len() >= self.rec_size {
warn!(target:"network", "Unexpected connection read");
return Ok(None);
}
let sock_ref = <Socket as Read>::by_ref(&mut self.socket);
loop {
@@ -355,7 +355,7 @@ impl EncryptedConnection {
self.encoder.encrypt(&mut RefReadBuffer::new(&header), &mut RefWriteBuffer::new(&mut packet), false).expect("Invalid length or padding");
EncryptedConnection::update_mac(&mut self.egress_mac, &mut self.mac_encoder, &packet[0..16]);
self.egress_mac.clone().finalize(&mut packet[16..32]);
self.encoder.encrypt(&mut RefReadBuffer::new(&payload), &mut RefWriteBuffer::new(&mut packet[32..(32 + len)]), padding == 0).expect("Invalid length or padding");
self.encoder.encrypt(&mut RefReadBuffer::new(payload), &mut RefWriteBuffer::new(&mut packet[32..(32 + len)]), padding == 0).expect("Invalid length or padding");
if padding != 0 {
let pad = [0u8; 16];
self.encoder.encrypt(&mut RefReadBuffer::new(&pad[0..padding]), &mut RefWriteBuffer::new(&mut packet[(32 + len)..(32 + len + padding)]), true).expect("Invalid length or padding");

4
util/src/network/discovery.rs
View File

@@ -167,7 +167,7 @@ impl Discovery {
}
fn clear_ping(&mut self, id: &NodeId) {
let mut bucket = self.node_buckets.get_mut(Discovery::distance(&self.id, &id) as usize).unwrap();
let mut bucket = self.node_buckets.get_mut(Discovery::distance(&self.id, id) as usize).unwrap();
if let Some(node) = bucket.nodes.iter_mut().find(|n| &n.address.id == id) {
node.timeout = None;
}
@@ -438,7 +438,7 @@ impl Discovery {
}
let mut packets = Discovery::prepare_neighbours_packets(&nearest);
for p in packets.drain(..) {
self.send_packet(PACKET_NEIGHBOURS, &from, &p);
self.send_packet(PACKET_NEIGHBOURS, from, &p);
}
trace!(target: "discovery", "Sent {} Neighbours to {:?}", nearest.len(), &from);
Ok(None)

58
util/src/network/handshake.rs
View File

@@ -128,31 +128,27 @@ impl Handshake {
/// Readable IO handler. Drives the state change.
pub fn readable<Message>(&mut self, io: &IoContext<Message>, host: &HostInfo) -> Result<(), UtilError> where Message: Send + Clone {
if !self.expired() {
match self.state {
HandshakeState::New => {}
HandshakeState::ReadingAuth => {
if let Some(data) = try!(self.connection.readable()) {
while let Some(data) = try!(self.connection.readable()) {
match self.state {
HandshakeState::New => {},
HandshakeState::StartSession => {},
HandshakeState::ReadingAuth => {
try!(self.read_auth(io, host.secret(), &data));
};
},
HandshakeState::ReadingAuthEip8 => {
if let Some(data) = try!(self.connection.readable()) {
},
HandshakeState::ReadingAuthEip8 => {
try!(self.read_auth_eip8(io, host.secret(), &data));
};
},
HandshakeState::ReadingAck => {
if let Some(data) = try!(self.connection.readable()) {
},
HandshakeState::ReadingAck => {
try!(self.read_ack(host.secret(), &data));
};
},
HandshakeState::ReadingAckEip8 => {
if let Some(data) = try!(self.connection.readable()) {
},
HandshakeState::ReadingAckEip8 => {
try!(self.read_ack_eip8(host.secret(), &data));
};
},
HandshakeState::StartSession => {
},
}
if self.state == HandshakeState::StartSession {
io.clear_timer(self.connection.token).ok();
},
break;
}
}
}
Ok(())
@@ -178,9 +174,9 @@ impl Handshake {
/// Parse, validate and confirm auth message
fn read_auth<Message>(&mut self, io: &IoContext<Message>, secret: &Secret, data: &[u8]) -> Result<(), UtilError> where Message: Send + Clone {
trace!(target:"network", "Received handshake auth from {:?}", self.connection.remote_addr_str());
trace!(target: "network", "Received handshake auth from {:?}", self.connection.remote_addr_str());
if data.len() != V4_AUTH_PACKET_SIZE {
debug!(target:"net", "Wrong auth packet size");
debug!(target: "network", "Wrong auth packet size");
return Err(From::from(NetworkError::BadProtocol));
}
self.auth_cipher = data.to_vec();
@@ -197,7 +193,7 @@ impl Handshake {
// Try to interpret as EIP-8 packet
let total = (((data[0] as u16) << 8 | (data[1] as u16)) as usize) + 2;
if total < V4_AUTH_PACKET_SIZE {
debug!(target:"net", "Wrong EIP8 auth packet size");
debug!(target: "network", "Wrong EIP8 auth packet size");
return Err(From::from(NetworkError::BadProtocol));
}
let rest = total - data.len();
@@ -209,7 +205,7 @@ impl Handshake {
}
fn read_auth_eip8<Message>(&mut self, io: &IoContext<Message>, secret: &Secret, data: &[u8]) -> Result<(), UtilError> where Message: Send + Clone {
trace!(target:"network", "Received EIP8 handshake auth from {:?}", self.connection.remote_addr_str());
trace!(target: "network", "Received EIP8 handshake auth from {:?}", self.connection.remote_addr_str());
self.auth_cipher.extend_from_slice(data);
let auth = try!(ecies::decrypt(secret, &self.auth_cipher[0..2], &self.auth_cipher[2..]));
let rlp = UntrustedRlp::new(&auth);
@@ -224,9 +220,9 @@ impl Handshake {
/// Parse and validate ack message
fn read_ack(&mut self, secret: &Secret, data: &[u8]) -> Result<(), UtilError> {
trace!(target:"network", "Received handshake ack from {:?}", self.connection.remote_addr_str());
trace!(target: "network", "Received handshake ack from {:?}", self.connection.remote_addr_str());
if data.len() != V4_ACK_PACKET_SIZE {
debug!(target:"net", "Wrong ack packet size");
debug!(target: "network", "Wrong ack packet size");
return Err(From::from(NetworkError::BadProtocol));
}
self.ack_cipher = data.to_vec();
@@ -240,7 +236,7 @@ impl Handshake {
// Try to interpret as EIP-8 packet
let total = (((data[0] as u16) << 8 | (data[1] as u16)) as usize) + 2;
if total < V4_ACK_PACKET_SIZE {
debug!(target:"net", "Wrong EIP8 ack packet size");
debug!(target: "network", "Wrong EIP8 ack packet size");
return Err(From::from(NetworkError::BadProtocol));
}
let rest = total - data.len();
@@ -252,7 +248,7 @@ impl Handshake {
}
fn read_ack_eip8(&mut self, secret: &Secret, data: &[u8]) -> Result<(), UtilError> {
trace!(target:"network", "Received EIP8 handshake auth from {:?}", self.connection.remote_addr_str());
trace!(target: "network", "Received EIP8 handshake auth from {:?}", self.connection.remote_addr_str());
self.ack_cipher.extend_from_slice(data);
let ack = try!(ecies::decrypt(secret, &self.ack_cipher[0..2], &self.ack_cipher[2..]));
let rlp = UntrustedRlp::new(&ack);
@@ -265,7 +261,7 @@ impl Handshake {
/// Sends auth message
fn write_auth<Message>(&mut self, io: &IoContext<Message>, secret: &Secret, public: &Public) -> Result<(), UtilError> where Message: Send + Clone {
trace!(target:"network", "Sending handshake auth to {:?}", self.connection.remote_addr_str());
trace!(target: "network", "Sending handshake auth to {:?}", self.connection.remote_addr_str());
let mut data = [0u8; /*Signature::SIZE*/ 65 + /*H256::SIZE*/ 32 + /*Public::SIZE*/ 64 + /*H256::SIZE*/ 32 + 1]; //TODO: use associated constants
let len = data.len();
{
@@ -292,7 +288,7 @@ impl Handshake {
/// Sends ack message
fn write_ack<Message>(&mut self, io: &IoContext<Message>) -> Result<(), UtilError> where Message: Send + Clone {
trace!(target:"network", "Sending handshake ack to {:?}", self.connection.remote_addr_str());
trace!(target: "network", "Sending handshake ack to {:?}", self.connection.remote_addr_str());
let mut data = [0u8; 1 + /*Public::SIZE*/ 64 + /*H256::SIZE*/ 32]; //TODO: use associated constants
let len = data.len();
{
@@ -311,7 +307,7 @@ impl Handshake {
/// Sends EIP8 ack message
fn write_ack_eip8<Message>(&mut self, io: &IoContext<Message>) -> Result<(), UtilError> where Message: Send + Clone {
trace!(target:"network", "Sending EIP8 handshake ack to {:?}", self.connection.remote_addr_str());
trace!(target: "network", "Sending EIP8 handshake ack to {:?}", self.connection.remote_addr_str());
let mut rlp = RlpStream::new_list(3);
rlp.append(self.ecdhe.public());
rlp.append(&self.nonce);

29
util/src/network/host.rs
View File

@@ -51,7 +51,7 @@ const MAX_HANDSHAKES: usize = 80;
const MAX_HANDSHAKES_PER_ROUND: usize = 32;
const MAINTENANCE_TIMEOUT: u64 = 1000;
#[derive(Debug, Clone)]
#[derive(Debug, PartialEq, Clone)]
/// Network service configuration
pub struct NetworkConfiguration {
/// Directory path to store network configuration. None means nothing will be saved
@@ -70,8 +70,10 @@ pub struct NetworkConfiguration {
pub boot_nodes: Vec<String>,
/// Use provided node key instead of default
pub use_secret: Option<Secret>,
/// Number of connected peers to maintain
pub ideal_peers: u32,
/// Minimum number of connected peers to maintain
pub min_peers: u32,
/// Maximum allowd number of peers
pub max_peers: u32,
/// List of reserved node addresses.
pub reserved_nodes: Vec<String>,
/// The non-reserved peer mode.
@@ -96,7 +98,8 @@ impl NetworkConfiguration {
discovery_enabled: true,
boot_nodes: Vec::new(),
use_secret: None,
ideal_peers: 25,
min_peers: 25,
max_peers: 50,
reserved_nodes: Vec::new(),
non_reserved_mode: NonReservedPeerMode::Accept,
}
@@ -355,11 +358,11 @@ impl Host {
let keys = if let Some(ref secret) = config.use_secret {
KeyPair::from_secret(secret.clone()).unwrap()
} else {
config.config_path.clone().and_then(|ref p| load_key(&Path::new(&p)))
config.config_path.clone().and_then(|ref p| load_key(Path::new(&p)))
.map_or_else(|| {
let key = KeyPair::create().unwrap();
if let Some(path) = config.config_path.clone() {
save_key(&Path::new(&path), &key.secret());
save_key(Path::new(&path), key.secret());
}
key
},
@@ -597,19 +600,19 @@ impl Host {
}
fn connect_peers(&self, io: &IoContext<NetworkIoMessage>) {
let (ideal_peers, mut pin) = {
let (min_peers, mut pin) = {
let info = self.info.read();
if info.capabilities.is_empty() {
return;
}
let config = &info.config;
(config.ideal_peers, config.non_reserved_mode == NonReservedPeerMode::Deny)
(config.min_peers, config.non_reserved_mode == NonReservedPeerMode::Deny)
};
let session_count = self.session_count();
let reserved_nodes = self.reserved_nodes.read();
if session_count >= ideal_peers as usize + reserved_nodes.len() {
if session_count >= min_peers as usize + reserved_nodes.len() {
// check if all pinned nodes are connected.
if reserved_nodes.iter().all(|n| self.have_session(n) && self.connecting_to(n)) {
return;
@@ -767,12 +770,12 @@ impl Host {
self.num_sessions.fetch_add(1, AtomicOrdering::SeqCst);
if !s.info.originated {
let session_count = self.session_count();
let (ideal_peers, reserved_only) = {
let (max_peers, reserved_only) = {
let info = self.info.read();
(info.config.ideal_peers, info.config.non_reserved_mode == NonReservedPeerMode::Deny)
(info.config.max_peers, info.config.non_reserved_mode == NonReservedPeerMode::Deny)
};
if session_count >= ideal_peers as usize || reserved_only {
if session_count >= max_peers as usize || reserved_only {
// only proceed if the connecting peer is reserved.
if !self.reserved_nodes.read().contains(s.id().unwrap()) {
s.disconnect(io, DisconnectReason::TooManyPeers);
@@ -1099,7 +1102,7 @@ fn save_key(path: &Path, key: &Secret) {
return;
}
};
if let Err(e) = restrict_permissions_owner(&path) {
if let Err(e) = restrict_permissions_owner(path) {
warn!(target: "network", "Failed to modify permissions of the file (chmod: {})", e);
}
if let Err(e) = file.write(&key.hex().into_bytes()) {

10
util/src/network/ip_utils.rs
View File

@@ -16,17 +16,12 @@
// Based on original work by David Levy https://raw.githubusercontent.com/dlevy47/rust-interfaces
use std::net::{Ipv4Addr, Ipv6Addr, SocketAddr, SocketAddrV4, SocketAddrV6};
use std::net::{IpAddr, Ipv4Addr, Ipv6Addr, SocketAddr, SocketAddrV4, SocketAddrV6};
use std::io;
use igd::{PortMappingProtocol, search_gateway_from_timeout};
use std::time::Duration;
use network::node_table::{NodeEndpoint};
pub enum IpAddr{
V4(Ipv4Addr),
V6(Ipv6Addr),
}
/// Socket address extension for rustc beta. To be replaces with now unstable API
pub trait SocketAddrExt {
/// Returns true for the special 'unspecified' address 0.0.0.0.
@@ -66,8 +61,7 @@ mod getinterfaces {
use std::{mem, io, ptr};
use libc::{AF_INET, AF_INET6};
use libc::{getifaddrs, freeifaddrs, ifaddrs, sockaddr, sockaddr_in, sockaddr_in6};
use std::net::{Ipv4Addr, Ipv6Addr};
use super::IpAddr;
use std::net::{Ipv4Addr, Ipv6Addr, IpAddr};
fn convert_sockaddr (sa: *mut sockaddr) -> Option<IpAddr> {
if sa == ptr::null_mut() { return None; }

6
util/src/network/session.rs
View File

@@ -128,7 +128,7 @@ impl Session {
nonce: &H256, stats: Arc<NetworkStats>, host: &HostInfo) -> Result<Session, UtilError>
where Message: Send + Clone {
let originated = id.is_some();
let mut handshake = Handshake::new(token, id, socket, &nonce, stats).expect("Can't create handshake");
let mut handshake = Handshake::new(token, id, socket, nonce, stats).expect("Can't create handshake");
try!(handshake.start(io, host, originated));
Ok(Session {
state: State::Handshake(handshake),
@@ -313,7 +313,7 @@ impl Session {
self.connection().token()
}
fn read_packet<Message>(&mut self, io: &IoContext<Message>, packet: Packet, host: &HostInfo) -> Result<SessionData, UtilError>
fn read_packet<Message>(&mut self, io: &IoContext<Message>, packet: Packet, host: &HostInfo) -> Result<SessionData, UtilError>
where Message: Send + Sync + Clone {
if packet.data.len() < 2 {
return Err(From::from(NetworkError::BadProtocol));
@@ -381,7 +381,7 @@ impl Session {
self.send(io, rlp)
}
fn read_hello<Message>(&mut self, io: &IoContext<Message>, rlp: &UntrustedRlp, host: &HostInfo) -> Result<(), UtilError>
fn read_hello<Message>(&mut self, io: &IoContext<Message>, rlp: &UntrustedRlp, host: &HostInfo) -> Result<(), UtilError>
where Message: Send + Sync + Clone {
let protocol = try!(rlp.val_at::<u32>(0));
let client_version = try!(rlp.val_at::<String>(1));

20
util/src/network_settings.rs
View File

@@ -16,13 +16,15 @@
//! Structure to hold network settings configured from CLI
/// Networking & RPC settings
#[derive(Debug, PartialEq)]
#[derive(Debug, PartialEq, Clone)]
pub struct NetworkSettings {
/// Node name
pub name: String,
/// Name of the chain we are connected to
pub chain: String,
/// Ideal number of peers
/// Min number of peers
pub min_peers: u32,
/// Max number of peers
pub max_peers: u32,
/// Networking port
pub network_port: u16,
@@ -34,3 +36,17 @@ pub struct NetworkSettings {
pub rpc_port: u16,
}
impl Default for NetworkSettings {
fn default() -> Self {
NetworkSettings {
name: "".into(),
chain: "homestead".into(),
min_peers: 25,
max_peers: 50,
network_port: 30303,
rpc_enabled: true,
rpc_interface: "local".into(),
rpc_port: 8545
}
}
}

149
util/src/overlaydb.rs
View File

@@ -24,7 +24,6 @@ use hashdb::*;
use memorydb::*;
use std::ops::*;
use std::sync::*;
use std::env;
use std::collections::HashMap;
use kvdb::{Database, DBTransaction};
@@ -40,22 +39,29 @@ use kvdb::{Database, DBTransaction};
pub struct OverlayDB {
overlay: MemoryDB,
backing: Arc<Database>,
column: Option<u32>,
}
impl OverlayDB {
/// Create a new instance of OverlayDB given a `backing` database.
pub fn new(backing: Database) -> OverlayDB { Self::new_with_arc(Arc::new(backing)) }
/// Create a new instance of OverlayDB given a `backing` database.
pub fn new_with_arc(backing: Arc<Database>) -> OverlayDB {
OverlayDB{ overlay: MemoryDB::new(), backing: backing }
pub fn new(backing: Arc<Database>, col: Option<u32>) -> OverlayDB {
OverlayDB{ overlay: MemoryDB::new(), backing: backing, column: col }
}
/// Create a new instance of OverlayDB with an anonymous temporary database.
#[cfg(test)]
pub fn new_temp() -> OverlayDB {
let mut dir = env::temp_dir();
let mut dir = ::std::env::temp_dir();
dir.push(H32::random().hex());
Self::new(Database::open_default(dir.to_str().unwrap()).unwrap())
Self::new(Arc::new(Database::open_default(dir.to_str().unwrap()).unwrap()), None)
}
/// Commit all operations in a single batch.
#[cfg(test)]
pub fn commit(&mut self) -> Result<u32, UtilError> {
let batch = self.backing.transaction();
let res = try!(self.commit_to_batch(&batch));
self.backing.write(batch).map(|_| res).map_err(|e| e.into())
}
/// Commit all operations to given batch.
@@ -88,91 +94,16 @@ impl OverlayDB {
Ok(ret)
}
/// Commit all memory operations to the backing database.
///
/// Returns either an error or the number of items changed in the backing database.
///
/// Will return an error if the number of `remove()`s ever exceeds the number of
/// `insert()`s for any key. This will leave the database in an undeterminate
/// state. Don't ever let it happen.
///
/// # Example
/// ```
/// extern crate ethcore_util;
/// use ethcore_util::hashdb::*;
/// use ethcore_util::overlaydb::*;
/// fn main() {
/// let mut m = OverlayDB::new_temp();
/// let key = m.insert(b"foo"); // insert item.
/// assert!(m.contains(&key)); // key exists (in memory).
/// assert_eq!(m.commit().unwrap(), 1); // 1 item changed.
/// assert!(m.contains(&key)); // key still exists (in backing).
/// m.remove(&key); // delete item.
/// assert!(!m.contains(&key)); // key "doesn't exist" (though still does in backing).
/// m.remove(&key); // oh dear... more removes than inserts for the key...
/// //m.commit().unwrap(); // this commit/unwrap would cause a panic.
/// m.revert(); // revert both removes.
/// assert!(m.contains(&key)); // key now still exists.
/// }
/// ```
pub fn commit(&mut self) -> Result<u32, UtilError> {
let mut ret = 0u32;
let mut deletes = 0usize;
for i in self.overlay.drain().into_iter() {
let (key, (value, rc)) = i;
if rc != 0 {
match self.payload(&key) {
Some(x) => {
let (back_value, back_rc) = x;
let total_rc: i32 = back_rc as i32 + rc;
if total_rc < 0 {
return Err(From::from(BaseDataError::NegativelyReferencedHash(key)));
}
deletes += if self.put_payload(&key, (back_value, total_rc as u32)) {1} else {0};
}
None => {
if rc < 0 {
return Err(From::from(BaseDataError::NegativelyReferencedHash(key)));
}
self.put_payload(&key, (value, rc as u32));
}
};
ret += 1;
}
}
trace!("OverlayDB::commit() deleted {} nodes", deletes);
Ok(ret)
}
/// Revert all operations on this object (i.e. `insert()`s and `remove()`s) since the
/// last `commit()`.
///
/// # Example
/// ```
/// extern crate ethcore_util;
/// use ethcore_util::hashdb::*;
/// use ethcore_util::overlaydb::*;
/// fn main() {
/// let mut m = OverlayDB::new_temp();
/// let foo = m.insert(b"foo"); // insert foo.
/// m.commit().unwrap(); // commit - new operations begin here...
/// let bar = m.insert(b"bar"); // insert bar.
/// m.remove(&foo); // remove foo.
/// assert!(!m.contains(&foo)); // foo is gone.
/// assert!(m.contains(&bar)); // bar is here.
/// m.revert(); // revert the last two operations.
/// assert!(m.contains(&foo)); // foo is here.
/// assert!(!m.contains(&bar)); // bar is gone.
/// }
/// ```
pub fn revert(&mut self) { self.overlay.clear(); }
/// Get the number of references that would be committed.
pub fn commit_refs(&self, key: &H256) -> i32 { self.overlay.raw(&key).map_or(0, |&(_, refs)| refs) }
pub fn commit_refs(&self, key: &H256) -> i32 { self.overlay.raw(key).map_or(0, |&(_, refs)| refs) }
/// Get the refs and value of the given key.
fn payload(&self, key: &H256) -> Option<(Bytes, u32)> {
self.backing.get(key)
self.backing.get(self.column, key)
.expect("Low-level database error. Some issue with your hard disk?")
.map(|d| {
let r = Rlp::new(&d);
@@ -186,24 +117,10 @@ impl OverlayDB {
let mut s = RlpStream::new_list(2);
s.append(&payload.1);
s.append(&payload.0);
batch.put(key, s.as_raw()).expect("Low-level database error. Some issue with your hard disk?");
batch.put(self.column, key, s.as_raw()).expect("Low-level database error. Some issue with your hard disk?");
false
} else {
batch.delete(key).expect("Low-level database error. Some issue with your hard disk?");
true
}
}
/// Put the refs and value of the given key, possibly deleting it from the db.
fn put_payload(&self, key: &H256, payload: (Bytes, u32)) -> bool {
if payload.1 > 0 {
let mut s = RlpStream::new_list(2);
s.append(&payload.1);
s.append(&payload.0);
self.backing.put(key, s.as_raw()).expect("Low-level database error. Some issue with your hard disk?");
false
} else {
self.backing.delete(key).expect("Low-level database error. Some issue with your hard disk?");
batch.delete(self.column, key).expect("Low-level database error. Some issue with your hard disk?");
true
}
}
@@ -212,7 +129,7 @@ impl OverlayDB {
impl HashDB for OverlayDB {
fn keys(&self) -> HashMap<H256, i32> {
let mut ret: HashMap<H256, i32> = HashMap::new();
for (key, _) in self.backing.iter() {
for (key, _) in self.backing.iter(self.column) {
let h = H256::from_slice(key.deref());
let r = self.payload(&h).unwrap().1;
ret.insert(h, r as i32);
@@ -274,6 +191,22 @@ impl HashDB for OverlayDB {
fn remove(&mut self, key: &H256) { self.overlay.remove(key); }
}
#[test]
fn overlaydb_revert() {
let mut m = OverlayDB::new_temp();
let foo = m.insert(b"foo"); // insert foo.
let batch = m.backing.transaction();
m.commit_to_batch(&batch).unwrap(); // commit - new operations begin here...
m.backing.write(batch).unwrap();
let bar = m.insert(b"bar"); // insert bar.
m.remove(&foo); // remove foo.
assert!(!m.contains(&foo)); // foo is gone.
assert!(m.contains(&bar)); // bar is here.
m.revert(); // revert the last two operations.
assert!(m.contains(&foo)); // foo is here.
assert!(!m.contains(&bar)); // bar is gone.
}
#[test]
fn overlaydb_overlay_insert_and_remove() {
let mut trie = OverlayDB::new_temp();
@@ -366,14 +299,18 @@ fn overlaydb_complex() {
fn playpen() {
use std::fs;
{
let db: Database = Database::open_default("/tmp/test").unwrap();
db.put(b"test", b"test2").unwrap();
match db.get(b"test") {
let db = Database::open_default("/tmp/test").unwrap();
let batch = db.transaction();
batch.put(None, b"test", b"test2").unwrap();
db.write(batch).unwrap();
match db.get(None, b"test") {
Ok(Some(value)) => println!("Got value {:?}", value.deref()),
Ok(None) => println!("No value for that key"),
Err(..) => println!("Gah"),
}
db.delete(b"test").unwrap();
let batch = db.transaction();
batch.delete(None, b"test").unwrap();
db.write(batch).unwrap();
}
fs::remove_dir_all("/tmp/test").unwrap();
}

106
util/src/rlp/commonrlps.rs Normal file
View File

@@ -0,0 +1,106 @@
// Copyright 2015, 2016 Ethcore (UK) Ltd.
// This file is part of Parity.
// Parity is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// Parity is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
// You should have received a copy of the GNU General Public License
// along with Parity. If not, see <http://www.gnu.org/licenses/>.
//! Contains RLPs used for compression.
use rlp::rlpcompression::InvalidRlpSwapper;
lazy_static! {
/// Swapper for snapshot compression.
pub static ref SNAPSHOT_RLP_SWAPPER: InvalidRlpSwapper<'static> = InvalidRlpSwapper::new(EMPTY_RLPS, INVALID_RLPS);
}
lazy_static! {
/// Swapper with common long RLPs, up to 127 can be added.
pub static ref BLOCKS_RLP_SWAPPER: InvalidRlpSwapper<'static> = InvalidRlpSwapper::new(COMMON_RLPS, INVALID_RLPS);
}
static EMPTY_RLPS: &'static [&'static [u8]] = &[
// RLP of SHA3_NULL_RLP
&[160, 86, 232, 31, 23, 27, 204, 85, 166, 255, 131, 69, 230, 146, 192, 248, 110, 91, 72, 224, 27, 153, 108, 173, 192, 1, 98, 47, 181, 227, 99, 180, 33],
// RLP of SHA3_EMPTY
&[160, 197, 210, 70, 1, 134, 247, 35, 60, 146, 126, 125, 178, 220, 199, 3, 192, 229, 0, 182, 83, 202, 130, 39, 59, 123, 250, 216, 4, 93, 133, 164, 112]
];
static COMMON_RLPS: &'static [&'static [u8]] = &[
// RLP of SHA3_NULL_RLP
&[160, 86, 232, 31, 23, 27, 204, 85, 166, 255, 131, 69, 230, 146, 192, 248, 110, 91, 72, 224, 27, 153, 108, 173, 192, 1, 98, 47, 181, 227, 99, 180, 33],
// RLP of SHA3_EMPTY
&[160, 197, 210, 70, 1, 134, 247, 35, 60, 146, 126, 125, 178, 220, 199, 3, 192, 229, 0, 182, 83, 202, 130, 39, 59, 123, 250, 216, 4, 93, 133, 164, 112],
// Other RLPs found in blocks DB using the test below.
&[160, 29, 204, 77, 232, 222, 199, 93, 122, 171, 133, 181, 103, 182, 204, 212, 26, 211, 18, 69, 27, 148, 138, 116, 19, 240, 161, 66, 253, 64, 212, 147, 71],
&[148, 50, 190, 52, 59, 148, 248, 96, 18, 77, 196, 254, 226, 120, 253, 203, 211, 140, 16, 45, 136],
&[148, 82, 188, 68, 213, 55, 131, 9, 238, 42, 191, 21, 57, 191, 113, 222, 27, 125, 123, 227, 181],
&[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
];
static INVALID_RLPS: &'static [&'static [u8]] = &[&[0x81, 0x0], &[0x81, 0x1], &[0x81, 0x2], &[0x81, 0x3], &[0x81, 0x4], &[0x81, 0x5], &[0x81, 0x6], &[0x81, 0x7], &[0x81, 0x8], &[0x81, 0x9], &[0x81, 0xa], &[0x81, 0xb], &[0x81, 0xc], &[0x81, 0xd], &[0x81, 0xe], &[0x81, 0xf], &[0x81, 0x10], &[0x81, 0x11], &[0x81, 0x12], &[0x81, 0x13], &[0x81, 0x14], &[0x81, 0x15], &[0x81, 0x16], &[0x81, 0x17], &[0x81, 0x18], &[0x81, 0x19], &[0x81, 0x1a], &[0x81, 0x1b], &[0x81, 0x1c], &[0x81, 0x1d], &[0x81, 0x1e], &[0x81, 0x1f], &[0x81, 0x20], &[0x81, 0x21], &[0x81, 0x22], &[0x81, 0x23], &[0x81, 0x24], &[0x81, 0x25], &[0x81, 0x26], &[0x81, 0x27], &[0x81, 0x28], &[0x81, 0x29], &[0x81, 0x2a], &[0x81, 0x2b], &[0x81, 0x2c], &[0x81, 0x2d], &[0x81, 0x2e], &[0x81, 0x2f], &[0x81, 0x30], &[0x81, 0x31], &[0x81, 0x32], &[0x81, 0x33], &[0x81, 0x34], &[0x81, 0x35], &[0x81, 0x36], &[0x81, 0x37], &[0x81, 0x38], &[0x81, 0x39], &[0x81, 0x3a], &[0x81, 0x3b], &[0x81, 0x3c], &[0x81, 0x3d], &[0x81, 0x3e], &[0x81, 0x3f], &[0x81, 0x40], &[0x81, 0x41], &[0x81, 0x42], &[0x81, 0x43], &[0x81, 0x44], &[0x81, 0x45], &[0x81, 0x46], &[0x81, 0x47], &[0x81, 0x48], &[0x81, 0x49], &[0x81, 0x4a], &[0x81, 0x4b], &[0x81, 0x4c], &[0x81, 0x4d], &[0x81, 0x4e], &[0x81, 0x4f], &[0x81, 0x50], &[0x81, 0x51], &[0x81, 0x52], &[0x81, 0x53], &[0x81, 0x54], &[0x81, 0x55], &[0x81, 0x56], &[0x81, 0x57], &[0x81, 0x58], &[0x81, 0x59], &[0x81, 0x5a], &[0x81, 0x5b], &[0x81, 0x5c], &[0x81, 0x5d], &[0x81, 0x5e], &[0x81, 0x5f], &[0x81, 0x60], &[0x81, 0x61], &[0x81, 0x62], &[0x81, 0x63], &[0x81, 0x64], &[0x81, 0x65], &[0x81, 0x66], &[0x81, 0x67], &[0x81, 0x68], &[0x81, 0x69], &[0x81, 0x6a], &[0x81, 0x6b], &[0x81, 0x6c], &[0x81, 0x6d], &[0x81, 0x6e], &[0x81, 0x6f], &[0x81, 0x70], &[0x81, 0x71], &[0x81, 0x72], &[0x81, 0x73], &[0x81, 0x74], &[0x81, 0x75], &[0x81, 0x76], &[0x81, 0x77], &[0x81, 0x78], &[0x81, 0x79], &[0x81, 0x7a], &[0x81, 0x7b], &[0x81, 0x7c], &[0x81, 0x7d], &[0x81, 0x7e]];
#[cfg(test)]
mod tests {
#[test]
#[ignore]
fn analyze_db() {
use rlp::{UntrustedRlp, View};
use std::collections::HashMap;
use kvdb::*;
let path = "db path".to_string();
let values: Vec<_> = Database::open_default(&path).unwrap().iter(Some(2)).map(|(_, v)| v).collect();
let mut rlp_counts: HashMap<_, u32> = HashMap::new();
let mut rlp_sizes: HashMap<_, u32> = HashMap::new();
fn flat_rlp<'a>(acc: &mut Vec<UntrustedRlp<'a>>, rlp: UntrustedRlp<'a>) {
match rlp.is_data() {
true => if rlp.size()>=70 {
match rlp.data() {
Ok(x) => flat_rlp(acc, UntrustedRlp::new(x)),
_ => acc.push(rlp),
}
} else {
acc.push(rlp);
},
false => for r in rlp.iter() { flat_rlp(acc, r); },
}
}
fn space_saving(bytes: &[u8]) -> u32 {
let l = bytes.len() as u32;
match l >= 2 {
true => l-2,
false => 0,
}
}
for v in values.iter() {
let rlp = UntrustedRlp::new(&v);
let mut flat = Vec::new();
flat_rlp(&mut flat, rlp);
for r in flat.iter() {
*rlp_counts.entry(r.as_raw()).or_insert(0) += 1;
*rlp_sizes.entry(r.as_raw()).or_insert(0) += space_saving(r.as_raw());
}
}
let mut size_vec: Vec<_> = rlp_sizes.iter().collect();
size_vec.sort_by(|a, b| b.1.cmp(a.1));
// Exclude rare large RLPs.
for v in size_vec.iter().filter(|v| rlp_counts.get(v.0).unwrap()>&100).take(20) {
println!("{:?}, {:?}", v, rlp_counts.get(v.0).unwrap());
}
println!("DONE");
}
}

7
util/src/rlp/mod.rs
View File

@@ -51,16 +51,19 @@ mod rlperrors;
mod rlpin;
mod untrusted_rlp;
mod rlpstream;
mod rlpcompression;
mod commonrlps;
mod bytes;
#[cfg(test)]
mod tests;
pub use self::rlperrors::DecoderError;
pub use self::rlptraits::{Decoder, Decodable, View, Stream, Encodable, Encoder, RlpEncodable, RlpDecodable};
pub use self::rlptraits::{Decoder, Decodable, View, Stream, Encodable, Encoder, RlpEncodable, RlpDecodable, Compressible};
pub use self::untrusted_rlp::{UntrustedRlp, UntrustedRlpIterator, PayloadInfo, Prototype};
pub use self::rlpin::{Rlp, RlpIterator};
pub use self::rlpstream::{RlpStream};
pub use self::rlpstream::RlpStream;
pub use self::rlpcompression::RlpType;
pub use elastic_array::ElasticArray1024;
use super::hash::H256;

245
util/src/rlp/rlpcompression.rs Normal file
View File

@@ -0,0 +1,245 @@
// Copyright 2015, 2016 Ethcore (UK) Ltd.
// This file is part of Parity.
// Parity is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// Parity is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
// You should have received a copy of the GNU General Public License
// along with Parity. If not, see <http://www.gnu.org/licenses/>.
use rlp::{UntrustedRlp, View, Compressible, encode, ElasticArray1024, Stream, RlpStream};
use rlp::commonrlps::{BLOCKS_RLP_SWAPPER, SNAPSHOT_RLP_SWAPPER};
use std::collections::HashMap;
/// Stores RLPs used for compression
pub struct InvalidRlpSwapper<'a> {
invalid_to_valid: HashMap<&'a [u8], &'a [u8]>,
valid_to_invalid: HashMap<&'a [u8], &'a [u8]>,
}
impl<'a> InvalidRlpSwapper<'a> {
/// Construct a swapper from a list of common RLPs
pub fn new(rlps_to_swap: &[&'a [u8]], invalid_rlps: &[&'a [u8]]) -> Self {
if rlps_to_swap.len() > 0x7e {
panic!("Invalid usage, only 127 RLPs can be swappable.");
}
let mut invalid_to_valid = HashMap::new();
let mut valid_to_invalid = HashMap::new();
for (&rlp, &invalid) in rlps_to_swap.iter().zip(invalid_rlps.iter()) {
invalid_to_valid.insert(invalid, rlp);
valid_to_invalid.insert(rlp, invalid);
}
InvalidRlpSwapper {
invalid_to_valid: invalid_to_valid,
valid_to_invalid: valid_to_invalid
}
}
/// Get a valid RLP corresponding to an invalid one
fn get_valid(&self, invalid_rlp: &[u8]) -> Option<&[u8]> {
self.invalid_to_valid.get(invalid_rlp).map(|r| r.clone())
}
/// Get an invalid RLP corresponding to a valid one
fn get_invalid(&self, valid_rlp: &[u8]) -> Option<&[u8]> {
self.valid_to_invalid.get(valid_rlp).map(|r| r.clone())
}
}
/// Type of RLP indicating its origin database.
pub enum RlpType {
/// RLP used in blocks database.
Blocks,
/// RLP used in snapshots.
Snapshot,
}
fn to_elastic(slice: &[u8]) -> ElasticArray1024<u8> {
let mut out = ElasticArray1024::new();
out.append_slice(slice);
out
}
fn map_rlp<F>(rlp: &UntrustedRlp, f: F) -> Option<ElasticArray1024<u8>> where
F: Fn(&UntrustedRlp) -> Option<ElasticArray1024<u8>> {
match rlp.iter()
.fold((false, RlpStream::new_list(rlp.item_count())),
|(is_some, mut acc), subrlp| {
let new = f(&subrlp);
if let Some(ref insert) = new {
acc.append_raw(&insert[..], 1);
} else {
acc.append_raw(subrlp.as_raw(), 1);
}
(is_some || new.is_some(), acc)
}) {
(true, s) => Some(s.drain()),
_ => None,
}
}
/// Replace common RLPs with invalid shorter ones.
fn simple_compress(rlp: &UntrustedRlp, swapper: &InvalidRlpSwapper) -> ElasticArray1024<u8> {
if rlp.is_data() {
to_elastic(swapper.get_invalid(rlp.as_raw()).unwrap_or(rlp.as_raw()))
} else {
map_rlp(rlp, |r| Some(simple_compress(r, swapper))).unwrap_or(to_elastic(rlp.as_raw()))
}
}
/// Recover valid RLP from a compressed form.
fn simple_decompress(rlp: &UntrustedRlp, swapper: &InvalidRlpSwapper) -> ElasticArray1024<u8> {
if rlp.is_data() {
to_elastic(swapper.get_valid(rlp.as_raw()).unwrap_or(rlp.as_raw()))
} else {
map_rlp(rlp, |r| Some(simple_decompress(r, swapper))).unwrap_or(to_elastic(rlp.as_raw()))
}
}
/// Replace common RLPs with invalid shorter ones, None if no compression achieved.
/// Tries to compress data insides.
fn deep_compress(rlp: &UntrustedRlp, swapper: &InvalidRlpSwapper) -> Option<ElasticArray1024<u8>> {
let simple_swap = ||
swapper.get_invalid(rlp.as_raw()).map(|b| to_elastic(&b));
if rlp.is_data() {
// Try to treat the inside as RLP.
return match rlp.payload_info() {
// Shortest decompressed account is 70, so simply try to swap the value.
Ok(ref p) if p.value_len < 70 => simple_swap(),
_ => {
if let Ok(d) = rlp.data() {
let internal_rlp = UntrustedRlp::new(d);
if let Some(new_d) = deep_compress(&internal_rlp, swapper) {
// If compressed put in a special list, with first element being invalid code.
let mut rlp = RlpStream::new_list(2);
rlp.append_raw(&[0x81, 0x7f], 1);
rlp.append_raw(&new_d[..], 1);
return Some(rlp.drain());
}
}
simple_swap()
},
};
}
// Iterate through RLP while checking if it has been compressed.
map_rlp(rlp, |r| deep_compress(r, swapper))
}
/// Recover valid RLP from a compressed form, None if no decompression achieved.
/// Tries to decompress compressed data insides.
fn deep_decompress(rlp: &UntrustedRlp, swapper: &InvalidRlpSwapper) -> Option<ElasticArray1024<u8>> {
let simple_swap = ||
swapper.get_valid(rlp.as_raw()).map(|b| to_elastic(&b));
// Simply decompress data.
if rlp.is_data() { return simple_swap(); }
match rlp.item_count() {
// Look for special compressed list, which contains nested data.
2 if rlp.at(0).map(|r| r.as_raw() == &[0x81, 0x7f]).unwrap_or(false) =>
rlp.at(1).ok().map_or(simple_swap(),
|r| deep_decompress(&r, swapper).map(|d| { let v = d.to_vec(); encode(&v) })),
// Iterate through RLP while checking if it has been compressed.
_ => map_rlp(rlp, |r| deep_decompress(r, swapper)),
}
}
impl<'a> Compressible for UntrustedRlp<'a> {
type DataType = RlpType;
fn compress(&self, t: RlpType) -> ElasticArray1024<u8> {
match t {
RlpType::Snapshot => simple_compress(self, &SNAPSHOT_RLP_SWAPPER),
RlpType::Blocks => deep_compress(self, &BLOCKS_RLP_SWAPPER).unwrap_or(to_elastic(self.as_raw())),
}
}
fn decompress(&self, t: RlpType) -> ElasticArray1024<u8> {
match t {
RlpType::Snapshot => simple_decompress(self, &SNAPSHOT_RLP_SWAPPER),
RlpType::Blocks => deep_decompress(self, &BLOCKS_RLP_SWAPPER).unwrap_or(to_elastic(self.as_raw())),
}
}
}
#[cfg(test)]
mod tests {
use rlp::{UntrustedRlp, Compressible, View, RlpType};
use rlp::rlpcompression::InvalidRlpSwapper;
#[test]
fn invalid_rlp_swapper() {
let to_swap: &[&[u8]] = &[&[0x83, b'c', b'a', b't'], &[0x83, b'd', b'o', b'g']];
let invalid_rlp: &[&[u8]] = &[&[0x81, 0x00], &[0x81, 0x01]];
let swapper = InvalidRlpSwapper::new(to_swap, invalid_rlp);
assert_eq!(Some(invalid_rlp[0]), swapper.get_invalid(&[0x83, b'c', b'a', b't']));
assert_eq!(None, swapper.get_invalid(&[0x83, b'b', b'a', b't']));
assert_eq!(Some(to_swap[1]), swapper.get_valid(invalid_rlp[1]));
}
#[test]
fn simple_compression() {
let basic_account_rlp = vec![248, 68, 4, 2, 160, 86, 232, 31, 23, 27, 204, 85, 166, 255, 131, 69, 230, 146, 192, 248, 110, 91, 72, 224, 27, 153, 108, 173, 192, 1, 98, 47, 181, 227, 99, 180, 33, 160, 197, 210, 70, 1, 134, 247, 35, 60, 146, 126, 125, 178, 220, 199, 3, 192, 229, 0, 182, 83, 202, 130, 39, 59, 123, 250, 216, 4, 93, 133, 164, 112];
let rlp = UntrustedRlp::new(&basic_account_rlp);
let compressed = rlp.compress(RlpType::Snapshot).to_vec();
assert_eq!(compressed, vec![198, 4, 2, 129, 0, 129, 1]);
let compressed_rlp = UntrustedRlp::new(&compressed);
assert_eq!(compressed_rlp.decompress(RlpType::Snapshot).to_vec(), basic_account_rlp);
}
#[test]
fn data_compression() {
let data_basic_account_rlp = vec![184, 70, 248, 68, 4, 2, 160, 86, 232, 31, 23, 27, 204, 85, 166, 255, 131, 69, 230, 146, 192, 248, 110, 91, 72, 224, 27, 153, 108, 173, 192, 1, 98, 47, 181, 227, 99, 180, 33, 160, 197, 210, 70, 1, 134, 247, 35, 60, 146, 126, 125, 178, 220, 199, 3, 192, 229, 0, 182, 83, 202, 130, 39, 59, 123, 250, 216, 4, 93, 133, 164, 112];
let data_rlp = UntrustedRlp::new(&data_basic_account_rlp);
let compressed = data_rlp.compress(RlpType::Blocks).to_vec();
assert_eq!(compressed, vec![201, 129, 127, 198, 4, 2, 129, 0, 129, 1]);
let compressed_rlp = UntrustedRlp::new(&compressed);
assert_eq!(compressed_rlp.decompress(RlpType::Blocks).to_vec(), data_basic_account_rlp);
}
#[test]
fn nested_list_rlp() {
let nested_basic_account_rlp = vec![228, 4, 226, 2, 160, 86, 232, 31, 23, 27, 204, 85, 166, 255, 131, 69, 230, 146, 192, 248, 110, 91, 72, 224, 27, 153, 108, 173, 192, 1, 98, 47, 181, 227, 99, 180, 33];
let nested_rlp = UntrustedRlp::new(&nested_basic_account_rlp);
let compressed = nested_rlp.compress(RlpType::Blocks).to_vec();
assert_eq!(compressed, vec![197, 4, 195, 2, 129, 0]);
let compressed_rlp = UntrustedRlp::new(&compressed);
assert_eq!(compressed_rlp.decompress(RlpType::Blocks).to_vec(), nested_basic_account_rlp);
let compressed = nested_rlp.compress(RlpType::Snapshot).to_vec();
assert_eq!(compressed, vec![197, 4, 195, 2, 129, 0]);
let compressed_rlp = UntrustedRlp::new(&compressed);
assert_eq!(compressed_rlp.decompress(RlpType::Snapshot).to_vec(), nested_basic_account_rlp);
}
#[test]
fn malformed_rlp() {
let malformed = vec![248, 81, 128, 128, 128, 128, 128, 160, 12, 51, 241, 93, 69, 218, 74, 138, 79, 115, 227, 44, 216, 81, 46, 132, 85, 235, 96, 45, 252, 48, 181, 29, 75, 141, 217, 215, 86, 160, 109, 130, 160, 140, 36, 93, 200, 109, 215, 100, 241, 246, 99, 135, 92, 168, 149, 170, 114, 9, 143, 4, 93, 25, 76, 54, 176, 119, 230, 170, 154, 105, 47, 121, 10, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128];
let malformed_rlp = UntrustedRlp::new(&malformed);
assert_eq!(malformed_rlp.decompress(RlpType::Blocks).to_vec(), malformed);
}
#[test]
#[ignore]
fn test_compression() {
use kvdb::*;
let path = "db to test".to_string();
let values: Vec<_> = Database::open_default(&path).unwrap().iter(Some(2)).map(|(_, v)| v).collect();
let mut decomp_size = 0;
let mut comp_size = 0;
for v in values.iter() {
let rlp = UntrustedRlp::new(&v);
let compressed = rlp.compress(RlpType::Blocks).to_vec();
comp_size += compressed.len();
let decompressed = rlp.decompress(RlpType::Blocks).to_vec();
decomp_size += decompressed.len();
}
println!("Decompressed bytes {:?}, compressed bytes: {:?}", decomp_size, comp_size);
assert!(decomp_size > comp_size);
}
}

17
util/src/rlp/rlptraits.rs
View File

@@ -26,8 +26,8 @@ use sha3::*;
/// Type is able to decode RLP.
pub trait Decoder: Sized {
/// Read a value from the RLP into a given type.
fn read_value<T, F>(&self, f: F) -> Result<T, DecoderError>
where F: FnOnce(&[u8]) -> Result<T, DecoderError>;
fn read_value<T, F>(&self, f: &F) -> Result<T, DecoderError>
where F: Fn(&[u8]) -> Result<T, DecoderError>;
/// Get underlying `UntrustedRLP` object.
fn as_rlp(&self) -> &UntrustedRlp;
@@ -63,7 +63,7 @@ pub trait View<'a, 'view>: Sized {
/// Creates a new instance of `Rlp` reader
fn new(bytes: &'a [u8]) -> Self;
/// The raw data of the RLP.
/// The raw data of the RLP as slice.
///
/// ```rust
/// extern crate ethcore_util as util;
@@ -365,3 +365,14 @@ pub trait Stream: Sized {
/// panic! if stream is not finished.
fn out(self) -> Vec<u8>;
}
/// Trait for compressing and decompressing RLP by replacement of common terms.
pub trait Compressible: Sized {
/// Indicates the origin of RLP to be compressed.
type DataType;
/// Compress given RLP type using appropriate methods.
fn compress(&self, t: Self::DataType) -> ElasticArray1024<u8>;
/// Decompress given RLP type using appropriate methods.
fn decompress(&self, t: Self::DataType) -> ElasticArray1024<u8>;
}

78
util/src/rlp/untrusted_rlp.rs
View File

@@ -55,6 +55,18 @@ pub struct PayloadInfo {
pub value_len: usize,
}
fn calculate_payload_info(header_bytes: &[u8], len_of_len: usize) -> Result<PayloadInfo, DecoderError> {
let header_len = 1 + len_of_len;
match header_bytes.get(1) {
Some(&0) => return Err(DecoderError::RlpDataLenWithZeroPrefix),
None => return Err(DecoderError::RlpIsTooShort),
_ => (),
}
if header_bytes.len() < header_len { return Err(DecoderError::RlpIsTooShort); }
let value_len = try!(usize::from_bytes(&header_bytes[1..header_len]));
Ok(PayloadInfo::new(header_len, value_len))
}
impl PayloadInfo {
fn new(header_len: usize, value_len: usize) -> PayloadInfo {
PayloadInfo {
@@ -68,28 +80,22 @@ impl PayloadInfo {
/// Create a new object from the given bytes RLP. The bytes
pub fn from(header_bytes: &[u8]) -> Result<PayloadInfo, DecoderError> {
Ok(match header_bytes.first().cloned() {
None => return Err(DecoderError::RlpIsTooShort),
Some(0...0x7f) => PayloadInfo::new(0, 1),
Some(l @ 0x80...0xb7) => PayloadInfo::new(1, l as usize - 0x80),
match header_bytes.first().cloned() {
None => Err(DecoderError::RlpIsTooShort),
Some(0...0x7f) => Ok(PayloadInfo::new(0, 1)),
Some(l @ 0x80...0xb7) => Ok(PayloadInfo::new(1, l as usize - 0x80)),
Some(l @ 0xb8...0xbf) => {
let len_of_len = l as usize - 0xb7;
let header_len = 1 + len_of_len;
if header_bytes[1] == 0 { return Err(DecoderError::RlpDataLenWithZeroPrefix); }
let value_len = try!(usize::from_bytes(&header_bytes[1..header_len]));
PayloadInfo::new(header_len, value_len)
calculate_payload_info(header_bytes, len_of_len)
}
Some(l @ 0xc0...0xf7) => PayloadInfo::new(1, l as usize - 0xc0),
Some(l @ 0xc0...0xf7) => Ok(PayloadInfo::new(1, l as usize - 0xc0)),
Some(l @ 0xf8...0xff) => {
let len_of_len = l as usize - 0xf7;
let header_len = 1 + len_of_len;
let value_len = try!(usize::from_bytes(&header_bytes[1..header_len]));
if header_bytes[1] == 0 { return Err(DecoderError::RlpListLenWithZeroPrefix); }
PayloadInfo::new(header_len, value_len)
calculate_payload_info(header_bytes, len_of_len)
},
// we cant reach this place, but rust requires _ to be implemented
_ => { unreachable!(); }
})
}
}
}
@@ -190,8 +196,8 @@ impl<'a, 'view> View<'a, 'view> for UntrustedRlp<'a> where 'a: 'view {
fn size(&self) -> usize {
match self.is_data() {
// we can safely unwrap (?) cause its data
true => BasicDecoder::payload_info(self.bytes).unwrap().value_len,
// TODO: No panic on malformed data, but ideally would Err on no PayloadInfo.
true => BasicDecoder::payload_info(self.bytes).map(|b| b.value_len).unwrap_or(0),
false => 0
}
}
@@ -342,15 +348,15 @@ impl<'a> BasicDecoder<'a> {
}
impl<'a> Decoder for BasicDecoder<'a> {
fn read_value<T, F>(&self, f: F) -> Result<T, DecoderError>
where F: FnOnce(&[u8]) -> Result<T, DecoderError> {
fn read_value<T, F>(&self, f: &F) -> Result<T, DecoderError>
where F: Fn(&[u8]) -> Result<T, DecoderError> {
let bytes = self.rlp.as_raw();
match bytes.first().cloned() {
// rlp is too short
// RLP is too short.
None => Err(DecoderError::RlpIsTooShort),
// single byt value
// Single byte value.
Some(l @ 0...0x7f) => Ok(try!(f(&[l]))),
// 0-55 bytes
Some(l @ 0x80...0xb7) => {
@@ -362,10 +368,9 @@ impl<'a> Decoder for BasicDecoder<'a> {
if l == 0x81 && d[0] < 0x80 {
return Err(DecoderError::RlpInvalidIndirection);
}
Ok(try!(f(d)))
},
// longer than 55 bytes
// Longer than 55 bytes.
Some(l @ 0xb8...0xbf) => {
let len_of_len = l as usize - 0xb7;
let begin_of_value = 1 as usize + len_of_len;
@@ -380,7 +385,7 @@ impl<'a> Decoder for BasicDecoder<'a> {
}
Ok(try!(f(&bytes[begin_of_value..last_index_of_value])))
}
// we are reading value, not a list!
// We are reading value, not a list!
_ => Err(DecoderError::RlpExpectedToBeData)
}
}
@@ -396,9 +401,7 @@ impl<'a> Decoder for BasicDecoder<'a> {
impl<T> Decodable for T where T: FromBytes {
fn decode<D>(decoder: &D) -> Result<Self, DecoderError> where D: Decoder {
decoder.read_value(| bytes | {
Ok(try!(T::from_bytes(bytes)))
})
decoder.read_value(&|bytes: &[u8]| Ok(try!(T::from_bytes(bytes))))
}
}
@@ -416,11 +419,7 @@ impl<T> Decodable for Option<T> where T: Decodable {
impl Decodable for Vec<u8> {
fn decode<D>(decoder: &D) -> Result<Self, DecoderError> where D: Decoder {
decoder.read_value(| bytes | {
let mut res = vec![];
res.extend_from_slice(bytes);
Ok(res)
})
decoder.read_value(&|bytes: &[u8]| Ok(bytes.to_vec()))
}
}
@@ -489,11 +488,14 @@ impl RlpDecodable for u8 {
}
}
#[test]
fn test_rlp_display() {
use rustc_serialize::hex::FromHex;
let data = "f84d0589010efbef67941f79b2a056e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421a0c5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470".from_hex().unwrap();
let rlp = UntrustedRlp::new(&data);
assert_eq!(format!("{}", rlp), "[\"0x05\", \"0x010efbef67941f79b2\", \"0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421\", \"0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470\"]");
#[cfg(test)]
mod tests {
use rlp::{UntrustedRlp, View};
#[test]
fn test_rlp_display() {
use rustc_serialize::hex::FromHex;
let data = "f84d0589010efbef67941f79b2a056e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421a0c5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470".from_hex().unwrap();
let rlp = UntrustedRlp::new(&data);
assert_eq!(format!("{}", rlp), "[\"0x05\", \"0x010efbef67941f79b2\", \"0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421\", \"0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470\"]");
}
}

2
util/src/trie/mod.rs
View File

@@ -63,7 +63,7 @@ impl fmt::Display for TrieError {
}
/// Trie types
#[derive(Debug, Clone)]
#[derive(Debug, PartialEq, Clone)]
pub enum TrieSpec {
/// Generic trie.
Generic,

2
util/src/trie/standardmap.rs
View File

@@ -106,7 +106,7 @@ impl StandardMap {
Alphabet::All => Self::random_bytes(self.min_key, self.journal_key, seed),
Alphabet::Low => Self::random_word(low, self.min_key, self.journal_key, seed),
Alphabet::Mid => Self::random_word(mid, self.min_key, self.journal_key, seed),
Alphabet::Custom(ref a) => Self::random_word(&a, self.min_key, self.journal_key, seed),
Alphabet::Custom(ref a) => Self::random_word(a, self.min_key, self.journal_key, seed),
};
let v = match self.value_mode {
ValueMode::Mirror => k.clone(),

10
util/src/trie/triedb.rs
View File

@@ -132,7 +132,7 @@ impl<'db> TrieDB<'db> {
/// Get the data of the root node.
fn root_data(&self) -> &[u8] {
self.db.get(&self.root).expect("Trie root not found!")
self.db.get(self.root).expect("Trie root not found!")
}
/// Get the root node as a `Node`.
@@ -184,7 +184,7 @@ impl<'db> TrieDB<'db> {
/// Return optional data for a key given as a `NibbleSlice`. Returns `None` if no data exists.
fn do_lookup<'a, 'key>(&'a self, key: &NibbleSlice<'key>) -> Option<&'a [u8]> where 'a: 'key {
let root_rlp = self.root_data();
self.get_from_node(&root_rlp, key)
self.get_from_node(root_rlp, key)
}
/// Recursible function to retrieve the value given a `node` and a partial `key`. `None` if no
@@ -340,7 +340,7 @@ impl<'db> Trie for TrieDB<'db> {
Box::new(TrieDB::iter(self))
}
fn root(&self) -> &H256 { &self.root }
fn root(&self) -> &H256 { self.root }
fn contains(&self, key: &[u8]) -> bool {
self.get(key).is_some()
@@ -354,7 +354,7 @@ impl<'db> Trie for TrieDB<'db> {
impl<'db> fmt::Debug for TrieDB<'db> {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
try!(writeln!(f, "c={:?} [", self.hash_count));
let root_rlp = self.db.get(&self.root).expect("Trie root not found!");
let root_rlp = self.db.get(self.root).expect("Trie root not found!");
try!(self.fmt_all(Node::decoded(root_rlp), f, 0));
writeln!(f, "]")
}
@@ -373,7 +373,7 @@ fn iterator() {
{
let mut t = TrieDBMut::new(&mut memdb, &mut root);
for x in &d {
t.insert(&x, &x);
t.insert(x, x);
}
}
assert_eq!(d.iter().map(|i|i.to_vec()).collect::<Vec<_>>(), TrieDB::new(&memdb, &root).unwrap().iter().map(|x|x.0).collect::<Vec<_>>());

8
util/src/trie/triedbmut.rs
View File

@@ -401,7 +401,7 @@ impl<'a> TrieDBMut<'a> {
/// Return optional data for a key given as a `NibbleSlice`. Returns `None` if no data exists.
fn do_db_lookup<'x, 'key>(&'x self, hash: &H256, key: NibbleSlice<'key>) -> Option<&'x [u8]> where 'x: 'key {
self.db.get(hash).and_then(|node_rlp| self.get_from_db_node(&node_rlp, key))
self.db.get(hash).and_then(|node_rlp| self.get_from_db_node(node_rlp, key))
}
/// Recursible function to retrieve the value given a `node` and a partial `key`. `None` if no
@@ -868,7 +868,7 @@ impl<'a> TrieDBMut<'a> {
impl<'a> TrieMut for TrieDBMut<'a> {
fn root(&mut self) -> &H256 {
self.commit();
&self.root
self.root
}
fn is_empty(&self) -> bool {
@@ -938,7 +938,7 @@ mod tests {
for i in 0..v.len() {
let key: &[u8]= &v[i].0;
let val: &[u8] = &v[i].1;
t.insert(&key, &val);
t.insert(key, val);
}
t
}
@@ -946,7 +946,7 @@ mod tests {
fn unpopulate_trie<'db>(t: &mut TrieDBMut<'db>, v: &[(Vec<u8>, Vec<u8>)]) {
for i in v {
let key: &[u8]= &i.0;
t.remove(&key);
t.remove(key);
}
}

2
util/src/triehash.rs
View File

@@ -213,7 +213,7 @@ fn hash256rlp(input: &[(Vec<u8>, Vec<u8>)], pre_len: usize, stream: &mut RlpStre
.skip(1)
// get minimum number of shared nibbles between first and each successive
.fold(key.len(), | acc, &(ref k, _) | {
cmp::min(key.shared_prefix_len(&k), acc)
cmp::min(key.shared_prefix_len(k), acc)
});
// if shared prefix is higher than current prefix append its