grassrootseconomics/openethereum
8
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Merge branch 'master' of github.com:ethcore/parity into jdb_option2

Browse Source
This commit is contained in:
arkpar
2016-03-08 18:35:25 +01:00
parent 744c4c7d8b 9780897fc7
commit ca7d3b91d0
9 changed files with 233 additions and 154 deletions
Download Patch File Download Diff File Expand all files Collapse all files

167
util/src/journaldb.rs
View File

@@ -46,17 +46,17 @@ use std::env;
/// data nodes that go out of history scope and must be written to disk.
///
/// Commit workflow:
/// Create a new journal record from the transaction overlay.
/// Inseart each node from the transaction overlay into the History overlay increasing reference
/// 1. Create a new journal record from the transaction overlay.
/// 2. Inseart each node from the transaction overlay into the History overlay increasing reference
/// count if it is already there. Note that the reference counting is managed by `MemoryDB`
/// Clear the transaction overlay.
/// For a canonical journal record that becomes ancient inserts its insertions into the disk DB
/// For each journal record that goes out of the history scope (becomes ancient) remove its
/// 3. Clear the transaction overlay.
/// 4. For a canonical journal record that becomes ancient inserts its insertions into the disk DB
/// 5. For each journal record that goes out of the history scope (becomes ancient) remove its
/// insertions from the history overlay, decreasing the reference counter and removing entry if
/// if reaches zero.
/// For a canonical journal record that becomes ancient delete its removals from the disk only if
/// 6. For a canonical journal record that becomes ancient delete its removals from the disk only if
/// the removed key is not present in the history overlay.
/// Delete ancient record from memory and disk.
/// 7. Delete ancient record from memory and disk.
///
pub struct JournalDB {
transaction_overlay: MemoryDB,
@@ -66,13 +66,11 @@ pub struct JournalDB {
struct JournalOverlay {
backing_overlay: MemoryDB,
journal: VecDeque<JournalEntry>
journal: HashMap<u64, Vec<JournalEntry>>
}
struct JournalEntry {
id: H256,
index: usize,
era: u64,
insertions: Vec<H256>,
deletions: Vec<H256>,
}
@@ -200,29 +198,6 @@ impl JournalDB {
let mut journal_overlay = self.journal_overlay.as_mut().unwrap().write().unwrap();
let batch = DBTransaction::new();
{
let mut index = 0usize;
let mut last;
while {
let record = try!(self.backing.get({
let mut r = RlpStream::new_list(3);
r.append(&now);
r.append(&index);
r.append(&&PADDING[..]);
last = r.drain();
&last
}));
match record {
Some(r) => {
assert!(&Rlp::new(&r).val_at::<H256>(0) != id);
true
},
None => false,
}
} {
index += 1;
}
let mut r = RlpStream::new_list(3);
let mut tx = self.transaction_overlay.drain();
let inserted_keys: Vec<_> = tx.iter().filter_map(|(k, &(_, c))| if c > 0 { Some(k.clone()) } else { None }).collect();
@@ -238,55 +213,64 @@ impl JournalDB {
journal_overlay.backing_overlay.emplace(k, v);
}
r.append(&removed_keys);
try!(batch.put(&last, r.as_raw()));
let mut k = RlpStream::new_list(3);
let index = journal_overlay.journal.get(&now).map(|j| j.len()).unwrap_or(0);
k.append(&now);
k.append(&index);
k.append(&&PADDING[..]);
try!(batch.put(&k.drain(), r.as_raw()));
try!(batch.put(&LATEST_ERA_KEY, &encode(&now)));
journal_overlay.journal.push_back(JournalEntry { id: id.clone(), index: index, era: now, insertions: inserted_keys, deletions: removed_keys });
journal_overlay.journal.entry(now).or_insert_with(Vec::new).push(JournalEntry { id: id.clone(), insertions: inserted_keys, deletions: removed_keys });
}
let journal_overlay = journal_overlay.deref_mut();
// apply old commits' details
if let Some((end_era, canon_id)) = end {
let mut canon_insertions: Vec<(H256, Bytes)> = Vec::new();
let mut canon_deletions: Vec<H256> = Vec::new();
let mut overlay_deletions: Vec<H256> = Vec::new();
while journal_overlay.journal.front().map_or(false, |e| e.era <= end_era) {
let mut journal = journal_overlay.journal.pop_front().unwrap();
//delete the record from the db
let mut r = RlpStream::new_list(3);
r.append(&journal.era);
r.append(&journal.index);
r.append(&&PADDING[..]);
try!(batch.delete(&r.drain()));
trace!("commit: Delete journal for time #{}.{}: {}, (canon was {}): +{} -{} entries", end_era, journal.index, journal.id, canon_id, journal.insertions.len(), journal.deletions.len());
{
if canon_id == journal.id {
for h in &journal.insertions {
match journal_overlay.backing_overlay.raw(&h) {
Some(&(ref d, rc)) if rc > 0 => canon_insertions.push((h.clone(), d.clone())), //TODO: optimizie this to avoid data copy
_ => ()
if let Some(ref mut records) = journal_overlay.journal.get_mut(&end_era) {
let mut canon_insertions: Vec<(H256, Bytes)> = Vec::new();
let mut canon_deletions: Vec<H256> = Vec::new();
let mut overlay_deletions: Vec<H256> = Vec::new();
let mut index = 0usize;
for mut journal in records.drain(..) {
//delete the record from the db
let mut r = RlpStream::new_list(3);
r.append(&end_era);
r.append(&index);
r.append(&&PADDING[..]);
try!(batch.delete(&r.drain()));
trace!("commit: Delete journal for time #{}.{}: {}, (canon was {}): +{} -{} entries", end_era, index, journal.id, canon_id, journal.insertions.len(), journal.deletions.len());
{
if canon_id == journal.id {
for h in &journal.insertions {
match journal_overlay.backing_overlay.raw(&h) {
Some(&(ref d, rc)) if rc > 0 => canon_insertions.push((h.clone(), d.clone())), //TODO: optimizie this to avoid data copy
_ => ()
}
}
canon_deletions = journal.deletions;
}
canon_deletions = journal.deletions;
overlay_deletions.append(&mut journal.insertions);
}
overlay_deletions.append(&mut journal.insertions);
index +=1;
}
if canon_id == journal.id {
// apply canon inserts first
for (k, v) in canon_insertions {
try!(batch.put(&k, &v));
}
}
// apply canon inserts first
for (k, v) in canon_insertions {
try!(batch.put(&k, &v));
}
// clean the overlay
for k in overlay_deletions {
journal_overlay.backing_overlay.kill(&k);
}
// apply removes
for k in canon_deletions {
if !journal_overlay.backing_overlay.exists(&k) {
try!(batch.delete(&k));
// clean the overlay
for k in overlay_deletions {
journal_overlay.backing_overlay.kill(&k);
}
// apply removes
for k in canon_deletions {
if !journal_overlay.backing_overlay.exists(&k) {
try!(batch.delete(&k));
}
}
journal_overlay.backing_overlay.purge();
}
journal_overlay.backing_overlay.purge();
journal_overlay.journal.remove(&end_era);
}
try!(self.backing.write(batch));
Ok(0 as u32)
@@ -297,7 +281,7 @@ impl JournalDB {
}
fn read_overlay(db: &Database) -> JournalOverlay {
let mut journal = VecDeque::new();
let mut journal = HashMap::new();
let mut overlay = MemoryDB::new();
let mut count = 0;
if let Some(val) = db.get(&LATEST_ERA_KEY).expect("Low-level database error.") {
@@ -311,6 +295,7 @@ impl JournalDB {
r.append(&&PADDING[..]);
&r.drain()
}).expect("Low-level database error.") {
trace!("read_counters: era={}, index={}", era, index);
let rlp = Rlp::new(&rlp_data);
let id: H256 = rlp.val_at(0);
let insertions = rlp.at(1);
@@ -323,10 +308,8 @@ impl JournalDB {
inserted_keys.push(k);
count += 1;
}
journal.push_front(JournalEntry {
journal.entry(era).or_insert_with(Vec::new).push(JournalEntry {
id: id,
index: index,
era: era,
insertions: inserted_keys,
deletions: deletions,
});
@@ -413,6 +396,28 @@ mod tests {
use super::*;
use hashdb::*;
#[test]
fn insert_same_in_fork() {
// history is 1
let mut jdb = JournalDB::new_temp();
let x = jdb.insert(b"X");
jdb.commit(1, &b"1".sha3(), None).unwrap();
jdb.commit(2, &b"2".sha3(), None).unwrap();
jdb.commit(3, &b"1002a".sha3(), Some((1, b"1".sha3()))).unwrap();
jdb.commit(4, &b"1003a".sha3(), Some((2, b"2".sha3()))).unwrap();
jdb.remove(&x);
jdb.commit(3, &b"1002b".sha3(), Some((1, b"1".sha3()))).unwrap();
let x = jdb.insert(b"X");
jdb.commit(4, &b"1003b".sha3(), Some((2, b"2".sha3()))).unwrap();
jdb.commit(5, &b"1004a".sha3(), Some((3, b"1002a".sha3()))).unwrap();
jdb.commit(6, &b"1005a".sha3(), Some((4, b"1003a".sha3()))).unwrap();
assert!(jdb.exists(&x));
}
#[test]
fn long_history() {
// history is 3
@@ -533,6 +538,7 @@ mod tests {
assert!(jdb.exists(&foo));
}
#[test]
fn reopen() {
let mut dir = ::std::env::temp_dir();
@@ -574,20 +580,27 @@ mod tests {
// history is 1
let foo = jdb.insert(b"foo");
jdb.commit(0, &b"0".sha3(), None).unwrap();
jdb.commit(1, &b"1".sha3(), Some((0, b"0".sha3()))).unwrap();
// foo is ancient history.
jdb.insert(b"foo");
jdb.commit(1, &b"1".sha3(), None).unwrap();
jdb.commit(2, &b"2".sha3(), Some((1, b"1".sha3()))).unwrap();
foo
};
{
let mut jdb = JournalDB::new(dir.to_str().unwrap());
jdb.remove(&foo);
jdb.commit(2, &b"2".sha3(), Some((1, b"1".sha3()))).unwrap();
assert!(jdb.exists(&foo));
jdb.commit(3, &b"3".sha3(), Some((2, b"2".sha3()))).unwrap();
assert!(jdb.exists(&foo));
jdb.remove(&foo);
jdb.commit(4, &b"4".sha3(), Some((3, b"3".sha3()))).unwrap();
jdb.commit(5, &b"5".sha3(), Some((4, b"4".sha3()))).unwrap();
assert!(!jdb.exists(&foo));
}
}
#[test]
fn reopen_fork() {
let mut dir = ::std::env::temp_dir();

10
util/src/network/connection.rs
View File

@@ -190,25 +190,25 @@ impl Connection {
/// Register this connection with the IO event loop.
pub fn register_socket<Host: Handler>(&self, reg: Token, event_loop: &mut EventLoop<Host>) -> io::Result<()> {
trace!(target: "net", "connection register; token={:?}", reg);
trace!(target: "network", "connection register; token={:?}", reg);
if let Err(e) = event_loop.register(&self.socket, reg, self.interest, PollOpt::edge() | PollOpt::oneshot()) {
debug!("Failed to register {:?}, {:?}", reg, e);
trace!(target: "network", "Failed to register {:?}, {:?}", reg, e);
}
Ok(())
}
/// Update connection registration. Should be called at the end of the IO handler.
pub fn update_socket<Host: Handler>(&self, reg: Token, event_loop: &mut EventLoop<Host>) -> io::Result<()> {
trace!(target: "net", "connection reregister; token={:?}", reg);
trace!(target: "network", "connection reregister; token={:?}", reg);
event_loop.reregister( &self.socket, reg, self.interest, PollOpt::edge() | PollOpt::oneshot()).or_else(|e| {
debug!("Failed to reregister {:?}, {:?}", reg, e);
trace!(target: "network", "Failed to reregister {:?}, {:?}", reg, e);
Ok(())
})
}
/// Delete connection registration. Should be called at the end of the IO handler.
pub fn deregister_socket<Host: Handler>(&self, event_loop: &mut EventLoop<Host>) -> io::Result<()> {
trace!(target: "net", "connection deregister; token={:?}", self.token);
trace!(target: "network", "connection deregister; token={:?}", self.token);
event_loop.deregister(&self.socket).ok(); // ignore errors here
Ok(())
}

14
util/src/network/handshake.rs
View File

@@ -222,7 +222,7 @@ impl Handshake {
/// Parse, validate and confirm auth message
fn read_auth(&mut self, secret: &Secret, data: &[u8]) -> Result<(), UtilError> {
trace!(target:"net", "Received handshake auth from {:?}", self.connection.socket.peer_addr());
trace!(target:"network", "Received handshake auth from {:?}", self.connection.socket.peer_addr());
if data.len() != V4_AUTH_PACKET_SIZE {
debug!(target:"net", "Wrong auth packet size");
return Err(From::from(NetworkError::BadProtocol));
@@ -253,7 +253,7 @@ impl Handshake {
}
fn read_auth_eip8(&mut self, secret: &Secret, data: &[u8]) -> Result<(), UtilError> {
trace!(target:"net", "Received EIP8 handshake auth from {:?}", self.connection.socket.peer_addr());
trace!(target:"network", "Received EIP8 handshake auth from {:?}", self.connection.socket.peer_addr());
self.auth_cipher.extend_from_slice(data);
let auth = try!(ecies::decrypt(secret, &self.auth_cipher[0..2], &self.auth_cipher[2..]));
let rlp = UntrustedRlp::new(&auth);
@@ -268,7 +268,7 @@ impl Handshake {
/// Parse and validate ack message
fn read_ack(&mut self, secret: &Secret, data: &[u8]) -> Result<(), UtilError> {
trace!(target:"net", "Received handshake auth to {:?}", self.connection.socket.peer_addr());
trace!(target:"network", "Received handshake auth to {:?}", self.connection.socket.peer_addr());
if data.len() != V4_ACK_PACKET_SIZE {
debug!(target:"net", "Wrong ack packet size");
return Err(From::from(NetworkError::BadProtocol));
@@ -296,7 +296,7 @@ impl Handshake {
}
fn read_ack_eip8(&mut self, secret: &Secret, data: &[u8]) -> Result<(), UtilError> {
trace!(target:"net", "Received EIP8 handshake auth from {:?}", self.connection.socket.peer_addr());
trace!(target:"network", "Received EIP8 handshake auth from {:?}", self.connection.socket.peer_addr());
self.ack_cipher.extend_from_slice(data);
let ack = try!(ecies::decrypt(secret, &self.ack_cipher[0..2], &self.ack_cipher[2..]));
let rlp = UntrustedRlp::new(&ack);
@@ -309,7 +309,7 @@ impl Handshake {
/// Sends auth message
fn write_auth(&mut self, secret: &Secret, public: &Public) -> Result<(), UtilError> {
trace!(target:"net", "Sending handshake auth to {:?}", self.connection.socket.peer_addr());
trace!(target:"network", "Sending handshake auth to {:?}", self.connection.socket.peer_addr());
let mut data = [0u8; /*Signature::SIZE*/ 65 + /*H256::SIZE*/ 32 + /*Public::SIZE*/ 64 + /*H256::SIZE*/ 32 + 1]; //TODO: use associated constants
let len = data.len();
{
@@ -336,7 +336,7 @@ impl Handshake {
/// Sends ack message
fn write_ack(&mut self) -> Result<(), UtilError> {
trace!(target:"net", "Sending handshake ack to {:?}", self.connection.socket.peer_addr());
trace!(target:"network", "Sending handshake ack to {:?}", self.connection.socket.peer_addr());
let mut data = [0u8; 1 + /*Public::SIZE*/ 64 + /*H256::SIZE*/ 32]; //TODO: use associated constants
let len = data.len();
{
@@ -355,7 +355,7 @@ impl Handshake {
/// Sends EIP8 ack message
fn write_ack_eip8(&mut self) -> Result<(), UtilError> {
trace!(target:"net", "Sending EIP8 handshake ack to {:?}", self.connection.socket.peer_addr());
trace!(target:"network", "Sending EIP8 handshake ack to {:?}", self.connection.socket.peer_addr());
let mut rlp = RlpStream::new_list(3);
rlp.append(self.ecdhe.public());
rlp.append(&self.nonce);

43
util/src/network/host.rs
View File

@@ -170,29 +170,37 @@ pub struct NetworkContext<'s, Message> where Message: Send + Sync + Clone + 'sta
io: &'s IoContext<NetworkIoMessage<Message>>,
protocol: ProtocolId,
sessions: Arc<RwLock<Slab<SharedSession>>>,
session: Option<StreamToken>,
session: Option<SharedSession>,
session_id: Option<StreamToken>,
}
impl<'s, Message> NetworkContext<'s, Message> where Message: Send + Sync + Clone + 'static, {
/// Create a new network IO access point. Takes references to all the data that can be updated within the IO handler.
fn new(io: &'s IoContext<NetworkIoMessage<Message>>,
protocol: ProtocolId,
session: Option<StreamToken>, sessions: Arc<RwLock<Slab<SharedSession>>>) -> NetworkContext<'s, Message> {
session: Option<SharedSession>, sessions: Arc<RwLock<Slab<SharedSession>>>) -> NetworkContext<'s, Message> {
let id = session.as_ref().map(|s| s.lock().unwrap().token());
NetworkContext {
io: io,
protocol: protocol,
session_id: id,
session: session,
sessions: sessions,
}
}
fn resolve_session(&self, peer: PeerId) -> Option<SharedSession> {
match self.session_id {
Some(id) if id == peer => self.session.clone(),
_ => self.sessions.read().unwrap().get(peer).cloned(),
}
}
/// Send a packet over the network to another peer.
pub fn send(&self, peer: PeerId, packet_id: PacketId, data: Vec<u8>) -> Result<(), UtilError> {
let session = { self.sessions.read().unwrap().get(peer).cloned() };
let session = self.resolve_session(peer);
if let Some(session) = session {
session.lock().unwrap().deref_mut().send_packet(self.protocol, packet_id as u8, &data).unwrap_or_else(|e| {
warn!(target: "network", "Send error: {:?}", e);
}); //TODO: don't copy vector data
try!(session.lock().unwrap().deref_mut().send_packet(self.protocol, packet_id as u8, &data));
try!(self.io.update_registration(peer));
} else {
trace!(target: "network", "Send: Peer no longer exist")
@@ -200,14 +208,10 @@ impl<'s, Message> NetworkContext<'s, Message> where Message: Send + Sync + Clone
Ok(())
}
/// Respond to a current network message. Panics if no there is no packet in the context.
/// Respond to a current network message. Panics if no there is no packet in the context. If the session is expired returns nothing.
pub fn respond(&self, packet_id: PacketId, data: Vec<u8>) -> Result<(), UtilError> {
match self.session {
Some(session) => self.send(session, packet_id, data),
None => {
panic!("Respond: Session does not exist")
}
}
assert!(self.session.is_some(), "Respond called without network context");
self.send(self.session_id.unwrap(), packet_id, data)
}
/// Send an IO message
@@ -215,7 +219,6 @@ impl<'s, Message> NetworkContext<'s, Message> where Message: Send + Sync + Clone
self.io.message(NetworkIoMessage::User(msg));
}
/// Disable current protocol capability for given peer. If no capabilities left peer gets disconnected.
pub fn disable_peer(&self, peer: PeerId) {
//TODO: remove capability, disconnect if no capabilities left
@@ -239,7 +242,7 @@ impl<'s, Message> NetworkContext<'s, Message> where Message: Send + Sync + Clone
/// Returns peer identification string
pub fn peer_info(&self, peer: PeerId) -> String {
let session = { self.sessions.read().unwrap().get(peer).cloned() };
let session = self.resolve_session(peer);
if let Some(session) = session {
return session.lock().unwrap().info.client_version.clone()
}
@@ -624,7 +627,7 @@ impl<Message> Host<Message> where Message: Send + Sync + Clone {
let mut packet_data: Option<(ProtocolId, PacketId, Vec<u8>)> = None;
let mut kill = false;
let session = { self.sessions.read().unwrap().get(token).cloned() };
if let Some(session) = session {
if let Some(session) = session.clone() {
let mut s = session.lock().unwrap();
match s.readable(io, &self.info.read().unwrap()) {
Err(e) => {
@@ -656,11 +659,11 @@ impl<Message> Host<Message> where Message: Send + Sync + Clone {
}
for p in ready_data {
let h = self.handlers.read().unwrap().get(p).unwrap().clone();
h.connected(&NetworkContext::new(io, p, Some(token), self.sessions.clone()), &token);
h.connected(&NetworkContext::new(io, p, session.clone(), self.sessions.clone()), &token);
}
if let Some((p, packet_id, data)) = packet_data {
let h = self.handlers.read().unwrap().get(p).unwrap().clone();
h.read(&NetworkContext::new(io, p, Some(token), self.sessions.clone()), &token, packet_id, &data[1..]);
h.read(&NetworkContext::new(io, p, session.clone(), self.sessions.clone()), &token, packet_id, &data[1..]);
}
io.update_registration(token).unwrap_or_else(|e| debug!(target: "network", "Token registration error: {:?}", e));
}
@@ -718,6 +721,7 @@ impl<Message> Host<Message> where Message: Send + Sync + Clone {
let mut to_disconnect: Vec<ProtocolId> = Vec::new();
let mut failure_id = None;
let mut deregister = false;
let mut expired_session = None;
match token {
FIRST_HANDSHAKE ... LAST_HANDSHAKE => {
let handshakes = self.handshakes.write().unwrap();
@@ -733,6 +737,7 @@ impl<Message> Host<Message> where Message: Send + Sync + Clone {
FIRST_SESSION ... LAST_SESSION => {
let sessions = self.sessions.write().unwrap();
if let Some(session) = sessions.get(token).cloned() {
expired_session = Some(session.clone());
let mut s = session.lock().unwrap();
if !s.expired() {
if s.is_ready() {
@@ -757,7 +762,7 @@ impl<Message> Host<Message> where Message: Send + Sync + Clone {
}
for p in to_disconnect {
let h = self.handlers.read().unwrap().get(p).unwrap().clone();
h.disconnected(&NetworkContext::new(io, p, Some(token), self.sessions.clone()), &token);
h.disconnected(&NetworkContext::new(io, p, expired_session.clone(), self.sessions.clone()), &token);
}
if deregister {
io.deregister_stream(token).expect("Error deregistering stream");

9
util/src/network/session.rs
View File

@@ -213,6 +213,9 @@ impl Session {
/// Send a protocol packet to peer.
pub fn send_packet(&mut self, protocol: &str, packet_id: u8, data: &[u8]) -> Result<(), UtilError> {
if self.expired() {
return Err(From::from(NetworkError::Expired));
}
let mut i = 0usize;
while protocol != self.info.capabilities[i].protocol {
i += 1;
@@ -351,15 +354,15 @@ impl Session {
offset += caps[i].packet_count;
i += 1;
}
trace!(target: "net", "Hello: {} v{} {} {:?}", client_version, protocol, id, caps);
trace!(target: "network", "Hello: {} v{} {} {:?}", client_version, protocol, id, caps);
self.info.client_version = client_version;
self.info.capabilities = caps;
if self.info.capabilities.is_empty() {
trace!("No common capabilities with peer.");
trace!(target: "network", "No common capabilities with peer.");
return Err(From::from(self.disconnect(DisconnectReason::UselessPeer)));
}
if protocol != host.protocol_version {
trace!("Peer protocol version mismatch: {}", protocol);
trace!(target: "network", "Peer protocol version mismatch: {}", protocol);
return Err(From::from(self.disconnect(DisconnectReason::UselessPeer)));
}
self.had_hello = true;