From 09b65037957df851400ba626a9892b8645a853ea Mon Sep 17 00:00:00 2001 From: arkpar Date: Fri, 12 Feb 2016 09:52:32 +0100 Subject: [PATCH 01/61] Discovery packets --- util/src/network/connection.rs | 5 + util/src/network/discovery.rs | 201 ++++++++++++++++++++++++++------- util/src/network/error.rs | 7 ++ util/src/network/handshake.rs | 10 ++ util/src/network/host.rs | 98 ++++++++++------ util/src/network/mod.rs | 1 + util/src/network/node_table.rs | 52 +++++++++ util/src/network/session.rs | 5 + 8 files changed, 303 insertions(+), 76 deletions(-) create mode 100644 util/src/network/node_table.rs diff --git a/util/src/network/connection.rs b/util/src/network/connection.rs index 746c745c4..44d429164 100644 --- a/util/src/network/connection.rs +++ b/util/src/network/connection.rs @@ -159,6 +159,11 @@ impl Connection { } } + /// Get socket token + pub fn token(&self) -> StreamToken { + self.token + } + /// Register this connection with the IO event loop. pub fn register_socket(&self, reg: Token, event_loop: &mut EventLoop) -> io::Result<()> { trace!(target: "net", "connection register; token={:?}", reg); diff --git a/util/src/network/discovery.rs b/util/src/network/discovery.rs index 32370b88d..da81920ff 100644 --- a/util/src/network/discovery.rs +++ b/util/src/network/discovery.rs @@ -14,26 +14,34 @@ // You should have received a copy of the GNU General Public License // along with Parity. If not, see . -// This module is a work in progress - -#![allow(dead_code)] //TODO: remove this after everything is done - -use std::collections::{HashSet, BTreeMap}; +use bytes::Bytes; +use std::net::SocketAddr; +use std::collections::{HashSet, HashMap, BTreeMap, VecDeque}; use std::cell::{RefCell}; use std::ops::{DerefMut}; +use std::mem; use mio::*; use mio::udp::*; use hash::*; use sha3::Hashable; use crypto::*; +use rlp::*; use network::node::*; +use network::error::NetworkError; +use io::StreamToken; -const ADDRESS_BYTES_SIZE: u32 = 32; ///< Size of address type in bytes. -const ADDRESS_BITS: u32 = 8 * ADDRESS_BYTES_SIZE; ///< Denoted by n in [Kademlia]. -const NODE_BINS: u32 = ADDRESS_BITS - 1; ///< Size of m_state (excludes root, which is us). -const DISCOVERY_MAX_STEPS: u16 = 8; ///< Max iterations of discovery. (discover) -const BUCKET_SIZE: u32 = 16; ///< Denoted by k in [Kademlia]. Number of nodes stored in each bucket. -const ALPHA: usize = 3; ///< Denoted by \alpha in [Kademlia]. Number of concurrent FindNode requests. +const ADDRESS_BYTES_SIZE: u32 = 32; // Size of address type in bytes. +const ADDRESS_BITS: u32 = 8 * ADDRESS_BYTES_SIZE; // Denoted by n in [Kademlia]. +const NODE_BINS: u32 = ADDRESS_BITS - 1; // Size of m_state (excludes root, which is us). +const DISCOVERY_MAX_STEPS: u16 = 8; // Max iterations of discovery. (discover) +const BUCKET_SIZE: u32 = 16; // Denoted by k in [Kademlia]. Number of nodes stored in each bucket. +const ALPHA: usize = 3; // Denoted by \alpha in [Kademlia]. Number of concurrent FindNode requests. +const MAX_DATAGRAM_SIZE: usize = 1280; + +const PACKET_PING: u8 = 1; +const PACKET_PONG: u8 = 2; +const PACKET_FIND_NODE: u8 = 3; +const PACKET_NEIGHBOURS: u8 = 4; struct NodeBucket { distance: u32, @@ -49,12 +57,25 @@ impl NodeBucket { } } -struct Discovery { +struct Datagramm { + payload: Bytes, + address: SocketAddr, +} + +pub struct Discovery { id: NodeId, + udp_socket: UdpSocket, + token: StreamToken, discovery_round: u16, discovery_id: NodeId, discovery_nodes: HashSet, node_buckets: Vec, + send_queue: VecDeque +} + +pub struct TableUpdates { + pub added: HashMap, + pub removed: HashSet, } struct FindNodePacket; @@ -72,13 +93,17 @@ impl FindNodePacket { } impl Discovery { - pub fn new(id: &NodeId) -> Discovery { + pub fn new(id: &NodeId, address: &SocketAddr, token: StreamToken) -> Discovery { + let socket = UdpSocket::bound(address).expect("Error binding UDP socket"); Discovery { id: id.clone(), + token: token, discovery_round: 0, discovery_id: NodeId::new(), discovery_nodes: HashSet::new(), node_buckets: (0..NODE_BINS).map(NodeBucket::new).collect(), + udp_socket: socket, + send_queue: VecDeque::new(), } } @@ -151,17 +176,13 @@ impl Discovery { // if d is 0, then we roll look forward, if last, we reverse, else, spread from d if head > 1 && tail != LAST_BIN { - while head != tail && head < NODE_BINS && count < BUCKET_SIZE - { - for n in &buckets[head as usize].nodes - { - if count < BUCKET_SIZE { - count += 1; - found.entry(Discovery::distance(target, &n)).or_insert_with(Vec::new).push(n); - } - else { - break; - } + while head != tail && head < NODE_BINS && count < BUCKET_SIZE { + for n in &buckets[head as usize].nodes { + if count < BUCKET_SIZE { + count += 1; + found.entry(Discovery::distance(target, &n)).or_insert_with(Vec::new).push(n); + } + else { break } } if count < BUCKET_SIZE && tail != 0 { for n in &buckets[tail as usize].nodes { @@ -169,9 +190,7 @@ impl Discovery { count += 1; found.entry(Discovery::distance(target, &n)).or_insert_with(Vec::new).push(n); } - else { - break; - } + else { break } } } @@ -184,13 +203,11 @@ impl Discovery { else if head < 2 { while head < NODE_BINS && count < BUCKET_SIZE { for n in &buckets[head as usize].nodes { - if count < BUCKET_SIZE { - count += 1; - found.entry(Discovery::distance(target, &n)).or_insert_with(Vec::new).push(n); - } - else { - break; - } + if count < BUCKET_SIZE { + count += 1; + found.entry(Discovery::distance(target, &n)).or_insert_with(Vec::new).push(n); + } + else { break } } head += 1; } @@ -198,13 +215,11 @@ impl Discovery { else { while tail > 0 && count < BUCKET_SIZE { for n in &buckets[tail as usize].nodes { - if count < BUCKET_SIZE { - count += 1; - found.entry(Discovery::distance(target, &n)).or_insert_with(Vec::new).push(n); - } - else { - break; - } + if count < BUCKET_SIZE { + count += 1; + found.entry(Discovery::distance(target, &n)).or_insert_with(Vec::new).push(n); + } + else { break } } tail -= 1; } @@ -220,4 +235,108 @@ impl Discovery { } ret } + + pub fn writable(&mut self) { + if self.send_queue.is_empty() { + return; + } + let data = self.send_queue.pop_front().unwrap(); + match self.udp_socket.send_to(&data.payload, &data.address) { + Ok(Some(size)) if size == data.payload.len() => { + }, + Ok(Some(size)) => { + warn!("UDP sent incomplete datagramm"); + }, + Ok(None) => { + self.send_queue.push_front(data); + } + Err(e) => { + warn!("UDP sent error: {:?}", e); + } + } + } + + fn send_to(&mut self, payload: Bytes, address: SocketAddr) { + self.send_queue.push_back(Datagramm { payload: payload, address: address }); + } + + pub fn readable(&mut self) -> Option { + let mut buf: [u8; MAX_DATAGRAM_SIZE] = unsafe { mem::uninitialized() }; + match self.udp_socket.recv_from(&mut buf) { + Ok(Some((len, address))) => self.on_packet(&buf[0..len], address).unwrap_or_else(|e| { + debug!("Error processing UDP packet: {:?}", e); + None + }), + Ok(_) => None, + Err(e) => { + warn!("Error reading UPD socket: {:?}", e); + None + } + } + } + + fn on_packet(&mut self, packet: &[u8], from: SocketAddr) -> Result, NetworkError> { + // validate packet + if packet.len() < 32 + 65 + 4 + 1 { + return Err(NetworkError::BadProtocol); + } + + let hash_signed = (&packet[32..]).sha3(); + if hash_signed[..] != packet[0..32] { + return Err(NetworkError::BadProtocol); + } + + let signed = &packet[(32 + 65)..]; + let signature = Signature::from_slice(&packet[32..(32 + 65)]); + let node_id = try!(ec::recover(&signature, &signed.sha3())); + + let packet_id = signed[0]; + let rlp = UntrustedRlp::new(&signed[1..]); + match packet_id { + PACKET_PING => self.on_ping(&rlp, &node_id, &from), + PACKET_PONG => self.on_pong(&rlp, &node_id, &from), + PACKET_FIND_NODE => self.on_find_node(&rlp, &node_id, &from), + PACKET_NEIGHBOURS => self.on_neighbours(&rlp, &node_id, &from), + _ => { + debug!("Unknown UDP packet: {}", packet_id); + Ok(None) + } + } + } + + fn on_ping(&mut self, rlp: &UntrustedRlp, node: &NodeId, from: &SocketAddr) -> Result, NetworkError> { + Ok(None) + } + + fn on_pong(&mut self, rlp: &UntrustedRlp, node: &NodeId, from: &SocketAddr) -> Result, NetworkError> { + Ok(None) + } + + fn on_find_node(&mut self, rlp: &UntrustedRlp, node: &NodeId, from: &SocketAddr) -> Result, NetworkError> { + Ok(None) + } + + fn on_neighbours(&mut self, rlp: &UntrustedRlp, node: &NodeId, from: &SocketAddr) -> Result, NetworkError> { + Ok(None) + } + + pub fn round(&mut self) { + } + + pub fn refresh(&mut self) { + } + + pub fn register_socket(&self, event_loop: &mut EventLoop) -> Result<(), NetworkError> { + event_loop.register(&self.udp_socket, Token(self.token), EventSet::all(), PollOpt::edge()).expect("Error registering UDP socket"); + Ok(()) + } + + pub fn update_registration(&self, event_loop: &mut EventLoop) -> Result<(), NetworkError> { + let mut registration = EventSet::readable(); + if !self.send_queue.is_empty() { + registration &= EventSet::writable(); + } + event_loop.reregister(&self.udp_socket, Token(self.token), registration, PollOpt::edge()).expect("Error reregistering UDP socket"); + Ok(()) + } } diff --git a/util/src/network/error.rs b/util/src/network/error.rs index 4d7fb483e..eb97e54b6 100644 --- a/util/src/network/error.rs +++ b/util/src/network/error.rs @@ -15,6 +15,7 @@ // along with Parity. If not, see . use io::IoError; +use crypto::CryptoError; use rlp::*; #[derive(Debug, Copy, Clone)] @@ -61,3 +62,9 @@ impl From for NetworkError { } } +impl From for NetworkError { + fn from(_err: CryptoError) -> NetworkError { + NetworkError::Auth + } +} + diff --git a/util/src/network/handshake.rs b/util/src/network/handshake.rs index 4b23c4e16..94650b2a7 100644 --- a/util/src/network/handshake.rs +++ b/util/src/network/handshake.rs @@ -87,6 +87,16 @@ impl Handshake { }) } + /// Get id of the remote node if known + pub fn id(&self) -> &NodeId { + &self.id + } + + /// Get stream token id + pub fn token(&self) -> StreamToken { + self.connection.token() + } + /// Start a handhsake pub fn start(&mut self, io: &IoContext, host: &HostInfo, originated: bool) -> Result<(), UtilError> where Message: Send + Clone{ self.originated = originated; diff --git a/util/src/network/host.rs b/util/src/network/host.rs index c1423dbb3..2ad949642 100644 --- a/util/src/network/host.rs +++ b/util/src/network/host.rs @@ -22,7 +22,6 @@ use std::sync::*; use std::ops::*; use mio::*; use mio::tcp::*; -use mio::udp::*; use target_info::Target; use hash::*; use crypto::*; @@ -37,6 +36,8 @@ use network::node::*; use network::stats::NetworkStats; use network::error::DisconnectReason; use igd::{PortMappingProtocol,search_gateway}; +use network::discovery::{Discovery, TableUpdates}; +use network::node_table::NodeTable; type Slab = ::slab::Slab; @@ -50,6 +51,8 @@ const MAINTENANCE_TIMEOUT: u64 = 1000; #[derive(Debug)] /// Network service configuration pub struct NetworkConfiguration { + /// Directory path to store network configuration. None means nothing will be saved + pub config_path: Option, /// IP address to listen for incoming connections pub listen_address: SocketAddr, /// IP address to advertise @@ -70,6 +73,7 @@ impl NetworkConfiguration { /// Create a new instance of default settings. pub fn new() -> NetworkConfiguration { NetworkConfiguration { + config_path: None, listen_address: SocketAddr::from_str("0.0.0.0:30304").unwrap(), public_address: SocketAddr::from_str("0.0.0.0:30304").unwrap(), nat_enabled: true, @@ -114,6 +118,7 @@ impl NetworkConfiguration { } NetworkConfiguration { + config_path: self.config_path, listen_address: listen, public_address: public, nat_enabled: false, @@ -126,14 +131,12 @@ impl NetworkConfiguration { } // Tokens -//const TOKEN_BEGIN: usize = USER_TOKEN_START; // TODO: ICE in rustc 1.7.0-nightly (49c382779 2016-01-12) -const TOKEN_BEGIN: usize = 32; -const TCP_ACCEPT: usize = TOKEN_BEGIN + 1; -const IDLE: usize = TOKEN_BEGIN + 2; -const NODETABLE_RECEIVE: usize = TOKEN_BEGIN + 3; -const NODETABLE_MAINTAIN: usize = TOKEN_BEGIN + 4; -const NODETABLE_DISCOVERY: usize = TOKEN_BEGIN + 5; -const FIRST_CONNECTION: usize = TOKEN_BEGIN + 16; +const TCP_ACCEPT: usize = MAX_CONNECTIONS + 1; +const IDLE: usize = MAX_CONNECTIONS + 2; +const DISCOVERY: usize = MAX_CONNECTIONS + 3; +const DISCOVERY_REFRESH: usize = MAX_CONNECTIONS + 4; +const DISCOVERY_ROUND: usize = MAX_CONNECTIONS + 5; +const FIRST_CONNECTION: usize = 0; const LAST_CONNECTION: usize = FIRST_CONNECTION + MAX_CONNECTIONS - 1; /// Protocol handler level packet id @@ -320,10 +323,10 @@ struct ProtocolTimer { /// Root IO handler. Manages protocol handlers, IO timers and network connections. pub struct Host where Message: Send + Sync + Clone { pub info: RwLock, - udp_socket: Mutex, tcp_listener: Mutex, connections: Arc>>, - nodes: RwLock>, + discovery: Mutex, + nodes: RwLock, handlers: RwLock>>>, timers: RwLock>, timer_counter: RwLock, @@ -338,10 +341,12 @@ impl Host where Message: Send + Sync + Clone { let addr = config.listen_address; // Setup the server socket let tcp_listener = TcpListener::bind(&addr).unwrap(); - let udp_socket = UdpSocket::bound(&addr).unwrap(); + let keys = if let Some(ref secret) = config.use_secret { KeyPair::from_secret(secret.clone()).unwrap() } else { KeyPair::create().unwrap() }; + let public = keys.public().clone(); + let path = config.config_path.clone(); let mut host = Host:: { info: RwLock::new(HostInfo { - keys: if let Some(ref secret) = config.use_secret { KeyPair::from_secret(secret.clone()).unwrap() } else { KeyPair::create().unwrap() }, + keys: keys, config: config, nonce: H256::random(), protocol_version: 4, @@ -349,10 +354,10 @@ impl Host where Message: Send + Sync + Clone { listen_port: 0, capabilities: Vec::new(), }), - udp_socket: Mutex::new(udp_socket), + discovery: Mutex::new(Discovery::new(&public, &addr, DISCOVERY)), tcp_listener: Mutex::new(tcp_listener), connections: Arc::new(RwLock::new(Slab::new_starting_at(FIRST_CONNECTION, MAX_CONNECTIONS))), - nodes: RwLock::new(HashMap::new()), + nodes: RwLock::new(NodeTable::new(path)), handlers: RwLock::new(HashMap::new()), timers: RwLock::new(HashMap::new()), timer_counter: RwLock::new(LAST_CONNECTION + 1), @@ -361,12 +366,6 @@ impl Host where Message: Send + Sync + Clone { let port = host.info.read().unwrap().config.listen_address.port(); host.info.write().unwrap().deref_mut().listen_port = port; - /* - match ::ifaces::Interface::get_all().unwrap().into_iter().filter(|x| x.kind == ::ifaces::Kind::Packet && x.addr.is_some()).next() { - Some(iface) => config.public_address = iface.addr.unwrap(), - None => warn!("No public network interface"), - */ - let boot_nodes = host.info.read().unwrap().config.boot_nodes.clone(); for n in boot_nodes { host.add_node(&n); @@ -382,7 +381,7 @@ impl Host where Message: Send + Sync + Clone { match Node::from_str(id) { Err(e) => { warn!("Could not add node: {:?}", e); }, Ok(n) => { - self.nodes.write().unwrap().insert(n.id.clone(), n); + self.nodes.write().unwrap().add_node(n); } } } @@ -430,12 +429,9 @@ impl Host where Message: Send + Sync + Clone { } let mut to_connect: Vec = Vec::new(); - let mut req_conn = 0; - //TODO: use nodes from discovery here - //for n in self.node_buckets.iter().flat_map(|n| &n.nodes).map(|id| NodeInfo { id: id.clone(), peer_type: self.nodes.get(id).unwrap().peer_type}) { let pin = self.info.read().unwrap().deref().config.pin; - for n in self.nodes.read().unwrap().values().map(|n| NodeInfo { id: n.id.clone(), peer_type: n.peer_type }) { + for n in self.nodes.read().unwrap().nodes().map(|n| NodeInfo { id: n.id.clone(), peer_type: n.peer_type }) { let connected = self.have_session(&n.id) || self.connecting_to(&n.id); let required = n.peer_type == PeerType::Required; if connected && required { @@ -685,15 +681,39 @@ impl Host where Message: Send + Sync + Clone { h.disconnected(&NetworkContext::new(io, p, Some(token), self.connections.clone()), &token); } } + + fn update_nodes(&self, io: &IoContext>, node_changes: TableUpdates) { + let connections = self.connections.write().unwrap(); + let mut to_remove: Vec = Vec::new(); + for c in connections.iter() { + match *c.lock().unwrap().deref_mut() { + ConnectionEntry::Handshake(ref h) => { + if node_changes.removed.contains(&h.id) { + to_remove.push(h.token()); + } + } + ConnectionEntry::Session(ref s) => { + if node_changes.removed.contains(&s.id()) { + to_remove.push(s.token()); + } + } + } + } + for i in to_remove { + self.kill_connection(i, io); + } + self.nodes.write().unwrap().update(node_changes); + } } impl IoHandler> for Host where Message: Send + Sync + Clone + 'static { /// Initialize networking fn initialize(&self, io: &IoContext>) { io.register_stream(TCP_ACCEPT).expect("Error registering TCP listener"); - io.register_stream(NODETABLE_RECEIVE).expect("Error registering UDP listener"); + io.register_stream(DISCOVERY).expect("Error registering UDP listener"); io.register_timer(IDLE, MAINTENANCE_TIMEOUT).expect("Error registering Network idle timer"); - //io.register_timer(NODETABLE_MAINTAIN, 7200); + io.register_timer(DISCOVERY_REFRESH, 7200).expect("Error registering discovery timer"); + io.register_timer(DISCOVERY_ROUND, 300).expect("Error registering discovery timer"); } fn stream_hup(&self, io: &IoContext>, stream: StreamToken) { @@ -707,7 +727,11 @@ impl IoHandler> for Host where Messa fn stream_readable(&self, io: &IoContext>, stream: StreamToken) { match stream { FIRST_CONNECTION ... LAST_CONNECTION => self.connection_readable(stream, io), - NODETABLE_RECEIVE => {}, + DISCOVERY => { + if let Some(node_changes) = self.discovery.lock().unwrap().readable() { + self.update_nodes(io, node_changes); + } + }, TCP_ACCEPT => self.accept(io), _ => panic!("Received unknown readable token"), } @@ -716,7 +740,7 @@ impl IoHandler> for Host where Messa fn stream_writable(&self, io: &IoContext>, stream: StreamToken) { match stream { FIRST_CONNECTION ... LAST_CONNECTION => self.connection_writable(stream, io), - NODETABLE_RECEIVE => {}, + DISCOVERY => self.discovery.lock().unwrap().writable(), _ => panic!("Received unknown writable token"), } } @@ -725,8 +749,12 @@ impl IoHandler> for Host where Messa match token { IDLE => self.maintain_network(io), FIRST_CONNECTION ... LAST_CONNECTION => self.connection_timeout(token, io), - NODETABLE_DISCOVERY => {}, - NODETABLE_MAINTAIN => {}, + DISCOVERY_REFRESH => { + self.discovery.lock().unwrap().refresh(); + }, + DISCOVERY_ROUND => { + self.discovery.lock().unwrap().round(); + }, _ => match self.timers.read().unwrap().get(&token).cloned() { Some(timer) => match self.handlers.read().unwrap().get(timer.protocol).cloned() { None => { warn!(target: "net", "No handler found for protocol: {:?}", timer.protocol) }, @@ -794,7 +822,7 @@ impl IoHandler> for Host where Messa } } else {} // expired } - NODETABLE_RECEIVE => event_loop.register(self.udp_socket.lock().unwrap().deref(), Token(NODETABLE_RECEIVE), EventSet::all(), PollOpt::edge()).expect("Error registering stream"), + DISCOVERY => self.discovery.lock().unwrap().register_socket(event_loop).expect("Error registering discovery socket"), TCP_ACCEPT => event_loop.register(self.tcp_listener.lock().unwrap().deref(), Token(TCP_ACCEPT), EventSet::all(), PollOpt::edge()).expect("Error registering stream"), _ => warn!("Unexpected stream registration") } @@ -812,7 +840,7 @@ impl IoHandler> for Host where Messa connections.remove(stream); } }, - NODETABLE_RECEIVE => event_loop.deregister(self.udp_socket.lock().unwrap().deref()).unwrap(), + DISCOVERY => (), TCP_ACCEPT => event_loop.deregister(self.tcp_listener.lock().unwrap().deref()).unwrap(), _ => warn!("Unexpected stream deregistration") } @@ -828,7 +856,7 @@ impl IoHandler> for Host where Messa } } else {} // expired } - NODETABLE_RECEIVE => event_loop.reregister(self.udp_socket.lock().unwrap().deref(), Token(NODETABLE_RECEIVE), EventSet::all(), PollOpt::edge()).expect("Error reregistering stream"), + DISCOVERY => self.discovery.lock().unwrap().update_registration(event_loop).expect("Error reregistering discovery socket"), TCP_ACCEPT => event_loop.reregister(self.tcp_listener.lock().unwrap().deref(), Token(TCP_ACCEPT), EventSet::all(), PollOpt::edge()).expect("Error reregistering stream"), _ => warn!("Unexpected stream update") } diff --git a/util/src/network/mod.rs b/util/src/network/mod.rs index 6b58c87eb..e5465c952 100644 --- a/util/src/network/mod.rs +++ b/util/src/network/mod.rs @@ -72,6 +72,7 @@ mod discovery; mod service; mod error; mod node; +mod node_table; mod stats; #[cfg(test)] diff --git a/util/src/network/node_table.rs b/util/src/network/node_table.rs new file mode 100644 index 000000000..0f1c2c5ad --- /dev/null +++ b/util/src/network/node_table.rs @@ -0,0 +1,52 @@ +// Copyright 2015, 2016 Ethcore (UK) Ltd. +// This file is part of Parity. + +// Parity is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Parity is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Parity. If not, see . + +use std::collections::HashMap; +use std::collections::hash_map::Values; +use network::node::*; +use network::discovery::TableUpdates; + +pub struct NodeTable { + nodes: HashMap +} + +impl NodeTable { + pub fn new(_path: Option) -> NodeTable { + NodeTable { + nodes: HashMap::new() + } + } + + pub fn add_node(&mut self, node: Node) { + self.nodes.insert(node.id.clone(), node); + } + + pub fn nodes(&self) -> Values { + self.nodes.values() + } + + pub fn get_mut(&mut self, id: &NodeId) -> Option<&mut Node> { + self.nodes.get_mut(id) + } + + pub fn update(&mut self, mut update: TableUpdates) { + self.nodes.extend(update.added.drain()); + for r in update.removed { + self.nodes.remove(&r); + } + } + +} diff --git a/util/src/network/session.rs b/util/src/network/session.rs index b38807c49..19e2cf08e 100644 --- a/util/src/network/session.rs +++ b/util/src/network/session.rs @@ -129,6 +129,11 @@ impl Session { Ok(session) } + /// Get id of the remote peer + pub fn id(&self) -> &NodeId { + &self.info.id + } + /// Check if session is ready to send/receive data pub fn is_ready(&self) -> bool { self.had_hello From 62b9f4b91db181f0d93ee1d76e78dbd526f82e25 Mon Sep 17 00:00:00 2001 From: arkpar Date: Sat, 13 Feb 2016 22:57:39 +0100 Subject: [PATCH 02/61] UDP discovery working --- util/src/bytes.rs | 24 +- util/src/hash.rs | 6 - util/src/lib.rs | 1 + util/src/network/discovery.rs | 394 ++++++++++++++++++++++----------- util/src/network/error.rs | 2 + util/src/network/host.rs | 43 ++-- util/src/network/mod.rs | 2 + util/src/network/node.rs | 78 ++++++- util/src/network/node_table.rs | 5 +- 9 files changed, 375 insertions(+), 180 deletions(-) diff --git a/util/src/bytes.rs b/util/src/bytes.rs index 5ad2660e8..4923e6eb4 100644 --- a/util/src/bytes.rs +++ b/util/src/bytes.rs @@ -170,28 +170,8 @@ pub trait BytesConvertable { fn to_bytes(&self) -> Bytes { self.as_slice().to_vec() } } -impl<'a> BytesConvertable for &'a [u8] { - fn bytes(&self) -> &[u8] { self } -} - -impl BytesConvertable for Vec { - fn bytes(&self) -> &[u8] { self } -} - -macro_rules! impl_bytes_convertable_for_array { - ($zero: expr) => (); - ($len: expr, $($idx: expr),*) => { - impl BytesConvertable for [u8; $len] { - fn bytes(&self) -> &[u8] { self } - } - impl_bytes_convertable_for_array! { $($idx),* } - } -} - -// -1 at the end is not expanded -impl_bytes_convertable_for_array! { - 32, 31, 30, 29, 28, 27, 26, 25, 24, 23, 22, 21, 20, 19, 18, 17, 16, - 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0, -1 +impl BytesConvertable for T where T: Deref { + fn bytes(&self) -> &[u8] { self.deref() } } #[test] diff --git a/util/src/hash.rs b/util/src/hash.rs index 75c39720e..c678d13a7 100644 --- a/util/src/hash.rs +++ b/util/src/hash.rs @@ -77,12 +77,6 @@ macro_rules! impl_hash { /// Unformatted binary data of fixed length. pub struct $from (pub [u8; $size]); - impl BytesConvertable for $from { - fn bytes(&self) -> &[u8] { - &self.0 - } - } - impl Deref for $from { type Target = [u8]; diff --git a/util/src/lib.rs b/util/src/lib.rs index bdd595014..05162bca7 100644 --- a/util/src/lib.rs +++ b/util/src/lib.rs @@ -19,6 +19,7 @@ #![feature(augmented_assignments)] #![feature(associated_consts)] #![feature(plugin)] +#![feature(ip)] #![plugin(clippy)] #![allow(needless_range_loop, match_bool)] #![feature(catch_panic)] diff --git a/util/src/network/discovery.rs b/util/src/network/discovery.rs index da81920ff..a214f5278 100644 --- a/util/src/network/discovery.rs +++ b/util/src/network/discovery.rs @@ -17,24 +17,26 @@ use bytes::Bytes; use std::net::SocketAddr; use std::collections::{HashSet, HashMap, BTreeMap, VecDeque}; -use std::cell::{RefCell}; -use std::ops::{DerefMut}; use std::mem; +use std::cmp; use mio::*; use mio::udp::*; +use sha3::*; +use time; use hash::*; -use sha3::Hashable; use crypto::*; use rlp::*; use network::node::*; use network::error::NetworkError; use io::StreamToken; +use network::PROTOCOL_VERSION; + const ADDRESS_BYTES_SIZE: u32 = 32; // Size of address type in bytes. const ADDRESS_BITS: u32 = 8 * ADDRESS_BYTES_SIZE; // Denoted by n in [Kademlia]. const NODE_BINS: u32 = ADDRESS_BITS - 1; // Size of m_state (excludes root, which is us). const DISCOVERY_MAX_STEPS: u16 = 8; // Max iterations of discovery. (discover) -const BUCKET_SIZE: u32 = 16; // Denoted by k in [Kademlia]. Number of nodes stored in each bucket. +const BUCKET_SIZE: usize = 16; // Denoted by k in [Kademlia]. Number of nodes stored in each bucket. const ALPHA: usize = 3; // Denoted by \alpha in [Kademlia]. Number of concurrent FindNode requests. const MAX_DATAGRAM_SIZE: usize = 1280; @@ -43,16 +45,27 @@ const PACKET_PONG: u8 = 2; const PACKET_FIND_NODE: u8 = 3; const PACKET_NEIGHBOURS: u8 = 4; +const PING_TIMEOUT_MS: u64 = 300; + +#[derive(Clone, Debug)] +pub struct NodeEntry { + pub id: NodeId, + pub endpoint: NodeEndpoint, +} + +pub struct BucketEntry { + pub address: NodeEntry, + pub timeout: Option, +} + struct NodeBucket { - distance: u32, - nodes: Vec + nodes: VecDeque, //sorted by last active } impl NodeBucket { - fn new(distance: u32) -> NodeBucket { + fn new() -> NodeBucket { NodeBucket { - distance: distance, - nodes: Vec::new() + nodes: VecDeque::new() } } } @@ -64,6 +77,8 @@ struct Datagramm { pub struct Discovery { id: NodeId, + secret: Secret, + address: NodeEndpoint, udp_socket: UdpSocket, token: StreamToken, discovery_round: u16, @@ -74,80 +89,90 @@ pub struct Discovery { } pub struct TableUpdates { - pub added: HashMap, + pub added: HashMap, pub removed: HashSet, } -struct FindNodePacket; - -impl FindNodePacket { - fn new(_endpoint: &NodeEndpoint, _id: &NodeId) -> FindNodePacket { - FindNodePacket - } - - fn sign(&mut self, _secret: &Secret) { - } - - fn send(& self, _socket: &mut UdpSocket) { - } -} - impl Discovery { - pub fn new(id: &NodeId, address: &SocketAddr, token: StreamToken) -> Discovery { - let socket = UdpSocket::bound(address).expect("Error binding UDP socket"); + pub fn new(key: &KeyPair, address: NodeEndpoint, token: StreamToken) -> Discovery { + let socket = UdpSocket::bound(&address.udp_address()).expect("Error binding UDP socket"); Discovery { - id: id.clone(), + id: key.public().clone(), + secret: key.secret().clone(), + address: address, token: token, discovery_round: 0, discovery_id: NodeId::new(), discovery_nodes: HashSet::new(), - node_buckets: (0..NODE_BINS).map(NodeBucket::new).collect(), + node_buckets: (0..NODE_BINS).map(|_| NodeBucket::new()).collect(), udp_socket: socket, send_queue: VecDeque::new(), } } - pub fn add_node(&mut self, id: &NodeId) { - self.node_buckets[Discovery::distance(&self.id, &id) as usize].nodes.push(id.clone()); + pub fn add_node(&mut self, e: NodeEntry) { + let endpoint = e.endpoint.clone(); + self.update_node(e); + self.ping(&endpoint); } - fn start_node_discovery(&mut self, event_loop: &mut EventLoop) { + fn update_node(&mut self, e: NodeEntry) { + trace!(target: "discovery", "Inserting {:?}", &e); + let ping = { + let mut bucket = self.node_buckets.get_mut(Discovery::distance(&self.id, &e.id) as usize).unwrap(); + let updated = if let Some(node) = bucket.nodes.iter_mut().find(|n| n.address.id == e.id) { + node.address = e.clone(); + node.timeout = None; + true + } else { false }; + + if !updated { + bucket.nodes.push_front(BucketEntry { address: e, timeout: None }); + } + + if bucket.nodes.len() > BUCKET_SIZE { + //ping least active node + bucket.nodes.back_mut().unwrap().timeout = Some(time::precise_time_ns()); + Some(bucket.nodes.back().unwrap().address.endpoint.clone()) + } else { None } + }; + if let Some(endpoint) = ping { + self.ping(&endpoint); + } + } + + fn start(&mut self) { + trace!(target: "discovery", "Starting discovery"); self.discovery_round = 0; - self.discovery_id.randomize(); + self.discovery_id.randomize(); //TODO: use cryptographic nonce self.discovery_nodes.clear(); - self.discover(event_loop); } - fn discover(&mut self, event_loop: &mut EventLoop) { - if self.discovery_round == DISCOVERY_MAX_STEPS - { - debug!("Restarting discovery"); - self.start_node_discovery(event_loop); + fn discover(&mut self) { + if self.discovery_round == DISCOVERY_MAX_STEPS { return; } + trace!(target: "discovery", "Starting round {:?}", self.discovery_round); let mut tried_count = 0; { - let nearest = Discovery::nearest_node_entries(&self.id, &self.discovery_id, &self.node_buckets).into_iter(); - let nodes = RefCell::new(&mut self.discovery_nodes); - let nearest = nearest.filter(|x| nodes.borrow().contains(&x)).take(ALPHA); + let nearest = Discovery::nearest_node_entries(&self.discovery_id, &self.node_buckets).into_iter(); + let nearest = nearest.filter(|x| !self.discovery_nodes.contains(&x.id)).take(ALPHA).collect::>(); for r in nearest { - //let mut p = FindNodePacket::new(&r.endpoint, &self.discovery_id); - //p.sign(&self.secret); - //p.send(&mut self.udp_socket); - let mut borrowed = nodes.borrow_mut(); - borrowed.deref_mut().insert(r.clone()); + let rlp = encode(&(&[self.discovery_id.clone()][..])); + self.send_packet(PACKET_FIND_NODE, &r.endpoint.udp_address(), &rlp); + self.discovery_nodes.insert(r.id.clone()); tried_count += 1; + trace!(target: "discovery", "Sent FindNode to {:?}", &r.endpoint); } } - if tried_count == 0 - { - debug!("Restarting discovery"); - self.start_node_discovery(event_loop); + if tried_count == 0 { + trace!(target: "discovery", "Completing discovery"); + self.discovery_round = DISCOVERY_MAX_STEPS; + self.discovery_nodes.clear(); return; } self.discovery_round += 1; - //event_loop.timeout_ms(Token(NODETABLE_DISCOVERY), 1200).unwrap(); } fn distance(a: &NodeId, b: &NodeId) -> u32 { @@ -163,75 +188,75 @@ impl Discovery { ret } - #[allow(cyclomatic_complexity)] - fn nearest_node_entries<'b>(source: &NodeId, target: &NodeId, buckets: &'b [NodeBucket]) -> Vec<&'b NodeId> - { - // send ALPHA FindNode packets to nodes we know, closest to target - const LAST_BIN: u32 = NODE_BINS - 1; - let mut head = Discovery::distance(source, target); - let mut tail = if head == 0 { LAST_BIN } else { (head - 1) % NODE_BINS }; + fn ping(&mut self, node: &NodeEndpoint) { + let mut rlp = RlpStream::new_list(3); + rlp.append(&PROTOCOL_VERSION); + self.address.to_rlp_list(&mut rlp); + node.to_rlp_list(&mut rlp); + trace!(target: "discovery", "Sent Ping to {:?}", &node); + self.send_packet(PACKET_PING, &node.udp_address(), &rlp.drain()); + } - let mut found: BTreeMap> = BTreeMap::new(); + fn send_packet(&mut self, packet_id: u8, address: &SocketAddr, payload: &[u8]) { + let mut rlp = RlpStream::new(); + rlp.append_raw(&[packet_id], 1); + let source = Rlp::new(payload); + rlp.begin_list(source.item_count() + 1); + for i in 0 .. source.item_count() { + rlp.append_raw(source.at(i).as_raw(), 1); + } + let timestamp = time::get_time().sec as u32 + 60; + rlp.append(×tamp); + + let bytes = rlp.drain(); + let hash = bytes.sha3(); + let signature = match ec::sign(&self.secret, &hash) { + Ok(s) => s, + Err(_) => { + warn!("Error signing UDP packet"); + return; + } + }; + let mut packet = Bytes::with_capacity(bytes.len() + 32 + 65); + packet.extend(hash.iter()); + packet.extend(signature.iter()); + packet.extend(bytes.iter()); + let signed_hash = (&packet[32..]).sha3(); + packet[0..32].clone_from_slice(&signed_hash); + self.send_to(packet, address.clone()); + } + + #[allow(map_clone)] + fn nearest_node_entries(target: &NodeId, buckets: &[NodeBucket]) -> Vec + { + let mut found: BTreeMap> = BTreeMap::new(); let mut count = 0; - // if d is 0, then we roll look forward, if last, we reverse, else, spread from d - if head > 1 && tail != LAST_BIN { - while head != tail && head < NODE_BINS && count < BUCKET_SIZE { - for n in &buckets[head as usize].nodes { - if count < BUCKET_SIZE { - count += 1; - found.entry(Discovery::distance(target, &n)).or_insert_with(Vec::new).push(n); - } - else { break } - } - if count < BUCKET_SIZE && tail != 0 { - for n in &buckets[tail as usize].nodes { - if count < BUCKET_SIZE { - count += 1; - found.entry(Discovery::distance(target, &n)).or_insert_with(Vec::new).push(n); - } - else { break } + // Sort nodes by distance to target + for bucket in buckets { + for node in &bucket.nodes { + let distance = Discovery::distance(target, &node.address.id); + found.entry(distance).or_insert_with(Vec::new).push(&node.address); + if count == BUCKET_SIZE { + // delete the most distant element + let remove = { + let (_, last) = found.iter_mut().next_back().unwrap(); + last.pop(); + last.is_empty() + }; + if remove { + found.remove(&distance); } } - - head += 1; - if tail > 0 { - tail -= 1; + else { + count += 1; } } } - else if head < 2 { - while head < NODE_BINS && count < BUCKET_SIZE { - for n in &buckets[head as usize].nodes { - if count < BUCKET_SIZE { - count += 1; - found.entry(Discovery::distance(target, &n)).or_insert_with(Vec::new).push(n); - } - else { break } - } - head += 1; - } - } - else { - while tail > 0 && count < BUCKET_SIZE { - for n in &buckets[tail as usize].nodes { - if count < BUCKET_SIZE { - count += 1; - found.entry(Discovery::distance(target, &n)).or_insert_with(Vec::new).push(n); - } - else { break } - } - tail -= 1; - } - } - let mut ret:Vec<&NodeId> = Vec::new(); + let mut ret:Vec = Vec::new(); for (_, nodes) in found { - for n in nodes { - if ret.len() < BUCKET_SIZE as usize /* && n->endpoint && n->endpoint.isAllowed() */ { - ret.push(n); - } - } + ret.extend(nodes.iter().map(|&n| n.clone())); } ret } @@ -240,18 +265,22 @@ impl Discovery { if self.send_queue.is_empty() { return; } - let data = self.send_queue.pop_front().unwrap(); - match self.udp_socket.send_to(&data.payload, &data.address) { - Ok(Some(size)) if size == data.payload.len() => { - }, - Ok(Some(size)) => { - warn!("UDP sent incomplete datagramm"); - }, - Ok(None) => { - self.send_queue.push_front(data); - } - Err(e) => { - warn!("UDP sent error: {:?}", e); + while !self.send_queue.is_empty() { + let data = self.send_queue.pop_front().unwrap(); + match self.udp_socket.send_to(&data.payload, &data.address) { + Ok(Some(size)) if size == data.payload.len() => { + }, + Ok(Some(_)) => { + warn!("UDP sent incomplete datagramm"); + }, + Ok(None) => { + self.send_queue.push_front(data); + return; + } + Err(e) => { + warn!("UDP send error: {:?}, address: {:?}", e, &data.address); + return; + } } } } @@ -305,25 +334,132 @@ impl Discovery { } fn on_ping(&mut self, rlp: &UntrustedRlp, node: &NodeId, from: &SocketAddr) -> Result, NetworkError> { - Ok(None) + trace!(target: "discovery", "Got Ping from {:?}", &from); + let version: u32 = try!(rlp.val_at(0)); + if version != PROTOCOL_VERSION { + debug!(target: "discovery", "Unexpected protocol version: {}", version); + return Err(NetworkError::BadProtocol); + } + let source = try!(NodeEndpoint::from_rlp(&try!(rlp.at(1)))); + let dest = try!(NodeEndpoint::from_rlp(&try!(rlp.at(2)))); + let timestamp: u64 = try!(rlp.val_at(3)); + if timestamp < time::get_time().sec as u64{ + debug!(target: "discovery", "Expired ping"); + return Err(NetworkError::Expired); + } + let mut entry = NodeEntry { id: node.clone(), endpoint: source.clone() }; + if !entry.endpoint.is_valid() { + debug!(target: "discovery", "Bad address: {:?}", entry); + entry.endpoint.address = from.clone(); + } + self.update_node(entry.clone()); + let hash = rlp.as_raw().sha3(); + let mut response = RlpStream::new_list(2); + dest.to_rlp_list(&mut response); + response.append(&hash); + self.send_packet(PACKET_PONG, &entry.endpoint.udp_address(), &response.drain()); + + let mut added_map = HashMap::new(); + added_map.insert(node.clone(), entry); + Ok(Some(TableUpdates { added: added_map, removed: HashSet::new() })) } fn on_pong(&mut self, rlp: &UntrustedRlp, node: &NodeId, from: &SocketAddr) -> Result, NetworkError> { + trace!(target: "discovery", "Got Pong from {:?}", &from); + // TODO: validate pong packet + let dest = try!(NodeEndpoint::from_rlp(&try!(rlp.at(0)))); + let timestamp: u64 = try!(rlp.val_at(2)); + if timestamp > time::get_time().sec as u64 { + return Err(NetworkError::Expired); + } + let mut entry = NodeEntry { id: node.clone(), endpoint: dest }; + if !entry.endpoint.is_valid() { + debug!(target: "discovery", "Bad address: {:?}", entry); + entry.endpoint.address = from.clone(); + } + self.update_node(entry.clone()); + let mut added_map = HashMap::new(); + added_map.insert(node.clone(), entry); + Ok(Some(TableUpdates { added: added_map, removed: HashSet::new() })) + } + + fn on_find_node(&mut self, rlp: &UntrustedRlp, _node: &NodeId, from: &SocketAddr) -> Result, NetworkError> { + trace!(target: "discovery", "Got FindNode from {:?}", &from); + let target: NodeId = try!(rlp.val_at(0)); + let timestamp: u64 = try!(rlp.val_at(1)); + if timestamp > time::get_time().sec as u64 { + return Err(NetworkError::Expired); + } + + let limit = (MAX_DATAGRAM_SIZE - 109) / 90; + let nearest = Discovery::nearest_node_entries(&target, &self.node_buckets); + if nearest.is_empty() { + return Ok(None); + } + let mut rlp = RlpStream::new_list(cmp::min(limit, nearest.len())); + rlp.begin_list(1); + for n in 0 .. nearest.len() { + rlp.begin_list(4); + nearest[n].endpoint.to_rlp(&mut rlp); + rlp.append(&nearest[n].id); + if (n + 1) % limit == 0 || n == nearest.len() - 1 { + self.send_packet(PACKET_NEIGHBOURS, &from, &rlp.drain()); + trace!(target: "discovery", "Sent {} Neighbours to {:?}", n, &from); + rlp = RlpStream::new_list(cmp::min(limit, nearest.len() - n)); + rlp.begin_list(1); + } + } Ok(None) } - fn on_find_node(&mut self, rlp: &UntrustedRlp, node: &NodeId, from: &SocketAddr) -> Result, NetworkError> { - Ok(None) + fn on_neighbours(&mut self, rlp: &UntrustedRlp, _node: &NodeId, from: &SocketAddr) -> Result, NetworkError> { + // TODO: validate packet + let mut added = HashMap::new(); + trace!(target: "discovery", "Got {} Neighbours from {:?}", try!(rlp.at(0)).item_count(), &from); + for r in try!(rlp.at(0)).iter() { + let endpoint = try!(NodeEndpoint::from_rlp(&r)); + if !endpoint.is_valid() { + debug!(target: "discovery", "Bad address: {:?}", endpoint); + continue; + } + let node_id: NodeId = try!(r.val_at(3)); + let entry = NodeEntry { id: node_id.clone(), endpoint: endpoint }; + added.insert(node_id, entry.clone()); + self.update_node(entry); + } + Ok(Some(TableUpdates { added: added, removed: HashSet::new() })) } - fn on_neighbours(&mut self, rlp: &UntrustedRlp, node: &NodeId, from: &SocketAddr) -> Result, NetworkError> { - Ok(None) + fn check_expired(&mut self) -> HashSet { + let now = time::precise_time_ns(); + let mut removed: HashSet = HashSet::new(); + for bucket in &mut self.node_buckets { + bucket.nodes.retain(|node| { + if let Some(timeout) = node.timeout { + if now - timeout < PING_TIMEOUT_MS * 1000_0000 { + true + } + else { + trace!(target: "discovery", "Removed expired node {:?}", &node.address); + removed.insert(node.address.id.clone()); + false + } + } else { true } + }); + } + removed } - pub fn round(&mut self) { + pub fn round(&mut self) -> Option { + let removed = self.check_expired(); + self.discover(); + if !removed.is_empty() { + Some(TableUpdates { added: HashMap::new(), removed: removed }) + } else { None } } pub fn refresh(&mut self) { + self.start(); } pub fn register_socket(&self, event_loop: &mut EventLoop) -> Result<(), NetworkError> { @@ -334,7 +470,7 @@ impl Discovery { pub fn update_registration(&self, event_loop: &mut EventLoop) -> Result<(), NetworkError> { let mut registration = EventSet::readable(); if !self.send_queue.is_empty() { - registration &= EventSet::writable(); + registration = registration | EventSet::writable(); } event_loop.reregister(&self.udp_socket, Token(self.token), registration, PollOpt::edge()).expect("Error reregistering UDP socket"); Ok(()) diff --git a/util/src/network/error.rs b/util/src/network/error.rs index eb97e54b6..74babb110 100644 --- a/util/src/network/error.rs +++ b/util/src/network/error.rs @@ -42,6 +42,8 @@ pub enum NetworkError { Auth, /// Unrecognised protocol. BadProtocol, + /// Message expired. + Expired, /// Peer not found. PeerNotFound, /// Peer is diconnected. diff --git a/util/src/network/host.rs b/util/src/network/host.rs index 05462be37..47a3d9986 100644 --- a/util/src/network/host.rs +++ b/util/src/network/host.rs @@ -31,21 +31,18 @@ use network::handshake::Handshake; use network::session::{Session, SessionData}; use error::*; use io::*; -use network::NetworkProtocolHandler; +use network::{NetworkProtocolHandler, PROTOCOL_VERSION}; use network::node::*; use network::stats::NetworkStats; use network::error::DisconnectReason; use igd::{PortMappingProtocol,search_gateway}; -use network::discovery::{Discovery, TableUpdates}; +use network::discovery::{Discovery, TableUpdates, NodeEntry}; use network::node_table::NodeTable; type Slab = ::slab::Slab; const _DEFAULT_PORT: u16 = 30304; - const MAX_CONNECTIONS: usize = 1024; -const IDEAL_PEERS: u32 = 10; - const MAINTENANCE_TIMEOUT: u64 = 1000; #[derive(Debug)] @@ -67,6 +64,8 @@ pub struct NetworkConfiguration { pub boot_nodes: Vec, /// Use provided node key instead of default pub use_secret: Option, + /// Number of connected peers to maintain + pub ideal_peers: u32, } impl NetworkConfiguration { @@ -81,6 +80,7 @@ impl NetworkConfiguration { pin: false, boot_nodes: Vec::new(), use_secret: None, + ideal_peers: 10, } } @@ -126,6 +126,7 @@ impl NetworkConfiguration { pin: self.pin, boot_nodes: self.boot_nodes, use_secret: self.use_secret, + ideal_peers: self.ideal_peers, } } } @@ -343,19 +344,20 @@ impl Host where Message: Send + Sync + Clone { // Setup the server socket let tcp_listener = TcpListener::bind(&addr).unwrap(); let keys = if let Some(ref secret) = config.use_secret { KeyPair::from_secret(secret.clone()).unwrap() } else { KeyPair::create().unwrap() }; - let public = keys.public().clone(); + let endpoint = NodeEndpoint { address: addr.clone(), udp_port: addr.port() }; + let discovery = Discovery::new(&keys, endpoint, DISCOVERY); let path = config.config_path.clone(); let mut host = Host:: { info: RwLock::new(HostInfo { keys: keys, config: config, nonce: H256::random(), - protocol_version: 4, + protocol_version: PROTOCOL_VERSION, client_version: format!("Parity/{}/{}-{}-{}", env!("CARGO_PKG_VERSION"), Target::arch(), Target::env(), Target::os()), listen_port: 0, capabilities: Vec::new(), }), - discovery: Mutex::new(Discovery::new(&public, &addr, DISCOVERY)), + discovery: Mutex::new(discovery), tcp_listener: Mutex::new(tcp_listener), connections: Arc::new(RwLock::new(Slab::new_starting_at(FIRST_CONNECTION, MAX_CONNECTIONS))), nodes: RwLock::new(NodeTable::new(path)), @@ -382,7 +384,9 @@ impl Host where Message: Send + Sync + Clone { match Node::from_str(id) { Err(e) => { warn!("Could not add node: {:?}", e); }, Ok(n) => { + let entry = NodeEntry { endpoint: n.endpoint.clone(), id: n.id.clone() }; self.nodes.write().unwrap().add_node(n); + self.discovery.lock().unwrap().add_node(entry); } } } @@ -432,6 +436,7 @@ impl Host where Message: Send + Sync + Clone { let mut to_connect: Vec = Vec::new(); let mut req_conn = 0; let pin = self.info.read().unwrap().deref().config.pin; + let ideal_peers = self.info.read().unwrap().deref().config.ideal_peers; for n in self.nodes.read().unwrap().nodes().map(|n| NodeInfo { id: n.id.clone(), peer_type: n.peer_type }) { let connected = self.have_session(&n.id) || self.connecting_to(&n.id); let required = n.peer_type == PeerType::Required; @@ -445,7 +450,7 @@ impl Host where Message: Send + Sync + Clone { for n in &to_connect { if n.peer_type == PeerType::Required { - if req_conn < IDEAL_PEERS { + if req_conn < ideal_peers { self.connect_peer(&n.id, io); } req_conn += 1; @@ -455,7 +460,7 @@ impl Host where Message: Send + Sync + Clone { if !pin { let pending_count = 0; //TODO: let peer_count = 0; - let mut open_slots = IDEAL_PEERS - peer_count - pending_count + req_conn; + let mut open_slots = ideal_peers - peer_count - pending_count + req_conn; if open_slots > 0 { for n in &to_connect { if n.peer_type == PeerType::Optional && open_slots > 0 { @@ -471,11 +476,11 @@ impl Host where Message: Send + Sync + Clone { fn connect_peer(&self, id: &NodeId, io: &IoContext>) { if self.have_session(id) { - warn!("Aborted connect. Node already connected."); + debug!("Aborted connect. Node already connected."); return; } if self.connecting_to(id) { - warn!("Aborted connect. Node already connecting."); + debug!("Aborted connect. Node already connecting."); return; } @@ -689,7 +694,7 @@ impl Host where Message: Send + Sync + Clone { for c in connections.iter() { match *c.lock().unwrap().deref_mut() { ConnectionEntry::Handshake(ref h) => { - if node_changes.removed.contains(&h.id) { + if node_changes.removed.contains(&h.id()) { to_remove.push(h.token()); } } @@ -732,6 +737,7 @@ impl IoHandler> for Host where Messa if let Some(node_changes) = self.discovery.lock().unwrap().readable() { self.update_nodes(io, node_changes); } + io.update_registration(DISCOVERY).expect("Error updating disicovery registration"); }, TCP_ACCEPT => self.accept(io), _ => panic!("Received unknown readable token"), @@ -741,7 +747,10 @@ impl IoHandler> for Host where Messa fn stream_writable(&self, io: &IoContext>, stream: StreamToken) { match stream { FIRST_CONNECTION ... LAST_CONNECTION => self.connection_writable(stream, io), - DISCOVERY => self.discovery.lock().unwrap().writable(), + DISCOVERY => { + self.discovery.lock().unwrap().writable(); + io.update_registration(DISCOVERY).expect("Error updating disicovery registration"); + } _ => panic!("Received unknown writable token"), } } @@ -752,9 +761,13 @@ impl IoHandler> for Host where Messa FIRST_CONNECTION ... LAST_CONNECTION => self.connection_timeout(token, io), DISCOVERY_REFRESH => { self.discovery.lock().unwrap().refresh(); + io.update_registration(DISCOVERY).expect("Error updating disicovery registration"); }, DISCOVERY_ROUND => { - self.discovery.lock().unwrap().round(); + if let Some(node_changes) = self.discovery.lock().unwrap().round() { + self.update_nodes(io, node_changes); + } + io.update_registration(DISCOVERY).expect("Error updating disicovery registration"); }, _ => match self.timers.read().unwrap().get(&token).cloned() { Some(timer) => match self.handlers.read().unwrap().get(timer.protocol).cloned() { diff --git a/util/src/network/mod.rs b/util/src/network/mod.rs index e5465c952..466ef4e6a 100644 --- a/util/src/network/mod.rs +++ b/util/src/network/mod.rs @@ -90,6 +90,8 @@ pub use network::stats::NetworkStats; use io::TimerToken; +const PROTOCOL_VERSION: u32 = 4; + /// Network IO protocol handler. This needs to be implemented for each new subprotocol. /// All the handler function are called from within IO event loop. /// `Message` is the type for message data. diff --git a/util/src/network/node.rs b/util/src/network/node.rs index e23dee9f5..d8370bc79 100644 --- a/util/src/network/node.rs +++ b/util/src/network/node.rs @@ -14,7 +14,9 @@ // You should have received a copy of the GNU General Public License // along with Parity. If not, see . -use std::net::{SocketAddr, ToSocketAddrs}; +use std::mem; +use std::slice::from_raw_parts; +use std::net::{SocketAddr, ToSocketAddrs, SocketAddrV4, SocketAddrV6, Ipv4Addr, Ipv6Addr}; use std::hash::{Hash, Hasher}; use std::str::{FromStr}; use hash::*; @@ -25,17 +27,69 @@ use error::*; /// Node public key pub type NodeId = H512; -#[derive(Debug)] -/// Noe address info +#[derive(Debug, Clone)] +/// Node address info pub struct NodeEndpoint { /// IP(V4 or V6) address pub address: SocketAddr, - /// Address as string (can be host name). - pub address_str: String, /// Conneciton port. pub udp_port: u16 } +impl NodeEndpoint { + pub fn udp_address(&self) -> SocketAddr { + match self.address { + SocketAddr::V4(a) => SocketAddr::V4(SocketAddrV4::new(a.ip().clone(), self.udp_port)), + SocketAddr::V6(a) => SocketAddr::V6(SocketAddrV6::new(a.ip().clone(), self.udp_port, a.flowinfo(), a.scope_id())), + } + } +} + +impl NodeEndpoint { + pub fn from_rlp(rlp: &UntrustedRlp) -> Result { + let tcp_port = try!(rlp.val_at::(2)); + let udp_port = try!(rlp.val_at::(1)); + let addr_bytes = try!(try!(rlp.at(0)).data()); + let address = try!(match addr_bytes.len() { + 4 => Ok(SocketAddr::V4(SocketAddrV4::new(Ipv4Addr::new(addr_bytes[0], addr_bytes[1], addr_bytes[2], addr_bytes[3]), tcp_port))), + 16 => unsafe { + let o: *const u16 = mem::transmute(addr_bytes.as_ptr()); + let o = from_raw_parts(o, 8); + Ok(SocketAddr::V6(SocketAddrV6::new(Ipv6Addr::new(o[0], o[1], o[2], o[3], o[4], o[5], o[6], o[7]), tcp_port, 0, 0))) + }, + _ => Err(DecoderError::RlpInconsistentLengthAndData) + }); + Ok(NodeEndpoint { address: address, udp_port: udp_port }) + } + + pub fn to_rlp(&self, rlp: &mut RlpStream) { + match self.address { + SocketAddr::V4(a) => { + rlp.append(&(&a.ip().octets()[..])); + } + SocketAddr::V6(a) => unsafe { + let o: *const u8 = mem::transmute(a.ip().segments().as_ptr()); + rlp.append(&from_raw_parts(o, 16)); + } + }; + rlp.append(&self.udp_port); + rlp.append(&self.address.port()); + } + + pub fn to_rlp_list(&self, rlp: &mut RlpStream) { + rlp.begin_list(3); + self.to_rlp(rlp); + } + + pub fn is_valid(&self) -> bool { + self.udp_port != 0 && self.address.port() != 0 && + match self.address { + SocketAddr::V4(a) => !a.ip().is_unspecified(), + SocketAddr::V6(a) => !a.ip().is_unspecified() + } + } +} + impl FromStr for NodeEndpoint { type Err = UtilError; @@ -45,7 +99,6 @@ impl FromStr for NodeEndpoint { match address { Ok(Some(a)) => Ok(NodeEndpoint { address: a, - address_str: s.to_owned(), udp_port: a.port() }), Ok(_) => Err(UtilError::AddressResolve(None)), @@ -67,6 +120,17 @@ pub struct Node { pub last_attempted: Option, } +impl Node { + pub fn new(id: NodeId, endpoint: NodeEndpoint) -> Node { + Node { + id: id, + endpoint: endpoint, + peer_type: PeerType::Optional, + last_attempted: None, + } + } +} + impl FromStr for Node { type Err = UtilError; fn from_str(s: &str) -> Result { @@ -91,7 +155,7 @@ impl PartialEq for Node { self.id == other.id } } -impl Eq for Node { } +impl Eq for Node {} impl Hash for Node { fn hash(&self, state: &mut H) where H: Hasher { diff --git a/util/src/network/node_table.rs b/util/src/network/node_table.rs index 0f1c2c5ad..d93057eb3 100644 --- a/util/src/network/node_table.rs +++ b/util/src/network/node_table.rs @@ -43,7 +43,10 @@ impl NodeTable { } pub fn update(&mut self, mut update: TableUpdates) { - self.nodes.extend(update.added.drain()); + for (_, node) in update.added.drain() { + let mut entry = self.nodes.entry(node.id.clone()).or_insert_with(|| Node::new(node.id.clone(), node.endpoint.clone())); + entry.endpoint = node.endpoint; + } for r in update.removed { self.nodes.remove(&r); } From 76ea030b7817a3f3c21f4674c987d817de4d7c38 Mon Sep 17 00:00:00 2001 From: arkpar Date: Sun, 14 Feb 2016 01:03:48 +0100 Subject: [PATCH 03/61] Small refactoring --- sync/src/chain.rs | 6 +- util/src/network/discovery.rs | 6 +- util/src/network/handshake.rs | 2 +- util/src/network/host.rs | 106 ++++++++--------- util/src/network/mod.rs | 1 - util/src/network/node.rs | 198 -------------------------------- util/src/network/node_table.rs | 203 ++++++++++++++++++++++++++++++++- util/src/network/session.rs | 2 +- 8 files changed, 257 insertions(+), 267 deletions(-) delete mode 100644 util/src/network/node.rs diff --git a/sync/src/chain.rs b/sync/src/chain.rs index f82162b79..671e2241e 100644 --- a/sync/src/chain.rs +++ b/sync/src/chain.rs @@ -82,7 +82,7 @@ const RECEIPTS_PACKET: u8 = 0x10; const NETWORK_ID: U256 = ONE_U256; //TODO: get this from parent -const CONNECTION_TIMEOUT_SEC: f64 = 30f64; +const CONNECTION_TIMEOUT_SEC: f64 = 10f64; struct Header { /// Header data @@ -309,7 +309,7 @@ impl ChainSync { } self.peers.insert(peer_id.clone(), peer); - info!(target: "sync", "Connected {}:{}", peer_id, io.peer_info(peer_id)); + debug!(target: "sync", "Connected {}:{}", peer_id, io.peer_info(peer_id)); self.sync_peer(io, peer_id, false); Ok(()) } @@ -537,7 +537,7 @@ impl ChainSync { pub fn on_peer_aborting(&mut self, io: &mut SyncIo, peer: PeerId) { trace!(target: "sync", "== Disconnecting {}", peer); if self.peers.contains_key(&peer) { - info!(target: "sync", "Disconnected {}", peer); + debug!(target: "sync", "Disconnected {}", peer); self.clear_peer_download(peer); self.peers.remove(&peer); self.continue_sync(io); diff --git a/util/src/network/discovery.rs b/util/src/network/discovery.rs index a214f5278..9feef9c74 100644 --- a/util/src/network/discovery.rs +++ b/util/src/network/discovery.rs @@ -26,7 +26,7 @@ use time; use hash::*; use crypto::*; use rlp::*; -use network::node::*; +use network::node_table::*; use network::error::NetworkError; use io::StreamToken; @@ -227,8 +227,7 @@ impl Discovery { } #[allow(map_clone)] - fn nearest_node_entries(target: &NodeId, buckets: &[NodeBucket]) -> Vec - { + fn nearest_node_entries(target: &NodeId, buckets: &[NodeBucket]) -> Vec { let mut found: BTreeMap> = BTreeMap::new(); let mut count = 0; @@ -425,6 +424,7 @@ impl Discovery { let node_id: NodeId = try!(r.val_at(3)); let entry = NodeEntry { id: node_id.clone(), endpoint: endpoint }; added.insert(node_id, entry.clone()); + self.ping(&entry.endpoint); self.update_node(entry); } Ok(Some(TableUpdates { added: added, removed: HashSet::new() })) diff --git a/util/src/network/handshake.rs b/util/src/network/handshake.rs index 94650b2a7..1fd830ea0 100644 --- a/util/src/network/handshake.rs +++ b/util/src/network/handshake.rs @@ -24,7 +24,7 @@ use crypto::*; use crypto; use network::connection::{Connection}; use network::host::{HostInfo}; -use network::node::NodeId; +use network::node_table::NodeId; use error::*; use network::error::NetworkError; use network::stats::NetworkStats; diff --git a/util/src/network/host.rs b/util/src/network/host.rs index 47a3d9986..321e965ec 100644 --- a/util/src/network/host.rs +++ b/util/src/network/host.rs @@ -32,18 +32,18 @@ use network::session::{Session, SessionData}; use error::*; use io::*; use network::{NetworkProtocolHandler, PROTOCOL_VERSION}; -use network::node::*; +use network::node_table::*; use network::stats::NetworkStats; use network::error::DisconnectReason; use igd::{PortMappingProtocol,search_gateway}; use network::discovery::{Discovery, TableUpdates, NodeEntry}; -use network::node_table::NodeTable; type Slab = ::slab::Slab; const _DEFAULT_PORT: u16 = 30304; const MAX_CONNECTIONS: usize = 1024; const MAINTENANCE_TIMEOUT: u64 = 1000; +const MAX_HANDSHAKES: usize = 100; #[derive(Debug)] /// Network service configuration @@ -226,7 +226,7 @@ impl<'s, Message> NetworkContext<'s, Message> where Message: Send + Sync + Clone _ => warn!(target: "net", "Send: Peer is not connected yet") } } else { - warn!(target: "net", "Send: Peer does not exist") + trace!(target: "net", "Send: Peer no longer exist") } Ok(()) } @@ -405,11 +405,23 @@ impl Host where Message: Send + Sync + Clone { } fn have_session(&self, id: &NodeId) -> bool { - self.connections.read().unwrap().iter().any(|e| match *e.lock().unwrap().deref() { ConnectionEntry::Session(ref s) => s.info.id.eq(&id), _ => false }) + self.connections.read().unwrap().iter().any(|e| + match *e.lock().unwrap().deref() { ConnectionEntry::Session(ref s) => s.info.id.eq(&id), _ => false }) + } + + fn session_count(&self) -> usize { + self.connections.read().unwrap().iter().filter(|e| + match *e.lock().unwrap().deref() { ConnectionEntry::Session(_) => true, _ => false }).count() } fn connecting_to(&self, id: &NodeId) -> bool { - self.connections.read().unwrap().iter().any(|e| match *e.lock().unwrap().deref() { ConnectionEntry::Handshake(ref h) => h.id.eq(&id), _ => false }) + self.connections.read().unwrap().iter().any(|e| + match *e.lock().unwrap().deref() { ConnectionEntry::Handshake(ref h) => h.id.eq(&id), _ => false }) + } + + fn handshake_count(&self) -> usize { + self.connections.read().unwrap().iter().filter(|e| + match *e.lock().unwrap().deref() { ConnectionEntry::Handshake(_) => true, _ => false }).count() } fn keep_alive(&self, io: &IoContext>) { @@ -423,64 +435,40 @@ impl Host where Message: Send + Sync + Clone { } } for p in to_kill { - self.kill_connection(p, io); + self.kill_connection(p, io, true); } } fn connect_peers(&self, io: &IoContext>) { - struct NodeInfo { - id: NodeId, - peer_type: PeerType + let ideal_peers = { self.info.read().unwrap().deref().config.ideal_peers }; + let connections = self.session_count(); + if connections >= ideal_peers as usize { + return; } - let mut to_connect: Vec = Vec::new(); - let mut req_conn = 0; - let pin = self.info.read().unwrap().deref().config.pin; - let ideal_peers = self.info.read().unwrap().deref().config.ideal_peers; - for n in self.nodes.read().unwrap().nodes().map(|n| NodeInfo { id: n.id.clone(), peer_type: n.peer_type }) { - let connected = self.have_session(&n.id) || self.connecting_to(&n.id); - let required = n.peer_type == PeerType::Required; - if connected && required { - req_conn += 1; - } - else if !connected && (!pin || required) { - to_connect.push(n); - } + let handshake_count = self.handshake_count(); + if handshake_count >= MAX_HANDSHAKES { + return; } - for n in &to_connect { - if n.peer_type == PeerType::Required { - if req_conn < ideal_peers { - self.connect_peer(&n.id, io); - } - req_conn += 1; - } - } - if !pin { - let pending_count = 0; //TODO: - let peer_count = 0; - let mut open_slots = ideal_peers - peer_count - pending_count + req_conn; - if open_slots > 0 { - for n in &to_connect { - if n.peer_type == PeerType::Optional && open_slots > 0 { - open_slots -= 1; - self.connect_peer(&n.id, io); - } - } - } + let nodes = { self.nodes.read().unwrap().nodes() }; + + for id in nodes.iter().filter(|ref id| !self.have_session(id) && !self.connecting_to(id)).take(MAX_HANDSHAKES - handshake_count) { + self.connect_peer(&id, io); } + debug!(target: "net", "Connecting peers: {} sessions, {} pending", self.session_count(), self.handshake_count()); } #[allow(single_match)] fn connect_peer(&self, id: &NodeId, io: &IoContext>) { if self.have_session(id) { - debug!("Aborted connect. Node already connected."); + trace!("Aborted connect. Node already connected."); return; } if self.connecting_to(id) { - debug!("Aborted connect. Node already connecting."); + trace!("Aborted connect. Node already connecting."); return; } @@ -542,7 +530,7 @@ impl Host where Message: Send + Sync + Clone { ConnectionEntry::Handshake(ref mut h) => { match h.writable(io, &self.info.read().unwrap()) { Err(e) => { - debug!(target: "net", "Handshake write error: {:?}", e); + debug!(target: "net", "Handshake write error: {}:{:?}", token, e); kill = true; }, Ok(_) => () @@ -554,7 +542,7 @@ impl Host where Message: Send + Sync + Clone { ConnectionEntry::Session(ref mut s) => { match s.writable(io, &self.info.read().unwrap()) { Err(e) => { - debug!(target: "net", "Session write error: {:?}", e); + debug!(target: "net", "Session write error: {}:{:?}", token, e); kill = true; }, Ok(_) => () @@ -564,7 +552,7 @@ impl Host where Message: Send + Sync + Clone { } } if kill { - self.kill_connection(token, io); //TODO: mark connection as dead an check in kill_connection + self.kill_connection(token, io, true); //TODO: mark connection as dead an check in kill_connection return; } else if create_session { self.start_session(token, io); @@ -573,7 +561,7 @@ impl Host where Message: Send + Sync + Clone { } fn connection_closed(&self, token: TimerToken, io: &IoContext>) { - self.kill_connection(token, io); + self.kill_connection(token, io, true); } fn connection_readable(&self, token: StreamToken, io: &IoContext>) { @@ -585,7 +573,7 @@ impl Host where Message: Send + Sync + Clone { match *connection.lock().unwrap().deref_mut() { ConnectionEntry::Handshake(ref mut h) => { if let Err(e) = h.readable(io, &self.info.read().unwrap()) { - debug!(target: "net", "Handshake read error: {:?}", e); + debug!(target: "net", "Handshake read error: {}:{:?}", token, e); kill = true; } if h.done() { @@ -595,7 +583,7 @@ impl Host where Message: Send + Sync + Clone { ConnectionEntry::Session(ref mut s) => { match s.readable(io, &self.info.read().unwrap()) { Err(e) => { - debug!(target: "net", "Handshake read error: {:?}", e); + debug!(target: "net", "Handshake read error: {}:{:?}", token, e); kill = true; }, Ok(SessionData::Ready) => { @@ -621,7 +609,7 @@ impl Host where Message: Send + Sync + Clone { } } if kill { - self.kill_connection(token, io); //TODO: mark connection as dead an check in kill_connection + self.kill_connection(token, io, true); //TODO: mark connection as dead an check in kill_connection return; } else if create_session { self.start_session(token, io); @@ -657,17 +645,20 @@ impl Host where Message: Send + Sync + Clone { } fn connection_timeout(&self, token: StreamToken, io: &IoContext>) { - self.kill_connection(token, io) + self.kill_connection(token, io, true) } - fn kill_connection(&self, token: StreamToken, io: &IoContext>) { + fn kill_connection(&self, token: StreamToken, io: &IoContext>, remote: bool) { let mut to_disconnect: Vec = Vec::new(); { let mut connections = self.connections.write().unwrap(); if let Some(connection) = connections.get(token).cloned() { match *connection.lock().unwrap().deref_mut() { - ConnectionEntry::Handshake(_) => { + ConnectionEntry::Handshake(ref h) => { connections.remove(token); + if remote { + self.nodes.write().unwrap().note_failure(h.id()); + } }, ConnectionEntry::Session(ref mut s) if s.is_ready() => { for (p, _) in self.handlers.read().unwrap().iter() { @@ -676,6 +667,9 @@ impl Host where Message: Send + Sync + Clone { } } connections.remove(token); + if remote { + self.nodes.write().unwrap().note_failure(s.id()); + } }, _ => {}, } @@ -706,7 +700,7 @@ impl Host where Message: Send + Sync + Clone { } } for i in to_remove { - self.kill_connection(i, io); + self.kill_connection(i, io, false); } self.nodes.write().unwrap().update(node_changes); } @@ -816,7 +810,7 @@ impl IoHandler> for Host where Messa ConnectionEntry::Session(ref mut s) => { s.disconnect(DisconnectReason::DisconnectRequested); } } } - self.kill_connection(*peer, io); + self.kill_connection(*peer, io, false); }, NetworkIoMessage::User(ref message) => { for (p, h) in self.handlers.read().unwrap().iter() { diff --git a/util/src/network/mod.rs b/util/src/network/mod.rs index 466ef4e6a..7d5aac8f7 100644 --- a/util/src/network/mod.rs +++ b/util/src/network/mod.rs @@ -71,7 +71,6 @@ mod session; mod discovery; mod service; mod error; -mod node; mod node_table; mod stats; diff --git a/util/src/network/node.rs b/util/src/network/node.rs deleted file mode 100644 index d8370bc79..000000000 --- a/util/src/network/node.rs +++ /dev/null @@ -1,198 +0,0 @@ -// Copyright 2015, 2016 Ethcore (UK) Ltd. -// This file is part of Parity. - -// Parity is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// Parity is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. - -// You should have received a copy of the GNU General Public License -// along with Parity. If not, see . - -use std::mem; -use std::slice::from_raw_parts; -use std::net::{SocketAddr, ToSocketAddrs, SocketAddrV4, SocketAddrV6, Ipv4Addr, Ipv6Addr}; -use std::hash::{Hash, Hasher}; -use std::str::{FromStr}; -use hash::*; -use rlp::*; -use time::Tm; -use error::*; - -/// Node public key -pub type NodeId = H512; - -#[derive(Debug, Clone)] -/// Node address info -pub struct NodeEndpoint { - /// IP(V4 or V6) address - pub address: SocketAddr, - /// Conneciton port. - pub udp_port: u16 -} - -impl NodeEndpoint { - pub fn udp_address(&self) -> SocketAddr { - match self.address { - SocketAddr::V4(a) => SocketAddr::V4(SocketAddrV4::new(a.ip().clone(), self.udp_port)), - SocketAddr::V6(a) => SocketAddr::V6(SocketAddrV6::new(a.ip().clone(), self.udp_port, a.flowinfo(), a.scope_id())), - } - } -} - -impl NodeEndpoint { - pub fn from_rlp(rlp: &UntrustedRlp) -> Result { - let tcp_port = try!(rlp.val_at::(2)); - let udp_port = try!(rlp.val_at::(1)); - let addr_bytes = try!(try!(rlp.at(0)).data()); - let address = try!(match addr_bytes.len() { - 4 => Ok(SocketAddr::V4(SocketAddrV4::new(Ipv4Addr::new(addr_bytes[0], addr_bytes[1], addr_bytes[2], addr_bytes[3]), tcp_port))), - 16 => unsafe { - let o: *const u16 = mem::transmute(addr_bytes.as_ptr()); - let o = from_raw_parts(o, 8); - Ok(SocketAddr::V6(SocketAddrV6::new(Ipv6Addr::new(o[0], o[1], o[2], o[3], o[4], o[5], o[6], o[7]), tcp_port, 0, 0))) - }, - _ => Err(DecoderError::RlpInconsistentLengthAndData) - }); - Ok(NodeEndpoint { address: address, udp_port: udp_port }) - } - - pub fn to_rlp(&self, rlp: &mut RlpStream) { - match self.address { - SocketAddr::V4(a) => { - rlp.append(&(&a.ip().octets()[..])); - } - SocketAddr::V6(a) => unsafe { - let o: *const u8 = mem::transmute(a.ip().segments().as_ptr()); - rlp.append(&from_raw_parts(o, 16)); - } - }; - rlp.append(&self.udp_port); - rlp.append(&self.address.port()); - } - - pub fn to_rlp_list(&self, rlp: &mut RlpStream) { - rlp.begin_list(3); - self.to_rlp(rlp); - } - - pub fn is_valid(&self) -> bool { - self.udp_port != 0 && self.address.port() != 0 && - match self.address { - SocketAddr::V4(a) => !a.ip().is_unspecified(), - SocketAddr::V6(a) => !a.ip().is_unspecified() - } - } -} - -impl FromStr for NodeEndpoint { - type Err = UtilError; - - /// Create endpoint from string. Performs name resolution if given a host name. - fn from_str(s: &str) -> Result { - let address = s.to_socket_addrs().map(|mut i| i.next()); - match address { - Ok(Some(a)) => Ok(NodeEndpoint { - address: a, - udp_port: a.port() - }), - Ok(_) => Err(UtilError::AddressResolve(None)), - Err(e) => Err(UtilError::AddressResolve(Some(e))) - } - } -} - -#[derive(PartialEq, Eq, Copy, Clone)] -pub enum PeerType { - Required, - Optional -} - -pub struct Node { - pub id: NodeId, - pub endpoint: NodeEndpoint, - pub peer_type: PeerType, - pub last_attempted: Option, -} - -impl Node { - pub fn new(id: NodeId, endpoint: NodeEndpoint) -> Node { - Node { - id: id, - endpoint: endpoint, - peer_type: PeerType::Optional, - last_attempted: None, - } - } -} - -impl FromStr for Node { - type Err = UtilError; - fn from_str(s: &str) -> Result { - let (id, endpoint) = if &s[0..8] == "enode://" && s.len() > 136 && &s[136..137] == "@" { - (try!(NodeId::from_str(&s[8..136])), try!(NodeEndpoint::from_str(&s[137..]))) - } - else { - (NodeId::new(), try!(NodeEndpoint::from_str(s))) - }; - - Ok(Node { - id: id, - endpoint: endpoint, - peer_type: PeerType::Optional, - last_attempted: None, - }) - } -} - -impl PartialEq for Node { - fn eq(&self, other: &Self) -> bool { - self.id == other.id - } -} -impl Eq for Node {} - -impl Hash for Node { - fn hash(&self, state: &mut H) where H: Hasher { - self.id.hash(state) - } -} - -#[cfg(test)] -mod tests { - use super::*; - use std::str::FromStr; - use std::net::*; - use hash::*; - - #[test] - fn endpoint_parse() { - let endpoint = NodeEndpoint::from_str("123.99.55.44:7770"); - assert!(endpoint.is_ok()); - let v4 = match endpoint.unwrap().address { - SocketAddr::V4(v4address) => v4address, - _ => panic!("should ve v4 address") - }; - assert_eq!(SocketAddrV4::new(Ipv4Addr::new(123, 99, 55, 44), 7770), v4); - } - - #[test] - fn node_parse() { - let node = Node::from_str("enode://a979fb575495b8d6db44f750317d0f4622bf4c2aa3365d6af7c284339968eef29b69ad0dce72a4d8db5ebb4968de0e3bec910127f134779fbcb0cb6d3331163c@22.99.55.44:7770"); - assert!(node.is_ok()); - let node = node.unwrap(); - let v4 = match node.endpoint.address { - SocketAddr::V4(v4address) => v4address, - _ => panic!("should ve v4 address") - }; - assert_eq!(SocketAddrV4::new(Ipv4Addr::new(22, 99, 55, 44), 7770), v4); - assert_eq!( - H512::from_str("a979fb575495b8d6db44f750317d0f4622bf4c2aa3365d6af7c284339968eef29b69ad0dce72a4d8db5ebb4968de0e3bec910127f134779fbcb0cb6d3331163c").unwrap(), - node.id); - } -} diff --git a/util/src/network/node_table.rs b/util/src/network/node_table.rs index d93057eb3..a3ee57481 100644 --- a/util/src/network/node_table.rs +++ b/util/src/network/node_table.rs @@ -14,11 +14,161 @@ // You should have received a copy of the GNU General Public License // along with Parity. If not, see . +use std::mem; +use std::slice::from_raw_parts; +use std::net::{SocketAddr, ToSocketAddrs, SocketAddrV4, SocketAddrV6, Ipv4Addr, Ipv6Addr}; +use std::hash::{Hash, Hasher}; +use std::str::{FromStr}; use std::collections::HashMap; -use std::collections::hash_map::Values; -use network::node::*; +use hash::*; +use rlp::*; +use time::Tm; +use error::*; use network::discovery::TableUpdates; +/// Node public key +pub type NodeId = H512; + +#[derive(Debug, Clone)] +/// Node address info +pub struct NodeEndpoint { + /// IP(V4 or V6) address + pub address: SocketAddr, + /// Conneciton port. + pub udp_port: u16 +} + +impl NodeEndpoint { + pub fn udp_address(&self) -> SocketAddr { + match self.address { + SocketAddr::V4(a) => SocketAddr::V4(SocketAddrV4::new(a.ip().clone(), self.udp_port)), + SocketAddr::V6(a) => SocketAddr::V6(SocketAddrV6::new(a.ip().clone(), self.udp_port, a.flowinfo(), a.scope_id())), + } + } +} + +impl NodeEndpoint { + pub fn from_rlp(rlp: &UntrustedRlp) -> Result { + let tcp_port = try!(rlp.val_at::(2)); + let udp_port = try!(rlp.val_at::(1)); + let addr_bytes = try!(try!(rlp.at(0)).data()); + let address = try!(match addr_bytes.len() { + 4 => Ok(SocketAddr::V4(SocketAddrV4::new(Ipv4Addr::new(addr_bytes[0], addr_bytes[1], addr_bytes[2], addr_bytes[3]), tcp_port))), + 16 => unsafe { + let o: *const u16 = mem::transmute(addr_bytes.as_ptr()); + let o = from_raw_parts(o, 8); + Ok(SocketAddr::V6(SocketAddrV6::new(Ipv6Addr::new(o[0], o[1], o[2], o[3], o[4], o[5], o[6], o[7]), tcp_port, 0, 0))) + }, + _ => Err(DecoderError::RlpInconsistentLengthAndData) + }); + Ok(NodeEndpoint { address: address, udp_port: udp_port }) + } + + pub fn to_rlp(&self, rlp: &mut RlpStream) { + match self.address { + SocketAddr::V4(a) => { + rlp.append(&(&a.ip().octets()[..])); + } + SocketAddr::V6(a) => unsafe { + let o: *const u8 = mem::transmute(a.ip().segments().as_ptr()); + rlp.append(&from_raw_parts(o, 16)); + } + }; + rlp.append(&self.udp_port); + rlp.append(&self.address.port()); + } + + pub fn to_rlp_list(&self, rlp: &mut RlpStream) { + rlp.begin_list(3); + self.to_rlp(rlp); + } + + pub fn is_valid(&self) -> bool { + self.udp_port != 0 && self.address.port() != 0 && + match self.address { + SocketAddr::V4(a) => !a.ip().is_unspecified(), + SocketAddr::V6(a) => !a.ip().is_unspecified() + } + } +} + +impl FromStr for NodeEndpoint { + type Err = UtilError; + + /// Create endpoint from string. Performs name resolution if given a host name. + fn from_str(s: &str) -> Result { + let address = s.to_socket_addrs().map(|mut i| i.next()); + match address { + Ok(Some(a)) => Ok(NodeEndpoint { + address: a, + udp_port: a.port() + }), + Ok(_) => Err(UtilError::AddressResolve(None)), + Err(e) => Err(UtilError::AddressResolve(Some(e))) + } + } +} + +#[derive(PartialEq, Eq, Copy, Clone)] +pub enum PeerType { + _Required, + Optional +} + +pub struct Node { + pub id: NodeId, + pub endpoint: NodeEndpoint, + pub peer_type: PeerType, + pub failures: u32, + pub last_attempted: Option, +} + +impl Node { + pub fn new(id: NodeId, endpoint: NodeEndpoint) -> Node { + Node { + id: id, + endpoint: endpoint, + peer_type: PeerType::Optional, + failures: 0, + last_attempted: None, + } + } +} + +impl FromStr for Node { + type Err = UtilError; + fn from_str(s: &str) -> Result { + let (id, endpoint) = if &s[0..8] == "enode://" && s.len() > 136 && &s[136..137] == "@" { + (try!(NodeId::from_str(&s[8..136])), try!(NodeEndpoint::from_str(&s[137..]))) + } + else { + (NodeId::new(), try!(NodeEndpoint::from_str(s))) + }; + + Ok(Node { + id: id, + endpoint: endpoint, + peer_type: PeerType::Optional, + last_attempted: None, + failures: 0, + }) + } +} + +impl PartialEq for Node { + fn eq(&self, other: &Self) -> bool { + self.id == other.id + } +} +impl Eq for Node {} + +impl Hash for Node { + fn hash(&self, state: &mut H) where H: Hasher { + self.id.hash(state) + } +} + +/// Node table backed by disk file. pub struct NodeTable { nodes: HashMap } @@ -30,18 +180,24 @@ impl NodeTable { } } + /// Add a node to table pub fn add_node(&mut self, node: Node) { self.nodes.insert(node.id.clone(), node); } - pub fn nodes(&self) -> Values { - self.nodes.values() + /// Returns node ids sorted by number of failures + pub fn nodes(&self) -> Vec { + let mut refs: Vec<&Node> = self.nodes.values().collect(); + refs.sort_by(|a, b| a.failures.cmp(&b.failures)); + refs.iter().map(|n| n.id.clone()).collect() } + /// Get particular node pub fn get_mut(&mut self, id: &NodeId) -> Option<&mut Node> { self.nodes.get_mut(id) } + /// Apply table changes coming from discovery pub fn update(&mut self, mut update: TableUpdates) { for (_, node) in update.added.drain() { let mut entry = self.nodes.entry(node.id.clone()).or_insert_with(|| Node::new(node.id.clone(), node.endpoint.clone())); @@ -52,4 +208,43 @@ impl NodeTable { } } + pub fn note_failure(&mut self, id: &NodeId) { + if let Some(node) = self.nodes.get_mut(id) { + node.failures += 1; + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::str::FromStr; + use std::net::*; + use hash::*; + + #[test] + fn endpoint_parse() { + let endpoint = NodeEndpoint::from_str("123.99.55.44:7770"); + assert!(endpoint.is_ok()); + let v4 = match endpoint.unwrap().address { + SocketAddr::V4(v4address) => v4address, + _ => panic!("should ve v4 address") + }; + assert_eq!(SocketAddrV4::new(Ipv4Addr::new(123, 99, 55, 44), 7770), v4); + } + + #[test] + fn node_parse() { + let node = Node::from_str("enode://a979fb575495b8d6db44f750317d0f4622bf4c2aa3365d6af7c284339968eef29b69ad0dce72a4d8db5ebb4968de0e3bec910127f134779fbcb0cb6d3331163c@22.99.55.44:7770"); + assert!(node.is_ok()); + let node = node.unwrap(); + let v4 = match node.endpoint.address { + SocketAddr::V4(v4address) => v4address, + _ => panic!("should ve v4 address") + }; + assert_eq!(SocketAddrV4::new(Ipv4Addr::new(22, 99, 55, 44), 7770), v4); + assert_eq!( + H512::from_str("a979fb575495b8d6db44f750317d0f4622bf4c2aa3365d6af7c284339968eef29b69ad0dce72a4d8db5ebb4968de0e3bec910127f134779fbcb0cb6d3331163c").unwrap(), + node.id); + } } diff --git a/util/src/network/session.rs b/util/src/network/session.rs index 19e2cf08e..3b49a8f5e 100644 --- a/util/src/network/session.rs +++ b/util/src/network/session.rs @@ -23,7 +23,7 @@ use error::*; use io::{IoContext, StreamToken}; use network::error::{NetworkError, DisconnectReason}; use network::host::*; -use network::node::NodeId; +use network::node_table::NodeId; use time; const PING_TIMEOUT_SEC: u64 = 30; From 9768fddb19e76a82bbee4c071d6fd4b88e181fe7 Mon Sep 17 00:00:00 2001 From: arkpar Date: Sun, 14 Feb 2016 01:05:54 +0100 Subject: [PATCH 04/61] Homestead block set to 1100000 --- ethcore/res/ethereum/frontier.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ethcore/res/ethereum/frontier.json b/ethcore/res/ethereum/frontier.json index 301441958..6e31a2fce 100644 --- a/ethcore/res/ethereum/frontier.json +++ b/ethcore/res/ethereum/frontier.json @@ -3,7 +3,7 @@ "engineName": "Ethash", "params": { "accountStartNonce": "0x00", - "frontierCompatibilityModeLimit": "0xf4240", + "frontierCompatibilityModeLimit": "0x10c8e0", "maximumExtraDataSize": "0x20", "tieBreakingGas": false, "minGasLimit": "0x1388", From 2d89708ea8cc034bc625df68c02ff6511fbfce90 Mon Sep 17 00:00:00 2001 From: arkpar Date: Sun, 14 Feb 2016 02:11:55 +0100 Subject: [PATCH 05/61] Reduced thread contention --- util/src/network/host.rs | 91 ++++++++++++++++++++++++---------------- 1 file changed, 55 insertions(+), 36 deletions(-) diff --git a/util/src/network/host.rs b/util/src/network/host.rs index 321e965ec..5c08ad5c8 100644 --- a/util/src/network/host.rs +++ b/util/src/network/host.rs @@ -215,7 +215,8 @@ impl<'s, Message> NetworkContext<'s, Message> where Message: Send + Sync + Clone /// Send a packet over the network to another peer. pub fn send(&self, peer: PeerId, packet_id: PacketId, data: Vec) -> Result<(), UtilError> { - if let Some(connection) = self.connections.read().unwrap().get(peer).cloned() { + let connection = { self.connections.read().unwrap().get(peer).cloned() }; + if let Some(connection) = connection { match *connection.lock().unwrap().deref_mut() { ConnectionEntry::Session(ref mut s) => { s.send_packet(self.protocol, packet_id as u8, &data).unwrap_or_else(|e| { @@ -264,7 +265,8 @@ impl<'s, Message> NetworkContext<'s, Message> where Message: Send + Sync + Clone /// Returns peer identification string pub fn peer_info(&self, peer: PeerId) -> String { - if let Some(connection) = self.connections.read().unwrap().get(peer).cloned() { + let connection = { self.connections.read().unwrap().get(peer).cloned() }; + if let Some(connection) = connection { if let ConnectionEntry::Session(ref s) = *connection.lock().unwrap().deref() { return s.info.client_version.clone() } @@ -525,7 +527,8 @@ impl Host where Message: Send + Sync + Clone { fn connection_writable(&self, token: StreamToken, io: &IoContext>) { let mut create_session = false; let mut kill = false; - if let Some(connection) = self.connections.read().unwrap().get(token).cloned() { + let connection = { self.connections.read().unwrap().get(token).cloned() }; + if let Some(connection) = connection { match *connection.lock().unwrap().deref_mut() { ConnectionEntry::Handshake(ref mut h) => { match h.writable(io, &self.info.read().unwrap()) { @@ -569,7 +572,8 @@ impl Host where Message: Send + Sync + Clone { let mut packet_data: Option<(ProtocolId, PacketId, Vec)> = None; let mut create_session = false; let mut kill = false; - if let Some(connection) = self.connections.read().unwrap().get(token).cloned() { + let connection = { self.connections.read().unwrap().get(token).cloned() }; + if let Some(connection) = connection { match *connection.lock().unwrap().deref_mut() { ConnectionEntry::Handshake(ref mut h) => { if let Err(e) = h.readable(io, &self.info.read().unwrap()) { @@ -628,20 +632,28 @@ impl Host where Message: Send + Sync + Clone { fn start_session(&self, token: StreamToken, io: &IoContext>) { let mut connections = self.connections.write().unwrap(); - if connections.get(token).is_none() { - return; // handshake expired + let replace = { + let connection = { connections.get(token).cloned() }; + if let Some(connection) = connection { + match *connection.lock().unwrap().deref_mut() { + ConnectionEntry::Handshake(_) => true, + _ => false, + } + } else { false } + }; + if replace { + connections.replace_with(token, |c| { + match Arc::try_unwrap(c).ok().unwrap().into_inner().unwrap() { + ConnectionEntry::Handshake(h) => { + let session = Session::new(h, io, &self.info.read().unwrap()).expect("Session creation error"); + io.update_registration(token).expect("Error updating session registration"); + self.stats.inc_sessions(); + Some(Arc::new(Mutex::new(ConnectionEntry::Session(session)))) + }, + _ => { None } // handshake expired + } + }).ok(); } - connections.replace_with(token, |c| { - match Arc::try_unwrap(c).ok().unwrap().into_inner().unwrap() { - ConnectionEntry::Handshake(h) => { - let session = Session::new(h, io, &self.info.read().unwrap()).expect("Session creation error"); - io.update_registration(token).expect("Error updating session registration"); - self.stats.inc_sessions(); - Some(Arc::new(Mutex::new(ConnectionEntry::Session(session)))) - }, - _ => { None } // handshake expired - } - }).ok(); } fn connection_timeout(&self, token: StreamToken, io: &IoContext>) { @@ -650,15 +662,14 @@ impl Host where Message: Send + Sync + Clone { fn kill_connection(&self, token: StreamToken, io: &IoContext>, remote: bool) { let mut to_disconnect: Vec = Vec::new(); + let mut failure_id = None; { let mut connections = self.connections.write().unwrap(); if let Some(connection) = connections.get(token).cloned() { match *connection.lock().unwrap().deref_mut() { ConnectionEntry::Handshake(ref h) => { connections.remove(token); - if remote { - self.nodes.write().unwrap().note_failure(h.id()); - } + failure_id = Some(h.id().clone()); }, ConnectionEntry::Session(ref mut s) if s.is_ready() => { for (p, _) in self.handlers.read().unwrap().iter() { @@ -667,15 +678,18 @@ impl Host where Message: Send + Sync + Clone { } } connections.remove(token); - if remote { - self.nodes.write().unwrap().note_failure(s.id()); - } + failure_id = Some(s.id().clone()); }, _ => {}, } } io.deregister_stream(token).expect("Error deregistering stream"); } + if let Some(id) = failure_id { + if remote { + self.nodes.write().unwrap().note_failure(&id); + } + } for p in to_disconnect { let h = self.handlers.read().unwrap().get(p).unwrap().clone(); h.disconnected(&NetworkContext::new(io, p, Some(token), self.connections.clone()), &token); @@ -683,18 +697,20 @@ impl Host where Message: Send + Sync + Clone { } fn update_nodes(&self, io: &IoContext>, node_changes: TableUpdates) { - let connections = self.connections.write().unwrap(); let mut to_remove: Vec = Vec::new(); - for c in connections.iter() { - match *c.lock().unwrap().deref_mut() { - ConnectionEntry::Handshake(ref h) => { - if node_changes.removed.contains(&h.id()) { - to_remove.push(h.token()); + { + let connections = self.connections.write().unwrap(); + for c in connections.iter() { + match *c.lock().unwrap().deref_mut() { + ConnectionEntry::Handshake(ref h) => { + if node_changes.removed.contains(&h.id()) { + to_remove.push(h.token()); + } } - } - ConnectionEntry::Session(ref s) => { - if node_changes.removed.contains(&s.id()) { - to_remove.push(s.token()); + ConnectionEntry::Session(ref s) => { + if node_changes.removed.contains(&s.id()) { + to_remove.push(s.token()); + } } } } @@ -804,7 +820,8 @@ impl IoHandler> for Host where Messa io.register_timer(handler_token, *delay).expect("Error registering timer"); }, NetworkIoMessage::Disconnect(ref peer) => { - if let Some(connection) = self.connections.read().unwrap().get(*peer).cloned() { + let connection = { self.connections.read().unwrap().get(*peer).cloned() }; + if let Some(connection) = connection { match *connection.lock().unwrap().deref_mut() { ConnectionEntry::Handshake(_) => {}, ConnectionEntry::Session(ref mut s) => { s.disconnect(DisconnectReason::DisconnectRequested); } @@ -823,7 +840,8 @@ impl IoHandler> for Host where Messa fn register_stream(&self, stream: StreamToken, reg: Token, event_loop: &mut EventLoop>>) { match stream { FIRST_CONNECTION ... LAST_CONNECTION => { - if let Some(connection) = self.connections.read().unwrap().get(stream).cloned() { + let connection = { self.connections.read().unwrap().get(stream).cloned() }; + if let Some(connection) = connection { match *connection.lock().unwrap().deref() { ConnectionEntry::Handshake(ref h) => h.register_socket(reg, event_loop).expect("Error registering socket"), ConnectionEntry::Session(_) => warn!("Unexpected session stream registration") @@ -857,7 +875,8 @@ impl IoHandler> for Host where Messa fn update_stream(&self, stream: StreamToken, reg: Token, event_loop: &mut EventLoop>>) { match stream { FIRST_CONNECTION ... LAST_CONNECTION => { - if let Some(connection) = self.connections.read().unwrap().get(stream).cloned() { + let connection = { self.connections.read().unwrap().get(stream).cloned() }; + if let Some(connection) = connection { match *connection.lock().unwrap().deref() { ConnectionEntry::Handshake(ref h) => h.update_socket(reg, event_loop).expect("Error updating socket"), ConnectionEntry::Session(ref s) => s.update_socket(reg, event_loop).expect("Error updating socket"), From 718646f943b1388fb68afa95f5e79aca13e53a93 Mon Sep 17 00:00:00 2001 From: arkpar Date: Sun, 14 Feb 2016 11:34:59 +0100 Subject: [PATCH 06/61] Refactored host to use different containers for handshakes and sessions --- util/src/network/connection.rs | 10 + util/src/network/host.rs | 435 +++++++++++++++++---------------- util/src/network/session.rs | 3 +- 3 files changed, 233 insertions(+), 215 deletions(-) diff --git a/util/src/network/connection.rs b/util/src/network/connection.rs index 44d429164..242e8935e 100644 --- a/util/src/network/connection.rs +++ b/util/src/network/connection.rs @@ -164,6 +164,11 @@ impl Connection { self.token } + /// Replace socket token + pub fn set_token(&mut self, token: StreamToken) { + self.token = token; + } + /// Register this connection with the IO event loop. pub fn register_socket(&self, reg: Token, event_loop: &mut EventLoop) -> io::Result<()> { trace!(target: "net", "connection register; token={:?}", reg); @@ -243,6 +248,11 @@ impl EncryptedConnection { self.connection.token } + /// Replace socket token + pub fn set_token(&mut self, token: StreamToken) { + self.connection.set_token(token); + } + /// Create an encrypted connection out of the handshake. Consumes a handshake object. pub fn new(mut handshake: Handshake) -> Result { let shared = try!(crypto::ecdh::agree(handshake.ecdhe.secret(), &handshake.remote_public)); diff --git a/util/src/network/host.rs b/util/src/network/host.rs index 5c08ad5c8..04dda02bd 100644 --- a/util/src/network/host.rs +++ b/util/src/network/host.rs @@ -20,6 +20,7 @@ use std::hash::{Hasher}; use std::str::{FromStr}; use std::sync::*; use std::ops::*; +use std::cmp::min; use mio::*; use mio::tcp::*; use target_info::Target; @@ -41,9 +42,10 @@ use network::discovery::{Discovery, TableUpdates, NodeEntry}; type Slab = ::slab::Slab; const _DEFAULT_PORT: u16 = 30304; -const MAX_CONNECTIONS: usize = 1024; +const MAX_SESSIONS: usize = 1024; +const MAX_HANDSHAKES: usize = 256; +const MAX_HANDSHAKES_PER_ROUND: usize = 64; const MAINTENANCE_TIMEOUT: u64 = 1000; -const MAX_HANDSHAKES: usize = 100; #[derive(Debug)] /// Network service configuration @@ -132,13 +134,16 @@ impl NetworkConfiguration { } // Tokens -const TCP_ACCEPT: usize = MAX_CONNECTIONS + 1; -const IDLE: usize = MAX_CONNECTIONS + 2; -const DISCOVERY: usize = MAX_CONNECTIONS + 3; -const DISCOVERY_REFRESH: usize = MAX_CONNECTIONS + 4; -const DISCOVERY_ROUND: usize = MAX_CONNECTIONS + 5; -const FIRST_CONNECTION: usize = 0; -const LAST_CONNECTION: usize = FIRST_CONNECTION + MAX_CONNECTIONS - 1; +const TCP_ACCEPT: usize = LAST_HANDSHAKE + 1; +const IDLE: usize = LAST_HANDSHAKE + 2; +const DISCOVERY: usize = LAST_HANDSHAKE + 3; +const DISCOVERY_REFRESH: usize = LAST_HANDSHAKE + 4; +const DISCOVERY_ROUND: usize = LAST_HANDSHAKE + 5; +const FIRST_SESSION: usize = 0; +const LAST_SESSION: usize = FIRST_SESSION + MAX_SESSIONS - 1; +const FIRST_HANDSHAKE: usize = LAST_SESSION + 1; +const LAST_HANDSHAKE: usize = FIRST_HANDSHAKE + MAX_HANDSHAKES - 1; +const USER_TIMER: usize = LAST_HANDSHAKE + 256; /// Protocol handler level packet id pub type PacketId = u8; @@ -196,7 +201,7 @@ impl Encodable for CapabilityInfo { pub struct NetworkContext<'s, Message> where Message: Send + Sync + Clone + 'static, 's { io: &'s IoContext>, protocol: ProtocolId, - connections: Arc>>, + sessions: Arc>>, session: Option, } @@ -204,28 +209,23 @@ impl<'s, Message> NetworkContext<'s, Message> where Message: Send + Sync + Clone /// Create a new network IO access point. Takes references to all the data that can be updated within the IO handler. fn new(io: &'s IoContext>, protocol: ProtocolId, - session: Option, connections: Arc>>) -> NetworkContext<'s, Message> { + session: Option, sessions: Arc>>) -> NetworkContext<'s, Message> { NetworkContext { io: io, protocol: protocol, session: session, - connections: connections, + sessions: sessions, } } /// Send a packet over the network to another peer. pub fn send(&self, peer: PeerId, packet_id: PacketId, data: Vec) -> Result<(), UtilError> { - let connection = { self.connections.read().unwrap().get(peer).cloned() }; - if let Some(connection) = connection { - match *connection.lock().unwrap().deref_mut() { - ConnectionEntry::Session(ref mut s) => { - s.send_packet(self.protocol, packet_id as u8, &data).unwrap_or_else(|e| { + let session = { self.sessions.read().unwrap().get(peer).cloned() }; + if let Some(session) = session { + session.lock().unwrap().deref_mut().send_packet(self.protocol, packet_id as u8, &data).unwrap_or_else(|e| { warn!(target: "net", "Send error: {:?}", e); }); //TODO: don't copy vector data - try!(self.io.update_registration(peer)); - }, - _ => warn!(target: "net", "Send: Peer is not connected yet") - } + try!(self.io.update_registration(peer)); } else { trace!(target: "net", "Send: Peer no longer exist") } @@ -265,11 +265,9 @@ impl<'s, Message> NetworkContext<'s, Message> where Message: Send + Sync + Clone /// Returns peer identification string pub fn peer_info(&self, peer: PeerId) -> String { - let connection = { self.connections.read().unwrap().get(peer).cloned() }; - if let Some(connection) = connection { - if let ConnectionEntry::Session(ref s) = *connection.lock().unwrap().deref() { - return s.info.client_version.clone() - } + let session = { self.sessions.read().unwrap().get(peer).cloned() }; + if let Some(session) = session { + return session.lock().unwrap().info.client_version.clone() } "unknown".to_owned() } @@ -311,12 +309,8 @@ impl HostInfo { } } -enum ConnectionEntry { - Handshake(Handshake), - Session(Session) -} - -type SharedConnectionEntry = Arc>; +type SharedSession = Arc>; +type SharedHandshake = Arc>; #[derive(Copy, Clone)] struct ProtocolTimer { @@ -328,7 +322,8 @@ struct ProtocolTimer { pub struct Host where Message: Send + Sync + Clone { pub info: RwLock, tcp_listener: Mutex, - connections: Arc>>, + handshakes: Arc>>, + sessions: Arc>>, discovery: Mutex, nodes: RwLock, handlers: RwLock>>>, @@ -361,11 +356,12 @@ impl Host where Message: Send + Sync + Clone { }), discovery: Mutex::new(discovery), tcp_listener: Mutex::new(tcp_listener), - connections: Arc::new(RwLock::new(Slab::new_starting_at(FIRST_CONNECTION, MAX_CONNECTIONS))), + handshakes: Arc::new(RwLock::new(Slab::new_starting_at(FIRST_HANDSHAKE, MAX_HANDSHAKES))), + sessions: Arc::new(RwLock::new(Slab::new_starting_at(FIRST_SESSION, MAX_SESSIONS))), nodes: RwLock::new(NodeTable::new(path)), handlers: RwLock::new(HashMap::new()), timers: RwLock::new(HashMap::new()), - timer_counter: RwLock::new(LAST_CONNECTION + 1), + timer_counter: RwLock::new(USER_TIMER), stats: Arc::new(NetworkStats::default()), }; let port = host.info.read().unwrap().config.listen_address.port(); @@ -407,33 +403,28 @@ impl Host where Message: Send + Sync + Clone { } fn have_session(&self, id: &NodeId) -> bool { - self.connections.read().unwrap().iter().any(|e| - match *e.lock().unwrap().deref() { ConnectionEntry::Session(ref s) => s.info.id.eq(&id), _ => false }) + self.sessions.read().unwrap().iter().any(|e| e.lock().unwrap().info.id.eq(&id)) } fn session_count(&self) -> usize { - self.connections.read().unwrap().iter().filter(|e| - match *e.lock().unwrap().deref() { ConnectionEntry::Session(_) => true, _ => false }).count() + self.sessions.read().unwrap().count() } fn connecting_to(&self, id: &NodeId) -> bool { - self.connections.read().unwrap().iter().any(|e| - match *e.lock().unwrap().deref() { ConnectionEntry::Handshake(ref h) => h.id.eq(&id), _ => false }) + self.handshakes.read().unwrap().iter().any(|e| e.lock().unwrap().id.eq(&id)) } fn handshake_count(&self) -> usize { - self.connections.read().unwrap().iter().filter(|e| - match *e.lock().unwrap().deref() { ConnectionEntry::Handshake(_) => true, _ => false }).count() + self.handshakes.read().unwrap().count() } fn keep_alive(&self, io: &IoContext>) { let mut to_kill = Vec::new(); - for e in self.connections.write().unwrap().iter_mut() { - if let ConnectionEntry::Session(ref mut s) = *e.lock().unwrap().deref_mut() { - if !s.keep_alive(io) { - s.disconnect(DisconnectReason::PingTimeout); - to_kill.push(s.token()); - } + for e in self.sessions.write().unwrap().iter_mut() { + let mut s = e.lock().unwrap(); + if !s.keep_alive(io) { + s.disconnect(DisconnectReason::PingTimeout); + to_kill.push(s.token()); } } for p in to_kill { @@ -443,8 +434,8 @@ impl Host where Message: Send + Sync + Clone { fn connect_peers(&self, io: &IoContext>) { let ideal_peers = { self.info.read().unwrap().deref().config.ideal_peers }; - let connections = self.session_count(); - if connections >= ideal_peers as usize { + let session_count = self.session_count(); + if session_count >= ideal_peers as usize { return; } @@ -453,10 +444,9 @@ impl Host where Message: Send + Sync + Clone { return; } - let nodes = { self.nodes.read().unwrap().nodes() }; - - for id in nodes.iter().filter(|ref id| !self.have_session(id) && !self.connecting_to(id)).take(MAX_HANDSHAKES - handshake_count) { + for id in nodes.iter().filter(|ref id| !self.have_session(id) && !self.connecting_to(id)) + .take(min(MAX_HANDSHAKES_PER_ROUND, MAX_HANDSHAKES - handshake_count)) { self.connect_peer(&id, io); } debug!(target: "net", "Connecting peers: {} sessions, {} pending", self.session_count(), self.handshake_count()); @@ -495,15 +485,15 @@ impl Host where Message: Send + Sync + Clone { #[allow(block_in_if_condition_stmt)] fn create_connection(&self, socket: TcpStream, id: Option<&NodeId>, io: &IoContext>) { let nonce = self.info.write().unwrap().next_nonce(); - let mut connections = self.connections.write().unwrap(); - if connections.insert_with(|token| { + let mut handshakes = self.handshakes.write().unwrap(); + if handshakes.insert_with(|token| { let mut handshake = Handshake::new(token, id, socket, &nonce, self.stats.clone()).expect("Can't create handshake"); handshake.start(io, &self.info.read().unwrap(), id.is_some()).and_then(|_| io.register_stream(token)).unwrap_or_else (|e| { debug!(target: "net", "Handshake create error: {:?}", e); }); - Arc::new(Mutex::new(ConnectionEntry::Handshake(handshake))) + Arc::new(Mutex::new(handshake)) }).is_none() { - warn!("Max connections reached"); + warn!("Max handshakes reached"); } } @@ -523,35 +513,18 @@ impl Host where Message: Send + Sync + Clone { io.update_registration(TCP_ACCEPT).expect("Error registering TCP listener"); } - #[allow(single_match)] - fn connection_writable(&self, token: StreamToken, io: &IoContext>) { + fn handshake_writable(&self, token: StreamToken, io: &IoContext>) { let mut create_session = false; let mut kill = false; - let connection = { self.connections.read().unwrap().get(token).cloned() }; - if let Some(connection) = connection { - match *connection.lock().unwrap().deref_mut() { - ConnectionEntry::Handshake(ref mut h) => { - match h.writable(io, &self.info.read().unwrap()) { - Err(e) => { - debug!(target: "net", "Handshake write error: {}:{:?}", token, e); - kill = true; - }, - Ok(_) => () - } - if h.done() { - create_session = true; - } - }, - ConnectionEntry::Session(ref mut s) => { - match s.writable(io, &self.info.read().unwrap()) { - Err(e) => { - debug!(target: "net", "Session write error: {}:{:?}", token, e); - kill = true; - }, - Ok(_) => () - } - io.update_registration(token).unwrap_or_else(|e| debug!(target: "net", "Session registration error: {:?}", e)); - } + let handshake = { self.handshakes.read().unwrap().get(token).cloned() }; + if let Some(handshake) = handshake { + let mut h = handshake.lock().unwrap(); + if let Err(e) = h.writable(io, &self.info.read().unwrap()) { + debug!(target: "net", "Handshake write error: {}:{:?}", token, e); + kill = true; + } + if h.done() { + create_session = true; } } if kill { @@ -563,55 +536,40 @@ impl Host where Message: Send + Sync + Clone { } } + fn session_writable(&self, token: StreamToken, io: &IoContext>) { + let mut kill = false; + let session = { self.sessions.read().unwrap().get(token).cloned() }; + if let Some(session) = session { + let mut s = session.lock().unwrap(); + if let Err(e) = s.writable(io, &self.info.read().unwrap()) { + debug!(target: "net", "Session write error: {}:{:?}", token, e); + kill = true; + } + io.update_registration(token).unwrap_or_else(|e| debug!(target: "net", "Session registration error: {:?}", e)); + } + if kill { + self.kill_connection(token, io, true); //TODO: mark connection as dead an check in kill_connection + } + } + fn connection_closed(&self, token: TimerToken, io: &IoContext>) { self.kill_connection(token, io, true); } - fn connection_readable(&self, token: StreamToken, io: &IoContext>) { - let mut ready_data: Vec = Vec::new(); - let mut packet_data: Option<(ProtocolId, PacketId, Vec)> = None; + fn handshake_readable(&self, token: StreamToken, io: &IoContext>) { let mut create_session = false; let mut kill = false; - let connection = { self.connections.read().unwrap().get(token).cloned() }; - if let Some(connection) = connection { - match *connection.lock().unwrap().deref_mut() { - ConnectionEntry::Handshake(ref mut h) => { - if let Err(e) = h.readable(io, &self.info.read().unwrap()) { - debug!(target: "net", "Handshake read error: {}:{:?}", token, e); - kill = true; - } - if h.done() { - create_session = true; - } - }, - ConnectionEntry::Session(ref mut s) => { - match s.readable(io, &self.info.read().unwrap()) { - Err(e) => { - debug!(target: "net", "Handshake read error: {}:{:?}", token, e); - kill = true; - }, - Ok(SessionData::Ready) => { - for (p, _) in self.handlers.read().unwrap().iter() { - if s.have_capability(p) { - ready_data.push(p); - } - } - }, - Ok(SessionData::Packet { - data, - protocol, - packet_id, - }) => { - match self.handlers.read().unwrap().get(protocol) { - None => { warn!(target: "net", "No handler found for protocol: {:?}", protocol) }, - Some(_) => packet_data = Some((protocol, packet_id, data)), - } - }, - Ok(SessionData::None) => {}, - } - } + let handshake = { self.handshakes.read().unwrap().get(token).cloned() }; + if let Some(handshake) = handshake { + let mut h = handshake.lock().unwrap(); + if let Err(e) = h.readable(io, &self.info.read().unwrap()) { + debug!(target: "net", "Handshake read error: {}:{:?}", token, e); + kill = true; } - } + if h.done() { + create_session = true; + } + } if kill { self.kill_connection(token, io, true); //TODO: mark connection as dead an check in kill_connection return; @@ -619,40 +577,74 @@ impl Host where Message: Send + Sync + Clone { self.start_session(token, io); io.update_registration(token).unwrap_or_else(|e| debug!(target: "net", "Session registration error: {:?}", e)); } + io.update_registration(token).unwrap_or_else(|e| debug!(target: "net", "Token registration error: {:?}", e)); + } + + fn session_readable(&self, token: StreamToken, io: &IoContext>) { + let mut ready_data: Vec = Vec::new(); + let mut packet_data: Option<(ProtocolId, PacketId, Vec)> = None; + let mut kill = false; + let session = { self.sessions.read().unwrap().get(token).cloned() }; + if let Some(session) = session { + let mut s = session.lock().unwrap(); + match s.readable(io, &self.info.read().unwrap()) { + Err(e) => { + debug!(target: "net", "Session read error: {}:{:?}", token, e); + kill = true; + }, + Ok(SessionData::Ready) => { + for (p, _) in self.handlers.read().unwrap().iter() { + if s.have_capability(p) { + ready_data.push(p); + } + } + }, + Ok(SessionData::Packet { + data, + protocol, + packet_id, + }) => { + match self.handlers.read().unwrap().get(protocol) { + None => { warn!(target: "net", "No handler found for protocol: {:?}", protocol) }, + Some(_) => packet_data = Some((protocol, packet_id, data)), + } + }, + Ok(SessionData::None) => {}, + } + } + if kill { + self.kill_connection(token, io, true); //TODO: mark connection as dead an check in kill_connection + return; + } for p in ready_data { let h = self.handlers.read().unwrap().get(p).unwrap().clone(); - h.connected(&NetworkContext::new(io, p, Some(token), self.connections.clone()), &token); + h.connected(&NetworkContext::new(io, p, Some(token), self.sessions.clone()), &token); } if let Some((p, packet_id, data)) = packet_data { let h = self.handlers.read().unwrap().get(p).unwrap().clone(); - h.read(&NetworkContext::new(io, p, Some(token), self.connections.clone()), &token, packet_id, &data[1..]); + h.read(&NetworkContext::new(io, p, Some(token), self.sessions.clone()), &token, packet_id, &data[1..]); } io.update_registration(token).unwrap_or_else(|e| debug!(target: "net", "Token registration error: {:?}", e)); } fn start_session(&self, token: StreamToken, io: &IoContext>) { - let mut connections = self.connections.write().unwrap(); - let replace = { - let connection = { connections.get(token).cloned() }; - if let Some(connection) = connection { - match *connection.lock().unwrap().deref_mut() { - ConnectionEntry::Handshake(_) => true, - _ => false, - } - } else { false } - }; - if replace { - connections.replace_with(token, |c| { - match Arc::try_unwrap(c).ok().unwrap().into_inner().unwrap() { - ConnectionEntry::Handshake(h) => { - let session = Session::new(h, io, &self.info.read().unwrap()).expect("Session creation error"); - io.update_registration(token).expect("Error updating session registration"); - self.stats.inc_sessions(); - Some(Arc::new(Mutex::new(ConnectionEntry::Session(session)))) - }, - _ => { None } // handshake expired - } - }).ok(); + let mut handshakes = self.handshakes.write().unwrap(); + if handshakes.get(token).is_none() { + return; + } + + // turn a handshake into a session + let mut sessions = self.sessions.write().unwrap(); + let h = handshakes.remove(token).unwrap(); + let h = Arc::try_unwrap(h).ok().unwrap().into_inner().unwrap(); + let result = sessions.insert_with(move |session_token| { + let session = Session::new(h, session_token, &self.info.read().unwrap()).expect("Session creation error"); + io.update_registration(session_token).expect("Error updating session registration"); + self.stats.inc_sessions(); + Arc::new(Mutex::new(session)) + }); + if result.is_none() { + warn!("Max sessions reached"); } } @@ -663,28 +655,32 @@ impl Host where Message: Send + Sync + Clone { fn kill_connection(&self, token: StreamToken, io: &IoContext>, remote: bool) { let mut to_disconnect: Vec = Vec::new(); let mut failure_id = None; - { - let mut connections = self.connections.write().unwrap(); - if let Some(connection) = connections.get(token).cloned() { - match *connection.lock().unwrap().deref_mut() { - ConnectionEntry::Handshake(ref h) => { - connections.remove(token); - failure_id = Some(h.id().clone()); - }, - ConnectionEntry::Session(ref mut s) if s.is_ready() => { + match token { + FIRST_HANDSHAKE ... LAST_HANDSHAKE => { + let mut handshakes = self.handshakes.write().unwrap(); + if let Some(handshake) = handshakes.get(token).cloned() { + failure_id = Some(handshake.lock().unwrap().id().clone()); + handshakes.remove(token); + } + }, + FIRST_SESSION ... LAST_SESSION => { + let mut sessions = self.sessions.write().unwrap(); + if let Some(session) = sessions.get(token).cloned() { + let s = session.lock().unwrap(); + if s.is_ready() { for (p, _) in self.handlers.read().unwrap().iter() { if s.have_capability(p) { to_disconnect.push(p); } } - connections.remove(token); - failure_id = Some(s.id().clone()); - }, - _ => {}, + } + failure_id = Some(s.id().clone()); + sessions.remove(token); } - } - io.deregister_stream(token).expect("Error deregistering stream"); + }, + _ => {}, } + io.deregister_stream(token).expect("Error deregistering stream"); if let Some(id) = failure_id { if remote { self.nodes.write().unwrap().note_failure(&id); @@ -692,25 +688,28 @@ impl Host where Message: Send + Sync + Clone { } for p in to_disconnect { let h = self.handlers.read().unwrap().get(p).unwrap().clone(); - h.disconnected(&NetworkContext::new(io, p, Some(token), self.connections.clone()), &token); + h.disconnected(&NetworkContext::new(io, p, Some(token), self.sessions.clone()), &token); } } fn update_nodes(&self, io: &IoContext>, node_changes: TableUpdates) { let mut to_remove: Vec = Vec::new(); { - let connections = self.connections.write().unwrap(); - for c in connections.iter() { - match *c.lock().unwrap().deref_mut() { - ConnectionEntry::Handshake(ref h) => { - if node_changes.removed.contains(&h.id()) { - to_remove.push(h.token()); - } + { + let handshakes = self.handshakes.write().unwrap(); + for c in handshakes.iter() { + let h = c.lock().unwrap(); + if node_changes.removed.contains(&h.id()) { + to_remove.push(h.token()); } - ConnectionEntry::Session(ref s) => { - if node_changes.removed.contains(&s.id()) { - to_remove.push(s.token()); - } + } + } + { + let sessions = self.sessions.write().unwrap(); + for c in sessions.iter() { + let s = c.lock().unwrap(); + if node_changes.removed.contains(&s.id()) { + to_remove.push(s.token()); } } } @@ -735,14 +734,16 @@ impl IoHandler> for Host where Messa fn stream_hup(&self, io: &IoContext>, stream: StreamToken) { trace!(target: "net", "Hup: {}", stream); match stream { - FIRST_CONNECTION ... LAST_CONNECTION => self.connection_closed(stream, io), + FIRST_SESSION ... LAST_SESSION => self.connection_closed(stream, io), + FIRST_HANDSHAKE ... LAST_HANDSHAKE => self.connection_closed(stream, io), _ => warn!(target: "net", "Unexpected hup"), }; } fn stream_readable(&self, io: &IoContext>, stream: StreamToken) { match stream { - FIRST_CONNECTION ... LAST_CONNECTION => self.connection_readable(stream, io), + FIRST_SESSION ... LAST_SESSION => self.session_readable(stream, io), + FIRST_HANDSHAKE ... LAST_HANDSHAKE => self.handshake_readable(stream, io), DISCOVERY => { if let Some(node_changes) = self.discovery.lock().unwrap().readable() { self.update_nodes(io, node_changes); @@ -756,7 +757,8 @@ impl IoHandler> for Host where Messa fn stream_writable(&self, io: &IoContext>, stream: StreamToken) { match stream { - FIRST_CONNECTION ... LAST_CONNECTION => self.connection_writable(stream, io), + FIRST_SESSION ... LAST_SESSION => self.session_writable(stream, io), + FIRST_HANDSHAKE ... LAST_HANDSHAKE => self.handshake_writable(stream, io), DISCOVERY => { self.discovery.lock().unwrap().writable(); io.update_registration(DISCOVERY).expect("Error updating disicovery registration"); @@ -768,7 +770,8 @@ impl IoHandler> for Host where Messa fn timeout(&self, io: &IoContext>, token: TimerToken) { match token { IDLE => self.maintain_network(io), - FIRST_CONNECTION ... LAST_CONNECTION => self.connection_timeout(token, io), + FIRST_SESSION ... LAST_SESSION => self.connection_timeout(token, io), + FIRST_HANDSHAKE ... LAST_HANDSHAKE => self.connection_timeout(token, io), DISCOVERY_REFRESH => { self.discovery.lock().unwrap().refresh(); io.update_registration(DISCOVERY).expect("Error updating disicovery registration"); @@ -782,7 +785,7 @@ impl IoHandler> for Host where Messa _ => match self.timers.read().unwrap().get(&token).cloned() { Some(timer) => match self.handlers.read().unwrap().get(timer.protocol).cloned() { None => { warn!(target: "net", "No handler found for protocol: {:?}", timer.protocol) }, - Some(h) => { h.timeout(&NetworkContext::new(io, timer.protocol, None, self.connections.clone()), timer.token); } + Some(h) => { h.timeout(&NetworkContext::new(io, timer.protocol, None, self.sessions.clone()), timer.token); } }, None => { warn!("Unknown timer token: {}", token); } // timer is not registerd through us } @@ -797,7 +800,7 @@ impl IoHandler> for Host where Messa ref versions } => { let h = handler.clone(); - h.initialize(&NetworkContext::new(io, protocol, None, self.connections.clone())); + h.initialize(&NetworkContext::new(io, protocol, None, self.sessions.clone())); self.handlers.write().unwrap().insert(protocol, h); let mut info = self.info.write().unwrap(); for v in versions { @@ -820,18 +823,15 @@ impl IoHandler> for Host where Messa io.register_timer(handler_token, *delay).expect("Error registering timer"); }, NetworkIoMessage::Disconnect(ref peer) => { - let connection = { self.connections.read().unwrap().get(*peer).cloned() }; - if let Some(connection) = connection { - match *connection.lock().unwrap().deref_mut() { - ConnectionEntry::Handshake(_) => {}, - ConnectionEntry::Session(ref mut s) => { s.disconnect(DisconnectReason::DisconnectRequested); } - } + let session = { self.sessions.read().unwrap().get(*peer).cloned() }; + if let Some(session) = session { + session.lock().unwrap().disconnect(DisconnectReason::DisconnectRequested); } self.kill_connection(*peer, io, false); }, NetworkIoMessage::User(ref message) => { for (p, h) in self.handlers.read().unwrap().iter() { - h.message(&NetworkContext::new(io, p, None, self.connections.clone()), &message); + h.message(&NetworkContext::new(io, p, None, self.sessions.clone()), &message); } } } @@ -839,14 +839,14 @@ impl IoHandler> for Host where Messa fn register_stream(&self, stream: StreamToken, reg: Token, event_loop: &mut EventLoop>>) { match stream { - FIRST_CONNECTION ... LAST_CONNECTION => { - let connection = { self.connections.read().unwrap().get(stream).cloned() }; + FIRST_SESSION ... LAST_SESSION => { + warn!("Unexpected session stream registration"); + } + FIRST_HANDSHAKE ... LAST_HANDSHAKE => { + let connection = { self.handshakes.read().unwrap().get(stream).cloned() }; if let Some(connection) = connection { - match *connection.lock().unwrap().deref() { - ConnectionEntry::Handshake(ref h) => h.register_socket(reg, event_loop).expect("Error registering socket"), - ConnectionEntry::Session(_) => warn!("Unexpected session stream registration") - } - } else {} // expired + connection.lock().unwrap().register_socket(reg, event_loop).expect("Error registering socket"); + } } DISCOVERY => self.discovery.lock().unwrap().register_socket(event_loop).expect("Error registering discovery socket"), TCP_ACCEPT => event_loop.register(self.tcp_listener.lock().unwrap().deref(), Token(TCP_ACCEPT), EventSet::all(), PollOpt::edge()).expect("Error registering stream"), @@ -856,16 +856,20 @@ impl IoHandler> for Host where Messa fn deregister_stream(&self, stream: StreamToken, event_loop: &mut EventLoop>>) { match stream { - FIRST_CONNECTION ... LAST_CONNECTION => { - let mut connections = self.connections.write().unwrap(); + FIRST_SESSION ... LAST_SESSION => { + let mut connections = self.sessions.write().unwrap(); if let Some(connection) = connections.get(stream).cloned() { - match *connection.lock().unwrap().deref() { - ConnectionEntry::Handshake(ref h) => h.deregister_socket(event_loop).expect("Error deregistering socket"), - ConnectionEntry::Session(ref s) => s.deregister_socket(event_loop).expect("Error deregistering session socket"), - } + connection.lock().unwrap().deregister_socket(event_loop).expect("Error deregistering socket"); connections.remove(stream); - } - }, + } + } + FIRST_HANDSHAKE ... LAST_HANDSHAKE => { + let mut connections = self.handshakes.write().unwrap(); + if let Some(connection) = connections.get(stream).cloned() { + connection.lock().unwrap().deregister_socket(event_loop).expect("Error deregistering socket"); + connections.remove(stream); + } + } DISCOVERY => (), TCP_ACCEPT => event_loop.deregister(self.tcp_listener.lock().unwrap().deref()).unwrap(), _ => warn!("Unexpected stream deregistration") @@ -874,14 +878,17 @@ impl IoHandler> for Host where Messa fn update_stream(&self, stream: StreamToken, reg: Token, event_loop: &mut EventLoop>>) { match stream { - FIRST_CONNECTION ... LAST_CONNECTION => { - let connection = { self.connections.read().unwrap().get(stream).cloned() }; - if let Some(connection) = connection { - match *connection.lock().unwrap().deref() { - ConnectionEntry::Handshake(ref h) => h.update_socket(reg, event_loop).expect("Error updating socket"), - ConnectionEntry::Session(ref s) => s.update_socket(reg, event_loop).expect("Error updating socket"), - } - } else {} // expired + FIRST_SESSION ... LAST_SESSION => { + let connection = { self.sessions.read().unwrap().get(stream).cloned() }; + if let Some(connection) = connection { + connection.lock().unwrap().update_socket(reg, event_loop).expect("Error updating socket"); + } + } + FIRST_HANDSHAKE ... LAST_HANDSHAKE => { + let connection = { self.handshakes.read().unwrap().get(stream).cloned() }; + if let Some(connection) = connection { + connection.lock().unwrap().update_socket(reg, event_loop).expect("Error updating socket"); + } } DISCOVERY => self.discovery.lock().unwrap().update_registration(event_loop).expect("Error reregistering discovery socket"), TCP_ACCEPT => event_loop.reregister(self.tcp_listener.lock().unwrap().deref(), Token(TCP_ACCEPT), EventSet::all(), PollOpt::edge()).expect("Error reregistering stream"), diff --git a/util/src/network/session.rs b/util/src/network/session.rs index 3b49a8f5e..2763dfd82 100644 --- a/util/src/network/session.rs +++ b/util/src/network/session.rs @@ -108,7 +108,7 @@ const PACKET_LAST: u8 = 0x7f; impl Session { /// Create a new session out of comepleted handshake. Consumes handshake object. - pub fn new(h: Handshake, _io: &IoContext, host: &HostInfo) -> Result where Message: Send + Sync + Clone { + pub fn new(h: Handshake, token: StreamToken, host: &HostInfo) -> Result { let id = h.id.clone(); let connection = try!(EncryptedConnection::new(h)); let mut session = Session { @@ -124,6 +124,7 @@ impl Session { ping_time_ns: 0, pong_time_ns: None, }; + session.connection.set_token(token); try!(session.write_hello(host)); try!(session.send_ping()); Ok(session) From 7503d6695a8ef22fa0aa4530745ceed95a260f63 Mon Sep 17 00:00:00 2001 From: arkpar Date: Sun, 14 Feb 2016 11:54:08 +0100 Subject: [PATCH 07/61] Fixed panic on session creation --- util/src/network/host.rs | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/util/src/network/host.rs b/util/src/network/host.rs index 04dda02bd..fe283e21a 100644 --- a/util/src/network/host.rs +++ b/util/src/network/host.rs @@ -635,7 +635,13 @@ impl Host where Message: Send + Sync + Clone { // turn a handshake into a session let mut sessions = self.sessions.write().unwrap(); - let h = handshakes.remove(token).unwrap(); + let mut h = handshakes.remove(token).unwrap(); + // wait for other threads to stop using it + { + while Arc::get_mut(&mut h).is_none() { + h.lock().ok(); + } + } let h = Arc::try_unwrap(h).ok().unwrap().into_inner().unwrap(); let result = sessions.insert_with(move |session_token| { let session = Session::new(h, session_token, &self.info.read().unwrap()).expect("Session creation error"); From dee375bfacc3010dadc97a8392f7f2d8632f8eac Mon Sep 17 00:00:00 2001 From: arkpar Date: Sun, 14 Feb 2016 12:11:18 +0100 Subject: [PATCH 08/61] Handle session creation error --- util/src/network/host.rs | 9 ++++++++- util/src/network/session.rs | 8 ++++++-- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/util/src/network/host.rs b/util/src/network/host.rs index fe283e21a..89cc4c225 100644 --- a/util/src/network/host.rs +++ b/util/src/network/host.rs @@ -643,8 +643,15 @@ impl Host where Message: Send + Sync + Clone { } } let h = Arc::try_unwrap(h).ok().unwrap().into_inner().unwrap(); + let mut session = match Session::new(h, &self.info.read().unwrap()) { + Ok(s) => s, + Err(e) => { + warn!("Session creation error: {:?}", e); + return; + } + }; let result = sessions.insert_with(move |session_token| { - let session = Session::new(h, session_token, &self.info.read().unwrap()).expect("Session creation error"); + session.set_token(session_token); io.update_registration(session_token).expect("Error updating session registration"); self.stats.inc_sessions(); Arc::new(Mutex::new(session)) diff --git a/util/src/network/session.rs b/util/src/network/session.rs index 2763dfd82..f501e9c79 100644 --- a/util/src/network/session.rs +++ b/util/src/network/session.rs @@ -108,7 +108,7 @@ const PACKET_LAST: u8 = 0x7f; impl Session { /// Create a new session out of comepleted handshake. Consumes handshake object. - pub fn new(h: Handshake, token: StreamToken, host: &HostInfo) -> Result { + pub fn new(h: Handshake, host: &HostInfo) -> Result { let id = h.id.clone(); let connection = try!(EncryptedConnection::new(h)); let mut session = Session { @@ -124,7 +124,6 @@ impl Session { ping_time_ns: 0, pong_time_ns: None, }; - session.connection.set_token(token); try!(session.write_hello(host)); try!(session.send_ping()); Ok(session) @@ -140,6 +139,11 @@ impl Session { self.had_hello } + /// Replace socket token + pub fn set_token(&mut self, token: StreamToken) { + self.connection.set_token(token); + } + /// Readable IO handler. Returns packet data if available. pub fn readable(&mut self, io: &IoContext, host: &HostInfo) -> Result where Message: Send + Sync + Clone { match try!(self.connection.readable(io)) { From fc7483ab87437e92751d676539eff93bcb774a7e Mon Sep 17 00:00:00 2001 From: arkpar Date: Sun, 14 Feb 2016 17:10:55 +0100 Subject: [PATCH 09/61] Propagate only one last hash for peers that are too far behind --- sync/src/chain.rs | 81 +++++++++++++++++++++++++++++------------------ 1 file changed, 51 insertions(+), 30 deletions(-) diff --git a/sync/src/chain.rs b/sync/src/chain.rs index 671e2241e..0de0bdcaf 100644 --- a/sync/src/chain.rs +++ b/sync/src/chain.rs @@ -155,7 +155,9 @@ struct PeerInfo { /// Peer network id network_id: U256, /// Peer best block hash - latest: H256, + latest_hash: H256, + /// Peer best block number if known + latest_number: Option, /// Peer total difficulty difficulty: U256, /// Type of data currenty being requested from peer. @@ -282,7 +284,8 @@ impl ChainSync { protocol_version: try!(r.val_at(0)), network_id: try!(r.val_at(1)), difficulty: try!(r.val_at(2)), - latest: try!(r.val_at(3)), + latest_hash: try!(r.val_at(3)), + latest_number: None, genesis: try!(r.val_at(4)), asking: PeerAsking::Nothing, asking_blocks: Vec::new(), @@ -290,7 +293,7 @@ impl ChainSync { ask_time: 0f64, }; - trace!(target: "sync", "New peer {} (protocol: {}, network: {:?}, difficulty: {:?}, latest:{}, genesis:{})", peer_id, peer.protocol_version, peer.network_id, peer.difficulty, peer.latest, peer.genesis); + trace!(target: "sync", "New peer {} (protocol: {}, network: {:?}, difficulty: {:?}, latest:{}, genesis:{})", peer_id, peer.protocol_version, peer.network_id, peer.difficulty, peer.latest_hash, peer.genesis); if self.peers.contains_key(&peer_id) { warn!("Unexpected status packet from {}:{}", peer_id, io.peer_info(peer_id)); @@ -450,7 +453,8 @@ impl ChainSync { let mut unknown = false; { let peer = self.peers.get_mut(&peer_id).unwrap(); - peer.latest = header.hash(); + peer.latest_hash = header.hash(); + peer.latest_number = Some(header.number()); } // TODO: Decompose block and add to self.headers and self.bodies instead if header.number == From::from(self.current_base_block() + 1) { @@ -516,7 +520,8 @@ impl ChainSync { if d > max_height { trace!(target: "sync", "New unknown block hash {:?}", h); let peer = self.peers.get_mut(&peer_id).unwrap(); - peer.latest = h.clone(); + peer.latest_hash = h.clone(); + peer.latest_number = Some(d); max_height = d; } }, @@ -583,7 +588,7 @@ impl ChainSync { trace!(target: "sync", "Waiting for block queue"); return; } - (peer.latest.clone(), peer.difficulty.clone()) + (peer.latest_hash.clone(), peer.difficulty.clone()) }; let td = io.chain().chain_info().pending_total_difficulty; @@ -1117,25 +1122,28 @@ impl ChainSync { } /// returns peer ids that have less blocks than our chain - fn get_lagging_peers(&self, io: &SyncIo) -> Vec { + fn get_lagging_peers(&mut self, io: &SyncIo) -> Vec<(PeerId, BlockNumber)> { let chain = io.chain(); let chain_info = chain.chain_info(); let latest_hash = chain_info.best_block_hash; let latest_number = chain_info.best_block_number; - self.peers.iter().filter(|&(_, peer_info)| - match io.chain().block_status(BlockId::Hash(peer_info.latest.clone())) { + self.peers.iter_mut().filter_map(|(&id, ref mut peer_info)| + match io.chain().block_status(BlockId::Hash(peer_info.latest_hash.clone())) { BlockStatus::InChain => { - let peer_number = HeaderView::new(&io.chain().block_header(BlockId::Hash(peer_info.latest.clone())).unwrap()).number(); - peer_info.latest != latest_hash && latest_number > peer_number + if peer_info.latest_number.is_none() { + peer_info.latest_number = Some(HeaderView::new(&io.chain().block_header(BlockId::Hash(peer_info.latest_hash.clone())).unwrap()).number()); + } + if peer_info.latest_hash != latest_hash && latest_number > peer_info.latest_number.unwrap() { + Some((id, peer_info.latest_number.unwrap())) + } else { None } }, - _ => false + _ => None }) - .map(|(peer_id, _)| peer_id) - .cloned().collect::>() + .collect::>() } /// propagades latest block to lagging peers - fn propagade_blocks(&mut self, local_best: &H256, io: &mut SyncIo) -> usize { + fn propagade_blocks(&mut self, local_best: &H256, best_number: BlockNumber, io: &mut SyncIo) -> usize { let updated_peers = { let lagging_peers = self.get_lagging_peers(io); @@ -1143,33 +1151,41 @@ impl ChainSync { let fraction = (self.peers.len() as f64).powf(-0.5).mul(u32::max_value() as f64).round() as u32; let lucky_peers = match lagging_peers.len() { 0 ... MIN_PEERS_PROPAGATION => lagging_peers, - _ => lagging_peers.iter().filter(|_| ::rand::random::() < fraction).cloned().collect::>() + _ => lagging_peers.into_iter().filter(|_| ::rand::random::() < fraction).collect::>() }; // taking at max of MAX_PEERS_PROPAGATION - lucky_peers.iter().take(min(lucky_peers.len(), MAX_PEERS_PROPAGATION)).cloned().collect::>() + lucky_peers.iter().map(|&(id, _)| id.clone()).take(min(lucky_peers.len(), MAX_PEERS_PROPAGATION)).collect::>() }; let mut sent = 0; for peer_id in updated_peers { let rlp = ChainSync::create_latest_block_rlp(io.chain()); self.send_packet(io, peer_id, NEW_BLOCK_PACKET, rlp); - self.peers.get_mut(&peer_id).unwrap().latest = local_best.clone(); + self.peers.get_mut(&peer_id).unwrap().latest_hash = local_best.clone(); + self.peers.get_mut(&peer_id).unwrap().latest_number = Some(best_number); sent = sent + 1; } sent } /// propagades new known hashes to all peers - fn propagade_new_hashes(&mut self, local_best: &H256, io: &mut SyncIo) -> usize { + fn propagade_new_hashes(&mut self, local_best: &H256, best_number: BlockNumber, io: &mut SyncIo) -> usize { let updated_peers = self.get_lagging_peers(io); let mut sent = 0; - for peer_id in updated_peers { - sent = sent + match ChainSync::create_new_hashes_rlp(io.chain(), &self.peers.get(&peer_id).unwrap().latest, &local_best) { + let last_parent = HeaderView::new(&io.chain().block_header(BlockId::Hash(local_best.clone())).unwrap()).parent_hash(); + for (peer_id, peer_number) in updated_peers { + let mut peer_best = self.peers.get(&peer_id).unwrap().latest_hash.clone(); + if best_number - peer_number > MAX_PEERS_PROPAGATION as BlockNumber { + // If we think peer is too far behind just end one latest hash + peer_best = last_parent.clone(); + } + sent = sent + match ChainSync::create_new_hashes_rlp(io.chain(), &peer_best, &local_best) { Some(rlp) => { { let peer = self.peers.get_mut(&peer_id).unwrap(); - peer.latest = local_best.clone(); + peer.latest_hash = local_best.clone(); + peer.latest_number = Some(best_number); } self.send_packet(io, peer_id, NEW_BLOCK_HASHES_PACKET, rlp); 1 @@ -1189,8 +1205,8 @@ impl ChainSync { pub fn chain_blocks_verified(&mut self, io: &mut SyncIo) { let chain = io.chain().chain_info(); if (((chain.best_block_number as i64) - (self.last_send_block_number as i64)).abs() as BlockNumber) < MAX_PEER_LAG_PROPAGATION { - let blocks = self.propagade_blocks(&chain.best_block_hash, io); - let hashes = self.propagade_new_hashes(&chain.best_block_hash, io); + let blocks = self.propagade_blocks(&chain.best_block_hash, chain.best_block_number, io); + let hashes = self.propagade_new_hashes(&chain.best_block_hash, chain.best_block_number, io); if blocks != 0 || hashes != 0 { trace!(target: "sync", "Sent latest {} blocks and {} hashes to peers.", blocks, hashes); } @@ -1322,7 +1338,8 @@ mod tests { protocol_version: 0, genesis: H256::zero(), network_id: U256::zero(), - latest: peer_latest_hash, + latest_hash: peer_latest_hash, + latest_number: None, difficulty: U256::zero(), asking: PeerAsking::Nothing, asking_blocks: Vec::::new(), @@ -1337,7 +1354,7 @@ mod tests { let mut client = TestBlockChainClient::new(); client.add_blocks(100, false); let mut queue = VecDeque::new(); - let sync = dummy_sync_with_peer(client.block_hash_delta_minus(10)); + let mut sync = dummy_sync_with_peer(client.block_hash_delta_minus(10)); let io = TestIo::new(&mut client, &mut queue, None); let lagging_peers = sync.get_lagging_peers(&io); @@ -1369,9 +1386,10 @@ mod tests { let mut queue = VecDeque::new(); let mut sync = dummy_sync_with_peer(client.block_hash_delta_minus(5)); let best_hash = client.chain_info().best_block_hash.clone(); + let best_number = client.chain_info().best_block_number; let mut io = TestIo::new(&mut client, &mut queue, None); - let peer_count = sync.propagade_new_hashes(&best_hash, &mut io); + let peer_count = sync.propagade_new_hashes(&best_hash, best_number, &mut io); // 1 message should be send assert_eq!(1, io.queue.len()); @@ -1388,9 +1406,10 @@ mod tests { let mut queue = VecDeque::new(); let mut sync = dummy_sync_with_peer(client.block_hash_delta_minus(5)); let best_hash = client.chain_info().best_block_hash.clone(); + let best_number = client.chain_info().best_block_number; let mut io = TestIo::new(&mut client, &mut queue, None); - let peer_count = sync.propagade_blocks(&best_hash, &mut io); + let peer_count = sync.propagade_blocks(&best_hash, best_number, &mut io); // 1 message should be send assert_eq!(1, io.queue.len()); @@ -1493,9 +1512,10 @@ mod tests { let mut queue = VecDeque::new(); let mut sync = dummy_sync_with_peer(client.block_hash_delta_minus(5)); let best_hash = client.chain_info().best_block_hash.clone(); + let best_number = client.chain_info().best_block_number; let mut io = TestIo::new(&mut client, &mut queue, None); - sync.propagade_new_hashes(&best_hash, &mut io); + sync.propagade_new_hashes(&best_hash, best_number, &mut io); let data = &io.queue[0].data.clone(); let result = sync.on_peer_new_hashes(&mut io, 0, &UntrustedRlp::new(&data)); @@ -1511,9 +1531,10 @@ mod tests { let mut queue = VecDeque::new(); let mut sync = dummy_sync_with_peer(client.block_hash_delta_minus(5)); let best_hash = client.chain_info().best_block_hash.clone(); + let best_number = client.chain_info().best_block_number; let mut io = TestIo::new(&mut client, &mut queue, None); - sync.propagade_blocks(&best_hash, &mut io); + sync.propagade_blocks(&best_hash, best_number, &mut io); let data = &io.queue[0].data.clone(); let result = sync.on_peer_new_block(&mut io, 0, &UntrustedRlp::new(&data)); From 61c52f15a3a2b81d718993668ba1a0a31e980ad2 Mon Sep 17 00:00:00 2001 From: arkpar Date: Sun, 14 Feb 2016 17:42:03 +0100 Subject: [PATCH 10/61] Fixed panic on accessing expired node --- util/src/network/host.rs | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/util/src/network/host.rs b/util/src/network/host.rs index 89cc4c225..454e8e802 100644 --- a/util/src/network/host.rs +++ b/util/src/network/host.rs @@ -82,7 +82,7 @@ impl NetworkConfiguration { pin: false, boot_nodes: Vec::new(), use_secret: None, - ideal_peers: 10, + ideal_peers: 25, } } @@ -467,9 +467,10 @@ impl Host where Message: Send + Sync + Clone { let socket = { let address = { let mut nodes = self.nodes.write().unwrap(); - let node = nodes.get_mut(id).unwrap(); - node.last_attempted = Some(::time::now()); - node.endpoint.address + if let Some(node) = nodes.get_mut(id) { + node.last_attempted = Some(::time::now()); + node.endpoint.address + } }; match TcpStream::connect(&address) { Ok(socket) => socket, From 38f4a06f1d248125e75ee07675a88a04f4d85617 Mon Sep 17 00:00:00 2001 From: arkpar Date: Sun, 14 Feb 2016 17:45:00 +0100 Subject: [PATCH 11/61] Fixed panic on accessing expired node --- util/src/network/host.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/util/src/network/host.rs b/util/src/network/host.rs index 454e8e802..7366e129b 100644 --- a/util/src/network/host.rs +++ b/util/src/network/host.rs @@ -471,6 +471,10 @@ impl Host where Message: Send + Sync + Clone { node.last_attempted = Some(::time::now()); node.endpoint.address } + else { + debug!("Connection to expired node aborted"); + return; + } }; match TcpStream::connect(&address) { Ok(socket) => socket, @@ -647,7 +651,7 @@ impl Host where Message: Send + Sync + Clone { let mut session = match Session::new(h, &self.info.read().unwrap()) { Ok(s) => s, Err(e) => { - warn!("Session creation error: {:?}", e); + debug!("Session creation error: {:?}", e); return; } }; From 186c7585d287a770ca0f45ed4f664e75093ff9b1 Mon Sep 17 00:00:00 2001 From: arkpar Date: Mon, 15 Feb 2016 11:54:38 +0100 Subject: [PATCH 12/61] Node table persistency --- util/src/bytes.rs | 4 +- util/src/hash.rs | 7 +++ util/src/io/service.rs | 5 ++ util/src/network/discovery.rs | 2 +- util/src/network/node_table.rs | 111 ++++++++++++++++++++++++++++++++- util/src/sha3.rs | 2 +- 6 files changed, 124 insertions(+), 7 deletions(-) diff --git a/util/src/bytes.rs b/util/src/bytes.rs index 4923e6eb4..4e1a11e4d 100644 --- a/util/src/bytes.rs +++ b/util/src/bytes.rs @@ -170,8 +170,8 @@ pub trait BytesConvertable { fn to_bytes(&self) -> Bytes { self.as_slice().to_vec() } } -impl BytesConvertable for T where T: Deref { - fn bytes(&self) -> &[u8] { self.deref() } +impl BytesConvertable for T where T: AsRef<[u8]> { + fn bytes(&self) -> &[u8] { self.as_ref() } } #[test] diff --git a/util/src/hash.rs b/util/src/hash.rs index 3e164098b..aefa7795b 100644 --- a/util/src/hash.rs +++ b/util/src/hash.rs @@ -86,6 +86,13 @@ macro_rules! impl_hash { } } + impl AsRef<[u8]> for $from { + #[inline] + fn as_ref(&self) -> &[u8] { + &self.0 + } + } + impl DerefMut for $from { #[inline] fn deref_mut(&mut self) -> &mut [u8] { diff --git a/util/src/io/service.rs b/util/src/io/service.rs index c5f4a6072..83fa71b8a 100644 --- a/util/src/io/service.rs +++ b/util/src/io/service.rs @@ -256,6 +256,11 @@ impl Handler for IoManager where Message: Send + Clone + Sync IoMessage::DeregisterStream { handler_id, token } => { let handler = self.handlers.get(handler_id).expect("Unknown handler id").clone(); handler.deregister_stream(token, event_loop); + // unregister a timer associated with the token (if any) + let timer_id = token + handler_id * TOKENS_PER_HANDLER; + if let Some(timer) = self.timers.write().unwrap().remove(&timer_id) { + event_loop.clear_timeout(timer.timeout); + } }, IoMessage::UpdateStreamRegistration { handler_id, token } => { let handler = self.handlers.get(handler_id).expect("Unknown handler id").clone(); diff --git a/util/src/network/discovery.rs b/util/src/network/discovery.rs index 9feef9c74..04c32cd33 100644 --- a/util/src/network/discovery.rs +++ b/util/src/network/discovery.rs @@ -209,7 +209,7 @@ impl Discovery { rlp.append(×tamp); let bytes = rlp.drain(); - let hash = bytes.sha3(); + let hash = bytes.as_ref().sha3(); let signature = match ec::sign(&self.secret, &hash) { Ok(s) => s, Err(_) => { diff --git a/util/src/network/node_table.rs b/util/src/network/node_table.rs index a3ee57481..dea18ab63 100644 --- a/util/src/network/node_table.rs +++ b/util/src/network/node_table.rs @@ -20,11 +20,17 @@ use std::net::{SocketAddr, ToSocketAddrs, SocketAddrV4, SocketAddrV6, Ipv4Addr, use std::hash::{Hash, Hasher}; use std::str::{FromStr}; use std::collections::HashMap; +use std::fmt::{Display, Formatter}; +use std::path::{PathBuf, Path}; +use std::fmt; +use std::fs; +use std::io::{Read, Write}; use hash::*; use rlp::*; use time::Tm; use error::*; use network::discovery::TableUpdates; +pub use rustc_serialize::json::Json; /// Node public key pub type NodeId = H512; @@ -135,6 +141,17 @@ impl Node { } } +impl Display for Node { + fn fmt(&self, f: &mut Formatter) -> fmt::Result { + if self.endpoint.udp_port != self.endpoint.address.port() { + write!(f, "enode://{}@{}+{}", self.id.hex(), self.endpoint.address, self.endpoint.udp_port); + } else { + write!(f, "enode://{}@{}", self.id.hex(), self.endpoint.address); + } + Ok(()) + } +} + impl FromStr for Node { type Err = UtilError; fn from_str(s: &str) -> Result { @@ -170,13 +187,15 @@ impl Hash for Node { /// Node table backed by disk file. pub struct NodeTable { - nodes: HashMap + nodes: HashMap, + path: Option, } impl NodeTable { - pub fn new(_path: Option) -> NodeTable { + pub fn new(path: Option) -> NodeTable { NodeTable { - nodes: HashMap::new() + path: path.clone(), + nodes: NodeTable::load(path), } } @@ -208,11 +227,86 @@ impl NodeTable { } } + /// Increase failure counte for a node pub fn note_failure(&mut self, id: &NodeId) { if let Some(node) = self.nodes.get_mut(id) { node.failures += 1; } } + + fn save(&self) { + if let Some(ref path) = self.path { + let mut path_buf = PathBuf::from(path); + path_buf.push("nodes.json"); + let mut json = String::new(); + json.push_str("{\n"); + json.push_str("nodes: [\n"); + let node_ids = self.nodes(); + for i in 0 .. node_ids.len() { + let node = self.nodes.get(&node_ids[i]).unwrap(); + json.push_str(&format!("\t{{ url: \"{}\", failures: {} }}{}\n", node, node.failures, if i == node_ids.len() - 1 {""} else {","})) + } + json.push_str("]\n"); + json.push_str("}"); + let mut file = match fs::File::create(path_buf.as_path()) { + Ok(file) => file, + Err(e) => { + warn!("Error creating node table file: {:?}", e); + return; + } + }; + if let Err(e) = file.write(&json.into_bytes()) { + warn!("Error writing node table file: {:?}", e); + } + } + } + + fn load(path: Option) -> HashMap { + let mut nodes: HashMap = HashMap::new(); + if let Some(path) = path { + let mut file = match fs::File::open(path.clone()) { + Ok(file) => file, + Err(e) => { + warn!("Error opening node table file: {:?}", e); + return nodes; + } + }; + let mut buf = String::new(); + match file.read_to_string(&mut buf) { + Ok(_) => {}, + Err(e) => { + warn!("Error reading node table file: {:?}", e); + return nodes; + } + } + let json = match Json::from_str(&buf) { + Ok(json) => json, + Err(e) => { + warn!("Error parsing node table file: {:?}", e); + return nodes; + } + }; + if let Some(list) = json.as_object().and_then(|o| o.get("nodes")).and_then(|n| n.as_array()) { + for n in list.iter().filter_map(|n| n.as_object()) { + if let Some(url) = n.get("url").and_then(|u| u.as_string()) { + if let Ok(mut node) = Node::from_str(url) { + if let Some(failures) = n.get("failures").and_then(|f| f.as_u64()) { + node.failures = failures as u32; + } + nodes.insert(node.id.clone(), node); + } + } + } + } + } + nodes + } +} + +impl Drop for NodeTable { + fn drop(&mut self) { + self.save(); + } } #[cfg(test)] @@ -247,4 +341,15 @@ mod tests { H512::from_str("a979fb575495b8d6db44f750317d0f4622bf4c2aa3365d6af7c284339968eef29b69ad0dce72a4d8db5ebb4968de0e3bec910127f134779fbcb0cb6d3331163c").unwrap(), node.id); } + + #[test] + fn table_failure_order() { + let node1 = Node::from_str("enode://a979fb575495b8d6db44f750317d0f4622bf4c2aa3365d6af7c284339968eef29b69ad0dce72a4d8db5ebb4968de0e3bec910127f134779fbcb0cb6d3331163c@22.99.55.44:7770"); + let node2 = Node::from_str("enode://b979fb575495b8d6db44f750317d0f4622bf4c2aa3365d6af7c284339968eef29b69ad0dce72a4d8db5ebb4968de0e3bec910127f134779fbcb0cb6d3331163c@22.99.55.44:7770"); + let node3 = Node::from_str("enode://c979fb575495b8d6db44f750317d0f4622bf4c2aa3365d6af7c284339968eef29b69ad0dce72a4d8db5ebb4968de0e3bec910127f134779fbcb0cb6d3331163c@22.99.55.44:7770"); + let id1 = H512::from_str("a979fb575495b8d6db44f750317d0f4622bf4c2aa3365d6af7c284339968eef29b69ad0dce72a4d8db5ebb4968de0e3bec910127f134779fbcb0cb6d3331163c").unwrap(); + let id2 = H512::from_str("b979fb575495b8d6db44f750317d0f4622bf4c2aa3365d6af7c284339968eef29b69ad0dce72a4d8db5ebb4968de0e3bec910127f134779fbcb0cb6d3331163c").unwrap(); + let id3 = H512::from_str("3979fb575495b8d6db44f750317d0f4622bf4c2aa3365d6af7c284339968eef29b69ad0dce72a4d8db5ebb4968de0e3bec910127f134779fbcb0cb6d3331163c").unwrap(); + let mut table = NodeTable::new(None); + } } diff --git a/util/src/sha3.rs b/util/src/sha3.rs index 7e8382250..4079e8ba4 100644 --- a/util/src/sha3.rs +++ b/util/src/sha3.rs @@ -66,7 +66,7 @@ impl Hashable for T where T: BytesConvertable { #[test] fn sha3_empty() { - assert_eq!([0u8; 0].sha3(), SHA3_EMPTY); + assert_eq!((&[0u8; 0]).sha3(), SHA3_EMPTY); } #[test] fn sha3_as() { From cf45d5914a3e355210f4fad7744d00c8e7b7976a Mon Sep 17 00:00:00 2001 From: arkpar Date: Mon, 15 Feb 2016 14:39:56 +0100 Subject: [PATCH 13/61] Node table tests --- util/src/network/discovery.rs | 2 +- util/src/network/mod.rs | 2 +- util/src/network/node_table.rs | 95 ++++++++++++++++++++++++++++++---- util/src/network/tests.rs | 2 +- 4 files changed, 88 insertions(+), 13 deletions(-) diff --git a/util/src/network/discovery.rs b/util/src/network/discovery.rs index 04c32cd33..c580c8507 100644 --- a/util/src/network/discovery.rs +++ b/util/src/network/discovery.rs @@ -254,7 +254,7 @@ impl Discovery { } let mut ret:Vec = Vec::new(); - for (_, nodes) in found { + for nodes in found.values() { ret.extend(nodes.iter().map(|&n| n.clone())); } ret diff --git a/util/src/network/mod.rs b/util/src/network/mod.rs index 7d5aac8f7..c5066ff99 100644 --- a/util/src/network/mod.rs +++ b/util/src/network/mod.rs @@ -56,7 +56,7 @@ //! } //! //! fn main () { -//! let mut service = NetworkService::::start(NetworkConfiguration::new()).expect("Error creating network service"); +//! let mut service = NetworkService::::start(NetworkConfiguration::new_with_port(40412)).expect("Error creating network service"); //! service.register_protocol(Arc::new(MyHandler), "myproto", &[1u8]); //! //! // Wait for quit condition diff --git a/util/src/network/node_table.rs b/util/src/network/node_table.rs index dea18ab63..2b02748b4 100644 --- a/util/src/network/node_table.rs +++ b/util/src/network/node_table.rs @@ -21,7 +21,7 @@ use std::hash::{Hash, Hasher}; use std::str::{FromStr}; use std::collections::HashMap; use std::fmt::{Display, Formatter}; -use std::path::{PathBuf, Path}; +use std::path::{PathBuf}; use std::fmt; use std::fs; use std::io::{Read, Write}; @@ -144,9 +144,9 @@ impl Node { impl Display for Node { fn fmt(&self, f: &mut Formatter) -> fmt::Result { if self.endpoint.udp_port != self.endpoint.address.port() { - write!(f, "enode://{}@{}+{}", self.id.hex(), self.endpoint.address, self.endpoint.udp_port); + try!(write!(f, "enode://{}@{}+{}", self.id.hex(), self.endpoint.address, self.endpoint.udp_port)); } else { - write!(f, "enode://{}@{}", self.id.hex(), self.endpoint.address); + try!(write!(f, "enode://{}@{}", self.id.hex(), self.endpoint.address)); } Ok(()) } @@ -237,14 +237,18 @@ impl NodeTable { fn save(&self) { if let Some(ref path) = self.path { let mut path_buf = PathBuf::from(path); + if let Err(e) = fs::create_dir_all(path_buf.as_path()) { + warn!("Error creating node table directory: {:?}", e); + return; + }; path_buf.push("nodes.json"); let mut json = String::new(); json.push_str("{\n"); - json.push_str("nodes: [\n"); + json.push_str("\"nodes\": [\n"); let node_ids = self.nodes(); for i in 0 .. node_ids.len() { let node = self.nodes.get(&node_ids[i]).unwrap(); - json.push_str(&format!("\t{{ url: \"{}\", failures: {} }}{}\n", node, node.failures, if i == node_ids.len() - 1 {""} else {","})) + json.push_str(&format!("\t{{ \"url\": \"{}\", \"failures\": {} }}{}\n", node, node.failures, if i == node_ids.len() - 1 {""} else {","})) } json.push_str("]\n"); json.push_str("}"); @@ -264,7 +268,9 @@ impl NodeTable { fn load(path: Option) -> HashMap { let mut nodes: HashMap = HashMap::new(); if let Some(path) = path { - let mut file = match fs::File::open(path.clone()) { + let mut path_buf = PathBuf::from(path); + path_buf.push("nodes.json"); + let mut file = match fs::File::open(path_buf.as_path()) { Ok(file) => file, Err(e) => { warn!("Error opening node table file: {:?}", e); @@ -344,12 +350,81 @@ mod tests { #[test] fn table_failure_order() { - let node1 = Node::from_str("enode://a979fb575495b8d6db44f750317d0f4622bf4c2aa3365d6af7c284339968eef29b69ad0dce72a4d8db5ebb4968de0e3bec910127f134779fbcb0cb6d3331163c@22.99.55.44:7770"); - let node2 = Node::from_str("enode://b979fb575495b8d6db44f750317d0f4622bf4c2aa3365d6af7c284339968eef29b69ad0dce72a4d8db5ebb4968de0e3bec910127f134779fbcb0cb6d3331163c@22.99.55.44:7770"); - let node3 = Node::from_str("enode://c979fb575495b8d6db44f750317d0f4622bf4c2aa3365d6af7c284339968eef29b69ad0dce72a4d8db5ebb4968de0e3bec910127f134779fbcb0cb6d3331163c@22.99.55.44:7770"); + let node1 = Node::from_str("enode://a979fb575495b8d6db44f750317d0f4622bf4c2aa3365d6af7c284339968eef29b69ad0dce72a4d8db5ebb4968de0e3bec910127f134779fbcb0cb6d3331163c@22.99.55.44:7770").unwrap(); + let node2 = Node::from_str("enode://b979fb575495b8d6db44f750317d0f4622bf4c2aa3365d6af7c284339968eef29b69ad0dce72a4d8db5ebb4968de0e3bec910127f134779fbcb0cb6d3331163c@22.99.55.44:7770").unwrap(); + let node3 = Node::from_str("enode://c979fb575495b8d6db44f750317d0f4622bf4c2aa3365d6af7c284339968eef29b69ad0dce72a4d8db5ebb4968de0e3bec910127f134779fbcb0cb6d3331163c@22.99.55.44:7770").unwrap(); let id1 = H512::from_str("a979fb575495b8d6db44f750317d0f4622bf4c2aa3365d6af7c284339968eef29b69ad0dce72a4d8db5ebb4968de0e3bec910127f134779fbcb0cb6d3331163c").unwrap(); let id2 = H512::from_str("b979fb575495b8d6db44f750317d0f4622bf4c2aa3365d6af7c284339968eef29b69ad0dce72a4d8db5ebb4968de0e3bec910127f134779fbcb0cb6d3331163c").unwrap(); - let id3 = H512::from_str("3979fb575495b8d6db44f750317d0f4622bf4c2aa3365d6af7c284339968eef29b69ad0dce72a4d8db5ebb4968de0e3bec910127f134779fbcb0cb6d3331163c").unwrap(); + let id3 = H512::from_str("c979fb575495b8d6db44f750317d0f4622bf4c2aa3365d6af7c284339968eef29b69ad0dce72a4d8db5ebb4968de0e3bec910127f134779fbcb0cb6d3331163c").unwrap(); let mut table = NodeTable::new(None); + table.add_node(node3); + table.add_node(node1); + table.add_node(node2); + + table.note_failure(&id1); + table.note_failure(&id1); + table.note_failure(&id2); + + let r = table.nodes(); + assert_eq!(r[0][..], id3[..]); + assert_eq!(r[1][..], id2[..]); + assert_eq!(r[2][..], id1[..]); + } + + use std::path::PathBuf; + use std::env; + use std::fs::{remove_dir_all}; + // TODO: use common impl + pub struct RandomTempPath { + path: PathBuf + } + + impl RandomTempPath { + pub fn new() -> RandomTempPath { + let mut dir = env::temp_dir(); + dir.push(H32::random().hex()); + RandomTempPath { + path: dir.clone() + } + } + + pub fn as_path(&self) -> &PathBuf { + &self.path + } + + pub fn as_str(&self) -> &str { + self.path.to_str().unwrap() + } + } + + impl Drop for RandomTempPath { + fn drop(&mut self) { + if let Err(e) = remove_dir_all(self.as_path()) { + panic!("failed to remove temp directory, probably something failed to destroyed ({})", e); + } + } + } + + + #[test] + fn table_save_load() { + let temp_path = RandomTempPath::new(); + let node1 = Node::from_str("enode://a979fb575495b8d6db44f750317d0f4622bf4c2aa3365d6af7c284339968eef29b69ad0dce72a4d8db5ebb4968de0e3bec910127f134779fbcb0cb6d3331163c@22.99.55.44:7770").unwrap(); + let node2 = Node::from_str("enode://b979fb575495b8d6db44f750317d0f4622bf4c2aa3365d6af7c284339968eef29b69ad0dce72a4d8db5ebb4968de0e3bec910127f134779fbcb0cb6d3331163c@22.99.55.44:7770").unwrap(); + let id1 = H512::from_str("a979fb575495b8d6db44f750317d0f4622bf4c2aa3365d6af7c284339968eef29b69ad0dce72a4d8db5ebb4968de0e3bec910127f134779fbcb0cb6d3331163c").unwrap(); + let id2 = H512::from_str("b979fb575495b8d6db44f750317d0f4622bf4c2aa3365d6af7c284339968eef29b69ad0dce72a4d8db5ebb4968de0e3bec910127f134779fbcb0cb6d3331163c").unwrap(); + { + let mut table = NodeTable::new(Some(temp_path.as_str().to_owned())); + table.add_node(node1); + table.add_node(node2); + table.note_failure(&id2); + } + + { + let table = NodeTable::new(Some(temp_path.as_str().to_owned())); + let r = table.nodes(); + assert_eq!(r[0][..], id1[..]); + assert_eq!(r[1][..], id2[..]); + } } } diff --git a/util/src/network/tests.rs b/util/src/network/tests.rs index c1b59df9b..a3d5290c9 100644 --- a/util/src/network/tests.rs +++ b/util/src/network/tests.rs @@ -85,7 +85,7 @@ impl NetworkProtocolHandler for TestProtocol { #[test] fn net_service() { - let mut service = NetworkService::::start(NetworkConfiguration::new()).expect("Error creating network service"); + let mut service = NetworkService::::start(NetworkConfiguration::new_with_port(40414)).expect("Error creating network service"); service.register_protocol(Arc::new(TestProtocol::default()), "myproto", &[1u8]).unwrap(); } From 4d40991c1a33effbc0b13fd2a0f71b6406118ff3 Mon Sep 17 00:00:00 2001 From: arkpar Date: Mon, 15 Feb 2016 16:01:45 +0100 Subject: [PATCH 14/61] Discovery test --- util/src/network/discovery.rs | 60 +++++++++++++++++++++++++++++++---- 1 file changed, 54 insertions(+), 6 deletions(-) diff --git a/util/src/network/discovery.rs b/util/src/network/discovery.rs index c580c8507..da8ce9e34 100644 --- a/util/src/network/discovery.rs +++ b/util/src/network/discovery.rs @@ -368,7 +368,7 @@ impl Discovery { // TODO: validate pong packet let dest = try!(NodeEndpoint::from_rlp(&try!(rlp.at(0)))); let timestamp: u64 = try!(rlp.val_at(2)); - if timestamp > time::get_time().sec as u64 { + if timestamp < time::get_time().sec as u64 { return Err(NetworkError::Expired); } let mut entry = NodeEntry { id: node.clone(), endpoint: dest }; @@ -386,7 +386,7 @@ impl Discovery { trace!(target: "discovery", "Got FindNode from {:?}", &from); let target: NodeId = try!(rlp.val_at(0)); let timestamp: u64 = try!(rlp.val_at(1)); - if timestamp > time::get_time().sec as u64 { + if timestamp < time::get_time().sec as u64 { return Err(NetworkError::Expired); } @@ -395,8 +395,8 @@ impl Discovery { if nearest.is_empty() { return Ok(None); } - let mut rlp = RlpStream::new_list(cmp::min(limit, nearest.len())); - rlp.begin_list(1); + let mut rlp = RlpStream::new_list(1); + rlp.begin_list(cmp::min(limit, nearest.len())); for n in 0 .. nearest.len() { rlp.begin_list(4); nearest[n].endpoint.to_rlp(&mut rlp); @@ -404,8 +404,8 @@ impl Discovery { if (n + 1) % limit == 0 || n == nearest.len() - 1 { self.send_packet(PACKET_NEIGHBOURS, &from, &rlp.drain()); trace!(target: "discovery", "Sent {} Neighbours to {:?}", n, &from); - rlp = RlpStream::new_list(cmp::min(limit, nearest.len() - n)); - rlp.begin_list(1); + rlp = RlpStream::new_list(1); + rlp.begin_list(cmp::min(limit, nearest.len() - n)); } } Ok(None) @@ -422,6 +422,9 @@ impl Discovery { continue; } let node_id: NodeId = try!(r.val_at(3)); + if node_id == self.id { + continue; + } let entry = NodeEntry { id: node_id.clone(), endpoint: endpoint }; added.insert(node_id, entry.clone()); self.ping(&entry.endpoint); @@ -476,3 +479,48 @@ impl Discovery { Ok(()) } } + +#[cfg(test)] +mod tests { + use super::*; + use hash::*; + use std::net::*; + use network::node_table::*; + use crypto::KeyPair; + use std::str::FromStr; + + #[test] + fn discovery() { + let key1 = KeyPair::create().unwrap(); + let key2 = KeyPair::create().unwrap(); + let ep1 = NodeEndpoint { address: SocketAddr::from_str("127.0.0.1:40444").unwrap(), udp_port: 40444 }; + let ep2 = NodeEndpoint { address: SocketAddr::from_str("127.0.0.1:40445").unwrap(), udp_port: 40445 }; + let mut discovery1 = Discovery::new(&key1, ep1.clone(), 0); + let mut discovery2 = Discovery::new(&key2, ep2.clone(), 0); + + let node1 = Node::from_str("enode://a979fb575495b8d6db44f750317d0f4622bf4c2aa3365d6af7c284339968eef29b69ad0dce72a4d8db5ebb4968de0e3bec910127f134779fbcb0cb6d3331163c@127.0.0.1:7770").unwrap(); + let node2 = Node::from_str("enode://b979fb575495b8d6db44f750317d0f4622bf4c2aa3365d6af7c284339968eef29b69ad0dce72a4d8db5ebb4968de0e3bec910127f134779fbcb0cb6d3331163c@127.0.0.1:7771").unwrap(); + discovery1.add_node(NodeEntry { id: node1.id.clone(), endpoint: node1. endpoint.clone() }); + discovery1.add_node(NodeEntry { id: node2.id.clone(), endpoint: node2. endpoint.clone() }); + + discovery2.add_node(NodeEntry { id: key1.public().clone(), endpoint: ep1.clone() }); + discovery2.refresh(); + + for _ in 0 .. 10 { + while !discovery1.send_queue.is_empty() { + let datagramm = discovery1.send_queue.pop_front().unwrap(); + if datagramm.address == ep2.address { + discovery2.on_packet(&datagramm.payload, ep1.address.clone()).ok(); + } + } + while !discovery2.send_queue.is_empty() { + let datagramm = discovery2.send_queue.pop_front().unwrap(); + if datagramm.address == ep1.address { + discovery1.on_packet(&datagramm.payload, ep2.address.clone()).ok(); + } + } + discovery2.round(); + } + assert_eq!(Discovery::nearest_node_entries(&NodeId::new(), &discovery2.node_buckets).len(), 3) + } +} From 0e1e80477a0efd49fac14a7e4ddfa6f86b216c8c Mon Sep 17 00:00:00 2001 From: arkpar Date: Mon, 15 Feb 2016 18:36:34 +0100 Subject: [PATCH 15/61] Save key to a file --- parity/main.rs | 4 +++ util/src/network/host.rs | 63 +++++++++++++++++++++++++++++++++- util/src/network/node_table.rs | 2 +- 3 files changed, 67 insertions(+), 2 deletions(-) diff --git a/parity/main.rs b/parity/main.rs index 460922b64..903b471c5 100644 --- a/parity/main.rs +++ b/parity/main.rs @@ -36,6 +36,7 @@ extern crate ethcore_rpc as rpc; use std::net::{SocketAddr}; use std::env; +use std::path::PathBuf; use rlog::{LogLevelFilter}; use env_logger::LogBuilder; use ctrlc::CtrlC; @@ -207,6 +208,9 @@ fn main() { net_settings.listen_address = listen; net_settings.public_address = public; net_settings.use_secret = conf.args.flag_node_key.as_ref().map(|s| Secret::from_str(&s).expect("Invalid key string")); + let mut net_path = PathBuf::from(&conf.path()); + net_path.push("network"); + net_settings.config_path = Some(net_path.to_str().unwrap().to_owned()); // Build client let mut service = ClientService::start(spec, net_settings, &Path::new(&conf.path())).unwrap(); diff --git a/util/src/network/host.rs b/util/src/network/host.rs index 7366e129b..fb3bcee62 100644 --- a/util/src/network/host.rs +++ b/util/src/network/host.rs @@ -21,6 +21,9 @@ use std::str::{FromStr}; use std::sync::*; use std::ops::*; use std::cmp::min; +use std::path::{Path, PathBuf}; +use std::io::{Read, Write}; +use std::fs; use mio::*; use mio::tcp::*; use target_info::Target; @@ -340,7 +343,19 @@ impl Host where Message: Send + Sync + Clone { let addr = config.listen_address; // Setup the server socket let tcp_listener = TcpListener::bind(&addr).unwrap(); - let keys = if let Some(ref secret) = config.use_secret { KeyPair::from_secret(secret.clone()).unwrap() } else { KeyPair::create().unwrap() }; + let keys = if let Some(ref secret) = config.use_secret { + KeyPair::from_secret(secret.clone()).unwrap() + } else { + config.config_path.clone().and_then(|ref p| load_key(&Path::new(&p))) + .map_or_else(|| { + let key = KeyPair::create().unwrap(); + if let Some(path) = config.config_path.clone() { + save_key(&Path::new(&path), &key.secret()); + } + key + }, + |s| KeyPair::from_secret(s).expect("Error creating node secret key")) + }; let endpoint = NodeEndpoint { address: addr.clone(), udp_port: addr.port() }; let discovery = Discovery::new(&keys, endpoint, DISCOVERY); let path = config.config_path.clone(); @@ -914,3 +929,49 @@ impl IoHandler> for Host where Messa } } } + +fn save_key(path: &Path, key: &Secret) { + let mut path_buf = PathBuf::from(path); + if let Err(e) = fs::create_dir_all(path_buf.as_path()) { + warn!("Error creating key directory: {:?}", e); + return; + }; + path_buf.push("key"); + let mut file = match fs::File::create(path_buf.as_path()) { + Ok(file) => file, + Err(e) => { + warn!("Error creating key file: {:?}", e); + return; + } + }; + if let Err(e) = file.write(&key.hex().into_bytes()) { + warn!("Error writing key file: {:?}", e); + } +} + +fn load_key(path: &Path) -> Option { + let mut path_buf = PathBuf::from(path); + path_buf.push("key"); + let mut file = match fs::File::open(path_buf.as_path()) { + Ok(file) => file, + Err(e) => { + debug!("Error opening key file: {:?}", e); + return None; + } + }; + let mut buf = String::new(); + match file.read_to_string(&mut buf) { + Ok(_) => {}, + Err(e) => { + warn!("Error reading key file: {:?}", e); + return None; + } + } + match Secret::from_str(&buf) { + Ok(key) => Some(key), + Err(e) => { + warn!("Error parsing key file: {:?}", e); + None + } + } +} diff --git a/util/src/network/node_table.rs b/util/src/network/node_table.rs index 2b02748b4..e5c47cafb 100644 --- a/util/src/network/node_table.rs +++ b/util/src/network/node_table.rs @@ -273,7 +273,7 @@ impl NodeTable { let mut file = match fs::File::open(path_buf.as_path()) { Ok(file) => file, Err(e) => { - warn!("Error opening node table file: {:?}", e); + debug!("Error opening node table file: {:?}", e); return nodes; } }; From 4b9c7f7517eb79e95d7c9326cca734508614b1e0 Mon Sep 17 00:00:00 2001 From: arkpar Date: Mon, 15 Feb 2016 19:54:27 +0100 Subject: [PATCH 16/61] Add incoming connection to node table --- util/src/network/connection.rs | 11 +++++++++++ util/src/network/host.rs | 9 +++++++++ util/src/network/node_table.rs | 5 ++++- util/src/network/session.rs | 7 +++++++ 4 files changed, 31 insertions(+), 1 deletion(-) diff --git a/util/src/network/connection.rs b/util/src/network/connection.rs index 242e8935e..9e9304ca6 100644 --- a/util/src/network/connection.rs +++ b/util/src/network/connection.rs @@ -16,6 +16,7 @@ use std::sync::Arc; use std::collections::VecDeque; +use std::net::SocketAddr; use mio::{Handler, Token, EventSet, EventLoop, PollOpt, TryRead, TryWrite}; use mio::tcp::*; use hash::*; @@ -169,6 +170,11 @@ impl Connection { self.token = token; } + /// Get remote peer address + pub fn remote_addr(&self) -> io::Result { + self.socket.peer_addr() + } + /// Register this connection with the IO event loop. pub fn register_socket(&self, reg: Token, event_loop: &mut EventLoop) -> io::Result<()> { trace!(target: "net", "connection register; token={:?}", reg); @@ -253,6 +259,11 @@ impl EncryptedConnection { self.connection.set_token(token); } + /// Get remote peer address + pub fn remote_addr(&self) -> io::Result { + self.connection.remote_addr() + } + /// Create an encrypted connection out of the handshake. Consumes a handshake object. pub fn new(mut handshake: Handshake) -> Result { let shared = try!(crypto::ecdh::agree(handshake.ecdhe.secret(), &handshake.remote_public)); diff --git a/util/src/network/host.rs b/util/src/network/host.rs index fb3bcee62..5f0bf19b9 100644 --- a/util/src/network/host.rs +++ b/util/src/network/host.rs @@ -663,6 +663,7 @@ impl Host where Message: Send + Sync + Clone { } } let h = Arc::try_unwrap(h).ok().unwrap().into_inner().unwrap(); + let originated = h.originated; let mut session = match Session::new(h, &self.info.read().unwrap()) { Ok(s) => s, Err(e) => { @@ -674,6 +675,14 @@ impl Host where Message: Send + Sync + Clone { session.set_token(session_token); io.update_registration(session_token).expect("Error updating session registration"); self.stats.inc_sessions(); + if !originated { + // Add it no node table + if let Ok(address) = session.remote_addr() { + let entry = NodeEntry { id: session.id().clone(), endpoint: NodeEndpoint { address: address, udp_port: address.port() } }; + self.nodes.write().unwrap().add_node(Node::new(entry.id.clone(), entry.endpoint.clone())); + self.discovery.lock().unwrap().add_node(entry); + } + } Arc::new(Mutex::new(session)) }); if result.is_none() { diff --git a/util/src/network/node_table.rs b/util/src/network/node_table.rs index e5c47cafb..40cc14743 100644 --- a/util/src/network/node_table.rs +++ b/util/src/network/node_table.rs @@ -200,7 +200,10 @@ impl NodeTable { } /// Add a node to table - pub fn add_node(&mut self, node: Node) { + pub fn add_node(&mut self, mut node: Node) { + // preserve failure counter + let failures = self.nodes.get(&node.id).map_or(0, |n| n.failures); + node.failures = failures; self.nodes.insert(node.id.clone(), node); } diff --git a/util/src/network/session.rs b/util/src/network/session.rs index 5cda00567..f08fef385 100644 --- a/util/src/network/session.rs +++ b/util/src/network/session.rs @@ -14,6 +14,8 @@ // You should have received a copy of the GNU General Public License // along with Parity. If not, see . +use std::net::SocketAddr; +use std::io; use mio::*; use hash::*; use rlp::*; @@ -144,6 +146,11 @@ impl Session { self.connection.set_token(token); } + /// Get remote peer address + pub fn remote_addr(&self) -> io::Result { + self.connection.remote_addr() + } + /// Readable IO handler. Returns packet data if available. pub fn readable(&mut self, io: &IoContext, host: &HostInfo) -> Result where Message: Send + Sync + Clone { match try!(self.connection.readable(io)) { From 01a83e603140e82faf9586c4c824fececa69a791 Mon Sep 17 00:00:00 2001 From: arkpar Date: Mon, 15 Feb 2016 20:28:27 +0100 Subject: [PATCH 17/61] Populate discovery from node table --- util/src/network/discovery.rs | 10 +++++++++- util/src/network/host.rs | 1 + util/src/network/node_table.rs | 8 +++++++- 3 files changed, 17 insertions(+), 2 deletions(-) diff --git a/util/src/network/discovery.rs b/util/src/network/discovery.rs index da8ce9e34..08f5e5cf1 100644 --- a/util/src/network/discovery.rs +++ b/util/src/network/discovery.rs @@ -110,12 +110,20 @@ impl Discovery { } } - pub fn add_node(&mut self, e: NodeEntry) { + /// Add a new node to discovery table. Pings the node. + pub fn add_node(&mut self, e: NodeEntry) { let endpoint = e.endpoint.clone(); self.update_node(e); self.ping(&endpoint); } + /// Add a list of known nodes to the table. + pub fn init_node_list(&mut self, mut nodes: Vec) { + for n in nodes.drain(..) { + self.update_node(n); + } + } + fn update_node(&mut self, e: NodeEntry) { trace!(target: "discovery", "Inserting {:?}", &e); let ping = { diff --git a/util/src/network/host.rs b/util/src/network/host.rs index 5f0bf19b9..c548d07cf 100644 --- a/util/src/network/host.rs +++ b/util/src/network/host.rs @@ -386,6 +386,7 @@ impl Host where Message: Send + Sync + Clone { for n in boot_nodes { host.add_node(&n); } + host.discovery.lock().unwrap().init_node_list(host.nodes.read().unwrap().unordered_entries()); host } diff --git a/util/src/network/node_table.rs b/util/src/network/node_table.rs index 40cc14743..9bce2d334 100644 --- a/util/src/network/node_table.rs +++ b/util/src/network/node_table.rs @@ -29,7 +29,7 @@ use hash::*; use rlp::*; use time::Tm; use error::*; -use network::discovery::TableUpdates; +use network::discovery::{TableUpdates, NodeEntry}; pub use rustc_serialize::json::Json; /// Node public key @@ -214,6 +214,12 @@ impl NodeTable { refs.iter().map(|n| n.id.clone()).collect() } + /// Unordered list of all entries + pub fn unordered_entries(&self) -> Vec { + // preserve failure counter + self.nodes.values().map(|n| NodeEntry { endpoint: n.endpoint.clone(), id: n.id.clone() }).collect() + } + /// Get particular node pub fn get_mut(&mut self, id: &NodeId) -> Option<&mut Node> { self.nodes.get_mut(id) From 0bef355494754507efc546bf80dc5634367c546e Mon Sep 17 00:00:00 2001 From: arkpar Date: Mon, 15 Feb 2016 20:34:05 +0100 Subject: [PATCH 18/61] Removed temp test code --- util/src/network/node_table.rs | 42 ++++------------------------------ 1 file changed, 4 insertions(+), 38 deletions(-) diff --git a/util/src/network/node_table.rs b/util/src/network/node_table.rs index 9bce2d334..69d12dc87 100644 --- a/util/src/network/node_table.rs +++ b/util/src/network/node_table.rs @@ -330,6 +330,7 @@ mod tests { use std::str::FromStr; use std::net::*; use hash::*; + use tests::helpers::*; #[test] fn endpoint_parse() { @@ -380,57 +381,22 @@ mod tests { assert_eq!(r[2][..], id1[..]); } - use std::path::PathBuf; - use std::env; - use std::fs::{remove_dir_all}; - // TODO: use common impl - pub struct RandomTempPath { - path: PathBuf - } - - impl RandomTempPath { - pub fn new() -> RandomTempPath { - let mut dir = env::temp_dir(); - dir.push(H32::random().hex()); - RandomTempPath { - path: dir.clone() - } - } - - pub fn as_path(&self) -> &PathBuf { - &self.path - } - - pub fn as_str(&self) -> &str { - self.path.to_str().unwrap() - } - } - - impl Drop for RandomTempPath { - fn drop(&mut self) { - if let Err(e) = remove_dir_all(self.as_path()) { - panic!("failed to remove temp directory, probably something failed to destroyed ({})", e); - } - } - } - - #[test] fn table_save_load() { - let temp_path = RandomTempPath::new(); + let temp_path = RandomTempPath::create_dir(); let node1 = Node::from_str("enode://a979fb575495b8d6db44f750317d0f4622bf4c2aa3365d6af7c284339968eef29b69ad0dce72a4d8db5ebb4968de0e3bec910127f134779fbcb0cb6d3331163c@22.99.55.44:7770").unwrap(); let node2 = Node::from_str("enode://b979fb575495b8d6db44f750317d0f4622bf4c2aa3365d6af7c284339968eef29b69ad0dce72a4d8db5ebb4968de0e3bec910127f134779fbcb0cb6d3331163c@22.99.55.44:7770").unwrap(); let id1 = H512::from_str("a979fb575495b8d6db44f750317d0f4622bf4c2aa3365d6af7c284339968eef29b69ad0dce72a4d8db5ebb4968de0e3bec910127f134779fbcb0cb6d3331163c").unwrap(); let id2 = H512::from_str("b979fb575495b8d6db44f750317d0f4622bf4c2aa3365d6af7c284339968eef29b69ad0dce72a4d8db5ebb4968de0e3bec910127f134779fbcb0cb6d3331163c").unwrap(); { - let mut table = NodeTable::new(Some(temp_path.as_str().to_owned())); + let mut table = NodeTable::new(Some(temp_path.as_path().to_str().unwrap().to_owned())); table.add_node(node1); table.add_node(node2); table.note_failure(&id2); } { - let table = NodeTable::new(Some(temp_path.as_str().to_owned())); + let table = NodeTable::new(Some(temp_path.as_path().to_str().unwrap().to_owned())); let r = table.nodes(); assert_eq!(r[0][..], id1[..]); assert_eq!(r[1][..], id2[..]); From 64913d5009816e23adff19fdb96e402baa8f6952 Mon Sep 17 00:00:00 2001 From: arkpar Date: Mon, 15 Feb 2016 21:43:30 +0100 Subject: [PATCH 19/61] Additional address filter --- util/src/network/discovery.rs | 6 +++++- util/src/network/host.rs | 8 ++++---- util/src/network/node_table.rs | 7 +++++++ 3 files changed, 16 insertions(+), 5 deletions(-) diff --git a/util/src/network/discovery.rs b/util/src/network/discovery.rs index 08f5e5cf1..8ed49b274 100644 --- a/util/src/network/discovery.rs +++ b/util/src/network/discovery.rs @@ -356,7 +356,11 @@ impl Discovery { } let mut entry = NodeEntry { id: node.clone(), endpoint: source.clone() }; if !entry.endpoint.is_valid() { - debug!(target: "discovery", "Bad address: {:?}", entry); + debug!(target: "discovery", "Got bad address: {:?}, using {:?} instead", entry, from); + entry.endpoint.address = from.clone(); + } + if !entry.endpoint.is_global() { + debug!(target: "discovery", "Got local address: {:?}, using {:?} instead", entry, from); entry.endpoint.address = from.clone(); } self.update_node(entry.clone()); diff --git a/util/src/network/host.rs b/util/src/network/host.rs index c548d07cf..806ef2b92 100644 --- a/util/src/network/host.rs +++ b/util/src/network/host.rs @@ -39,7 +39,7 @@ use network::{NetworkProtocolHandler, PROTOCOL_VERSION}; use network::node_table::*; use network::stats::NetworkStats; use network::error::DisconnectReason; -use igd::{PortMappingProtocol,search_gateway}; +use igd::{PortMappingProtocol, search_gateway}; use network::discovery::{Discovery, TableUpdates, NodeEntry}; type Slab = ::slab::Slab; @@ -105,12 +105,12 @@ impl NetworkConfiguration { if self.nat_enabled { info!("Enabling NAT..."); match search_gateway() { - Err(ref err) => info!("Error: {}", err), + Err(ref err) => warn!("Port mapping error: {}", err), Ok(gateway) => { let int_addr = SocketAddrV4::from_str("127.0.0.1:30304").unwrap(); match gateway.get_any_address(PortMappingProtocol::TCP, int_addr, 0, "Parity Node/TCP") { Err(ref err) => { - info!("There was an error! {}", err); + warn!("Port mapping error: {}", err); }, Ok(ext_addr) => { info!("Local gateway: {}, External ip address: {}", gateway, ext_addr); @@ -356,7 +356,7 @@ impl Host where Message: Send + Sync + Clone { }, |s| KeyPair::from_secret(s).expect("Error creating node secret key")) }; - let endpoint = NodeEndpoint { address: addr.clone(), udp_port: addr.port() }; + let endpoint = NodeEndpoint { address: config.public_address.clone(), udp_port: addr.port() }; let discovery = Discovery::new(&keys, endpoint, DISCOVERY); let path = config.config_path.clone(); let mut host = Host:: { diff --git a/util/src/network/node_table.rs b/util/src/network/node_table.rs index 69d12dc87..c152efd4f 100644 --- a/util/src/network/node_table.rs +++ b/util/src/network/node_table.rs @@ -96,6 +96,13 @@ impl NodeEndpoint { SocketAddr::V6(a) => !a.ip().is_unspecified() } } + + pub fn is_global(&self) -> bool { + match self.address { + SocketAddr::V4(a) => a.ip().is_global(), + SocketAddr::V6(a) => a.ip().is_global() + } + } } impl FromStr for NodeEndpoint { From fb0b5b2e5ba49ed0799e9efed0978be3cf981273 Mon Sep 17 00:00:00 2001 From: arkpar Date: Tue, 16 Feb 2016 00:22:44 +0100 Subject: [PATCH 20/61] Raise fd limit in linux --- util/fdlimit/src/raise_fd_limit.rs | 25 ++++++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/util/fdlimit/src/raise_fd_limit.rs b/util/fdlimit/src/raise_fd_limit.rs index f57ac2785..92127da35 100644 --- a/util/fdlimit/src/raise_fd_limit.rs +++ b/util/fdlimit/src/raise_fd_limit.rs @@ -57,5 +57,28 @@ pub unsafe fn raise_fd_limit() { } } -#[cfg(not(any(target_os = "macos", target_os = "ios")))] +#[cfg(any(target_os = "linux"))] +#[allow(non_camel_case_types)] +pub unsafe fn raise_fd_limit() { + use libc; + use std::io; + + // Fetch the current resource limits + let mut rlim = libc::rlimit{rlim_cur: 0, rlim_max: 0}; + if libc::getrlimit(libc::RLIMIT_NOFILE, &mut rlim) != 0 { + let err = io::Error::last_os_error(); + panic!("raise_fd_limit: error calling getrlimit: {}", err); + } + + // Set soft limit to hard imit + rlim.rlim_cur = rlim.rlim_max; + + // Set our newly-increased resource limit + if libc::setrlimit(libc::RLIMIT_NOFILE, &rlim) != 0 { + let err = io::Error::last_os_error(); + panic!("raise_fd_limit: error calling setrlimit: {}", err); + } +} + +#[cfg(not(any(target_os = "macos", target_os = "ios", target_os = "linux")))] pub unsafe fn raise_fd_limit() {} From dbf3691c22870e7079d5976c19658c70232ad363 Mon Sep 17 00:00:00 2001 From: arkpar Date: Tue, 16 Feb 2016 01:13:13 +0100 Subject: [PATCH 21/61] Return nothing on state requests instead of panicing --- ethcore/src/client.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ethcore/src/client.rs b/ethcore/src/client.rs index 09f7417e8..03c470bdb 100644 --- a/ethcore/src/client.rs +++ b/ethcore/src/client.rs @@ -419,11 +419,11 @@ impl BlockChainClient for Client { } fn state_data(&self, _hash: &H256) -> Option { - unimplemented!(); + None } fn block_receipts(&self, _hash: &H256) -> Option { - unimplemented!(); + None } fn import_block(&self, bytes: Bytes) -> ImportResult { From 203947388b9f7fe6a6dda5e5e55c76c3fe66646a Mon Sep 17 00:00:00 2001 From: arkpar Date: Tue, 16 Feb 2016 02:05:36 +0100 Subject: [PATCH 22/61] Get public address/UPNP refactoring --- Cargo.lock | 1 + parity/main.rs | 40 ++++++++----- util/Cargo.toml | 1 + util/src/lib.rs | 1 + util/src/network/discovery.rs | 14 ++--- util/src/network/host.rs | 105 +++++++++++++++------------------- util/src/network/mod.rs | 1 + util/src/network/service.rs | 2 +- 8 files changed, 84 insertions(+), 81 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index c451a6477..2e38f5aeb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -214,6 +214,7 @@ dependencies = [ "itertools 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", "json-tests 0.1.0", "lazy_static 0.1.15 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.7 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)", "mio 0.5.0 (registry+https://github.com/rust-lang/crates.io-index)", "rand 0.3.14 (registry+https://github.com/rust-lang/crates.io-index)", diff --git a/parity/main.rs b/parity/main.rs index 903b471c5..5fec05eb4 100644 --- a/parity/main.rs +++ b/parity/main.rs @@ -64,8 +64,8 @@ Options: -d --db-path PATH Specify the database & configuration directory path [default: $HOME/.parity] --no-bootstrap Don't bother trying to connect to any nodes initially. - --listen-address URL Specify the IP/port on which to listen for peers [default: 0.0.0.0:30304]. - --public-address URL Specify the IP/port on which peers may connect [default: 0.0.0.0:30304]. + --listen-address URL Specify the IP/port on which to listen for peers. + --public-address URL Specify the IP/port on which peers may connect. --address URL Equivalent to --listen-address URL --public-address URL. --upnp Use UPnP to try to figure out the correct network settings. --node-key KEY Specify node secret key as hex string. @@ -79,7 +79,12 @@ Options: -l --logging LOGGING Specify the logging level. -v --version Show information about version. -h --help Show this screen. -", flag_cache_pref_size: usize, flag_cache_max_size: usize, flag_address: Option, flag_node_key: Option); +", flag_cache_pref_size: usize, + flag_cache_max_size: usize, + flag_address: Option, + flag_listen_address: Option, + flag_public_address: Option, + flag_node_key: Option); fn setup_log(init: &str) { let mut builder = LogBuilder::new(); @@ -155,21 +160,26 @@ impl Configuration { } } - fn net_addresses(&self) -> (SocketAddr, SocketAddr) { - let listen_address; - let public_address; + fn net_addresses(&self) -> (Option, Option) { + let mut listen_address = None; + let mut public_address = None; - match self.args.flag_address { - None => { - listen_address = SocketAddr::from_str(self.args.flag_listen_address.as_ref()).expect("Invalid listen address given with --listen-address"); - public_address = SocketAddr::from_str(self.args.flag_public_address.as_ref()).expect("Invalid public address given with --public-address"); + if let Some(ref a) = self.args.flag_address { + public_address = Some(SocketAddr::from_str(a.as_ref()).expect("Invalid listen/public address given with --address")); + listen_address = public_address; + } + if let Some(ref a) = self.args.flag_listen_address { + if listen_address.is_some() { + panic!("Conflicting flags: --address and --listen-address"); } - Some(ref a) => { - public_address = SocketAddr::from_str(a.as_ref()).expect("Invalid listen/public address given with --address"); - listen_address = public_address; + listen_address = Some(SocketAddr::from_str(a.as_ref()).expect("Invalid listen address given with --listen-address")); + } + if let Some(ref a) = self.args.flag_public_address { + if public_address.is_some() { + panic!("Conflicting flags: --address and --public-address"); } - }; - + public_address = Some(SocketAddr::from_str(a.as_ref()).expect("Invalid listen address given with --public-address")); + } (listen_address, public_address) } } diff --git a/util/Cargo.toml b/util/Cargo.toml index b1e9bbc1e..5bdf8c5a6 100644 --- a/util/Cargo.toml +++ b/util/Cargo.toml @@ -30,3 +30,4 @@ clippy = "0.0.41" json-tests = { path = "json-tests" } target_info = "0.1.0" igd = "0.4.2" +libc = "0.2.7" diff --git a/util/src/lib.rs b/util/src/lib.rs index 6dde49a01..7592cd17a 100644 --- a/util/src/lib.rs +++ b/util/src/lib.rs @@ -110,6 +110,7 @@ extern crate serde; #[macro_use] extern crate log as rlog; extern crate igd; +extern crate libc; pub mod standard; #[macro_use] diff --git a/util/src/network/discovery.rs b/util/src/network/discovery.rs index 8ed49b274..c15dbbbc4 100644 --- a/util/src/network/discovery.rs +++ b/util/src/network/discovery.rs @@ -78,7 +78,7 @@ struct Datagramm { pub struct Discovery { id: NodeId, secret: Secret, - address: NodeEndpoint, + public_endpoint: NodeEndpoint, udp_socket: UdpSocket, token: StreamToken, discovery_round: u16, @@ -94,12 +94,12 @@ pub struct TableUpdates { } impl Discovery { - pub fn new(key: &KeyPair, address: NodeEndpoint, token: StreamToken) -> Discovery { - let socket = UdpSocket::bound(&address.udp_address()).expect("Error binding UDP socket"); + pub fn new(key: &KeyPair, listen: SocketAddr, public: NodeEndpoint, token: StreamToken) -> Discovery { + let socket = UdpSocket::bound(&listen).expect("Error binding UDP socket"); Discovery { id: key.public().clone(), secret: key.secret().clone(), - address: address, + public_endpoint: public, token: token, discovery_round: 0, discovery_id: NodeId::new(), @@ -199,7 +199,7 @@ impl Discovery { fn ping(&mut self, node: &NodeEndpoint) { let mut rlp = RlpStream::new_list(3); rlp.append(&PROTOCOL_VERSION); - self.address.to_rlp_list(&mut rlp); + self.public_endpoint.to_rlp_list(&mut rlp); node.to_rlp_list(&mut rlp); trace!(target: "discovery", "Sent Ping to {:?}", &node); self.send_packet(PACKET_PING, &node.udp_address(), &rlp.drain()); @@ -507,8 +507,8 @@ mod tests { let key2 = KeyPair::create().unwrap(); let ep1 = NodeEndpoint { address: SocketAddr::from_str("127.0.0.1:40444").unwrap(), udp_port: 40444 }; let ep2 = NodeEndpoint { address: SocketAddr::from_str("127.0.0.1:40445").unwrap(), udp_port: 40445 }; - let mut discovery1 = Discovery::new(&key1, ep1.clone(), 0); - let mut discovery2 = Discovery::new(&key2, ep2.clone(), 0); + let mut discovery1 = Discovery::new(&key1, ep1.address.clone(), ep1.clone(), 0); + let mut discovery2 = Discovery::new(&key2, ep2.address.clone(), ep2.clone(), 0); let node1 = Node::from_str("enode://a979fb575495b8d6db44f750317d0f4622bf4c2aa3365d6af7c284339968eef29b69ad0dce72a4d8db5ebb4968de0e3bec910127f134779fbcb0cb6d3331163c@127.0.0.1:7770").unwrap(); let node2 = Node::from_str("enode://b979fb575495b8d6db44f750317d0f4622bf4c2aa3365d6af7c284339968eef29b69ad0dce72a4d8db5ebb4968de0e3bec910127f134779fbcb0cb6d3331163c@127.0.0.1:7771").unwrap(); diff --git a/util/src/network/host.rs b/util/src/network/host.rs index 806ef2b92..3d55430bd 100644 --- a/util/src/network/host.rs +++ b/util/src/network/host.rs @@ -14,7 +14,7 @@ // You should have received a copy of the GNU General Public License // along with Parity. If not, see . -use std::net::{SocketAddr, SocketAddrV4}; +use std::net::{SocketAddr}; use std::collections::{HashMap}; use std::hash::{Hasher}; use std::str::{FromStr}; @@ -39,8 +39,8 @@ use network::{NetworkProtocolHandler, PROTOCOL_VERSION}; use network::node_table::*; use network::stats::NetworkStats; use network::error::DisconnectReason; -use igd::{PortMappingProtocol, search_gateway}; use network::discovery::{Discovery, TableUpdates, NodeEntry}; +use network::ip_utils::{map_external_address, select_public_address}; type Slab = ::slab::Slab; @@ -55,10 +55,12 @@ const MAINTENANCE_TIMEOUT: u64 = 1000; pub struct NetworkConfiguration { /// Directory path to store network configuration. None means nothing will be saved pub config_path: Option, - /// IP address to listen for incoming connections - pub listen_address: SocketAddr, - /// IP address to advertise - pub public_address: SocketAddr, + /// IP address to listen for incoming connections. Listen to all connections by default + pub listen_address: Option, + /// IP address to advertise. Detected automatically if none. + pub public_address: Option, + /// Port for UDP connections, same as TCP by default + pub udp_port: Option, /// Enable NAT configuration pub nat_enabled: bool, /// Enable discovery @@ -78,8 +80,9 @@ impl NetworkConfiguration { pub fn new() -> NetworkConfiguration { NetworkConfiguration { config_path: None, - listen_address: SocketAddr::from_str("0.0.0.0:30304").unwrap(), - public_address: SocketAddr::from_str("0.0.0.0:30304").unwrap(), + listen_address: None, + public_address: None, + udp_port: None, nat_enabled: true, discovery_enabled: true, pin: false, @@ -92,48 +95,9 @@ impl NetworkConfiguration { /// Create new default configuration with sepcified listen port. pub fn new_with_port(port: u16) -> NetworkConfiguration { let mut config = NetworkConfiguration::new(); - config.listen_address = SocketAddr::from_str(&format!("0.0.0.0:{}", port)).unwrap(); - config.public_address = SocketAddr::from_str(&format!("0.0.0.0:{}", port)).unwrap(); + config.listen_address = Some(SocketAddr::from_str(&format!("0.0.0.0:{}", port)).unwrap()); config } - - /// Conduct NAT if needed. - pub fn prepared(self) -> Self { - let mut listen = self.listen_address; - let mut public = self.public_address; - - if self.nat_enabled { - info!("Enabling NAT..."); - match search_gateway() { - Err(ref err) => warn!("Port mapping error: {}", err), - Ok(gateway) => { - let int_addr = SocketAddrV4::from_str("127.0.0.1:30304").unwrap(); - match gateway.get_any_address(PortMappingProtocol::TCP, int_addr, 0, "Parity Node/TCP") { - Err(ref err) => { - warn!("Port mapping error: {}", err); - }, - Ok(ext_addr) => { - info!("Local gateway: {}, External ip address: {}", gateway, ext_addr); - public = SocketAddr::V4(ext_addr); - listen = SocketAddr::V4(int_addr); - }, - } - }, - } - } - - NetworkConfiguration { - config_path: self.config_path, - listen_address: listen, - public_address: public, - nat_enabled: false, - discovery_enabled: self.discovery_enabled, - pin: self.pin, - boot_nodes: self.boot_nodes, - use_secret: self.use_secret, - ideal_peers: self.ideal_peers, - } - } } // Tokens @@ -333,16 +297,39 @@ pub struct Host where Message: Send + Sync + Clone { timers: RwLock>, timer_counter: RwLock, stats: Arc, + public_endpoint: NodeEndpoint, } impl Host where Message: Send + Sync + Clone { /// Create a new instance pub fn new(config: NetworkConfiguration) -> Host { - let config = config.prepared(); + let listen_address = match config.listen_address { + None => SocketAddr::from_str("0.0.0.0:30304").unwrap(), + Some(addr) => addr, + }; + + let udp_port = config.udp_port.unwrap_or(listen_address.port()); + let public_endpoint = match config.public_address { + None => { + let public_address = select_public_address(listen_address.port()); + let local_endpoint = NodeEndpoint { address: public_address, udp_port: udp_port }; + if config.nat_enabled { + match map_external_address(&local_endpoint) { + Some(endpoint) => { + info!("NAT Mappped to external address {}", endpoint.address); + endpoint + }, + None => local_endpoint + } + } else { + local_endpoint + } + } + Some(addr) => NodeEndpoint { address: addr, udp_port: udp_port } + }; - let addr = config.listen_address; // Setup the server socket - let tcp_listener = TcpListener::bind(&addr).unwrap(); + let tcp_listener = TcpListener::bind(&listen_address).unwrap(); let keys = if let Some(ref secret) = config.use_secret { KeyPair::from_secret(secret.clone()).unwrap() } else { @@ -356,8 +343,7 @@ impl Host where Message: Send + Sync + Clone { }, |s| KeyPair::from_secret(s).expect("Error creating node secret key")) }; - let endpoint = NodeEndpoint { address: config.public_address.clone(), udp_port: addr.port() }; - let discovery = Discovery::new(&keys, endpoint, DISCOVERY); + let discovery = Discovery::new(&keys, listen_address.clone(), public_endpoint.clone(), DISCOVERY); let path = config.config_path.clone(); let mut host = Host:: { info: RwLock::new(HostInfo { @@ -378,8 +364,9 @@ impl Host where Message: Send + Sync + Clone { timers: RwLock::new(HashMap::new()), timer_counter: RwLock::new(USER_TIMER), stats: Arc::new(NetworkStats::default()), + public_endpoint: public_endpoint, }; - let port = host.info.read().unwrap().config.listen_address.port(); + let port = listen_address.port(); host.info.write().unwrap().deref_mut().listen_port = port; let boot_nodes = host.info.read().unwrap().config.boot_nodes.clone(); @@ -409,8 +396,8 @@ impl Host where Message: Send + Sync + Clone { self.info.read().unwrap().client_version.clone() } - pub fn client_id(&self) -> NodeId { - self.info.read().unwrap().id().clone() + pub fn client_url(&self) -> String { + format!("{}", Node::new(self.info.read().unwrap().id().clone(), self.public_endpoint.clone())) } fn maintain_network(&self, io: &IoContext>) { @@ -456,13 +443,15 @@ impl Host where Message: Send + Sync + Clone { } let handshake_count = self.handshake_count(); - if handshake_count >= MAX_HANDSHAKES { + // allow 16 slots for incoming connections + let handshake_limit = MAX_HANDSHAKES - 16; + if handshake_count >= handshake_limit { return; } let nodes = { self.nodes.read().unwrap().nodes() }; for id in nodes.iter().filter(|ref id| !self.have_session(id) && !self.connecting_to(id)) - .take(min(MAX_HANDSHAKES_PER_ROUND, MAX_HANDSHAKES - handshake_count)) { + .take(min(MAX_HANDSHAKES_PER_ROUND, handshake_limit - handshake_count)) { self.connect_peer(&id, io); } debug!(target: "net", "Connecting peers: {} sessions, {} pending", self.session_count(), self.handshake_count()); diff --git a/util/src/network/mod.rs b/util/src/network/mod.rs index c5066ff99..ff52212af 100644 --- a/util/src/network/mod.rs +++ b/util/src/network/mod.rs @@ -73,6 +73,7 @@ mod service; mod error; mod node_table; mod stats; +mod ip_utils; #[cfg(test)] mod tests; diff --git a/util/src/network/service.rs b/util/src/network/service.rs index 60f0ec415..1cd48abe1 100644 --- a/util/src/network/service.rs +++ b/util/src/network/service.rs @@ -42,7 +42,7 @@ impl NetworkService where Message: Send + Sync + Clone + 'stat let host = Arc::new(Host::new(config)); let stats = host.stats().clone(); let host_info = host.client_version(); - info!("Host ID={:?}", host.client_id()); + info!("Node URL: {}", host.client_url()); try!(io_service.register_handler(host)); Ok(NetworkService { io_service: io_service, From f771306867121d4dbe30c289305994dba50c9fed Mon Sep 17 00:00:00 2001 From: arkpar Date: Tue, 16 Feb 2016 02:05:45 +0100 Subject: [PATCH 23/61] Get public address/UPNP refactoring --- util/src/network/ip_utils.rs | 176 +++++++++++++++++++++++++++++++++++ 1 file changed, 176 insertions(+) create mode 100644 util/src/network/ip_utils.rs diff --git a/util/src/network/ip_utils.rs b/util/src/network/ip_utils.rs new file mode 100644 index 000000000..8f94073b3 --- /dev/null +++ b/util/src/network/ip_utils.rs @@ -0,0 +1,176 @@ +// Copyright 2015, 2016 Ethcore (UK) Ltd. +// This file is part of Parity. + +// Parity is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Parity is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Parity. If not, see . + +// Based on original work by David Levy https://raw.githubusercontent.com/dlevy47/rust-interfaces + +use std::net::{Ipv4Addr, Ipv6Addr, SocketAddr, SocketAddrV4, SocketAddrV6}; +use std::io; +use igd::{PortMappingProtocol, search_gateway_from_timeout}; +use std::time::Duration; +use network::node_table::{NodeEndpoint}; + +pub enum IpAddr{ + V4(Ipv4Addr), + V6(Ipv6Addr), +} + +#[cfg(not(windows))] +mod getinterfaces { + use std::{mem, io, ptr}; + use libc::{AF_INET, AF_INET6}; + use libc::{getifaddrs, freeifaddrs, ifaddrs, sockaddr, sockaddr_in, sockaddr_in6}; + use std::net::{Ipv4Addr, Ipv6Addr}; + use super::IpAddr; + + fn convert_sockaddr (sa: *mut sockaddr) -> Option { + if sa == ptr::null_mut() { return None; } + + let (addr, _) = match unsafe { *sa }.sa_family as i32 { + AF_INET => { + let sa: *const sockaddr_in = unsafe { mem::transmute(sa) }; + let sa = & unsafe { *sa }; + let (addr, port) = (sa.sin_addr.s_addr, sa.sin_port); + ( + IpAddr::V4(Ipv4Addr::new( + (addr & 0x000000FF) as u8, + ((addr & 0x0000FF00) >> 8) as u8, + ((addr & 0x00FF0000) >> 16) as u8, + ((addr & 0xFF000000) >> 24) as u8, + )), + port + ) + }, + AF_INET6 => { + let sa: *const sockaddr_in6 = unsafe { mem::transmute(sa) }; + let sa = & unsafe { *sa }; + let (addr, port) = (sa.sin6_addr.s6_addr, sa.sin6_port); + let addr: [u16; 8] = unsafe { mem::transmute(addr) }; + ( + IpAddr::V6(Ipv6Addr::new( + addr[0], + addr[1], + addr[2], + addr[3], + addr[4], + addr[5], + addr[6], + addr[7], + )), + port + ) + }, + _ => return None, + }; + Some(addr) + } + + fn convert_ifaddrs (ifa: *mut ifaddrs) -> Option { + let ifa = unsafe { &mut *ifa }; + convert_sockaddr(ifa.ifa_addr) + } + + pub fn get_all() -> io::Result> { + let mut ifap: *mut ifaddrs = unsafe { mem::zeroed() }; + if unsafe { getifaddrs(&mut ifap as *mut _) } != 0 { + return Err(io::Error::last_os_error()); + } + + let mut ret = Vec::new(); + let mut cur: *mut ifaddrs = ifap; + while cur != ptr::null_mut() { + if let Some(ip_addr) = convert_ifaddrs(cur) { + ret.push(ip_addr); + } + + //TODO: do something else maybe? + cur = unsafe { (*cur).ifa_next }; + } + + unsafe { freeifaddrs(ifap) }; + Ok(ret) + } +} + +#[cfg(not(windows))] +fn get_if_addrs() -> io::Result> { + getinterfaces::get_all() +} + +#[cfg(windows)] +fn get_if_addrs() -> io::Result> { + Ok(Vec::new()) +} + +/// Select the best available public address +pub fn select_public_address(port: u16) -> SocketAddr { + match get_if_addrs() { + Ok(list) => { + //prefer IPV4 bindings + for addr in &list { //TODO: use better criteria than just the first in the list + match *addr { + IpAddr::V4(a) if !a.is_unspecified() && !a.is_loopback() && !a.is_link_local() => { + return SocketAddr::V4(SocketAddrV4::new(a, port)); + }, + _ => {}, + } + } + for addr in list { + match addr { + IpAddr::V6(a) if !a.is_unspecified() && !a.is_loopback() => { + return SocketAddr::V6(SocketAddrV6::new(a, port, 0, 0)); + }, + _ => {}, + } + } + }, + Err(e) => debug!("Error listing public interfaces: {:?}", e) + } + SocketAddr::V4(SocketAddrV4::new(Ipv4Addr::new(127, 0, 0, 1), port)) +} + +pub fn map_external_address(local: &NodeEndpoint) -> Option { + if let SocketAddr::V4(ref local_addr) = local.address { + match search_gateway_from_timeout(local_addr.ip().clone(), Duration::new(5, 0)) { + Err(ref err) => debug!("Gateway search error: {}", err), + Ok(gateway) => { + match gateway.get_external_ip() { + Err(ref err) => { + debug!("IP request error: {}", err); + }, + Ok(external_addr) => { + match gateway.add_any_port(PortMappingProtocol::TCP, SocketAddrV4::new(local_addr.ip().clone(), local_addr.port()), 0, "Parity Node/TCP") { + Err(ref err) => { + debug!("Port mapping error: {}", err); + }, + Ok(tcp_port) => { + match gateway.add_any_port(PortMappingProtocol::UDP, SocketAddrV4::new(local_addr.ip().clone(), local.udp_port), 0, "Parity Node/UDP") { + Err(ref err) => { + debug!("Port mapping error: {}", err); + }, + Ok(udp_port) => { + return Some(NodeEndpoint { address: SocketAddr::V4(SocketAddrV4::new(external_addr, tcp_port)), udp_port: udp_port }); + }, + } + }, + } + }, + } + }, + } + } + None +} + From 58fdfe77d3c93a2599805bd718401f7ebedba645 Mon Sep 17 00:00:00 2001 From: arkpar Date: Tue, 16 Feb 2016 02:31:17 +0100 Subject: [PATCH 24/61] Handle pinning and enable_discovery options --- util/src/network/host.rs | 44 ++++++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 15 deletions(-) diff --git a/util/src/network/host.rs b/util/src/network/host.rs index 3d55430bd..004410466 100644 --- a/util/src/network/host.rs +++ b/util/src/network/host.rs @@ -291,13 +291,14 @@ pub struct Host where Message: Send + Sync + Clone { tcp_listener: Mutex, handshakes: Arc>>, sessions: Arc>>, - discovery: Mutex, + discovery: Option>, nodes: RwLock, handlers: RwLock>>>, timers: RwLock>, timer_counter: RwLock, stats: Arc, public_endpoint: NodeEndpoint, + pinned_nodes: Vec, } impl Host where Message: Send + Sync + Clone { @@ -343,7 +344,9 @@ impl Host where Message: Send + Sync + Clone { }, |s| KeyPair::from_secret(s).expect("Error creating node secret key")) }; - let discovery = Discovery::new(&keys, listen_address.clone(), public_endpoint.clone(), DISCOVERY); + let discovery = if config.discovery_enabled && !config.pin { + Some(Discovery::new(&keys, listen_address.clone(), public_endpoint.clone(), DISCOVERY)) + } else { None }; let path = config.config_path.clone(); let mut host = Host:: { info: RwLock::new(HostInfo { @@ -355,7 +358,7 @@ impl Host where Message: Send + Sync + Clone { listen_port: 0, capabilities: Vec::new(), }), - discovery: Mutex::new(discovery), + discovery: discovery.map(Mutex::new), tcp_listener: Mutex::new(tcp_listener), handshakes: Arc::new(RwLock::new(Slab::new_starting_at(FIRST_HANDSHAKE, MAX_HANDSHAKES))), sessions: Arc::new(RwLock::new(Slab::new_starting_at(FIRST_SESSION, MAX_SESSIONS))), @@ -365,6 +368,7 @@ impl Host where Message: Send + Sync + Clone { timer_counter: RwLock::new(USER_TIMER), stats: Arc::new(NetworkStats::default()), public_endpoint: public_endpoint, + pinned_nodes: Vec::new(), }; let port = listen_address.port(); host.info.write().unwrap().deref_mut().listen_port = port; @@ -373,7 +377,9 @@ impl Host where Message: Send + Sync + Clone { for n in boot_nodes { host.add_node(&n); } - host.discovery.lock().unwrap().init_node_list(host.nodes.read().unwrap().unordered_entries()); + if let Some(ref mut discovery) = host.discovery { + discovery.lock().unwrap().init_node_list(host.nodes.read().unwrap().unordered_entries()); + } host } @@ -386,8 +392,11 @@ impl Host where Message: Send + Sync + Clone { Err(e) => { warn!("Could not add node: {:?}", e); }, Ok(n) => { let entry = NodeEntry { endpoint: n.endpoint.clone(), id: n.id.clone() }; + self.pinned_nodes.push(n.id.clone()); self.nodes.write().unwrap().add_node(n); - self.discovery.lock().unwrap().add_node(entry); + if let Some(ref mut discovery) = self.discovery { + discovery.lock().unwrap().add_node(entry); + } } } } @@ -437,6 +446,7 @@ impl Host where Message: Send + Sync + Clone { fn connect_peers(&self, io: &IoContext>) { let ideal_peers = { self.info.read().unwrap().deref().config.ideal_peers }; + let pin = { self.info.read().unwrap().deref().config.pin }; let session_count = self.session_count(); if session_count >= ideal_peers as usize { return; @@ -449,7 +459,7 @@ impl Host where Message: Send + Sync + Clone { return; } - let nodes = { self.nodes.read().unwrap().nodes() }; + let nodes = if pin { self.pinned_nodes.clone() } else { self.nodes.read().unwrap().nodes() }; for id in nodes.iter().filter(|ref id| !self.have_session(id) && !self.connecting_to(id)) .take(min(MAX_HANDSHAKES_PER_ROUND, handshake_limit - handshake_count)) { self.connect_peer(&id, io); @@ -670,7 +680,9 @@ impl Host where Message: Send + Sync + Clone { if let Ok(address) = session.remote_addr() { let entry = NodeEntry { id: session.id().clone(), endpoint: NodeEndpoint { address: address, udp_port: address.port() } }; self.nodes.write().unwrap().add_node(Node::new(entry.id.clone(), entry.endpoint.clone())); - self.discovery.lock().unwrap().add_node(entry); + if let Some(ref discovery) = self.discovery { + discovery.lock().unwrap().add_node(entry); + } } } Arc::new(Mutex::new(session)) @@ -759,8 +771,10 @@ impl IoHandler> for Host where Messa io.register_stream(TCP_ACCEPT).expect("Error registering TCP listener"); io.register_stream(DISCOVERY).expect("Error registering UDP listener"); io.register_timer(IDLE, MAINTENANCE_TIMEOUT).expect("Error registering Network idle timer"); - io.register_timer(DISCOVERY_REFRESH, 7200).expect("Error registering discovery timer"); - io.register_timer(DISCOVERY_ROUND, 300).expect("Error registering discovery timer"); + if self.discovery.is_some() { + io.register_timer(DISCOVERY_REFRESH, 7200).expect("Error registering discovery timer"); + io.register_timer(DISCOVERY_ROUND, 300).expect("Error registering discovery timer"); + } } fn stream_hup(&self, io: &IoContext>, stream: StreamToken) { @@ -777,7 +791,7 @@ impl IoHandler> for Host where Messa FIRST_SESSION ... LAST_SESSION => self.session_readable(stream, io), FIRST_HANDSHAKE ... LAST_HANDSHAKE => self.handshake_readable(stream, io), DISCOVERY => { - if let Some(node_changes) = self.discovery.lock().unwrap().readable() { + if let Some(node_changes) = self.discovery.as_ref().unwrap().lock().unwrap().readable() { self.update_nodes(io, node_changes); } io.update_registration(DISCOVERY).expect("Error updating disicovery registration"); @@ -792,7 +806,7 @@ impl IoHandler> for Host where Messa FIRST_SESSION ... LAST_SESSION => self.session_writable(stream, io), FIRST_HANDSHAKE ... LAST_HANDSHAKE => self.handshake_writable(stream, io), DISCOVERY => { - self.discovery.lock().unwrap().writable(); + self.discovery.as_ref().unwrap().lock().unwrap().writable(); io.update_registration(DISCOVERY).expect("Error updating disicovery registration"); } _ => panic!("Received unknown writable token"), @@ -805,11 +819,11 @@ impl IoHandler> for Host where Messa FIRST_SESSION ... LAST_SESSION => self.connection_timeout(token, io), FIRST_HANDSHAKE ... LAST_HANDSHAKE => self.connection_timeout(token, io), DISCOVERY_REFRESH => { - self.discovery.lock().unwrap().refresh(); + self.discovery.as_ref().unwrap().lock().unwrap().refresh(); io.update_registration(DISCOVERY).expect("Error updating disicovery registration"); }, DISCOVERY_ROUND => { - if let Some(node_changes) = self.discovery.lock().unwrap().round() { + if let Some(node_changes) = self.discovery.as_ref().unwrap().lock().unwrap().round() { self.update_nodes(io, node_changes); } io.update_registration(DISCOVERY).expect("Error updating disicovery registration"); @@ -880,7 +894,7 @@ impl IoHandler> for Host where Messa connection.lock().unwrap().register_socket(reg, event_loop).expect("Error registering socket"); } } - DISCOVERY => self.discovery.lock().unwrap().register_socket(event_loop).expect("Error registering discovery socket"), + DISCOVERY => self.discovery.as_ref().unwrap().lock().unwrap().register_socket(event_loop).expect("Error registering discovery socket"), TCP_ACCEPT => event_loop.register(self.tcp_listener.lock().unwrap().deref(), Token(TCP_ACCEPT), EventSet::all(), PollOpt::edge()).expect("Error registering stream"), _ => warn!("Unexpected stream registration") } @@ -922,7 +936,7 @@ impl IoHandler> for Host where Messa connection.lock().unwrap().update_socket(reg, event_loop).expect("Error updating socket"); } } - DISCOVERY => self.discovery.lock().unwrap().update_registration(event_loop).expect("Error reregistering discovery socket"), + DISCOVERY => self.discovery.as_ref().unwrap().lock().unwrap().update_registration(event_loop).expect("Error reregistering discovery socket"), TCP_ACCEPT => event_loop.reregister(self.tcp_listener.lock().unwrap().deref(), Token(TCP_ACCEPT), EventSet::all(), PollOpt::edge()).expect("Error reregistering stream"), _ => warn!("Unexpected stream update") } From 33e8d749d2da4bc56213d4348bd163bccfea3c88 Mon Sep 17 00:00:00 2001 From: arkpar Date: Tue, 16 Feb 2016 03:05:05 +0100 Subject: [PATCH 25/61] Max handhsakes reached is now a debug warning --- util/src/network/host.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/util/src/network/host.rs b/util/src/network/host.rs index 004410466..24f24fc6f 100644 --- a/util/src/network/host.rs +++ b/util/src/network/host.rs @@ -513,7 +513,7 @@ impl Host where Message: Send + Sync + Clone { }); Arc::new(Mutex::new(handshake)) }).is_none() { - warn!("Max handshakes reached"); + debug!("Max handshakes reached"); } } From d95e9710306e95552a2ba3ad4df66067c77bea96 Mon Sep 17 00:00:00 2001 From: arkpar Date: Tue, 16 Feb 2016 17:53:31 +0100 Subject: [PATCH 26/61] Prevent deadlocks --- ethcore/src/block_queue.rs | 8 ++++---- util/src/io/worker.rs | 15 +++++++++++---- 2 files changed, 15 insertions(+), 8 deletions(-) diff --git a/ethcore/src/block_queue.rs b/ethcore/src/block_queue.rs index 1a1dee48e..f11519067 100644 --- a/ethcore/src/block_queue.rs +++ b/ethcore/src/block_queue.rs @@ -153,7 +153,7 @@ impl BlockQueue { } fn verify(verification: Arc>, engine: Arc>, wait: Arc, ready: Arc, deleting: Arc, empty: Arc) { - while !deleting.load(AtomicOrdering::Relaxed) { + while !deleting.load(AtomicOrdering::Acquire) { { let mut lock = verification.lock().unwrap(); @@ -161,11 +161,11 @@ impl BlockQueue { empty.notify_all(); } - while lock.unverified.is_empty() && !deleting.load(AtomicOrdering::Relaxed) { + while lock.unverified.is_empty() && !deleting.load(AtomicOrdering::Acquire) { lock = wait.wait(lock).unwrap(); } - if deleting.load(AtomicOrdering::Relaxed) { + if deleting.load(AtomicOrdering::Acquire) { return; } } @@ -347,7 +347,7 @@ impl MayPanic for BlockQueue { impl Drop for BlockQueue { fn drop(&mut self) { self.clear(); - self.deleting.store(true, AtomicOrdering::Relaxed); + self.deleting.store(true, AtomicOrdering::Release); self.more_to_verify.notify_all(); for t in self.verifiers.drain(..) { t.join().unwrap(); diff --git a/util/src/io/worker.rs b/util/src/io/worker.rs index 1ba0318bc..b874ea0a4 100644 --- a/util/src/io/worker.rs +++ b/util/src/io/worker.rs @@ -44,6 +44,7 @@ pub struct Worker { thread: Option>, wait: Arc, deleting: Arc, + wait_mutex: Arc>, } impl Worker { @@ -61,6 +62,7 @@ impl Worker { thread: None, wait: wait.clone(), deleting: deleting.clone(), + wait_mutex: wait_mutex.clone(), }; worker.thread = Some(thread::Builder::new().name(format!("IO Worker #{}", index)).spawn( move || { @@ -77,13 +79,17 @@ impl Worker { wait_mutex: Arc>, deleting: Arc) where Message: Send + Sync + Clone + 'static { - while !deleting.load(AtomicOrdering::Relaxed) { + loop { { let lock = wait_mutex.lock().unwrap(); - let _ = wait.wait(lock).unwrap(); - if deleting.load(AtomicOrdering::Relaxed) { + if deleting.load(AtomicOrdering::Acquire) { return; } + let _ = wait.wait(lock).unwrap(); + } + + if deleting.load(AtomicOrdering::Acquire) { + return; } while let chase_lev::Steal::Data(work) = stealer.steal() { Worker::do_work(work, channel.clone()); @@ -114,7 +120,8 @@ impl Worker { impl Drop for Worker { fn drop(&mut self) { - self.deleting.store(true, AtomicOrdering::Relaxed); + let _ = self.wait_mutex.lock(); + self.deleting.store(true, AtomicOrdering::Release); self.wait.notify_all(); let thread = mem::replace(&mut self.thread, None).unwrap(); thread.join().ok(); From a4ea0737b25ae4953e16f3b87ebaa84e8994c745 Mon Sep 17 00:00:00 2001 From: arkpar Date: Tue, 16 Feb 2016 17:54:34 +0100 Subject: [PATCH 27/61] Fixed some tests --- util/src/network/host.rs | 6 +++--- util/src/network/tests.rs | 10 ++++++---- util/src/sha3.rs | 2 +- 3 files changed, 10 insertions(+), 8 deletions(-) diff --git a/util/src/network/host.rs b/util/src/network/host.rs index 24f24fc6f..140625eea 100644 --- a/util/src/network/host.rs +++ b/util/src/network/host.rs @@ -493,8 +493,8 @@ impl Host where Message: Send + Sync + Clone { }; match TcpStream::connect(&address) { Ok(socket) => socket, - Err(_) => { - warn!("Cannot connect to node"); + Err(e) => { + warn!("Can't connect to node: {:?}", e); return; } } @@ -769,9 +769,9 @@ impl IoHandler> for Host where Messa /// Initialize networking fn initialize(&self, io: &IoContext>) { io.register_stream(TCP_ACCEPT).expect("Error registering TCP listener"); - io.register_stream(DISCOVERY).expect("Error registering UDP listener"); io.register_timer(IDLE, MAINTENANCE_TIMEOUT).expect("Error registering Network idle timer"); if self.discovery.is_some() { + io.register_stream(DISCOVERY).expect("Error registering UDP listener"); io.register_timer(DISCOVERY_REFRESH, 7200).expect("Error registering discovery timer"); io.register_timer(DISCOVERY_ROUND, 300).expect("Error registering discovery timer"); } diff --git a/util/src/network/tests.rs b/util/src/network/tests.rs index a3d5290c9..925c95396 100644 --- a/util/src/network/tests.rs +++ b/util/src/network/tests.rs @@ -92,16 +92,18 @@ fn net_service() { #[test] fn net_connect() { let key1 = KeyPair::create().unwrap(); - let mut config1 = NetworkConfiguration::new_with_port(30344); + let mut config1 = NetworkConfiguration::new_with_port(30354); config1.use_secret = Some(key1.secret().clone()); + config1.nat_enabled = false; config1.boot_nodes = vec![ ]; - let mut config2 = NetworkConfiguration::new_with_port(30345); - config2.boot_nodes = vec![ format!("enode://{}@127.0.0.1:30344", key1.public().hex()) ]; + let mut config2 = NetworkConfiguration::new_with_port(30355); + config2.boot_nodes = vec![ format!("enode://{}@127.0.0.1:30354", key1.public().hex()) ]; + config2.nat_enabled = false; let mut service1 = NetworkService::::start(config1).unwrap(); let mut service2 = NetworkService::::start(config2).unwrap(); let handler1 = TestProtocol::register(&mut service1); let handler2 = TestProtocol::register(&mut service2); - while !handler1.got_packet() && !handler2.got_packet() { + while !handler1.got_packet() && !handler2.got_packet() && (service1.stats().sessions() == 0 || service2.stats().sessions() == 0) { thread::sleep(Duration::from_millis(50)); } assert!(service1.stats().sessions() >= 1); diff --git a/util/src/sha3.rs b/util/src/sha3.rs index 4079e8ba4..7e8382250 100644 --- a/util/src/sha3.rs +++ b/util/src/sha3.rs @@ -66,7 +66,7 @@ impl Hashable for T where T: BytesConvertable { #[test] fn sha3_empty() { - assert_eq!((&[0u8; 0]).sha3(), SHA3_EMPTY); + assert_eq!([0u8; 0].sha3(), SHA3_EMPTY); } #[test] fn sha3_as() { From f4fa747cd0683b16d670ae4f446dd5155ffd1df4 Mon Sep 17 00:00:00 2001 From: arkpar Date: Tue, 16 Feb 2016 17:55:37 +0100 Subject: [PATCH 28/61] ip_utils tests --- util/src/network/ip_utils.rs | 50 ++++++++++++++++++++---------------- 1 file changed, 28 insertions(+), 22 deletions(-) diff --git a/util/src/network/ip_utils.rs b/util/src/network/ip_utils.rs index 8f94073b3..da50f8ddf 100644 --- a/util/src/network/ip_utils.rs +++ b/util/src/network/ip_utils.rs @@ -43,34 +43,28 @@ mod getinterfaces { let sa: *const sockaddr_in = unsafe { mem::transmute(sa) }; let sa = & unsafe { *sa }; let (addr, port) = (sa.sin_addr.s_addr, sa.sin_port); - ( - IpAddr::V4(Ipv4Addr::new( - (addr & 0x000000FF) as u8, - ((addr & 0x0000FF00) >> 8) as u8, - ((addr & 0x00FF0000) >> 16) as u8, - ((addr & 0xFF000000) >> 24) as u8, - )), - port - ) + (IpAddr::V4(Ipv4Addr::new( + (addr & 0x000000FF) as u8, + ((addr & 0x0000FF00) >> 8) as u8, + ((addr & 0x00FF0000) >> 16) as u8, + ((addr & 0xFF000000) >> 24) as u8)), + port) }, AF_INET6 => { let sa: *const sockaddr_in6 = unsafe { mem::transmute(sa) }; let sa = & unsafe { *sa }; let (addr, port) = (sa.sin6_addr.s6_addr, sa.sin6_port); let addr: [u16; 8] = unsafe { mem::transmute(addr) }; - ( - IpAddr::V6(Ipv6Addr::new( - addr[0], - addr[1], - addr[2], - addr[3], - addr[4], - addr[5], - addr[6], - addr[7], - )), - port - ) + (IpAddr::V6(Ipv6Addr::new( + addr[0], + addr[1], + addr[2], + addr[3], + addr[4], + addr[5], + addr[6], + addr[7])), + port) }, _ => return None, }; @@ -174,3 +168,15 @@ pub fn map_external_address(local: &NodeEndpoint) -> Option { None } +#[test] +fn can_select_public_address() { + let pub_address = select_public_address(40477); + assert!(pub_address.port() == 40477); +} + +#[test] +fn can_map_external_address_or_fail() { + let pub_address = select_public_address(40478); + let _ = map_external_address(&NodeEndpoint { address: pub_address, udp_port: 40478 }); +} + From 217cbec50ed6c2cccec386dd3daedbf95b92c64e Mon Sep 17 00:00:00 2001 From: arkpar Date: Tue, 16 Feb 2016 19:08:58 +0100 Subject: [PATCH 29/61] Disconnect test --- util/src/network/host.rs | 1 - util/src/network/session.rs | 5 +++ util/src/network/tests.rs | 65 +++++++++++++++++++++++++++---------- 3 files changed, 53 insertions(+), 18 deletions(-) diff --git a/util/src/network/host.rs b/util/src/network/host.rs index 140625eea..4afb790ae 100644 --- a/util/src/network/host.rs +++ b/util/src/network/host.rs @@ -634,7 +634,6 @@ impl Host where Message: Send + Sync + Clone { } if kill { self.kill_connection(token, io, true); //TODO: mark connection as dead an check in kill_connection - return; } for p in ready_data { let h = self.handlers.read().unwrap().get(p).unwrap().clone(); diff --git a/util/src/network/session.rs b/util/src/network/session.rs index f08fef385..04fb6d930 100644 --- a/util/src/network/session.rs +++ b/util/src/network/session.rs @@ -318,7 +318,12 @@ impl Session { trace!(target: "net", "Hello: {} v{} {} {:?}", client_version, protocol, id, caps); self.info.client_version = client_version; self.info.capabilities = caps; + if self.info.capabilities.is_empty() { + trace!("No common capabilities with peer."); + return Err(From::from(self.disconnect(DisconnectReason::UselessPeer))); + } if protocol != host.protocol_version { + trace!("Peer protocol version mismatch: {}", protocol); return Err(From::from(self.disconnect(DisconnectReason::UselessPeer))); } self.had_hello = true; diff --git a/util/src/network/tests.rs b/util/src/network/tests.rs index 925c95396..dc80936d2 100644 --- a/util/src/network/tests.rs +++ b/util/src/network/tests.rs @@ -23,17 +23,10 @@ use io::TimerToken; use crypto::KeyPair; pub struct TestProtocol { + drop_session: bool, pub packet: Mutex, pub got_timeout: AtomicBool, -} - -impl Default for TestProtocol { - fn default() -> Self { - TestProtocol { - packet: Mutex::new(Vec::new()), - got_timeout: AtomicBool::new(false), - } - } + pub got_disconnect: AtomicBool, } #[derive(Clone)] @@ -42,9 +35,17 @@ pub struct TestProtocolMessage { } impl TestProtocol { + pub fn new(drop_session: bool) -> Self { + TestProtocol { + packet: Mutex::new(Vec::new()), + got_timeout: AtomicBool::new(false), + got_disconnect: AtomicBool::new(false), + drop_session: drop_session, + } + } /// Creates and register protocol with the network service - pub fn register(service: &mut NetworkService) -> Arc { - let handler = Arc::new(TestProtocol::default()); + pub fn register(service: &mut NetworkService, drop_session: bool) -> Arc { + let handler = Arc::new(TestProtocol::new(drop_session)); service.register_protocol(handler.clone(), "test", &[42u8, 43u8]).expect("Error registering test protocol handler"); handler } @@ -56,6 +57,10 @@ impl TestProtocol { pub fn got_timeout(&self) -> bool { self.got_timeout.load(AtomicOrdering::Relaxed) } + + pub fn got_disconnect(&self) -> bool { + self.got_disconnect.load(AtomicOrdering::Relaxed) + } } impl NetworkProtocolHandler for TestProtocol { @@ -68,11 +73,16 @@ impl NetworkProtocolHandler for TestProtocol { self.packet.lock().unwrap().extend(data); } - fn connected(&self, io: &NetworkContext, _peer: &PeerId) { - io.respond(33, "hello".to_owned().into_bytes()).unwrap(); + fn connected(&self, io: &NetworkContext, peer: &PeerId) { + if self.drop_session { + io.disconnect_peer(*peer) + } else { + io.respond(33, "hello".to_owned().into_bytes()).unwrap(); + } } fn disconnected(&self, _io: &NetworkContext, _peer: &PeerId) { + self.got_disconnect.store(true, AtomicOrdering::Relaxed); } /// Timer function called after a timeout created with `NetworkContext::timeout`. @@ -86,7 +96,7 @@ impl NetworkProtocolHandler for TestProtocol { #[test] fn net_service() { let mut service = NetworkService::::start(NetworkConfiguration::new_with_port(40414)).expect("Error creating network service"); - service.register_protocol(Arc::new(TestProtocol::default()), "myproto", &[1u8]).unwrap(); + service.register_protocol(Arc::new(TestProtocol::new(false)), "myproto", &[1u8]).unwrap(); } #[test] @@ -101,8 +111,8 @@ fn net_connect() { config2.nat_enabled = false; let mut service1 = NetworkService::::start(config1).unwrap(); let mut service2 = NetworkService::::start(config2).unwrap(); - let handler1 = TestProtocol::register(&mut service1); - let handler2 = TestProtocol::register(&mut service2); + let handler1 = TestProtocol::register(&mut service1, false); + let handler2 = TestProtocol::register(&mut service2, false); while !handler1.got_packet() && !handler2.got_packet() && (service1.stats().sessions() == 0 || service2.stats().sessions() == 0) { thread::sleep(Duration::from_millis(50)); } @@ -110,11 +120,32 @@ fn net_connect() { assert!(service2.stats().sessions() >= 1); } +#[test] +fn net_disconnect() { + let key1 = KeyPair::create().unwrap(); + let mut config1 = NetworkConfiguration::new_with_port(30364); + config1.use_secret = Some(key1.secret().clone()); + config1.nat_enabled = false; + config1.boot_nodes = vec![ ]; + let mut config2 = NetworkConfiguration::new_with_port(30365); + config2.boot_nodes = vec![ format!("enode://{}@127.0.0.1:30364", key1.public().hex()) ]; + config2.nat_enabled = false; + let mut service1 = NetworkService::::start(config1).unwrap(); + let mut service2 = NetworkService::::start(config2).unwrap(); + let handler1 = TestProtocol::register(&mut service1, false); + let handler2 = TestProtocol::register(&mut service2, true); + while !(handler1.got_disconnect() && handler2.got_disconnect()) { + thread::sleep(Duration::from_millis(50)); + } + assert!(handler1.got_disconnect()); + assert!(handler2.got_disconnect()); +} + #[test] fn net_timeout() { let config = NetworkConfiguration::new_with_port(30346); let mut service = NetworkService::::start(config).unwrap(); - let handler = TestProtocol::register(&mut service); + let handler = TestProtocol::register(&mut service, false); while !handler.got_timeout() { thread::sleep(Duration::from_millis(50)); } From 4f73d63f90154140d6dadeaed3700ca623eed0ab Mon Sep 17 00:00:00 2001 From: arkpar Date: Tue, 16 Feb 2016 19:51:51 +0100 Subject: [PATCH 30/61] Tweaked CLI options --- parity/main.rs | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/parity/main.rs b/parity/main.rs index 5fec05eb4..516609615 100644 --- a/parity/main.rs +++ b/parity/main.rs @@ -64,9 +64,11 @@ Options: -d --db-path PATH Specify the database & configuration directory path [default: $HOME/.parity] --no-bootstrap Don't bother trying to connect to any nodes initially. - --listen-address URL Specify the IP/port on which to listen for peers. + --listen-address URL Specify the IP/port on which to listen for peers [default: 0.0.0.0:30304]. --public-address URL Specify the IP/port on which peers may connect. --address URL Equivalent to --listen-address URL --public-address URL. + --peers NUM Try to manintain that many peers [default: 25]. + --no-discovery Disable new peer discovery. --upnp Use UPnP to try to figure out the correct network settings. --node-key KEY Specify node secret key as hex string. @@ -81,8 +83,8 @@ Options: -h --help Show this screen. ", flag_cache_pref_size: usize, flag_cache_max_size: usize, + flag_peers: u32, flag_address: Option, - flag_listen_address: Option, flag_public_address: Option, flag_node_key: Option); @@ -168,11 +170,8 @@ impl Configuration { public_address = Some(SocketAddr::from_str(a.as_ref()).expect("Invalid listen/public address given with --address")); listen_address = public_address; } - if let Some(ref a) = self.args.flag_listen_address { - if listen_address.is_some() { - panic!("Conflicting flags: --address and --listen-address"); - } - listen_address = Some(SocketAddr::from_str(a.as_ref()).expect("Invalid listen address given with --listen-address")); + if listen_address.is_none() { + listen_address = Some(SocketAddr::from_str(self.args.flag_listen_address.as_ref()).expect("Invalid listen address given with --listen-address")); } if let Some(ref a) = self.args.flag_public_address { if public_address.is_some() { @@ -218,6 +217,8 @@ fn main() { net_settings.listen_address = listen; net_settings.public_address = public; net_settings.use_secret = conf.args.flag_node_key.as_ref().map(|s| Secret::from_str(&s).expect("Invalid key string")); + net_settings.discovery_enabled = !conf.args.flag_no_discovery; + net_settings.ideal_peers = conf.args.flag_peers; let mut net_path = PathBuf::from(&conf.path()); net_path.push("network"); net_settings.config_path = Some(net_path.to_str().unwrap().to_owned()); From fbe06d3f2f85275cc6ec4c9c3b17344a3778def0 Mon Sep 17 00:00:00 2001 From: arkpar Date: Tue, 16 Feb 2016 21:25:01 +0100 Subject: [PATCH 31/61] More tests --- util/src/network/discovery.rs | 22 +++++++++++++++++----- util/src/network/host.rs | 10 ++++++++++ 2 files changed, 27 insertions(+), 5 deletions(-) diff --git a/util/src/network/discovery.rs b/util/src/network/discovery.rs index c15dbbbc4..0064a34db 100644 --- a/util/src/network/discovery.rs +++ b/util/src/network/discovery.rs @@ -445,13 +445,13 @@ impl Discovery { Ok(Some(TableUpdates { added: added, removed: HashSet::new() })) } - fn check_expired(&mut self) -> HashSet { + fn check_expired(&mut self, force: bool) -> HashSet { let now = time::precise_time_ns(); let mut removed: HashSet = HashSet::new(); for bucket in &mut self.node_buckets { bucket.nodes.retain(|node| { if let Some(timeout) = node.timeout { - if now - timeout < PING_TIMEOUT_MS * 1000_0000 { + if !force && now - timeout < PING_TIMEOUT_MS * 1000_0000 { true } else { @@ -466,7 +466,7 @@ impl Discovery { } pub fn round(&mut self) -> Option { - let removed = self.check_expired(); + let removed = self.check_expired(false); self.discover(); if !removed.is_empty() { Some(TableUpdates { added: HashMap::new(), removed: removed }) @@ -512,8 +512,8 @@ mod tests { let node1 = Node::from_str("enode://a979fb575495b8d6db44f750317d0f4622bf4c2aa3365d6af7c284339968eef29b69ad0dce72a4d8db5ebb4968de0e3bec910127f134779fbcb0cb6d3331163c@127.0.0.1:7770").unwrap(); let node2 = Node::from_str("enode://b979fb575495b8d6db44f750317d0f4622bf4c2aa3365d6af7c284339968eef29b69ad0dce72a4d8db5ebb4968de0e3bec910127f134779fbcb0cb6d3331163c@127.0.0.1:7771").unwrap(); - discovery1.add_node(NodeEntry { id: node1.id.clone(), endpoint: node1. endpoint.clone() }); - discovery1.add_node(NodeEntry { id: node2.id.clone(), endpoint: node2. endpoint.clone() }); + discovery1.add_node(NodeEntry { id: node1.id.clone(), endpoint: node1.endpoint.clone() }); + discovery1.add_node(NodeEntry { id: node2.id.clone(), endpoint: node2.endpoint.clone() }); discovery2.add_node(NodeEntry { id: key1.public().clone(), endpoint: ep1.clone() }); discovery2.refresh(); @@ -535,4 +535,16 @@ mod tests { } assert_eq!(Discovery::nearest_node_entries(&NodeId::new(), &discovery2.node_buckets).len(), 3) } + + #[test] + fn removes_expired() { + let key = KeyPair::create().unwrap(); + let ep = NodeEndpoint { address: SocketAddr::from_str("127.0.0.1:40446").unwrap(), udp_port: 40444 }; + let mut discovery = Discovery::new(&key, ep.address.clone(), ep.clone(), 0); + for _ in 0..1200 { + discovery.add_node(NodeEntry { id: NodeId::random(), endpoint: ep.clone() }); + } + let removed = discovery.check_expired(true).len(); + assert!(removed > 0); + } } diff --git a/util/src/network/host.rs b/util/src/network/host.rs index 4afb790ae..080a8b5cc 100644 --- a/util/src/network/host.rs +++ b/util/src/network/host.rs @@ -987,3 +987,13 @@ fn load_key(path: &Path) -> Option { } } } + +#[test] +fn key_save_load() { + use tests::helpers::RandomTempPath; + let temp_path = RandomTempPath::create_dir(); + let key = H256::random(); + save_key(temp_path.as_path(), &key); + let r = load_key(temp_path.as_path()); + assert_eq!(key, r.unwrap()); +} From 7e0dfb41d08bc4a47084a61a7b745eed00de1690 Mon Sep 17 00:00:00 2001 From: arkpar Date: Tue, 16 Feb 2016 23:33:32 +0100 Subject: [PATCH 32/61] Minor test tweaks and code cleanup --- util/src/network/discovery.rs | 1 + util/src/network/host.rs | 26 ++++---------------------- 2 files changed, 5 insertions(+), 22 deletions(-) diff --git a/util/src/network/discovery.rs b/util/src/network/discovery.rs index 0064a34db..f383b74f7 100644 --- a/util/src/network/discovery.rs +++ b/util/src/network/discovery.rs @@ -544,6 +544,7 @@ mod tests { for _ in 0..1200 { discovery.add_node(NodeEntry { id: NodeId::random(), endpoint: ep.clone() }); } + assert!(Discovery::nearest_node_entries(&NodeId::new(), &discovery.node_buckets).len() <= 16); let removed = discovery.check_expired(true).len(); assert!(removed > 0); } diff --git a/util/src/network/host.rs b/util/src/network/host.rs index 080a8b5cc..bcb1c7585 100644 --- a/util/src/network/host.rs +++ b/util/src/network/host.rs @@ -534,42 +534,24 @@ impl Host where Message: Send + Sync + Clone { } fn handshake_writable(&self, token: StreamToken, io: &IoContext>) { - let mut create_session = false; - let mut kill = false; let handshake = { self.handshakes.read().unwrap().get(token).cloned() }; if let Some(handshake) = handshake { let mut h = handshake.lock().unwrap(); if let Err(e) = h.writable(io, &self.info.read().unwrap()) { debug!(target: "net", "Handshake write error: {}:{:?}", token, e); - kill = true; - } - if h.done() { - create_session = true; } } - if kill { - self.kill_connection(token, io, true); //TODO: mark connection as dead an check in kill_connection - return; - } else if create_session { - self.start_session(token, io); - io.update_registration(token).unwrap_or_else(|e| debug!(target: "net", "Session registration error: {:?}", e)); - } } fn session_writable(&self, token: StreamToken, io: &IoContext>) { - let mut kill = false; let session = { self.sessions.read().unwrap().get(token).cloned() }; if let Some(session) = session { let mut s = session.lock().unwrap(); if let Err(e) = s.writable(io, &self.info.read().unwrap()) { debug!(target: "net", "Session write error: {}:{:?}", token, e); - kill = true; } io.update_registration(token).unwrap_or_else(|e| debug!(target: "net", "Session registration error: {:?}", e)); } - if kill { - self.kill_connection(token, io, true); //TODO: mark connection as dead an check in kill_connection - } } fn connection_closed(&self, token: TimerToken, io: &IoContext>) { @@ -793,7 +775,7 @@ impl IoHandler> for Host where Messa if let Some(node_changes) = self.discovery.as_ref().unwrap().lock().unwrap().readable() { self.update_nodes(io, node_changes); } - io.update_registration(DISCOVERY).expect("Error updating disicovery registration"); + io.update_registration(DISCOVERY).expect("Error updating discovery registration"); }, TCP_ACCEPT => self.accept(io), _ => panic!("Received unknown readable token"), @@ -806,7 +788,7 @@ impl IoHandler> for Host where Messa FIRST_HANDSHAKE ... LAST_HANDSHAKE => self.handshake_writable(stream, io), DISCOVERY => { self.discovery.as_ref().unwrap().lock().unwrap().writable(); - io.update_registration(DISCOVERY).expect("Error updating disicovery registration"); + io.update_registration(DISCOVERY).expect("Error updating discovery registration"); } _ => panic!("Received unknown writable token"), } @@ -819,13 +801,13 @@ impl IoHandler> for Host where Messa FIRST_HANDSHAKE ... LAST_HANDSHAKE => self.connection_timeout(token, io), DISCOVERY_REFRESH => { self.discovery.as_ref().unwrap().lock().unwrap().refresh(); - io.update_registration(DISCOVERY).expect("Error updating disicovery registration"); + io.update_registration(DISCOVERY).expect("Error updating discovery registration"); }, DISCOVERY_ROUND => { if let Some(node_changes) = self.discovery.as_ref().unwrap().lock().unwrap().round() { self.update_nodes(io, node_changes); } - io.update_registration(DISCOVERY).expect("Error updating disicovery registration"); + io.update_registration(DISCOVERY).expect("Error updating discovery registration"); }, _ => match self.timers.read().unwrap().get(&token).cloned() { Some(timer) => match self.handlers.read().unwrap().get(timer.protocol).cloned() { From b6ccbdb6949a3d2a3a9b85b34fa7a6ece0fd1e0c Mon Sep 17 00:00:00 2001 From: arkpar Date: Tue, 16 Feb 2016 23:37:24 +0100 Subject: [PATCH 33/61] Lower max handshakes to reduce network load --- util/src/network/host.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/util/src/network/host.rs b/util/src/network/host.rs index bcb1c7585..9560ca81e 100644 --- a/util/src/network/host.rs +++ b/util/src/network/host.rs @@ -46,8 +46,8 @@ type Slab = ::slab::Slab; const _DEFAULT_PORT: u16 = 30304; const MAX_SESSIONS: usize = 1024; -const MAX_HANDSHAKES: usize = 256; -const MAX_HANDSHAKES_PER_ROUND: usize = 64; +const MAX_HANDSHAKES: usize = 64; +const MAX_HANDSHAKES_PER_ROUND: usize = 8; const MAINTENANCE_TIMEOUT: u64 = 1000; #[derive(Debug)] From 39a98cd555dbb48caf1c9ff1632f5ff9993a9170 Mon Sep 17 00:00:00 2001 From: arkpar Date: Wed, 17 Feb 2016 01:39:16 +0100 Subject: [PATCH 34/61] Prevent connection deletion until unregister is called; minor tweaks --- sync/src/chain.rs | 2 +- util/src/network/host.rs | 6 ++---- util/src/network/session.rs | 2 +- 3 files changed, 4 insertions(+), 6 deletions(-) diff --git a/sync/src/chain.rs b/sync/src/chain.rs index 8f3d7440b..bbfefb78d 100644 --- a/sync/src/chain.rs +++ b/sync/src/chain.rs @@ -1179,7 +1179,7 @@ impl ChainSync { for (peer_id, peer_number) in updated_peers { let mut peer_best = self.peers.get(&peer_id).unwrap().latest_hash.clone(); if best_number - peer_number > MAX_PEERS_PROPAGATION as BlockNumber { - // If we think peer is too far behind just end one latest hash + // If we think peer is too far behind just send one latest hash peer_best = last_parent.clone(); } sent = sent + match ChainSync::create_new_hashes_rlp(io.chain(), &peer_best, &local_best) { diff --git a/util/src/network/host.rs b/util/src/network/host.rs index 9560ca81e..fc4e07306 100644 --- a/util/src/network/host.rs +++ b/util/src/network/host.rs @@ -682,14 +682,13 @@ impl Host where Message: Send + Sync + Clone { let mut failure_id = None; match token { FIRST_HANDSHAKE ... LAST_HANDSHAKE => { - let mut handshakes = self.handshakes.write().unwrap(); + let handshakes = self.handshakes.write().unwrap(); if let Some(handshake) = handshakes.get(token).cloned() { failure_id = Some(handshake.lock().unwrap().id().clone()); - handshakes.remove(token); } }, FIRST_SESSION ... LAST_SESSION => { - let mut sessions = self.sessions.write().unwrap(); + let sessions = self.sessions.write().unwrap(); if let Some(session) = sessions.get(token).cloned() { let s = session.lock().unwrap(); if s.is_ready() { @@ -700,7 +699,6 @@ impl Host where Message: Send + Sync + Clone { } } failure_id = Some(s.id().clone()); - sessions.remove(token); } }, _ => {}, diff --git a/util/src/network/session.rs b/util/src/network/session.rs index 04fb6d930..572b91a18 100644 --- a/util/src/network/session.rs +++ b/util/src/network/session.rs @@ -91,7 +91,7 @@ impl Decodable for PeerCapabilityInfo { } } -#[derive(Debug, PartialEq, Eq)] +#[derive(Debug)] struct SessionCapabilityInfo { pub protocol: &'static str, pub version: u8, From e4baf37bf87fe29aa9355412a41016ade2897129 Mon Sep 17 00:00:00 2001 From: arkpar Date: Wed, 17 Feb 2016 02:55:46 +0100 Subject: [PATCH 35/61] Fixed adding boot nodes to discovery table; Ping optimization --- util/src/network/discovery.rs | 11 +++++++++-- util/src/network/host.rs | 6 +++--- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/util/src/network/discovery.rs b/util/src/network/discovery.rs index f383b74f7..950c66dae 100644 --- a/util/src/network/discovery.rs +++ b/util/src/network/discovery.rs @@ -149,6 +149,13 @@ impl Discovery { } } + fn clear_ping(&mut self, id: &NodeId) { + let mut bucket = self.node_buckets.get_mut(Discovery::distance(&self.id, &id) as usize).unwrap(); + if let Some(node) = bucket.nodes.iter_mut().find(|n| &n.address.id == id) { + node.timeout = None; + } + } + fn start(&mut self) { trace!(target: "discovery", "Starting discovery"); self.discovery_round = 0; @@ -388,10 +395,10 @@ impl Discovery { debug!(target: "discovery", "Bad address: {:?}", entry); entry.endpoint.address = from.clone(); } - self.update_node(entry.clone()); + self.clear_ping(node); let mut added_map = HashMap::new(); added_map.insert(node.clone(), entry); - Ok(Some(TableUpdates { added: added_map, removed: HashSet::new() })) + Ok(None) } fn on_find_node(&mut self, rlp: &UntrustedRlp, _node: &NodeId, from: &SocketAddr) -> Result, NetworkError> { diff --git a/util/src/network/host.rs b/util/src/network/host.rs index fc4e07306..e470d47d8 100644 --- a/util/src/network/host.rs +++ b/util/src/network/host.rs @@ -374,12 +374,12 @@ impl Host where Message: Send + Sync + Clone { host.info.write().unwrap().deref_mut().listen_port = port; let boot_nodes = host.info.read().unwrap().config.boot_nodes.clone(); - for n in boot_nodes { - host.add_node(&n); - } if let Some(ref mut discovery) = host.discovery { discovery.lock().unwrap().init_node_list(host.nodes.read().unwrap().unordered_entries()); } + for n in boot_nodes { + host.add_node(&n); + } host } From eef193e8bd377b1d9483b814a2999980b5bfb613 Mon Sep 17 00:00:00 2001 From: arkpar Date: Wed, 17 Feb 2016 14:06:19 +0100 Subject: [PATCH 36/61] Don't add useless peers to table --- util/src/network/discovery.rs | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/util/src/network/discovery.rs b/util/src/network/discovery.rs index 950c66dae..b12e3b2a1 100644 --- a/util/src/network/discovery.rs +++ b/util/src/network/discovery.rs @@ -361,24 +361,21 @@ impl Discovery { debug!(target: "discovery", "Expired ping"); return Err(NetworkError::Expired); } - let mut entry = NodeEntry { id: node.clone(), endpoint: source.clone() }; - if !entry.endpoint.is_valid() { - debug!(target: "discovery", "Got bad address: {:?}, using {:?} instead", entry, from); - entry.endpoint.address = from.clone(); + let mut added_map = HashMap::new(); + let entry = NodeEntry { id: node.clone(), endpoint: source.clone() }; + if !entry.endpoint.is_valid() || !entry.endpoint.is_global() { + debug!(target: "discovery", "Got bad address: {:?}", entry); } - if !entry.endpoint.is_global() { - debug!(target: "discovery", "Got local address: {:?}, using {:?} instead", entry, from); - entry.endpoint.address = from.clone(); + else { + self.update_node(entry.clone()); + added_map.insert(node.clone(), entry); } - self.update_node(entry.clone()); let hash = rlp.as_raw().sha3(); let mut response = RlpStream::new_list(2); dest.to_rlp_list(&mut response); response.append(&hash); - self.send_packet(PACKET_PONG, &entry.endpoint.udp_address(), &response.drain()); + self.send_packet(PACKET_PONG, from, &response.drain()); - let mut added_map = HashMap::new(); - added_map.insert(node.clone(), entry); Ok(Some(TableUpdates { added: added_map, removed: HashSet::new() })) } From c9f3f5e54482e592fb8e36e149dd596b46078efe Mon Sep 17 00:00:00 2001 From: arkpar Date: Wed, 17 Feb 2016 14:07:11 +0100 Subject: [PATCH 37/61] Tweaked connection limits to be a bit more aggressive --- util/src/network/handshake.rs | 2 +- util/src/network/host.rs | 14 ++++++++++++-- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/util/src/network/handshake.rs b/util/src/network/handshake.rs index 1fd830ea0..5d43decd7 100644 --- a/util/src/network/handshake.rs +++ b/util/src/network/handshake.rs @@ -68,7 +68,7 @@ pub struct Handshake { const AUTH_PACKET_SIZE: usize = 307; const ACK_PACKET_SIZE: usize = 210; -const HANDSHAKE_TIMEOUT: u64 = 30000; +const HANDSHAKE_TIMEOUT: u64 = 5000; impl Handshake { /// Create a new handshake object diff --git a/util/src/network/host.rs b/util/src/network/host.rs index e470d47d8..460cabc7c 100644 --- a/util/src/network/host.rs +++ b/util/src/network/host.rs @@ -46,8 +46,8 @@ type Slab = ::slab::Slab; const _DEFAULT_PORT: u16 = 30304; const MAX_SESSIONS: usize = 1024; -const MAX_HANDSHAKES: usize = 64; -const MAX_HANDSHAKES_PER_ROUND: usize = 8; +const MAX_HANDSHAKES: usize = 80; +const MAX_HANDSHAKES_PER_ROUND: usize = 32; const MAINTENANCE_TIMEOUT: u64 = 1000; #[derive(Debug)] @@ -977,3 +977,13 @@ fn key_save_load() { let r = load_key(temp_path.as_path()); assert_eq!(key, r.unwrap()); } + + +#[test] +fn host_client_url() { + let mut config = NetworkConfiguration::new(); + let key = h256_from_hex("6f7b0d801bc7b5ce7bbd930b84fd0369b3eb25d09be58d64ba811091046f3aa2"); + config.use_secret = Some(key); + let host: Host = Host::new(config); + assert!(host.client_url().starts_with("enode://101b3ef5a4ea7a1c7928e24c4c75fd053c235d7b80c22ae5c03d145d0ac7396e2a4ffff9adee3133a7b05044a5cee08115fd65145e5165d646bde371010d803c@")); +} From 0cfc4cbb3437d9547fc78e85ef5e5b08f08490fb Mon Sep 17 00:00:00 2001 From: arkpar Date: Wed, 17 Feb 2016 14:07:26 +0100 Subject: [PATCH 38/61] More tests --- util/src/network/error.rs | 60 ++++++++++++++++++++++++++++++------- util/src/network/session.rs | 6 +++- 2 files changed, 55 insertions(+), 11 deletions(-) diff --git a/util/src/network/error.rs b/util/src/network/error.rs index 74babb110..31e1d785b 100644 --- a/util/src/network/error.rs +++ b/util/src/network/error.rs @@ -18,21 +18,42 @@ use io::IoError; use crypto::CryptoError; use rlp::*; -#[derive(Debug, Copy, Clone)] +#[derive(Debug, Copy, Clone, PartialEq, Eq)] pub enum DisconnectReason { DisconnectRequested, - _TCPError, - _BadProtocol, + TCPError, + BadProtocol, UselessPeer, - _TooManyPeers, - _DuplicatePeer, - _IncompatibleProtocol, - _NullIdentity, - _ClientQuit, - _UnexpectedIdentity, - _LocalIdentity, + TooManyPeers, + DuplicatePeer, + IncompatibleProtocol, + NullIdentity, + ClientQuit, + UnexpectedIdentity, + LocalIdentity, PingTimeout, + Unknown, +} + +impl DisconnectReason { + pub fn from_u8(n: u8) -> DisconnectReason { + match n { + 0 => DisconnectReason::DisconnectRequested, + 1 => DisconnectReason::TCPError, + 2 => DisconnectReason::BadProtocol, + 3 => DisconnectReason::UselessPeer, + 4 => DisconnectReason::TooManyPeers, + 5 => DisconnectReason::DuplicatePeer, + 6 => DisconnectReason::IncompatibleProtocol, + 7 => DisconnectReason::NullIdentity, + 8 => DisconnectReason::ClientQuit, + 9 => DisconnectReason::UnexpectedIdentity, + 10 => DisconnectReason::LocalIdentity, + 11 => DisconnectReason::PingTimeout, + _ => DisconnectReason::Unknown, + } + } } #[derive(Debug)] @@ -70,3 +91,22 @@ impl From for NetworkError { } } +#[test] +fn test_errors() { + assert_eq!(DisconnectReason::ClientQuit, DisconnectReason::from_u8(8)); + let mut r = DisconnectReason::DisconnectRequested; + for i in 0 .. 20 { + r = DisconnectReason::from_u8(i); + } + assert_eq!(DisconnectReason::Unknown, r); + + match >::from(DecoderError::RlpIsTooBig) { + NetworkError::Auth => {}, + _ => panic!("Unexpeceted error"), + } + + match >::from(CryptoError::InvalidSecret) { + NetworkError::Auth => {}, + _ => panic!("Unexpeceted error"), + } +} diff --git a/util/src/network/session.rs b/util/src/network/session.rs index 572b91a18..b0db5f7ef 100644 --- a/util/src/network/session.rs +++ b/util/src/network/session.rs @@ -231,7 +231,11 @@ impl Session { try!(self.read_hello(&rlp, host)); Ok(SessionData::Ready) }, - PACKET_DISCONNECT => Err(From::from(NetworkError::Disconnect(DisconnectReason::DisconnectRequested))), + PACKET_DISCONNECT => { + let rlp = UntrustedRlp::new(&packet.data[1..]); + let reason: u8 = try!(rlp.val_at(0)); + Err(From::from(NetworkError::Disconnect(DisconnectReason::from_u8(reason)))) + } PACKET_PING => { try!(self.send_pong()); Ok(SessionData::None) From 2cc690f31f27adb5e761cc676d90b7c94c1afec9 Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Fri, 19 Feb 2016 12:54:51 +0100 Subject: [PATCH 39/61] Better user errors. Fixed up README. --- Cargo.lock | 2 ++ Cargo.toml | 2 ++ README.md | 2 +- parity/main.rs | 64 ++++++++++++++++++++++++++++++++++++-------------- 4 files changed, 52 insertions(+), 18 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index b69736c2e..34b618c2b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -12,7 +12,9 @@ dependencies = [ "ethcore-util 0.9.99", "ethsync 0.9.99", "fdlimit 0.1.0", + "lazy_static 0.1.15 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)", + "regex 0.1.51 (registry+https://github.com/rust-lang/crates.io-index)", "rustc-serialize 0.3.18 (registry+https://github.com/rust-lang/crates.io-index)", "target_info 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", ] diff --git a/Cargo.toml b/Cargo.toml index 5b59b26f1..f4163253b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -19,6 +19,8 @@ ethcore-rpc = { path = "rpc", optional = true } fdlimit = { path = "util/fdlimit" } target_info = "0.1" daemonize = "0.2" +regex = "0.1" +lazy_static = "0.1" [features] default = ["rpc"] diff --git a/README.md b/README.md index b840195d8..4e238fd6f 100644 --- a/README.md +++ b/README.md @@ -50,7 +50,7 @@ cd .. curl -sf https://raw.githubusercontent.com/brson/multirust/master/blastoff.sh | sudo sh -s -- --yes # install beta and make it default -sudo multirust default beta +multirust default beta # download and build parity git clone https://github.com/ethcore/parity diff --git a/parity/main.rs b/parity/main.rs index 3279e1fed..aca2b9c03 100644 --- a/parity/main.rs +++ b/parity/main.rs @@ -26,17 +26,21 @@ extern crate ethcore; extern crate ethsync; #[macro_use] extern crate log as rlog; +#[macro_use] +extern crate lazy_static; extern crate env_logger; extern crate ctrlc; extern crate fdlimit; extern crate target_info; extern crate daemonize; +extern crate regex; #[cfg(feature = "rpc")] extern crate ethcore_rpc as rpc; use std::net::{SocketAddr}; use std::env; +use std::process::exit; use rlog::{LogLevelFilter}; use env_logger::LogBuilder; use ctrlc::CtrlC; @@ -51,6 +55,7 @@ use ethsync::EthSync; use docopt::Docopt; use target_info::Target; use daemonize::Daemonize; +use regex::Regex; const USAGE: &'static str = " Parity. Ethereum Client. @@ -150,6 +155,16 @@ By Wood/Paronyan/Kotewicz/DrwiÄ™ga/Volf.\ ", env!("CARGO_PKG_VERSION"), Target::arch(), Target::env(), Target::os()); } +fn die_with_message(msg: &str) -> ! { + println!("ERROR: {}", msg); + exit(1); +} + +#[macro_export] +macro_rules! die { + ($($arg:tt)*) => (die_with_message(&format!("{}", format_args!($($arg)*)))); +} + struct Configuration { args: Args } @@ -174,7 +189,17 @@ impl Configuration { "frontier" | "mainnet" => ethereum::new_frontier(), "morden" | "testnet" => ethereum::new_morden(), "olympic" => ethereum::new_olympic(), - f => Spec::from_json_utf8(contents(f).expect("Couldn't read chain specification file. Sure it exists?").as_ref()), + f => Spec::from_json_utf8(contents(f).unwrap_or_else(|_| die!("{}: Couldn't read chain specification file. Sure it exists?", f)).as_ref()), + } + } + + fn normalize_enode(e: &str) -> Option { + lazy_static! { + static ref RE: Regex = Regex::new(r"^enode://([0-9a-fA-F]{64})@(\d+\.\d+\.\d+\.\d+):(\d+)$").unwrap(); + } + match RE.is_match(e) { + true => Some(e.to_owned()), + false => None, } } @@ -182,7 +207,7 @@ impl Configuration { if self.args.flag_no_bootstrap { Vec::new() } else { match self.args.arg_enode.len() { 0 => spec.nodes().clone(), - _ => self.args.arg_enode.clone(), // TODO check format first. + _ => self.args.arg_enode.iter().map(|s| Self::normalize_enode(s).unwrap_or_else(||die!("{}: Invalid node address format given for a boot node.", s))).collect(), } } } @@ -197,7 +222,7 @@ impl Configuration { public_address = SocketAddr::from_str(self.args.flag_public_address.as_ref()).expect("Invalid public address given with --public-address"); } Some(ref a) => { - public_address = SocketAddr::from_str(a.as_ref()).expect("Invalid listen/public address given with --address"); + public_address = SocketAddr::from_str(a.as_ref()).unwrap_or_else(|_|die!("{}: Invalid listen/public address given with --address", a)); listen_address = public_address; } }; @@ -205,17 +230,28 @@ impl Configuration { (listen_address, public_address) } + fn net_settings(&self, spec: &Spec) -> NetworkConfiguration { + let mut ret = NetworkConfiguration::new(); + ret.nat_enabled = self.args.flag_upnp; + ret.boot_nodes = self.init_nodes(spec); + let (listen, public) = self.net_addresses(); + ret.listen_address = listen; + ret.public_address = public; + ret.use_secret = self.args.flag_node_key.as_ref().map(|s| Secret::from_str(&s).unwrap_or_else(|_| s.as_bytes().sha3())); + ret + } + fn execute(&self) { if self.args.flag_version { print_version(); return; } if self.args.cmd_daemon { - let daemonize = Daemonize::new().pid_file(self.args.arg_pid_file.clone()).chown_pid_file(true); - match daemonize.start() { - Ok(_) => info!("Daemonized"), - Err(e) => { error!("{}", e); return; }, - } + Daemonize::new() + .pid_file(self.args.arg_pid_file.clone()) + .chown_pid_file(true) + .start() + .unwrap_or_else(|e| die!("Couldn't daemonize; {}", e)); } self.execute_client(); } @@ -227,15 +263,7 @@ impl Configuration { unsafe { ::fdlimit::raise_fd_limit(); } let spec = self.spec(); - - // Configure network - let mut net_settings = NetworkConfiguration::new(); - net_settings.nat_enabled = self.args.flag_upnp; - net_settings.boot_nodes = self.init_nodes(&spec); - let (listen, public) = self.net_addresses(); - net_settings.listen_address = listen; - net_settings.public_address = public; - net_settings.use_secret = self.args.flag_node_key.as_ref().map(|s| Secret::from_str(&s).expect("Invalid key string")); + let net_settings = self.net_settings(&spec); // Build client let mut service = ClientService::start(spec, net_settings, &Path::new(&self.path())).unwrap(); @@ -265,11 +293,13 @@ impl Configuration { fn wait_for_exit(client_service: &ClientService) { let exit = Arc::new(Condvar::new()); + // Handle possible exits let e = exit.clone(); CtrlC::set_handler(move || { e.notify_all(); }); let e = exit.clone(); client_service.on_panic(move |_reason| { e.notify_all(); }); + // Wait for signal let mutex = Mutex::new(()); let _ = exit.wait(mutex.lock().unwrap()).unwrap(); From 85c842b7fd52c08215ff09eca88d28db18eba103 Mon Sep 17 00:00:00 2001 From: arkpar Date: Fri, 19 Feb 2016 13:47:13 +0100 Subject: [PATCH 40/61] Restored service test --- ethcore/src/service.rs | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/ethcore/src/service.rs b/ethcore/src/service.rs index 9389f1db1..db0260b06 100644 --- a/ethcore/src/service.rs +++ b/ethcore/src/service.rs @@ -124,8 +124,6 @@ impl IoHandler for ClientIoHandler { } } -// TODO: rewrite into something that doesn't dependent on the testing environment having a particular port ready for use. -/* #[cfg(test)] mod tests { use super::*; @@ -136,8 +134,7 @@ mod tests { fn it_can_be_started() { let spec = get_test_spec(); let temp_path = RandomTempPath::new(); - let service = ClientService::start(spec, NetworkConfiguration::new(), &temp_path.as_path()); + let service = ClientService::start(spec, NetworkConfiguration::new_with_port(40456), &temp_path.as_path()); assert!(service.is_ok()); } } -*/ \ No newline at end of file From beab90c70715aaddf6c325e40aa8f125d062a4ef Mon Sep 17 00:00:00 2001 From: arkpar Date: Fri, 19 Feb 2016 14:13:20 +0100 Subject: [PATCH 41/61] Added is_valid_node_url --- util/src/network/mod.rs | 1 + util/src/network/node_table.rs | 6 ++++++ 2 files changed, 7 insertions(+) diff --git a/util/src/network/mod.rs b/util/src/network/mod.rs index ff52212af..50645f2be 100644 --- a/util/src/network/mod.rs +++ b/util/src/network/mod.rs @@ -89,6 +89,7 @@ pub use network::host::NetworkConfiguration; pub use network::stats::NetworkStats; use io::TimerToken; +pub use network::node_table::is_valid_node_url; const PROTOCOL_VERSION: u32 = 4; diff --git a/util/src/network/node_table.rs b/util/src/network/node_table.rs index 065990a63..f528f7134 100644 --- a/util/src/network/node_table.rs +++ b/util/src/network/node_table.rs @@ -332,6 +332,11 @@ impl Drop for NodeTable { } } +pub fn is_valid_node_url(url: &str) -> bool { + use std::str::FromStr; + Node::from_str(url).is_ok() +} + #[cfg(test)] mod tests { use super::*; @@ -353,6 +358,7 @@ mod tests { #[test] fn node_parse() { + assert!(is_valid_node_url("enode://a979fb575495b8d6db44f750317d0f4622bf4c2aa3365d6af7c284339968eef29b69ad0dce72a4d8db5ebb4968de0e3bec910127f134779fbcb0cb6d3331163c@22.99.55.44:7770")); let node = Node::from_str("enode://a979fb575495b8d6db44f750317d0f4622bf4c2aa3365d6af7c284339968eef29b69ad0dce72a4d8db5ebb4968de0e3bec910127f134779fbcb0cb6d3331163c@22.99.55.44:7770"); assert!(node.is_ok()); let node = node.unwrap(); From fed90c126ec80ba67fb3d6b16338866dc320de14 Mon Sep 17 00:00:00 2001 From: Nikolay Volf Date: Fri, 19 Feb 2016 17:18:20 +0300 Subject: [PATCH 42/61] dev/test tools to separate crate --- devtools/Cargo.toml | 16 + devtools/LICENSE.txt | 675 ++++++++++++++++++++++++++++++++ devtools/README.md | 1 + devtools/src/lib.rs | 24 ++ devtools/src/random_path.rs | 71 ++++ ethcore/Cargo.toml | 3 + ethcore/src/blockchain.rs | 5 +- ethcore/src/json_tests/chain.rs | 1 + ethcore/src/lib.rs | 1 + ethcore/src/state.rs | 3 +- ethcore/src/tests/client.rs | 1 + ethcore/src/tests/helpers.rs | 34 +- util/Cargo.toml | 4 + util/src/keys/directory.rs | 4 +- util/src/keys/store.rs | 4 +- util/src/lib.rs | 4 +- util/src/tests/helpers.rs | 31 -- util/src/tests/mod.rs | 1 - 18 files changed, 809 insertions(+), 74 deletions(-) create mode 100644 devtools/Cargo.toml create mode 100644 devtools/LICENSE.txt create mode 100644 devtools/README.md create mode 100644 devtools/src/lib.rs create mode 100644 devtools/src/random_path.rs delete mode 100644 util/src/tests/helpers.rs delete mode 100644 util/src/tests/mod.rs diff --git a/devtools/Cargo.toml b/devtools/Cargo.toml new file mode 100644 index 000000000..ce0260936 --- /dev/null +++ b/devtools/Cargo.toml @@ -0,0 +1,16 @@ +[package] +description = "Ethcore development/test/build tools" +homepage = "http://ethcore.io" +license = "GPL-3.0" +name = "ethcore-devtools" +version = "0.9.99" +authors = ["Ethcore "] + +[dependencies] +rand = "0.3" + +[features] + +[lib] +path = "src/lib.rs" +test = true diff --git a/devtools/LICENSE.txt b/devtools/LICENSE.txt new file mode 100644 index 000000000..733c07236 --- /dev/null +++ b/devtools/LICENSE.txt @@ -0,0 +1,675 @@ + GNU GENERAL PUBLIC LICENSE + Version 3, 29 June 2007 + + Copyright (C) 2007 Free Software Foundation, Inc. + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The GNU General Public License is a free, copyleft license for +software and other kinds of works. + + The licenses for most software and other practical works are designed +to take away your freedom to share and change the works. By contrast, +the GNU General Public License is intended to guarantee your freedom to +share and change all versions of a program--to make sure it remains free +software for all its users. We, the Free Software Foundation, use the +GNU General Public License for most of our software; it applies also to +any other work released this way by its authors. You can apply it to +your programs, too. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +them if you wish), that you receive source code or can get it if you +want it, that you can change the software or use pieces of it in new +free programs, and that you know you can do these things. + + To protect your rights, we need to prevent others from denying you +these rights or asking you to surrender the rights. Therefore, you have +certain responsibilities if you distribute copies of the software, or if +you modify it: responsibilities to respect the freedom of others. + + For example, if you distribute copies of such a program, whether +gratis or for a fee, you must pass on to the recipients the same +freedoms that you received. You must make sure that they, too, receive +or can get the source code. And you must show them these terms so they +know their rights. + + Developers that use the GNU GPL protect your rights with two steps: +(1) assert copyright on the software, and (2) offer you this License +giving you legal permission to copy, distribute and/or modify it. + + For the developers' and authors' protection, the GPL clearly explains +that there is no warranty for this free software. For both users' and +authors' sake, the GPL requires that modified versions be marked as +changed, so that their problems will not be attributed erroneously to +authors of previous versions. + + Some devices are designed to deny users access to install or run +modified versions of the software inside them, although the manufacturer +can do so. This is fundamentally incompatible with the aim of +protecting users' freedom to change the software. The systematic +pattern of such abuse occurs in the area of products for individuals to +use, which is precisely where it is most unacceptable. Therefore, we +have designed this version of the GPL to prohibit the practice for those +products. If such problems arise substantially in other domains, we +stand ready to extend this provision to those domains in future versions +of the GPL, as needed to protect the freedom of users. + + Finally, every program is threatened constantly by software patents. +States should not allow patents to restrict development and use of +software on general-purpose computers, but in those that do, we wish to +avoid the special danger that patents applied to a free program could +make it effectively proprietary. To prevent this, the GPL assures that +patents cannot be used to render the program non-free. + + The precise terms and conditions for copying, distribution and +modification follow. + + TERMS AND CONDITIONS + + 0. Definitions. + + "This License" refers to version 3 of the GNU General Public License. + + "Copyright" also means copyright-like laws that apply to other kinds of +works, such as semiconductor masks. + + "The Program" refers to any copyrightable work licensed under this +License. Each licensee is addressed as "you". "Licensees" and +"recipients" may be individuals or organizations. + + To "modify" a work means to copy from or adapt all or part of the work +in a fashion requiring copyright permission, other than the making of an +exact copy. The resulting work is called a "modified version" of the +earlier work or a work "based on" the earlier work. + + A "covered work" means either the unmodified Program or a work based +on the Program. + + To "propagate" a work means to do anything with it that, without +permission, would make you directly or secondarily liable for +infringement under applicable copyright law, except executing it on a +computer or modifying a private copy. Propagation includes copying, +distribution (with or without modification), making available to the +public, and in some countries other activities as well. + + To "convey" a work means any kind of propagation that enables other +parties to make or receive copies. Mere interaction with a user through +a computer network, with no transfer of a copy, is not conveying. + + An interactive user interface displays "Appropriate Legal Notices" +to the extent that it includes a convenient and prominently visible +feature that (1) displays an appropriate copyright notice, and (2) +tells the user that there is no warranty for the work (except to the +extent that warranties are provided), that licensees may convey the +work under this License, and how to view a copy of this License. If +the interface presents a list of user commands or options, such as a +menu, a prominent item in the list meets this criterion. + + 1. Source Code. + + The "source code" for a work means the preferred form of the work +for making modifications to it. "Object code" means any non-source +form of a work. + + A "Standard Interface" means an interface that either is an official +standard defined by a recognized standards body, or, in the case of +interfaces specified for a particular programming language, one that +is widely used among developers working in that language. + + The "System Libraries" of an executable work include anything, other +than the work as a whole, that (a) is included in the normal form of +packaging a Major Component, but which is not part of that Major +Component, and (b) serves only to enable use of the work with that +Major Component, or to implement a Standard Interface for which an +implementation is available to the public in source code form. A +"Major Component", in this context, means a major essential component +(kernel, window system, and so on) of the specific operating system +(if any) on which the executable work runs, or a compiler used to +produce the work, or an object code interpreter used to run it. + + The "Corresponding Source" for a work in object code form means all +the source code needed to generate, install, and (for an executable +work) run the object code and to modify the work, including scripts to +control those activities. However, it does not include the work's +System Libraries, or general-purpose tools or generally available free +programs which are used unmodified in performing those activities but +which are not part of the work. For example, Corresponding Source +includes interface definition files associated with source files for +the work, and the source code for shared libraries and dynamically +linked subprograms that the work is specifically designed to require, +such as by intimate data communication or control flow between those +subprograms and other parts of the work. + + The Corresponding Source need not include anything that users +can regenerate automatically from other parts of the Corresponding +Source. + + The Corresponding Source for a work in source code form is that +same work. + + 2. Basic Permissions. + + All rights granted under this License are granted for the term of +copyright on the Program, and are irrevocable provided the stated +conditions are met. This License explicitly affirms your unlimited +permission to run the unmodified Program. The output from running a +covered work is covered by this License only if the output, given its +content, constitutes a covered work. This License acknowledges your +rights of fair use or other equivalent, as provided by copyright law. + + You may make, run and propagate covered works that you do not +convey, without conditions so long as your license otherwise remains +in force. You may convey covered works to others for the sole purpose +of having them make modifications exclusively for you, or provide you +with facilities for running those works, provided that you comply with +the terms of this License in conveying all material for which you do +not control copyright. Those thus making or running the covered works +for you must do so exclusively on your behalf, under your direction +and control, on terms that prohibit them from making any copies of +your copyrighted material outside their relationship with you. + + Conveying under any other circumstances is permitted solely under +the conditions stated below. Sublicensing is not allowed; section 10 +makes it unnecessary. + + 3. Protecting Users' Legal Rights From Anti-Circumvention Law. + + No covered work shall be deemed part of an effective technological +measure under any applicable law fulfilling obligations under article +11 of the WIPO copyright treaty adopted on 20 December 1996, or +similar laws prohibiting or restricting circumvention of such +measures. + + When you convey a covered work, you waive any legal power to forbid +circumvention of technological measures to the extent such circumvention +is effected by exercising rights under this License with respect to +the covered work, and you disclaim any intention to limit operation or +modification of the work as a means of enforcing, against the work's +users, your or third parties' legal rights to forbid circumvention of +technological measures. + + 4. Conveying Verbatim Copies. + + You may convey verbatim copies of the Program's source code as you +receive it, in any medium, provided that you conspicuously and +appropriately publish on each copy an appropriate copyright notice; +keep intact all notices stating that this License and any +non-permissive terms added in accord with section 7 apply to the code; +keep intact all notices of the absence of any warranty; and give all +recipients a copy of this License along with the Program. + + You may charge any price or no price for each copy that you convey, +and you may offer support or warranty protection for a fee. + + 5. Conveying Modified Source Versions. + + You may convey a work based on the Program, or the modifications to +produce it from the Program, in the form of source code under the +terms of section 4, provided that you also meet all of these conditions: + + a) The work must carry prominent notices stating that you modified + it, and giving a relevant date. + + b) The work must carry prominent notices stating that it is + released under this License and any conditions added under section + 7. This requirement modifies the requirement in section 4 to + "keep intact all notices". + + c) You must license the entire work, as a whole, under this + License to anyone who comes into possession of a copy. This + License will therefore apply, along with any applicable section 7 + additional terms, to the whole of the work, and all its parts, + regardless of how they are packaged. This License gives no + permission to license the work in any other way, but it does not + invalidate such permission if you have separately received it. + + d) If the work has interactive user interfaces, each must display + Appropriate Legal Notices; however, if the Program has interactive + interfaces that do not display Appropriate Legal Notices, your + work need not make them do so. + + A compilation of a covered work with other separate and independent +works, which are not by their nature extensions of the covered work, +and which are not combined with it such as to form a larger program, +in or on a volume of a storage or distribution medium, is called an +"aggregate" if the compilation and its resulting copyright are not +used to limit the access or legal rights of the compilation's users +beyond what the individual works permit. Inclusion of a covered work +in an aggregate does not cause this License to apply to the other +parts of the aggregate. + + 6. Conveying Non-Source Forms. + + You may convey a covered work in object code form under the terms +of sections 4 and 5, provided that you also convey the +machine-readable Corresponding Source under the terms of this License, +in one of these ways: + + a) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by the + Corresponding Source fixed on a durable physical medium + customarily used for software interchange. + + b) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by a + written offer, valid for at least three years and valid for as + long as you offer spare parts or customer support for that product + model, to give anyone who possesses the object code either (1) a + copy of the Corresponding Source for all the software in the + product that is covered by this License, on a durable physical + medium customarily used for software interchange, for a price no + more than your reasonable cost of physically performing this + conveying of source, or (2) access to copy the + Corresponding Source from a network server at no charge. + + c) Convey individual copies of the object code with a copy of the + written offer to provide the Corresponding Source. This + alternative is allowed only occasionally and noncommercially, and + only if you received the object code with such an offer, in accord + with subsection 6b. + + d) Convey the object code by offering access from a designated + place (gratis or for a charge), and offer equivalent access to the + Corresponding Source in the same way through the same place at no + further charge. You need not require recipients to copy the + Corresponding Source along with the object code. If the place to + copy the object code is a network server, the Corresponding Source + may be on a different server (operated by you or a third party) + that supports equivalent copying facilities, provided you maintain + clear directions next to the object code saying where to find the + Corresponding Source. Regardless of what server hosts the + Corresponding Source, you remain obligated to ensure that it is + available for as long as needed to satisfy these requirements. + + e) Convey the object code using peer-to-peer transmission, provided + you inform other peers where the object code and Corresponding + Source of the work are being offered to the general public at no + charge under subsection 6d. + + A separable portion of the object code, whose source code is excluded +from the Corresponding Source as a System Library, need not be +included in conveying the object code work. + + A "User Product" is either (1) a "consumer product", which means any +tangible personal property which is normally used for personal, family, +or household purposes, or (2) anything designed or sold for incorporation +into a dwelling. In determining whether a product is a consumer product, +doubtful cases shall be resolved in favor of coverage. For a particular +product received by a particular user, "normally used" refers to a +typical or common use of that class of product, regardless of the status +of the particular user or of the way in which the particular user +actually uses, or expects or is expected to use, the product. A product +is a consumer product regardless of whether the product has substantial +commercial, industrial or non-consumer uses, unless such uses represent +the only significant mode of use of the product. + + "Installation Information" for a User Product means any methods, +procedures, authorization keys, or other information required to install +and execute modified versions of a covered work in that User Product from +a modified version of its Corresponding Source. The information must +suffice to ensure that the continued functioning of the modified object +code is in no case prevented or interfered with solely because +modification has been made. + + If you convey an object code work under this section in, or with, or +specifically for use in, a User Product, and the conveying occurs as +part of a transaction in which the right of possession and use of the +User Product is transferred to the recipient in perpetuity or for a +fixed term (regardless of how the transaction is characterized), the +Corresponding Source conveyed under this section must be accompanied +by the Installation Information. But this requirement does not apply +if neither you nor any third party retains the ability to install +modified object code on the User Product (for example, the work has +been installed in ROM). + + The requirement to provide Installation Information does not include a +requirement to continue to provide support service, warranty, or updates +for a work that has been modified or installed by the recipient, or for +the User Product in which it has been modified or installed. Access to a +network may be denied when the modification itself materially and +adversely affects the operation of the network or violates the rules and +protocols for communication across the network. + + Corresponding Source conveyed, and Installation Information provided, +in accord with this section must be in a format that is publicly +documented (and with an implementation available to the public in +source code form), and must require no special password or key for +unpacking, reading or copying. + + 7. Additional Terms. + + "Additional permissions" are terms that supplement the terms of this +License by making exceptions from one or more of its conditions. +Additional permissions that are applicable to the entire Program shall +be treated as though they were included in this License, to the extent +that they are valid under applicable law. If additional permissions +apply only to part of the Program, that part may be used separately +under those permissions, but the entire Program remains governed by +this License without regard to the additional permissions. + + When you convey a copy of a covered work, you may at your option +remove any additional permissions from that copy, or from any part of +it. (Additional permissions may be written to require their own +removal in certain cases when you modify the work.) You may place +additional permissions on material, added by you to a covered work, +for which you have or can give appropriate copyright permission. + + Notwithstanding any other provision of this License, for material you +add to a covered work, you may (if authorized by the copyright holders of +that material) supplement the terms of this License with terms: + + a) Disclaiming warranty or limiting liability differently from the + terms of sections 15 and 16 of this License; or + + b) Requiring preservation of specified reasonable legal notices or + author attributions in that material or in the Appropriate Legal + Notices displayed by works containing it; or + + c) Prohibiting misrepresentation of the origin of that material, or + requiring that modified versions of such material be marked in + reasonable ways as different from the original version; or + + d) Limiting the use for publicity purposes of names of licensors or + authors of the material; or + + e) Declining to grant rights under trademark law for use of some + trade names, trademarks, or service marks; or + + f) Requiring indemnification of licensors and authors of that + material by anyone who conveys the material (or modified versions of + it) with contractual assumptions of liability to the recipient, for + any liability that these contractual assumptions directly impose on + those licensors and authors. + + All other non-permissive additional terms are considered "further +restrictions" within the meaning of section 10. If the Program as you +received it, or any part of it, contains a notice stating that it is +governed by this License along with a term that is a further +restriction, you may remove that term. If a license document contains +a further restriction but permits relicensing or conveying under this +License, you may add to a covered work material governed by the terms +of that license document, provided that the further restriction does +not survive such relicensing or conveying. + + If you add terms to a covered work in accord with this section, you +must place, in the relevant source files, a statement of the +additional terms that apply to those files, or a notice indicating +where to find the applicable terms. + + Additional terms, permissive or non-permissive, may be stated in the +form of a separately written license, or stated as exceptions; +the above requirements apply either way. + + 8. Termination. + + You may not propagate or modify a covered work except as expressly +provided under this License. Any attempt otherwise to propagate or +modify it is void, and will automatically terminate your rights under +this License (including any patent licenses granted under the third +paragraph of section 11). + + However, if you cease all violation of this License, then your +license from a particular copyright holder is reinstated (a) +provisionally, unless and until the copyright holder explicitly and +finally terminates your license, and (b) permanently, if the copyright +holder fails to notify you of the violation by some reasonable means +prior to 60 days after the cessation. + + Moreover, your license from a particular copyright holder is +reinstated permanently if the copyright holder notifies you of the +violation by some reasonable means, this is the first time you have +received notice of violation of this License (for any work) from that +copyright holder, and you cure the violation prior to 30 days after +your receipt of the notice. + + Termination of your rights under this section does not terminate the +licenses of parties who have received copies or rights from you under +this License. If your rights have been terminated and not permanently +reinstated, you do not qualify to receive new licenses for the same +material under section 10. + + 9. Acceptance Not Required for Having Copies. + + You are not required to accept this License in order to receive or +run a copy of the Program. Ancillary propagation of a covered work +occurring solely as a consequence of using peer-to-peer transmission +to receive a copy likewise does not require acceptance. However, +nothing other than this License grants you permission to propagate or +modify any covered work. These actions infringe copyright if you do +not accept this License. Therefore, by modifying or propagating a +covered work, you indicate your acceptance of this License to do so. + + 10. Automatic Licensing of Downstream Recipients. + + Each time you convey a covered work, the recipient automatically +receives a license from the original licensors, to run, modify and +propagate that work, subject to this License. You are not responsible +for enforcing compliance by third parties with this License. + + An "entity transaction" is a transaction transferring control of an +organization, or substantially all assets of one, or subdividing an +organization, or merging organizations. If propagation of a covered +work results from an entity transaction, each party to that +transaction who receives a copy of the work also receives whatever +licenses to the work the party's predecessor in interest had or could +give under the previous paragraph, plus a right to possession of the +Corresponding Source of the work from the predecessor in interest, if +the predecessor has it or can get it with reasonable efforts. + + You may not impose any further restrictions on the exercise of the +rights granted or affirmed under this License. For example, you may +not impose a license fee, royalty, or other charge for exercise of +rights granted under this License, and you may not initiate litigation +(including a cross-claim or counterclaim in a lawsuit) alleging that +any patent claim is infringed by making, using, selling, offering for +sale, or importing the Program or any portion of it. + + 11. Patents. + + A "contributor" is a copyright holder who authorizes use under this +License of the Program or a work on which the Program is based. The +work thus licensed is called the contributor's "contributor version". + + A contributor's "essential patent claims" are all patent claims +owned or controlled by the contributor, whether already acquired or +hereafter acquired, that would be infringed by some manner, permitted +by this License, of making, using, or selling its contributor version, +but do not include claims that would be infringed only as a +consequence of further modification of the contributor version. For +purposes of this definition, "control" includes the right to grant +patent sublicenses in a manner consistent with the requirements of +this License. + + Each contributor grants you a non-exclusive, worldwide, royalty-free +patent license under the contributor's essential patent claims, to +make, use, sell, offer for sale, import and otherwise run, modify and +propagate the contents of its contributor version. + + In the following three paragraphs, a "patent license" is any express +agreement or commitment, however denominated, not to enforce a patent +(such as an express permission to practice a patent or covenant not to +sue for patent infringement). To "grant" such a patent license to a +party means to make such an agreement or commitment not to enforce a +patent against the party. + + If you convey a covered work, knowingly relying on a patent license, +and the Corresponding Source of the work is not available for anyone +to copy, free of charge and under the terms of this License, through a +publicly available network server or other readily accessible means, +then you must either (1) cause the Corresponding Source to be so +available, or (2) arrange to deprive yourself of the benefit of the +patent license for this particular work, or (3) arrange, in a manner +consistent with the requirements of this License, to extend the patent +license to downstream recipients. "Knowingly relying" means you have +actual knowledge that, but for the patent license, your conveying the +covered work in a country, or your recipient's use of the covered work +in a country, would infringe one or more identifiable patents in that +country that you have reason to believe are valid. + + If, pursuant to or in connection with a single transaction or +arrangement, you convey, or propagate by procuring conveyance of, a +covered work, and grant a patent license to some of the parties +receiving the covered work authorizing them to use, propagate, modify +or convey a specific copy of the covered work, then the patent license +you grant is automatically extended to all recipients of the covered +work and works based on it. + + A patent license is "discriminatory" if it does not include within +the scope of its coverage, prohibits the exercise of, or is +conditioned on the non-exercise of one or more of the rights that are +specifically granted under this License. You may not convey a covered +work if you are a party to an arrangement with a third party that is +in the business of distributing software, under which you make payment +to the third party based on the extent of your activity of conveying +the work, and under which the third party grants, to any of the +parties who would receive the covered work from you, a discriminatory +patent license (a) in connection with copies of the covered work +conveyed by you (or copies made from those copies), or (b) primarily +for and in connection with specific products or compilations that +contain the covered work, unless you entered into that arrangement, +or that patent license was granted, prior to 28 March 2007. + + Nothing in this License shall be construed as excluding or limiting +any implied license or other defenses to infringement that may +otherwise be available to you under applicable patent law. + + 12. No Surrender of Others' Freedom. + + If conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot convey a +covered work so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you may +not convey it at all. For example, if you agree to terms that obligate you +to collect a royalty for further conveying from those to whom you convey +the Program, the only way you could satisfy both those terms and this +License would be to refrain entirely from conveying the Program. + + 13. Use with the GNU Affero General Public License. + + Notwithstanding any other provision of this License, you have +permission to link or combine any covered work with a work licensed +under version 3 of the GNU Affero General Public License into a single +combined work, and to convey the resulting work. The terms of this +License will continue to apply to the part which is the covered work, +but the special requirements of the GNU Affero General Public License, +section 13, concerning interaction through a network will apply to the +combination as such. + + 14. Revised Versions of this License. + + The Free Software Foundation may publish revised and/or new versions of +the GNU General Public License from time to time. Such new versions will +be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + + Each version is given a distinguishing version number. If the +Program specifies that a certain numbered version of the GNU General +Public License "or any later version" applies to it, you have the +option of following the terms and conditions either of that numbered +version or of any later version published by the Free Software +Foundation. If the Program does not specify a version number of the +GNU General Public License, you may choose any version ever published +by the Free Software Foundation. + + If the Program specifies that a proxy can decide which future +versions of the GNU General Public License can be used, that proxy's +public statement of acceptance of a version permanently authorizes you +to choose that version for the Program. + + Later license versions may give you additional or different +permissions. However, no additional obligations are imposed on any +author or copyright holder as a result of your choosing to follow a +later version. + + 15. Disclaimer of Warranty. + + THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY +APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT +HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY +OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, +THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM +IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF +ALL NECESSARY SERVICING, REPAIR OR CORRECTION. + + 16. Limitation of Liability. + + IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS +THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY +GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE +USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF +DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD +PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), +EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF +SUCH DAMAGES. + + 17. Interpretation of Sections 15 and 16. + + If the disclaimer of warranty and limitation of liability provided +above cannot be given local legal effect according to their terms, +reviewing courts shall apply local law that most closely approximates +an absolute waiver of all civil liability in connection with the +Program, unless a warranty or assumption of liability accompanies a +copy of the Program in return for a fee. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +state the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + {one line to give the program's name and a brief idea of what it does.} + Copyright (C) {year} {name of author} + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + +Also add information on how to contact you by electronic and paper mail. + + If the program does terminal interaction, make it output a short +notice like this when it starts in an interactive mode: + + {project} Copyright (C) {year} {fullname} + This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'. + This is free software, and you are welcome to redistribute it + under certain conditions; type `show c' for details. + +The hypothetical commands `show w' and `show c' should show the appropriate +parts of the General Public License. Of course, your program's commands +might be different; for a GUI interface, you would use an "about box". + + You should also get your employer (if you work as a programmer) or school, +if any, to sign a "copyright disclaimer" for the program, if necessary. +For more information on this, and how to apply and follow the GNU GPL, see +. + + The GNU General Public License does not permit incorporating your program +into proprietary programs. If your program is a subroutine library, you +may consider it more useful to permit linking proprietary applications with +the library. If this is what you want to do, use the GNU Lesser General +Public License instead of this License. But first, please read +. + diff --git a/devtools/README.md b/devtools/README.md new file mode 100644 index 000000000..5d5144689 --- /dev/null +++ b/devtools/README.md @@ -0,0 +1 @@ +# ethcore dev tools diff --git a/devtools/src/lib.rs b/devtools/src/lib.rs new file mode 100644 index 000000000..f310cca30 --- /dev/null +++ b/devtools/src/lib.rs @@ -0,0 +1,24 @@ +// Copyright 2015, 2016 Ethcore (UK) Ltd. +// This file is part of Parity. + +// Parity is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Parity is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Parity. If not, see . + +//! dev-tools + + +extern crate rand; + +pub mod random_path; + +pub use random_path::*; diff --git a/devtools/src/random_path.rs b/devtools/src/random_path.rs new file mode 100644 index 000000000..3cbd19b96 --- /dev/null +++ b/devtools/src/random_path.rs @@ -0,0 +1,71 @@ +use std::path::*; +use std::fs; +use std::env; +use rand::random; + +pub struct RandomTempPath { + path: PathBuf +} + +pub fn random_filename() -> String { + (0..8).map(|_| ((random::() * 26.0) as u8 + 97) as char).collect() +} + +impl RandomTempPath { + pub fn new() -> RandomTempPath { + let mut dir = env::temp_dir(); + dir.push(random_filename()); + RandomTempPath { + path: dir.clone() + } + } + + pub fn create_dir() -> RandomTempPath { + let mut dir = env::temp_dir(); + dir.push(random_filename()); + fs::create_dir_all(dir.as_path()).unwrap(); + RandomTempPath { + path: dir.clone() + } + } + + pub fn as_path(&self) -> &PathBuf { + &self.path + } + + pub fn as_str(&self) -> &str { + self.path.to_str().unwrap() + } +} + +impl Drop for RandomTempPath { + fn drop(&mut self) { + if let Err(e) = fs::remove_dir_all(self.as_path()) { + panic!("failed to remove temp directory, probably something failed to destroyed ({})", e); + } + } +} + +#[test] +fn creates_dir() { + let temp = RandomTempPath::create_dir(); + assert!(fs::metadata(temp.as_path()).unwrap().is_dir()); +} + +#[test] +fn destroys_dir() { + let path_buf = { + let temp = RandomTempPath::create_dir(); + assert!(fs::metadata(temp.as_path()).unwrap().is_dir()); + let path_buf = temp.as_path().to_path_buf(); + path_buf + }; + + assert!(fs::metadata(&path_buf).is_err()); +} + +#[test] +fn provides_random() { + let temp = RandomTempPath::create_dir(); + assert!(temp.as_path().to_str().is_some()); +} diff --git a/ethcore/Cargo.toml b/ethcore/Cargo.toml index 090280cae..323da3e09 100644 --- a/ethcore/Cargo.toml +++ b/ethcore/Cargo.toml @@ -22,6 +22,9 @@ clippy = { version = "0.0.42", optional = true } crossbeam = "0.1.5" lazy_static = "0.1" +[dev-dependencies] +ethcore-devtools = { path = "../devtools" } + [features] jit = ["evmjit"] evm-debug = [] diff --git a/ethcore/src/blockchain.rs b/ethcore/src/blockchain.rs index 9240ff800..fc823411f 100644 --- a/ethcore/src/blockchain.rs +++ b/ethcore/src/blockchain.rs @@ -664,6 +664,7 @@ mod tests { use util::hash::*; use blockchain::*; use tests::helpers::*; + use devtools::*; #[test] fn valid_tests_extra32() { @@ -679,7 +680,7 @@ mod tests { assert_eq!(bc.best_block_hash(), genesis_hash.clone()); assert_eq!(bc.block_hash(0), Some(genesis_hash.clone())); assert_eq!(bc.block_hash(1), None); - + let first = "f90285f90219a03caa2203f3d7c136c0295ed128a7d31cea520b1ca5e27afe17d0853331798942a01dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347948888f1f195afa192cfee860698584c030f4c9db1a0bac6177a79e910c98d86ec31a09ae37ac2de15b754fd7bed1ba52362c49416bfa0d45893a296c1490a978e0bd321b5f2635d8280365c1fe9f693d65f233e791344a0c7778a7376099ee2e5c455791c1885b5c361b95713fddcbe32d97fd01334d296b90100000000000000000010000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000200000000000000000008000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000040000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000200000000000400000000000000000000000000000000000000000000000000000008302000001832fefd882560b845627cb99a00102030405060708091011121314151617181920212223242526272829303132a08ccb2837fb2923bd97e8f2d08ea32012d6e34be018c73e49a0f98843e8f47d5d88e53be49fec01012ef866f864800a82c35094095e7baea6a6c7c4c2dfeb977efac326af552d8785012a05f200801ba0cb088b8d2ff76a7b2c6616c9d02fb6b7a501afbf8b69d7180b09928a1b80b5e4a06448fe7476c606582039bb72a9f6f4b4fad18507b8dfbd00eebbe151cc573cd2c0".from_hex().unwrap(); bc.insert_block(&first); @@ -855,7 +856,7 @@ mod tests { let temp = RandomTempPath::new(); let bc = BlockChain::new(&genesis, temp.as_path()); bc.insert_block(&b1); - + let transactions = bc.transactions(&b1_hash).unwrap(); assert_eq!(transactions.len(), 7); for t in transactions { diff --git a/ethcore/src/json_tests/chain.rs b/ethcore/src/json_tests/chain.rs index 6a9f84073..a386e2854 100644 --- a/ethcore/src/json_tests/chain.rs +++ b/ethcore/src/json_tests/chain.rs @@ -20,6 +20,7 @@ use pod_state::*; use block::Block; use ethereum; use tests::helpers::*; +use devtools::*; pub fn json_chain_test(json_data: &[u8], era: ChainEra) -> Vec { init_log(); diff --git a/ethcore/src/lib.rs b/ethcore/src/lib.rs index 8e59cf7a2..9deb3a691 100644 --- a/ethcore/src/lib.rs +++ b/ethcore/src/lib.rs @@ -92,6 +92,7 @@ extern crate env_logger; extern crate num_cpus; extern crate crossbeam; +#[cfg(test)] extern crate ethcore_devtools as devtools; #[cfg(feature = "jit" )] extern crate evmjit; pub mod block; diff --git a/ethcore/src/state.rs b/ethcore/src/state.rs index bbec8cd37..8bbf317c1 100644 --- a/ethcore/src/state.rs +++ b/ethcore/src/state.rs @@ -163,7 +163,7 @@ impl State { /// Mutate storage of account `address` so that it is `value` for `key`. pub fn storage_at(&self, address: &Address, key: &H256) -> H256 { - self.get(address, false).as_ref().map_or(H256::new(), |a|a.storage_at(&AccountDB::new(&self.db, address), key)) + self.get(address, false).as_ref().map_or(H256::new(), |a|a.storage_at(&AccountDB::new(&self.db, address), key)) } /// Mutate storage of account `a` so that it is `value` for `key`. @@ -341,6 +341,7 @@ use util::rlp::*; use util::uint::*; use account::*; use tests::helpers::*; +use devtools::*; #[test] fn code_from_database() { diff --git a/ethcore/src/tests/client.rs b/ethcore/src/tests/client.rs index d025b4b78..af25d1b72 100644 --- a/ethcore/src/tests/client.rs +++ b/ethcore/src/tests/client.rs @@ -17,6 +17,7 @@ use client::{BlockChainClient, Client, BlockId}; use tests::helpers::*; use common::*; +use devtools::*; #[test] fn created() { diff --git a/ethcore/src/tests/helpers.rs b/ethcore/src/tests/helpers.rs index 93e3e0a0d..56653e820 100644 --- a/ethcore/src/tests/helpers.rs +++ b/ethcore/src/tests/helpers.rs @@ -15,17 +15,15 @@ // along with Parity. If not, see . use client::{BlockChainClient, Client}; -use std::env; use common::*; -use std::path::PathBuf; use spec::*; -use std::fs::{remove_dir_all}; use blockchain::{BlockChain}; use state::*; use rocksdb::*; use evm::{Schedule, Factory}; use engine::*; use ethereum; +use devtools::*; #[cfg(feature = "json-tests")] pub enum ChainEra { @@ -33,36 +31,6 @@ pub enum ChainEra { Homestead, } -pub struct RandomTempPath { - path: PathBuf -} - -impl RandomTempPath { - pub fn new() -> RandomTempPath { - let mut dir = env::temp_dir(); - dir.push(H32::random().hex()); - RandomTempPath { - path: dir.clone() - } - } - - pub fn as_path(&self) -> &PathBuf { - &self.path - } - - pub fn as_str(&self) -> &str { - self.path.to_str().unwrap() - } -} - -impl Drop for RandomTempPath { - fn drop(&mut self) { - if let Err(e) = remove_dir_all(self.as_path()) { - panic!("failed to remove temp directory, probably something failed to destroyed ({})", e); - } - } -} - #[cfg(test)] pub struct GuardedTempResult { result: Option, diff --git a/util/Cargo.toml b/util/Cargo.toml index 18e9a0a75..d01b0e495 100644 --- a/util/Cargo.toml +++ b/util/Cargo.toml @@ -31,6 +31,10 @@ json-tests = { path = "json-tests" } target_info = "0.1.0" igd = "0.4.2" +[dev-dependencies] +ethcore-devtools = { path = "../devtools" } + [features] default = [] dev = ["clippy"] +test = [] diff --git a/util/src/keys/directory.rs b/util/src/keys/directory.rs index 23c483482..bc875db3f 100644 --- a/util/src/keys/directory.rs +++ b/util/src/keys/directory.rs @@ -1030,7 +1030,7 @@ mod file_tests { mod directory_tests { use super::{KeyDirectory, new_uuid, uuid_to_string, KeyFileContent, KeyFileCrypto, MAX_CACHE_USAGE_TRACK}; use common::*; - use tests::helpers::*; + use devtools::*; #[test] fn key_directory_locates_keys() { @@ -1110,7 +1110,7 @@ mod directory_tests { mod specs { use super::*; use common::*; - use tests::helpers::*; + use devtools::*; #[test] fn can_initiate_key_directory() { diff --git a/util/src/keys/store.rs b/util/src/keys/store.rs index b8b0b0a47..ae44d567a 100644 --- a/util/src/keys/store.rs +++ b/util/src/keys/store.rs @@ -70,7 +70,7 @@ impl SecretStore { } #[cfg(test)] - fn new_test(path: &::tests::helpers::RandomTempPath) -> SecretStore { + fn new_test(path: &::devtools::RandomTempPath) -> SecretStore { SecretStore { directory: KeyDirectory::new(path.as_path()) } @@ -203,7 +203,7 @@ mod vector_tests { #[cfg(test)] mod tests { use super::*; - use tests::helpers::*; + use devtools::*; use common::*; #[test] diff --git a/util/src/lib.rs b/util/src/lib.rs index 59713a107..8f354b7c7 100644 --- a/util/src/lib.rs +++ b/util/src/lib.rs @@ -106,6 +106,8 @@ extern crate serde; #[macro_use] extern crate log as rlog; extern crate igd; +#[cfg(test)] +extern crate ethcore_devtools as devtools; pub mod standard; #[macro_use] @@ -160,5 +162,3 @@ pub use network::*; pub use io::*; pub use log::*; -#[cfg(test)] -mod tests; diff --git a/util/src/tests/helpers.rs b/util/src/tests/helpers.rs deleted file mode 100644 index fee3d2cbb..000000000 --- a/util/src/tests/helpers.rs +++ /dev/null @@ -1,31 +0,0 @@ -use common::*; -use std::path::PathBuf; -use std::fs::{remove_dir_all}; -use std::env; - -pub struct RandomTempPath { - path: PathBuf -} - -impl RandomTempPath { - pub fn create_dir() -> RandomTempPath { - let mut dir = env::temp_dir(); - dir.push(H32::random().hex()); - fs::create_dir_all(dir.as_path()).unwrap(); - RandomTempPath { - path: dir.clone() - } - } - - pub fn as_path(&self) -> &PathBuf { - &self.path - } -} - -impl Drop for RandomTempPath { - fn drop(&mut self) { - if let Err(e) = remove_dir_all(self.as_path()) { - panic!("failed to remove temp directory, probably something failed to destroyed ({})", e); - } - } -} diff --git a/util/src/tests/mod.rs b/util/src/tests/mod.rs deleted file mode 100644 index 1630fabcd..000000000 --- a/util/src/tests/mod.rs +++ /dev/null @@ -1 +0,0 @@ -pub mod helpers; From bd18b4930d18af803d7afd6c61437975f3258fbf Mon Sep 17 00:00:00 2001 From: Nikolay Volf Date: Fri, 19 Feb 2016 17:20:02 +0300 Subject: [PATCH 43/61] license for random_path.rs --- devtools/src/random_path.rs | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/devtools/src/random_path.rs b/devtools/src/random_path.rs index 3cbd19b96..b037867fa 100644 --- a/devtools/src/random_path.rs +++ b/devtools/src/random_path.rs @@ -1,3 +1,21 @@ +// Copyright 2015, 2016 Ethcore (UK) Ltd. +// This file is part of Parity. + +// Parity is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Parity is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Parity. If not, see . + +//! Random path + use std::path::*; use std::fs; use std::env; From ab0fe65f3f1cc6387c63947da8236f76325330b4 Mon Sep 17 00:00:00 2001 From: Nikolay Volf Date: Fri, 19 Feb 2016 18:09:31 +0300 Subject: [PATCH 44/61] unlisting as dev-dependencies --- Cargo.lock | 9 +++++++++ Cargo.toml | 1 + ethcore/Cargo.toml | 2 -- util/Cargo.toml | 4 ---- 4 files changed, 10 insertions(+), 6 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index b69736c2e..6ae4a06ca 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8,6 +8,7 @@ dependencies = [ "docopt 0.6.78 (registry+https://github.com/rust-lang/crates.io-index)", "env_logger 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)", "ethcore 0.9.99", + "ethcore-devtools 0.9.99", "ethcore-rpc 0.9.99", "ethcore-util 0.9.99", "ethsync 0.9.99", @@ -184,6 +185,13 @@ dependencies = [ "time 0.1.34 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "ethcore-devtools" +version = "0.9.99" +dependencies = [ + "rand 0.3.14 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "ethcore-rpc" version = "0.9.99" @@ -212,6 +220,7 @@ dependencies = [ "elastic-array 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", "env_logger 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)", "eth-secp256k1 0.5.4 (git+https://github.com/arkpar/rust-secp256k1.git)", + "ethcore-devtools 0.9.99", "heapsize 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)", "igd 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", "itertools 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", diff --git a/Cargo.toml b/Cargo.toml index 5b59b26f1..3cb158df7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -19,6 +19,7 @@ ethcore-rpc = { path = "rpc", optional = true } fdlimit = { path = "util/fdlimit" } target_info = "0.1" daemonize = "0.2" +ethcore-devtools = { path = "devtools" } [features] default = ["rpc"] diff --git a/ethcore/Cargo.toml b/ethcore/Cargo.toml index 323da3e09..d34a5478b 100644 --- a/ethcore/Cargo.toml +++ b/ethcore/Cargo.toml @@ -21,8 +21,6 @@ num_cpus = "0.2" clippy = { version = "0.0.42", optional = true } crossbeam = "0.1.5" lazy_static = "0.1" - -[dev-dependencies] ethcore-devtools = { path = "../devtools" } [features] diff --git a/util/Cargo.toml b/util/Cargo.toml index d01b0e495..0ff7fe318 100644 --- a/util/Cargo.toml +++ b/util/Cargo.toml @@ -30,11 +30,7 @@ clippy = { version = "0.0.42", optional = true } json-tests = { path = "json-tests" } target_info = "0.1.0" igd = "0.4.2" - -[dev-dependencies] ethcore-devtools = { path = "../devtools" } - [features] default = [] dev = ["clippy"] -test = [] From ab233a941f83fd4f162de2c6a8a10ca5fc917ddd Mon Sep 17 00:00:00 2001 From: arkpar Date: Fri, 19 Feb 2016 16:34:31 +0100 Subject: [PATCH 45/61] Slightly improved tests --- util/src/network/host.rs | 7 +++++++ util/src/network/node_table.rs | 1 + util/src/network/tests.rs | 4 +++- 3 files changed, 11 insertions(+), 1 deletion(-) diff --git a/util/src/network/host.rs b/util/src/network/host.rs index b2be68a32..78fb274fa 100644 --- a/util/src/network/host.rs +++ b/util/src/network/host.rs @@ -209,6 +209,12 @@ impl<'s, Message> NetworkContext<'s, Message> where Message: Send + Sync + Clone } } + /// Send an IO message + pub fn message(&self, msg: Message) { + self.io.message(NetworkIoMessage::User(msg)); + } + + /// Disable current protocol capability for given peer. If no capabilities left peer gets disconnected. pub fn disable_peer(&self, peer: PeerId) { //TODO: remove capability, disconnect if no capabilities left @@ -754,6 +760,7 @@ impl IoHandler> for Host where Messa io.register_timer(DISCOVERY_REFRESH, 7200).expect("Error registering discovery timer"); io.register_timer(DISCOVERY_ROUND, 300).expect("Error registering discovery timer"); } + self.maintain_network(io) } fn stream_hup(&self, io: &IoContext>, stream: StreamToken) { diff --git a/util/src/network/node_table.rs b/util/src/network/node_table.rs index f528f7134..7ca060f75 100644 --- a/util/src/network/node_table.rs +++ b/util/src/network/node_table.rs @@ -332,6 +332,7 @@ impl Drop for NodeTable { } } +/// Check if node url is valid pub fn is_valid_node_url(url: &str) -> bool { use std::str::FromStr; Node::from_str(url).is_ok() diff --git a/util/src/network/tests.rs b/util/src/network/tests.rs index dc80936d2..44d53bdbe 100644 --- a/util/src/network/tests.rs +++ b/util/src/network/tests.rs @@ -74,6 +74,7 @@ impl NetworkProtocolHandler for TestProtocol { } fn connected(&self, io: &NetworkContext, peer: &PeerId) { + assert!(io.peer_info(*peer).contains("parity")); if self.drop_session { io.disconnect_peer(*peer) } else { @@ -86,7 +87,8 @@ impl NetworkProtocolHandler for TestProtocol { } /// Timer function called after a timeout created with `NetworkContext::timeout`. - fn timeout(&self, _io: &NetworkContext, timer: TimerToken) { + fn timeout(&self, io: &NetworkContext, timer: TimerToken) { + io.message(TestProtocolMessage { payload: 22 }); assert_eq!(timer, 0); self.got_timeout.store(true, AtomicOrdering::Relaxed); } From 5572d1792d3546df050785e34ebfe3f8712d0961 Mon Sep 17 00:00:00 2001 From: arkpar Date: Fri, 19 Feb 2016 18:42:54 +0100 Subject: [PATCH 46/61] Back to original slab crate --- Cargo.lock | 7 +------ util/Cargo.toml | 2 +- 2 files changed, 2 insertions(+), 7 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 250acb5a8..d4d1ceaff 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -226,7 +226,7 @@ dependencies = [ "rustc-serialize 0.3.18 (registry+https://github.com/rust-lang/crates.io-index)", "serde 0.6.14 (registry+https://github.com/rust-lang/crates.io-index)", "sha3 0.1.0", - "slab 0.1.4 (git+https://github.com/arkpar/slab.git)", + "slab 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)", "target_info 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", "time 0.1.34 (registry+https://github.com/rust-lang/crates.io-index)", "tiny-keccak 1.0.5 (registry+https://github.com/rust-lang/crates.io-index)", @@ -643,11 +643,6 @@ name = "slab" version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -[[package]] -name = "slab" -version = "0.1.4" -source = "git+https://github.com/arkpar/slab.git#3c9284e1f010e394c9d0359b27464e8fb5c87bf0" - [[package]] name = "solicit" version = "0.4.4" diff --git a/util/Cargo.toml b/util/Cargo.toml index 2675ba56c..c27d4bdc3 100644 --- a/util/Cargo.toml +++ b/util/Cargo.toml @@ -23,7 +23,7 @@ elastic-array = "0.4" heapsize = "0.3" itertools = "0.4" crossbeam = "0.2" -slab = { git = "https://github.com/arkpar/slab.git" } +slab = "0.1" sha3 = { path = "sha3" } serde = "0.6.7" clippy = { version = "0.0.42", optional = true } From 6c82e405ddae5de03b784d2608c66a9dc4f8b683 Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Fri, 19 Feb 2016 19:42:23 +0100 Subject: [PATCH 47/61] Remove regex &c., use network code for enode ID. --- Cargo.lock | 1 - Cargo.toml | 2 -- parity/main.rs | 12 +++--------- 3 files changed, 3 insertions(+), 12 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 34b618c2b..8891e4658 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -12,7 +12,6 @@ dependencies = [ "ethcore-util 0.9.99", "ethsync 0.9.99", "fdlimit 0.1.0", - "lazy_static 0.1.15 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)", "regex 0.1.51 (registry+https://github.com/rust-lang/crates.io-index)", "rustc-serialize 0.3.18 (registry+https://github.com/rust-lang/crates.io-index)", diff --git a/Cargo.toml b/Cargo.toml index f4163253b..5b59b26f1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -19,8 +19,6 @@ ethcore-rpc = { path = "rpc", optional = true } fdlimit = { path = "util/fdlimit" } target_info = "0.1" daemonize = "0.2" -regex = "0.1" -lazy_static = "0.1" [features] default = ["rpc"] diff --git a/parity/main.rs b/parity/main.rs index aca2b9c03..0f1f77606 100644 --- a/parity/main.rs +++ b/parity/main.rs @@ -26,8 +26,6 @@ extern crate ethcore; extern crate ethsync; #[macro_use] extern crate log as rlog; -#[macro_use] -extern crate lazy_static; extern crate env_logger; extern crate ctrlc; extern crate fdlimit; @@ -40,11 +38,13 @@ extern crate ethcore_rpc as rpc; use std::net::{SocketAddr}; use std::env; +use std::from_str::FromStr; use std::process::exit; use rlog::{LogLevelFilter}; use env_logger::LogBuilder; use ctrlc::CtrlC; use util::*; +use util::network::node::Node; use util::panics::MayPanic; use ethcore::spec::*; use ethcore::client::*; @@ -194,13 +194,7 @@ impl Configuration { } fn normalize_enode(e: &str) -> Option { - lazy_static! { - static ref RE: Regex = Regex::new(r"^enode://([0-9a-fA-F]{64})@(\d+\.\d+\.\d+\.\d+):(\d+)$").unwrap(); - } - match RE.is_match(e) { - true => Some(e.to_owned()), - false => None, - } + Node::from_str(e).ok().map(|_| e.to_owned()) } fn init_nodes(&self, spec: &Spec) -> Vec { From dc3ceeb5bb89d731d9ad2b0bf2431d76cabd84c0 Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Fri, 19 Feb 2016 20:02:23 +0100 Subject: [PATCH 48/61] Use new is_valid_node_url function. --- Cargo.lock | 1 - parity/main.rs | 10 +++++----- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index d475f81e1..250acb5a8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -13,7 +13,6 @@ dependencies = [ "ethsync 0.9.99", "fdlimit 0.1.0", "log 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)", - "regex 0.1.51 (registry+https://github.com/rust-lang/crates.io-index)", "rustc-serialize 0.3.18 (registry+https://github.com/rust-lang/crates.io-index)", "target_info 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", ] diff --git a/parity/main.rs b/parity/main.rs index 8930c60e9..58d3a6f4c 100644 --- a/parity/main.rs +++ b/parity/main.rs @@ -31,21 +31,18 @@ extern crate ctrlc; extern crate fdlimit; extern crate target_info; extern crate daemonize; -extern crate regex; #[cfg(feature = "rpc")] extern crate ethcore_rpc as rpc; use std::net::{SocketAddr}; use std::env; -use std::from_str::FromStr; use std::process::exit; use std::path::PathBuf; use rlog::{LogLevelFilter}; use env_logger::LogBuilder; use ctrlc::CtrlC; use util::*; -use util::network::node::Node; use util::panics::MayPanic; use ethcore::spec::*; use ethcore::client::*; @@ -56,7 +53,6 @@ use ethsync::EthSync; use docopt::Docopt; use target_info::Target; use daemonize::Daemonize; -use regex::Regex; const USAGE: &'static str = " Parity. Ethereum Client. @@ -199,7 +195,10 @@ impl Configuration { } fn normalize_enode(e: &str) -> Option { - Node::from_str(e).ok().map(|_| e.to_owned()) + match is_valid_node_url(e) { + true => Some(e.to_owned()), + false => None, + } } fn init_nodes(&self, spec: &Spec) -> Vec { @@ -244,6 +243,7 @@ impl Configuration { let mut net_path = PathBuf::from(&self.path()); net_path.push("network"); ret.config_path = Some(net_path.to_str().unwrap().to_owned()); + ret } fn execute(&self) { From 69df91de68fe1e345b0fd5e4202079e21bb7f3ae Mon Sep 17 00:00:00 2001 From: arkpar Date: Fri, 19 Feb 2016 22:09:06 +0100 Subject: [PATCH 49/61] Deregister handshake properly when converting to session --- util/src/network/connection.rs | 29 ++++++++++++++++++++++++----- util/src/network/host.rs | 29 +++++++++++++---------------- util/src/network/session.rs | 10 ++++++++-- 3 files changed, 45 insertions(+), 23 deletions(-) diff --git a/util/src/network/connection.rs b/util/src/network/connection.rs index 9e9304ca6..4b7886698 100644 --- a/util/src/network/connection.rs +++ b/util/src/network/connection.rs @@ -175,6 +175,18 @@ impl Connection { self.socket.peer_addr() } + pub fn try_clone(&self) -> io::Result { + Ok(Connection { + token: self.token, + socket: try!(self.socket.try_clone()), + rec_buf: Vec::new(), + rec_size: 0, + send_queue: VecDeque::new(), + interest: EventSet::hup() | EventSet::readable(), + stats: self.stats.clone(), + }) + } + /// Register this connection with the IO event loop. pub fn register_socket(&self, reg: Token, event_loop: &mut EventLoop) -> io::Result<()> { trace!(target: "net", "connection register; token={:?}", reg); @@ -265,7 +277,7 @@ impl EncryptedConnection { } /// Create an encrypted connection out of the handshake. Consumes a handshake object. - pub fn new(mut handshake: Handshake) -> Result { + pub fn new(handshake: &mut Handshake) -> Result { let shared = try!(crypto::ecdh::agree(handshake.ecdhe.secret(), &handshake.remote_public)); let mut nonce_material = H512::new(); if handshake.originated { @@ -300,9 +312,8 @@ impl EncryptedConnection { ingress_mac.update(&mac_material); ingress_mac.update(if handshake.originated { &handshake.ack_cipher } else { &handshake.auth_cipher }); - handshake.connection.expect(ENCRYPTED_HEADER_LEN); - Ok(EncryptedConnection { - connection: handshake.connection, + let mut enc = EncryptedConnection { + connection: try!(handshake.connection.try_clone()), encoder: encoder, decoder: decoder, mac_encoder: mac_encoder, @@ -311,7 +322,9 @@ impl EncryptedConnection { read_state: EncryptedConnectionState::Header, protocol_id: 0, payload_len: 0 - }) + }; + enc.connection.expect(ENCRYPTED_HEADER_LEN); + Ok(enc) } /// Send a packet @@ -440,6 +453,12 @@ impl EncryptedConnection { Ok(()) } + /// Register socket with the event lpop. This should be called at the end of the event loop. + pub fn register_socket(&self, reg: Token, event_loop: &mut EventLoop) -> Result<(), UtilError> { + try!(self.connection.register_socket(reg, event_loop)); + Ok(()) + } + /// Update connection registration. This should be called at the end of the event loop. pub fn update_socket(&self, reg: Token, event_loop: &mut EventLoop) -> Result<(), UtilError> { try!(self.connection.update_socket(reg, event_loop)); diff --git a/util/src/network/host.rs b/util/src/network/host.rs index 78fb274fa..f4f3b4e1b 100644 --- a/util/src/network/host.rs +++ b/util/src/network/host.rs @@ -544,7 +544,7 @@ impl Host where Message: Send + Sync + Clone { if let Some(handshake) = handshake { let mut h = handshake.lock().unwrap(); if let Err(e) = h.writable(io, &self.info.read().unwrap()) { - debug!(target: "net", "Handshake write error: {}:{:?}", token, e); + trace!(target: "net", "Handshake write error: {}: {:?}", token, e); } } } @@ -554,7 +554,7 @@ impl Host where Message: Send + Sync + Clone { if let Some(session) = session { let mut s = session.lock().unwrap(); if let Err(e) = s.writable(io, &self.info.read().unwrap()) { - debug!(target: "net", "Session write error: {}:{:?}", token, e); + trace!(target: "net", "Session write error: {}: {:?}", token, e); } io.update_registration(token).unwrap_or_else(|e| debug!(target: "net", "Session registration error: {:?}", e)); } @@ -571,7 +571,7 @@ impl Host where Message: Send + Sync + Clone { if let Some(handshake) = handshake { let mut h = handshake.lock().unwrap(); if let Err(e) = h.readable(io, &self.info.read().unwrap()) { - debug!(target: "net", "Handshake read error: {}:{:?}", token, e); + debug!(target: "net", "Handshake read error: {}: {:?}", token, e); kill = true; } if h.done() { @@ -583,7 +583,7 @@ impl Host where Message: Send + Sync + Clone { return; } else if create_session { self.start_session(token, io); - io.update_registration(token).unwrap_or_else(|e| debug!(target: "net", "Session registration error: {:?}", e)); + return; } io.update_registration(token).unwrap_or_else(|e| debug!(target: "net", "Token registration error: {:?}", e)); } @@ -597,7 +597,7 @@ impl Host where Message: Send + Sync + Clone { let mut s = session.lock().unwrap(); match s.readable(io, &self.info.read().unwrap()) { Err(e) => { - debug!(target: "net", "Session read error: {}:{:?}", token, e); + debug!(target: "net", "Session read error: {}: {:?}", token, e); kill = true; }, Ok(SessionData::Ready) => { @@ -642,16 +642,9 @@ impl Host where Message: Send + Sync + Clone { // turn a handshake into a session let mut sessions = self.sessions.write().unwrap(); - let mut h = handshakes.remove(token).unwrap(); - // wait for other threads to stop using it - { - while Arc::get_mut(&mut h).is_none() { - h.lock().ok(); - } - } - let h = Arc::try_unwrap(h).ok().unwrap().into_inner().unwrap(); + let mut h = handshakes.get_mut(token).unwrap().lock().unwrap(); let originated = h.originated; - let mut session = match Session::new(h, &self.info.read().unwrap()) { + let mut session = match Session::new(&mut h, &self.info.read().unwrap()) { Ok(s) => s, Err(e) => { debug!("Session creation error: {:?}", e); @@ -660,7 +653,8 @@ impl Host where Message: Send + Sync + Clone { }; let result = sessions.insert_with(move |session_token| { session.set_token(session_token); - io.update_registration(session_token).expect("Error updating session registration"); + io.deregister_stream(token).expect("Error deleting handshake registration"); + io.register_stream(session_token).expect("Error creating session registration"); self.stats.inc_sessions(); if !originated { // Add it no node table @@ -872,7 +866,10 @@ impl IoHandler> for Host where Messa fn register_stream(&self, stream: StreamToken, reg: Token, event_loop: &mut EventLoop>>) { match stream { FIRST_SESSION ... LAST_SESSION => { - warn!("Unexpected session stream registration"); + let session = { self.sessions.read().unwrap().get(stream).cloned() }; + if let Some(session) = session { + session.lock().unwrap().register_socket(reg, event_loop).expect("Error registering socket"); + } } FIRST_HANDSHAKE ... LAST_HANDSHAKE => { let connection = { self.handshakes.read().unwrap().get(stream).cloned() }; diff --git a/util/src/network/session.rs b/util/src/network/session.rs index b0db5f7ef..159dc795d 100644 --- a/util/src/network/session.rs +++ b/util/src/network/session.rs @@ -109,8 +109,8 @@ const PACKET_USER: u8 = 0x10; const PACKET_LAST: u8 = 0x7f; impl Session { - /// Create a new session out of comepleted handshake. Consumes handshake object. - pub fn new(h: Handshake, host: &HostInfo) -> Result { + /// Create a new session out of comepleted handshake. + pub fn new(h: &mut Handshake, host: &HostInfo) -> Result { let id = h.id.clone(); let connection = try!(EncryptedConnection::new(h)); let mut session = Session { @@ -169,6 +169,12 @@ impl Session { self.info.capabilities.iter().any(|c| c.protocol == protocol) } + /// Register the session socket with the event loop + pub fn register_socket>(&self, reg: Token, event_loop: &mut EventLoop) -> Result<(), UtilError> { + try!(self.connection.register_socket(reg, event_loop)); + Ok(()) + } + /// Update registration with the event loop. Should be called at the end of the IO handler. pub fn update_socket(&self, reg:Token, event_loop: &mut EventLoop) -> Result<(), UtilError> { self.connection.update_socket(reg, event_loop) From 0b48507d399423efbfa7732e95bc03122f9e4cc3 Mon Sep 17 00:00:00 2001 From: Nikolay Volf Date: Sat, 20 Feb 2016 12:38:16 +0300 Subject: [PATCH 50/61] Delete LICENSE.txt --- devtools/LICENSE.txt | 675 ------------------------------------------- 1 file changed, 675 deletions(-) delete mode 100644 devtools/LICENSE.txt diff --git a/devtools/LICENSE.txt b/devtools/LICENSE.txt deleted file mode 100644 index 733c07236..000000000 --- a/devtools/LICENSE.txt +++ /dev/null @@ -1,675 +0,0 @@ - GNU GENERAL PUBLIC LICENSE - Version 3, 29 June 2007 - - Copyright (C) 2007 Free Software Foundation, Inc. - Everyone is permitted to copy and distribute verbatim copies - of this license document, but changing it is not allowed. - - Preamble - - The GNU General Public License is a free, copyleft license for -software and other kinds of works. - - The licenses for most software and other practical works are designed -to take away your freedom to share and change the works. By contrast, -the GNU General Public License is intended to guarantee your freedom to -share and change all versions of a program--to make sure it remains free -software for all its users. We, the Free Software Foundation, use the -GNU General Public License for most of our software; it applies also to -any other work released this way by its authors. You can apply it to -your programs, too. - - When we speak of free software, we are referring to freedom, not -price. Our General Public Licenses are designed to make sure that you -have the freedom to distribute copies of free software (and charge for -them if you wish), that you receive source code or can get it if you -want it, that you can change the software or use pieces of it in new -free programs, and that you know you can do these things. - - To protect your rights, we need to prevent others from denying you -these rights or asking you to surrender the rights. Therefore, you have -certain responsibilities if you distribute copies of the software, or if -you modify it: responsibilities to respect the freedom of others. - - For example, if you distribute copies of such a program, whether -gratis or for a fee, you must pass on to the recipients the same -freedoms that you received. You must make sure that they, too, receive -or can get the source code. And you must show them these terms so they -know their rights. - - Developers that use the GNU GPL protect your rights with two steps: -(1) assert copyright on the software, and (2) offer you this License -giving you legal permission to copy, distribute and/or modify it. - - For the developers' and authors' protection, the GPL clearly explains -that there is no warranty for this free software. For both users' and -authors' sake, the GPL requires that modified versions be marked as -changed, so that their problems will not be attributed erroneously to -authors of previous versions. - - Some devices are designed to deny users access to install or run -modified versions of the software inside them, although the manufacturer -can do so. This is fundamentally incompatible with the aim of -protecting users' freedom to change the software. The systematic -pattern of such abuse occurs in the area of products for individuals to -use, which is precisely where it is most unacceptable. Therefore, we -have designed this version of the GPL to prohibit the practice for those -products. If such problems arise substantially in other domains, we -stand ready to extend this provision to those domains in future versions -of the GPL, as needed to protect the freedom of users. - - Finally, every program is threatened constantly by software patents. -States should not allow patents to restrict development and use of -software on general-purpose computers, but in those that do, we wish to -avoid the special danger that patents applied to a free program could -make it effectively proprietary. To prevent this, the GPL assures that -patents cannot be used to render the program non-free. - - The precise terms and conditions for copying, distribution and -modification follow. - - TERMS AND CONDITIONS - - 0. Definitions. - - "This License" refers to version 3 of the GNU General Public License. - - "Copyright" also means copyright-like laws that apply to other kinds of -works, such as semiconductor masks. - - "The Program" refers to any copyrightable work licensed under this -License. Each licensee is addressed as "you". "Licensees" and -"recipients" may be individuals or organizations. - - To "modify" a work means to copy from or adapt all or part of the work -in a fashion requiring copyright permission, other than the making of an -exact copy. The resulting work is called a "modified version" of the -earlier work or a work "based on" the earlier work. - - A "covered work" means either the unmodified Program or a work based -on the Program. - - To "propagate" a work means to do anything with it that, without -permission, would make you directly or secondarily liable for -infringement under applicable copyright law, except executing it on a -computer or modifying a private copy. Propagation includes copying, -distribution (with or without modification), making available to the -public, and in some countries other activities as well. - - To "convey" a work means any kind of propagation that enables other -parties to make or receive copies. Mere interaction with a user through -a computer network, with no transfer of a copy, is not conveying. - - An interactive user interface displays "Appropriate Legal Notices" -to the extent that it includes a convenient and prominently visible -feature that (1) displays an appropriate copyright notice, and (2) -tells the user that there is no warranty for the work (except to the -extent that warranties are provided), that licensees may convey the -work under this License, and how to view a copy of this License. If -the interface presents a list of user commands or options, such as a -menu, a prominent item in the list meets this criterion. - - 1. Source Code. - - The "source code" for a work means the preferred form of the work -for making modifications to it. "Object code" means any non-source -form of a work. - - A "Standard Interface" means an interface that either is an official -standard defined by a recognized standards body, or, in the case of -interfaces specified for a particular programming language, one that -is widely used among developers working in that language. - - The "System Libraries" of an executable work include anything, other -than the work as a whole, that (a) is included in the normal form of -packaging a Major Component, but which is not part of that Major -Component, and (b) serves only to enable use of the work with that -Major Component, or to implement a Standard Interface for which an -implementation is available to the public in source code form. A -"Major Component", in this context, means a major essential component -(kernel, window system, and so on) of the specific operating system -(if any) on which the executable work runs, or a compiler used to -produce the work, or an object code interpreter used to run it. - - The "Corresponding Source" for a work in object code form means all -the source code needed to generate, install, and (for an executable -work) run the object code and to modify the work, including scripts to -control those activities. However, it does not include the work's -System Libraries, or general-purpose tools or generally available free -programs which are used unmodified in performing those activities but -which are not part of the work. For example, Corresponding Source -includes interface definition files associated with source files for -the work, and the source code for shared libraries and dynamically -linked subprograms that the work is specifically designed to require, -such as by intimate data communication or control flow between those -subprograms and other parts of the work. - - The Corresponding Source need not include anything that users -can regenerate automatically from other parts of the Corresponding -Source. - - The Corresponding Source for a work in source code form is that -same work. - - 2. Basic Permissions. - - All rights granted under this License are granted for the term of -copyright on the Program, and are irrevocable provided the stated -conditions are met. This License explicitly affirms your unlimited -permission to run the unmodified Program. The output from running a -covered work is covered by this License only if the output, given its -content, constitutes a covered work. This License acknowledges your -rights of fair use or other equivalent, as provided by copyright law. - - You may make, run and propagate covered works that you do not -convey, without conditions so long as your license otherwise remains -in force. You may convey covered works to others for the sole purpose -of having them make modifications exclusively for you, or provide you -with facilities for running those works, provided that you comply with -the terms of this License in conveying all material for which you do -not control copyright. Those thus making or running the covered works -for you must do so exclusively on your behalf, under your direction -and control, on terms that prohibit them from making any copies of -your copyrighted material outside their relationship with you. - - Conveying under any other circumstances is permitted solely under -the conditions stated below. Sublicensing is not allowed; section 10 -makes it unnecessary. - - 3. Protecting Users' Legal Rights From Anti-Circumvention Law. - - No covered work shall be deemed part of an effective technological -measure under any applicable law fulfilling obligations under article -11 of the WIPO copyright treaty adopted on 20 December 1996, or -similar laws prohibiting or restricting circumvention of such -measures. - - When you convey a covered work, you waive any legal power to forbid -circumvention of technological measures to the extent such circumvention -is effected by exercising rights under this License with respect to -the covered work, and you disclaim any intention to limit operation or -modification of the work as a means of enforcing, against the work's -users, your or third parties' legal rights to forbid circumvention of -technological measures. - - 4. Conveying Verbatim Copies. - - You may convey verbatim copies of the Program's source code as you -receive it, in any medium, provided that you conspicuously and -appropriately publish on each copy an appropriate copyright notice; -keep intact all notices stating that this License and any -non-permissive terms added in accord with section 7 apply to the code; -keep intact all notices of the absence of any warranty; and give all -recipients a copy of this License along with the Program. - - You may charge any price or no price for each copy that you convey, -and you may offer support or warranty protection for a fee. - - 5. Conveying Modified Source Versions. - - You may convey a work based on the Program, or the modifications to -produce it from the Program, in the form of source code under the -terms of section 4, provided that you also meet all of these conditions: - - a) The work must carry prominent notices stating that you modified - it, and giving a relevant date. - - b) The work must carry prominent notices stating that it is - released under this License and any conditions added under section - 7. This requirement modifies the requirement in section 4 to - "keep intact all notices". - - c) You must license the entire work, as a whole, under this - License to anyone who comes into possession of a copy. This - License will therefore apply, along with any applicable section 7 - additional terms, to the whole of the work, and all its parts, - regardless of how they are packaged. This License gives no - permission to license the work in any other way, but it does not - invalidate such permission if you have separately received it. - - d) If the work has interactive user interfaces, each must display - Appropriate Legal Notices; however, if the Program has interactive - interfaces that do not display Appropriate Legal Notices, your - work need not make them do so. - - A compilation of a covered work with other separate and independent -works, which are not by their nature extensions of the covered work, -and which are not combined with it such as to form a larger program, -in or on a volume of a storage or distribution medium, is called an -"aggregate" if the compilation and its resulting copyright are not -used to limit the access or legal rights of the compilation's users -beyond what the individual works permit. Inclusion of a covered work -in an aggregate does not cause this License to apply to the other -parts of the aggregate. - - 6. Conveying Non-Source Forms. - - You may convey a covered work in object code form under the terms -of sections 4 and 5, provided that you also convey the -machine-readable Corresponding Source under the terms of this License, -in one of these ways: - - a) Convey the object code in, or embodied in, a physical product - (including a physical distribution medium), accompanied by the - Corresponding Source fixed on a durable physical medium - customarily used for software interchange. - - b) Convey the object code in, or embodied in, a physical product - (including a physical distribution medium), accompanied by a - written offer, valid for at least three years and valid for as - long as you offer spare parts or customer support for that product - model, to give anyone who possesses the object code either (1) a - copy of the Corresponding Source for all the software in the - product that is covered by this License, on a durable physical - medium customarily used for software interchange, for a price no - more than your reasonable cost of physically performing this - conveying of source, or (2) access to copy the - Corresponding Source from a network server at no charge. - - c) Convey individual copies of the object code with a copy of the - written offer to provide the Corresponding Source. This - alternative is allowed only occasionally and noncommercially, and - only if you received the object code with such an offer, in accord - with subsection 6b. - - d) Convey the object code by offering access from a designated - place (gratis or for a charge), and offer equivalent access to the - Corresponding Source in the same way through the same place at no - further charge. You need not require recipients to copy the - Corresponding Source along with the object code. If the place to - copy the object code is a network server, the Corresponding Source - may be on a different server (operated by you or a third party) - that supports equivalent copying facilities, provided you maintain - clear directions next to the object code saying where to find the - Corresponding Source. Regardless of what server hosts the - Corresponding Source, you remain obligated to ensure that it is - available for as long as needed to satisfy these requirements. - - e) Convey the object code using peer-to-peer transmission, provided - you inform other peers where the object code and Corresponding - Source of the work are being offered to the general public at no - charge under subsection 6d. - - A separable portion of the object code, whose source code is excluded -from the Corresponding Source as a System Library, need not be -included in conveying the object code work. - - A "User Product" is either (1) a "consumer product", which means any -tangible personal property which is normally used for personal, family, -or household purposes, or (2) anything designed or sold for incorporation -into a dwelling. In determining whether a product is a consumer product, -doubtful cases shall be resolved in favor of coverage. For a particular -product received by a particular user, "normally used" refers to a -typical or common use of that class of product, regardless of the status -of the particular user or of the way in which the particular user -actually uses, or expects or is expected to use, the product. A product -is a consumer product regardless of whether the product has substantial -commercial, industrial or non-consumer uses, unless such uses represent -the only significant mode of use of the product. - - "Installation Information" for a User Product means any methods, -procedures, authorization keys, or other information required to install -and execute modified versions of a covered work in that User Product from -a modified version of its Corresponding Source. The information must -suffice to ensure that the continued functioning of the modified object -code is in no case prevented or interfered with solely because -modification has been made. - - If you convey an object code work under this section in, or with, or -specifically for use in, a User Product, and the conveying occurs as -part of a transaction in which the right of possession and use of the -User Product is transferred to the recipient in perpetuity or for a -fixed term (regardless of how the transaction is characterized), the -Corresponding Source conveyed under this section must be accompanied -by the Installation Information. But this requirement does not apply -if neither you nor any third party retains the ability to install -modified object code on the User Product (for example, the work has -been installed in ROM). - - The requirement to provide Installation Information does not include a -requirement to continue to provide support service, warranty, or updates -for a work that has been modified or installed by the recipient, or for -the User Product in which it has been modified or installed. Access to a -network may be denied when the modification itself materially and -adversely affects the operation of the network or violates the rules and -protocols for communication across the network. - - Corresponding Source conveyed, and Installation Information provided, -in accord with this section must be in a format that is publicly -documented (and with an implementation available to the public in -source code form), and must require no special password or key for -unpacking, reading or copying. - - 7. Additional Terms. - - "Additional permissions" are terms that supplement the terms of this -License by making exceptions from one or more of its conditions. -Additional permissions that are applicable to the entire Program shall -be treated as though they were included in this License, to the extent -that they are valid under applicable law. If additional permissions -apply only to part of the Program, that part may be used separately -under those permissions, but the entire Program remains governed by -this License without regard to the additional permissions. - - When you convey a copy of a covered work, you may at your option -remove any additional permissions from that copy, or from any part of -it. (Additional permissions may be written to require their own -removal in certain cases when you modify the work.) You may place -additional permissions on material, added by you to a covered work, -for which you have or can give appropriate copyright permission. - - Notwithstanding any other provision of this License, for material you -add to a covered work, you may (if authorized by the copyright holders of -that material) supplement the terms of this License with terms: - - a) Disclaiming warranty or limiting liability differently from the - terms of sections 15 and 16 of this License; or - - b) Requiring preservation of specified reasonable legal notices or - author attributions in that material or in the Appropriate Legal - Notices displayed by works containing it; or - - c) Prohibiting misrepresentation of the origin of that material, or - requiring that modified versions of such material be marked in - reasonable ways as different from the original version; or - - d) Limiting the use for publicity purposes of names of licensors or - authors of the material; or - - e) Declining to grant rights under trademark law for use of some - trade names, trademarks, or service marks; or - - f) Requiring indemnification of licensors and authors of that - material by anyone who conveys the material (or modified versions of - it) with contractual assumptions of liability to the recipient, for - any liability that these contractual assumptions directly impose on - those licensors and authors. - - All other non-permissive additional terms are considered "further -restrictions" within the meaning of section 10. If the Program as you -received it, or any part of it, contains a notice stating that it is -governed by this License along with a term that is a further -restriction, you may remove that term. If a license document contains -a further restriction but permits relicensing or conveying under this -License, you may add to a covered work material governed by the terms -of that license document, provided that the further restriction does -not survive such relicensing or conveying. - - If you add terms to a covered work in accord with this section, you -must place, in the relevant source files, a statement of the -additional terms that apply to those files, or a notice indicating -where to find the applicable terms. - - Additional terms, permissive or non-permissive, may be stated in the -form of a separately written license, or stated as exceptions; -the above requirements apply either way. - - 8. Termination. - - You may not propagate or modify a covered work except as expressly -provided under this License. Any attempt otherwise to propagate or -modify it is void, and will automatically terminate your rights under -this License (including any patent licenses granted under the third -paragraph of section 11). - - However, if you cease all violation of this License, then your -license from a particular copyright holder is reinstated (a) -provisionally, unless and until the copyright holder explicitly and -finally terminates your license, and (b) permanently, if the copyright -holder fails to notify you of the violation by some reasonable means -prior to 60 days after the cessation. - - Moreover, your license from a particular copyright holder is -reinstated permanently if the copyright holder notifies you of the -violation by some reasonable means, this is the first time you have -received notice of violation of this License (for any work) from that -copyright holder, and you cure the violation prior to 30 days after -your receipt of the notice. - - Termination of your rights under this section does not terminate the -licenses of parties who have received copies or rights from you under -this License. If your rights have been terminated and not permanently -reinstated, you do not qualify to receive new licenses for the same -material under section 10. - - 9. Acceptance Not Required for Having Copies. - - You are not required to accept this License in order to receive or -run a copy of the Program. Ancillary propagation of a covered work -occurring solely as a consequence of using peer-to-peer transmission -to receive a copy likewise does not require acceptance. However, -nothing other than this License grants you permission to propagate or -modify any covered work. These actions infringe copyright if you do -not accept this License. Therefore, by modifying or propagating a -covered work, you indicate your acceptance of this License to do so. - - 10. Automatic Licensing of Downstream Recipients. - - Each time you convey a covered work, the recipient automatically -receives a license from the original licensors, to run, modify and -propagate that work, subject to this License. You are not responsible -for enforcing compliance by third parties with this License. - - An "entity transaction" is a transaction transferring control of an -organization, or substantially all assets of one, or subdividing an -organization, or merging organizations. If propagation of a covered -work results from an entity transaction, each party to that -transaction who receives a copy of the work also receives whatever -licenses to the work the party's predecessor in interest had or could -give under the previous paragraph, plus a right to possession of the -Corresponding Source of the work from the predecessor in interest, if -the predecessor has it or can get it with reasonable efforts. - - You may not impose any further restrictions on the exercise of the -rights granted or affirmed under this License. For example, you may -not impose a license fee, royalty, or other charge for exercise of -rights granted under this License, and you may not initiate litigation -(including a cross-claim or counterclaim in a lawsuit) alleging that -any patent claim is infringed by making, using, selling, offering for -sale, or importing the Program or any portion of it. - - 11. Patents. - - A "contributor" is a copyright holder who authorizes use under this -License of the Program or a work on which the Program is based. The -work thus licensed is called the contributor's "contributor version". - - A contributor's "essential patent claims" are all patent claims -owned or controlled by the contributor, whether already acquired or -hereafter acquired, that would be infringed by some manner, permitted -by this License, of making, using, or selling its contributor version, -but do not include claims that would be infringed only as a -consequence of further modification of the contributor version. For -purposes of this definition, "control" includes the right to grant -patent sublicenses in a manner consistent with the requirements of -this License. - - Each contributor grants you a non-exclusive, worldwide, royalty-free -patent license under the contributor's essential patent claims, to -make, use, sell, offer for sale, import and otherwise run, modify and -propagate the contents of its contributor version. - - In the following three paragraphs, a "patent license" is any express -agreement or commitment, however denominated, not to enforce a patent -(such as an express permission to practice a patent or covenant not to -sue for patent infringement). To "grant" such a patent license to a -party means to make such an agreement or commitment not to enforce a -patent against the party. - - If you convey a covered work, knowingly relying on a patent license, -and the Corresponding Source of the work is not available for anyone -to copy, free of charge and under the terms of this License, through a -publicly available network server or other readily accessible means, -then you must either (1) cause the Corresponding Source to be so -available, or (2) arrange to deprive yourself of the benefit of the -patent license for this particular work, or (3) arrange, in a manner -consistent with the requirements of this License, to extend the patent -license to downstream recipients. "Knowingly relying" means you have -actual knowledge that, but for the patent license, your conveying the -covered work in a country, or your recipient's use of the covered work -in a country, would infringe one or more identifiable patents in that -country that you have reason to believe are valid. - - If, pursuant to or in connection with a single transaction or -arrangement, you convey, or propagate by procuring conveyance of, a -covered work, and grant a patent license to some of the parties -receiving the covered work authorizing them to use, propagate, modify -or convey a specific copy of the covered work, then the patent license -you grant is automatically extended to all recipients of the covered -work and works based on it. - - A patent license is "discriminatory" if it does not include within -the scope of its coverage, prohibits the exercise of, or is -conditioned on the non-exercise of one or more of the rights that are -specifically granted under this License. You may not convey a covered -work if you are a party to an arrangement with a third party that is -in the business of distributing software, under which you make payment -to the third party based on the extent of your activity of conveying -the work, and under which the third party grants, to any of the -parties who would receive the covered work from you, a discriminatory -patent license (a) in connection with copies of the covered work -conveyed by you (or copies made from those copies), or (b) primarily -for and in connection with specific products or compilations that -contain the covered work, unless you entered into that arrangement, -or that patent license was granted, prior to 28 March 2007. - - Nothing in this License shall be construed as excluding or limiting -any implied license or other defenses to infringement that may -otherwise be available to you under applicable patent law. - - 12. No Surrender of Others' Freedom. - - If conditions are imposed on you (whether by court order, agreement or -otherwise) that contradict the conditions of this License, they do not -excuse you from the conditions of this License. If you cannot convey a -covered work so as to satisfy simultaneously your obligations under this -License and any other pertinent obligations, then as a consequence you may -not convey it at all. For example, if you agree to terms that obligate you -to collect a royalty for further conveying from those to whom you convey -the Program, the only way you could satisfy both those terms and this -License would be to refrain entirely from conveying the Program. - - 13. Use with the GNU Affero General Public License. - - Notwithstanding any other provision of this License, you have -permission to link or combine any covered work with a work licensed -under version 3 of the GNU Affero General Public License into a single -combined work, and to convey the resulting work. The terms of this -License will continue to apply to the part which is the covered work, -but the special requirements of the GNU Affero General Public License, -section 13, concerning interaction through a network will apply to the -combination as such. - - 14. Revised Versions of this License. - - The Free Software Foundation may publish revised and/or new versions of -the GNU General Public License from time to time. Such new versions will -be similar in spirit to the present version, but may differ in detail to -address new problems or concerns. - - Each version is given a distinguishing version number. If the -Program specifies that a certain numbered version of the GNU General -Public License "or any later version" applies to it, you have the -option of following the terms and conditions either of that numbered -version or of any later version published by the Free Software -Foundation. If the Program does not specify a version number of the -GNU General Public License, you may choose any version ever published -by the Free Software Foundation. - - If the Program specifies that a proxy can decide which future -versions of the GNU General Public License can be used, that proxy's -public statement of acceptance of a version permanently authorizes you -to choose that version for the Program. - - Later license versions may give you additional or different -permissions. However, no additional obligations are imposed on any -author or copyright holder as a result of your choosing to follow a -later version. - - 15. Disclaimer of Warranty. - - THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY -APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT -HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY -OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, -THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM -IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF -ALL NECESSARY SERVICING, REPAIR OR CORRECTION. - - 16. Limitation of Liability. - - IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING -WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS -THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY -GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE -USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF -DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD -PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), -EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF -SUCH DAMAGES. - - 17. Interpretation of Sections 15 and 16. - - If the disclaimer of warranty and limitation of liability provided -above cannot be given local legal effect according to their terms, -reviewing courts shall apply local law that most closely approximates -an absolute waiver of all civil liability in connection with the -Program, unless a warranty or assumption of liability accompanies a -copy of the Program in return for a fee. - - END OF TERMS AND CONDITIONS - - How to Apply These Terms to Your New Programs - - If you develop a new program, and you want it to be of the greatest -possible use to the public, the best way to achieve this is to make it -free software which everyone can redistribute and change under these terms. - - To do so, attach the following notices to the program. It is safest -to attach them to the start of each source file to most effectively -state the exclusion of warranty; and each file should have at least -the "copyright" line and a pointer to where the full notice is found. - - {one line to give the program's name and a brief idea of what it does.} - Copyright (C) {year} {name of author} - - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program. If not, see . - -Also add information on how to contact you by electronic and paper mail. - - If the program does terminal interaction, make it output a short -notice like this when it starts in an interactive mode: - - {project} Copyright (C) {year} {fullname} - This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'. - This is free software, and you are welcome to redistribute it - under certain conditions; type `show c' for details. - -The hypothetical commands `show w' and `show c' should show the appropriate -parts of the General Public License. Of course, your program's commands -might be different; for a GUI interface, you would use an "about box". - - You should also get your employer (if you work as a programmer) or school, -if any, to sign a "copyright disclaimer" for the program, if necessary. -For more information on this, and how to apply and follow the GNU GPL, see -. - - The GNU General Public License does not permit incorporating your program -into proprietary programs. If your program is a subroutine library, you -may consider it more useful to permit linking proprietary applications with -the library. If this is what you want to do, use the GNU Lesser General -Public License instead of this License. But first, please read -. - From b1bfd00875be0230763ce3a98c89e789e01dc2c0 Mon Sep 17 00:00:00 2001 From: arkpar Date: Sat, 20 Feb 2016 01:10:27 +0100 Subject: [PATCH 51/61] Zombie connections --- ethcore/res/ethereum/tests | 2 +- util/src/network/connection.rs | 8 ++-- util/src/network/handshake.rs | 73 ++++++++++++++++++++++------------ util/src/network/host.rs | 71 +++++++++++++++++++-------------- util/src/network/session.rs | 25 ++++++++++++ util/src/network/tests.rs | 2 +- 6 files changed, 120 insertions(+), 61 deletions(-) diff --git a/ethcore/res/ethereum/tests b/ethcore/res/ethereum/tests index 3116f85a4..f32954b3d 160000 --- a/ethcore/res/ethereum/tests +++ b/ethcore/res/ethereum/tests @@ -1 +1 @@ -Subproject commit 3116f85a499ceaf4dfdc46726060fc056e2d7829 +Subproject commit f32954b3ddb5af2dc3dc9ec6d9a28bee848fdf70 diff --git a/util/src/network/connection.rs b/util/src/network/connection.rs index 4b7886698..0135fc333 100644 --- a/util/src/network/connection.rs +++ b/util/src/network/connection.rs @@ -181,7 +181,7 @@ impl Connection { socket: try!(self.socket.try_clone()), rec_buf: Vec::new(), rec_size: 0, - send_queue: VecDeque::new(), + send_queue: self.send_queue.clone(), interest: EventSet::hup() | EventSet::readable(), stats: self.stats.clone(), }) @@ -190,10 +190,10 @@ impl Connection { /// Register this connection with the IO event loop. pub fn register_socket(&self, reg: Token, event_loop: &mut EventLoop) -> io::Result<()> { trace!(target: "net", "connection register; token={:?}", reg); - event_loop.register(&self.socket, reg, self.interest, PollOpt::edge() | PollOpt::oneshot()).or_else(|e| { + if let Err(e) = event_loop.register(&self.socket, reg, self.interest, PollOpt::edge() | PollOpt::oneshot()) { debug!("Failed to register {:?}, {:?}", reg, e); - Ok(()) - }) + } + Ok(()) } /// Update connection registration. Should be called at the end of the IO handler. diff --git a/util/src/network/handshake.rs b/util/src/network/handshake.rs index 5d43decd7..a50dd6ba2 100644 --- a/util/src/network/handshake.rs +++ b/util/src/network/handshake.rs @@ -63,7 +63,9 @@ pub struct Handshake { /// A copy of received encryped auth packet pub auth_cipher: Bytes, /// A copy of received encryped ack packet - pub ack_cipher: Bytes + pub ack_cipher: Bytes, + /// This Handshake is marked for deleteion flag + pub expired: bool, } const AUTH_PACKET_SIZE: usize = 307; @@ -84,6 +86,7 @@ impl Handshake { remote_nonce: H256::new(), auth_cipher: Bytes::new(), ack_cipher: Bytes::new(), + expired: false, }) } @@ -97,6 +100,16 @@ impl Handshake { self.connection.token() } + /// Mark this handshake as inactive to be deleted lated. + pub fn set_expired(&mut self) { + self.expired = true; + } + + /// Check if this handshake is expired. + pub fn expired(&self) -> bool { + self.expired + } + /// Start a handhsake pub fn start(&mut self, io: &IoContext, host: &HostInfo, originated: bool) -> Result<(), UtilError> where Message: Send + Clone{ self.originated = originated; @@ -118,47 +131,55 @@ impl Handshake { /// Readable IO handler. Drives the state change. pub fn readable(&mut self, io: &IoContext, host: &HostInfo) -> Result<(), UtilError> where Message: Send + Clone { - io.clear_timer(self.connection.token).unwrap(); - match self.state { - HandshakeState::ReadingAuth => { - if let Some(data) = try!(self.connection.readable()) { - try!(self.read_auth(host, &data)); - try!(self.write_ack()); - }; - }, - HandshakeState::ReadingAck => { - if let Some(data) = try!(self.connection.readable()) { - try!(self.read_ack(host, &data)); - self.state = HandshakeState::StartSession; - }; - }, - HandshakeState::StartSession => {}, - _ => { panic!("Unexpected state"); } - } - if self.state != HandshakeState::StartSession { - try!(io.update_registration(self.connection.token)); + if !self.expired() { + io.clear_timer(self.connection.token).unwrap(); + match self.state { + HandshakeState::ReadingAuth => { + if let Some(data) = try!(self.connection.readable()) { + try!(self.read_auth(host, &data)); + try!(self.write_ack()); + }; + }, + HandshakeState::ReadingAck => { + if let Some(data) = try!(self.connection.readable()) { + try!(self.read_ack(host, &data)); + self.state = HandshakeState::StartSession; + }; + }, + HandshakeState::StartSession => {}, + _ => { panic!("Unexpected state"); } + } + if self.state != HandshakeState::StartSession { + try!(io.update_registration(self.connection.token)); + } } Ok(()) } /// Writabe IO handler. pub fn writable(&mut self, io: &IoContext, _host: &HostInfo) -> Result<(), UtilError> where Message: Send + Clone { - io.clear_timer(self.connection.token).unwrap(); - try!(self.connection.writable()); - if self.state != HandshakeState::StartSession { - io.update_registration(self.connection.token).unwrap(); + if !self.expired() { + io.clear_timer(self.connection.token).unwrap(); + try!(self.connection.writable()); + if self.state != HandshakeState::StartSession { + io.update_registration(self.connection.token).unwrap(); + } } Ok(()) } /// Register the socket with the event loop pub fn register_socket>(&self, reg: Token, event_loop: &mut EventLoop) -> Result<(), UtilError> { - try!(self.connection.register_socket(reg, event_loop)); + if !self.expired() { + try!(self.connection.register_socket(reg, event_loop)); + } Ok(()) } pub fn update_socket>(&self, reg: Token, event_loop: &mut EventLoop) -> Result<(), UtilError> { - try!(self.connection.update_socket(reg, event_loop)); + if !self.expired() { + try!(self.connection.update_socket(reg, event_loop)); + } Ok(()) } diff --git a/util/src/network/host.rs b/util/src/network/host.rs index f4f3b4e1b..95b778531 100644 --- a/util/src/network/host.rs +++ b/util/src/network/host.rs @@ -190,11 +190,11 @@ impl<'s, Message> NetworkContext<'s, Message> where Message: Send + Sync + Clone let session = { self.sessions.read().unwrap().get(peer).cloned() }; if let Some(session) = session { session.lock().unwrap().deref_mut().send_packet(self.protocol, packet_id as u8, &data).unwrap_or_else(|e| { - warn!(target: "net", "Send error: {:?}", e); + warn!(target: "network", "Send error: {:?}", e); }); //TODO: don't copy vector data try!(self.io.update_registration(peer)); } else { - trace!(target: "net", "Send: Peer no longer exist") + trace!(target: "network", "Send: Peer no longer exist") } Ok(()) } @@ -470,18 +470,18 @@ impl Host where Message: Send + Sync + Clone { .take(min(MAX_HANDSHAKES_PER_ROUND, handshake_limit - handshake_count)) { self.connect_peer(&id, io); } - debug!(target: "net", "Connecting peers: {} sessions, {} pending", self.session_count(), self.handshake_count()); + debug!(target: "network", "Connecting peers: {} sessions, {} pending", self.session_count(), self.handshake_count()); } #[cfg_attr(feature="dev", allow(single_match))] fn connect_peer(&self, id: &NodeId, io: &IoContext>) { if self.have_session(id) { - trace!("Aborted connect. Node already connected."); + trace!(target: "network", "Aborted connect. Node already connected."); return; } if self.connecting_to(id) { - trace!("Aborted connect. Node already connecting."); + trace!(target: "network", "Aborted connect. Node already connecting."); return; } @@ -493,7 +493,7 @@ impl Host where Message: Send + Sync + Clone { node.endpoint.address } else { - debug!("Connection to expired node aborted"); + debug!(target: "network", "Connection to expired node aborted"); return; } }; @@ -515,16 +515,16 @@ impl Host where Message: Send + Sync + Clone { if handshakes.insert_with(|token| { let mut handshake = Handshake::new(token, id, socket, &nonce, self.stats.clone()).expect("Can't create handshake"); handshake.start(io, &self.info.read().unwrap(), id.is_some()).and_then(|_| io.register_stream(token)).unwrap_or_else (|e| { - debug!(target: "net", "Handshake create error: {:?}", e); + debug!(target: "network", "Handshake create error: {:?}", e); }); Arc::new(Mutex::new(handshake)) }).is_none() { - debug!("Max handshakes reached"); + debug!(target: "network", "Max handshakes reached"); } } fn accept(&self, io: &IoContext>) { - trace!(target: "net", "accept"); + trace!(target: "network", "Accepting incoming connection"); loop { let socket = match self.tcp_listener.lock().unwrap().accept() { Ok(None) => break, @@ -544,7 +544,7 @@ impl Host where Message: Send + Sync + Clone { if let Some(handshake) = handshake { let mut h = handshake.lock().unwrap(); if let Err(e) = h.writable(io, &self.info.read().unwrap()) { - trace!(target: "net", "Handshake write error: {}: {:?}", token, e); + trace!(target: "network", "Handshake write error: {}: {:?}", token, e); } } } @@ -554,9 +554,9 @@ impl Host where Message: Send + Sync + Clone { if let Some(session) = session { let mut s = session.lock().unwrap(); if let Err(e) = s.writable(io, &self.info.read().unwrap()) { - trace!(target: "net", "Session write error: {}: {:?}", token, e); + trace!(target: "network", "Session write error: {}: {:?}", token, e); } - io.update_registration(token).unwrap_or_else(|e| debug!(target: "net", "Session registration error: {:?}", e)); + io.update_registration(token).unwrap_or_else(|e| debug!(target: "network", "Session registration error: {:?}", e)); } } @@ -571,7 +571,7 @@ impl Host where Message: Send + Sync + Clone { if let Some(handshake) = handshake { let mut h = handshake.lock().unwrap(); if let Err(e) = h.readable(io, &self.info.read().unwrap()) { - debug!(target: "net", "Handshake read error: {}: {:?}", token, e); + debug!(target: "network", "Handshake read error: {}: {:?}", token, e); kill = true; } if h.done() { @@ -585,7 +585,7 @@ impl Host where Message: Send + Sync + Clone { self.start_session(token, io); return; } - io.update_registration(token).unwrap_or_else(|e| debug!(target: "net", "Token registration error: {:?}", e)); + io.update_registration(token).unwrap_or_else(|e| debug!(target: "network", "Token registration error: {:?}", e)); } fn session_readable(&self, token: StreamToken, io: &IoContext>) { @@ -597,7 +597,7 @@ impl Host where Message: Send + Sync + Clone { let mut s = session.lock().unwrap(); match s.readable(io, &self.info.read().unwrap()) { Err(e) => { - debug!(target: "net", "Session read error: {}: {:?}", token, e); + debug!(target: "network", "Session read error: {}: {:?}", token, e); kill = true; }, Ok(SessionData::Ready) => { @@ -613,7 +613,7 @@ impl Host where Message: Send + Sync + Clone { packet_id, }) => { match self.handlers.read().unwrap().get(protocol) { - None => { warn!(target: "net", "No handler found for protocol: {:?}", protocol) }, + None => { warn!(target: "network", "No handler found for protocol: {:?}", protocol) }, Some(_) => packet_data = Some((protocol, packet_id, data)), } }, @@ -631,7 +631,7 @@ impl Host where Message: Send + Sync + Clone { let h = self.handlers.read().unwrap().get(p).unwrap().clone(); h.read(&NetworkContext::new(io, p, Some(token), self.sessions.clone()), &token, packet_id, &data[1..]); } - io.update_registration(token).unwrap_or_else(|e| debug!(target: "net", "Token registration error: {:?}", e)); + io.update_registration(token).unwrap_or_else(|e| debug!(target: "network", "Token registration error: {:?}", e)); } fn start_session(&self, token: StreamToken, io: &IoContext>) { @@ -643,19 +643,24 @@ impl Host where Message: Send + Sync + Clone { // turn a handshake into a session let mut sessions = self.sessions.write().unwrap(); let mut h = handshakes.get_mut(token).unwrap().lock().unwrap(); + if h.expired { + return; + } let originated = h.originated; let mut session = match Session::new(&mut h, &self.info.read().unwrap()) { Ok(s) => s, Err(e) => { - debug!("Session creation error: {:?}", e); + debug!(target: "network", "Session creation error: {:?}", e); return; } }; let result = sessions.insert_with(move |session_token| { session.set_token(session_token); io.deregister_stream(token).expect("Error deleting handshake registration"); + h.set_expired(); io.register_stream(session_token).expect("Error creating session registration"); self.stats.inc_sessions(); + trace!(target: "network", "Creating session {} -> {}", token, session_token); if !originated { // Add it no node table if let Ok(address) = session.remote_addr() { @@ -684,26 +689,34 @@ impl Host where Message: Send + Sync + Clone { FIRST_HANDSHAKE ... LAST_HANDSHAKE => { let handshakes = self.handshakes.write().unwrap(); if let Some(handshake) = handshakes.get(token).cloned() { - failure_id = Some(handshake.lock().unwrap().id().clone()); + let mut handshake = handshake.lock().unwrap(); + if !handshake.expired() { + handshake.set_expired(); + failure_id = Some(handshake.id().clone()); + io.deregister_stream(token).expect("Error deregistering stream"); + } } }, FIRST_SESSION ... LAST_SESSION => { let sessions = self.sessions.write().unwrap(); if let Some(session) = sessions.get(token).cloned() { - let s = session.lock().unwrap(); - if s.is_ready() { - for (p, _) in self.handlers.read().unwrap().iter() { - if s.have_capability(p) { - to_disconnect.push(p); + let mut s = session.lock().unwrap(); + if !s.expired() { + if s.is_ready() { + for (p, _) in self.handlers.read().unwrap().iter() { + if s.have_capability(p) { + to_disconnect.push(p); + } } } + s.set_expired(); + failure_id = Some(s.id().clone()); + io.deregister_stream(token).expect("Error deregistering stream"); } - failure_id = Some(s.id().clone()); } }, _ => {}, } - io.deregister_stream(token).expect("Error deregistering stream"); if let Some(id) = failure_id { if remote { self.nodes.write().unwrap().note_failure(&id); @@ -758,11 +771,11 @@ impl IoHandler> for Host where Messa } fn stream_hup(&self, io: &IoContext>, stream: StreamToken) { - trace!(target: "net", "Hup: {}", stream); + trace!(target: "network", "Hup: {}", stream); match stream { FIRST_SESSION ... LAST_SESSION => self.connection_closed(stream, io), FIRST_HANDSHAKE ... LAST_HANDSHAKE => self.connection_closed(stream, io), - _ => warn!(target: "net", "Unexpected hup"), + _ => warn!(target: "network", "Unexpected hup"), }; } @@ -810,7 +823,7 @@ impl IoHandler> for Host where Messa }, _ => match self.timers.read().unwrap().get(&token).cloned() { Some(timer) => match self.handlers.read().unwrap().get(timer.protocol).cloned() { - None => { warn!(target: "net", "No handler found for protocol: {:?}", timer.protocol) }, + None => { warn!(target: "network", "No handler found for protocol: {:?}", timer.protocol) }, Some(h) => { h.timeout(&NetworkContext::new(io, timer.protocol, None, self.sessions.clone()), timer.token); } }, None => { warn!("Unknown timer token: {}", token); } // timer is not registerd through us diff --git a/util/src/network/session.rs b/util/src/network/session.rs index 159dc795d..6be1dbfe4 100644 --- a/util/src/network/session.rs +++ b/util/src/network/session.rs @@ -41,6 +41,8 @@ pub struct Session { connection: EncryptedConnection, /// Session ready flag. Set after successfull Hello packet exchange had_hello: bool, + /// Session is no longer active flag. + expired: bool, ping_time_ns: u64, pong_time_ns: Option, } @@ -125,6 +127,7 @@ impl Session { }, ping_time_ns: 0, pong_time_ns: None, + expired: false, }; try!(session.write_hello(host)); try!(session.send_ping()); @@ -141,6 +144,16 @@ impl Session { self.had_hello } + /// Mark this session as inactive to be deleted lated. + pub fn set_expired(&mut self) { + self.expired = true; + } + + /// Check if this session is expired. + pub fn expired(&self) -> bool { + self.expired + } + /// Replace socket token pub fn set_token(&mut self, token: StreamToken) { self.connection.set_token(token); @@ -153,6 +166,9 @@ impl Session { /// Readable IO handler. Returns packet data if available. pub fn readable(&mut self, io: &IoContext, host: &HostInfo) -> Result where Message: Send + Sync + Clone { + if self.expired() { + return Ok(SessionData::None) + } match try!(self.connection.readable(io)) { Some(data) => Ok(try!(self.read_packet(data, host))), None => Ok(SessionData::None) @@ -161,6 +177,9 @@ impl Session { /// Writable IO handler. Sends pending packets. pub fn writable(&mut self, io: &IoContext, _host: &HostInfo) -> Result<(), UtilError> where Message: Send + Sync + Clone { + if self.expired() { + return Ok(()) + } self.connection.writable(io) } @@ -171,12 +190,18 @@ impl Session { /// Register the session socket with the event loop pub fn register_socket>(&self, reg: Token, event_loop: &mut EventLoop) -> Result<(), UtilError> { + if self.expired() { + return Ok(()); + } try!(self.connection.register_socket(reg, event_loop)); Ok(()) } /// Update registration with the event loop. Should be called at the end of the IO handler. pub fn update_socket(&self, reg:Token, event_loop: &mut EventLoop) -> Result<(), UtilError> { + if self.expired() { + return Ok(()); + } self.connection.update_socket(reg, event_loop) } diff --git a/util/src/network/tests.rs b/util/src/network/tests.rs index 44d53bdbe..f8ef588f6 100644 --- a/util/src/network/tests.rs +++ b/util/src/network/tests.rs @@ -74,7 +74,7 @@ impl NetworkProtocolHandler for TestProtocol { } fn connected(&self, io: &NetworkContext, peer: &PeerId) { - assert!(io.peer_info(*peer).contains("parity")); + assert!(io.peer_info(*peer).contains("Parity")); if self.drop_session { io.disconnect_peer(*peer) } else { From 8bd052b9869de1853c1251a28887d2213211a5b2 Mon Sep 17 00:00:00 2001 From: arkpar Date: Sat, 20 Feb 2016 11:54:12 +0100 Subject: [PATCH 52/61] Fixed warnings --- ethcore/res/ethereum/tests | 2 +- ethcore/src/blockchain.rs | 2 +- ethcore/src/evm/tests.rs | 2 +- util/src/hash.rs | 2 +- util/src/panics.rs | 10 +--------- util/src/uint.rs | 2 +- 6 files changed, 6 insertions(+), 14 deletions(-) diff --git a/ethcore/res/ethereum/tests b/ethcore/res/ethereum/tests index 3116f85a4..f32954b3d 160000 --- a/ethcore/res/ethereum/tests +++ b/ethcore/res/ethereum/tests @@ -1 +1 @@ -Subproject commit 3116f85a499ceaf4dfdc46726060fc056e2d7829 +Subproject commit f32954b3ddb5af2dc3dc9ec6d9a28bee848fdf70 diff --git a/ethcore/src/blockchain.rs b/ethcore/src/blockchain.rs index 9240ff800..2e9a867c4 100644 --- a/ethcore/src/blockchain.rs +++ b/ethcore/src/blockchain.rs @@ -696,7 +696,7 @@ mod tests { } #[test] - #[allow(cyclomatic_complexity)] + #[cfg_attr(feature="dev", allow(cyclomatic_complexity))] fn test_small_fork() { let genesis = "f901fcf901f7a00000000000000000000000000000000000000000000000000000000000000000a01dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347948888f1f195afa192cfee860698584c030f4c9db1a07dba07d6b448a186e9612e5f737d1c909dce473e53199901a302c00646d523c1a056e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421a056e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421b90100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000008302000080832fefd8808454c98c8142a059262c330941f3fe2a34d16d6e3c7b30d2ceb37c6a0e9a994c494ee1a61d2410885aa4c8bf8e56e264c0c0".from_hex().unwrap(); let b1 = "f90261f901f9a05716670833ec874362d65fea27a7cd35af5897d275b31a44944113111e4e96d2a01dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347948888f1f195afa192cfee860698584c030f4c9db1a0cb52de543653d86ccd13ba3ddf8b052525b04231c6884a4db3188a184681d878a0e78628dd45a1f8dc495594d83b76c588a3ee67463260f8b7d4a42f574aeab29aa0e9244cf7503b79c03d3a099e07a80d2dbc77bb0b502d8a89d51ac0d68dd31313b90100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000008302000001832fefd882520884562791e580a051b3ecba4e3f2b49c11d42dd0851ec514b1be3138080f72a2b6e83868275d98f8877671f479c414b47f862f86080018304cb2f94095e7baea6a6c7c4c2dfeb977efac326af552d870a801ca09e2709d7ec9bbe6b1bbbf0b2088828d14cd5e8642a1fee22dc74bfa89761a7f9a04bd8813dee4be989accdb708b1c2e325a7e9c695a8024e30e89d6c644e424747c0".from_hex().unwrap(); diff --git a/ethcore/src/evm/tests.rs b/ethcore/src/evm/tests.rs index 02f929192..9d4dd3bc4 100644 --- a/ethcore/src/evm/tests.rs +++ b/ethcore/src/evm/tests.rs @@ -25,7 +25,7 @@ struct FakeLogEntry { } #[derive(PartialEq, Eq, Hash, Debug)] -#[allow(enum_variant_names)] // Common prefix is C ;) +#[cfg_attr(feature="dev", allow(enum_variant_names))] // Common prefix is C ;) enum FakeCallType { CALL, CREATE } diff --git a/util/src/hash.rs b/util/src/hash.rs index d436c2d81..71c690ef6 100644 --- a/util/src/hash.rs +++ b/util/src/hash.rs @@ -635,7 +635,7 @@ mod tests { use std::str::FromStr; #[test] - #[allow(eq_op)] + #[cfg_attr(feature="dev", allow(eq_op))] fn hash() { let h = H64([0x01, 0x23, 0x45, 0x67, 0x89, 0xab, 0xcd, 0xef]); assert_eq!(H64::from_str("0123456789abcdef").unwrap(), h); diff --git a/util/src/panics.rs b/util/src/panics.rs index 18ed9ecb1..60d85ef14 100644 --- a/util/src/panics.rs +++ b/util/src/panics.rs @@ -18,7 +18,6 @@ use std::thread; use std::ops::DerefMut; -use std::any::Any; use std::sync::{Arc, Mutex}; /// Thread-safe closure for handling possible panics @@ -75,7 +74,7 @@ impl PanicHandler { #[cfg_attr(feature="dev", allow(deprecated))] // TODO [todr] catch_panic is deprecated but panic::recover has different bounds (not allowing mutex) pub fn catch_panic(&self, g: G) -> thread::Result where G: FnOnce() -> R + Send + 'static { - let guard = PanicGuard { handler: self }; + let _guard = PanicGuard { handler: self }; let result = g(); Ok(result) } @@ -108,13 +107,6 @@ impl OnPanicListener for F } } -fn convert_to_string(t: &Box) -> Option { - let as_str = t.downcast_ref::<&'static str>().cloned().map(|t| t.to_owned()); - let as_string = t.downcast_ref::().cloned(); - - as_str.or(as_string) -} - #[test] #[ignore] // panic forwarding doesnt work on the same thread in beta fn should_notify_listeners_about_panic () { diff --git a/util/src/uint.rs b/util/src/uint.rs index 912088fb9..7206a521e 100644 --- a/util/src/uint.rs +++ b/util/src/uint.rs @@ -991,7 +991,7 @@ mod tests { } #[test] - #[allow(eq_op)] + #[cfg_attr(feature="dev", allow(eq_op))] pub fn uint256_comp_test() { let small = U256([10u64, 0, 0, 0]); let big = U256([0x8C8C3EE70C644118u64, 0x0209E7378231E632, 0, 0]); From 7bc2853de96b672a64b5f56e405ded1c8e375830 Mon Sep 17 00:00:00 2001 From: arkpar Date: Sat, 20 Feb 2016 15:14:54 +0100 Subject: [PATCH 53/61] Removed TODO --- util/src/panics.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/util/src/panics.rs b/util/src/panics.rs index 60d85ef14..05d266b8b 100644 --- a/util/src/panics.rs +++ b/util/src/panics.rs @@ -72,7 +72,6 @@ impl PanicHandler { /// Invoke closure and catch any possible panics. /// In case of panic notifies all listeners about it. #[cfg_attr(feature="dev", allow(deprecated))] - // TODO [todr] catch_panic is deprecated but panic::recover has different bounds (not allowing mutex) pub fn catch_panic(&self, g: G) -> thread::Result where G: FnOnce() -> R + Send + 'static { let _guard = PanicGuard { handler: self }; let result = g(); From fe84eb4ff65b843cf0f56e1d4b038574ce144e53 Mon Sep 17 00:00:00 2001 From: arkpar Date: Sun, 21 Feb 2016 13:58:01 +0100 Subject: [PATCH 54/61] Fix locking --- util/src/network/host.rs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/util/src/network/host.rs b/util/src/network/host.rs index feddf1952..70915bee3 100644 --- a/util/src/network/host.rs +++ b/util/src/network/host.rs @@ -777,7 +777,8 @@ impl IoHandler> for Host where Messa FIRST_SESSION ... LAST_SESSION => self.session_readable(stream, io), FIRST_HANDSHAKE ... LAST_HANDSHAKE => self.handshake_readable(stream, io), DISCOVERY => { - if let Some(node_changes) = self.discovery.as_ref().unwrap().lock().unwrap().readable() { + let node_changes = { self.discovery.as_ref().unwrap().lock().unwrap().readable() }; + if let Some(node_changes) = node_changes { self.update_nodes(io, node_changes); } io.update_registration(DISCOVERY).expect("Error updating discovery registration"); @@ -809,7 +810,8 @@ impl IoHandler> for Host where Messa io.update_registration(DISCOVERY).expect("Error updating discovery registration"); }, DISCOVERY_ROUND => { - if let Some(node_changes) = self.discovery.as_ref().unwrap().lock().unwrap().round() { + let node_changes = { self.discovery.as_ref().unwrap().lock().unwrap().round() }; + if let Some(node_changes) = node_changes { self.update_nodes(io, node_changes); } io.update_registration(DISCOVERY).expect("Error updating discovery registration"); From 8b50fa658fa22d3c7ac71aba36fb5b96c73dd5ce Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Sun, 21 Feb 2016 16:32:11 +0100 Subject: [PATCH 55/61] Fix typo in deps script. --- install-deps.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/install-deps.sh b/install-deps.sh index 214b6748c..f5e1e44cf 100755 --- a/install-deps.sh +++ b/install-deps.sh @@ -425,7 +425,7 @@ function run_installer() depFound=$((depFound+1)) check "multirust" isMultirust=true - if [[ $(multirust show-default 2>/dev/null | grep beta | wc -l) == 4 ]]; then + if [[ $(multirust show-default 2>/dev/null | grep beta | wc -l) == 3 ]]; then depFound=$((depFound+1)) check "rust beta" isMultirustBeta=true From 91276ad82e702dca8f3183511fcfed7703c70a5c Mon Sep 17 00:00:00 2001 From: arkpar Date: Sun, 21 Feb 2016 16:52:25 +0100 Subject: [PATCH 56/61] Added comments --- util/src/network/connection.rs | 1 + util/src/network/handshake.rs | 1 + util/src/network/session.rs | 5 +++-- 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/util/src/network/connection.rs b/util/src/network/connection.rs index 0135fc333..a7396ff33 100644 --- a/util/src/network/connection.rs +++ b/util/src/network/connection.rs @@ -175,6 +175,7 @@ impl Connection { self.socket.peer_addr() } + /// Clone this connection. Clears the receiving buffer of the returned connection. pub fn try_clone(&self) -> io::Result { Ok(Connection { token: self.token, diff --git a/util/src/network/handshake.rs b/util/src/network/handshake.rs index a50dd6ba2..087de63b3 100644 --- a/util/src/network/handshake.rs +++ b/util/src/network/handshake.rs @@ -176,6 +176,7 @@ impl Handshake { Ok(()) } + /// Update socket registration with the event loop. pub fn update_socket>(&self, reg: Token, event_loop: &mut EventLoop) -> Result<(), UtilError> { if !self.expired() { try!(self.connection.update_socket(reg, event_loop)); diff --git a/util/src/network/session.rs b/util/src/network/session.rs index 6be1dbfe4..edf929a9a 100644 --- a/util/src/network/session.rs +++ b/util/src/network/session.rs @@ -111,7 +111,8 @@ const PACKET_USER: u8 = 0x10; const PACKET_LAST: u8 = 0x7f; impl Session { - /// Create a new session out of comepleted handshake. + /// Create a new session out of comepleted handshake. This clones the handshake connection object + /// and leaves the handhsake in limbo to be deregistered from the event loop. pub fn new(h: &mut Handshake, host: &HostInfo) -> Result { let id = h.id.clone(); let connection = try!(EncryptedConnection::new(h)); @@ -189,7 +190,7 @@ impl Session { } /// Register the session socket with the event loop - pub fn register_socket>(&self, reg: Token, event_loop: &mut EventLoop) -> Result<(), UtilError> { + pub fn register_socket>(&self, reg: Token, event_loop: &mut EventLoop) -> Result<(), UtilError> { if self.expired() { return Ok(()); } From 63bbd0ccd8b592bc8260ed41e8d558848ebdc088 Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Sun, 21 Feb 2016 20:00:45 +0100 Subject: [PATCH 57/61] Use proper version string. --- Cargo.lock | 2 ++ parity/main.rs | 7 +++---- rpc/Cargo.toml | 1 + rpc/src/lib.rs | 1 - rpc/src/v1/impls/web3.rs | 6 ++++-- util/Cargo.toml | 1 + util/src/lib.rs | 1 + util/src/misc.rs | 7 +++++++ util/src/network/host.rs | 4 ++-- 9 files changed, 21 insertions(+), 9 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 259a0c4d8..acdb241ac 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -204,6 +204,7 @@ dependencies = [ "jsonrpc-core 1.1.4 (registry+https://github.com/rust-lang/crates.io-index)", "jsonrpc-http-server 1.1.2 (registry+https://github.com/rust-lang/crates.io-index)", "rustc-serialize 0.3.18 (registry+https://github.com/rust-lang/crates.io-index)", + "rustc_version 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)", "serde 0.6.14 (registry+https://github.com/rust-lang/crates.io-index)", "serde_codegen 0.6.14 (registry+https://github.com/rust-lang/crates.io-index)", "serde_json 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)", @@ -234,6 +235,7 @@ dependencies = [ "rocksdb 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)", "rust-crypto 0.2.34 (registry+https://github.com/rust-lang/crates.io-index)", "rustc-serialize 0.3.18 (registry+https://github.com/rust-lang/crates.io-index)", + "rustc_version 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)", "serde 0.6.14 (registry+https://github.com/rust-lang/crates.io-index)", "sha3 0.1.0", "slab 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)", diff --git a/parity/main.rs b/parity/main.rs index 58d3a6f4c..f4b7880ab 100644 --- a/parity/main.rs +++ b/parity/main.rs @@ -29,7 +29,6 @@ extern crate log as rlog; extern crate env_logger; extern crate ctrlc; extern crate fdlimit; -extern crate target_info; extern crate daemonize; #[cfg(feature = "rpc")] @@ -51,7 +50,6 @@ use ethcore::ethereum; use ethcore::blockchain::CacheSize; use ethsync::EthSync; use docopt::Docopt; -use target_info::Target; use daemonize::Daemonize; const USAGE: &'static str = " @@ -146,14 +144,15 @@ fn setup_rpc_server(_client: Arc, _sync: Arc, _url: &str) { fn print_version() { println!("\ -Parity version {} ({}-{}-{}) +Parity + version {} Copyright 2015, 2016 Ethcore (UK) Limited License GPLv3+: GNU GPL version 3 or later . This is free software: you are free to change and redistribute it. There is NO WARRANTY, to the extent permitted by law. By Wood/Paronyan/Kotewicz/DrwiÄ™ga/Volf.\ -", env!("CARGO_PKG_VERSION"), Target::arch(), Target::env(), Target::os()); +", version()); } fn die_with_message(msg: &str) -> ! { diff --git a/rpc/Cargo.toml b/rpc/Cargo.toml index 0b7c17383..bce5b609c 100644 --- a/rpc/Cargo.toml +++ b/rpc/Cargo.toml @@ -20,6 +20,7 @@ clippy = { version = "0.0.42", optional = true } target_info = "0.1.0" rustc-serialize = "0.3" serde_macros = { version = "0.6.13", optional = true } +rustc_version = "0.1.0" [build-dependencies] serde_codegen = { version = "0.6.13", optional = true } diff --git a/rpc/src/lib.rs b/rpc/src/lib.rs index c9542ade7..d7b5bdc3b 100644 --- a/rpc/src/lib.rs +++ b/rpc/src/lib.rs @@ -20,7 +20,6 @@ #![cfg_attr(nightly, plugin(serde_macros, clippy))] extern crate rustc_serialize; -extern crate target_info; extern crate serde; extern crate serde_json; extern crate jsonrpc_core; diff --git a/rpc/src/v1/impls/web3.rs b/rpc/src/v1/impls/web3.rs index f10150a30..0a237d56b 100644 --- a/rpc/src/v1/impls/web3.rs +++ b/rpc/src/v1/impls/web3.rs @@ -15,8 +15,8 @@ // along with Parity. If not, see . //! Web3 rpc implementation. -use target_info::Target; use jsonrpc_core::*; +use util::version; use v1::traits::Web3; /// Web3 rpc implementation. @@ -30,7 +30,9 @@ impl Web3Client { impl Web3 for Web3Client { fn client_version(&self, params: Params) -> Result { match params { - Params::None => Ok(Value::String(format!("Parity/-/{}/{}-{}-{}/rust1.8-nightly", env!("CARGO_PKG_VERSION"), Target::arch(), Target::env(), Target::os()))), + Params::None => { + Ok(Value::String(version())), + } _ => Err(Error::invalid_params()) } } diff --git a/util/Cargo.toml b/util/Cargo.toml index 5d7fa697f..29d820760 100644 --- a/util/Cargo.toml +++ b/util/Cargo.toml @@ -29,6 +29,7 @@ serde = "0.6.7" clippy = { version = "0.0.42", optional = true } json-tests = { path = "json-tests" } target_info = "0.1.0" +rustc_version = "0.1.0" igd = "0.4.2" ethcore-devtools = { path = "../devtools" } libc = "0.2.7" diff --git a/util/src/lib.rs b/util/src/lib.rs index 9527341ad..b91e2d65b 100644 --- a/util/src/lib.rs +++ b/util/src/lib.rs @@ -108,6 +108,7 @@ extern crate log as rlog; extern crate igd; extern crate ethcore_devtools as devtools; extern crate libc; +extern crate rustc_version; pub mod standard; #[macro_use] diff --git a/util/src/misc.rs b/util/src/misc.rs index ae3dbc5bf..fdbaa0b49 100644 --- a/util/src/misc.rs +++ b/util/src/misc.rs @@ -18,6 +18,8 @@ use std::fs::File; use common::*; +use target_info::Target; +use rustc_version; #[derive(Debug,Clone,PartialEq,Eq)] /// Diff type for specifying a change (or not). @@ -62,3 +64,8 @@ pub fn contents(name: &str) -> Result { try!(file.read_to_end(&mut ret)); Ok(ret) } + +/// Get the standard version string for this software. +pub fn version() -> String { + format!("Parity/-/{}/{}-{}-{}/{}", env!("CARGO_PKG_VERSION"), Target::arch(), Target::env(), Target::os(), rustc_version::version()) +} \ No newline at end of file diff --git a/util/src/network/host.rs b/util/src/network/host.rs index 326cd2715..fb3150ad3 100644 --- a/util/src/network/host.rs +++ b/util/src/network/host.rs @@ -26,8 +26,8 @@ use std::io::{Read, Write}; use std::fs; use mio::*; use mio::tcp::*; -use target_info::Target; use hash::*; +use misc::version; use crypto::*; use sha3::Hashable; use rlp::*; @@ -360,7 +360,7 @@ impl Host where Message: Send + Sync + Clone { config: config, nonce: H256::random(), protocol_version: PROTOCOL_VERSION, - client_version: format!("Parity/{}/{}-{}-{}", env!("CARGO_PKG_VERSION"), Target::arch(), Target::env(), Target::os()), + client_version: version(), listen_port: 0, capabilities: Vec::new(), }), From 6fa2284c6889ee3815d18481043c897a928eb4ab Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Sun, 21 Feb 2016 20:03:01 +0100 Subject: [PATCH 58/61] Remove unneeded deps. --- Cargo.lock | 3 --- Cargo.toml | 1 - rpc/Cargo.toml | 2 -- 3 files changed, 6 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index acdb241ac..16b41d0e8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -15,7 +15,6 @@ dependencies = [ "fdlimit 0.1.0", "log 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)", "rustc-serialize 0.3.18 (registry+https://github.com/rust-lang/crates.io-index)", - "target_info 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -204,12 +203,10 @@ dependencies = [ "jsonrpc-core 1.1.4 (registry+https://github.com/rust-lang/crates.io-index)", "jsonrpc-http-server 1.1.2 (registry+https://github.com/rust-lang/crates.io-index)", "rustc-serialize 0.3.18 (registry+https://github.com/rust-lang/crates.io-index)", - "rustc_version 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)", "serde 0.6.14 (registry+https://github.com/rust-lang/crates.io-index)", "serde_codegen 0.6.14 (registry+https://github.com/rust-lang/crates.io-index)", "serde_json 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)", "syntex 0.29.0 (registry+https://github.com/rust-lang/crates.io-index)", - "target_info 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 3cb158df7..7fdfc2bee 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -17,7 +17,6 @@ ethcore = { path = "ethcore" } ethsync = { path = "sync" } ethcore-rpc = { path = "rpc", optional = true } fdlimit = { path = "util/fdlimit" } -target_info = "0.1" daemonize = "0.2" ethcore-devtools = { path = "devtools" } diff --git a/rpc/Cargo.toml b/rpc/Cargo.toml index bce5b609c..be06316a4 100644 --- a/rpc/Cargo.toml +++ b/rpc/Cargo.toml @@ -17,10 +17,8 @@ ethcore-util = { path = "../util" } ethcore = { path = "../ethcore" } ethsync = { path = "../sync" } clippy = { version = "0.0.42", optional = true } -target_info = "0.1.0" rustc-serialize = "0.3" serde_macros = { version = "0.6.13", optional = true } -rustc_version = "0.1.0" [build-dependencies] serde_codegen = { version = "0.6.13", optional = true } From ea187253a23d7394720d3b071d5888d30f2227d0 Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Sun, 21 Feb 2016 21:14:09 +0100 Subject: [PATCH 59/61] Include git commit date & hash. --- Cargo.lock | 21 +++++++++++++++------ util/Cargo.toml | 6 +++++- util/build.rs | 6 ++++++ util/src/lib.rs | 2 +- util/src/misc.rs | 5 +++-- 5 files changed, 30 insertions(+), 10 deletions(-) create mode 100644 util/build.rs diff --git a/Cargo.lock b/Cargo.lock index 16b41d0e8..8bf4a3764 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -52,6 +52,11 @@ name = "bitflags" version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" +[[package]] +name = "blastfig" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" + [[package]] name = "bytes" version = "0.3.0" @@ -236,9 +241,9 @@ dependencies = [ "serde 0.6.14 (registry+https://github.com/rust-lang/crates.io-index)", "sha3 0.1.0", "slab 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)", - "target_info 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", "time 0.1.34 (registry+https://github.com/rust-lang/crates.io-index)", "tiny-keccak 1.0.5 (registry+https://github.com/rust-lang/crates.io-index)", + "vergen 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -687,11 +692,6 @@ dependencies = [ "unicode-xid 0.0.3 (registry+https://github.com/rust-lang/crates.io-index)", ] -[[package]] -name = "target_info" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" - [[package]] name = "term" version = "0.2.14" @@ -788,6 +788,15 @@ dependencies = [ "rustc-serialize 0.3.18 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "vergen" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "blastfig 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)", + "time 0.1.34 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "winapi" version = "0.2.5" diff --git a/util/Cargo.toml b/util/Cargo.toml index 29d820760..aae6a3d85 100644 --- a/util/Cargo.toml +++ b/util/Cargo.toml @@ -5,6 +5,7 @@ license = "GPL-3.0" name = "ethcore-util" version = "0.9.99" authors = ["Ethcore "] +build = "build.rs" [dependencies] log = "0.3" @@ -28,12 +29,15 @@ sha3 = { path = "sha3" } serde = "0.6.7" clippy = { version = "0.0.42", optional = true } json-tests = { path = "json-tests" } -target_info = "0.1.0" rustc_version = "0.1.0" igd = "0.4.2" ethcore-devtools = { path = "../devtools" } libc = "0.2.7" +vergen = "*" [features] default = [] dev = ["clippy"] + +[build-dependencies] +vergen = "*" diff --git a/util/build.rs b/util/build.rs new file mode 100644 index 000000000..32ee30472 --- /dev/null +++ b/util/build.rs @@ -0,0 +1,6 @@ +extern crate vergen; +use vergen::*; + +fn main() { + vergen(OutputFns::all()).unwrap(); +} \ No newline at end of file diff --git a/util/src/lib.rs b/util/src/lib.rs index b91e2d65b..07593e5eb 100644 --- a/util/src/lib.rs +++ b/util/src/lib.rs @@ -82,7 +82,6 @@ //! cargo build --release //! ``` -extern crate target_info; extern crate slab; extern crate rustc_serialize; extern crate mio; @@ -109,6 +108,7 @@ extern crate igd; extern crate ethcore_devtools as devtools; extern crate libc; extern crate rustc_version; +extern crate vergen; pub mod standard; #[macro_use] diff --git a/util/src/misc.rs b/util/src/misc.rs index fdbaa0b49..075da476d 100644 --- a/util/src/misc.rs +++ b/util/src/misc.rs @@ -18,9 +18,10 @@ use std::fs::File; use common::*; -use target_info::Target; use rustc_version; +include!(concat!(env!("OUT_DIR"), "/version.rs")); + #[derive(Debug,Clone,PartialEq,Eq)] /// Diff type for specifying a change (or not). pub enum Diff where T: Eq { @@ -67,5 +68,5 @@ pub fn contents(name: &str) -> Result { /// Get the standard version string for this software. pub fn version() -> String { - format!("Parity/-/{}/{}-{}-{}/{}", env!("CARGO_PKG_VERSION"), Target::arch(), Target::env(), Target::os(), rustc_version::version()) + format!("Parity/{}/{}-{}/{}/rustc{}", env!("CARGO_PKG_VERSION"), short_sha(), commit_date(), target(), rustc_version::version()) } \ No newline at end of file From 0c832853b6624fd9df0e36cee94ace83c48d23a9 Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Sun, 21 Feb 2016 21:22:11 +0100 Subject: [PATCH 60/61] Avoid the dep = "*" --- util/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/util/Cargo.toml b/util/Cargo.toml index aae6a3d85..da4f8e34e 100644 --- a/util/Cargo.toml +++ b/util/Cargo.toml @@ -33,7 +33,7 @@ rustc_version = "0.1.0" igd = "0.4.2" ethcore-devtools = { path = "../devtools" } libc = "0.2.7" -vergen = "*" +vergen = "0.1" [features] default = [] From fbee46d69d56580eaebd3d074bfcc849fd8c9803 Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Sun, 21 Feb 2016 21:45:56 +0100 Subject: [PATCH 61/61] Fix netstats. --- util/src/misc.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/util/src/misc.rs b/util/src/misc.rs index 075da476d..22a2bb673 100644 --- a/util/src/misc.rs +++ b/util/src/misc.rs @@ -68,5 +68,5 @@ pub fn contents(name: &str) -> Result { /// Get the standard version string for this software. pub fn version() -> String { - format!("Parity/{}/{}-{}/{}/rustc{}", env!("CARGO_PKG_VERSION"), short_sha(), commit_date(), target(), rustc_version::version()) + format!("Parity//{}/{}-{}/{}/rustc{}", env!("CARGO_PKG_VERSION"), short_sha(), commit_date(), target(), rustc_version::version()) } \ No newline at end of file