Secretstore over network (#4974)

* ECDKG protocol prototype

* added test for enc/dec math

* get rid of decryption_session

* added licenses

* fix after merge

* get rid of unused serde dependency

* doc

* decryption session [without commutative enc]

* failed_dec_session

* fixed tests

* added commen

* added more decryption session tests

* helper to localize an issue

* more computations to localize error

* decryption_session::SessionParams

* added tests for EC math to localize problem

* secretstore network transport

* encryption_session_works_over_network

* network errors processing

* connecting to KeyServer

* licenses

* get rid of debug println-s

* fixed secretstore args

* encryption results are stored in KS database

* decryption protocol works over network

* enc/dec Session traits

* fixing warnings

* fix after merge

* finally fixed <odd>-of-N-scheme

* temporary commented test

* 1-of-N works in math

* scheme 1-of-N works

* remove unnecessary unsafety

* fixed grumbles

* fix grumbles

* lost files

Cargo.lock

@@ -23,6 +23,7 @@ dependencies = [
  "ethcore-signer 1.7.0",
  "ethcore-stratum 1.7.0",
  "ethcore-util 1.7.0",
+ "ethkey 0.2.0",
  "ethsync 1.7.0",
  "evmbin 0.1.0",
  "fdlimit 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -633,6 +634,7 @@ dependencies = [
 name = "ethcore-secretstore"
 version = "1.0.0"
 dependencies = [
+ "byteorder 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "ethcore-devtools 1.7.0",
  "ethcore-ipc 1.7.0",
  "ethcore-ipc-codegen 1.7.0",
@@ -640,9 +642,18 @@ dependencies = [
  "ethcore-util 1.7.0",
  "ethcrypto 0.1.0",
  "ethkey 0.2.0",
+ "futures 0.1.11 (registry+https://github.com/rust-lang/crates.io-index)",
+ "futures-cpupool 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)",
  "hyper 0.10.5 (registry+https://github.com/rust-lang/crates.io-index)",
  "log 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)",
  "parking_lot 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "rustc-serialize 0.3.19 (registry+https://github.com/rust-lang/crates.io-index)",
+ "serde 0.9.6 (registry+https://github.com/rust-lang/crates.io-index)",
+ "serde_derive 0.9.6 (registry+https://github.com/rust-lang/crates.io-index)",
+ "serde_json 0.9.5 (registry+https://github.com/rust-lang/crates.io-index)",
+ "tokio-core 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)",
+ "tokio-proto 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "tokio-service 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "url 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 

Cargo.toml

@@ -40,6 +40,7 @@ ethcore-ipc-hypervisor = { path = "ipc/hypervisor" }
 ethcore-light = { path = "ethcore/light" }
 ethcore-logger = { path = "logger" }
 ethcore-stratum = { path = "stratum" }
+ethkey = { path = "ethkey" }
 evmbin = { path = "evmbin" }
 rlp = { path = "util/rlp" }
 rpc-cli = { path = "rpc_cli" }

ethcrypto/src/lib.rs

@@ -78,6 +78,12 @@ impl fmt::Display for Error {
 	}
 }
 
+impl Into<String> for Error {
+	fn into(self) -> String {
+		format!("{}", self)
+	}
+}
+
 impl From<SecpError> for Error {
 	fn from(e: SecpError) -> Self {
 		Error::Secp(e)

ethkey/src/keypair.rs

@@ -27,6 +27,7 @@ pub fn public_to_address(public: &Public) -> Address {
 	result
 }
 
+#[derive(Clone)]
 /// secp256k1 key pair
 pub struct KeyPair {
 	secret: Secret,

parity/main.rs

@@ -53,6 +53,7 @@ extern crate ethcore_logger;
 extern crate ethcore_rpc;
 extern crate ethcore_signer;
 extern crate ethcore_util as util;
+extern crate ethkey;
 extern crate ethsync;
 extern crate parity_hash_fetch as hash_fetch;
 extern crate parity_ipfs_api;

parity/secretstore.rs

@@ -53,6 +53,7 @@ mod server {
 
 #[cfg(feature="secretstore")]
 mod server {
+	use ethkey;
 	use ethcore_secretstore;
 	use super::{Configuration, Dependencies};
 
@@ -64,10 +65,35 @@ mod server {
 	impl KeyServer {
 		/// Create new key server
 		pub fn new(conf: Configuration, _deps: Dependencies) -> Result<Self, String> {
+			let key_pairs = vec![
+				ethkey::KeyPair::from_secret("6c26a76e9b31048d170873a791401c7e799a11f0cefc0171cc31a49800967509".parse().unwrap()).unwrap(),
+				ethkey::KeyPair::from_secret("7e94018b3731afdb3b4e6f4c3e179475640166da12e1d1b0c7d80729b1a5b452".parse().unwrap()).unwrap(),
+				ethkey::KeyPair::from_secret("5ab6ed2a52c33142380032c39a03a86b12eacb3fa4b53bc16d84f51318156f8c".parse().unwrap()).unwrap(),
+			];
 			let conf = ethcore_secretstore::ServiceConfiguration {
-				listener_addr: conf.interface,
+				listener_address: ethcore_secretstore::NodeAddress {
-				listener_port: conf.port,
+					address: conf.interface.clone(),
-				data_path: conf.data_path,
+					port: conf.port,
+				},
+				data_path: conf.data_path.clone(),
+				// TODO: this is test configuration. how it will be configured in production?
+				cluster_config: ethcore_secretstore::ClusterConfiguration {
+					threads: 4,
+					self_private: (***key_pairs[(conf.port - 8082) as usize].secret()).into(),
+					listener_address: ethcore_secretstore::NodeAddress {
+						address: conf.interface.clone(),
+						port: conf.port + 10,
+					},
+					nodes: key_pairs.iter().enumerate().map(|(i, kp)| (kp.public().clone(),
+						ethcore_secretstore::NodeAddress {
+							address: conf.interface.clone(),
+							port: 8082 + 10 + (i as u16),
+						})).collect(),
+					allow_connecting_to_higher_nodes: true,
+					encryption_config: ethcore_secretstore::EncryptionConfiguration {
+						key_check_timeout_ms: 1000,
+					},
+				}
 			};
 
 			let key_server = ethcore_secretstore::start(conf)

secret_store/Cargo.toml

@@ -10,9 +10,19 @@ build = "build.rs"
 ethcore-ipc-codegen = { path = "../ipc/codegen" }
 
 [dependencies]
+byteorder = "1.0"
 log = "0.3"
 parking_lot = "0.4"
 hyper = { version = "0.10", default-features = false }
+serde = "0.9"
+serde_json = "0.9"
+serde_derive = "0.9"
+futures = "0.1"
+futures-cpupool = "0.1"
+rustc-serialize = "0.3"
+tokio-core = "0.1"
+tokio-service = "0.1"
+tokio-proto = "0.1"
 url = "1.0"
 ethcore-devtools = { path = "../devtools" }
 ethcore-util = { path = "../util" }

secret_store/src/http_listener.rs

@@ -14,7 +14,6 @@
 // You should have received a copy of the GNU General Public License
 // along with Parity.  If not, see <http://www.gnu.org/licenses/>.
 
-use std::str::FromStr;
 use std::sync::Arc;
 use hyper::header;
 use hyper::uri::RequestUri;
@@ -39,7 +38,9 @@ pub struct KeyServerHttpListener<T: KeyServer + 'static> {
 enum Request {
 	/// Invalid request
 	Invalid,
-	/// Request encryption key of given document for given requestor
+	/// Generate encryption key.
+	GenerateDocumentKey(DocumentAddress, RequestSignature, usize),
+	/// Request encryption key of given document for given requestor.
 	GetDocumentKey(DocumentAddress, RequestSignature),
 }
 
@@ -63,9 +64,9 @@ impl<T> KeyServerHttpListener<T> where T: KeyServer + 'static {
 			handler: shared_handler.clone(),
 		};
 
-		let listener_addr: &str = &format!("{}:{}", config.listener_addr, config.listener_port);
+		let listener_addr: &str = &format!("{}:{}", config.listener_address.address, config.listener_address.port);
-		let http_server = HttpServer::http(&listener_addr).unwrap();
+		let http_server = HttpServer::http(&listener_addr).expect("cannot start HttpServer");
-		let http_server = http_server.handle(handler).unwrap();
+		let http_server = http_server.handle(handler).expect("cannot start HttpServer");
 		let listener = KeyServerHttpListener {
 			_http_server: http_server,
 			handler: shared_handler,
@@ -75,6 +76,10 @@ impl<T> KeyServerHttpListener<T> where T: KeyServer + 'static {
 }
 
 impl<T> KeyServer for KeyServerHttpListener<T> where T: KeyServer + 'static {
+	fn generate_document_key(&self, signature: &RequestSignature, document: &DocumentAddress, threshold: usize) -> Result<DocumentEncryptedKey, Error> {
+		self.handler.key_server.generate_document_key(signature, document, threshold)
+	}
+
 	fn document_key(&self, signature: &RequestSignature, document: &DocumentAddress) -> Result<DocumentEncryptedKey, Error> {
 		self.handler.key_server.document_key(signature, document)
 	}
@@ -82,33 +87,51 @@ impl<T> KeyServer for KeyServerHttpListener<T> where T: KeyServer + 'static {
 
 impl<T> HttpHandler for KeyServerHttpHandler<T> where T: KeyServer + 'static {
 	fn handle(&self, req: HttpRequest, mut res: HttpResponse) {
-		if req.method != HttpMethod::Get {
-			warn!(target: "secretstore", "Ignoring {}-request {}", req.method, req.uri);
-			*res.status_mut() = HttpStatusCode::NotFound;
-			return;
-		}
-
 		if req.headers.has::<header::Origin>() {
 			warn!(target: "secretstore", "Ignoring {}-request {} with Origin header", req.method, req.uri);
 			*res.status_mut() = HttpStatusCode::NotFound;
 			return;
 		}
 
-		match req.uri {
+		let req_method = req.method.clone();
-			RequestUri::AbsolutePath(ref path) => match parse_request(&path) {
+		let req_uri = req.uri.clone();
-				Request::GetDocumentKey(document, signature) => {
+		match &req_uri {
-					let document_key = self.handler.key_server.document_key(&signature, &document)
+			&RequestUri::AbsolutePath(ref path) => match parse_request(&req_method, &path) {
+				Request::GenerateDocumentKey(document, signature, threshold) => {
+					return_document_key(req, res, self.handler.key_server.generate_document_key(&signature, &document, threshold)
 						.map_err(|err| {
-							warn!(target: "secretstore", "GetDocumentKey request {} has failed with: {}", req.uri, err);
+							warn!(target: "secretstore", "GenerateDocumentKey request {} has failed with: {}", req_uri, err);
 							err
-						});
+						}));
+				},
+				Request::GetDocumentKey(document, signature) => {
+					return_document_key(req, res, self.handler.key_server.document_key(&signature, &document)
+						.map_err(|err| {
+							warn!(target: "secretstore", "GetDocumentKey request {} has failed with: {}", req_uri, err);
+							err
+						}));
+				},
+				Request::Invalid => {
+					warn!(target: "secretstore", "Ignoring invalid {}-request {}", req_method, req_uri);
+					*res.status_mut() = HttpStatusCode::BadRequest;
+				},
+			},
+			_ => {
+				warn!(target: "secretstore", "Ignoring invalid {}-request {}", req_method, req_uri);
+				*res.status_mut() = HttpStatusCode::NotFound;
+			},
+		};
+	}
+}
+
+fn return_document_key(req: HttpRequest, mut res: HttpResponse, document_key: Result<DocumentEncryptedKey, Error>) {
 	match document_key {
 		Ok(document_key) => {
 			let document_key = document_key.to_hex().into_bytes();
 			res.headers_mut().set(header::ContentType::plaintext());
 			if let Err(err) = res.send(&document_key) {
-								// nothing to do, but log error
+				// nothing to do, but to log an error
-								warn!(target: "secretstore", "GetDocumentKey request {} response has failed with: {}", req.uri, err);
+				warn!(target: "secretstore", "response to request {} has failed with: {}", req.uri, err);
 			}
 		},
 		Err(Error::BadSignature) => *res.status_mut() = HttpStatusCode::BadRequest,
@@ -117,60 +140,50 @@ impl<T> HttpHandler for KeyServerHttpHandler<T> where T: KeyServer + 'static {
 		Err(Error::Database(_)) => *res.status_mut() = HttpStatusCode::InternalServerError,
 		Err(Error::Internal(_)) => *res.status_mut() = HttpStatusCode::InternalServerError,
 	}
-				},
-				Request::Invalid => {
-					warn!(target: "secretstore", "Ignoring invalid {}-request {}", req.method, req.uri);
-					*res.status_mut() = HttpStatusCode::BadRequest;
-				},
-			},
-			_ => {
-				warn!(target: "secretstore", "Ignoring invalid {}-request {}", req.method, req.uri);
-				*res.status_mut() = HttpStatusCode::NotFound;
-			},
-		};
-	}
 }
 
-fn parse_request(uri_path: &str) -> Request {
+fn parse_request(method: &HttpMethod, uri_path: &str) -> Request {
 	let uri_path = match percent_decode(uri_path.as_bytes()).decode_utf8() {
 		Ok(path) => path,
 		Err(_) => return Request::Invalid,
 	};
 
 	let path: Vec<String> = uri_path.trim_left_matches('/').split('/').map(Into::into).collect();
-	if path.len() != 2 || path[0].is_empty() || path[1].is_empty() {
+	if path.len() < 2 || path[0].is_empty() || path[1].is_empty() {
 		return Request::Invalid;
 	}
 
-	let document = DocumentAddress::from_str(&path[0]);
+	let args_len = path.len();
-	let signature = RequestSignature::from_str(&path[1]);
+	let document = path[0].parse();
-	match (document, signature) {
+	let signature = path[1].parse();
-		(Ok(document), Ok(signature)) => Request::GetDocumentKey(document, signature),
+	let threshold = (if args_len > 2 { &path[2] } else { "" }).parse();
+	match (args_len, method, document, signature, threshold) {
+		(3, &HttpMethod::Post, Ok(document), Ok(signature), Ok(threshold)) => Request::GenerateDocumentKey(document, signature, threshold),
+		(2, &HttpMethod::Get, Ok(document), Ok(signature), _) => Request::GetDocumentKey(document, signature),
 		_ => Request::Invalid,
 	}
 }
 
 #[cfg(test)]
 mod tests {
-	use std::str::FromStr;
+	use hyper::method::Method as HttpMethod;
-	use super::super::RequestSignature;
 	use super::{parse_request, Request};
 
 	#[test]
 	fn parse_request_successful() {
-		assert_eq!(parse_request("/0000000000000000000000000000000000000000000000000000000000000001/a199fb39e11eefb61c78a4074a53c0d4424600a3e74aad4fb9d93a26c30d067e1d4d29936de0c73f19827394a1dd049480a0d581aee7ae7546968da7d3d1c2fd01"),
+		assert_eq!(parse_request(&HttpMethod::Get, "/0000000000000000000000000000000000000000000000000000000000000001/a199fb39e11eefb61c78a4074a53c0d4424600a3e74aad4fb9d93a26c30d067e1d4d29936de0c73f19827394a1dd049480a0d581aee7ae7546968da7d3d1c2fd01"),
 			Request::GetDocumentKey("0000000000000000000000000000000000000000000000000000000000000001".into(),
-				RequestSignature::from_str("a199fb39e11eefb61c78a4074a53c0d4424600a3e74aad4fb9d93a26c30d067e1d4d29936de0c73f19827394a1dd049480a0d581aee7ae7546968da7d3d1c2fd01").unwrap()));
+				"a199fb39e11eefb61c78a4074a53c0d4424600a3e74aad4fb9d93a26c30d067e1d4d29936de0c73f19827394a1dd049480a0d581aee7ae7546968da7d3d1c2fd01".parse().unwrap()));
-		assert_eq!(parse_request("/%30000000000000000000000000000000000000000000000000000000000000001/a199fb39e11eefb61c78a4074a53c0d4424600a3e74aad4fb9d93a26c30d067e1d4d29936de0c73f19827394a1dd049480a0d581aee7ae7546968da7d3d1c2fd01"),
+		assert_eq!(parse_request(&HttpMethod::Get, "/%30000000000000000000000000000000000000000000000000000000000000001/a199fb39e11eefb61c78a4074a53c0d4424600a3e74aad4fb9d93a26c30d067e1d4d29936de0c73f19827394a1dd049480a0d581aee7ae7546968da7d3d1c2fd01"),
 			Request::GetDocumentKey("0000000000000000000000000000000000000000000000000000000000000001".into(),
-				RequestSignature::from_str("a199fb39e11eefb61c78a4074a53c0d4424600a3e74aad4fb9d93a26c30d067e1d4d29936de0c73f19827394a1dd049480a0d581aee7ae7546968da7d3d1c2fd01").unwrap()));
+				"a199fb39e11eefb61c78a4074a53c0d4424600a3e74aad4fb9d93a26c30d067e1d4d29936de0c73f19827394a1dd049480a0d581aee7ae7546968da7d3d1c2fd01".parse().unwrap()));
 	}
 
 	#[test]
 	fn parse_request_failed() {
-		assert_eq!(parse_request("/0000000000000000000000000000000000000000000000000000000000000001"), Request::Invalid);
+		assert_eq!(parse_request(&HttpMethod::Get, "/0000000000000000000000000000000000000000000000000000000000000001"), Request::Invalid);
-		assert_eq!(parse_request("/0000000000000000000000000000000000000000000000000000000000000001/"), Request::Invalid);
+		assert_eq!(parse_request(&HttpMethod::Get, "/0000000000000000000000000000000000000000000000000000000000000001/"), Request::Invalid);
-		assert_eq!(parse_request("/a/b"), Request::Invalid);
+		assert_eq!(parse_request(&HttpMethod::Get, "/a/b"), Request::Invalid);
-		assert_eq!(parse_request("/0000000000000000000000000000000000000000000000000000000000000001/a199fb39e11eefb61c78a4074a53c0d4424600a3e74aad4fb9d93a26c30d067e1d4d29936de0c73f19827394a1dd049480a0d581aee7ae7546968da7d3d1c2fd01/0000000000000000000000000000000000000000000000000000000000000002"), Request::Invalid);
+		assert_eq!(parse_request(&HttpMethod::Get, "/0000000000000000000000000000000000000000000000000000000000000001/a199fb39e11eefb61c78a4074a53c0d4424600a3e74aad4fb9d93a26c30d067e1d4d29936de0c73f19827394a1dd049480a0d581aee7ae7546968da7d3d1c2fd01/0000000000000000000000000000000000000000000000000000000000000002"), Request::Invalid);
 	}
 }

secret_store/src/key_server.rs

@@ -14,42 +14,78 @@
 // You should have received a copy of the GNU General Public License
 // along with Parity.  If not, see <http://www.gnu.org/licenses/>.
 
+use std::thread;
+use std::sync::Arc;
+use std::sync::mpsc;
+use futures::{self, Future};
+use parking_lot::Mutex;
+use tokio_core::reactor::Core;
 use ethcrypto;
 use ethkey;
 use super::acl_storage::AclStorage;
 use super::key_storage::KeyStorage;
+use key_server_cluster::ClusterCore;
 use traits::KeyServer;
-use types::all::{Error, RequestSignature, DocumentAddress, DocumentEncryptedKey};
+use types::all::{Error, RequestSignature, DocumentAddress, DocumentEncryptedKey, ClusterConfiguration};
+use key_server_cluster::{ClusterClient, ClusterConfiguration as NetClusterConfiguration};
 
 /// Secret store key server implementation
-pub struct KeyServerImpl<T: AclStorage, U: KeyStorage> {
+pub struct KeyServerImpl {
-	acl_storage: T,
+	data: Arc<Mutex<KeyServerCore>>,
-	key_storage: U,
 }
 
-impl<T, U> KeyServerImpl<T, U> where T: AclStorage, U: KeyStorage {
+/// Secret store key server data.
+pub struct KeyServerCore {
+	close: Option<futures::Complete<()>>,
+	handle: Option<thread::JoinHandle<()>>,
+	cluster: Option<Arc<ClusterClient>>,
+}
+
+impl KeyServerImpl {
 	/// Create new key server instance
-	pub fn new(acl_storage: T, key_storage: U) -> Self {
+	pub fn new(config: &ClusterConfiguration, acl_storage: Arc<AclStorage>, key_storage: Arc<KeyStorage>) -> Result<Self, Error> {
-		KeyServerImpl {
+		Ok(KeyServerImpl {
-			acl_storage: acl_storage,
+			data: Arc::new(Mutex::new(KeyServerCore::new(config, acl_storage, key_storage)?)),
-			key_storage: key_storage,
+		})
 	}
+
+	#[cfg(test)]
+	/// Get cluster client reference.
+	pub fn cluster(&self) -> Arc<ClusterClient> {
+		self.data.lock().cluster.clone()
+			.expect("cluster can be None in test cfg only; test cfg is for correct tests; qed")
 	}
 }
 
-impl<T, U> KeyServer for KeyServerImpl<T, U> where T: AclStorage, U: KeyStorage {
+impl KeyServer for KeyServerImpl {
+	fn generate_document_key(&self, signature: &RequestSignature, document: &DocumentAddress, threshold: usize) -> Result<DocumentEncryptedKey, Error> {
+		// recover requestor' public key from signature
+		let public = ethkey::recover(signature, document)
+			.map_err(|_| Error::BadSignature)?;
+
+		// generate document key
+		let data = self.data.lock();
+		let encryption_session = data.cluster.as_ref().expect("cluster can be None in test cfg only; test cfg is for correct tests; qed")
+			.new_encryption_session(document.clone(), threshold)?;
+		let document_key = encryption_session.wait()?;
+
+		// encrypt document key with requestor public key
+		let document_key = ethcrypto::ecies::encrypt_single_message(&public, &document_key)
+			.map_err(|err| Error::Internal(format!("Error encrypting document key: {}", err)))?;
+		Ok(document_key)
+	}
+
 	fn document_key(&self, signature: &RequestSignature, document: &DocumentAddress) -> Result<DocumentEncryptedKey, Error> {
 		// recover requestor' public key from signature
 		let public = ethkey::recover(signature, document)
 			.map_err(|_| Error::BadSignature)?;
 
-		// check that requestor has access to the document
+		// decrypt document key
-		if !self.acl_storage.check(&public, document)? {
+		let data = self.data.lock();
-			return Err(Error::AccessDenied);
+		let decryption_session = data.cluster.as_ref().expect("cluster can be None in test cfg only; test cfg is for correct tests; qed")
-		}
+			.new_decryption_session(document.clone(), signature.clone())?;
+		let document_key = decryption_session.wait()?;
 
-		// read unencrypted document key
-		let document_key = self.key_storage.get(document)?;
 		// encrypt document key with requestor public key
 		let document_key = ethcrypto::ecies::encrypt_single_message(&public, &document_key)
 			.map_err(|err| Error::Internal(format!("Error encrypting document key: {}", err)))?;
@@ -57,68 +93,132 @@ impl<T, U> KeyServer for KeyServerImpl<T, U> where T: AclStorage, U: KeyStorage
 	}
 }
 
+impl KeyServerCore {
+	pub fn new(config: &ClusterConfiguration, acl_storage: Arc<AclStorage>, key_storage: Arc<KeyStorage>) -> Result<Self, Error> {
+		let config = NetClusterConfiguration {
+			threads: config.threads,
+			self_key_pair: ethkey::KeyPair::from_secret_slice(&config.self_private)?,
+			listen_address: (config.listener_address.address.clone(), config.listener_address.port),
+			nodes: config.nodes.iter()
+				.map(|(node_id, node_address)| (node_id.clone(), (node_address.address.clone(), node_address.port)))
+				.collect(),
+			allow_connecting_to_higher_nodes: config.allow_connecting_to_higher_nodes,
+			encryption_config: config.encryption_config.clone(),
+			acl_storage: acl_storage,
+			key_storage: key_storage,
+		};
+
+		let (stop, stopped) = futures::oneshot();
+		let (tx, rx) = mpsc::channel();
+		let handle = thread::spawn(move || {
+			let mut el = match Core::new() {
+				Ok(el) => el,
+				Err(e) => {
+					tx.send(Err(Error::Internal(format!("error initializing event loop: {}", e)))).expect("Rx is blocking upper thread.");
+					return;
+				},
+			};
+			
+			let cluster = ClusterCore::new(el.handle(), config);
+			let cluster_client = cluster.and_then(|c| c.run().map(|_| c.client()));
+			tx.send(cluster_client.map_err(Into::into)).expect("Rx is blocking upper thread.");
+			let _ = el.run(futures::empty().select(stopped));
+		});
+		let cluster = rx.recv().map_err(|e| Error::Internal(format!("error initializing event loop: {}", e)))??;
+
+		Ok(KeyServerCore {
+			close: Some(stop),
+			handle: Some(handle),
+			cluster: Some(cluster),
+		})
+	}
+}
+
+impl Drop for KeyServerCore {
+	fn drop(&mut self) {
+		self.close.take().map(|v| v.send(()));
+		self.handle.take().map(|h| h.join());
+	}
+}
+
 #[cfg(test)]
 mod tests {
-	use std::str::FromStr;
+	use std::time;
+	use std::sync::Arc;
 	use ethcrypto;
-	use ethkey::{self, Secret};
+	use ethkey::{self, Random, Generator};
 	use acl_storage::DummyAclStorage;
-	use key_storage::KeyStorage;
 	use key_storage::tests::DummyKeyStorage;
-	use super::super::{Error, RequestSignature, DocumentAddress};
+	use types::all::{ClusterConfiguration, NodeAddress, EncryptionConfiguration, DocumentEncryptedKey, DocumentKey};
+	use super::super::{RequestSignature, DocumentAddress};
 	use super::{KeyServer, KeyServerImpl};
 
 	const DOCUMENT1: &'static str = "0000000000000000000000000000000000000000000000000000000000000001";
-	const DOCUMENT2: &'static str = "0000000000000000000000000000000000000000000000000000000000000002";
-	const KEY1: &'static str = "key1";
 	const PRIVATE1: &'static str = "03055e18a8434dcc9061cc1b81c4ef84dc7cf4574d755e52cdcf0c8898b25b11";
-	const PUBLIC2: &'static str = "dfe62f56bb05fbd85b485bac749f3410309e24b352bac082468ce151e9ddb94fa7b5b730027fe1c7c5f3d5927621d269f91aceb5caa3c7fe944677a22f88a318";
-	const PRIVATE2: &'static str = "0eb3816f4f705fa0fd952fb27b71b8c0606f09f4743b5b65cbc375bd569632f2";
-
-	fn create_key_server() -> KeyServerImpl<DummyAclStorage, DummyKeyStorage> {
-		let acl_storage = DummyAclStorage::default();
-		let key_storage = DummyKeyStorage::default();
-		key_storage.insert(DOCUMENT1.into(), KEY1.into()).unwrap();
-		acl_storage.prohibit(PUBLIC2.into(), DOCUMENT1.into());
-		KeyServerImpl::new(acl_storage, key_storage)
-	}
 
 	fn make_signature(secret: &str, document: &'static str) -> RequestSignature {
-		let secret = Secret::from_str(secret).unwrap();
+		let secret = secret.parse().unwrap();
 		let document: DocumentAddress = document.into();
 		ethkey::sign(&secret, &document).unwrap()
 	}
 
+	fn decrypt_document_key(secret: &str, document_key: DocumentEncryptedKey) -> DocumentKey {
+		let secret = secret.parse().unwrap();
+		ethcrypto::ecies::decrypt_single_message(&secret, &document_key).unwrap()
+	}
+
 	#[test]
-	fn document_key_succeeds() {
+	fn document_key_generation_and_retrievement_works_over_network() {
-		let key_server = create_key_server();
+		//::util::log::init_log();
+
+		let num_nodes = 3;
+		let key_pairs: Vec<_> = (0..num_nodes).map(|_| Random.generate().unwrap()).collect();
+		let configs: Vec<_> = (0..num_nodes).map(|i| ClusterConfiguration {
+				threads: 1,
+				self_private: (***key_pairs[i].secret()).into(),
+				listener_address: NodeAddress {
+					address: "127.0.0.1".into(),
+					port: 6060 + (i as u16),
+				},
+				nodes: key_pairs.iter().enumerate().map(|(j, kp)| (kp.public().clone(),
+					NodeAddress {
+						address: "127.0.0.1".into(),
+						port: 6060 + (j as u16),
+					})).collect(),
+				allow_connecting_to_higher_nodes: false,
+				encryption_config: EncryptionConfiguration {
+					key_check_timeout_ms: 10,
+				},
+			}).collect();
+		let key_servers: Vec<_> = configs.into_iter().map(|cfg|
+			KeyServerImpl::new(&cfg, Arc::new(DummyAclStorage::default()), Arc::new(DummyKeyStorage::default())).unwrap()
+		).collect();
+
+		// wait until connections are established
+		let start = time::Instant::now();
+		loop {
+			if key_servers.iter().all(|ks| ks.cluster().cluster_state().connected.len() == num_nodes - 1) {
+				break;
+			}
+			if time::Instant::now() - start > time::Duration::from_millis(30000) {
+				panic!("connections are not established in 30000ms");
+			}
+		}
+
+		let test_cases = [0, 1, 2];
+		for threshold in &test_cases {
+			// generate document key
+			// TODO: it is an error that we can regenerate key for the same DOCUMENT
 			let signature = make_signature(PRIVATE1, DOCUMENT1);
-		let document_key = key_server.document_key(&signature, &DOCUMENT1.into()).unwrap();
+			let generated_key = key_servers[0].generate_document_key(&signature, &DOCUMENT1.into(), *threshold).unwrap();
-		let document_key = ethcrypto::ecies::decrypt_single_message(&Secret::from_str(PRIVATE1).unwrap(), &document_key);
+			let generated_key = decrypt_document_key(PRIVATE1, generated_key);
-		assert_eq!(document_key, Ok(KEY1.into()));
-	}
 
-	#[test]
+			// now let's try to retrieve key back
-	fn document_key_fails_when_bad_signature() {
+			for key_server in key_servers.iter() {
-		let key_server = create_key_server();
+				let retrieved_key = key_server.document_key(&signature, &DOCUMENT1.into()).unwrap();
-		let signature = RequestSignature::default();
+				let retrieved_key = decrypt_document_key(PRIVATE1, retrieved_key);
-		let document_key = key_server.document_key(&signature, &DOCUMENT1.into());
+				assert_eq!(retrieved_key, generated_key);
-		assert_eq!(document_key, Err(Error::BadSignature));
+			}
-	}
+		}
-
-	#[test]
-	fn document_key_fails_when_acl_check_fails() {
-		let key_server = create_key_server();
-		let signature = make_signature(PRIVATE2, DOCUMENT1);
-		let document_key = key_server.document_key(&signature, &DOCUMENT1.into());
-		assert_eq!(document_key, Err(Error::AccessDenied));
-	}
-
-	#[test]
-	fn document_key_fails_when_document_not_found() {
-		let key_server = create_key_server();
-		let signature = make_signature(PRIVATE1, DOCUMENT2);
-		let document_key = key_server.document_key(&signature, &DOCUMENT2.into());
-		assert_eq!(document_key, Err(Error::DocumentNotFound));
 	}
 }

secret_store/src/key_server_cluster/cluster.rs

@@ -14,11 +14,42 @@
 // You should have received a copy of the GNU General Public License
 // along with Parity.  If not, see <http://www.gnu.org/licenses/>.
 
-use key_server_cluster::{Error, NodeId};
+use std::io;
-use key_server_cluster::message::Message;
+use std::time;
+use std::sync::Arc;
+use std::collections::{BTreeMap, BTreeSet, VecDeque};
+use std::collections::btree_map::Entry;
+use std::net::{SocketAddr, IpAddr};
+use futures::{finished, failed, Future, Stream, BoxFuture};
+use futures_cpupool::CpuPool;
+use parking_lot::{RwLock, Mutex};
+use tokio_core::io::IoFuture;
+use tokio_core::reactor::{Handle, Remote, Timeout, Interval};
+use tokio_core::net::{TcpListener, TcpStream};
+use ethkey::{Secret, KeyPair, Signature, Random, Generator};
+use key_server_cluster::{Error, NodeId, SessionId, EncryptionConfiguration, AclStorage, KeyStorage};
+use key_server_cluster::message::{self, Message, ClusterMessage, EncryptionMessage, DecryptionMessage};
+use key_server_cluster::decryption_session::{SessionImpl as DecryptionSessionImpl, DecryptionSessionId,
+	SessionParams as DecryptionSessionParams, Session as DecryptionSession};
+use key_server_cluster::encryption_session::{SessionImpl as EncryptionSessionImpl, SessionState as EncryptionSessionState,
+	SessionParams as EncryptionSessionParams, Session as EncryptionSession};
+use key_server_cluster::io::{DeadlineStatus, ReadMessage, SharedTcpStream, read_encrypted_message, WriteMessage, write_encrypted_message};
+use key_server_cluster::net::{accept_connection as net_accept_connection, connect as net_connect, Connection as NetConnection};
+
+pub type BoxedEmptyFuture = BoxFuture<(), ()>;
+
+/// Cluster interface for external clients.
+pub trait ClusterClient: Send + Sync {
+	/// Get cluster state.
+	fn cluster_state(&self) -> ClusterState;
+	/// Start new encryption session.
+	fn new_encryption_session(&self, session_id: SessionId, threshold: usize) -> Result<Arc<EncryptionSession>, Error>;
+	/// Start new decryption session.
+	fn new_decryption_session(&self, session_id: SessionId, requestor_signature: Signature) -> Result<Arc<DecryptionSession>, Error>;
+}
 
 /// Cluster access for single encryption/decryption participant.
-pub trait Cluster {
+pub trait Cluster: Send + Sync {
 	/// Broadcast message to all other nodes.
 	fn broadcast(&self, message: Message) -> Result<(), Error>;
 	/// Send message to given node.
@@ -27,13 +58,841 @@ pub trait Cluster {
 	fn blacklist(&self, node: &NodeId);
 }
 
+#[derive(Clone)]
+/// Cluster initialization parameters.
+pub struct ClusterConfiguration {
+	/// Number of threads reserved by cluster.
+	pub threads: usize,
+	/// Allow connecting to 'higher' nodes.
+	pub allow_connecting_to_higher_nodes: bool,
+	/// KeyPair this node holds.
+	pub self_key_pair: KeyPair,
+	/// Interface to listen to.
+	pub listen_address: (String, u16),
+	/// Cluster nodes.
+	pub nodes: BTreeMap<NodeId, (String, u16)>,
+	/// Encryption session configuration.
+	pub encryption_config: EncryptionConfiguration,
+	/// Reference to key storage
+	pub key_storage: Arc<KeyStorage>,
+	/// Reference to ACL storage
+	pub acl_storage: Arc<AclStorage>,
+}
+
+/// Cluster state.
+pub struct ClusterState {
+	/// Nodes, to which connections are established.
+	pub connected: BTreeSet<NodeId>,
+}
+
+/// Network cluster implementation.
+pub struct ClusterCore {
+	/// Handle to the event loop.
+	handle: Handle,
+	/// Listen address.
+	listen_address: SocketAddr,
+	/// Cluster data.
+	data: Arc<ClusterData>,
+}
+
+/// Network cluster client interface implementation.
+pub struct ClusterClientImpl {
+	/// Cluster data.
+	data: Arc<ClusterData>,
+}
+
+/// Network cluster view. It is a communication channel, required in single session.
+pub struct ClusterView {
+	core: Arc<Mutex<ClusterViewCore>>,
+}
+
+/// Cross-thread shareable cluster data.
+pub struct ClusterData {
+	/// Cluster configuration.
+	config: ClusterConfiguration,
+	/// Handle to the event loop.
+	handle: Remote,
+	/// Handle to the cpu thread pool.
+	pool: CpuPool,
+	/// KeyPair this node holds.
+	self_key_pair: KeyPair,
+	/// Connections data.
+	connections: ClusterConnections,
+	/// Active sessions data.
+	sessions: ClusterSessions,
+}
+
+/// Connections that are forming the cluster.
+pub struct ClusterConnections {
+	/// Self node id.
+	pub self_node_id: NodeId,
+	/// All known other key servers.
+	pub nodes: BTreeMap<NodeId, SocketAddr>,
+	/// Active connections to key servers.
+	pub connections: RwLock<BTreeMap<NodeId, Arc<Connection>>>,
+}
+
+/// Active sessions on this cluster.
+pub struct ClusterSessions {
+	/// Self node id.
+	pub self_node_id: NodeId,
+	/// Reference to key storage
+	pub key_storage: Arc<KeyStorage>,
+	/// Reference to ACL storage
+	pub acl_storage: Arc<AclStorage>,
+	/// Active encryption sessions.
+	pub encryption_sessions: RwLock<BTreeMap<SessionId, QueuedEncryptionSession>>,
+	/// Active decryption sessions.
+	pub decryption_sessions: RwLock<BTreeMap<DecryptionSessionId, QueuedDecryptionSession>>,
+}
+
+/// Encryption session and its message queue.
+pub struct QueuedEncryptionSession {
+	/// Encryption session.
+	pub session: Arc<EncryptionSessionImpl>,
+	/// Messages queue.
+	pub queue: VecDeque<(NodeId, EncryptionMessage)>,
+}
+
+/// Decryption session and its message queue.
+pub struct QueuedDecryptionSession {
+	/// Decryption session.
+	pub session: Arc<DecryptionSessionImpl>,
+	/// Messages queue.
+	pub queue: VecDeque<(NodeId, DecryptionMessage)>,
+}
+
+/// Cluster view core.
+struct ClusterViewCore {
+	/// Cluster reference.
+	cluster: Arc<ClusterData>,
+	/// Subset of nodes, required for this session.
+	nodes: BTreeSet<NodeId>,
+}
+
+/// Connection to single node.
+pub struct Connection {
+	/// Node id.
+	node_id: NodeId,
+	/// Node address.
+	node_address: SocketAddr,
+	/// Is inbound connection?
+	is_inbound: bool,
+	/// Tcp stream.
+	stream: SharedTcpStream,
+	/// Connection key.
+	key: Secret,
+	/// Last message time.
+	last_message_time: Mutex<time::Instant>,
+}
+
+impl ClusterCore {
+	pub fn new(handle: Handle, config: ClusterConfiguration) -> Result<Arc<Self>, Error> {
+		let listen_address = make_socket_address(&config.listen_address.0, config.listen_address.1)?;
+		let connections = ClusterConnections::new(&config)?;
+		let sessions = ClusterSessions::new(&config);
+		let data = ClusterData::new(&handle, config, connections, sessions);
+
+		Ok(Arc::new(ClusterCore {
+			handle: handle,
+			listen_address: listen_address,
+			data: data,
+		}))
+	}
+
+	/// Create new client interface.
+	pub fn client(&self) -> Arc<ClusterClient> {
+		Arc::new(ClusterClientImpl::new(self.data.clone()))
+	}
+
+	#[cfg(test)]
+	/// Get cluster configuration.
+	pub fn config(&self) -> &ClusterConfiguration {
+		&self.data.config
+	}
+
+	#[cfg(test)]
+	/// Get connection to given node.
+	pub fn connection(&self, node: &NodeId) -> Option<Arc<Connection>> {
+		self.data.connection(node)
+	}
+
+	/// Run cluster
+	pub fn run(&self) -> Result<(), Error> {
+		// try to connect to every other peer
+		ClusterCore::connect_disconnected_nodes(self.data.clone());
+
+		// schedule maintain procedures
+		ClusterCore::schedule_maintain(&self.handle, self.data.clone());
+
+		// start listening for incoming connections
+		self.handle.spawn(ClusterCore::listen(&self.handle, self.data.clone(), self.listen_address.clone())?);
+
+		Ok(())
+	}
+
+	/// Connect to peer.
+	fn connect(data: Arc<ClusterData>, node_address: SocketAddr) {
+		data.handle.clone().spawn(move |handle| {
+			data.pool.clone().spawn(ClusterCore::connect_future(handle, data, node_address))
+		})
+	}
+
+	/// Connect to socket using given context and handle.
+	fn connect_future(handle: &Handle, data: Arc<ClusterData>, node_address: SocketAddr) -> BoxedEmptyFuture {
+		let disconnected_nodes = data.connections.disconnected_nodes().keys().cloned().collect();
+		net_connect(&node_address, handle, data.self_key_pair.clone(), disconnected_nodes)
+			.then(move |result| ClusterCore::process_connection_result(data, false, result))
+			.then(|_| finished(()))
+			.boxed()
+	}
+
+	/// Start listening for incoming connections.
+	fn listen(handle: &Handle, data: Arc<ClusterData>, listen_address: SocketAddr) -> Result<BoxedEmptyFuture, Error> {
+		Ok(TcpListener::bind(&listen_address, &handle)?
+			.incoming()
+			.and_then(move |(stream, node_address)| {
+				ClusterCore::accept_connection(data.clone(), stream, node_address);
+				Ok(())
+			})
+			.for_each(|_| Ok(()))
+			.then(|_| finished(()))
+			.boxed())
+	}
+
+	/// Accept connection.
+	fn accept_connection(data: Arc<ClusterData>, stream: TcpStream, node_address: SocketAddr) {
+		data.handle.clone().spawn(move |handle| {
+			data.pool.clone().spawn(ClusterCore::accept_connection_future(handle, data, stream, node_address))
+		})
+	}
+
+	/// Accept connection future.
+	fn accept_connection_future(handle: &Handle, data: Arc<ClusterData>, stream: TcpStream, node_address: SocketAddr) -> BoxedEmptyFuture {
+		let disconnected_nodes = data.connections.disconnected_nodes().keys().cloned().collect();
+		net_accept_connection(node_address, stream, handle, data.self_key_pair.clone(), disconnected_nodes)
+			.then(move |result| ClusterCore::process_connection_result(data, true, result))
+			.then(|_| finished(()))
+			.boxed()
+	}
+
+	/// Schedule mainatain procedures.
+	fn schedule_maintain(handle: &Handle, data: Arc<ClusterData>) {
+		// TODO: per-session timeouts (node can respond to messages, but ignore sessions messages)
+		let (d1, d2, d3) = (data.clone(), data.clone(), data.clone());
+		let interval: BoxedEmptyFuture = Interval::new(time::Duration::new(10, 0), handle)
+			.expect("failed to create interval")
+			.and_then(move |_| Ok(trace!(target: "secretstore_net", "{}: executing maintain procedures", d1.self_key_pair.public())))
+			.and_then(move |_| Ok(ClusterCore::keep_alive(d2.clone())))
+			.and_then(move |_| Ok(ClusterCore::connect_disconnected_nodes(d3.clone())))
+			.for_each(|_| Ok(()))
+			.then(|_| finished(()))
+			.boxed();
+
+		data.spawn(interval);
+	}
+
+	/// Called for every incomming mesage.
+	fn process_connection_messages(data: Arc<ClusterData>, connection: Arc<Connection>) -> IoFuture<Result<(), Error>> {
+		connection
+			.read_message()
+			.then(move |result|
+				match result {
+					Ok((_, Ok(message))) => {
+						ClusterCore::process_connection_message(data.clone(), connection.clone(), message);
+						// continue serving connection
+						data.spawn(ClusterCore::process_connection_messages(data.clone(), connection));
+						finished(Ok(())).boxed()
+					},
+					Ok((_, Err(err))) => {
+						warn!(target: "secretstore_net", "{}: protocol error {} when reading message from node {}", data.self_key_pair.public(), err, connection.node_id());
+						// continue serving connection
+						data.spawn(ClusterCore::process_connection_messages(data.clone(), connection));
+						finished(Err(err)).boxed()
+					},
+					Err(err) => {
+						warn!(target: "secretstore_net", "{}: network error {} when reading message from node {}", data.self_key_pair.public(), err, connection.node_id());
+						// close connection
+						data.connections.remove(connection.node_id(), connection.is_inbound());
+						failed(err).boxed()
+					},
+				}
+			).boxed()
+	}
+
+	/// Send keepalive messages to every othe node.
+	fn keep_alive(data: Arc<ClusterData>) {
+		for connection in data.connections.active_connections() {
+			let last_message_diff = time::Instant::now() - connection.last_message_time();
+			if last_message_diff > time::Duration::from_secs(60) {
+				data.connections.remove(connection.node_id(), connection.is_inbound());
+				data.sessions.on_connection_timeout(connection.node_id());
+			}
+			else if last_message_diff > time::Duration::from_secs(30) {
+				data.spawn(connection.send_message(Message::Cluster(ClusterMessage::KeepAlive(message::KeepAlive {}))));
+			}
+		}
+	}
+
+	/// Try to connect to every disconnected node.
+	fn connect_disconnected_nodes(data: Arc<ClusterData>) {
+		for (node_id, node_address) in data.connections.disconnected_nodes() {
+			if data.config.allow_connecting_to_higher_nodes || data.self_key_pair.public() < &node_id {
+				ClusterCore::connect(data.clone(), node_address);
+			}
+		}
+	}
+
+	/// Process connection future result.
+	fn process_connection_result(data: Arc<ClusterData>, is_inbound: bool, result: Result<DeadlineStatus<Result<NetConnection, Error>>, io::Error>) -> IoFuture<Result<(), Error>> {
+		match result {
+			Ok(DeadlineStatus::Meet(Ok(connection))) => {
+				let connection = Connection::new(is_inbound, connection);
+				if data.connections.insert(connection.clone()) {
+					ClusterCore::process_connection_messages(data.clone(), connection)
+				} else {
+					finished(Ok(())).boxed()
+				}
+			},
+			Ok(DeadlineStatus::Meet(Err(_))) => {
+				finished(Ok(())).boxed()
+			},
+			Ok(DeadlineStatus::Timeout) => {
+				finished(Ok(())).boxed()
+			},
+			Err(_) => {
+				// network error
+				finished(Ok(())).boxed()
+			},
+		}
+	}
+
+	/// Process single message from the connection.
+	fn process_connection_message(data: Arc<ClusterData>, connection: Arc<Connection>, message: Message) {
+		connection.set_last_message_time(time::Instant::now());
+		trace!(target: "secretstore_net", "{}: processing message {} from {}", data.self_key_pair.public(), message, connection.node_id());
+		match message {
+			Message::Encryption(message) => ClusterCore::process_encryption_message(data, connection, message),
+			Message::Decryption(message) => ClusterCore::process_decryption_message(data, connection, message),
+			Message::Cluster(message) => ClusterCore::process_cluster_message(data, connection, message),
+		}
+	}
+
+	/// Process single encryption message from the connection.
+	fn process_encryption_message(data: Arc<ClusterData>, connection: Arc<Connection>, mut message: EncryptionMessage) {
+		let mut sender = connection.node_id().clone();
+		let mut is_queued_message = false;
+		let session_id = message.session_id().clone();
+		let key_check_timeout_ms = data.config.encryption_config.key_check_timeout_ms;
+		loop {
+			let result = match message {
+				EncryptionMessage::InitializeSession(ref message) => {
+					let mut connected_nodes = data.connections.connected_nodes();
+					connected_nodes.insert(data.self_key_pair.public().clone());
+
+					let cluster = Arc::new(ClusterView::new(data.clone(), connected_nodes));
+					let session_id: SessionId = message.session.clone().into();
+					data.sessions.new_encryption_session(sender.clone(), session_id.clone(), cluster)
+						.and_then(|s| s.on_initialize_session(sender.clone(), message))
+				},
+				EncryptionMessage::ConfirmInitialization(ref message) => data.sessions.encryption_session(&*message.session)
+					.ok_or(Error::InvalidSessionId)
+					.and_then(|s| s.on_confirm_initialization(sender.clone(), message)),
+				EncryptionMessage::CompleteInitialization(ref message) => data.sessions.encryption_session(&*message.session)
+					.ok_or(Error::InvalidSessionId)
+					.and_then(|s| s.on_complete_initialization(sender.clone(), message)),
+				EncryptionMessage::KeysDissemination(ref message) => data.sessions.encryption_session(&*message.session)
+					.ok_or(Error::InvalidSessionId)
+					.and_then(|s| {
+						// TODO: move this logic to session (or session connector)
+						let is_in_key_check_state = s.state() == EncryptionSessionState::KeyCheck;
+						let result = s.on_keys_dissemination(sender.clone(), message);
+						if !is_in_key_check_state && s.state() == EncryptionSessionState::KeyCheck {
+							let session = s.clone();
+							let d = data.clone();
+							data.handle.spawn(move |handle|
+								Timeout::new(time::Duration::new(key_check_timeout_ms / 1000, 0), handle)
+									.expect("failed to create timeout")
+									.and_then(move |_| {
+										if let Err(error) = session.start_key_generation_phase() {
+											session.on_session_error(d.self_key_pair.public().clone(), &message::SessionError {
+												session: session.id().clone().into(),
+												error: error.into(),
+											});
+										}
+										Ok(())
+									})
+									.then(|_| finished(()))
+							);
+						}
+
+						result
+					}),
+				EncryptionMessage::Complaint(ref message) => data.sessions.encryption_session(&*message.session)
+					.ok_or(Error::InvalidSessionId)
+					.and_then(|s| s.on_complaint(sender.clone(), message)),
+				EncryptionMessage::ComplaintResponse(ref message) => data.sessions.encryption_session(&*message.session)
+					.ok_or(Error::InvalidSessionId)
+					.and_then(|s| s.on_complaint_response(sender.clone(), message)),
+				EncryptionMessage::PublicKeyShare(ref message) => data.sessions.encryption_session(&*message.session)
+					.ok_or(Error::InvalidSessionId)
+					.and_then(|s| s.on_public_key_share(sender.clone(), message)),
+				EncryptionMessage::SessionError(ref message) => {
+						if let Some(s) = data.sessions.encryption_session(&*message.session) {
+							data.sessions.remove_encryption_session(s.id());
+							s.on_session_error(sender.clone(), message);
+						}
+						Ok(())
+					},
+				EncryptionMessage::SessionCompleted(ref message) => data.sessions.encryption_session(&*message.session)
+					.ok_or(Error::InvalidSessionId)
+					.and_then(|s| {
+						let result = s.on_session_completed(sender.clone(), message);
+						if result.is_ok() && s.state() == EncryptionSessionState::Finished {
+							data.sessions.remove_encryption_session(s.id());
+						}
+
+						result
+					}),
+			};
+
+			match result {
+				Err(Error::TooEarlyForRequest) => {
+					data.sessions.enqueue_encryption_message(&session_id, sender, message, is_queued_message);
+					break;
+				},
+				Err(err) => {
+					warn!(target: "secretstore_net", "{}: error {} when processing message {} from node {}", data.self_key_pair.public(), err, message, sender);
+					if let Some(connection) = data.connections.get(&sender) {
+						data.spawn(connection.send_message(Message::Encryption(EncryptionMessage::SessionError(message::SessionError {
+							session: session_id.clone().into(),
+							error: format!("{:?}", err),
+						}))));
+					}
+
+					if err != Error::InvalidSessionId {
+						data.sessions.remove_encryption_session(&session_id);
+					}
+					break;
+				},
+				_ => {
+					match data.sessions.dequeue_encryption_message(&session_id) {
+						Some((msg_sender, msg)) => {
+							is_queued_message = true;
+							sender = msg_sender;
+							message = msg;
+						},
+						None => break,
+					}
+				},
+			}
+		}
+	}
+
+	/// Process single decryption message from the connection.
+	fn process_decryption_message(data: Arc<ClusterData>, connection: Arc<Connection>, mut message: DecryptionMessage) {
+		let mut sender = connection.node_id().clone();
+		let mut is_queued_message = false;
+		let session_id = message.session_id().clone();
+		let sub_session_id = message.sub_session_id().clone();
+		loop {
+			let result = match message {
+				DecryptionMessage::InitializeDecryptionSession(ref message) => {
+					let mut connected_nodes = data.connections.connected_nodes();
+					connected_nodes.insert(data.self_key_pair.public().clone());
+
+					let cluster = Arc::new(ClusterView::new(data.clone(), connected_nodes));
+					data.sessions.new_decryption_session(sender.clone(), session_id.clone(), sub_session_id.clone(), cluster)
+						.and_then(|s| s.on_initialize_session(sender.clone(), message))
+				},
+				DecryptionMessage::ConfirmDecryptionInitialization(ref message) => data.sessions.decryption_session(&*message.session, &*message.sub_session)
+					.ok_or(Error::InvalidSessionId)
+					.and_then(|s| s.on_confirm_initialization(sender.clone(), message)),
+				DecryptionMessage::RequestPartialDecryption(ref message) => data.sessions.decryption_session(&*message.session, &*message.sub_session)
+					.ok_or(Error::InvalidSessionId)
+					.and_then(|s| s.on_partial_decryption_requested(sender.clone(), message)),
+				DecryptionMessage::PartialDecryption(ref message) => data.sessions.decryption_session(&*message.session, &*message.sub_session)
+					.ok_or(Error::InvalidSessionId)
+					.and_then(|s| s.on_partial_decryption(sender.clone(), message)),
+				DecryptionMessage::DecryptionSessionError(ref message) => {
+						if let Some(s) = data.sessions.decryption_session(&*message.session, &*message.sub_session) {
+							data.sessions.remove_decryption_session(&session_id, &sub_session_id);
+							s.on_session_error(sender.clone(), message);
+						}
+						Ok(())
+					},
+			};
+
+			match result {
+				Err(Error::TooEarlyForRequest) => {
+					data.sessions.enqueue_decryption_message(&session_id, &sub_session_id, sender, message, is_queued_message);
+					break;
+				},
+				Err(err) => {
+					if let Some(connection) = data.connections.get(&sender) {
+						data.spawn(connection.send_message(Message::Decryption(DecryptionMessage::DecryptionSessionError(message::DecryptionSessionError {
+							session: session_id.clone().into(),
+							sub_session: sub_session_id.clone().into(),
+							error: format!("{:?}", err),
+						}))));
+					}
+
+					if err != Error::InvalidSessionId {
+						data.sessions.remove_decryption_session(&session_id, &sub_session_id);
+					}
+					break;
+				},
+				_ => {
+					match data.sessions.dequeue_decryption_message(&session_id, &sub_session_id) {
+						Some((msg_sender, msg)) => {
+							is_queued_message = true;
+							sender = msg_sender;
+							message = msg;
+						},
+						None => break,
+					}
+				},
+			}
+		}
+	}
+
+	/// Process single cluster message from the connection.
+	fn process_cluster_message(data: Arc<ClusterData>, connection: Arc<Connection>, message: ClusterMessage) {
+		match message {
+			ClusterMessage::KeepAlive(_) => data.spawn(connection.send_message(Message::Cluster(ClusterMessage::KeepAliveResponse(message::KeepAliveResponse {})))),
+			ClusterMessage::KeepAliveResponse(_) => (),
+			_ => warn!(target: "secretstore_net", "{}: received unexpected message {} from node {} at {}", data.self_key_pair.public(), message, connection.node_id(), connection.node_address()),
+		}
+	}
+}
+
+impl ClusterConnections {
+	pub fn new(config: &ClusterConfiguration) -> Result<Self, Error> {
+		let mut connections = ClusterConnections {
+			self_node_id: config.self_key_pair.public().clone(),
+			nodes: BTreeMap::new(),
+			connections: RwLock::new(BTreeMap::new()),
+		};
+
+		for (node_id, &(ref node_addr, node_port)) in config.nodes.iter().filter(|&(node_id, _)| node_id != config.self_key_pair.public()) {
+			let socket_address = make_socket_address(&node_addr, node_port)?;
+			connections.nodes.insert(node_id.clone(), socket_address);
+		}
+
+		Ok(connections)
+	}
+
+	pub fn cluster_state(&self) -> ClusterState {
+		ClusterState {
+			connected: self.connections.read().keys().cloned().collect(),
+		}
+	}
+
+	pub fn get(&self, node: &NodeId) -> Option<Arc<Connection>> {
+		self.connections.read().get(node).cloned()
+	}
+
+	pub fn insert(&self, connection: Arc<Connection>) -> bool {
+		let mut connections = self.connections.write();
+		if connections.contains_key(connection.node_id()) {
+			// we have already connected to the same node
+			// the agreement is that node with lower id must establish connection to node with higher id
+			if (&self.self_node_id < connection.node_id() && connection.is_inbound())
+				|| (&self.self_node_id > connection.node_id() && !connection.is_inbound()) {
+				return false;
+			}
+		}
+		trace!(target: "secretstore_net", "{}: inserting connection to {} at {}", self.self_node_id, connection.node_id(), connection.node_address());
+		connections.insert(connection.node_id().clone(), connection);
+		true
+	}
+
+	pub fn remove(&self, node: &NodeId, is_inbound: bool) {
+		let mut connections = self.connections.write();
+		if let Entry::Occupied(entry) = connections.entry(node.clone()) {
+			if entry.get().is_inbound() != is_inbound {
+				return;
+			}
+
+			trace!(target: "secretstore_net", "{}: removing connection to {} at {}", self.self_node_id, entry.get().node_id(), entry.get().node_address());
+			entry.remove_entry();
+		}
+	}
+
+	pub fn connected_nodes(&self) -> BTreeSet<NodeId> {
+		self.connections.read().keys().cloned().collect()
+	}
+
+	pub fn active_connections(&self)-> Vec<Arc<Connection>> {
+		self.connections.read().values().cloned().collect()
+	}
+
+	pub fn disconnected_nodes(&self) -> BTreeMap<NodeId, SocketAddr> {
+		let connections = self.connections.read();
+		self.nodes.iter()
+			.filter(|&(node_id, _)| !connections.contains_key(node_id))
+			.map(|(node_id, node_address)| (node_id.clone(), node_address.clone()))
+			.collect()
+	}
+}
+
+impl ClusterSessions {
+	pub fn new(config: &ClusterConfiguration) -> Self {
+		ClusterSessions {
+			self_node_id: config.self_key_pair.public().clone(),
+			acl_storage: config.acl_storage.clone(),
+			key_storage: config.key_storage.clone(),
+			encryption_sessions: RwLock::new(BTreeMap::new()),
+			decryption_sessions: RwLock::new(BTreeMap::new()),
+		}
+	}
+
+	pub fn new_encryption_session(&self, _master: NodeId, session_id: SessionId, cluster: Arc<Cluster>) -> Result<Arc<EncryptionSessionImpl>, Error> {
+		let mut encryption_sessions = self.encryption_sessions.write();
+		if encryption_sessions.contains_key(&session_id) {
+			return Err(Error::DuplicateSessionId);
+		}
+
+		let session = Arc::new(EncryptionSessionImpl::new(EncryptionSessionParams {
+			id: session_id.clone(),
+			self_node_id: self.self_node_id.clone(),
+			key_storage: self.key_storage.clone(),
+			cluster: cluster,
+		}));
+		let encryption_session = QueuedEncryptionSession {
+			session: session.clone(),
+			queue: VecDeque::new()
+		};
+		encryption_sessions.insert(session_id, encryption_session);
+		Ok(session)
+	}
+
+	pub fn remove_encryption_session(&self, session_id: &SessionId) {
+		self.encryption_sessions.write().remove(session_id);
+	}
+
+	pub fn encryption_session(&self, session_id: &SessionId) -> Option<Arc<EncryptionSessionImpl>> {
+		self.encryption_sessions.read().get(session_id).map(|s| s.session.clone())
+	}
+
+	pub fn enqueue_encryption_message(&self, session_id: &SessionId, sender: NodeId, message: EncryptionMessage, is_queued_message: bool) {
+		self.encryption_sessions.write().get_mut(session_id)
+			.map(|session| if is_queued_message { session.queue.push_front((sender, message)) }
+				else { session.queue.push_back((sender, message)) });
+	}
+
+	pub fn dequeue_encryption_message(&self, session_id: &SessionId) -> Option<(NodeId, EncryptionMessage)> {
+		self.encryption_sessions.write().get_mut(session_id)
+			.and_then(|session| session.queue.pop_front())
+	}
+
+	pub fn new_decryption_session(&self, _master: NodeId, session_id: SessionId, sub_session_id: Secret, cluster: Arc<Cluster>) -> Result<Arc<DecryptionSessionImpl>, Error> {
+		let mut decryption_sessions = self.decryption_sessions.write();
+		let session_id = DecryptionSessionId::new(session_id, sub_session_id);
+		if decryption_sessions.contains_key(&session_id) {
+			return Err(Error::DuplicateSessionId);
+		}
+
+		let session = Arc::new(DecryptionSessionImpl::new(DecryptionSessionParams {
+			id: session_id.id.clone(),
+			access_key: session_id.access_key.clone(),
+			self_node_id: self.self_node_id.clone(),
+			encrypted_data: self.key_storage.get(&session_id.id).map_err(|e| Error::KeyStorage(e.into()))?,
+			acl_storage: self.acl_storage.clone(),
+			cluster: cluster,
+		})?);
+		let decryption_session = QueuedDecryptionSession {
+			session: session.clone(),
+			queue: VecDeque::new()
+		};
+		decryption_sessions.insert(session_id, decryption_session);
+		Ok(session)
+	}
+
+	pub fn remove_decryption_session(&self, session_id: &SessionId, sub_session_id: &Secret) {
+		let session_id = DecryptionSessionId::new(session_id.clone(), sub_session_id.clone());
+		self.decryption_sessions.write().remove(&session_id);
+	}
+
+	pub fn decryption_session(&self, session_id: &SessionId, sub_session_id: &Secret) -> Option<Arc<DecryptionSessionImpl>> {
+		let session_id = DecryptionSessionId::new(session_id.clone(), sub_session_id.clone());
+		self.decryption_sessions.read().get(&session_id).map(|s| s.session.clone())
+	}
+
+	pub fn enqueue_decryption_message(&self, session_id: &SessionId, sub_session_id: &Secret, sender: NodeId, message: DecryptionMessage, is_queued_message: bool) {
+		let session_id = DecryptionSessionId::new(session_id.clone(), sub_session_id.clone());
+		self.decryption_sessions.write().get_mut(&session_id)
+			.map(|session| if is_queued_message { session.queue.push_front((sender, message)) }
+				else { session.queue.push_back((sender, message)) });
+	}
+
+	pub fn dequeue_decryption_message(&self, session_id: &SessionId, sub_session_id: &Secret) -> Option<(NodeId, DecryptionMessage)> {
+		let session_id = DecryptionSessionId::new(session_id.clone(), sub_session_id.clone());
+		self.decryption_sessions.write().get_mut(&session_id)
+			.and_then(|session| session.queue.pop_front())
+	}
+
+	pub fn on_connection_timeout(&self, node_id: &NodeId) {
+		for encryption_session in self.encryption_sessions.read().values() {
+			encryption_session.session.on_session_timeout(node_id);
+		}
+		for decryption_session in self.decryption_sessions.read().values() {
+			decryption_session.session.on_session_timeout(node_id);
+		}
+	}
+}
+
+impl ClusterData {
+	pub fn new(handle: &Handle, config: ClusterConfiguration, connections: ClusterConnections, sessions: ClusterSessions) -> Arc<Self> {
+		Arc::new(ClusterData {
+			handle: handle.remote().clone(),
+			pool: CpuPool::new(config.threads),
+			self_key_pair: config.self_key_pair.clone(),
+			connections: connections,
+			sessions: sessions,
+			config: config,
+		})
+	}
+
+	/// Get connection to given node.
+	pub fn connection(&self, node: &NodeId) -> Option<Arc<Connection>> {
+		self.connections.get(node)
+	}
+
+	/// Spawns a future using thread pool and schedules execution of it with event loop handle.
+	pub fn spawn<F>(&self, f: F) where F: Future + Send + 'static, F::Item: Send + 'static, F::Error: Send + 'static {
+		let pool_work = self.pool.spawn(f);
+		self.handle.spawn(move |_handle| {
+			pool_work.then(|_| finished(()))
+		})
+	}
+}
+
+impl Connection {
+	pub fn new(is_inbound: bool, connection: NetConnection) -> Arc<Connection> {
+		Arc::new(Connection {
+			node_id: connection.node_id,
+			node_address: connection.address,
+			is_inbound: is_inbound,
+			stream: connection.stream,
+			key: connection.key,
+			last_message_time: Mutex::new(time::Instant::now()),
+		})
+	}
+
+	pub fn is_inbound(&self) -> bool {
+		self.is_inbound
+	}
+
+	pub fn node_id(&self) -> &NodeId {
+		&self.node_id
+	}
+
+	pub fn last_message_time(&self) -> time::Instant {
+		*self.last_message_time.lock()
+	}
+
+	pub fn set_last_message_time(&self, last_message_time: time::Instant) {
+		*self.last_message_time.lock() = last_message_time;
+	}
+
+	pub fn node_address(&self) -> &SocketAddr {
+		&self.node_address
+	}
+
+	pub fn send_message(&self, message: Message) -> WriteMessage<SharedTcpStream> {
+		write_encrypted_message(self.stream.clone(), &self.key, message)
+	}
+
+	pub fn read_message(&self) -> ReadMessage<SharedTcpStream> {
+		read_encrypted_message(self.stream.clone(), self.key.clone())
+	}
+}
+
+impl ClusterView {
+	pub fn new(cluster: Arc<ClusterData>, nodes: BTreeSet<NodeId>) -> Self {
+		ClusterView {
+			core: Arc::new(Mutex::new(ClusterViewCore {
+				cluster: cluster,
+				nodes: nodes,
+			})),
+		}
+	}
+}
+
+impl Cluster for ClusterView {
+	fn broadcast(&self, message: Message) -> Result<(), Error> {
+		let core = self.core.lock();
+		for node in core.nodes.iter().filter(|n| *n != core.cluster.self_key_pair.public()) {
+			let connection = core.cluster.connection(node).ok_or(Error::NodeDisconnected)?;
+			core.cluster.spawn(connection.send_message(message.clone()))
+		}
+		Ok(())
+	}
+
+	fn send(&self, to: &NodeId, message: Message) -> Result<(), Error> {
+		let core = self.core.lock();
+		let connection = core.cluster.connection(to).ok_or(Error::NodeDisconnected)?;
+		core.cluster.spawn(connection.send_message(message));
+		Ok(())
+	}
+
+	fn blacklist(&self, _node: &NodeId) {
+		// TODO: unimplemented!()
+	}
+}
+
+impl ClusterClientImpl {
+	pub fn new(data: Arc<ClusterData>) -> Self {
+		ClusterClientImpl {
+			data: data,
+		}
+	}
+}
+
+impl ClusterClient for ClusterClientImpl {
+	fn cluster_state(&self) -> ClusterState {
+		self.data.connections.cluster_state()
+	}
+
+	fn new_encryption_session(&self, session_id: SessionId, threshold: usize) -> Result<Arc<EncryptionSession>, Error> {
+		let mut connected_nodes = self.data.connections.connected_nodes();
+		connected_nodes.insert(self.data.self_key_pair.public().clone());
+
+		let cluster = Arc::new(ClusterView::new(self.data.clone(), connected_nodes.clone()));
+		let session = self.data.sessions.new_encryption_session(self.data.self_key_pair.public().clone(), session_id, cluster)?;
+		session.initialize(threshold, connected_nodes)?;
+		Ok(session)
+	}
+
+	fn new_decryption_session(&self, session_id: SessionId, requestor_signature: Signature) -> Result<Arc<DecryptionSession>, Error> {
+		let mut connected_nodes = self.data.connections.connected_nodes();
+		connected_nodes.insert(self.data.self_key_pair.public().clone());
+
+		let access_key = Random.generate()?.secret().clone();
+		let cluster = Arc::new(ClusterView::new(self.data.clone(), connected_nodes.clone()));
+		let session = self.data.sessions.new_decryption_session(self.data.self_key_pair.public().clone(), session_id, access_key, cluster)?;
+		session.initialize(requestor_signature)?;
+		Ok(session)
+	}
+}
+
+fn make_socket_address(address: &str, port: u16) -> Result<SocketAddr, Error> {
+	let ip_address: IpAddr = address.parse().map_err(|_| Error::InvalidNodeAddress)?;
+	Ok(SocketAddr::new(ip_address, port))
+}
+
 #[cfg(test)]
 pub mod tests {
+	use std::sync::Arc;
+	use std::time;
 	use std::collections::VecDeque;
 	use parking_lot::Mutex;
-	use key_server_cluster::{NodeId, Error};
+	use tokio_core::reactor::Core;
+	use ethkey::{Random, Generator};
+	use key_server_cluster::{NodeId, Error, EncryptionConfiguration, DummyAclStorage, DummyKeyStorage};
 	use key_server_cluster::message::Message;
-	use key_server_cluster::cluster::Cluster;
+	use key_server_cluster::cluster::{Cluster, ClusterCore, ClusterConfiguration};
 
 	#[derive(Debug)]
 	pub struct DummyCluster {
@@ -87,4 +946,61 @@ pub mod tests {
 		fn blacklist(&self, _node: &NodeId) {
 		}
 	}
+
+	pub fn loop_until<F>(core: &mut Core, timeout: time::Duration, predicate: F) where F: Fn() -> bool {
+		let start = time::Instant::now();
+		loop {
+			core.turn(Some(time::Duration::from_millis(1)));
+			if predicate() {
+				break;
+			}
+
+			if time::Instant::now() - start > timeout {
+				panic!("no result in {:?}", timeout);
+			}
+		}
+	}
+
+	pub fn all_connections_established(cluster: &Arc<ClusterCore>) -> bool {
+		cluster.config().nodes.keys()
+			.filter(|p| *p != cluster.config().self_key_pair.public())
+			.all(|p| cluster.connection(p).is_some())
+	}
+
+	pub fn make_clusters(core: &Core, ports_begin: u16, num_nodes: usize) -> Vec<Arc<ClusterCore>> {
+		let key_pairs: Vec<_> = (0..num_nodes).map(|_| Random.generate().unwrap()).collect();
+		let cluster_params: Vec<_> = (0..num_nodes).map(|i| ClusterConfiguration {
+			threads: 1,
+			self_key_pair: key_pairs[i].clone(),
+			listen_address: ("127.0.0.1".to_owned(), ports_begin + i as u16),
+			nodes: key_pairs.iter().enumerate()
+				.map(|(j, kp)| (kp.public().clone(), ("127.0.0.1".into(), ports_begin + j as u16)))
+				.collect(),
+			allow_connecting_to_higher_nodes: false,
+			encryption_config: EncryptionConfiguration {
+				key_check_timeout_ms: 10,
+			},
+			key_storage: Arc::new(DummyKeyStorage::default()),
+			acl_storage: Arc::new(DummyAclStorage::default()),
+		}).collect();
+		let clusters: Vec<_> = cluster_params.into_iter().enumerate()
+			.map(|(_, params)| ClusterCore::new(core.handle(), params).unwrap())
+			.collect();
+
+		clusters
+	}
+
+	pub fn run_clusters(clusters: &[Arc<ClusterCore>]) {
+		for cluster in clusters {
+			cluster.run().unwrap();
+		}
+	}
+
+	#[test]
+	fn cluster_connects_to_other_nodes() {
+		let mut core = Core::new().unwrap();
+		let clusters = make_clusters(&core, 6010, 3);
+		run_clusters(&clusters);
+		loop_until(&mut core, time::Duration::from_millis(300), || clusters.iter().all(all_connections_established));
+	}
 }

secret_store/src/key_server_cluster/decryption_session.rs

@@ -14,15 +14,22 @@
 // You should have received a copy of the GNU General Public License
 // along with Parity.  If not, see <http://www.gnu.org/licenses/>.
 
+use std::cmp::{Ord, PartialOrd, Ordering};
 use std::collections::{BTreeSet, BTreeMap};
 use std::sync::Arc;
-use parking_lot::Mutex;
+use parking_lot::{Mutex, Condvar};
 use ethkey::{self, Secret, Public, Signature};
-use key_server_cluster::{Error, AclStorage, EncryptedData, NodeId, SessionId};
+use key_server_cluster::{Error, AclStorage, DocumentKeyShare, NodeId, SessionId};
 use key_server_cluster::cluster::Cluster;
 use key_server_cluster::math;
-use key_server_cluster::message::{Message, InitializeDecryptionSession, ConfirmDecryptionInitialization,
+use key_server_cluster::message::{Message, DecryptionMessage, InitializeDecryptionSession, ConfirmDecryptionInitialization,
-	RequestPartialDecryption, PartialDecryption};
+	RequestPartialDecryption, PartialDecryption, DecryptionSessionError};
+
+/// Decryption session API.
+pub trait Session: Send + Sync + 'static {
+	/// Wait until session is completed. Returns distributely restored secret key.
+	fn wait(&self) -> Result<Public, Error>;
+}
 
 /// Distributed decryption session.
 /// Based on "ECDKG: A Distributed Key Generation Protocol Based on Elliptic Curve Discrete Logarithm" paper:
@@ -32,7 +39,7 @@ use key_server_cluster::message::{Message, InitializeDecryptionSession, ConfirmD
 /// 2) ACL check: all nodes which have received the request are querying ACL-contract to check if requestor has access to the document
 /// 3) partial decryption: every node which has succussfully checked access for the requestor do a partial decryption
 /// 4) decryption: master node receives all partial decryptions of the secret and restores the secret
-pub struct Session {
+pub struct SessionImpl {
 	/// Encryption session id.
 	id: SessionId,
 	/// Decryption session access key.
@@ -40,25 +47,36 @@ pub struct Session {
 	/// Public identifier of this node.
 	self_node_id: NodeId,
 	/// Encrypted data.
-	encrypted_data: EncryptedData,
+	encrypted_data: DocumentKeyShare,
 	/// ACL storate to check access to the resource.
 	acl_storage: Arc<AclStorage>,
 	/// Cluster which allows this node to send messages to other nodes in the cluster.
 	cluster: Arc<Cluster>,
+	/// SessionImpl completion condvar.
+	completed: Condvar,
 	/// Mutable session data.
 	data: Mutex<SessionData>,
 }
 
-/// Session creation parameters
+/// Decryption session Id.
-pub struct SessionParams {
+#[derive(Debug, Clone, PartialEq, Eq)]
-	/// Session identifier.
+pub struct DecryptionSessionId {
+	/// Encryption session id.
 	pub id: SessionId,
-	/// Session access key.
+	/// Decryption session access key.
+	pub access_key: Secret,
+}
+
+/// SessionImpl creation parameters
+pub struct SessionParams {
+	/// SessionImpl identifier.
+	pub id: SessionId,
+	/// SessionImpl access key.
 	pub access_key: Secret,
 	/// Id of node, on which this session is running.
 	pub self_node_id: Public,
-	/// Encrypted data (result of running encryption_session::Session).
+	/// Encrypted data (result of running encryption_session::SessionImpl).
-	pub encrypted_data: EncryptedData,
+	pub encrypted_data: DocumentKeyShare,
 	/// ACL storage.
 	pub acl_storage: Arc<AclStorage>,
 	/// Cluster
@@ -91,16 +109,11 @@ struct SessionData {
 
 	/// === Values, filled during final decryption ===
 	/// Decrypted secret
-	decrypted_secret: Option<Public>,
+	decrypted_secret: Option<Result<Public, Error>>,
-}
-
-#[derive(Debug)]
-struct NodeData {
-	/// Node-generated shadow point.
-	shadow_point: Option<Public>,
 }
 
 #[derive(Debug, Clone, PartialEq)]
+/// Decryption session data.
 pub enum SessionState {
 	/// Every node starts in this state.
 	WaitingForInitialization,
@@ -116,18 +129,19 @@ pub enum SessionState {
 	Failed,
 }
 
-impl Session {
+impl SessionImpl {
 	/// Create new decryption session.
 	pub fn new(params: SessionParams) -> Result<Self, Error> {
 		check_encrypted_data(&params.self_node_id, &params.encrypted_data)?;
 
-		Ok(Session {
+		Ok(SessionImpl {
 			id: params.id,
 			access_key: params.access_key,
 			self_node_id: params.self_node_id,
 			encrypted_data: params.encrypted_data,
 			acl_storage: params.acl_storage,
 			cluster: params.cluster,
+			completed: Condvar::new(),
 			data: Mutex::new(SessionData {
 				state: SessionState::WaitingForInitialization,
 				master: None,
@@ -146,19 +160,22 @@ impl Session {
 		&self.self_node_id
 	}
 
+	#[cfg(test)]
 	/// Get this session access key.
 	pub fn access_key(&self) -> &Secret {
 		&self.access_key
 	}
 
+	#[cfg(test)]
 	/// Get current session state.
 	pub fn state(&self) -> SessionState {
 		self.data.lock().state.clone()
 	}
 
+	#[cfg(test)]
 	/// Get decrypted secret
 	pub fn decrypted_secret(&self) -> Option<Public> {
-		self.data.lock().decrypted_secret.clone()
+		self.data.lock().decrypted_secret.clone().and_then(|r| r.ok())
 	}
 
 	/// Initialize decryption session.
@@ -188,15 +205,20 @@ impl Session {
 			// not enough nodes => pass initialization message to all other nodes
 			SessionState::WaitingForInitializationConfirm => {
 				for node in self.encrypted_data.id_numbers.keys().filter(|n| *n != self.node()) {
-					self.cluster.send(node, Message::InitializeDecryptionSession(InitializeDecryptionSession {
+					self.cluster.send(node, Message::Decryption(DecryptionMessage::InitializeDecryptionSession(InitializeDecryptionSession {
-							session: self.id.clone(),
+							session: self.id.clone().into(),
-							sub_session: self.access_key.clone(),
+							sub_session: self.access_key.clone().into(),
-							requestor_signature: requestor_signature.clone(),
+							requestor_signature: requestor_signature.clone().into(),
-						}))?;
+						})))?;
 				}
 			},
 			// we can decrypt data on our own
-			SessionState::WaitingForPartialDecryption => unimplemented!(),
+			SessionState::WaitingForPartialDecryption => {
+				data.confirmed_nodes.insert(self.node().clone());
+				SessionImpl::start_waiting_for_partial_decryption(self.node().clone(), self.id.clone(), self.access_key.clone(), &self.cluster, &self.encrypted_data, &mut *data)?;
+				SessionImpl::do_decryption(self.access_key.clone(), &self.encrypted_data, &mut *data)?;
+				self.completed.notify_all();
+			},
 			// we can not decrypt data
 			SessionState::Failed => (),
 			// cannot reach other states
@@ -207,9 +229,9 @@ impl Session {
 	}
 
 	/// When session initialization message is received.
-	pub fn on_initialize_session(&self, sender: NodeId, message: InitializeDecryptionSession) -> Result<(), Error> {
+	pub fn on_initialize_session(&self, sender: NodeId, message: &InitializeDecryptionSession) -> Result<(), Error> {
-		debug_assert!(self.id == message.session);
+		debug_assert!(self.id == *message.session);
-		debug_assert!(self.access_key == message.sub_session);
+		debug_assert!(self.access_key == *message.sub_session);
 		debug_assert!(&sender != self.node());
 
 		let mut data = self.data.lock();
@@ -230,17 +252,17 @@ impl Session {
 
 		// respond to master node
 		data.master = Some(sender.clone());
-		self.cluster.send(&sender, Message::ConfirmDecryptionInitialization(ConfirmDecryptionInitialization {
+		self.cluster.send(&sender, Message::Decryption(DecryptionMessage::ConfirmDecryptionInitialization(ConfirmDecryptionInitialization {
-			session: self.id.clone(),
+			session: self.id.clone().into(),
-			sub_session: self.access_key.clone(),
+			sub_session: self.access_key.clone().into(),
 			is_confirmed: is_requestor_allowed_to_read,
-		}))
+		})))
 	}
 
 	/// When session initialization confirmation message is reeived.
-	pub fn on_confirm_initialization(&self, sender: NodeId, message: ConfirmDecryptionInitialization) -> Result<(), Error> {
+	pub fn on_confirm_initialization(&self, sender: NodeId, message: &ConfirmDecryptionInitialization) -> Result<(), Error> {
-		debug_assert!(self.id == message.session);
+		debug_assert!(self.id == *message.session);
-		debug_assert!(self.access_key == message.sub_session);
+		debug_assert!(self.access_key == *message.sub_session);
 		debug_assert!(&sender != self.node());
 
 		let mut data = self.data.lock();
@@ -260,26 +282,8 @@ impl Session {
 			// we do not yet have enough nodes for decryption
 			SessionState::WaitingForInitializationConfirm => Ok(()),
 			// we have enough nodes for decryption
-			SessionState::WaitingForPartialDecryption => {
+			SessionState::WaitingForPartialDecryption =>
-				let confirmed_nodes: BTreeSet<_> = data.confirmed_nodes.clone();
+				SessionImpl::start_waiting_for_partial_decryption(self.node().clone(), self.id.clone(), self.access_key.clone(), &self.cluster, &self.encrypted_data, &mut *data),
-				for node in data.confirmed_nodes.iter().filter(|n| n != &self.node()) {
-					self.cluster.send(node, Message::RequestPartialDecryption(RequestPartialDecryption {
-						session: self.id.clone(),
-						sub_session: self.access_key.clone(),
-						nodes: confirmed_nodes.clone(),
-					}))?;
-				}
-
-				assert!(data.confirmed_nodes.remove(self.node()));
-
-				let shadow_point = {
-					let requestor = data.requestor.as_ref().expect("requestor public is filled during initialization; WaitingForPartialDecryption follows initialization; qed");
-					do_partial_decryption(self.node(), &requestor, &data.confirmed_nodes, &self.access_key, &self.encrypted_data)?
-				};
-				data.shadow_points.insert(self.node().clone(), shadow_point);
-
-				Ok(())
-			},
 			// we can not have enough nodes for decryption
 			SessionState::Failed => Ok(()),
 			// cannot reach other states
@@ -288,9 +292,9 @@ impl Session {
 	}
 
 	/// When partial decryption is requested.
-	pub fn on_partial_decryption_requested(&self, sender: NodeId, message: RequestPartialDecryption) -> Result<(), Error> {
+	pub fn on_partial_decryption_requested(&self, sender: NodeId, message: &RequestPartialDecryption) -> Result<(), Error> {
-		debug_assert!(self.id == message.session);
+		debug_assert!(self.id == *message.session);
-		debug_assert!(self.access_key == message.sub_session);
+		debug_assert!(self.access_key == *message.sub_session);
 		debug_assert!(&sender != self.node());
 
 		// check message
@@ -311,13 +315,13 @@ impl Session {
 		// calculate shadow point
 		let shadow_point = {
 			let requestor = data.requestor.as_ref().expect("requestor public is filled during initialization; WaitingForPartialDecryptionRequest follows initialization; qed");
-			do_partial_decryption(self.node(), &requestor, &message.nodes, &self.access_key, &self.encrypted_data)?
+			do_partial_decryption(self.node(), &requestor, &message.nodes.iter().cloned().map(Into::into).collect(), &self.access_key, &self.encrypted_data)?
 		};
-		self.cluster.send(&sender, Message::PartialDecryption(PartialDecryption {
+		self.cluster.send(&sender, Message::Decryption(DecryptionMessage::PartialDecryption(PartialDecryption {
-			session: self.id.clone(),
+			session: self.id.clone().into(),
-			sub_session: self.access_key.clone(),
+			sub_session: self.access_key.clone().into(),
-			shadow_point: shadow_point,
+			shadow_point: shadow_point.into(),
-		}))?;
+		})))?;
 
 		// update sate
 		data.state = SessionState::Finished;
@@ -326,9 +330,9 @@ impl Session {
 	}
 
 	/// When partial decryption is received.
-	pub fn on_partial_decryption(&self, sender: NodeId, message: PartialDecryption) -> Result<(), Error> {
+	pub fn on_partial_decryption(&self, sender: NodeId, message: &PartialDecryption) -> Result<(), Error> {
-		debug_assert!(self.id == message.session);
+		debug_assert!(self.id == *message.session);
-		debug_assert!(self.access_key == message.sub_session);
+		debug_assert!(self.access_key == *message.sub_session);
 		debug_assert!(&sender != self.node());
 
 		let mut data = self.data.lock();
@@ -341,24 +345,113 @@ impl Session {
 		if !data.confirmed_nodes.remove(&sender) {
 			return Err(Error::InvalidStateForRequest);
 		}
-		data.shadow_points.insert(sender, message.shadow_point);
+		data.shadow_points.insert(sender, message.shadow_point.clone().into());
 
 		// check if we have enough shadow points to decrypt the secret
 		if data.shadow_points.len() != self.encrypted_data.threshold + 1 {
 			return Ok(());
 		}
 
+		SessionImpl::do_decryption(self.access_key.clone(), &self.encrypted_data, &mut *data)?;
+		self.completed.notify_all();
+
+		Ok(())
+	}
+
+	/// When error has occured on another node.
+	pub fn on_session_error(&self, sender: NodeId, message: &DecryptionSessionError) {
+		warn!("{}: decryption session error: {:?} from {}", self.node(), message, sender);
+		let mut data = self.data.lock();
+		data.state = SessionState::Failed;
+		data.decrypted_secret = Some(Err(Error::Io(message.error.clone())));
+		self.completed.notify_all();
+	}
+
+	/// When session timeout has occured.
+	pub fn on_session_timeout(&self, _node: &NodeId) {
+		warn!("{}: decryption session timeout", self.node());
+		let mut data = self.data.lock();
+		// TODO: check that node is a part of decryption process
+		data.state = SessionState::Failed;
+		data.decrypted_secret = Some(Err(Error::Io("session expired".into())));
+		self.completed.notify_all();
+	}
+
+	fn start_waiting_for_partial_decryption(self_node_id: NodeId, session_id: SessionId, access_key: Secret, cluster: &Arc<Cluster>, encrypted_data: &DocumentKeyShare, data: &mut SessionData) -> Result<(), Error> {
+		let confirmed_nodes: BTreeSet<_> = data.confirmed_nodes.clone();
+		for node in data.confirmed_nodes.iter().filter(|n| n != &&self_node_id) {
+			cluster.send(node, Message::Decryption(DecryptionMessage::RequestPartialDecryption(RequestPartialDecryption {
+				session: session_id.clone().into(),
+				sub_session: access_key.clone().into(),
+				nodes: confirmed_nodes.iter().cloned().map(Into::into).collect(),
+			})))?;
+		}
+
+		assert!(data.confirmed_nodes.remove(&self_node_id));
+
+		let shadow_point = {
+			let requestor = data.requestor.as_ref().expect("requestor public is filled during initialization; WaitingForPartialDecryption follows initialization; qed");
+			do_partial_decryption(&self_node_id, &requestor, &data.confirmed_nodes, &access_key, &encrypted_data)?
+		};
+		data.shadow_points.insert(self_node_id.clone(), shadow_point);
+
+		Ok(())
+	}
+
+	fn do_decryption(access_key: Secret, encrypted_data: &DocumentKeyShare, data: &mut SessionData) -> Result<(), Error> {
 		// decrypt the secret using shadow points
 		let joint_shadow_point = math::compute_joint_shadow_point(data.shadow_points.values())?;
-		let decrypted_secret = math::decrypt_with_joint_shadow(&self.access_key, &self.encrypted_data.encrypted_point, &joint_shadow_point)?;
+		let decrypted_secret = math::decrypt_with_joint_shadow(encrypted_data.threshold, &access_key, &encrypted_data.encrypted_point, &joint_shadow_point)?;
-		data.decrypted_secret = Some(decrypted_secret);
+		data.decrypted_secret = Some(Ok(decrypted_secret));
+
+		// switch to completed state
 		data.state = SessionState::Finished;
 
 		Ok(())
 	}
 }
 
-fn check_encrypted_data(self_node_id: &Public, encrypted_data: &EncryptedData) -> Result<(), Error> {
+impl Session for SessionImpl {
+	fn wait(&self) -> Result<Public, Error> {
+		let mut data = self.data.lock();
+		if !data.decrypted_secret.is_some() {
+			self.completed.wait(&mut data);
+		}
+
+		data.decrypted_secret.as_ref()
+			.expect("checked above or waited for completed; completed is only signaled when decrypted_secret.is_some(); qed")
+			.clone()
+	}
+}
+
+impl DecryptionSessionId {
+	/// Create new decryption session Id.
+	pub fn new(session_id: SessionId, sub_session_id: Secret) -> Self {
+		DecryptionSessionId {
+			id: session_id,
+			access_key: sub_session_id,
+		}
+	}
+}
+
+impl PartialOrd for DecryptionSessionId {
+	fn partial_cmp(&self, other: &Self) -> Option<Ordering> {
+		Some(self.cmp(other))
+	}
+}
+
+
+impl Ord for DecryptionSessionId {
+	fn cmp(&self, other: &Self) -> Ordering {
+		match self.id.cmp(&other.id) {
+			Ordering::Equal => self.access_key.cmp(&other.access_key),
+			r @ _ => r,
+		}
+	}
+}
+
+
+fn check_encrypted_data(self_node_id: &Public, encrypted_data: &DocumentKeyShare) -> Result<(), Error> {
 	use key_server_cluster::encryption_session::{check_cluster_nodes, check_threshold};
 
 	let nodes = encrypted_data.id_numbers.keys().cloned().collect();
@@ -368,7 +461,7 @@ fn check_encrypted_data(self_node_id: &Public, encrypted_data: &EncryptedData) -
 	Ok(())
 }
 
-fn process_initialization_response(encrypted_data: &EncryptedData, data: &mut SessionData, node: &NodeId, check_result: bool) -> Result<(), Error> {
+fn process_initialization_response(encrypted_data: &DocumentKeyShare, data: &mut SessionData, node: &NodeId, check_result: bool) -> Result<(), Error> {
 	if !data.requested_nodes.remove(node) {
 		return Err(Error::InvalidMessage);
 	}
@@ -395,7 +488,7 @@ fn process_initialization_response(encrypted_data: &EncryptedData, data: &mut Se
 	Ok(())
 }
 
-fn do_partial_decryption(node: &NodeId, _requestor_public: &Public, participants: &BTreeSet<NodeId>, access_key: &Secret, encrypted_data: &EncryptedData) -> Result<Public, Error> {
+fn do_partial_decryption(node: &NodeId, _requestor_public: &Public, participants: &BTreeSet<NodeId>, access_key: &Secret, encrypted_data: &DocumentKeyShare) -> Result<Public, Error> {
 	let node_id_number = &encrypted_data.id_numbers[node];
 	let node_secret_share = &encrypted_data.secret_share;
 	let other_id_numbers = participants.iter()
@@ -409,43 +502,42 @@ fn do_partial_decryption(node: &NodeId, _requestor_public: &Public, participants
 #[cfg(test)]
 mod tests {
 	use std::sync::Arc;
-	use std::str::FromStr;
 	use std::collections::BTreeMap;
 	use super::super::super::acl_storage::DummyAclStorage;
 	use ethkey::{self, Random, Generator, Public, Secret};
-	use key_server_cluster::{NodeId, EncryptedData, SessionId, Error};
+	use key_server_cluster::{NodeId, DocumentKeyShare, SessionId, Error};
 	use key_server_cluster::cluster::tests::DummyCluster;
-	use key_server_cluster::decryption_session::{Session, SessionParams, 	SessionState};
+	use key_server_cluster::decryption_session::{SessionImpl, SessionParams, SessionState};
-	use key_server_cluster::message::{self, Message};
+	use key_server_cluster::message::{self, Message, DecryptionMessage};
 
 	const SECRET_PLAIN: &'static str = "d2b57ae7619e070af0af6bc8c703c0cd27814c54d5d6a999cacac0da34ede279ca0d9216e85991029e54e2f0c92ee0bd30237725fa765cbdbfc4529489864c5f";
 
-	fn prepare_decryption_sessions() -> (Vec<Arc<DummyCluster>>, Vec<Arc<DummyAclStorage>>, Vec<Session>) {
+	fn prepare_decryption_sessions() -> (Vec<Arc<DummyCluster>>, Vec<Arc<DummyAclStorage>>, Vec<SessionImpl>) {
 		// prepare encrypted data + cluster configuration for scheme 4-of-5
 		let session_id = SessionId::default();
 		let access_key = Random.generate().unwrap().secret().clone();
-		let secret_shares = vec![
+		let secret_shares: Vec<Secret> = vec![
-			Secret::from_str("834cb736f02d9c968dfaf0c37658a1d86ff140554fc8b59c9fdad5a8cf810eec").unwrap(),
+			"834cb736f02d9c968dfaf0c37658a1d86ff140554fc8b59c9fdad5a8cf810eec".parse().unwrap(),
-			Secret::from_str("5a3c1d90fafafa66bb808bcc464354a98b05e6b2c95b5f609d4511cdd1b17a0b").unwrap(),
+			"5a3c1d90fafafa66bb808bcc464354a98b05e6b2c95b5f609d4511cdd1b17a0b".parse().unwrap(),
-			Secret::from_str("71bf61e7848e08e3a8486c308ce521bdacfebcf9116a0151447eb301f3a2d0e9").unwrap(),
+			"71bf61e7848e08e3a8486c308ce521bdacfebcf9116a0151447eb301f3a2d0e9".parse().unwrap(),
-			Secret::from_str("80c0e5e2bea66fa9b2e07f7ce09630a9563e8242446d5ee63221feb09c4338f4").unwrap(),
+			"80c0e5e2bea66fa9b2e07f7ce09630a9563e8242446d5ee63221feb09c4338f4".parse().unwrap(),
-			Secret::from_str("c06546b5669877ba579ca437a5602e89425c53808c708d44ccd6afcaa4610fad").unwrap(),
+			"c06546b5669877ba579ca437a5602e89425c53808c708d44ccd6afcaa4610fad".parse().unwrap(),
 		];
 		let id_numbers: Vec<(NodeId, Secret)> = vec![
 			("b486d3840218837b035c66196ecb15e6b067ca20101e11bd5e626288ab6806ecc70b8307012626bd512bad1559112d11d21025cef48cc7a1d2f3976da08f36c8".into(),
-				Secret::from_str("281b6bf43cb86d0dc7b98e1b7def4a80f3ce16d28d2308f934f116767306f06c").unwrap()),
+				"281b6bf43cb86d0dc7b98e1b7def4a80f3ce16d28d2308f934f116767306f06c".parse().unwrap()),
 			("1395568277679f7f583ab7c0992da35f26cde57149ee70e524e49bdae62db3e18eb96122501e7cbb798b784395d7bb5a499edead0706638ad056d886e56cf8fb".into(),
-				Secret::from_str("00125d85a05e5e63e214cb60fe63f132eec8a103aa29266b7e6e6c5b7597230b").unwrap()),
+				"00125d85a05e5e63e214cb60fe63f132eec8a103aa29266b7e6e6c5b7597230b".parse().unwrap()),
 			("99e82b163b062d55a64085bacfd407bb55f194ba5fb7a1af9c34b84435455520f1372e0e650a4f91aed0058cb823f62146ccb5599c8d13372c300dea866b69fc".into(),
-				Secret::from_str("f43ac0fba42a5b6ed95707d2244659e89ba877b1c9b82c0d0a9dcf834e80fc62").unwrap()),
+				"f43ac0fba42a5b6ed95707d2244659e89ba877b1c9b82c0d0a9dcf834e80fc62".parse().unwrap()),
 			("7e05df9dd077ec21ed4bc45c9fe9e0a43d65fa4be540630de615ced5e95cf5c3003035eb713317237d7667feeeb64335525158f5f7411f67aca9645169ea554c".into(),
-				Secret::from_str("5a324938dfb2516800487d25ab7289ba8ec38811f77c3df602e4e65e3c9acd9f").unwrap()),
+				"5a324938dfb2516800487d25ab7289ba8ec38811f77c3df602e4e65e3c9acd9f".parse().unwrap()),
 			("321977760d1d8e15b047a309e4c7fe6f355c10bb5a06c68472b676926427f69f229024fa2692c10da167d14cdc77eb95d0fce68af0a0f704f0d3db36baa83bb2".into(),
-				Secret::from_str("12cf422d50002d04e52bd4906fd7f5f235f051ca36abfe37e061f8da248008d8").unwrap()),
+				"12cf422d50002d04e52bd4906fd7f5f235f051ca36abfe37e061f8da248008d8".parse().unwrap()),
 		];
 		let common_point: Public = "6962be696e1bcbba8e64cc7fddf140f854835354b5804f3bb95ae5a2799130371b589a131bd39699ac7174ccb35fc4342dab05331202209582fc8f3a40916ab0".into();
 		let encrypted_point: Public = "b07031982bde9890e12eff154765f03c56c3ab646ad47431db5dd2d742a9297679c4c65b998557f8008469afd0c43d40b6c5f6c6a1c7354875da4115237ed87a".into();
-		let encrypted_datas: Vec<_> = (0..5).map(|i| EncryptedData {
+		let encrypted_datas: Vec<_> = (0..5).map(|i| DocumentKeyShare {
 			threshold: 3,
 			id_numbers: id_numbers.clone().into_iter().collect(),
 			secret_share: secret_shares[i].clone(),
@@ -454,7 +546,7 @@ mod tests {
 		}).collect();
 		let acl_storages: Vec<_> = (0..5).map(|_| Arc::new(DummyAclStorage::default())).collect();
 		let clusters: Vec<_> = (0..5).map(|i| Arc::new(DummyCluster::new(id_numbers.iter().nth(i).clone().unwrap().0))).collect();
-		let sessions: Vec<_> = (0..5).map(|i| Session::new(SessionParams {
+		let sessions: Vec<_> = (0..5).map(|i| SessionImpl::new(SessionParams {
 			id: session_id.clone(),
 			access_key: access_key.clone(),
 			self_node_id: id_numbers.iter().nth(i).clone().unwrap().0,
@@ -466,11 +558,11 @@ mod tests {
 		(clusters, acl_storages, sessions)
 	}
 
-	fn do_messages_exchange(clusters: &[Arc<DummyCluster>], sessions: &[Session]) {
+	fn do_messages_exchange(clusters: &[Arc<DummyCluster>], sessions: &[SessionImpl]) {
 		do_messages_exchange_until(clusters, sessions, |_, _, _| false);
 	}
 
-	fn do_messages_exchange_until<F>(clusters: &[Arc<DummyCluster>], sessions: &[Session], mut cond: F) where F: FnMut(&NodeId, &NodeId, &Message) -> bool {
+	fn do_messages_exchange_until<F>(clusters: &[Arc<DummyCluster>], sessions: &[SessionImpl], mut cond: F) where F: FnMut(&NodeId, &NodeId, &Message) -> bool {
 		while let Some((from, to, message)) = clusters.iter().filter_map(|c| c.take_message().map(|(to, msg)| (c.node(), to, msg))).next() {
 			let session = &sessions[sessions.iter().position(|s| s.node() == &to).unwrap()];
 			if cond(&from, &to, &message) {
@@ -478,10 +570,10 @@ mod tests {
 			}
 
 			match message {
-				Message::InitializeDecryptionSession(message) => session.on_initialize_session(from, message).unwrap(),
+				Message::Decryption(DecryptionMessage::InitializeDecryptionSession(message)) => session.on_initialize_session(from, &message).unwrap(),
-				Message::ConfirmDecryptionInitialization(message) => session.on_confirm_initialization(from, message).unwrap(),
+				Message::Decryption(DecryptionMessage::ConfirmDecryptionInitialization(message)) => session.on_confirm_initialization(from, &message).unwrap(),
-				Message::RequestPartialDecryption(message) => session.on_partial_decryption_requested(from, message).unwrap(),
+				Message::Decryption(DecryptionMessage::RequestPartialDecryption(message)) => session.on_partial_decryption_requested(from, &message).unwrap(),
-				Message::PartialDecryption(message) => session.on_partial_decryption(from, message).unwrap(),
+				Message::Decryption(DecryptionMessage::PartialDecryption(message)) => session.on_partial_decryption(from, &message).unwrap(),
 				_ => panic!("unexpected"),
 			}
 		}
@@ -492,11 +584,11 @@ mod tests {
 		let mut nodes = BTreeMap::new();
 		let self_node_id = Random.generate().unwrap().public().clone();
 		nodes.insert(self_node_id, Random.generate().unwrap().secret().clone());
-		match Session::new(SessionParams {
+		match SessionImpl::new(SessionParams {
 			id: SessionId::default(),
 			access_key: Random.generate().unwrap().secret().clone(),
 			self_node_id: self_node_id.clone(),
-			encrypted_data: EncryptedData {
+			encrypted_data: DocumentKeyShare {
 				threshold: 0,
 				id_numbers: nodes,
 				secret_share: Random.generate().unwrap().secret().clone(),
@@ -517,11 +609,11 @@ mod tests {
 		let self_node_id = Random.generate().unwrap().public().clone();
 		nodes.insert(Random.generate().unwrap().public().clone(), Random.generate().unwrap().secret().clone());
 		nodes.insert(Random.generate().unwrap().public().clone(), Random.generate().unwrap().secret().clone());
-		match Session::new(SessionParams {
+		match SessionImpl::new(SessionParams {
 			id: SessionId::default(),
 			access_key: Random.generate().unwrap().secret().clone(),
 			self_node_id: self_node_id.clone(),
-			encrypted_data: EncryptedData {
+			encrypted_data: DocumentKeyShare {
 				threshold: 0,
 				id_numbers: nodes,
 				secret_share: Random.generate().unwrap().secret().clone(),
@@ -542,11 +634,11 @@ mod tests {
 		let self_node_id = Random.generate().unwrap().public().clone();
 		nodes.insert(self_node_id.clone(), Random.generate().unwrap().secret().clone());
 		nodes.insert(Random.generate().unwrap().public().clone(), Random.generate().unwrap().secret().clone());
-		match Session::new(SessionParams {
+		match SessionImpl::new(SessionParams {
 			id: SessionId::default(),
 			access_key: Random.generate().unwrap().secret().clone(),
 			self_node_id: self_node_id.clone(),
-			encrypted_data: EncryptedData {
+			encrypted_data: DocumentKeyShare {
 				threshold: 2,
 				id_numbers: nodes,
 				secret_share: Random.generate().unwrap().secret().clone(),
@@ -572,70 +664,70 @@ mod tests {
 	fn fails_to_accept_initialization_when_already_initialized() {
 		let (_, _, sessions) = prepare_decryption_sessions();
 		assert_eq!(sessions[0].initialize(ethkey::sign(Random.generate().unwrap().secret(), &SessionId::default()).unwrap()).unwrap(), ());
-		assert_eq!(sessions[0].on_initialize_session(sessions[1].node().clone(), message::InitializeDecryptionSession {
+		assert_eq!(sessions[0].on_initialize_session(sessions[1].node().clone(), &message::InitializeDecryptionSession {
-			session: SessionId::default(),
+			session: SessionId::default().into(),
-			sub_session: sessions[0].access_key().clone(),
+			sub_session: sessions[0].access_key().clone().into(),
-			requestor_signature: ethkey::sign(Random.generate().unwrap().secret(), &SessionId::default()).unwrap(),
+			requestor_signature: ethkey::sign(Random.generate().unwrap().secret(), &SessionId::default()).unwrap().into(),
 		}).unwrap_err(), Error::InvalidStateForRequest);
 	}
 
 	#[test]
 	fn fails_to_partial_decrypt_if_not_waiting() {
 		let (_, _, sessions) = prepare_decryption_sessions();
-		assert_eq!(sessions[1].on_initialize_session(sessions[0].node().clone(), message::InitializeDecryptionSession {
+		assert_eq!(sessions[1].on_initialize_session(sessions[0].node().clone(), &message::InitializeDecryptionSession {
-			session: SessionId::default(),
+			session: SessionId::default().into(),
-			sub_session: sessions[0].access_key().clone(),
+			sub_session: sessions[0].access_key().clone().into(),
-			requestor_signature: ethkey::sign(Random.generate().unwrap().secret(), &SessionId::default()).unwrap(),
+			requestor_signature: ethkey::sign(Random.generate().unwrap().secret(), &SessionId::default()).unwrap().into(),
 		}).unwrap(), ());
-		assert_eq!(sessions[1].on_partial_decryption_requested(sessions[0].node().clone(), message::RequestPartialDecryption {
+		assert_eq!(sessions[1].on_partial_decryption_requested(sessions[0].node().clone(), &message::RequestPartialDecryption {
-			session: SessionId::default(),
+			session: SessionId::default().into(),
-			sub_session: sessions[0].access_key().clone(),
+			sub_session: sessions[0].access_key().clone().into(),
-			nodes: sessions.iter().map(|s| s.node().clone()).take(4).collect(),
+			nodes: sessions.iter().map(|s| s.node().clone().into()).take(4).collect(),
 		}).unwrap(), ());
-		assert_eq!(sessions[1].on_partial_decryption_requested(sessions[0].node().clone(), message::RequestPartialDecryption {
+		assert_eq!(sessions[1].on_partial_decryption_requested(sessions[0].node().clone(), &message::RequestPartialDecryption {
-			session: SessionId::default(),
+			session: SessionId::default().into(),
-			sub_session: sessions[0].access_key().clone(),
+			sub_session: sessions[0].access_key().clone().into(),
-			nodes: sessions.iter().map(|s| s.node().clone()).take(4).collect(),
+			nodes: sessions.iter().map(|s| s.node().clone().into()).take(4).collect(),
 		}).unwrap_err(), Error::InvalidStateForRequest);
 	}
 
 	#[test]
 	fn fails_to_partial_decrypt_if_requested_by_slave() {
 		let (_, _, sessions) = prepare_decryption_sessions();
-		assert_eq!(sessions[1].on_initialize_session(sessions[0].node().clone(), message::InitializeDecryptionSession {
+		assert_eq!(sessions[1].on_initialize_session(sessions[0].node().clone(), &message::InitializeDecryptionSession {
-			session: SessionId::default(),
+			session: SessionId::default().into(),
-			sub_session: sessions[0].access_key().clone(),
+			sub_session: sessions[0].access_key().clone().into(),
-			requestor_signature: ethkey::sign(Random.generate().unwrap().secret(), &SessionId::default()).unwrap(),
+			requestor_signature: ethkey::sign(Random.generate().unwrap().secret(), &SessionId::default()).unwrap().into(),
 		}).unwrap(), ());
-		assert_eq!(sessions[1].on_partial_decryption_requested(sessions[2].node().clone(), message::RequestPartialDecryption {
+		assert_eq!(sessions[1].on_partial_decryption_requested(sessions[2].node().clone(), &message::RequestPartialDecryption {
-			session: SessionId::default(),
+			session: SessionId::default().into(),
-			sub_session: sessions[0].access_key().clone(),
+			sub_session: sessions[0].access_key().clone().into(),
-			nodes: sessions.iter().map(|s| s.node().clone()).take(4).collect(),
+			nodes: sessions.iter().map(|s| s.node().clone().into()).take(4).collect(),
 		}).unwrap_err(), Error::InvalidMessage);
 	}
 
 	#[test]
 	fn fails_to_partial_decrypt_if_wrong_number_of_nodes_participating() {
 		let (_, _, sessions) = prepare_decryption_sessions();
-		assert_eq!(sessions[1].on_initialize_session(sessions[0].node().clone(), message::InitializeDecryptionSession {
+		assert_eq!(sessions[1].on_initialize_session(sessions[0].node().clone(), &message::InitializeDecryptionSession {
-			session: SessionId::default(),
+			session: SessionId::default().into(),
-			sub_session: sessions[0].access_key().clone(),
+			sub_session: sessions[0].access_key().clone().into(),
-			requestor_signature: ethkey::sign(Random.generate().unwrap().secret(), &SessionId::default()).unwrap(),
+			requestor_signature: ethkey::sign(Random.generate().unwrap().secret(), &SessionId::default()).unwrap().into(),
 		}).unwrap(), ());
-		assert_eq!(sessions[1].on_partial_decryption_requested(sessions[0].node().clone(), message::RequestPartialDecryption {
+		assert_eq!(sessions[1].on_partial_decryption_requested(sessions[0].node().clone(), &message::RequestPartialDecryption {
-			session: SessionId::default(),
+			session: SessionId::default().into(),
-			sub_session: sessions[0].access_key().clone(),
+			sub_session: sessions[0].access_key().clone().into(),
-			nodes: sessions.iter().map(|s| s.node().clone()).take(2).collect(),
+			nodes: sessions.iter().map(|s| s.node().clone().into()).take(2).collect(),
 		}).unwrap_err(), Error::InvalidMessage);
 	}
 
 	#[test]
 	fn fails_to_accept_partial_decrypt_if_not_waiting() {
 		let (_, _, sessions) = prepare_decryption_sessions();
-		assert_eq!(sessions[0].on_partial_decryption(sessions[1].node().clone(), message::PartialDecryption {
+		assert_eq!(sessions[0].on_partial_decryption(sessions[1].node().clone(), &message::PartialDecryption {
-			session: SessionId::default(),
+			session: SessionId::default().into(),
-			sub_session: sessions[0].access_key().clone(),
+			sub_session: sessions[0].access_key().clone().into(),
-			shadow_point: Random.generate().unwrap().public().clone(),
+			shadow_point: Random.generate().unwrap().public().clone().into(),
 		}).unwrap_err(), Error::InvalidStateForRequest);
 	}
 
@@ -647,7 +739,7 @@ mod tests {
 		let mut pd_from = None;
 		let mut pd_msg = None;
 		do_messages_exchange_until(&clusters, &sessions, |from, _, msg| match msg {
-			&Message::PartialDecryption(ref msg) => {
+			&Message::Decryption(DecryptionMessage::PartialDecryption(ref msg)) => {
 				pd_from = Some(from.clone());
 				pd_msg = Some(msg.clone());
 				true
@@ -655,8 +747,8 @@ mod tests {
 			_ => false,
 		});
 
-		assert_eq!(sessions[0].on_partial_decryption(pd_from.clone().unwrap(), pd_msg.clone().unwrap()).unwrap(), ());
+		assert_eq!(sessions[0].on_partial_decryption(pd_from.clone().unwrap(), &pd_msg.clone().unwrap()).unwrap(), ());
-		assert_eq!(sessions[0].on_partial_decryption(pd_from.unwrap(), pd_msg.unwrap()).unwrap_err(), Error::InvalidStateForRequest);
+		assert_eq!(sessions[0].on_partial_decryption(pd_from.unwrap(), &pd_msg.unwrap()).unwrap_err(), Error::InvalidStateForRequest);
 	}
 
 	#[test]
@@ -704,4 +796,9 @@ mod tests {
 		// 3) 0 sessions have decrypted key value
 		assert!(sessions.iter().all(|s| s.decrypted_secret().is_none()));
 	}
+
+	#[test]
+	fn decryption_session_works_over_network() {
+		// TODO
+	}
 }

secret_store/src/key_server_cluster/io/deadline.rs

@@ -0,0 +1,85 @@
+// Copyright 2015-2017 Parity Technologies (UK) Ltd.
+// This file is part of Parity.
+
+// Parity is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Parity is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Parity.  If not, see <http://www.gnu.org/licenses/>.
+
+use std::io;
+use std::time::Duration;
+use futures::{Future, Select, BoxFuture, Poll, Async};
+use tokio_core::reactor::{Handle, Timeout};
+
+type DeadlineBox<F> where F: Future = BoxFuture<DeadlineStatus<F::Item>, F::Error>;
+
+/// Complete a passed future or fail if it is not completed within timeout.
+pub fn deadline<F, T>(duration: Duration, handle: &Handle, future: F) -> Result<Deadline<F>, io::Error>
+	where F: Future<Item = T, Error = io::Error> + Send + 'static, T: 'static {
+	let timeout = try!(Timeout::new(duration, handle)).map(|_| DeadlineStatus::Timeout).boxed();
+	let future = future.map(DeadlineStatus::Meet).boxed();
+	let deadline = Deadline {
+		future: timeout.select(future),
+	};
+	Ok(deadline)
+}
+
+#[derive(Debug, PartialEq)]
+/// Deadline future completion status.
+pub enum DeadlineStatus<T> {
+	/// Completed a future.
+	Meet(T),
+	/// Faled with timeout.
+	Timeout,
+}
+
+/// Future, which waits for passed future completion within given period, or fails with timeout.
+pub struct Deadline<F> where F: Future {
+	future: Select<DeadlineBox<F>, DeadlineBox<F>>,
+}
+
+impl<F, T> Future for Deadline<F> where F: Future<Item = T, Error = io::Error> {
+	type Item = DeadlineStatus<T>;
+	type Error = io::Error;
+
+	fn poll(&mut self) -> Poll<Self::Item, Self::Error> {
+		match self.future.poll() {
+			Ok(Async::Ready((result, _other))) => Ok(Async::Ready(result)),
+			Ok(Async::NotReady) => Ok(Async::NotReady),
+			Err((err, _other)) => Err(err),
+		}
+	}
+}
+
+#[cfg(test)]
+mod tests {
+	use std::io;
+	use std::time::Duration;
+	use futures::{Future, empty, done};
+	use tokio_core::reactor::Core;
+	use super::{deadline, DeadlineStatus};
+
+	//#[test] TODO: not working
+	fn _deadline_timeout_works() {
+		let mut core = Core::new().unwrap();
+		let deadline = deadline(Duration::from_millis(1), &core.handle(), empty::<(), io::Error>()).unwrap();
+		core.turn(Some(Duration::from_millis(3)));
+		assert_eq!(deadline.wait().unwrap(), DeadlineStatus::Timeout);
+	}
+
+	#[test]
+	fn deadline_result_works() {
+		let mut core = Core::new().unwrap();
+		let deadline = deadline(Duration::from_millis(1000), &core.handle(), done(Ok(()))).unwrap();
+		core.turn(Some(Duration::from_millis(3)));
+		assert_eq!(deadline.wait().unwrap(), DeadlineStatus::Meet(()));
+	}
+}

secret_store/src/key_server_cluster/io/handshake.rs

@@ -0,0 +1,320 @@
+// Copyright 2015-2017 Parity Technologies (UK) Ltd.
+// This file is part of Parity.
+
+// Parity is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Parity is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Parity.  If not, see <http://www.gnu.org/licenses/>.
+
+use std::io;
+use std::collections::BTreeSet;
+use futures::{Future, Poll, Async};
+use ethkey::{Random, Generator, KeyPair, Secret, sign, verify_public};
+use util::H256;
+use key_server_cluster::{NodeId, Error};
+use key_server_cluster::message::{Message, ClusterMessage, NodePublicKey, NodePrivateKeySignature};
+use key_server_cluster::io::{write_message, write_encrypted_message, WriteMessage, ReadMessage,
+	read_message, compute_shared_key};
+
+/// Start handshake procedure with another node from the cluster.
+pub fn handshake<A>(a: A, self_key_pair: KeyPair, trusted_nodes: BTreeSet<NodeId>) -> Handshake<A> where A: io::Write + io::Read {
+	let self_confirmation_plain = Random.generate().map(|kp| *kp.secret().clone()).map_err(Into::into);
+	handshake_with_plain_confirmation(a, self_confirmation_plain, self_key_pair, trusted_nodes)
+}
+
+/// Start handshake procedure with another node from the cluster and given plain confirmation.
+pub fn handshake_with_plain_confirmation<A>(a: A, self_confirmation_plain: Result<H256, Error>, self_key_pair: KeyPair, trusted_nodes: BTreeSet<NodeId>) -> Handshake<A> where A: io::Write + io::Read {
+	let (error, state) = match self_confirmation_plain.clone()
+		.and_then(|c| Handshake::<A>::make_public_key_message(self_key_pair.public().clone(), c)) {
+		Ok(message) => (None, HandshakeState::SendPublicKey(write_message(a, message))),
+		Err(err) => (Some((a, Err(err))), HandshakeState::Finished),
+	};
+
+	Handshake {
+		is_active: true,
+		error: error,
+		state: state,
+		self_key_pair: self_key_pair,
+		self_confirmation_plain: self_confirmation_plain.unwrap_or(Default::default()),
+		trusted_nodes: trusted_nodes,
+		other_node_id: None,
+		other_confirmation_plain: None,
+		shared_key: None,
+	}
+}
+
+/// Wait for handshake procedure to be started by another node from the cluster.
+pub fn accept_handshake<A>(a: A, self_key_pair: KeyPair, trusted_nodes: BTreeSet<NodeId>) -> Handshake<A> where A: io::Write + io::Read {
+	let self_confirmation_plain = Random.generate().map(|kp| *kp.secret().clone()).map_err(Into::into);
+	let (error, state) = match self_confirmation_plain.clone() {
+		Ok(_) => (None, HandshakeState::ReceivePublicKey(read_message(a))),
+		Err(err) => (Some((a, Err(err))), HandshakeState::Finished),
+	};
+
+	Handshake {
+		is_active: false,
+		error: error,
+		state: state,
+		self_key_pair: self_key_pair,
+		self_confirmation_plain: self_confirmation_plain.unwrap_or(Default::default()),
+		trusted_nodes: trusted_nodes,
+		other_node_id: None,
+		other_confirmation_plain: None,
+		shared_key: None,
+	}
+}
+
+#[derive(Debug, PartialEq)]
+/// Result of handshake procedure.
+pub struct HandshakeResult {
+	/// Node id.
+	pub node_id: NodeId,
+	/// Shared key.
+	pub shared_key: Secret,
+}
+
+/// Future handshake procedure.
+pub struct Handshake<A> {
+	is_active: bool,
+	error: Option<(A, Result<HandshakeResult, Error>)>,
+	state: HandshakeState<A>,
+	self_key_pair: KeyPair,
+	self_confirmation_plain: H256,
+	trusted_nodes: BTreeSet<NodeId>,
+	other_node_id: Option<NodeId>,
+	other_confirmation_plain: Option<H256>,
+	shared_key: Option<Secret>,
+}
+
+/// Active handshake state.
+enum HandshakeState<A> {
+	SendPublicKey(WriteMessage<A>),
+	ReceivePublicKey(ReadMessage<A>),
+	SendPrivateKeySignature(WriteMessage<A>),
+	ReceivePrivateKeySignature(ReadMessage<A>),
+	Finished,
+}
+
+impl<A> Handshake<A> where A: io::Read + io::Write {
+	#[cfg(test)]
+	pub fn set_self_confirmation_plain(&mut self, self_confirmation_plain: H256) {
+		self.self_confirmation_plain = self_confirmation_plain;
+	}
+
+	pub fn make_public_key_message(self_node_id: NodeId, confirmation_plain: H256) -> Result<Message, Error> {
+		Ok(Message::Cluster(ClusterMessage::NodePublicKey(NodePublicKey {
+			node_id: self_node_id.into(),
+			confirmation_plain: confirmation_plain.into(),
+		})))
+	}
+
+	fn make_private_key_signature_message(secret: &Secret, confirmation_plain: &H256) -> Result<Message, Error> {
+		Ok(Message::Cluster(ClusterMessage::NodePrivateKeySignature(NodePrivateKeySignature {
+			confirmation_signed: sign(secret, confirmation_plain)?.into(),
+		})))
+	}
+}
+
+impl<A> Future for Handshake<A> where A: io::Read + io::Write {
+	type Item = (A, Result<HandshakeResult, Error>);
+	type Error = io::Error;
+
+	fn poll(&mut self) -> Poll<Self::Item, Self::Error> {
+		if let Some(error_result) = self.error.take() {
+			return Ok(error_result.into());
+		}
+
+		let (next, result) = match self.state {
+			HandshakeState::SendPublicKey(ref mut future) => {
+				let (stream, _) = try_ready!(future.poll());
+
+				if self.is_active {
+					(HandshakeState::ReceivePublicKey(
+						read_message(stream)
+					), Async::NotReady)
+				} else {
+					self.shared_key = match compute_shared_key(self.self_key_pair.secret(),
+						self.other_node_id.as_ref().expect("we are in passive mode; in passive mode SendPublicKey follows ReceivePublicKey; other_node_id is filled in ReceivePublicKey; qed")
+					) {
+						Ok(shared_key) => Some(shared_key),
+						Err(err) => return Ok((stream, Err(err)).into()),
+					};
+
+					let message = match Handshake::<A>::make_private_key_signature_message(
+						self.self_key_pair.secret(),
+						self.other_confirmation_plain.as_ref().expect("we are in passive mode; in passive mode SendPublicKey follows ReceivePublicKey; other_confirmation_plain is filled in ReceivePublicKey; qed")
+					) {
+						Ok(message) => message,
+						Err(err) => return Ok((stream, Err(err)).into()),
+					};
+					(HandshakeState::SendPrivateKeySignature(write_encrypted_message(stream,
+						self.shared_key.as_ref().expect("filled couple of lines above; qed"),
+					message)), Async::NotReady)
+				}
+			},
+			HandshakeState::ReceivePublicKey(ref mut future) => {
+				let (stream, message) = try_ready!(future.poll());
+
+				let message = match message {
+					Ok(message) => match message {
+						Message::Cluster(ClusterMessage::NodePublicKey(message)) => message,
+						_ => return Ok((stream, Err(Error::InvalidMessage)).into()),
+					},
+					Err(err) => return Ok((stream, Err(err.into())).into()),
+				};
+
+				if !self.trusted_nodes.contains(&*message.node_id) {
+					return Ok((stream, Err(Error::InvalidNodeId)).into());
+				}
+
+				self.other_node_id = Some(message.node_id.into());
+				self.other_confirmation_plain = Some(message.confirmation_plain.into());
+				if self.is_active {
+					self.shared_key = match compute_shared_key(self.self_key_pair.secret(),
+						self.other_node_id.as_ref().expect("filled couple of lines above; qed")
+					) {
+						Ok(shared_key) => Some(shared_key),
+						Err(err) => return Ok((stream, Err(err)).into()),
+					};
+
+					let message = match Handshake::<A>::make_private_key_signature_message(
+						self.self_key_pair.secret(),
+						self.other_confirmation_plain.as_ref().expect("filled couple of lines above; qed")
+					) {
+						Ok(message) => message,
+						Err(err) => return Ok((stream, Err(err)).into()),
+					};
+					(HandshakeState::SendPrivateKeySignature(write_encrypted_message(stream,
+						self.shared_key.as_ref().expect("filled couple of lines above; qed"),
+					message)), Async::NotReady)
+				} else {
+					let message = match Handshake::<A>::make_public_key_message(self.self_key_pair.public().clone(), self.self_confirmation_plain.clone()) {
+						Ok(message) => message,
+						Err(err) => return Ok((stream, Err(err)).into()),
+					};
+					(HandshakeState::SendPublicKey(write_message(stream, message)), Async::NotReady)
+				}
+			},
+			HandshakeState::SendPrivateKeySignature(ref mut future) => {
+				let (stream, _) = try_ready!(future.poll());
+
+				(HandshakeState::ReceivePrivateKeySignature(
+					read_message(stream)
+				), Async::NotReady)
+			},
+			HandshakeState::ReceivePrivateKeySignature(ref mut future) => {
+				let (stream, message) = try_ready!(future.poll());
+
+				let message = match message {
+					Ok(message) => match message {
+						Message::Cluster(ClusterMessage::NodePrivateKeySignature(message)) => message,
+						_ => return Ok((stream, Err(Error::InvalidMessage)).into()),
+					},
+					Err(err) => return Ok((stream, Err(err.into())).into()),
+				};
+
+				let other_node_public = self.other_node_id.as_ref().expect("other_node_id is filled in ReceivePublicKey; ReceivePrivateKeySignature follows ReceivePublicKey; qed");
+				if !verify_public(other_node_public, &*message.confirmation_signed, &self.self_confirmation_plain).unwrap_or(false) {
+					return Ok((stream, Err(Error::InvalidMessage)).into());
+				}
+
+				(HandshakeState::Finished, Async::Ready((stream, Ok(HandshakeResult {
+					node_id: self.other_node_id.expect("other_node_id is filled in ReceivePublicKey; ReceivePrivateKeySignature follows ReceivePublicKey; qed"),
+					shared_key: self.shared_key.clone().expect("shared_key is filled in Send/ReceivePublicKey; ReceivePrivateKeySignature follows Send/ReceivePublicKey; qed"),
+				}))))
+			},
+			HandshakeState::Finished => panic!("poll Handshake after it's done"),
+		};
+
+		self.state = next;
+		match result {
+			// by polling again, we register new future
+			Async::NotReady => self.poll(),
+			result => Ok(result)
+		}
+	}
+}
+
+#[cfg(test)]
+mod tests {
+	use std::collections::BTreeSet;
+	use futures::Future;
+	use ethcrypto::ecdh::agree;
+	use ethkey::{Random, Generator, sign};
+	use util::H256;
+	use key_server_cluster::io::message::tests::TestIo;
+	use key_server_cluster::message::{Message, ClusterMessage, NodePublicKey, NodePrivateKeySignature};
+	use super::{handshake_with_plain_confirmation, accept_handshake, HandshakeResult};
+
+	fn prepare_test_io() -> (H256, TestIo) {
+		let self_key_pair = Random.generate().unwrap();
+		let peer_key_pair = Random.generate().unwrap();
+		let mut io = TestIo::new(self_key_pair.clone(), peer_key_pair.public().clone());
+
+		let self_confirmation_plain = *Random.generate().unwrap().secret().clone();
+		let peer_confirmation_plain = *Random.generate().unwrap().secret().clone();
+
+		let self_confirmation_signed = sign(peer_key_pair.secret(), &self_confirmation_plain).unwrap();
+		let peer_confirmation_signed = sign(self_key_pair.secret(), &peer_confirmation_plain).unwrap();
+
+		io.add_input_message(Message::Cluster(ClusterMessage::NodePublicKey(NodePublicKey {
+			node_id: peer_key_pair.public().clone().into(),
+			confirmation_plain: peer_confirmation_plain.into(),
+		})));
+		io.add_input_message(Message::Cluster(ClusterMessage::NodePrivateKeySignature(NodePrivateKeySignature {
+			confirmation_signed: self_confirmation_signed.into(),
+		})));
+
+		io.add_output_message(Message::Cluster(ClusterMessage::NodePublicKey(NodePublicKey {
+			node_id: self_key_pair.public().clone().into(),
+			confirmation_plain: self_confirmation_plain.clone().into(),
+		})));
+		io.add_output_message(Message::Cluster(ClusterMessage::NodePrivateKeySignature(NodePrivateKeySignature {
+			confirmation_signed: peer_confirmation_signed.into(),
+		})));
+
+		(self_confirmation_plain, io)
+	}
+
+	#[test]
+	fn active_handshake_works() {
+		let (self_confirmation_plain, io) = prepare_test_io();
+		let self_key_pair = io.self_key_pair().clone();
+		let trusted_nodes: BTreeSet<_> = vec![io.peer_public().clone()].into_iter().collect();
+		let shared_key = agree(self_key_pair.secret(), trusted_nodes.iter().nth(0).unwrap()).unwrap();
+
+		let handshake = handshake_with_plain_confirmation(io, Ok(self_confirmation_plain), self_key_pair, trusted_nodes);
+		let handshake_result = handshake.wait().unwrap();
+		assert_eq!(handshake_result.1, Ok(HandshakeResult {
+			node_id: handshake_result.0.peer_public().clone(),
+			shared_key: shared_key,
+		}));
+		handshake_result.0.assert_output();
+	}
+
+	#[test]
+	fn passive_handshake_works() {
+		let (self_confirmation_plain, io) = prepare_test_io();
+		let self_key_pair = io.self_key_pair().clone();
+		let trusted_nodes: BTreeSet<_> = vec![io.peer_public().clone()].into_iter().collect();
+		let shared_key = agree(self_key_pair.secret(), io.peer_public()).unwrap();
+
+		let mut handshake = accept_handshake(io, self_key_pair, trusted_nodes);
+		handshake.set_self_confirmation_plain(self_confirmation_plain);
+
+		let handshake_result = handshake.wait().unwrap();
+		assert_eq!(handshake_result.1, Ok(HandshakeResult {
+			node_id: handshake_result.0.peer_public().clone(),
+			shared_key: shared_key,
+		}));
+		handshake_result.0.assert_output();
+	}
+}

secret_store/src/key_server_cluster/io/message.rs

@@ -0,0 +1,247 @@
+// Copyright 2015-2017 Parity Technologies (UK) Ltd.
+// This file is part of Parity.
+
+// Parity is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Parity is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Parity.  If not, see <http://www.gnu.org/licenses/>.
+
+use std::io::Cursor;
+use std::u16;
+use std::ops::Deref;
+use byteorder::{LittleEndian, ReadBytesExt, WriteBytesExt};
+use serde_json;
+use ethcrypto::ecdh::agree;
+use ethkey::{Public, Secret};
+use key_server_cluster::Error;
+use key_server_cluster::message::{Message, ClusterMessage, EncryptionMessage, DecryptionMessage};
+
+/// Size of serialized header.
+pub const MESSAGE_HEADER_SIZE: usize = 4;
+
+#[derive(Debug, PartialEq)]
+/// Message header.
+pub struct MessageHeader {
+	/// Message/Header version.
+	pub version: u8,
+	/// Message kind.
+	pub kind: u8,
+	/// Message payload size (without header).
+	pub size: u16,
+}
+
+#[derive(Debug, Clone, PartialEq)]
+/// Serialized message.
+pub struct SerializedMessage(Vec<u8>);
+
+impl Deref for SerializedMessage {
+	type Target = [u8];
+
+	fn deref(&self) -> &[u8] {
+		&self.0
+	}
+}
+
+impl Into<Vec<u8>> for SerializedMessage {
+	fn into(self) -> Vec<u8> {
+		self.0
+	}
+}
+
+/// Serialize message.
+pub fn serialize_message(message: Message) -> Result<SerializedMessage, Error> {
+	let (message_kind, payload) = match message {
+		Message::Cluster(ClusterMessage::NodePublicKey(payload))							=> (1, serde_json::to_vec(&payload)),
+		Message::Cluster(ClusterMessage::NodePrivateKeySignature(payload))					=> (2, serde_json::to_vec(&payload)),
+		Message::Cluster(ClusterMessage::KeepAlive(payload))								=> (3, serde_json::to_vec(&payload)),
+		Message::Cluster(ClusterMessage::KeepAliveResponse(payload))						=> (4, serde_json::to_vec(&payload)),
+
+		Message::Encryption(EncryptionMessage::InitializeSession(payload))					=> (50, serde_json::to_vec(&payload)),
+		Message::Encryption(EncryptionMessage::ConfirmInitialization(payload))				=> (51, serde_json::to_vec(&payload)),
+		Message::Encryption(EncryptionMessage::CompleteInitialization(payload))				=> (52, serde_json::to_vec(&payload)),
+		Message::Encryption(EncryptionMessage::KeysDissemination(payload))					=> (53, serde_json::to_vec(&payload)),
+		Message::Encryption(EncryptionMessage::Complaint(payload))							=> (54, serde_json::to_vec(&payload)),
+		Message::Encryption(EncryptionMessage::ComplaintResponse(payload))					=> (55, serde_json::to_vec(&payload)),
+		Message::Encryption(EncryptionMessage::PublicKeyShare(payload))						=> (56, serde_json::to_vec(&payload)),
+		Message::Encryption(EncryptionMessage::SessionError(payload))						=> (57, serde_json::to_vec(&payload)),
+		Message::Encryption(EncryptionMessage::SessionCompleted(payload))					=> (58, serde_json::to_vec(&payload)),
+
+		Message::Decryption(DecryptionMessage::InitializeDecryptionSession(payload))		=> (100, serde_json::to_vec(&payload)),
+		Message::Decryption(DecryptionMessage::ConfirmDecryptionInitialization(payload))	=> (101, serde_json::to_vec(&payload)),
+		Message::Decryption(DecryptionMessage::RequestPartialDecryption(payload))			=> (102, serde_json::to_vec(&payload)),
+		Message::Decryption(DecryptionMessage::PartialDecryption(payload))					=> (103, serde_json::to_vec(&payload)),
+		Message::Decryption(DecryptionMessage::DecryptionSessionError(payload))				=> (104, serde_json::to_vec(&payload)),
+	};
+
+	let payload = payload.map_err(|err| Error::Serde(err.to_string()))?;
+	let payload_len = payload.len();
+	if payload_len > u16::MAX as usize {
+		return Err(Error::InvalidMessage);
+	}
+
+	let header = MessageHeader {
+		kind: message_kind,
+		version: 1,
+		size: payload_len as u16,
+	};
+
+	let mut serialized_message = serialize_header(&header)?;
+	serialized_message.extend(payload);
+	Ok(SerializedMessage(serialized_message))
+}
+
+/// Deserialize message.
+pub fn deserialize_message(header: &MessageHeader, payload: Vec<u8>) -> Result<Message, Error> {
+	Ok(match header.kind {
+		1	=> Message::Cluster(ClusterMessage::NodePublicKey(serde_json::from_slice(&payload).map_err(|err| Error::Serde(err.to_string()))?)),
+		2	=> Message::Cluster(ClusterMessage::NodePrivateKeySignature(serde_json::from_slice(&payload).map_err(|err| Error::Serde(err.to_string()))?)),
+		3	=> Message::Cluster(ClusterMessage::KeepAlive(serde_json::from_slice(&payload).map_err(|err| Error::Serde(err.to_string()))?)),
+		4	=> Message::Cluster(ClusterMessage::KeepAliveResponse(serde_json::from_slice(&payload).map_err(|err| Error::Serde(err.to_string()))?)),
+
+		50	=> Message::Encryption(EncryptionMessage::InitializeSession(serde_json::from_slice(&payload).map_err(|err| Error::Serde(err.to_string()))?)),
+		51	=> Message::Encryption(EncryptionMessage::ConfirmInitialization(serde_json::from_slice(&payload).map_err(|err| Error::Serde(err.to_string()))?)),
+		52	=> Message::Encryption(EncryptionMessage::CompleteInitialization(serde_json::from_slice(&payload).map_err(|err| Error::Serde(err.to_string()))?)),
+		53	=> Message::Encryption(EncryptionMessage::KeysDissemination(serde_json::from_slice(&payload).map_err(|err| Error::Serde(err.to_string()))?)),
+		54	=> Message::Encryption(EncryptionMessage::Complaint(serde_json::from_slice(&payload).map_err(|err| Error::Serde(err.to_string()))?)),
+		55	=> Message::Encryption(EncryptionMessage::ComplaintResponse(serde_json::from_slice(&payload).map_err(|err| Error::Serde(err.to_string()))?)),
+		56	=> Message::Encryption(EncryptionMessage::PublicKeyShare(serde_json::from_slice(&payload).map_err(|err| Error::Serde(err.to_string()))?)),
+		57	=> Message::Encryption(EncryptionMessage::SessionError(serde_json::from_slice(&payload).map_err(|err| Error::Serde(err.to_string()))?)),
+		58	=> Message::Encryption(EncryptionMessage::SessionCompleted(serde_json::from_slice(&payload).map_err(|err| Error::Serde(err.to_string()))?)),
+
+		100	=> Message::Decryption(DecryptionMessage::InitializeDecryptionSession(serde_json::from_slice(&payload).map_err(|err| Error::Serde(err.to_string()))?)),
+		101	=> Message::Decryption(DecryptionMessage::ConfirmDecryptionInitialization(serde_json::from_slice(&payload).map_err(|err| Error::Serde(err.to_string()))?)),
+		102	=> Message::Decryption(DecryptionMessage::RequestPartialDecryption(serde_json::from_slice(&payload).map_err(|err| Error::Serde(err.to_string()))?)),
+		103	=> Message::Decryption(DecryptionMessage::PartialDecryption(serde_json::from_slice(&payload).map_err(|err| Error::Serde(err.to_string()))?)),
+		104	=> Message::Decryption(DecryptionMessage::DecryptionSessionError(serde_json::from_slice(&payload).map_err(|err| Error::Serde(err.to_string()))?)),
+
+		_ => return Err(Error::Serde(format!("unknown message type {}", header.kind))),
+	})
+}
+
+/// Encrypt serialized message.
+pub fn encrypt_message(_key: &Secret, message: SerializedMessage) -> Result<SerializedMessage, Error> {
+	Ok(message) // TODO: implement me
+}
+
+/// Decrypt serialized message.
+pub fn decrypt_message(_key: &Secret, payload: Vec<u8>) -> Result<Vec<u8>, Error> {
+	Ok(payload) // TODO: implement me
+}
+
+/// Compute shared encryption key.
+pub fn compute_shared_key(self_secret: &Secret, other_public: &Public) -> Result<Secret, Error> {
+	Ok(agree(self_secret, other_public)?)
+}
+
+/// Serialize message header.
+fn serialize_header(header: &MessageHeader) -> Result<Vec<u8>, Error> {
+	let mut buffer = Vec::with_capacity(MESSAGE_HEADER_SIZE);
+	buffer.write_u8(header.version)?;
+	buffer.write_u8(header.kind)?;
+	buffer.write_u16::<LittleEndian>(header.size)?;
+	Ok(buffer)
+}
+
+/// Deserialize message header.
+pub fn deserialize_header(data: &[u8]) -> Result<MessageHeader, Error> {
+	let mut reader = Cursor::new(data);
+	Ok(MessageHeader {
+		version: reader.read_u8()?,
+		kind: reader.read_u8()?,
+		size: reader.read_u16::<LittleEndian>()?,
+	})
+}
+
+#[cfg(test)]
+pub mod tests {
+	use std::io;
+	use ethkey::{KeyPair, Public};
+	use key_server_cluster::message::Message;
+	use super::{MESSAGE_HEADER_SIZE, MessageHeader, serialize_message, serialize_header, deserialize_header};
+
+	pub struct TestIo {
+		self_key_pair: KeyPair,
+		peer_public: Public,
+		input_buffer: io::Cursor<Vec<u8>>,
+		output_buffer: Vec<u8>,
+		expected_output_buffer: Vec<u8>,
+	}
+
+	impl TestIo {
+		pub fn new(self_key_pair: KeyPair, peer_public: Public) -> Self {
+			TestIo {
+				self_key_pair: self_key_pair,
+				peer_public: peer_public,
+				input_buffer: io::Cursor::new(Vec::new()),
+				output_buffer: Vec::new(),
+				expected_output_buffer: Vec::new(),
+			}
+		}
+
+		pub fn self_key_pair(&self) -> &KeyPair {
+			&self.self_key_pair
+		}
+
+		pub fn peer_public(&self) -> &Public {
+			&self.peer_public
+		}
+
+		pub fn add_input_message(&mut self, message: Message) {
+			let serialized_message = serialize_message(message).unwrap();
+			let serialized_message: Vec<_> = serialized_message.into();
+			let input_buffer = self.input_buffer.get_mut();
+			for b in serialized_message {
+				input_buffer.push(b);
+			}
+		}
+
+		pub fn add_output_message(&mut self, message: Message) {
+			let serialized_message = serialize_message(message).unwrap();
+			let serialized_message: Vec<_> = serialized_message.into();
+			self.expected_output_buffer.extend(serialized_message);
+		}
+
+		pub fn assert_output(&self) {
+			assert_eq!(self.output_buffer, self.expected_output_buffer);
+		}
+	}
+
+	impl io::Read for TestIo {
+		fn read(&mut self, buf: &mut [u8]) -> io::Result<usize> {
+			io::Read::read(&mut self.input_buffer, buf)
+		}
+	}
+
+	impl io::Write for TestIo {
+		fn write(&mut self, buf: &[u8]) -> io::Result<usize> {
+			io::Write::write(&mut self.output_buffer, buf)
+		}
+
+		fn flush(&mut self) -> io::Result<()> {
+			io::Write::flush(&mut self.output_buffer)
+		}
+	}
+
+	#[test]
+	fn header_serialization_works() {
+		let header = MessageHeader {
+			kind: 1,
+			version: 2,
+			size: 3,
+		};
+
+		let serialized_header = serialize_header(&header).unwrap();
+		assert_eq!(serialized_header.len(), MESSAGE_HEADER_SIZE);
+
+		let deserialized_header = deserialize_header(&serialized_header).unwrap();
+		assert_eq!(deserialized_header, header);
+	}
+}

secret_store/src/key_server_cluster/io/mod.rs

@@ -0,0 +1,34 @@
+// Copyright 2015-2017 Parity Technologies (UK) Ltd.
+// This file is part of Parity.
+
+// Parity is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Parity is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Parity.  If not, see <http://www.gnu.org/licenses/>.
+
+mod deadline;
+mod handshake;
+mod message;
+mod read_header;
+mod read_payload;
+mod read_message;
+mod shared_tcp_stream;
+mod write_message;
+
+pub use self::deadline::{deadline, Deadline, DeadlineStatus};
+pub use self::handshake::{handshake, accept_handshake, Handshake, HandshakeResult};
+pub use self::message::{MessageHeader, SerializedMessage, serialize_message, deserialize_message,
+	encrypt_message, compute_shared_key};
+pub use self::read_header::{read_header, ReadHeader};
+pub use self::read_payload::{read_payload, read_encrypted_payload, ReadPayload};
+pub use self::read_message::{read_message, read_encrypted_message, ReadMessage};
+pub use self::shared_tcp_stream::SharedTcpStream;
+pub use self::write_message::{write_message, write_encrypted_message, WriteMessage};

secret_store/src/key_server_cluster/io/read_header.rs

@@ -0,0 +1,44 @@
+// Copyright 2015-2017 Parity Technologies (UK) Ltd.
+// This file is part of Parity.
+
+// Parity is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Parity is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Parity.  If not, see <http://www.gnu.org/licenses/>.
+
+use std::io;
+use futures::{Future, Poll, Async};
+use tokio_core::io::{ReadExact, read_exact};
+use key_server_cluster::Error;
+use key_server_cluster::io::message::{MESSAGE_HEADER_SIZE, MessageHeader, deserialize_header};
+
+/// Create future for read single message header from the stream.
+pub fn read_header<A>(a: A) -> ReadHeader<A> where A: io::Read {
+	ReadHeader {
+		reader: read_exact(a, [0; MESSAGE_HEADER_SIZE]),
+	}
+}
+
+/// Future for read single message header from the stream.
+pub struct ReadHeader<A> {
+	reader: ReadExact<A, [u8; MESSAGE_HEADER_SIZE]>,
+}
+
+impl<A> Future for ReadHeader<A> where A: io::Read {
+	type Item = (A, Result<MessageHeader, Error>);
+	type Error = io::Error;
+
+	fn poll(&mut self) -> Poll<Self::Item, Self::Error> {
+		let (read, data) = try_ready!(self.reader.poll());
+		let header = deserialize_header(&data);
+		Ok(Async::Ready((read, header)))
+	}
+}

secret_store/src/key_server_cluster/io/read_message.rs

@@ -0,0 +1,86 @@
+// Copyright 2015-2017 Parity Technologies (UK) Ltd.
+// This file is part of Parity.
+
+// Parity is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Parity is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Parity.  If not, see <http://www.gnu.org/licenses/>.
+
+use std::io;
+use futures::{Poll, Future, Async};
+use ethkey::Secret;
+use key_server_cluster::Error;
+use key_server_cluster::message::Message;
+use key_server_cluster::io::{read_header, ReadHeader, read_payload, read_encrypted_payload, ReadPayload};
+
+/// Create future for read single message from the stream.
+pub fn read_message<A>(a: A) -> ReadMessage<A> where A: io::Read {
+	ReadMessage {
+		key: None,
+		state: ReadMessageState::ReadHeader(read_header(a)),
+	}
+}
+
+/// Create future for read single encrypted message from the stream.
+pub fn read_encrypted_message<A>(a: A, key: Secret) -> ReadMessage<A> where A: io::Read {
+	ReadMessage {
+		key: Some(key),
+		state: ReadMessageState::ReadHeader(read_header(a)),
+	}
+}
+
+enum ReadMessageState<A> {
+	ReadHeader(ReadHeader<A>),
+	ReadPayload(ReadPayload<A>),
+	Finished,
+}
+
+/// Future for read single message from the stream.
+pub struct ReadMessage<A> {
+	key: Option<Secret>,
+	state: ReadMessageState<A>,
+}
+
+impl<A> Future for ReadMessage<A> where A: io::Read {
+	type Item = (A, Result<Message, Error>);
+	type Error = io::Error;
+
+	fn poll(&mut self) -> Poll<Self::Item, Self::Error> {
+		let (next, result) = match self.state {
+			ReadMessageState::ReadHeader(ref mut future) => {
+				let (read, header) = try_ready!(future.poll());
+				let header = match header {
+					Ok(header) => header,
+					Err(err) => return Ok((read, Err(err)).into()),
+				};
+
+				let future = match self.key.take() {
+					Some(key) => read_encrypted_payload(read, header, key),
+					None => read_payload(read, header),
+				};
+				let next = ReadMessageState::ReadPayload(future);
+				(next, Async::NotReady)
+			},
+			ReadMessageState::ReadPayload(ref mut future) => {
+				let (read, payload) = try_ready!(future.poll());
+				(ReadMessageState::Finished, Async::Ready((read, payload)))
+			},
+			ReadMessageState::Finished => panic!("poll ReadMessage after it's done"),
+		};
+
+		self.state = next;
+		match result {
+			// by polling again, we register new future
+			Async::NotReady => self.poll(),
+			result => Ok(result)
+		}
+	}
+}

secret_store/src/key_server_cluster/io/read_payload.rs

@@ -0,0 +1,64 @@
+// Copyright 2015-2017 Parity Technologies (UK) Ltd.
+// This file is part of Parity.
+
+// Parity is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Parity is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Parity.  If not, see <http://www.gnu.org/licenses/>.
+
+use std::io;
+use futures::{Poll, Future};
+use tokio_core::io::{read_exact, ReadExact};
+use ethkey::Secret;
+use key_server_cluster::Error;
+use key_server_cluster::message::Message;
+use key_server_cluster::io::message::{MessageHeader, deserialize_message, decrypt_message};
+
+/// Create future for read single message payload from the stream.
+pub fn read_payload<A>(a: A, header: MessageHeader) -> ReadPayload<A> where A: io::Read {
+	ReadPayload {
+		reader: read_exact(a, vec![0; header.size as usize]),
+		header: header,
+		key: None,
+	}
+}
+
+/// Create future for read single encrypted message payload from the stream.
+pub fn read_encrypted_payload<A>(a: A, header: MessageHeader, key: Secret) -> ReadPayload<A> where A: io::Read {
+	ReadPayload {
+		reader: read_exact(a, vec![0; header.size as usize]),
+		header: header,
+		key: Some(key),
+	}
+}
+
+/// Future for read single message payload from the stream.
+pub struct ReadPayload<A> {
+	reader: ReadExact<A, Vec<u8>>,
+	header: MessageHeader,
+	key: Option<Secret>,
+}
+
+impl<A> Future for ReadPayload<A> where A: io::Read {
+	type Item = (A, Result<Message, Error>);
+	type Error = io::Error;
+
+	fn poll(&mut self) -> Poll<Self::Item, Self::Error> {
+		let (read, data) = try_ready!(self.reader.poll());
+		let payload = if let Some(key) = self.key.take() {
+			decrypt_message(&key, data)
+				.and_then(|data| deserialize_message(&self.header, data))
+		} else {
+			deserialize_message(&self.header, data)
+		};
+		Ok((read, payload).into())
+	}
+}

secret_store/src/key_server_cluster/io/shared_tcp_stream.rs

@@ -0,0 +1,60 @@
+// Copyright 2015-2017 Parity Technologies (UK) Ltd.
+// This file is part of Parity.
+
+// Parity is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Parity is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Parity.  If not, see <http://www.gnu.org/licenses/>.
+
+use std::sync::Arc;
+use std::io::{Read, Write, Error};
+use tokio_core::net::TcpStream;
+
+/// Read+Write implementation for Arc<TcpStream>.
+pub struct SharedTcpStream {
+	io: Arc<TcpStream>,
+}
+
+impl SharedTcpStream {
+	pub fn new(a: Arc<TcpStream>) -> Self {
+		SharedTcpStream {
+			io: a,
+		}
+	}
+}
+
+impl From<TcpStream> for SharedTcpStream {
+	fn from(a: TcpStream) -> Self {
+		SharedTcpStream::new(Arc::new(a))
+	}
+}
+
+impl Read for SharedTcpStream {
+	fn read(&mut self, buf: &mut [u8]) -> Result<usize, Error> {
+		Read::read(&mut (&*self.io as &TcpStream), buf)
+	}
+}
+
+impl Write for SharedTcpStream {
+	fn write(&mut self, buf: &[u8]) -> Result<usize, Error> {
+		Write::write(&mut (&*self.io as &TcpStream), buf)
+	}
+
+	fn flush(&mut self) -> Result<(), Error> {
+		Write::flush(&mut (&*self.io as &TcpStream))
+	}
+}
+
+impl Clone for SharedTcpStream {
+	fn clone(&self) -> Self {
+		SharedTcpStream::new(self.io.clone())
+	}
+}

secret_store/src/key_server_cluster/io/write_message.rs

@@ -0,0 +1,70 @@
+// Copyright 2015-2017 Parity Technologies (UK) Ltd.
+// This file is part of Parity.
+
+// Parity is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Parity is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Parity.  If not, see <http://www.gnu.org/licenses/>.
+
+use std::io;
+use futures::{Future, Poll};
+use tokio_core::io::{WriteAll, write_all};
+use ethkey::Secret;
+use key_server_cluster::message::Message;
+use key_server_cluster::io::{serialize_message, encrypt_message};
+
+/// Write plain message to the channel.
+pub fn write_message<A>(a: A, message: Message) -> WriteMessage<A> where A: io::Write {
+	let (error, future) = match serialize_message(message)
+		.map_err(|e| io::Error::new(io::ErrorKind::Other, e.to_string())) {
+		Ok(message) => (None, write_all(a, message.into())),
+		Err(error) => (Some(error), write_all(a, Vec::new())),
+	};
+	WriteMessage {
+		error: error,
+		future: future,
+	}
+}
+
+/// Write encrypted message to the channel.
+pub fn write_encrypted_message<A>(a: A, key: &Secret, message: Message) -> WriteMessage<A> where A: io::Write {
+	let (error, future) = match serialize_message(message)
+		.and_then(|message| encrypt_message(key, message))
+		.map_err(|e| io::Error::new(io::ErrorKind::Other, e.to_string())) {
+		Ok(message) => (None, write_all(a, message.into())),
+		Err(error) => (Some(error), write_all(a, Vec::new())),
+	};
+
+
+	WriteMessage {
+		error: error,
+		future: future,
+	}
+}
+
+/// Future message write.
+pub struct WriteMessage<A> {
+	error: Option<io::Error>,
+	future: WriteAll<A, Vec<u8>>,
+}
+
+impl<A> Future for WriteMessage<A> where A: io::Write {
+	type Item = (A, Vec<u8>);
+	type Error = io::Error;
+
+	fn poll(&mut self) -> Poll<Self::Item, Self::Error> {
+		if let Some(err) = self.error.take() {
+			return Err(err);
+		}
+
+		self.future.poll()
+	}
+}

secret_store/src/key_server_cluster/math.rs

@@ -160,7 +160,7 @@ pub fn compute_joint_secret<'a, I>(mut secret_coeffs: I) -> Result<Secret, Error
 }
 
 /// Encrypt secret with joint public key.
-pub fn encrypt_secret(secret: Public, joint_public: &Public) -> Result<EncryptedSecret, Error> {
+pub fn encrypt_secret(secret: &Public, joint_public: &Public) -> Result<EncryptedSecret, Error> {
 	// this is performed by KS-cluster client (or KS master)
 	let key_pair = Random.generate()?;
 
@@ -171,7 +171,7 @@ pub fn encrypt_secret(secret: Public, joint_public: &Public) -> Result<Encrypted
 	// M + k * y
 	let mut encrypted_point = joint_public.clone();
 	math::public_mul_secret(&mut encrypted_point, key_pair.secret())?;
-	math::public_add(&mut encrypted_point, &secret)?;
+	math::public_add(&mut encrypted_point, secret)?;
 
 	Ok(EncryptedSecret {
 		common_point: common_point,
@@ -181,7 +181,11 @@ pub fn encrypt_secret(secret: Public, joint_public: &Public) -> Result<Encrypted
 
 /// Compute shadow for the node.
 pub fn compute_node_shadow<'a, I>(node_number: &Secret, node_secret_share: &Secret, mut other_nodes_numbers: I) -> Result<Secret, Error> where I: Iterator<Item=&'a Secret> {
-	let other_node_number = other_nodes_numbers.next().expect("compute_node_shadow is called when at least two nodes are required to decrypt secret; qed");
+	let other_node_number = match other_nodes_numbers.next() {
+		Some(other_node_number) => other_node_number,
+		None => return Ok(node_secret_share.clone()),
+	};
+
 	let mut shadow = node_number.clone();
 	shadow.sub(other_node_number)?;
 	shadow.inv()?;
@@ -231,17 +235,24 @@ pub fn compute_joint_shadow_point_test<'a, I>(access_key: &Secret, common_point:
 }
 
 /// Decrypt data using joint shadow point.
-pub fn decrypt_with_joint_shadow(access_key: &Secret, encrypted_point: &Public, joint_shadow_point: &Public) -> Result<Public, Error> {
+pub fn decrypt_with_joint_shadow(threshold: usize, access_key: &Secret, encrypted_point: &Public, joint_shadow_point: &Public) -> Result<Public, Error> {
 	let mut inv_access_key = access_key.clone();
 	inv_access_key.inv()?;
 
-	let mut decrypted_point = joint_shadow_point.clone();
+	let mut mul = joint_shadow_point.clone();
-	math::public_mul_secret(&mut decrypted_point, &inv_access_key)?;
+	math::public_mul_secret(&mut mul, &inv_access_key)?;
-	math::public_add(&mut decrypted_point, encrypted_point)?;
+
+	let mut decrypted_point = encrypted_point.clone();
+	if threshold % 2 != 0 {
+		math::public_add(&mut decrypted_point, &mul)?;
+	} else {
+		math::public_sub(&mut decrypted_point, &mul)?;
+	}
 
 	Ok(decrypted_point)
 }
 
+#[cfg(test)]
 /// Decrypt data using joint secret (version for tests).
 pub fn decrypt_with_joint_secret(encrypted_point: &Public, common_point: &Public, joint_secret: &Secret) -> Result<Public, Error> {
 	let mut common_point_mul = common_point.clone();
@@ -262,7 +273,7 @@ pub mod tests {
 		// === PART2: encryption using joint public key ===
 
 		// the next line is executed on KeyServer-client
-		let encrypted_secret = encrypt_secret(document_secret_plain.clone(), &joint_public).unwrap();
+		let encrypted_secret = encrypt_secret(&document_secret_plain, &joint_public).unwrap();
 
 		// === PART3: decryption ===
 
@@ -285,7 +296,7 @@ pub mod tests {
 		assert_eq!(joint_shadow_point, joint_shadow_point_test);
 
 		// decrypt encrypted secret using joint shadow point
-		let document_secret_decrypted = decrypt_with_joint_shadow(&access_key, &encrypted_secret.encrypted_point, &joint_shadow_point).unwrap();
+		let document_secret_decrypted = decrypt_with_joint_shadow(t, &access_key, &encrypted_secret.encrypted_point, &joint_shadow_point).unwrap();
 
 		// decrypt encrypted secret using joint secret [just for test]
 		let document_secret_decrypted_test = match joint_secret {
@@ -298,7 +309,8 @@ pub mod tests {
 
 	#[test]
 	fn full_encryption_math_session() {
-		let test_cases = [(1, 3)];
+		let test_cases = [(0, 2), (1, 2), (1, 3), (2, 3), (1, 4), (2, 4), (3, 4), (1, 5), (2, 5), (3, 5), (4, 5),
+			(1, 10), (2, 10), (3, 10), (4, 10), (5, 10), (6, 10), (7, 10), (8, 10), (9, 10)];
 		for &(t, n) in &test_cases {
 			// === PART1: DKG ===
 			

secret_store/src/key_server_cluster/message.rs

@@ -14,13 +14,42 @@
 // You should have received a copy of the GNU General Public License
 // along with Parity.  If not, see <http://www.gnu.org/licenses/>.
 
+use std::fmt;
 use std::collections::{BTreeSet, BTreeMap};
-use ethkey::{Public, Secret, Signature};
+use ethkey::Secret;
-use key_server_cluster::{NodeId, SessionId};
+use key_server_cluster::SessionId;
+use super::{SerializableH256, SerializablePublic, SerializableSecret, SerializableSignature};
+
+pub type MessageSessionId = SerializableH256;
+pub type MessageNodeId = SerializablePublic;
 
 #[derive(Clone, Debug)]
-/// All possible messages that can be sent during DKG.
+/// All possible messages that can be sent during encryption/decryption sessions.
 pub enum Message {
+	/// Cluster message.
+	Cluster(ClusterMessage),
+	/// Encryption message.
+	Encryption(EncryptionMessage),
+	/// Decryption message.
+	Decryption(DecryptionMessage),
+}
+
+#[derive(Clone, Debug)]
+/// All possible cluster-level messages.
+pub enum ClusterMessage {
+	/// Introduce node public key.
+	NodePublicKey(NodePublicKey),
+	/// Confirm that node owns its private key.
+	NodePrivateKeySignature(NodePrivateKeySignature),
+	/// Keep alive message.
+	KeepAlive(KeepAlive),
+	/// Keep alive message response.
+	KeepAliveResponse(KeepAliveResponse),
+}
+
+#[derive(Clone, Debug)]
+/// All possible messages that can be sent during encryption session.
+pub enum EncryptionMessage {
 	/// Initialize new DKG session.
 	InitializeSession(InitializeSession),
 	/// Confirm DKG session initialization.
@@ -35,7 +64,15 @@ pub enum Message {
 	ComplaintResponse(ComplaintResponse),
 	/// Broadcast self public key portion.
 	PublicKeyShare(PublicKeyShare),
+	/// When session error has occured.
+	SessionError(SessionError),
+	/// When session is completed.
+	SessionCompleted(SessionCompleted),
+}
 
+#[derive(Clone, Debug)]
+/// All possible messages that can be sent during decryption session.
+pub enum DecryptionMessage {
 	/// Initialize decryption session.
 	InitializeDecryptionSession(InitializeDecryptionSession),
 	/// Confirm/reject decryption session initialization.
@@ -44,125 +81,272 @@ pub enum Message {
 	RequestPartialDecryption(RequestPartialDecryption),
 	/// Partial decryption is completed
 	PartialDecryption(PartialDecryption),
+	/// When decryption session error has occured.
+	DecryptionSessionError(DecryptionSessionError),
 }
 
-#[derive(Clone, Debug)]
+#[derive(Clone, Debug, Serialize, Deserialize)]
+/// Introduce node public key.
+pub struct NodePublicKey {
+	/// Node identifier (aka node public key).
+	pub node_id: MessageNodeId,
+	/// Data, which must be signed by peer to prove that he owns the corresponding private key. 
+	pub confirmation_plain: SerializableH256,
+}
+
+#[derive(Clone, Debug, Serialize, Deserialize)]
+/// Confirm that node owns the private key of previously passed public key (aka node id).
+pub struct NodePrivateKeySignature {
+	/// Previously passed `confirmation_plain`, signed with node private key.
+	pub confirmation_signed: SerializableSignature,
+}
+
+
+#[derive(Clone, Debug, Serialize, Deserialize)]
+/// Ask if the node is still alive.
+pub struct KeepAlive {
+}
+
+#[derive(Clone, Debug, Serialize, Deserialize)]
+/// Confirm that the node is still alive.
+pub struct KeepAliveResponse {
+}
+
+#[derive(Clone, Debug, Serialize, Deserialize)]
 /// Initialize new DKG session.
 pub struct InitializeSession {
 	/// Session Id.
-	pub session: SessionId,
+	pub session: MessageSessionId,
 	/// Derived generation point. Starting from originator, every node must multiply this
 	/// point by random scalar (unknown by other nodes). At the end of initialization
 	/// `point` will be some (k1 * k2 * ... * kn) * G = `point` where `(k1 * k2 * ... * kn)`
 	/// is unknown for every node.
-	pub derived_point: Public,
+	pub derived_point: SerializablePublic,
 }
 
-#[derive(Clone, Debug)]
+#[derive(Clone, Debug, Serialize, Deserialize)]
 /// Confirm DKG session initialization.
 pub struct ConfirmInitialization {
 	/// Session Id.
-	pub session: SessionId,
+	pub session: MessageSessionId,
 	/// Derived generation point.
-	pub derived_point: Public,
+	pub derived_point: SerializablePublic,
 }
 
-#[derive(Clone, Debug)]
+#[derive(Clone, Debug, Serialize, Deserialize)]
 /// Broadcast generated point to every other node.
 pub struct CompleteInitialization {
 	/// Session Id.
-	pub session: SessionId,
+	pub session: MessageSessionId,
 	/// All session participants along with their identification numbers.
-	pub nodes: BTreeMap<NodeId, Secret>,
+	pub nodes: BTreeMap<MessageNodeId, SerializableSecret>,
 	/// Decryption threshold. During decryption threshold-of-route.len() nodes must came to
 	/// consensus to successfully decrypt message.
 	pub threshold: usize,
 	/// Derived generation point.
-	pub derived_point: Public,
+	pub derived_point: SerializablePublic,
 }
 
-#[derive(Clone, Debug)]
+#[derive(Clone, Debug, Serialize, Deserialize)]
 /// Generated keys are sent to every node.
 pub struct KeysDissemination {
 	/// Session Id.
-	pub session: SessionId,
+	pub session: MessageSessionId,
 	/// Secret 1.
-	pub secret1: Secret,
+	pub secret1: SerializableSecret,
 	/// Secret 2.
-	pub secret2: Secret,
+	pub secret2: SerializableSecret,
 	/// Public values.
-	pub publics: Vec<Public>,
+	pub publics: Vec<SerializablePublic>,
 }
 
-#[derive(Clone, Debug)]
+#[derive(Clone, Debug, Serialize, Deserialize)]
 /// Complaint against node is broadcasted.
 pub struct Complaint {
 	/// Session Id.
-	pub session: SessionId,
+	pub session: MessageSessionId,
 	/// Public values.
-	pub against: NodeId,
+	pub against: MessageNodeId,
 }
 
-#[derive(Clone, Debug)]
+#[derive(Clone, Debug, Serialize, Deserialize)]
 /// Node is responding to complaint.
 pub struct ComplaintResponse {
 	/// Session Id.
-	pub session: SessionId,
+	pub session: MessageSessionId,
 	/// Secret 1.
-	pub secret1: Secret,
+	pub secret1: SerializableSecret,
 	/// Secret 2.
-	pub secret2: Secret,
+	pub secret2: SerializableSecret,
 }
 
-#[derive(Clone, Debug)]
+#[derive(Clone, Debug, Serialize, Deserialize)]
 /// Node is sharing its public key share.
 pub struct PublicKeyShare {
 	/// Session Id.
-	pub session: SessionId,
+	pub session: MessageSessionId,
 	/// Public key share.
-	pub public_share: Public,
+	pub public_share: SerializablePublic,
 }
 
-#[derive(Clone, Debug)]
+#[derive(Clone, Debug, Serialize, Deserialize)]
+/// When session error has occured.
+pub struct SessionError {
+	/// Session Id.
+	pub session: MessageSessionId,
+	/// Public key share.
+	pub error: String,
+}
+
+#[derive(Clone, Debug, Serialize, Deserialize)]
+/// When session is completed.
+pub struct SessionCompleted {
+	/// Session Id.
+	pub session: MessageSessionId,
+	/// Common (shared) encryption point.
+	pub common_point: SerializablePublic,
+	/// Encrypted point.
+	pub encrypted_point: SerializablePublic,
+}
+
+#[derive(Clone, Debug, Serialize, Deserialize)]
 /// Node is requested to decrypt data, encrypted in given session.
 pub struct InitializeDecryptionSession {
 	/// Encryption session Id.
-	pub session: SessionId,
+	pub session: MessageSessionId,
 	/// Decryption session Id.
-	pub sub_session: Secret,
+	pub sub_session: SerializableSecret,
 	/// Requestor signature.
-	pub requestor_signature: Signature,
+	pub requestor_signature: SerializableSignature,
 }
 
-#[derive(Clone, Debug)]
+#[derive(Clone, Debug, Serialize, Deserialize)]
 /// Node is responding to decryption request.
 pub struct ConfirmDecryptionInitialization {
 	/// Encryption session Id.
-	pub session: SessionId,
+	pub session: MessageSessionId,
 	/// Decryption session Id.
-	pub sub_session: Secret,
+	pub sub_session: SerializableSecret,
 	/// Is node confirmed to make a decryption?.
 	pub is_confirmed: bool,
 }
 
-#[derive(Clone, Debug)]
+#[derive(Clone, Debug, Serialize, Deserialize)]
 /// Node is requested to do a partial decryption.
 pub struct RequestPartialDecryption {
 	/// Encryption session Id.
-	pub session: SessionId,
+	pub session: MessageSessionId,
 	/// Decryption session Id.
-	pub sub_session: Secret,
+	pub sub_session: SerializableSecret,
 	/// Nodes that are agreed to do a decryption.
-	pub nodes: BTreeSet<NodeId>,
+	pub nodes: BTreeSet<MessageNodeId>,
 }
 
-#[derive(Clone, Debug)]
+#[derive(Clone, Debug, Serialize, Deserialize)]
 /// Node has partially decrypted the secret.
 pub struct PartialDecryption {
 	/// Encryption session Id.
-	pub session: SessionId,
+	pub session: MessageSessionId,
 	/// Decryption session Id.
-	pub sub_session: Secret,
+	pub sub_session: SerializableSecret,
 	/// Partially decrypted secret.
-	pub shadow_point: Public,
+	pub shadow_point: SerializablePublic,
+}
+
+#[derive(Clone, Debug, Serialize, Deserialize)]
+/// When decryption session error has occured.
+pub struct DecryptionSessionError {
+	/// Encryption session Id.
+	pub session: MessageSessionId,
+	/// Decryption session Id.
+	pub sub_session: SerializableSecret,
+	/// Public key share.
+	pub error: String,
+}
+
+impl EncryptionMessage {
+	pub fn session_id(&self) -> &SessionId {
+		match *self {
+			EncryptionMessage::InitializeSession(ref msg) => &msg.session,
+			EncryptionMessage::ConfirmInitialization(ref msg) => &msg.session,
+			EncryptionMessage::CompleteInitialization(ref msg) => &msg.session,
+			EncryptionMessage::KeysDissemination(ref msg) => &msg.session,
+			EncryptionMessage::Complaint(ref msg) => &msg.session,
+			EncryptionMessage::ComplaintResponse(ref msg) => &msg.session,
+			EncryptionMessage::PublicKeyShare(ref msg) => &msg.session,
+			EncryptionMessage::SessionError(ref msg) => &msg.session,
+			EncryptionMessage::SessionCompleted(ref msg) => &msg.session,
+		}
+	}
+}
+
+impl DecryptionMessage {
+	pub fn session_id(&self) -> &SessionId {
+		match *self {
+			DecryptionMessage::InitializeDecryptionSession(ref msg) => &msg.session,
+			DecryptionMessage::ConfirmDecryptionInitialization(ref msg) => &msg.session,
+			DecryptionMessage::RequestPartialDecryption(ref msg) => &msg.session,
+			DecryptionMessage::PartialDecryption(ref msg) => &msg.session,
+			DecryptionMessage::DecryptionSessionError(ref msg) => &msg.session,
+		}
+	}
+
+	pub fn sub_session_id(&self) -> &Secret {
+		match *self {
+			DecryptionMessage::InitializeDecryptionSession(ref msg) => &msg.sub_session,
+			DecryptionMessage::ConfirmDecryptionInitialization(ref msg) => &msg.sub_session,
+			DecryptionMessage::RequestPartialDecryption(ref msg) => &msg.sub_session,
+			DecryptionMessage::PartialDecryption(ref msg) => &msg.sub_session,
+			DecryptionMessage::DecryptionSessionError(ref msg) => &msg.sub_session,
+		}
+	}
+}
+
+impl fmt::Display for Message {
+	fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+		match *self {
+			Message::Cluster(ref message) => write!(f, "Cluster.{}", message),
+			Message::Encryption(ref message) => write!(f, "Encryption.{}", message),
+			Message::Decryption(ref message) => write!(f, "Decryption.{}", message),
+		}
+	}
+}
+
+impl fmt::Display for ClusterMessage {
+	fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+		match *self {
+			ClusterMessage::NodePublicKey(_) => write!(f, "NodePublicKey"),
+			ClusterMessage::NodePrivateKeySignature(_) => write!(f, "NodePrivateKeySignature"),
+			ClusterMessage::KeepAlive(_) => write!(f, "KeepAlive"),
+			ClusterMessage::KeepAliveResponse(_) => write!(f, "KeepAliveResponse"),
+		}
+	}
+}
+
+impl fmt::Display for EncryptionMessage {
+	fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+		match *self {
+			EncryptionMessage::InitializeSession(_) => write!(f, "InitializeSession"),
+			EncryptionMessage::ConfirmInitialization(_) => write!(f, "ConfirmInitialization"),
+			EncryptionMessage::CompleteInitialization(_) => write!(f, "CompleteInitialization"),
+			EncryptionMessage::KeysDissemination(_) => write!(f, "KeysDissemination"),
+			EncryptionMessage::Complaint(_) => write!(f, "Complaint"),
+			EncryptionMessage::ComplaintResponse(_) => write!(f, "ComplaintResponse"),
+			EncryptionMessage::PublicKeyShare(_) => write!(f, "PublicKeyShare"),
+			EncryptionMessage::SessionError(ref msg) => write!(f, "SessionError({})", msg.error),
+			EncryptionMessage::SessionCompleted(_) => write!(f, "SessionCompleted"),
+		}
+	}
+}
+
+impl fmt::Display for DecryptionMessage {
+	fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+		match *self {
+			DecryptionMessage::InitializeDecryptionSession(_) => write!(f, "InitializeDecryptionSession"),
+			DecryptionMessage::ConfirmDecryptionInitialization(_) => write!(f, "ConfirmDecryptionInitialization"),
+			DecryptionMessage::RequestPartialDecryption(_) => write!(f, "RequestPartialDecryption"),
+			DecryptionMessage::PartialDecryption(_) => write!(f, "PartialDecryption"),
+			DecryptionMessage::DecryptionSessionError(_) => write!(f, "DecryptionSessionError"),
+		}
+	}
 }

secret_store/src/key_server_cluster/mod.rs

@@ -14,21 +14,36 @@
 // You should have received a copy of the GNU General Public License
 // along with Parity.  If not, see <http://www.gnu.org/licenses/>.
 
-#![allow(dead_code)] // TODO: remove me
+use std::fmt;
-
+use std::io::Error as IoError;
-use std::collections::BTreeMap;
+use ethkey;
-use ethkey::{self, Public, Secret, Signature};
+use ethcrypto;
 use super::types::all::DocumentAddress;
 
-pub use super::acl_storage::AclStorage;
+pub use super::types::all::{NodeId, EncryptionConfiguration};
+pub use super::acl_storage::{AclStorage, DummyAclStorage};
+pub use super::key_storage::{KeyStorage, DocumentKeyShare};
+pub use super::serialization::{SerializableSignature, SerializableH256, SerializableSecret, SerializablePublic};
+pub use self::cluster::{ClusterCore, ClusterConfiguration, ClusterClient};
+pub use self::encryption_session::Session as EncryptionSession;
+pub use self::decryption_session::Session as DecryptionSession;
+
+#[cfg(test)]
+pub use super::key_storage::tests::DummyKeyStorage;
 
-pub type NodeId = Public;
 pub type SessionId = DocumentAddress;
-pub type SessionIdSignature = Signature;
 
-#[derive(Debug, PartialEq)]
+#[derive(Clone, Debug, PartialEq)]
 /// Errors which can occur during encryption/decryption session
 pub enum Error {
+	/// Invalid node address has been passed.
+	InvalidNodeAddress,
+	/// Invalid node id has been passed.
+	InvalidNodeId,
+	/// Session with the given id already exists.
+	DuplicateSessionId,
+	/// Session with the given id is unknown.
+	InvalidSessionId,
 	/// Invalid number of nodes.
 	/// There must be at least two nodes participating in encryption.
 	/// There must be at least one node participating in decryption.
@@ -39,28 +54,24 @@ pub enum Error {
 	/// Threshold value must be in [0; n - 1], where n is a number of nodes participating in the encryption.
 	InvalidThreshold,
 	/// Current state of encryption/decryption session does not allow to proceed request.
+	/// Reschedule this request for later processing.
+	TooEarlyForRequest,
+	/// Current state of encryption/decryption session does not allow to proceed request.
 	/// This means that either there is some comm-failure or node is misbehaving/cheating.
 	InvalidStateForRequest,
-	/// Some data in passed message was recognized as invalid.
+	/// Message or some data in the message was recognized as invalid.
 	/// This means that node is misbehaving/cheating.
 	InvalidMessage,
+	/// Connection to node, required for this session is not established.
+	NodeDisconnected,
 	/// Cryptographic error.
 	EthKey(String),
-}
+	/// I/O error has occured.
-
+	Io(String),
-#[derive(Debug, Clone)]
+	/// Deserialization error has occured.
-/// Data, which is stored on every node after DKG && encryption is completed.
+	Serde(String),
-pub struct EncryptedData {
+	/// Key storage error.
-	/// Decryption threshold (at least threshold + 1 nodes are required to decrypt data).
+	KeyStorage(String),
-	threshold: usize,
-	/// Nodes ids numbers.
-	id_numbers: BTreeMap<NodeId, Secret>,
-	/// Node secret share.
-	secret_share: Secret,
-	/// Common (shared) encryption point.
-	common_point: Public,
-	/// Encrypted point.
-	encrypted_point: Public,
 }
 
 impl From<ethkey::Error> for Error {
@@ -69,8 +80,50 @@ impl From<ethkey::Error> for Error {
 	}
 }
 
+impl From<ethcrypto::Error> for Error {
+	fn from(err: ethcrypto::Error) -> Self {
+		Error::EthKey(err.into())
+	}
+}
+
+impl From<IoError> for Error {
+	fn from(err: IoError) -> Self {
+		Error::Io(err.to_string())
+	}
+}
+
+impl fmt::Display for Error {
+	fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+		match *self {
+			Error::InvalidNodeAddress => write!(f, "invalid node address has been passed"),
+			Error::InvalidNodeId => write!(f, "invalid node id has been passed"),
+			Error::DuplicateSessionId => write!(f, "session with the same id is already registered"),
+			Error::InvalidSessionId => write!(f, "invalid session id has been passed"),
+			Error::InvalidNodesCount => write!(f, "invalid nodes count"),
+			Error::InvalidNodesConfiguration => write!(f, "invalid nodes configuration"),
+			Error::InvalidThreshold => write!(f, "invalid threshold value has been passed"),
+			Error::TooEarlyForRequest => write!(f, "session is not yet ready to process this request"),
+			Error::InvalidStateForRequest => write!(f, "session is in invalid state for processing this request"),
+			Error::InvalidMessage => write!(f, "invalid message is received"),
+			Error::NodeDisconnected => write!(f, "node required for this operation is currently disconnected"),
+			Error::EthKey(ref e) => write!(f, "cryptographic error {}", e),
+			Error::Io(ref e) => write!(f, "i/o error {}", e),
+			Error::Serde(ref e) => write!(f, "serde error {}", e),
+			Error::KeyStorage(ref e) => write!(f, "key storage error {}", e),
+		}
+	}
+}
+
+impl Into<String> for Error {
+	fn into(self) -> String {
+		format!("{}", self)
+	}
+}
+
 mod cluster;
 mod decryption_session;
 mod encryption_session;
+mod io;
 mod math;
 mod message;
+mod net;

secret_store/src/key_server_cluster/net/accept_connection.rs

@@ -0,0 +1,63 @@
+// Copyright 2015-2017 Parity Technologies (UK) Ltd.
+// This file is part of Parity.
+
+// Parity is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Parity is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Parity.  If not, see <http://www.gnu.org/licenses/>.
+
+use std::io;
+use std::net::SocketAddr;
+use std::time::Duration;
+use std::collections::BTreeSet;
+use futures::{Future, Poll};
+use tokio_core::reactor::Handle;
+use tokio_core::net::TcpStream;
+use ethkey::KeyPair;
+use key_server_cluster::{Error, NodeId};
+use key_server_cluster::io::{accept_handshake, Handshake, Deadline, deadline};
+use key_server_cluster::net::Connection;
+
+/// Create future for accepting incoming connection.
+pub fn accept_connection(address: SocketAddr, stream: TcpStream, handle: &Handle, self_key_pair: KeyPair, trusted_nodes: BTreeSet<NodeId>) -> Deadline<AcceptConnection> {
+	let accept = AcceptConnection {
+		handshake: accept_handshake(stream, self_key_pair, trusted_nodes),
+		address: address,
+	};
+
+	deadline(Duration::new(5, 0), handle, accept).expect("Failed to create timeout")
+}
+
+/// Future for accepting incoming connection.
+pub struct AcceptConnection {
+	handshake: Handshake<TcpStream>,
+	address: SocketAddr,
+}
+
+impl Future for AcceptConnection {
+	type Item = Result<Connection, Error>;
+	type Error = io::Error;
+
+	fn poll(&mut self) -> Poll<Self::Item, Self::Error> {
+		let (stream, result) = try_ready!(self.handshake.poll());
+		let result = match result {
+			Ok(result) => result,
+			Err(err) => return Ok(Err(err).into()),
+		};
+		let connection = Connection {
+			stream: stream.into(),
+			address: self.address,
+			node_id: result.node_id,
+			key: result.shared_key,
+		};
+		Ok(Ok(connection).into())
+	}
+}

secret_store/src/key_server_cluster/net/connect.rs

@@ -0,0 +1,90 @@
+// Copyright 2015-2017 Parity Technologies (UK) Ltd.
+// This file is part of Parity.
+
+// Parity is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Parity is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Parity.  If not, see <http://www.gnu.org/licenses/>.
+
+use std::collections::BTreeSet;
+use std::io;
+use std::time::Duration;
+use std::net::SocketAddr;
+use futures::{Future, Poll, Async};
+use tokio_core::reactor::Handle;
+use tokio_core::net::{TcpStream, TcpStreamNew};
+use ethkey::KeyPair;
+use key_server_cluster::{Error, NodeId};
+use key_server_cluster::io::{handshake, Handshake, Deadline, deadline};
+use key_server_cluster::net::Connection;
+
+/// Create future for connecting to other node.
+pub fn connect(address: &SocketAddr, handle: &Handle, self_key_pair: KeyPair, trusted_nodes: BTreeSet<NodeId>) -> Deadline<Connect> {
+	let connect = Connect {
+		state: ConnectState::TcpConnect(TcpStream::connect(address, handle)),
+		address: address.clone(),
+		self_key_pair: self_key_pair,
+		trusted_nodes: trusted_nodes,
+	};
+
+	deadline(Duration::new(5, 0), handle, connect).expect("Failed to create timeout")
+}
+
+enum ConnectState {
+	TcpConnect(TcpStreamNew),
+	Handshake(Handshake<TcpStream>),
+	Connected,
+}
+
+/// Future for connecting to other node.
+pub struct Connect {
+	state: ConnectState,
+	address: SocketAddr,
+	self_key_pair: KeyPair,
+	trusted_nodes: BTreeSet<NodeId>,
+}
+
+impl Future for Connect {
+	type Item = Result<Connection, Error>;
+	type Error = io::Error;
+
+	fn poll(&mut self) -> Poll<Self::Item, Self::Error> {
+		let (next, result) = match self.state {
+			ConnectState::TcpConnect(ref mut future) => {
+				let stream = try_ready!(future.poll());
+				let handshake = handshake(stream, self.self_key_pair.clone(), self.trusted_nodes.clone());
+				(ConnectState::Handshake(handshake), Async::NotReady)
+			},
+			ConnectState::Handshake(ref mut future) => {
+				let (stream, result) = try_ready!(future.poll());
+				let result = match result {
+					Ok(result) => result,
+					Err(err) => return Ok(Async::Ready(Err(err))),
+				};
+				let connection = Connection {
+					stream: stream.into(),
+					address: self.address,
+					node_id: result.node_id,
+					key: result.shared_key,
+				};
+				(ConnectState::Connected, Async::Ready(Ok(connection)))
+			},
+			ConnectState::Connected => panic!("poll Connect after it's done"),
+		};
+
+		self.state = next;
+		match result {
+			// by polling again, we register new future
+			Async::NotReady => self.poll(),
+			result => Ok(result)
+		}
+	}
+}

secret_store/src/key_server_cluster/net/connection.rs

@@ -0,0 +1,32 @@
+// Copyright 2015-2017 Parity Technologies (UK) Ltd.
+// This file is part of Parity.
+
+// Parity is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Parity is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Parity.  If not, see <http://www.gnu.org/licenses/>.
+
+use std::net;
+use ethkey::Secret;
+use key_server_cluster::NodeId;
+use key_server_cluster::io::SharedTcpStream;
+
+/// Established connection data
+pub struct Connection {
+	/// Peer address.
+	pub address: net::SocketAddr,
+	/// Connection stream.
+	pub stream: SharedTcpStream,
+	/// Peer node id.
+	pub node_id: NodeId,
+	/// Encryption key.
+	pub key: Secret,
+}

secret_store/src/key_server_cluster/net/mod.rs

@@ -0,0 +1,23 @@
+// Copyright 2015-2017 Parity Technologies (UK) Ltd.
+// This file is part of Parity.
+
+// Parity is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Parity is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Parity.  If not, see <http://www.gnu.org/licenses/>.
+
+mod accept_connection;
+mod connect;
+mod connection;
+
+pub use self::accept_connection::{AcceptConnection, accept_connection};
+pub use self::connect::{Connect, connect};
+pub use self::connection::Connection;

secret_store/src/key_storage.rs

@@ -15,15 +15,34 @@
 // along with Parity.  If not, see <http://www.gnu.org/licenses/>.
 
 use std::path::PathBuf;
+use std::collections::BTreeMap;
+use serde_json;
+use ethkey::{Secret, Public};
 use util::Database;
-use types::all::{Error, ServiceConfiguration, DocumentAddress, DocumentKey};
+use types::all::{Error, ServiceConfiguration, DocumentAddress, NodeId};
+use serialization::{SerializablePublic, SerializableSecret};
+
+#[derive(Debug, Clone, PartialEq)]
+/// Encrypted key share, stored by key storage on the single key server.
+pub struct DocumentKeyShare {
+	/// Decryption threshold (at least threshold + 1 nodes are required to decrypt data).
+	pub threshold: usize,
+	/// Nodes ids numbers.
+	pub id_numbers: BTreeMap<NodeId, Secret>,
+	/// Node secret share.
+	pub secret_share: Secret,
+	/// Common (shared) encryption point.
+	pub common_point: Public,
+	/// Encrypted point.
+	pub encrypted_point: Public,
+}
 
 /// Document encryption keys storage
 pub trait KeyStorage: Send + Sync {
 	/// Insert document encryption key
-	fn insert(&self, document: DocumentAddress, key: DocumentKey) -> Result<(), Error>;
+	fn insert(&self, document: DocumentAddress, key: DocumentKeyShare) -> Result<(), Error>;
 	/// Get document encryption key
-	fn get(&self, document: &DocumentAddress) -> Result<DocumentKey, Error>;
+	fn get(&self, document: &DocumentAddress) -> Result<DocumentKeyShare, Error>;
 }
 
 /// Persistent document encryption keys storage
@@ -31,6 +50,21 @@ pub struct PersistentKeyStorage {
 	db: Database,
 }
 
+#[derive(Serialize, Deserialize)]
+/// Encrypted key share, as it is stored by key storage on the single key server.
+struct SerializableDocumentKeyShare {
+	/// Decryption threshold (at least threshold + 1 nodes are required to decrypt data).
+	pub threshold: usize,
+	/// Nodes ids numbers.
+	pub id_numbers: BTreeMap<SerializablePublic, SerializableSecret>,
+	/// Node secret share.
+	pub secret_share: SerializableSecret,
+	/// Common (shared) encryption point.
+	pub common_point: SerializablePublic,
+	/// Encrypted point.
+	pub encrypted_point: SerializablePublic,
+}
+
 impl PersistentKeyStorage {
 	/// Create new persistent document encryption keys storage
 	pub fn new(config: &ServiceConfiguration) -> Result<Self, Error> {
@@ -45,41 +79,71 @@ impl PersistentKeyStorage {
 }
 
 impl KeyStorage for PersistentKeyStorage {
-	fn insert(&self, document: DocumentAddress, key: DocumentKey) -> Result<(), Error> {
+	fn insert(&self, document: DocumentAddress, key: DocumentKeyShare) -> Result<(), Error> {
+		let key: SerializableDocumentKeyShare = key.into();
+		let key = serde_json::to_vec(&key).map_err(|e| Error::Database(e.to_string()))?;
 		let mut batch = self.db.transaction();
 		batch.put(None, &document, &key);
 		self.db.write(batch).map_err(Error::Database)
 	}
 
-	fn get(&self, document: &DocumentAddress) -> Result<DocumentKey, Error> {
+	fn get(&self, document: &DocumentAddress) -> Result<DocumentKeyShare, Error> {
 		self.db.get(None, document)
 			.map_err(Error::Database)?
 			.ok_or(Error::DocumentNotFound)
 			.map(|key| key.to_vec())
+			.and_then(|key| serde_json::from_slice::<SerializableDocumentKeyShare>(&key).map_err(|e| Error::Database(e.to_string())))
+			.map(Into::into)
+	}
+}
+
+impl From<DocumentKeyShare> for SerializableDocumentKeyShare {
+	fn from(key: DocumentKeyShare) -> Self {
+		SerializableDocumentKeyShare {
+			threshold: key.threshold,
+			id_numbers: key.id_numbers.into_iter().map(|(k, v)| (k.into(), v.into())).collect(),
+			secret_share: key.secret_share.into(),
+			common_point: key.common_point.into(),
+			encrypted_point: key.encrypted_point.into(),
+		}
+	}
+}
+
+impl From<SerializableDocumentKeyShare> for DocumentKeyShare {
+	fn from(key: SerializableDocumentKeyShare) -> Self {
+		DocumentKeyShare {
+			threshold: key.threshold,
+			id_numbers: key.id_numbers.into_iter().map(|(k, v)| (k.into(), v.into())).collect(),
+			secret_share: key.secret_share.into(),
+			common_point: key.common_point.into(),
+			encrypted_point: key.encrypted_point.into(),
+		}
 	}
 }
 
 #[cfg(test)]
 pub mod tests {
-	use std::collections::HashMap;
+	use std::collections::{BTreeMap, HashMap};
 	use parking_lot::RwLock;
 	use devtools::RandomTempPath;
-	use super::super::types::all::{Error, ServiceConfiguration, DocumentAddress, DocumentKey};
+	use ethkey::{Random, Generator};
-	use super::{KeyStorage, PersistentKeyStorage};
+	use super::super::types::all::{Error, NodeAddress, ServiceConfiguration, ClusterConfiguration,
+		DocumentAddress, EncryptionConfiguration};
+	use super::{KeyStorage, PersistentKeyStorage, DocumentKeyShare};
 
 	#[derive(Default)]
 	/// In-memory document encryption keys storage
 	pub struct DummyKeyStorage {
-		keys: RwLock<HashMap<DocumentAddress, DocumentKey>>,
+		keys: RwLock<HashMap<DocumentAddress, DocumentKeyShare>>,
 	}
 
 	impl KeyStorage for DummyKeyStorage {
-		fn insert(&self, document: DocumentAddress, key: DocumentKey) -> Result<(), Error> {
+		fn insert(&self, document: DocumentAddress, key: DocumentKeyShare) -> Result<(), Error> {
 			self.keys.write().insert(document, key);
 			Ok(())
 		}
 
-		fn get(&self, document: &DocumentAddress) -> Result<DocumentKey, Error> {
+		fn get(&self, document: &DocumentAddress) -> Result<DocumentKeyShare, Error> {
 			self.keys.read().get(document).cloned().ok_or(Error::DocumentNotFound)
 		}
 	}
@@ -88,15 +152,46 @@ pub mod tests {
 	fn persistent_key_storage() {
 		let path = RandomTempPath::create_dir();
 		let config = ServiceConfiguration {
-			listener_addr: "0.0.0.0".to_owned(),
+			listener_address: NodeAddress {
-			listener_port: 8082,
+				address: "0.0.0.0".to_owned(),
+				port: 8082,
+			},
 			data_path: path.as_str().to_owned(),
+			cluster_config: ClusterConfiguration {
+				threads: 1,
+				self_private: (**Random.generate().unwrap().secret().clone()).into(),
+				listener_address: NodeAddress {
+					address: "0.0.0.0".to_owned(),
+					port: 8083,
+				},
+				nodes: BTreeMap::new(),
+				allow_connecting_to_higher_nodes: false,
+				encryption_config: EncryptionConfiguration {
+					key_check_timeout_ms: 10,
+				},
+			},
 		};
 		
 		let key1 = DocumentAddress::from(1);
-		let value1: DocumentKey = vec![0x77, 0x88];
+		let value1 = DocumentKeyShare {
+			threshold: 100,
+			id_numbers: vec![
+				(Random.generate().unwrap().public().clone(), Random.generate().unwrap().secret().clone())
+			].into_iter().collect(),
+			secret_share: Random.generate().unwrap().secret().clone(),
+			common_point: Random.generate().unwrap().public().clone(),
+			encrypted_point: Random.generate().unwrap().public().clone(),
+		};
 		let key2 = DocumentAddress::from(2);
-		let value2: DocumentKey = vec![0x11, 0x22];
+		let value2 = DocumentKeyShare {
+			threshold: 200,
+			id_numbers: vec![
+				(Random.generate().unwrap().public().clone(), Random.generate().unwrap().secret().clone())
+			].into_iter().collect(),
+			secret_share: Random.generate().unwrap().secret().clone(),
+			common_point: Random.generate().unwrap().public().clone(),
+			encrypted_point: Random.generate().unwrap().public().clone(),
+		};
 		let key3 = DocumentAddress::from(3);
 
 		let key_storage = PersistentKeyStorage::new(&config).unwrap();

secret_store/src/lib.rs

@@ -14,10 +14,22 @@
 // You should have received a copy of the GNU General Public License
 // along with Parity.  If not, see <http://www.gnu.org/licenses/>.
 
+extern crate byteorder;
 #[macro_use]
 extern crate log;
+#[macro_use]
+extern crate futures;
+extern crate futures_cpupool;
 extern crate hyper;
 extern crate parking_lot;
+extern crate rustc_serialize;
+extern crate serde;
+extern crate serde_json;
+#[macro_use]
+extern crate serde_derive;
+extern crate tokio_core;
+extern crate tokio_service;
+extern crate tokio_proto;
 extern crate url;
 
 extern crate ethcore_devtools as devtools;
@@ -38,16 +50,19 @@ mod acl_storage;
 mod http_listener;
 mod key_server;
 mod key_storage;
+mod serialization;
 
 pub use types::all::{DocumentAddress, DocumentKey, DocumentEncryptedKey, RequestSignature, Public,
-	Error, ServiceConfiguration};
+	Error, NodeAddress, ServiceConfiguration, ClusterConfiguration, EncryptionConfiguration};
 pub use traits::{KeyServer};
 
 /// Start new key server instance
 pub fn start(config: ServiceConfiguration) -> Result<Box<KeyServer>, Error> {
-	let acl_storage = acl_storage::DummyAclStorage::default();
+	use std::sync::Arc;
-	let key_storage = key_storage::PersistentKeyStorage::new(&config)?;
+
-	let key_server = key_server::KeyServerImpl::new(acl_storage, key_storage);
+	let acl_storage = Arc::new(acl_storage::DummyAclStorage::default());
+	let key_storage = Arc::new(key_storage::PersistentKeyStorage::new(&config)?);
+	let key_server = key_server::KeyServerImpl::new(&config.cluster_config, acl_storage, key_storage)?;
 	let listener = http_listener::KeyServerHttpListener::start(config, key_server)?;
 	Ok(Box::new(listener))
 }

secret_store/src/serialization.rs

@@ -0,0 +1,260 @@
+// Copyright 2015-2017 Parity Technologies (UK) Ltd.
+// This file is part of Parity.
+
+// Parity is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Parity is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Parity.  If not, see <http://www.gnu.org/licenses/>.
+
+use std::fmt;
+use std::cmp::{Ord, PartialOrd, Ordering};
+use std::ops::Deref;
+use rustc_serialize::hex::ToHex;
+use serde::{Serialize, Deserialize, Serializer, Deserializer};
+use serde::de::{Visitor, Error as SerdeError};
+use ethkey::{Public, Secret, Signature};
+use util::H256;
+
+#[derive(Clone, Debug)]
+/// Serializable Signature.
+pub struct SerializableSignature(Signature);
+
+impl<T> From<T> for SerializableSignature where Signature: From<T> {
+	fn from(s: T) -> SerializableSignature {
+		SerializableSignature(s.into())
+	}
+}
+
+impl Into<Signature> for SerializableSignature {
+	fn into(self) -> Signature {
+		self.0
+	}
+}
+
+impl Deref for SerializableSignature {
+	type Target = Signature;
+
+	fn deref(&self) -> &Signature {
+		&self.0
+	}
+}
+
+impl Serialize for SerializableSignature {
+	fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error> where S: Serializer {
+		serializer.serialize_str(&(*self.0).to_hex())
+	}
+}
+
+impl Deserialize for SerializableSignature {
+	fn deserialize<D>(deserializer: D) -> Result<Self, D::Error> where D: Deserializer {
+		struct HashVisitor;
+
+		impl Visitor for HashVisitor {
+			type Value = SerializableSignature;
+
+			fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result {
+				write!(formatter, "a hex-encoded Signature")
+			}
+
+			fn visit_str<E>(self, value: &str) -> Result<Self::Value, E> where E: SerdeError {
+				value.parse().map(|s| SerializableSignature(s)).map_err(SerdeError::custom)
+			}
+
+			fn visit_string<E>(self, value: String) -> Result<Self::Value, E> where E: SerdeError {
+				self.visit_str(value.as_ref())
+			}
+		}
+
+		deserializer.deserialize(HashVisitor)
+	}
+}
+
+#[derive(Clone, Debug)]
+/// Serializable H256.
+pub struct SerializableH256(H256);
+
+impl<T> From<T> for SerializableH256 where H256: From<T> {
+	fn from(s: T) -> SerializableH256 {
+		SerializableH256(s.into())
+	}
+}
+
+impl Into<H256> for SerializableH256 {
+	fn into(self) -> H256 {
+		self.0
+	}
+}
+
+impl Deref for SerializableH256 {
+	type Target = H256;
+
+	fn deref(&self) -> &H256 {
+		&self.0
+	}
+}
+
+impl Serialize for SerializableH256 {
+	fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error> where S: Serializer {
+		serializer.serialize_str(&(*self.0).to_hex())
+	}
+}
+
+impl Deserialize for SerializableH256 {
+	fn deserialize<D>(deserializer: D) -> Result<Self, D::Error> where D: Deserializer {
+		struct HashVisitor;
+
+		impl Visitor for HashVisitor {
+			type Value = SerializableH256;
+
+			fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result {
+				write!(formatter, "a hex-encoded H256")
+			}
+
+			fn visit_str<E>(self, value: &str) -> Result<Self::Value, E> where E: SerdeError {
+				value.parse().map(|s| SerializableH256(s)).map_err(SerdeError::custom)
+			}
+
+			fn visit_string<E>(self, value: String) -> Result<Self::Value, E> where E: SerdeError {
+				self.visit_str(value.as_ref())
+			}
+		}
+
+		deserializer.deserialize(HashVisitor)
+	}
+}
+
+#[derive(Clone, Debug)]
+/// Serializable EC scalar/secret key.
+pub struct SerializableSecret(Secret);
+
+impl<T> From<T> for SerializableSecret where Secret: From<T> {
+	fn from(s: T) -> SerializableSecret {
+		SerializableSecret(s.into())
+	}
+}
+
+impl Into<Secret> for SerializableSecret {
+	fn into(self) -> Secret {
+		self.0
+	}
+}
+
+impl Deref for SerializableSecret {
+	type Target = Secret;
+
+	fn deref(&self) -> &Secret {
+		&self.0
+	}
+}
+
+impl Serialize for SerializableSecret {
+	fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error> where S: Serializer {
+		serializer.serialize_str(&(*self.0).to_hex())
+	}
+}
+
+impl Deserialize for SerializableSecret {
+	fn deserialize<D>(deserializer: D) -> Result<Self, D::Error> where D: Deserializer {
+		struct HashVisitor;
+
+		impl Visitor for HashVisitor {
+			type Value = SerializableSecret;
+
+			fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result {
+				write!(formatter, "a hex-encoded EC scalar")
+			}
+
+			fn visit_str<E>(self, value: &str) -> Result<Self::Value, E> where E: SerdeError {
+				value.parse().map(|s| SerializableSecret(s)).map_err(SerdeError::custom)
+			}
+
+			fn visit_string<E>(self, value: String) -> Result<Self::Value, E> where E: SerdeError {
+				self.visit_str(value.as_ref())
+			}
+		}
+
+		deserializer.deserialize(HashVisitor)
+	}
+}
+
+#[derive(Clone, Debug)]
+/// Serializable EC point/public key.
+pub struct SerializablePublic(Public);
+
+impl<T> From<T> for SerializablePublic where Public: From<T> {
+	fn from(p: T) -> SerializablePublic {
+		SerializablePublic(p.into())
+	}
+}
+
+impl Into<Public> for SerializablePublic {
+	fn into(self) -> Public {
+		self.0
+	}
+}
+
+impl Deref for SerializablePublic {
+	type Target = Public;
+
+	fn deref(&self) -> &Public {
+		&self.0
+	}
+}
+
+impl Eq for SerializablePublic { }
+
+impl PartialEq for SerializablePublic {
+	fn eq(&self, other: &SerializablePublic) -> bool {
+		self.0.eq(&other.0)
+	}
+}
+
+impl Ord for SerializablePublic {
+	fn cmp(&self, other: &SerializablePublic) -> Ordering {
+		self.0.cmp(&other.0)
+	}
+}
+
+impl PartialOrd for SerializablePublic {
+	fn partial_cmp(&self, other: &SerializablePublic) -> Option<Ordering> {
+		self.0.partial_cmp(&other.0)
+	}
+}
+
+impl Serialize for SerializablePublic {
+	fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error> where S: Serializer {
+		serializer.serialize_str(&(*self.0).to_hex())
+	}
+}
+
+impl Deserialize for SerializablePublic {
+	fn deserialize<D>(deserializer: D) -> Result<Self, D::Error> where D: Deserializer {
+		struct HashVisitor;
+
+		impl Visitor for HashVisitor {
+			type Value = SerializablePublic;
+
+			fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result {
+				write!(formatter, "a hex-encoded EC point")
+			}
+
+			fn visit_str<E>(self, value: &str) -> Result<Self::Value, E> where E: SerdeError {
+				value.parse().map(|s| SerializablePublic(s)).map_err(SerdeError::custom)
+			}
+
+			fn visit_string<E>(self, value: String) -> Result<Self::Value, E> where E: SerdeError {
+				self.visit_str(value.as_ref())
+			}
+		}
+
+		deserializer.deserialize(HashVisitor)
+	}
+}

secret_store/src/traits.rs

@@ -19,6 +19,8 @@ use types::all::{Error, RequestSignature, DocumentAddress, DocumentEncryptedKey}
 #[ipc(client_ident="RemoteKeyServer")]
 /// Secret store key server
 pub trait KeyServer: Send + Sync {
+	/// Generate encryption key for given document.
+	fn generate_document_key(&self, signature: &RequestSignature, document: &DocumentAddress, threshold: usize) -> Result<DocumentEncryptedKey, Error>;
 	/// Request encryption key of given document for given requestor
 	fn document_key(&self, signature: &RequestSignature, document: &DocumentAddress) -> Result<DocumentEncryptedKey, Error>;
 }

secret_store/src/types/all.rs

@@ -15,10 +15,14 @@
 // along with Parity.  If not, see <http://www.gnu.org/licenses/>.
 
 use std::fmt;
+use std::collections::BTreeMap;
 
 use ethkey;
 use util;
+use key_server_cluster;
 
+/// Node id.
+pub type NodeId = ethkey::Public;
 /// Document address type.
 pub type DocumentAddress = util::H256;
 /// Document key type.
@@ -46,16 +50,53 @@ pub enum Error {
 	Internal(String),
 }
 
+#[derive(Debug)]
+#[binary]
+/// Secret store configuration
+pub struct NodeAddress {
+	/// IP address.
+	pub address: String,
+	/// IP port.
+	pub port: u16,
+}
+
 #[derive(Debug)]
 #[binary]
 /// Secret store configuration
 pub struct ServiceConfiguration {
-	/// Interface to listen to
+	/// HTTP listener address.
-	pub listener_addr: String,
+	pub listener_address: NodeAddress,
-	/// Port to listen to
-	pub listener_port: u16,
 	/// Data directory path for secret store
 	pub data_path: String,
+	/// Cluster configuration.
+	pub cluster_config: ClusterConfiguration,
+}
+
+#[derive(Debug)]
+#[binary]
+/// Key server cluster configuration
+pub struct ClusterConfiguration {
+	/// Number of threads reserved by cluster.
+	pub threads: usize,
+	/// Private key this node holds.
+	pub self_private: Vec<u8>, // holds ethkey::Secret
+	/// This node address.
+	pub listener_address: NodeAddress,
+	/// All cluster nodes addresses.
+	pub nodes: BTreeMap<ethkey::Public, NodeAddress>,
+	/// Allow outbound connections to 'higher' nodes.
+	/// This is useful for tests, but slower a bit for production.
+	pub allow_connecting_to_higher_nodes: bool,
+	/// Encryption session configuration.
+	pub encryption_config: EncryptionConfiguration,
+}
+
+#[derive(Clone, Debug)]
+#[binary]
+/// Encryption parameters.
+pub struct EncryptionConfiguration {
+	/// Key check timeout.
+	pub key_check_timeout_ms: u64,
 }
 
 impl fmt::Display for Error {
@@ -70,6 +111,18 @@ impl fmt::Display for Error {
 	}
 }
 
+impl From<ethkey::Error> for Error {
+	fn from(err: ethkey::Error) -> Self {
+		Error::Internal(err.into())
+	}
+}
+
+impl From<key_server_cluster::Error> for Error {
+	fn from(err: key_server_cluster::Error) -> Self {
+		Error::Internal(err.into())
+	}
+}
+
 impl Into<String> for Error {
 	fn into(self) -> String {
 		format!("{}", self)