Fetching content from HTTPS using rustls (#2024)

* Fetching https

* Fetching dispatcher (HTTP, HTTPS)

* Fetching from github

* Chunked encoding parser

* Abort support

* Fixing tests and review comments

* Cargo.lock order [ci skip]

* Relaxed -> SeqCst

util/https-fetch/Cargo.toml

@@ -0,0 +1,18 @@
+[package]
+description = "HTTPS fetching library"
+homepage = "http://ethcore.io"
+license = "GPL-3.0"
+name = "https-fetch"
+version = "0.1.0"
+authors = ["Ethcore <admin@ethcore.io>"]
+
+[dependencies]
+log = "0.3"
+mio = { git = "https://github.com/ethcore/mio", branch = "v0.5.x" }
+rustls = { git = "https://github.com/ctz/rustls" }
+clippy = { version = "0.0.85", optional = true}
+
+[features]
+default = []
+ca-github-only = []
+dev = ["clippy"]

util/https-fetch/examples/fetch.rs

@@ -0,0 +1,15 @@
+extern crate https_fetch;
+
+use std::io;
+use std::sync::Arc;
+use std::sync::atomic::AtomicBool;
+use https_fetch::*;
+
+fn main() {
+	let client = Client::new().unwrap();
+	let aborted = Arc::new(AtomicBool::new(false));
+
+	client.fetch(Url::new("github.com", 443, "/").unwrap(), Box::new(io::stdout()), aborted, |result| {
+		assert!(result.is_ok());
+	}).unwrap();
+}

util/https-fetch/src/ca-github.crt

@@ -0,0 +1,93 @@
+-----BEGIN CERTIFICATE-----
+MIIDxTCCAq2gAwIBAgIQAqxcJmoLQJuPC3nyrkYldzANBgkqhkiG9w0BAQUFADBs
+MQswCQYDVQQGEwJVUzEVMBMGA1UEChMMRGlnaUNlcnQgSW5jMRkwFwYDVQQLExB3
+d3cuZGlnaWNlcnQuY29tMSswKQYDVQQDEyJEaWdpQ2VydCBIaWdoIEFzc3VyYW5j
+ZSBFViBSb290IENBMB4XDTA2MTExMDAwMDAwMFoXDTMxMTExMDAwMDAwMFowbDEL
+MAkGA1UEBhMCVVMxFTATBgNVBAoTDERpZ2lDZXJ0IEluYzEZMBcGA1UECxMQd3d3
+LmRpZ2ljZXJ0LmNvbTErMCkGA1UEAxMiRGlnaUNlcnQgSGlnaCBBc3N1cmFuY2Ug
+RVYgUm9vdCBDQTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAMbM5XPm
++9S75S0tMqbf5YE/yc0lSbZxKsPVlDRnogocsF9ppkCxxLeyj9CYpKlBWTrT3JTW
+PNt0OKRKzE0lgvdKpVMSOO7zSW1xkX5jtqumX8OkhPhPYlG++MXs2ziS4wblCJEM
+xChBVfvLWokVfnHoNb9Ncgk9vjo4UFt3MRuNs8ckRZqnrG0AFFoEt7oT61EKmEFB
+Ik5lYYeBQVCmeVyJ3hlKV9Uu5l0cUyx+mM0aBhakaHPQNAQTXKFx01p8VdteZOE3
+hzBWBOURtCmAEvF5OYiiAhF8J2a3iLd48soKqDirCmTCv2ZdlYTBoSUeh10aUAsg
+EsxBu24LUTi4S8sCAwEAAaNjMGEwDgYDVR0PAQH/BAQDAgGGMA8GA1UdEwEB/wQF
+MAMBAf8wHQYDVR0OBBYEFLE+w2kD+L9HAdSYJhoIAu9jZCvDMB8GA1UdIwQYMBaA
+FLE+w2kD+L9HAdSYJhoIAu9jZCvDMA0GCSqGSIb3DQEBBQUAA4IBAQAcGgaX3Nec
+nzyIZgYIVyHbIUf4KmeqvxgydkAQV8GK83rZEWWONfqe/EW1ntlMMUu4kehDLI6z
+eM7b41N5cdblIZQB2lWHmiRk9opmzN6cN82oNLFpmyPInngiK3BD41VHMWEZ71jF
+hS9OMPagMRYjyOfiZRYzy78aG6A9+MpeizGLYAiJLQwGXFK3xPkKmNEVX58Svnw2
+Yzi9RKR/5CYrCsSXaQ3pjOLAEFe4yHYSkVXySGnYvCoCWw9E1CAx2/S6cCZdkGCe
+vEsXCS+0yx5DaMkHJ8HSXPfqIbloEpw8nL+e/IBcm2PN7EeqJSdnoDfzAIJ9VNep
++OkuE6N36B9K
+-----END CERTIFICATE-----
+-----BEGIN CERTIFICATE-----
+MIIEtjCCA56gAwIBAgIQDHmpRLCMEZUgkmFf4msdgzANBgkqhkiG9w0BAQsFADBs
+MQswCQYDVQQGEwJVUzEVMBMGA1UEChMMRGlnaUNlcnQgSW5jMRkwFwYDVQQLExB3
+d3cuZGlnaWNlcnQuY29tMSswKQYDVQQDEyJEaWdpQ2VydCBIaWdoIEFzc3VyYW5j
+ZSBFViBSb290IENBMB4XDTEzMTAyMjEyMDAwMFoXDTI4MTAyMjEyMDAwMFowdTEL
+MAkGA1UEBhMCVVMxFTATBgNVBAoTDERpZ2lDZXJ0IEluYzEZMBcGA1UECxMQd3d3
+LmRpZ2ljZXJ0LmNvbTE0MDIGA1UEAxMrRGlnaUNlcnQgU0hBMiBFeHRlbmRlZCBW
+YWxpZGF0aW9uIFNlcnZlciBDQTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoC
+ggEBANdTpARR+JmmFkhLZyeqk0nQOe0MsLAAh/FnKIaFjI5j2ryxQDji0/XspQUY
+uD0+xZkXMuwYjPrxDKZkIYXLBxA0sFKIKx9om9KxjxKws9LniB8f7zh3VFNfgHk/
+LhqqqB5LKw2rt2O5Nbd9FLxZS99RStKh4gzikIKHaq7q12TWmFXo/a8aUGxUvBHy
+/Urynbt/DvTVvo4WiRJV2MBxNO723C3sxIclho3YIeSwTQyJ3DkmF93215SF2AQh
+cJ1vb/9cuhnhRctWVyh+HA1BV6q3uCe7seT6Ku8hI3UarS2bhjWMnHe1c63YlC3k
+8wyd7sFOYn4XwHGeLN7x+RAoGTMCAwEAAaOCAUkwggFFMBIGA1UdEwEB/wQIMAYB
+Af8CAQAwDgYDVR0PAQH/BAQDAgGGMB0GA1UdJQQWMBQGCCsGAQUFBwMBBggrBgEF
+BQcDAjA0BggrBgEFBQcBAQQoMCYwJAYIKwYBBQUHMAGGGGh0dHA6Ly9vY3NwLmRp
+Z2ljZXJ0LmNvbTBLBgNVHR8ERDBCMECgPqA8hjpodHRwOi8vY3JsNC5kaWdpY2Vy
+dC5jb20vRGlnaUNlcnRIaWdoQXNzdXJhbmNlRVZSb290Q0EuY3JsMD0GA1UdIAQ2
+MDQwMgYEVR0gADAqMCgGCCsGAQUFBwIBFhxodHRwczovL3d3dy5kaWdpY2VydC5j
+b20vQ1BTMB0GA1UdDgQWBBQ901Cl1qCt7vNKYApl0yHU+PjWDzAfBgNVHSMEGDAW
+gBSxPsNpA/i/RwHUmCYaCALvY2QrwzANBgkqhkiG9w0BAQsFAAOCAQEAnbbQkIbh
+hgLtxaDwNBx0wY12zIYKqPBKikLWP8ipTa18CK3mtlC4ohpNiAexKSHc59rGPCHg
+4xFJcKx6HQGkyhE6V6t9VypAdP3THYUYUN9XR3WhfVUgLkc3UHKMf4Ib0mKPLQNa
+2sPIoc4sUqIAY+tzunHISScjl2SFnjgOrWNoPLpSgVh5oywM395t6zHyuqB8bPEs
+1OG9d4Q3A84ytciagRpKkk47RpqF/oOi+Z6Mo8wNXrM9zwR4jxQUezKcxwCmXMS1
+oVWNWlZopCJwqjyBcdmdqEU79OX2olHdx3ti6G8MdOu42vi/hw15UJGQmxg7kVkn
+8TUoE6smftX3eg==
+-----END CERTIFICATE-----
+-----BEGIN CERTIFICATE-----
+MIIHeTCCBmGgAwIBAgIQC/20CQrXteZAwwsWyVKaJzANBgkqhkiG9w0BAQsFADB1
+MQswCQYDVQQGEwJVUzEVMBMGA1UEChMMRGlnaUNlcnQgSW5jMRkwFwYDVQQLExB3
+d3cuZGlnaWNlcnQuY29tMTQwMgYDVQQDEytEaWdpQ2VydCBTSEEyIEV4dGVuZGVk
+IFZhbGlkYXRpb24gU2VydmVyIENBMB4XDTE2MDMxMDAwMDAwMFoXDTE4MDUxNzEy
+MDAwMFowgf0xHTAbBgNVBA8MFFByaXZhdGUgT3JnYW5pemF0aW9uMRMwEQYLKwYB
+BAGCNzwCAQMTAlVTMRkwFwYLKwYBBAGCNzwCAQITCERlbGF3YXJlMRAwDgYDVQQF
+Ewc1MTU3NTUwMSQwIgYDVQQJExs4OCBDb2xpbiBQIEtlbGx5LCBKciBTdHJlZXQx
+DjAMBgNVBBETBTk0MTA3MQswCQYDVQQGEwJVUzETMBEGA1UECBMKQ2FsaWZvcm5p
+YTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEVMBMGA1UEChMMR2l0SHViLCBJbmMu
+MRMwEQYDVQQDEwpnaXRodWIuY29tMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIB
+CgKCAQEA54hc8pZclxgcupjiA/F/OZGRwm/ZlucoQGTNTKmBEgNsrn/mxhngWmPw
+bAvUaLP//T79Jc+1WXMpxMiz9PK6yZRRFuIo0d2bx423NA6hOL2RTtbnfs+y0PFS
+/YTpQSelTuq+Fuwts5v6aAweNyMcYD0HBybkkdosFoDccBNzJ92Ac8I5EVDUc3Or
+/4jSyZwzxu9kdmBlBzeHMvsqdH8SX9mNahXtXxRpwZnBiUjw36PgN+s9GLWGrafd
+02T0ux9Yzd5ezkMxukqEAQ7AKIIijvaWPAJbK/52XLhIy2vpGNylyni/DQD18bBP
+T+ZG1uv0QQP9LuY/joO+FKDOTler4wIDAQABo4IDejCCA3YwHwYDVR0jBBgwFoAU
+PdNQpdagre7zSmAKZdMh1Pj41g8wHQYDVR0OBBYEFIhcSGcZzKB2WS0RecO+oqyH
+IidbMCUGA1UdEQQeMByCCmdpdGh1Yi5jb22CDnd3dy5naXRodWIuY29tMA4GA1Ud
+DwEB/wQEAwIFoDAdBgNVHSUEFjAUBggrBgEFBQcDAQYIKwYBBQUHAwIwdQYDVR0f
+BG4wbDA0oDKgMIYuaHR0cDovL2NybDMuZGlnaWNlcnQuY29tL3NoYTItZXYtc2Vy
+dmVyLWcxLmNybDA0oDKgMIYuaHR0cDovL2NybDQuZGlnaWNlcnQuY29tL3NoYTIt
+ZXYtc2VydmVyLWcxLmNybDBLBgNVHSAERDBCMDcGCWCGSAGG/WwCATAqMCgGCCsG
+AQUFBwIBFhxodHRwczovL3d3dy5kaWdpY2VydC5jb20vQ1BTMAcGBWeBDAEBMIGI
+BggrBgEFBQcBAQR8MHowJAYIKwYBBQUHMAGGGGh0dHA6Ly9vY3NwLmRpZ2ljZXJ0
+LmNvbTBSBggrBgEFBQcwAoZGaHR0cDovL2NhY2VydHMuZGlnaWNlcnQuY29tL0Rp
+Z2lDZXJ0U0hBMkV4dGVuZGVkVmFsaWRhdGlvblNlcnZlckNBLmNydDAMBgNVHRMB
+Af8EAjAAMIIBfwYKKwYBBAHWeQIEAgSCAW8EggFrAWkAdgCkuQmQtBhYFIe7E6LM
+Z3AKPDWYBPkb37jjd80OyA3cEAAAAVNhieoeAAAEAwBHMEUCIQCHHSEY/ROK2/sO
+ljbKaNEcKWz6BxHJNPOtjSyuVnSn4QIgJ6RqvYbSX1vKLeX7vpnOfCAfS2Y8lB5R
+NMwk6us2QiAAdgBo9pj4H2SCvjqM7rkoHUz8cVFdZ5PURNEKZ6y7T0/7xAAAAVNh
+iennAAAEAwBHMEUCIQDZpd5S+3to8k7lcDeWBhiJASiYTk2rNAT26lVaM3xhWwIg
+NUqrkIODZpRg+khhp8ag65B8mu0p4JUAmkRDbiYnRvYAdwBWFAaaL9fC7NP14b1E
+sj7HRna5vJkRXMDvlJhV1onQ3QAAAVNhieqZAAAEAwBIMEYCIQDnm3WStlvE99GC
+izSx+UGtGmQk2WTokoPgo1hfiv8zIAIhAPrYeXrBgseA9jUWWoB4IvmcZtshjXso
+nT8MIG1u1zF8MA0GCSqGSIb3DQEBCwUAA4IBAQCLbNtkxuspqycq8h1EpbmAX0wM
+5DoW7hM/FVdz4LJ3Kmftyk1yd8j/PSxRrAQN2Mr/frKeK8NE1cMji32mJbBqpWtK
+/+wC+avPplBUbNpzP53cuTMF/QssxItPGNP5/OT9Aj1BxA/NofWZKh4ufV7cz3pY
+RDS4BF+EEFQ4l5GY+yp4WJA/xSvYsTHWeWxRD1/nl62/Rd9FN2NkacRVozCxRVle
+FrBHTFxqIP6kDnxiLElBrZngtY07ietaYZVLQN/ETyqLQftsf8TecwTklbjvm8NT
+JqbaIVifYwqwNN+4lRxS3F5lNlA/il12IOgbRioLI62o8G0DaEUQgHNf8vSG
+-----END CERTIFICATE-----

util/https-fetch/src/client.rs

@@ -0,0 +1,210 @@
+// Copyright 2015, 2016 Ethcore (UK) Ltd.
+// This file is part of Parity.
+
+// Parity is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Parity is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Parity.  If not, see <http://www.gnu.org/licenses/>.
+
+use std::cell::RefCell;
+use std::{fs, str, thread};
+use std::sync::Arc;
+use std::sync::atomic::AtomicBool;
+use std::path::PathBuf;
+use std::io::{self, Write};
+use std::collections::HashMap;
+
+use mio;
+use tlsclient::{TlsClient, TlsClientError};
+
+use url::Url;
+
+#[derive(Debug)]
+pub enum FetchError {
+	InvalidAddress,
+	ReadingCaCertificates,
+	CaCertificates(io::Error),
+	Io(io::Error),
+	Notify(mio::NotifyError<ClientMessage>),
+	Client(TlsClientError),
+}
+
+impl From<io::Error> for FetchError {
+	fn from(e: io::Error) -> Self {
+		FetchError::Io(e)
+	}
+}
+
+impl From<mio::NotifyError<ClientMessage>> for FetchError {
+	fn from(e: mio::NotifyError<ClientMessage>) -> Self {
+		FetchError::Notify(e)
+	}
+}
+
+impl From<TlsClientError> for FetchError {
+	fn from(e: TlsClientError) -> Self {
+		FetchError::Client(e)
+	}
+}
+
+pub type FetchResult = Result<(), FetchError>;
+
+pub enum ClientMessage {
+	Fetch(Url, Box<io::Write + Send>, Arc<AtomicBool>, Box<FnMut(FetchResult) + Send>),
+	Shutdown,
+}
+
+pub struct Client {
+	channel: mio::Sender<ClientMessage>,
+	thread: Option<thread::JoinHandle<()>>,
+}
+
+impl Drop for Client {
+	fn drop(&mut self) {
+		self.close_internal();
+		if let Some(thread) = self.thread.take() {
+			thread.join().expect("Clean shutdown.");
+		}
+	}
+}
+
+impl Client {
+	pub fn new() -> Result<Self, FetchError> {
+		let mut event_loop = try!(mio::EventLoop::new());
+		let channel = event_loop.channel();
+
+		let thread = thread::spawn(move || {
+			let mut client = ClientLoop {
+				next_token: 0,
+				sessions: HashMap::new(),
+			};
+			event_loop.run(&mut client).unwrap();
+		});
+
+		Ok(Client {
+			channel: channel,
+			thread: Some(thread),
+		})
+	}
+
+	pub fn fetch_to_file<F: FnOnce(FetchResult) + Send + 'static>(&self, url: Url, path: PathBuf, abort: Arc<AtomicBool>, callback: F) -> Result<(), FetchError> {
+		let file = try!(fs::File::create(&path));
+		self.fetch(url, Box::new(file), abort, move |result| {
+			if let Err(_) = result {
+				// remove temporary file
+				let _ = fs::remove_file(&path);
+			}
+			callback(result);
+		})
+	}
+
+	pub fn fetch<F: FnOnce(FetchResult) + Send + 'static>(&self, url: Url, writer: Box<io::Write + Send>, abort: Arc<AtomicBool>, callback: F) -> Result<(), FetchError> {
+		let cell = RefCell::new(Some(callback));
+		try!(self.channel.send(ClientMessage::Fetch(url, writer, abort, Box::new(move |res| {
+			cell.borrow_mut().take().expect("Called only once.")(res);
+		}))));
+		Ok(())
+	}
+
+	pub fn close(mut self) {
+		self.close_internal()
+	}
+
+	fn close_internal(&mut self) {
+		if let Err(e) = self.channel.send(ClientMessage::Shutdown) {
+			warn!("Error while closing client: {:?}. Already stopped?", e);
+		}
+	}
+}
+
+pub struct ClientLoop {
+	next_token: usize,
+	sessions: HashMap<usize, TlsClient>,
+}
+
+impl mio::Handler for ClientLoop {
+	type Timeout = ();
+	type Message = ClientMessage;
+
+	fn ready(&mut self, event_loop: &mut mio::EventLoop<ClientLoop>, token: mio::Token, events: mio::EventSet) {
+		let utoken = token.as_usize();
+		let remove = if let Some(mut tlsclient) = self.sessions.get_mut(&utoken) {
+			tlsclient.ready(event_loop, token, events)
+		} else {
+			false
+		};
+
+		if remove {
+			self.sessions.remove(&utoken);
+		}
+	}
+
+	fn notify(&mut self, event_loop: &mut mio::EventLoop<Self>, msg: Self::Message) {
+		match msg {
+			ClientMessage::Shutdown => event_loop.shutdown(),
+			ClientMessage::Fetch(url, writer, abort, callback) => {
+				let token = self.next_token;
+				self.next_token += 1;
+
+				if let Ok(mut tlsclient) = TlsClient::new(mio::Token(token), &url, writer, abort, callback) {
+					let httpreq = format!(
+						"GET {} HTTP/1.1\r\nHost: {}\r\nConnection: close\r\nAccept-Encoding: identity\r\n\r\n",
+						url.path(),
+						url.hostname()
+					);
+					let _ = tlsclient.write(httpreq.as_bytes());
+					tlsclient.register(event_loop);
+
+					self.sessions.insert(token, tlsclient);
+				}
+			}
+		}
+	}
+}
+
+#[test]
+fn should_successfuly_fetch_a_page() {
+	use std::io::{self, Cursor};
+	use std::sync::{mpsc, Arc};
+	use std::sync::atomic::{AtomicUsize, AtomicBool, Ordering};
+
+	struct Writer {
+		wrote: Arc<AtomicUsize>,
+		data: Cursor<Vec<u8>>,
+	}
+
+	impl io::Write for Writer {
+		fn write(&mut self, buf: &[u8]) -> io::Result<usize> {
+			let res = self.data.write(buf);
+			if let Ok(count) = res {
+				self.wrote.fetch_add(count, Ordering::Relaxed);
+			}
+			res
+		}
+		fn flush(&mut self) -> io::Result<()> { Ok(()) }
+	}
+
+	let client = Client::new().unwrap();
+
+	let wrote = Arc::new(AtomicUsize::new(0));
+	let writer = Writer {
+		wrote: wrote.clone(),
+		data: Cursor::new(Vec::new()),
+	};
+	let (tx, rx) = mpsc::channel();
+	client.fetch(Url::new("github.com", 443, "/").unwrap(), Box::new(writer), Arc::new(AtomicBool::new(false)), move |result| {
+		assert!(result.is_ok());
+		assert!(wrote.load(Ordering::Relaxed) > 0);
+		tx.send(result).unwrap();
+	}).unwrap();
+	let _ = rx.recv().unwrap();
+}
+

util/https-fetch/src/http.rs

@@ -0,0 +1,334 @@
+// Copyright 2015, 2016 Ethcore (UK) Ltd.
+// This file is part of Parity.
+
+// Parity is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Parity is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Parity.  If not, see <http://www.gnu.org/licenses/>.
+
+//! HTTP format processor
+
+use std::io::{self, Cursor, Write};
+use std::cmp;
+
+#[derive(Debug, Clone, Copy, PartialEq)]
+pub enum State {
+	WaitingForStatus,
+	WaitingForHeaders,
+	WaitingForChunk,
+	WritingBody,
+	WritingChunk(usize),
+	Finished,
+}
+
+pub struct HttpProcessor {
+	state: State,
+	buffer: Cursor<Vec<u8>>,
+	status: Option<String>,
+	headers: Vec<String>,
+	body_writer: io::BufWriter<Box<io::Write>>,
+}
+
+const BREAK_LEN: usize = 2;
+
+impl HttpProcessor {
+	pub fn new(body_writer: Box<io::Write>) -> Self {
+		HttpProcessor {
+			state: State::WaitingForStatus,
+			buffer: Cursor::new(Vec::new()),
+			status: None,
+			headers: Vec::new(),
+			body_writer: io::BufWriter::new(body_writer)
+		}
+	}
+
+	#[cfg(test)]
+	pub fn status(&self) -> Option<&String> {
+		self.status.as_ref()
+	}
+
+	#[cfg(test)]
+	pub fn headers(&self) -> &[String] {
+		&self.headers
+	}
+
+	fn find_break_index(&mut self) -> Option<usize> {
+		let data = self.buffer.get_ref();
+		let mut idx = 0;
+		let mut got_r = false;
+		// looks for \r\n in data
+		for b in data {
+			idx += 1;
+			if got_r && b == &10u8 {
+				return Some(idx);
+			} else if !got_r && b == &13u8 {
+				got_r = true;
+			} else {
+				got_r = false;
+			}
+		}
+		None
+	}
+
+	// Consumes bytes from internal buffer
+	fn buffer_consume(&mut self, bytes: usize) {
+		let bytes = cmp::min(bytes, self.buffer.get_ref().len());
+		// Drain data
+		self.buffer.get_mut().drain(0..bytes);
+		let len = self.buffer.position();
+		self.buffer.set_position(len - bytes as u64);
+	}
+
+	fn buffer_to_string(&mut self, bytes: usize) -> String {
+		let val = String::from_utf8_lossy(&self.buffer.get_ref()[0..bytes-BREAK_LEN]).into_owned();
+		self.buffer_consume(bytes);
+		val
+	}
+
+	fn is_chunked(&self) -> bool {
+		self.headers
+			.iter()
+			.find(|item| item.to_lowercase().contains("transfer-encoding: chunked"))
+			.is_some()
+	}
+	fn set_state(&mut self, state: State) {
+		self.state = state;
+		trace!("Changing state to {:?}", state);
+	}
+
+	fn process_buffer(&mut self) -> io::Result<()> {
+		// consume data and perform state transitions
+		loop {
+			match self.state {
+				State::WaitingForStatus => {
+					if let Some(break_index) = self.find_break_index() {
+						let status = self.buffer_to_string(break_index);
+						debug!("Read status: {:?}", status);
+						self.status = Some(status);
+						self.set_state(State::WaitingForHeaders);
+					} else {
+						// wait for more data
+						return Ok(());
+					}
+				},
+				State::WaitingForHeaders => {
+					match self.find_break_index() {
+						// Last header - got empty line, body starts
+						Some(BREAK_LEN) => {
+							self.buffer_consume(BREAK_LEN);
+							let is_chunked = self.is_chunked();
+							self.set_state(match is_chunked {
+								true => State::WaitingForChunk,
+								false => State::WritingBody,
+							});
+						},
+						Some(break_index) => {
+							let header = self.buffer_to_string(break_index);
+							debug!("Found header: {:?}", header);
+							self.headers.push(header);
+						},
+						None => return Ok(()),
+					}
+				},
+				State::WritingBody => {
+					let len = self.buffer.get_ref().len();
+					try!(self.body_writer.write_all(self.buffer.get_ref()));
+					self.buffer_consume(len);
+					return Ok(());
+				},
+				State::WaitingForChunk => {
+					match self.find_break_index() {
+						None => return Ok(()),
+						// last chunk - size 0
+						Some(BREAK_LEN) => {
+							self.state = State::Finished;
+						},
+						Some(break_index) => {
+							let chunk_size = self.buffer_to_string(break_index);
+							self.set_state(if let Ok(size) = usize::from_str_radix(&chunk_size, 16) {
+								State::WritingChunk(size)
+							} else {
+								warn!("Error parsing server chunked response. Invalid chunk size.");
+								State::Finished
+							});
+						}
+					}
+				},
+				State::WritingChunk(0) => {
+					self.set_state(State::Finished);
+				},
+				// Buffers the data until we have a full chunk
+				State::WritingChunk(left) if self.buffer.get_ref().len() >= left => {
+					try!(self.body_writer.write_all(&self.buffer.get_ref()[0..left]));
+					self.buffer_consume(left + BREAK_LEN);
+
+					self.set_state(State::WaitingForChunk);
+				},
+				// Wait for more data
+				State::WritingChunk(_) => return Ok(()),
+				// Just consume buffer
+				State::Finished => {
+					let len = self.buffer.get_ref().len();
+					self.buffer_consume(len);
+					return Ok(());
+				},
+			}
+		}
+	}
+
+	#[cfg(test)]
+	pub fn state(&self) -> State {
+		self.state
+	}
+}
+
+impl io::Write for HttpProcessor {
+	fn write(&mut self, bytes: &[u8]) -> io::Result<usize> {
+		let result = self.buffer.write(bytes);
+		try!(self.process_buffer());
+		result
+	}
+
+	fn flush(&mut self) -> io::Result<()> {
+		self.buffer.flush().and_then(|_| {
+			self.body_writer.flush()
+		})
+	}
+}
+
+#[cfg(test)]
+mod tests {
+	use std::rc::Rc;
+	use std::cell::RefCell;
+	use std::io::{self, Write, Cursor};
+	use super::*;
+
+	struct Writer {
+		data: Rc<RefCell<Cursor<Vec<u8>>>>,
+	}
+
+	impl Writer {
+		fn new() -> (Self, Rc<RefCell<Cursor<Vec<u8>>>>) {
+			let data = Rc::new(RefCell::new(Cursor::new(Vec::new())));
+			(Writer { data: data.clone() }, data)
+		}
+	}
+
+	impl Write for Writer {
+		fn write(&mut self, buf: &[u8]) -> io::Result<usize> { self.data.borrow_mut().write(buf) }
+		fn flush(&mut self) -> io::Result<()> { self.data.borrow_mut().flush() }
+	}
+
+	#[test]
+	fn should_be_able_to_process_status_line() {
+		// given
+		let mut http = HttpProcessor::new(Box::new(Cursor::new(Vec::new())));
+
+		// when
+		let out =
+			"\
+				HTTP/1.1 200 OK\r\n\
+				Server: Pari
+			";
+		http.write_all(out.as_bytes()).unwrap();
+		http.flush().unwrap();
+
+		// then
+		assert_eq!(http.status().unwrap(), "HTTP/1.1 200 OK");
+		assert_eq!(http.state(), State::WaitingForHeaders);
+	}
+
+	#[test]
+	fn should_be_able_to_process_headers() {
+		// given
+		let mut http = HttpProcessor::new(Box::new(Cursor::new(Vec::new())));
+
+		// when
+		let out =
+			"\
+				HTTP/1.1 200 OK\r\n\
+				Server: Parity/1.1.1\r\n\
+				Connection: close\r\n\
+				Content-Length: 2\r\n\
+				Content-Type: application/json\r\n\
+				\r\n\
+			";
+		http.write_all(out.as_bytes()).unwrap();
+		http.flush().unwrap();
+
+		// then
+		assert_eq!(http.status().unwrap(), "HTTP/1.1 200 OK");
+		assert_eq!(http.headers().len(), 4);
+		assert_eq!(http.state(), State::WritingBody);
+	}
+
+	#[test]
+	fn should_be_able_to_consume_body() {
+		// given
+		let (writer, data) = Writer::new();
+		let mut http = HttpProcessor::new(Box::new(writer));
+
+		// when
+		let out =
+			"\
+				HTTP/1.1 200 OK\r\n\
+				Server: Parity/1.1.1\r\n\
+				Connection: close\r\n\
+				Content-Length: 2\r\n\
+				Content-Type: application/json\r\n\
+				\r\n\
+				Some data\
+			";
+		http.write_all(out.as_bytes()).unwrap();
+		http.flush().unwrap();
+
+		// then
+		assert_eq!(http.status().unwrap(), "HTTP/1.1 200 OK");
+		assert_eq!(http.headers().len(), 4);
+		assert_eq!(http.state(), State::WritingBody);
+		assert_eq!(data.borrow().get_ref()[..], b"Some data"[..]);
+	}
+
+	#[test]
+	fn should_correctly_handle_chunked_content() {
+		// given
+		let (writer, data) = Writer::new();
+		let mut http = HttpProcessor::new(Box::new(writer));
+
+		// when
+		let out =
+			"\
+				HTTP/1.1 200 OK\r\n\
+				Host: 127.0.0.1:8080\r\n\
+				Transfer-Encoding: chunked\r\n\
+				Connection: close\r\n\
+				\r\n\
+				4\r\n\
+				Pari\r\n\
+				3\r\n\
+				ty \r\n\
+				D\r\n\
+				in\r\n\
+				\r\n\
+				chunks.\r\n\
+				0\r\n\
+				\r\n\
+			";
+		http.write_all(out.as_bytes()).unwrap();
+		http.flush().unwrap();
+
+		// then
+		assert_eq!(http.status().unwrap(), "HTTP/1.1 200 OK");
+		assert_eq!(http.headers().len(), 3);
+		assert_eq!(data.borrow().get_ref()[..], b"Parity in\r\n\r\nchunks."[..]);
+		assert_eq!(http.state(), State::Finished);
+	}
+}

util/https-fetch/src/lib.rs

@@ -0,0 +1,28 @@
+// Copyright 2015, 2016 Ethcore (UK) Ltd.
+// This file is part of Parity.
+
+// Parity is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Parity is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Parity.  If not, see <http://www.gnu.org/licenses/>.
+
+extern crate rustls;
+extern crate mio;
+#[macro_use] extern crate log;
+
+mod tlsclient;
+mod client;
+mod url;
+mod http;
+
+pub use self::client::{Client, FetchError, FetchResult};
+pub use self::url::{Url, UrlError};
+

util/https-fetch/src/tlsclient.rs

@@ -0,0 +1,247 @@
+// Copyright 2015, 2016 Ethcore (UK) Ltd.
+// This file is part of Parity.
+
+// Parity is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Parity is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Parity.  If not, see <http://www.gnu.org/licenses/>.
+
+use std::str;
+use std::sync::Arc;
+use std::sync::atomic::{AtomicBool, Ordering};
+use std::io::{self, Write, Read, Cursor, BufReader};
+
+use mio;
+use mio::tcp::TcpStream;
+use rustls::{self, Session};
+
+use url::Url;
+use http::HttpProcessor;
+use client::{FetchError, ClientLoop, FetchResult};
+
+#[derive(Debug)]
+pub enum TlsClientError {
+	Aborted,
+	Initialization,
+	UnexpectedEof,
+	Connection(io::Error),
+	Writer(io::Error),
+	Tls(rustls::TLSError),
+}
+
+/// This encapsulates the TCP-level connection, some connection
+/// state, and the underlying TLS-level session.
+pub struct TlsClient {
+	abort: Arc<AtomicBool>,
+	token: mio::Token,
+	socket: TcpStream,
+	tls_session: rustls::ClientSession,
+	writer: HttpProcessor,
+	error: Option<TlsClientError>,
+	closing: bool,
+	callback: Box<FnMut(FetchResult) + Send>,
+}
+
+impl io::Write for TlsClient {
+	fn write(&mut self, bytes: &[u8]) -> io::Result<usize> {
+		self.tls_session.write(bytes)
+	}
+
+	fn flush(&mut self) -> io::Result<()> {
+		self.tls_session.flush()
+	}
+}
+
+impl io::Read for TlsClient {
+	fn read(&mut self, bytes: &mut [u8]) -> io::Result<usize> {
+		self.tls_session.read(bytes)
+	}
+}
+
+#[cfg(feature = "ca-github-only")]
+static CA_CERTS: &'static [u8] = include_bytes!("./ca-github.crt");
+#[cfg(not(feature = "ca-github-only"))]
+static CA_CERTS: &'static [u8] = include_bytes!("./ca-certificates.crt");
+
+impl TlsClient {
+	pub fn make_config() -> Result<Arc<rustls::ClientConfig>, FetchError> {
+		let mut config = rustls::ClientConfig::new();
+		let mut cursor = Cursor::new(CA_CERTS.to_vec());
+		let mut reader = BufReader::new(&mut cursor);
+		try!(config.root_store.add_pem_file(&mut reader).map_err(|_| FetchError::ReadingCaCertificates));
+		// TODO [ToDr] client certificate?
+		Ok(Arc::new(config))
+	}
+
+	pub fn new(
+		token: mio::Token,
+		url: &Url,
+		writer: Box<io::Write + Send>,
+		abort: Arc<AtomicBool>,
+		mut callback: Box<FnMut(FetchResult) + Send>,
+		) -> Result<Self, FetchError> {
+			let res = TlsClient::make_config().and_then(|cfg| {
+				TcpStream::connect(url.address()).map(|sock| {
+					(cfg, sock)
+				}).map_err(Into::into)
+			});
+
+			match res {
+				Ok((cfg, sock)) => Ok(TlsClient {
+					abort: abort,
+					token: token,
+					writer: HttpProcessor::new(writer),
+					socket: sock,
+					closing: false,
+					error: None,
+					tls_session: rustls::ClientSession::new(&cfg, url.hostname()),
+					callback: callback,
+				}),
+				Err(e) => {
+					callback(Err(e));
+					Err(FetchError::Client(TlsClientError::Initialization))
+				}
+			}
+		}
+
+	/// Called by mio each time events we register() for happen.
+	/// Return false if reregistering again.
+	pub fn ready(&mut self, event_loop: &mut mio::EventLoop<ClientLoop>, token: mio::Token, events: mio::EventSet) -> bool {
+		assert_eq!(token, self.token);
+
+		let aborted = self.is_aborted();
+		if aborted {
+			// do_write needs to be invoked after that
+			self.tls_session.send_close_notify();
+			self.error = Some(TlsClientError::Aborted);
+		}
+
+		if events.is_readable() {
+			self.do_read();
+		}
+
+		if events.is_writable() {
+			self.do_write();
+		}
+
+		if self.is_closed() || aborted {
+			trace!("Connection closed");
+			let callback = &mut self.callback;
+			callback(match self.error.take() {
+				Some(err) => Err(err.into()),
+				None => Ok(()),
+			});
+
+			return true;
+		}
+
+		self.reregister(event_loop);
+		false
+	}
+
+	pub fn register(&mut self, event_loop: &mut mio::EventLoop<ClientLoop>) {
+		event_loop.register(
+			&self.socket,
+			self.token,
+			self.event_set(),
+			mio::PollOpt::level() | mio::PollOpt::oneshot()
+			).unwrap_or_else(|e| self.error = Some(TlsClientError::Connection(e)));
+	}
+
+	fn reregister(&mut self, event_loop: &mut mio::EventLoop<ClientLoop>) {
+		event_loop.reregister(
+			&self.socket,
+			self.token,
+			self.event_set(),
+			mio::PollOpt::level() | mio::PollOpt::oneshot()
+			).unwrap_or_else(|e| self.error = Some(TlsClientError::Connection(e)));
+	}
+
+	/// We're ready to do a read.
+	fn do_read(&mut self) {
+		// Read TLS data.  This fails if the underlying TCP connection is broken.
+		let rc = self.tls_session.read_tls(&mut self.socket);
+		if let Err(e) = rc {
+			trace!("TLS read error: {:?}", e);
+			self.closing = true;
+			self.error = Some(TlsClientError::Connection(e));
+			return;
+		}
+
+		// If we're ready but there's no data: EOF.
+		if rc.unwrap() == 0 {
+			trace!("Unexpected EOF");
+			self.error = Some(TlsClientError::UnexpectedEof);
+			self.closing = true;
+			return;
+		}
+
+		// Reading some TLS data might have yielded new TLS messages to process.
+		// Errors from this indicate TLS protocol problems and are fatal.
+		let processed = self.tls_session.process_new_packets();
+		if let Err(e) = processed {
+			trace!("TLS error: {:?}", e);
+			self.error = Some(TlsClientError::Tls(e));
+			self.closing = true;
+			return;
+		}
+
+		// Having read some TLS data, and processed any new messages, we might have new plaintext as a result.
+		// Read it and then write it to stdout.
+		let mut plaintext = Vec::new();
+		let rc = self.tls_session.read_to_end(&mut plaintext);
+		if !plaintext.is_empty() {
+			self.writer.write(&plaintext).unwrap_or_else(|e| {
+				trace!("Write error: {:?}", e);
+				self.error = Some(TlsClientError::Writer(e));
+				0
+			});
+		}
+
+		// If that fails, the peer might have started a clean TLS-level session closure.
+		if let Err(err) = rc {
+			if err.kind() != io::ErrorKind::ConnectionAborted {
+				self.error = Some(TlsClientError::Connection(err));
+			}
+			self.closing = true;
+		}
+	}
+
+	fn do_write(&mut self) {
+		self.tls_session.write_tls(&mut self.socket).unwrap_or_else(|e| {
+			warn!("TLS write error: {:?}", e);
+			0
+		});
+	}
+
+	// Use wants_read/wants_write to register for different mio-level IO readiness events.
+	fn event_set(&self) -> mio::EventSet {
+		let rd = self.tls_session.wants_read();
+		let wr = self.tls_session.wants_write();
+
+		if rd && wr {
+			mio::EventSet::readable() | mio::EventSet::writable()
+		} else if wr {
+			mio::EventSet::writable()
+		} else {
+			mio::EventSet::readable()
+		}
+	}
+
+	fn is_closed(&self) -> bool {
+		self.closing
+	}
+
+	fn is_aborted(&self) -> bool {
+		self.abort.load(Ordering::Relaxed)
+	}
+}
+

util/https-fetch/src/url.rs

@@ -0,0 +1,81 @@
+// Copyright 2015, 2016 Ethcore (UK) Ltd.
+// This file is part of Parity.
+
+// Parity is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Parity is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Parity.  If not, see <http://www.gnu.org/licenses/>.
+
+use std::net::SocketAddr;
+
+#[derive(Debug)]
+pub enum UrlError {
+	InvalidAddress
+}
+
+/// Build a ClientConfig from our arguments
+pub struct Url {
+	address: SocketAddr,
+	hostname: String,
+	port: u16,
+	path: String,
+}
+
+impl Url {
+	pub fn new(hostname: &str, port: u16, path: &str) -> Result<Self, UrlError> {
+		let addr = try!(Self::lookup_ipv4(hostname, port));
+		Ok(Url {
+			address: addr,
+			hostname: hostname.into(),
+			port: port,
+			path: path.into(),
+		})
+	}
+
+	fn lookup_ipv4(host: &str, port: u16) -> Result<SocketAddr, UrlError> {
+		use std::net::ToSocketAddrs;
+
+		let addrs = try!((host, port).to_socket_addrs().map_err(|_| UrlError::InvalidAddress));
+		for addr in addrs {
+			if let SocketAddr::V4(_) = addr {
+				return Ok(addr.clone());
+			}
+		}
+		Err(UrlError::InvalidAddress)
+	}
+
+	pub fn address(&self) -> &SocketAddr {
+		&self.address
+	}
+
+	pub fn hostname(&self) -> &str {
+		&self.hostname
+	}
+
+	pub fn port(&self) -> u16 {
+		self.port
+	}
+
+	pub fn path(&self) -> &str {
+		&self.path
+	}
+}
+
+#[cfg(test)]
+#[test]
+fn should_parse_url() {
+	// given
+	let url = Url::new("github.com", 443, "/").unwrap();
+
+	assert_eq!(url.hostname(), "github.com");
+	assert_eq!(url.port(), 443);
+	assert_eq!(url.path(), "/");
+}

util/src/sha3.rs

@@ -69,7 +69,7 @@ impl<T> Hashable for T where T: AsRef<[u8]> {
 }
 
 /// Calculate SHA3 of given stream.
-pub fn sha3<R: io::Read>(r: &mut R) -> Result<H256, io::Error> {
+pub fn sha3(r: &mut io::BufRead) -> Result<H256, io::Error> {
 	let mut output = [0u8; 32];
 	let mut input = [0u8; 1024];
 	let mut sha3 = Keccak::new_keccak256();
@@ -90,7 +90,7 @@ pub fn sha3<R: io::Read>(r: &mut R) -> Result<H256, io::Error> {
 #[cfg(test)]
 mod tests {
 	use std::fs;
-	use std::io::Write;
+	use std::io::{Write, BufReader};
 	use super::*;
 
 	#[test]
@@ -113,7 +113,7 @@ mod tests {
 			file.write_all(b"something").unwrap();
 		}
 
-		let mut file = fs::File::open(&path).unwrap();
+		let mut file = BufReader::new(fs::File::open(&path).unwrap());
 		// when
 		let hash = sha3(&mut file).unwrap();