Implementing PanicHandlers for all places when new thread is spawned. Handling Client panics

ethcore/src/block_queue.rs

@@ -26,6 +26,7 @@ use views::*;
 use header::*;
 use service::*;
 use client::BlockStatus;
+use util::panics::*;
 
 /// Block queue status
 #[derive(Debug)]
@@ -59,6 +60,7 @@ impl BlockQueueInfo {
 /// A queue of blocks. Sits between network or other I/O and the BlockChain.
 /// Sorts them ready for blockchain insertion.
 pub struct BlockQueue {
+	panic_handler: SafeStringPanicHandler,
 	engine: Arc<Box<Engine>>,
 	more_to_verify: Arc<Condvar>,
 	verification: Arc<Mutex<Verification>>,
@@ -113,6 +115,7 @@ impl BlockQueue {
 		let ready_signal = Arc::new(QueueSignal { signalled: AtomicBool::new(false), message_channel: message_channel });
 		let deleting = Arc::new(AtomicBool::new(false));
 		let empty = Arc::new(Condvar::new());
+		let panic_handler = StringPanicHandler::new_thread_safe();
 
 		let mut verifiers: Vec<JoinHandle<()>> = Vec::new();
 		let thread_count = max(::num_cpus::get(), 3) - 2;
@@ -123,11 +126,22 @@ impl BlockQueue {
 			let ready_signal = ready_signal.clone();
 			let empty = empty.clone();
 			let deleting = deleting.clone();
-			verifiers.push(thread::Builder::new().name(format!("Verifier #{}", i)).spawn(move || BlockQueue::verify(verification, engine, more_to_verify, ready_signal, deleting, empty))
-				.expect("Error starting block verification thread"));
+			let panic_handler = panic_handler.clone();
+			verifiers.push(
+				thread::Builder::new()
+				.name(format!("Verifier #{}", i))
+				.spawn(move || {
+					let mut panic = panic_handler.lock().unwrap();
+					panic.catch_panic(move || {
+					  BlockQueue::verify(verification, engine, more_to_verify, ready_signal, deleting, empty)
+					}).unwrap()
+				})
+				.expect("Error starting block verification thread")
+			);
 		}
 		BlockQueue {
 			engine: engine,
+			panic_handler: panic_handler,
 			ready_signal: ready_signal.clone(),
 			more_to_verify: more_to_verify.clone(),
 			verification: verification.clone(),
@@ -150,7 +164,7 @@ impl BlockQueue {
 				while lock.unverified.is_empty() && !deleting.load(AtomicOrdering::Relaxed) {
 					lock = wait.wait(lock).unwrap();
 				}
-				
+
 				if deleting.load(AtomicOrdering::Relaxed) {
 					return;
 				}
@@ -324,6 +338,12 @@ impl BlockQueue {
 	}
 }
 
+impl MayPanic<String> for BlockQueue {
+	fn on_panic<F>(&self, closure: F) where F: OnPanicListener<String> {
+		self.panic_handler.on_panic(closure);
+	}
+}
+
 impl Drop for BlockQueue {
 	fn drop(&mut self) {
 		self.clear();

ethcore/src/client.rs

@@ -17,6 +17,7 @@
 //! Blockchain database client.
 
 use util::*;
+use util::panics::*;
 use rocksdb::{Options, DB, DBCompactionStyle};
 use blockchain::{BlockChain, BlockProvider, CacheSize};
 use views::BlockView;
@@ -157,7 +158,8 @@ pub struct Client {
 	state_db: Mutex<JournalDB>,
 	block_queue: RwLock<BlockQueue>,
 	report: RwLock<ClientReport>,
-	import_lock: Mutex<()>
+	import_lock: Mutex<()>,
+	panic_handler: SafeStringPanicHandler,
 }
 
 const HISTORY: u64 = 1000;
@@ -198,19 +200,26 @@ impl Client {
 		let mut state_path = path.to_path_buf();
 		state_path.push("state");
 		let db = Arc::new(DB::open(&opts, state_path.to_str().unwrap()).unwrap());
-		
+
 		let engine = Arc::new(try!(spec.to_engine()));
 		let mut state_db = JournalDB::new_with_arc(db.clone());
 		if state_db.is_empty() && engine.spec().ensure_db_good(&mut state_db) {
 			state_db.commit(0, &engine.spec().genesis_header().hash(), None).expect("Error commiting genesis state to state DB");
 		}
+
+		let block_queue = BlockQueue::new(engine.clone(), message_channel);
+		let panic_handler = StringPanicHandler::new_thread_safe();
+		let panic = panic_handler.clone();
+		block_queue.on_panic(move |t: &String| panic.lock().unwrap().notify_all(t));
+
 		Ok(Arc::new(Client {
 			chain: chain,
-			engine: engine.clone(),
+			engine: engine,
 			state_db: Mutex::new(state_db),
-			block_queue: RwLock::new(BlockQueue::new(engine, message_channel)),
+			block_queue: RwLock::new(block_queue),
 			report: RwLock::new(Default::default()),
 			import_lock: Mutex::new(()),
+			panic_handler: panic_handler
 		}))
 	}
 
@@ -348,12 +357,12 @@ impl BlockChainClient for Client {
 
 	fn block_status(&self, hash: &H256) -> BlockStatus {
 		if self.chain.read().unwrap().is_known(&hash) {
-			BlockStatus::InChain 
-		} else { 
-			self.block_queue.read().unwrap().block_status(hash) 
+			BlockStatus::InChain
+		} else {
+			self.block_queue.read().unwrap().block_status(hash)
 		}
 	}
-	
+
 	fn block_total_difficulty(&self, hash: &H256) -> Option<U256> {
 		self.chain.read().unwrap().block_details(hash).map(|d| d.total_difficulty)
 	}
@@ -423,3 +432,9 @@ impl BlockChainClient for Client {
 		}
 	}
 }
+
+impl MayPanic<String> for Client {
+	fn on_panic<F>(&self, closure: F) where F: OnPanicListener<String> {
+		self.panic_handler.on_panic(closure);
+	}
+}