Control service for IPC (#2013)

* hypervisor extension

* sorted with shutdown-wait

* hypervisor lifecycle alter

ipc/hypervisor/src/lib.rs

@@ -33,7 +33,7 @@ use service::{HypervisorService, IpcModuleId};
 use std::process::{Command,Child};
 use std::collections::HashMap;
 
-pub use service::{HypervisorServiceClient, CLIENT_MODULE_ID, SYNC_MODULE_ID};
+pub use service::{HypervisorServiceClient, ControlService, CLIENT_MODULE_ID, SYNC_MODULE_ID};
 
 pub type BinaryId = &'static str;
 
@@ -174,6 +174,10 @@ impl Hypervisor {
 		self.service.unchecked_count() == 0
 	}
 
+	pub fn modules_shutdown(&self) -> bool {
+		self.service.running_count() == 0
+	}
+
 	/// Waits for every required module to check in
 	pub fn wait_for_startup(&self) {
 		let mut worker = self.ipc_worker.write().unwrap();
@@ -182,21 +186,30 @@ impl Hypervisor {
 		}
 	}
 
-	/// Shutdown the ipc and all managed child processes
-	pub fn shutdown(&self, wait_time: Option<std::time::Duration>) {
-		if wait_time.is_some() { std::thread::sleep(wait_time.unwrap()) }
-
-		let mut childs = self.processes.write().unwrap();
-		for (ref mut module, ref mut child) in childs.iter_mut() {
-			trace!(target: "hypervisor", "Stopping process module: {}", module);
-			child.kill().unwrap();
+	/// Waits for every required module to check in
+	pub fn wait_for_shutdown(&self) {
+		let mut worker = self.ipc_worker.write().unwrap();
+		while !self.modules_shutdown() {
+			worker.poll()
 		}
 	}
+
+	/// Shutdown the ipc and all managed child processes
+	pub fn shutdown(&self) {
+		let mut childs = self.processes.write().unwrap();
+		for (ref mut module, _) in childs.iter_mut() {
+			trace!(target: "hypervisor", "Stopping process module: {}", module);
+			self.service.send_shutdown(**module);
+		}
+		trace!(target: "hypervisor", "Waiting for shutdown...");
+		self.wait_for_shutdown();
+		trace!(target: "hypervisor", "All modules reported shutdown");
+	}
 }
 
 impl Drop for Hypervisor {
 	fn drop(&mut self) {
-		self.shutdown(Some(std::time::Duration::new(1, 0)));
+		self.shutdown();
 	}
 }
 

ipc/hypervisor/src/service.rs.in

@@ -17,6 +17,7 @@
 use std::sync::{RwLock,Arc};
 use ipc::IpcConfig;
 use std::collections::HashMap;
+use nanoipc;
 
 pub type IpcModuleId = u64;
 
@@ -28,15 +29,43 @@ pub const SYNC_MODULE_ID: IpcModuleId = 2100;
 
 /// IPC service that handles module management
 pub struct HypervisorService {
-	check_list: RwLock<HashMap<IpcModuleId, bool>>,
+	modules: RwLock<HashMap<IpcModuleId, ModuleState>>,
+}
+
+#[derive(Default)]
+pub struct ModuleState {
+	started: bool,
+	control_url: String,
+	shutdown: bool,
+}
+
+
+#[derive(Ipc)]
+pub trait ControlService {
+	fn shutdown(&self);
 }
 
 #[derive(Ipc)]
 impl HypervisorService {
-	fn module_ready(&self, module_id: u64) -> bool {
-		let mut check_list = self.check_list.write().unwrap();
-		check_list.get_mut(&module_id).map(|mut status| *status = true);
-		check_list.iter().any(|(_, status)| !status)
+	// return type for making method synchronous
+	fn module_ready(&self, module_id: u64, control_url: String) -> bool {
+		let mut modules = self.modules.write().unwrap();
+		modules.get_mut(&module_id).map(|mut module| {
+			module.started = true;
+			module.control_url = control_url;
+		});
+		trace!(target: "hypervisor", "Module ready: {}", module_id);
+		true
+	}
+
+	// return type for making method synchronous
+	fn module_shutdown(&self, module_id: u64) -> bool {
+		let mut modules = self.modules.write().unwrap();
+		modules.get_mut(&module_id).map(|mut module| {
+			module.shutdown = true;
+		});
+		trace!(target: "hypervisor", "Module shutdown: {}", module_id);
+		true
 	}
 }
 
@@ -48,29 +77,46 @@ impl HypervisorService {
 
 	/// New service with list of modules that will report for being ready
 	pub fn with_modules(module_ids: Vec<IpcModuleId>) -> Arc<HypervisorService> {
-		let mut check_list = HashMap::new();
+		let mut modules = HashMap::new();
 		for module_id in module_ids {
-			check_list.insert(module_id, false);
+			modules.insert(module_id, ModuleState::default());
 		}
 		Arc::new(HypervisorService {
-			check_list: RwLock::new(check_list),
+			modules: RwLock::new(modules),
 		})
 	}
 
 	/// Add the module to the check-list
 	pub fn add_module(&self, module_id: IpcModuleId) {
-		self.check_list.write().unwrap().insert(module_id, false);
+		self.modules.write().unwrap().insert(module_id, ModuleState::default());
 	}
 
 	/// Number of modules still being waited for check-in
 	pub fn unchecked_count(&self) -> usize {
-		self.check_list.read().unwrap().iter().filter(|&(_, status)| !status).count()
+		self.modules.read().unwrap().iter().filter(|&(_, module)| !module.started).count()
 	}
 
 	/// List of all modules within this service
 	pub fn module_ids(&self) -> Vec<IpcModuleId> {
-		self.check_list.read().unwrap().iter().map(|(module_id, _)| module_id).cloned().collect()
+		self.modules.read().unwrap().iter().map(|(module_id, _)| module_id).cloned().collect()
+	}
+
+	/// Number of modules started and running
+	pub fn running_count(&self) -> usize {
+		self.modules.read().unwrap().iter().filter(|&(_, module)| module.started && !module.shutdown).count()
+	}
+
+	pub fn send_shutdown(&self, module_id: IpcModuleId) {
+		let modules = self.modules.read().unwrap();
+		modules.get(&module_id).map(|module| {
+			trace!(target: "hypervisor", "Sending shutdown to {}({})", module_id, &module.control_url);
+			let client = nanoipc::init_client::<ControlServiceClient<_>>(&module.control_url).unwrap();
+			client.shutdown();
+			trace!(target: "hypervisor", "Sent shutdown to {}", module_id);
+		});
 	}
 }
 
 impl ::ipc::IpcConfig for HypervisorService {}
+
+impl ::ipc::IpcConfig for ControlService {}