diff --git a/util/src/keys/directory.rs b/util/src/keys/directory.rs
index d0d3393cd..a92bf4593 100644
--- a/util/src/keys/directory.rs
+++ b/util/src/keys/directory.rs
@@ -542,6 +542,8 @@ impl KeyDirectory {
 		if removes.is_empty() { return; }
 		let mut cache = self.cache.write().unwrap();
 		for key in removes { cache.remove(&key); }
+
+		cache.shrink_to_fit();
 	}
 
 	/// Reports how many keys are currently cached.
diff --git a/util/src/keys/store.rs b/util/src/keys/store.rs
index e0a505f79..78540bdb0 100644
--- a/util/src/keys/store.rs
+++ b/util/src/keys/store.rs
@@ -273,13 +273,16 @@ impl SecretStore {
 
 	/// Makes account unlocks expire and removes unused key files from memory
 	pub fn collect_garbage(&mut self) {
+		let mut garbage_lock = self.unlocks.write().unwrap();
 		self.directory.collect_garbage();
 		let utc = UTC::now();
-		let expired_addresses = self.unlocks.read().unwrap().iter()
+		let expired_addresses = garbage_lock.iter()
 			.filter(|&(_, unlock)| unlock.expires < utc)
 			.map(|(address, _)| address.clone()).collect::<Vec<Address>>();
 
-		for expired in expired_addresses { self.unlocks.write().unwrap().remove(&expired); }
+		for expired in expired_addresses { garbage_lock.remove(&expired); }
+
+		garbage_lock.shrink_to_fit();
 	}
 }