Automatic compaction selection on Linux (#2785)

* add auto compaction types

* pass db paths

* detect drive type on Linux

* use base db path

* add docstring

* limit the test to be side effect free

* use base db path

* more docs

* fix parsing test

* update error

* detect only on Linux

* make test Linux only

* add second device letter, update cli doc

* use spaces in cli doc

* import only on linux

* default->auto

util/src/kvdb.rs

@@ -24,6 +24,12 @@ use std::path::PathBuf;
 use rlp::{UntrustedRlp, RlpType, View, Compressible};
 use rocksdb::{DB, Writable, WriteBatch, WriteOptions, IteratorMode, DBIterator,
 	Options, DBCompactionStyle, BlockBasedOptions, Direction, Cache, Column, ReadOptions};
+#[cfg(target_os = "linux")]
+use regex::Regex;
+#[cfg(target_os = "linux")]
+use std::process::Command;
+#[cfg(target_os = "linux")]
+use std::fs::File;
 
 const DB_BACKGROUND_FLUSHES: i32 = 2;
 const DB_BACKGROUND_COMPACTIONS: i32 = 2;
@@ -110,7 +116,7 @@ enum KeyState {
 }
 
 /// Compaction profile for the database settings
-#[derive(Clone, Copy)]
+#[derive(Clone, Copy, PartialEq, Debug)]
 pub struct CompactionProfile {
 	/// L0-L1 target file size
 	pub initial_file_size: u64,
@@ -123,16 +129,73 @@ pub struct CompactionProfile {
 impl Default for CompactionProfile {
 	/// Default profile suitable for most storage
 	fn default() -> CompactionProfile {
+		CompactionProfile::ssd()
+	}
+}
+
+/// Given output of df command return Linux rotational flag file path.
+#[cfg(target_os = "linux")]
+pub fn rotational_from_df_output(df_out: Vec<u8>) -> Option<PathBuf> {
+	str::from_utf8(df_out.as_slice())
+		.ok()
+		// Get the drive name.
+		.and_then(|df_str| Regex::new(r"/dev/(sd[:alpha:]{1,2})")
+			.ok()
+			.and_then(|re| re.captures(df_str))
+			.and_then(|captures| captures.at(1)))
+		// Generate path e.g. /sys/block/sda/queue/rotational
+		.map(|drive_path| {
+			let mut p = PathBuf::from("/sys/block");
+			p.push(drive_path);
+			p.push("queue/rotational");
+			p
+		})
+}
+
+impl CompactionProfile {
+	/// Attempt to determine the best profile automatically, only Linux for now.
+	#[cfg(target_os = "linux")]
+	pub fn auto(db_path: &Path) -> CompactionProfile {
+		let hdd_check_file = db_path
+			.to_str()
+			.and_then(|path_str| Command::new("df").arg(path_str).output().ok())
+			.and_then(|df_res| match df_res.status.success() {
+				true => Some(df_res.stdout),
+				false => None,
+			})
+			.and_then(rotational_from_df_output);
+		// Read out the file and match compaction profile.
+		if let Some(hdd_check) = hdd_check_file {
+			if let Ok(mut file) = File::open(hdd_check.as_path()) {
+				let mut buffer = [0; 1];
+				if file.read_exact(&mut buffer).is_ok() {
+					// 0 means not rotational.
+					if buffer == [48] { return Self::ssd(); }
+					// 1 means rotational.
+					if buffer == [49] { return Self::hdd(); }
+				}
+			}
+		}
+		// Fallback if drive type was not determined.
+		Self::default()
+	}
+
+	/// Just default for other platforms.
+	#[cfg(not(target_os = "linux"))]
+	pub fn auto(_db_path: &Path) -> CompactionProfile {
+		Self::default()
+	}
+
+	/// Default profile suitable for SSD storage
+	pub fn ssd() -> CompactionProfile {
 		CompactionProfile {
 			initial_file_size: 32 * 1024 * 1024,
 			file_size_multiplier: 2,
 			write_rate_limit: None,
 		}
 	}
-}
 
-impl CompactionProfile {
-	/// Slow hdd compaction profile
+	/// Slow HDD compaction profile
 	pub fn hdd() -> CompactionProfile {
 		CompactionProfile {
 			initial_file_size: 192 * 1024 * 1024,
@@ -538,6 +601,7 @@ mod tests {
 	use super::*;
 	use devtools::*;
 	use std::str::FromStr;
+	use std::path::PathBuf;
 
 	fn test_db(config: &DatabaseConfig) {
 		let path = RandomTempPath::create_dir();
@@ -598,4 +662,13 @@ mod tests {
 		let _ = Database::open_default(path.as_path().to_str().unwrap()).unwrap();
 		test_db(&DatabaseConfig::default());
 	}
+
+	#[test]
+	#[cfg(target_os = "linux")]
+	fn df_to_rotational() {
+		// Example df output.
+		let example_df = vec![70, 105, 108, 101, 115, 121, 115, 116, 101, 109, 32, 32, 32, 32, 32, 49, 75, 45, 98, 108, 111, 99, 107, 115, 32, 32, 32, 32, 32, 85, 115, 101, 100, 32, 65, 118, 97, 105, 108, 97, 98, 108, 101, 32, 85, 115, 101, 37, 32, 77, 111, 117, 110, 116, 101, 100, 32, 111, 110, 10, 47, 100, 101, 118, 47, 115, 100, 97, 49, 32, 32, 32, 32, 32, 32, 32, 54, 49, 52, 48, 57, 51, 48, 48, 32, 51, 56, 56, 50, 50, 50, 51, 54, 32, 32, 49, 57, 52, 52, 52, 54, 49, 54, 32, 32, 54, 55, 37, 32, 47, 10];
+		let expected_output = Some(PathBuf::from("/sys/block/sda/queue/rotational"));
+		assert_eq!(rotational_from_df_output(example_df), expected_output);
+	}
 }

util/src/lib.rs

@@ -104,6 +104,7 @@ extern crate parking_lot;
 extern crate ansi_term;
 extern crate tiny_keccak;
 extern crate rlp;
+extern crate regex;
 
 #[macro_use]
 extern crate heapsize;