Memory-based pruning history size (#4114)

* prune states based on memory param

* pruning memory CLI and usage in sync

* return purged value from memorydb

* calculate memory used incrementally in overlayrecentdb

* refactor shared history pruning code in client

* Fixed usage alignment

* journal_size function for fast memory calculation

parity/blockchain.rs

@@ -85,6 +85,7 @@ pub struct ImportBlockchain {
 	pub format: Option<DataFormat>,
 	pub pruning: Pruning,
 	pub pruning_history: u64,
+	pub pruning_memory: usize,
 	pub compaction: DatabaseCompactionProfile,
 	pub wal: bool,
 	pub tracing: Switch,
@@ -104,6 +105,7 @@ pub struct ExportBlockchain {
 	pub format: Option<DataFormat>,
 	pub pruning: Pruning,
 	pub pruning_history: u64,
+	pub pruning_memory: usize,
 	pub compaction: DatabaseCompactionProfile,
 	pub wal: bool,
 	pub fat_db: Switch,
@@ -122,6 +124,7 @@ pub struct ExportState {
 	pub format: Option<DataFormat>,
 	pub pruning: Pruning,
 	pub pruning_history: u64,
+	pub pruning_memory: usize,
 	pub compaction: DatabaseCompactionProfile,
 	pub wal: bool,
 	pub fat_db: Switch,
@@ -196,6 +199,7 @@ fn execute_import(cmd: ImportBlockchain) -> Result<(), String> {
 		"".into(),
 		algorithm,
 		cmd.pruning_history,
+		cmd.pruning_memory,
 		cmd.check_seal
 	);
 
@@ -310,6 +314,7 @@ fn start_client(
 	spec: SpecType,
 	pruning: Pruning,
 	pruning_history: u64,
+	pruning_memory: usize,
 	tracing: Switch,
 	fat_db: Switch,
 	compaction: DatabaseCompactionProfile,
@@ -354,7 +359,20 @@ fn start_client(
 	dirs.create_dirs(false, false)?;
 
 	// prepare client config
-	let client_config = to_client_config(&cache_config, Mode::Active, tracing, fat_db, compaction, wal, VMType::default(), "".into(), algorithm, pruning_history, true);
+	let client_config = to_client_config(
+		&cache_config,
+		Mode::Active,
+		tracing,
+		fat_db,
+		compaction,
+		wal,
+		VMType::default(),
+		"".into(),
+		algorithm,
+		pruning_history,
+		pruning_memory,
+		true,
+	);
 
 	let service = ClientService::start(
 		client_config,
@@ -371,7 +389,18 @@ fn start_client(
 
 fn execute_export(cmd: ExportBlockchain) -> Result<(), String> {
 	// Setup panic handler
-	let service = start_client(cmd.dirs, cmd.spec, cmd.pruning, cmd.pruning_history, cmd.tracing, cmd.fat_db, cmd.compaction, cmd.wal, cmd.cache_config)?;
+	let service = start_client(
+		cmd.dirs,
+		cmd.spec,
+		cmd.pruning,
+		cmd.pruning_history,
+		cmd.pruning_memory,
+		cmd.tracing,
+		cmd.fat_db,
+		cmd.compaction,
+		cmd.wal,
+		cmd.cache_config
+	)?;
 	let panic_handler = PanicHandler::new_in_arc();
 	let format = cmd.format.unwrap_or_default();
 
@@ -403,7 +432,19 @@ fn execute_export(cmd: ExportBlockchain) -> Result<(), String> {
 
 fn execute_export_state(cmd: ExportState) -> Result<(), String> {
 	// Setup panic handler
-	let service = start_client(cmd.dirs, cmd.spec, cmd.pruning, cmd.pruning_history, cmd.tracing, cmd.fat_db, cmd.compaction, cmd.wal, cmd.cache_config)?;
+	let service = start_client(
+		cmd.dirs,
+		cmd.spec,
+		cmd.pruning,
+		cmd.pruning_history,
+		cmd.pruning_memory,
+		cmd.tracing,
+		cmd.fat_db,
+		cmd.compaction,
+		cmd.wal,
+		cmd.cache_config
+	)?;
+
 	let panic_handler = PanicHandler::new_in_arc();
 
 	panic_handler.forward_from(&service);

parity/cli/config.full.toml

@@ -95,6 +95,7 @@ notify_work = ["http://localhost:3001"]
 tracing = "auto"
 pruning = "auto"
 pruning_history = 1200
+pruning_memory = 500
 cache_size_db = 64
 cache_size_blocks = 8
 cache_size_queue = 50

parity/cli/mod.rs

@@ -238,6 +238,8 @@ usage! {
 			or |c: &Config| otry!(c.footprint).pruning.clone(),
 		flag_pruning_history: u64 = 1200u64,
 			or |c: &Config| otry!(c.footprint).pruning_history.clone(),
+		flag_pruning_memory: usize = 150usize,
+			or |c: &Config| otry!(c.footprint).pruning_memory.clone(),
 		flag_cache_size_db: u32 = 64u32,
 			or |c: &Config| otry!(c.footprint).cache_size_db.clone(),
 		flag_cache_size_blocks: u32 = 8u32,
@@ -421,6 +423,7 @@ struct Footprint {
 	tracing: Option<String>,
 	pruning: Option<String>,
 	pruning_history: Option<u64>,
+	pruning_memory: Option<usize>,
 	fast_and_loose: Option<bool>,
 	cache_size: Option<u32>,
 	cache_size_db: Option<u32>,
@@ -635,6 +638,7 @@ mod tests {
 			flag_tracing: "auto".into(),
 			flag_pruning: "auto".into(),
 			flag_pruning_history: 1200u64,
+			flag_pruning_memory: 500usize,
 			flag_cache_size_db: 64u32,
 			flag_cache_size_blocks: 8u32,
 			flag_cache_size_queue: 50u32,
@@ -812,6 +816,7 @@ mod tests {
 				tracing: Some("on".into()),
 				pruning: Some("fast".into()),
 				pruning_history: Some(64),
+				pruning_memory: None,
 				fast_and_loose: None,
 				cache_size: None,
 				cache_size_db: Some(128),

parity/cli/usage.txt

@@ -271,8 +271,12 @@ Footprint Options:
                            fast - maintain journal overlay. Fast but 50MB used.
                            auto - use the method most recently synced or
                            default to fast if none synced (default: {flag_pruning}).
-  --pruning-history NUM    Set a number of recent states to keep when pruning
+  --pruning-history NUM    Set a minimum number of recent states to keep when pruning
                            is active. (default: {flag_pruning_history}).
+  --pruning-memory MB      The ideal amount of memory in megabytes to use to store
+                           recent states. As many states as possible will be kept
+                           within this limit, and at least --pruning-history states
+                           will always be kept. (default: {flag_pruning_memory})
   --cache-size-db MB       Override database cache size (default: {flag_cache_size_db}).
   --cache-size-blocks MB   Specify the prefered size of the blockchain cache in
                            megabytes (default: {flag_cache_size_blocks}).

parity/configuration.rs

@@ -214,6 +214,7 @@ impl Configuration {
 				format: format,
 				pruning: pruning,
 				pruning_history: pruning_history,
+				pruning_memory: self.args.flag_pruning_memory,
 				compaction: compaction,
 				wal: wal,
 				tracing: tracing,
@@ -234,6 +235,7 @@ impl Configuration {
 					format: format,
 					pruning: pruning,
 					pruning_history: pruning_history,
+					pruning_memory: self.args.flag_pruning_memory,
 					compaction: compaction,
 					wal: wal,
 					tracing: tracing,
@@ -252,6 +254,7 @@ impl Configuration {
 					format: format,
 					pruning: pruning,
 					pruning_history: pruning_history,
+					pruning_memory: self.args.flag_pruning_memory,
 					compaction: compaction,
 					wal: wal,
 					tracing: tracing,
@@ -273,6 +276,7 @@ impl Configuration {
 				spec: spec,
 				pruning: pruning,
 				pruning_history: pruning_history,
+				pruning_memory: self.args.flag_pruning_memory,
 				tracing: tracing,
 				fat_db: fat_db,
 				compaction: compaction,
@@ -289,6 +293,7 @@ impl Configuration {
 				spec: spec,
 				pruning: pruning,
 				pruning_history: pruning_history,
+				pruning_memory: self.args.flag_pruning_memory,
 				tracing: tracing,
 				fat_db: fat_db,
 				compaction: compaction,
@@ -313,6 +318,7 @@ impl Configuration {
 				spec: spec,
 				pruning: pruning,
 				pruning_history: pruning_history,
+				pruning_memory: self.args.flag_pruning_memory,
 				daemon: daemon,
 				logger_config: logger_config.clone(),
 				miner_options: miner_options,
@@ -943,6 +949,7 @@ mod tests {
 			format: Default::default(),
 			pruning: Default::default(),
 			pruning_history: 1200,
+			pruning_memory: 150,
 			compaction: Default::default(),
 			wal: true,
 			tracing: Default::default(),
@@ -965,6 +972,7 @@ mod tests {
 			file_path: Some("blockchain.json".into()),
 			pruning: Default::default(),
 			pruning_history: 1200,
+			pruning_memory: 150,
 			format: Default::default(),
 			compaction: Default::default(),
 			wal: true,
@@ -987,6 +995,7 @@ mod tests {
 			file_path: Some("state.json".into()),
 			pruning: Default::default(),
 			pruning_history: 1200,
+			pruning_memory: 150,
 			format: Default::default(),
 			compaction: Default::default(),
 			wal: true,
@@ -1011,6 +1020,7 @@ mod tests {
 			file_path: Some("blockchain.json".into()),
 			pruning: Default::default(),
 			pruning_history: 1200,
+			pruning_memory: 150,
 			format: Some(DataFormat::Hex),
 			compaction: Default::default(),
 			wal: true,
@@ -1046,6 +1056,7 @@ mod tests {
 			spec: Default::default(),
 			pruning: Default::default(),
 			pruning_history: 1200,
+			pruning_memory: 150,
 			daemon: None,
 			logger_config: Default::default(),
 			miner_options: Default::default(),

parity/helpers.rs

@@ -224,6 +224,7 @@ pub fn to_client_config(
 		name: String,
 		pruning: Algorithm,
 		pruning_history: u64,
+		pruning_memory: usize,
 		check_seal: bool,
 	) -> ClientConfig {
 	let mut client_config = ClientConfig::default();
@@ -247,6 +248,8 @@ pub fn to_client_config(
 	client_config.state_cache_size = cache_config.state() as usize * mb;
 	// in bytes
 	client_config.jump_table_size = cache_config.jump_tables() as usize * mb;
+	// in bytes
+	client_config.history_mem = pruning_memory * mb;
 
 	client_config.mode = mode;
 	client_config.tracing.enabled = tracing;

parity/run.rs

@@ -68,6 +68,7 @@ pub struct RunCmd {
 	pub spec: SpecType,
 	pub pruning: Pruning,
 	pub pruning_history: u64,
+	pub pruning_memory: usize,
 	/// Some if execution should be daemonized. Contains pid_file path.
 	pub daemon: Option<String>,
 	pub logger_config: LogConfig,
@@ -273,6 +274,7 @@ pub fn execute(cmd: RunCmd, can_restart: bool, logger: Arc<RotatingLogger>) -> R
 		cmd.name,
 		algorithm,
 		cmd.pruning_history,
+		cmd.pruning_memory,
 		cmd.check_seal,
 	);
 

parity/snapshot.rs

@@ -54,6 +54,7 @@ pub struct SnapshotCommand {
 	pub spec: SpecType,
 	pub pruning: Pruning,
 	pub pruning_history: u64,
+	pub pruning_memory: usize,
 	pub tracing: Switch,
 	pub fat_db: Switch,
 	pub compaction: DatabaseCompactionProfile,
@@ -170,7 +171,20 @@ impl SnapshotCommand {
 		execute_upgrades(&self.dirs.base, &db_dirs, algorithm, self.compaction.compaction_profile(db_dirs.db_root_path().as_path()))?;
 
 		// prepare client config
-		let client_config = to_client_config(&self.cache_config, Mode::Active, tracing, fat_db, self.compaction, self.wal, VMType::default(), "".into(), algorithm, self.pruning_history, true);
+		let client_config = to_client_config(
+			&self.cache_config,
+			Mode::Active,
+			tracing,
+			fat_db,
+			self.compaction,
+			self.wal,
+			VMType::default(),
+			"".into(),
+			algorithm,
+			self.pruning_history,
+			self.pruning_memory,
+			true
+		);
 
 		let service = ClientService::start(
 			client_config,