service: storage monitor added

Storage monitor added. It uses `notify` create to get notifications
about any changes to monitored path (which is database path).
Notifications are consumed in essential task which terminates when
available storage space drops below given threshold.

Closes: #12399

bin/node/cli/benches/block_production.rs

@@ -113,6 +113,7 @@ fn new_node(tokio_handle: Handle) -> node_cli::service::NewFullBase {
 		base_path: Some(base_path),
 		informant_output_format: Default::default(),
 		wasm_runtime_overrides: None,
+		available_storage_theshold: 1000,
 	};
 
 	node_cli::service::new_full_base(config, false, |_, _| ())

bin/node/cli/benches/transaction_pool.rs

@@ -106,6 +106,7 @@ fn new_node(tokio_handle: Handle) -> node_cli::service::NewFullBase {
 		base_path: Some(base_path),
 		informant_output_format: Default::default(),
 		wasm_runtime_overrides: None,
+		available_storage_theshold: 1000,
 	};
 
 	node_cli::service::new_full_base(config, false, |_, _| ()).expect("Creates node")

client/cli/src/config.rs

@@ -191,6 +191,16 @@ pub trait CliConfiguration<DCV: DefaultConfigurationValues = ()>: Sized {
 			.unwrap_or_else(|| Ok((None, KeystoreConfig::InMemory)))
 	}
 
+	/// Get the database available storage space threshold.
+	///
+	/// By default this is retrieved from `DatabaseParams` if it is available.
+	fn database_storage_threshold(&self) -> Result<u64> {
+		Ok(self
+			.database_params()
+			.map(|x| x.database_storage_threshold())
+			.unwrap_or_default())
+	}
+
 	/// Get the database cache size.
 	///
 	/// By default this is retrieved from `DatabaseParams` if it is available. Otherwise its `None`.
@@ -562,6 +572,7 @@ pub trait CliConfiguration<DCV: DefaultConfigurationValues = ()>: Sized {
 			base_path: Some(base_path),
 			informant_output_format: Default::default(),
 			runtime_cache_size,
+			available_storage_theshold: self.database_storage_threshold()?,
 		})
 	}
 

client/cli/src/params/database_params.rs

@@ -19,7 +19,7 @@
 use crate::arg_enums::Database;
 use clap::Args;
 
-/// Parameters for block import.
+/// Parameters for database
 #[derive(Debug, Clone, PartialEq, Args)]
 pub struct DatabaseParams {
 	/// Select database backend to use.
@@ -29,10 +29,16 @@ pub struct DatabaseParams {
 	/// Limit the memory the database cache can use.
 	#[arg(long = "db-cache", value_name = "MiB")]
 	pub database_cache_size: Option<usize>,
+
+	/// Required available space on database storage. If available space for DB storage drops below
+	/// the given threshold, node will be gracefully terminated. If `0` is given monitoring will be
+	/// disabled.
+	#[arg(long = "db-storage-threshold", value_name = "MB", default_value_t = 1000)]
+	pub database_storage_threshold: u64,
 }
 
 impl DatabaseParams {
-	/// Limit the memory the database cache can use.
+	/// Database backend
 	pub fn database(&self) -> Option<Database> {
 		self.database
 	}
@@ -41,4 +47,9 @@ impl DatabaseParams {
 	pub fn database_cache_size(&self) -> Option<usize> {
 		self.database_cache_size
 	}
+
+	/// Available storage space threshold
+	pub fn database_storage_threshold(&self) -> u64 {
+		self.database_storage_threshold
+	}
 }

client/cli/src/runner.rs

@@ -367,6 +367,7 @@ mod tests {
 				base_path: None,
 				informant_output_format: Default::default(),
 				runtime_cache_size: 2,
+				available_storage_theshold: 1000,
 			},
 			runtime,
 		)

client/service/Cargo.toml

@@ -80,6 +80,8 @@ tokio = { version = "1.22.0", features = ["time", "rt-multi-thread", "parking_lo
 tempfile = "3.1.0"
 directories = "4.0.1"
 static_init = "1.0.3"
+nix = { version = "0.26.1", features = ["fs"] }
+notify = { version = "5.0.0", default-features = false, feature=["macos_kqueue"] }
 
 [dev-dependencies]
 substrate-test-runtime-client = { version = "2.0.0", path = "../../test-utils/runtime/client" }

client/service/src/builder.rs

@@ -22,8 +22,10 @@ use crate::{
 	config::{Configuration, KeystoreConfig, PrometheusConfig},
 	error::Error,
 	metrics::MetricsService,
-	start_rpc_servers, BuildGenesisBlock, GenesisBlockBuilder, RpcHandlers, SpawnTaskHandle,
-	TaskManager, TransactionPoolAdapter,
+	start_rpc_servers,
+	storage_monitor::StorageMonitorService,
+	BuildGenesisBlock, GenesisBlockBuilder, RpcHandlers, SpawnTaskHandle, TaskManager,
+	TransactionPoolAdapter,
 };
 use futures::{channel::oneshot, future::ready, FutureExt, StreamExt};
 use jsonrpsee::RpcModule;
@@ -534,6 +536,14 @@ where
 		metrics_service.run(client.clone(), transaction_pool.clone(), network.clone()),
 	);
 
+	if let Some(storage_monitor_service) = StorageMonitorService::new_for_config(&config)? {
+		task_manager.spawn_essential_handle().spawn(
+			"storage-monitor",
+			None,
+			storage_monitor_service.run(),
+		)
+	}
+
 	let rpc_id_provider = config.rpc_id_provider.take();
 
 	// jsonrpsee RPC

client/service/src/config.rs

@@ -149,6 +149,9 @@ pub struct Configuration {
 	pub informant_output_format: sc_informant::OutputFormat,
 	/// Maximum number of different runtime versions that can be cached.
 	pub runtime_cache_size: u8,
+	/// Threshold (in megabytes) for available storage space associated with `base_path`. `0` means
+	/// no storage monitoring.
+	pub available_storage_theshold: u64,
 }
 
 /// Type for tasks spawned by the executor.

client/service/src/lib.rs

@@ -32,6 +32,7 @@ pub mod client;
 #[cfg(not(feature = "test-helpers"))]
 mod client;
 mod metrics;
+mod storage_monitor;
 mod task_manager;
 
 use std::{collections::HashMap, net::SocketAddr};