CP-13995: Periodic Archiving of SR-level RRDs

1) backup_rrds need to backup sr rrds on a host.
2) Xapi_periodic_scheduler performs backup of SR-level RRDs too.
3) Asks xcp-rrdd to archive RRDs and copy them to filesystem.
4) Xapi copies these archived RRDs from filesystem to sr-stats vdi

Signed-off-by: Sharad Yadav <[email protected]>

Author: sharady

ocaml/xapi/xapi_host.ml

@@ -950,10 +950,19 @@ let has_extension ~__context ~name =
 let sync_data ~__context ~host =
   Xapi_sync.sync_host __context host (* Nb, no attempt to wrap exceptions yet *)
 
+let copy_srs_rrds ~__context () =
+	let srs = Xapi_sr.srs_with_rrds ~__context in
+	if !srs <> [] then begin
+		List.iter (fun sr ->
+			let vdi = Xapi_vdi_helpers.create_rrd_vdi ~__context ~sr:sr in
+			Xapi_vdi_helpers.copy_sr_rdds ~__context ~sr:sr ~vdi:vdi ~archive:false) !srs
+	end
+
 let backup_rrds ~__context ~host ~delay =
 	Xapi_periodic_scheduler.add_to_queue "RRD backup" Xapi_periodic_scheduler.OneShot
 	delay (fun _ ->
-		log_and_ignore_exn (Rrdd.backup_rrds ~save_stats_locally:(Pool_role.is_master ()))
+		log_and_ignore_exn (Rrdd.backup_rrds ~save_stats_locally:(Pool_role.is_master ()));
+		log_and_ignore_exn (copy_srs_rrds ~__context)
 	)
 
 let get_servertime ~__context ~host =

ocaml/xapi/xapi_host.mli

@@ -164,7 +164,7 @@ val call_plugin :
   host:[ `host ] Ref.t ->
   plugin:string -> fn:string -> args:(string * string) list -> string
 val sync_data : __context:Context.t -> host:API.ref_host -> unit
-val backup_rrds : __context:'a -> host:'b -> delay:float -> unit
+val backup_rrds : __context:Context.t -> host:'b -> delay:float -> unit
 val get_servertime : __context:'a -> host:'b -> Date.iso8601
 val get_server_localtime : __context:'a -> host:'b -> Date.iso8601
 val enable_binary_storage :

ocaml/xapi/xapi_periodic_scheduler_init.ml

@@ -39,14 +39,15 @@ let register () =
   (* Periodic backup of RRDs *)
   let rrdbackup_timer = 
     if Xapi_fist.reduce_rrd_backup_interval then 60.0 *. 5.0 else !Xapi_globs.rrd_backup_interval in
-  let rrdbackup_func () =
-    Server_helpers.exec_with_new_task "rrdbackup_func"
-      (fun __context ->
-	let hosts = Db.Host.get_all ~__context in
-	Helpers.call_api_functions ~__context 
-	  (fun rpc session_id -> 
-	    ignore(List.fold_left (fun delay host -> Client.Client.Host.backup_rrds rpc session_id host delay; (delay +. 60.0)) 0.0 hosts)))
-  in
+	let rrdbackup_func () =
+		Server_helpers.exec_with_new_task "rrdbackup_func"
+			(fun __context ->
+				let hosts = Db.Host.get_all ~__context in
+				Helpers.call_api_functions ~__context 
+					(fun rpc session_id -> 
+						ignore(List.fold_left (fun delay host -> Client.Client.Host.backup_rrds rpc session_id host delay; (delay +. 60.0)) 0.0 hosts))
+			)
+	in
   let rrdbackup_delay = 
     if Xapi_fist.reduce_rrd_backup_interval then 60.0 *. 6.0 else 3600.0 in
 

ocaml/xapi/xapi_sr.ml

@@ -126,23 +126,27 @@ let scanning_thread () = Debug.with_thread_named "scanning_thread" (fun () ->
 
 let sr_cache = ref []
 
-let update_physical_utilisation ~__context =
+let srs_with_rrds ~__context =
 	let srs = Helpers.get_all_plugged_srs ~__context in
 	(* Remove SRs from sr_cache if they are no more available *)
 	sr_cache := List.filter (fun sr -> (List.mem sr srs)) !sr_cache;
 	(* Checks whether SR is sr_master and has SR_STATS capability *)
 	let is_stats_sr ~__context sr =
 		let sr_record = Db.SR.get_record_internal ~__context ~self:sr in
 		if (Smint.(has_capability Sr_stats (Xapi_sr_operations.features_of_sr ~__context sr_record)) &&
-			(Helpers.i_am_srmaster ~__context ~sr))
-			then true else false
+			(Helpers.i_am_srmaster ~__context ~sr)) then true else false
 	in
 	let srs_to_be_added = List.filter (fun sr -> (List.mem sr !sr_cache = false) && (is_stats_sr ~__context sr)) srs in
 	sr_cache := !sr_cache @ srs_to_be_added;
+	sr_cache
+
+let update_physical_utilisation ~__context =
+	let srs = srs_with_rrds ~__context in
 	(* Update the physical utilisation db field of cached SRs *)
 	List.iter (fun sr ->
 		let new_value = Rrdd.query_sr_ds ~sr_uuid:(Db.SR.get_uuid ~__context ~self:sr) ~ds_name:"physical_utilisation" in
-		Db.SR.set_physical_utilisation ~__context ~self:sr ~value:(Int64.of_float new_value)) !sr_cache
+		Db.SR.set_physical_utilisation ~__context ~self:sr ~value:(Int64.of_float new_value)
+		) !srs
 
 let physical_utilisation_thread ~__context () =
 	while true do