Merge pull request #3093 from gaborigloi/feature/CBT_merge

Merge first part of CBT feature branch into master

Author: robhoes

ocaml/idl/datamodel.ml

@@ -18,7 +18,7 @@ open Datamodel_types
 (* IMPORTANT: Please bump schema vsn if you change/add/remove a _field_.
               You do not have to bump vsn if you change/add/remove a message *)
 let schema_major_vsn = 5
-let schema_minor_vsn = 120
+let schema_minor_vsn = 131
 
 (* Historical schema versions just in case this is useful later *)
 let rio_schema_major_vsn = 5
@@ -88,7 +88,7 @@ let falcon_release_schema_major_vsn = 5
 let falcon_release_schema_minor_vsn = 120
 
 let inverness_release_schema_major_vsn = 5
-let inverness_release_schema_minor_vsn = 130
+let inverness_release_schema_minor_vsn = 131
 
 (* List of tech-preview releases. Fields in these releases are not guaranteed to be retained when
  * upgrading to a full release. *)
@@ -1105,10 +1105,12 @@ let _ =
     ~doc:"This operation cannot be performed because the system does not manage this VDI" ();
   error Api_errors.vdi_not_in_map [ "vdi" ]
     ~doc:"This VDI was not mapped to a destination SR in VM.migrate_send operation" () ;
+  error Api_errors.vdi_cbt_enabled [ "vdi" ]
+    ~doc:"The requested operation is not allowed for VDIs with CBT enabled or VMs having such VDIs, and CBT is enabled for the specified VDI." ();
   error Api_errors.vdi_copy_failed []
     ~doc:"The VDI copy action has failed" ();
   error Api_errors.vdi_on_boot_mode_incompatible_with_operation []
-    ~doc:"This operation is not permitted on VMs containing VDIs in the 'on-boot=reset' mode" ();
+    ~doc:"This operation is not permitted on VDIs in the 'on-boot=reset' mode, or on VMs having such VDIs." ();
   error Api_errors.cannot_create_state_file []
     ~doc:"An HA statefile could not be created, perhaps because no SR with the appropriate capability was found." ();
   error Api_errors.vif_not_in_map [ "vif" ]
@@ -5985,6 +5987,9 @@ let storage_operations =
           "vdi_clone", "Cloneing a VDI";
           "vdi_snapshot", "Snapshotting a VDI";
           "vdi_mirror", "Mirroring a VDI";
+          "vdi_enable_cbt", "Enabling changed block tracking for a VDI";
+          "vdi_disable_cbt", "Disabling changed block tracking for a VDI";
+          "vdi_set_on_boot", "Setting the on_boot field of the VDI";
           "pbd_create", "Creating a PBD for this SR";
           "pbd_destroy", "Destroying one of this SR's PBDs"; ])
 
@@ -6199,6 +6204,7 @@ let vdi_type = Enum ("vdi_type", [ "system",    "a disk that may be replaced on
                                    "redo_log", "a disk used for a general metadata redo-log";
                                    "rrd", "a disk that stores SR-level RRDs";
                                    "pvs_cache", "a disk that stores PVS cache data";
+                                   "cbt_metadata", "Metadata about a snapshot VDI that has been deleted: the set of blocks that changed between some previous version of the disk and the version tracked by the snapshot.";
                                  ])
 
 let vdi_introduce_params first_rel =
@@ -6230,7 +6236,10 @@ let vdi_pool_introduce = call
     ~name:"pool_introduce"
     ~in_oss_since:None
     ~in_product_since:rel_rio
-    ~versioned_params:(vdi_introduce_params miami_release)
+    ~versioned_params:(
+      (vdi_introduce_params miami_release) @
+      [{ param_type=Bool; param_name="cbt_enabled"; param_doc="True if changed blocks are tracked for this VDI"; param_release=inverness_release; param_default= Some(VBool false) }]
+    )
     ~doc:"Create a new VDI record in the database only"
     ~result:(Ref _vdi, "The ref of the newly created VDI record.")
     ~hide_from_docs:true
@@ -6303,6 +6312,9 @@ let vdi_operations =
           "update", "Refreshing the fields of the VDI";
           "force_unlock", "Forcibly unlocking the VDI";
           "generate_config", "Generating static configuration";
+          "enable_cbt", "Enabling changed block tracking for a VDI";
+          "disable_cbt", "Disabling changed block tracking for a VDI";
+          "set_on_boot", "Setting the on_boot field of the VDI";
           "blocked", "Operations on this VDI are temporarily blocked";
         ])
 
@@ -6508,6 +6520,42 @@ let vdi_read_database_pool_uuid = call
     ~allowed_roles:_R_READ_ONLY
     ()
 
+let vdi_enable_cbt = call
+    ~name:"enable_cbt"
+    ~in_oss_since:None
+    ~in_product_since:rel_inverness
+    ~params:[Ref _vdi, "self", "The VDI for which CBT should be enabled"]
+    ~errs:[
+      Api_errors.sr_operation_not_supported;
+      Api_errors.vdi_missing;
+      Api_errors.sr_not_attached;
+      Api_errors.sr_no_pbds;
+      Api_errors.operation_not_allowed;
+      Api_errors.vdi_incompatible_type;
+      Api_errors.vdi_on_boot_mode_incompatible_with_operation;
+    ]
+    ~doc:"Enable changed block tracking for the VDI. This call is idempotent - enabling CBT for a VDI for which CBT is already enabled results in a no-op, and no error will be thrown."
+    ~allowed_roles:_R_VM_ADMIN
+    ()
+
+let vdi_disable_cbt = call
+    ~name:"disable_cbt"
+    ~in_oss_since:None
+    ~in_product_since:rel_inverness
+    ~params:[Ref _vdi, "self", "The VDI for which CBT should be disabled"]
+    ~errs:[
+      Api_errors.sr_operation_not_supported;
+      Api_errors.vdi_missing;
+      Api_errors.sr_not_attached;
+      Api_errors.sr_no_pbds;
+      Api_errors.operation_not_allowed;
+      Api_errors.vdi_incompatible_type;
+      Api_errors.vdi_on_boot_mode_incompatible_with_operation;
+    ]
+    ~doc:"Disable changed block tracking for the VDI. This call is only allowed on VDIs that support enabling CBT. It is an idempotent operation - disabling CBT for a VDI for which CBT is not enabled results in a no-op, and no error will be thrown."
+    ~allowed_roles:_R_VM_ADMIN
+    ()
+
 (** A virtual disk *)
 let vdi =
   create_obj ~in_db:true ~in_product_since:rel_rio ~in_oss_since:oss_since_303 ~internal_deprecated_since:None ~persist:PersistEverything ~gen_constructor_destructor:true ~name:_vdi ~descr:"A virtual disk image"
@@ -6540,6 +6588,8 @@ let vdi =
                vdi_checksum;
                vdi_read_database_pool_uuid;
                vdi_pool_migrate;
+               vdi_enable_cbt;
+               vdi_disable_cbt;
               ]
     ~contents:
       ([ uid _vdi;
@@ -6573,6 +6623,7 @@ let vdi =
          field ~in_product_since:rel_boston ~qualifier:DynamicRO ~ty:(Ref _pool) ~default_value:(Some (VRef null_ref)) "metadata_of_pool" "The pool whose metadata is contained in this VDI";
          field ~in_product_since:rel_boston ~qualifier:DynamicRO ~ty:Bool ~default_value:(Some (VBool false)) "metadata_latest" "Whether this VDI contains the latest known accessible metadata for the pool";
          field ~lifecycle:[Published, rel_dundee, ""] ~qualifier:DynamicRO ~ty:Bool ~default_value:(Some (VBool false)) "is_tools_iso" "Whether this VDI is a Tools ISO";
+         field ~lifecycle:[Published, rel_inverness, ""] ~qualifier:DynamicRO ~ty:Bool ~default_value:(Some (VBool false)) "cbt_enabled" "True if changed blocks are tracked for this VDI" ~doc_tags:[Snapshots];
        ])
     ()
 

ocaml/xapi-consts/api_errors.ml

@@ -239,6 +239,7 @@ let vdi_not_managed = "VDI_NOT_MANAGED"
 let vdi_io_error = "VDI_IO_ERROR"
 let vdi_on_boot_mode_incompatible_with_operation = "VDI_ON_BOOT_MODE_INCOMPATIBLE_WITH_OPERATION"
 let vdi_not_in_map = "VDI_NOT_IN_MAP"
+let vdi_cbt_enabled = "VDI_CBT_ENABLED"
 let vif_not_in_map = "VIF_NOT_IN_MAP"
 let cannot_create_state_file = "CANNOT_CREATE_STATE_FILE"
 

ocaml/xapi/cli_frontend.ml

@@ -1969,6 +1969,22 @@ let rec cmdtable_data : (string*cmd_spec) list =
       implementation=No_fd Cli_operations.vdi_unlock;
       flags=[];
     };
+    "vdi-enable-cbt",
+    {
+      reqd=["uuid"];
+      optn=[];
+      help="Enable changed block tracking on a VDI.";
+      implementation=No_fd Cli_operations.vdi_enable_cbt;
+      flags=[];
+    };
+    "vdi-disable-cbt",
+    {
+      reqd=["uuid"];
+      optn=[];
+      help="Disable changed block tracking on a VDI.";
+      implementation=No_fd Cli_operations.vdi_disable_cbt;
+      flags=[];
+    };
     "diagnostic-vdi-status",
     {
       reqd=["uuid"];

ocaml/xapi/cli_operations.ml

@@ -1248,6 +1248,14 @@ let vdi_unlock printer rpc session_id params =
   then failwith "This operation is extremely dangerous and may cause data loss. This operation must be forced (use --force).";
   Client.VDI.force_unlock rpc session_id vdi
 
+let vdi_enable_cbt printer rpc session_id params =
+  let vdi = Client.VDI.get_by_uuid rpc session_id (List.assoc "uuid" params) in
+  Client.VDI.enable_cbt rpc session_id vdi
+
+let vdi_disable_cbt printer rpc session_id params =
+  let vdi = Client.VDI.get_by_uuid rpc session_id (List.assoc "uuid" params) in
+  Client.VDI.disable_cbt rpc session_id vdi
+
 let diagnostic_vdi_status printer rpc session_id params =
   let vdi = Client.VDI.get_by_uuid rpc session_id (List.assoc "uuid" params) in
   let vdi_r = vdi_record rpc session_id vdi in

ocaml/xapi/debug_populate.ml

@@ -67,8 +67,9 @@ let rec make_vdis_and_vbds __context vmref i =
       let snapshot_time = Stdext.Date.never in
       let snapshot_of = Ref.null in
       let sharable = false in
+      let cbt_enabled = false in
       let vdi = Xapi_vdi.pool_introduce
-          ~__context ~uuid ~name_label ~name_description ~sR ~_type ~sharable ~read_only ~other_config ~location ~xenstore_data ~sm_config ~managed ~virtual_size ~physical_utilisation ~metadata_of_pool ~is_a_snapshot ~snapshot_time ~snapshot_of in
+          ~__context ~uuid ~name_label ~name_description ~sR ~_type ~sharable ~read_only ~other_config ~location ~xenstore_data ~sm_config ~managed ~virtual_size ~physical_utilisation ~metadata_of_pool ~is_a_snapshot ~snapshot_time ~snapshot_of ~cbt_enabled in
 
       let _ =
         Xapi_vbd.create ~__context ~vM:vmref ~vDI:vdi ~userdevice:(string_of_int i) ~bootable:true ~mode:`RW ~_type:`Disk ~empty:false

ocaml/xapi/message_forwarding.ml

@@ -3393,8 +3393,11 @@ module Forward = functor(Local: Custom_actions.CUSTOM_ACTIONS) -> struct
     let set_on_boot ~__context ~self ~value =
       ensure_vdi_not_on_running_vm ~__context ~self;
       let local_fn = Local.VDI.set_on_boot ~self ~value in
-      forward_vdi_op ~local_fn ~__context ~self
-        (fun session_id rpc -> Client.VDI.set_on_boot rpc session_id self value)
+      let sr = Db.VDI.get_SR ~__context ~self in
+      with_sr_andor_vdi ~__context ~sr:(sr, `vdi_set_on_boot) ~vdi:(self, `set_on_boot) ~doc:"VDI.set_on_boot"
+        (fun () ->
+           forward_vdi_op ~local_fn ~__context ~self
+             (fun session_id rpc -> Client.VDI.set_on_boot rpc session_id self value))
 
     let set_allow_caching ~__context ~self ~value =
       ensure_vdi_not_on_running_vm ~__context ~self;
@@ -3416,13 +3419,12 @@ module Forward = functor(Local: Custom_actions.CUSTOM_ACTIONS) -> struct
              (fun session_id rpc -> Client.VDI.create ~rpc ~session_id ~name_label ~name_description ~sR ~virtual_size ~_type ~sharable ~read_only ~other_config ~xenstore_data ~sm_config ~tags))
 
     (* Hidden call used in pool join only *)
-    let pool_introduce ~__context ~uuid ~name_label ~name_description ~sR ~_type ~sharable ~read_only ~other_config ~location =
-      Local.VDI.pool_introduce ~__context ~uuid ~name_label ~name_description ~sR ~_type ~sharable ~read_only ~other_config ~location
+    let pool_introduce = Local.VDI.pool_introduce
 
     (* Called from the SM backend *)
-    let db_introduce ~__context ~uuid ~name_label ~name_description ~sR ~_type ~sharable ~read_only ~other_config ~location =
+    let db_introduce ~__context ~uuid ~name_label ~name_description ~sR ~_type ~sharable ~read_only ~other_config ~location ~xenstore_data ~sm_config  ~managed ~virtual_size ~physical_utilisation ~metadata_of_pool ~is_a_snapshot ~snapshot_time ~snapshot_of ~cbt_enabled =
       Sm.assert_session_has_internal_sr_access ~__context ~sr:sR;
-      Local.VDI.db_introduce ~__context ~uuid ~name_label ~name_description ~sR ~_type ~sharable ~read_only ~other_config ~location
+      Local.VDI.db_introduce ~__context ~uuid ~name_label ~name_description ~sR ~_type ~sharable ~read_only ~other_config ~location ~xenstore_data ~sm_config  ~managed ~virtual_size ~physical_utilisation ~metadata_of_pool ~is_a_snapshot ~snapshot_time ~snapshot_of ~cbt_enabled
 
     (* Called from the SM backend *)
     let db_forget ~__context ~vdi =
@@ -3576,6 +3578,24 @@ module Forward = functor(Local: Custom_actions.CUSTOM_ACTIONS) -> struct
         ~extra_sr:(Db.VDI.get_SR ~__context ~self)
         (fun session_id rpc -> Client.VDI.checksum rpc session_id self)
 
+    let enable_cbt ~__context ~self =
+      info "VDI.enable_cbt: VDI = '%s'" (vdi_uuid ~__context self);
+      let local_fn = Local.VDI.enable_cbt ~self in
+      let sR = Db.VDI.get_SR ~__context ~self in
+      with_sr_andor_vdi ~__context ~sr:(sR, `vdi_enable_cbt) ~vdi:(self, `enable_cbt) ~doc:"VDI.enable_cbt"
+        (fun () ->
+           forward_vdi_op ~local_fn ~__context ~self
+             (fun session_id rpc -> Client.VDI.enable_cbt rpc session_id self))
+
+    let disable_cbt ~__context ~self =
+      info "VDI.disable_cbt: VDI = '%s'" (vdi_uuid ~__context self);
+      let local_fn = Local.VDI.disable_cbt ~self in
+      let sR = Db.VDI.get_SR ~__context ~self in
+      with_sr_andor_vdi ~__context ~sr:(sR, `vdi_disable_cbt) ~vdi:(self, `disable_cbt) ~doc:"VDI.disable_cbt"
+        (fun () ->
+           forward_vdi_op ~local_fn ~__context ~self
+             (fun session_id rpc -> Client.VDI.disable_cbt rpc session_id self))
+
   end
   module VBD = struct
 

ocaml/xapi/record_util.ml

@@ -97,7 +97,7 @@ let host_operation_to_string = function
   | `vm_resume -> "VM.resume"
   | `vm_migrate -> "VM.migrate"
 
-let vdi_operation_to_string = function
+let vdi_operation_to_string: API.vdi_operations -> string = function
   | `scan -> "scan"
   | `clone -> "clone"
   | `copy -> "copy"
@@ -110,9 +110,12 @@ let vdi_operation_to_string = function
   | `forget -> "forget"
   | `update -> "update"
   | `generate_config -> "generate_config"
+  | `enable_cbt -> "enable_cbt"
+  | `disable_cbt -> "disable_cbt"
+  | `set_on_boot -> "set_on_boot"
   | `blocked -> "blocked"
 
-let sr_operation_to_string = function
+let sr_operation_to_string: API.storage_operations -> string = function
   | `scan -> "scan"
   | `destroy -> "destroy"
   | `forget -> "forget"
@@ -126,6 +129,9 @@ let sr_operation_to_string = function
   | `vdi_clone -> "VDI.clone"
   | `vdi_snapshot -> "VDI.snapshot"
   | `vdi_mirror -> "VDI.mirror"
+  | `vdi_enable_cbt -> "VDI.enable_cbt"
+  | `vdi_disable_cbt -> "VDI.disable_cbt"
+  | `vdi_set_on_boot -> "VDI.set_on_boot"
   | `pbd_create -> "PBD.create"
   | `pbd_destroy -> "PBD.destroy"
 
@@ -387,6 +393,7 @@ let vdi_type_to_string t =
   | `redo_log -> "Redo log"
   | `rrd -> "rrd"
   | `pvs_cache -> "PVS cache"
+  | `cbt_metadata -> "CBT metadata"
 
 let ip_configuration_mode_to_string = function
   | `None -> "None"

ocaml/xapi/records.ml

@@ -1308,6 +1308,7 @@ let vdi_record rpc session_id vdi =
         ~get_set:(fun () -> (x ()).API.vDI_tags)
         ~add_to_set:(fun tag -> Client.VDI.add_tags rpc session_id vdi tag)
         ~remove_from_set:(fun tag -> Client.VDI.remove_tags rpc session_id vdi tag) ();
+      make_field ~name:"cbt-enabled" ~get:(fun () -> string_of_bool (x ()).API.vDI_cbt_enabled) ();
     ]}
 
 let vbd_record rpc session_id vbd =

ocaml/xapi/sm.ml

@@ -222,6 +222,18 @@ let vdi_epoch_end dconf driver sr vdi =
   let call = Sm_exec.make_call ~sr_ref:sr ~vdi_ref:vdi dconf "vdi_epoch_end" [] in
   Sm_exec.parse_unit (Sm_exec.exec_xmlrpc (driver_filename driver) call)
 
+let vdi_enable_cbt dconf driver sr vdi =
+  debug "vdi_enable_cbt" driver (sprintf "sr=%s vdi=%s" (Ref.string_of sr) (Ref.string_of vdi));
+  srmaster_only dconf;
+  let call = Sm_exec.make_call ~sr_ref:sr ~vdi_ref:vdi dconf "vdi_enable_cbt" [] in
+  Sm_exec.parse_unit (Sm_exec.exec_xmlrpc (driver_filename driver) call)
+
+let vdi_disable_cbt dconf driver sr vdi =
+  debug "vdi_disable_cbt" driver (sprintf "sr=%s vdi=%s" (Ref.string_of sr) (Ref.string_of vdi));
+  srmaster_only dconf;
+  let call = Sm_exec.make_call ~sr_ref:sr ~vdi_ref:vdi dconf "vdi_disable_cbt" [] in
+  Sm_exec.parse_unit (Sm_exec.exec_xmlrpc (driver_filename driver) call)
+
 let session_has_internal_sr_access ~__context ~sr =
   let session_id = Context.get_session_id __context in
   (* XXX: need to move this somewhere else eventually *)

ocaml/xapi/smint.ml

@@ -42,6 +42,7 @@ type capability =
   | Vdi_generate_config
   | Vdi_attach_offline
   | Vdi_reset_on_boot
+  | Vdi_configure_cbt
 
 type feature = capability * int64
 
@@ -67,6 +68,7 @@ let string_to_capability_table = [
   "VDI_GENERATE_CONFIG", Vdi_generate_config;
   "VDI_ATTACH_OFFLINE", Vdi_attach_offline;
   "VDI_RESET_ON_BOOT", Vdi_reset_on_boot;
+  "VDI_CONFIG_CBT", Vdi_configure_cbt;
   "SR_STATS", Sr_stats;
 ]
 let capability_to_string_table = List.map (fun (k, v) -> v, k) string_to_capability_table