Merge pull request xapi-project#3164 from gaborigloi/feature/CBT_merge

robhoes · web-flow · commit 01a87ac5cdf0 · 2017-08-08T16:50:29.000+01:00
Merge feature/CBT branch into master
diff --git a/ocaml/idl/datamodel.ml b/ocaml/idl/datamodel.ml
@@ -1110,6 +1110,8 @@ let _ =
     ~doc:"This VDI was not mapped to a destination SR in VM.migrate_send operation" () ;
   error Api_errors.vdi_cbt_enabled [ "vdi" ]
     ~doc:"The requested operation is not allowed for VDIs with CBT enabled or VMs having such VDIs, and CBT is enabled for the specified VDI." ();
+  error Api_errors.vdi_no_cbt_metadata [ "vdi" ]
+    ~doc:"The requested operation is not allowed because the specified VDI does not have changed block tracking metadata." ();
   error Api_errors.vdi_copy_failed []
     ~doc:"The VDI copy action has failed" ();
   error Api_errors.vdi_on_boot_mode_incompatible_with_operation []
@@ -6005,6 +6007,8 @@ let storage_operations =
           "vdi_mirror", "Mirroring a VDI";
           "vdi_enable_cbt", "Enabling changed block tracking for a VDI";
           "vdi_disable_cbt", "Disabling changed block tracking for a VDI";
+          "vdi_data_destroy", "Deleting the data of the VDI";
+          "vdi_export_changed_blocks", "Exporting a bitmap that shows the changed blocks between two VDIs";
           "vdi_set_on_boot", "Setting the on_boot field of the VDI";
           "pbd_create", "Creating a PBD for this SR";
           "pbd_destroy", "Destroying one of this SR's PBDs"; ])
@@ -6329,6 +6333,8 @@ let vdi_operations =
           "generate_config", "Generating static configuration";
           "enable_cbt", "Enabling changed block tracking for a VDI";
           "disable_cbt", "Disabling changed block tracking for a VDI";
+          "data_destroy", "Deleting the data of the VDI";
+          "export_changed_blocks", "Exporting a bitmap that shows the changed blocks between two VDIs";
           "set_on_boot", "Setting the on_boot field of the VDI";
           "blocked", "Operations on this VDI are temporarily blocked";
         ])
@@ -6571,6 +6577,57 @@ let vdi_disable_cbt = call
     ~allowed_roles:_R_VM_ADMIN
     ()
 
+let vdi_data_destroy = call
+    ~name:"data_destroy"
+    ~in_oss_since:None
+    ~in_product_since:rel_inverness
+    ~params:[Ref _vdi, "self", "The VDI whose data should be deleted."]
+    ~errs:[
+      Api_errors.sr_operation_not_supported;
+      Api_errors.vdi_missing;
+      Api_errors.sr_not_attached;
+      Api_errors.sr_no_pbds;
+      Api_errors.operation_not_allowed;
+      Api_errors.vdi_incompatible_type;
+      Api_errors.vdi_no_cbt_metadata;
+      Api_errors.vdi_in_use;
+      Api_errors.vdi_is_a_physical_device;
+    ]
+    ~doc:"Delete the data of the snapshot VDI, but keep its changed block tracking metadata. When successful, this call changes the type of the VDI to cbt_metadata. This operation is idempotent: calling it on a VDI of type cbt_metadata results in a no-op, and no error will be thrown."
+    ~allowed_roles:_R_VM_ADMIN
+    ()
+
+let vdi_export_changed_blocks = call
+    ~name:"export_changed_blocks"
+    ~in_oss_since:None
+    ~in_product_since:rel_inverness
+    ~params:
+      [ Ref _vdi, "vdi_from", "The first VDI."
+      ; Ref _vdi, "vdi_to", "The second VDI."
+      ]
+    ~errs:
+      [ Api_errors.sr_operation_not_supported
+      ; Api_errors.vdi_missing
+      ; Api_errors.sr_not_attached
+      ; Api_errors.sr_no_pbds
+      ; Api_errors.vdi_in_use
+      ]
+    ~result:(String, "A base64 string-encoding of the bitmap showing which blocks differ in the two VDIs.")
+    ~doc:"Reports which blocks differ in the two VDIs. This operation is not allowed when vdi_to is attached to a VM."
+    ~allowed_roles:_R_VM_OP
+    ()
+
+let vdi_get_nbd_info = call
+    ~name:"get_nbd_info"
+    ~in_oss_since:None
+    ~in_product_since:rel_inverness
+    ~params:[Ref _vdi, "self", "The VDI to access via NBD."]
+    ~errs: [Api_errors.vdi_incompatible_type]
+    ~result:(Set String, "The list of URIs.")
+    ~doc:"Get a list of URIs specifying how to access this VDI via the NBD server of XenServer. A URI will be returned for each PIF of each host that is connected to the VDI's SR. An empty list is returned in case no network has a PIF on a host with access to the relevant SR. To access the given VDI, any of the returned URIs can be passed to the NBD server running at the IP address and port specified by that URI as the export name."
+    ~allowed_roles:_R_VM_ADMIN
+    ()
+
 (** A virtual disk *)
 let vdi =
   create_obj ~in_db:true ~in_product_since:rel_rio ~in_oss_since:oss_since_303 ~internal_deprecated_since:None ~persist:PersistEverything ~gen_constructor_destructor:true ~name:_vdi ~descr:"A virtual disk image"
@@ -6605,6 +6662,9 @@ let vdi =
                vdi_pool_migrate;
                vdi_enable_cbt;
                vdi_disable_cbt;
+               vdi_data_destroy;
+               vdi_export_changed_blocks;
+               vdi_get_nbd_info;
               ]
     ~contents:
       ([ uid _vdi;
diff --git a/ocaml/xapi-consts/api_errors.ml b/ocaml/xapi-consts/api_errors.ml
@@ -240,6 +240,7 @@ let vdi_io_error = "VDI_IO_ERROR"
 let vdi_on_boot_mode_incompatible_with_operation = "VDI_ON_BOOT_MODE_INCOMPATIBLE_WITH_OPERATION"
 let vdi_not_in_map = "VDI_NOT_IN_MAP"
 let vdi_cbt_enabled = "VDI_CBT_ENABLED"
+let vdi_no_cbt_metadata = "VDI_NO_CBT_METADATA"
 let vif_not_in_map = "VIF_NOT_IN_MAP"
 let cannot_create_state_file = "CANNOT_CREATE_STATE_FILE"
 
diff --git a/ocaml/xapi/cli_frontend.ml b/ocaml/xapi/cli_frontend.ml
@@ -1985,6 +1985,22 @@ let rec cmdtable_data : (string*cmd_spec) list =
       implementation=No_fd Cli_operations.vdi_disable_cbt;
       flags=[];
     };
+    "vdi-data-destroy",
+    {
+      reqd=["uuid"];
+      optn=[];
+      help="Delete the data of the VDI, but keep its changed block tracking metadata.";
+      implementation=No_fd Cli_operations.vdi_data_destroy;
+      flags=[];
+    };
+    "vdi-export-changed-blocks",
+    {
+      reqd=["vdi-from"; "vdi-to"];
+      optn=[];
+      help="Write the changed blocks between the two given VDIs to the standard output as a base64-encoded bitmap string.";
+      implementation=With_fd Cli_operations.vdi_export_changed_blocks;
+      flags=[];
+    };
     "diagnostic-vdi-status",
     {
       reqd=["uuid"];
diff --git a/ocaml/xapi/cli_operations.ml b/ocaml/xapi/cli_operations.ml
@@ -1256,6 +1256,16 @@ let vdi_disable_cbt printer rpc session_id params =
   let vdi = Client.VDI.get_by_uuid rpc session_id (List.assoc "uuid" params) in
   Client.VDI.disable_cbt rpc session_id vdi
 
+let vdi_data_destroy printer rpc session_id params =
+  let vdi = Client.VDI.get_by_uuid rpc session_id (List.assoc "uuid" params) in
+  Client.VDI.data_destroy rpc session_id vdi
+
+let vdi_export_changed_blocks socket _ rpc session_id params =
+  let vdi_from = Client.VDI.get_by_uuid rpc session_id (List.assoc "vdi-from" params) in
+  let vdi_to = Client.VDI.get_by_uuid rpc session_id (List.assoc "vdi-to" params) in
+  let bitmap = Client.VDI.export_changed_blocks ~rpc ~session_id ~vdi_from ~vdi_to in
+  marshal socket (Command (Print bitmap))
+
 let diagnostic_vdi_status printer rpc session_id params =
   let vdi = Client.VDI.get_by_uuid rpc session_id (List.assoc "uuid" params) in
   let vdi_r = vdi_record rpc session_id vdi in
diff --git a/ocaml/xapi/message_forwarding.ml b/ocaml/xapi/message_forwarding.ml
@@ -3108,8 +3108,9 @@ module Forward = functor(Local: Custom_actions.CUSTOM_ACTIONS) -> struct
       let host = Db.PBD.get_host ~__context ~self:pbd in
       do_op_on ~local_fn ~__context ~host op
 
-    (* do op on a host that can view multiple SRs, if none is found, an
-       		   exception of Not_found will be raised *)
+    (** Do op on a host that can view multiple SRs. If none is found, the
+        Not_found exception will be raised.
+        WARNING: this may forward the call to a host that is NOT the SR master. *)
     let forward_sr_multiple_op ~local_fn ~__context ~srs ?(prefer_slaves=false) op =
       let choose_fn ~host =
         Xapi_vm_helpers.assert_can_see_specified_SRs ~__context ~reqd_srs:srs ~host in
@@ -3596,6 +3597,28 @@ module Forward = functor(Local: Custom_actions.CUSTOM_ACTIONS) -> struct
            forward_vdi_op ~local_fn ~__context ~self
              (fun session_id rpc -> Client.VDI.disable_cbt rpc session_id self))
 
+    let data_destroy ~__context ~self =
+      info "VDI.data_destroy: VDI = '%s'" (vdi_uuid ~__context self);
+      let local_fn = Local.VDI.data_destroy ~self in
+      let sR = Db.VDI.get_SR ~__context ~self in
+      with_sr_andor_vdi ~__context ~sr:(sR, `vdi_data_destroy) ~vdi:(self, `data_destroy) ~doc:"VDI.data_destroy"
+        (fun () ->
+           forward_vdi_op ~local_fn ~__context ~self
+             (fun session_id rpc -> Client.VDI.data_destroy rpc session_id self))
+
+    let export_changed_blocks ~__context ~vdi_from ~vdi_to =
+      info "VDI.export_changed_blocks: vdi_from  = '%s'; vdi_to = '%s'" (vdi_uuid ~__context vdi_from) (vdi_uuid ~__context vdi_to);
+      let local_fn = Local.VDI.export_changed_blocks ~vdi_from ~vdi_to in
+      let vdi_to_sr = Db.VDI.get_SR ~__context ~self:vdi_to in
+      with_sr_andor_vdi ~__context ~sr:(vdi_to_sr, `vdi_export_changed_blocks) ~vdi:(vdi_to, `export_changed_blocks) ~doc:"VDI.export_changed_blocks"
+        (fun () ->
+           forward_vdi_op ~local_fn ~__context ~self:vdi_to
+             (fun session_id rpc -> Client.VDI.export_changed_blocks ~rpc ~session_id ~vdi_from ~vdi_to))
+
+    let get_nbd_info ~__context ~self =
+      info "VDI.get_nbd_info: vdi  = '%s'" (vdi_uuid ~__context self);
+      Local.VDI.get_nbd_info ~__context ~self
+
   end
   module VBD = struct
 
diff --git a/ocaml/xapi/ounit_comparators.ml b/ocaml/xapi/ounit_comparators.ml
@@ -0,0 +1,9 @@
+module StringDiff =
+struct
+  type t = string
+  let compare = String.compare
+  let pp_printer = Format.pp_print_string
+  let pp_print_sep = OUnitDiff.pp_comma_separator
+end
+
+module StringSet = OUnitDiff.SetMake(StringDiff)
diff --git a/ocaml/xapi/record_util.ml b/ocaml/xapi/record_util.ml
@@ -111,6 +111,8 @@ let vdi_operation_to_string: API.vdi_operations -> string = function
   | `generate_config -> "generate_config"
   | `enable_cbt -> "enable_cbt"
   | `disable_cbt -> "disable_cbt"
+  | `data_destroy -> "data_destroy"
+  | `export_changed_blocks -> "export_changed_blocks"
   | `set_on_boot -> "set_on_boot"
   | `blocked -> "blocked"
 
@@ -131,6 +133,8 @@ let sr_operation_to_string: API.storage_operations -> string = function
   | `vdi_enable_cbt -> "VDI.enable_cbt"
   | `vdi_disable_cbt -> "VDI.disable_cbt"
   | `vdi_set_on_boot -> "VDI.set_on_boot"
+  | `vdi_data_destroy -> "VDI.data_destroy"
+  | `vdi_export_changed_blocks -> "VDI.export_changed_blocks"
   | `pbd_create -> "PBD.create"
   | `pbd_destroy -> "PBD.destroy"
 
diff --git a/ocaml/xapi/sm.ml b/ocaml/xapi/sm.ml
@@ -214,6 +214,18 @@ let vdi_disable_cbt dconf driver sr vdi =
   let call = Sm_exec.make_call ~sr_ref:sr ~vdi_ref:vdi dconf "vdi_disable_cbt" [] in
   Sm_exec.parse_unit (Sm_exec.exec_xmlrpc (driver_filename driver) call)
 
+let vdi_data_destroy dconf driver sr vdi =
+  debug "vdi_data_destroy" driver (sprintf "sr=%s vdi=%s" (Ref.string_of sr) (Ref.string_of vdi));
+  srmaster_only dconf;
+  let call = Sm_exec.make_call ~sr_ref:sr ~vdi_ref:vdi dconf "vdi_data_destroy" [] in
+  Sm_exec.parse_unit (Sm_exec.exec_xmlrpc (driver_filename driver) call)
+
+let vdi_export_changed_blocks dconf driver sr ~vdi_from ~vdi_to =
+  debug "vdi_export_changed_blocks" driver (sprintf "sr=%s vdi_from=%s vdi_to=%s" (Ref.string_of sr) (Ref.string_of vdi_from) (Ref.string_of vdi_to));
+  srmaster_only dconf;
+  let call = Sm_exec.make_call ~sr_ref:sr ~vdi_ref:vdi_from dconf "vdi_export_changed_blocks" [ Ref.string_of vdi_to ] in
+  Sm_exec.parse_string (Sm_exec.exec_xmlrpc (driver_filename driver) call)
+
 let session_has_internal_sr_access ~__context ~sr =
   let session_id = Context.get_session_id __context in
   (* XXX: need to move this somewhere else eventually *)
diff --git a/ocaml/xapi/storage_access.ml b/ocaml/xapi/storage_access.ml
@@ -844,6 +844,26 @@ module SMAPIv1 = struct
     let disable_cbt context =
       call_cbt_function context ~f:Sm.vdi_disable_cbt ~f_name:"VDI.disable_cbt"
 
+    let data_destroy context ~dbg ~sr ~vdi =
+      call_cbt_function context ~f:Sm.vdi_data_destroy ~f_name:"VDI.data_destroy" ~dbg ~sr ~vdi;
+      set_content_id context ~dbg ~sr ~vdi ~content_id:"/No content: this is a cbt_metadata VDI/"
+
+    let export_changed_blocks context ~dbg ~sr ~vdi_from ~vdi_to =
+      try
+        Server_helpers.exec_with_new_task "VDI.export_changed_blocks" ~subtask_of:(Ref.of_string dbg)
+          (fun __context ->
+             let vdi_from = find_vdi ~__context sr vdi_from |> fst in
+             for_vdi ~dbg ~sr ~vdi:vdi_to "VDI.export_changed_blocks"
+               (fun device_config _type sr vdi_to ->
+                  Sm.vdi_export_changed_blocks device_config _type sr ~vdi_from ~vdi_to
+               ))
+      with
+      | Smint.Not_implemented_in_backend ->
+        raise (Unimplemented "VDI.export_changed_blocks")
+      | Api_errors.Server_error(code, params) ->
+        raise (Backend_error(code, params))
+      | Sm.MasterOnly -> redirect sr
+
   end
 
   let get_by_name context ~dbg ~name = assert false
diff --git a/ocaml/xapi/storage_impl.ml b/ocaml/xapi/storage_impl.ml
@@ -522,16 +522,19 @@ module Wrapper = functor(Impl: Server_impl) -> struct
            Impl.VDI.resize context ~dbg ~sr ~vdi ~new_size
         )
 
-    let destroy context ~dbg ~sr ~vdi =
-      info "VDI.destroy dbg:%s sr:%s vdi:%s" dbg sr vdi;
+    let destroy_and_data_destroy call_name call_f context ~dbg ~sr ~vdi =
+      info "%s dbg:%s sr:%s vdi:%s" call_name dbg sr vdi;
       with_vdi sr vdi
         (fun () ->
            remove_datapaths_andthen_nolock context ~dbg ~sr ~vdi Vdi.all
              (fun () ->
-                Impl.VDI.destroy context ~dbg ~sr ~vdi
+                call_f context ~dbg ~sr ~vdi
              )
         )
 
+    let destroy = destroy_and_data_destroy "VDI.destroy" Impl.VDI.destroy
+    let data_destroy = destroy_and_data_destroy "VDI.data_destroy" Impl.VDI.data_destroy
+
     let stat context ~dbg ~sr ~vdi =
       info "VDI.stat dbg:%s sr:%s vdi:%s" dbg sr vdi;
       Impl.VDI.stat context ~dbg ~sr ~vdi
@@ -589,6 +592,14 @@ module Wrapper = functor(Impl: Server_impl) -> struct
            Impl.VDI.disable_cbt context ~dbg ~sr ~vdi
         )
 
+    (** The [sr] parameter is the SR of VDI [vdi_to]. *)
+    let export_changed_blocks context ~dbg ~sr ~vdi_from ~vdi_to =
+      info "VDI.export_changed_blocks dbg:%s sr:%s vdi_from:%s vdi_to:%s" dbg sr vdi_from vdi_to;
+      with_vdi sr vdi_to
+        (fun () ->
+           Impl.VDI.export_changed_blocks context ~dbg ~sr ~vdi_from ~vdi_to
+        )
+
   end
 
   let get_by_name context ~dbg ~name =
diff --git a/ocaml/xapi/storage_impl_test.ml b/ocaml/xapi/storage_impl_test.ml
@@ -168,6 +168,8 @@ module Debug_print_impl = struct
     let similar_content context ~dbg ~sr ~vdi = assert false
     let enable_cbt context ~dbg ~sr ~vdi = assert false
     let disable_cbt context ~dbg ~sr ~vdi = assert false
+    let data_destroy context ~dbg ~sr ~vdi = assert false
+    let export_changed_blocks context ~dbg ~sr ~vdi_from ~vdi_to = assert false
 
 
   end
diff --git a/ocaml/xapi/storage_mux.ml b/ocaml/xapi/storage_mux.ml
@@ -261,6 +261,12 @@ module Mux = struct
     let disable_cbt context ~dbg ~sr ~vdi =
       let module C = Client(struct let rpc = of_sr sr end) in
       C.VDI.disable_cbt ~dbg ~sr ~vdi
+    let data_destroy context ~dbg ~sr ~vdi =
+      let module C = Client(struct let rpc = of_sr sr end) in
+      C.VDI.data_destroy ~dbg ~sr ~vdi
+    let export_changed_blocks context ~dbg ~sr ~vdi_from ~vdi_to =
+      let module C = Client(struct let rpc = of_sr sr end) in
+      C.VDI.export_changed_blocks ~dbg ~sr ~vdi_from ~vdi_to
 
   end
 
diff --git a/ocaml/xapi/test_common.ml b/ocaml/xapi/test_common.ml
@@ -154,7 +154,7 @@ let make_pif ~__context ~network ~host ?(device="eth0") ?(mAC="C0:FF:EE:C0:FF:EE
     ?(vLAN=(-1L)) ?(physical=true) ?(ip_configuration_mode=`None) ?(iP="") ?(netmask="")
     ?(gateway="") ?(dNS="") ?(bond_slave_of=Ref.null) ?(vLAN_master_of=Ref.null)
     ?(management=false) ?(other_config=[]) ?(disallow_unplug=false)
-    ?(ipv6_configuration_mode=`None) ?(iPv6=[""]) ?(ipv6_gateway="") ?(primary_address_type=`IPv4) ?(managed=true)
+    ?(ipv6_configuration_mode=`None) ?(iPv6=[]) ?(ipv6_gateway="") ?(primary_address_type=`IPv4) ?(managed=true)
     ?(properties=["gro", "on"]) () =
   Xapi_pif.pool_introduce ~__context
     ~device ~network ~host ~mAC ~mTU ~vLAN ~physical ~ip_configuration_mode
@@ -378,3 +378,7 @@ let make_pool_update ~__context
     } in
   Xapi_pool_update.create_update_record ~__context ~update:ref ~update_info ~vdi;
   ref
+
+let make_session ~__context ?(ref=Ref.make ()) ?(uuid=make_uuid ()) ?(this_host=Ref.null) ?(this_user=Ref.null) ?(last_active=API.Date.never) ?(pool=false) ?(other_config=[]) ?(is_local_superuser=false) ?(subject=Ref.null) ?(validation_time=API.Date.never) ?(auth_user_sid="") ?(auth_user_name="") ?(rbac_permissions=[]) ?(parent=Ref.null) ?(originator="test") () =
+  Db.Session.create ~__context ~ref ~uuid ~this_host ~this_user ~last_active ~pool ~other_config ~is_local_superuser ~subject ~validation_time ~auth_user_sid ~auth_user_name ~rbac_permissions ~parent ~originator;
+  ref
diff --git a/ocaml/xapi/test_event.ml b/ocaml/xapi/test_event.ml
@@ -27,34 +27,12 @@ let start_periodic_scheduler () =
     ps_start := true
   end
 
-let create_session ~__context =
-  let session_id = Ref.make () in
-  let uuid = Uuid.to_string (Uuid.make_uuid ()) in
-  let user = Ref.null in (* always return a null reference to the deprecated user object *)
-  let parent = try Context.get_session_id __context with _ -> Ref.null in
-  (*match uname with   (* the user object is deprecated in favor of subject *)
-      Some uname -> Helpers.get_user ~__context uname
-      | None -> Ref.null in*)
-  (* This info line is important for tracking, auditability and client accountability purposes on XenServer *)
-  (* Never print the session id nor uuid: they are secret values that should be known only to the user that *)
-  (* has just logged in. Instead, we print a non-invertible hash as the tracking id for the session id *)
-  (* see also task creation in context.ml *)
-  (* CP-982: promote tracking debug line to info status *)
-  (* CP-982: create tracking id in log files to link username to actions *)
-  Db.Session.create ~__context ~ref:session_id ~uuid
-    ~this_user:user ~this_host:(Helpers.get_localhost ~__context) ~pool:false
-    ~last_active:(Date.of_float (Unix.time ())) ~other_config:[]
-    ~subject:(Ref.null) ~is_local_superuser:true
-    ~auth_user_sid:"" ~validation_time:(Date.of_float (Unix.time ()))
-    ~auth_user_name:"root" ~rbac_permissions:[] ~parent ~originator:"test";
-  session_id
-
 let parse_event_from rpc = rpc |> Xmlrpc.to_string |> Xmlrpc.of_string |> event_from_of_rpc
 
 let event_setup_common () =
   start_periodic_scheduler ();
-  let c1 = make_test_database () in
-  let session_id = create_session c1 in
+  let __context = make_test_database () in
+  let session_id = Test_common.make_session ~__context () in
   let __context = Mock.Context.make ~session_id "session context" in
   (__context, session_id)
   
diff --git a/ocaml/xapi/test_pr1510.ml b/ocaml/xapi/test_pr1510.ml
@@ -4,16 +4,7 @@ open Fun
 open OUnit
 open Network_utils
 
-(* Example of using OUnitDiff with a String Set *)
-module StringDiff =
-struct
-  type t = string
-  let compare = String.compare
-  let pp_printer = Format.pp_print_string
-  let pp_print_sep = OUnitDiff.pp_comma_separator
-end
-
-module OSSet = OUnitDiff.SetMake(StringDiff)
+module OSSet = Ounit_comparators.StringSet
 
 let run_bond_prop_test props c_props c_per_iface =
   let props, per_iface_props =
diff --git a/ocaml/xapi/test_vdi_cbt.ml b/ocaml/xapi/test_vdi_cbt.ml
diff --git a/ocaml/xapi/xapi_sr_operations.ml b/ocaml/xapi/xapi_sr_operations.ml
diff --git a/ocaml/xapi/xapi_vdi.ml b/ocaml/xapi/xapi_vdi.ml
diff --git a/ocaml/xapi/xapi_vdi.mli b/ocaml/xapi/xapi_vdi.mli
