Merge pull request xapi-project#226 from gaborigloi/ely-merge

Merge commits from the (current) master branch that should go into Ely. Commits have been re-based and old merge commits have been removed.

Author: lindig

.gitarchive-info

@@ -0,0 +1,2 @@
+Changeset: $Format:%H$
+Commit date: $Format:%cD$

.gitattributes

@@ -0,0 +1 @@
+.gitarchive-info export-subst

ocaml/idl/datamodel.ml

@@ -1208,11 +1208,11 @@ let _ =
     ~doc:"This update has already been applied to all hosts in the pool." ();
   error Api_errors.update_precheck_failed_unknown_error [ "update"; "info" ]
     ~doc:"The update precheck stage failed with an unknown error." ();
-  error Api_errors.update_precheck_failed_prerequisite_missing [ "update"; "info" ]
+  error Api_errors.update_precheck_failed_prerequisite_missing [ "update"; "prerequisite_update" ]
     ~doc:"The update precheck stage failed: prerequisite update(s) are missing." ();
-  error Api_errors.update_precheck_failed_conflict_present ["update"; "info"]
-    ~doc:"The update precheck stage failed: conflicting updates are present." ();
-  error Api_errors.update_precheck_failed_wrong_server_version ["update"; "info"]
+  error Api_errors.update_precheck_failed_conflict_present ["update"; "conflict_update"]
+    ~doc:"The update precheck stage failed: conflicting update(s) are present." ();
+  error Api_errors.update_precheck_failed_wrong_server_version ["update"; "installed_version"; "required_version "]
     ~doc:"The update precheck stage failed: the server is of an incorrect version." ();
   error Api_errors.update_precheck_failed_out_of_space ["update"; "available_space"; "required_space "]
     ~doc:"The update precheck stage failed: the server does not have enough space." ();
@@ -3241,10 +3241,11 @@ let host_set_localdb_key = call
 
 let host_refresh_pack_info = call
     ~name:"refresh_pack_info"
-    ~in_product_since:rel_midnight_ride
     ~doc:"Refresh the list of installed Supplemental Packs."
     ~params:[Ref _host, "host", "The Host to modify"]
     ~allowed_roles:_R_POOL_OP
+    ~lifecycle:[Published, rel_midnight_ride, "";
+                Deprecated, rel_ely, "Use Pool_update.resync_host instead"]
     ()
 
 (* ------------------------------------------------------------------------------------------------------------

ocaml/xapi/cli_frontend.ml

@@ -959,6 +959,15 @@ let rec cmdtable_data : (string*cmd_spec) list =
       flags=[];
     };
 
+    "update-resync-host",
+    {
+      reqd=["host-uuid"];
+      optn=[];
+      help="Refreshes Host.updates";
+      implementation=No_fd Cli_operations.update_resync_host;
+      flags=[Hidden];
+    };
+
     "user-password-change",
     {
       reqd=["new"];

ocaml/xapi/cli_operations.ml

@@ -4837,3 +4837,8 @@ let update_destroy printer rpc session_id params =
   let uuid = List.assoc "uuid" params in
   let ref = Client.Pool_update.get_by_uuid rpc session_id uuid in
   Client.Pool_update.destroy rpc session_id ref
+
+let update_resync_host printer rpc session_id params =
+  let uuid = List.assoc "host-uuid" params in
+  let host = Client.Host.get_by_uuid rpc session_id uuid in
+  Client.Pool_update.resync_host rpc session_id host

ocaml/xapi/xapi_pool_patch.ml

@@ -65,6 +65,7 @@ let pool_patch_upload_handler (req: Http.Request.t) s _ =
                      Db.Task.set_result ~__context ~self:(Context.get_task_id __context) ~value:(Ref.string_of patch);
                      TaskHelper.complete ~__context None
                    with e ->
+                     Client.VDI.destroy rpc session_id vdi;
                      TaskHelper.failed ~__context e
                  end
                | None ->

ocaml/xapi/xapi_pool_update.ml

@@ -144,20 +144,27 @@ let detach ~__context ~self =
   detach_helper ~__context ~uuid ~vdi
 
 let with_api_errors f x =
+  let errinfo = "Please upload a valid package." in
   try f x
   with
   | Smint.Command_failed(ret, status, stdout_log, stderr_log)
   | Smint.Command_killed(ret, status, stdout_log, stderr_log) ->
     let msg = Printf.sprintf "Smint.Command_{failed,killed} ret = %d; status = %s; stdout = %s; stderr = %s"
         ret status stdout_log stderr_log in
-    raise (Api_errors.Server_error (Api_errors.internal_error, [msg]))
+    error "%s" msg;
+    raise (Api_errors.Server_error (Api_errors.invalid_update, [errinfo]))
+  | e ->
+    let msg = ExnHelper.string_of_exn e in
+    error "%s" msg;
+    raise (Api_errors.Server_error (Api_errors.invalid_update, [errinfo]))
 
 (* yum confif example
    [main]
    keepcache=0
    reposdir=/dev/null
    gpgcheck=$signed
    repo_gpgcheck=$signed
+   installonlypkgs=
 
    [$label]
    name=$label
@@ -170,9 +177,19 @@ let create_yum_config ~__context ~self ~url =
   let signed = String.length key <> 0 in
   let signed_index = if signed then 1 else 0 in
   let name_label = Db.Pool_update.get_name_label ~__context ~self in
-  (Printf.sprintf ("[main]\nkeepcache=0\nreposdir=/dev/null\ngpgcheck=%d\nrepo_gpgcheck=%d\n\n") signed_index signed_index)
-  ^(Printf.sprintf ("[%s]\nname=%s\nbaseurl=%s\n") name_label name_label url)
-  ^(if signed then Printf.sprintf ("gpgkey=file:///etc/pki/rpm-gpg/%s") key else "")
+  String.concat "\n"
+  [ "[main]"
+  ; "keepcache=0"
+  ; "reposdir=/dev/null"
+  ; Printf.sprintf "gpgcheck=%d" signed_index
+  ; Printf.sprintf "repo_gpgcheck=%d" signed_index
+  ; "installonlypkgs="
+  ; ""
+  ; Printf.sprintf "[%s]" name_label
+  ; Printf.sprintf "name=%s" name_label
+  ; Printf.sprintf "baseurl=%s" url
+  ; if signed then Printf.sprintf ("gpgkey=file:///etc/pki/rpm-gpg/%s") key else ""
+  ]
 
 let attach_helper ~__context ~uuid ~vdi =
   let host = Helpers.get_localhost ~__context in
@@ -445,7 +462,17 @@ let resync_host ~__context ~host =
       ) update_refs;
     Create_misc.create_updates_requiring_reboot_info ~__context ~host
   end
-  else Db.Host.set_updates ~__context ~self:host ~value:[]
+  else Db.Host.set_updates ~__context ~self:host ~value:[];
+
+  (* Remove any pool_patch objects that don't have a corresponding pool_update object *)
+  Db.Pool_patch.get_all ~__context
+  |> List.filter (fun self -> Db.Pool_patch.get_pool_update ~__context ~self = Ref.null)
+  |> List.iter (fun self -> Db.Pool_patch.destroy ~__context ~self);
+
+  (* Clean updates that don't have a corresponding patch record *)
+  Db.Pool_update.get_all ~__context
+  |> List.filter (fun self -> Xapi_pool_patch.pool_patch_of_update ~__context self = Ref.null)
+  |> List.iter (fun self -> destroy ~__context ~self)
 
 let pool_update_download_handler (req: Request.t) s _ =
   debug "pool_update.pool_update_download_handler URL %s" req.Request.uri;

scripts/OMakefile

@@ -132,6 +132,3 @@ install:
 	mkdir -p $(DESTDIR)/etc/cron.d
 	$(IDATA) xapi-logrotate.cron $(DESTDIR)/etc/cron.d/xapi-logrotate.cron
 	mkdir -p $(DESTDIR)/opt/xensource/gpg
-	$(IDATA) gpg/secring.gpg $(DESTDIR)/opt/xensource/gpg/
-	$(IDATA) gpg/pubring.gpg $(DESTDIR)/opt/xensource/gpg/
-	$(IDATA) gpg/trustdb.gpg $(DESTDIR)/opt/xensource/gpg/

scripts/extensions/pool_update.apply

@@ -45,7 +45,9 @@ def failure_message(code, params):
 
 def execute_apply(session, update_package, yum_conf_file):
     cmd = ['yum', 'install', '-y', '--noplugins', '-c', yum_conf_file, update_package]
-    p = subprocess.Popen(cmd, shell=False, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, close_fds=True)
+    yum_env = os.environ.copy()
+    yum_env['LANG'] = 'C'
+    p = subprocess.Popen(cmd, shell=False, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, close_fds=True, env=yum_env)
     output, _ = p.communicate()
     xcp.logger.info('pool_update.apply %r returncode=%r output=%r', cmd, p.returncode, output)
     if p.returncode != 0:

scripts/extensions/pool_update.precheck

@@ -25,11 +25,14 @@ UPDATE_PRECHECK_FAILED_PREREQUISITE_MISSING = 'UPDATE_PRECHECK_FAILED_PREREQUISI
 UPDATE_PRECHECK_FAILED_CONFLICT_PRESENT = 'UPDATE_PRECHECK_FAILED_CONFLICT_PRESENT'
 UPDATE_PRECHECK_FAILED_WRONG_SERVER_VERSION = 'UPDATE_PRECHECK_FAILED_WRONG_SERVER_VERSION'
 UPDATE_PRECHECK_FAILED_OUT_OF_SPACE = 'UPDATE_PRECHECK_FAILED_OUT_OF_SPACE'
+UPDATE_PREFIX = 'update-'
 INVALID_UPDATE = 'INVALID_UPDATE'
 CANNOT_FIND_UPDATE = 'CANNOT_FIND_UPDATE'
 ERROR_MESSAGE_START = 'Error: '
 ERROR_MESSAGE_END = 'You could try '
-ERROR_MESSAGE_CONFLICT = ' conflicts with '
+ERROR_MESSAGE_CONFLICTS_WITH = ' conflicts with '
+ERROR_MESSAGE_CONFLICTS = 'conflicts '
+ERROR_MESSAGE_PROCESSING_CONFLICT = '--> Processing Conflict:'
 ERROR_MESSAGE_PREREQUISITE = 'Requires: '
 ERROR_MESSAGE_VERSION_REQUIRED = 'Requires: '
 ERROR_MESSAGE_VERSION_INSTALLED = 'Installed: '
@@ -80,7 +83,9 @@ def execute_precheck(session, control_package, yum_conf_file, update_precheck_fi
                      'PATCH_PRECHECK_LIVEPATCH_INCOMPLETE': 'ok_livepatch_incomplete',
                      'PATCH_PRECHECK_LIVEPATCH_NOT_APPLICABLE': 'ok'}
     cmd = ['yum', 'install', '-y', '--noplugins', '-c', yum_conf_file, control_package]
-    p = subprocess.Popen(cmd, shell=False, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, close_fds=True)
+    yum_env = os.environ.copy()
+    yum_env['LANG'] = 'C'
+    p = subprocess.Popen(cmd, shell=False, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, close_fds=True, env=yum_env)
     output, _ = p.communicate()
     xcp.logger.info('pool_update.precheck %r returncode=%r output=%r', cmd, p.returncode, output)
     if p.returncode != 0:
@@ -91,15 +96,38 @@ def execute_precheck(session, control_package, yum_conf_file, update_precheck_fi
         if m:
             errmsg = m.group()
             errmsg = re.sub(ERROR_MESSAGE_END + '.+', '', errmsg, flags=re.DOTALL)
-            errmsg = re.sub('\n +', ' ', errmsg, flags=re.DOTALL)
-            if ERROR_MESSAGE_CONFLICT in errmsg:
-                raise ConflictPresent(errmsg)
+            precheckfailure_msg = re.sub('\n +', ' ', errmsg, flags=re.DOTALL)
+            if ERROR_MESSAGE_CONFLICTS_WITH in errmsg and ERROR_MESSAGE_PROCESSING_CONFLICT in output:
+                regex = ERROR_MESSAGE_PROCESSING_CONFLICT + '(.*)' + ERROR_MESSAGE_CONFLICTS + UPDATE_PREFIX + '(.+?)\n'
+                conflict_tuples = re.findall(regex, output)
+                if len(conflict_tuples) > 0:
+                    conflict_updates = ''
+                    for tuple in conflict_tuples:
+                        conflict_updates += tuple[1] + ' '
+                    raise ConflictPresent(conflict_updates.rstrip())
+                else:
+                    raise PrecheckFailure(precheckfailure_msg)
             elif ERROR_MESSAGE_VERSION_REQUIRED in errmsg and ERROR_MESSAGE_VERSION_INSTALLED in errmsg:
-                 raise WrongServerVersion(errmsg)
+                regex = ERROR_MESSAGE_VERSION_REQUIRED + '(.+?)\n.+ {2,2}(.+)$'
+                match = re.search(regex, errmsg, flags=re.DOTALL)
+                if match:
+                    required_version = match.group(1).rstrip()
+                    installed_version = match.group(2).rstrip()
+                    raise WrongServerVersion(required_version, installed_version)
+                else:
+                    raise PrecheckFailure(precheckfailure_msg)
             elif ERROR_MESSAGE_PREREQUISITE in errmsg:
-                raise PrerequisiteMissing(errmsg)
+                regex = ERROR_MESSAGE_PREREQUISITE + UPDATE_PREFIX + '(.+?)\n'
+                prerequisite_list = re.findall(regex, errmsg)
+                if len(prerequisite_list) > 0:
+                    prerequisite_updates = ''
+                    for prerequisite in prerequisite_list:
+                        prerequisite_updates += prerequisite + ' '
+                    raise PrerequisiteMissing(prerequisite_updates.rstrip())
+                else:
+                    raise PrecheckFailure(precheckfailure_msg)
             else:
-                raise PrecheckFailure(errmsg)
+                raise PrecheckFailure(precheckfailure_msg)
         else:
             raise PrecheckFailure()
     else:
@@ -180,7 +208,8 @@ if __name__ == '__main__':
     except ConflictPresent as e:
         print(failure_message(UPDATE_PRECHECK_FAILED_CONFLICT_PRESENT, [update_package, str(e)]))
     except WrongServerVersion as e:
-        print(failure_message(UPDATE_PRECHECK_FAILED_WRONG_SERVER_VERSION, [update_package, str(e)]))
+        required_version, installed_version = e.args
+        print(failure_message(UPDATE_PRECHECK_FAILED_WRONG_SERVER_VERSION, [update_package, installed_version, required_version]))
     except InvalidUpdate as e:
         print(failure_message(INVALID_UPDATE, [update_package, str(e)]))
     except Exception as e: