fixes

sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_common/rai_service.py

@@ -13,12 +13,10 @@
 import jwt
 import json
 
-from azure.ai.inference._model_base import SdkJSONEncoder
-from azure.ai.inference.models import ChatRequestMessage, SystemMessage, AssistantMessage
-
+from promptflow.core._errors import MissingRequiredPackage
 from azure.ai.evaluation._exceptions import ErrorBlame, ErrorCategory, ErrorTarget, EvaluationException
 from azure.ai.evaluation._http_utils import AsyncHttpPipeline, get_async_http_client
-from azure.ai.evaluation._model_configurations import AzureAIProject
+from azure.ai.evaluation._model_configurations import AzureAIProject, Message
 from azure.core.credentials import TokenCredential
 from azure.core.pipeline.policies import AsyncRetryPolicy
 
@@ -499,19 +497,20 @@ async def submit_multimodal_request(messages, metric: str, rai_svc_url: str, tok
     :rtype: str
     """
     ## handle json payload and payload from inference sdk strongly type messages
-    if len(messages) > 0 and isinstance(messages[0], ChatRequestMessage):
-        filtered_messages = [message for message in messages if not isinstance(message, SystemMessage)]
-        assistant_messages = [message for message in messages if isinstance(message, AssistantMessage)]
-        content_type = retrieve_content_type(assistant_messages, metric)
-        json_text = generate_payload_multimodal(content_type, filtered_messages, metric)
-        messages_text = json.dumps(json_text, cls=SdkJSONEncoder, exclude_readonly=True)
-        payload = json.loads(messages_text)
-
-    else:
-        filtered_messages = [message for message in messages if message["role"] != "system"]
-        assistant_messages = [message for message in messages if message["role"] == "assistant"]
-        content_type = retrieve_content_type(assistant_messages, metric)
-        payload = generate_payload_multimodal(content_type, filtered_messages, metric)
+    if len(messages) > 0 and not isinstance(messages[0], Dict):
+        try:
+            from azure.ai.inference.models import ChatRequestMessage
+        except ImportError:
+            error_message = "Please install 'azure-ai-inference' package to use SystemMessage, UserMessage, AssistantMessage"
+            raise MissingRequiredPackage(message=error_message)
+        else:
+            if len(messages) > 0 and isinstance(messages[0], ChatRequestMessage):
+                messages = [message.as_dict() for message in messages]
+
+    filtered_messages = [message for message in messages if message["role"] != "system"]
+    assistant_messages = [message for message in messages if message["role"] == "assistant"]
+    content_type = retrieve_content_type(assistant_messages, metric)
+    payload = generate_payload_multimodal(content_type, filtered_messages, metric)
 
     ## calling rai service for annotation
     url = rai_svc_url + "/submitannotation"

sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluate/_utils.py

@@ -88,14 +88,14 @@ def _store_multimodal_content(messages, tmpdir: str):
     os.makedirs(images_folder_path, exist_ok=True)
 
     # traverse all messages and replace base64 image data with new file name.
-    for item in messages:
-        if "content" in item:
-            for content in item["content"]:
+    for message in messages:
+        if "content" in message:
+            for content in message["content"]:
                 if content.get("type") == "image_url":
                     image_url = content.get("image_url")
-                    if image_url and 'url' in image_url and image_url['url'].startswith("data:image/jpeg;base64,"):
+                    if image_url and 'url' in image_url and image_url['url'].startswith("data:image/jpg;base64,"):
                         # Extract the base64 string
-                        base64image = image_url['url'].replace("data:image/jpeg;base64,", "")
+                        base64image = image_url['url'].replace("data:image/jpg;base64,", "")
 
                         # Generate a unique filename
                         image_file_name = f"{str(uuid.uuid4())}.jpg"
@@ -139,10 +139,12 @@ def _log_metrics_and_instance_results(
 
         with tempfile.TemporaryDirectory() as tmpdir:
             # storing multi_modal images if exists
-            col_name = "inputs.messages"
+            col_name = "inputs.conversation"
             if col_name in instance_results.columns:
-                instance_results[col_name].apply(lambda messages: _store_multimodal_content(messages, tmpdir))
-
+                for key, item in instance_results[col_name].items():
+                    if "messages" in item:
+                        _store_multimodal_content(item["messages"], tmpdir)
+
             # storing artifact result
             tmp_path = os.path.join(tmpdir, artifact_name)
 

sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_multimodal/_content_safety_multimodal.py

@@ -5,14 +5,14 @@
 import math
 from concurrent.futures import as_completed
 from typing import Callable, Dict, List, Union
-from azure.ai.inference.models import ChatRequestMessage, UserMessage, AssistantMessage, SystemMessage, ToolMessage, ContentItem, ImageContentItem
 
 from promptflow.tracing import ThreadPoolExecutorWithContext as ThreadPoolExecutor
+from promptflow.core._errors import MissingRequiredPackage
 from azure.ai.evaluation._common._experimental import experimental
 from azure.ai.evaluation._common.constants import HarmSeverityLevel
 from azure.ai.evaluation._common.math import list_mean_nan_safe
 from azure.ai.evaluation._exceptions import ErrorBlame, ErrorCategory, ErrorTarget, EvaluationException
-
+from azure.ai.evaluation._model_configurations import Conversation
 from ._hate_unfairness import HateUnfairnessMultimodalEvaluator
 from ._self_harm import SelfHarmMultimodalEvaluator
 from ._sexual import SexualMultimodalEvaluator
@@ -47,27 +47,29 @@ class ContentSafetyMultimodalEvaluator:
         }
         eval_fn = ContentSafetyMultimodalEvaluator(azure_ai_project)
         result = eval_fn(
-            messages= [
-                {
-                    "role": "user",
-                    "content": [
-                        {
-                            "type": "text",
-                            "text": "What's in this image?"
-                        },
-                        {
-                            "type": "image_url",
-                            "image_url": {
-                                "url": "<image url or base64 encoded image>"
+            {
+                "messages": [
+                    {
+                        "role": "user",
+                        "content": [
+                            {
+                                "type": "text",
+                                "text": "What's in this image?"
+                            },
+                            {
+                                "type": "image_url",
+                                "image_url": {
+                                    "url": "<image url or base64 encoded image>"
+                                }
                             }
-                        }
-                    ]
-                },
-                {
-                    "role": "assistant",
-                    "content": "This picture shows an astronaut standing in desert."
-                }
-            ]
+                        ]
+                    },
+                    {
+                        "role": "assistant",
+                        "content": "This picture shows an astronaut standing in desert."
+                    }
+                ]
+            }
         )
 
     **Output format**
@@ -105,57 +107,58 @@ def __init__(
     def __call__(
             self, 
             *, 
-            messages, 
+            conversation, 
             **kwargs):
         """
         Evaluates content-safety metrics for list of messages.
-        :keyword messages: The messages to be evaluated. Each message should have "role" and "content" keys.
-        :paramtype messages: ~azure.ai.evaluation.Conversation
+        :keyword conversation: The conversation contains list of messages to be evaluated. Each message should have "role" and "content" keys.
+        :paramtype conversation: ~azure.ai.evaluation.Conversation
         :return: The scores for messages.
         :rtype: Dict
         """
-        self._validate_messages(messages)
+        self._validate_conversation(conversation)
 
         results: Dict[str, Union[str, float]] = {}
         if self._parallel:
             with ThreadPoolExecutor() as executor:
                 futures = {
-                    executor.submit(evaluator, messages=messages, **kwargs): evaluator
+                    executor.submit(evaluator, conversation=conversation, **kwargs): evaluator
                     for evaluator in self._evaluators
                 }
 
                 for future in as_completed(futures):
                     results.update(future.result())
         else:
             for evaluator in self._evaluators:
-                result = evaluator(messages=messages, **kwargs)
+                result = evaluator(conversation=conversation, **kwargs)
                 results.update(result)
 
         return results
 
-    def _validate_messages(self, messages):
+    def _validate_conversation(self, conversation):
+        if conversation is None or "messages" not in conversation:
+            msg = "Attribute messages is missing in the request"
+            raise EvaluationException(
+                message=msg,
+                internal_message=msg,
+                target=ErrorTarget.CONTENT_SAFETY_CHAT_EVALUATOR,
+                category=ErrorCategory.INVALID_VALUE,
+                blame=ErrorBlame.USER_ERROR,
+            )
+        messages = conversation["messages"]   
         if messages is None or not isinstance(messages, list):
-            msg = "messages parameter must be a list of JSON representation of chat messages or strong typed child class of ChatRequestMessage"
+            msg = "messages parameter must be a list of JSON representation of chat messages"
             raise EvaluationException(
                 message=msg,
                 internal_message=msg,
                 target=ErrorTarget.CONTENT_SAFETY_MULTIMODAL_EVALUATOR,
                 category=ErrorCategory.INVALID_VALUE,
                 blame=ErrorBlame.USER_ERROR,
             )
-        expected_roles = [ "user", "assistant", "system", "tool" ]
+        expected_roles = [ "user", "assistant", "system"]
         image_found = False
         for num, message in enumerate(messages):
             msg_num = num + 1
-            if not isinstance(message, dict) and not isinstance(message, ChatRequestMessage):
-                msg = f"Messsage in array must be a dictionary or class of ChatRequestMessage [UserMessage, SystemMessage, AssistantMessage, ToolMessage]. Message number: {msg_num}"
-                raise EvaluationException(
-                    message=msg,
-                    internal_message=msg,
-                    target=ErrorTarget.CONTENT_SAFETY_MULTIMODAL_EVALUATOR,
-                    category=ErrorCategory.INVALID_VALUE,
-                    blame=ErrorBlame.USER_ERROR,
-                )
             if isinstance(message, dict):
                 if "role" in message or "content" in message:
                     if message["role"] not in expected_roles:
@@ -192,22 +195,29 @@ def _validate_messages(self, messages):
                             category=ErrorCategory.INVALID_VALUE,
                             blame=ErrorBlame.USER_ERROR,
                         )         
-            if isinstance(message, ChatRequestMessage):
-                if not isinstance(message, UserMessage) and not isinstance(message, AssistantMessage) and not isinstance(message, SystemMessage) and not isinstance(message, ToolMessage):
-                    msg = f"Messsage in array must be a strongly typed class of ChatRequestMessage [UserMessage, SystemMessage, AssistantMessage, ToolMessage]. Message number: {msg_num}"
-                    raise EvaluationException(
-                        message=msg,
-                        internal_message=msg,
-                        target=ErrorTarget.CONTENT_SAFETY_MULTIMODAL_EVALUATOR,
-                        category=ErrorCategory.INVALID_VALUE,
-                        blame=ErrorBlame.USER_ERROR,
-                    )  
-                if message.content and isinstance(message.content, list):
-                    image_items = [item for item in message.content if isinstance(item, ImageContentItem)]
-                    if len(image_items) > 0:
-                        image_found = True
+            else:
+                try:
+                    from azure.ai.inference.models import ChatRequestMessage, UserMessage, AssistantMessage, SystemMessage, ImageContentItem
+                except ImportError:
+                    error_message = "Please install 'azure-ai-inference' package to use SystemMessage, AssistantMessage"
+                    raise MissingRequiredPackage(message=error_message)
+                else:
+                    if isinstance(messages[0], ChatRequestMessage):
+                        if not isinstance(message, UserMessage) and not isinstance(message, AssistantMessage) and not isinstance(message, SystemMessage):
+                            msg = f"Messsage in array must be a strongly typed class of [UserMessage, SystemMessage, AssistantMessage]. Message number: {msg_num}"
+                            raise EvaluationException(
+                                message=msg,
+                                internal_message=msg,
+                                target=ErrorTarget.CONTENT_SAFETY_MULTIMODAL_EVALUATOR,
+                                category=ErrorCategory.INVALID_VALUE,
+                                blame=ErrorBlame.USER_ERROR,
+                            )  
+                        if message.content and isinstance(message.content, list):
+                            image_items = [item for item in message.content if isinstance(item, ImageContentItem)]
+                            if len(image_items) > 0:
+                                image_found = True
         if image_found is False:    
-            msg = f"Message needs to have multimodal input like images"
+            msg = f"Message needs to have multimodal input like images."
             raise EvaluationException(
                 message=msg,
                 internal_message=msg,

sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_multimodal/_content_safety_multimodal_base.py

@@ -2,7 +2,7 @@
 # Copyright (c) Microsoft Corporation. All rights reserved.
 # ---------------------------------------------------------
 from abc import ABC
-from typing import Dict, List, Union
+from typing import Union
 from azure.ai.evaluation._common.constants import EvaluationMetrics
 from azure.ai.evaluation._common.rai_service import evaluate_with_rai_service_multimodal
 from azure.ai.evaluation._common.constants import EvaluationMetrics, _InternalEvaluationMetrics
@@ -35,16 +35,17 @@ def __init__(
     async def __call__(
         self, 
         *, 
-        messages, 
+        conversation, 
         **kwargs
     ):
         """
         Evaluates content according to this evaluator's metric.
-        :keyword messages: The messages to be evaluated. Each message should have "role" and "content" keys.
-        :paramtype messages: ~azure.ai.evaluation.Conversation
+        :keyword conversation: The conversation contains list of messages to be evaluated. Each message should have "role" and "content" keys.
+        :paramtype conversation: ~azure.ai.evaluation.Conversation
         :return: The evaluation score computation based on the Content Safety metric (self.metric).
         :rtype: Any
         """
+        messages = conversation["messages"]
         # Run score computation based on supplied metric.
         result = await evaluate_with_rai_service_multimodal(
             messages=messages,

sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_multimodal/_hate_unfairness.py

@@ -4,7 +4,6 @@
 from promptflow._utils.async_utils import async_run_allowing_running_loop
 from azure.ai.evaluation._common._experimental import experimental
 from azure.ai.evaluation._common.constants import EvaluationMetrics
-from typing import List, Dict
 from ._content_safety_multimodal_base import ContentSafetyMultimodalEvaluatorBase
 
 @experimental
@@ -26,27 +25,29 @@ class HateUnfairnessMultimodalEvaluator:
         }
         eval_fn = HateUnfairnessEvaluator(azure_ai_project)
         result = eval_fn(
-            messages= [
-                {
-                    "role": "user",
-                    "content": [
-                        {
-                            "type": "text",
-                            "text": "What's in this image?"
-                        },
-                        {
-                            "type": "image_url",
-                            "image_url": {
-                                "url": "<image url or base64 encoded image>"
+            {
+                "messages": [
+                    {
+                        "role": "user",
+                        "content": [
+                            {
+                                "type": "text",
+                                "text": "What's in this image?"
+                            },
+                            {
+                                "type": "image_url",
+                                "image_url": {
+                                    "url": "<image url or base64 encoded image>"
+                                }
                             }
-                        }
-                    ]
-                },
-                {
-                    "role": "assistant",
-                    "content": "This picture shows an astronaut standing in desert."
-                }
-            ]
+                        ]
+                    },
+                    {
+                        "role": "assistant",
+                        "content": "This picture shows an astronaut standing in desert."
+                    }
+                ]
+            }
         )
 
     **Output format**
@@ -68,17 +69,18 @@ def __init__(
     def __call__(
         self, 
         *, 
-        messages, 
+        conversation, 
         **kwargs
     ):
         """
         Evaluates hate unfairness content.
-        :keyword messages: The messages to be evaluated. Each message should have "role" and "content" keys.
-        :paramtype messages: ~azure.ai.evaluation.Conversation
+        :keyword conversation: The conversation contains list of messages to be evaluated. Each message should have "role" and "content" keys.
+        :paramtype conversation: ~azure.ai.evaluation.Conversation
         :return: The hate unfairness score.
         :rtype: Dict
         """
-        return async_run_allowing_running_loop(self._async_evaluator, messages=messages, **kwargs)
+        self._validate_conversation(conversation)
+        return async_run_allowing_running_loop(self._async_evaluator, conversation=conversation, **kwargs)
 
     def _to_async(self):
         return self._async_evaluator
@@ -99,7 +101,7 @@ def __init__(
     async def __call__(
         self, 
         *, 
-        messages, 
+        conversation, 
         **kwargs
     ):
-        return await super().__call__(messages=messages, **kwargs)
+        return await super().__call__(conversation=conversation, **kwargs)