added all tests

sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_common/rai_service.py

@@ -270,8 +270,8 @@ def _parse_content_harm_response(batch_response: List[Dict], metric_name: str) -
 
     result: Dict[str, Union[str, float]] = {
     (key.value if hasattr(key, 'value') else key): math.nan,
-    f"{key}_score": math.nan,
-    f"{key}_reason": ""
+    f"{(key.value if hasattr(key, 'value') else key)}_score": math.nan,
+    f"{(key.value if hasattr(key, 'value') else key)}_reason": math.nan
 }
 
     response = batch_response[0]

sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_content_safety/_content_safety_chat.py

@@ -99,10 +99,10 @@ def __init__(
         self._eval_last_turn = eval_last_turn
         self._parallel = parallel
         self._evaluators: List[Callable[..., Dict[str, Union[str, float]]]] = [
-            ViolenceEvaluator(azure_ai_project, credential),
-            SexualEvaluator(azure_ai_project, credential),
-            SelfHarmEvaluator(azure_ai_project, credential),
-            HateUnfairnessEvaluator(azure_ai_project, credential),
+            ViolenceEvaluator(credential, azure_ai_project),
+            SexualEvaluator(credential, azure_ai_project),
+            SelfHarmEvaluator(credential, azure_ai_project),
+            HateUnfairnessEvaluator(credential, azure_ai_project),
         ]
 
     def __call__(self, *, conversation: list, **kwargs):

sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_multimodal/_content_safety_multimodal.py

@@ -24,18 +24,21 @@
 class ContentSafetyMultimodalEvaluator:
     """
     Initialize a content safety multimodal evaluator configured to evaluate content safety metrics for multimodal scenario.
+
+    :param credential: The credential for connecting to Azure AI project. Required
+    :type credential: ~azure.core.credentials.TokenCredential
     :param azure_ai_project: The scope of the Azure AI project.
         It contains subscription id, resource group, and project name.
     :type azure_ai_project: ~azure.ai.evaluation.AzureAIProject
     :param parallel: If True, use parallel execution for evaluators. Else, use sequential execution.
         Default is True.
     :type parallel: bool
-    :param credential: The credential for connecting to Azure AI project.
-    :type credential: ~azure.core.credentials.TokenCredential
+
     :return: A function that evaluates multimodal chat messages and generates metrics.
     :rtype: Callable
 
     **Usage**
+
     .. code-block:: python
         azure_ai_project = {
             "subscription_id": "<subscription_id>",
@@ -85,13 +88,18 @@ class ContentSafetyMultimodalEvaluator:
         }
     """
 
-    def __init__(self, credential, azure_ai_project: dict, parallel: bool = False):
+    def __init__(
+        self, 
+        credential, 
+        azure_ai_project, 
+        parallel: bool = False
+    ):
         self._parallel = parallel
         self._evaluators: List[Callable[..., Dict[str, Union[str, float]]]] = [
-            ViolenceMultimodalEvaluator(azure_ai_project, credential),
-            SexualMultimodalEvaluator(azure_ai_project, credential),
-            SelfHarmMultimodalEvaluator(azure_ai_project, credential),
-            HateUnfairnessMultimodalEvaluator(azure_ai_project, credential),
+            ViolenceMultimodalEvaluator(credential, azure_ai_project),
+            SexualMultimodalEvaluator(credential, azure_ai_project),
+            SelfHarmMultimodalEvaluator(credential, azure_ai_project),
+            HateUnfairnessMultimodalEvaluator(credential, azure_ai_project),
         ]
 
     def __call__(
@@ -102,7 +110,7 @@ def __call__(
         """
         Evaluates content-safety metrics for list of messages.
         :keyword messages: The messages to be evaluated. Each message should have "role" and "content" keys.
-        :paramtype messages: Dict
+        :paramtype messages: ~azure.ai.evaluation.Conversation
         :return: The scores for messages.
         :rtype: Dict
         """
@@ -208,7 +216,6 @@ def _validate_messages(self, messages):
                 blame=ErrorBlame.USER_ERROR,
             )
 
-
     def _get_harm_severity_level(self, harm_score: float) -> Union[HarmSeverityLevel, float]:
         HARM_SEVERITY_LEVEL_MAPPING = {
             HarmSeverityLevel.VeryLow: (0, 1),

sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_multimodal/_content_safety_multimodal_base.py

@@ -12,30 +12,36 @@ class ContentSafetyMultimodalEvaluatorBase(ABC):
     """
     Initialize a evaluator for a specified Evaluation Metric. Base class that is not
     meant to be instantiated by users.
+
     :param metric: The metric to be evaluated.
     :type metric: ~azure.ai.evaluation._evaluators._content_safety.flow.constants.EvaluationMetrics
+    :param credential: The credential for connecting to Azure AI project. Required
+    :type credential: ~azure.core.credentials.TokenCredential
     :param azure_ai_project: The scope of the Azure AI project.
         It contains subscription id, resource group, and project name.
     :type azure_ai_project: ~azure.ai.evaluation.AzureAIProject
-    :param credential: The credential for connecting to Azure AI project.
-    :type credential: ~azure.core.credentials.TokenCredential
     """
 
     def __init__(
         self, 
         metric: Union[EvaluationMetrics, _InternalEvaluationMetrics],
-        azure_ai_project: Dict, 
-        credential: TokenCredential
+        credential: TokenCredential,
+        azure_ai_project, 
     ):
         self._metric = metric
         self._azure_ai_project = azure_ai_project
         self._credential = credential
 
-    async def __call__(self, *, messages, **kwargs):
+    async def __call__(
+        self, 
+        *, 
+        messages, 
+        **kwargs
+    ):
         """
         Evaluates content according to this evaluator's metric.
         :keyword messages: The messages to be evaluated. Each message should have "role" and "content" keys.
-        :paramtype messages: List[Dict]
+        :paramtype messages: ~azure.ai.evaluation.Conversation
         :return: The evaluation score computation based on the Content Safety metric (self.metric).
         :rtype: Any
         """

sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_multimodal/_hate_unfairness.py

@@ -58,14 +58,23 @@ class HateUnfairnessMultimodalEvaluator:
         }
     """
 
-    def __init__(self, azure_ai_project: dict, credential=None) -> None:
-        self._async_evaluator = _AsyncHateUnfairnessMultimodalEvaluator(azure_ai_project, credential)
+    def __init__(
+        self, 
+        credential,
+        azure_ai_project
+    ):
+        self._async_evaluator = _AsyncHateUnfairnessMultimodalEvaluator(credential, azure_ai_project)
 
-    def __call__(self, *, messages, **kwargs):
+    def __call__(
+        self, 
+        *, 
+        messages, 
+        **kwargs
+    ):
         """
         Evaluates hate unfairness content.
         :keyword messages: The messages to be evaluated. Each message should have "role" and "content" keys.
-        :paramtype messages: List[Dict]
+        :paramtype messages: ~azure.ai.evaluation.Conversation
         :return: The hate unfairness score.
         :rtype: Dict
         """
@@ -76,12 +85,21 @@ def _to_async(self):
 
 
 class _AsyncHateUnfairnessMultimodalEvaluator(ContentSafetyMultimodalEvaluatorBase):
-    def __init__(self, azure_ai_project: dict, credential=None):
+    def __init__(
+        self, 
+        credential,
+        azure_ai_project
+    ):
         super().__init__(
             metric=EvaluationMetrics.HATE_FAIRNESS,
-            azure_ai_project=azure_ai_project,
             credential=credential,
+            azure_ai_project=azure_ai_project,
         )
 
-    async def __call__(self, *, messages, **kwargs):
+    async def __call__(
+        self, 
+        *, 
+        messages, 
+        **kwargs
+    ):
         return await super().__call__(messages=messages, **kwargs)

sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_multimodal/_protected_material.py

@@ -18,6 +18,7 @@ class ProtectedMaterialMultimodalEvaluator:
     :param azure_ai_project: The scope of the Azure AI project.
         It contains subscription id, resource group, and project name.
     :type azure_ai_project: ~azure.ai.evaluation.AzureAIProject
+
     :return: Whether or not protected material was found in the response, with AI-generated reasoning.
     :rtype: Dict[str, str]
 
@@ -61,15 +62,24 @@ class ProtectedMaterialMultimodalEvaluator:
         }
     """
 
-    def __init__(self, azure_ai_project: dict, credential=None):
-        self._async_evaluator = _AsyncProtectedMaterialMultimodalEvaluator(azure_ai_project, credential)
+    def __init__(
+        self,
+        credential, 
+        azure_ai_project,
+    ):
+        self._async_evaluator = _AsyncProtectedMaterialMultimodalEvaluator(credential, azure_ai_project)
 
-    def __call__(self, *, messages, **kwargs):
+    def __call__(
+        self, 
+        *, 
+        messages, 
+        **kwargs
+    ):
         """
         Evaluates protected materials content.
 
         :keyword messages: The messages to be evaluated. Each message should have "role" and "content" keys.
-        :paramtype messages: List[Dict]
+        :paramtype messages: ~azure.ai.evaluation.Conversation
         :return: A dictionary containing a boolean label and reasoning.
         :rtype: dict
         """
@@ -79,15 +89,19 @@ def _to_async(self):
         return self._async_evaluator
 
 class _AsyncProtectedMaterialMultimodalEvaluator:
-    def __init__(self, azure_ai_project: dict, credential: None):
-        self._azure_ai_project = azure_ai_project
+    def __init__(
+        self, 
+        credential,
+        azure_ai_project
+    ):
         self._credential = credential
+        self._azure_ai_project = azure_ai_project
 
     async def __call__(self, *, messages, **kwargs):
         """
         Evaluates content according to this evaluator's metric.
         :keyword messages: The messages to be evaluated. Each message should have "role" and "content" keys.
-        :paramtype messages: List[Dict]
+        :paramtype messages: ~azure.ai.evaluation.Conversation
         :return: The evaluation score computation based on the Content Safety metric (self.metric).
         :rtype: Any
         """
@@ -97,7 +111,7 @@ async def __call__(self, *, messages, **kwargs):
         result = await evaluate_with_rai_service_multimodal(
             messages=messages,
             metric_name=EvaluationMetrics.PROTECTED_MATERIAL,
-            project_scope=self._azure_ai_project,
             credential=self._credential,
+            project_scope=self._azure_ai_project,
         )
         return result

sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_multimodal/_self_harm.py

@@ -11,12 +11,15 @@
 class SelfHarmMultimodalEvaluator:
     """
     Initialize a self harm multimodal evaluator for self harm score.
+
+    :param credential: The credential for connecting to Azure AI project. Required
+    :type credential: ~azure.core.credentials.TokenCredential
     :param azure_ai_project: The scope of the Azure AI project.
         It contains subscription id, resource group, and project name.
     :type azure_ai_project: ~azure.ai.evaluation.AzureAIProject
-    :param credential: The credential for connecting to Azure AI project.
-    :type credential: ~azure.core.credentials.TokenCredential
+
     **Usage**
+
     .. code-block:: python
         azure_ai_project = {
             "subscription_id": "<subscription_id>",
@@ -56,14 +59,23 @@ class SelfHarmMultimodalEvaluator:
         }
     """
 
-    def __init__(self, azure_ai_project: dict, credential=None):
-        self._async_evaluator = _AsyncSelfHarmMultimodalEvaluator(azure_ai_project, credential)
+    def __init__(
+        self,
+        credential, 
+        azure_ai_project
+    ):
+        self._async_evaluator = _AsyncSelfHarmMultimodalEvaluator(credential, azure_ai_project)
 
-    def __call__(self, *, messages, **kwargs):
+    def __call__(
+        self, 
+        *, 
+        messages, 
+        **kwargs
+    ):
         """
         Evaluates self harm content.
         :keyword messages: The messages to be evaluated. Each message should have "role" and "content" keys.
-        :paramtype messages: List[Dict]
+        :paramtype messages: ~azure.ai.evaluation.Conversation
         :return: The self harm score.
         :rtype: Dict
         """
@@ -73,11 +85,15 @@ def _to_async(self):
         return self._async_evaluator
 
 class _AsyncSelfHarmMultimodalEvaluator(ContentSafetyMultimodalEvaluatorBase):
-    def __init__(self, azure_ai_project: dict, credential=None):
+    def __init__(
+        self, 
+        credential,
+        azure_ai_project
+    ):
         super().__init__(
             metric=EvaluationMetrics.SELF_HARM,
-            azure_ai_project=azure_ai_project,
             credential=credential,
+            azure_ai_project=azure_ai_project,
         )
     async def __call__(self, *, messages, **kwargs):
         return await super().__call__(messages=messages, **kwargs)

sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_multimodal/_sexual.py

@@ -12,13 +12,15 @@
 class SexualMultimodalEvaluator:
     """
     Initialize a sexual multimodal evaluator for sexual score.
+
+    :param credential: The credential for connecting to Azure AI project. Required
+    :type credential: ~azure.core.credentials.TokenCredential
     :param azure_ai_project: The scope of the Azure AI project.
         It contains subscription id, resource group, and project name.
     :type azure_ai_project: ~azure.ai.evaluation.AzureAIProject
-    :param credential: The credential for connecting to Azure AI project.
-    :type credential: ~azure.core.credentials.TokenCredential
 
     **Usage**
+
     .. code-block:: python
         azure_ai_project = {
             "subscription_id": "<subscription_id>",
@@ -58,14 +60,23 @@ class SexualMultimodalEvaluator:
         }
     """
 
-    def __init__(self, azure_ai_project: dict, credential=None):
-        self._async_evaluator = _AsyncSexualMultimodalEvaluator(azure_ai_project, credential)
+    def __init__(
+        self,
+        credential, 
+        azure_ai_project
+    ):
+        self._async_evaluator = _AsyncSexualMultimodalEvaluator(credential, azure_ai_project)
 
-    def __call__(self, *, messages, **kwargs):
+    def __call__(
+        self, 
+        *, 
+        messages, 
+        **kwargs
+    ):
         """
         Evaluates sexual content.
         :keyword messages: The messages to be evaluated. Each message should have "role" and "content" keys.
-        :paramtype messages: List[Dict]
+        :paramtype messages: ~azure.ai.evaluation.Conversation
         :return: The sexual score.
         :rtype: Dict
         """
@@ -75,11 +86,15 @@ def _to_async(self):
         return self._async_evaluator
 
 class _AsyncSexualMultimodalEvaluator(ContentSafetyMultimodalEvaluatorBase):
-    def __init__(self, azure_ai_project: dict, credential=None):
+    def __init__(
+        self, 
+        credential,
+        azure_ai_project
+    ):
         super().__init__(
             metric=EvaluationMetrics.SEXUAL,
-            azure_ai_project=azure_ai_project,
             credential=credential,
+            azure_ai_project=azure_ai_project,
         )
     async def __call__(self, *, messages, **kwargs):
         return await super().__call__(messages=messages, **kwargs)