first pass at writing tests

sdk/ai/azure-ai-inference/tests/README.md

@@ -0,0 +1,58 @@
+# Azure Image Analysis client library tests for Python
+
+## Running tests locally, on a Windows PC, against the live service
+
+### Prerequisites
+
+See [Prerequisites](https://github.com/Azure/azure-sdk-for-python/blob/main/sdk/vision/azure-ai-vision-imageanalysis/README.md#prerequisites). Create an Azure resource in one of the GPU-supported regions, otherwise some of the tests will fail.
+
+### Setup
+
+* Clone or download this sample repository.
+* Open a command prompt window in the folder `sdk\vision\azure-ai-vision-imageanalysis`.
+* If you want to run tests against the latest public Image Analysis client library, install it by running:
+   ```bash
+   pip install azure-ai-vision-imageanalysis
+   ```
+* If you want to run tests against a locally built Image Analysis client library:
+    * First build the wheel:
+        ```bash
+        pip install wheel
+        pip install -r dev_requirements.txt
+        python setup.py bdist_wheel
+        ```
+    * Then install the resulting local wheel (update version `1.0.0b1` to the current one):
+        ```bash
+        pip install dist\azure_ai_vision_imageanalysis-1.0.0b1-py3-none-any.whl --user --force-reinstall
+        ```
+
+
+### Set environment variables
+
+See [Set environment variables](https://github.com/Azure/azure-sdk-for-python/blob/main/sdk/vision/azure-ai-vision-imageanalysis/README.md#set-environment-variables).
+
+In addition, the following environment values **must be** defined, although not used. Assign any value to them:
+```
+set VISION_TENANT_ID=not-used
+set VISION_CLIENT_ID=not-used
+set VISION_CLIENT_SECRET=not-used
+```
+
+### Configure test proxy
+
+Configure the test proxy to run live service tests without recordings:
+```
+set AZURE_TEST_RUN_LIVE=true
+set AZURE_SKIP_LIVE_RECORDING=true
+```
+
+### Run tests
+
+To run all tests, type:
+```
+pytest
+```
+
+### Additional information
+
+See [test documentation](https://github.com/Azure/azure-sdk-for-python/blob/main/doc/dev/tests.md) for additional information, including how to set proxy recordings and run tests using recordings.

sdk/ai/azure-ai-inference/tests/conftest.py

@@ -0,0 +1,12 @@
+# ------------------------------------
+# Copyright (c) Microsoft Corporation.
+# Licensed under the MIT License.
+# ------------------------------------
+
+import pytest
+from devtools_testutils import test_proxy
+
+# autouse=True will trigger this fixture on each pytest run, even if it's not explicitly used by a test method
+@pytest.fixture(scope="session", autouse=True)
+def start_proxy(test_proxy):
+    return

sdk/ai/azure-ai-inference/tests/model_inference_test_base.py

@@ -0,0 +1,189 @@
+# ------------------------------------
+# Copyright (c) Microsoft Corporation.
+# Licensed under the MIT License.
+# ------------------------------------
+import functools
+import logging
+import sys
+import azure.ai.inference as sdk
+import azure.ai.inference as async_sdk
+
+from os import path
+from typing import List, Optional, Union
+from devtools_testutils import AzureRecordedTestCase, EnvironmentVariableLoader
+from azure.core.credentials import AzureKeyCredential
+from azure.core.exceptions import AzureError
+from azure.core.pipeline import PipelineRequest
+
+# Set to True to enable SDK logging
+LOGGING_ENABLED = True
+
+if LOGGING_ENABLED:
+    # Create a logger for the 'azure' SDK
+    # See https://docs.python.org/3/library/logging.html
+    logger = logging.getLogger("azure")
+    logger.setLevel(logging.INFO)  # INFO or DEBUG
+
+    # Configure a console output
+    handler = logging.StreamHandler(stream=sys.stdout)
+    logger.addHandler(handler)
+
+ServicePreparer = functools.partial(
+    EnvironmentVariableLoader,
+    "model",
+    model_endpoint="https://your-azure-resource-name.your-azure-region.inference.ai.azure.com",
+    model_key="00000000000000000000000000000000"
+)
+
+
+# The test class name needs to start with "Test" to get collected by pytest
+class ModelInferenceTestBase(AzureRecordedTestCase):
+
+    client: sdk.ModelClient
+    async_client: async_sdk.ModelClient
+    connection_url: str
+
+    # Set to True to print out all analysis results
+    PRINT_CHAT_COMPLETION_RESULTS = True
+
+    def _create_client_for_standard_test(self, sync: bool, get_connection_url: bool = False, **kwargs):
+        endpoint = kwargs.pop("model_endpoint")
+        key = kwargs.pop("model_key")
+        self._create_client(endpoint, key, sync, get_connection_url)
+
+    def _create_client_for_authentication_failure(self, sync: bool, **kwargs):
+        endpoint = kwargs.pop("model_endpoint")
+        key = "00000000000000000000000000000000"
+        self._create_client(endpoint, key, sync, False)
+
+    def _create_client(self, endpoint: str, key: str, sync: bool, get_connection_url: bool):
+        credential = AzureKeyCredential(key)
+        if sync:
+            self.client = sdk.ModelClient(
+                endpoint=endpoint,
+                credential=credential,
+                logging_enable=LOGGING_ENABLED,
+                raw_request_hook=self._raw_request_check if get_connection_url else None,
+            )
+            assert self.client is not None
+        else:
+            self.async_client = async_sdk.ModelClient(
+                endpoint=endpoint,
+                credential=credential,
+                logging_enable=LOGGING_ENABLED,
+                raw_request_hook=self._raw_request_check if get_connection_url else None,
+            )
+            assert self.async_client is not None
+
+    def _raw_request_check(self, request: PipelineRequest):
+        self.connection_url = request.http_request.url
+        print(f"Connection URL: {request.http_request.url}")
+
+    def _do_chat_completions(
+        self,
+        options: sdk.models.ChatCompletionsOptions,
+        query_params: Optional[dict] = None,
+        **kwargs,
+    ):
+
+        result = self.client.get_chat_completions(options=options, params=query_params)
+
+        # Optional: console printout of all results
+        if ModelInferenceTestBase.PRINT_CHAT_COMPLETION_RESULTS:
+            ModelInferenceTestBase._print_chat_completion_results(result)
+
+        # Validate all results
+        ModelInferenceTestBase._validate_chat_completion_results(result)
+
+        # Validate that additional query parameters exists in the connection URL, if specify
+        if query_params is not None:
+            ModelInferenceTestBase._validate_query_parameters(query_params, self.connection_url)
+
+    async def _do_async_chat_completion(
+        self,
+        options: sdk.models.ChatCompletionsOptions,
+        query_params: Optional[dict] = None,
+        **kwargs,
+    ):
+
+        result = await self.async_client.chat_completions(options=options, params=query_params)
+
+        # Optional: console printout of all results
+        if ModelInferenceTestBase.PRINT_CHAT_COMPLETION_RESULTS:
+            ModelInferenceTestBase._print_chat_completion_results(result)
+
+        # Validate all results
+        ModelInferenceTestBase._validate_chat_completion_results(result)
+
+        # Validate that additional query parameters exists in the connection URL, if specify
+        if query_params is not None:
+            ModelInferenceTestBase._validate_query_parameters(query_params, self.connection_url)
+
+    def _do_chat_completion_with_error(
+        self,
+        options: sdk.models.ChatCompletionsOptions,
+        expected_status_code: int,
+        expected_message_contains: str,
+        **kwargs,
+    ):
+
+        try:
+            result = self.client.get_chat_completions(options=options)
+
+        except AzureError as e:
+            print(e)
+            assert hasattr(e, "status_code")
+            assert e.status_code == expected_status_code
+            assert expected_message_contains in e.message
+            return
+        assert False  # We should not get here
+
+    async def _do_async_chat_completion_with_error(
+        self,
+        options: sdk.models.ChatCompletionsOptions,
+        expected_status_code: int,
+        expected_message_contains: str,
+        **kwargs,
+    ):
+
+        try:
+            result = await self.async_client.get_chat_completions(options=options)
+
+        except AzureError as e:
+            print(e)
+            assert hasattr(e, "status_code")
+            assert e.status_code == expected_status_code
+            assert expected_message_contains in e.message
+            return
+        assert False  # We should not get here
+
+    @staticmethod
+    def _validate_query_parameters(query_params: dict, connection_url: str):
+        assert len(query_params) > 0
+        query_string = ""
+        for key, value in query_params.items():
+            query_string += "&" + key + "=" + value
+        query_string = "?" + query_string[1:]
+        assert query_string in connection_url
+
+    @staticmethod
+    def _validate_result(result: sdk.models.ChatCompletions):
+        assert True
+
+    @staticmethod
+    def _print_analysis_results(result: sdk.models.ChatCompletions):
+
+        for choice in result.choices:
+            print(" choices[0].message.content: {}".format(choice.message.content))
+            print(" choices[0].message.role: {}".format(choice.message.role))
+            print(" choices[0].finish_reason: {}".format(choice.finish_reason))
+            print(" choices[0].index: {}".format(choice.index))
+
+        print(" id: {}".format(result.id))
+        print(" created: {}".format(result.created))
+        print(" model: {}".format(result.model))
+        print(" object: {}".format(result.object))
+        print(" usage.completion_tokens: {}".format(result.usage.completion_tokens))
+        print(" usage.prompt_tokens: {}".format(result.usage.prompt_tokens))
+        print(" usage.completion_tokens: {}".format(result.usage.completion_tokens))
+

sdk/ai/azure-ai-inference/tests/test_model_inference_async_client.py

@@ -0,0 +1,90 @@
+# ------------------------------------
+# Copyright (c) Microsoft Corporation.
+# Licensed under the MIT License.
+# ------------------------------------
+import inspect
+import azure.ai.vision.imageanalysis as sdk
+
+from model_inference_test_base import ModelInferenceTestBase, ServicePreparer
+from devtools_testutils.aio import recorded_by_proxy_async
+
+# The test class name needs to start with "Test" to get collected by pytest
+class TestImageAnalysisAsyncClient(ModelInferenceTestBase):
+
+    # **********************************************************************************
+    #
+    #                            HAPPY PATH TESTS
+    #
+    # **********************************************************************************
+
+    # Test all visual features from a local image, using default settings
+    @ServicePreparer()
+    @recorded_by_proxy_async
+    async def test_async_chat_completion(self, **kwargs):
+
+        self._create_client_for_standard_analysis(sync=False, **kwargs)
+
+        await self._do_async_chat_completions(
+            options=[
+                messages=[
+                    role=sdk.models.ChatRole.USER,
+                    content="How many feet are in a mile?"
+                ]
+            ],
+            **kwargs
+        )
+
+        await self.async_client.close()
+
+    # Test some visual features, one after the other, from image URL, with relevant settings specified
+"""     @ServicePreparer()
+    @recorded_by_proxy_async
+    async def test_analyze_async_single_feature_from_url(self, **kwargs):
+
+        self._create_client_for_standard_analysis(sync=False, **kwargs)
+
+        await self._do_async_analysis(
+            image_source=self.IMAGE_URL,
+            visual_features=[sdk.models.VisualFeatures.DENSE_CAPTIONS],
+            gender_neutral_caption=True,
+            **kwargs
+        )
+
+        await self._do_async_analysis(
+            image_source=self.IMAGE_URL,
+            visual_features=[sdk.models.VisualFeatures.SMART_CROPS],
+            smart_crops_aspect_ratios=[0.9, 1.33],
+            **kwargs
+        )
+
+        await self._do_async_analysis(
+            image_source=self.IMAGE_URL, visual_features=[sdk.models.VisualFeatures.TAGS], language="en", **kwargs
+        )
+
+        await self._do_async_analysis(
+            image_source=self.IMAGE_URL, visual_features=[sdk.models.VisualFeatures.PEOPLE], **kwargs
+        )
+
+        await self.async_client.close() """
+
+    # **********************************************************************************
+    #
+    #                            ERROR TESTS
+    #
+    # **********************************************************************************
+
+ """    @ServicePreparer()
+    @recorded_by_proxy_async
+    async def test_analyze_async_authentication_failure(self, **kwargs):
+
+        self._create_client_for_authentication_failure(sync=False, **kwargs)
+
+        await self._do_async_analysis_with_error(
+            image_source=self.IMAGE_URL,
+            visual_features=[sdk.models.VisualFeatures.TAGS],
+            expected_status_code=401,
+            expected_message_contains="Access denied",
+            **kwargs
+        )
+
+        await self.async_client.close() """