Add handling for image files for chat completions

Signed-off-by: trangevi <[email protected]>

sdk/ai/Azure.AI.Inference/src/Customized/ChatCompletionsClient.cs

@@ -49,11 +49,6 @@ public ChatCompletionsClient(Uri endpoint, TokenCredential credential, AzureAIIn
             Argument.AssertNotNull(credential, nameof(credential));
             options ??= new AzureAIInferenceClientOptions();
 
-            // CUSTOM CODE NOTE: This mimics the TokenRequestContext internal to the BearerTokenAuthenticationPolicy used in the pipeline
-            TokenRequestContext context = new TokenRequestContext(AuthorizationScopes, null, null, null, isCaeEnabled: true, isProofOfPossessionEnabled: false, null, null, null);
-            var token = credential.GetToken(context, CancellationToken.None).Token;
-            options.AddPolicy(new AddApiKeyHeaderPolicy(token), HttpPipelinePosition.PerCall);
-
             ClientDiagnostics = new ClientDiagnostics(options, true);
             _tokenCredential = credential;
             _pipeline = HttpPipelineBuilder.Build(options, Array.Empty<HttpPipelinePolicy>(), new HttpPipelinePolicy[] { new BearerTokenAuthenticationPolicy(_tokenCredential, AuthorizationScopes) }, new ResponseClassifier());

sdk/ai/Azure.AI.Inference/src/Customized/ChatMessageImageContentItem.cs

@@ -3,6 +3,7 @@
 
 using System;
 using System.IO;
+using System.Runtime.InteropServices.ComTypes;
 
 namespace Azure.AI.Inference
 {
@@ -50,6 +51,21 @@ public ChatMessageImageContentItem(Stream stream, string mimeType, ChatMessageIm
             : this(new ChatMessageImageUrl(stream, mimeType, detailLevel))
         { }
 
+        /// <summary>
+        /// Initializes a new instance of ChatMessageImageContentItem from a file pointer to an image
+        /// in a known format.
+        /// </summary>
+        /// <param name="imageFilePath"> The path to the image to use. </param>
+        /// <param name="mimeType"> The MIME type, e.g. <c>image/png</c>, matching the format of the image data. </param>
+        /// <param name="detailLevel"> The image detail level the model should use when evaluating the image. </param>
+        public ChatMessageImageContentItem(string imageFilePath, string mimeType, ChatMessageImageDetailLevel? detailLevel = null)
+        {
+            Stream fileStream = File.OpenRead(imageFilePath);
+
+            Type = "image_url";
+            ImageUrl = new ChatMessageImageUrl(fileStream, mimeType, detailLevel);
+        }
+
         /// <summary> Initializes a new instance of <see cref="ChatMessageImageContentItem"/>. </summary>
         /// <param name="imageUrl"> An internet location, which must be accessible to the model,from which the image may be retrieved. </param>
         /// <exception cref="ArgumentNullException"> <paramref name="imageUrl"/> is null. </exception>

sdk/ai/Azure.AI.Inference/tests/ChatCompletionsClientTest.cs

@@ -41,9 +41,10 @@ public enum ImageTestSourceKind
             UsingInternetLocation,
             UsingStream,
             UsingBinaryData,
+            UsingFilePath
         }
 
-        public ChatCompletionsClientTest(bool isAsync) : base(isAsync)
+        public ChatCompletionsClientTest(bool isAsync) : base(isAsync, RecordedTestMode.Record)
         {
             TestDiagnostics = false;
             JsonPathSanitizers.Add("$.messages[*].content[*].image_url.url");
@@ -460,8 +461,14 @@ Dictionary<string, string> arguments
         [TestCase(ImageTestSourceKind.UsingInternetLocation)]
         [TestCase(ImageTestSourceKind.UsingStream)]
         [TestCase(ImageTestSourceKind.UsingBinaryData)]
+        [TestCase(ImageTestSourceKind.UsingFilePath)]
         public async Task TestChatCompletionsWithImages(ImageTestSourceKind imageSourceKind)
         {
+            if (imageSourceKind == ImageTestSourceKind.UsingFilePath && Mode == RecordedTestMode.Playback)
+            {
+                Assert.Inconclusive("Unable to run test with file path in playback mode.");
+            }
+
             var aoaiEndpoint = new Uri(TestEnvironment.AoaiEndpoint);
             var aoaiKey = new AzureKeyCredential(TestEnvironment.AoaiKey);
 
@@ -480,6 +487,7 @@ public async Task TestChatCompletionsWithImages(ImageTestSourceKind imageSourceK
                 ImageTestSourceKind.UsingInternetLocation => new(GetTestImageInternetUri(), ChatMessageImageDetailLevel.Low),
                 ImageTestSourceKind.UsingStream => new(GetTestImageStream("image/jpg"), "image/jpg", ChatMessageImageDetailLevel.Low),
                 ImageTestSourceKind.UsingBinaryData => new(GetTestImageData("image/jpg"), "image/jpg", ChatMessageImageDetailLevel.Low),
+                ImageTestSourceKind.UsingFilePath => new(TestEnvironment.TestImageJpgInputPath, "image/jpg", ChatMessageImageDetailLevel.Low),
                 _ => throw new ArgumentException(nameof(imageSourceKind)),
             };