From 6c40bc19b3e7157cd732ceecea03544719582842 Mon Sep 17 00:00:00 2001
From: Stephen Toub <stoub@microsoft.com>
Date: Wed, 30 Oct 2024 11:21:10 -0400
Subject: [PATCH] Add ImageContent integration test

---
 .../ChatClientIntegrationTests.cs             |  31 ++++++++++++++++++
 ...oft.Extensions.AI.Integration.Tests.csproj |   4 +++
 .../dotnet.png                                | Bin 0 -> 2140 bytes
 .../OllamaChatClientIntegrationTests.cs       |   2 ++
 4 files changed, 37 insertions(+)
 create mode 100644 test/Libraries/Microsoft.Extensions.AI.Integration.Tests/dotnet.png

diff --git a/test/Libraries/Microsoft.Extensions.AI.Integration.Tests/ChatClientIntegrationTests.cs b/test/Libraries/Microsoft.Extensions.AI.Integration.Tests/ChatClientIntegrationTests.cs
index 0863e31db37..e9c2bd57d65 100644
--- a/test/Libraries/Microsoft.Extensions.AI.Integration.Tests/ChatClientIntegrationTests.cs
+++ b/test/Libraries/Microsoft.Extensions.AI.Integration.Tests/ChatClientIntegrationTests.cs
@@ -6,6 +6,7 @@
 using System.ComponentModel;
 using System.Diagnostics;
 using System.Diagnostics.CodeAnalysis;
+using System.IO;
 using System.Linq;
 using System.Runtime.InteropServices;
 using System.Text;
@@ -132,6 +133,27 @@ public virtual async Task CompleteStreamingAsync_UsageDataAvailable()
         Assert.Equal(usage.Details.InputTokenCount + usage.Details.OutputTokenCount, usage.Details.TotalTokenCount);
     }
 
+    protected virtual string? GetModel_MultiModal_DescribeImage() => null;
+
+    [ConditionalFact]
+    public virtual async Task MultiModal_DescribeImage()
+    {
+        SkipIfNotEnabled();
+
+        var response = await _chatClient.CompleteAsync(
+            [
+                new(ChatRole.User,
+                [
+                    new TextContent("What does this logo say?"),
+                    new ImageContent(GetImageDataUri()),
+                ])
+            ],
+            new() { ModelId = GetModel_MultiModal_DescribeImage() });
+
+        Assert.Single(response.Choices);
+        Assert.True(response.Message.Text?.IndexOf("net", StringComparison.OrdinalIgnoreCase) >= 0, response.Message.Text);
+    }
+
     [ConditionalFact]
     public virtual async Task FunctionInvocation_AutomaticallyInvokeFunction_Parameterless()
     {
@@ -714,6 +736,15 @@ private enum JobType
         Unknown,
     }
 
+    private static Uri GetImageDataUri()
+    {
+        using Stream? s = typeof(ChatClientIntegrationTests).Assembly.GetManifestResourceStream("Microsoft.Extensions.AI.dotnet.png");
+        Assert.NotNull(s);
+        MemoryStream ms = new();
+        s.CopyTo(ms);
+        return new Uri($"data:image/png;base64,{Convert.ToBase64String(ms.ToArray())}");
+    }
+
     [MemberNotNull(nameof(_chatClient))]
     protected void SkipIfNotEnabled()
     {
diff --git a/test/Libraries/Microsoft.Extensions.AI.Integration.Tests/Microsoft.Extensions.AI.Integration.Tests.csproj b/test/Libraries/Microsoft.Extensions.AI.Integration.Tests/Microsoft.Extensions.AI.Integration.Tests.csproj
index e38ccd3268b..04d9bc6d29f 100644
--- a/test/Libraries/Microsoft.Extensions.AI.Integration.Tests/Microsoft.Extensions.AI.Integration.Tests.csproj
+++ b/test/Libraries/Microsoft.Extensions.AI.Integration.Tests/Microsoft.Extensions.AI.Integration.Tests.csproj
@@ -15,6 +15,10 @@
     <InjectSharedThrow>true</InjectSharedThrow>
   </PropertyGroup>
 
+  <ItemGroup>
+    <EmbeddedResource Include="dotnet.png" />
+  </ItemGroup>
+
   <ItemGroup>
     <Compile Include="..\Microsoft.Extensions.AI.Abstractions.Tests\CapturingLogger.cs" />
     <Compile Include="..\Microsoft.Extensions.AI.Abstractions.Tests\TestChatClient.cs" />
diff --git a/test/Libraries/Microsoft.Extensions.AI.Integration.Tests/dotnet.png b/test/Libraries/Microsoft.Extensions.AI.Integration.Tests/dotnet.png
new file mode 100644
index 0000000000000000000000000000000000000000..fb00ecf91e4b78804c636194bb323bf3710fa1c6
GIT binary patch
literal 2140
zcmeHIX;4#F6iz}QBp3(*v25~!F*qzP0ojoU1Wja7KpD}12oa>Xrfece2to*M4{<@1
zP%%ms&>BGn2e5_86S0m&0z?*tFd77$B7y>vzNa&tPJedV>7UNLH{U(qJ@=gNJNL&G
zZwy{XCYg~i7z~-iW`$xfSQ!0vwGifWRzWuc0UHB1`G?p&M?Q^4^TQdnylqlFJQMHV
zlMy}8cyoMm;xpH^t5o#5@y2-k+Mdkle$k#mS^4OrhKW<@s@vsbjW^%%!+QI>rn#<)
z{_bgV=B_HFEH)`LIBec*h>(fF5SlnFpG|4X(PvmP2D6~~`@Sai?5+nm4-!M>!#e`&
z78+VFVXe(SMlq!^eg7xWd6boUv<F$_l`pGWbm|T#Z^yDLv)e~l=_4yfyH}OhA9jx?
zOjg=2=TqrG4xR@I7EmD{9Fq;|5Mcmo`j3)L25#hQYZFH^)80H`mgdEmkgQ%kn(sDJ
zgZh;VwU`$us=2r;f_4VpjazLD^_&JoP2sxm?odb`GmAYRMB;ypM^R6fbPsnvXn|}c
zBfFiU6Y)(E7#Eh)YxNbpxd=&c<Jwz6J(IxnL;4?pr=_BiF|Em=%MN&btKMK%?QT+H
zq|u4J)G1*bGQAnwv2IE`Gm$8Gs*O%Nd5i;iao{CcT)-;`){3RFHyHDys+fRR2t2cj
zD6Jqxcrmgkoq*OUG{PRVD)DxKBs!{LrRNvkd*DJV9VNbv^_`I0;yUC4&z`#EK%Dv-
zsH{GKU;?-|=J9jqui5QhW;Zf1YBi{=O+faDK9bjGk-|3kgv7_i_>M4P&8=5T0wgi!
zp<$6Qus?iG)+d;(0n8C!5))oCLL24Oym*Gn(Wz_qt3FCV776EaQ0@8?Z;*X?j^|Sm
zn!Z?CE$ZK^Bel^@7%D^$=pOWDB0`L52FJhnR;@@*4VX&YV4t-$6D&!6S(50;t4kpO
zSu@uc7lG;JP{4;;gTKszM|=8RA1W@lCalj952n{cQ<2H{5Ho%XwCWD_{Z4eMJK>$z
zk!wqDgEouoAc>PS^6@?wtqyq}w+)qQ-k}&oy@>2Jq0>$(&(2yz!kQ$B-aeP1JVAwi
zLdDN$6HyHca*;h+GLwCO>;wmB&<D;Xyfek2i!qrCVklkEYl*aq1ji8t4a<Q&Pc%Fo
zX%|6yya%z}MO1ltAX)C&HDTkMd%*MtFb=u2s1?wZ#1uK!2Pz%VTb{{okD9%wRo7`C
zjbz47km2eRfE>|!(*}Njk+((AJ3{PGOd<ohTQ96x<;I8yp3DdXPd*~DZ#N=|e0)ch
zBfN7xI|G}bFy&3`AKukBLS}xjUD{Z?H<?h^4qhG6?+6Dpw^Ip~<mQbOf71w=t$Ak)
z=k1*4T#6xS={xOSp7D>x_ow6Z(=O$zl0DfhA9qWN%*0384ZX-%PdlCVsU;^WuLWS6
zFXEf8|L9P?|BxkhK<E@gYpof2#>eqYq++65zuqrIXB_=k7X0>One(W+8@jAr|KOqE
zA{8rpb%xTK*LyH?wH7r3VwmvsXBR83fP-{+TLwEzY04n+9E2=$_+^Svj1xydpdM_|
zOAWL@lsIQ|g;s%d*cN6$i7RV!TL3t`Nippk%z?;2>&ui}v9{Qry++zPRCad)IAFrl
zVrN|@85!_xg;Kwn%BW~--xx!>Wk=-d&Bet~zEI_-Jkz#l_a=BNo+EN{T-sGgLK>2R
zTl1?679M`Te?VAqflxs-Xlp(?+z^mfcBjCASoPc0Eo+$1Ukv);wnZsCeH&`cr6V+-
zlv98Q2P&n*!Bn0NQC5WS;Rxq;oYdzbZwp4})3$-jUb&QY)+bmNVpr+``XIZdFn@{R
z-y&lERCUNMmld3Uk>W<<`>Kw>#6lx$oBxuCIFtk;kF{VWJb!KMr(kW0zjXo2SiFoH
kLN8~t3iGWE|8=58=$f)MBl>~e*<^I~9RFa}4c}<t@AoBRN&o-=

literal 0
HcmV?d00001

diff --git a/test/Libraries/Microsoft.Extensions.AI.Ollama.Tests/OllamaChatClientIntegrationTests.cs b/test/Libraries/Microsoft.Extensions.AI.Ollama.Tests/OllamaChatClientIntegrationTests.cs
index 891378c0e86..ac941623124 100644
--- a/test/Libraries/Microsoft.Extensions.AI.Ollama.Tests/OllamaChatClientIntegrationTests.cs
+++ b/test/Libraries/Microsoft.Extensions.AI.Ollama.Tests/OllamaChatClientIntegrationTests.cs
@@ -30,6 +30,8 @@ public override Task FunctionInvocation_RequireAny() =>
     public override Task FunctionInvocation_RequireSpecific() =>
         throw new SkipTestException("Ollama does not currently support requiring function invocation.");
 
+    protected override string? GetModel_MultiModal_DescribeImage() => "llava";
+
     [ConditionalFact]
     public async Task PromptBasedFunctionCalling_NoArgs()
     {