Merge branch 'main' into dbohdanov/metadata-resolver-api

Author: rainsxng

.nuget/nuget.exe

@@ -168,5 +168,8 @@
       Starting with 3.0.0, xunit.runner.visualstudio only supports net472, but we target net462
      -->
     <XUnitRunnerVisualStudioVersion>2.8.2</XUnitRunnerVisualStudioVersion>
+    <!-- MEVD is still part of the Semantic Kernel repo -->
+    <MicrosoftExtensionsVectorDataAbstractionsVersion>9.7.0</MicrosoftExtensionsVectorDataAbstractionsVersion>
+    <MicrosoftSemanticKernelConnectorsVersion>1.66.0-preview</MicrosoftSemanticKernelConnectorsVersion>
   </PropertyGroup>
 </Project>

eng/Versions.props

@@ -13,6 +13,7 @@
     <PackageVersion Include="Microsoft.CodeAnalysis.CSharp.Workspaces" Version="$(MicrosoftCodeAnalysisVersion)" />
     <PackageVersion Include="Microsoft.CodeAnalysis.CSharp" Version="$(MicrosoftCodeAnalysisVersion)" />
     <PackageVersion Include="Microsoft.CodeAnalysis" Version="$(MicrosoftCodeAnalysisVersion)" />
+    <PackageVersion Include="Microsoft.Extensions.VectorData.Abstractions" Version="$(MicrosoftExtensionsVectorDataAbstractionsVersion)" />
     <PackageVersion Include="Microsoft.IO.RecyclableMemoryStream" Version="3.0.0" />
     <PackageVersion Include="Microsoft.ML.Tokenizers" Version="$(MicrosoftMLTokenizersVersion)" />
     <PackageVersion Include="Newtonsoft.Json" Version="13.0.3" />

eng/packages/General.props

@@ -12,6 +12,8 @@
     <PackageVersion Include="Microsoft.Extensions.Configuration.EnvironmentVariables" Version="9.0.0" />
     <PackageVersion Include="Microsoft.Extensions.Configuration.UserSecrets" Version="9.0.0" />
     <PackageVersion Include="Microsoft.ML.Tokenizers.Data.O200kBase" Version="$(MicrosoftMLTokenizersVersion)" />
+    <PackageVersion Include="Microsoft.SemanticKernel.Connectors.InMemory" Version="$(MicrosoftSemanticKernelConnectorsVersion)" />
+    <PackageVersion Include="Microsoft.SemanticKernel.Connectors.SqliteVec" Version="$(MicrosoftSemanticKernelConnectorsVersion)" />
     <PackageVersion Include="Microsoft.TemplateEngine.Authoring.TemplateVerifier" Version="9.0.201" />
     <PackageVersion Include="Microsoft.TemplateEngine.TestHelper" Version="9.0.200-rtm.25066.4" />
     <PackageVersion Include="Moq.AutoMock" Version="3.1.0" />

eng/packages/TestOnly.props

@@ -0,0 +1,14 @@
+-SourceToolsList @("policheck","credscan")
+-ArtifactToolsList @("binskim")
+-TsaInstanceURL https://devdiv.visualstudio.com/
+-TsaProjectName DEVDIV
+-TsaNotificationEmail [email protected]
+-TsaCodebaseAdmin REDMOND\aspnetcore-build
+-TsaBugAreaPath "DevDiv\ASP.NET Core\Policy Violations"
+-TsaIterationPath DevDiv
+-TsaRepositoryName dotnetextensions
+-TsaCodebaseName dotnetextensions
+-TsaOnboard $True
+-TsaPublish $True
+-PoliCheckAdditionalRunConfigParams @("UserExclusionPath < $(Build.SourcesDirectory)/.config/PoliCheckExclusions.xml")
+-CrScanAdditionalRunConfigParams @("SuppressionsPath < $(Build.SourcesDirectory)/.config/CredScanSuppressions.json")

eng/sdl-tsa-vars.config

@@ -2,8 +2,15 @@
 
 ## NOT YET RELEASED
 
+- Updated `AIFunctionFactory` to respect `[DisplayName(...)]` on functions as a way to override the function name.
+- Updated `AIFunctionFactory` to respect `[DefaultValue(...)]` on function parameters as a way to specify default values.
+
+## 9.10.1
+
 - Updated `HostedMcpServerTool` to allow for non-`Uri` server addresses, in order to enable built-in names.
 - Updated `HostedMcpServerTool` to replace the header collection with an `AuthorizationToken` property.
+- Fixed `ToChatResponse{Async}` to not discard `TextReasoningContent.ProtectedData` when coalescing messages.
+- Fixed `AIFunctionFactory.Create` to special-case return types of `AIContent` and `IEnumerable<AIContent>` to not automatically JSON serialize them.
 
 ## 9.10.0
 

src/Libraries/Microsoft.Extensions.AI.Abstractions/CHANGELOG.md

@@ -5,7 +5,11 @@
 using System.Collections.Generic;
 using System.Diagnostics;
 using System.Diagnostics.CodeAnalysis;
+using System.IO;
 using System.Linq;
+#if !NET
+using System.Runtime.InteropServices;
+#endif
 using System.Text;
 using System.Threading;
 using System.Threading.Tasks;
@@ -181,7 +185,7 @@ static async Task<ChatResponse> ToChatResponseAsync(
     }
 
     /// <summary>Coalesces sequential <see cref="AIContent"/> content elements.</summary>
-    internal static void CoalesceTextContent(IList<AIContent> contents)
+    internal static void CoalesceContent(IList<AIContent> contents)
     {
         Coalesce<TextContent>(
             contents,
@@ -215,6 +219,110 @@ internal static void CoalesceTextContent(IList<AIContent> contents)
                 return content;
             });
 
+        Coalesce<DataContent>(
+            contents,
+            mergeSingle: false,
+            canMerge: static (r1, r2) => r1.MediaType == r2.MediaType && r1.HasTopLevelMediaType("text") && r1.Name == r2.Name,
+            static (contents, start, end) =>
+            {
+                Debug.Assert(end - start > 1, "Expected multiple contents to merge");
+
+                MemoryStream ms = new();
+                for (int i = start; i < end; i++)
+                {
+                    var current = (DataContent)contents[i];
+#if NET
+                    ms.Write(current.Data.Span);
+#else
+                    if (!MemoryMarshal.TryGetArray(current.Data, out var segment))
+                    {
+                        segment = new(current.Data.ToArray());
+                    }
+
+                    ms.Write(segment.Array!, segment.Offset, segment.Count);
+#endif
+                }
+
+                var first = (DataContent)contents[start];
+                return new DataContent(new ReadOnlyMemory<byte>(ms.GetBuffer(), 0, (int)ms.Length), first.MediaType) { Name = first.Name };
+            });
+
+        Coalesce<CodeInterpreterToolCallContent>(
+            contents,
+            mergeSingle: true,
+            canMerge: static (r1, r2) => r1.CallId == r2.CallId,
+            static (contents, start, end) =>
+            {
+                var firstContent = (CodeInterpreterToolCallContent)contents[start];
+
+                if (start == end - 1)
+                {
+                    if (firstContent.Inputs is not null)
+                    {
+                        CoalesceContent(firstContent.Inputs);
+                    }
+
+                    return firstContent;
+                }
+
+                List<AIContent>? inputs = null;
+
+                for (int i = start; i < end; i++)
+                {
+                    (inputs ??= []).AddRange(((CodeInterpreterToolCallContent)contents[i]).Inputs ?? []);
+                }
+
+                if (inputs is not null)
+                {
+                    CoalesceContent(inputs);
+                }
+
+                return new()
+                {
+                    CallId = firstContent.CallId,
+                    Inputs = inputs,
+                    AdditionalProperties = firstContent.AdditionalProperties?.Clone(),
+                };
+            });
+
+        Coalesce<CodeInterpreterToolResultContent>(
+            contents,
+            mergeSingle: true,
+            canMerge: static (r1, r2) => r1.CallId is not null && r2.CallId is not null && r1.CallId == r2.CallId,
+            static (contents, start, end) =>
+            {
+                var firstContent = (CodeInterpreterToolResultContent)contents[start];
+
+                if (start == end - 1)
+                {
+                    if (firstContent.Outputs is not null)
+                    {
+                        CoalesceContent(firstContent.Outputs);
+                    }
+
+                    return firstContent;
+                }
+
+                List<AIContent>? output = null;
+
+                for (int i = start; i < end; i++)
+                {
+                    (output ??= []).AddRange(((CodeInterpreterToolResultContent)contents[i]).Outputs ?? []);
+                }
+
+                if (output is not null)
+                {
+                    CoalesceContent(output);
+                }
+
+                return new()
+                {
+                    CallId = firstContent.CallId,
+                    Outputs = output,
+                    AdditionalProperties = firstContent.AdditionalProperties?.Clone(),
+                };
+            });
+
         static string MergeText(IList<AIContent> contents, int start, int end)
         {
             Debug.Assert(end - start > 1, "Expected multiple contents to merge");
@@ -318,7 +426,7 @@ private static void FinalizeResponse(ChatResponse response)
         int count = response.Messages.Count;
         for (int i = 0; i < count; i++)
         {
-            CoalesceTextContent((List<AIContent>)response.Messages[i].Contents);
+            CoalesceContent((List<AIContent>)response.Messages[i].Contents);
         }
     }
 

src/Libraries/Microsoft.Extensions.AI.Abstractions/ChatCompletion/ChatResponseExtensions.cs

@@ -30,6 +30,8 @@ namespace Microsoft.Extensions.AI;
 // [JsonDerivedType(typeof(McpServerToolResultContent), typeDiscriminator: "mcpServerToolResult")]
 // [JsonDerivedType(typeof(McpServerToolApprovalRequestContent), typeDiscriminator: "mcpServerToolApprovalRequest")]
 // [JsonDerivedType(typeof(McpServerToolApprovalResponseContent), typeDiscriminator: "mcpServerToolApprovalResponse")]
+// [JsonDerivedType(typeof(CodeInterpreterToolCallContent), typeDiscriminator: "codeInterpreterToolCall")]
+// [JsonDerivedType(typeof(CodeInterpreterToolResultContent), typeDiscriminator: "codeInterpreterToolResult")]
 
 public class AIContent
 {

src/Libraries/Microsoft.Extensions.AI.Abstractions/Contents/AIContent.cs

@@ -0,0 +1,41 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using System.Collections.Generic;
+using System.Diagnostics.CodeAnalysis;
+
+namespace Microsoft.Extensions.AI;
+
+/// <summary>
+/// Represents a code interpreter tool call invocation by a hosted service.
+/// </summary>
+/// <remarks>
+/// This content type represents when a hosted AI service invokes a code interpreter tool.
+/// It is informational only and represents the call itself, not the result.
+/// </remarks>
+[Experimental("MEAI001")]
+public sealed class CodeInterpreterToolCallContent : AIContent
+{
+    /// <summary>
+    /// Initializes a new instance of the <see cref="CodeInterpreterToolCallContent"/> class.
+    /// </summary>
+    public CodeInterpreterToolCallContent()
+    {
+    }
+
+    /// <summary>
+    /// Gets or sets the tool call ID.
+    /// </summary>
+    public string? CallId { get; set; }
+
+    /// <summary>
+    /// Gets or sets the inputs to the code interpreter tool.
+    /// </summary>
+    /// <remarks>
+    /// Inputs can include various types of content such as <see cref="HostedFileContent"/> for files,
+    /// <see cref="DataContent"/> for binary data, or other <see cref="AIContent"/> types as supported
+    /// by the service. Typically <see cref="Inputs"/> includes a <see cref="DataContent"/> with a "text/x-python"
+    /// media type representing the code for execution by the code interpreter tool.
+    /// </remarks>
+    public IList<AIContent>? Inputs { get; set; }
+}

src/Libraries/Microsoft.Extensions.AI.Abstractions/Contents/CodeInterpreterToolCallContent.cs

@@ -0,0 +1,36 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using System.Collections.Generic;
+using System.Diagnostics.CodeAnalysis;
+
+namespace Microsoft.Extensions.AI;
+
+/// <summary>
+/// Represents the result of a code interpreter tool invocation by a hosted service.
+/// </summary>
+[Experimental("MEAI001")]
+public sealed class CodeInterpreterToolResultContent : AIContent
+{
+    /// <summary>
+    /// Initializes a new instance of the <see cref="CodeInterpreterToolResultContent"/> class.
+    /// </summary>
+    public CodeInterpreterToolResultContent()
+    {
+    }
+
+    /// <summary>
+    /// Gets or sets the tool call ID that this result corresponds to.
+    /// </summary>
+    public string? CallId { get; set; }
+
+    /// <summary>
+    /// Gets or sets the output of code interpreter tool.
+    /// </summary>
+    /// <remarks>
+    /// Outputs can include various types of content such as <see cref="HostedFileContent"/> for files,
+    /// <see cref="DataContent"/> for binary data, <see cref="TextContent"/> for standard output text,
+    /// or other <see cref="AIContent"/> types as supported by the service.
+    /// </remarks>
+    public IList<AIContent>? Outputs { get; set; }
+}