From 003b8a7dd54557d2abf919439d66323dc0eab5ab Mon Sep 17 00:00:00 2001 From: Prachi Gandhi Date: Fri, 30 May 2025 17:13:03 -0400 Subject: [PATCH 1/6] eval target typespec --- .../Azure.AI.Projects/evaluations/models.tsp | 75 +++++++++ .../2025-05-15-preview/azure-ai-projects.json | 142 ++++++++++++++++++ .../stable/v1/azure-ai-projects.json | 138 +++++++++++++++++ 3 files changed, 355 insertions(+) diff --git a/specification/ai/Azure.AI.Projects/evaluations/models.tsp b/specification/ai/Azure.AI.Projects/evaluations/models.tsp index 1819c1b83bc0..61369e49dc03 100644 --- a/specification/ai/Azure.AI.Projects/evaluations/models.tsp +++ b/specification/ai/Azure.AI.Projects/evaluations/models.tsp @@ -78,6 +78,9 @@ model Evaluation { @doc("Evaluators to be used for the evaluation.") evaluators: Record; + + @doc("Evaluation target specifies the type and configuration of the target to be used for this evaluation.") + target?: EvaluationTarget; } @doc("Definition for sampling strategy.") @@ -176,3 +179,75 @@ model AgentEvaluation { @doc("The agent evaluation result.") result?: Array; } + +@doc("Abstract base model representing a single message in a conversation.") +@discriminator("role") +@added(Versions.v2025_05_15_preview) +model Message { + @doc("The role of the message author: 'system', 'assistant', 'developer', or 'user'.") + role: string; +} + +@doc("A message authored by the system to guide model behavior.") +@added(Versions.v2025_05_15_preview) +model SystemMessage extends Message { + @doc("Indicates this is a system message.") + role: "system"; + + @doc("Plain text instructions provided by the system to steer model behavior.") + content: string; +} + +@doc("A message authored by a developer to guide the model during evaluation.") +@added(Versions.v2025_05_15_preview) +model DeveloperMessage extends Message { + @doc("Indicates this is a developer message.") + role: "developer"; + + @doc("Content provided by a developer to guide model behavior in an evaluation context.") + content: string; +} + +@doc("A message authored by the end user as input to the model.") +@added(Versions.v2025_05_15_preview) +model UserMessage extends Message { + @doc("Indicates this is a user message.") + role: "user"; + + @doc("Input content or question provided by the end user.") + content: string; +} + +@doc("A message generated by the assistant in response to previous messages.") +@added(Versions.v2025_05_15_preview) +model AssistantMessage extends Message { + @doc("Indicates this is an assistant message.") + role: "assistant"; + + @doc("Response content generated by the assistant.") + content: string; +} + +@doc("Abstract base model for defining evaluation targets.") +@discriminator("type") +@added(Versions.v2025_05_15_preview) +model EvaluationTarget { + @doc("Discriminator that defines the type of the evaluation target.") + type: string; +} + +@doc("Evaluation target that uses the chat completion API.") +@added(Versions.v2025_05_15_preview) +model ChatCompletionTarget extends EvaluationTarget { + @doc("The type of evaluation target. Always 'chatCompletion'.") + type: "chatCompletion"; + + @doc("A list of messages comprising the conversation so far.") + messages: Message[]; + + @doc("The name of the model deployment to be evaluated.") + modelDeploymentName: string; + + @doc("Optional parameters passed to the model for evaluation.") + modelParams: Record; +} diff --git a/specification/ai/data-plane/Azure.AI.Projects/preview/2025-05-15-preview/azure-ai-projects.json b/specification/ai/data-plane/Azure.AI.Projects/preview/2025-05-15-preview/azure-ai-projects.json index cfccd5ccbb5c..b4f47ffc07ae 100644 --- a/specification/ai/data-plane/Azure.AI.Projects/preview/2025-05-15-preview/azure-ai-projects.json +++ b/specification/ai/data-plane/Azure.AI.Projects/preview/2025-05-15-preview/azure-ai-projects.json @@ -1695,6 +1695,25 @@ "blobReference" ] }, + "AssistantMessage": { + "type": "object", + "description": "A message generated by the assistant in response to previous messages.", + "properties": { + "content": { + "type": "string", + "description": "Response content generated by the assistant." + } + }, + "required": [ + "content" + ], + "allOf": [ + { + "$ref": "#/definitions/Message" + } + ], + "x-ms-discriminator-value": "assistant" + }, "AttackStrategy": { "type": "string", "description": "Strategies for attacks.", @@ -2005,6 +2024,40 @@ "credential" ] }, + "ChatCompletionTarget": { + "type": "object", + "description": "Evaluation target that uses the chat completion API.", + "properties": { + "messages": { + "type": "array", + "description": "A list of messages comprising the conversation so far.", + "items": { + "$ref": "#/definitions/Message" + }, + "x-ms-identifiers": [] + }, + "modelDeploymentName": { + "type": "string", + "description": "The name of the model deployment to be evaluated." + }, + "modelParams": { + "type": "object", + "description": "Optional parameters passed to the model for evaluation.", + "additionalProperties": {} + } + }, + "required": [ + "messages", + "modelDeploymentName", + "modelParams" + ], + "allOf": [ + { + "$ref": "#/definitions/EvaluationTarget" + } + ], + "x-ms-discriminator-value": "chatCompletion" + }, "Connection": { "type": "object", "description": "Response from the list and get connections operations", @@ -2378,6 +2431,25 @@ ] } }, + "DeveloperMessage": { + "type": "object", + "description": "A message authored by a developer to guide the model during evaluation.", + "properties": { + "content": { + "type": "string", + "description": "Content provided by a developer to guide model behavior in an evaluation context." + } + }, + "required": [ + "content" + ], + "allOf": [ + { + "$ref": "#/definitions/Message" + } + ], + "x-ms-discriminator-value": "developer" + }, "EmbeddingConfiguration": { "type": "object", "description": "Embedding configuration class", @@ -2459,6 +2531,10 @@ "additionalProperties": { "$ref": "#/definitions/EvaluatorConfiguration" } + }, + "target": { + "$ref": "#/definitions/EvaluationTarget", + "description": "Evaluation target specifies the type and configuration of the target to be used for this evaluation." } }, "required": [ @@ -2467,6 +2543,20 @@ "evaluators" ] }, + "EvaluationTarget": { + "type": "object", + "description": "Abstract base model for defining evaluation targets.", + "properties": { + "type": { + "type": "string", + "description": "Discriminator that defines the type of the evaluation target." + } + }, + "discriminator": "type", + "required": [ + "type" + ] + }, "EvaluatorConfiguration": { "type": "object", "description": "Evaluator Configuration", @@ -2706,6 +2796,20 @@ ], "x-ms-discriminator-value": "ManagedAzureSearch" }, + "Message": { + "type": "object", + "description": "Abstract base model representing a single message in a conversation.", + "properties": { + "role": { + "type": "string", + "description": "The role of the message author: 'system', 'assistant', 'developer', or 'user'." + } + }, + "discriminator": "role", + "required": [ + "role" + ] + }, "ModelDeployment": { "type": "object", "description": "Model Deployment Definition", @@ -3181,6 +3285,25 @@ "tier" ] }, + "SystemMessage": { + "type": "object", + "description": "A message authored by the system to guide model behavior.", + "properties": { + "content": { + "type": "string", + "description": "Plain text instructions provided by the system to steer model behavior." + } + }, + "required": [ + "content" + ], + "allOf": [ + { + "$ref": "#/definitions/Message" + } + ], + "x-ms-discriminator-value": "system" + }, "TargetConfig": { "type": "object", "description": "Abstract class for target configuration.", @@ -3194,6 +3317,25 @@ "required": [ "type" ] + }, + "UserMessage": { + "type": "object", + "description": "A message authored by the end user as input to the model.", + "properties": { + "content": { + "type": "string", + "description": "Input content or question provided by the end user." + } + }, + "required": [ + "content" + ], + "allOf": [ + { + "$ref": "#/definitions/Message" + } + ], + "x-ms-discriminator-value": "user" } }, "parameters": { diff --git a/specification/ai/data-plane/Azure.AI.Projects/stable/v1/azure-ai-projects.json b/specification/ai/data-plane/Azure.AI.Projects/stable/v1/azure-ai-projects.json index f0591f750a4d..17fcc719c7bf 100644 --- a/specification/ai/data-plane/Azure.AI.Projects/stable/v1/azure-ai-projects.json +++ b/specification/ai/data-plane/Azure.AI.Projects/stable/v1/azure-ai-projects.json @@ -1090,6 +1090,25 @@ "blobReference" ] }, + "AssistantMessage": { + "type": "object", + "description": "A message generated by the assistant in response to previous messages.", + "properties": { + "content": { + "type": "string", + "description": "Response content generated by the assistant." + } + }, + "required": [ + "content" + ], + "allOf": [ + { + "$ref": "#/definitions/Message" + } + ], + "x-ms-discriminator-value": "assistant" + }, "Azure.Core.Foundations.Error": { "type": "object", "description": "The error object.", @@ -1225,6 +1244,40 @@ "credential" ] }, + "ChatCompletionTarget": { + "type": "object", + "description": "Evaluation target that uses the chat completion API.", + "properties": { + "messages": { + "type": "array", + "description": "A list of messages comprising the conversation so far.", + "items": { + "$ref": "#/definitions/Message" + }, + "x-ms-identifiers": [] + }, + "modelDeploymentName": { + "type": "string", + "description": "The name of the model deployment to be evaluated." + }, + "modelParams": { + "type": "object", + "description": "Optional parameters passed to the model for evaluation.", + "additionalProperties": {} + } + }, + "required": [ + "messages", + "modelDeploymentName", + "modelParams" + ], + "allOf": [ + { + "$ref": "#/definitions/EvaluationTarget" + } + ], + "x-ms-discriminator-value": "chatCompletion" + }, "Connection": { "type": "object", "description": "Response from the list and get connections operations", @@ -1598,6 +1651,25 @@ ] } }, + "DeveloperMessage": { + "type": "object", + "description": "A message authored by a developer to guide the model during evaluation.", + "properties": { + "content": { + "type": "string", + "description": "Content provided by a developer to guide model behavior in an evaluation context." + } + }, + "required": [ + "content" + ], + "allOf": [ + { + "$ref": "#/definitions/Message" + } + ], + "x-ms-discriminator-value": "developer" + }, "EmbeddingConfiguration": { "type": "object", "description": "Embedding configuration class", @@ -1632,6 +1704,20 @@ ], "x-ms-discriminator-value": "AAD" }, + "EvaluationTarget": { + "type": "object", + "description": "Abstract base model for defining evaluation targets.", + "properties": { + "type": { + "type": "string", + "description": "Discriminator that defines the type of the evaluation target." + } + }, + "discriminator": "type", + "required": [ + "type" + ] + }, "FieldMapping": { "type": "object", "description": "Field mapping configuration class", @@ -1813,6 +1899,20 @@ ], "x-ms-discriminator-value": "ManagedAzureSearch" }, + "Message": { + "type": "object", + "description": "Abstract base model representing a single message in a conversation.", + "properties": { + "role": { + "type": "string", + "description": "The role of the message author: 'system', 'assistant', 'developer', or 'user'." + } + }, + "discriminator": "role", + "required": [ + "role" + ] + }, "ModelDeployment": { "type": "object", "description": "Model Deployment Definition", @@ -2138,6 +2238,44 @@ "size", "tier" ] + }, + "SystemMessage": { + "type": "object", + "description": "A message authored by the system to guide model behavior.", + "properties": { + "content": { + "type": "string", + "description": "Plain text instructions provided by the system to steer model behavior." + } + }, + "required": [ + "content" + ], + "allOf": [ + { + "$ref": "#/definitions/Message" + } + ], + "x-ms-discriminator-value": "system" + }, + "UserMessage": { + "type": "object", + "description": "A message authored by the end user as input to the model.", + "properties": { + "content": { + "type": "string", + "description": "Input content or question provided by the end user." + } + }, + "required": [ + "content" + ], + "allOf": [ + { + "$ref": "#/definitions/Message" + } + ], + "x-ms-discriminator-value": "user" } }, "parameters": { From 355a6551a4e0660b36ca341fe7da19ad82f441fb Mon Sep 17 00:00:00 2001 From: Prachi Gandhi Date: Thu, 12 Jun 2025 18:54:02 -0400 Subject: [PATCH 2/6] review changes --- .../Azure.AI.Projects/evaluations/models.tsp | 38 +-- .../2025-05-15-preview/azure-ai-projects.json | 255 ++++++++++-------- .../stable/v1/azure-ai-projects.json | 253 +++++++++-------- 3 files changed, 304 insertions(+), 242 deletions(-) diff --git a/specification/ai/Azure.AI.Projects/evaluations/models.tsp b/specification/ai/Azure.AI.Projects/evaluations/models.tsp index 61369e49dc03..33ee847bc521 100644 --- a/specification/ai/Azure.AI.Projects/evaluations/models.tsp +++ b/specification/ai/Azure.AI.Projects/evaluations/models.tsp @@ -79,7 +79,7 @@ model Evaluation { @doc("Evaluators to be used for the evaluation.") evaluators: Record; - @doc("Evaluation target specifies the type and configuration of the target to be used for this evaluation.") + @doc("Specifies the type and configuration of the entity used for this evaluation.") target?: EvaluationTarget; } @@ -183,14 +183,14 @@ model AgentEvaluation { @doc("Abstract base model representing a single message in a conversation.") @discriminator("role") @added(Versions.v2025_05_15_preview) -model Message { - @doc("The role of the message author: 'system', 'assistant', 'developer', or 'user'.") - role: string; +model EvaluationMessage { + @doc("The role of the message author. Known values: 'system', 'assistant', 'developer', 'user'.") + role: "system" | "assistant" | "developer" | "user" | string; } @doc("A message authored by the system to guide model behavior.") @added(Versions.v2025_05_15_preview) -model SystemMessage extends Message { +model EvaluationSystemMessage extends EvaluationMessage { @doc("Indicates this is a system message.") role: "system"; @@ -200,7 +200,7 @@ model SystemMessage extends Message { @doc("A message authored by a developer to guide the model during evaluation.") @added(Versions.v2025_05_15_preview) -model DeveloperMessage extends Message { +model EvaluationDeveloperMessage extends EvaluationMessage { @doc("Indicates this is a developer message.") role: "developer"; @@ -210,7 +210,7 @@ model DeveloperMessage extends Message { @doc("A message authored by the end user as input to the model.") @added(Versions.v2025_05_15_preview) -model UserMessage extends Message { +model EvaluationUserMessage extends EvaluationMessage { @doc("Indicates this is a user message.") role: "user"; @@ -220,7 +220,7 @@ model UserMessage extends Message { @doc("A message generated by the assistant in response to previous messages.") @added(Versions.v2025_05_15_preview) -model AssistantMessage extends Message { +model EvaluationAssistantMessage extends EvaluationMessage { @doc("Indicates this is an assistant message.") role: "assistant"; @@ -228,24 +228,32 @@ model AssistantMessage extends Message { content: string; } +@doc("Allowed types of evaluation targets.") +@added(Versions.v2025_05_15_preview) +union EvaluationTargetType { + @doc("Evaluation target that uses a model and prompt.") + modelPrompt: "modelPrompt"; + string; +} + @doc("Abstract base model for defining evaluation targets.") @discriminator("type") @added(Versions.v2025_05_15_preview) model EvaluationTarget { @doc("Discriminator that defines the type of the evaluation target.") - type: string; + type: EvaluationTargetType; } -@doc("Evaluation target that uses the chat completion API.") +@doc("Evaluation target that uses the chat completion API to generate responses.") @added(Versions.v2025_05_15_preview) -model ChatCompletionTarget extends EvaluationTarget { - @doc("The type of evaluation target. Always 'chatCompletion'.") - type: "chatCompletion"; +model modelPromptEvaluationTarget extends EvaluationTarget { + @doc("The type of evaluation target. Always 'modelPrompt'.") + type: "modelPrompt"; @doc("A list of messages comprising the conversation so far.") - messages: Message[]; + messages: EvaluationMessage[]; - @doc("The name of the model deployment to be evaluated.") + @doc("The model deployment to be evaluated. Accepts either the deployment name alone or with the connection name as '{connectionName}/modelDeploymentName'.") modelDeploymentName: string; @doc("Optional parameters passed to the model for evaluation.") diff --git a/specification/ai/data-plane/Azure.AI.Projects/preview/2025-05-15-preview/azure-ai-projects.json b/specification/ai/data-plane/Azure.AI.Projects/preview/2025-05-15-preview/azure-ai-projects.json index b4f47ffc07ae..eb95b7617565 100644 --- a/specification/ai/data-plane/Azure.AI.Projects/preview/2025-05-15-preview/azure-ai-projects.json +++ b/specification/ai/data-plane/Azure.AI.Projects/preview/2025-05-15-preview/azure-ai-projects.json @@ -1695,25 +1695,6 @@ "blobReference" ] }, - "AssistantMessage": { - "type": "object", - "description": "A message generated by the assistant in response to previous messages.", - "properties": { - "content": { - "type": "string", - "description": "Response content generated by the assistant." - } - }, - "required": [ - "content" - ], - "allOf": [ - { - "$ref": "#/definitions/Message" - } - ], - "x-ms-discriminator-value": "assistant" - }, "AttackStrategy": { "type": "string", "description": "Strategies for attacks.", @@ -2024,40 +2005,6 @@ "credential" ] }, - "ChatCompletionTarget": { - "type": "object", - "description": "Evaluation target that uses the chat completion API.", - "properties": { - "messages": { - "type": "array", - "description": "A list of messages comprising the conversation so far.", - "items": { - "$ref": "#/definitions/Message" - }, - "x-ms-identifiers": [] - }, - "modelDeploymentName": { - "type": "string", - "description": "The name of the model deployment to be evaluated." - }, - "modelParams": { - "type": "object", - "description": "Optional parameters passed to the model for evaluation.", - "additionalProperties": {} - } - }, - "required": [ - "messages", - "modelDeploymentName", - "modelParams" - ], - "allOf": [ - { - "$ref": "#/definitions/EvaluationTarget" - } - ], - "x-ms-discriminator-value": "chatCompletion" - }, "Connection": { "type": "object", "description": "Response from the list and get connections operations", @@ -2431,25 +2378,6 @@ ] } }, - "DeveloperMessage": { - "type": "object", - "description": "A message authored by a developer to guide the model during evaluation.", - "properties": { - "content": { - "type": "string", - "description": "Content provided by a developer to guide model behavior in an evaluation context." - } - }, - "required": [ - "content" - ], - "allOf": [ - { - "$ref": "#/definitions/Message" - } - ], - "x-ms-discriminator-value": "developer" - }, "EmbeddingConfiguration": { "type": "object", "description": "Embedding configuration class", @@ -2534,7 +2462,7 @@ }, "target": { "$ref": "#/definitions/EvaluationTarget", - "description": "Evaluation target specifies the type and configuration of the target to be used for this evaluation." + "description": "Specifies the type and configuration of the entity used for this evaluation." } }, "required": [ @@ -2543,12 +2471,92 @@ "evaluators" ] }, + "EvaluationAssistantMessage": { + "type": "object", + "description": "A message generated by the assistant in response to previous messages.", + "properties": { + "content": { + "type": "string", + "description": "Response content generated by the assistant." + } + }, + "required": [ + "content" + ], + "allOf": [ + { + "$ref": "#/definitions/EvaluationMessage" + } + ], + "x-ms-discriminator-value": "assistant" + }, + "EvaluationDeveloperMessage": { + "type": "object", + "description": "A message authored by a developer to guide the model during evaluation.", + "properties": { + "content": { + "type": "string", + "description": "Content provided by a developer to guide model behavior in an evaluation context." + } + }, + "required": [ + "content" + ], + "allOf": [ + { + "$ref": "#/definitions/EvaluationMessage" + } + ], + "x-ms-discriminator-value": "developer" + }, + "EvaluationMessage": { + "type": "object", + "description": "Abstract base model representing a single message in a conversation.", + "properties": { + "role": { + "type": "string", + "description": "The role of the message author. Known values: 'system', 'assistant', 'developer', 'user'.", + "enum": [ + "system", + "assistant", + "developer", + "user" + ], + "x-ms-enum": { + "modelAsString": true + } + } + }, + "discriminator": "role", + "required": [ + "role" + ] + }, + "EvaluationSystemMessage": { + "type": "object", + "description": "A message authored by the system to guide model behavior.", + "properties": { + "content": { + "type": "string", + "description": "Plain text instructions provided by the system to steer model behavior." + } + }, + "required": [ + "content" + ], + "allOf": [ + { + "$ref": "#/definitions/EvaluationMessage" + } + ], + "x-ms-discriminator-value": "system" + }, "EvaluationTarget": { "type": "object", "description": "Abstract base model for defining evaluation targets.", "properties": { "type": { - "type": "string", + "$ref": "#/definitions/EvaluationTargetType", "description": "Discriminator that defines the type of the evaluation target." } }, @@ -2557,6 +2565,43 @@ "type" ] }, + "EvaluationTargetType": { + "type": "string", + "description": "Allowed types of evaluation targets.", + "enum": [ + "modelPrompt" + ], + "x-ms-enum": { + "name": "EvaluationTargetType", + "modelAsString": true, + "values": [ + { + "name": "modelPrompt", + "value": "modelPrompt", + "description": "Evaluation target that uses a model and prompt." + } + ] + } + }, + "EvaluationUserMessage": { + "type": "object", + "description": "A message authored by the end user as input to the model.", + "properties": { + "content": { + "type": "string", + "description": "Input content or question provided by the end user." + } + }, + "required": [ + "content" + ], + "allOf": [ + { + "$ref": "#/definitions/EvaluationMessage" + } + ], + "x-ms-discriminator-value": "user" + }, "EvaluatorConfiguration": { "type": "object", "description": "Evaluator Configuration", @@ -2796,20 +2841,6 @@ ], "x-ms-discriminator-value": "ManagedAzureSearch" }, - "Message": { - "type": "object", - "description": "Abstract base model representing a single message in a conversation.", - "properties": { - "role": { - "type": "string", - "description": "The role of the message author: 'system', 'assistant', 'developer', or 'user'." - } - }, - "discriminator": "role", - "required": [ - "role" - ] - }, "ModelDeployment": { "type": "object", "description": "Model Deployment Definition", @@ -3285,25 +3316,6 @@ "tier" ] }, - "SystemMessage": { - "type": "object", - "description": "A message authored by the system to guide model behavior.", - "properties": { - "content": { - "type": "string", - "description": "Plain text instructions provided by the system to steer model behavior." - } - }, - "required": [ - "content" - ], - "allOf": [ - { - "$ref": "#/definitions/Message" - } - ], - "x-ms-discriminator-value": "system" - }, "TargetConfig": { "type": "object", "description": "Abstract class for target configuration.", @@ -3318,24 +3330,39 @@ "type" ] }, - "UserMessage": { + "modelPromptEvaluationTarget": { "type": "object", - "description": "A message authored by the end user as input to the model.", + "description": "Evaluation target that uses the chat completion API to generate responses.", "properties": { - "content": { + "messages": { + "type": "array", + "description": "A list of messages comprising the conversation so far.", + "items": { + "$ref": "#/definitions/EvaluationMessage" + }, + "x-ms-identifiers": [] + }, + "modelDeploymentName": { "type": "string", - "description": "Input content or question provided by the end user." + "description": "The model deployment to be evaluated. Accepts either the deployment name alone or with the connection name as '{connectionName}/modelDeploymentName'." + }, + "modelParams": { + "type": "object", + "description": "Optional parameters passed to the model for evaluation.", + "additionalProperties": {} } }, "required": [ - "content" + "messages", + "modelDeploymentName", + "modelParams" ], "allOf": [ { - "$ref": "#/definitions/Message" + "$ref": "#/definitions/EvaluationTarget" } ], - "x-ms-discriminator-value": "user" + "x-ms-discriminator-value": "modelPrompt" } }, "parameters": { diff --git a/specification/ai/data-plane/Azure.AI.Projects/stable/v1/azure-ai-projects.json b/specification/ai/data-plane/Azure.AI.Projects/stable/v1/azure-ai-projects.json index 17fcc719c7bf..39c784f9eb9a 100644 --- a/specification/ai/data-plane/Azure.AI.Projects/stable/v1/azure-ai-projects.json +++ b/specification/ai/data-plane/Azure.AI.Projects/stable/v1/azure-ai-projects.json @@ -1090,25 +1090,6 @@ "blobReference" ] }, - "AssistantMessage": { - "type": "object", - "description": "A message generated by the assistant in response to previous messages.", - "properties": { - "content": { - "type": "string", - "description": "Response content generated by the assistant." - } - }, - "required": [ - "content" - ], - "allOf": [ - { - "$ref": "#/definitions/Message" - } - ], - "x-ms-discriminator-value": "assistant" - }, "Azure.Core.Foundations.Error": { "type": "object", "description": "The error object.", @@ -1244,40 +1225,6 @@ "credential" ] }, - "ChatCompletionTarget": { - "type": "object", - "description": "Evaluation target that uses the chat completion API.", - "properties": { - "messages": { - "type": "array", - "description": "A list of messages comprising the conversation so far.", - "items": { - "$ref": "#/definitions/Message" - }, - "x-ms-identifiers": [] - }, - "modelDeploymentName": { - "type": "string", - "description": "The name of the model deployment to be evaluated." - }, - "modelParams": { - "type": "object", - "description": "Optional parameters passed to the model for evaluation.", - "additionalProperties": {} - } - }, - "required": [ - "messages", - "modelDeploymentName", - "modelParams" - ], - "allOf": [ - { - "$ref": "#/definitions/EvaluationTarget" - } - ], - "x-ms-discriminator-value": "chatCompletion" - }, "Connection": { "type": "object", "description": "Response from the list and get connections operations", @@ -1651,25 +1598,6 @@ ] } }, - "DeveloperMessage": { - "type": "object", - "description": "A message authored by a developer to guide the model during evaluation.", - "properties": { - "content": { - "type": "string", - "description": "Content provided by a developer to guide model behavior in an evaluation context." - } - }, - "required": [ - "content" - ], - "allOf": [ - { - "$ref": "#/definitions/Message" - } - ], - "x-ms-discriminator-value": "developer" - }, "EmbeddingConfiguration": { "type": "object", "description": "Embedding configuration class", @@ -1704,12 +1632,92 @@ ], "x-ms-discriminator-value": "AAD" }, + "EvaluationAssistantMessage": { + "type": "object", + "description": "A message generated by the assistant in response to previous messages.", + "properties": { + "content": { + "type": "string", + "description": "Response content generated by the assistant." + } + }, + "required": [ + "content" + ], + "allOf": [ + { + "$ref": "#/definitions/EvaluationMessage" + } + ], + "x-ms-discriminator-value": "assistant" + }, + "EvaluationDeveloperMessage": { + "type": "object", + "description": "A message authored by a developer to guide the model during evaluation.", + "properties": { + "content": { + "type": "string", + "description": "Content provided by a developer to guide model behavior in an evaluation context." + } + }, + "required": [ + "content" + ], + "allOf": [ + { + "$ref": "#/definitions/EvaluationMessage" + } + ], + "x-ms-discriminator-value": "developer" + }, + "EvaluationMessage": { + "type": "object", + "description": "Abstract base model representing a single message in a conversation.", + "properties": { + "role": { + "type": "string", + "description": "The role of the message author. Known values: 'system', 'assistant', 'developer', 'user'.", + "enum": [ + "system", + "assistant", + "developer", + "user" + ], + "x-ms-enum": { + "modelAsString": true + } + } + }, + "discriminator": "role", + "required": [ + "role" + ] + }, + "EvaluationSystemMessage": { + "type": "object", + "description": "A message authored by the system to guide model behavior.", + "properties": { + "content": { + "type": "string", + "description": "Plain text instructions provided by the system to steer model behavior." + } + }, + "required": [ + "content" + ], + "allOf": [ + { + "$ref": "#/definitions/EvaluationMessage" + } + ], + "x-ms-discriminator-value": "system" + }, "EvaluationTarget": { "type": "object", "description": "Abstract base model for defining evaluation targets.", "properties": { "type": { - "type": "string", + "$ref": "#/definitions/EvaluationTargetType", "description": "Discriminator that defines the type of the evaluation target." } }, @@ -1718,6 +1726,43 @@ "type" ] }, + "EvaluationTargetType": { + "type": "string", + "description": "Allowed types of evaluation targets.", + "enum": [ + "modelPrompt" + ], + "x-ms-enum": { + "name": "EvaluationTargetType", + "modelAsString": true, + "values": [ + { + "name": "modelPrompt", + "value": "modelPrompt", + "description": "Evaluation target that uses a model and prompt." + } + ] + } + }, + "EvaluationUserMessage": { + "type": "object", + "description": "A message authored by the end user as input to the model.", + "properties": { + "content": { + "type": "string", + "description": "Input content or question provided by the end user." + } + }, + "required": [ + "content" + ], + "allOf": [ + { + "$ref": "#/definitions/EvaluationMessage" + } + ], + "x-ms-discriminator-value": "user" + }, "FieldMapping": { "type": "object", "description": "Field mapping configuration class", @@ -1899,20 +1944,6 @@ ], "x-ms-discriminator-value": "ManagedAzureSearch" }, - "Message": { - "type": "object", - "description": "Abstract base model representing a single message in a conversation.", - "properties": { - "role": { - "type": "string", - "description": "The role of the message author: 'system', 'assistant', 'developer', or 'user'." - } - }, - "discriminator": "role", - "required": [ - "role" - ] - }, "ModelDeployment": { "type": "object", "description": "Model Deployment Definition", @@ -2239,43 +2270,39 @@ "tier" ] }, - "SystemMessage": { - "type": "object", - "description": "A message authored by the system to guide model behavior.", - "properties": { - "content": { - "type": "string", - "description": "Plain text instructions provided by the system to steer model behavior." - } - }, - "required": [ - "content" - ], - "allOf": [ - { - "$ref": "#/definitions/Message" - } - ], - "x-ms-discriminator-value": "system" - }, - "UserMessage": { + "modelPromptEvaluationTarget": { "type": "object", - "description": "A message authored by the end user as input to the model.", + "description": "Evaluation target that uses the chat completion API to generate responses.", "properties": { - "content": { + "messages": { + "type": "array", + "description": "A list of messages comprising the conversation so far.", + "items": { + "$ref": "#/definitions/EvaluationMessage" + }, + "x-ms-identifiers": [] + }, + "modelDeploymentName": { "type": "string", - "description": "Input content or question provided by the end user." + "description": "The model deployment to be evaluated. Accepts either the deployment name alone or with the connection name as '{connectionName}/modelDeploymentName'." + }, + "modelParams": { + "type": "object", + "description": "Optional parameters passed to the model for evaluation.", + "additionalProperties": {} } }, "required": [ - "content" + "messages", + "modelDeploymentName", + "modelParams" ], "allOf": [ { - "$ref": "#/definitions/Message" + "$ref": "#/definitions/EvaluationTarget" } ], - "x-ms-discriminator-value": "user" + "x-ms-discriminator-value": "modelPrompt" } }, "parameters": { From 431dfca18a3cac8e3a029b4f13463a819deaafa6 Mon Sep 17 00:00:00 2001 From: Prachi Gandhi Date: Mon, 23 Jun 2025 13:38:31 -0400 Subject: [PATCH 3/6] review comments after api review --- .../Azure.AI.Projects/evaluations/models.tsp | 24 +- .../2025-05-15-preview/azure-ai-projects.json | 218 +++++++++--------- .../stable/v1/azure-ai-projects.json | 218 +++++++++--------- 3 files changed, 230 insertions(+), 230 deletions(-) diff --git a/specification/ai/Azure.AI.Projects/evaluations/models.tsp b/specification/ai/Azure.AI.Projects/evaluations/models.tsp index 33ee847bc521..82bdc08b73d6 100644 --- a/specification/ai/Azure.AI.Projects/evaluations/models.tsp +++ b/specification/ai/Azure.AI.Projects/evaluations/models.tsp @@ -183,14 +183,14 @@ model AgentEvaluation { @doc("Abstract base model representing a single message in a conversation.") @discriminator("role") @added(Versions.v2025_05_15_preview) -model EvaluationMessage { +model Message { @doc("The role of the message author. Known values: 'system', 'assistant', 'developer', 'user'.") role: "system" | "assistant" | "developer" | "user" | string; } @doc("A message authored by the system to guide model behavior.") @added(Versions.v2025_05_15_preview) -model EvaluationSystemMessage extends EvaluationMessage { +model SystemMessage extends Message { @doc("Indicates this is a system message.") role: "system"; @@ -200,7 +200,7 @@ model EvaluationSystemMessage extends EvaluationMessage { @doc("A message authored by a developer to guide the model during evaluation.") @added(Versions.v2025_05_15_preview) -model EvaluationDeveloperMessage extends EvaluationMessage { +model DeveloperMessage extends Message { @doc("Indicates this is a developer message.") role: "developer"; @@ -210,7 +210,7 @@ model EvaluationDeveloperMessage extends EvaluationMessage { @doc("A message authored by the end user as input to the model.") @added(Versions.v2025_05_15_preview) -model EvaluationUserMessage extends EvaluationMessage { +model UserMessage extends Message { @doc("Indicates this is a user message.") role: "user"; @@ -220,7 +220,7 @@ model EvaluationUserMessage extends EvaluationMessage { @doc("A message generated by the assistant in response to previous messages.") @added(Versions.v2025_05_15_preview) -model EvaluationAssistantMessage extends EvaluationMessage { +model AssistantMessage extends Message { @doc("Indicates this is an assistant message.") role: "assistant"; @@ -231,8 +231,8 @@ model EvaluationAssistantMessage extends EvaluationMessage { @doc("Allowed types of evaluation targets.") @added(Versions.v2025_05_15_preview) union EvaluationTargetType { - @doc("Evaluation target that uses a model and prompt.") - modelPrompt: "modelPrompt"; + @doc("Evaluation target that uses a model for response generation.") + modelResponseGeneration: "modelResponseGeneration"; string; } @@ -244,14 +244,14 @@ model EvaluationTarget { type: EvaluationTargetType; } -@doc("Evaluation target that uses the chat completion API to generate responses.") +@doc("Evaluation target for generating responses using a given model and dataset.") @added(Versions.v2025_05_15_preview) -model modelPromptEvaluationTarget extends EvaluationTarget { - @doc("The type of evaluation target. Always 'modelPrompt'.") - type: "modelPrompt"; +model modelResponseGenerationTarget extends EvaluationTarget { + @doc("The type of evaluation target. Always 'modelResponseGeneration'.") + type: EvaluationTargetType.modelResponseGeneration; @doc("A list of messages comprising the conversation so far.") - messages: EvaluationMessage[]; + baseMessages: Message[]; @doc("The model deployment to be evaluated. Accepts either the deployment name alone or with the connection name as '{connectionName}/modelDeploymentName'.") modelDeploymentName: string; diff --git a/specification/ai/data-plane/Azure.AI.Projects/preview/2025-05-15-preview/azure-ai-projects.json b/specification/ai/data-plane/Azure.AI.Projects/preview/2025-05-15-preview/azure-ai-projects.json index eb95b7617565..f7d6c513196b 100644 --- a/specification/ai/data-plane/Azure.AI.Projects/preview/2025-05-15-preview/azure-ai-projects.json +++ b/specification/ai/data-plane/Azure.AI.Projects/preview/2025-05-15-preview/azure-ai-projects.json @@ -1695,6 +1695,25 @@ "blobReference" ] }, + "AssistantMessage": { + "type": "object", + "description": "A message generated by the assistant in response to previous messages.", + "properties": { + "content": { + "type": "string", + "description": "Response content generated by the assistant." + } + }, + "required": [ + "content" + ], + "allOf": [ + { + "$ref": "#/definitions/Message" + } + ], + "x-ms-discriminator-value": "assistant" + }, "AttackStrategy": { "type": "string", "description": "Strategies for attacks.", @@ -2378,6 +2397,25 @@ ] } }, + "DeveloperMessage": { + "type": "object", + "description": "A message authored by a developer to guide the model during evaluation.", + "properties": { + "content": { + "type": "string", + "description": "Content provided by a developer to guide model behavior in an evaluation context." + } + }, + "required": [ + "content" + ], + "allOf": [ + { + "$ref": "#/definitions/Message" + } + ], + "x-ms-discriminator-value": "developer" + }, "EmbeddingConfiguration": { "type": "object", "description": "Embedding configuration class", @@ -2471,86 +2509,6 @@ "evaluators" ] }, - "EvaluationAssistantMessage": { - "type": "object", - "description": "A message generated by the assistant in response to previous messages.", - "properties": { - "content": { - "type": "string", - "description": "Response content generated by the assistant." - } - }, - "required": [ - "content" - ], - "allOf": [ - { - "$ref": "#/definitions/EvaluationMessage" - } - ], - "x-ms-discriminator-value": "assistant" - }, - "EvaluationDeveloperMessage": { - "type": "object", - "description": "A message authored by a developer to guide the model during evaluation.", - "properties": { - "content": { - "type": "string", - "description": "Content provided by a developer to guide model behavior in an evaluation context." - } - }, - "required": [ - "content" - ], - "allOf": [ - { - "$ref": "#/definitions/EvaluationMessage" - } - ], - "x-ms-discriminator-value": "developer" - }, - "EvaluationMessage": { - "type": "object", - "description": "Abstract base model representing a single message in a conversation.", - "properties": { - "role": { - "type": "string", - "description": "The role of the message author. Known values: 'system', 'assistant', 'developer', 'user'.", - "enum": [ - "system", - "assistant", - "developer", - "user" - ], - "x-ms-enum": { - "modelAsString": true - } - } - }, - "discriminator": "role", - "required": [ - "role" - ] - }, - "EvaluationSystemMessage": { - "type": "object", - "description": "A message authored by the system to guide model behavior.", - "properties": { - "content": { - "type": "string", - "description": "Plain text instructions provided by the system to steer model behavior." - } - }, - "required": [ - "content" - ], - "allOf": [ - { - "$ref": "#/definitions/EvaluationMessage" - } - ], - "x-ms-discriminator-value": "system" - }, "EvaluationTarget": { "type": "object", "description": "Abstract base model for defining evaluation targets.", @@ -2569,39 +2527,20 @@ "type": "string", "description": "Allowed types of evaluation targets.", "enum": [ - "modelPrompt" + "modelResponseGeneration" ], "x-ms-enum": { "name": "EvaluationTargetType", "modelAsString": true, "values": [ { - "name": "modelPrompt", - "value": "modelPrompt", - "description": "Evaluation target that uses a model and prompt." + "name": "modelResponseGeneration", + "value": "modelResponseGeneration", + "description": "Evaluation target that uses a model for response generation." } ] } }, - "EvaluationUserMessage": { - "type": "object", - "description": "A message authored by the end user as input to the model.", - "properties": { - "content": { - "type": "string", - "description": "Input content or question provided by the end user." - } - }, - "required": [ - "content" - ], - "allOf": [ - { - "$ref": "#/definitions/EvaluationMessage" - } - ], - "x-ms-discriminator-value": "user" - }, "EvaluatorConfiguration": { "type": "object", "description": "Evaluator Configuration", @@ -2841,6 +2780,29 @@ ], "x-ms-discriminator-value": "ManagedAzureSearch" }, + "Message": { + "type": "object", + "description": "Abstract base model representing a single message in a conversation.", + "properties": { + "role": { + "type": "string", + "description": "The role of the message author. Known values: 'system', 'assistant', 'developer', 'user'.", + "enum": [ + "system", + "assistant", + "developer", + "user" + ], + "x-ms-enum": { + "modelAsString": true + } + } + }, + "discriminator": "role", + "required": [ + "role" + ] + }, "ModelDeployment": { "type": "object", "description": "Model Deployment Definition", @@ -3316,6 +3278,25 @@ "tier" ] }, + "SystemMessage": { + "type": "object", + "description": "A message authored by the system to guide model behavior.", + "properties": { + "content": { + "type": "string", + "description": "Plain text instructions provided by the system to steer model behavior." + } + }, + "required": [ + "content" + ], + "allOf": [ + { + "$ref": "#/definitions/Message" + } + ], + "x-ms-discriminator-value": "system" + }, "TargetConfig": { "type": "object", "description": "Abstract class for target configuration.", @@ -3330,15 +3311,34 @@ "type" ] }, - "modelPromptEvaluationTarget": { + "UserMessage": { + "type": "object", + "description": "A message authored by the end user as input to the model.", + "properties": { + "content": { + "type": "string", + "description": "Input content or question provided by the end user." + } + }, + "required": [ + "content" + ], + "allOf": [ + { + "$ref": "#/definitions/Message" + } + ], + "x-ms-discriminator-value": "user" + }, + "modelResponseGenerationTarget": { "type": "object", - "description": "Evaluation target that uses the chat completion API to generate responses.", + "description": "Evaluation target for generating responses using a given model and dataset.", "properties": { - "messages": { + "baseMessages": { "type": "array", "description": "A list of messages comprising the conversation so far.", "items": { - "$ref": "#/definitions/EvaluationMessage" + "$ref": "#/definitions/Message" }, "x-ms-identifiers": [] }, @@ -3353,7 +3353,7 @@ } }, "required": [ - "messages", + "baseMessages", "modelDeploymentName", "modelParams" ], @@ -3362,7 +3362,7 @@ "$ref": "#/definitions/EvaluationTarget" } ], - "x-ms-discriminator-value": "modelPrompt" + "x-ms-discriminator-value": "modelResponseGeneration" } }, "parameters": { diff --git a/specification/ai/data-plane/Azure.AI.Projects/stable/v1/azure-ai-projects.json b/specification/ai/data-plane/Azure.AI.Projects/stable/v1/azure-ai-projects.json index 39c784f9eb9a..3bc829b0524f 100644 --- a/specification/ai/data-plane/Azure.AI.Projects/stable/v1/azure-ai-projects.json +++ b/specification/ai/data-plane/Azure.AI.Projects/stable/v1/azure-ai-projects.json @@ -1090,6 +1090,25 @@ "blobReference" ] }, + "AssistantMessage": { + "type": "object", + "description": "A message generated by the assistant in response to previous messages.", + "properties": { + "content": { + "type": "string", + "description": "Response content generated by the assistant." + } + }, + "required": [ + "content" + ], + "allOf": [ + { + "$ref": "#/definitions/Message" + } + ], + "x-ms-discriminator-value": "assistant" + }, "Azure.Core.Foundations.Error": { "type": "object", "description": "The error object.", @@ -1598,6 +1617,25 @@ ] } }, + "DeveloperMessage": { + "type": "object", + "description": "A message authored by a developer to guide the model during evaluation.", + "properties": { + "content": { + "type": "string", + "description": "Content provided by a developer to guide model behavior in an evaluation context." + } + }, + "required": [ + "content" + ], + "allOf": [ + { + "$ref": "#/definitions/Message" + } + ], + "x-ms-discriminator-value": "developer" + }, "EmbeddingConfiguration": { "type": "object", "description": "Embedding configuration class", @@ -1632,86 +1670,6 @@ ], "x-ms-discriminator-value": "AAD" }, - "EvaluationAssistantMessage": { - "type": "object", - "description": "A message generated by the assistant in response to previous messages.", - "properties": { - "content": { - "type": "string", - "description": "Response content generated by the assistant." - } - }, - "required": [ - "content" - ], - "allOf": [ - { - "$ref": "#/definitions/EvaluationMessage" - } - ], - "x-ms-discriminator-value": "assistant" - }, - "EvaluationDeveloperMessage": { - "type": "object", - "description": "A message authored by a developer to guide the model during evaluation.", - "properties": { - "content": { - "type": "string", - "description": "Content provided by a developer to guide model behavior in an evaluation context." - } - }, - "required": [ - "content" - ], - "allOf": [ - { - "$ref": "#/definitions/EvaluationMessage" - } - ], - "x-ms-discriminator-value": "developer" - }, - "EvaluationMessage": { - "type": "object", - "description": "Abstract base model representing a single message in a conversation.", - "properties": { - "role": { - "type": "string", - "description": "The role of the message author. Known values: 'system', 'assistant', 'developer', 'user'.", - "enum": [ - "system", - "assistant", - "developer", - "user" - ], - "x-ms-enum": { - "modelAsString": true - } - } - }, - "discriminator": "role", - "required": [ - "role" - ] - }, - "EvaluationSystemMessage": { - "type": "object", - "description": "A message authored by the system to guide model behavior.", - "properties": { - "content": { - "type": "string", - "description": "Plain text instructions provided by the system to steer model behavior." - } - }, - "required": [ - "content" - ], - "allOf": [ - { - "$ref": "#/definitions/EvaluationMessage" - } - ], - "x-ms-discriminator-value": "system" - }, "EvaluationTarget": { "type": "object", "description": "Abstract base model for defining evaluation targets.", @@ -1730,39 +1688,20 @@ "type": "string", "description": "Allowed types of evaluation targets.", "enum": [ - "modelPrompt" + "modelResponseGeneration" ], "x-ms-enum": { "name": "EvaluationTargetType", "modelAsString": true, "values": [ { - "name": "modelPrompt", - "value": "modelPrompt", - "description": "Evaluation target that uses a model and prompt." + "name": "modelResponseGeneration", + "value": "modelResponseGeneration", + "description": "Evaluation target that uses a model for response generation." } ] } }, - "EvaluationUserMessage": { - "type": "object", - "description": "A message authored by the end user as input to the model.", - "properties": { - "content": { - "type": "string", - "description": "Input content or question provided by the end user." - } - }, - "required": [ - "content" - ], - "allOf": [ - { - "$ref": "#/definitions/EvaluationMessage" - } - ], - "x-ms-discriminator-value": "user" - }, "FieldMapping": { "type": "object", "description": "Field mapping configuration class", @@ -1944,6 +1883,29 @@ ], "x-ms-discriminator-value": "ManagedAzureSearch" }, + "Message": { + "type": "object", + "description": "Abstract base model representing a single message in a conversation.", + "properties": { + "role": { + "type": "string", + "description": "The role of the message author. Known values: 'system', 'assistant', 'developer', 'user'.", + "enum": [ + "system", + "assistant", + "developer", + "user" + ], + "x-ms-enum": { + "modelAsString": true + } + } + }, + "discriminator": "role", + "required": [ + "role" + ] + }, "ModelDeployment": { "type": "object", "description": "Model Deployment Definition", @@ -2270,15 +2232,53 @@ "tier" ] }, - "modelPromptEvaluationTarget": { + "SystemMessage": { + "type": "object", + "description": "A message authored by the system to guide model behavior.", + "properties": { + "content": { + "type": "string", + "description": "Plain text instructions provided by the system to steer model behavior." + } + }, + "required": [ + "content" + ], + "allOf": [ + { + "$ref": "#/definitions/Message" + } + ], + "x-ms-discriminator-value": "system" + }, + "UserMessage": { + "type": "object", + "description": "A message authored by the end user as input to the model.", + "properties": { + "content": { + "type": "string", + "description": "Input content or question provided by the end user." + } + }, + "required": [ + "content" + ], + "allOf": [ + { + "$ref": "#/definitions/Message" + } + ], + "x-ms-discriminator-value": "user" + }, + "modelResponseGenerationTarget": { "type": "object", - "description": "Evaluation target that uses the chat completion API to generate responses.", + "description": "Evaluation target for generating responses using a given model and dataset.", "properties": { - "messages": { + "baseMessages": { "type": "array", "description": "A list of messages comprising the conversation so far.", "items": { - "$ref": "#/definitions/EvaluationMessage" + "$ref": "#/definitions/Message" }, "x-ms-identifiers": [] }, @@ -2293,7 +2293,7 @@ } }, "required": [ - "messages", + "baseMessages", "modelDeploymentName", "modelParams" ], @@ -2302,7 +2302,7 @@ "$ref": "#/definitions/EvaluationTarget" } ], - "x-ms-discriminator-value": "modelPrompt" + "x-ms-discriminator-value": "modelResponseGeneration" } }, "parameters": { From 7631ac50135d0fc52525f03aff5609c5e925c0f5 Mon Sep 17 00:00:00 2001 From: Prachi Gandhi Date: Mon, 7 Jul 2025 11:10:13 -0500 Subject: [PATCH 4/6] rebase and compile --- .../2025-05-15-preview/azure-ai-projects.json | 15 ++++++++++----- .../stable/2025-05-01/azure-ai-projects.json | 6 ++++-- .../stable/v1/azure-ai-projects.json | 6 ++++-- 3 files changed, 18 insertions(+), 9 deletions(-) diff --git a/specification/ai/data-plane/Azure.AI.Projects/preview/2025-05-15-preview/azure-ai-projects.json b/specification/ai/data-plane/Azure.AI.Projects/preview/2025-05-15-preview/azure-ai-projects.json index f7d6c513196b..17b8240276b5 100644 --- a/specification/ai/data-plane/Azure.AI.Projects/preview/2025-05-15-preview/azure-ai-projects.json +++ b/specification/ai/data-plane/Azure.AI.Projects/preview/2025-05-15-preview/azure-ai-projects.json @@ -1526,7 +1526,8 @@ "description": "The agent evaluation result.", "items": { "$ref": "#/definitions/AgentEvaluationResult" - } + }, + "x-ms-identifiers": [] } }, "required": [ @@ -1891,7 +1892,8 @@ "description": "An array of details about specific errors that led to this reported error.", "items": { "$ref": "#/definitions/Azure.Core.Foundations.Error" - } + }, + "x-ms-identifiers": [] }, "innererror": { "$ref": "#/definitions/Azure.Core.Foundations.InnerError", @@ -2916,7 +2918,8 @@ "description": "The Deployment items on this page", "items": { "$ref": "#/definitions/Deployment" - } + }, + "x-ms-identifiers": [] }, "nextLink": { "type": "string", @@ -2937,7 +2940,8 @@ "description": "The Evaluation items on this page", "items": { "$ref": "#/definitions/Evaluation" - } + }, + "x-ms-identifiers": [] }, "nextLink": { "type": "string", @@ -2979,7 +2983,8 @@ "description": "The RedTeam items on this page", "items": { "$ref": "#/definitions/RedTeam" - } + }, + "x-ms-identifiers": [] }, "nextLink": { "type": "string", diff --git a/specification/ai/data-plane/Azure.AI.Projects/stable/2025-05-01/azure-ai-projects.json b/specification/ai/data-plane/Azure.AI.Projects/stable/2025-05-01/azure-ai-projects.json index 4fc40d517d90..f0e91994f31d 100644 --- a/specification/ai/data-plane/Azure.AI.Projects/stable/2025-05-01/azure-ai-projects.json +++ b/specification/ai/data-plane/Azure.AI.Projects/stable/2025-05-01/azure-ai-projects.json @@ -1111,7 +1111,8 @@ "description": "An array of details about specific errors that led to this reported error.", "items": { "$ref": "#/definitions/Azure.Core.Foundations.Error" - } + }, + "x-ms-identifiers": [] }, "innererror": { "$ref": "#/definitions/Azure.Core.Foundations.InnerError", @@ -1926,7 +1927,8 @@ "description": "The Deployment items on this page", "items": { "$ref": "#/definitions/Deployment" - } + }, + "x-ms-identifiers": [] }, "nextLink": { "type": "string", diff --git a/specification/ai/data-plane/Azure.AI.Projects/stable/v1/azure-ai-projects.json b/specification/ai/data-plane/Azure.AI.Projects/stable/v1/azure-ai-projects.json index 3bc829b0524f..b403c354e25e 100644 --- a/specification/ai/data-plane/Azure.AI.Projects/stable/v1/azure-ai-projects.json +++ b/specification/ai/data-plane/Azure.AI.Projects/stable/v1/azure-ai-projects.json @@ -1130,7 +1130,8 @@ "description": "An array of details about specific errors that led to this reported error.", "items": { "$ref": "#/definitions/Azure.Core.Foundations.Error" - } + }, + "x-ms-identifiers": [] }, "innererror": { "$ref": "#/definitions/Azure.Core.Foundations.InnerError", @@ -2019,7 +2020,8 @@ "description": "The Deployment items on this page", "items": { "$ref": "#/definitions/Deployment" - } + }, + "x-ms-identifiers": [] }, "nextLink": { "type": "string", From 4a37129cc45d2ab36cb4884bb48a489baae58a38 Mon Sep 17 00:00:00 2001 From: Prachi Gandhi Date: Wed, 16 Jul 2025 14:23:34 -0500 Subject: [PATCH 5/6] remove from v1 version --- .../Azure.AI.Projects/evaluations/models.tsp | 8 + .../2025-05-15-preview/azure-ai-projects.json | 18 +- .../stable/2025-05-01/azure-ai-projects.json | 6 +- .../stable/v1/azure-ai-projects.json | 171 +----------------- 4 files changed, 18 insertions(+), 185 deletions(-) diff --git a/specification/ai/Azure.AI.Projects/evaluations/models.tsp b/specification/ai/Azure.AI.Projects/evaluations/models.tsp index 82bdc08b73d6..8d1e77b709a6 100644 --- a/specification/ai/Azure.AI.Projects/evaluations/models.tsp +++ b/specification/ai/Azure.AI.Projects/evaluations/models.tsp @@ -182,6 +182,7 @@ model AgentEvaluation { @doc("Abstract base model representing a single message in a conversation.") @discriminator("role") +@removed(Versions.v1) @added(Versions.v2025_05_15_preview) model Message { @doc("The role of the message author. Known values: 'system', 'assistant', 'developer', 'user'.") @@ -189,6 +190,7 @@ model Message { } @doc("A message authored by the system to guide model behavior.") +@removed(Versions.v1) @added(Versions.v2025_05_15_preview) model SystemMessage extends Message { @doc("Indicates this is a system message.") @@ -199,6 +201,7 @@ model SystemMessage extends Message { } @doc("A message authored by a developer to guide the model during evaluation.") +@removed(Versions.v1) @added(Versions.v2025_05_15_preview) model DeveloperMessage extends Message { @doc("Indicates this is a developer message.") @@ -209,6 +212,7 @@ model DeveloperMessage extends Message { } @doc("A message authored by the end user as input to the model.") +@removed(Versions.v1) @added(Versions.v2025_05_15_preview) model UserMessage extends Message { @doc("Indicates this is a user message.") @@ -219,6 +223,7 @@ model UserMessage extends Message { } @doc("A message generated by the assistant in response to previous messages.") +@removed(Versions.v1) @added(Versions.v2025_05_15_preview) model AssistantMessage extends Message { @doc("Indicates this is an assistant message.") @@ -229,6 +234,7 @@ model AssistantMessage extends Message { } @doc("Allowed types of evaluation targets.") +@removed(Versions.v1) @added(Versions.v2025_05_15_preview) union EvaluationTargetType { @doc("Evaluation target that uses a model for response generation.") @@ -238,6 +244,7 @@ union EvaluationTargetType { @doc("Abstract base model for defining evaluation targets.") @discriminator("type") +@removed(Versions.v1) @added(Versions.v2025_05_15_preview) model EvaluationTarget { @doc("Discriminator that defines the type of the evaluation target.") @@ -245,6 +252,7 @@ model EvaluationTarget { } @doc("Evaluation target for generating responses using a given model and dataset.") +@removed(Versions.v1) @added(Versions.v2025_05_15_preview) model modelResponseGenerationTarget extends EvaluationTarget { @doc("The type of evaluation target. Always 'modelResponseGeneration'.") diff --git a/specification/ai/data-plane/Azure.AI.Projects/preview/2025-05-15-preview/azure-ai-projects.json b/specification/ai/data-plane/Azure.AI.Projects/preview/2025-05-15-preview/azure-ai-projects.json index 17b8240276b5..b4cfc17440bb 100644 --- a/specification/ai/data-plane/Azure.AI.Projects/preview/2025-05-15-preview/azure-ai-projects.json +++ b/specification/ai/data-plane/Azure.AI.Projects/preview/2025-05-15-preview/azure-ai-projects.json @@ -1526,8 +1526,7 @@ "description": "The agent evaluation result.", "items": { "$ref": "#/definitions/AgentEvaluationResult" - }, - "x-ms-identifiers": [] + } } }, "required": [ @@ -1892,8 +1891,7 @@ "description": "An array of details about specific errors that led to this reported error.", "items": { "$ref": "#/definitions/Azure.Core.Foundations.Error" - }, - "x-ms-identifiers": [] + } }, "innererror": { "$ref": "#/definitions/Azure.Core.Foundations.InnerError", @@ -2918,8 +2916,7 @@ "description": "The Deployment items on this page", "items": { "$ref": "#/definitions/Deployment" - }, - "x-ms-identifiers": [] + } }, "nextLink": { "type": "string", @@ -2940,8 +2937,7 @@ "description": "The Evaluation items on this page", "items": { "$ref": "#/definitions/Evaluation" - }, - "x-ms-identifiers": [] + } }, "nextLink": { "type": "string", @@ -2983,8 +2979,7 @@ "description": "The RedTeam items on this page", "items": { "$ref": "#/definitions/RedTeam" - }, - "x-ms-identifiers": [] + } }, "nextLink": { "type": "string", @@ -3344,8 +3339,7 @@ "description": "A list of messages comprising the conversation so far.", "items": { "$ref": "#/definitions/Message" - }, - "x-ms-identifiers": [] + } }, "modelDeploymentName": { "type": "string", diff --git a/specification/ai/data-plane/Azure.AI.Projects/stable/2025-05-01/azure-ai-projects.json b/specification/ai/data-plane/Azure.AI.Projects/stable/2025-05-01/azure-ai-projects.json index f0e91994f31d..4fc40d517d90 100644 --- a/specification/ai/data-plane/Azure.AI.Projects/stable/2025-05-01/azure-ai-projects.json +++ b/specification/ai/data-plane/Azure.AI.Projects/stable/2025-05-01/azure-ai-projects.json @@ -1111,8 +1111,7 @@ "description": "An array of details about specific errors that led to this reported error.", "items": { "$ref": "#/definitions/Azure.Core.Foundations.Error" - }, - "x-ms-identifiers": [] + } }, "innererror": { "$ref": "#/definitions/Azure.Core.Foundations.InnerError", @@ -1927,8 +1926,7 @@ "description": "The Deployment items on this page", "items": { "$ref": "#/definitions/Deployment" - }, - "x-ms-identifiers": [] + } }, "nextLink": { "type": "string", diff --git a/specification/ai/data-plane/Azure.AI.Projects/stable/v1/azure-ai-projects.json b/specification/ai/data-plane/Azure.AI.Projects/stable/v1/azure-ai-projects.json index b403c354e25e..f0591f750a4d 100644 --- a/specification/ai/data-plane/Azure.AI.Projects/stable/v1/azure-ai-projects.json +++ b/specification/ai/data-plane/Azure.AI.Projects/stable/v1/azure-ai-projects.json @@ -1090,25 +1090,6 @@ "blobReference" ] }, - "AssistantMessage": { - "type": "object", - "description": "A message generated by the assistant in response to previous messages.", - "properties": { - "content": { - "type": "string", - "description": "Response content generated by the assistant." - } - }, - "required": [ - "content" - ], - "allOf": [ - { - "$ref": "#/definitions/Message" - } - ], - "x-ms-discriminator-value": "assistant" - }, "Azure.Core.Foundations.Error": { "type": "object", "description": "The error object.", @@ -1130,8 +1111,7 @@ "description": "An array of details about specific errors that led to this reported error.", "items": { "$ref": "#/definitions/Azure.Core.Foundations.Error" - }, - "x-ms-identifiers": [] + } }, "innererror": { "$ref": "#/definitions/Azure.Core.Foundations.InnerError", @@ -1618,25 +1598,6 @@ ] } }, - "DeveloperMessage": { - "type": "object", - "description": "A message authored by a developer to guide the model during evaluation.", - "properties": { - "content": { - "type": "string", - "description": "Content provided by a developer to guide model behavior in an evaluation context." - } - }, - "required": [ - "content" - ], - "allOf": [ - { - "$ref": "#/definitions/Message" - } - ], - "x-ms-discriminator-value": "developer" - }, "EmbeddingConfiguration": { "type": "object", "description": "Embedding configuration class", @@ -1671,38 +1632,6 @@ ], "x-ms-discriminator-value": "AAD" }, - "EvaluationTarget": { - "type": "object", - "description": "Abstract base model for defining evaluation targets.", - "properties": { - "type": { - "$ref": "#/definitions/EvaluationTargetType", - "description": "Discriminator that defines the type of the evaluation target." - } - }, - "discriminator": "type", - "required": [ - "type" - ] - }, - "EvaluationTargetType": { - "type": "string", - "description": "Allowed types of evaluation targets.", - "enum": [ - "modelResponseGeneration" - ], - "x-ms-enum": { - "name": "EvaluationTargetType", - "modelAsString": true, - "values": [ - { - "name": "modelResponseGeneration", - "value": "modelResponseGeneration", - "description": "Evaluation target that uses a model for response generation." - } - ] - } - }, "FieldMapping": { "type": "object", "description": "Field mapping configuration class", @@ -1884,29 +1813,6 @@ ], "x-ms-discriminator-value": "ManagedAzureSearch" }, - "Message": { - "type": "object", - "description": "Abstract base model representing a single message in a conversation.", - "properties": { - "role": { - "type": "string", - "description": "The role of the message author. Known values: 'system', 'assistant', 'developer', 'user'.", - "enum": [ - "system", - "assistant", - "developer", - "user" - ], - "x-ms-enum": { - "modelAsString": true - } - } - }, - "discriminator": "role", - "required": [ - "role" - ] - }, "ModelDeployment": { "type": "object", "description": "Model Deployment Definition", @@ -2020,8 +1926,7 @@ "description": "The Deployment items on this page", "items": { "$ref": "#/definitions/Deployment" - }, - "x-ms-identifiers": [] + } }, "nextLink": { "type": "string", @@ -2233,78 +2138,6 @@ "size", "tier" ] - }, - "SystemMessage": { - "type": "object", - "description": "A message authored by the system to guide model behavior.", - "properties": { - "content": { - "type": "string", - "description": "Plain text instructions provided by the system to steer model behavior." - } - }, - "required": [ - "content" - ], - "allOf": [ - { - "$ref": "#/definitions/Message" - } - ], - "x-ms-discriminator-value": "system" - }, - "UserMessage": { - "type": "object", - "description": "A message authored by the end user as input to the model.", - "properties": { - "content": { - "type": "string", - "description": "Input content or question provided by the end user." - } - }, - "required": [ - "content" - ], - "allOf": [ - { - "$ref": "#/definitions/Message" - } - ], - "x-ms-discriminator-value": "user" - }, - "modelResponseGenerationTarget": { - "type": "object", - "description": "Evaluation target for generating responses using a given model and dataset.", - "properties": { - "baseMessages": { - "type": "array", - "description": "A list of messages comprising the conversation so far.", - "items": { - "$ref": "#/definitions/Message" - }, - "x-ms-identifiers": [] - }, - "modelDeploymentName": { - "type": "string", - "description": "The model deployment to be evaluated. Accepts either the deployment name alone or with the connection name as '{connectionName}/modelDeploymentName'." - }, - "modelParams": { - "type": "object", - "description": "Optional parameters passed to the model for evaluation.", - "additionalProperties": {} - } - }, - "required": [ - "baseMessages", - "modelDeploymentName", - "modelParams" - ], - "allOf": [ - { - "$ref": "#/definitions/EvaluationTarget" - } - ], - "x-ms-discriminator-value": "modelResponseGeneration" } }, "parameters": { From 080f2b50aeb4097bb1b9e3112dee5c2c41a7c723 Mon Sep 17 00:00:00 2001 From: Prachi Gandhi Date: Wed, 16 Jul 2025 14:41:06 -0500 Subject: [PATCH 6/6] tsp format --- specification/ai/Azure.AI.Projects/evaluations/models.tsp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/specification/ai/Azure.AI.Projects/evaluations/models.tsp b/specification/ai/Azure.AI.Projects/evaluations/models.tsp index 8d1e77b709a6..ab1ad5338c1a 100644 --- a/specification/ai/Azure.AI.Projects/evaluations/models.tsp +++ b/specification/ai/Azure.AI.Projects/evaluations/models.tsp @@ -238,8 +238,9 @@ model AssistantMessage extends Message { @added(Versions.v2025_05_15_preview) union EvaluationTargetType { @doc("Evaluation target that uses a model for response generation.") - modelResponseGeneration: "modelResponseGeneration"; - string; + modelResponseGeneration: "modelResponseGeneration", + + string, } @doc("Abstract base model for defining evaluation targets.")