Skip to content

[Evals Svc Team] Evaluation Target typespec #35021

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
gandhipr merged 6 commits into from
Jul 16, 2025
Merged

[Evals Svc Team] Evaluation Target typespec #35021

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
003b8a7
eval target typespec
May 30, 2025
355a655
review changes
Jun 12, 2025
431dfca
review comments after api review
Jun 23, 2025
7631ac5
rebase and compile
Jul 7, 2025
4a37129
remove from v1 version
Jul 16, 2025
080f2b5
tsp format
Jul 16, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
remove from v1 version
  • Loading branch information
@gandhipr
Prachi Gandhi authored and gandhipr committed Jul 16, 2025
commit 4a37129cc45d2ab36cb4884bb48a489baae58a38
8 changes: 8 additions & 0 deletions specification/ai/Azure.AI.Projects/evaluations/models.tsp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -182,13 +182,15 @@ model AgentEvaluation {

@doc("Abstract base model representing a single message in a conversation.")
@discriminator("role")
@removed(Versions.v1)
@added(Versions.v2025_05_15_preview)
model Message {
@doc("The role of the message author. Known values: 'system', 'assistant', 'developer', 'user'.")
role: "system" | "assistant" | "developer" | "user" | string;
}

@doc("A message authored by the system to guide model behavior.")
@removed(Versions.v1)
@added(Versions.v2025_05_15_preview)
model SystemMessage extends Message {
@doc("Indicates this is a system message.")
Expand All @@ -199,6 +201,7 @@ model SystemMessage extends Message {
}

@doc("A message authored by a developer to guide the model during evaluation.")
@removed(Versions.v1)
@added(Versions.v2025_05_15_preview)
model DeveloperMessage extends Message {
@doc("Indicates this is a developer message.")
Expand All @@ -209,6 +212,7 @@ model DeveloperMessage extends Message {
}

@doc("A message authored by the end user as input to the model.")
@removed(Versions.v1)
@added(Versions.v2025_05_15_preview)
model UserMessage extends Message {
@doc("Indicates this is a user message.")
Expand All @@ -219,6 +223,7 @@ model UserMessage extends Message {
}

@doc("A message generated by the assistant in response to previous messages.")
@removed(Versions.v1)
@added(Versions.v2025_05_15_preview)
model AssistantMessage extends Message {
@doc("Indicates this is an assistant message.")
Expand All @@ -229,6 +234,7 @@ model AssistantMessage extends Message {
}

@doc("Allowed types of evaluation targets.")
@removed(Versions.v1)
@added(Versions.v2025_05_15_preview)
union EvaluationTargetType {
@doc("Evaluation target that uses a model for response generation.")
Expand All @@ -238,13 +244,15 @@ union EvaluationTargetType {

@doc("Abstract base model for defining evaluation targets.")
@discriminator("type")
@removed(Versions.v1)
@added(Versions.v2025_05_15_preview)
model EvaluationTarget {
@doc("Discriminator that defines the type of the evaluation target.")
type: EvaluationTargetType;
}

@doc("Evaluation target for generating responses using a given model and dataset.")
@removed(Versions.v1)
@added(Versions.v2025_05_15_preview)
model modelResponseGenerationTarget extends EvaluationTarget {
@doc("The type of evaluation target. Always 'modelResponseGeneration'.")
Expand Down
18 changes: 6 additions & 12 deletions ...ication/ai/data-plane/Azure.AI.Projects/preview/2025-05-15-preview/azure-ai-projects.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1526,8 +1526,7 @@
"description": "The agent evaluation result.",
"items": {
"$ref": "#/definitions/AgentEvaluationResult"
},
"x-ms-identifiers": []
}
}
},
"required": [
Expand Down Expand Up @@ -1892,8 +1891,7 @@
"description": "An array of details about specific errors that led to this reported error.",
"items": {
"$ref": "#/definitions/Azure.Core.Foundations.Error"
},
"x-ms-identifiers": []
}
},
"innererror": {
"$ref": "#/definitions/Azure.Core.Foundations.InnerError",
Expand Down Expand Up @@ -2918,8 +2916,7 @@
"description": "The Deployment items on this page",
"items": {
"$ref": "#/definitions/Deployment"
},
"x-ms-identifiers": []
}
},
"nextLink": {
"type": "string",
Expand All @@ -2940,8 +2937,7 @@
"description": "The Evaluation items on this page",
"items": {
"$ref": "#/definitions/Evaluation"
},
"x-ms-identifiers": []
}
},
"nextLink": {
"type": "string",
Expand Down Expand Up @@ -2983,8 +2979,7 @@
"description": "The RedTeam items on this page",
"items": {
"$ref": "#/definitions/RedTeam"
},
"x-ms-identifiers": []
}
},
"nextLink": {
"type": "string",
Expand Down Expand Up @@ -3344,8 +3339,7 @@
"description": "A list of messages comprising the conversation so far.",
"items": {
"$ref": "#/definitions/Message"
},
"x-ms-identifiers": []
}
},
"modelDeploymentName": {
"type": "string",
Expand Down
6 changes: 2 additions & 4 deletions specification/ai/data-plane/Azure.AI.Projects/stable/2025-05-01/azure-ai-projects.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1111,8 +1111,7 @@
"description": "An array of details about specific errors that led to this reported error.",
"items": {
"$ref": "#/definitions/Azure.Core.Foundations.Error"
},
"x-ms-identifiers": []
}
},
"innererror": {
"$ref": "#/definitions/Azure.Core.Foundations.InnerError",
Expand Down Expand Up @@ -1927,8 +1926,7 @@
"description": "The Deployment items on this page",
"items": {
"$ref": "#/definitions/Deployment"
},
"x-ms-identifiers": []
}
},
"nextLink": {
"type": "string",
Expand Down
171 changes: 2 additions & 169 deletions specification/ai/data-plane/Azure.AI.Projects/stable/v1/azure-ai-projects.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1090,25 +1090,6 @@
"blobReference"
]
},
"AssistantMessage": {
"type": "object",
"description": "A message generated by the assistant in response to previous messages.",
"properties": {
"content": {
"type": "string",
"description": "Response content generated by the assistant."
}
},
"required": [
"content"
],
"allOf": [
{
"$ref": "#/definitions/Message"
}
],
"x-ms-discriminator-value": "assistant"
},
"Azure.Core.Foundations.Error": {
"type": "object",
"description": "The error object.",
Expand All @@ -1130,8 +1111,7 @@
"description": "An array of details about specific errors that led to this reported error.",
"items": {
"$ref": "#/definitions/Azure.Core.Foundations.Error"
},
"x-ms-identifiers": []
}
},
"innererror": {
"$ref": "#/definitions/Azure.Core.Foundations.InnerError",
Expand Down Expand Up @@ -1618,25 +1598,6 @@
]
}
},
"DeveloperMessage": {
"type": "object",
"description": "A message authored by a developer to guide the model during evaluation.",
"properties": {
"content": {
"type": "string",
"description": "Content provided by a developer to guide model behavior in an evaluation context."
}
},
"required": [
"content"
],
"allOf": [
{
"$ref": "#/definitions/Message"
}
],
"x-ms-discriminator-value": "developer"
},
"EmbeddingConfiguration": {
"type": "object",
"description": "Embedding configuration class",
Expand Down Expand Up @@ -1671,38 +1632,6 @@
],
"x-ms-discriminator-value": "AAD"
},
"EvaluationTarget": {
"type": "object",
"description": "Abstract base model for defining evaluation targets.",
"properties": {
"type": {
"$ref": "#/definitions/EvaluationTargetType",
"description": "Discriminator that defines the type of the evaluation target."
}
},
"discriminator": "type",
"required": [
"type"
]
},
"EvaluationTargetType": {
"type": "string",
"description": "Allowed types of evaluation targets.",
"enum": [
"modelResponseGeneration"
],
"x-ms-enum": {
"name": "EvaluationTargetType",
"modelAsString": true,
"values": [
{
"name": "modelResponseGeneration",
"value": "modelResponseGeneration",
"description": "Evaluation target that uses a model for response generation."
}
]
}
},
"FieldMapping": {
"type": "object",
"description": "Field mapping configuration class",
Expand Down Expand Up @@ -1884,29 +1813,6 @@
],
"x-ms-discriminator-value": "ManagedAzureSearch"
},
"Message": {
"type": "object",
"description": "Abstract base model representing a single message in a conversation.",
"properties": {
"role": {
"type": "string",
"description": "The role of the message author. Known values: 'system', 'assistant', 'developer', 'user'.",
"enum": [
"system",
"assistant",
"developer",
"user"
],
"x-ms-enum": {
"modelAsString": true
}
}
},
"discriminator": "role",
"required": [
"role"
]
},
"ModelDeployment": {
"type": "object",
"description": "Model Deployment Definition",
Expand Down Expand Up @@ -2020,8 +1926,7 @@
"description": "The Deployment items on this page",
"items": {
"$ref": "#/definitions/Deployment"
},
"x-ms-identifiers": []
}
},
"nextLink": {
"type": "string",
Expand Down Expand Up @@ -2233,78 +2138,6 @@
"size",
"tier"
]
},
"SystemMessage": {
"type": "object",
"description": "A message authored by the system to guide model behavior.",
"properties": {
"content": {
"type": "string",
"description": "Plain text instructions provided by the system to steer model behavior."
}
},
"required": [
"content"
],
"allOf": [
{
"$ref": "#/definitions/Message"
}
],
"x-ms-discriminator-value": "system"
},
"UserMessage": {
"type": "object",
"description": "A message authored by the end user as input to the model.",
"properties": {
"content": {
"type": "string",
"description": "Input content or question provided by the end user."
}
},
"required": [
"content"
],
"allOf": [
{
"$ref": "#/definitions/Message"
}
],
"x-ms-discriminator-value": "user"
},
"modelResponseGenerationTarget": {
"type": "object",
"description": "Evaluation target for generating responses using a given model and dataset.",
"properties": {
"baseMessages": {
"type": "array",
"description": "A list of messages comprising the conversation so far.",
"items": {
"$ref": "#/definitions/Message"
},
"x-ms-identifiers": []
},
"modelDeploymentName": {
"type": "string",
"description": "The model deployment to be evaluated. Accepts either the deployment name alone or with the connection name as '{connectionName}/modelDeploymentName'."
},
"modelParams": {
"type": "object",
"description": "Optional parameters passed to the model for evaluation.",
"additionalProperties": {}
}
},
"required": [
"baseMessages",
"modelDeploymentName",
"modelParams"
],
"allOf": [
{
"$ref": "#/definitions/EvaluationTarget"
}
],
"x-ms-discriminator-value": "modelResponseGeneration"
}
},
"parameters": {
Expand Down