Skip to content

[Evals Svc Team] Evaluation Target typespec #35021

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
gandhipr merged 6 commits into from
Jul 16, 2025
Merged

[Evals Svc Team] Evaluation Target typespec #35021

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
003b8a7
eval target typespec
May 30, 2025
355a655
review changes
Jun 12, 2025
431dfca
review comments after api review
Jun 23, 2025
7631ac5
rebase and compile
Jul 7, 2025
4a37129
remove from v1 version
Jul 16, 2025
080f2b5
tsp format
Jul 16, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
92 changes: 92 additions & 0 deletions specification/ai/Azure.AI.Projects/evaluations/models.tsp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -78,6 +78,9 @@ model Evaluation {

@doc("Evaluators to be used for the evaluation.")
evaluators: Record<EvaluatorConfiguration>;

@doc("Specifies the type and configuration of the entity used for this evaluation.")
target?: EvaluationTarget;
}

@doc("Definition for sampling strategy.")
Expand Down Expand Up @@ -176,3 +179,92 @@ model AgentEvaluation {
@doc("The agent evaluation result.")
result?: Array<AgentEvaluationResult>;
}

@doc("Abstract base model representing a single message in a conversation.")
@discriminator("role")
@removed(Versions.v1)
@added(Versions.v2025_05_15_preview)
model Message {
@doc("The role of the message author. Known values: 'system', 'assistant', 'developer', 'user'.")
role: "system" | "assistant" | "developer" | "user" | string;
}

@doc("A message authored by the system to guide model behavior.")
@removed(Versions.v1)
@added(Versions.v2025_05_15_preview)
model SystemMessage extends Message {
@doc("Indicates this is a system message.")
role: "system";

@doc("Plain text instructions provided by the system to steer model behavior.")
content: string;
}

@doc("A message authored by a developer to guide the model during evaluation.")
@removed(Versions.v1)
@added(Versions.v2025_05_15_preview)
model DeveloperMessage extends Message {
@doc("Indicates this is a developer message.")
role: "developer";

@doc("Content provided by a developer to guide model behavior in an evaluation context.")
content: string;
}

@doc("A message authored by the end user as input to the model.")
@removed(Versions.v1)
@added(Versions.v2025_05_15_preview)
model UserMessage extends Message {
@doc("Indicates this is a user message.")
role: "user";

@doc("Input content or question provided by the end user.")
content: string;
}

@doc("A message generated by the assistant in response to previous messages.")
@removed(Versions.v1)
@added(Versions.v2025_05_15_preview)
model AssistantMessage extends Message {
@doc("Indicates this is an assistant message.")
role: "assistant";

@doc("Response content generated by the assistant.")
content: string;
}

@doc("Allowed types of evaluation targets.")
@removed(Versions.v1)
@added(Versions.v2025_05_15_preview)
union EvaluationTargetType {
@doc("Evaluation target that uses a model for response generation.")
modelResponseGeneration: "modelResponseGeneration",

string,
}

@doc("Abstract base model for defining evaluation targets.")
@discriminator("type")
@removed(Versions.v1)
@added(Versions.v2025_05_15_preview)
model EvaluationTarget {
@doc("Discriminator that defines the type of the evaluation target.")
type: EvaluationTargetType;
}

@doc("Evaluation target for generating responses using a given model and dataset.")
@removed(Versions.v1)
@added(Versions.v2025_05_15_preview)
model modelResponseGenerationTarget extends EvaluationTarget {
@doc("The type of evaluation target. Always 'modelResponseGeneration'.")
type: EvaluationTargetType.modelResponseGeneration;

@doc("A list of messages comprising the conversation so far.")
baseMessages: Message[];

@doc("The model deployment to be evaluated. Accepts either the deployment name alone or with the connection name as '{connectionName}/modelDeploymentName'.")
modelDeploymentName: string;

@doc("Optional parameters passed to the model for evaluation.")
modelParams: Record<unknown>;
}
168 changes: 168 additions & 0 deletions ...ication/ai/data-plane/Azure.AI.Projects/preview/2025-05-15-preview/azure-ai-projects.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1695,6 +1695,25 @@
"blobReference"
]
},
"AssistantMessage": {
"type": "object",
"description": "A message generated by the assistant in response to previous messages.",
"properties": {
"content": {
"type": "string",
"description": "Response content generated by the assistant."
}
},
"required": [
"content"
],
"allOf": [
{
"$ref": "#/definitions/Message"
}
],
"x-ms-discriminator-value": "assistant"
},
"AttackStrategy": {
"type": "string",
"description": "Strategies for attacks.",
Expand Down Expand Up @@ -2378,6 +2397,25 @@
]
}
},
"DeveloperMessage": {
"type": "object",
"description": "A message authored by a developer to guide the model during evaluation.",
"properties": {
"content": {
"type": "string",
"description": "Content provided by a developer to guide model behavior in an evaluation context."
}
},
"required": [
"content"
],
"allOf": [
{
"$ref": "#/definitions/Message"
}
],
"x-ms-discriminator-value": "developer"
},
"EmbeddingConfiguration": {
"type": "object",
"description": "Embedding configuration class",
Expand Down Expand Up @@ -2459,6 +2497,10 @@
"additionalProperties": {
"$ref": "#/definitions/EvaluatorConfiguration"
}
},
"target": {
"$ref": "#/definitions/EvaluationTarget",
"description": "Specifies the type and configuration of the entity used for this evaluation."
}
},
"required": [
Expand All @@ -2467,6 +2509,38 @@
"evaluators"
]
},
"EvaluationTarget": {
"type": "object",
"description": "Abstract base model for defining evaluation targets.",
"properties": {
"type": {
"$ref": "#/definitions/EvaluationTargetType",
"description": "Discriminator that defines the type of the evaluation target."
}
},
"discriminator": "type",
"required": [
"type"
]
},
"EvaluationTargetType": {
"type": "string",
"description": "Allowed types of evaluation targets.",
"enum": [
"modelResponseGeneration"
],
"x-ms-enum": {
"name": "EvaluationTargetType",
"modelAsString": true,
"values": [
{
"name": "modelResponseGeneration",
"value": "modelResponseGeneration",
"description": "Evaluation target that uses a model for response generation."
}
]
}
},
"EvaluatorConfiguration": {
"type": "object",
"description": "Evaluator Configuration",
Expand Down Expand Up @@ -2706,6 +2780,29 @@
],
"x-ms-discriminator-value": "ManagedAzureSearch"
},
"Message": {
"type": "object",
"description": "Abstract base model representing a single message in a conversation.",
"properties": {
"role": {
"type": "string",
"description": "The role of the message author. Known values: 'system', 'assistant', 'developer', 'user'.",
"enum": [
"system",
"assistant",
"developer",
"user"
],
"x-ms-enum": {
"modelAsString": true
}
}
},
"discriminator": "role",
"required": [
"role"
]
},
"ModelDeployment": {
"type": "object",
"description": "Model Deployment Definition",
Expand Down Expand Up @@ -3181,6 +3278,25 @@
"tier"
]
},
"SystemMessage": {
"type": "object",
"description": "A message authored by the system to guide model behavior.",
"properties": {
"content": {
"type": "string",
"description": "Plain text instructions provided by the system to steer model behavior."
}
},
"required": [
"content"
],
"allOf": [
{
"$ref": "#/definitions/Message"
}
],
"x-ms-discriminator-value": "system"
},
"TargetConfig": {
"type": "object",
"description": "Abstract class for target configuration.",
Expand All @@ -3194,6 +3310,58 @@
"required": [
"type"
]
},
"UserMessage": {
"type": "object",
"description": "A message authored by the end user as input to the model.",
"properties": {
"content": {
"type": "string",
"description": "Input content or question provided by the end user."
}
},
"required": [
"content"
],
"allOf": [
{
"$ref": "#/definitions/Message"
}
],
"x-ms-discriminator-value": "user"
},
"modelResponseGenerationTarget": {
"type": "object",
"description": "Evaluation target for generating responses using a given model and dataset.",
"properties": {
"baseMessages": {
"type": "array",
"description": "A list of messages comprising the conversation so far.",
"items": {
"$ref": "#/definitions/Message"
}
},
"modelDeploymentName": {
"type": "string",
"description": "The model deployment to be evaluated. Accepts either the deployment name alone or with the connection name as '{connectionName}/modelDeploymentName'."
},
"modelParams": {
"type": "object",
"description": "Optional parameters passed to the model for evaluation.",
"additionalProperties": {}
}
},
"required": [
"baseMessages",
"modelDeploymentName",
"modelParams"
],
"allOf": [
{
"$ref": "#/definitions/EvaluationTarget"
}
],
"x-ms-discriminator-value": "modelResponseGeneration"
}
},
"parameters": {
Expand Down
Loading