From 97cff401b3822e99f34b69646140158c58e53f11 Mon Sep 17 00:00:00 2001 From: David Wu Date: Wed, 23 Apr 2025 15:02:14 -0700 Subject: [PATCH 1/4] Switch AppInsights connection string to be mandatory for SDKs and add top level error field for get agent evaluation results API --- specification/ai/Azure.AI.Projects/evaluations/models.tsp | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/specification/ai/Azure.AI.Projects/evaluations/models.tsp b/specification/ai/Azure.AI.Projects/evaluations/models.tsp index 990f9fc35093..29a2cfee3784 100644 --- a/specification/ai/Azure.AI.Projects/evaluations/models.tsp +++ b/specification/ai/Azure.AI.Projects/evaluations/models.tsp @@ -121,7 +121,7 @@ model AgentEvaluationRequest { redactionConfiguration?: AgentEvaluationRedactionConfiguration; @doc("Optional and temporary way to pass the AppInsights connection string to the evaluator. When this string is not null, the evaluation results will be logged to Azure AppInsights.") - appInsightsConnectionString?: string; + appInsightsConnectionString: string; } @doc("Result for the agent evaluation evaluator run.") @@ -166,6 +166,9 @@ model AgentEvaluation { @doc("Status of the agent evaluation. Options: Running, Completed, Failed.") status: string; + @doc("The reason of the request failure for the long running process, if applicable.") + error?: string; + @doc("The agent evaluation result.") result?: Array; } From d672dd8587d1fa7e0ae089aabaf2852116bc8fa6 Mon Sep 17 00:00:00 2001 From: David Wu Date: Wed, 23 Apr 2025 21:56:59 -0700 Subject: [PATCH 2/4] Updated generated JSON --- .../preview/2025-05-15-preview/azure-ai-projects-1dp.json | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/specification/ai/data-plane/Azure.AI.Projects/preview/2025-05-15-preview/azure-ai-projects-1dp.json b/specification/ai/data-plane/Azure.AI.Projects/preview/2025-05-15-preview/azure-ai-projects-1dp.json index 5f4a16d2309c..3856e73cbf39 100644 --- a/specification/ai/data-plane/Azure.AI.Projects/preview/2025-05-15-preview/azure-ai-projects-1dp.json +++ b/specification/ai/data-plane/Azure.AI.Projects/preview/2025-05-15-preview/azure-ai-projects-1dp.json @@ -1492,6 +1492,10 @@ "type": "string", "description": "Status of the agent evaluation. Options: Running, Completed, Failed." }, + "error": { + "type": "string", + "description": "The reason of the request failure for the long running process, if applicable." + }, "result": { "type": "array", "description": "The agent evaluation result.", @@ -1550,7 +1554,8 @@ }, "required": [ "runId", - "evaluators" + "evaluators", + "appInsightsConnectionString" ] }, "AgentEvaluationResult": { From 51bdf66f6f1c21a874ee58cdc913f69f661e431c Mon Sep 17 00:00:00 2001 From: David Wu Date: Thu, 24 Apr 2025 08:26:05 -0700 Subject: [PATCH 3/4] Update docs to address PR feedbacks --- specification/ai/Azure.AI.Projects/evaluations/models.tsp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specification/ai/Azure.AI.Projects/evaluations/models.tsp b/specification/ai/Azure.AI.Projects/evaluations/models.tsp index 29a2cfee3784..f72bf164ab54 100644 --- a/specification/ai/Azure.AI.Projects/evaluations/models.tsp +++ b/specification/ai/Azure.AI.Projects/evaluations/models.tsp @@ -120,7 +120,7 @@ model AgentEvaluationRequest { @doc("Redaction configuration for the evaluation.") redactionConfiguration?: AgentEvaluationRedactionConfiguration; - @doc("Optional and temporary way to pass the AppInsights connection string to the evaluator. When this string is not null, the evaluation results will be logged to Azure AppInsights.") + @doc("Pass the AppInsights connection string to the agent evaluation for the evaluation results and the errors logs.") appInsightsConnectionString: string; } From 06ce8d39091cbfa0fe39e986fe0e3250257e6605 Mon Sep 17 00:00:00 2001 From: David Wu Date: Thu, 24 Apr 2025 08:29:34 -0700 Subject: [PATCH 4/4] Update the generated JSON file --- .../preview/2025-05-15-preview/azure-ai-projects-1dp.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specification/ai/data-plane/Azure.AI.Projects/preview/2025-05-15-preview/azure-ai-projects-1dp.json b/specification/ai/data-plane/Azure.AI.Projects/preview/2025-05-15-preview/azure-ai-projects-1dp.json index 3856e73cbf39..bd7eac5443e3 100644 --- a/specification/ai/data-plane/Azure.AI.Projects/preview/2025-05-15-preview/azure-ai-projects-1dp.json +++ b/specification/ai/data-plane/Azure.AI.Projects/preview/2025-05-15-preview/azure-ai-projects-1dp.json @@ -1549,7 +1549,7 @@ }, "appInsightsConnectionString": { "type": "string", - "description": "Optional and temporary way to pass the AppInsights connection string to the evaluator. When this string is not null, the evaluation results will be logged to Azure AppInsights." + "description": "Pass the AppInsights connection string to the agent evaluation for the evaluation results and the errors logs." } }, "required": [