diff --git a/specification/ai/Azure.AI.Projects/evaluations/models.tsp b/specification/ai/Azure.AI.Projects/evaluations/models.tsp index 3edf36b1b57e..bac75805e47a 100644 --- a/specification/ai/Azure.AI.Projects/evaluations/models.tsp +++ b/specification/ai/Azure.AI.Projects/evaluations/models.tsp @@ -6,6 +6,7 @@ import "@azure-tools/typespec-azure-core"; import "../common/models.tsp"; import "../main.tsp"; import "@typespec/openapi"; +import "../red-teams/models.tsp"; using TypeSpec.Rest; using TypeSpec.Versioning; @@ -91,7 +92,20 @@ model MAASModelConfig extends TargetModelConfig { @doc("Target for the evaluation process.") @added(Versions.v2025_05_15_preview) @removed(Versions.v_latest) +@discriminator("type") model EvaluationTarget { + @doc("Type of the evaluation target.") + type: string; +} + +#suppress "@azure-tools/typespec-azure-core/no-string-discriminator" +@doc("Evaluation target for model-based evaluations.") +@added(Versions.v2025_05_15_preview) +@removed(Versions.v_latest) +model EvaluationModelTarget extends EvaluationTarget { + @visibility(Lifecycle.Read, Lifecycle.Create) + type: "model"; + @doc("System message related to the evaluation target.") systemMessage: string; @@ -103,6 +117,30 @@ model EvaluationTarget { modelParams?: Record; } +#suppress "@azure-tools/typespec-azure-core/no-string-discriminator" +@doc("Evaluation target for model-based evaluations.") +@added(Versions.v2025_05_15_preview) +@removed(Versions.v_latest) +model RedTeamTarget extends EvaluationTarget { + @visibility(Lifecycle.Read, Lifecycle.Create) + type: "redteam"; + + @doc("Number of simulation rounds.") + numTurns: int32; + + @doc("List of attack strategies or nested lists of attack strategies.") + attackStrategies: AttackStrategy[]; + + @doc("Simulation-only or Simulation + Evaluation. Default false, if true the scan outputs conversation not evaluation result.") + simulationOnly: boolean; + + @doc("List of risk categories to generate attack objectives for.") + riskCategories: RiskCategory[]; + + @doc("Application scenario for the red team operation, to generate scenario specific attacks.") + applicationScenario?: string; +} + @doc("Evaluation Definition") @resource("runs") @added(Versions.v2025_05_15_preview) diff --git a/specification/ai/Azure.AI.Projects/evaluations/routes.tsp b/specification/ai/Azure.AI.Projects/evaluations/routes.tsp index 9b99dbfc7c9e..24c52b80c0b2 100644 --- a/specification/ai/Azure.AI.Projects/evaluations/routes.tsp +++ b/specification/ai/Azure.AI.Projects/evaluations/routes.tsp @@ -121,6 +121,7 @@ interface Evaluations { #suppress "@azure-tools/typespec-azure-core/use-standard-operations" #suppress "@azure-tools/typespec-azure-core/no-closed-literal-union" + #suppress "@azure-tools/typespec-autorest/union-unsupported" "" @doc("Poll for the operation results.") @route("/operations/{operationId}") @get @@ -138,7 +139,8 @@ interface Evaluations { @doc("The operation results.") @body - operationResults: Record[]; + + operationResults: Record[] | ChatCompletions; }; // Upload API set @@ -158,6 +160,7 @@ interface Evaluations { Evaluation >; + #suppress "@typespec/http/patch-implicit-optional" "" #suppress "@azure-tools/typespec-azure-core/use-standard-operations" @doc("Update the uploaded the result to an evaluation run.") @route("runs:updateUpload/{name}") @@ -177,4 +180,91 @@ interface Evaluations { }, Evaluation >; + + // Simulation Private APIs + #suppress "@azure-tools/typespec-azure-core/use-standard-operations" "Waiting for bug fix: https://github.com/Azure/typespec-azure-pr/issues/3739." + @doc("Get the jailbreak dataset with type.") + @route("/simulation/jailbreak/{type}") + @get + getJailBreakDatasetWithType is Azure.Core.Foundations.Operation< + { + @doc("Type of jailbreak dataset") + @path + type: string; + }, + Array + >; + + #suppress "@azure-tools/typespec-azure-core/use-standard-operations" "Waiting for bug fix: https://github.com/Azure/typespec-azure-pr/issues/3739." + #suppress "@azure-tools/typespec-azure-core/no-query-explode" + @doc("Get the attack objectives.") + @route("/simulation/attackobjectives") + @get + getAttackObjectives is Azure.Core.Foundations.Operation< + { + @doc("Risk types for the attack objectives dataset") + @query(#{ explode: true }) + riskTypes?: string[]; + + @doc("The language for the attack objectives dataset, defaults to 'en'") + @query + lang?: string; + + @doc("The strategy") + @query + strategy?: string; + }, + Array + >; + + #suppress "@azure-tools/typespec-azure-core/use-standard-operations" "Waiting for bug fix: https://github.com/Azure/typespec-azure-pr/issues/3739." + @doc("Get the jailbreak dataset.") + @route("/simulation/jailbreak/") + @get + getJailBreakDataset is Azure.Core.Foundations.Operation<{}, Array>; + + #suppress "@azure-tools/typespec-azure-core/use-standard-operations" "Waiting for bug fix: https://github.com/Azure/typespec-azure-pr/issues/3739." + @doc("Get template parameters with type.") + @route("/simulation/template/parameters/{type}") + @get + getTemplateParametersWithType is Azure.Core.Foundations.Operation< + { + @doc("Type for the template parameters") + @path + type: string; + }, + string + >; + + #suppress "@azure-tools/typespec-azure-core/use-standard-operations" "Waiting for bug fix: https://github.com/Azure/typespec-azure-pr/issues/3739." + @doc("Get template parameters.") + @route("/simulation/template/parameters/") + @get + getTemplateParameters is Azure.Core.Foundations.Operation<{}, string>; + + #suppress "@azure-tools/typespec-azure-core/use-standard-operations" "Waiting for bug fix: https://github.com/Azure/typespec-azure-pr/issues/3739." + @doc("Get the template parameters image.") + @route("/simulation/template/parameters/image") + @get + getTemplateParametersImage is Azure.Core.Foundations.Operation< + { + @doc("Image path.") + @query + path: string; + }, + string + >; + + #suppress "@azure-tools/typespec-azure-core/use-standard-operations" "Waiting for bug fix: https://github.com/Azure/typespec-azure-pr/issues/3739." + @doc("Submit a request for simulation.") + @route("/simulation/chat/completions/submit") + @post + submitSimulation is Azure.Core.Foundations.Operation< + { + @doc("Properties of a Prompt Version.") + @body + body: SimulationDTO; + }, + ResourceCreatedOrOkResponse + >; } diff --git a/specification/ai/Azure.AI.Projects/red-teams/routes.tsp b/specification/ai/Azure.AI.Projects/red-teams/routes.tsp index a78e0788269c..18d8e2ed1f60 100644 --- a/specification/ai/Azure.AI.Projects/red-teams/routes.tsp +++ b/specification/ai/Azure.AI.Projects/red-teams/routes.tsp @@ -62,6 +62,7 @@ interface RedTeams { RedTeam >; + #suppress "@typespec/http/patch-implicit-optional" "" #suppress "@azure-tools/typespec-azure-core/use-standard-operations" @doc("Update the uploaded the result to an redteam run.") @route("runs:updateUpload/{name}") diff --git a/specification/ai/Azure.AI.Projects/servicepatterns.tsp b/specification/ai/Azure.AI.Projects/servicepatterns.tsp index 376184787b32..f08d48532718 100644 --- a/specification/ai/Azure.AI.Projects/servicepatterns.tsp +++ b/specification/ai/Azure.AI.Projects/servicepatterns.tsp @@ -72,6 +72,7 @@ namespace Azure.AI.Projects.ServicePatterns { >; */ + #suppress "@typespec/http/patch-implicit-optional" "" @doc( "Create a new or update an existing {name} with the given version id", TEntityType