Merge RedTeams with Evaluations

specification/ai/Azure.AI.Projects/evaluations/models.tsp

@@ -6,6 +6,7 @@ import "@azure-tools/typespec-azure-core";
 import "../common/models.tsp";
 import "../main.tsp";
 import "@typespec/openapi";
+import "../red-teams/models.tsp";
 
 using TypeSpec.Rest;
 using TypeSpec.Versioning;
@@ -91,7 +92,20 @@ model MAASModelConfig extends TargetModelConfig {
 @doc("Target for the evaluation process.")
 @added(Versions.v2025_05_15_preview)
 @removed(Versions.v_latest)
+@discriminator("type")
 model EvaluationTarget {
+  @doc("Type of the evaluation target.")
+  type: string;
+}
+
+#suppress "@azure-tools/typespec-azure-core/no-string-discriminator"
+@doc("Evaluation target for model-based evaluations.")
+@added(Versions.v2025_05_15_preview)
+@removed(Versions.v_latest)
+model EvaluationModelTarget extends EvaluationTarget {
+  @visibility(Lifecycle.Read, Lifecycle.Create)
+  type: "model";
+
   @doc("System message related to the evaluation target.")
   systemMessage: string;
 
@@ -103,6 +117,30 @@ model EvaluationTarget {
   modelParams?: Record<unknown>;
 }
 
+#suppress "@azure-tools/typespec-azure-core/no-string-discriminator"
+@doc("Evaluation target for model-based evaluations.")
+@added(Versions.v2025_05_15_preview)
+@removed(Versions.v_latest)
+model RedTeamTarget extends EvaluationTarget {
+  @visibility(Lifecycle.Read, Lifecycle.Create)
+  type: "redteam";
+
+  @doc("Number of simulation rounds.")
+  numTurns: int32;
+
+  @doc("List of attack strategies or nested lists of attack strategies.")
+  attackStrategies: AttackStrategy[];
+
+  @doc("Simulation-only or Simulation + Evaluation. Default false, if true the scan outputs conversation not evaluation result.")
+  simulationOnly: boolean;
+
+  @doc("List of risk categories to generate attack objectives for.")
+  riskCategories: RiskCategory[];
+
+  @doc("Application scenario for the red team operation, to generate scenario specific attacks.")
+  applicationScenario?: string;
+}
+
 @doc("Evaluation Definition")
 @resource("runs")
 @added(Versions.v2025_05_15_preview)

specification/ai/Azure.AI.Projects/evaluations/routes.tsp

@@ -121,6 +121,7 @@ interface Evaluations {
 
   #suppress "@azure-tools/typespec-azure-core/use-standard-operations"
   #suppress "@azure-tools/typespec-azure-core/no-closed-literal-union"
+  #suppress "@azure-tools/typespec-autorest/union-unsupported" ""
   @doc("Poll for the operation results.")
   @route("/operations/{operationId}")
   @get
@@ -138,7 +139,8 @@ interface Evaluations {
 
     @doc("The operation results.")
     @body
-    operationResults: Record<unknown>[];
+
+    operationResults: Record<unknown>[] | ChatCompletions;
   };
 
   // Upload API set
@@ -158,6 +160,7 @@ interface Evaluations {
     Evaluation
   >;
 
+  #suppress "@typespec/http/patch-implicit-optional" ""
   #suppress "@azure-tools/typespec-azure-core/use-standard-operations"
   @doc("Update the uploaded the result to an evaluation run.")
   @route("runs:updateUpload/{name}")
@@ -177,4 +180,91 @@ interface Evaluations {
     },
     Evaluation
   >;
+
+  // Simulation Private APIs
+  #suppress "@azure-tools/typespec-azure-core/use-standard-operations" "Waiting for bug fix: https://github.com/Azure/typespec-azure-pr/issues/3739."
+  @doc("Get the jailbreak dataset with type.")
+  @route("/simulation/jailbreak/{type}")
+  @get
+  getJailBreakDatasetWithType is Azure.Core.Foundations.Operation<
+    {
+      @doc("Type of jailbreak dataset")
+      @path
+      type: string;
+    },
+    Array<string>
+  >;
+
+  #suppress "@azure-tools/typespec-azure-core/use-standard-operations" "Waiting for bug fix: https://github.com/Azure/typespec-azure-pr/issues/3739."
+  #suppress "@azure-tools/typespec-azure-core/no-query-explode"
+  @doc("Get the attack objectives.")
+  @route("/simulation/attackobjectives")
+  @get
+  getAttackObjectives is Azure.Core.Foundations.Operation<
+    {
+      @doc("Risk types for the attack objectives dataset")
+      @query(#{ explode: true })
+      riskTypes?: string[];
+
+      @doc("The language for the attack objectives dataset, defaults to 'en'")
+      @query
+      lang?: string;
+
+      @doc("The strategy")
+      @query
+      strategy?: string;
+    },
+    Array<AttackObjective>
+  >;
+
+  #suppress "@azure-tools/typespec-azure-core/use-standard-operations" "Waiting for bug fix: https://github.com/Azure/typespec-azure-pr/issues/3739."
+  @doc("Get the jailbreak dataset.")
+  @route("/simulation/jailbreak/")
+  @get
+  getJailBreakDataset is Azure.Core.Foundations.Operation<{}, Array<string>>;
+
+  #suppress "@azure-tools/typespec-azure-core/use-standard-operations" "Waiting for bug fix: https://github.com/Azure/typespec-azure-pr/issues/3739."
+  @doc("Get template parameters with type.")
+  @route("/simulation/template/parameters/{type}")
+  @get
+  getTemplateParametersWithType is Azure.Core.Foundations.Operation<
+    {
+      @doc("Type for the template parameters")
+      @path
+      type: string;
+    },
+    string
+  >;
+
+  #suppress "@azure-tools/typespec-azure-core/use-standard-operations" "Waiting for bug fix: https://github.com/Azure/typespec-azure-pr/issues/3739."
+  @doc("Get template parameters.")
+  @route("/simulation/template/parameters/")
+  @get
+  getTemplateParameters is Azure.Core.Foundations.Operation<{}, string>;
+
+  #suppress "@azure-tools/typespec-azure-core/use-standard-operations" "Waiting for bug fix: https://github.com/Azure/typespec-azure-pr/issues/3739."
+  @doc("Get the template parameters image.")
+  @route("/simulation/template/parameters/image")
+  @get
+  getTemplateParametersImage is Azure.Core.Foundations.Operation<
+    {
+      @doc("Image path.")
+      @query
+      path: string;
+    },
+    string
+  >;
+
+  #suppress "@azure-tools/typespec-azure-core/use-standard-operations" "Waiting for bug fix: https://github.com/Azure/typespec-azure-pr/issues/3739."
+  @doc("Submit a request for simulation.")
+  @route("/simulation/chat/completions/submit")
+  @post
+  submitSimulation is Azure.Core.Foundations.Operation<
+    {
+      @doc("Properties of a Prompt Version.")
+      @body
+      body: SimulationDTO;
+    },
+    ResourceCreatedOrOkResponse<LongRunningResponse>
+  >;
 }

specification/ai/Azure.AI.Projects/red-teams/routes.tsp

@@ -62,6 +62,7 @@ interface RedTeams {
     RedTeam
   >;
 
+  #suppress "@typespec/http/patch-implicit-optional" ""
   #suppress "@azure-tools/typespec-azure-core/use-standard-operations"
   @doc("Update the uploaded the result to an redteam run.")
   @route("runs:updateUpload/{name}")

specification/ai/Azure.AI.Projects/servicepatterns.tsp

@@ -72,6 +72,7 @@ namespace Azure.AI.Projects.ServicePatterns {
     >;
     */
 
+    #suppress "@typespec/http/patch-implicit-optional" ""
     @doc(
       "Create a new or update an existing {name} with the given version id",
       TEntityType