From c86b79a528a58573a1b8c6363c68ffeac6152c01 Mon Sep 17 00:00:00 2001 From: Pratibha Shrivastav Date: Mon, 8 Dec 2025 20:28:50 +0530 Subject: [PATCH 1/6] add resume test --- .../tests/FineTuning/FineTuningTests.cs | 27 +++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/sdk/ai/Azure.AI.Projects/tests/FineTuning/FineTuningTests.cs b/sdk/ai/Azure.AI.Projects/tests/FineTuning/FineTuningTests.cs index 999850b89087..48ad815bb6fa 100644 --- a/sdk/ai/Azure.AI.Projects/tests/FineTuning/FineTuningTests.cs +++ b/sdk/ai/Azure.AI.Projects/tests/FineTuning/FineTuningTests.cs @@ -455,6 +455,33 @@ public async Task Test_FineTuning_Pause_Job() Assert.That(pausedJob.JobId, Is.EqualTo(runningJobId)); } + [RecordedTest] + public async Task Test_FineTuning_Resume_Job() + { + // Note: This test uses a paused fine-tuning job ID to test resume functionality. + // Resume is only valid for jobs that are currently paused. + // When re-recording this test, ensure the job is in a paused state. + string pausedJobId = "ftjob-5053b0f026604ac59b4a0ef2dbece2fc"; + + var (_, fineTuningClient) = GetClients(); + + // Retrieve the job first + FineTuningJob job = await fineTuningClient.GetJobAsync(pausedJobId); + Console.WriteLine($"Retrieved job: {job.JobId}, Status: {job.Status}"); + + // Resume the job + Console.WriteLine($"Resuming fine-tuning job with ID: {pausedJobId}"); + await fineTuningClient.ResumeFineTuningJobAsync(pausedJobId, options: null); + + // Retrieve the job again to verify status + FineTuningJob resumedJob = await fineTuningClient.GetJobAsync(pausedJobId); + Console.WriteLine($"Resumed job: {resumedJob.JobId}, Status: {resumedJob.Status}"); + + // Verify the job is resumed (status should be "running" or "queued") + Assert.That(resumedJob, Is.Not.Null); + Assert.That(resumedJob.JobId, Is.EqualTo(pausedJobId)); + } + [RecordedTest] public async Task Test_FineTuning_List_Events() { From 7ac643705aec881e0b2648448ce4f7f4fe8b1b8a Mon Sep 17 00:00:00 2001 From: Pratibha Shrivastav <164305667+PratibhaShrivastav18@users.noreply.github.com> Date: Tue, 9 Dec 2025 14:08:01 +0530 Subject: [PATCH 2/6] Update sdk/ai/Azure.AI.Projects/tests/FineTuning/FineTuningTests.cs Co-authored-by: Nikolay Rovinskiy <30440255+nick863@users.noreply.github.com> --- sdk/ai/Azure.AI.Projects/tests/FineTuning/FineTuningTests.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdk/ai/Azure.AI.Projects/tests/FineTuning/FineTuningTests.cs b/sdk/ai/Azure.AI.Projects/tests/FineTuning/FineTuningTests.cs index 48ad815bb6fa..65ec209fbf5c 100644 --- a/sdk/ai/Azure.AI.Projects/tests/FineTuning/FineTuningTests.cs +++ b/sdk/ai/Azure.AI.Projects/tests/FineTuning/FineTuningTests.cs @@ -475,7 +475,7 @@ public async Task Test_FineTuning_Resume_Job() // Retrieve the job again to verify status FineTuningJob resumedJob = await fineTuningClient.GetJobAsync(pausedJobId); - Console.WriteLine($"Resumed job: {resumedJob.JobId}, Status: {resumedJob.Status}"); + Assert.That(resumedJob.Status == FineTuningStatus.Running || resumedJob.Status == FineTuningStatus.Queued, $"The job has wrong status {resumedJob.Status}"); // Verify the job is resumed (status should be "running" or "queued") Assert.That(resumedJob, Is.Not.Null); From dcf9676b9464767b8472aa47d25438968698d0dc Mon Sep 17 00:00:00 2001 From: Pratibha Shrivastav Date: Thu, 11 Dec 2025 01:02:42 +0530 Subject: [PATCH 3/6] add diff training type tests --- sdk/ai/Azure.AI.Projects/assets.json | 2 +- .../tests/FineTuning/FineTuningTests.cs | 376 +++++++++++++----- 2 files changed, 287 insertions(+), 91 deletions(-) diff --git a/sdk/ai/Azure.AI.Projects/assets.json b/sdk/ai/Azure.AI.Projects/assets.json index baf544df834d..b1170ea6bd8c 100644 --- a/sdk/ai/Azure.AI.Projects/assets.json +++ b/sdk/ai/Azure.AI.Projects/assets.json @@ -2,5 +2,5 @@ "AssetsRepo": "Azure/azure-sdk-assets", "AssetsRepoPrefixPath": "net", "TagPrefix": "net/ai/Azure.AI.Projects", - "Tag": "net/ai/Azure.AI.Projects_824ac44755" + "Tag": "net/ai/Azure.AI.Projects_d39afa2707" } diff --git a/sdk/ai/Azure.AI.Projects/tests/FineTuning/FineTuningTests.cs b/sdk/ai/Azure.AI.Projects/tests/FineTuning/FineTuningTests.cs index 65ec209fbf5c..f9405a19bd2d 100644 --- a/sdk/ai/Azure.AI.Projects/tests/FineTuning/FineTuningTests.cs +++ b/sdk/ai/Azure.AI.Projects/tests/FineTuning/FineTuningTests.cs @@ -98,46 +98,23 @@ private async Task CleanupTestFilesAsync(OpenAIFileClient fileClient, OpenAIFile } } - private async Task CreateSupervisedFineTuningJobAsync( - FineTuningClient fineTuningClient, - string modelName, - string trainFileId, - string validationFileId, - int epochCount = 1, - int batchSize = 4, - double learningRate = 0.0001) - { - return await fineTuningClient.FineTuneAsync( - modelName, - trainFileId, - waitUntilCompleted: false, - new() - { - TrainingMethod = FineTuningTrainingMethod.CreateSupervised( - epochCount: epochCount, - batchSize: batchSize, - learningRate: learningRate), - ValidationFile = validationFileId - }); - } - - private async Task CreateSupervisedFineTuningJobForOssModelAsync( + /// + /// Creates a SFT or DPO fine-tuning job using raw JSON (for trainingType support). + /// + private async Task CreateFineTuningJobWithRawJsonAsync( FineTuningClient fineTuningClient, string modelName, string trainFileId, string validationFileId, - string trainingType, + string jobType, + string trainingType = null, int epochCount = 1, int batchSize = 4, double learningRate = 0.0001) { - var requestJson = new + object methodObject = jobType.ToLowerInvariant() switch { - model = modelName, - training_file = trainFileId, - validation_file = validationFileId, - trainingType = trainingType, - method = new + "supervised" or "sft" => new { type = "supervised", supervised = new @@ -149,23 +126,95 @@ private async Task CreateSupervisedFineTuningJobForOssModelAsync( learning_rate_multiplier = learningRate } } - } + }, + "dpo" => new + { + type = "dpo", + dpo = new + { + hyperparameters = new + { + n_epochs = epochCount, + batch_size = batchSize, + learning_rate_multiplier = learningRate + } + } + }, + _ => throw new ArgumentException($"Unknown job type: {jobType}. Use CreateRftFineTuningJobAsync for RFT jobs.", nameof(jobType)) + }; + + var requestJson = new Dictionary + { + ["model"] = modelName, + ["training_file"] = trainFileId, + ["validation_file"] = validationFileId, + ["method"] = methodObject }; + if (!string.IsNullOrEmpty(trainingType)) + { + requestJson["trainingType"] = trainingType; + } + string jsonString = JsonSerializer.Serialize(requestJson); BinaryContent content = BinaryContent.Create(BinaryData.FromString(jsonString)); return await fineTuningClient.FineTuneAsync(content, waitUntilCompleted: false, options: null); } + private async Task CreateSupervisedFineTuningJobAsync( + FineTuningClient fineTuningClient, + string modelName, + string trainFileId, + string validationFileId, + string trainingType = null, + int epochCount = 1, + int batchSize = 4, + double learningRate = 0.0001) + { + // Use raw JSON if trainingType is specified (required for Azure-specific field and OSS models) + if (!string.IsNullOrEmpty(trainingType)) + { + return await CreateFineTuningJobWithRawJsonAsync( + fineTuningClient, modelName, trainFileId, validationFileId, + jobType: "sft", trainingType: trainingType, + epochCount: epochCount, batchSize: batchSize, learningRate: learningRate); + } + + // Default: use the SDK's typed API + return await fineTuningClient.FineTuneAsync( + modelName, + trainFileId, + waitUntilCompleted: false, + new() + { + TrainingMethod = FineTuningTrainingMethod.CreateSupervised( + epochCount: epochCount, + batchSize: batchSize, + learningRate: learningRate), + ValidationFile = validationFileId + }); + } + private async Task CreateDpoFineTuningJobAsync( FineTuningClient fineTuningClient, string modelName, string trainFileId, string validationFileId, + string trainingType = null, int epochCount = 1, int batchSize = 4, double learningRate = 0.0001) { + // Use raw JSON if trainingType is specified (required for Azure-specific field) + if (!string.IsNullOrEmpty(trainingType)) + { + return await CreateFineTuningJobWithRawJsonAsync( + fineTuningClient, modelName, trainFileId, validationFileId, + jobType: "dpo", trainingType: trainingType, + epochCount: epochCount, batchSize: batchSize, learningRate: learningRate); + } + + // Default: use the SDK's typed API return await fineTuningClient.FineTuneAsync( modelName, trainFileId, @@ -180,6 +229,159 @@ private async Task CreateDpoFineTuningJobAsync( }); } + private async Task CreateRftFineTuningJobAsync( + FineTuningClient fineTuningClient, + string modelName, + string trainFileId, + string validationFileId, + string trainingType = null, + int epochCount = 1, + int batchSize = 4, + double learningRate = 2, + int evalInterval = 5, + int evalSamples = 2, + string reasoningEffort = "medium") + { + // RFT uses raw JSON with its own structure (grader + RFT-specific hyperparameters) + var methodObject = new + { + type = "reinforcement", + reinforcement = new + { + grader = new + { + type = "score_model", + name = "o3-mini", + model = "o3-mini", + input = new[] + { + new + { + role = "user", + content = "Evaluate the model's response based on correctness and quality. Rate from 0 to 10." + } + }, + range = new[] { 0.0, 10.0 } + }, + hyperparameters = new + { + n_epochs = epochCount, + batch_size = batchSize, + learning_rate_multiplier = learningRate, + eval_interval = evalInterval, + eval_samples = evalSamples, + reasoning_effort = reasoningEffort + } + } + }; + + var requestJson = new Dictionary + { + ["model"] = modelName, + ["training_file"] = trainFileId, + ["validation_file"] = validationFileId, + ["method"] = methodObject + }; + + if (!string.IsNullOrEmpty(trainingType)) + { + requestJson["trainingType"] = trainingType; + } + + string jsonString = JsonSerializer.Serialize(requestJson); + BinaryContent content = BinaryContent.Create(BinaryData.FromString(jsonString)); + return await fineTuningClient.FineTuneAsync(content, waitUntilCompleted: false, options: null); + } + + private async Task RunSftCreateJobTestAsync(string trainingType) + { + var (fileClient, fineTuningClient) = GetClients(); + var (trainFile, validationFile) = await UploadTestFilesAsync(fileClient, "sft"); + + try + { + FineTuningJob fineTuningJob = await CreateSupervisedFineTuningJobAsync( + fineTuningClient, + "gpt-4.1", + trainFile.Id, + validationFile.Id, + trainingType: trainingType, + epochCount: 1, + batchSize: 4, + learningRate: 0.0001); + + Console.WriteLine($"Created SFT job with {trainingType} training type: {fineTuningJob.JobId}"); + ValidateFineTuningJob(fineTuningJob); + + // Cancel the job + await fineTuningJob.CancelAndUpdateAsync(); + Console.WriteLine($"Cancelled job: {fineTuningJob.JobId}"); + } + finally + { + await CleanupTestFilesAsync(fileClient, trainFile, validationFile); + } + } + + private async Task RunDpoCreateJobTestAsync(string trainingType) + { + var (fileClient, fineTuningClient) = GetClients(); + var (trainFile, validationFile) = await UploadTestFilesAsync(fileClient, "dpo"); + + try + { + FineTuningJob fineTuningJob = await CreateDpoFineTuningJobAsync( + fineTuningClient, + "gpt-4o-mini", + trainFile.Id, + validationFile.Id, + trainingType: trainingType, + epochCount: 1, + batchSize: 4, + learningRate: 0.0001); + + Console.WriteLine($"Created DPO job with {trainingType} training type: {fineTuningJob.JobId}"); + ValidateFineTuningJob(fineTuningJob); + + // Cancel the job + await fineTuningJob.CancelAndUpdateAsync(); + Console.WriteLine($"Cancelled job: {fineTuningJob.JobId}"); + } + finally + { + await CleanupTestFilesAsync(fileClient, trainFile, validationFile); + } + } + + private async Task RunRftCreateJobTestAsync(string trainingType) + { + TestTimeoutInSeconds = 120; // Increase timeout to 2 minutes for RFT job operations + + var (fileClient, fineTuningClient) = GetClients(); + var (trainFile, validationFile) = await UploadTestFilesAsync(fileClient, "rft"); + + try + { + FineTuningJob fineTuningJob = await CreateRftFineTuningJobAsync( + fineTuningClient, + "o4-mini", + trainFile.Id, + validationFile.Id, + trainingType: trainingType); + + Console.WriteLine($"Created RFT job with {trainingType} training type: {fineTuningJob.JobId}"); + ValidateFineTuningJob(fineTuningJob); + + // Cancel the job + await fineTuningJob.CancelAndUpdateAsync(); + Console.WriteLine($"Cancelled job: {fineTuningJob.JobId}"); + } + finally + { + await CleanupTestFilesAsync(fileClient, trainFile, validationFile); + } + } + [RecordedTest] public async Task Test_Sft_FineTuning_Create_Job() { @@ -210,6 +412,24 @@ public async Task Test_Sft_FineTuning_Create_Job() } } + [RecordedTest] + public async Task Test_Sft_FineTuning_Create_Job_OpenAI_Standard() + { + await RunSftCreateJobTestAsync("Standard"); + } + + [RecordedTest] + public async Task Test_Sft_FineTuning_Create_Job_OpenAI_Developer() + { + await RunSftCreateJobTestAsync("developerTier"); + } + + [RecordedTest] + public async Task Test_Sft_FineTuning_Create_Job_OpenAI_GlobalStandard() + { + await RunSftCreateJobTestAsync("GlobalStandard"); + } + [RecordedTest] public async Task Test_Sft_FineTuning_Create_Job_Oss_Model() { @@ -218,7 +438,7 @@ public async Task Test_Sft_FineTuning_Create_Job_Oss_Model() try { - FineTuningJob fineTuningJob = await CreateSupervisedFineTuningJobForOssModelAsync( + FineTuningJob fineTuningJob = await CreateSupervisedFineTuningJobAsync( fineTuningClient, "Ministral-3B", trainFile.Id, @@ -272,69 +492,45 @@ public async Task Test_Dpo_FineTuning_Create_Job() } [RecordedTest] - public async Task Test_Rft_FineTuning_Create_Job() + public async Task Test_Dpo_FineTuning_Create_Job_OpenAI_Standard() { - TestTimeoutInSeconds = 120; // Increase timeout to 2 minutes for RFT job operations + await RunDpoCreateJobTestAsync("Standard"); + } - var (fileClient, fineTuningClient) = GetClients(); - var (trainFile, validationFile) = await UploadTestFilesAsync(fileClient, "rft"); + [RecordedTest] + public async Task Test_Dpo_FineTuning_Create_Job_OpenAI_Developer() + { + await RunDpoCreateJobTestAsync("developerTier"); + } - // Build the JSON request manually since RL APIs are internal - var requestJson = new - { - model = "o4-mini", - training_file = trainFile.Id, - validation_file = validationFile.Id, - method = new - { - type = "reinforcement", - reinforcement = new - { - grader = new - { - type = "score_model", - name = "o3-mini", - model = "o3-mini", - input = new[] - { - new - { - role = "user", - content = "Evaluate the model's response based on correctness and quality. Rate from 0 to 10." - } - }, - range = new[] { 0.0, 10.0 } - }, - hyperparameters = new - { - n_epochs = 1, - batch_size = 4, - learning_rate_multiplier = 2, - eval_interval = 5, - eval_samples = 2, - reasoning_effort = "medium" - } - } - } - }; + [RecordedTest] + public async Task Test_Dpo_FineTuning_Create_Job_OpenAI_GlobalStandard() + { + await RunDpoCreateJobTestAsync("GlobalStandard"); + } - try - { - string jsonString = JsonSerializer.Serialize(requestJson); - BinaryContent content = BinaryContent.Create(BinaryData.FromString(jsonString)); - FineTuningJob fineTuningJob = await fineTuningClient.FineTuneAsync(content, waitUntilCompleted: false, options: null); + [RecordedTest] + public async Task Test_Rft_FineTuning_Create_Job() + { + await RunRftCreateJobTestAsync(null); + } - Console.WriteLine($"Created RFT job: {fineTuningJob.JobId}"); - ValidateFineTuningJob(fineTuningJob); + [RecordedTest] + public async Task Test_Rft_FineTuning_Create_Job_OpenAI_Standard() + { + await RunRftCreateJobTestAsync("Standard"); + } - // Cancel the job - await fineTuningJob.CancelAndUpdateAsync(); - Console.WriteLine($"Cancelled job: {fineTuningJob.JobId}"); - } - finally - { - await CleanupTestFilesAsync(fileClient, trainFile, validationFile); - } + [RecordedTest] + public async Task Test_Rft_FineTuning_Create_Job_OpenAI_Developer() + { + await RunRftCreateJobTestAsync("developerTier"); + } + + [RecordedTest] + public async Task Test_Rft_FineTuning_Create_Job_OpenAI_GlobalStandard() + { + await RunRftCreateJobTestAsync("GlobalStandard"); } [RecordedTest] From 5975e40f992bd5e9debec63a28d295f545fd8760 Mon Sep 17 00:00:00 2001 From: Pratibha Shrivastav Date: Thu, 11 Dec 2025 01:29:19 +0530 Subject: [PATCH 4/6] update recordings --- sdk/ai/Azure.AI.Projects/assets.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdk/ai/Azure.AI.Projects/assets.json b/sdk/ai/Azure.AI.Projects/assets.json index b1170ea6bd8c..fcec46525574 100644 --- a/sdk/ai/Azure.AI.Projects/assets.json +++ b/sdk/ai/Azure.AI.Projects/assets.json @@ -2,5 +2,5 @@ "AssetsRepo": "Azure/azure-sdk-assets", "AssetsRepoPrefixPath": "net", "TagPrefix": "net/ai/Azure.AI.Projects", - "Tag": "net/ai/Azure.AI.Projects_d39afa2707" + "Tag": "net/ai/Azure.AI.Projects_582571cc09" } From 469ee48d64caa4607f2681f00bd239ad5aca1bb6 Mon Sep 17 00:00:00 2001 From: Pratibha Shrivastav Date: Thu, 11 Dec 2025 12:51:26 +0530 Subject: [PATCH 5/6] update status to resuming also --- sdk/ai/Azure.AI.Projects/tests/FineTuning/FineTuningTests.cs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/sdk/ai/Azure.AI.Projects/tests/FineTuning/FineTuningTests.cs b/sdk/ai/Azure.AI.Projects/tests/FineTuning/FineTuningTests.cs index f9405a19bd2d..e25c4ebe9c11 100644 --- a/sdk/ai/Azure.AI.Projects/tests/FineTuning/FineTuningTests.cs +++ b/sdk/ai/Azure.AI.Projects/tests/FineTuning/FineTuningTests.cs @@ -671,9 +671,10 @@ public async Task Test_FineTuning_Resume_Job() // Retrieve the job again to verify status FineTuningJob resumedJob = await fineTuningClient.GetJobAsync(pausedJobId); - Assert.That(resumedJob.Status == FineTuningStatus.Running || resumedJob.Status == FineTuningStatus.Queued, $"The job has wrong status {resumedJob.Status}"); + var validStatuses = new[] { "running", "queued", "resuming" }; // Using string comparison directly since resuming status is not available in FineTuningStatus enum + Assert.That(validStatuses.Contains(resumedJob.Status.ToString().ToLowerInvariant()), $"The job has wrong status {resumedJob.Status}"); - // Verify the job is resumed (status should be "running" or "queued") + // Verify the job is resumed (status should be "running", "queued", or "resuming") Assert.That(resumedJob, Is.Not.Null); Assert.That(resumedJob.JobId, Is.EqualTo(pausedJobId)); } From 3444d0a215a56d5ff0560750929cd707cba3f7c9 Mon Sep 17 00:00:00 2001 From: Pratibha Shrivastav Date: Thu, 11 Dec 2025 14:27:48 +0530 Subject: [PATCH 6/6] fix list jobs test --- sdk/ai/Azure.AI.Projects/tests/FineTuning/FineTuningTests.cs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/sdk/ai/Azure.AI.Projects/tests/FineTuning/FineTuningTests.cs b/sdk/ai/Azure.AI.Projects/tests/FineTuning/FineTuningTests.cs index e25c4ebe9c11..23d2dae956aa 100644 --- a/sdk/ai/Azure.AI.Projects/tests/FineTuning/FineTuningTests.cs +++ b/sdk/ai/Azure.AI.Projects/tests/FineTuning/FineTuningTests.cs @@ -565,6 +565,8 @@ public async Task Test_FineTuning_Retrieve_Job() [RecordedTest] public async Task Test_FineTuning_List_Jobs() { + TestTimeoutInSeconds = 120; // Increase timeout for listing jobs + var (fileClient, fineTuningClient) = GetClients(); var (trainFile, validationFile) = await UploadTestFilesAsync(fileClient, "sft");