From 2f04c662f31d9b7b74e35621a1eba5c0fd9102a4 Mon Sep 17 00:00:00 2001 From: "Ray Chen (from Dev Box)" Date: Fri, 15 Mar 2024 17:49:06 +0800 Subject: [PATCH 01/64] Added pipeline to build embeddings --- .../.pipelines/BuildDocumentEmbeddings.yml | 47 +++++++++++++++++++ 1 file changed, 47 insertions(+) create mode 100644 tools/sdk-ai-bots/.pipelines/BuildDocumentEmbeddings.yml diff --git a/tools/sdk-ai-bots/.pipelines/BuildDocumentEmbeddings.yml b/tools/sdk-ai-bots/.pipelines/BuildDocumentEmbeddings.yml new file mode 100644 index 00000000000..3d225b2ba50 --- /dev/null +++ b/tools/sdk-ai-bots/.pipelines/BuildDocumentEmbeddings.yml @@ -0,0 +1,47 @@ +trigger: none + +schedules: +- cron: "0 0 * * *" + displayName: Daily midnight build + branches: + include: + - main + always: true + +pool: + vmImage: 'windows-latest' + +variables: + aoai-endpoint: '' + aoai-key: '' + asch-key: '' + asch-endpoint: '' + asch-index-name: '' + aoai-embedding-model: '' + +parameters: +- name: incrementalEmbedding + displayName: 'Incremental Embedding Build?' + type: boolean + default: true + +stages: +- stage: BuildEngHubDocEmbeddings + displayName: 'Build EngHub Document Embeddings' + jobs: + - job: Build + steps: + - task: UsePythonVersion@0 + inputs: + versionSpec: '3.x' + addToPath: true + - script: | + "/tools/sdk-ai-bots/Scripts/Build-EngHubDocEmbeddings.ps1 -IncrementalEmbedding $(incrementalEmbedding)" + displayName: 'Run embeddings build script' + env: + AZURE_OPENAI_ENDPOINT: $(aoai-endpoint) + AZURE_OPENAI_API_KEY: $(aoai-key) + AZURE_SEARCH_ENDPOINT: $(asch-key) + AZURE_SEARCH_KEY: $(asch-endpoint) + AZURE_SEARCH_INDEX_NAME: $(asch-index-name) + AZURE_OPENAI_EMBEDDING_MODEL: $(aoai-embedding-model) \ No newline at end of file From e50b402a5d3dececfc8e62d5b1edefa283fa7edc Mon Sep 17 00:00:00 2001 From: "Ray Chen (from Dev Box)" Date: Fri, 15 Mar 2024 18:15:12 +0800 Subject: [PATCH 02/64] Added secret variables --- .../.pipelines/BuildDocumentEmbeddings.yml | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/tools/sdk-ai-bots/.pipelines/BuildDocumentEmbeddings.yml b/tools/sdk-ai-bots/.pipelines/BuildDocumentEmbeddings.yml index 3d225b2ba50..908c067613d 100644 --- a/tools/sdk-ai-bots/.pipelines/BuildDocumentEmbeddings.yml +++ b/tools/sdk-ai-bots/.pipelines/BuildDocumentEmbeddings.yml @@ -12,12 +12,10 @@ pool: vmImage: 'windows-latest' variables: - aoai-endpoint: '' - aoai-key: '' - asch-key: '' - asch-endpoint: '' - asch-index-name: '' - aoai-embedding-model: '' + aoai-endpoint: $(azure-openai-endpoint) + asch-endpoint: $(azure-search-endpoint) + asch-index-name: $(azure-search-index-name) + aoai-embedding-model: $(azure-openai-embedding-model) parameters: - name: incrementalEmbedding @@ -40,8 +38,9 @@ stages: displayName: 'Run embeddings build script' env: AZURE_OPENAI_ENDPOINT: $(aoai-endpoint) - AZURE_OPENAI_API_KEY: $(aoai-key) - AZURE_SEARCH_ENDPOINT: $(asch-key) - AZURE_SEARCH_KEY: $(asch-endpoint) + AZURE_SEARCH_ENDPOINT: $(asch-endpoint) AZURE_SEARCH_INDEX_NAME: $(asch-index-name) - AZURE_OPENAI_EMBEDDING_MODEL: $(aoai-embedding-model) \ No newline at end of file + AZURE_OPENAI_EMBEDDING_MODEL: $(aoai-embedding-model) + AZURE_OPENAI_API_KEY: $(azure-openapi-key) + AZURE_SEARCH_KEY: $(azure-search-key) + AZURE_STORAGE_ACCOUNT_KEY: $(storage-account-key) \ No newline at end of file From 299a6f0a011a4657f4853d264c9e70a1515e1699 Mon Sep 17 00:00:00 2001 From: "Ray Chen (from Dev Box)" Date: Fri, 15 Mar 2024 21:47:42 +0800 Subject: [PATCH 03/64] Updated nuget.config --- tools/sdk-ai-bots/AzureSdkQaBot/Nuget.Config | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/tools/sdk-ai-bots/AzureSdkQaBot/Nuget.Config b/tools/sdk-ai-bots/AzureSdkQaBot/Nuget.Config index c05ac833aa3..6de7d4c1286 100644 --- a/tools/sdk-ai-bots/AzureSdkQaBot/Nuget.Config +++ b/tools/sdk-ai-bots/AzureSdkQaBot/Nuget.Config @@ -2,14 +2,7 @@ - - - + + - - - - - - \ No newline at end of file From b60c178a12a7b4c26c63f2c167f91a565808559c Mon Sep 17 00:00:00 2001 From: "Ray Chen (from Dev Box)" Date: Fri, 15 Mar 2024 22:05:24 +0800 Subject: [PATCH 04/64] Updated yml file --- tools/sdk-ai-bots/.pipelines/BuildDocumentEmbeddings.yml | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/tools/sdk-ai-bots/.pipelines/BuildDocumentEmbeddings.yml b/tools/sdk-ai-bots/.pipelines/BuildDocumentEmbeddings.yml index 908c067613d..856a05c7d0d 100644 --- a/tools/sdk-ai-bots/.pipelines/BuildDocumentEmbeddings.yml +++ b/tools/sdk-ai-bots/.pipelines/BuildDocumentEmbeddings.yml @@ -33,8 +33,13 @@ stages: inputs: versionSpec: '3.x' addToPath: true - - script: | - "/tools/sdk-ai-bots/Scripts/Build-EngHubDocEmbeddings.ps1 -IncrementalEmbedding $(incrementalEmbedding)" + - task: Powershell@2 + inputs: + filePath: $(Build.SourcesDirectory)/tools/sdk-ai-bots/Scripts/Build-EngHubDocEmbeddings.ps1 + arguments: > + -IncrementalEmbedding "${{ parameters.incrementalEmbedding }}" + pwsh: true + workingDirectory: $(Pipeline.Workspace) displayName: 'Run embeddings build script' env: AZURE_OPENAI_ENDPOINT: $(aoai-endpoint) From b1cf977f88cc76f8eaeccc4b6e026f11e7278446 Mon Sep 17 00:00:00 2001 From: "Ray Chen (from Dev Box)" Date: Fri, 15 Mar 2024 22:17:45 +0800 Subject: [PATCH 05/64] Updated working directory in the script --- tools/sdk-ai-bots/Scripts/Build-EngHubDocEmbeddings.ps1 | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/tools/sdk-ai-bots/Scripts/Build-EngHubDocEmbeddings.ps1 b/tools/sdk-ai-bots/Scripts/Build-EngHubDocEmbeddings.ps1 index 7169ac70c36..164204edb58 100644 --- a/tools/sdk-ai-bots/Scripts/Build-EngHubDocEmbeddings.ps1 +++ b/tools/sdk-ai-bots/Scripts/Build-EngHubDocEmbeddings.ps1 @@ -18,11 +18,7 @@ param ( [string] $IncrementalEmbedding = $true ) -$workingDirectory = Get-Location -if($env:AGENT_ID) { - $workingDirectory = $(System.DefaultWorkingDirectory) -} -$workingDirectory = Join-Path $workingDirectory "tools\sdk-ai-bots" +$workingDirectory = Join-Path Get-Location "tools\sdk-ai-bots" $scriptsRoot = Join-Path $workingDirectory "Scripts" $embeddingToolFolder = Join-Path $workingDirectory "Embeddings" From f2e8a090a0d620beebb0ed9add916797ec5a6ae7 Mon Sep 17 00:00:00 2001 From: "Ray Chen (from Dev Box)" Date: Fri, 15 Mar 2024 22:26:36 +0800 Subject: [PATCH 06/64] Fixed script issue --- tools/sdk-ai-bots/Scripts/Build-EngHubDocEmbeddings.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/sdk-ai-bots/Scripts/Build-EngHubDocEmbeddings.ps1 b/tools/sdk-ai-bots/Scripts/Build-EngHubDocEmbeddings.ps1 index 164204edb58..867160dd762 100644 --- a/tools/sdk-ai-bots/Scripts/Build-EngHubDocEmbeddings.ps1 +++ b/tools/sdk-ai-bots/Scripts/Build-EngHubDocEmbeddings.ps1 @@ -18,7 +18,7 @@ param ( [string] $IncrementalEmbedding = $true ) -$workingDirectory = Join-Path Get-Location "tools\sdk-ai-bots" +$workingDirectory = Join-Path (Get-Location) "tools\sdk-ai-bots" $scriptsRoot = Join-Path $workingDirectory "Scripts" $embeddingToolFolder = Join-Path $workingDirectory "Embeddings" From 0e78a03a8846da36cda2290dddac5a9321f19974 Mon Sep 17 00:00:00 2001 From: "Ray Chen (from Dev Box)" Date: Fri, 15 Mar 2024 22:41:44 +0800 Subject: [PATCH 07/64] Fixed script name case issue --- .../sdk-ai-bots/Scripts/Build-CustomizedDocEmbeddings.ps1 | 8 ++------ tools/sdk-ai-bots/Scripts/Build-EngHubDocEmbeddings.ps1 | 2 +- .../Scripts/Build-TypeSpecAzureDocEmbeddings.ps1 | 8 ++------ 3 files changed, 5 insertions(+), 13 deletions(-) diff --git a/tools/sdk-ai-bots/Scripts/Build-CustomizedDocEmbeddings.ps1 b/tools/sdk-ai-bots/Scripts/Build-CustomizedDocEmbeddings.ps1 index d0e217c87e3..67c4e5bb61e 100644 --- a/tools/sdk-ai-bots/Scripts/Build-CustomizedDocEmbeddings.ps1 +++ b/tools/sdk-ai-bots/Scripts/Build-CustomizedDocEmbeddings.ps1 @@ -62,15 +62,11 @@ function Download-GitHubFile { } -$workingDirectory = Get-Location -if($env:AGENT_ID) { - $workingDirectory = $(System.DefaultWorkingDirectory) -} -$workingDirectory = Join-Path $workingDirectory "tools\sdk-ai-bots" +$workingDirectory = Join-Path (Get-Location) "tools\sdk-ai-bots" $scriptsRoot = Join-Path $workingDirectory "Scripts" $embeddingToolFolder = Join-Path $workingDirectory "Embeddings" -. (Join-Path $scriptsRoot common.ps1) +. (Join-Path $scriptsRoot Common.ps1) Write-Host "scriptsRoot: $scriptsRoot" Write-Host "embeddingToolFolder: $embeddingToolFolder" diff --git a/tools/sdk-ai-bots/Scripts/Build-EngHubDocEmbeddings.ps1 b/tools/sdk-ai-bots/Scripts/Build-EngHubDocEmbeddings.ps1 index 867160dd762..1a4b1588425 100644 --- a/tools/sdk-ai-bots/Scripts/Build-EngHubDocEmbeddings.ps1 +++ b/tools/sdk-ai-bots/Scripts/Build-EngHubDocEmbeddings.ps1 @@ -22,7 +22,7 @@ $workingDirectory = Join-Path (Get-Location) "tools\sdk-ai-bots" $scriptsRoot = Join-Path $workingDirectory "Scripts" $embeddingToolFolder = Join-Path $workingDirectory "Embeddings" -. (Join-Path $scriptsRoot common.ps1) +. (Join-Path $scriptsRoot Common.ps1) Write-Host "scriptsRoot: $scriptsRoot" Write-Host "embeddingToolFolder: $embeddingToolFolder" diff --git a/tools/sdk-ai-bots/Scripts/Build-TypeSpecAzureDocEmbeddings.ps1 b/tools/sdk-ai-bots/Scripts/Build-TypeSpecAzureDocEmbeddings.ps1 index 8a9addd3ca8..0a9f6dd261e 100644 --- a/tools/sdk-ai-bots/Scripts/Build-TypeSpecAzureDocEmbeddings.ps1 +++ b/tools/sdk-ai-bots/Scripts/Build-TypeSpecAzureDocEmbeddings.ps1 @@ -19,15 +19,11 @@ param ( [string] $IncrementalEmbedding = $true ) -$workingDirectory = Get-Location -if($env:AGENT_ID) { - $workingDirectory = $(System.DefaultWorkingDirectory) -} -$workingDirectory = Join-Path $workingDirectory "tools\sdk-ai-bots" +$workingDirectory = Join-Path (Get-Location) "tools\sdk-ai-bots" $scriptsRoot = Join-Path $workingDirectory "Scripts" $embeddingToolFolder = Join-Path $workingDirectory "Embeddings" -. (Join-Path $scriptsRoot common.ps1) +. (Join-Path $scriptsRoot Common.ps1) Write-Host "scriptsRoot: $scriptsRoot" Write-Host "embeddingToolFolder: $embeddingToolFolder" From bcf84d1746ae6fc691b3197f0900fe27e5ac902d Mon Sep 17 00:00:00 2001 From: "Ray Chen (from Dev Box)" Date: Fri, 15 Mar 2024 22:49:02 +0800 Subject: [PATCH 08/64] Fixed working directory value --- tools/sdk-ai-bots/.pipelines/BuildDocumentEmbeddings.yml | 2 +- tools/sdk-ai-bots/Scripts/Build-EngHubDocEmbeddings.ps1 | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/tools/sdk-ai-bots/.pipelines/BuildDocumentEmbeddings.yml b/tools/sdk-ai-bots/.pipelines/BuildDocumentEmbeddings.yml index 856a05c7d0d..06152f22782 100644 --- a/tools/sdk-ai-bots/.pipelines/BuildDocumentEmbeddings.yml +++ b/tools/sdk-ai-bots/.pipelines/BuildDocumentEmbeddings.yml @@ -39,7 +39,7 @@ stages: arguments: > -IncrementalEmbedding "${{ parameters.incrementalEmbedding }}" pwsh: true - workingDirectory: $(Pipeline.Workspace) + workingDirectory: $(Build.SourcesDirectory) displayName: 'Run embeddings build script' env: AZURE_OPENAI_ENDPOINT: $(aoai-endpoint) diff --git a/tools/sdk-ai-bots/Scripts/Build-EngHubDocEmbeddings.ps1 b/tools/sdk-ai-bots/Scripts/Build-EngHubDocEmbeddings.ps1 index 1a4b1588425..e7946a11402 100644 --- a/tools/sdk-ai-bots/Scripts/Build-EngHubDocEmbeddings.ps1 +++ b/tools/sdk-ai-bots/Scripts/Build-EngHubDocEmbeddings.ps1 @@ -22,10 +22,9 @@ $workingDirectory = Join-Path (Get-Location) "tools\sdk-ai-bots" $scriptsRoot = Join-Path $workingDirectory "Scripts" $embeddingToolFolder = Join-Path $workingDirectory "Embeddings" -. (Join-Path $scriptsRoot Common.ps1) - Write-Host "scriptsRoot: $scriptsRoot" Write-Host "embeddingToolFolder: $embeddingToolFolder" +. (Join-Path $scriptsRoot Common.ps1) # Create 'repos' folder on current location $reposFolder = Join-Path -Path $workingDirectory -ChildPath "repos" From c0ca0c1b9a870649fca874c61c831741e809c4f2 Mon Sep 17 00:00:00 2001 From: "Ray Chen (from Dev Box)" Date: Fri, 15 Mar 2024 23:07:24 +0800 Subject: [PATCH 09/64] checkout enghub repo in yml --- tools/sdk-ai-bots/.pipelines/BuildDocumentEmbeddings.yml | 3 +++ tools/sdk-ai-bots/Scripts/Build-EngHubDocEmbeddings.ps1 | 6 ------ 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/tools/sdk-ai-bots/.pipelines/BuildDocumentEmbeddings.yml b/tools/sdk-ai-bots/.pipelines/BuildDocumentEmbeddings.yml index 06152f22782..c22abcdd831 100644 --- a/tools/sdk-ai-bots/.pipelines/BuildDocumentEmbeddings.yml +++ b/tools/sdk-ai-bots/.pipelines/BuildDocumentEmbeddings.yml @@ -33,6 +33,9 @@ stages: inputs: versionSpec: '3.x' addToPath: true + - checkout: git://azure-sdk/internal/_git/azure-sdk-docs-eng.ms + path: s/tools/sdk-ai-bots/repos/azure-sdk-docs-eng.ms + displayName: 'Checkout azure-sdk-docs-eng.ms repository' - task: Powershell@2 inputs: filePath: $(Build.SourcesDirectory)/tools/sdk-ai-bots/Scripts/Build-EngHubDocEmbeddings.ps1 diff --git a/tools/sdk-ai-bots/Scripts/Build-EngHubDocEmbeddings.ps1 b/tools/sdk-ai-bots/Scripts/Build-EngHubDocEmbeddings.ps1 index e7946a11402..a5596047144 100644 --- a/tools/sdk-ai-bots/Scripts/Build-EngHubDocEmbeddings.ps1 +++ b/tools/sdk-ai-bots/Scripts/Build-EngHubDocEmbeddings.ps1 @@ -32,12 +32,6 @@ if (-not (Test-Path -Path $reposFolder)) { New-Item -ItemType Directory -Path $reposFolder } -# Clone azure-sdk-docs-eng.ms repository -Write-Host "Cloning azure-sdk-docs-eng.ms repository at $reposFolder" -if(-not (Clone-Repository -RepoUrl "https://azure-sdk@dev.azure.com/azure-sdk/internal/_git/azure-sdk-docs-eng.ms" -RootFolder $reposFolder)) { - exit 1 -} - # Create embeddingSource folder on current location $embeddingSourceFolder = Join-Path -Path $workingDirectory -ChildPath "embeddingSource" if (-not (Test-Path -Path $embeddingSourceFolder)) { From c951e6ead53ad197c98fea5cdc8cc66c9ee421ca Mon Sep 17 00:00:00 2001 From: "Ray Chen (from Dev Box)" Date: Fri, 15 Mar 2024 23:16:30 +0800 Subject: [PATCH 10/64] updated the enghub checkout url --- tools/sdk-ai-bots/.pipelines/BuildDocumentEmbeddings.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/sdk-ai-bots/.pipelines/BuildDocumentEmbeddings.yml b/tools/sdk-ai-bots/.pipelines/BuildDocumentEmbeddings.yml index c22abcdd831..06261870ebd 100644 --- a/tools/sdk-ai-bots/.pipelines/BuildDocumentEmbeddings.yml +++ b/tools/sdk-ai-bots/.pipelines/BuildDocumentEmbeddings.yml @@ -33,7 +33,7 @@ stages: inputs: versionSpec: '3.x' addToPath: true - - checkout: git://azure-sdk/internal/_git/azure-sdk-docs-eng.ms + - checkout: git://internal/_git/azure-sdk-docs-eng.ms path: s/tools/sdk-ai-bots/repos/azure-sdk-docs-eng.ms displayName: 'Checkout azure-sdk-docs-eng.ms repository' - task: Powershell@2 From f8efa44650bd77fe48ea5ae4ae7533796403d18d Mon Sep 17 00:00:00 2001 From: "Ray Chen (from Dev Box)" Date: Fri, 15 Mar 2024 23:21:08 +0800 Subject: [PATCH 11/64] Checkout sdk tools repo --- tools/sdk-ai-bots/.pipelines/BuildDocumentEmbeddings.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/tools/sdk-ai-bots/.pipelines/BuildDocumentEmbeddings.yml b/tools/sdk-ai-bots/.pipelines/BuildDocumentEmbeddings.yml index 06261870ebd..54436bee171 100644 --- a/tools/sdk-ai-bots/.pipelines/BuildDocumentEmbeddings.yml +++ b/tools/sdk-ai-bots/.pipelines/BuildDocumentEmbeddings.yml @@ -33,6 +33,7 @@ stages: inputs: versionSpec: '3.x' addToPath: true + - checkout: self - checkout: git://internal/_git/azure-sdk-docs-eng.ms path: s/tools/sdk-ai-bots/repos/azure-sdk-docs-eng.ms displayName: 'Checkout azure-sdk-docs-eng.ms repository' From 48a99805f9787b17d15569b55d8f69bd08961639 Mon Sep 17 00:00:00 2001 From: "Ray Chen (from Dev Box)" Date: Sat, 16 Mar 2024 15:16:16 +0800 Subject: [PATCH 12/64] Updated repo folders --- .../Scripts/Build-EngHubDocEmbeddings.ps1 | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/tools/sdk-ai-bots/Scripts/Build-EngHubDocEmbeddings.ps1 b/tools/sdk-ai-bots/Scripts/Build-EngHubDocEmbeddings.ps1 index a5596047144..7423164c2b3 100644 --- a/tools/sdk-ai-bots/Scripts/Build-EngHubDocEmbeddings.ps1 +++ b/tools/sdk-ai-bots/Scripts/Build-EngHubDocEmbeddings.ps1 @@ -17,8 +17,8 @@ param ( [ValidateNotNullOrEmpty()] [string] $IncrementalEmbedding = $true ) - -$workingDirectory = Join-Path (Get-Location) "tools\sdk-ai-bots" +$buildSourceDirectory = Get-Location +$workingDirectory = Join-Path $buildSourceDirectory "azure-sdk-tools\tools\sdk-ai-bots" $scriptsRoot = Join-Path $workingDirectory "Scripts" $embeddingToolFolder = Join-Path $workingDirectory "Embeddings" @@ -26,12 +26,6 @@ Write-Host "scriptsRoot: $scriptsRoot" Write-Host "embeddingToolFolder: $embeddingToolFolder" . (Join-Path $scriptsRoot Common.ps1) -# Create 'repos' folder on current location -$reposFolder = Join-Path -Path $workingDirectory -ChildPath "repos" -if (-not (Test-Path -Path $reposFolder)) { - New-Item -ItemType Directory -Path $reposFolder -} - # Create embeddingSource folder on current location $embeddingSourceFolder = Join-Path -Path $workingDirectory -ChildPath "embeddingSource" if (-not (Test-Path -Path $embeddingSourceFolder)) { @@ -44,7 +38,7 @@ if (-not (Test-Path -Path $enghubDocsDestFolder)) { New-Item -ItemType Directory -Path $enghubDocsDestFolder } -$enghubDocsSrcFolder = Join-Path -Path $reposFolder -ChildPath "azure-sdk-docs-eng.ms/docs" +$enghubDocsSrcFolder = Join-Path -Path $buildSourceDirectory -ChildPath "azure-sdk-docs-eng.ms/docs" # Call the script to build the metadata.json file Write-Host "Building metadata.json file for enghub documents" From 9ebadac32089d809e539b1408007dd6a5b3d8f01 Mon Sep 17 00:00:00 2001 From: "Ray Chen (from Dev Box)" Date: Sat, 16 Mar 2024 15:24:10 +0800 Subject: [PATCH 13/64] Updated path of script file --- tools/sdk-ai-bots/.pipelines/BuildDocumentEmbeddings.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/sdk-ai-bots/.pipelines/BuildDocumentEmbeddings.yml b/tools/sdk-ai-bots/.pipelines/BuildDocumentEmbeddings.yml index 54436bee171..a49dbffd113 100644 --- a/tools/sdk-ai-bots/.pipelines/BuildDocumentEmbeddings.yml +++ b/tools/sdk-ai-bots/.pipelines/BuildDocumentEmbeddings.yml @@ -39,7 +39,7 @@ stages: displayName: 'Checkout azure-sdk-docs-eng.ms repository' - task: Powershell@2 inputs: - filePath: $(Build.SourcesDirectory)/tools/sdk-ai-bots/Scripts/Build-EngHubDocEmbeddings.ps1 + filePath: $(Build.SourcesDirectory)/azure-sdk-tools/tools/sdk-ai-bots/Scripts/Build-EngHubDocEmbeddings.ps1 arguments: > -IncrementalEmbedding "${{ parameters.incrementalEmbedding }}" pwsh: true From 74dfa3288aa078e1e9d2c7bcac85b72d5472b9ec Mon Sep 17 00:00:00 2001 From: "Ray Chen (from Dev Box)" Date: Sat, 16 Mar 2024 15:29:15 +0800 Subject: [PATCH 14/64] Fixed checkout path in the yml file --- tools/sdk-ai-bots/.pipelines/BuildDocumentEmbeddings.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/tools/sdk-ai-bots/.pipelines/BuildDocumentEmbeddings.yml b/tools/sdk-ai-bots/.pipelines/BuildDocumentEmbeddings.yml index a49dbffd113..d060334cf27 100644 --- a/tools/sdk-ai-bots/.pipelines/BuildDocumentEmbeddings.yml +++ b/tools/sdk-ai-bots/.pipelines/BuildDocumentEmbeddings.yml @@ -35,7 +35,6 @@ stages: addToPath: true - checkout: self - checkout: git://internal/_git/azure-sdk-docs-eng.ms - path: s/tools/sdk-ai-bots/repos/azure-sdk-docs-eng.ms displayName: 'Checkout azure-sdk-docs-eng.ms repository' - task: Powershell@2 inputs: From 94328e628c766b0b2d0d87ea74debe4ccbaa9cf7 Mon Sep 17 00:00:00 2001 From: "Ray Chen (from Dev Box)" Date: Sat, 16 Mar 2024 16:31:03 +0800 Subject: [PATCH 15/64] Introduce common jobs to setup env --- .../.pipelines/BuildDocumentEmbeddings.yml | 16 ++++++++++------ .../Scripts/Build-EngHubDocEmbeddings.ps1 | 5 +++++ 2 files changed, 15 insertions(+), 6 deletions(-) diff --git a/tools/sdk-ai-bots/.pipelines/BuildDocumentEmbeddings.yml b/tools/sdk-ai-bots/.pipelines/BuildDocumentEmbeddings.yml index d060334cf27..59a9834a2cb 100644 --- a/tools/sdk-ai-bots/.pipelines/BuildDocumentEmbeddings.yml +++ b/tools/sdk-ai-bots/.pipelines/BuildDocumentEmbeddings.yml @@ -23,19 +23,23 @@ parameters: type: boolean default: true -stages: -- stage: BuildEngHubDocEmbeddings - displayName: 'Build EngHub Document Embeddings' - jobs: - - job: Build +jobs: + - job: SetupEnvironment steps: - task: UsePythonVersion@0 inputs: versionSpec: '3.x' addToPath: true - checkout: self +stages: +- stage: BuildEngHubDocEmbeddings + displayName: 'Build EngHub Document Embeddings' + jobs: + - job: Build EngHub Document Embeddings + dependsOn: SetupEnvironment + steps: - checkout: git://internal/_git/azure-sdk-docs-eng.ms - displayName: 'Checkout azure-sdk-docs-eng.ms repository' + displayName: 'Checkout azure-sdk-docs-eng.ms repository' - task: Powershell@2 inputs: filePath: $(Build.SourcesDirectory)/azure-sdk-tools/tools/sdk-ai-bots/Scripts/Build-EngHubDocEmbeddings.ps1 diff --git a/tools/sdk-ai-bots/Scripts/Build-EngHubDocEmbeddings.ps1 b/tools/sdk-ai-bots/Scripts/Build-EngHubDocEmbeddings.ps1 index 7423164c2b3..76e33c316a8 100644 --- a/tools/sdk-ai-bots/Scripts/Build-EngHubDocEmbeddings.ps1 +++ b/tools/sdk-ai-bots/Scripts/Build-EngHubDocEmbeddings.ps1 @@ -26,6 +26,11 @@ Write-Host "scriptsRoot: $scriptsRoot" Write-Host "embeddingToolFolder: $embeddingToolFolder" . (Join-Path $scriptsRoot Common.ps1) +# Install Az.Storage module +if (-not (Get-Module -ListAvailable -Name Az.Storage)) { + Install-Module -Name Az.Storage -Force -AllowClobber -Scope CurrentUser +} + # Create embeddingSource folder on current location $embeddingSourceFolder = Join-Path -Path $workingDirectory -ChildPath "embeddingSource" if (-not (Test-Path -Path $embeddingSourceFolder)) { From 869776e1b9b111e37dc904f9a5ac1fdd01e6dd71 Mon Sep 17 00:00:00 2001 From: "Ray Chen (from Dev Box)" Date: Sat, 16 Mar 2024 16:34:38 +0800 Subject: [PATCH 16/64] Fixed syntax error in yml --- tools/sdk-ai-bots/.pipelines/BuildDocumentEmbeddings.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/tools/sdk-ai-bots/.pipelines/BuildDocumentEmbeddings.yml b/tools/sdk-ai-bots/.pipelines/BuildDocumentEmbeddings.yml index 59a9834a2cb..77a76a697cc 100644 --- a/tools/sdk-ai-bots/.pipelines/BuildDocumentEmbeddings.yml +++ b/tools/sdk-ai-bots/.pipelines/BuildDocumentEmbeddings.yml @@ -31,6 +31,7 @@ jobs: versionSpec: '3.x' addToPath: true - checkout: self + stages: - stage: BuildEngHubDocEmbeddings displayName: 'Build EngHub Document Embeddings' From 2040a01c45fdeab6a8eb2f227ab96203bb802362 Mon Sep 17 00:00:00 2001 From: "Ray Chen (from Dev Box)" Date: Sat, 16 Mar 2024 16:38:59 +0800 Subject: [PATCH 17/64] Use stages --- .../.pipelines/BuildDocumentEmbeddings.yml | 20 ++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/tools/sdk-ai-bots/.pipelines/BuildDocumentEmbeddings.yml b/tools/sdk-ai-bots/.pipelines/BuildDocumentEmbeddings.yml index 77a76a697cc..a8a47e9884c 100644 --- a/tools/sdk-ai-bots/.pipelines/BuildDocumentEmbeddings.yml +++ b/tools/sdk-ai-bots/.pipelines/BuildDocumentEmbeddings.yml @@ -23,16 +23,18 @@ parameters: type: boolean default: true -jobs: - - job: SetupEnvironment - steps: - - task: UsePythonVersion@0 - inputs: - versionSpec: '3.x' - addToPath: true - - checkout: self - stages: +- stage: SetupEnvironment + displayName: 'Setup Environment' + jobs: + - job: Setup + steps: + - task: UsePythonVersion@0 + inputs: + versionSpec: '3.x' + addToPath: true + - checkout: self + - stage: BuildEngHubDocEmbeddings displayName: 'Build EngHub Document Embeddings' jobs: From cd23ba4da07835d88fee6c6dbc5f23a7e126d973 Mon Sep 17 00:00:00 2001 From: "Ray Chen (from Dev Box)" Date: Sat, 16 Mar 2024 16:44:37 +0800 Subject: [PATCH 18/64] Fixed dependsOn error --- .../sdk-ai-bots/.pipelines/BuildDocumentEmbeddings.yml | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/tools/sdk-ai-bots/.pipelines/BuildDocumentEmbeddings.yml b/tools/sdk-ai-bots/.pipelines/BuildDocumentEmbeddings.yml index a8a47e9884c..4a96d04455c 100644 --- a/tools/sdk-ai-bots/.pipelines/BuildDocumentEmbeddings.yml +++ b/tools/sdk-ai-bots/.pipelines/BuildDocumentEmbeddings.yml @@ -27,7 +27,7 @@ stages: - stage: SetupEnvironment displayName: 'Setup Environment' jobs: - - job: Setup + - job: SetupEnvironment steps: - task: UsePythonVersion@0 inputs: @@ -38,11 +38,13 @@ stages: - stage: BuildEngHubDocEmbeddings displayName: 'Build EngHub Document Embeddings' jobs: - - job: Build EngHub Document Embeddings - dependsOn: SetupEnvironment + - job: checkout steps: - checkout: git://internal/_git/azure-sdk-docs-eng.ms - displayName: 'Checkout azure-sdk-docs-eng.ms repository' + displayName: 'Checkout azure-sdk-docs-eng.ms repository' + - job: BuildEngHubDocumentEmbeddings + dependsOn: SetupEnvironment + steps: - task: Powershell@2 inputs: filePath: $(Build.SourcesDirectory)/azure-sdk-tools/tools/sdk-ai-bots/Scripts/Build-EngHubDocEmbeddings.ps1 From f5bb2ddfe14404ff2a3920d939e97fa1ce5bc865 Mon Sep 17 00:00:00 2001 From: "Ray Chen (from Dev Box)" Date: Sat, 16 Mar 2024 16:48:32 +0800 Subject: [PATCH 19/64] Fixed stage dependson --- tools/sdk-ai-bots/.pipelines/BuildDocumentEmbeddings.yml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/tools/sdk-ai-bots/.pipelines/BuildDocumentEmbeddings.yml b/tools/sdk-ai-bots/.pipelines/BuildDocumentEmbeddings.yml index 4a96d04455c..a72891b6533 100644 --- a/tools/sdk-ai-bots/.pipelines/BuildDocumentEmbeddings.yml +++ b/tools/sdk-ai-bots/.pipelines/BuildDocumentEmbeddings.yml @@ -37,14 +37,12 @@ stages: - stage: BuildEngHubDocEmbeddings displayName: 'Build EngHub Document Embeddings' + dependsOn: SetupEnvironment jobs: - - job: checkout + - job: BuildEngHubDocumentEmbeddings steps: - checkout: git://internal/_git/azure-sdk-docs-eng.ms displayName: 'Checkout azure-sdk-docs-eng.ms repository' - - job: BuildEngHubDocumentEmbeddings - dependsOn: SetupEnvironment - steps: - task: Powershell@2 inputs: filePath: $(Build.SourcesDirectory)/azure-sdk-tools/tools/sdk-ai-bots/Scripts/Build-EngHubDocEmbeddings.ps1 From f1e45ef0bc58080d4fb57c36759afa80f3fc431d Mon Sep 17 00:00:00 2001 From: "Ray Chen (from Dev Box)" Date: Sun, 17 Mar 2024 07:01:21 +0800 Subject: [PATCH 20/64] Updated pipeline file name --- ...Embeddings.yml => build-document-embeddings.yml} | 13 +------------ tools/sdk-ai-bots/.pipelines/setup-pipeline.yml | 11 +++++++++++ 2 files changed, 12 insertions(+), 12 deletions(-) rename tools/sdk-ai-bots/.pipelines/{BuildDocumentEmbeddings.yml => build-document-embeddings.yml} (84%) create mode 100644 tools/sdk-ai-bots/.pipelines/setup-pipeline.yml diff --git a/tools/sdk-ai-bots/.pipelines/BuildDocumentEmbeddings.yml b/tools/sdk-ai-bots/.pipelines/build-document-embeddings.yml similarity index 84% rename from tools/sdk-ai-bots/.pipelines/BuildDocumentEmbeddings.yml rename to tools/sdk-ai-bots/.pipelines/build-document-embeddings.yml index a72891b6533..89d7030e900 100644 --- a/tools/sdk-ai-bots/.pipelines/BuildDocumentEmbeddings.yml +++ b/tools/sdk-ai-bots/.pipelines/build-document-embeddings.yml @@ -24,23 +24,12 @@ parameters: default: true stages: -- stage: SetupEnvironment - displayName: 'Setup Environment' - jobs: - - job: SetupEnvironment - steps: - - task: UsePythonVersion@0 - inputs: - versionSpec: '3.x' - addToPath: true - - checkout: self - - stage: BuildEngHubDocEmbeddings displayName: 'Build EngHub Document Embeddings' - dependsOn: SetupEnvironment jobs: - job: BuildEngHubDocumentEmbeddings steps: + - template: .pipelines/setup-pipeline.yml - checkout: git://internal/_git/azure-sdk-docs-eng.ms displayName: 'Checkout azure-sdk-docs-eng.ms repository' - task: Powershell@2 diff --git a/tools/sdk-ai-bots/.pipelines/setup-pipeline.yml b/tools/sdk-ai-bots/.pipelines/setup-pipeline.yml new file mode 100644 index 00000000000..ebcd2c16de2 --- /dev/null +++ b/tools/sdk-ai-bots/.pipelines/setup-pipeline.yml @@ -0,0 +1,11 @@ +parameters: +- name: pythonVersion + type: string + default: '3.x' + +steps: +- task: UsePythonVersion@0 + inputs: + versionSpec: ${{ parameters.pythonVersion }} + addToPath: true +- checkout: self \ No newline at end of file From 00d8ff51febc424b589e527f2913379be4a658a8 Mon Sep 17 00:00:00 2001 From: "Ray Chen (from Dev Box)" Date: Sun, 17 Mar 2024 07:06:44 +0800 Subject: [PATCH 21/64] Added common pipeline template --- tools/sdk-ai-bots/.pipelines/build-document-embeddings.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/sdk-ai-bots/.pipelines/build-document-embeddings.yml b/tools/sdk-ai-bots/.pipelines/build-document-embeddings.yml index 89d7030e900..f58c102b1b7 100644 --- a/tools/sdk-ai-bots/.pipelines/build-document-embeddings.yml +++ b/tools/sdk-ai-bots/.pipelines/build-document-embeddings.yml @@ -29,7 +29,7 @@ stages: jobs: - job: BuildEngHubDocumentEmbeddings steps: - - template: .pipelines/setup-pipeline.yml + - template: setup-pipeline.yml - checkout: git://internal/_git/azure-sdk-docs-eng.ms displayName: 'Checkout azure-sdk-docs-eng.ms repository' - task: Powershell@2 From a6ee1850e2b4f92339dc112917690ce1ef3473eb Mon Sep 17 00:00:00 2001 From: "Ray Chen (from Dev Box)" Date: Mon, 18 Mar 2024 12:35:46 +0800 Subject: [PATCH 22/64] Enable local run for the script --- .../.pipelines/build-document-embeddings.yml | 2 ++ .../Scripts/Build-EngHubDocEmbeddings.ps1 | 19 ++++++++++++++++++- tools/sdk-ai-bots/Scripts/Common.ps1 | 4 ++++ 3 files changed, 24 insertions(+), 1 deletion(-) diff --git a/tools/sdk-ai-bots/.pipelines/build-document-embeddings.yml b/tools/sdk-ai-bots/.pipelines/build-document-embeddings.yml index f58c102b1b7..513e03f1c4a 100644 --- a/tools/sdk-ai-bots/.pipelines/build-document-embeddings.yml +++ b/tools/sdk-ai-bots/.pipelines/build-document-embeddings.yml @@ -30,6 +30,8 @@ stages: - job: BuildEngHubDocumentEmbeddings steps: - template: setup-pipeline.yml + parameters: + pythonVersion: '3.11.3' - checkout: git://internal/_git/azure-sdk-docs-eng.ms displayName: 'Checkout azure-sdk-docs-eng.ms repository' - task: Powershell@2 diff --git a/tools/sdk-ai-bots/Scripts/Build-EngHubDocEmbeddings.ps1 b/tools/sdk-ai-bots/Scripts/Build-EngHubDocEmbeddings.ps1 index 76e33c316a8..7b2047dd3a1 100644 --- a/tools/sdk-ai-bots/Scripts/Build-EngHubDocEmbeddings.ps1 +++ b/tools/sdk-ai-bots/Scripts/Build-EngHubDocEmbeddings.ps1 @@ -17,8 +17,13 @@ param ( [ValidateNotNullOrEmpty()] [string] $IncrementalEmbedding = $true ) +# Set the working directory, current location is supposed to be the root of the repository $buildSourceDirectory = Get-Location -$workingDirectory = Join-Path $buildSourceDirectory "azure-sdk-tools\tools\sdk-ai-bots" +$workingDirectory = Join-Path $buildSourceDirectory "tools\sdk-ai-bots" +if($env:AGENT_ID) { + # Running in Azure DevOps + $workingDirectory = Join-Path $buildSourceDirectory "azure-sdk-tools\tools\sdk-ai-bots" +} $scriptsRoot = Join-Path $workingDirectory "Scripts" $embeddingToolFolder = Join-Path $workingDirectory "Embeddings" @@ -43,7 +48,19 @@ if (-not (Test-Path -Path $enghubDocsDestFolder)) { New-Item -ItemType Directory -Path $enghubDocsDestFolder } +$reposFolder = Join-Path -Path $buildSourceDirectory -ChildPath "azure-sdk-docs-eng.ms" +if(-not (Test-Path $reposFolder)) { + # Clone eng hub repository + Write-Host "Cloning azure-sdk-docs-eng.ms repository at $reposFolder" + if(-not (Clone-Repository -RepoUrl "https://azure-sdk@dev.azure.com/azure-sdk/internal/_git/azure-sdk-docs-eng.ms" -RootFolder $buildSourceDirectory)) { + exit 1 + } +} $enghubDocsSrcFolder = Join-Path -Path $buildSourceDirectory -ChildPath "azure-sdk-docs-eng.ms/docs" +if(-not (Test-Path $enghubDocsSrcFolder)) { + Write-Error "Failed to find the enghub documents folder at $enghubDocsSrcFolder" + exit 1 +} # Call the script to build the metadata.json file Write-Host "Building metadata.json file for enghub documents" diff --git a/tools/sdk-ai-bots/Scripts/Common.ps1 b/tools/sdk-ai-bots/Scripts/Common.ps1 index 1b086c27e9a..a4e8ebb9079 100644 --- a/tools/sdk-ai-bots/Scripts/Common.ps1 +++ b/tools/sdk-ai-bots/Scripts/Common.ps1 @@ -13,6 +13,10 @@ function Clone-Repository { [string] $RootFolder ) try { + if(-not (Test-Path $RootFolder)) { + New-Item -ItemType Directory -Path $RootFolder + } + Push-Location $RootFolder # Clone repository git clone $RepoUrl From 4802758fd27d4d5f466e2f49fba59c9f05074917 Mon Sep 17 00:00:00 2001 From: "Ray Chen (from Dev Box)" Date: Mon, 18 Mar 2024 16:21:10 +0800 Subject: [PATCH 23/64] Added two more stages in pipeline yml --- .../.pipelines/build-document-embeddings.yml | 46 +++++++++++++++ .../settings/metadata_customized_docs.json | 56 ------------------- .../Scripts/Build-CustomizedDocEmbeddings.ps1 | 16 +++++- .../Scripts/Build-EngHubDocEmbeddings.ps1 | 2 +- .../Build-TypeSpecAzureDocEmbeddings.ps1 | 39 ++++++++----- tools/sdk-ai-bots/Scripts/Common.ps1 | 4 +- 6 files changed, 87 insertions(+), 76 deletions(-) diff --git a/tools/sdk-ai-bots/.pipelines/build-document-embeddings.yml b/tools/sdk-ai-bots/.pipelines/build-document-embeddings.yml index 513e03f1c4a..04525d50d56 100644 --- a/tools/sdk-ai-bots/.pipelines/build-document-embeddings.yml +++ b/tools/sdk-ai-bots/.pipelines/build-document-embeddings.yml @@ -42,6 +42,52 @@ stages: pwsh: true workingDirectory: $(Build.SourcesDirectory) displayName: 'Run embeddings build script' + env: + AZURE_OPENAI_ENDPOINT: $(aoai-endpoint) + AZURE_SEARCH_ENDPOINT: $(asch-endpoint) + AZURE_SEARCH_INDEX_NAME: $(asch-index-name) + AZURE_OPENAI_EMBEDDING_MODEL: $(aoai-embedding-model) + AZURE_OPENAI_API_KEY: $(azure-openapi-key) + AZURE_SEARCH_KEY: $(azure-search-key) + AZURE_STORAGE_ACCOUNT_KEY: $(storage-account-key) +- stage: BuildTypeSpecDocEmbeddings + displayName: 'Build TypeSpec Document Embeddings' + jobs: + - job: BuildTypeSpecDocumentEmbeddings + steps: + - template: setup-pipeline.yml + - checkout: git://Azure/typespec-azure.git@main + displayName: 'Checkout azure typespec repository' + - task: Powershell@2 + inputs: + filePath: $(Build.SourcesDirectory)/azure-sdk-tools/tools/sdk-ai-bots/Scripts/Build-TypeSpecAzureDocEmbeddings.ps1 + arguments: > + -IncrementalEmbedding "${{ parameters.incrementalEmbedding }}" + pwsh: true + workingDirectory: $(Build.SourcesDirectory) + displayName: 'Run embeddings build script' + env: + AZURE_OPENAI_ENDPOINT: $(aoai-endpoint) + AZURE_SEARCH_ENDPOINT: $(asch-endpoint) + AZURE_SEARCH_INDEX_NAME: $(asch-index-name) + AZURE_OPENAI_EMBEDDING_MODEL: $(aoai-embedding-model) + AZURE_OPENAI_API_KEY: $(azure-openapi-key) + AZURE_SEARCH_KEY: $(azure-search-key) + AZURE_STORAGE_ACCOUNT_KEY: $(storage-account-key) +- stage: BuildCustomizedDocEmbeddings + displayName: 'Build Customized Document Embeddings' + jobs: + - job: BuildCustomizedDocumentEmbeddings + steps: + - template: setup-pipeline.yml + - task: Powershell@2 + inputs: + filePath: $(Build.SourcesDirectory)/azure-sdk-tools/tools/sdk-ai-bots/Scripts/Build-EngCustomizedDocEmbeddings.ps1 + arguments: > + -IncrementalEmbedding "${{ parameters.incrementalEmbedding }}" + pwsh: true + workingDirectory: $(Build.SourcesDirectory) + displayName: 'Run embeddings build script' env: AZURE_OPENAI_ENDPOINT: $(aoai-endpoint) AZURE_SEARCH_ENDPOINT: $(asch-endpoint) diff --git a/tools/sdk-ai-bots/Embeddings/settings/metadata_customized_docs.json b/tools/sdk-ai-bots/Embeddings/settings/metadata_customized_docs.json index 4a3cd8cd39a..33109691477 100644 --- a/tools/sdk-ai-bots/Embeddings/settings/metadata_customized_docs.json +++ b/tools/sdk-ai-bots/Embeddings/settings/metadata_customized_docs.json @@ -1,60 +1,4 @@ { - "ci-fix.md": { - "title": "CI Fix Guide", - "url": "https://github.com/Azure/azure-rest-api-specs/blob/main/documentation/ci-fix.md" - }, - "Getting-started-with-TypeSpec-specifications.md": { - "title": "What is a TypeSpec Specification?", - "url": "https://github.com/Azure/azure-rest-api-specs/blob/main/documentation/Getting-started-with-TypeSpec-specifications.md" - }, - "Semantic-and-Model-Violations-Reference.md": { - "title": "Semantic and Model Violations Reference", - "url": "https://github.com/Azure/azure-rest-api-specs/blob/main/documentation/Semantic-and-Model-Violations-Reference.md" - }, - "lintdiff-rules.md": { - "title": "Azure Rules", - "url": "https://github.com/Azure/azure-openapi-validator/blob/main/docs/rules.md" - }, - "typespec-rest-api-dev-process.md": { - "title": "Azure Rest API, SDK development process with TypeSpec", - "url": "https://github.com/Azure/azure-rest-api-specs/blob/main/documentation/typespec-rest-api-dev-process.md" - }, - "typespec-structure-guidelines.md": { - "title": "Repository Guidelines for TypeSpec projects", - "url": "https://github.com/Azure/azure-rest-api-specs/blob/main/documentation/typespec-structure-guidelines.md" - }, - "onboard-dpg-in-sdkautomation-README.md": { - "title": "Service Onboard DPG with Swagger CI Pipeline", - "url": "https://github.com/Azure/azure-rest-api-specs/blob/main/documentation/onboard-dpg-in-sdkautomation/README.md" - }, - "rules-for-release-request.md": { - "title": "Some guidance and rules when you fill the release issue", - "url": "https://github.com/Azure/azure-rest-api-specs/blob/main/documentation/release-request/rules-for-release-request.md" - }, - "configure-go-sdk.md": { - "title": "Readme Configuration Guide for Azure SDK for Go", - "url": "https://github.com/Azure/azure-rest-api-specs/blob/main/documentation/code-gen/configure-go-sdk.md" - }, - "configure-python-sdk.md": { - "title": "Readme Configuration Guide for Azure SDK for Python", - "url": "https://github.com/Azure/azure-rest-api-specs/blob/main/documentation/code-gen/configure-python-sdk.md" - }, - "configure-typescript-sdk.md": { - "title": "Readme Configuration Guide for Azure SDK for Javascript (Typescript)", - "url": "https://github.com/Azure/azure-rest-api-specs/blob/main/documentation/code-gen/configure-typescript-sdk.md" - }, - "autorest-extensions-readme.md": { - "title": "AutoRest Extensions for OpenAPI 2.0", - "url": "https://github.com/Azure/autorest/blob/main/docs/extensions/readme.md" - }, - "avocado-readme.md": { - "title": "Avocado Readme", - "url": "https://github.com/Azure/avocado/blob/main/README.md" - }, - "control_plane_template.md": { - "title": "Control plane pull request template", - "url": "https://github.com/Azure/azure-rest-api-specs/blob/main/.github/PULL_REQUEST_TEMPLATE/control_plane_template.md" - }, "data_plane_template.md": { "title": "Data plane pull request template", "url": "https://github.com/Azure/azure-rest-api-specs/blob/main/.github/PULL_REQUEST_TEMPLATE/data_plane_template.md" diff --git a/tools/sdk-ai-bots/Scripts/Build-CustomizedDocEmbeddings.ps1 b/tools/sdk-ai-bots/Scripts/Build-CustomizedDocEmbeddings.ps1 index 67c4e5bb61e..40b0efae4a9 100644 --- a/tools/sdk-ai-bots/Scripts/Build-CustomizedDocEmbeddings.ps1 +++ b/tools/sdk-ai-bots/Scripts/Build-CustomizedDocEmbeddings.ps1 @@ -62,14 +62,24 @@ function Download-GitHubFile { } -$workingDirectory = Join-Path (Get-Location) "tools\sdk-ai-bots" +# Set the working directory, current location is supposed to be the root of the repository +$buildSourceDirectory = Get-Location +$workingDirectory = Join-Path $buildSourceDirectory "tools\sdk-ai-bots" +if($env:AGENT_ID) { + # Running in Azure DevOps + $workingDirectory = Join-Path $buildSourceDirectory "azure-sdk-tools\tools\sdk-ai-bots" +} $scriptsRoot = Join-Path $workingDirectory "Scripts" $embeddingToolFolder = Join-Path $workingDirectory "Embeddings" -. (Join-Path $scriptsRoot Common.ps1) - Write-Host "scriptsRoot: $scriptsRoot" Write-Host "embeddingToolFolder: $embeddingToolFolder" +. (Join-Path $scriptsRoot Common.ps1) + +# Install Az.Storage module +if (-not (Get-Module -ListAvailable -Name Az.Storage)) { + Install-Module -Name Az.Storage -Force -AllowClobber -Scope CurrentUser +} # Create embeddingSource folder on current location $embeddingSourceFolder = Join-Path -Path $workingDirectory -ChildPath "embeddingSource" diff --git a/tools/sdk-ai-bots/Scripts/Build-EngHubDocEmbeddings.ps1 b/tools/sdk-ai-bots/Scripts/Build-EngHubDocEmbeddings.ps1 index 7b2047dd3a1..6bb76ca48fd 100644 --- a/tools/sdk-ai-bots/Scripts/Build-EngHubDocEmbeddings.ps1 +++ b/tools/sdk-ai-bots/Scripts/Build-EngHubDocEmbeddings.ps1 @@ -51,7 +51,7 @@ if (-not (Test-Path -Path $enghubDocsDestFolder)) { $reposFolder = Join-Path -Path $buildSourceDirectory -ChildPath "azure-sdk-docs-eng.ms" if(-not (Test-Path $reposFolder)) { # Clone eng hub repository - Write-Host "Cloning azure-sdk-docs-eng.ms repository at $reposFolder" + Write-Host "Cloning azure-sdk-docs-eng.ms repository at $buildSourceDirectory" if(-not (Clone-Repository -RepoUrl "https://azure-sdk@dev.azure.com/azure-sdk/internal/_git/azure-sdk-docs-eng.ms" -RootFolder $buildSourceDirectory)) { exit 1 } diff --git a/tools/sdk-ai-bots/Scripts/Build-TypeSpecAzureDocEmbeddings.ps1 b/tools/sdk-ai-bots/Scripts/Build-TypeSpecAzureDocEmbeddings.ps1 index 0a9f6dd261e..9b5d10d768f 100644 --- a/tools/sdk-ai-bots/Scripts/Build-TypeSpecAzureDocEmbeddings.ps1 +++ b/tools/sdk-ai-bots/Scripts/Build-TypeSpecAzureDocEmbeddings.ps1 @@ -18,26 +18,23 @@ param ( [ValidateNotNullOrEmpty()] [string] $IncrementalEmbedding = $true ) - -$workingDirectory = Join-Path (Get-Location) "tools\sdk-ai-bots" +# Set the working directory, current location is supposed to be the root of the repository +$buildSourceDirectory = Get-Location +$workingDirectory = Join-Path $buildSourceDirectory "tools\sdk-ai-bots" +if($env:AGENT_ID) { + # Running in Azure DevOps + $workingDirectory = Join-Path $buildSourceDirectory "azure-sdk-tools\tools\sdk-ai-bots" +} $scriptsRoot = Join-Path $workingDirectory "Scripts" $embeddingToolFolder = Join-Path $workingDirectory "Embeddings" -. (Join-Path $scriptsRoot Common.ps1) - Write-Host "scriptsRoot: $scriptsRoot" Write-Host "embeddingToolFolder: $embeddingToolFolder" +. (Join-Path $scriptsRoot Common.ps1) -# Create 'repos' folder on current location -$reposFolder = Join-Path -Path $workingDirectory -ChildPath "repos" -if (-not (Test-Path -Path $reposFolder)) { - New-Item -ItemType Directory -Path $reposFolder -} - -# Clone Azure/typespec-azure repository -Write-Host "Cloning Azure/typespec-azure repository at $reposFolder" -if(-not (Clone-Repository -RepoUrl "https://github.com/Azure/typespec-azure.git" -RootFolder $reposFolder)) { - exit 1 +# Install Az.Storage module +if (-not (Get-Module -ListAvailable -Name Az.Storage)) { + Install-Module -Name Az.Storage -Force -AllowClobber -Scope CurrentUser } # Create embeddingSource folder on current location @@ -52,7 +49,19 @@ if (-not (Test-Path -Path $typespecDocsDestFolder)) { New-Item -ItemType Directory -Path $typespecDocsDestFolder } -$typespecDocsSrcFolder = Join-Path -Path $reposFolder -ChildPath "typespec-azure/docs" +$reposFolder = Join-Path -Path $buildSourceDirectory -ChildPath "typespec-azure" +if(-not (Test-Path $reposFolder)) { + # Clone Azure/typespec-azure repository + Write-Host "Cloning Azure/typespec-azure repository at $buildSourceDirectory" + if(-not (Clone-Repository -RepoUrl "https://github.com/Azure/typespec-azure.git" -RootFolder $buildSourceDirectory)) { + exit 1 + } +} +$typespecDocsSrcFolder = Join-Path -Path $buildSourceDirectory -ChildPath "typespec-azure/docs" +if(-not (Test-Path $typespecDocsSrcFolder)) { + Write-Error "Failed to find the typespec documents folder at $typespecDocsSrcFolder" + exit 1 +} # Call the script to build the metadata.json file Write-Host "Building metadata.json file for typespec documents" diff --git a/tools/sdk-ai-bots/Scripts/Common.ps1 b/tools/sdk-ai-bots/Scripts/Common.ps1 index a4e8ebb9079..77a22469ecb 100644 --- a/tools/sdk-ai-bots/Scripts/Common.ps1 +++ b/tools/sdk-ai-bots/Scripts/Common.ps1 @@ -92,7 +92,9 @@ function Build-Embeddings { Write-Host "Installing required packages" pip install -r requirements.txt - + + Write-Host "List package versions..." + pip List Write-Host "Starts building" python main.py } From d5ce93b745aafdfafd5eec3950a98f0fa33593fc Mon Sep 17 00:00:00 2001 From: "Ray Chen (from Dev Box)" Date: Mon, 18 Mar 2024 16:30:36 +0800 Subject: [PATCH 24/64] Updated github repo url --- tools/sdk-ai-bots/.pipelines/build-document-embeddings.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/sdk-ai-bots/.pipelines/build-document-embeddings.yml b/tools/sdk-ai-bots/.pipelines/build-document-embeddings.yml index 04525d50d56..83285010f5d 100644 --- a/tools/sdk-ai-bots/.pipelines/build-document-embeddings.yml +++ b/tools/sdk-ai-bots/.pipelines/build-document-embeddings.yml @@ -56,7 +56,7 @@ stages: - job: BuildTypeSpecDocumentEmbeddings steps: - template: setup-pipeline.yml - - checkout: git://Azure/typespec-azure.git@main + - checkout: github://Azure/typespec-azure.git@main displayName: 'Checkout azure typespec repository' - task: Powershell@2 inputs: From acba8ddb8d096602c1207c3b7e4af2aa34857f53 Mon Sep 17 00:00:00 2001 From: "Ray Chen (from Dev Box)" Date: Mon, 18 Mar 2024 16:37:55 +0800 Subject: [PATCH 25/64] Removed the typespec repo stage temporaily --- .../.pipelines/build-document-embeddings.yml | 25 +------------------ tools/sdk-ai-bots/AzureSdkQaBot/Nuget.Config | 8 ------ 2 files changed, 1 insertion(+), 32 deletions(-) delete mode 100644 tools/sdk-ai-bots/AzureSdkQaBot/Nuget.Config diff --git a/tools/sdk-ai-bots/.pipelines/build-document-embeddings.yml b/tools/sdk-ai-bots/.pipelines/build-document-embeddings.yml index 83285010f5d..b184bf8cc1e 100644 --- a/tools/sdk-ai-bots/.pipelines/build-document-embeddings.yml +++ b/tools/sdk-ai-bots/.pipelines/build-document-embeddings.yml @@ -50,30 +50,7 @@ stages: AZURE_OPENAI_API_KEY: $(azure-openapi-key) AZURE_SEARCH_KEY: $(azure-search-key) AZURE_STORAGE_ACCOUNT_KEY: $(storage-account-key) -- stage: BuildTypeSpecDocEmbeddings - displayName: 'Build TypeSpec Document Embeddings' - jobs: - - job: BuildTypeSpecDocumentEmbeddings - steps: - - template: setup-pipeline.yml - - checkout: github://Azure/typespec-azure.git@main - displayName: 'Checkout azure typespec repository' - - task: Powershell@2 - inputs: - filePath: $(Build.SourcesDirectory)/azure-sdk-tools/tools/sdk-ai-bots/Scripts/Build-TypeSpecAzureDocEmbeddings.ps1 - arguments: > - -IncrementalEmbedding "${{ parameters.incrementalEmbedding }}" - pwsh: true - workingDirectory: $(Build.SourcesDirectory) - displayName: 'Run embeddings build script' - env: - AZURE_OPENAI_ENDPOINT: $(aoai-endpoint) - AZURE_SEARCH_ENDPOINT: $(asch-endpoint) - AZURE_SEARCH_INDEX_NAME: $(asch-index-name) - AZURE_OPENAI_EMBEDDING_MODEL: $(aoai-embedding-model) - AZURE_OPENAI_API_KEY: $(azure-openapi-key) - AZURE_SEARCH_KEY: $(azure-search-key) - AZURE_STORAGE_ACCOUNT_KEY: $(storage-account-key) + - stage: BuildCustomizedDocEmbeddings displayName: 'Build Customized Document Embeddings' jobs: diff --git a/tools/sdk-ai-bots/AzureSdkQaBot/Nuget.Config b/tools/sdk-ai-bots/AzureSdkQaBot/Nuget.Config deleted file mode 100644 index 6de7d4c1286..00000000000 --- a/tools/sdk-ai-bots/AzureSdkQaBot/Nuget.Config +++ /dev/null @@ -1,8 +0,0 @@ - - - - - - - - \ No newline at end of file From 93d19080f95ef6c37b37bac21b16f4be248d1ffa Mon Sep 17 00:00:00 2001 From: "Ray Chen (from Dev Box)" Date: Mon, 18 Mar 2024 16:47:26 +0800 Subject: [PATCH 26/64] list python package installed --- tools/sdk-ai-bots/Scripts/Common.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/sdk-ai-bots/Scripts/Common.ps1 b/tools/sdk-ai-bots/Scripts/Common.ps1 index 77a22469ecb..2d6a3c2f204 100644 --- a/tools/sdk-ai-bots/Scripts/Common.ps1 +++ b/tools/sdk-ai-bots/Scripts/Common.ps1 @@ -94,7 +94,7 @@ function Build-Embeddings { pip install -r requirements.txt Write-Host "List package versions..." - pip List + pip list Write-Host "Starts building" python main.py } From d69483a890f728c50d21484855a58d95aba4cf3d Mon Sep 17 00:00:00 2001 From: "Ray Chen (from Dev Box)" Date: Mon, 18 Mar 2024 17:06:44 +0800 Subject: [PATCH 27/64] Use python -m command --- tools/sdk-ai-bots/Scripts/Common.ps1 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/sdk-ai-bots/Scripts/Common.ps1 b/tools/sdk-ai-bots/Scripts/Common.ps1 index 2d6a3c2f204..e7f9ea3d617 100644 --- a/tools/sdk-ai-bots/Scripts/Common.ps1 +++ b/tools/sdk-ai-bots/Scripts/Common.ps1 @@ -91,10 +91,10 @@ function Build-Embeddings { python -m pip install --upgrade pip Write-Host "Installing required packages" - pip install -r requirements.txt + python -m pip install -r requirements.txt Write-Host "List package versions..." - pip list + python -m pip list Write-Host "Starts building" python main.py } From 8185ce9739ca7a9b54b69b38e1aa5ce72e8dfdef Mon Sep 17 00:00:00 2001 From: "Ray Chen (from Dev Box)" Date: Mon, 18 Mar 2024 17:16:24 +0800 Subject: [PATCH 28/64] Print content of installed python packages --- tools/sdk-ai-bots/Scripts/Common.ps1 | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tools/sdk-ai-bots/Scripts/Common.ps1 b/tools/sdk-ai-bots/Scripts/Common.ps1 index e7f9ea3d617..cbafb1e7716 100644 --- a/tools/sdk-ai-bots/Scripts/Common.ps1 +++ b/tools/sdk-ai-bots/Scripts/Common.ps1 @@ -94,7 +94,10 @@ function Build-Embeddings { python -m pip install -r requirements.txt Write-Host "List package versions..." - python -m pip list + python -m pip list > pip_list.txt + Write-Host "Print the content of pip_list.txt" + Get-Content -Path "pip_list.txt" + Write-Host "Starts building" python main.py } From e42b50473a3c62c8e3bb7d9527a945943a39edc3 Mon Sep 17 00:00:00 2001 From: "Ray Chen (from Dev Box)" Date: Mon, 18 Mar 2024 17:42:12 +0800 Subject: [PATCH 29/64] Added debug script --- tools/sdk-ai-bots/Scripts/Common.ps1 | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/tools/sdk-ai-bots/Scripts/Common.ps1 b/tools/sdk-ai-bots/Scripts/Common.ps1 index cbafb1e7716..343084fa32c 100644 --- a/tools/sdk-ai-bots/Scripts/Common.ps1 +++ b/tools/sdk-ai-bots/Scripts/Common.ps1 @@ -86,15 +86,28 @@ function Build-Embeddings { Write-Host "Building embeddings..." try { Push-Location $embeddingToolFolder + + Write-Host "Check Python environment" + python --version + python -m pip --version + # setup python environment and install required packages Write-Host "Setting up python environment" python -m pip install --upgrade pip + Write-Host "Check requirements.txt file" + Get-Content -Path "requirements.txt" + Write-Host "Installing required packages" python -m pip install -r requirements.txt + Write-Host "Check current directory" + Get-Location + Write-Host "List package versions..." python -m pip list > pip_list.txt + Write-Host "Check pip_list.txt file" + Test-Path "pip_list.txt" Write-Host "Print the content of pip_list.txt" Get-Content -Path "pip_list.txt" From ceb04713c7e3a5a1cb53f3775f2160cc69ea059f Mon Sep 17 00:00:00 2001 From: "Ray Chen (from Dev Box)" Date: Mon, 18 Mar 2024 18:18:21 +0800 Subject: [PATCH 30/64] Redirect log stream --- tools/sdk-ai-bots/Scripts/Common.ps1 | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/tools/sdk-ai-bots/Scripts/Common.ps1 b/tools/sdk-ai-bots/Scripts/Common.ps1 index 343084fa32c..3bdf325c63e 100644 --- a/tools/sdk-ai-bots/Scripts/Common.ps1 +++ b/tools/sdk-ai-bots/Scripts/Common.ps1 @@ -88,28 +88,28 @@ function Build-Embeddings { Push-Location $embeddingToolFolder Write-Host "Check Python environment" - python --version - python -m pip --version + python --version 2>&1 + python -m pip --version 2>&1 # setup python environment and install required packages Write-Host "Setting up python environment" - python -m pip install --upgrade pip + python -m pip install --upgrade pip 2>&1 Write-Host "Check requirements.txt file" - Get-Content -Path "requirements.txt" + Get-Content -Path "requirements.txt" 2>&1 Write-Host "Installing required packages" - python -m pip install -r requirements.txt + python -m pip install -r requirements.txt 2>&1 Write-Host "Check current directory" - Get-Location + Get-Location 2>&1 Write-Host "List package versions..." - python -m pip list > pip_list.txt + python -m pip list > pip_list.txt 2>&1 Write-Host "Check pip_list.txt file" - Test-Path "pip_list.txt" + Test-Path "pip_list.txt" 2>&1 Write-Host "Print the content of pip_list.txt" - Get-Content -Path "pip_list.txt" + Get-Content -Path "pip_list.txt" 2>&1 Write-Host "Starts building" python main.py From 0673c86502d1a0f4367fe604c727bc5e8d44e00a Mon Sep 17 00:00:00 2001 From: "Ray Chen (from Dev Box)" Date: Mon, 18 Mar 2024 19:42:17 +0800 Subject: [PATCH 31/64] comment measure-command --- tools/sdk-ai-bots/Scripts/Common.ps1 | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tools/sdk-ai-bots/Scripts/Common.ps1 b/tools/sdk-ai-bots/Scripts/Common.ps1 index 3bdf325c63e..9b9cd56b07b 100644 --- a/tools/sdk-ai-bots/Scripts/Common.ps1 +++ b/tools/sdk-ai-bots/Scripts/Common.ps1 @@ -82,18 +82,18 @@ function Build-Embeddings { Write-Error "The embedding tool folder does not exist: $embeddingToolFolder" return $false } - $stopwatch = Measure-Command { + #$stopwatch = Measure-Command { Write-Host "Building embeddings..." try { Push-Location $embeddingToolFolder Write-Host "Check Python environment" - python --version 2>&1 - python -m pip --version 2>&1 + python --version + python -m pip --version # setup python environment and install required packages Write-Host "Setting up python environment" - python -m pip install --upgrade pip 2>&1 + python -m pip install --upgrade pip Write-Host "Check requirements.txt file" Get-Content -Path "requirements.txt" 2>&1 @@ -121,9 +121,9 @@ function Build-Embeddings { finally { Pop-Location } - } + #} - Write-Host "Finishes building with time: $($stopwatch.TotalSeconds) seconds" + #Write-Host "Finishes building with time: $($stopwatch.TotalSeconds) seconds" return $true } From 136147d5afe6cf2bb0e9923bbd0507a99c7a5f52 Mon Sep 17 00:00:00 2001 From: "Ray Chen (from Dev Box)" Date: Mon, 18 Mar 2024 20:23:24 +0800 Subject: [PATCH 32/64] Output log by variables --- tools/sdk-ai-bots/Scripts/Common.ps1 | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/tools/sdk-ai-bots/Scripts/Common.ps1 b/tools/sdk-ai-bots/Scripts/Common.ps1 index 9b9cd56b07b..40d14b70b42 100644 --- a/tools/sdk-ai-bots/Scripts/Common.ps1 +++ b/tools/sdk-ai-bots/Scripts/Common.ps1 @@ -96,20 +96,20 @@ function Build-Embeddings { python -m pip install --upgrade pip Write-Host "Check requirements.txt file" - Get-Content -Path "requirements.txt" 2>&1 + Get-Content -Path "requirements.txt" Write-Host "Installing required packages" - python -m pip install -r requirements.txt 2>&1 + python -m pip install -r requirements.txt - Write-Host "Check current directory" - Get-Location 2>&1 + $currentDir = Get-Location + Write-Host "Current directory: $currentDir" Write-Host "List package versions..." - python -m pip list > pip_list.txt 2>&1 - Write-Host "Check pip_list.txt file" - Test-Path "pip_list.txt" 2>&1 + python -m pip list > pip_list.txt + Write-Host "Print the content of pip_list.txt" - Get-Content -Path "pip_list.txt" 2>&1 + $installedPkg = Get-Content -Path "pip_list.txt" + Write-Host $installedPkg Write-Host "Starts building" python main.py From 5a004ecc02ddc4ceadbeecb57f780f44abe5e865 Mon Sep 17 00:00:00 2001 From: "Ray Chen (from Dev Box)" Date: Tue, 19 Mar 2024 09:33:15 +0800 Subject: [PATCH 33/64] Use liunx agent pool --- tools/sdk-ai-bots/.pipelines/build-document-embeddings.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/sdk-ai-bots/.pipelines/build-document-embeddings.yml b/tools/sdk-ai-bots/.pipelines/build-document-embeddings.yml index b184bf8cc1e..54a39191a85 100644 --- a/tools/sdk-ai-bots/.pipelines/build-document-embeddings.yml +++ b/tools/sdk-ai-bots/.pipelines/build-document-embeddings.yml @@ -9,7 +9,7 @@ schedules: always: true pool: - vmImage: 'windows-latest' + vmImage: 'ubuntu-latest' variables: aoai-endpoint: $(azure-openai-endpoint) From 5eb4c1b5bba5524d424c92bf6a92970005784b3b Mon Sep 17 00:00:00 2001 From: "Ray Chen (from Dev Box)" Date: Tue, 19 Mar 2024 10:10:28 +0800 Subject: [PATCH 34/64] Added conda python env --- .../.pipelines/build-document-embeddings.yml | 2 -- .../sdk-ai-bots/.pipelines/setup-pipeline.yml | 19 +++++++-------- .../Scripts/Build-EngHubDocEmbeddings.ps1 | 1 + tools/sdk-ai-bots/Scripts/Common.ps1 | 23 +++++++++++++++---- 4 files changed, 29 insertions(+), 16 deletions(-) diff --git a/tools/sdk-ai-bots/.pipelines/build-document-embeddings.yml b/tools/sdk-ai-bots/.pipelines/build-document-embeddings.yml index 54a39191a85..1dcfbbcc798 100644 --- a/tools/sdk-ai-bots/.pipelines/build-document-embeddings.yml +++ b/tools/sdk-ai-bots/.pipelines/build-document-embeddings.yml @@ -30,8 +30,6 @@ stages: - job: BuildEngHubDocumentEmbeddings steps: - template: setup-pipeline.yml - parameters: - pythonVersion: '3.11.3' - checkout: git://internal/_git/azure-sdk-docs-eng.ms displayName: 'Checkout azure-sdk-docs-eng.ms repository' - task: Powershell@2 diff --git a/tools/sdk-ai-bots/.pipelines/setup-pipeline.yml b/tools/sdk-ai-bots/.pipelines/setup-pipeline.yml index ebcd2c16de2..24c2defa60b 100644 --- a/tools/sdk-ai-bots/.pipelines/setup-pipeline.yml +++ b/tools/sdk-ai-bots/.pipelines/setup-pipeline.yml @@ -1,11 +1,12 @@ -parameters: -- name: pythonVersion - type: string - default: '3.x' - steps: -- task: UsePythonVersion@0 - inputs: - versionSpec: ${{ parameters.pythonVersion }} - addToPath: true +- script: | + echo "Installing Miniconda" + Invoke-WebRequest -Uri "https://repo.anaconda.com/miniconda/Miniconda3-latest-Windows-x86_64.exe" -OutFile "miniconda.exe" + Start-Process "miniconda.exe" -ArgumentList "/S /D=C:\Miniconda" -Wait + echo "Adding Miniconda to PATH" + $env:Path += ";C:\Miniconda" + $env:Path += ";C:\Miniconda\Scripts" + $env:Path += ";C:\Miniconda\Library\bin" + displayName: 'Install Miniconda' + shell: powershell - checkout: self \ No newline at end of file diff --git a/tools/sdk-ai-bots/Scripts/Build-EngHubDocEmbeddings.ps1 b/tools/sdk-ai-bots/Scripts/Build-EngHubDocEmbeddings.ps1 index 6bb76ca48fd..6051de52ad3 100644 --- a/tools/sdk-ai-bots/Scripts/Build-EngHubDocEmbeddings.ps1 +++ b/tools/sdk-ai-bots/Scripts/Build-EngHubDocEmbeddings.ps1 @@ -94,6 +94,7 @@ $env:RAG_CHUNK_PATH = $ragChunkPath $env:METADATA_PATH = "$embeddingSourceFolder/metadata_enghub_docs.json" $env:DOCUMENT_PATH = $enghubDocsDestFolder $env:INCREMENTAL_EMBEDDING = $IncrementalEmbedding +Initialize-PythonEnv if(-not (Build-Embeddings -EmbeddingToolFolder $embeddingToolFolder)) { exit 1 } diff --git a/tools/sdk-ai-bots/Scripts/Common.ps1 b/tools/sdk-ai-bots/Scripts/Common.ps1 index 40d14b70b42..049bca0b194 100644 --- a/tools/sdk-ai-bots/Scripts/Common.ps1 +++ b/tools/sdk-ai-bots/Scripts/Common.ps1 @@ -88,15 +88,21 @@ function Build-Embeddings { Push-Location $embeddingToolFolder Write-Host "Check Python environment" - python --version - python -m pip --version + $pythonVersion = python --version + Write-Host "Python version: $pythonVersion" + $pipVersion = python -m pip --version + Write-Host "Pip version: $pipVersion" + + # Print Python version + $pythonVersion = python -c "import sys; print(sys.version)" + Write-Host "Python version: $pythonVersion" + # Print Python executable path + $pythonEnvExePath = python -c "import sys; print(sys.executable)" + Write-Host "Python executable path: $pythonEnvExePath" # setup python environment and install required packages Write-Host "Setting up python environment" python -m pip install --upgrade pip - - Write-Host "Check requirements.txt file" - Get-Content -Path "requirements.txt" Write-Host "Installing required packages" python -m pip install -r requirements.txt @@ -162,3 +168,10 @@ function Upload-AzureBlob { } return $false } + +function Initialize-PythonEnv { + Write-Host "Create Conda environment" + conda create -n myenv python=3.11 -y + Write-Host "Activate Conda environment" + conda activate myenv +} From e1578411e8d948a1a1c07746e6644792f5097898 Mon Sep 17 00:00:00 2001 From: "Ray Chen (from Dev Box)" Date: Tue, 19 Mar 2024 10:14:03 +0800 Subject: [PATCH 35/64] Use pwsh shell --- tools/sdk-ai-bots/.pipelines/setup-pipeline.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/sdk-ai-bots/.pipelines/setup-pipeline.yml b/tools/sdk-ai-bots/.pipelines/setup-pipeline.yml index 24c2defa60b..238c42768f3 100644 --- a/tools/sdk-ai-bots/.pipelines/setup-pipeline.yml +++ b/tools/sdk-ai-bots/.pipelines/setup-pipeline.yml @@ -8,5 +8,5 @@ steps: $env:Path += ";C:\Miniconda\Scripts" $env:Path += ";C:\Miniconda\Library\bin" displayName: 'Install Miniconda' - shell: powershell + shell: pwsh - checkout: self \ No newline at end of file From a7672b6db9606ffa04aef510c90fd1c0366c7421 Mon Sep 17 00:00:00 2001 From: "Ray Chen (from Dev Box)" Date: Tue, 19 Mar 2024 10:16:08 +0800 Subject: [PATCH 36/64] Use windows os --- tools/sdk-ai-bots/.pipelines/build-document-embeddings.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/sdk-ai-bots/.pipelines/build-document-embeddings.yml b/tools/sdk-ai-bots/.pipelines/build-document-embeddings.yml index 1dcfbbcc798..45d5c05df60 100644 --- a/tools/sdk-ai-bots/.pipelines/build-document-embeddings.yml +++ b/tools/sdk-ai-bots/.pipelines/build-document-embeddings.yml @@ -9,7 +9,7 @@ schedules: always: true pool: - vmImage: 'ubuntu-latest' + vmImage: 'windows-latest' variables: aoai-endpoint: $(azure-openai-endpoint) From 7cff8ac092406496a560499eed08582000bfd008 Mon Sep 17 00:00:00 2001 From: "Ray Chen (from Dev Box)" Date: Tue, 19 Mar 2024 10:22:22 +0800 Subject: [PATCH 37/64] Use pwsh step --- tools/sdk-ai-bots/.pipelines/setup-pipeline.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tools/sdk-ai-bots/.pipelines/setup-pipeline.yml b/tools/sdk-ai-bots/.pipelines/setup-pipeline.yml index 238c42768f3..f645b959810 100644 --- a/tools/sdk-ai-bots/.pipelines/setup-pipeline.yml +++ b/tools/sdk-ai-bots/.pipelines/setup-pipeline.yml @@ -1,5 +1,5 @@ steps: -- script: | +- pwsh: | echo "Installing Miniconda" Invoke-WebRequest -Uri "https://repo.anaconda.com/miniconda/Miniconda3-latest-Windows-x86_64.exe" -OutFile "miniconda.exe" Start-Process "miniconda.exe" -ArgumentList "/S /D=C:\Miniconda" -Wait @@ -8,5 +8,4 @@ steps: $env:Path += ";C:\Miniconda\Scripts" $env:Path += ";C:\Miniconda\Library\bin" displayName: 'Install Miniconda' - shell: pwsh - checkout: self \ No newline at end of file From 37a019e79ff30b274e4fca37385b4ccc3e4e35cf Mon Sep 17 00:00:00 2001 From: "Ray Chen (from Dev Box)" Date: Tue, 19 Mar 2024 10:44:13 +0800 Subject: [PATCH 38/64] Use full path to conda --- tools/sdk-ai-bots/.pipelines/setup-pipeline.yml | 1 + tools/sdk-ai-bots/Scripts/Build-EngHubDocEmbeddings.ps1 | 4 +++- tools/sdk-ai-bots/Scripts/Common.ps1 | 9 +++++++-- 3 files changed, 11 insertions(+), 3 deletions(-) diff --git a/tools/sdk-ai-bots/.pipelines/setup-pipeline.yml b/tools/sdk-ai-bots/.pipelines/setup-pipeline.yml index f645b959810..1619011c588 100644 --- a/tools/sdk-ai-bots/.pipelines/setup-pipeline.yml +++ b/tools/sdk-ai-bots/.pipelines/setup-pipeline.yml @@ -7,5 +7,6 @@ steps: $env:Path += ";C:\Miniconda" $env:Path += ";C:\Miniconda\Scripts" $env:Path += ";C:\Miniconda\Library\bin" + conda --version displayName: 'Install Miniconda' - checkout: self \ No newline at end of file diff --git a/tools/sdk-ai-bots/Scripts/Build-EngHubDocEmbeddings.ps1 b/tools/sdk-ai-bots/Scripts/Build-EngHubDocEmbeddings.ps1 index 6051de52ad3..6147d2b4907 100644 --- a/tools/sdk-ai-bots/Scripts/Build-EngHubDocEmbeddings.ps1 +++ b/tools/sdk-ai-bots/Scripts/Build-EngHubDocEmbeddings.ps1 @@ -94,7 +94,9 @@ $env:RAG_CHUNK_PATH = $ragChunkPath $env:METADATA_PATH = "$embeddingSourceFolder/metadata_enghub_docs.json" $env:DOCUMENT_PATH = $enghubDocsDestFolder $env:INCREMENTAL_EMBEDDING = $IncrementalEmbedding -Initialize-PythonEnv +if(-not (Initialize-PythonEnv)) { + exit 1 +} if(-not (Build-Embeddings -EmbeddingToolFolder $embeddingToolFolder)) { exit 1 } diff --git a/tools/sdk-ai-bots/Scripts/Common.ps1 b/tools/sdk-ai-bots/Scripts/Common.ps1 index 049bca0b194..2342cfdd0bc 100644 --- a/tools/sdk-ai-bots/Scripts/Common.ps1 +++ b/tools/sdk-ai-bots/Scripts/Common.ps1 @@ -171,7 +171,12 @@ function Upload-AzureBlob { function Initialize-PythonEnv { Write-Host "Create Conda environment" - conda create -n myenv python=3.11 -y + $condaPath = "C:\Miniconda\Scripts\conda.exe" + if(-not (Test-Path $condaPath)) { + Write-Error "Conda is not installed at $condaPath" + return $false + } + & $condaPath create -n myenv python=3.11 -y Write-Host "Activate Conda environment" - conda activate myenv + & $condaPath activate myenv } From e3e75503192e0169fd09dde35c0982f7282b96f2 Mon Sep 17 00:00:00 2001 From: "Ray Chen (from Dev Box)" Date: Tue, 19 Mar 2024 11:02:27 +0800 Subject: [PATCH 39/64] Added conda init --- tools/sdk-ai-bots/Scripts/Common.ps1 | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tools/sdk-ai-bots/Scripts/Common.ps1 b/tools/sdk-ai-bots/Scripts/Common.ps1 index 2342cfdd0bc..d9845eacfd9 100644 --- a/tools/sdk-ai-bots/Scripts/Common.ps1 +++ b/tools/sdk-ai-bots/Scripts/Common.ps1 @@ -177,6 +177,8 @@ function Initialize-PythonEnv { return $false } & $condaPath create -n myenv python=3.11 -y + Write-Host "Initialize Conda environment" + & $condaPath init Write-Host "Activate Conda environment" & $condaPath activate myenv } From 825b5fd0473ccf1d35b5b18eda9e954484e0399c Mon Sep 17 00:00:00 2001 From: "Ray Chen (from Dev Box)" Date: Tue, 19 Mar 2024 11:26:45 +0800 Subject: [PATCH 40/64] Run conda with full path --- tools/sdk-ai-bots/Scripts/Common.ps1 | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/tools/sdk-ai-bots/Scripts/Common.ps1 b/tools/sdk-ai-bots/Scripts/Common.ps1 index d9845eacfd9..789ad2f8c00 100644 --- a/tools/sdk-ai-bots/Scripts/Common.ps1 +++ b/tools/sdk-ai-bots/Scripts/Common.ps1 @@ -118,7 +118,13 @@ function Build-Embeddings { Write-Host $installedPkg Write-Host "Starts building" - python main.py + $condaPath = "C:\Miniconda\Scripts\conda.exe" + if(-not (Test-Path $condaPath)) { + Write-Error "Conda is not installed at $condaPath" + return $false + } + & $condaPath create -n myenv python=3.11 -y + & $condaPath run -n myenv python main.py } catch { Write-Error "Failed to build embeddings with exception:`n$_" @@ -178,7 +184,7 @@ function Initialize-PythonEnv { } & $condaPath create -n myenv python=3.11 -y Write-Host "Initialize Conda environment" - & $condaPath init + & $condaPath init powershell Write-Host "Activate Conda environment" & $condaPath activate myenv } From d774e31b7b5fa5215c959cda6b8076ecc474d672 Mon Sep 17 00:00:00 2001 From: "Ray Chen (from Dev Box)" Date: Tue, 19 Mar 2024 14:11:21 +0800 Subject: [PATCH 41/64] Call python thur conda env --- .../sdk-ai-bots/.pipelines/setup-pipeline.yml | 10 ----- .../Scripts/Build-EngHubDocEmbeddings.ps1 | 2 +- tools/sdk-ai-bots/Scripts/Common.ps1 | 42 ++++++++++++++----- 3 files changed, 32 insertions(+), 22 deletions(-) diff --git a/tools/sdk-ai-bots/.pipelines/setup-pipeline.yml b/tools/sdk-ai-bots/.pipelines/setup-pipeline.yml index 1619011c588..8a657813f2c 100644 --- a/tools/sdk-ai-bots/.pipelines/setup-pipeline.yml +++ b/tools/sdk-ai-bots/.pipelines/setup-pipeline.yml @@ -1,12 +1,2 @@ steps: -- pwsh: | - echo "Installing Miniconda" - Invoke-WebRequest -Uri "https://repo.anaconda.com/miniconda/Miniconda3-latest-Windows-x86_64.exe" -OutFile "miniconda.exe" - Start-Process "miniconda.exe" -ArgumentList "/S /D=C:\Miniconda" -Wait - echo "Adding Miniconda to PATH" - $env:Path += ";C:\Miniconda" - $env:Path += ";C:\Miniconda\Scripts" - $env:Path += ";C:\Miniconda\Library\bin" - conda --version - displayName: 'Install Miniconda' - checkout: self \ No newline at end of file diff --git a/tools/sdk-ai-bots/Scripts/Build-EngHubDocEmbeddings.ps1 b/tools/sdk-ai-bots/Scripts/Build-EngHubDocEmbeddings.ps1 index 6147d2b4907..e9c3c2589ce 100644 --- a/tools/sdk-ai-bots/Scripts/Build-EngHubDocEmbeddings.ps1 +++ b/tools/sdk-ai-bots/Scripts/Build-EngHubDocEmbeddings.ps1 @@ -94,7 +94,7 @@ $env:RAG_CHUNK_PATH = $ragChunkPath $env:METADATA_PATH = "$embeddingSourceFolder/metadata_enghub_docs.json" $env:DOCUMENT_PATH = $enghubDocsDestFolder $env:INCREMENTAL_EMBEDDING = $IncrementalEmbedding -if(-not (Initialize-PythonEnv)) { +if(-not (Initialize-CondaEnv)) { exit 1 } if(-not (Build-Embeddings -EmbeddingToolFolder $embeddingToolFolder)) { diff --git a/tools/sdk-ai-bots/Scripts/Common.ps1 b/tools/sdk-ai-bots/Scripts/Common.ps1 index 789ad2f8c00..b4d7c1c6240 100644 --- a/tools/sdk-ai-bots/Scripts/Common.ps1 +++ b/tools/sdk-ai-bots/Scripts/Common.ps1 @@ -82,6 +82,7 @@ function Build-Embeddings { Write-Error "The embedding tool folder does not exist: $embeddingToolFolder" return $false } + #$stopwatch = Measure-Command { Write-Host "Building embeddings..." try { @@ -100,30 +101,39 @@ function Build-Embeddings { $pythonEnvExePath = python -c "import sys; print(sys.executable)" Write-Host "Python executable path: $pythonEnvExePath" + $condaPath = "C:\Miniconda\Scripts\conda.exe" + if(-not (Test-Path $condaPath)) { + Write-Error "Conda is not installed at $condaPath" + return $false + } + Write-Host "Create Conda environment" + & $condaPath create -n myenv python=3.11 -y + + # Print Python version + $pythonVersion = & $condaPath run -n myenv python -c "import sys; print(sys.version)" + Write-Host "Python version: $pythonVersion" + # Print Python executable path + $pythonEnvExePath = & $condaPath run -n myenv python -c "import sys; print(sys.executable)" + Write-Host "Python executable path: $pythonEnvExePath" + # setup python environment and install required packages Write-Host "Setting up python environment" - python -m pip install --upgrade pip + & $condaPath run -n myenv python -m pip install --upgrade pip Write-Host "Installing required packages" - python -m pip install -r requirements.txt + & $condaPath run -n myenv python -m pip install -r requirements.txt $currentDir = Get-Location Write-Host "Current directory: $currentDir" Write-Host "List package versions..." - python -m pip list > pip_list.txt + & $condaPath run -n myenv python -m pip list > pip_list.txt Write-Host "Print the content of pip_list.txt" $installedPkg = Get-Content -Path "pip_list.txt" Write-Host $installedPkg Write-Host "Starts building" - $condaPath = "C:\Miniconda\Scripts\conda.exe" - if(-not (Test-Path $condaPath)) { - Write-Error "Conda is not installed at $condaPath" - return $false - } - & $condaPath create -n myenv python=3.11 -y & $condaPath run -n myenv python main.py } catch { @@ -175,7 +185,17 @@ function Upload-AzureBlob { return $false } -function Initialize-PythonEnv { +function Initialize-CondaEnv { + if(-not (Test-Path "C:\Miniconda" -PathType Container)) { + Write-Host "Installing Miniconda" + Invoke-WebRequest -Uri "https://repo.anaconda.com/miniconda/Miniconda3-latest-Windows-x86_64.exe" -OutFile "miniconda.exe" + Start-Process "miniconda.exe" -ArgumentList "/S /D=C:\Miniconda" -Wait + Write-Host "Adding Miniconda to PATH" + $env:Path += ";C:\Miniconda" + $env:Path += ";C:\Miniconda\Scripts" + $env:Path += ";C:\Miniconda\Library\bin" + conda --version + }<# Write-Host "Create Conda environment" $condaPath = "C:\Miniconda\Scripts\conda.exe" if(-not (Test-Path $condaPath)) { @@ -186,5 +206,5 @@ function Initialize-PythonEnv { Write-Host "Initialize Conda environment" & $condaPath init powershell Write-Host "Activate Conda environment" - & $condaPath activate myenv + & $condaPath activate myenv*/#> } From 1968ac9e887d6864f0e5199c6eb4caf78979a248 Mon Sep 17 00:00:00 2001 From: "Ray Chen (from Dev Box)" Date: Tue, 19 Mar 2024 14:29:57 +0800 Subject: [PATCH 42/64] Install conda in pwsh --- tools/sdk-ai-bots/Scripts/Common.ps1 | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/tools/sdk-ai-bots/Scripts/Common.ps1 b/tools/sdk-ai-bots/Scripts/Common.ps1 index b4d7c1c6240..781a7d7655b 100644 --- a/tools/sdk-ai-bots/Scripts/Common.ps1 +++ b/tools/sdk-ai-bots/Scripts/Common.ps1 @@ -186,7 +186,11 @@ function Upload-AzureBlob { } function Initialize-CondaEnv { - if(-not (Test-Path "C:\Miniconda" -PathType Container)) { + try { + Get-Command conda -ErrorAction Stop >$null + Write-Host "Conda is installed." + } catch { + Write-Host "Conda is not installed." Write-Host "Installing Miniconda" Invoke-WebRequest -Uri "https://repo.anaconda.com/miniconda/Miniconda3-latest-Windows-x86_64.exe" -OutFile "miniconda.exe" Start-Process "miniconda.exe" -ArgumentList "/S /D=C:\Miniconda" -Wait @@ -195,7 +199,9 @@ function Initialize-CondaEnv { $env:Path += ";C:\Miniconda\Scripts" $env:Path += ";C:\Miniconda\Library\bin" conda --version - }<# + } + return $true + <# Write-Host "Create Conda environment" $condaPath = "C:\Miniconda\Scripts\conda.exe" if(-not (Test-Path $condaPath)) { From bfa51c14378b3a0875d288d16d0155192a082acc Mon Sep 17 00:00:00 2001 From: "Ray Chen (from Dev Box)" Date: Wed, 20 Mar 2024 16:58:44 +0800 Subject: [PATCH 43/64] debug variables --- .../Embeddings/embedding/embedding.py | 8 +++---- .../Scripts/Build-EngHubDocEmbeddings.ps1 | 23 +++++++++++++++++++ 2 files changed, 27 insertions(+), 4 deletions(-) diff --git a/tools/sdk-ai-bots/Embeddings/embedding/embedding.py b/tools/sdk-ai-bots/Embeddings/embedding/embedding.py index 1622af447b1..95d463d21c8 100644 --- a/tools/sdk-ai-bots/Embeddings/embedding/embedding.py +++ b/tools/sdk-ai-bots/Embeddings/embedding/embedding.py @@ -18,10 +18,10 @@ class Embedding: def __init__(self): - os.environ["AZURESEARCH_FIELDS_CONTENT"] = "Text" - os.environ["AZURESEARCH_FIELDS_CONTENT_VECTOR"] = "Embedding" - os.environ["AZURESEARCH_FIELDS_TAG"] = "AdditionalMetadata" - os.environ["AZURESEARCH_FIELDS_ID"] = "Id" + #os.environ["AZURESEARCH_FIELDS_CONTENT"] = "Text" + #os.environ["AZURESEARCH_FIELDS_CONTENT_VECTOR"] = "Embedding" + #os.environ["AZURESEARCH_FIELDS_TAG"] = "AdditionalMetadata" + #os.environ["AZURESEARCH_FIELDS_ID"] = "Id" embeddings: OpenAIEmbeddings = OpenAIEmbeddings( openai_api_type="azure", diff --git a/tools/sdk-ai-bots/Scripts/Build-EngHubDocEmbeddings.ps1 b/tools/sdk-ai-bots/Scripts/Build-EngHubDocEmbeddings.ps1 index e9c3c2589ce..dc116a37bb9 100644 --- a/tools/sdk-ai-bots/Scripts/Build-EngHubDocEmbeddings.ps1 +++ b/tools/sdk-ai-bots/Scripts/Build-EngHubDocEmbeddings.ps1 @@ -94,9 +94,32 @@ $env:RAG_CHUNK_PATH = $ragChunkPath $env:METADATA_PATH = "$embeddingSourceFolder/metadata_enghub_docs.json" $env:DOCUMENT_PATH = $enghubDocsDestFolder $env:INCREMENTAL_EMBEDDING = $IncrementalEmbedding + +$env:AZURESEARCH_FIELDS_CONTENT = "Text" +$env:AZURESEARCH_FIELDS_CONTENT_VECTOR = "Embedding" +$env:AZURESEARCH_FIELDS_TAG = "AdditionalMetadata" +$env:AZURESEARCH_FIELDS_ID = "Id" + if(-not (Initialize-CondaEnv)) { exit 1 } + +# print the variables +Write-Host "RAG_CHUNK_PATH: $env:RAG_CHUNK_PATH" +Write-Host "METADATA_PATH: $env:METADATA_PATH" +Write-Host "DOCUMENT_PATH: $env:DOCUMENT_PATH" +Write-Host "INCREMENTAL_EMBEDDING: $env:INCREMENTAL_EMBEDDING" +Write-Host "AZURE_OPENAI_ENDPOINT: $env:AZURE_OPENAI_ENDPOINT" +Write-Host "AZURE_SEARCH_ENDPOINT: $env:AZURE_SEARCH_ENDPOINT" +Write-Host "AZURE_SEARCH_INDEX_NAME: $env:AZURE_SEARCH_INDEX_NAME" +Write-Host "AZURE_OPENAI_EMBEDDING_MODEL: $env:AZURE_OPENAI_EMBEDDING_MODEL" +Write-Host "AZURE_OPENAI_API_KEY: $env:AZURE_OPENAI_API_KEY" +Write-Host "AZURE_SEARCH_KEY: $env:AZURE_SEARCH_KEY" +Write-Host "AZURE_STORAGE_ACCOUNT_KEY: $env:AZURE_STORAGE_ACCOUNT_KEY" +Write-Host "AZURESEARCH_FIELDS_CONTENT: $env:AZURESEARCH_FIELDS_CONTENT" +Write-Host "AZURESEARCH_FIELDS_CONTENT_VECTOR: $env:AZURESEARCH_FIELDS_CONTENT_VECTOR" +Write-Host "AZURESEARCH_FIELDS_TAG: $env:AZURESEARCH_FIELDS_TAG" +Write-Host "AZURESEARCH_FIELDS_ID: $env:AZURESEARCH_FIELDS_ID" if(-not (Build-Embeddings -EmbeddingToolFolder $embeddingToolFolder)) { exit 1 } From 139b63f20820cab5a504a6afb3937bef37905f91 Mon Sep 17 00:00:00 2001 From: "Ray Chen (from Dev Box)" Date: Wed, 20 Mar 2024 18:11:13 +0800 Subject: [PATCH 44/64] Use conda for local run too --- .../Scripts/Build-EngHubDocEmbeddings.ps1 | 22 ++---- tools/sdk-ai-bots/Scripts/Common.ps1 | 69 ++++++------------- 2 files changed, 25 insertions(+), 66 deletions(-) diff --git a/tools/sdk-ai-bots/Scripts/Build-EngHubDocEmbeddings.ps1 b/tools/sdk-ai-bots/Scripts/Build-EngHubDocEmbeddings.ps1 index dc116a37bb9..697eb1c2420 100644 --- a/tools/sdk-ai-bots/Scripts/Build-EngHubDocEmbeddings.ps1 +++ b/tools/sdk-ai-bots/Scripts/Build-EngHubDocEmbeddings.ps1 @@ -100,27 +100,13 @@ $env:AZURESEARCH_FIELDS_CONTENT_VECTOR = "Embedding" $env:AZURESEARCH_FIELDS_TAG = "AdditionalMetadata" $env:AZURESEARCH_FIELDS_ID = "Id" -if(-not (Initialize-CondaEnv)) { +$CondaPath = Initialize-CondaEnv +if(-not $CondaPath) { + Write-Error "Failed to initialize conda environment at $CondaPath." exit 1 } -# print the variables -Write-Host "RAG_CHUNK_PATH: $env:RAG_CHUNK_PATH" -Write-Host "METADATA_PATH: $env:METADATA_PATH" -Write-Host "DOCUMENT_PATH: $env:DOCUMENT_PATH" -Write-Host "INCREMENTAL_EMBEDDING: $env:INCREMENTAL_EMBEDDING" -Write-Host "AZURE_OPENAI_ENDPOINT: $env:AZURE_OPENAI_ENDPOINT" -Write-Host "AZURE_SEARCH_ENDPOINT: $env:AZURE_SEARCH_ENDPOINT" -Write-Host "AZURE_SEARCH_INDEX_NAME: $env:AZURE_SEARCH_INDEX_NAME" -Write-Host "AZURE_OPENAI_EMBEDDING_MODEL: $env:AZURE_OPENAI_EMBEDDING_MODEL" -Write-Host "AZURE_OPENAI_API_KEY: $env:AZURE_OPENAI_API_KEY" -Write-Host "AZURE_SEARCH_KEY: $env:AZURE_SEARCH_KEY" -Write-Host "AZURE_STORAGE_ACCOUNT_KEY: $env:AZURE_STORAGE_ACCOUNT_KEY" -Write-Host "AZURESEARCH_FIELDS_CONTENT: $env:AZURESEARCH_FIELDS_CONTENT" -Write-Host "AZURESEARCH_FIELDS_CONTENT_VECTOR: $env:AZURESEARCH_FIELDS_CONTENT_VECTOR" -Write-Host "AZURESEARCH_FIELDS_TAG: $env:AZURESEARCH_FIELDS_TAG" -Write-Host "AZURESEARCH_FIELDS_ID: $env:AZURESEARCH_FIELDS_ID" -if(-not (Build-Embeddings -EmbeddingToolFolder $embeddingToolFolder)) { +if(-not (Build-Embeddings -EmbeddingToolFolder $embeddingToolFolder -CondaPath $CondaPath)) { exit 1 } diff --git a/tools/sdk-ai-bots/Scripts/Common.ps1 b/tools/sdk-ai-bots/Scripts/Common.ps1 index 781a7d7655b..7ce19ebb17e 100644 --- a/tools/sdk-ai-bots/Scripts/Common.ps1 +++ b/tools/sdk-ai-bots/Scripts/Common.ps1 @@ -75,7 +75,10 @@ function Build-Embeddings { param ( [Parameter(Position = 0)] [ValidateNotNullOrEmpty()] - [string] $EmbeddingToolFolder + [string] $EmbeddingToolFolder, + [Parameter(Position = 1)] + [ValidateNotNullOrEmpty()] + [string] $CondaPath ) if(-not (Test-Path $embeddingToolFolder)){ @@ -83,58 +86,37 @@ function Build-Embeddings { return $false } - #$stopwatch = Measure-Command { + $stopwatch = Measure-Command { Write-Host "Building embeddings..." try { - Push-Location $embeddingToolFolder - - Write-Host "Check Python environment" - $pythonVersion = python --version - Write-Host "Python version: $pythonVersion" - $pipVersion = python -m pip --version - Write-Host "Pip version: $pipVersion" - - # Print Python version - $pythonVersion = python -c "import sys; print(sys.version)" - Write-Host "Python version: $pythonVersion" - # Print Python executable path - $pythonEnvExePath = python -c "import sys; print(sys.executable)" - Write-Host "Python executable path: $pythonEnvExePath" - - $condaPath = "C:\Miniconda\Scripts\conda.exe" - if(-not (Test-Path $condaPath)) { - Write-Error "Conda is not installed at $condaPath" - return $false - } + Push-Location $embeddingToolFolder + Write-Host "Create Conda environment" - & $condaPath create -n myenv python=3.11 -y + & $CondaPath create -n myenv python=3.11 -y # Print Python version - $pythonVersion = & $condaPath run -n myenv python -c "import sys; print(sys.version)" + $pythonVersion = & $CondaPath run -n myenv python -c "import sys; print(sys.version)" Write-Host "Python version: $pythonVersion" # Print Python executable path - $pythonEnvExePath = & $condaPath run -n myenv python -c "import sys; print(sys.executable)" + $pythonEnvExePath = & $CondaPath run -n myenv python -c "import sys; print(sys.executable)" Write-Host "Python executable path: $pythonEnvExePath" # setup python environment and install required packages Write-Host "Setting up python environment" - & $condaPath run -n myenv python -m pip install --upgrade pip + & $CondaPath run -n myenv python -m pip install --upgrade pip Write-Host "Installing required packages" - & $condaPath run -n myenv python -m pip install -r requirements.txt - - $currentDir = Get-Location - Write-Host "Current directory: $currentDir" + & $CondaPath run -n myenv python -m pip install -r requirements.txt Write-Host "List package versions..." - & $condaPath run -n myenv python -m pip list > pip_list.txt + & $CondaPath run -n myenv python -m pip list > pip_list.txt Write-Host "Print the content of pip_list.txt" $installedPkg = Get-Content -Path "pip_list.txt" Write-Host $installedPkg Write-Host "Starts building" - & $condaPath run -n myenv python main.py + & $CondaPath run -n myenv python main.py } catch { Write-Error "Failed to build embeddings with exception:`n$_" @@ -142,10 +124,9 @@ function Build-Embeddings { } finally { Pop-Location - } - #} + } - #Write-Host "Finishes building with time: $($stopwatch.TotalSeconds) seconds" + Write-Host "Finishes building with time: $($stopwatch.TotalSeconds) seconds" return $true } @@ -185,10 +166,14 @@ function Upload-AzureBlob { return $false } +# only support windows platform function Initialize-CondaEnv { + $condaPath = "C:\Miniconda\Scripts\conda.exe" try { Get-Command conda -ErrorAction Stop >$null Write-Host "Conda is installed." + $condaPath = (Get-Command conda -All | Where-Object { $_.CommandType -eq 'Application' -and $_.Source -like '*conda.exe' }).Source + Write-Host $condaPath } catch { Write-Host "Conda is not installed." Write-Host "Installing Miniconda" @@ -200,17 +185,5 @@ function Initialize-CondaEnv { $env:Path += ";C:\Miniconda\Library\bin" conda --version } - return $true - <# - Write-Host "Create Conda environment" - $condaPath = "C:\Miniconda\Scripts\conda.exe" - if(-not (Test-Path $condaPath)) { - Write-Error "Conda is not installed at $condaPath" - return $false - } - & $condaPath create -n myenv python=3.11 -y - Write-Host "Initialize Conda environment" - & $condaPath init powershell - Write-Host "Activate Conda environment" - & $condaPath activate myenv*/#> + return $condaPath } From 8ad1029522d2955da11c5c2f87db9d5eb659613a Mon Sep 17 00:00:00 2001 From: "Ray Chen (from Dev Box)" Date: Wed, 20 Mar 2024 18:22:42 +0800 Subject: [PATCH 45/64] make blob container configurable --- .../.pipelines/build-document-embeddings.yml | 5 ++++- .../Scripts/Build-CustomizedDocEmbeddings.ps1 | 19 +++++++++++++++++-- .../Scripts/Build-EngHubDocEmbeddings.ps1 | 6 +++++- .../Build-TypeSpecAzureDocEmbeddings.ps1 | 19 +++++++++++++++++-- 4 files changed, 43 insertions(+), 6 deletions(-) diff --git a/tools/sdk-ai-bots/.pipelines/build-document-embeddings.yml b/tools/sdk-ai-bots/.pipelines/build-document-embeddings.yml index 45d5c05df60..1b4a37ea76a 100644 --- a/tools/sdk-ai-bots/.pipelines/build-document-embeddings.yml +++ b/tools/sdk-ai-bots/.pipelines/build-document-embeddings.yml @@ -12,6 +12,7 @@ pool: vmImage: 'windows-latest' variables: + st-container-name: $(storage-account-container) aoai-endpoint: $(azure-openai-endpoint) asch-endpoint: $(azure-search-endpoint) asch-index-name: $(azure-search-index-name) @@ -48,6 +49,7 @@ stages: AZURE_OPENAI_API_KEY: $(azure-openapi-key) AZURE_SEARCH_KEY: $(azure-search-key) AZURE_STORAGE_ACCOUNT_KEY: $(storage-account-key) + AZURE_STORAGE_ACCOUNT_CONTAINER: $(st-container-name) - stage: BuildCustomizedDocEmbeddings displayName: 'Build Customized Document Embeddings' @@ -70,4 +72,5 @@ stages: AZURE_OPENAI_EMBEDDING_MODEL: $(aoai-embedding-model) AZURE_OPENAI_API_KEY: $(azure-openapi-key) AZURE_SEARCH_KEY: $(azure-search-key) - AZURE_STORAGE_ACCOUNT_KEY: $(storage-account-key) \ No newline at end of file + AZURE_STORAGE_ACCOUNT_KEY: $(storage-account-key) + AZURE_STORAGE_ACCOUNT_CONTAINER: $(st-container-name) \ No newline at end of file diff --git a/tools/sdk-ai-bots/Scripts/Build-CustomizedDocEmbeddings.ps1 b/tools/sdk-ai-bots/Scripts/Build-CustomizedDocEmbeddings.ps1 index 40b0efae4a9..6acd6621dfc 100644 --- a/tools/sdk-ai-bots/Scripts/Build-CustomizedDocEmbeddings.ps1 +++ b/tools/sdk-ai-bots/Scripts/Build-CustomizedDocEmbeddings.ps1 @@ -113,10 +113,14 @@ else { # Download previous saved embeddings(last_rag_chunks_customized_docs.json) from Azure Blob Storage $storageAccountName = "saazuresdkbot" -$containerName = "rag-contents" $blobName = "last_rag_chunks_customized_docs.json" $destinationPath = $embeddingSourceFolder $ragChunkPath = Join-Path -Path $embeddingSourceFolder -ChildPath $blobName +$containerName = $env:AZURE_STORAGE_ACCOUNT_CONTAINER +if(-not $containerName) { + Write-Error "Please set the environment variable 'AZURE_STORAGE_ACCOUNT_CONTAINER'." + exit 1 +} if($IncrementalEmbedding -eq $true) { Write-Host "Downloading previous saved embeddings $blobName from Azure Blob Storage" if(-not (Download-AzureBlob -StorageAccountName $storageAccountName -ContainerName $containerName -BlobName $blobName -DestinationPath $destinationPath)) { @@ -130,7 +134,18 @@ $env:RAG_CHUNK_PATH = $ragChunkPath $env:METADATA_PATH = $customizedDocsMetadataFile $env:DOCUMENT_PATH = $customizedDocsDestFolder $env:INCREMENTAL_EMBEDDING = $IncrementalEmbedding -if(-not (Build-Embeddings -EmbeddingToolFolder $embeddingToolFolder)) { +$env:AZURESEARCH_FIELDS_CONTENT = "Text" +$env:AZURESEARCH_FIELDS_CONTENT_VECTOR = "Embedding" +$env:AZURESEARCH_FIELDS_TAG = "AdditionalMetadata" +$env:AZURESEARCH_FIELDS_ID = "Id" + +$CondaPath = Initialize-CondaEnv +if(-not $CondaPath) { + Write-Error "Failed to initialize conda environment at $CondaPath." + exit 1 +} + +if(-not (Build-Embeddings -EmbeddingToolFolder $embeddingToolFolder -CondaPath $CondaPath)) { exit 1 } diff --git a/tools/sdk-ai-bots/Scripts/Build-EngHubDocEmbeddings.ps1 b/tools/sdk-ai-bots/Scripts/Build-EngHubDocEmbeddings.ps1 index 697eb1c2420..8d07e56b9b1 100644 --- a/tools/sdk-ai-bots/Scripts/Build-EngHubDocEmbeddings.ps1 +++ b/tools/sdk-ai-bots/Scripts/Build-EngHubDocEmbeddings.ps1 @@ -77,10 +77,14 @@ else { # Download previous saved embeddings(last_rag_chunks_enghub_docs.json) from Azure Blob Storage $storageAccountName = "saazuresdkbot" -$containerName = "rag-contents" $blobName = "last_rag_chunks_enghub_docs.json" $destinationPath = $embeddingSourceFolder $ragChunkPath = Join-Path -Path $embeddingSourceFolder -ChildPath $blobName +$containerName = $env:AZURE_STORAGE_ACCOUNT_CONTAINER +if(-not $containerName) { + Write-Error "Please set the environment variable 'AZURE_STORAGE_ACCOUNT_CONTAINER'." + exit 1 +} if($IncrementalEmbedding -eq $true) { Write-Host "Downloading previous saved embeddings $blobName from Azure Blob Storage" if(-not (Download-AzureBlob -StorageAccountName $storageAccountName -ContainerName $containerName -BlobName $blobName -DestinationPath $destinationPath)) { diff --git a/tools/sdk-ai-bots/Scripts/Build-TypeSpecAzureDocEmbeddings.ps1 b/tools/sdk-ai-bots/Scripts/Build-TypeSpecAzureDocEmbeddings.ps1 index 9b5d10d768f..461a8cec5c7 100644 --- a/tools/sdk-ai-bots/Scripts/Build-TypeSpecAzureDocEmbeddings.ps1 +++ b/tools/sdk-ai-bots/Scripts/Build-TypeSpecAzureDocEmbeddings.ps1 @@ -78,10 +78,14 @@ else { # Download previous saved embeddings(last_rag_chunks_typespec_docs.json) from Azure Blob Storage $storageAccountName = "saazuresdkbot" -$containerName = "rag-contents" $blobName = "last_rag_chunks_typespec_docs.json" $destinationPath = $embeddingSourceFolder $ragChunkPath = Join-Path -Path $embeddingSourceFolder -ChildPath $blobName +$containerName = $env:AZURE_STORAGE_ACCOUNT_CONTAINER +if(-not $containerName) { + Write-Error "Please set the environment variable 'AZURE_STORAGE_ACCOUNT_CONTAINER'." + exit 1 +} if($IncrementalEmbedding -eq $true) { Write-Host "Downloading previous saved embeddings $blobName from Azure Blob Storage" if(-not (Download-AzureBlob -StorageAccountName $storageAccountName -ContainerName $containerName -BlobName $blobName -DestinationPath $destinationPath)) { @@ -95,7 +99,18 @@ $env:RAG_CHUNK_PATH = $ragChunkPath $env:METADATA_PATH = "$embeddingSourceFolder/metadata_typespec_docs.json" $env:DOCUMENT_PATH = $typespecDocsDestFolder $env:INCREMENTAL_EMBEDDING = $IncrementalEmbedding -if(-not (Build-Embeddings -EmbeddingToolFolder $embeddingToolFolder)) { +$env:AZURESEARCH_FIELDS_CONTENT = "Text" +$env:AZURESEARCH_FIELDS_CONTENT_VECTOR = "Embedding" +$env:AZURESEARCH_FIELDS_TAG = "AdditionalMetadata" +$env:AZURESEARCH_FIELDS_ID = "Id" + +$CondaPath = Initialize-CondaEnv +if(-not $CondaPath) { + Write-Error "Failed to initialize conda environment at $CondaPath." + exit 1 +} + +if(-not (Build-Embeddings -EmbeddingToolFolder $embeddingToolFolder -CondaPath $CondaPath)) { exit 1 } From 99b3620967ea06865dd5077882dc6dab79e33b5a Mon Sep 17 00:00:00 2001 From: "Ray Chen (from Dev Box)" Date: Wed, 20 Mar 2024 18:36:38 +0800 Subject: [PATCH 46/64] Add missing ending brace --- tools/sdk-ai-bots/Scripts/Common.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/sdk-ai-bots/Scripts/Common.ps1 b/tools/sdk-ai-bots/Scripts/Common.ps1 index 7ce19ebb17e..05ea0f04aaf 100644 --- a/tools/sdk-ai-bots/Scripts/Common.ps1 +++ b/tools/sdk-ai-bots/Scripts/Common.ps1 @@ -125,7 +125,7 @@ function Build-Embeddings { finally { Pop-Location } - + } Write-Host "Finishes building with time: $($stopwatch.TotalSeconds) seconds" return $true } From 4013183ebf9df52febb40547aace8427520efe2a Mon Sep 17 00:00:00 2001 From: "Ray Chen (from Dev Box)" Date: Wed, 20 Mar 2024 18:58:12 +0800 Subject: [PATCH 47/64] Added condapath to output --- .../Scripts/Build-EngHubDocEmbeddings.ps1 | 2 +- tools/sdk-ai-bots/Scripts/Common.ps1 | 12 +++++------- 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/tools/sdk-ai-bots/Scripts/Build-EngHubDocEmbeddings.ps1 b/tools/sdk-ai-bots/Scripts/Build-EngHubDocEmbeddings.ps1 index 8d07e56b9b1..522f9385bce 100644 --- a/tools/sdk-ai-bots/Scripts/Build-EngHubDocEmbeddings.ps1 +++ b/tools/sdk-ai-bots/Scripts/Build-EngHubDocEmbeddings.ps1 @@ -109,7 +109,7 @@ if(-not $CondaPath) { Write-Error "Failed to initialize conda environment at $CondaPath." exit 1 } - +Write-Host "CondaPath: $CondaPath" if(-not (Build-Embeddings -EmbeddingToolFolder $embeddingToolFolder -CondaPath $CondaPath)) { exit 1 } diff --git a/tools/sdk-ai-bots/Scripts/Common.ps1 b/tools/sdk-ai-bots/Scripts/Common.ps1 index 05ea0f04aaf..0c3446121dc 100644 --- a/tools/sdk-ai-bots/Scripts/Common.ps1 +++ b/tools/sdk-ai-bots/Scripts/Common.ps1 @@ -168,22 +168,20 @@ function Upload-AzureBlob { # only support windows platform function Initialize-CondaEnv { - $condaPath = "C:\Miniconda\Scripts\conda.exe" + $condaPath = "" try { Get-Command conda -ErrorAction Stop >$null Write-Host "Conda is installed." $condaPath = (Get-Command conda -All | Where-Object { $_.CommandType -eq 'Application' -and $_.Source -like '*conda.exe' }).Source - Write-Host $condaPath + Write-Host "Conda path: $condaPath" } catch { Write-Host "Conda is not installed." Write-Host "Installing Miniconda" Invoke-WebRequest -Uri "https://repo.anaconda.com/miniconda/Miniconda3-latest-Windows-x86_64.exe" -OutFile "miniconda.exe" Start-Process "miniconda.exe" -ArgumentList "/S /D=C:\Miniconda" -Wait - Write-Host "Adding Miniconda to PATH" - $env:Path += ";C:\Miniconda" - $env:Path += ";C:\Miniconda\Scripts" - $env:Path += ";C:\Miniconda\Library\bin" - conda --version + $condaPath = "C:\Miniconda\Scripts\conda.exe" + Write-Host "Conda path: $condaPath" } + return $condaPath } From 8946f6158c7c3e03a2d19c7a53f249bf8b298bb8 Mon Sep 17 00:00:00 2001 From: "Ray Chen (from Dev Box)" Date: Fri, 22 Mar 2024 10:44:16 +0800 Subject: [PATCH 48/64] Test without conda --- .../.pipelines/build-document-embeddings.yml | 4 + .../sdk-ai-bots/.pipelines/setup-pipeline.yml | 9 ++ .../Scripts/Build-EngHubDocEmbeddings.ps1 | 5 +- tools/sdk-ai-bots/Scripts/Common.ps1 | 117 ++++++++++++------ 4 files changed, 96 insertions(+), 39 deletions(-) diff --git a/tools/sdk-ai-bots/.pipelines/build-document-embeddings.yml b/tools/sdk-ai-bots/.pipelines/build-document-embeddings.yml index 1b4a37ea76a..ca24c11595a 100644 --- a/tools/sdk-ai-bots/.pipelines/build-document-embeddings.yml +++ b/tools/sdk-ai-bots/.pipelines/build-document-embeddings.yml @@ -31,6 +31,8 @@ stages: - job: BuildEngHubDocumentEmbeddings steps: - template: setup-pipeline.yml + parameters: + pythonVersion: '3.11.3' - checkout: git://internal/_git/azure-sdk-docs-eng.ms displayName: 'Checkout azure-sdk-docs-eng.ms repository' - task: Powershell@2 @@ -57,6 +59,8 @@ stages: - job: BuildCustomizedDocumentEmbeddings steps: - template: setup-pipeline.yml + parameters: + pythonVersion: '3.11.3' - task: Powershell@2 inputs: filePath: $(Build.SourcesDirectory)/azure-sdk-tools/tools/sdk-ai-bots/Scripts/Build-EngCustomizedDocEmbeddings.ps1 diff --git a/tools/sdk-ai-bots/.pipelines/setup-pipeline.yml b/tools/sdk-ai-bots/.pipelines/setup-pipeline.yml index 8a657813f2c..ebcd2c16de2 100644 --- a/tools/sdk-ai-bots/.pipelines/setup-pipeline.yml +++ b/tools/sdk-ai-bots/.pipelines/setup-pipeline.yml @@ -1,2 +1,11 @@ +parameters: +- name: pythonVersion + type: string + default: '3.x' + steps: +- task: UsePythonVersion@0 + inputs: + versionSpec: ${{ parameters.pythonVersion }} + addToPath: true - checkout: self \ No newline at end of file diff --git a/tools/sdk-ai-bots/Scripts/Build-EngHubDocEmbeddings.ps1 b/tools/sdk-ai-bots/Scripts/Build-EngHubDocEmbeddings.ps1 index 522f9385bce..db54529a850 100644 --- a/tools/sdk-ai-bots/Scripts/Build-EngHubDocEmbeddings.ps1 +++ b/tools/sdk-ai-bots/Scripts/Build-EngHubDocEmbeddings.ps1 @@ -104,7 +104,7 @@ $env:AZURESEARCH_FIELDS_CONTENT_VECTOR = "Embedding" $env:AZURESEARCH_FIELDS_TAG = "AdditionalMetadata" $env:AZURESEARCH_FIELDS_ID = "Id" -$CondaPath = Initialize-CondaEnv +<#$CondaPath = Initialize-CondaEnv if(-not $CondaPath) { Write-Error "Failed to initialize conda environment at $CondaPath." exit 1 @@ -112,6 +112,9 @@ if(-not $CondaPath) { Write-Host "CondaPath: $CondaPath" if(-not (Build-Embeddings -EmbeddingToolFolder $embeddingToolFolder -CondaPath $CondaPath)) { exit 1 +}#> +if(-not (Build-Embeddings -EmbeddingToolFolder $embeddingToolFolder)) { + exit 1 } # Upload embeddings output to Azure Blob Storage diff --git a/tools/sdk-ai-bots/Scripts/Common.ps1 b/tools/sdk-ai-bots/Scripts/Common.ps1 index 0c3446121dc..b521b45f977 100644 --- a/tools/sdk-ai-bots/Scripts/Common.ps1 +++ b/tools/sdk-ai-bots/Scripts/Common.ps1 @@ -77,7 +77,6 @@ function Build-Embeddings { [ValidateNotNullOrEmpty()] [string] $EmbeddingToolFolder, [Parameter(Position = 1)] - [ValidateNotNullOrEmpty()] [string] $CondaPath ) @@ -85,47 +84,89 @@ function Build-Embeddings { Write-Error "The embedding tool folder does not exist: $embeddingToolFolder" return $false } + $stopwatch = $null + if ($CondaPath -and (Test-Path $CondaPath)) { + $stopwatch = Measure-Command { + Write-Host "Building embeddings..." + try { + Push-Location $embeddingToolFolder + + Write-Host "Create Conda environment" + & $CondaPath create -n myenv python=3.11 -y - $stopwatch = Measure-Command { - Write-Host "Building embeddings..." - try { - Push-Location $embeddingToolFolder - - Write-Host "Create Conda environment" - & $CondaPath create -n myenv python=3.11 -y - - # Print Python version - $pythonVersion = & $CondaPath run -n myenv python -c "import sys; print(sys.version)" - Write-Host "Python version: $pythonVersion" - # Print Python executable path - $pythonEnvExePath = & $CondaPath run -n myenv python -c "import sys; print(sys.executable)" - Write-Host "Python executable path: $pythonEnvExePath" - - # setup python environment and install required packages - Write-Host "Setting up python environment" - & $CondaPath run -n myenv python -m pip install --upgrade pip - - Write-Host "Installing required packages" - & $CondaPath run -n myenv python -m pip install -r requirements.txt - - Write-Host "List package versions..." - & $CondaPath run -n myenv python -m pip list > pip_list.txt - - Write-Host "Print the content of pip_list.txt" - $installedPkg = Get-Content -Path "pip_list.txt" - Write-Host $installedPkg - - Write-Host "Starts building" - & $CondaPath run -n myenv python main.py + # Print Python version + $pythonVersion = & $CondaPath run -n myenv python -c "import sys; print(sys.version)" + Write-Host "Python version: $pythonVersion" + # Print Python executable path + $pythonEnvExePath = & $CondaPath run -n myenv python -c "import sys; print(sys.executable)" + Write-Host "Python executable path: $pythonEnvExePath" + + # setup python environment and install required packages + Write-Host "Setting up python environment" + & $CondaPath run -n myenv python -m pip install --upgrade pip + + Write-Host "Installing required packages" + & $CondaPath run -n myenv python -m pip install -r requirements.txt + + Write-Host "List package versions..." + & $CondaPath run -n myenv python -m pip list > pip_list.txt + + Write-Host "Print the content of pip_list.txt" + $installedPkg = Get-Content -Path "pip_list.txt" + Write-Host $installedPkg + + Write-Host "Starts building" + & $CondaPath run -n myenv python main.py + } + catch { + Write-Error "Failed to build embeddings with exception:`n$_" + return $false + } + finally { + Pop-Location + } } - catch { - Write-Error "Failed to build embeddings with exception:`n$_" - return $false + } + else { + $stopwatch = Measure-Command { + Write-Host "Building embeddings..." + try { + Push-Location $embeddingToolFolder + + # Print Python version + $pythonVersion = python -c "import sys; print(sys.version)" + Write-Host "Python version: $pythonVersion" + # Print Python executable path + $pythonEnvExePath = python -c "import sys; print(sys.executable)" + Write-Host "Python executable path: $pythonEnvExePath" + + # setup python environment and install required packages + Write-Host "Setting up python environment" + python -m pip install --upgrade pip + + Write-Host "Installing required packages" + python -m pip install -r requirements.txt + + Write-Host "List package versions..." + python -m pip list > pip_list.txt + + Write-Host "Print the content of pip_list.txt" + $installedPkg = Get-Content -Path "pip_list.txt" + Write-Host $installedPkg + + Write-Host "Starts building" + python main.py + } + catch { + Write-Error "Failed to build embeddings with exception:`n$_" + return $false + } + finally { + Pop-Location + } } - finally { - Pop-Location - } } + Write-Host "Finishes building with time: $($stopwatch.TotalSeconds) seconds" return $true } From 489ab6ffe3c7b69c6beef96fc492270fff73fd82 Mon Sep 17 00:00:00 2001 From: "Ray Chen (from Dev Box)" Date: Fri, 22 Mar 2024 11:59:53 +0800 Subject: [PATCH 49/64] Delete python version parameter --- tools/sdk-ai-bots/.pipelines/build-document-embeddings.yml | 2 -- tools/sdk-ai-bots/Scripts/Build-CustomizedDocEmbeddings.ps1 | 5 +++-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/tools/sdk-ai-bots/.pipelines/build-document-embeddings.yml b/tools/sdk-ai-bots/.pipelines/build-document-embeddings.yml index ca24c11595a..0adc9ebea84 100644 --- a/tools/sdk-ai-bots/.pipelines/build-document-embeddings.yml +++ b/tools/sdk-ai-bots/.pipelines/build-document-embeddings.yml @@ -59,8 +59,6 @@ stages: - job: BuildCustomizedDocumentEmbeddings steps: - template: setup-pipeline.yml - parameters: - pythonVersion: '3.11.3' - task: Powershell@2 inputs: filePath: $(Build.SourcesDirectory)/azure-sdk-tools/tools/sdk-ai-bots/Scripts/Build-EngCustomizedDocEmbeddings.ps1 diff --git a/tools/sdk-ai-bots/Scripts/Build-CustomizedDocEmbeddings.ps1 b/tools/sdk-ai-bots/Scripts/Build-CustomizedDocEmbeddings.ps1 index 6acd6621dfc..e51f3f17e08 100644 --- a/tools/sdk-ai-bots/Scripts/Build-CustomizedDocEmbeddings.ps1 +++ b/tools/sdk-ai-bots/Scripts/Build-CustomizedDocEmbeddings.ps1 @@ -139,13 +139,14 @@ $env:AZURESEARCH_FIELDS_CONTENT_VECTOR = "Embedding" $env:AZURESEARCH_FIELDS_TAG = "AdditionalMetadata" $env:AZURESEARCH_FIELDS_ID = "Id" +<# $CondaPath = Initialize-CondaEnv if(-not $CondaPath) { Write-Error "Failed to initialize conda environment at $CondaPath." exit 1 } - -if(-not (Build-Embeddings -EmbeddingToolFolder $embeddingToolFolder -CondaPath $CondaPath)) { +#> +if(-not (Build-Embeddings -EmbeddingToolFolder $embeddingToolFolder)) { exit 1 } From 7ddf6e4363831a82ad347fb748f99f62dff5476d Mon Sep 17 00:00:00 2001 From: "Ray Chen (from Dev Box)" Date: Fri, 22 Mar 2024 12:10:42 +0800 Subject: [PATCH 50/64] Use same source path when checkout one repo only --- tools/sdk-ai-bots/Scripts/Build-CustomizedDocEmbeddings.ps1 | 4 ---- tools/sdk-ai-bots/Scripts/Build-EngHubDocEmbeddings.ps1 | 2 +- .../sdk-ai-bots/Scripts/Build-TypeSpecAzureDocEmbeddings.ps1 | 2 +- 3 files changed, 2 insertions(+), 6 deletions(-) diff --git a/tools/sdk-ai-bots/Scripts/Build-CustomizedDocEmbeddings.ps1 b/tools/sdk-ai-bots/Scripts/Build-CustomizedDocEmbeddings.ps1 index e51f3f17e08..de6990e737f 100644 --- a/tools/sdk-ai-bots/Scripts/Build-CustomizedDocEmbeddings.ps1 +++ b/tools/sdk-ai-bots/Scripts/Build-CustomizedDocEmbeddings.ps1 @@ -65,10 +65,6 @@ function Download-GitHubFile { # Set the working directory, current location is supposed to be the root of the repository $buildSourceDirectory = Get-Location $workingDirectory = Join-Path $buildSourceDirectory "tools\sdk-ai-bots" -if($env:AGENT_ID) { - # Running in Azure DevOps - $workingDirectory = Join-Path $buildSourceDirectory "azure-sdk-tools\tools\sdk-ai-bots" -} $scriptsRoot = Join-Path $workingDirectory "Scripts" $embeddingToolFolder = Join-Path $workingDirectory "Embeddings" diff --git a/tools/sdk-ai-bots/Scripts/Build-EngHubDocEmbeddings.ps1 b/tools/sdk-ai-bots/Scripts/Build-EngHubDocEmbeddings.ps1 index db54529a850..e3d88e303ec 100644 --- a/tools/sdk-ai-bots/Scripts/Build-EngHubDocEmbeddings.ps1 +++ b/tools/sdk-ai-bots/Scripts/Build-EngHubDocEmbeddings.ps1 @@ -21,7 +21,7 @@ param ( $buildSourceDirectory = Get-Location $workingDirectory = Join-Path $buildSourceDirectory "tools\sdk-ai-bots" if($env:AGENT_ID) { - # Running in Azure DevOps + # Running in Azure DevOps, pipeline would checkout two repositories, azure-sdk-tools and enginerring hub repository, so the working directory should be azure-sdk-tools $workingDirectory = Join-Path $buildSourceDirectory "azure-sdk-tools\tools\sdk-ai-bots" } $scriptsRoot = Join-Path $workingDirectory "Scripts" diff --git a/tools/sdk-ai-bots/Scripts/Build-TypeSpecAzureDocEmbeddings.ps1 b/tools/sdk-ai-bots/Scripts/Build-TypeSpecAzureDocEmbeddings.ps1 index 461a8cec5c7..9ea9e04b0a7 100644 --- a/tools/sdk-ai-bots/Scripts/Build-TypeSpecAzureDocEmbeddings.ps1 +++ b/tools/sdk-ai-bots/Scripts/Build-TypeSpecAzureDocEmbeddings.ps1 @@ -22,7 +22,7 @@ param ( $buildSourceDirectory = Get-Location $workingDirectory = Join-Path $buildSourceDirectory "tools\sdk-ai-bots" if($env:AGENT_ID) { - # Running in Azure DevOps + # Running in Azure DevOps, pipeline would checkout two repositories, azure-sdk-tools and typespec repository, so the working directory should be azure-sdk-tools $workingDirectory = Join-Path $buildSourceDirectory "azure-sdk-tools\tools\sdk-ai-bots" } $scriptsRoot = Join-Path $workingDirectory "Scripts" From c571ac5c2f99cfd0407982ac394bb0b9d510b996 Mon Sep 17 00:00:00 2001 From: "Ray Chen (from Dev Box)" Date: Mon, 25 Mar 2024 15:49:46 +0800 Subject: [PATCH 51/64] Remove conda env --- .../.pipelines/build-document-embeddings.yml | 5 +- .../Scripts/Build-CustomizedDocEmbeddings.ps1 | 9 +- .../Scripts/Build-EngHubDocEmbeddings.ps1 | 11 +- .../Build-TypeSpecAzureDocEmbeddings.ps1 | 2 +- tools/sdk-ai-bots/Scripts/Common.ps1 | 138 +++++------------- 5 files changed, 42 insertions(+), 123 deletions(-) diff --git a/tools/sdk-ai-bots/.pipelines/build-document-embeddings.yml b/tools/sdk-ai-bots/.pipelines/build-document-embeddings.yml index 0adc9ebea84..41cbd980430 100644 --- a/tools/sdk-ai-bots/.pipelines/build-document-embeddings.yml +++ b/tools/sdk-ai-bots/.pipelines/build-document-embeddings.yml @@ -12,6 +12,7 @@ pool: vmImage: 'windows-latest' variables: + st-account-name: $(storage-account-name) st-container-name: $(storage-account-container) aoai-endpoint: $(azure-openai-endpoint) asch-endpoint: $(azure-search-endpoint) @@ -51,6 +52,7 @@ stages: AZURE_OPENAI_API_KEY: $(azure-openapi-key) AZURE_SEARCH_KEY: $(azure-search-key) AZURE_STORAGE_ACCOUNT_KEY: $(storage-account-key) + AZURE_STORAGE_ACCOUNT_NAME: $(st-account-name) AZURE_STORAGE_ACCOUNT_CONTAINER: $(st-container-name) - stage: BuildCustomizedDocEmbeddings @@ -61,7 +63,7 @@ stages: - template: setup-pipeline.yml - task: Powershell@2 inputs: - filePath: $(Build.SourcesDirectory)/azure-sdk-tools/tools/sdk-ai-bots/Scripts/Build-EngCustomizedDocEmbeddings.ps1 + filePath: $(Build.SourcesDirectory)/tools/sdk-ai-bots/Scripts/Build-EngCustomizedDocEmbeddings.ps1 arguments: > -IncrementalEmbedding "${{ parameters.incrementalEmbedding }}" pwsh: true @@ -75,4 +77,5 @@ stages: AZURE_OPENAI_API_KEY: $(azure-openapi-key) AZURE_SEARCH_KEY: $(azure-search-key) AZURE_STORAGE_ACCOUNT_KEY: $(storage-account-key) + AZURE_STORAGE_ACCOUNT_NAME: $(st-account-name) AZURE_STORAGE_ACCOUNT_CONTAINER: $(st-container-name) \ No newline at end of file diff --git a/tools/sdk-ai-bots/Scripts/Build-CustomizedDocEmbeddings.ps1 b/tools/sdk-ai-bots/Scripts/Build-CustomizedDocEmbeddings.ps1 index de6990e737f..d1c6f5d9e58 100644 --- a/tools/sdk-ai-bots/Scripts/Build-CustomizedDocEmbeddings.ps1 +++ b/tools/sdk-ai-bots/Scripts/Build-CustomizedDocEmbeddings.ps1 @@ -108,10 +108,10 @@ else { } # Download previous saved embeddings(last_rag_chunks_customized_docs.json) from Azure Blob Storage -$storageAccountName = "saazuresdkbot" $blobName = "last_rag_chunks_customized_docs.json" $destinationPath = $embeddingSourceFolder $ragChunkPath = Join-Path -Path $embeddingSourceFolder -ChildPath $blobName +$storageAccountName = $env:AZURE_STORAGE_ACCOUNT_NAME $containerName = $env:AZURE_STORAGE_ACCOUNT_CONTAINER if(-not $containerName) { Write-Error "Please set the environment variable 'AZURE_STORAGE_ACCOUNT_CONTAINER'." @@ -135,13 +135,6 @@ $env:AZURESEARCH_FIELDS_CONTENT_VECTOR = "Embedding" $env:AZURESEARCH_FIELDS_TAG = "AdditionalMetadata" $env:AZURESEARCH_FIELDS_ID = "Id" -<# -$CondaPath = Initialize-CondaEnv -if(-not $CondaPath) { - Write-Error "Failed to initialize conda environment at $CondaPath." - exit 1 -} -#> if(-not (Build-Embeddings -EmbeddingToolFolder $embeddingToolFolder)) { exit 1 } diff --git a/tools/sdk-ai-bots/Scripts/Build-EngHubDocEmbeddings.ps1 b/tools/sdk-ai-bots/Scripts/Build-EngHubDocEmbeddings.ps1 index e3d88e303ec..895ddfbd865 100644 --- a/tools/sdk-ai-bots/Scripts/Build-EngHubDocEmbeddings.ps1 +++ b/tools/sdk-ai-bots/Scripts/Build-EngHubDocEmbeddings.ps1 @@ -76,10 +76,10 @@ else { } # Download previous saved embeddings(last_rag_chunks_enghub_docs.json) from Azure Blob Storage -$storageAccountName = "saazuresdkbot" $blobName = "last_rag_chunks_enghub_docs.json" $destinationPath = $embeddingSourceFolder $ragChunkPath = Join-Path -Path $embeddingSourceFolder -ChildPath $blobName +$storageAccountName = $env:AZURE_STORAGE_ACCOUNT_NAME $containerName = $env:AZURE_STORAGE_ACCOUNT_CONTAINER if(-not $containerName) { Write-Error "Please set the environment variable 'AZURE_STORAGE_ACCOUNT_CONTAINER'." @@ -104,15 +104,6 @@ $env:AZURESEARCH_FIELDS_CONTENT_VECTOR = "Embedding" $env:AZURESEARCH_FIELDS_TAG = "AdditionalMetadata" $env:AZURESEARCH_FIELDS_ID = "Id" -<#$CondaPath = Initialize-CondaEnv -if(-not $CondaPath) { - Write-Error "Failed to initialize conda environment at $CondaPath." - exit 1 -} -Write-Host "CondaPath: $CondaPath" -if(-not (Build-Embeddings -EmbeddingToolFolder $embeddingToolFolder -CondaPath $CondaPath)) { - exit 1 -}#> if(-not (Build-Embeddings -EmbeddingToolFolder $embeddingToolFolder)) { exit 1 } diff --git a/tools/sdk-ai-bots/Scripts/Build-TypeSpecAzureDocEmbeddings.ps1 b/tools/sdk-ai-bots/Scripts/Build-TypeSpecAzureDocEmbeddings.ps1 index 9ea9e04b0a7..2fb08cc39b5 100644 --- a/tools/sdk-ai-bots/Scripts/Build-TypeSpecAzureDocEmbeddings.ps1 +++ b/tools/sdk-ai-bots/Scripts/Build-TypeSpecAzureDocEmbeddings.ps1 @@ -77,10 +77,10 @@ else { } # Download previous saved embeddings(last_rag_chunks_typespec_docs.json) from Azure Blob Storage -$storageAccountName = "saazuresdkbot" $blobName = "last_rag_chunks_typespec_docs.json" $destinationPath = $embeddingSourceFolder $ragChunkPath = Join-Path -Path $embeddingSourceFolder -ChildPath $blobName +$storageAccountName = $env:AZURE_STORAGE_ACCOUNT_NAME $containerName = $env:AZURE_STORAGE_ACCOUNT_CONTAINER if(-not $containerName) { Write-Error "Please set the environment variable 'AZURE_STORAGE_ACCOUNT_CONTAINER'." diff --git a/tools/sdk-ai-bots/Scripts/Common.ps1 b/tools/sdk-ai-bots/Scripts/Common.ps1 index b521b45f977..b604cbb39b7 100644 --- a/tools/sdk-ai-bots/Scripts/Common.ps1 +++ b/tools/sdk-ai-bots/Scripts/Common.ps1 @@ -75,98 +75,50 @@ function Build-Embeddings { param ( [Parameter(Position = 0)] [ValidateNotNullOrEmpty()] - [string] $EmbeddingToolFolder, - [Parameter(Position = 1)] - [string] $CondaPath + [string] $EmbeddingToolFolder ) if(-not (Test-Path $embeddingToolFolder)){ Write-Error "The embedding tool folder does not exist: $embeddingToolFolder" return $false } - $stopwatch = $null - if ($CondaPath -and (Test-Path $CondaPath)) { - $stopwatch = Measure-Command { - Write-Host "Building embeddings..." - try { - Push-Location $embeddingToolFolder - - Write-Host "Create Conda environment" - & $CondaPath create -n myenv python=3.11 -y - - # Print Python version - $pythonVersion = & $CondaPath run -n myenv python -c "import sys; print(sys.version)" - Write-Host "Python version: $pythonVersion" - # Print Python executable path - $pythonEnvExePath = & $CondaPath run -n myenv python -c "import sys; print(sys.executable)" - Write-Host "Python executable path: $pythonEnvExePath" - - # setup python environment and install required packages - Write-Host "Setting up python environment" - & $CondaPath run -n myenv python -m pip install --upgrade pip - - Write-Host "Installing required packages" - & $CondaPath run -n myenv python -m pip install -r requirements.txt - - Write-Host "List package versions..." - & $CondaPath run -n myenv python -m pip list > pip_list.txt - - Write-Host "Print the content of pip_list.txt" - $installedPkg = Get-Content -Path "pip_list.txt" - Write-Host $installedPkg - - Write-Host "Starts building" - & $CondaPath run -n myenv python main.py - } - catch { - Write-Error "Failed to build embeddings with exception:`n$_" - return $false - } - finally { - Pop-Location - } + $stopwatch = Measure-Command { + Write-Host "Building embeddings..." + try { + Push-Location $embeddingToolFolder + + # Print Python version + $pythonVersion = python -c "import sys; print(sys.version)" + Write-Host "Python version: $pythonVersion" + # Print Python executable path + $pythonEnvExePath = python -c "import sys; print(sys.executable)" + Write-Host "Python executable path: $pythonEnvExePath" + + # setup python environment and install required packages + Write-Host "Setting up python environment" + python -m pip install --upgrade pip + + Write-Host "Installing required packages" + python -m pip install -r requirements.txt + + Write-Host "List package versions..." + python -m pip list > pip_list.txt + + Write-Host "Print the content of pip_list.txt" + $installedPkg = Get-Content -Path "pip_list.txt" + Write-Host $installedPkg + + Write-Host "Starts building" + python main.py } - } - else { - $stopwatch = Measure-Command { - Write-Host "Building embeddings..." - try { - Push-Location $embeddingToolFolder - - # Print Python version - $pythonVersion = python -c "import sys; print(sys.version)" - Write-Host "Python version: $pythonVersion" - # Print Python executable path - $pythonEnvExePath = python -c "import sys; print(sys.executable)" - Write-Host "Python executable path: $pythonEnvExePath" - - # setup python environment and install required packages - Write-Host "Setting up python environment" - python -m pip install --upgrade pip - - Write-Host "Installing required packages" - python -m pip install -r requirements.txt - - Write-Host "List package versions..." - python -m pip list > pip_list.txt - - Write-Host "Print the content of pip_list.txt" - $installedPkg = Get-Content -Path "pip_list.txt" - Write-Host $installedPkg - - Write-Host "Starts building" - python main.py - } - catch { - Write-Error "Failed to build embeddings with exception:`n$_" - return $false - } - finally { - Pop-Location - } + catch { + Write-Error "Failed to build embeddings with exception:`n$_" + return $false + } + finally { + Pop-Location } } - Write-Host "Finishes building with time: $($stopwatch.TotalSeconds) seconds" return $true } @@ -206,23 +158,3 @@ function Upload-AzureBlob { } return $false } - -# only support windows platform -function Initialize-CondaEnv { - $condaPath = "" - try { - Get-Command conda -ErrorAction Stop >$null - Write-Host "Conda is installed." - $condaPath = (Get-Command conda -All | Where-Object { $_.CommandType -eq 'Application' -and $_.Source -like '*conda.exe' }).Source - Write-Host "Conda path: $condaPath" - } catch { - Write-Host "Conda is not installed." - Write-Host "Installing Miniconda" - Invoke-WebRequest -Uri "https://repo.anaconda.com/miniconda/Miniconda3-latest-Windows-x86_64.exe" -OutFile "miniconda.exe" - Start-Process "miniconda.exe" -ArgumentList "/S /D=C:\Miniconda" -Wait - $condaPath = "C:\Miniconda\Scripts\conda.exe" - Write-Host "Conda path: $condaPath" - } - - return $condaPath -} From 9f7bc6b9af3ee4bc7a45654b5128dacc429baae9 Mon Sep 17 00:00:00 2001 From: "Ray Chen (from Dev Box)" Date: Mon, 25 Mar 2024 15:58:33 +0800 Subject: [PATCH 52/64] Updated customizedDocEmbeddings script path --- tools/sdk-ai-bots/.pipelines/build-document-embeddings.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/sdk-ai-bots/.pipelines/build-document-embeddings.yml b/tools/sdk-ai-bots/.pipelines/build-document-embeddings.yml index 41cbd980430..5ae957c698e 100644 --- a/tools/sdk-ai-bots/.pipelines/build-document-embeddings.yml +++ b/tools/sdk-ai-bots/.pipelines/build-document-embeddings.yml @@ -63,7 +63,7 @@ stages: - template: setup-pipeline.yml - task: Powershell@2 inputs: - filePath: $(Build.SourcesDirectory)/tools/sdk-ai-bots/Scripts/Build-EngCustomizedDocEmbeddings.ps1 + filePath: $(Build.SourcesDirectory)/tools/sdk-ai-bots/Scripts/Build-CustomizedDocEmbeddings.ps1 arguments: > -IncrementalEmbedding "${{ parameters.incrementalEmbedding }}" pwsh: true From bad10b2a9c0438383962331d58a8fddfc2239b1a Mon Sep 17 00:00:00 2001 From: "Ray Chen (from Dev Box)" Date: Mon, 25 Mar 2024 16:38:02 +0800 Subject: [PATCH 53/64] Use 3.11 python version --- tools/sdk-ai-bots/.pipelines/build-document-embeddings.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tools/sdk-ai-bots/.pipelines/build-document-embeddings.yml b/tools/sdk-ai-bots/.pipelines/build-document-embeddings.yml index 5ae957c698e..27270e10cf3 100644 --- a/tools/sdk-ai-bots/.pipelines/build-document-embeddings.yml +++ b/tools/sdk-ai-bots/.pipelines/build-document-embeddings.yml @@ -61,6 +61,8 @@ stages: - job: BuildCustomizedDocumentEmbeddings steps: - template: setup-pipeline.yml + parameters: + pythonVersion: '3.11.3' - task: Powershell@2 inputs: filePath: $(Build.SourcesDirectory)/tools/sdk-ai-bots/Scripts/Build-CustomizedDocEmbeddings.ps1 From 9b3f69b024c597426517b8165bccca170a617fcc Mon Sep 17 00:00:00 2001 From: "Ray Chen (from Dev Box)" Date: Mon, 25 Mar 2024 20:09:02 +0800 Subject: [PATCH 54/64] Added typespec embeddings build step --- .../.pipelines/build-document-embeddings.yml | 25 +++++++++++++++++++ .../Build-TypeSpecAzureDocEmbeddings.ps1 | 3 ++- 2 files changed, 27 insertions(+), 1 deletion(-) diff --git a/tools/sdk-ai-bots/.pipelines/build-document-embeddings.yml b/tools/sdk-ai-bots/.pipelines/build-document-embeddings.yml index 27270e10cf3..2d8167ae23a 100644 --- a/tools/sdk-ai-bots/.pipelines/build-document-embeddings.yml +++ b/tools/sdk-ai-bots/.pipelines/build-document-embeddings.yml @@ -55,6 +55,31 @@ stages: AZURE_STORAGE_ACCOUNT_NAME: $(st-account-name) AZURE_STORAGE_ACCOUNT_CONTAINER: $(st-container-name) +- stage: BuildTypeSpecDocEmbeddings + displayName: 'Build TypeSpec Document Embeddings' + jobs: + - job: BuildTypeSpecDocumentEmbeddings + steps: + - template: setup-pipeline.yml + - task: Powershell@2 + inputs: + filePath: $(Build.SourcesDirectory)/tools/sdk-ai-bots/Scripts/Build-TypeSpecAzureDocEmbeddings.ps1 + arguments: > + -IncrementalEmbedding "${{ parameters.incrementalEmbedding }}" + pwsh: true + workingDirectory: $(Build.SourcesDirectory) + displayName: 'Run embeddings build script' + env: + AZURE_OPENAI_ENDPOINT: $(aoai-endpoint) + AZURE_SEARCH_ENDPOINT: $(asch-endpoint) + AZURE_SEARCH_INDEX_NAME: $(asch-index-name) + AZURE_OPENAI_EMBEDDING_MODEL: $(aoai-embedding-model) + AZURE_OPENAI_API_KEY: $(azure-openapi-key) + AZURE_SEARCH_KEY: $(azure-search-key) + AZURE_STORAGE_ACCOUNT_KEY: $(storage-account-key) + AZURE_STORAGE_ACCOUNT_NAME: $(st-account-name) + AZURE_STORAGE_ACCOUNT_CONTAINER: $(st-container-name) + - stage: BuildCustomizedDocEmbeddings displayName: 'Build Customized Document Embeddings' jobs: diff --git a/tools/sdk-ai-bots/Scripts/Build-TypeSpecAzureDocEmbeddings.ps1 b/tools/sdk-ai-bots/Scripts/Build-TypeSpecAzureDocEmbeddings.ps1 index 2fb08cc39b5..c7f5fbe903c 100644 --- a/tools/sdk-ai-bots/Scripts/Build-TypeSpecAzureDocEmbeddings.ps1 +++ b/tools/sdk-ai-bots/Scripts/Build-TypeSpecAzureDocEmbeddings.ps1 @@ -21,10 +21,11 @@ param ( # Set the working directory, current location is supposed to be the root of the repository $buildSourceDirectory = Get-Location $workingDirectory = Join-Path $buildSourceDirectory "tools\sdk-ai-bots" +<# if($env:AGENT_ID) { # Running in Azure DevOps, pipeline would checkout two repositories, azure-sdk-tools and typespec repository, so the working directory should be azure-sdk-tools $workingDirectory = Join-Path $buildSourceDirectory "azure-sdk-tools\tools\sdk-ai-bots" -} +}#> $scriptsRoot = Join-Path $workingDirectory "Scripts" $embeddingToolFolder = Join-Path $workingDirectory "Embeddings" From 800dae70eb1e2796e58d02b2bd8bf07ffbdbfa02 Mon Sep 17 00:00:00 2001 From: "Ray Chen (from Dev Box)" Date: Mon, 25 Mar 2024 20:17:08 +0800 Subject: [PATCH 55/64] Remove conda call --- .../Scripts/Build-TypeSpecAzureDocEmbeddings.ps1 | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/tools/sdk-ai-bots/Scripts/Build-TypeSpecAzureDocEmbeddings.ps1 b/tools/sdk-ai-bots/Scripts/Build-TypeSpecAzureDocEmbeddings.ps1 index c7f5fbe903c..e30f52a726d 100644 --- a/tools/sdk-ai-bots/Scripts/Build-TypeSpecAzureDocEmbeddings.ps1 +++ b/tools/sdk-ai-bots/Scripts/Build-TypeSpecAzureDocEmbeddings.ps1 @@ -105,13 +105,7 @@ $env:AZURESEARCH_FIELDS_CONTENT_VECTOR = "Embedding" $env:AZURESEARCH_FIELDS_TAG = "AdditionalMetadata" $env:AZURESEARCH_FIELDS_ID = "Id" -$CondaPath = Initialize-CondaEnv -if(-not $CondaPath) { - Write-Error "Failed to initialize conda environment at $CondaPath." - exit 1 -} - -if(-not (Build-Embeddings -EmbeddingToolFolder $embeddingToolFolder -CondaPath $CondaPath)) { +if(-not (Build-Embeddings -EmbeddingToolFolder $embeddingToolFolder)) { exit 1 } From 5178dadb191949968d50dfd7b38410ad1b36fe18 Mon Sep 17 00:00:00 2001 From: "Ray Chen (from Dev Box)" Date: Tue, 26 Mar 2024 10:29:20 +0800 Subject: [PATCH 56/64] Use raw content url for downloading --- tools/sdk-ai-bots/Scripts/Build-CustomizedDocEmbeddings.ps1 | 4 ++-- .../sdk-ai-bots/Scripts/Build-TypeSpecAzureDocEmbeddings.ps1 | 5 ----- 2 files changed, 2 insertions(+), 7 deletions(-) diff --git a/tools/sdk-ai-bots/Scripts/Build-CustomizedDocEmbeddings.ps1 b/tools/sdk-ai-bots/Scripts/Build-CustomizedDocEmbeddings.ps1 index d1c6f5d9e58..2c5f91dd94e 100644 --- a/tools/sdk-ai-bots/Scripts/Build-CustomizedDocEmbeddings.ps1 +++ b/tools/sdk-ai-bots/Scripts/Build-CustomizedDocEmbeddings.ps1 @@ -52,11 +52,11 @@ function Download-GitHubFile { } try { - Invoke-WebRequest -Uri $url -OutFile $DestinationFilePath + Invoke-WebRequest -Uri $FileUrl -OutFile $DestinationFilePath Write-Host "File downloaded successfully to: $DestinationFilePath" } catch { - Write-Error "Failed to download file from GitHub: $url" + Write-Error "Failed to download file from GitHub: $FileUrl" exit 1 } } diff --git a/tools/sdk-ai-bots/Scripts/Build-TypeSpecAzureDocEmbeddings.ps1 b/tools/sdk-ai-bots/Scripts/Build-TypeSpecAzureDocEmbeddings.ps1 index e30f52a726d..6ad37c7ef88 100644 --- a/tools/sdk-ai-bots/Scripts/Build-TypeSpecAzureDocEmbeddings.ps1 +++ b/tools/sdk-ai-bots/Scripts/Build-TypeSpecAzureDocEmbeddings.ps1 @@ -21,11 +21,6 @@ param ( # Set the working directory, current location is supposed to be the root of the repository $buildSourceDirectory = Get-Location $workingDirectory = Join-Path $buildSourceDirectory "tools\sdk-ai-bots" -<# -if($env:AGENT_ID) { - # Running in Azure DevOps, pipeline would checkout two repositories, azure-sdk-tools and typespec repository, so the working directory should be azure-sdk-tools - $workingDirectory = Join-Path $buildSourceDirectory "azure-sdk-tools\tools\sdk-ai-bots" -}#> $scriptsRoot = Join-Path $workingDirectory "Scripts" $embeddingToolFolder = Join-Path $workingDirectory "Embeddings" From f576de9a83c1ba146a8dff41821276c8c4d4c6b8 Mon Sep 17 00:00:00 2001 From: "Ray Chen (from Dev Box)" Date: Tue, 26 Mar 2024 10:37:10 +0800 Subject: [PATCH 57/64] Removed python version setting --- tools/sdk-ai-bots/.pipelines/build-document-embeddings.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/tools/sdk-ai-bots/.pipelines/build-document-embeddings.yml b/tools/sdk-ai-bots/.pipelines/build-document-embeddings.yml index 2d8167ae23a..3d4308d9add 100644 --- a/tools/sdk-ai-bots/.pipelines/build-document-embeddings.yml +++ b/tools/sdk-ai-bots/.pipelines/build-document-embeddings.yml @@ -32,8 +32,6 @@ stages: - job: BuildEngHubDocumentEmbeddings steps: - template: setup-pipeline.yml - parameters: - pythonVersion: '3.11.3' - checkout: git://internal/_git/azure-sdk-docs-eng.ms displayName: 'Checkout azure-sdk-docs-eng.ms repository' - task: Powershell@2 @@ -86,8 +84,6 @@ stages: - job: BuildCustomizedDocumentEmbeddings steps: - template: setup-pipeline.yml - parameters: - pythonVersion: '3.11.3' - task: Powershell@2 inputs: filePath: $(Build.SourcesDirectory)/tools/sdk-ai-bots/Scripts/Build-CustomizedDocEmbeddings.ps1 From 206579475ad200b6effb7b985ce06ec6ff8a455e Mon Sep 17 00:00:00 2001 From: "Ray Chen (from Dev Box)" Date: Tue, 26 Mar 2024 11:01:12 +0800 Subject: [PATCH 58/64] Recover the list of customized document --- .../settings/metadata_customized_docs.json | 56 +++++++++++++++++++ 1 file changed, 56 insertions(+) diff --git a/tools/sdk-ai-bots/Embeddings/settings/metadata_customized_docs.json b/tools/sdk-ai-bots/Embeddings/settings/metadata_customized_docs.json index 33109691477..4a3cd8cd39a 100644 --- a/tools/sdk-ai-bots/Embeddings/settings/metadata_customized_docs.json +++ b/tools/sdk-ai-bots/Embeddings/settings/metadata_customized_docs.json @@ -1,4 +1,60 @@ { + "ci-fix.md": { + "title": "CI Fix Guide", + "url": "https://github.com/Azure/azure-rest-api-specs/blob/main/documentation/ci-fix.md" + }, + "Getting-started-with-TypeSpec-specifications.md": { + "title": "What is a TypeSpec Specification?", + "url": "https://github.com/Azure/azure-rest-api-specs/blob/main/documentation/Getting-started-with-TypeSpec-specifications.md" + }, + "Semantic-and-Model-Violations-Reference.md": { + "title": "Semantic and Model Violations Reference", + "url": "https://github.com/Azure/azure-rest-api-specs/blob/main/documentation/Semantic-and-Model-Violations-Reference.md" + }, + "lintdiff-rules.md": { + "title": "Azure Rules", + "url": "https://github.com/Azure/azure-openapi-validator/blob/main/docs/rules.md" + }, + "typespec-rest-api-dev-process.md": { + "title": "Azure Rest API, SDK development process with TypeSpec", + "url": "https://github.com/Azure/azure-rest-api-specs/blob/main/documentation/typespec-rest-api-dev-process.md" + }, + "typespec-structure-guidelines.md": { + "title": "Repository Guidelines for TypeSpec projects", + "url": "https://github.com/Azure/azure-rest-api-specs/blob/main/documentation/typespec-structure-guidelines.md" + }, + "onboard-dpg-in-sdkautomation-README.md": { + "title": "Service Onboard DPG with Swagger CI Pipeline", + "url": "https://github.com/Azure/azure-rest-api-specs/blob/main/documentation/onboard-dpg-in-sdkautomation/README.md" + }, + "rules-for-release-request.md": { + "title": "Some guidance and rules when you fill the release issue", + "url": "https://github.com/Azure/azure-rest-api-specs/blob/main/documentation/release-request/rules-for-release-request.md" + }, + "configure-go-sdk.md": { + "title": "Readme Configuration Guide for Azure SDK for Go", + "url": "https://github.com/Azure/azure-rest-api-specs/blob/main/documentation/code-gen/configure-go-sdk.md" + }, + "configure-python-sdk.md": { + "title": "Readme Configuration Guide for Azure SDK for Python", + "url": "https://github.com/Azure/azure-rest-api-specs/blob/main/documentation/code-gen/configure-python-sdk.md" + }, + "configure-typescript-sdk.md": { + "title": "Readme Configuration Guide for Azure SDK for Javascript (Typescript)", + "url": "https://github.com/Azure/azure-rest-api-specs/blob/main/documentation/code-gen/configure-typescript-sdk.md" + }, + "autorest-extensions-readme.md": { + "title": "AutoRest Extensions for OpenAPI 2.0", + "url": "https://github.com/Azure/autorest/blob/main/docs/extensions/readme.md" + }, + "avocado-readme.md": { + "title": "Avocado Readme", + "url": "https://github.com/Azure/avocado/blob/main/README.md" + }, + "control_plane_template.md": { + "title": "Control plane pull request template", + "url": "https://github.com/Azure/azure-rest-api-specs/blob/main/.github/PULL_REQUEST_TEMPLATE/control_plane_template.md" + }, "data_plane_template.md": { "title": "Data plane pull request template", "url": "https://github.com/Azure/azure-rest-api-specs/blob/main/.github/PULL_REQUEST_TEMPLATE/data_plane_template.md" From 0f80819c3129f53e76d378186de86e3070df7937 Mon Sep 17 00:00:00 2001 From: "Ray Chen (from Dev Box)" Date: Tue, 26 Mar 2024 11:57:19 +0800 Subject: [PATCH 59/64] Output key in metadata file --- tools/sdk-ai-bots/Scripts/Markdown-BuildIndexMetadata.ps1 | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tools/sdk-ai-bots/Scripts/Markdown-BuildIndexMetadata.ps1 b/tools/sdk-ai-bots/Scripts/Markdown-BuildIndexMetadata.ps1 index ff616774b46..e806fb1326e 100644 --- a/tools/sdk-ai-bots/Scripts/Markdown-BuildIndexMetadata.ps1 +++ b/tools/sdk-ai-bots/Scripts/Markdown-BuildIndexMetadata.ps1 @@ -62,15 +62,14 @@ function Generate-Metadata([string]$rootFolder, [string]$outputFolder) $pagePath = $_.DirectoryName.Substring($_.DirectoryName.IndexOf("\docs\")+"\docs\".Length).Replace('\','/') $url = $DocBaseUrl + '/' + $pagePath + '/' + $fileName $url = $url.Replace(' ', '%20') - #Write-Host "filename:" $fileName - Write-Host "url:" $url + Write-Host "The URL of the Markdown file is: $url" $title = Get-TitleFromMarkdown $_.FullName Write-Host "The title of the Markdown file is: $title" - #$key = $_.Name.Replace(' ','-') # adding path path to key to avoid name conflict $key = $pagePath + '/' + $_.Name $key = $key.Replace(' ', '-') $key = $key.Replace('/', '-') + Write-Host "The key of the Markdown file is: $key" $fileData = @{ "title" = $title "url" = $url From bb8ad6036e5cd2c40c9cb63282e7031bbb3f8f9d Mon Sep 17 00:00:00 2001 From: "Ray Chen (from Dev Box)" Date: Tue, 26 Mar 2024 14:17:28 +0800 Subject: [PATCH 60/64] Variables have to be set in env rather than .py --- tools/sdk-ai-bots/Embeddings/embedding/embedding.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/tools/sdk-ai-bots/Embeddings/embedding/embedding.py b/tools/sdk-ai-bots/Embeddings/embedding/embedding.py index 95d463d21c8..5dce2fbfe85 100644 --- a/tools/sdk-ai-bots/Embeddings/embedding/embedding.py +++ b/tools/sdk-ai-bots/Embeddings/embedding/embedding.py @@ -18,10 +18,6 @@ class Embedding: def __init__(self): - #os.environ["AZURESEARCH_FIELDS_CONTENT"] = "Text" - #os.environ["AZURESEARCH_FIELDS_CONTENT_VECTOR"] = "Embedding" - #os.environ["AZURESEARCH_FIELDS_TAG"] = "AdditionalMetadata" - #os.environ["AZURESEARCH_FIELDS_ID"] = "Id" embeddings: OpenAIEmbeddings = OpenAIEmbeddings( openai_api_type="azure", From 82ee5d984975422afbd41503a89ef6c1f20e55b8 Mon Sep 17 00:00:00 2001 From: "Ray Chen (from Dev Box)" Date: Tue, 26 Mar 2024 14:30:04 +0800 Subject: [PATCH 61/64] Added codeowner --- .github/CODEOWNERS | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 63c4f38cc92..6687704d140 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -39,6 +39,7 @@ /tools/perf-automation/ @mikeharder @benbp /tools/pipeline-generator/ @weshaggard @benbp /tools/pipeline-witness/ @praveenkuttappan @weshaggard +/tools/sdk-ai-bots/ @raych1 @lirenhe /tools/sdk-generation-pipeline/ @weshaggard @praveenkuttappan @maririos /tools/sdk-testgen/ @raych1 @tadelesh /tools/test-proxy/ @scbedd @mikeharder From 7dd251e74f516e2ac867dd629c20fbc298c51f85 Mon Sep 17 00:00:00 2001 From: "Ray Chen (from Dev Box)" Date: Tue, 26 Mar 2024 14:41:01 +0800 Subject: [PATCH 62/64] Disable pr trigger --- tools/sdk-ai-bots/.pipelines/build-document-embeddings.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/tools/sdk-ai-bots/.pipelines/build-document-embeddings.yml b/tools/sdk-ai-bots/.pipelines/build-document-embeddings.yml index 3d4308d9add..52937f725e3 100644 --- a/tools/sdk-ai-bots/.pipelines/build-document-embeddings.yml +++ b/tools/sdk-ai-bots/.pipelines/build-document-embeddings.yml @@ -1,4 +1,5 @@ trigger: none +pr: none schedules: - cron: "0 0 * * *" From d64c40ba7a993bd61851a4baaaf34624631bdb64 Mon Sep 17 00:00:00 2001 From: "Ray Chen (from Dev Box)" Date: Tue, 26 Mar 2024 15:44:51 +0800 Subject: [PATCH 63/64] Updated readme.md --- tools/sdk-ai-bots/AzureSdkQaBot/README.md | 4 +-- tools/sdk-ai-bots/Embeddings/README.md | 23 ++++++++++++-- tools/sdk-ai-bots/README.md | 37 ++++++++++++++++++++++- 3 files changed, 59 insertions(+), 5 deletions(-) diff --git a/tools/sdk-ai-bots/AzureSdkQaBot/README.md b/tools/sdk-ai-bots/AzureSdkQaBot/README.md index 96b6d1cd4c4..551b7ba18a5 100644 --- a/tools/sdk-ai-bots/AzureSdkQaBot/README.md +++ b/tools/sdk-ai-bots/AzureSdkQaBot/README.md @@ -1,5 +1,5 @@ -# Azure SDK Assistant +# Azure SDK Teams Assistant ## Summary -This is a conversational bot for Microsoft Teams that answers the question related to Azure rest api spec repo document and pull request review. +This is a conversational bot for Microsoft Teams that answers the question related to the Azure domain area. diff --git a/tools/sdk-ai-bots/Embeddings/README.md b/tools/sdk-ai-bots/Embeddings/README.md index 31eb844174c..e4940360c31 100644 --- a/tools/sdk-ai-bots/Embeddings/README.md +++ b/tools/sdk-ai-bots/Embeddings/README.md @@ -1,4 +1,23 @@ -## Get Started +## How to Run This Tool 1. Run `pip install -r requirements.txt` to install requirements. 1. Set the environment variables that are defined in `settings/settings.py` in `.env` file. -1. Run `python main.py`. \ No newline at end of file +1. Run `python main.py`. + +## Environment Variables +INCREMENTAL_EMBEDDING: the option to build embedding incrementally. +METADATA_PATH: the file path of the metadata file which contains the document URL and title. +DOCUMENT_PATH: the folder path of the document which need to build embeddings. +RAG_CHUNK_PATH: the file path of the RAG chunk file which is the last version or just the file name if it doesn't exist. + +AZURE_OPENAI_API_KEY: Azure OpenAI api key +AZURE_OPENAI_ENDPOINT: Azure OpenAI endpoint +AZURE_SEARCH_KEY: Azure search service key +AZURE_SEARCH_ENDPOINT: Azure serach service endpoint +AZURE_SEARCH_INDEX_NAME: Azure serach service index name +AZURE_OPENAI_EMBEDDING_MODEL: the deployed model name in Azure OpenAI service + +##### DO NOT CHANGE BELOW VARIABLES' VALUE +AZURESEARCH_FIELDS_CONTENT=Text +AZURESEARCH_FIELDS_CONTENT_VECTOR=Embedding +AZURESEARCH_FIELDS_TAG=AdditionalMetadata +AZURESEARCH_FIELDS_ID=Id diff --git a/tools/sdk-ai-bots/README.md b/tools/sdk-ai-bots/README.md index a840efbb4c6..ed77b3a2b7f 100644 --- a/tools/sdk-ai-bots/README.md +++ b/tools/sdk-ai-bots/README.md @@ -1,2 +1,37 @@ -# sdk-ai-bots +## Overview This folder contains a collection of tools that utilize AI techniques. + +#### AzureSdkQaBot +AzureSdkQaBot is a Teams bot which can answer the questions related to the Azure SDK domain. It is written in C#. + +#### Embeddings +It is a tool written in Python that uses `langchain` library to create embeddings in Azure Search Service. + +#### Scripts +This folder contains some scripts to build embeddings by calling the `Embeddings` tool. + + +## How to Refresh the Document Embeddings Used by Teams Bot +We have an [Azure DevOps pipeline](https://dev.azure.com/azure-sdk/internal/_build?definitionId=6811&_a=summary) which can help create or refresh the embeddings. + +1. This pipeline contains three stages: + - Build EngHub Document Embeddings + This stage builds embeddings for all the documents under the [engineering hub site](https://dev.azure.com/azure-sdk/internal/_git/azure-sdk-docs-eng.ms?path=/docs) + - Build TypeSpec Document Embeddings + This stage builds embeddings for all the documents under the [typespec-azure site](https://github.com/Azure/typespec-azure) + - Build Customized Document Embeddings + This stage builds embeddings for some markdown documents which are publicly accessible. + +2. The user can select specific stages when running the pipeline. By default, all three stages are included. + +3. The pipeline has an option to refresh the embeddings incrementally. By default, `Incremental Embedding Build` is selected when the pipeline is triggered. If the user wants to create embeddings from scratch, they should unselect this option when triggering the pipeline. + +### How to Add a New Document to the Customized Document List +If you have a publicly accessbile markdown file that you want the Teams bot to understand, you can add the information to [this file](https://github.com/Azure/azure-sdk-tools/blob/main/tools/sdk-ai-bots/Embeddings/settings/metadata_customized_docs.json) in the following format. +```JSON +"ci-fix.md": { + "title": "CI Fix Guide", + "url": "https://github.com/Azure/azure-rest-api-specs/blob/main/documentation/ci-fix.md" + } +``` + This file is a `JSON`, and you must ensure that the `key` in this `JSON` is not duplicated when adding a new document. \ No newline at end of file From ce791763a4c935efc8231679636b27814c95406f Mon Sep 17 00:00:00 2001 From: "Ray Chen (from Dev Box)" Date: Tue, 26 Mar 2024 19:24:34 +0800 Subject: [PATCH 64/64] Remove renhe temporarily --- .github/CODEOWNERS | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 6687704d140..8deca237a9d 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -39,7 +39,7 @@ /tools/perf-automation/ @mikeharder @benbp /tools/pipeline-generator/ @weshaggard @benbp /tools/pipeline-witness/ @praveenkuttappan @weshaggard -/tools/sdk-ai-bots/ @raych1 @lirenhe +/tools/sdk-ai-bots/ @raych1 /tools/sdk-generation-pipeline/ @weshaggard @praveenkuttappan @maririos /tools/sdk-testgen/ @raych1 @tadelesh /tools/test-proxy/ @scbedd @mikeharder