From 4e3c6ac414058ce51d34c1e5afc5db3bbb2665a9 Mon Sep 17 00:00:00 2001 From: Ben Broderick Phillips Date: Wed, 3 Jul 2024 17:32:44 -0400 Subject: [PATCH 1/5] Support storage network access and worm removal in remove test resources script --- .../TestResources/New-TestResources.ps1 | 293 +--------------- .../TestResources/Remove-TestResources.ps1 | 19 + .../TestResources/TestResources-Helpers.ps1 | 327 ++++++++++++++++++ .../TestResources/remove-test-resources.yml | 10 +- .../scripts/Helpers/Resource-Helpers.ps1 | 79 +++++ 5 files changed, 435 insertions(+), 293 deletions(-) create mode 100644 eng/common/TestResources/TestResources-Helpers.ps1 diff --git a/eng/common/TestResources/New-TestResources.ps1 b/eng/common/TestResources/New-TestResources.ps1 index 6ee09ff3b23e..1af87d50de1f 100644 --- a/eng/common/TestResources/New-TestResources.ps1 +++ b/eng/common/TestResources/New-TestResources.ps1 @@ -117,6 +117,7 @@ param ( $NewTestResourcesRemainingArguments ) +. $PSScriptRoot/TestResources-Helpers.ps1 . $PSScriptRoot/SubConfig-Helpers.ps1 if (!$ServicePrincipalAuth) { @@ -131,272 +132,6 @@ if (!$PSBoundParameters.ContainsKey('ErrorAction')) { $ErrorActionPreference = 'Stop' } -function Log($Message) -{ - Write-Host ('{0} - {1}' -f [DateTime]::Now.ToLongTimeString(), $Message) -} - -# vso commands are specially formatted log lines that are parsed by Azure Pipelines -# to perform additional actions, most commonly marking values as secrets. -# https://docs.microsoft.com/en-us/azure/devops/pipelines/scripts/logging-commands -function LogVsoCommand([string]$message) -{ - if (!$CI -or $SuppressVsoCommands) { - return - } - Write-Host $message -} - -function Retry([scriptblock] $Action, [int] $Attempts = 5) -{ - $attempt = 0 - $sleep = 5 - - while ($attempt -lt $Attempts) { - try { - $attempt++ - return $Action.Invoke() - } catch { - if ($attempt -lt $Attempts) { - $sleep *= 2 - - Write-Warning "Attempt $attempt failed: $_. Trying again in $sleep seconds..." - Start-Sleep -Seconds $sleep - } else { - throw - } - } - } -} - -# NewServicePrincipalWrapper creates an object from an AAD graph or Microsoft Graph service principal object type. -# This is necessary to work around breaking changes introduced in Az version 7.0.0: -# https://azure.microsoft.com/en-us/updates/update-your-apps-to-use-microsoft-graph-before-30-june-2022/ -function NewServicePrincipalWrapper([string]$subscription, [string]$resourceGroup, [string]$displayName) -{ - if ((Get-Module Az.Resources).Version -eq "5.3.0") { - # https://github.com/Azure/azure-powershell/issues/17040 - # New-AzAdServicePrincipal calls will fail with: - # "You cannot call a method on a null-valued expression." - Write-Warning "Az.Resources version 5.3.0 is not supported. Please update to >= 5.3.1" - Write-Warning "Update-Module Az.Resources -RequiredVersion 5.3.1" - exit 1 - } - - try { - $servicePrincipal = Retry { - New-AzADServicePrincipal -Role "Owner" -Scope "/subscriptions/$SubscriptionId/resourceGroups/$ResourceGroupName" -DisplayName $displayName - } - } catch { - # The underlying error "The directory object quota limit for the Principal has been exceeded" gets overwritten by the module trying - # to call New-AzADApplication with a null object instead of stopping execution, which makes this case hard to diagnose because it prints the following: - # "Cannot bind argument to parameter 'ObjectId' because it is an empty string." - # Provide a more helpful diagnostic prompt to the user if appropriate: - $totalApps = (Get-AzADApplication -OwnedApplication).Length - $msg = "App Registrations owned by you total $totalApps and may exceed the max quota (likely around 135)." + ` - "`nTry removing some at https://ms.portal.azure.com/#view/Microsoft_AAD_IAM/ActiveDirectoryMenuBlade/~/RegisteredApps" + ` - " or by running the following command to remove apps created by this script:" + ` - "`n Get-AzADApplication -DisplayNameStartsWith '$baseName' | Remove-AzADApplication" + ` - "`nNOTE: You may need to wait for the quota number to be updated after removing unused applications." - Write-Warning $msg - throw - } - - $spPassword = "" - $appId = "" - if (Get-Member -Name "Secret" -InputObject $servicePrincipal -MemberType property) { - Write-Verbose "Using legacy PSADServicePrincipal object type from AAD graph API" - # Secret property exists on PSADServicePrincipal type from AAD graph in Az # module versions < 7.0.0 - $spPassword = $servicePrincipal.Secret - $appId = $servicePrincipal.ApplicationId - } else { - if ((Get-Module Az.Resources).Version -eq "5.1.0") { - Write-Verbose "Creating password and credential for service principal via MS Graph API" - Write-Warning "Please update Az.Resources to >= 5.2.0 by running 'Update-Module Az'" - # Microsoft graph objects (Az.Resources version == 5.1.0) do not provision a secret on creation so it must be added separately. - # Submitting a password credential object without specifying a password will result in one being generated on the server side. - $password = New-Object -TypeName "Microsoft.Azure.PowerShell.Cmdlets.Resources.MSGraph.Models.ApiV10.MicrosoftGraphPasswordCredential" - $password.DisplayName = "Password for $displayName" - $credential = Retry { New-AzADSpCredential -PasswordCredentials $password -ServicePrincipalObject $servicePrincipal -ErrorAction 'Stop' } - $spPassword = ConvertTo-SecureString $credential.SecretText -AsPlainText -Force - $appId = $servicePrincipal.AppId - } else { - Write-Verbose "Creating service principal credential via MS Graph API" - # In 5.2.0 the password credential issue was fixed (see https://github.com/Azure/azure-powershell/pull/16690) but the - # parameter set was changed making the above call fail due to a missing ServicePrincipalId parameter. - $credential = Retry { $servicePrincipal | New-AzADSpCredential -ErrorAction 'Stop' } - $spPassword = ConvertTo-SecureString $credential.SecretText -AsPlainText -Force - $appId = $servicePrincipal.AppId - } - } - - return @{ - AppId = $appId - ApplicationId = $appId - # This is the ObjectId/OID but most return objects use .Id so keep it consistent to prevent confusion - Id = $servicePrincipal.Id - DisplayName = $servicePrincipal.DisplayName - Secret = $spPassword - } -} - -function LoadCloudConfig([string] $env) -{ - $configPath = "$PSScriptRoot/clouds/$env.json" - if (!(Test-Path $configPath)) { - Write-Warning "Could not find cloud configuration for environment '$env'" - return @{} - } - - $config = Get-Content $configPath | ConvertFrom-Json -AsHashtable - return $config -} - -function MergeHashes([hashtable] $source, [psvariable] $dest) -{ - foreach ($key in $source.Keys) { - if ($dest.Value.Contains($key) -and $dest.Value[$key] -ne $source[$key]) { - Write-Warning ("Overwriting '$($dest.Name).$($key)' with value '$($dest.Value[$key])' " + - "to new value '$($source[$key])'") - } - $dest.Value[$key] = $source[$key] - } -} - -function BuildBicepFile([System.IO.FileSystemInfo] $file) -{ - if (!(Get-Command bicep -ErrorAction Ignore)) { - Write-Error "A bicep file was found at '$($file.FullName)' but the Azure Bicep CLI is not installed. See aka.ms/bicep-install" - throw - } - - $tmp = $env:TEMP ? $env:TEMP : [System.IO.Path]::GetTempPath() - $templateFilePath = Join-Path $tmp "$ResourceType-resources.$(New-Guid).compiled.json" - - # Az can deploy bicep files natively, but by compiling here it becomes easier to parse the - # outputted json for mismatched parameter declarations. - bicep build $file.FullName --outfile $templateFilePath - if ($LASTEXITCODE) { - Write-Error "Failure building bicep file '$($file.FullName)'" - throw - } - - return $templateFilePath -} - -function BuildDeploymentOutputs([string]$serviceName, [object]$azContext, [object]$deployment, [hashtable]$environmentVariables) { - $serviceDirectoryPrefix = BuildServiceDirectoryPrefix $serviceName - # Add default values - $deploymentOutputs = [Ordered]@{ - "${serviceDirectoryPrefix}SUBSCRIPTION_ID" = $azContext.Subscription.Id; - "${serviceDirectoryPrefix}RESOURCE_GROUP" = $resourceGroup.ResourceGroupName; - "${serviceDirectoryPrefix}LOCATION" = $resourceGroup.Location; - "${serviceDirectoryPrefix}ENVIRONMENT" = $azContext.Environment.Name; - "${serviceDirectoryPrefix}AZURE_AUTHORITY_HOST" = $azContext.Environment.ActiveDirectoryAuthority; - "${serviceDirectoryPrefix}RESOURCE_MANAGER_URL" = $azContext.Environment.ResourceManagerUrl; - "${serviceDirectoryPrefix}SERVICE_MANAGEMENT_URL" = $azContext.Environment.ServiceManagementUrl; - "AZURE_SERVICE_DIRECTORY" = $serviceName.ToUpperInvariant(); - } - - if ($ServicePrincipalAuth) { - $deploymentOutputs["${serviceDirectoryPrefix}CLIENT_ID"] = $TestApplicationId; - $deploymentOutputs["${serviceDirectoryPrefix}CLIENT_SECRET"] = $TestApplicationSecret; - $deploymentOutputs["${serviceDirectoryPrefix}TENANT_ID"] = $azContext.Tenant.Id; - } - - MergeHashes $environmentVariables $(Get-Variable deploymentOutputs) - - foreach ($key in $deployment.Outputs.Keys) { - $variable = $deployment.Outputs[$key] - - # Work around bug that makes the first few characters of environment variables be lowercase. - $key = $key.ToUpperInvariant() - - if ($variable.Type -eq 'String' -or $variable.Type -eq 'SecureString') { - $deploymentOutputs[$key] = $variable.Value - } - } - - # Force capitalization of all keys to avoid Azure Pipelines confusion with - # variable auto-capitalization and OS env var capitalization differences - $capitalized = @{} - foreach ($item in $deploymentOutputs.GetEnumerator()) { - $capitalized[$item.Name.ToUpperInvariant()] = $item.Value - } - - return $capitalized -} - -function SetDeploymentOutputs( - [string]$serviceName, - [object]$azContext, - [object]$deployment, - [object]$templateFile, - [hashtable]$environmentVariables = @{} -) { - $deploymentEnvironmentVariables = $environmentVariables.Clone() - $deploymentOutputs = BuildDeploymentOutputs $serviceName $azContext $deployment $deploymentEnvironmentVariables - - if ($OutFile) { - if (!$IsWindows) { - Write-Host 'File option is supported only on Windows' - } - - $outputFile = "$($templateFile.originalFilePath).env" - - $environmentText = $deploymentOutputs | ConvertTo-Json; - $bytes = [System.Text.Encoding]::UTF8.GetBytes($environmentText) - $protectedBytes = [Security.Cryptography.ProtectedData]::Protect($bytes, $null, [Security.Cryptography.DataProtectionScope]::CurrentUser) - - Set-Content $outputFile -Value $protectedBytes -AsByteStream -Force - - Write-Host "Test environment settings`n $environmentText`nstored into encrypted $outputFile" - } else { - if (!$CI) { - # Write an extra new line to isolate the environment variables for easy reading. - Log "Persist the following environment variables based on your detected shell ($shell):`n" - } - - # Write overwrite warnings first, since local execution prints a runnable command to export variables - foreach ($key in $deploymentOutputs.Keys) { - if ([Environment]::GetEnvironmentVariable($key)) { - Write-Warning "Deployment outputs will overwrite pre-existing environment variable '$key'" - } - } - - # Marking values as secret by allowed keys below is not sufficient, as there may be outputs set in the ARM/bicep - # file that re-mark those values as secret (since all user-provided deployment outputs are treated as secret by default). - # This variable supports a second check on not marking previously allowed keys/values as secret. - $notSecretValues = @() - foreach ($key in $deploymentOutputs.Keys) { - $value = $deploymentOutputs[$key] - $deploymentEnvironmentVariables[$key] = $value - - if ($CI) { - if (ShouldMarkValueAsSecret $serviceName $key $value $notSecretValues) { - # Treat all ARM template output variables as secrets since "SecureString" variables do not set values. - # In order to mask secrets but set environment variables for any given ARM template, we set variables twice as shown below. - LogVsoCommand "##vso[task.setvariable variable=_$key;issecret=true;]$value" - Write-Host "Setting variable as secret '$key'" - } else { - Write-Host "Setting variable '$key': $value" - $notSecretValues += $value - } - LogVsoCommand "##vso[task.setvariable variable=$key;]$value" - } else { - Write-Host ($shellExportFormat -f $key, $value) - } - } - - if ($key) { - # Isolate the environment variables for easy reading. - Write-Host "`n" - $key = $null - } - } - - return $deploymentEnvironmentVariables, $deploymentOutputs -} # Support actions to invoke on exit. $exitActions = @({ @@ -843,31 +578,7 @@ try { -templateFile $templateFile ` -environmentVariables $EnvironmentVariables - $storageAccounts = Retry { Get-AzResource -ResourceGroupName $ResourceGroupName -ResourceType "Microsoft.Storage/storageAccounts" } - # Add client IP to storage account when running as local user. Pipeline's have their own vnet with access - if ($storageAccounts) { - foreach ($account in $storageAccounts) { - $rules = Get-AzStorageAccountNetworkRuleSet -ResourceGroupName $ResourceGroupName -AccountName $account.Name - if ($rules -and $rules.DefaultAction -eq "Allow") { - Write-Host "Restricting network rules in storage account '$($account.Name)' to deny access by default" - Retry { Update-AzStorageAccountNetworkRuleSet -ResourceGroupName $ResourceGroupName -Name $account.Name -DefaultAction Deny } - if ($CI -and $env:PoolSubnet) { - Write-Host "Enabling access to '$($account.Name)' from pipeline subnet $($env:PoolSubnet)" - Retry { Add-AzStorageAccountNetworkRule -ResourceGroupName $ResourceGroupName -Name $account.Name -VirtualNetworkResourceId $env:PoolSubnet } - } elseif ($AllowIpRanges) { - Write-Host "Enabling access to '$($account.Name)' to $($AllowIpRanges.Length) IP ranges" - $ipRanges = $AllowIpRanges | ForEach-Object { - @{ Action = 'allow'; IPAddressOrRange = $_ } - } - Retry { Update-AzStorageAccountNetworkRuleSet -ResourceGroupName $ResourceGroupName -Name $account.Name -IPRule $ipRanges | Out-Null } - } elseif (!$CI) { - Write-Host "Enabling access to '$($account.Name)' from client IP" - $clientIp ??= Retry { Invoke-RestMethod -Uri 'https://icanhazip.com/' } # cloudflare owned ip site - Retry { Add-AzStorageAccountNetworkRule -ResourceGroupName $ResourceGroupName -Name $account.Name -IPAddressOrRange $clientIp | Out-Null } - } - } - } - } + SetResourceNetworkAccessRules -ResourceGroupName $ResourceGroupName -AllowIpRanges $AllowIpRanges -CI:$CI $postDeploymentScript = $templateFile.originalFilePath | Split-Path | Join-Path -ChildPath "$ResourceType-resources-post.ps1" if (Test-Path $postDeploymentScript) { diff --git a/eng/common/TestResources/Remove-TestResources.ps1 b/eng/common/TestResources/Remove-TestResources.ps1 index 490b41b8ebe9..b5d10fb11f50 100644 --- a/eng/common/TestResources/Remove-TestResources.ps1 +++ b/eng/common/TestResources/Remove-TestResources.ps1 @@ -61,6 +61,19 @@ param ( [Parameter()] [switch] $ServicePrincipalAuth, + # List of CIDR ranges to add to specific resource firewalls, e.g. @(10.100.0.0/16, 10.200.0.0/16) + [Parameter()] + [ValidateCount(0,399)] + [Validatescript({ + foreach ($range in $PSItem) { + if ($range -like '*/31' -or $range -like '*/32') { + throw "Firewall IP Ranges cannot contain a /31 or /32 CIDR" + } + } + return $true + })] + [array] $AllowIpRanges = @(), + [Parameter()] [switch] $Force, @@ -69,6 +82,9 @@ param ( $RemoveTestResourcesRemainingArguments ) +. (Join-Path $PSScriptRoot .. scripts Helpers Resource-Helpers.ps1) +. (Join-Path $PSScriptRoot TestResources-Helpers.ps1) + # By default stop for any error. if (!$PSBoundParameters.ContainsKey('ErrorAction')) { $ErrorActionPreference = 'Stop' @@ -241,6 +257,9 @@ $verifyDeleteScript = { # Get any resources that can be purged after the resource group is deleted coerced into a collection even if empty. $purgeableResources = Get-PurgeableGroupResources $ResourceGroupName +SetStorageNetworkAccessRules -ResourceGroupName $ResourceGroupName -AllowIpRanges $AllowIpRanges -Override -CI:$CI +Remove-WormStorageAccounts -GroupPrefix $ResourceGroupName + Log "Deleting resource group '$ResourceGroupName'" if ($Force -and !$purgeableResources) { Remove-AzResourceGroup -Name "$ResourceGroupName" -Force:$Force -AsJob diff --git a/eng/common/TestResources/TestResources-Helpers.ps1 b/eng/common/TestResources/TestResources-Helpers.ps1 new file mode 100644 index 000000000000..c148e5c2dd1d --- /dev/null +++ b/eng/common/TestResources/TestResources-Helpers.ps1 @@ -0,0 +1,327 @@ +function Log($Message) { + Write-Host ('{0} - {1}' -f [DateTime]::Now.ToLongTimeString(), $Message) +} + +# vso commands are specially formatted log lines that are parsed by Azure Pipelines +# to perform additional actions, most commonly marking values as secrets. +# https://docs.microsoft.com/en-us/azure/devops/pipelines/scripts/logging-commands +function LogVsoCommand([string]$message) { + if (!$CI -or $SuppressVsoCommands) { + return + } + Write-Host $message +} + +function Retry([scriptblock] $Action, [int] $Attempts = 5) { + $attempt = 0 + $sleep = 5 + + while ($attempt -lt $Attempts) { + try { + $attempt++ + return $Action.Invoke() + } + catch { + if ($attempt -lt $Attempts) { + $sleep *= 2 + + Write-Warning "Attempt $attempt failed: $_. Trying again in $sleep seconds..." + Start-Sleep -Seconds $sleep + } + else { + throw + } + } + } +} + +# NewServicePrincipalWrapper creates an object from an AAD graph or Microsoft Graph service principal object type. +# This is necessary to work around breaking changes introduced in Az version 7.0.0: +# https://azure.microsoft.com/en-us/updates/update-your-apps-to-use-microsoft-graph-before-30-june-2022/ +function NewServicePrincipalWrapper([string]$subscription, [string]$resourceGroup, [string]$displayName) { + if ((Get-Module Az.Resources).Version -eq "5.3.0") { + # https://github.com/Azure/azure-powershell/issues/17040 + # New-AzAdServicePrincipal calls will fail with: + # "You cannot call a method on a null-valued expression." + Write-Warning "Az.Resources version 5.3.0 is not supported. Please update to >= 5.3.1" + Write-Warning "Update-Module Az.Resources -RequiredVersion 5.3.1" + exit 1 + } + + try { + $servicePrincipal = Retry { + New-AzADServicePrincipal -Role "Owner" -Scope "/subscriptions/$SubscriptionId/resourceGroups/$ResourceGroupName" -DisplayName $displayName + } + } + catch { + # The underlying error "The directory object quota limit for the Principal has been exceeded" gets overwritten by the module trying + # to call New-AzADApplication with a null object instead of stopping execution, which makes this case hard to diagnose because it prints the following: + # "Cannot bind argument to parameter 'ObjectId' because it is an empty string." + # Provide a more helpful diagnostic prompt to the user if appropriate: + $totalApps = (Get-AzADApplication -OwnedApplication).Length + $msg = "App Registrations owned by you total $totalApps and may exceed the max quota (likely around 135)." + ` + "`nTry removing some at https://ms.portal.azure.com/#view/Microsoft_AAD_IAM/ActiveDirectoryMenuBlade/~/RegisteredApps" + ` + " or by running the following command to remove apps created by this script:" + ` + "`n Get-AzADApplication -DisplayNameStartsWith '$baseName' | Remove-AzADApplication" + ` + "`nNOTE: You may need to wait for the quota number to be updated after removing unused applications." + Write-Warning $msg + throw + } + + $spPassword = "" + $appId = "" + if (Get-Member -Name "Secret" -InputObject $servicePrincipal -MemberType property) { + Write-Verbose "Using legacy PSADServicePrincipal object type from AAD graph API" + # Secret property exists on PSADServicePrincipal type from AAD graph in Az # module versions < 7.0.0 + $spPassword = $servicePrincipal.Secret + $appId = $servicePrincipal.ApplicationId + } + else { + if ((Get-Module Az.Resources).Version -eq "5.1.0") { + Write-Verbose "Creating password and credential for service principal via MS Graph API" + Write-Warning "Please update Az.Resources to >= 5.2.0 by running 'Update-Module Az'" + # Microsoft graph objects (Az.Resources version == 5.1.0) do not provision a secret on creation so it must be added separately. + # Submitting a password credential object without specifying a password will result in one being generated on the server side. + $password = New-Object -TypeName "Microsoft.Azure.PowerShell.Cmdlets.Resources.MSGraph.Models.ApiV10.MicrosoftGraphPasswordCredential" + $password.DisplayName = "Password for $displayName" + $credential = Retry { New-AzADSpCredential -PasswordCredentials $password -ServicePrincipalObject $servicePrincipal -ErrorAction 'Stop' } + $spPassword = ConvertTo-SecureString $credential.SecretText -AsPlainText -Force + $appId = $servicePrincipal.AppId + } + else { + Write-Verbose "Creating service principal credential via MS Graph API" + # In 5.2.0 the password credential issue was fixed (see https://github.com/Azure/azure-powershell/pull/16690) but the + # parameter set was changed making the above call fail due to a missing ServicePrincipalId parameter. + $credential = Retry { $servicePrincipal | New-AzADSpCredential -ErrorAction 'Stop' } + $spPassword = ConvertTo-SecureString $credential.SecretText -AsPlainText -Force + $appId = $servicePrincipal.AppId + } + } + + return @{ + AppId = $appId + ApplicationId = $appId + # This is the ObjectId/OID but most return objects use .Id so keep it consistent to prevent confusion + Id = $servicePrincipal.Id + DisplayName = $servicePrincipal.DisplayName + Secret = $spPassword + } +} + +function LoadCloudConfig([string] $env) { + $configPath = "$PSScriptRoot/clouds/$env.json" + if (!(Test-Path $configPath)) { + Write-Warning "Could not find cloud configuration for environment '$env'" + return @{} + } + + $config = Get-Content $configPath | ConvertFrom-Json -AsHashtable + return $config +} + +function MergeHashes([hashtable] $source, [psvariable] $dest) { + foreach ($key in $source.Keys) { + if ($dest.Value.Contains($key) -and $dest.Value[$key] -ne $source[$key]) { + Write-Warning ("Overwriting '$($dest.Name).$($key)' with value '$($dest.Value[$key])' " + + "to new value '$($source[$key])'") + } + $dest.Value[$key] = $source[$key] + } +} + +function BuildBicepFile([System.IO.FileSystemInfo] $file) { + if (!(Get-Command bicep -ErrorAction Ignore)) { + Write-Error "A bicep file was found at '$($file.FullName)' but the Azure Bicep CLI is not installed. See aka.ms/bicep-install" + throw + } + + $tmp = $env:TEMP ? $env:TEMP : [System.IO.Path]::GetTempPath() + $templateFilePath = Join-Path $tmp "$ResourceType-resources.$(New-Guid).compiled.json" + + # Az can deploy bicep files natively, but by compiling here it becomes easier to parse the + # outputted json for mismatched parameter declarations. + bicep build $file.FullName --outfile $templateFilePath + if ($LASTEXITCODE) { + Write-Error "Failure building bicep file '$($file.FullName)'" + throw + } + + return $templateFilePath +} + +function BuildDeploymentOutputs([string]$serviceName, [object]$azContext, [object]$deployment, [hashtable]$environmentVariables) { + $serviceDirectoryPrefix = BuildServiceDirectoryPrefix $serviceName + # Add default values + $deploymentOutputs = [Ordered]@{ + "${serviceDirectoryPrefix}SUBSCRIPTION_ID" = $azContext.Subscription.Id; + "${serviceDirectoryPrefix}RESOURCE_GROUP" = $resourceGroup.ResourceGroupName; + "${serviceDirectoryPrefix}LOCATION" = $resourceGroup.Location; + "${serviceDirectoryPrefix}ENVIRONMENT" = $azContext.Environment.Name; + "${serviceDirectoryPrefix}AZURE_AUTHORITY_HOST" = $azContext.Environment.ActiveDirectoryAuthority; + "${serviceDirectoryPrefix}RESOURCE_MANAGER_URL" = $azContext.Environment.ResourceManagerUrl; + "${serviceDirectoryPrefix}SERVICE_MANAGEMENT_URL" = $azContext.Environment.ServiceManagementUrl; + "AZURE_SERVICE_DIRECTORY" = $serviceName.ToUpperInvariant(); + } + + if ($ServicePrincipalAuth) { + $deploymentOutputs["${serviceDirectoryPrefix}CLIENT_ID"] = $TestApplicationId; + $deploymentOutputs["${serviceDirectoryPrefix}CLIENT_SECRET"] = $TestApplicationSecret; + $deploymentOutputs["${serviceDirectoryPrefix}TENANT_ID"] = $azContext.Tenant.Id; + } + + MergeHashes $environmentVariables $(Get-Variable deploymentOutputs) + + foreach ($key in $deployment.Outputs.Keys) { + $variable = $deployment.Outputs[$key] + + # Work around bug that makes the first few characters of environment variables be lowercase. + $key = $key.ToUpperInvariant() + + if ($variable.Type -eq 'String' -or $variable.Type -eq 'SecureString') { + $deploymentOutputs[$key] = $variable.Value + } + } + + # Force capitalization of all keys to avoid Azure Pipelines confusion with + # variable auto-capitalization and OS env var capitalization differences + $capitalized = @{} + foreach ($item in $deploymentOutputs.GetEnumerator()) { + $capitalized[$item.Name.ToUpperInvariant()] = $item.Value + } + + return $capitalized +} + +function SetDeploymentOutputs( + [string]$serviceName, + [object]$azContext, + [object]$deployment, + [object]$templateFile, + [hashtable]$environmentVariables = @{} +) { + $deploymentEnvironmentVariables = $environmentVariables.Clone() + $deploymentOutputs = BuildDeploymentOutputs $serviceName $azContext $deployment $deploymentEnvironmentVariables + + if ($OutFile) { + if (!$IsWindows) { + Write-Host 'File option is supported only on Windows' + } + + $outputFile = "$($templateFile.originalFilePath).env" + + $environmentText = $deploymentOutputs | ConvertTo-Json; + $bytes = [System.Text.Encoding]::UTF8.GetBytes($environmentText) + $protectedBytes = [Security.Cryptography.ProtectedData]::Protect($bytes, $null, [Security.Cryptography.DataProtectionScope]::CurrentUser) + + Set-Content $outputFile -Value $protectedBytes -AsByteStream -Force + + Write-Host "Test environment settings`n $environmentText`nstored into encrypted $outputFile" + } + else { + if (!$CI) { + # Write an extra new line to isolate the environment variables for easy reading. + Log "Persist the following environment variables based on your detected shell ($shell):`n" + } + + # Write overwrite warnings first, since local execution prints a runnable command to export variables + foreach ($key in $deploymentOutputs.Keys) { + if ([Environment]::GetEnvironmentVariable($key)) { + Write-Warning "Deployment outputs will overwrite pre-existing environment variable '$key'" + } + } + + # Marking values as secret by allowed keys below is not sufficient, as there may be outputs set in the ARM/bicep + # file that re-mark those values as secret (since all user-provided deployment outputs are treated as secret by default). + # This variable supports a second check on not marking previously allowed keys/values as secret. + $notSecretValues = @() + foreach ($key in $deploymentOutputs.Keys) { + $value = $deploymentOutputs[$key] + $deploymentEnvironmentVariables[$key] = $value + + if ($CI) { + if (ShouldMarkValueAsSecret $serviceName $key $value $notSecretValues) { + # Treat all ARM template output variables as secrets since "SecureString" variables do not set values. + # In order to mask secrets but set environment variables for any given ARM template, we set variables twice as shown below. + LogVsoCommand "##vso[task.setvariable variable=_$key;issecret=true;]$value" + Write-Host "Setting variable as secret '$key'" + } + else { + Write-Host "Setting variable '$key': $value" + $notSecretValues += $value + } + LogVsoCommand "##vso[task.setvariable variable=$key;]$value" + } + else { + Write-Host ($shellExportFormat -f $key, $value) + } + } + + if ($key) { + # Isolate the environment variables for easy reading. + Write-Host "`n" + $key = $null + } + } + + return $deploymentEnvironmentVariables, $deploymentOutputs +} + +function SetResourceNetworkAccessRules([string]$ResourceGroupName, [array]$AllowIpRanges, [switch]$CI) { + SetStorageNetworkAccessRules -ResourceGroupName $ResourceGroupName -AllowIpRanges $AllowIpRanges -CI:$CI +} + +function SetStorageNetworkAccessRules([string]$ResourceGroupName, [array]$AllowIpRanges, [switch]$CI, [switch]$Override) { + $storageAccounts = Retry { Get-AzResource -ResourceGroupName $ResourceGroupName -ResourceType "Microsoft.Storage/storageAccounts" } + # Add client IP to storage account when running as local user. Pipeline's have their own vnet with access + if ($storageAccounts) { + foreach ($account in $storageAccounts) { + $rules = Get-AzStorageAccountNetworkRuleSet -ResourceGroupName $ResourceGroupName -AccountName $account.Name + if ($rules -and ($Override -or $rules.DefaultAction -eq "Allow")) { + Write-Host "Restricting network rules in storage account '$($account.Name)' to deny access by default" + Retry { Update-AzStorageAccountNetworkRuleSet -ResourceGroupName $ResourceGroupName -Name $account.Name -DefaultAction Deny } + if ($CI -and $env:PoolSubnet) { + Write-Host "Enabling access to '$($account.Name)' from pipeline subnet $($env:PoolSubnet)" + Retry { Add-AzStorageAccountNetworkRule -ResourceGroupName $ResourceGroupName -Name $account.Name -VirtualNetworkResourceId $env:PoolSubnet } + } + elseif ($AllowIpRanges) { + Write-Host "Enabling access to '$($account.Name)' to $($AllowIpRanges.Length) IP ranges" + $ipRanges = $AllowIpRanges | ForEach-Object { + @{ Action = 'allow'; IPAddressOrRange = $_ } + } + Retry { Update-AzStorageAccountNetworkRuleSet -ResourceGroupName $ResourceGroupName -Name $account.Name -IPRule $ipRanges | Out-Null } + } + elseif (!$CI) { + Write-Host "Enabling access to '$($account.Name)' from client IP" + $clientIp ??= Retry { Invoke-RestMethod -Uri 'https://icanhazip.com/' } # cloudflare owned ip site + $clientIp = $clientIp.Trim() + $ipRanges = Get-AzStorageAccountNetworkRuleSet -ResourceGroupName $ResourceGroupName -Name $account.Name + foreach ($range in $ipRanges.IpRules) { + if (DoesSubnetOverlap $range.IPAddressOrRange $clientIp) { + return + } + } + Retry { Add-AzStorageAccountNetworkRule -ResourceGroupName $ResourceGroupName -Name $account.Name -IPAddressOrRange $clientIp | Out-Null } + } + } + } + } +} + +function DoesSubnetOverlap([string]$ipOrCidr, [string]$overlapIp) { + [System.Net.IPAddress]$overlapIpAddress = $overlapIp + [System.Net.IPAddress]$baseIp = ($ipOrCidr -split '/')[0] + $subnet = ($ipOrCidr -split '/')[1] + if (!$subnet) { + return $baseIp -eq $overlapIpAddress + } + $subnetNum = [int]$subnet + + $baseMask = [math]::pow(2, 31) + $mask = 0 + for ($i = 0; $i -lt $subnetNum; $i++) { + $mask = $mask + $baseMask; + $baseMask = $baseMask / 2 + } + + return $baseIp.Address -eq ($overlapIpAddress.Address -band ([System.Net.IPAddress]$mask).Address) +} \ No newline at end of file diff --git a/eng/common/TestResources/remove-test-resources.yml b/eng/common/TestResources/remove-test-resources.yml index b877d72139a2..a892e8c2ddfd 100644 --- a/eng/common/TestResources/remove-test-resources.yml +++ b/eng/common/TestResources/remove-test-resources.yml @@ -29,7 +29,9 @@ steps: displayName: Remove test resources condition: and(eq(variables['CI_HAS_DEPLOYED_RESOURCES'], 'true'), ne(variables['Skip.RemoveTestResources'], 'true')) continueOnError: true - env: ${{ parameters.EnvVars }} + env: + PoolSubnet: $(PoolSubnet) + ${{ insert }}: ${{ parameters.EnvVars }} inputs: azureSubscription: ${{ parameters.ServiceConnection }} azurePowerShellVersion: LatestVersion @@ -46,6 +48,7 @@ steps: @subscriptionConfiguration ` -ResourceType '${{ parameters.ResourceType }}' ` -ServiceDirectory "${{ parameters.ServiceDirectory }}" ` + -AllowIpRanges ('$(azsdk-corp-net-ip-ranges)' -split ',') ` -CI ` -Force ` -Verbose @@ -63,10 +66,13 @@ steps: -ResourceType '${{ parameters.ResourceType }}' ` -ServiceDirectory "${{ parameters.ServiceDirectory }}" ` -ServicePrincipalAuth ` + -AllowIpRanges ('$(azsdk-corp-net-ip-ranges)' -split ',') ` -CI ` -Force ` -Verbose displayName: Remove test resources condition: and(eq(variables['CI_HAS_DEPLOYED_RESOURCES'], 'true'), ne(variables['Skip.RemoveTestResources'], 'true')) continueOnError: true - env: ${{ parameters.EnvVars }} + env: + PoolSubnet: $(PoolSubnet) + ${{ insert }}: ${{ parameters.EnvVars }} \ No newline at end of file diff --git a/eng/common/scripts/Helpers/Resource-Helpers.ps1 b/eng/common/scripts/Helpers/Resource-Helpers.ps1 index 6c02e9150e24..0e6cd36678b0 100644 --- a/eng/common/scripts/Helpers/Resource-Helpers.ps1 +++ b/eng/common/scripts/Helpers/Resource-Helpers.ps1 @@ -204,3 +204,82 @@ function Wait-PurgeableResourceJob { } } } + +# Helper function for removing storage accounts with WORM that sometimes get leaked from live tests not set up to clean +# up their resource policies +function Remove-WormStorageAccounts() { + [CmdletBinding(SupportsShouldProcess=$True)] + param( + [string]$GroupPrefix + ) + + $ErrorActionPreference = 'Stop' + + # Be a little defensive so we don't delete non-live test groups via naming convention + # DO NOT REMOVE THIS + # We call this script from live test pipelines as well, and a string mismatch/error could blow away + # some static storage accounts we rely on + if (!$groupPrefix -or !$GroupPrefix.StartsWith('rg-')) { + throw "The -GroupPrefix parameter must start with 'rg-'" + } + + $groups = Get-AzResourceGroup | ? { $_.ResourceGroupName.StartsWith($GroupPrefix) } | ? { $_.ProvisioningState -ne 'Deleting' } + + foreach ($group in $groups) { + Write-Host "=========================================" + $accounts = Get-AzStorageAccount -ResourceGroupName $group.ResourceGroupName + if ($accounts) { + foreach ($account in $accounts) { + if ($WhatIfPreference) { + Write-Host "What if: Removing $($account.StorageAccountName) in $($account.ResourceGroupName)" + } else { + Write-Host "Removing $($account.StorageAccountName) in $($account.ResourceGroupName)" + } + + $hasContainers = ($account.Kind -ne "FileStorage") + + # If it doesn't have containers then we can skip the explicit clean-up of this storage account + if (!$hasContainers) { continue } + + $ctx = New-AzStorageContext -StorageAccountName $account.StorageAccountName + + $immutableBlobs = $ctx ` + | Get-AzStorageContainer ` + | Where-Object { $_.BlobContainerProperties.HasImmutableStorageWithVersioning } ` + | Get-AzStorageBlob + try { + foreach ($blob in $immutableBlobs) { + Write-Host "Removing legal hold - blob: $($blob.Name), account: $($account.StorageAccountName), group: $($group.ResourceGroupName)" + $blob | Set-AzStorageBlobLegalHold -DisableLegalHold | Out-Null + } + } catch { + Write-Warning "User must have 'Storage Blob Data Owner' RBAC permission on subscription or resource group" + Write-Error $_ + throw + } + # Sometimes we get a 404 blob not found but can still delete containers, + # and sometimes we must delete the blob if there's a legal hold. + # Try to remove the blob, but keep running regardless. + try { + Write-Host "Removing immutability policies and blobs - account: $($ctx.StorageAccountName), group: $($group.ResourceGroupName)" + $null = $ctx | Get-AzStorageContainer | Get-AzStorageBlob | Remove-AzStorageBlobImmutabilityPolicy + $ctx | Get-AzStorageContainer | Get-AzStorageBlob | Remove-AzStorageBlob -Force + } catch {} + + try { + # Use AzRm cmdlet as deletion will only work through ARM with the immutability policies defined on the blobs + $ctx | Get-AzStorageContainer | % { Remove-AzRmStorageContainer -Name $_.Name -StorageAccountName $ctx.StorageAccountName -ResourceGroupName $group.ResourceGroupName -Force } + } catch { + Write-Warning "Container removal failed. Ignoring the error and trying to delete the storage account." + Write-Warning $_ + } + Remove-AzStorageAccount -StorageAccountName $account.StorageAccountName -ResourceGroupName $account.ResourceGroupName -Force + } + } + if ($WhatIfPreference) { + Write-Host "What if: Removing resource group $($group.ResourceGroupName)" + } else { + Remove-AzResourceGroup -ResourceGroupName $group.ResourceGroupName -Force -AsJob + } + } +} From 9afd9fb7a59c6713762f6ae4aa3769b53d92039b Mon Sep 17 00:00:00 2001 From: Ben Broderick Phillips Date: Wed, 3 Jul 2024 18:41:38 -0400 Subject: [PATCH 2/5] Move storage network access script to common resource helpers file --- .../TestResources/TestResources-Helpers.ps1 | 60 ----------------- .../scripts/Helpers/Resource-Helpers.ps1 | 67 ++++++++++++++++++- 2 files changed, 66 insertions(+), 61 deletions(-) diff --git a/eng/common/TestResources/TestResources-Helpers.ps1 b/eng/common/TestResources/TestResources-Helpers.ps1 index c148e5c2dd1d..6dee017aec9a 100644 --- a/eng/common/TestResources/TestResources-Helpers.ps1 +++ b/eng/common/TestResources/TestResources-Helpers.ps1 @@ -265,63 +265,3 @@ function SetDeploymentOutputs( return $deploymentEnvironmentVariables, $deploymentOutputs } - -function SetResourceNetworkAccessRules([string]$ResourceGroupName, [array]$AllowIpRanges, [switch]$CI) { - SetStorageNetworkAccessRules -ResourceGroupName $ResourceGroupName -AllowIpRanges $AllowIpRanges -CI:$CI -} - -function SetStorageNetworkAccessRules([string]$ResourceGroupName, [array]$AllowIpRanges, [switch]$CI, [switch]$Override) { - $storageAccounts = Retry { Get-AzResource -ResourceGroupName $ResourceGroupName -ResourceType "Microsoft.Storage/storageAccounts" } - # Add client IP to storage account when running as local user. Pipeline's have their own vnet with access - if ($storageAccounts) { - foreach ($account in $storageAccounts) { - $rules = Get-AzStorageAccountNetworkRuleSet -ResourceGroupName $ResourceGroupName -AccountName $account.Name - if ($rules -and ($Override -or $rules.DefaultAction -eq "Allow")) { - Write-Host "Restricting network rules in storage account '$($account.Name)' to deny access by default" - Retry { Update-AzStorageAccountNetworkRuleSet -ResourceGroupName $ResourceGroupName -Name $account.Name -DefaultAction Deny } - if ($CI -and $env:PoolSubnet) { - Write-Host "Enabling access to '$($account.Name)' from pipeline subnet $($env:PoolSubnet)" - Retry { Add-AzStorageAccountNetworkRule -ResourceGroupName $ResourceGroupName -Name $account.Name -VirtualNetworkResourceId $env:PoolSubnet } - } - elseif ($AllowIpRanges) { - Write-Host "Enabling access to '$($account.Name)' to $($AllowIpRanges.Length) IP ranges" - $ipRanges = $AllowIpRanges | ForEach-Object { - @{ Action = 'allow'; IPAddressOrRange = $_ } - } - Retry { Update-AzStorageAccountNetworkRuleSet -ResourceGroupName $ResourceGroupName -Name $account.Name -IPRule $ipRanges | Out-Null } - } - elseif (!$CI) { - Write-Host "Enabling access to '$($account.Name)' from client IP" - $clientIp ??= Retry { Invoke-RestMethod -Uri 'https://icanhazip.com/' } # cloudflare owned ip site - $clientIp = $clientIp.Trim() - $ipRanges = Get-AzStorageAccountNetworkRuleSet -ResourceGroupName $ResourceGroupName -Name $account.Name - foreach ($range in $ipRanges.IpRules) { - if (DoesSubnetOverlap $range.IPAddressOrRange $clientIp) { - return - } - } - Retry { Add-AzStorageAccountNetworkRule -ResourceGroupName $ResourceGroupName -Name $account.Name -IPAddressOrRange $clientIp | Out-Null } - } - } - } - } -} - -function DoesSubnetOverlap([string]$ipOrCidr, [string]$overlapIp) { - [System.Net.IPAddress]$overlapIpAddress = $overlapIp - [System.Net.IPAddress]$baseIp = ($ipOrCidr -split '/')[0] - $subnet = ($ipOrCidr -split '/')[1] - if (!$subnet) { - return $baseIp -eq $overlapIpAddress - } - $subnetNum = [int]$subnet - - $baseMask = [math]::pow(2, 31) - $mask = 0 - for ($i = 0; $i -lt $subnetNum; $i++) { - $mask = $mask + $baseMask; - $baseMask = $baseMask / 2 - } - - return $baseIp.Address -eq ($overlapIpAddress.Address -band ([System.Net.IPAddress]$mask).Address) -} \ No newline at end of file diff --git a/eng/common/scripts/Helpers/Resource-Helpers.ps1 b/eng/common/scripts/Helpers/Resource-Helpers.ps1 index 0e6cd36678b0..cf8128a5dcac 100644 --- a/eng/common/scripts/Helpers/Resource-Helpers.ps1 +++ b/eng/common/scripts/Helpers/Resource-Helpers.ps1 @@ -240,7 +240,7 @@ function Remove-WormStorageAccounts() { # If it doesn't have containers then we can skip the explicit clean-up of this storage account if (!$hasContainers) { continue } - + $ctx = New-AzStorageContext -StorageAccountName $account.StorageAccountName $immutableBlobs = $ctx ` @@ -283,3 +283,68 @@ function Remove-WormStorageAccounts() { } } } + +function SetResourceNetworkAccessRules([string]$ResourceGroupName, [array]$AllowIpRanges, [switch]$CI) { + SetStorageNetworkAccessRules -ResourceGroupName $ResourceGroupName -AllowIpRanges $AllowIpRanges -CI:$CI +} + +function SetStorageNetworkAccessRules([string]$ResourceGroupName, [array]$AllowIpRanges, [switch]$CI, [switch]$Override) { + $clientIp = $null + $storageAccounts = Retry { Get-AzResource -ResourceGroupName $ResourceGroupName -ResourceType "Microsoft.Storage/storageAccounts" } + # Add client IP to storage account when running as local user. Pipeline's have their own vnet with access + if ($storageAccounts) { + foreach ($account in $storageAccounts) { + $rules = Get-AzStorageAccountNetworkRuleSet -ResourceGroupName $ResourceGroupName -AccountName $account.Name + if ($rules -and ($Override -or $rules.DefaultAction -eq "Allow")) { + Write-Host "Restricting network rules in storage account '$($account.Name)' to deny access by default" + Retry { Update-AzStorageAccountNetworkRuleSet -ResourceGroupName $ResourceGroupName -Name $account.Name -DefaultAction Deny } + if ($CI -and $env:PoolSubnet) { + Write-Host "Enabling access to '$($account.Name)' from pipeline subnet $($env:PoolSubnet)" + Retry { Add-AzStorageAccountNetworkRule -ResourceGroupName $ResourceGroupName -Name $account.Name -VirtualNetworkResourceId $env:PoolSubnet } + } + elseif ($AllowIpRanges) { + Write-Host "Enabling access to '$($account.Name)' to $($AllowIpRanges.Length) IP ranges" + $ipRanges = $AllowIpRanges | ForEach-Object { + @{ Action = 'allow'; IPAddressOrRange = $_ } + } + Retry { Update-AzStorageAccountNetworkRuleSet -ResourceGroupName $ResourceGroupName -Name $account.Name -IPRule $ipRanges | Out-Null } + } + elseif (!$CI) { + Write-Host "Enabling access to '$($account.Name)' from client IP" + $clientIp ??= Retry { Invoke-RestMethod -Uri 'https://icanhazip.com/' } # cloudflare owned ip site + $clientIp = $clientIp.Trim() + $ipRanges = Get-AzStorageAccountNetworkRuleSet -ResourceGroupName $ResourceGroupName -Name $account.Name + if ($ipRanges) { + foreach ($range in $ipRanges.IpRules) { + if (DoesSubnetOverlap $range.IPAddressOrRange $clientIp) { + return + } + } + } + Retry { Add-AzStorageAccountNetworkRule -ResourceGroupName $ResourceGroupName -Name $account.Name -IPAddressOrRange $clientIp | Out-Null } + } + } + } + } +} + +function DoesSubnetOverlap([string]$ipOrCidr, [string]$overlapIp) { + [System.Net.IPAddress]$overlapIpAddress = $overlapIp + $parsed = $ipOrCidr -split '/' + [System.Net.IPAddress]$baseIp = $parsed[0] + if ($parsed.Length -eq 1) { + return $baseIp -eq $overlapIpAddress + } + + $subnet = $parsed[1] + $subnetNum = [int]$subnet + + $baseMask = [math]::pow(2, 31) + $mask = 0 + for ($i = 0; $i -lt $subnetNum; $i++) { + $mask = $mask + $baseMask; + $baseMask = $baseMask / 2 + } + + return $baseIp.Address -eq ($overlapIpAddress.Address -band ([System.Net.IPAddress]$mask).Address) +} From de824ae19cab1daf287d2eeffc71070d58e3762c Mon Sep 17 00:00:00 2001 From: Ben Broderick Phillips Date: Wed, 3 Jul 2024 19:04:16 -0400 Subject: [PATCH 3/5] Improve storage container deletion resilience --- .../scripts/Helpers/Resource-Helpers.ps1 | 317 ++++++++++-------- 1 file changed, 169 insertions(+), 148 deletions(-) diff --git a/eng/common/scripts/Helpers/Resource-Helpers.ps1 b/eng/common/scripts/Helpers/Resource-Helpers.ps1 index cf8128a5dcac..a60a17df5af2 100644 --- a/eng/common/scripts/Helpers/Resource-Helpers.ps1 +++ b/eng/common/scripts/Helpers/Resource-Helpers.ps1 @@ -4,7 +4,7 @@ function Get-PurgeableGroupResources { param ( - [Parameter(Mandatory=$true, Position=0)] + [Parameter(Mandatory = $true, Position = 0)] [string] $ResourceGroupName ) @@ -27,8 +27,8 @@ function Get-PurgeableGroupResources { # Get any Key Vaults that will be deleted so they can be purged later if soft delete is enabled. $deletedKeyVaults = @(Get-AzKeyVault -ResourceGroupName $ResourceGroupName -ErrorAction Ignore | ForEach-Object { - # Enumerating vaults from a resource group does not return all properties we required. - Get-AzKeyVault -VaultName $_.VaultName -ErrorAction Ignore | Where-Object { $_.EnableSoftDelete } ` + # Enumerating vaults from a resource group does not return all properties we required. + Get-AzKeyVault -VaultName $_.VaultName -ErrorAction Ignore | Where-Object { $_.EnableSoftDelete } ` | Add-Member -MemberType NoteProperty -Name AzsdkResourceType -Value 'Key Vault' -PassThru ` | Add-Member -MemberType AliasProperty -Name AzsdkName -Value VaultName -PassThru }) @@ -56,13 +56,13 @@ function Get-PurgeableResources { $deletedHsms = @() foreach ($r in $content.value) { $deletedHsms += [pscustomobject] @{ - AzsdkResourceType = 'Managed HSM' - AzsdkName = $r.name - Id = $r.id - Name = $r.name - Location = $r.properties.location - DeletionDate = $r.properties.deletionDate -as [DateTime] - ScheduledPurgeDate = $r.properties.scheduledPurgeDate -as [DateTime] + AzsdkResourceType = 'Managed HSM' + AzsdkName = $r.name + Id = $r.id + Name = $r.name + Location = $r.properties.location + DeletionDate = $r.properties.deletionDate -as [DateTime] + ScheduledPurgeDate = $r.properties.scheduledPurgeDate -as [DateTime] EnablePurgeProtection = $r.properties.purgeProtectionEnabled } } @@ -91,7 +91,8 @@ function Get-PurgeableResources { Write-Verbose "Found $($deletedKeyVaults.Count) deleted Key Vaults to potentially purge." $purgeableResources += $deletedKeyVaults } - } catch { } + } + catch { } return $purgeableResources } @@ -100,7 +101,7 @@ function Get-PurgeableResources { # This allows you to pipe a collection and process each item in the collection. filter Remove-PurgeableResources { param ( - [Parameter(Position=0, ValueFromPipeline=$true)] + [Parameter(Position = 0, ValueFromPipeline = $true)] [object[]] $Resource, [Parameter()] @@ -128,7 +129,7 @@ filter Remove-PurgeableResources { # Use `-AsJob` to start a lightweight, cancellable job and pass to `Wait-PurgeableResoruceJob` for consistent behavior. Remove-AzKeyVault -VaultName $r.VaultName -Location $r.Location -InRemovedState -Force -ErrorAction Continue -AsJob ` - | Wait-PurgeableResourceJob -Resource $r -Timeout $Timeout -PassThru:$PassThru + | Wait-PurgeableResourceJob -Resource $r -Timeout $Timeout -PassThru:$PassThru } 'Managed HSM' { @@ -139,18 +140,19 @@ filter Remove-PurgeableResources { # Use `GetNewClosure()` on the `-Action` ScriptBlock to make sure variables are captured. Invoke-AzRestMethod -Method POST -Path "/subscriptions/$subscriptionId/providers/Microsoft.KeyVault/locations/$($r.Location)/deletedManagedHSMs/$($r.Name)/purge?api-version=2023-02-01" -ErrorAction Ignore -AsJob ` - | Wait-PurgeableResourceJob -Resource $r -Timeout $Timeout -PassThru:$PassThru -Action { - param ( $response ) - if ($response.StatusCode -ge 200 -and $response.StatusCode -lt 300) { - Write-Warning "Successfully requested that Managed HSM '$($r.Name)' be purged, but may take a few minutes before it is actually purged." - } elseif ($response.Content) { - $content = $response.Content | ConvertFrom-Json - if ($content.error) { - $err = $content.error - Write-Warning "Failed to deleted Managed HSM '$($r.Name)': ($($err.code)) $($err.message)" - } - } - }.GetNewClosure() + | Wait-PurgeableResourceJob -Resource $r -Timeout $Timeout -PassThru:$PassThru -Action { + param ( $response ) + if ($response.StatusCode -ge 200 -and $response.StatusCode -lt 300) { + Write-Warning "Successfully requested that Managed HSM '$($r.Name)' be purged, but may take a few minutes before it is actually purged." + } + elseif ($response.Content) { + $content = $response.Content | ConvertFrom-Json + if ($content.error) { + $err = $content.error + Write-Warning "Failed to deleted Managed HSM '$($r.Name)': ($($err.code)) $($err.message)" + } + } + }.GetNewClosure() } default { @@ -167,12 +169,12 @@ function Log($Message) { function Wait-PurgeableResourceJob { param ( - [Parameter(Mandatory=$true, ValueFromPipeline=$true)] + [Parameter(Mandatory = $true, ValueFromPipeline = $true)] $Job, # The resource is used for logging and to return if `-PassThru` is specified # so we can easily see all resources that may be in a bad state when the script has completed. - [Parameter(Mandatory=$true)] + [Parameter(Mandatory = $true)] $Resource, # Optional ScriptBlock should define params corresponding to the associated job's `Output` property. @@ -195,7 +197,8 @@ function Wait-PurgeableResourceJob { if ($Action) { $null = $Action.Invoke($result) } - } else { + } + else { Write-Warning "Timed out waiting to purge $($Resource.AzsdkResourceType) '$($Resource.AzsdkName)'. Cancelling job." $Job.Cancel() @@ -208,143 +211,161 @@ function Wait-PurgeableResourceJob { # Helper function for removing storage accounts with WORM that sometimes get leaked from live tests not set up to clean # up their resource policies function Remove-WormStorageAccounts() { - [CmdletBinding(SupportsShouldProcess=$True)] - param( - [string]$GroupPrefix - ) - - $ErrorActionPreference = 'Stop' - - # Be a little defensive so we don't delete non-live test groups via naming convention - # DO NOT REMOVE THIS - # We call this script from live test pipelines as well, and a string mismatch/error could blow away - # some static storage accounts we rely on - if (!$groupPrefix -or !$GroupPrefix.StartsWith('rg-')) { - throw "The -GroupPrefix parameter must start with 'rg-'" - } + [CmdletBinding(SupportsShouldProcess = $True)] + param( + [string]$GroupPrefix + ) - $groups = Get-AzResourceGroup | ? { $_.ResourceGroupName.StartsWith($GroupPrefix) } | ? { $_.ProvisioningState -ne 'Deleting' } - - foreach ($group in $groups) { - Write-Host "=========================================" - $accounts = Get-AzStorageAccount -ResourceGroupName $group.ResourceGroupName - if ($accounts) { - foreach ($account in $accounts) { - if ($WhatIfPreference) { - Write-Host "What if: Removing $($account.StorageAccountName) in $($account.ResourceGroupName)" - } else { - Write-Host "Removing $($account.StorageAccountName) in $($account.ResourceGroupName)" - } - - $hasContainers = ($account.Kind -ne "FileStorage") - - # If it doesn't have containers then we can skip the explicit clean-up of this storage account - if (!$hasContainers) { continue } - - $ctx = New-AzStorageContext -StorageAccountName $account.StorageAccountName - - $immutableBlobs = $ctx ` - | Get-AzStorageContainer ` - | Where-Object { $_.BlobContainerProperties.HasImmutableStorageWithVersioning } ` - | Get-AzStorageBlob - try { - foreach ($blob in $immutableBlobs) { - Write-Host "Removing legal hold - blob: $($blob.Name), account: $($account.StorageAccountName), group: $($group.ResourceGroupName)" - $blob | Set-AzStorageBlobLegalHold -DisableLegalHold | Out-Null - } - } catch { - Write-Warning "User must have 'Storage Blob Data Owner' RBAC permission on subscription or resource group" - Write-Error $_ - throw - } - # Sometimes we get a 404 blob not found but can still delete containers, - # and sometimes we must delete the blob if there's a legal hold. - # Try to remove the blob, but keep running regardless. - try { - Write-Host "Removing immutability policies and blobs - account: $($ctx.StorageAccountName), group: $($group.ResourceGroupName)" - $null = $ctx | Get-AzStorageContainer | Get-AzStorageBlob | Remove-AzStorageBlobImmutabilityPolicy - $ctx | Get-AzStorageContainer | Get-AzStorageBlob | Remove-AzStorageBlob -Force - } catch {} - - try { - # Use AzRm cmdlet as deletion will only work through ARM with the immutability policies defined on the blobs - $ctx | Get-AzStorageContainer | % { Remove-AzRmStorageContainer -Name $_.Name -StorageAccountName $ctx.StorageAccountName -ResourceGroupName $group.ResourceGroupName -Force } - } catch { - Write-Warning "Container removal failed. Ignoring the error and trying to delete the storage account." - Write-Warning $_ - } - Remove-AzStorageAccount -StorageAccountName $account.StorageAccountName -ResourceGroupName $account.ResourceGroupName -Force - } - } + $ErrorActionPreference = 'Stop' + + # Be a little defensive so we don't delete non-live test groups via naming convention + # DO NOT REMOVE THIS + # We call this script from live test pipelines as well, and a string mismatch/error could blow away + # some static storage accounts we rely on + if (!$groupPrefix -or !$GroupPrefix.StartsWith('rg-')) { + throw "The -GroupPrefix parameter must start with 'rg-'" + } + + $groups = Get-AzResourceGroup | Where-Object { $_.ResourceGroupName.StartsWith($GroupPrefix) } | Where-Object { $_.ProvisioningState -ne 'Deleting' } + + foreach ($group in $groups) { + Write-Host "=========================================" + $accounts = Get-AzStorageAccount -ResourceGroupName $group.ResourceGroupName + if ($accounts) { + foreach ($account in $accounts) { if ($WhatIfPreference) { - Write-Host "What if: Removing resource group $($group.ResourceGroupName)" - } else { - Remove-AzResourceGroup -ResourceGroupName $group.ResourceGroupName -Force -AsJob + Write-Host "What if: Removing $($account.StorageAccountName) in $($account.ResourceGroupName)" + } + else { + Write-Host "Removing $($account.StorageAccountName) in $($account.ResourceGroupName)" + } + + $hasContainers = ($account.Kind -ne "FileStorage") + + # If it doesn't have containers then we can skip the explicit clean-up of this storage account + if (!$hasContainers) { continue } + + $ctx = New-AzStorageContext -StorageAccountName $account.StorageAccountName + + $immutableBlobs = $ctx ` + | Get-AzStorageContainer ` + | Where-Object { $_.BlobContainerProperties.HasImmutableStorageWithVersioning } ` + | Get-AzStorageBlob + try { + foreach ($blob in $immutableBlobs) { + Write-Host "Removing legal hold - blob: $($blob.Name), account: $($account.StorageAccountName), group: $($group.ResourceGroupName)" + $blob | Set-AzStorageBlobLegalHold -DisableLegalHold | Out-Null + } } + catch { + Write-Warning "User must have 'Storage Blob Data Owner' RBAC permission on subscription or resource group" + Write-Error $_ + throw + } + # Sometimes we get a 404 blob not found but can still delete containers, + # and sometimes we must delete the blob if there's a legal hold. + # Try to remove the blob, but keep running regardless. + $succeeded = $false + for ($attempt = 0; $attempt -lt 2; $attempt++) { + if ($succeeded) { + break + } + + try { + Write-Host "Removing immutability policies - account: $($ctx.StorageAccountName), group: $($group.ResourceGroupName)" + $null = $ctx | Get-AzStorageContainer | Get-AzStorageBlob | Remove-AzStorageBlobImmutabilityPolicy + } catch {} + + try { + $ctx | Get-AzStorageContainer | Get-AzStorageBlob | Remove-AzStorageBlob -Force + $succeeded = $true + } catch { + Write-Warning "Failed to remove blobs - account: $($ctx.StorageAccountName), group: $($group.ResourceGroupName)" + Write-Warning $_ + } + } + + try { + # Use AzRm cmdlet as deletion will only work through ARM with the immutability policies defined on the blobs + $ctx | Get-AzStorageContainer | ForEach-Object { Remove-AzRmStorageContainer -Name $_.Name -StorageAccountName $ctx.StorageAccountName -ResourceGroupName $group.ResourceGroupName -Force } + } + catch { + Write-Warning "Container removal failed. Ignoring the error and trying to delete the storage account." + Write-Warning $_ + } + Remove-AzStorageAccount -StorageAccountName $account.StorageAccountName -ResourceGroupName $account.ResourceGroupName -Force + } + } + if ($WhatIfPreference) { + Write-Host "What if: Removing resource group $($group.ResourceGroupName)" } + else { + Remove-AzResourceGroup -ResourceGroupName $group.ResourceGroupName -Force -AsJob + } + } } function SetResourceNetworkAccessRules([string]$ResourceGroupName, [array]$AllowIpRanges, [switch]$CI) { - SetStorageNetworkAccessRules -ResourceGroupName $ResourceGroupName -AllowIpRanges $AllowIpRanges -CI:$CI + SetStorageNetworkAccessRules -ResourceGroupName $ResourceGroupName -AllowIpRanges $AllowIpRanges -CI:$CI } function SetStorageNetworkAccessRules([string]$ResourceGroupName, [array]$AllowIpRanges, [switch]$CI, [switch]$Override) { - $clientIp = $null - $storageAccounts = Retry { Get-AzResource -ResourceGroupName $ResourceGroupName -ResourceType "Microsoft.Storage/storageAccounts" } - # Add client IP to storage account when running as local user. Pipeline's have their own vnet with access - if ($storageAccounts) { - foreach ($account in $storageAccounts) { - $rules = Get-AzStorageAccountNetworkRuleSet -ResourceGroupName $ResourceGroupName -AccountName $account.Name - if ($rules -and ($Override -or $rules.DefaultAction -eq "Allow")) { - Write-Host "Restricting network rules in storage account '$($account.Name)' to deny access by default" - Retry { Update-AzStorageAccountNetworkRuleSet -ResourceGroupName $ResourceGroupName -Name $account.Name -DefaultAction Deny } - if ($CI -and $env:PoolSubnet) { - Write-Host "Enabling access to '$($account.Name)' from pipeline subnet $($env:PoolSubnet)" - Retry { Add-AzStorageAccountNetworkRule -ResourceGroupName $ResourceGroupName -Name $account.Name -VirtualNetworkResourceId $env:PoolSubnet } - } - elseif ($AllowIpRanges) { - Write-Host "Enabling access to '$($account.Name)' to $($AllowIpRanges.Length) IP ranges" - $ipRanges = $AllowIpRanges | ForEach-Object { - @{ Action = 'allow'; IPAddressOrRange = $_ } - } - Retry { Update-AzStorageAccountNetworkRuleSet -ResourceGroupName $ResourceGroupName -Name $account.Name -IPRule $ipRanges | Out-Null } - } - elseif (!$CI) { - Write-Host "Enabling access to '$($account.Name)' from client IP" - $clientIp ??= Retry { Invoke-RestMethod -Uri 'https://icanhazip.com/' } # cloudflare owned ip site - $clientIp = $clientIp.Trim() - $ipRanges = Get-AzStorageAccountNetworkRuleSet -ResourceGroupName $ResourceGroupName -Name $account.Name - if ($ipRanges) { - foreach ($range in $ipRanges.IpRules) { - if (DoesSubnetOverlap $range.IPAddressOrRange $clientIp) { - return - } - } - } - Retry { Add-AzStorageAccountNetworkRule -ResourceGroupName $ResourceGroupName -Name $account.Name -IPAddressOrRange $clientIp | Out-Null } - } + $clientIp = $null + $storageAccounts = Retry { Get-AzResource -ResourceGroupName $ResourceGroupName -ResourceType "Microsoft.Storage/storageAccounts" } + # Add client IP to storage account when running as local user. Pipeline's have their own vnet with access + if ($storageAccounts) { + foreach ($account in $storageAccounts) { + $rules = Get-AzStorageAccountNetworkRuleSet -ResourceGroupName $ResourceGroupName -AccountName $account.Name + if ($rules -and ($Override -or $rules.DefaultAction -eq "Allow")) { + Write-Host "Restricting network rules in storage account '$($account.Name)' to deny access by default" + Retry { Update-AzStorageAccountNetworkRuleSet -ResourceGroupName $ResourceGroupName -Name $account.Name -DefaultAction Deny } + if ($CI -and $env:PoolSubnet) { + Write-Host "Enabling access to '$($account.Name)' from pipeline subnet $($env:PoolSubnet)" + Retry { Add-AzStorageAccountNetworkRule -ResourceGroupName $ResourceGroupName -Name $account.Name -VirtualNetworkResourceId $env:PoolSubnet } + } + elseif ($AllowIpRanges) { + Write-Host "Enabling access to '$($account.Name)' to $($AllowIpRanges.Length) IP ranges" + $ipRanges = $AllowIpRanges | ForEach-Object { + @{ Action = 'allow'; IPAddressOrRange = $_ } + } + Retry { Update-AzStorageAccountNetworkRuleSet -ResourceGroupName $ResourceGroupName -Name $account.Name -IPRule $ipRanges | Out-Null } + } + elseif (!$CI) { + Write-Host "Enabling access to '$($account.Name)' from client IP" + $clientIp ??= Retry { Invoke-RestMethod -Uri 'https://icanhazip.com/' } # cloudflare owned ip site + $clientIp = $clientIp.Trim() + $ipRanges = Get-AzStorageAccountNetworkRuleSet -ResourceGroupName $ResourceGroupName -Name $account.Name + if ($ipRanges) { + foreach ($range in $ipRanges.IpRules) { + if (DoesSubnetOverlap $range.IPAddressOrRange $clientIp) { + return + } } + } + Retry { Add-AzStorageAccountNetworkRule -ResourceGroupName $ResourceGroupName -Name $account.Name -IPAddressOrRange $clientIp | Out-Null } } + } } + } } function DoesSubnetOverlap([string]$ipOrCidr, [string]$overlapIp) { - [System.Net.IPAddress]$overlapIpAddress = $overlapIp - $parsed = $ipOrCidr -split '/' - [System.Net.IPAddress]$baseIp = $parsed[0] - if ($parsed.Length -eq 1) { - return $baseIp -eq $overlapIpAddress - } + [System.Net.IPAddress]$overlapIpAddress = $overlapIp + $parsed = $ipOrCidr -split '/' + [System.Net.IPAddress]$baseIp = $parsed[0] + if ($parsed.Length -eq 1) { + return $baseIp -eq $overlapIpAddress + } - $subnet = $parsed[1] - $subnetNum = [int]$subnet + $subnet = $parsed[1] + $subnetNum = [int]$subnet - $baseMask = [math]::pow(2, 31) - $mask = 0 - for ($i = 0; $i -lt $subnetNum; $i++) { - $mask = $mask + $baseMask; - $baseMask = $baseMask / 2 - } + $baseMask = [math]::pow(2, 31) + $mask = 0 + for ($i = 0; $i -lt $subnetNum; $i++) { + $mask = $mask + $baseMask; + $baseMask = $baseMask / 2 + } - return $baseIp.Address -eq ($overlapIpAddress.Address -band ([System.Net.IPAddress]$mask).Address) + return $baseIp.Address -eq ($overlapIpAddress.Address -band ([System.Net.IPAddress]$mask).Address) } From ed8912db020ac5910b351be3842e089c2d635e52 Mon Sep 17 00:00:00 2001 From: Ben Broderick Phillips Date: Wed, 3 Jul 2024 19:26:06 -0400 Subject: [PATCH 4/5] Plumb through pool variable to live test cleanup template --- eng/common/TestResources/remove-test-resources.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/common/TestResources/remove-test-resources.yml b/eng/common/TestResources/remove-test-resources.yml index a892e8c2ddfd..025e90dd4c29 100644 --- a/eng/common/TestResources/remove-test-resources.yml +++ b/eng/common/TestResources/remove-test-resources.yml @@ -75,4 +75,4 @@ steps: continueOnError: true env: PoolSubnet: $(PoolSubnet) - ${{ insert }}: ${{ parameters.EnvVars }} \ No newline at end of file + ${{ insert }}: ${{ parameters.EnvVars }} From 308a10c9eabb360d1017a2a2a9cfa5a149d790c4 Mon Sep 17 00:00:00 2001 From: Ben Broderick Phillips Date: Wed, 3 Jul 2024 19:37:07 -0400 Subject: [PATCH 5/5] Add sleep for network rule application --- .../TestResources/New-TestResources.ps1 | 1 + .../TestResources/Remove-TestResources.ps1 | 4 ++-- .../scripts/Helpers/Resource-Helpers.ps1 | 21 ++++++++++++++----- 3 files changed, 19 insertions(+), 7 deletions(-) diff --git a/eng/common/TestResources/New-TestResources.ps1 b/eng/common/TestResources/New-TestResources.ps1 index 1af87d50de1f..6ccf55a781c1 100644 --- a/eng/common/TestResources/New-TestResources.ps1 +++ b/eng/common/TestResources/New-TestResources.ps1 @@ -117,6 +117,7 @@ param ( $NewTestResourcesRemainingArguments ) +. (Join-Path $PSScriptRoot .. scripts Helpers Resource-Helpers.ps1) . $PSScriptRoot/TestResources-Helpers.ps1 . $PSScriptRoot/SubConfig-Helpers.ps1 diff --git a/eng/common/TestResources/Remove-TestResources.ps1 b/eng/common/TestResources/Remove-TestResources.ps1 index b5d10fb11f50..08ca9d8f5a54 100644 --- a/eng/common/TestResources/Remove-TestResources.ps1 +++ b/eng/common/TestResources/Remove-TestResources.ps1 @@ -257,8 +257,8 @@ $verifyDeleteScript = { # Get any resources that can be purged after the resource group is deleted coerced into a collection even if empty. $purgeableResources = Get-PurgeableGroupResources $ResourceGroupName -SetStorageNetworkAccessRules -ResourceGroupName $ResourceGroupName -AllowIpRanges $AllowIpRanges -Override -CI:$CI -Remove-WormStorageAccounts -GroupPrefix $ResourceGroupName +SetResourceNetworkAccessRules -ResourceGroupName $ResourceGroupName -AllowIpRanges $AllowIpRanges -Override -CI:$CI +Remove-WormStorageAccounts -GroupPrefix $ResourceGroupName -CI:$CI Log "Deleting resource group '$ResourceGroupName'" if ($Force -and !$purgeableResources) { diff --git a/eng/common/scripts/Helpers/Resource-Helpers.ps1 b/eng/common/scripts/Helpers/Resource-Helpers.ps1 index a60a17df5af2..938ccfa4b55f 100644 --- a/eng/common/scripts/Helpers/Resource-Helpers.ps1 +++ b/eng/common/scripts/Helpers/Resource-Helpers.ps1 @@ -213,7 +213,8 @@ function Wait-PurgeableResourceJob { function Remove-WormStorageAccounts() { [CmdletBinding(SupportsShouldProcess = $True)] param( - [string]$GroupPrefix + [string]$GroupPrefix, + [switch]$CI ) $ErrorActionPreference = 'Stop' @@ -222,8 +223,8 @@ function Remove-WormStorageAccounts() { # DO NOT REMOVE THIS # We call this script from live test pipelines as well, and a string mismatch/error could blow away # some static storage accounts we rely on - if (!$groupPrefix -or !$GroupPrefix.StartsWith('rg-')) { - throw "The -GroupPrefix parameter must start with 'rg-'" + if (!$groupPrefix -or ($CI -and !$GroupPrefix.StartsWith('rg-'))) { + throw "The -GroupPrefix parameter must not be empty, or must start with 'rg-' in CI contexts" } $groups = Get-AzResourceGroup | Where-Object { $_.ResourceGroupName.StartsWith($GroupPrefix) } | Where-Object { $_.ProvisioningState -ne 'Deleting' } @@ -274,12 +275,14 @@ function Remove-WormStorageAccounts() { try { Write-Host "Removing immutability policies - account: $($ctx.StorageAccountName), group: $($group.ResourceGroupName)" $null = $ctx | Get-AzStorageContainer | Get-AzStorageBlob | Remove-AzStorageBlobImmutabilityPolicy - } catch {} + } + catch {} try { $ctx | Get-AzStorageContainer | Get-AzStorageBlob | Remove-AzStorageBlob -Force $succeeded = $true - } catch { + } + catch { Write-Warning "Failed to remove blobs - account: $($ctx.StorageAccountName), group: $($group.ResourceGroupName)" Write-Warning $_ } @@ -314,6 +317,7 @@ function SetStorageNetworkAccessRules([string]$ResourceGroupName, [array]$AllowI $storageAccounts = Retry { Get-AzResource -ResourceGroupName $ResourceGroupName -ResourceType "Microsoft.Storage/storageAccounts" } # Add client IP to storage account when running as local user. Pipeline's have their own vnet with access if ($storageAccounts) { + $appliedRule = $false foreach ($account in $storageAccounts) { $rules = Get-AzStorageAccountNetworkRuleSet -ResourceGroupName $ResourceGroupName -AccountName $account.Name if ($rules -and ($Override -or $rules.DefaultAction -eq "Allow")) { @@ -322,6 +326,7 @@ function SetStorageNetworkAccessRules([string]$ResourceGroupName, [array]$AllowI if ($CI -and $env:PoolSubnet) { Write-Host "Enabling access to '$($account.Name)' from pipeline subnet $($env:PoolSubnet)" Retry { Add-AzStorageAccountNetworkRule -ResourceGroupName $ResourceGroupName -Name $account.Name -VirtualNetworkResourceId $env:PoolSubnet } + $appliedRule = $true } elseif ($AllowIpRanges) { Write-Host "Enabling access to '$($account.Name)' to $($AllowIpRanges.Length) IP ranges" @@ -329,6 +334,7 @@ function SetStorageNetworkAccessRules([string]$ResourceGroupName, [array]$AllowI @{ Action = 'allow'; IPAddressOrRange = $_ } } Retry { Update-AzStorageAccountNetworkRuleSet -ResourceGroupName $ResourceGroupName -Name $account.Name -IPRule $ipRanges | Out-Null } + $appliedRule = $true } elseif (!$CI) { Write-Host "Enabling access to '$($account.Name)' from client IP" @@ -343,9 +349,14 @@ function SetStorageNetworkAccessRules([string]$ResourceGroupName, [array]$AllowI } } Retry { Add-AzStorageAccountNetworkRule -ResourceGroupName $ResourceGroupName -Name $account.Name -IPAddressOrRange $clientIp | Out-Null } + $appliedRule = $true } } } + if ($appliedRule) { + Write-Host "Sleeping for 15 seconds to allow network rules to take effect" + Start-Sleep 15 + } } }