Skip to content

Sync eng/common directory with azure-sdk-tools for PR 8558 #36350

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
azure-sdk merged 5 commits into from
Jul 8, 2024
Merged

Sync eng/common directory with azure-sdk-tools for PR 8558 #36350

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
4e3c6ac
Support storage network access and worm removal in remove test resour…
benbp Jul 3, 2024
9afd9fb
Move storage network access script to common resource helpers file
benbp Jul 3, 2024
de824ae
Improve storage container deletion resilience
benbp Jul 3, 2024
ed8912d
Plumb through pool variable to live test cleanup template
benbp Jul 3, 2024
308a10c
Add sleep for network rule application
benbp Jul 3, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Move storage network access script to common resource helpers file
  • Loading branch information
@benbp @azure-sdk
benbp authored and azure-sdk committed Jul 4, 2024
commit 9afd9fb7a59c6713762f6ae4aa3769b53d92039b
60 changes: 0 additions & 60 deletions eng/common/TestResources/TestResources-Helpers.ps1
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -265,63 +265,3 @@ function SetDeploymentOutputs(

return $deploymentEnvironmentVariables, $deploymentOutputs
}

function SetResourceNetworkAccessRules([string]$ResourceGroupName, [array]$AllowIpRanges, [switch]$CI) {
SetStorageNetworkAccessRules -ResourceGroupName $ResourceGroupName -AllowIpRanges $AllowIpRanges -CI:$CI
}

function SetStorageNetworkAccessRules([string]$ResourceGroupName, [array]$AllowIpRanges, [switch]$CI, [switch]$Override) {
$storageAccounts = Retry { Get-AzResource -ResourceGroupName $ResourceGroupName -ResourceType "Microsoft.Storage/storageAccounts" }
# Add client IP to storage account when running as local user. Pipeline's have their own vnet with access
if ($storageAccounts) {
foreach ($account in $storageAccounts) {
$rules = Get-AzStorageAccountNetworkRuleSet -ResourceGroupName $ResourceGroupName -AccountName $account.Name
if ($rules -and ($Override -or $rules.DefaultAction -eq "Allow")) {
Write-Host "Restricting network rules in storage account '$($account.Name)' to deny access by default"
Retry { Update-AzStorageAccountNetworkRuleSet -ResourceGroupName $ResourceGroupName -Name $account.Name -DefaultAction Deny }
if ($CI -and $env:PoolSubnet) {
Write-Host "Enabling access to '$($account.Name)' from pipeline subnet $($env:PoolSubnet)"
Retry { Add-AzStorageAccountNetworkRule -ResourceGroupName $ResourceGroupName -Name $account.Name -VirtualNetworkResourceId $env:PoolSubnet }
}
elseif ($AllowIpRanges) {
Write-Host "Enabling access to '$($account.Name)' to $($AllowIpRanges.Length) IP ranges"
$ipRanges = $AllowIpRanges | ForEach-Object {
@{ Action = 'allow'; IPAddressOrRange = $_ }
}
Retry { Update-AzStorageAccountNetworkRuleSet -ResourceGroupName $ResourceGroupName -Name $account.Name -IPRule $ipRanges | Out-Null }
}
elseif (!$CI) {
Write-Host "Enabling access to '$($account.Name)' from client IP"
$clientIp ??= Retry { Invoke-RestMethod -Uri 'https://icanhazip.com/' } # cloudflare owned ip site
$clientIp = $clientIp.Trim()
$ipRanges = Get-AzStorageAccountNetworkRuleSet -ResourceGroupName $ResourceGroupName -Name $account.Name
foreach ($range in $ipRanges.IpRules) {
if (DoesSubnetOverlap $range.IPAddressOrRange $clientIp) {
return
}
}
Retry { Add-AzStorageAccountNetworkRule -ResourceGroupName $ResourceGroupName -Name $account.Name -IPAddressOrRange $clientIp | Out-Null }
}
}
}
}
}

function DoesSubnetOverlap([string]$ipOrCidr, [string]$overlapIp) {
[System.Net.IPAddress]$overlapIpAddress = $overlapIp
[System.Net.IPAddress]$baseIp = ($ipOrCidr -split '/')[0]
$subnet = ($ipOrCidr -split '/')[1]
if (!$subnet) {
return $baseIp -eq $overlapIpAddress
}
$subnetNum = [int]$subnet

$baseMask = [math]::pow(2, 31)
$mask = 0
for ($i = 0; $i -lt $subnetNum; $i++) {
$mask = $mask + $baseMask;
$baseMask = $baseMask / 2
}

return $baseIp.Address -eq ($overlapIpAddress.Address -band ([System.Net.IPAddress]$mask).Address)
}
67 changes: 66 additions & 1 deletion eng/common/scripts/Helpers/Resource-Helpers.ps1
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -240,7 +240,7 @@ function Remove-WormStorageAccounts() {

# If it doesn't have containers then we can skip the explicit clean-up of this storage account
if (!$hasContainers) { continue }

$ctx = New-AzStorageContext -StorageAccountName $account.StorageAccountName

$immutableBlobs = $ctx `
Expand Down Expand Up @@ -283,3 +283,68 @@ function Remove-WormStorageAccounts() {
}
}
}

function SetResourceNetworkAccessRules([string]$ResourceGroupName, [array]$AllowIpRanges, [switch]$CI) {
SetStorageNetworkAccessRules -ResourceGroupName $ResourceGroupName -AllowIpRanges $AllowIpRanges -CI:$CI
}

function SetStorageNetworkAccessRules([string]$ResourceGroupName, [array]$AllowIpRanges, [switch]$CI, [switch]$Override) {
$clientIp = $null
$storageAccounts = Retry { Get-AzResource -ResourceGroupName $ResourceGroupName -ResourceType "Microsoft.Storage/storageAccounts" }
# Add client IP to storage account when running as local user. Pipeline's have their own vnet with access
if ($storageAccounts) {
foreach ($account in $storageAccounts) {
$rules = Get-AzStorageAccountNetworkRuleSet -ResourceGroupName $ResourceGroupName -AccountName $account.Name
if ($rules -and ($Override -or $rules.DefaultAction -eq "Allow")) {
Write-Host "Restricting network rules in storage account '$($account.Name)' to deny access by default"
Retry { Update-AzStorageAccountNetworkRuleSet -ResourceGroupName $ResourceGroupName -Name $account.Name -DefaultAction Deny }
if ($CI -and $env:PoolSubnet) {
Write-Host "Enabling access to '$($account.Name)' from pipeline subnet $($env:PoolSubnet)"
Retry { Add-AzStorageAccountNetworkRule -ResourceGroupName $ResourceGroupName -Name $account.Name -VirtualNetworkResourceId $env:PoolSubnet }
}
elseif ($AllowIpRanges) {
Write-Host "Enabling access to '$($account.Name)' to $($AllowIpRanges.Length) IP ranges"
$ipRanges = $AllowIpRanges | ForEach-Object {
@{ Action = 'allow'; IPAddressOrRange = $_ }
}
Retry { Update-AzStorageAccountNetworkRuleSet -ResourceGroupName $ResourceGroupName -Name $account.Name -IPRule $ipRanges | Out-Null }
}
elseif (!$CI) {
Write-Host "Enabling access to '$($account.Name)' from client IP"
$clientIp ??= Retry { Invoke-RestMethod -Uri 'https://icanhazip.com/' } # cloudflare owned ip site
$clientIp = $clientIp.Trim()
$ipRanges = Get-AzStorageAccountNetworkRuleSet -ResourceGroupName $ResourceGroupName -Name $account.Name
if ($ipRanges) {
foreach ($range in $ipRanges.IpRules) {
if (DoesSubnetOverlap $range.IPAddressOrRange $clientIp) {
return
}
}
}
Retry { Add-AzStorageAccountNetworkRule -ResourceGroupName $ResourceGroupName -Name $account.Name -IPAddressOrRange $clientIp | Out-Null }
}
}
}
}
}

function DoesSubnetOverlap([string]$ipOrCidr, [string]$overlapIp) {
[System.Net.IPAddress]$overlapIpAddress = $overlapIp
$parsed = $ipOrCidr -split '/'
[System.Net.IPAddress]$baseIp = $parsed[0]
if ($parsed.Length -eq 1) {
return $baseIp -eq $overlapIpAddress
}

$subnet = $parsed[1]
$subnetNum = [int]$subnet

$baseMask = [math]::pow(2, 31)
$mask = 0
for ($i = 0; $i -lt $subnetNum; $i++) {
$mask = $mask + $baseMask;
$baseMask = $baseMask / 2
}

return $baseIp.Address -eq ($overlapIpAddress.Address -band ([System.Net.IPAddress]$mask).Address)
}