Skip to content

Cherry-Pick from main -> release/6.0: Add support for dotnetbuilds-published dotnet cli packages #8191

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
alexperovich merged 1 commit into from
Nov 16, 2021
Merged

Cherry-Pick from main -> release/6.0: Add support for dotnetbuilds-published dotnet cli packages #8191

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
55 changes: 34 additions & 21 deletions eng/common/tools.ps1
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -301,31 +301,44 @@ function InstallDotNet([string] $dotnetRoot,
if ($skipNonVersionedFiles) { $installParameters.SkipNonVersionedFiles = $skipNonVersionedFiles }
if ($noPath) { $installParameters.NoPath = $True }

try {
& $installScript @installParameters
}
catch {
if ($runtimeSourceFeed -or $runtimeSourceFeedKey) {
Write-Host "Failed to install dotnet from public location. Trying from '$runtimeSourceFeed'"
if ($runtimeSourceFeed) { $installParameters.AzureFeed = $runtimeSourceFeed }
$variations = @()
$variations += @($installParameters)

if ($runtimeSourceFeedKey) {
$decodedBytes = [System.Convert]::FromBase64String($runtimeSourceFeedKey)
$decodedString = [System.Text.Encoding]::UTF8.GetString($decodedBytes)
$installParameters.FeedCredential = $decodedString
}
$dotnetBuilds = $installParameters.Clone()
$dotnetbuilds.AzureFeed = "https://dotnetbuilds.azureedge.net/public"
$variations += @($dotnetBuilds)

try {
& $installScript @installParameters
}
catch {
Write-PipelineTelemetryError -Category 'InitializeToolset' -Message "Failed to install dotnet from custom location '$runtimeSourceFeed'."
ExitWithExitCode 1
}
if ($runtimeSourceFeed) {
$runtimeSource = $installParameters.Clone()
$runtimeSource.AzureFeed = $runtimeSourceFeed
if ($runtimeSourceFeedKey) {
$decodedBytes = [System.Convert]::FromBase64String($runtimeSourceFeedKey)
$decodedString = [System.Text.Encoding]::UTF8.GetString($decodedBytes)
$runtimeSource.FeedCredential = $decodedString
}
$variations += @($runtimeSource)
}

$installSuccess = $false
foreach ($variation in $variations) {
if ($variation | Get-Member AzureFeed) {
$location = $variation.AzureFeed
} else {
Write-PipelineTelemetryError -Category 'InitializeToolset' -Message "Failed to install dotnet from public location."
ExitWithExitCode 1
$location = "public location";
}
Write-Host "Attempting to install dotnet from $location."
try {
& $installScript @variation
$installSuccess = $true
break
}
catch {
Write-Host "Failed to install dotnet from $location."
}
}
if (-not $installSuccess) {
Write-PipelineTelemetryError -Category 'InitializeToolset' -Message "Failed to install dotnet from any of the specified locations."
ExitWithExitCode 1
}
}

Expand Down
60 changes: 33 additions & 27 deletions eng/common/tools.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -188,28 +188,29 @@ function InstallDotNet {
GetDotNetInstallScript "$root"
local install_script=$_GetDotNetInstallScript

local archArg=''
local installParameters=(--version $version --install-dir "$root")

if [[ -n "${3:-}" ]] && [ "$3" != 'unset' ]; then
archArg="--architecture $3"
installParameters+=(--architecture $3)
fi
local runtimeArg=''
if [[ -n "${4:-}" ]] && [ "$4" != 'sdk' ]; then
runtimeArg="--runtime $4"
installParameters+=(--runtime $4)
fi
local skipNonVersionedFilesArg=""
if [[ "$#" -ge "5" ]] && [[ "$5" != 'false' ]]; then
skipNonVersionedFilesArg="--skip-non-versioned-files"
installParameters+=(--skip-non-versioned-files)
fi
bash "$install_script" --version $version --install-dir "$root" $archArg $runtimeArg $skipNonVersionedFilesArg || {
local exit_code=$?
echo "Failed to install dotnet SDK from public location (exit code '$exit_code')."

local runtimeSourceFeed=''
if [[ -n "${6:-}" ]]; then
runtimeSourceFeed="--azure-feed $6"
fi
local variations=() # list of variable names with parameter arrays in them

local public_location=("${installParameters[@]}")
variations+=(public_location)

local dotnetbuilds=("${installParameters[@]}" --azure-feed "https://dotnetbuilds.azureedge.net/public")
variations+=(dotnetbuilds)

local runtimeSourceFeedKey=''
if [[ -n "${6:-}" ]]; then
variations+=(private_feed)
local private_feed=("${installParameters[@]}" --azure-feed $6)
if [[ -n "${7:-}" ]]; then
# The 'base64' binary on alpine uses '-d' and doesn't support '--decode'
# '-d'. To work around this, do a simple detection and switch the parameter
Expand All @@ -219,22 +220,27 @@ function InstallDotNet {
decodeArg="-d"
fi
decodedFeedKey=`echo $7 | base64 $decodeArg`
runtimeSourceFeedKey="--feed-credential $decodedFeedKey"
private_feed+=(--feed-credential $decodedFeedKey)
fi
fi

if [[ -n "$runtimeSourceFeed" || -n "$runtimeSourceFeedKey" ]]; then
bash "$install_script" --version $version --install-dir "$root" $archArg $runtimeArg $skipNonVersionedFilesArg $runtimeSourceFeed $runtimeSourceFeedKey || {
local exit_code=$?
Write-PipelineTelemetryError -category 'InitializeToolset' "Failed to install dotnet SDK from custom location '$runtimeSourceFeed' (exit code '$exit_code')."
ExitWithExitCode $exit_code
}
else
if [[ $exit_code != 0 ]]; then
Write-PipelineTelemetryError -category 'InitializeToolset' "Failed to install dotnet SDK from public location (exit code '$exit_code')."
fi
ExitWithExitCode $exit_code
local installSuccess=0
for variationName in "${variations[@]}"; do
local name="$variationName[@]"
local variation=("${!name}")
echo "Attempting to install dotnet from $variationName."
bash "$install_script" "${variation[@]}" && installSuccess=1
if [[ "$installSuccess" -eq 1 ]]; then
break
fi
}

echo "Failed to install dotnet from $variationName."
done

if [[ "$installSuccess" -eq 0 ]]; then
Write-PipelineTelemetryError -category 'InitializeToolset' "Failed to install dotnet SDK from any of the specified locations."
ExitWithExitCode 1
fi
}

function with_retries {
Expand Down
120 changes: 78 additions & 42 deletions src/Microsoft.DotNet.Helix/Sdk/FindDotNetCliPackage.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,6 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Net;
using System.Net.Http;
using System.Threading.Tasks;
Expand All @@ -17,7 +19,6 @@ public class FindDotNetCliPackage : BaseTask
DelayBase = 3.0
};
private static readonly HttpClient _client = new HttpClient(new HttpClientHandler { CheckCertificateRevocationList = true });
private const string DotNetCliAzureFeed = "https://dotnetcli.blob.core.windows.net/dotnet";

/// <summary>
/// 'LTS' or 'Current'
Expand Down Expand Up @@ -55,82 +56,114 @@ public override bool Execute()
private async Task ExecuteAsync()
{
NormalizeParameters();
await ResolveVersionAsync();

string downloadUrl = await GetDownloadUrlAsync();

Log.LogMessage($"Retrieved dotnet cli {PackageType} version {Version} package uri {downloadUrl}, testing...");

try
var feeds = new List<string>
{
using HttpResponseMessage res = await HeadRequestWithRetry(downloadUrl);
"https://dotnetcli.azureedge.net/dotnet",
"https://dotnetbuilds.azureedge.net/public",
};

if (res.StatusCode == HttpStatusCode.NotFound)
string finalDownloadUrl = null;
foreach (var feed in feeds)
{
string downloadUrl = await GetDownloadUrlAsync(feed);
if (downloadUrl == null)
{
// 404 means that we successfully hit the server, and it returned 404. This cannot be a network hiccup
Log.LogError(FailureCategory.Build, $"Unable to find dotnet cli {PackageType} version {Version}, tried {downloadUrl}");
Log.LogMessage($"Could not retrieve dotnet cli {PackageType} version {Version} package uri from feed {feed}");
continue;
}
else

Log.LogMessage($"Retrieved dotnet cli {PackageType} version {Version} package uri {downloadUrl} from feed {feed}, testing...");

try
{
using HttpResponseMessage res = await HeadRequestWithRetry(downloadUrl);

if (res.StatusCode == HttpStatusCode.NotFound)
{
// 404 means that we successfully hit the server, and it returned 404. This cannot be a network hiccup
Log.LogMessage($"Unable to find dotnet cli {PackageType} version {Version} from feed {feed}");
continue;
}

res.EnsureSuccessStatusCode();
finalDownloadUrl = downloadUrl;
}
catch (Exception ex)
{
Log.LogMessage($"Unable to access dotnet cli {PackageType} version {Version} from feed {feed}, {ex.Message}");
}
}
catch (Exception ex)

if (finalDownloadUrl == null)
{
Log.LogError(FailureCategory.Build, $"Unable to access dotnet cli {PackageType} version {Version} at {downloadUrl}, {ex.Message}");
Log.LogError(FailureCategory.Build, $"Unable to find dotnet cli {PackageType} version {Version} from any of the specified feeds.");
}


if (!Log.HasLoggedErrors)
{
Log.LogMessage($"Url {downloadUrl} is valid.");
PackageUri = downloadUrl;
Log.LogMessage($"Url {finalDownloadUrl} is valid.");
PackageUri = finalDownloadUrl;
}
}

private async Task<string> GetDownloadUrlAsync()
private async Task<string> GetDownloadUrlAsync(string feed)
{
string extension = Runtime.StartsWith("win") ? "zip" : "tar.gz";
string effectiveVersion = await GetEffectiveVersion();
var oldVersion = Version; // ResolveVersionAsync will adjust the Version property, but we need it set back for other feeds to see the same initial Version
try
{
var version = await ResolveVersionAsync(feed);
string extension = Runtime.StartsWith("win") ? "zip" : "tar.gz";
string effectiveVersion = await GetEffectiveVersion(feed, version);

return PackageType switch
return PackageType switch
{
"sdk" => $"{feed}/Sdk/{version}/dotnet-sdk-{effectiveVersion}-{Runtime}.{extension}",
"aspnetcore-runtime" =>
$"{feed}/aspnetcore/Runtime/{version}/aspnetcore-runtime-{effectiveVersion}-{Runtime}.{extension}",
_ => $"{feed}/Runtime/{version}/dotnet-runtime-{effectiveVersion}-{Runtime}.{extension}"
};
}
catch (Exception ex)
{
"sdk" => $"{DotNetCliAzureFeed}/Sdk/{Version}/dotnet-sdk-{effectiveVersion}-{Runtime}.{extension}",
"aspnetcore-runtime" => $"{DotNetCliAzureFeed}/aspnetcore/Runtime/{Version}/aspnetcore-runtime-{effectiveVersion}-{Runtime}.{extension}",
_ => $"{DotNetCliAzureFeed}/Runtime/{Version}/dotnet-runtime-{effectiveVersion}-{Runtime}.{extension}"
};
Log.LogWarning($"Unable to resolve download link from feed {feed}; {ex.Message}");
return null;
}
finally
{
Version = oldVersion;
}
}

private async Task<string> GetEffectiveVersion()
private async Task<string> GetEffectiveVersion(string feed, string version)
{
if (NuGetVersion.TryParse(Version, out NuGetVersion semanticVersion))
if (NuGetVersion.TryParse(version, out NuGetVersion semanticVersion))
{
// Pared down version of the logic from https://github.com/dotnet/install-scripts/blob/main/src/dotnet-install.ps1
// If this functionality stops working, review changes made there.
// Current strategy is to start with a runtime-specific name then fall back to 'productVersion.txt'
string effectiveVersion = Version;
string effectiveVersion = version;

// Do nothing for older runtimes; the file won't exist
if (semanticVersion >= new NuGetVersion("5.0.0"))
{
var productVersionText = PackageType switch
{
"sdk" => await GetMatchingProductVersionTxtContents($"{DotNetCliAzureFeed}/Sdk/{Version}", "sdk-productVersion.txt"),
"aspnetcore-runtime" => await GetMatchingProductVersionTxtContents($"{DotNetCliAzureFeed}/aspnetcore/Runtime/{Version}", "aspnetcore-productVersion.txt"),
_ => await GetMatchingProductVersionTxtContents($"{DotNetCliAzureFeed}/Runtime/{Version}", "runtime-productVersion.txt")
"sdk" => await GetMatchingProductVersionTxtContents($"{feed}/Sdk/{version}", "sdk-productVersion.txt"),
"aspnetcore-runtime" => await GetMatchingProductVersionTxtContents($"{feed}/aspnetcore/Runtime/{version}", "aspnetcore-productVersion.txt"),
_ => await GetMatchingProductVersionTxtContents($"{feed}/Runtime/{version}", "runtime-productVersion.txt")
};

if (!productVersionText.Equals(Version))
if (!productVersionText.Equals(version))
{
effectiveVersion = productVersionText;
Log.LogMessage($"Switched to effective .NET Core version '{productVersionText}' from matching productVersion.txt");
}
}
return effectiveVersion;
}
else
{
throw new ArgumentException($"'{Version}' is not a valid semantic version.");
}

throw new ArgumentException($"'{version}' is not a valid semantic version.");
}
private async Task<string> GetMatchingProductVersionTxtContents(string baseUri, string customVersionTextFileName)
{
Expand Down Expand Up @@ -243,16 +276,17 @@ private void NormalizeParameters()
}
}

private async Task ResolveVersionAsync()
private async Task<string> ResolveVersionAsync(string feed)
{
string version = Version;
if (Version == "latest")
{
Log.LogMessage(MessageImportance.Low, "Resolving latest dotnet cli version.");
string latestVersionUrl = PackageType switch
{
"sdk" => $"{DotNetCliAzureFeed}/Sdk/{Channel}/latest.version",
"aspnetcore-runtime" => $"{DotNetCliAzureFeed}/aspnetcore/Runtime/{Channel}/latest.version",
_ => $"{DotNetCliAzureFeed}/Runtime/{Channel}/latest.version"
"sdk" => $"{feed}/Sdk/{Channel}/latest.version",
"aspnetcore-runtime" => $"{feed}/aspnetcore/Runtime/{Channel}/latest.version",
_ => $"{feed}/Runtime/{Channel}/latest.version"
};

Log.LogMessage(MessageImportance.Low, $"Resolving latest version from url {latestVersionUrl}");
Expand All @@ -261,9 +295,11 @@ private async Task ResolveVersionAsync()
versionResponse.EnsureSuccessStatusCode();
string latestVersionContent = await versionResponse.Content.ReadAsStringAsync();
string[] versionData = latestVersionContent.Split(Array.Empty<char>(), StringSplitOptions.RemoveEmptyEntries);
Version = versionData[1];
Log.LogMessage(MessageImportance.Low, $"Got latest dotnet cli version {Version}");
version = versionData[1];
Log.LogMessage(MessageImportance.Low, $"Got latest dotnet cli version {version}");
}

return version;
}
}
}