diff --git a/eng/common/build.sh b/eng/common/build.sh index bc07a1c6848..a16e18b174a 100755 --- a/eng/common/build.sh +++ b/eng/common/build.sh @@ -188,9 +188,6 @@ function InitializeCustomToolset { function Build { - if [[ "$ci" == true ]]; then - TryLogClientIpAddress - fi InitializeToolset InitializeCustomToolset diff --git a/eng/common/sdk-task.ps1 b/eng/common/sdk-task.ps1 index 7ab9baac5c8..b1bca63ab1d 100644 --- a/eng/common/sdk-task.ps1 +++ b/eng/common/sdk-task.ps1 @@ -83,9 +83,6 @@ try { } if ($restore) { - if ($ci) { - Try-LogClientIpAddress - } Build 'Restore' } diff --git a/eng/common/templates/job/onelocbuild.yml b/eng/common/templates/job/onelocbuild.yml index 2b55a567f82..069098b0a06 100644 --- a/eng/common/templates/job/onelocbuild.yml +++ b/eng/common/templates/job/onelocbuild.yml @@ -12,6 +12,7 @@ parameters: SourcesDirectory: $(Build.SourcesDirectory) CreatePr: true AutoCompletePr: false + ReusePr: true UseLfLineEndings: true UseCheckedInLocProjectJson: false LanguageSet: VS_Main_Languages @@ -64,6 +65,8 @@ jobs: ${{ if eq(parameters.CreatePr, true) }}: isAutoCompletePrSelected: ${{ parameters.AutoCompletePr }} isUseLfLineEndingsSelected: ${{ parameters.UseLfLineEndings }} + ${{ if eq(parameters.RepoType, 'gitHub') }}: + isShouldReusePrSelected: ${{ parameters.ReusePr }} packageSourceAuth: patAuth patVariable: ${{ parameters.CeapexPat }} ${{ if eq(parameters.RepoType, 'gitHub') }}: diff --git a/eng/common/tools.ps1 b/eng/common/tools.ps1 index 44484289943..f1e1cb53953 100644 --- a/eng/common/tools.ps1 +++ b/eng/common/tools.ps1 @@ -163,9 +163,6 @@ function InitializeDotNetCli([bool]$install, [bool]$createSdkLocationFile) { # Disable telemetry on CI. if ($ci) { $env:DOTNET_CLI_TELEMETRY_OPTOUT=1 - - # In case of network error, try to log the current IP for reference - Try-LogClientIpAddress } # Source Build uses DotNetCoreSdkDir variable @@ -301,32 +298,45 @@ function InstallDotNet([string] $dotnetRoot, if ($skipNonVersionedFiles) { $installParameters.SkipNonVersionedFiles = $skipNonVersionedFiles } if ($noPath) { $installParameters.NoPath = $True } - try { - & $installScript @installParameters - } - catch { - if ($runtimeSourceFeed -or $runtimeSourceFeedKey) { - Write-Host "Failed to install dotnet from public location. Trying from '$runtimeSourceFeed'" - if ($runtimeSourceFeed) { $installParameters.AzureFeed = $runtimeSourceFeed } + $variations = @() + $variations += @($installParameters) - if ($runtimeSourceFeedKey) { - $decodedBytes = [System.Convert]::FromBase64String($runtimeSourceFeedKey) - $decodedString = [System.Text.Encoding]::UTF8.GetString($decodedBytes) - $installParameters.FeedCredential = $decodedString - } + $dotnetBuilds = $installParameters.Clone() + $dotnetbuilds.AzureFeed = "https://dotnetbuilds.azureedge.net/public" + $variations += @($dotnetBuilds) - try { - & $installScript @installParameters - } - catch { - Write-PipelineTelemetryError -Category 'InitializeToolset' -Message "Failed to install dotnet from custom location '$runtimeSourceFeed'." - ExitWithExitCode 1 - } + if ($runtimeSourceFeed) { + $runtimeSource = $installParameters.Clone() + $runtimeSource.AzureFeed = $runtimeSourceFeed + if ($runtimeSourceFeedKey) { + $decodedBytes = [System.Convert]::FromBase64String($runtimeSourceFeedKey) + $decodedString = [System.Text.Encoding]::UTF8.GetString($decodedBytes) + $runtimeSource.FeedCredential = $decodedString + } + $variations += @($runtimeSource) + } + + $installSuccess = $false + foreach ($variation in $variations) { + if ($variation | Get-Member AzureFeed) { + $location = $variation.AzureFeed } else { - Write-PipelineTelemetryError -Category 'InitializeToolset' -Message "Failed to install dotnet from public location." - ExitWithExitCode 1 + $location = "public location"; + } + Write-Host "Attempting to install dotnet from $location." + try { + & $installScript @variation + $installSuccess = $true + break + } + catch { + Write-Host "Failed to install dotnet from $location." } } + if (-not $installSuccess) { + Write-PipelineTelemetryError -Category 'InitializeToolset' -Message "Failed to install dotnet from any of the specified locations." + ExitWithExitCode 1 + } } # @@ -882,24 +892,6 @@ if (!$disableConfigureToolsetImport) { } } -function Try-LogClientIpAddress() -{ - Write-Host "Attempting to log this client's IP for Azure Package feed telemetry purposes" - try - { - $result = Invoke-WebRequest -Uri "http://co1.msedge.net/fdv2/diagnostics.aspx" -UseBasicParsing - $lines = $result.Content.Split([Environment]::NewLine) - $socketIp = $lines | Select-String -Pattern "^Socket IP:.*" - Write-Host $socketIp - $clientIp = $lines | Select-String -Pattern "^Client IP:.*" - Write-Host $clientIp - } - catch - { - Write-Host "Unable to get this machine's effective IP address for logging: $_" - } -} - # # If $ci flag is set, turn on (and log that we did) special environment variables for improved Nuget client retry logic. # diff --git a/eng/common/tools.sh b/eng/common/tools.sh index 6a4871ef72b..e555c34269f 100755 --- a/eng/common/tools.sh +++ b/eng/common/tools.sh @@ -188,28 +188,29 @@ function InstallDotNet { GetDotNetInstallScript "$root" local install_script=$_GetDotNetInstallScript - local archArg='' + local installParameters=(--version $version --install-dir "$root") + if [[ -n "${3:-}" ]] && [ "$3" != 'unset' ]; then - archArg="--architecture $3" + installParameters+=(--architecture $3) fi - local runtimeArg='' if [[ -n "${4:-}" ]] && [ "$4" != 'sdk' ]; then - runtimeArg="--runtime $4" + installParameters+=(--runtime $4) fi - local skipNonVersionedFilesArg="" if [[ "$#" -ge "5" ]] && [[ "$5" != 'false' ]]; then - skipNonVersionedFilesArg="--skip-non-versioned-files" + installParameters+=(--skip-non-versioned-files) fi - bash "$install_script" --version $version --install-dir "$root" $archArg $runtimeArg $skipNonVersionedFilesArg || { - local exit_code=$? - echo "Failed to install dotnet SDK from public location (exit code '$exit_code')." - local runtimeSourceFeed='' - if [[ -n "${6:-}" ]]; then - runtimeSourceFeed="--azure-feed $6" - fi + local variations=() # list of variable names with parameter arrays in them + + local public_location=("${installParameters[@]}") + variations+=(public_location) - local runtimeSourceFeedKey='' + local dotnetbuilds=("${installParameters[@]}" --azure-feed "https://dotnetbuilds.azureedge.net/public") + variations+=(dotnetbuilds) + + if [[ -n "${6:-}" ]]; then + variations+=(private_feed) + local private_feed=("${installParameters[@]}" --azure-feed $6) if [[ -n "${7:-}" ]]; then # The 'base64' binary on alpine uses '-d' and doesn't support '--decode' # '-d'. To work around this, do a simple detection and switch the parameter @@ -219,22 +220,27 @@ function InstallDotNet { decodeArg="-d" fi decodedFeedKey=`echo $7 | base64 $decodeArg` - runtimeSourceFeedKey="--feed-credential $decodedFeedKey" + private_feed+=(--feed-credential $decodedFeedKey) fi + fi - if [[ -n "$runtimeSourceFeed" || -n "$runtimeSourceFeedKey" ]]; then - bash "$install_script" --version $version --install-dir "$root" $archArg $runtimeArg $skipNonVersionedFilesArg $runtimeSourceFeed $runtimeSourceFeedKey || { - local exit_code=$? - Write-PipelineTelemetryError -category 'InitializeToolset' "Failed to install dotnet SDK from custom location '$runtimeSourceFeed' (exit code '$exit_code')." - ExitWithExitCode $exit_code - } - else - if [[ $exit_code != 0 ]]; then - Write-PipelineTelemetryError -category 'InitializeToolset' "Failed to install dotnet SDK from public location (exit code '$exit_code')." - fi - ExitWithExitCode $exit_code + local installSuccess=0 + for variationName in "${variations[@]}"; do + local name="$variationName[@]" + local variation=("${!name}") + echo "Attempting to install dotnet from $variationName." + bash "$install_script" "${variation[@]}" && installSuccess=1 + if [[ "$installSuccess" -eq 1 ]]; then + break fi - } + + echo "Failed to install dotnet from $variationName." + done + + if [[ "$installSuccess" -eq 0 ]]; then + Write-PipelineTelemetryError -category 'InitializeToolset' "Failed to install dotnet SDK from any of the specified locations." + ExitWithExitCode 1 + fi } function with_retries { @@ -399,13 +405,6 @@ function StopProcesses { return 0 } -function TryLogClientIpAddress () { - echo 'Attempting to log this client''s IP for Azure Package feed telemetry purposes' - if command -v curl > /dev/null; then - curl -s 'http://co1.msedge.net/fdv2/diagnostics.aspx' | grep ' IP: ' || true - fi -} - function MSBuild { local args=$@ if [[ "$pipelines_log" == true ]]; then diff --git a/src/Microsoft.DotNet.Build.Tasks.Installers/build/installer.singlerid.targets b/src/Microsoft.DotNet.Build.Tasks.Installers/build/installer.singlerid.targets index 2651a4588fd..6c53ad557f9 100644 --- a/src/Microsoft.DotNet.Build.Tasks.Installers/build/installer.singlerid.targets +++ b/src/Microsoft.DotNet.Build.Tasks.Installers/build/installer.singlerid.targets @@ -458,6 +458,7 @@ $([System.IO.File]::ReadAllText('$(LicenseFile)').Replace('%0A', '\n').Replace('"', '\"')) + /usr/share/dotnet @@ -494,7 +495,7 @@ <_CommonLinuxPackageProperty Include="short_description" String="$(_ShortDescription)" /> <_CommonLinuxPackageProperty Include="maintainer_name" String=".NET Team" /> <_CommonLinuxPackageProperty Include="maintainer_email" String="dotnetpackages@dotnetfoundation.org" /> - <_CommonLinuxPackageProperty Include="install_root" String="/usr/share/dotnet" /> + <_CommonLinuxPackageProperty Include="install_root" String="$(LinuxInstallRoot)" /> <_CommonLinuxPackageProperty Include="long_description" String=".NET is a development platform that you can use to build command-line applications, microservices and modern websites. It is open source, cross-platform and is supported by Microsoft. We hope you enjoy using it! If you do, please consider joining the active community of developers that are contributing to the project on GitHub (https://github.com/dotnet/core). We happily accept issues and PRs." /> <_CommonLinuxPackageProperty Include="homepage" String="https://github.com/dotnet/core" /> <_CommonLinuxPackageProperty Include="copyright" String="2017 Microsoft" /> diff --git a/src/Microsoft.DotNet.Helix/Sdk/FindDotNetCliPackage.cs b/src/Microsoft.DotNet.Helix/Sdk/FindDotNetCliPackage.cs index a7f8947e11e..1d4f62a78c1 100644 --- a/src/Microsoft.DotNet.Helix/Sdk/FindDotNetCliPackage.cs +++ b/src/Microsoft.DotNet.Helix/Sdk/FindDotNetCliPackage.cs @@ -1,4 +1,6 @@ using System; +using System.Collections.Generic; +using System.Linq; using System.Net; using System.Net.Http; using System.Threading.Tasks; @@ -17,7 +19,6 @@ public class FindDotNetCliPackage : BaseTask DelayBase = 3.0 }; private static readonly HttpClient _client = new HttpClient(new HttpClientHandler { CheckCertificateRevocationList = true }); - private const string DotNetCliAzureFeed = "https://dotnetcli.blob.core.windows.net/dotnet"; /// /// 'LTS' or 'Current' @@ -55,71 +56,105 @@ public override bool Execute() private async Task ExecuteAsync() { NormalizeParameters(); - await ResolveVersionAsync(); - - string downloadUrl = await GetDownloadUrlAsync(); - - Log.LogMessage($"Retrieved dotnet cli {PackageType} version {Version} package uri {downloadUrl}, testing..."); - - try + var feeds = new List { - using HttpResponseMessage res = await HeadRequestWithRetry(downloadUrl); + "https://dotnetcli.azureedge.net/dotnet", + "https://dotnetbuilds.azureedge.net/public", + }; - if (res.StatusCode == HttpStatusCode.NotFound) + string finalDownloadUrl = null; + foreach (var feed in feeds) + { + string downloadUrl = await GetDownloadUrlAsync(feed); + if (downloadUrl == null) { - // 404 means that we successfully hit the server, and it returned 404. This cannot be a network hiccup - Log.LogError(FailureCategory.Build, $"Unable to find dotnet cli {PackageType} version {Version}, tried {downloadUrl}"); + Log.LogMessage($"Could not retrieve dotnet cli {PackageType} version {Version} package uri from feed {feed}"); + continue; } - else + + Log.LogMessage($"Retrieved dotnet cli {PackageType} version {Version} package uri {downloadUrl} from feed {feed}, testing..."); + + try { + using HttpResponseMessage res = await HeadRequestWithRetry(downloadUrl); + + if (res.StatusCode == HttpStatusCode.NotFound) + { + // 404 means that we successfully hit the server, and it returned 404. This cannot be a network hiccup + Log.LogMessage($"Unable to find dotnet cli {PackageType} version {Version} from feed {feed}"); + continue; + } + res.EnsureSuccessStatusCode(); + finalDownloadUrl = downloadUrl; + } + catch (Exception ex) + { + Log.LogMessage($"Unable to access dotnet cli {PackageType} version {Version} from feed {feed}, {ex.Message}"); } } - catch (Exception ex) + + if (finalDownloadUrl == null) { - Log.LogError(FailureCategory.Build, $"Unable to access dotnet cli {PackageType} version {Version} at {downloadUrl}, {ex.Message}"); + Log.LogError(FailureCategory.Build, $"Unable to find dotnet cli {PackageType} version {Version} from any of the specified feeds."); } + if (!Log.HasLoggedErrors) { - Log.LogMessage($"Url {downloadUrl} is valid."); - PackageUri = downloadUrl; + Log.LogMessage($"Url {finalDownloadUrl} is valid."); + PackageUri = finalDownloadUrl; } } - private async Task GetDownloadUrlAsync() + private async Task GetDownloadUrlAsync(string feed) { - string extension = Runtime.StartsWith("win") ? "zip" : "tar.gz"; - string effectiveVersion = await GetEffectiveVersion(); + var oldVersion = Version; // ResolveVersionAsync will adjust the Version property, but we need it set back for other feeds to see the same initial Version + try + { + var version = await ResolveVersionAsync(feed); + string extension = Runtime.StartsWith("win") ? "zip" : "tar.gz"; + string effectiveVersion = await GetEffectiveVersion(feed, version); - return PackageType switch + return PackageType switch + { + "sdk" => $"{feed}/Sdk/{version}/dotnet-sdk-{effectiveVersion}-{Runtime}.{extension}", + "aspnetcore-runtime" => + $"{feed}/aspnetcore/Runtime/{version}/aspnetcore-runtime-{effectiveVersion}-{Runtime}.{extension}", + _ => $"{feed}/Runtime/{version}/dotnet-runtime-{effectiveVersion}-{Runtime}.{extension}" + }; + } + catch (Exception ex) { - "sdk" => $"{DotNetCliAzureFeed}/Sdk/{Version}/dotnet-sdk-{effectiveVersion}-{Runtime}.{extension}", - "aspnetcore-runtime" => $"{DotNetCliAzureFeed}/aspnetcore/Runtime/{Version}/aspnetcore-runtime-{effectiveVersion}-{Runtime}.{extension}", - _ => $"{DotNetCliAzureFeed}/Runtime/{Version}/dotnet-runtime-{effectiveVersion}-{Runtime}.{extension}" - }; + Log.LogWarning($"Unable to resolve download link from feed {feed}; {ex.Message}"); + return null; + } + finally + { + Version = oldVersion; + } } - private async Task GetEffectiveVersion() + private async Task GetEffectiveVersion(string feed, string version) { - if (NuGetVersion.TryParse(Version, out NuGetVersion semanticVersion)) + if (NuGetVersion.TryParse(version, out NuGetVersion semanticVersion)) { // Pared down version of the logic from https://github.com/dotnet/install-scripts/blob/main/src/dotnet-install.ps1 // If this functionality stops working, review changes made there. // Current strategy is to start with a runtime-specific name then fall back to 'productVersion.txt' - string effectiveVersion = Version; + string effectiveVersion = version; // Do nothing for older runtimes; the file won't exist if (semanticVersion >= new NuGetVersion("5.0.0")) { var productVersionText = PackageType switch { - "sdk" => await GetMatchingProductVersionTxtContents($"{DotNetCliAzureFeed}/Sdk/{Version}", "sdk-productVersion.txt"), - "aspnetcore-runtime" => await GetMatchingProductVersionTxtContents($"{DotNetCliAzureFeed}/aspnetcore/Runtime/{Version}", "aspnetcore-productVersion.txt"), - _ => await GetMatchingProductVersionTxtContents($"{DotNetCliAzureFeed}/Runtime/{Version}", "runtime-productVersion.txt") + "sdk" => await GetMatchingProductVersionTxtContents($"{feed}/Sdk/{version}", "sdk-productVersion.txt"), + "aspnetcore-runtime" => await GetMatchingProductVersionTxtContents($"{feed}/aspnetcore/Runtime/{version}", "aspnetcore-productVersion.txt"), + _ => await GetMatchingProductVersionTxtContents($"{feed}/Runtime/{version}", "runtime-productVersion.txt") }; - if (!productVersionText.Equals(Version)) + if (!productVersionText.Equals(version)) { effectiveVersion = productVersionText; Log.LogMessage($"Switched to effective .NET Core version '{productVersionText}' from matching productVersion.txt"); @@ -127,10 +162,8 @@ private async Task GetEffectiveVersion() } return effectiveVersion; } - else - { - throw new ArgumentException($"'{Version}' is not a valid semantic version."); - } + + throw new ArgumentException($"'{version}' is not a valid semantic version."); } private async Task GetMatchingProductVersionTxtContents(string baseUri, string customVersionTextFileName) { @@ -243,16 +276,17 @@ private void NormalizeParameters() } } - private async Task ResolveVersionAsync() + private async Task ResolveVersionAsync(string feed) { + string version = Version; if (Version == "latest") { Log.LogMessage(MessageImportance.Low, "Resolving latest dotnet cli version."); string latestVersionUrl = PackageType switch { - "sdk" => $"{DotNetCliAzureFeed}/Sdk/{Channel}/latest.version", - "aspnetcore-runtime" => $"{DotNetCliAzureFeed}/aspnetcore/Runtime/{Channel}/latest.version", - _ => $"{DotNetCliAzureFeed}/Runtime/{Channel}/latest.version" + "sdk" => $"{feed}/Sdk/{Channel}/latest.version", + "aspnetcore-runtime" => $"{feed}/aspnetcore/Runtime/{Channel}/latest.version", + _ => $"{feed}/Runtime/{Channel}/latest.version" }; Log.LogMessage(MessageImportance.Low, $"Resolving latest version from url {latestVersionUrl}"); @@ -261,9 +295,11 @@ private async Task ResolveVersionAsync() versionResponse.EnsureSuccessStatusCode(); string latestVersionContent = await versionResponse.Content.ReadAsStringAsync(); string[] versionData = latestVersionContent.Split(Array.Empty(), StringSplitOptions.RemoveEmptyEntries); - Version = versionData[1]; - Log.LogMessage(MessageImportance.Low, $"Got latest dotnet cli version {Version}"); + version = versionData[1]; + Log.LogMessage(MessageImportance.Low, $"Got latest dotnet cli version {version}"); } + + return version; } } }