refactor stress test builds

.gitignore

@@ -354,3 +354,7 @@ src/coreclr/System.Private.CoreLib/common
 !src/coreclr/inc/obj/
 !src/coreclr/vm/.vscode/
 !src/coreclr/vm/.vscode/c_cpp_properties.json
+
+# Temporary artifacts from local libraries stress builds
+.dotnet-daily/
+run-stress-*

eng/docker/build-docker-sdk.ps1

@@ -7,7 +7,7 @@ Param(
   [string][Alias('t')]$imageName = "dotnet-sdk-libs-current",
   [string][Alias('c')]$configuration = "Release",
   [switch][Alias('w')]$buildWindowsContainers,
-  [switch][Alias('pa')]$privateAspNetCore
+  [switch][Alias('ds')]$dailySdk
 )
 
 $ErrorActionPreference = "Stop"
@@ -16,9 +16,9 @@ $REPO_ROOT_DIR=$(git -C "$PSScriptRoot" rev-parse --show-toplevel)
 
 $dockerFilePrefix="$PSScriptRoot/libraries-sdk"
 
-if ($privateAspNetCore)
+if ($dailySdk)
 {
-  $dockerFilePrefix="$PSScriptRoot/libraries-sdk-aspnetcore"
+  $dockerFilePrefix="$PSScriptRoot/libraries-sdk-daily"
 }
 
 if ($buildWindowsContainers)
@@ -33,13 +33,30 @@ if ($buildWindowsContainers)
     exit $LASTEXITCODE
   }
 
-  $dockerFile="$dockerFilePrefix.windows.Dockerfile"
+  $dockerFile="$dockerFilePrefix-daily.windows.Dockerfile"
+
+  ## Collect the following artifacts under to a special folder that and copy it to the container,
+  ## so projects can build and test against the live-built runtime:
+  ## 1. Reference assembly pack (microsoft.netcore.app.ref)
+  ## 2. Runtime pack (microsoft.netcore.app.runtime.linux-x64)
+  ## 3. targetingpacks.targets, so stress test builds can target the live-built runtime instead of the one in the pre-installed SDK
+  ## 4. testhost
+  $binArtifacts = "$REPO_ROOT_DIR\artifacts\bin"
+  $dockerArtifacts = "$REPO_ROOT_DIR\artifacts\docker-context"
 
+  if (Test-Path $dockerArtifacts) {
+      Remove-Item -Recurse -Force $dockerArtifacts
+  }
+
+  Copy-Item -Recurse -Destination $dockerArtifacts\microsoft.netcore.app.ref -Path $binArtifacts\microsoft.netcore.app.ref
+  Copy-Item -Recurse -Destination $dockerArtifacts\microsoft.netcore.app.runtime.win-x64 -Path $binArtifacts\microsoft.netcore.app.runtime.win-x64
+  Copy-Item -Recurse -Destination $dockerArtifacts\testhost -Path $binArtifacts\testhost
+  Copy-Item -Recurse -Destination $dockerArtifacts\targetingpacks.targets -Path $REPO_ROOT_DIR\eng\targetingpacks.targets
+
   docker build --tag $imageName `
     --build-arg CONFIGURATION=$configuration `
-    --build-arg TESTHOST_LOCATION=. `
     --file $dockerFile `
-    "$REPO_ROOT_DIR/artifacts/bin/testhost"
+    $dockerArtifacts
 }
 else
 {

eng/docker/build-docker-sdk.sh

@@ -23,7 +23,6 @@ scriptroot="$( cd -P "$( dirname "$source" )" && pwd )"
 
 imagename="dotnet-sdk-libs-current"
 configuration="Release"
-privateaspnetcore=0
 
 while [[ $# > 0 ]]; do
   opt="$(echo "${1/#--/-}" | tr "[:upper:]" "[:lower:]")"
@@ -36,10 +35,6 @@ while [[ $# > 0 ]]; do
       configuration=$2
       shift 2
       ;;
-    -privateaspnetcore|-pa)
-      privateaspnetcore=1
-      shift 1
-      ;;
     *)
       shift 1
       ;;
@@ -49,13 +44,9 @@ done
 repo_root=$(git rev-parse --show-toplevel)
 docker_file="$scriptroot/libraries-sdk.linux.Dockerfile"
 
-if [[ $privateaspnetcore -eq 1 ]]; then
-    docker_file="$scriptroot/libraries-sdk-aspnetcore.linux.Dockerfile"
-fi
-
 docker build --tag $imagename \
     --build-arg CONFIGURATION=$configuration \
     --file $docker_file \
     $repo_root
 
-exit $?
+exit $?

eng/docker/libraries-sdk-daily.windows.Dockerfile

@@ -0,0 +1,28 @@
+# escape=`
+# Simple Dockerfile which copies clr and library build artifacts into target dotnet sdk image
+ARG SDK_BASE_IMAGE=mcr.microsoft.com/dotnet/nightly/sdk:6.0-nanoserver-1809
+FROM $SDK_BASE_IMAGE as target
+
+SHELL ["pwsh", "-Command", "$ErrorActionPreference = 'Stop'; $ProgressPreference = 'SilentlyContinue';"]
+
+USER ContainerAdministrator
+
+RUN Invoke-WebRequest -Uri https://dot.net/v1/dotnet-install.ps1 -OutFile .\dotnet-install.ps1
+RUN & .\dotnet-install.ps1 -Channel 7.0.1xx -Quality daily -InstallDir 'C:/Program Files/dotnet'
+
+USER ContainerUser
+
+COPY . /live-runtime-artifacts
+
+ARG VERSION=7.0
+ARG CONFIGURATION=Release
+
+ENV _ASPNETCORE_SOURCE_DIR="C:/Program Files/dotnet/shared/Microsoft.AspNetCore.App/$VERSION*"
+ENV _ASPNETCORE_DEST_DIR="C:/live-runtime-artifacts/testhost/net$VERSION-windows-$CONFIGURATION-x64/shared/Microsoft.AspNetCore.App"
+RUN & New-Item -ItemType Directory -Path $env:_ASPNETCORE_DEST_DIR
+RUN Copy-Item -Recurse -Path $env:_ASPNETCORE_SOURCE_DIR -Destination $env:_ASPNETCORE_DEST_DIR
+
+# COPY ./bin/microsoft.netcore.app.ref /live-runtime-artifacts/microsoft.netcore.app.ref
+# COPY ./bin/microsoft.netcore.app.runtime.win-$ARCH /live-runtime-artifacts/microsoft.netcore.app.runtime.win-$ARCH
+# COPY ./targetingpacks.targets /live-runtime-artifacts/targetingpacks.targets
+# COPY ./bin/testhost /live-runtime-artifacts/testhost

eng/docker/libraries-sdk.linux.Dockerfile

@@ -3,26 +3,55 @@ ARG BUILD_BASE_IMAGE=mcr.microsoft.com/dotnet-buildtools/prereqs:centos-7-f39df2
 ARG SDK_BASE_IMAGE=mcr.microsoft.com/dotnet/nightly/sdk:6.0-bullseye-slim
 
 FROM $BUILD_BASE_IMAGE as corefxbuild
-
+ARG CONFIGURATION=Release
 WORKDIR /repo
 COPY . .
 
-ARG CONFIGURATION=Release
-RUN ./build.sh -ci -subset clr+libs -runtimeconfiguration release -c $CONFIGURATION
+RUN ./build.sh clr+libs -runtimeconfiguration Release -configuration $CONFIGURATION -ci
+
+# Mocks:
+# RUN mkdir -p /repo/artifacts/bin/testhost/net7.0-Linux-Release-x64
+# RUN echo 'lol' > /repo/artifacts/bin/testhost/lol.txt
+# 
+# RUN mkdir -p /repo/artifacts/bin/microsoft.netcore.app.ref
+# RUN echo 'rofl' > /repo/artifacts/bin/microsoft.netcore.app.ref/rofl.txt
+# 
+# RUN mkdir -p /repo/artifacts/bin/microsoft.netcore.app.runtime.linux-x64
+# RUN echo 'haha' > /repo/artifacts/bin/microsoft.netcore.app.runtime.linux-x64/haha.txt
 
 FROM $SDK_BASE_IMAGE as target
 
-ARG TESTHOST_LOCATION=/repo/artifacts/bin/testhost
-ARG TFM=net7.0
-ARG OS=Linux
+# Install 7.0 SDK:
+RUN wget https://dot.net/v1/dotnet-install.sh
+RUN bash ./dotnet-install.sh --channel 7.0.1xx --quality daily --install-dir /usr/share/dotnet
+
+## Collect the following artifacts under /live-runtime-artifacts,
+## so projects can build and test against the live-built runtime:
+## 1. Reference assembly pack (microsoft.netcore.app.ref)
+## 2. Runtime pack (microsoft.netcore.app.runtime.linux-x64)
+## 3. targetingpacks.targets, so stress test builds can target the live-built runtime instead of the one in the pre-installed SDK
+## 4. testhost
+
+ARG VERSION=7.0
 ARG ARCH=x64
 ARG CONFIGURATION=Release
 
-ARG COREFX_SHARED_FRAMEWORK_NAME=Microsoft.NETCore.App
-ARG SOURCE_COREFX_VERSION=7.0.0
-ARG TARGET_SHARED_FRAMEWORK=/usr/share/dotnet/shared
-ARG TARGET_COREFX_VERSION=$DOTNET_VERSION
+COPY --from=corefxbuild \
+    /repo/artifacts/bin/microsoft.netcore.app.ref \
+    /live-runtime-artifacts/microsoft.netcore.app.ref
+
+COPY --from=corefxbuild \
+    /repo/artifacts/bin/microsoft.netcore.app.runtime.linux-$ARCH \
+    /live-runtime-artifacts/microsoft.netcore.app.runtime.linux-$ARCH
 
 COPY --from=corefxbuild \
-    $TESTHOST_LOCATION/$TFM-$OS-$CONFIGURATION-$ARCH/shared/$COREFX_SHARED_FRAMEWORK_NAME/$SOURCE_COREFX_VERSION/* \
-    $TARGET_SHARED_FRAMEWORK/$COREFX_SHARED_FRAMEWORK_NAME/$TARGET_COREFX_VERSION/
+    /repo/eng/targetingpacks.targets \
+    /live-runtime-artifacts/targetingpacks.targets
+
+COPY --from=corefxbuild \
+    /repo/artifacts/bin/testhost \
+    /live-runtime-artifacts/testhost
+
+# Add AspNetCore bits to testhost:
+RUN mkdir -p /live-runtime-artifacts/testhost/net$VERSION-Linux-$CONFIGURATION-$ARCH/shared/Microsoft.AspNetCore.App
+RUN cp -r /usr/share/dotnet/shared/Microsoft.AspNetCore.App/$VERSION* /live-runtime-artifacts/testhost/net$VERSION-Linux-$CONFIGURATION-$ARCH/shared/Microsoft.AspNetCore.App

eng/pipelines/libraries/stress/http.yml

@@ -50,15 +50,15 @@ jobs:
     name: buildStress
     displayName: Build HttpStress
 
-  - bash: |
-      cd '$(httpStressProject)'
-      export CLIENT_DUMPS_SHARE="$(Build.ArtifactStagingDirectory)/dumps/client/3.0"
-      export SERVER_DUMPS_SHARE="$(Build.ArtifactStagingDirectory)/dumps/server/3.0"
-      export HTTPSTRESS_CLIENT_ARGS="$HTTPSTRESS_CLIENT_ARGS -http 3.0"
-      export HTTPSTRESS_SERVER_ARGS="$HTTPSTRESS_SERVER_ARGS -http 3.0"
-      docker-compose up --abort-on-container-exit --no-color
-    displayName: Run HttpStress - HTTP 3.0
-    condition: and(eq(variables['buildRuntime.succeeded'], 'true'), eq(variables['buildStress.succeeded'], 'true'))
+  # - bash: |
+  #     cd '$(httpStressProject)'
+  #     export CLIENT_DUMPS_SHARE="$(Build.ArtifactStagingDirectory)/dumps/client/3.0"
+  #     export SERVER_DUMPS_SHARE="$(Build.ArtifactStagingDirectory)/dumps/server/3.0"
+  #     export HTTPSTRESS_CLIENT_ARGS="$HTTPSTRESS_CLIENT_ARGS -http 3.0"
+  #     export HTTPSTRESS_SERVER_ARGS="$HTTPSTRESS_SERVER_ARGS -http 3.0"
+  #     docker-compose up --abort-on-container-exit --no-color
+  #   displayName: Run HttpStress - HTTP 3.0
+  #   condition: and(eq(variables['buildRuntime.succeeded'], 'true'), eq(variables['buildStress.succeeded'], 'true'))
 
   - bash: |
       cd '$(httpStressProject)'
@@ -149,4 +149,4 @@ jobs:
   - powershell: |
       Set-NetFirewallProfile -Profile Domain, Public, Private -Enabled True
     name: enableFirewall
-    displayName: Enable Firewall
+    displayName: Enable Firewall

src/libraries/System.Net.Http/ref/System.Net.Http.cs

@@ -51,6 +51,7 @@ public partial class HttpClient : System.Net.Http.HttpMessageInvoker
         public HttpClient() : base (default(System.Net.Http.HttpMessageHandler)) { }
         public HttpClient(System.Net.Http.HttpMessageHandler handler) : base (default(System.Net.Http.HttpMessageHandler)) { }
         public HttpClient(System.Net.Http.HttpMessageHandler handler, bool disposeHandler) : base (default(System.Net.Http.HttpMessageHandler)) { }
+        public static bool DummyPropertyToTestLiveBuild { get; set; }
         public System.Uri? BaseAddress { get { throw null; } set { } }
         public static System.Net.IWebProxy DefaultProxy { get { throw null; } set { } }
         public System.Net.Http.Headers.HttpRequestHeaders DefaultRequestHeaders { get { throw null; } }

src/libraries/System.Net.Http/src/System/Net/Http/HttpClient.cs

@@ -36,6 +36,9 @@ public partial class HttpClient : HttpMessageInvoker
         #endregion Fields
 
         #region Properties
+
+        public static bool DummyPropertyToTestLiveBuild { get; set; }
+
         public static IWebProxy DefaultProxy
         {
             get => LazyInitializer.EnsureInitialized(ref s_defaultProxy, () => SystemProxyInfo.Proxy);

src/libraries/System.Net.Http/tests/StressTests/HttpStress/Build-Local.ps1

@@ -0,0 +1,56 @@
+## This is a helper script for non-containerized local build and test execution.
+## It downloads and uses the daily SDK which contains the compatible AspNetCore bits.
+## Usage:
+## ./build-local.ps1 [StressConfiguration] [LibrariesConfiguration]
+
+$Version="7.0"
+$RepoRoot="$(git rev-parse --show-toplevel)"
+$DailyDotnetRoot= "./.dotnet-daily"
+
+$StressConfiguration = "Release"
+if (-not ([string]::IsNullOrEmpty($args[0]))) {
+    $StressConfiguration = $args[0]
+}
+
+$LibrariesConfiguration = "Release"
+if (-not ([string]::IsNullOrEmpty($args[1]))) {
+    $LibrariesConfiguration = $args[0]
+}
+
+Write-Host "StressConfiguration: $StressConfiguration, LibrariesConfiguration: $LibrariesConfiguration"
+
+if (-not (Test-Path -Path $DailyDotnetRoot)) {
+    Write-Host "Downloading daily SDK to: $DailyDotnetRoot"
+    New-Item -ItemType Directory -Path $DailyDotnetRoot
+    Invoke-WebRequest -Uri https://dot.net/v1/dotnet-install.ps1 -OutFile "$DailyDotnetRoot\dotnet-install.ps1"
+    & "$DailyDotnetRoot\dotnet-install.ps1" -NoPath -Channel "$Version.1xx" -Quality daily -InstallDir $DailyDotnetRoot
+}
+else {
+    Write-Host "Daily SDK found in $DailyDotnetRoot"
+}
+
+$TestHostRoot="$RepoRoot/artifacts/bin/testhost/net$Version-windows-$LibrariesConfiguration-x64"
+
+$env:DOTNET_ROOT=$DailyDotnetRoot
+$env:PATH="$DailyDotnetRoot;$env:PATH"
+$env:DOTNET_MULTILEVEL_LOOKUP=0
+
+if (-not (Test-Path -Path "$TestHostRoot/shared/Microsoft.AspNetCore.App")) {
+    Write-Host "Copying Microsoft.AspNetCore.App bits from daily SDK to testhost: $TestHostRoot"
+    Copy-Item -Recurse -Path "$DailyDotnetRoot/shared/Microsoft.AspNetCore.App" -Destination "$TestHostRoot/shared"
+}
+else {
+    Write-Host "Microsoft.AspNetCore.App found in testhost: $TestHostRoot"
+}
+
+Write-Host "Building solution."
+dotnet build -c $StressConfiguration
+
+$Runscript=".\run-stress-$LibrariesConfiguration-$StressConfiguration.ps1"
+if (-not (Test-Path $Runscript)) {
+    Write-Host "Generating Runscript."
+    Add-Content -Path $Runscript -Value "& '$TestHostRoot/dotnet' exec ./bin/$StressConfiguration/net$Version/HttpStress.dll `$args"
+}
+
+Write-Host "To run tests type:"
+Write-Host "$Runscript [stress test args]"

src/libraries/System.Net.Http/tests/StressTests/HttpStress/Directory.Build.props

@@ -1 +1,16 @@
-<Project/>
+<Project>
+    <PropertyGroup>
+        <PackageRid>linux-x64</PackageRid>
+        <PackageRid Condition="$([MSBuild]::IsOSPlatform('WINDOWS'))">win-x64</PackageRid>
+        <PackageRid Condition="$([MSBuild]::IsOSPlatform('OSX'))">osx-x64</PackageRid>
+
+        <TargetingPacksTargetsLocation Condition="'$(TargetingPacksTargetsLocation)' == ''">../../../../../../eng/targetingpacks.targets</TargetingPacksTargetsLocation>
+        <ProductVersion>7.0.0</ProductVersion>
+        <NetCoreAppCurrent>net7.0</NetCoreAppCurrent>
+        <NetCoreAppCurrentVersion>7.0</NetCoreAppCurrentVersion>
+        <MicrosoftNetCoreAppFrameworkName>Microsoft.NETCore.App</MicrosoftNetCoreAppFrameworkName>
+        <MicrosoftNetCoreAppRefPackDir Condition="'$(MicrosoftNetCoreAppRefPackDir)' == ''" >../../../../../../artifacts/bin/microsoft.netcore.app.ref/</MicrosoftNetCoreAppRefPackDir>
+        <MicrosoftNetCoreAppRuntimePackDir Condition="'$(MicrosoftNetCoreAppRuntimePackDir)' == ''">../../../../../../artifacts/bin/microsoft.netcore.app.runtime.$(PackageRid)/$(Configuration)/</MicrosoftNetCoreAppRuntimePackDir>
+    </PropertyGroup>
+</Project>
+

src/libraries/System.Net.Http/tests/StressTests/HttpStress/Directory.Build.targets

@@ -1 +1,11 @@
-<Project/>
+<Project>
+  <Import Project="$(TargetingPacksTargetsLocation)" />
+
+  <PropertyGroup>
+    <!--
+      Define this here because the SDK resets it
+      unconditionally in Microsoft.NETCoreSdk.BundledVersions.props.
+    -->
+    <NETCoreAppMaximumVersion>7.0</NETCoreAppMaximumVersion>
+  </PropertyGroup>
+</Project>