Build HttpStress and SslStress with live-built runtime using current TFM

.gitignore

@@ -354,3 +354,7 @@ src/coreclr/System.Private.CoreLib/common
 !src/coreclr/inc/obj/
 !src/coreclr/vm/.vscode/
 !src/coreclr/vm/.vscode/c_cpp_properties.json
+
+# Temporary artifacts from local libraries stress builds
+.dotnet-daily/
+run-stress-*

eng/docker/build-docker-sdk.ps1

@@ -6,21 +6,16 @@
 Param(
   [string][Alias('t')]$imageName = "dotnet-sdk-libs-current",
   [string][Alias('c')]$configuration = "Release",
-  [switch][Alias('w')]$buildWindowsContainers,
-  [switch][Alias('pa')]$privateAspNetCore
+  [switch][Alias('w')]$buildWindowsContainers
 )
 
+$dotNetVersion="7.0"
 $ErrorActionPreference = "Stop"
 
 $REPO_ROOT_DIR=$(git -C "$PSScriptRoot" rev-parse --show-toplevel)
 
 $dockerFilePrefix="$PSScriptRoot/libraries-sdk"
 
-if ($privateAspNetCore)
-{
-  $dockerFilePrefix="$PSScriptRoot/libraries-sdk-aspnetcore"
-}
-
 if ($buildWindowsContainers)
 {
   # Due to size concerns, we don't currently do docker builds on windows.
@@ -34,12 +29,35 @@ if ($buildWindowsContainers)
   }
 
   $dockerFile="$dockerFilePrefix.windows.Dockerfile"
+
+  # Collect the following artifacts to folder, that will be used as build context for the container,
+  # so projects can build and test against the live-built runtime:
+  # 1. Reference assembly pack (microsoft.netcore.app.ref)
+  # 2. Runtime pack (microsoft.netcore.app.runtime.linux-x64)
+  # 3. targetingpacks.targets, so stress test builds can target the live-built runtime instead of the one in the pre-installed SDK
+  # 4. testhost
+  $binArtifacts = "$REPO_ROOT_DIR\artifacts\bin"
+  $dockerContext = "$REPO_ROOT_DIR\artifacts\docker-context"
 
+  if (Test-Path $dockerContext) {
+      Remove-Item -Recurse -Force $dockerContext
+  }
+
+  Copy-Item -Recurse -Destination $dockerContext\microsoft.netcore.app.ref -Path $binArtifacts\microsoft.netcore.app.ref
+  Copy-Item -Recurse -Destination $dockerContext\microsoft.netcore.app.runtime.win-x64 -Path $binArtifacts\microsoft.netcore.app.runtime.win-x64
+  Copy-Item -Recurse -Destination $dockerContext\testhost -Path $binArtifacts\testhost
+  Copy-Item -Recurse -Destination $dockerContext\targetingpacks.targets -Path $REPO_ROOT_DIR\eng\targetingpacks.targets
+
+  # In case of non-CI builds, testhost may already contain Microsoft.AspNetCore.App (see Build-Local.ps1 in HttpStress):
+  $testHostAspNetCorePath="$dockerContext\testhost\net$dotNetVersion-windows-$configuration-x64/shared/Microsoft.AspNetCore.App"
+  if (Test-Path $testHostAspNetCorePath) {
+    Remove-Item -Recurse -Force $testHostAspNetCorePath
+  }
+
   docker build --tag $imageName `
     --build-arg CONFIGURATION=$configuration `
-    --build-arg TESTHOST_LOCATION=. `
     --file $dockerFile `
-    "$REPO_ROOT_DIR/artifacts/bin/testhost"
+    $dockerContext
 }
 else
 {

eng/docker/build-docker-sdk.sh

@@ -23,7 +23,6 @@ scriptroot="$( cd -P "$( dirname "$source" )" && pwd )"
 
 imagename="dotnet-sdk-libs-current"
 configuration="Release"
-privateaspnetcore=0
 
 while [[ $# > 0 ]]; do
   opt="$(echo "${1/#--/-}" | tr "[:upper:]" "[:lower:]")"
@@ -36,10 +35,6 @@ while [[ $# > 0 ]]; do
       configuration=$2
       shift 2
       ;;
-    -privateaspnetcore|-pa)
-      privateaspnetcore=1
-      shift 1
-      ;;
     *)
       shift 1
       ;;
@@ -49,13 +44,9 @@ done
 repo_root=$(git rev-parse --show-toplevel)
 docker_file="$scriptroot/libraries-sdk.linux.Dockerfile"
 
-if [[ $privateaspnetcore -eq 1 ]]; then
-    docker_file="$scriptroot/libraries-sdk-aspnetcore.linux.Dockerfile"
-fi
-
 docker build --tag $imagename \
     --build-arg CONFIGURATION=$configuration \
     --file $docker_file \
     $repo_root
 
-exit $?
+exit $?

eng/docker/libraries-sdk.linux.Dockerfile

@@ -3,26 +3,45 @@ ARG BUILD_BASE_IMAGE=mcr.microsoft.com/dotnet-buildtools/prereqs:centos-7-f39df2
 ARG SDK_BASE_IMAGE=mcr.microsoft.com/dotnet/nightly/sdk:6.0-bullseye-slim
 
 FROM $BUILD_BASE_IMAGE as corefxbuild
-
+ARG CONFIGURATION=Release
 WORKDIR /repo
 COPY . .
 
-ARG CONFIGURATION=Release
-RUN ./build.sh -ci -subset clr+libs -runtimeconfiguration release -c $CONFIGURATION
+RUN ./build.sh clr+libs -runtimeconfiguration Release -configuration $CONFIGURATION -ci
 
 FROM $SDK_BASE_IMAGE as target
 
-ARG TESTHOST_LOCATION=/repo/artifacts/bin/testhost
-ARG TFM=net7.0
-ARG OS=Linux
+# Install 7.0 SDK:
+RUN wget https://dot.net/v1/dotnet-install.sh
+RUN bash ./dotnet-install.sh --channel 7.0.1xx --quality daily --install-dir /usr/share/dotnet
+
+# Collect the following artifacts under /live-runtime-artifacts,
+# so projects can build and test against the live-built runtime:
+# 1. Reference assembly pack (microsoft.netcore.app.ref)
+# 2. Runtime pack (microsoft.netcore.app.runtime.linux-x64)
+# 3. targetingpacks.targets, so stress test builds can target the live-built runtime instead of the one in the pre-installed SDK
+# 4. testhost
+
+ARG VERSION=7.0
 ARG ARCH=x64
 ARG CONFIGURATION=Release
 
-ARG COREFX_SHARED_FRAMEWORK_NAME=Microsoft.NETCore.App
-ARG SOURCE_COREFX_VERSION=7.0.0
-ARG TARGET_SHARED_FRAMEWORK=/usr/share/dotnet/shared
-ARG TARGET_COREFX_VERSION=$DOTNET_VERSION
+COPY --from=corefxbuild \
+    /repo/artifacts/bin/microsoft.netcore.app.ref \
+    /live-runtime-artifacts/microsoft.netcore.app.ref
 
 COPY --from=corefxbuild \
-    $TESTHOST_LOCATION/$TFM-$OS-$CONFIGURATION-$ARCH/shared/$COREFX_SHARED_FRAMEWORK_NAME/$SOURCE_COREFX_VERSION/* \
-    $TARGET_SHARED_FRAMEWORK/$COREFX_SHARED_FRAMEWORK_NAME/$TARGET_COREFX_VERSION/
+    /repo/artifacts/bin/microsoft.netcore.app.runtime.linux-$ARCH \
+    /live-runtime-artifacts/microsoft.netcore.app.runtime.linux-$ARCH
+
+COPY --from=corefxbuild \
+    /repo/eng/targetingpacks.targets \
+    /live-runtime-artifacts/targetingpacks.targets
+
+COPY --from=corefxbuild \
+    /repo/artifacts/bin/testhost \
+    /live-runtime-artifacts/testhost
+
+# Add AspNetCore bits to testhost:
+RUN mkdir -p /live-runtime-artifacts/testhost/net$VERSION-Linux-$CONFIGURATION-$ARCH/shared/Microsoft.AspNetCore.App
+RUN cp -r /usr/share/dotnet/shared/Microsoft.AspNetCore.App/$VERSION* /live-runtime-artifacts/testhost/net$VERSION-Linux-$CONFIGURATION-$ARCH/shared/Microsoft.AspNetCore.App

eng/docker/libraries-sdk.windows.Dockerfile

@@ -3,17 +3,21 @@
 ARG SDK_BASE_IMAGE=mcr.microsoft.com/dotnet/nightly/sdk:6.0-nanoserver-1809
 FROM $SDK_BASE_IMAGE as target
 
-ARG TESTHOST_LOCATION=".\\artifacts\\bin\\testhost"
-ARG TFM=net7.0
-ARG OS=windows
-ARG ARCH=x64
-ARG CONFIGURATION=Release
+SHELL ["pwsh", "-Command", "$ErrorActionPreference = 'Stop'; $ProgressPreference = 'SilentlyContinue';"]
+
+USER ContainerAdministrator
+
+RUN Invoke-WebRequest -Uri https://dot.net/v1/dotnet-install.ps1 -OutFile .\dotnet-install.ps1
+RUN & .\dotnet-install.ps1 -Channel 7.0.1xx -Quality daily -InstallDir 'C:/Program Files/dotnet'
 
-ARG COREFX_SHARED_FRAMEWORK_NAME=Microsoft.NETCore.App
-ARG SOURCE_COREFX_VERSION=7.0.0
-ARG TARGET_SHARED_FRAMEWORK="C:\\Program Files\\dotnet\\shared"
-ARG TARGET_COREFX_VERSION=$DOTNET_VERSION
+USER ContainerUser
+
+COPY . /live-runtime-artifacts
+
+ARG VERSION=7.0
+ARG CONFIGURATION=Release
 
-COPY `
-    $TESTHOST_LOCATION\$TFM-$OS-$CONFIGURATION-$ARCH\shared\$COREFX_SHARED_FRAMEWORK_NAME\$SOURCE_COREFX_VERSION\ `
-    $TARGET_SHARED_FRAMEWORK\$COREFX_SHARED_FRAMEWORK_NAME\$TARGET_COREFX_VERSION\
+ENV _ASPNETCORE_SOURCE_DIR="C:/Program Files/dotnet/shared/Microsoft.AspNetCore.App/$VERSION*"
+ENV _ASPNETCORE_DEST_DIR="C:/live-runtime-artifacts/testhost/net$VERSION-windows-$CONFIGURATION-x64/shared/Microsoft.AspNetCore.App"
+RUN & New-Item -ItemType Directory -Path $env:_ASPNETCORE_DEST_DIR
+RUN Copy-Item -Recurse -Path $env:_ASPNETCORE_SOURCE_DIR -Destination $env:_ASPNETCORE_DEST_DIR

eng/pipelines/libraries/stress/http.yml

@@ -149,4 +149,4 @@ jobs:
   - powershell: |
       Set-NetFirewallProfile -Profile Domain, Public, Private -Enabled True
     name: enableFirewall
-    displayName: Enable Firewall
+    displayName: Enable Firewall

src/libraries/System.Net.Http/tests/StressTests/HttpStress/Build-Local.ps1

@@ -0,0 +1,56 @@
+## This is a helper script for non-containerized local build and test execution.
+## It downloads and uses the daily SDK which contains the compatible AspNetCore bits.
+## Usage:
+## ./build-local.ps1 [StressConfiguration] [LibrariesConfiguration]
+
+$Version="7.0"
+$RepoRoot="$(git rev-parse --show-toplevel)"
+$DailyDotnetRoot= "./.dotnet-daily"
+
+$StressConfiguration = "Release"
+if (-not ([string]::IsNullOrEmpty($args[0]))) {
+    $StressConfiguration = $args[0]
+}
+
+$LibrariesConfiguration = "Release"
+if (-not ([string]::IsNullOrEmpty($args[1]))) {
+    $LibrariesConfiguration = $args[0]
+}
+
+Write-Host "StressConfiguration: $StressConfiguration, LibrariesConfiguration: $LibrariesConfiguration"
+
+if (-not (Test-Path -Path $DailyDotnetRoot)) {
+    Write-Host "Downloading daily SDK to: $DailyDotnetRoot"
+    New-Item -ItemType Directory -Path $DailyDotnetRoot
+    Invoke-WebRequest -Uri https://dot.net/v1/dotnet-install.ps1 -OutFile "$DailyDotnetRoot\dotnet-install.ps1"
+    & "$DailyDotnetRoot\dotnet-install.ps1" -NoPath -Channel "$Version.1xx" -Quality daily -InstallDir $DailyDotnetRoot
+}
+else {
+    Write-Host "Daily SDK found in $DailyDotnetRoot"
+}
+
+$TestHostRoot="$RepoRoot/artifacts/bin/testhost/net$Version-windows-$LibrariesConfiguration-x64"
+
+$env:DOTNET_ROOT=$DailyDotnetRoot
+$env:PATH="$DailyDotnetRoot;$env:PATH"
+$env:DOTNET_MULTILEVEL_LOOKUP=0
+
+if (-not (Test-Path -Path "$TestHostRoot/shared/Microsoft.AspNetCore.App")) {
+    Write-Host "Copying Microsoft.AspNetCore.App bits from daily SDK to testhost: $TestHostRoot"
+    Copy-Item -Recurse -Path "$DailyDotnetRoot/shared/Microsoft.AspNetCore.App" -Destination "$TestHostRoot/shared"
+}
+else {
+    Write-Host "Microsoft.AspNetCore.App found in testhost: $TestHostRoot"
+}
+
+Write-Host "Building solution."
+dotnet build -c $StressConfiguration
+
+$Runscript=".\run-stress-$LibrariesConfiguration-$StressConfiguration.ps1"
+if (-not (Test-Path $Runscript)) {
+    Write-Host "Generating Runscript."
+    Add-Content -Path $Runscript -Value "& '$TestHostRoot/dotnet' exec ./bin/$StressConfiguration/net$Version/HttpStress.dll `$args"
+}
+
+Write-Host "To run tests type:"
+Write-Host "$Runscript [stress test args]"

src/libraries/System.Net.Http/tests/StressTests/HttpStress/Directory.Build.props

@@ -1 +1,15 @@
-<Project/>
+<Project>
+    <PropertyGroup>
+        <PackageRid>linux-x64</PackageRid>
+        <PackageRid Condition="$([MSBuild]::IsOSPlatform('WINDOWS'))">win-x64</PackageRid>
+        <PackageRid Condition="$([MSBuild]::IsOSPlatform('OSX'))">osx-x64</PackageRid>
+
+        <TargetingPacksTargetsLocation Condition="'$(TargetingPacksTargetsLocation)' == ''">../../../../../../eng/targetingpacks.targets</TargetingPacksTargetsLocation>
+        <ProductVersion>7.0.0</ProductVersion>
+        <NetCoreAppCurrent>net7.0</NetCoreAppCurrent>
+        <NetCoreAppCurrentVersion>7.0</NetCoreAppCurrentVersion>
+        <MicrosoftNetCoreAppFrameworkName>Microsoft.NETCore.App</MicrosoftNetCoreAppFrameworkName>
+        <MicrosoftNetCoreAppRefPackDir Condition="'$(MicrosoftNetCoreAppRefPackDir)' == ''" >../../../../../../artifacts/bin/microsoft.netcore.app.ref/</MicrosoftNetCoreAppRefPackDir>
+        <MicrosoftNetCoreAppRuntimePackDir Condition="'$(MicrosoftNetCoreAppRuntimePackDir)' == ''">../../../../../../artifacts/bin/microsoft.netcore.app.runtime.$(PackageRid)/$(Configuration)/</MicrosoftNetCoreAppRuntimePackDir>
+    </PropertyGroup>
+</Project>

src/libraries/System.Net.Http/tests/StressTests/HttpStress/Directory.Build.targets

@@ -1 +1,11 @@
-<Project/>
+<Project>
+  <Import Project="$(TargetingPacksTargetsLocation)" />
+
+  <PropertyGroup>
+    <!--
+      Define this here because the SDK resets it
+      unconditionally in Microsoft.NETCoreSdk.BundledVersions.props.
+    -->
+    <NETCoreAppMaximumVersion>7.0</NETCoreAppMaximumVersion>
+  </PropertyGroup>
+</Project>

src/libraries/System.Net.Http/tests/StressTests/HttpStress/Dockerfile

@@ -1,9 +1,6 @@
 ARG SDK_BASE_IMAGE=mcr.microsoft.com/dotnet/nightly/sdk:6.0-bullseye-slim
 FROM $SDK_BASE_IMAGE
 
-RUN echo "DOTNET_SDK_VERSION="$DOTNET_SDK_VERSION
-RUN echo "DOTNET_VERSION="$DOTNET_VERSION
-
 WORKDIR /app
 COPY . .
 
@@ -19,8 +16,13 @@ RUN unzip $PACKAGES_DIR.zip
 RUN dpkg -i $PACKAGES_DIR/$MSQUIC_PACKAGE
 RUN rm -rf $PACKAGES_DIR*
 
+ARG VERSION=7.0
 ARG CONFIGURATION=Release
-RUN dotnet build -c $CONFIGURATION
+
+RUN dotnet build -c $CONFIGURATION \
+    -p:TargetingPacksTargetsLocation=/live-runtime-artifacts/targetingpacks.targets \
+    -p:MicrosoftNetCoreAppRefPackDir=/live-runtime-artifacts/microsoft.netcore.app.ref/ \
+    -p:MicrosoftNetCoreAppRuntimePackDir=/live-runtime-artifacts/microsoft.netcore.app.runtime.linux-x64/$CONFIGURATION/
 
 # Enable dump collection
 ENV COMPlus_DbgEnableMiniDump=1
@@ -29,6 +31,8 @@ ENV COMPlus_DbgMiniDumpName="/share/coredump.%p"
 
 EXPOSE 5001
 
+ENV VERSION=$VERSION
 ENV CONFIGURATION=$CONFIGURATION
 ENV HTTPSTRESS_ARGS=''
-CMD dotnet run --no-build -c $CONFIGURATION -- $HTTPSTRESS_ARGS
+CMD /live-runtime-artifacts/testhost/net$VERSION-Linux-$CONFIGURATION-x64/dotnet exec \
+    ./bin/$CONFIGURATION/net$VERSION/HttpStress.dll $HTTPSTRESS_ARGS