From f50900428226fdd9045a56bef1b4b065b1c23ffb Mon Sep 17 00:00:00 2001 From: Logan Bussell Date: Fri, 11 Nov 2022 17:38:13 +0000 Subject: [PATCH 1/4] Add CI for dotnet-security-partners Remove nuget configs from all repo submodules Address code review Move security-partners ci to tarball/eng dir Copy out installer NuGet config before build Fix relative path for installer template Switch to 20.04 image --- .../job/source-build-build-tarball.yml | 70 +++++++++++-------- .../tools/SourceBuildArcadeTarball.targets | 4 ++ .../pipelines/security-partners-dotnet.yml | 18 +++++ 3 files changed, 61 insertions(+), 31 deletions(-) create mode 100644 src/SourceBuild/tarball/content/eng/pipelines/security-partners-dotnet.yml diff --git a/src/SourceBuild/Arcade/eng/common/templates/job/source-build-build-tarball.yml b/src/SourceBuild/Arcade/eng/common/templates/job/source-build-build-tarball.yml index 9de76621fb08..9ae170d1c95c 100644 --- a/src/SourceBuild/Arcade/eng/common/templates/job/source-build-build-tarball.yml +++ b/src/SourceBuild/Arcade/eng/common/templates/job/source-build-build-tarball.yml @@ -43,7 +43,15 @@ jobs: - name: additionalBuildArgs value: --with-sdk /tarball/.dotnet - name: tarballDir - value: $(Build.StagingDirectory)/tarball + ${{ if eq(parameters.installerBuildResourceId, '') }}: + value: $(Build.SourcesDirectory) + ${{ else }}: + value: $(Build.StagingDirectory)/tarball + - name: installerSrcDir + ${{ if eq(parameters.installerBuildResourceId, '') }}: + value: $(Build.SourcesDirectory)/src/installer + ${{ else }}: + value: $(Build.SourcesDirectory) workspace: clean: all @@ -51,19 +59,28 @@ jobs: - checkout: self clean: true - - ${{ if ne(variables['System.TeamProject'], 'public') }}: - - task: Bash@3 - displayName: Setup Private Feeds Credentials - inputs: - filePath: $(Build.SourcesDirectory)/eng/common/SetupNugetSources.sh - arguments: $(Build.SourcesDirectory)/NuGet.config $Token - env: - Token: $(dn-bot-dnceng-artifact-feeds-rw) + - ${{ if ne(parameters.installerBuildResourceId, '') }}: + - download: ${{ parameters.installerBuildResourceId }} + artifact: BlobArtifacts + patterns: '**/dotnet-sdk-source*.tar.gz' + displayName: Download Source Tarball + + - script: | + set -x - - download: ${{ parameters.installerBuildResourceId }} - artifact: BlobArtifacts - patterns: '**/dotnet-sdk-source*.tar.gz' - displayName: Download Source Tarball + resourceIdPathSegment= + if [[ '${{ parameters.installerBuildResourceId }}' != 'current' ]]; then + resourceIdPathSegment='${{ parameters.installerBuildResourceId }}/' + fi + + mkdir -p "$(tarballDir)" + tarballFilePath="$(PIPELINE.WORKSPACE)/${resourceIdPathSegment}BlobArtifacts/dotnet-sdk-source*.tar.gz" + eval tar -ozxf "$tarballFilePath" -C "$(tarballDir)" + eval rm -f "$tarballFilePath" + displayName: Extract Tarball + + - script: cp $(installerSrcDir)/NuGet.config $(tarballDir)/test/Microsoft.DotNet.SourceBuild.SmokeTests/assets/online.NuGet.Config + displayName: Copy Test NuGet Config - ${{ if ne(parameters.excludeSdkContentTests, 'true') }}: - download: ${{ parameters.installerBuildResourceId }} @@ -84,20 +101,6 @@ jobs: Contents: '*.tar.gz' TargetFolder: $(tarballDir)/packages/archive/ - - script: | - set -x - - resourceIdPathSegment= - if [[ '${{ parameters.installerBuildResourceId }}' != 'current' ]]; then - resourceIdPathSegment='${{ parameters.installerBuildResourceId }}/' - fi - - mkdir -p "$(tarballDir)" - tarballFilePath="$(PIPELINE.WORKSPACE)/${resourceIdPathSegment}BlobArtifacts/dotnet-sdk-source*.tar.gz" - eval tar -ozxf "$tarballFilePath" -C "$(tarballDir)" - eval rm -f "$tarballFilePath" - displayName: Extract Tarball - - script: | set -x @@ -135,13 +138,18 @@ jobs: docker run --rm -v $(tarballDir):/tarball -w /tarball ${networkArgs} $(_Container) ./build.sh --clean-while-building ${customBuildArgs} $(additionalBuildArgs) displayName: Build Tarball + - ${{ if ne(variables['System.TeamProject'], 'public') }}: + - task: Bash@3 + displayName: Setup Private Feeds Credentials + inputs: + filePath: $(installerSrcDir)/eng/common/SetupNugetSources.sh + arguments: $(tarballDir)/test/Microsoft.DotNet.SourceBuild.SmokeTests/assets/online.NuGet.Config $Token + env: + Token: $(dn-bot-dnceng-artifact-feeds-rw) + - script: | set -x - # Use installer repo's NuGet.config during online testing to utilize internal feeds - rm -f $(tarballDir)/test/Microsoft.DotNet.SourceBuild.SmokeTests/assets/online.NuGet.Config - cp $(Build.SourcesDirectory)/NuGet.config $(tarballDir)/test/Microsoft.DotNet.SourceBuild.SmokeTests/assets/online.NuGet.Config - dockerVolumeArgs="-v $(tarballDir):/tarball" dockerEnvArgs="-e SMOKE_TESTS_EXCLUDE_OMNISHARP=$(_ExcludeOmniSharpTests) -e SMOKE_TESTS_WARN_SDK_CONTENT_DIFFS=true" poisonArg='' diff --git a/src/SourceBuild/Arcade/tools/SourceBuildArcadeTarball.targets b/src/SourceBuild/Arcade/tools/SourceBuildArcadeTarball.targets index e7a9698c658d..7c82620b667c 100644 --- a/src/SourceBuild/Arcade/tools/SourceBuildArcadeTarball.targets +++ b/src/SourceBuild/Arcade/tools/SourceBuildArcadeTarball.targets @@ -176,6 +176,10 @@ Command="git submodule update --init --recursive --depth 1" WorkingDirectory="$(TarballRepoSourceDir)" /> + + Date: Thu, 17 Nov 2022 01:55:55 +0000 Subject: [PATCH 2/4] Add nuget-client patch to avoid nuget security scan errors --- ...01-Remove-package-feed-with-upstream.patch | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 src/SourceBuild/tarball/patches/nuget-client/0001-Remove-package-feed-with-upstream.patch diff --git a/src/SourceBuild/tarball/patches/nuget-client/0001-Remove-package-feed-with-upstream.patch b/src/SourceBuild/tarball/patches/nuget-client/0001-Remove-package-feed-with-upstream.patch new file mode 100644 index 000000000000..ae7222d5ceb9 --- /dev/null +++ b/src/SourceBuild/tarball/patches/nuget-client/0001-Remove-package-feed-with-upstream.patch @@ -0,0 +1,21 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: Logan Bussell +Date: Thu, 17 Nov 2022 01:38:04 +0000 +Subject: [PATCH] Remove package feed with upstream + +--- + NuGet.Config | 1 - + 1 file changed, 1 deletion(-) + +diff --git a/NuGet.Config b/NuGet.Config +index 5b91c9af1..7b913598f 100644 +--- a/NuGet.Config ++++ b/NuGet.Config +@@ -4,7 +4,6 @@ + + + +- + + + From f3d0faa25e5fd5fda1a18b5c3ea356bd653944ad Mon Sep 17 00:00:00 2001 From: Logan Bussell Date: Thu, 17 Nov 2022 10:46:29 -0800 Subject: [PATCH 3/4] Revert "Add nuget-client patch to avoid nuget security scan errors" This reverts commit 6eca00a41813171019b24597386a9074e2faa07b. --- ...01-Remove-package-feed-with-upstream.patch | 21 ------------------- 1 file changed, 21 deletions(-) delete mode 100644 src/SourceBuild/tarball/patches/nuget-client/0001-Remove-package-feed-with-upstream.patch diff --git a/src/SourceBuild/tarball/patches/nuget-client/0001-Remove-package-feed-with-upstream.patch b/src/SourceBuild/tarball/patches/nuget-client/0001-Remove-package-feed-with-upstream.patch deleted file mode 100644 index ae7222d5ceb9..000000000000 --- a/src/SourceBuild/tarball/patches/nuget-client/0001-Remove-package-feed-with-upstream.patch +++ /dev/null @@ -1,21 +0,0 @@ -From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 -From: Logan Bussell -Date: Thu, 17 Nov 2022 01:38:04 +0000 -Subject: [PATCH] Remove package feed with upstream - ---- - NuGet.Config | 1 - - 1 file changed, 1 deletion(-) - -diff --git a/NuGet.Config b/NuGet.Config -index 5b91c9af1..7b913598f 100644 ---- a/NuGet.Config -+++ b/NuGet.Config -@@ -4,7 +4,6 @@ - - - -- - - - From e93a096444467a35de56db311dfb35798598baa6 Mon Sep 17 00:00:00 2001 From: Logan Bussell Date: Thu, 17 Nov 2022 10:47:17 -0800 Subject: [PATCH 4/4] Remove NuGet.config workaround for security-partners ci --- src/SourceBuild/Arcade/tools/SourceBuildArcadeTarball.targets | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/SourceBuild/Arcade/tools/SourceBuildArcadeTarball.targets b/src/SourceBuild/Arcade/tools/SourceBuildArcadeTarball.targets index 7c82620b667c..e7a9698c658d 100644 --- a/src/SourceBuild/Arcade/tools/SourceBuildArcadeTarball.targets +++ b/src/SourceBuild/Arcade/tools/SourceBuildArcadeTarball.targets @@ -176,10 +176,6 @@ Command="git submodule update --init --recursive --depth 1" WorkingDirectory="$(TarballRepoSourceDir)" /> - -