From 882749267fda873edab89aff50194835572baafe Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" Date: Mon, 9 Aug 2021 12:11:58 +0000 Subject: [PATCH 1/3] Update dependencies from https://github.com/dotnet/arcade build 20210806.6 Microsoft.DotNet.Arcade.Sdk From Version 6.0.0-beta.21379.2 -> To Version 6.0.0-beta.21406.6 --- eng/Version.Details.xml | 4 +- eng/common/cross/arm/sources.list.trusty | 11 -- eng/common/cross/arm/trusty-lttng-2.4.patch | 71 ---------- eng/common/cross/arm/trusty.patch | 97 ------------- eng/common/cross/arm64/sources.list.trusty | 11 -- eng/common/cross/build-rootfs.sh | 36 +++-- eng/common/cross/x86/sources.list.trusty | 11 -- eng/common/internal/Tools.csproj | 1 - .../templates/job/publish-build-assets.yml | 26 +++- eng/common/templates/phases/base.yml | 130 ------------------ .../templates/phases/publish-build-assets.yml | 52 ------- eng/common/templates/steps/source-build.yml | 35 ++++- global.json | 4 +- 13 files changed, 76 insertions(+), 413 deletions(-) delete mode 100644 eng/common/cross/arm/sources.list.trusty delete mode 100644 eng/common/cross/arm/trusty-lttng-2.4.patch delete mode 100644 eng/common/cross/arm/trusty.patch delete mode 100644 eng/common/cross/arm64/sources.list.trusty delete mode 100644 eng/common/cross/x86/sources.list.trusty delete mode 100644 eng/common/templates/phases/base.yml delete mode 100644 eng/common/templates/phases/publish-build-assets.yml diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 0b89c2ef..53a98e40 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -3,9 +3,9 @@ - + https://github.com/dotnet/arcade - 62a8aafffd4c68ef887680f6837abdff906a662c + 382667fff0b58c362855a42c3529ba294fd0514c https://github.com/dotnet/source-build-reference-packages diff --git a/eng/common/cross/arm/sources.list.trusty b/eng/common/cross/arm/sources.list.trusty deleted file mode 100644 index 07d8f88d..00000000 --- a/eng/common/cross/arm/sources.list.trusty +++ /dev/null @@ -1,11 +0,0 @@ -deb http://ports.ubuntu.com/ubuntu-ports/ trusty main restricted universe -deb-src http://ports.ubuntu.com/ubuntu-ports/ trusty main restricted universe - -deb http://ports.ubuntu.com/ubuntu-ports/ trusty-updates main restricted universe -deb-src http://ports.ubuntu.com/ubuntu-ports/ trusty-updates main restricted universe - -deb http://ports.ubuntu.com/ubuntu-ports/ trusty-backports main restricted -deb-src http://ports.ubuntu.com/ubuntu-ports/ trusty-backports main restricted - -deb http://ports.ubuntu.com/ubuntu-ports/ trusty-security main restricted universe multiverse -deb-src http://ports.ubuntu.com/ubuntu-ports/ trusty-security main restricted universe multiverse \ No newline at end of file diff --git a/eng/common/cross/arm/trusty-lttng-2.4.patch b/eng/common/cross/arm/trusty-lttng-2.4.patch deleted file mode 100644 index 8e4dd7ae..00000000 --- a/eng/common/cross/arm/trusty-lttng-2.4.patch +++ /dev/null @@ -1,71 +0,0 @@ -From e72c9d7ead60e3317bd6d1fade995c07021c947b Mon Sep 17 00:00:00 2001 -From: Mathieu Desnoyers -Date: Thu, 7 May 2015 13:25:04 -0400 -Subject: [PATCH] Fix: building probe providers with C++ compiler - -Robert Daniels wrote: -> > I'm attempting to use lttng userspace tracing with a C++ application -> > on an ARM platform. I'm using GCC 4.8.4 on Linux 3.14 with the 2.6 -> > release of lttng. I've compiled lttng-modules, lttng-ust, and -> > lttng-tools and have been able to get a simple test working with C -> > code. When I attempt to run the hello.cxx test on my target it will -> > segfault. -> -> -> I spent a little time digging into this issue and finally discovered the -> cause of my segfault with ARM C++ tracepoints. -> -> There is a struct called 'lttng_event' in ust-events.h which contains an -> empty union 'u'. This was the cause of my issue. Under C, this empty union -> compiles to a zero byte member while under C++ it compiles to a one byte -> member, and in my case was four-byte aligned which caused my C++ code to -> have the 'cds_list_head node' offset incorrectly by four bytes. This lead -> to an incorrect linked list structure which caused my issue. -> -> Since this union is empty, I simply removed it from the struct and everything -> worked correctly. -> -> I don't know the history or purpose behind this empty union so I'd like to -> know if this is a safe fix. If it is I can submit a patch with the union -> removed. - -That's a very nice catch! - -We do not support building tracepoint probe provider with -g++ yet, as stated in lttng-ust(3): - -"- Note for C++ support: although an application instrumented with - tracepoints can be compiled with g++, tracepoint probes should be - compiled with gcc (only tested with gcc so far)." - -However, if it works fine with this fix, then I'm tempted to take it, -especially because removing the empty union does not appear to affect -the layout of struct lttng_event as seen from liblttng-ust, which must -be compiled with a C compiler, and from probe providers compiled with -a C compiler. So all we are changing is the layout of a probe provider -compiled with a C++ compiler, which is anyway buggy at the moment, -because it is not compatible with the layout expected by liblttng-ust -compiled with a C compiler. - -Reported-by: Robert Daniels -Signed-off-by: Mathieu Desnoyers ---- - include/lttng/ust-events.h | 2 -- - 1 file changed, 2 deletions(-) - -diff --git a/usr/include/lttng/ust-events.h b/usr/include/lttng/ust-events.h -index 328a875..3d7a274 100644 ---- a/usr/include/lttng/ust-events.h -+++ b/usr/include/lttng/ust-events.h -@@ -407,8 +407,6 @@ struct lttng_event { - void *_deprecated1; - struct lttng_ctx *ctx; - enum lttng_ust_instrumentation instrumentation; -- union { -- } u; - struct cds_list_head node; /* Event list in session */ - struct cds_list_head _deprecated2; - void *_deprecated3; --- -2.7.4 - diff --git a/eng/common/cross/arm/trusty.patch b/eng/common/cross/arm/trusty.patch deleted file mode 100644 index 2f2972f8..00000000 --- a/eng/common/cross/arm/trusty.patch +++ /dev/null @@ -1,97 +0,0 @@ -diff -u -r a/usr/include/urcu/uatomic/generic.h b/usr/include/urcu/uatomic/generic.h ---- a/usr/include/urcu/uatomic/generic.h 2014-03-28 06:04:42.000000000 +0900 -+++ b/usr/include/urcu/uatomic/generic.h 2017-02-13 10:35:21.189927116 +0900 -@@ -65,17 +65,17 @@ - switch (len) { - #ifdef UATOMIC_HAS_ATOMIC_BYTE - case 1: -- return __sync_val_compare_and_swap_1(addr, old, _new); -+ return __sync_val_compare_and_swap_1((uint8_t *) addr, old, _new); - #endif - #ifdef UATOMIC_HAS_ATOMIC_SHORT - case 2: -- return __sync_val_compare_and_swap_2(addr, old, _new); -+ return __sync_val_compare_and_swap_2((uint16_t *) addr, old, _new); - #endif - case 4: -- return __sync_val_compare_and_swap_4(addr, old, _new); -+ return __sync_val_compare_and_swap_4((uint32_t *) addr, old, _new); - #if (CAA_BITS_PER_LONG == 64) - case 8: -- return __sync_val_compare_and_swap_8(addr, old, _new); -+ return __sync_val_compare_and_swap_8((uint64_t *) addr, old, _new); - #endif - } - _uatomic_link_error(); -@@ -100,20 +100,20 @@ - switch (len) { - #ifdef UATOMIC_HAS_ATOMIC_BYTE - case 1: -- __sync_and_and_fetch_1(addr, val); -+ __sync_and_and_fetch_1((uint8_t *) addr, val); - return; - #endif - #ifdef UATOMIC_HAS_ATOMIC_SHORT - case 2: -- __sync_and_and_fetch_2(addr, val); -+ __sync_and_and_fetch_2((uint16_t *) addr, val); - return; - #endif - case 4: -- __sync_and_and_fetch_4(addr, val); -+ __sync_and_and_fetch_4((uint32_t *) addr, val); - return; - #if (CAA_BITS_PER_LONG == 64) - case 8: -- __sync_and_and_fetch_8(addr, val); -+ __sync_and_and_fetch_8((uint64_t *) addr, val); - return; - #endif - } -@@ -139,20 +139,20 @@ - switch (len) { - #ifdef UATOMIC_HAS_ATOMIC_BYTE - case 1: -- __sync_or_and_fetch_1(addr, val); -+ __sync_or_and_fetch_1((uint8_t *) addr, val); - return; - #endif - #ifdef UATOMIC_HAS_ATOMIC_SHORT - case 2: -- __sync_or_and_fetch_2(addr, val); -+ __sync_or_and_fetch_2((uint16_t *) addr, val); - return; - #endif - case 4: -- __sync_or_and_fetch_4(addr, val); -+ __sync_or_and_fetch_4((uint32_t *) addr, val); - return; - #if (CAA_BITS_PER_LONG == 64) - case 8: -- __sync_or_and_fetch_8(addr, val); -+ __sync_or_and_fetch_8((uint64_t *) addr, val); - return; - #endif - } -@@ -180,17 +180,17 @@ - switch (len) { - #ifdef UATOMIC_HAS_ATOMIC_BYTE - case 1: -- return __sync_add_and_fetch_1(addr, val); -+ return __sync_add_and_fetch_1((uint8_t *) addr, val); - #endif - #ifdef UATOMIC_HAS_ATOMIC_SHORT - case 2: -- return __sync_add_and_fetch_2(addr, val); -+ return __sync_add_and_fetch_2((uint16_t *) addr, val); - #endif - case 4: -- return __sync_add_and_fetch_4(addr, val); -+ return __sync_add_and_fetch_4((uint32_t *) addr, val); - #if (CAA_BITS_PER_LONG == 64) - case 8: -- return __sync_add_and_fetch_8(addr, val); -+ return __sync_add_and_fetch_8((uint64_t *) addr, val); - #endif - } - _uatomic_link_error(); diff --git a/eng/common/cross/arm64/sources.list.trusty b/eng/common/cross/arm64/sources.list.trusty deleted file mode 100644 index 07d8f88d..00000000 --- a/eng/common/cross/arm64/sources.list.trusty +++ /dev/null @@ -1,11 +0,0 @@ -deb http://ports.ubuntu.com/ubuntu-ports/ trusty main restricted universe -deb-src http://ports.ubuntu.com/ubuntu-ports/ trusty main restricted universe - -deb http://ports.ubuntu.com/ubuntu-ports/ trusty-updates main restricted universe -deb-src http://ports.ubuntu.com/ubuntu-ports/ trusty-updates main restricted universe - -deb http://ports.ubuntu.com/ubuntu-ports/ trusty-backports main restricted -deb-src http://ports.ubuntu.com/ubuntu-ports/ trusty-backports main restricted - -deb http://ports.ubuntu.com/ubuntu-ports/ trusty-security main restricted universe multiverse -deb-src http://ports.ubuntu.com/ubuntu-ports/ trusty-security main restricted universe multiverse \ No newline at end of file diff --git a/eng/common/cross/build-rootfs.sh b/eng/common/cross/build-rootfs.sh index 735a4c82..5c05b39f 100755 --- a/eng/common/cross/build-rootfs.sh +++ b/eng/common/cross/build-rootfs.sh @@ -6,10 +6,10 @@ usage() { echo "Usage: $0 [BuildArch] [CodeName] [lldbx.y] [--skipunmount] --rootfsdir ]" echo "BuildArch can be: arm(default), armel, arm64, x86" - echo "CodeName - optional, Code name for Linux, can be: trusty, xenial(default), zesty, bionic, alpine, alpine3.9 or alpine3.13. If BuildArch is armel, LinuxCodeName is jessie(default) or tizen." - echo " for FreeBSD can be: freebsd11 or freebsd12." + echo "CodeName - optional, Code name for Linux, can be: xenial(default), zesty, bionic, alpine, alpine3.9 or alpine3.13. If BuildArch is armel, LinuxCodeName is jessie(default) or tizen." + echo " for FreeBSD can be: freebsd11, freebsd12, freebsd13" echo " for illumos can be: illumos." - echo "lldbx.y - optional, LLDB version, can be: lldb3.9(default), lldb4.0, lldb5.0, lldb6.0 no-lldb. Ignored for alpine and FReeBSD" + echo "lldbx.y - optional, LLDB version, can be: lldb3.9(default), lldb4.0, lldb5.0, lldb6.0 no-lldb. Ignored for alpine and FreeBSD" echo "--skipunmount - optional, will skip the unmount of rootfs folder." echo "--use-mirror - optional, use mirror URL to fetch resources, when available." exit 1 @@ -60,13 +60,15 @@ __AlpinePackages+=" krb5-dev" __AlpinePackages+=" openssl-dev" __AlpinePackages+=" zlib-dev" -__FreeBSDBase="12.1-RELEASE" +__FreeBSDBase="12.2-RELEASE" __FreeBSDPkg="1.12.0" +__FreeBSDABI="12" __FreeBSDPackages="libunwind" __FreeBSDPackages+=" icu" __FreeBSDPackages+=" libinotify" __FreeBSDPackages+=" lttng-ust" __FreeBSDPackages+=" krb5" +__FreeBSDPackages+=" terminfo-db" __IllumosPackages="icu-64.2nb2" __IllumosPackages+=" mit-krb5-1.16.2nb4" @@ -144,11 +146,6 @@ while :; do no-lldb) unset __LLDB_Package ;; - trusty) # Ubuntu 14.04 - if [ "$__CodeName" != "jessie" ]; then - __CodeName=trusty - fi - ;; xenial) # Ubuntu 16.04 if [ "$__CodeName" != "jessie" ]; then __CodeName=xenial @@ -208,12 +205,20 @@ while :; do ;; freebsd11) __FreeBSDBase="11.3-RELEASE" + __FreeBSDABI="11" ;& freebsd12) __CodeName=freebsd __BuildArch=x64 __SkipUnmount=1 ;; + freebsd13) + __CodeName=freebsd + __FreeBSDBase="13.0-RELEASE" + __FreeBSDABI="13" + __BuildArch=x64 + __SkipUnmount=1 + ;; illumos) __CodeName=illumos __BuildArch=x64 @@ -291,9 +296,9 @@ if [[ "$__CodeName" == "alpine" ]]; then rm -r $__ApkToolsDir elif [[ "$__CodeName" == "freebsd" ]]; then mkdir -p $__RootfsDir/usr/local/etc + JOBS="$(getconf _NPROCESSORS_ONLN)" wget -O - https://download.freebsd.org/ftp/releases/amd64/${__FreeBSDBase}/base.txz | tar -C $__RootfsDir -Jxf - ./lib ./usr/lib ./usr/libdata ./usr/include ./usr/share/keys ./etc ./bin/freebsd-version - # For now, ask for 11 ABI even on 12. This can be revisited later. - echo "ABI = \"FreeBSD:11:amd64\"; FINGERPRINTS = \"${__RootfsDir}/usr/share/keys\"; REPOS_DIR = [\"${__RootfsDir}/etc/pkg\"]; REPO_AUTOUPDATE = NO; RUN_SCRIPTS = NO;" > ${__RootfsDir}/usr/local/etc/pkg.conf + echo "ABI = \"FreeBSD:${__FreeBSDABI}:amd64\"; FINGERPRINTS = \"${__RootfsDir}/usr/share/keys\"; REPOS_DIR = [\"${__RootfsDir}/etc/pkg\"]; REPO_AUTOUPDATE = NO; RUN_SCRIPTS = NO;" > ${__RootfsDir}/usr/local/etc/pkg.conf echo "FreeBSD: { url: "pkg+http://pkg.FreeBSD.org/\${ABI}/quarterly", mirror_type: \"srv\", signature_type: \"fingerprints\", fingerprints: \"${__RootfsDir}/usr/share/keys/pkg\", enabled: yes }" > ${__RootfsDir}/etc/pkg/FreeBSD.conf mkdir -p $__RootfsDir/tmp # get and build package manager @@ -301,7 +306,7 @@ elif [[ "$__CodeName" == "freebsd" ]]; then cd $__RootfsDir/tmp/pkg-${__FreeBSDPkg} # needed for install to succeed mkdir -p $__RootfsDir/host/etc - ./autogen.sh && ./configure --prefix=$__RootfsDir/host && make && make install + ./autogen.sh && ./configure --prefix=$__RootfsDir/host && make -j "$JOBS" && make install rm -rf $__RootfsDir/tmp/pkg-${__FreeBSDPkg} # install packages we need. INSTALL_AS_USER=$(whoami) $__RootfsDir/host/sbin/pkg -r $__RootfsDir -C $__RootfsDir/usr/local/etc/pkg.conf update @@ -364,13 +369,6 @@ elif [[ -n $__CodeName ]]; then umount $__RootfsDir/* || true fi - if [[ "$__BuildArch" == "arm" && "$__CodeName" == "trusty" ]]; then - pushd $__RootfsDir - patch -p1 < $__CrossDir/$__BuildArch/trusty.patch - patch -p1 < $__CrossDir/$__BuildArch/trusty-lttng-2.4.patch - popd - fi - if [[ "$__BuildArch" == "armel" && "$__CodeName" == "jessie" ]]; then pushd $__RootfsDir patch -p1 < $__CrossDir/$__BuildArch/armel.jessie.patch diff --git a/eng/common/cross/x86/sources.list.trusty b/eng/common/cross/x86/sources.list.trusty deleted file mode 100644 index 9b308543..00000000 --- a/eng/common/cross/x86/sources.list.trusty +++ /dev/null @@ -1,11 +0,0 @@ -deb http://archive.ubuntu.com/ubuntu/ trusty main restricted universe -deb-src http://archive.ubuntu.com/ubuntu/ trusty main restricted universe - -deb http://archive.ubuntu.com/ubuntu/ trusty-updates main restricted universe -deb-src http://archive.ubuntu.com/ubuntu/ trusty-updates main restricted universe - -deb http://archive.ubuntu.com/ubuntu/ trusty-backports main restricted -deb-src http://archive.ubuntu.com/ubuntu/ trusty-backports main restricted - -deb http://archive.ubuntu.com/ubuntu/ trusty-security main restricted universe multiverse -deb-src http://archive.ubuntu.com/ubuntu/ trusty-security main restricted universe multiverse diff --git a/eng/common/internal/Tools.csproj b/eng/common/internal/Tools.csproj index 2067b8df..beb9c464 100644 --- a/eng/common/internal/Tools.csproj +++ b/eng/common/internal/Tools.csproj @@ -1,5 +1,4 @@ - net472 diff --git a/eng/common/templates/job/publish-build-assets.yml b/eng/common/templates/job/publish-build-assets.yml index 3b9e2524..fe9dfdf7 100644 --- a/eng/common/templates/job/publish-build-assets.yml +++ b/eng/common/templates/job/publish-build-assets.yml @@ -94,7 +94,31 @@ jobs: PathtoPublish: '$(Build.StagingDirectory)/ReleaseConfigs.txt' PublishLocation: Container ArtifactName: ReleaseConfigs - + + - task: powershell@2 + displayName: Check if SymbolPublishingExclusionsFile.txt exists + inputs: + targetType: inline + script: | + $symbolExclusionfile = "$(Build.SourcesDirectory)/eng/SymbolPublishingExclusionsFile.txt" + if(Test-Path -Path $symbolExclusionfile) + { + Write-Host "SymbolExclusionFile exists" + Write-Host "##vso[task.setvariable variable=SymbolExclusionFile]true" + } + else{ + Write-Host "Symbols Exclusion file does not exists" + Write-Host "##vso[task.setvariable variable=SymbolExclusionFile]false" + } + + - task: PublishBuildArtifacts@1 + displayName: Publish SymbolPublishingExclusionsFile Artifact + condition: eq(variables['SymbolExclusionFile'], 'true') + inputs: + PathtoPublish: '$(Build.SourcesDirectory)/eng/SymbolPublishingExclusionsFile.txt' + PublishLocation: Container + ArtifactName: ReleaseConfigs + - ${{ if eq(parameters.enablePublishBuildArtifacts, 'true') }}: - template: /eng/common/templates/steps/publish-logs.yml parameters: diff --git a/eng/common/templates/phases/base.yml b/eng/common/templates/phases/base.yml deleted file mode 100644 index 0123cf43..00000000 --- a/eng/common/templates/phases/base.yml +++ /dev/null @@ -1,130 +0,0 @@ -parameters: - # Optional: Clean sources before building - clean: true - - # Optional: Git fetch depth - fetchDepth: '' - - # Optional: name of the phase (not specifying phase name may cause name collisions) - name: '' - # Optional: display name of the phase - displayName: '' - - # Optional: condition for the job to run - condition: '' - - # Optional: dependencies of the phase - dependsOn: '' - - # Required: A defined YAML queue - queue: {} - - # Required: build steps - steps: [] - - # Optional: variables - variables: {} - - # Optional: should run as a public build even in the internal project - # if 'true', the build won't run any of the internal only steps, even if it is running in non-public projects. - runAsPublic: false - - ## Telemetry variables - - # Optional: enable sending telemetry - # if 'true', these "variables" must be specified in the variables object or as part of the queue matrix - # _HelixBuildConfig - differentiate between Debug, Release, other - # _HelixSource - Example: build/product - # _HelixType - Example: official/dotnet/arcade/$(Build.SourceBranch) - enableTelemetry: false - - # Optional: Enable installing Microbuild plugin - # if 'true', these "variables" must be specified in the variables object or as part of the queue matrix - # _TeamName - the name of your team - # _SignType - 'test' or 'real' - enableMicrobuild: false - -# Internal resources (telemetry, microbuild) can only be accessed from non-public projects, -# and some (Microbuild) should only be applied to non-PR cases for internal builds. - -phases: -- phase: ${{ parameters.name }} - - ${{ if ne(parameters.displayName, '') }}: - displayName: ${{ parameters.displayName }} - - ${{ if ne(parameters.condition, '') }}: - condition: ${{ parameters.condition }} - - ${{ if ne(parameters.dependsOn, '') }}: - dependsOn: ${{ parameters.dependsOn }} - - queue: ${{ parameters.queue }} - - ${{ if ne(parameters.variables, '') }}: - variables: - ${{ insert }}: ${{ parameters.variables }} - - steps: - - checkout: self - clean: ${{ parameters.clean }} - ${{ if ne(parameters.fetchDepth, '') }}: - fetchDepth: ${{ parameters.fetchDepth }} - - - ${{ if eq(parameters.enableTelemetry, 'true') }}: - - template: /eng/common/templates/steps/telemetry-start.yml - parameters: - buildConfig: $(_HelixBuildConfig) - helixSource: $(_HelixSource) - helixType: $(_HelixType) - runAsPublic: ${{ parameters.runAsPublic }} - - - ${{ if eq(parameters.enableMicrobuild, 'true') }}: - # Internal only resource, and Microbuild signing shouldn't be applied to PRs. - - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - - task: MicroBuildSigningPlugin@2 - displayName: Install MicroBuild plugin - inputs: - signType: $(_SignType) - zipSources: false - feedSource: https://dnceng.pkgs.visualstudio.com/_packaging/MicroBuildToolset/nuget/v3/index.json - - env: - TeamName: $(_TeamName) - continueOnError: false - condition: and(succeeded(), in(variables['_SignType'], 'real', 'test'), eq(variables['Agent.Os'], 'Windows_NT')) - - # Run provided build steps - - ${{ parameters.steps }} - - - ${{ if eq(parameters.enableMicrobuild, 'true') }}: - # Internal only resources - - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - - task: MicroBuildCleanup@1 - displayName: Execute Microbuild cleanup tasks - condition: and(always(), in(variables['_SignType'], 'real', 'test'), eq(variables['Agent.Os'], 'Windows_NT')) - env: - TeamName: $(_TeamName) - - - ${{ if eq(parameters.enableTelemetry, 'true') }}: - - template: /eng/common/templates/steps/telemetry-end.yml - parameters: - helixSource: $(_HelixSource) - helixType: $(_HelixType) - - - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - - task: CopyFiles@2 - displayName: Gather Asset Manifests - inputs: - SourceFolder: '$(Build.SourcesDirectory)/artifacts/log/$(_BuildConfig)/AssetManifest' - TargetFolder: '$(Build.StagingDirectory)/AssetManifests' - continueOnError: false - condition: and(succeeded(), eq(variables['_DotNetPublishToBlobFeed'], 'true')) - - task: PublishBuildArtifacts@1 - displayName: Push Asset Manifests - inputs: - PathtoPublish: '$(Build.StagingDirectory)/AssetManifests' - PublishLocation: Container - ArtifactName: AssetManifests - continueOnError: false - condition: and(succeeded(), eq(variables['_DotNetPublishToBlobFeed'], 'true')) diff --git a/eng/common/templates/phases/publish-build-assets.yml b/eng/common/templates/phases/publish-build-assets.yml deleted file mode 100644 index 4e51e472..00000000 --- a/eng/common/templates/phases/publish-build-assets.yml +++ /dev/null @@ -1,52 +0,0 @@ -parameters: - dependsOn: '' - queue: {} - configuration: 'Debug' - condition: succeeded() - continueOnError: false - runAsPublic: false - publishUsingPipelines: false -phases: - - phase: Asset_Registry_Publish - displayName: Publish to Build Asset Registry - dependsOn: ${{ parameters.dependsOn }} - queue: ${{ parameters.queue }} - variables: - _BuildConfig: ${{ parameters.configuration }} - steps: - - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - - task: DownloadBuildArtifacts@0 - displayName: Download artifact - inputs: - artifactName: AssetManifests - downloadPath: '$(Build.StagingDirectory)/Download' - checkDownloadedFiles: true - condition: ${{ parameters.condition }} - continueOnError: ${{ parameters.continueOnError }} - - task: AzureKeyVault@1 - inputs: - azureSubscription: 'DotNet-Engineering-Services_KeyVault' - KeyVaultName: EngKeyVault - SecretsFilter: 'MaestroAccessToken' - condition: ${{ parameters.condition }} - continueOnError: ${{ parameters.continueOnError }} - - task: PowerShell@2 - displayName: Publish Build Assets - inputs: - filePath: eng\common\sdk-task.ps1 - arguments: -task PublishBuildAssets -restore -msbuildEngine dotnet - /p:ManifestsPath='$(Build.StagingDirectory)/Download/AssetManifests' - /p:BuildAssetRegistryToken=$(MaestroAccessToken) - /p:MaestroApiEndpoint=https://maestro-prod.westus2.cloudapp.azure.com - /p:PublishUsingPipelines=${{ parameters.publishUsingPipelines }} - /p:Configuration=$(_BuildConfig) - condition: ${{ parameters.condition }} - continueOnError: ${{ parameters.continueOnError }} - - task: PublishBuildArtifacts@1 - displayName: Publish Logs to VSTS - inputs: - PathtoPublish: '$(Build.SourcesDirectory)/artifacts/log/$(_BuildConfig)' - PublishLocation: Container - ArtifactName: $(Agent.Os)_Asset_Registry_Publish - continueOnError: true - condition: always() diff --git a/eng/common/templates/steps/source-build.yml b/eng/common/templates/steps/source-build.yml index 705b7a1c..ba40dc82 100644 --- a/eng/common/templates/steps/source-build.yml +++ b/eng/common/templates/steps/source-build.yml @@ -18,6 +18,35 @@ steps: set -x df -h + # If building on the internal project, the artifact feeds variable may be available (usually only if needed) + # In that case, call the feed setup script to add internal feeds corresponding to public ones. + # In addition, add an msbuild argument to copy the WIP from the repo to the target build location. + # This is because SetupNuGetSources.sh will alter the current NuGet.config file, and we need to preserve those + # changes. + $internalRestoreArgs= + if [ '$(dn-bot-dnceng-artifact-feeds-rw)' != '$''(dn-bot-dnceng-artifact-feeds-rw)' ]; then + # Temporarily work around https://github.com/dotnet/arcade/issues/7709 + chmod +x $(Build.SourcesDirectory)/eng/common/SetupNugetSources.sh + $(Build.SourcesDirectory)/eng/common/SetupNugetSources.sh $(Build.SourcesDirectory)/NuGet.config $(dn-bot-dnceng-artifact-feeds-rw) + internalRestoreArgs='/p:CopyWipIntoInnerSourceBuildRepo=true' + + # The 'Copy WIP' feature of source build uses git stash to apply changes from the original repo. + # This only works if there is a username/email configured, which won't be the case in most CI runs. + git config --get user.email + if [ $? -ne 0 ]; then + git config user.email dn-bot@microsoft.com + git config user.name dn-bot + fi + fi + + # If building on the internal project, the internal storage variable may be available (usually only if needed) + # In that case, add variables to allow the download of internal runtimes if the specified versions are not found + # in the default public locations. + internalRuntimeDownloadArgs= + if [ '$(dotnetclimsrc-read-sas-token-base64)' != '$''(dotnetclimsrc-read-sas-token-base64)' ]; then + internalRuntimeDownloadArgs='/p:DotNetRuntimeSourceFeed=https://dotnetclimsrc.blob.core.windows.net/dotnet /p:DotNetRuntimeSourceFeedKey=$(dotnetclimsrc-read-sas-token-base64) --runtimesourcefeed https://dotnetclimsrc.blob.core.windows.net/dotnet --runtimesourcefeedkey $(dotnetclimsrc-read-sas-token-base64)' + fi + buildConfig=Release # Check if AzDO substitutes in a build config from a variable, and use it if so. if [ '$(_BuildConfig)' != '$''(_BuildConfig)' ]; then @@ -29,11 +58,6 @@ steps: officialBuildArgs='/p:DotNetPublishUsingPipelines=true /p:OfficialBuildId=$(BUILD.BUILDNUMBER)' fi - internalRuntimeDownloadArgs= - if [ '$(dotnetclimsrc-read-sas-token-base64)' != '$''(dotnetclimsrc-read-sas-token-base64)' ]; then - internalRuntimeDownloadArgs='--runtimesourcefeed https://dotnetclimsrc.blob.core.windows.net/dotnet --runtimesourcefeedkey $(dotnetclimsrc-read-sas-token-base64)' - fi - targetRidArgs= if [ '${{ parameters.platform.targetRID }}' != '' ]; then targetRidArgs='/p:TargetRid=${{ parameters.platform.targetRID }}' @@ -49,6 +73,7 @@ steps: --restore --build --pack $publishArgs -bl \ $officialBuildArgs \ $internalRuntimeDownloadArgs \ + $internalRestoreArgs \ $targetRidArgs \ /p:SourceBuildNonPortable=${{ parameters.platform.nonPortable }} \ /p:ArcadeBuildFromSource=true diff --git a/global.json b/global.json index dfc9bf3c..f547b111 100644 --- a/global.json +++ b/global.json @@ -1,8 +1,8 @@ { "tools": { - "dotnet": "6.0.100-preview.6.21355.2" + "dotnet": "6.0.100-rc.1.21379.2" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "6.0.0-beta.21379.2" + "Microsoft.DotNet.Arcade.Sdk": "6.0.0-beta.21406.6" } } From 7d469d74890d18a10f33363aa2fe96edfd4a8e17 Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" Date: Mon, 16 Aug 2021 12:11:42 +0000 Subject: [PATCH 2/3] Update dependencies from https://github.com/dotnet/arcade build 20210812.1 Microsoft.DotNet.Arcade.Sdk From Version 6.0.0-beta.21379.2 -> To Version 6.0.0-beta.21412.1 --- eng/Version.Details.xml | 4 ++-- global.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 53a98e40..8e857842 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -3,9 +3,9 @@ - + https://github.com/dotnet/arcade - 382667fff0b58c362855a42c3529ba294fd0514c + 58ac7035021bd7277ef7582338afd25403fc9aea https://github.com/dotnet/source-build-reference-packages diff --git a/global.json b/global.json index f547b111..838d6a76 100644 --- a/global.json +++ b/global.json @@ -3,6 +3,6 @@ "dotnet": "6.0.100-rc.1.21379.2" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "6.0.0-beta.21406.6" + "Microsoft.DotNet.Arcade.Sdk": "6.0.0-beta.21412.1" } } From 293f98368a19357f1795e4d4c2c668976c700401 Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" Date: Mon, 23 Aug 2021 12:12:29 +0000 Subject: [PATCH 3/3] Update dependencies from https://github.com/dotnet/arcade build 20210820.4 Microsoft.DotNet.Arcade.Sdk From Version 6.0.0-beta.21379.2 -> To Version 6.0.0-beta.21420.4 --- eng/Version.Details.xml | 4 ++-- eng/common/build.sh | 1 + eng/common/init-tools-native.sh | 2 +- eng/common/native/common-library.sh | 8 +++++-- .../post-build/sourcelink-validation.ps1 | 17 ++++++++++----- eng/common/sdk-task.ps1 | 1 + .../templates/job/source-index-stage1.yml | 21 +++++++------------ eng/common/tools.ps1 | 21 +++++++++++++++++++ eng/common/tools.sh | 7 +++++++ global.json | 2 +- 10 files changed, 60 insertions(+), 24 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 8e857842..48d4c88e 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -3,9 +3,9 @@ - + https://github.com/dotnet/arcade - 58ac7035021bd7277ef7582338afd25403fc9aea + fe787bd48ed72e51a98eb5e4e5e5af74edb531e5 https://github.com/dotnet/source-build-reference-packages diff --git a/eng/common/build.sh b/eng/common/build.sh index 55b298f1..9d3042a9 100755 --- a/eng/common/build.sh +++ b/eng/common/build.sh @@ -187,6 +187,7 @@ function InitializeCustomToolset { } function Build { + TryLogClientIpAddress InitializeToolset InitializeCustomToolset diff --git a/eng/common/init-tools-native.sh b/eng/common/init-tools-native.sh index 5bd205b5..3e6a8d6a 100755 --- a/eng/common/init-tools-native.sh +++ b/eng/common/init-tools-native.sh @@ -10,7 +10,7 @@ force=false download_retries=5 retry_wait_time_seconds=30 global_json_file="$(dirname "$(dirname "${scriptroot}")")/global.json" -declare -A native_assets +declare -a native_assets . $scriptroot/pipeline-logging-functions.sh . $scriptroot/native/common-library.sh diff --git a/eng/common/native/common-library.sh b/eng/common/native/common-library.sh index bf272dcf..080c2c28 100755 --- a/eng/common/native/common-library.sh +++ b/eng/common/native/common-library.sh @@ -148,8 +148,12 @@ function NewScriptShim { fi if [[ ! -f $tool_file_path ]]; then - Write-PipelineTelemetryError -category 'NativeToolsBootstrap' "Specified tool file path:'$tool_file_path' does not exist" - return 1 + # try to see if the path is lower cased + tool_file_path="$(echo $tool_file_path | tr "[:upper:]" "[:lower:]")" + if [[ ! -f $tool_file_path ]]; then + Write-PipelineTelemetryError -category 'NativeToolsBootstrap' "Specified tool file path:'$tool_file_path' does not exist" + return 1 + fi fi local shim_contents=$'#!/usr/bin/env bash\n' diff --git a/eng/common/post-build/sourcelink-validation.ps1 b/eng/common/post-build/sourcelink-validation.ps1 index 85c89861..3b6fc953 100644 --- a/eng/common/post-build/sourcelink-validation.ps1 +++ b/eng/common/post-build/sourcelink-validation.ps1 @@ -17,6 +17,7 @@ $global:RepoFiles = @{} $MaxParallelJobs = 16 $MaxRetries = 5 +$RetryWaitTimeInSeconds = 30 # Wait time between check for system load $SecondsBetweenLoadChecks = 10 @@ -99,9 +100,9 @@ $ValidatePackage = { $Status = 200 $Cache = $using:RepoFiles - $totalRetries = 0 + $attempts = 0 - while ($totalRetries -lt $using:MaxRetries) { + while ($attempts -lt $using:MaxRetries) { if ( !($Cache.ContainsKey($FilePath)) ) { try { $Uri = $Link -as [System.URI] @@ -113,7 +114,7 @@ $ValidatePackage = { else { # If it's not a github link, we want to break out of the loop and not retry. $Status = 0 - $totalRetries = $using:MaxRetries + $attempts = $using:MaxRetries } } catch { @@ -123,9 +124,15 @@ $ValidatePackage = { } if ($Status -ne 200) { - $totalRetries++ + $attempts++ - if ($totalRetries -ge $using:MaxRetries) { + if ($attempts -lt $using:MaxRetries) + { + $attemptsLeft = $using:MaxRetries - $attempts + Write-Warning "Download failed, $attemptsLeft attempts remaining, will retry in $using:RetryWaitTimeInSeconds seconds" + Start-Sleep -Seconds $using:RetryWaitTimeInSeconds + } + else { if ($NumFailedLinks -eq 0) { if ($FailedFiles.Value -eq 0) { Write-Host diff --git a/eng/common/sdk-task.ps1 b/eng/common/sdk-task.ps1 index b1bca63a..7ffa3591 100644 --- a/eng/common/sdk-task.ps1 +++ b/eng/common/sdk-task.ps1 @@ -83,6 +83,7 @@ try { } if ($restore) { + Try-LogClientIpAddress Build 'Restore' } diff --git a/eng/common/templates/job/source-index-stage1.yml b/eng/common/templates/job/source-index-stage1.yml index b58d4236..1cc0c29e 100644 --- a/eng/common/templates/job/source-index-stage1.yml +++ b/eng/common/templates/job/source-index-stage1.yml @@ -34,29 +34,24 @@ jobs: inputs: packageType: sdk version: 3.1.x - - - task: UseDotNet@2 - displayName: Use .NET Core sdk - inputs: - useGlobalJson: true + installationPath: $(Agent.TempDirectory)/dotnet + workingDirectory: $(Agent.TempDirectory) - script: | - dotnet tool install BinLogToSln --version $(SourceIndexPackageVersion) --add-source $(SourceIndexPackageSource) --tool-path .source-index/tools - dotnet tool install UploadIndexStage1 --version $(SourceIndexPackageVersion) --add-source $(SourceIndexPackageSource) --tool-path .source-index/tools - echo ##vso[task.prependpath]$(Build.SourcesDirectory)/.source-index/tools + $(Agent.TempDirectory)/dotnet/dotnet tool install BinLogToSln --version $(SourceIndexPackageVersion) --add-source $(SourceIndexPackageSource) --tool-path $(Agent.TempDirectory)/.source-index/tools + $(Agent.TempDirectory)/dotnet/dotnet tool install UploadIndexStage1 --version $(SourceIndexPackageVersion) --add-source $(SourceIndexPackageSource) --tool-path $(Agent.TempDirectory)/.source-index/tools displayName: Download Tools + # Set working directory to temp directory so 'dotnet' doesn't try to use global.json and use the repo's sdk. + workingDirectory: $(Agent.TempDirectory) - script: ${{ parameters.sourceIndexBuildCommand }} displayName: Build Repository - - script: BinLogToSln -i $(BinlogPath) -r $(Build.SourcesDirectory) -n $(Build.Repository.Name) -o .source-index/stage1output + - script: $(Agent.TempDirectory)/.source-index/tools/BinLogToSln -i $(BinlogPath) -r $(Build.SourcesDirectory) -n $(Build.Repository.Name) -o .source-index/stage1output displayName: Process Binlog into indexable sln - env: - DOTNET_ROLL_FORWARD_ON_NO_CANDIDATE_FX: 2 - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - - script: UploadIndexStage1 -i .source-index/stage1output -n $(Build.Repository.Name) + - script: $(Agent.TempDirectory)/.source-index/tools/UploadIndexStage1 -i .source-index/stage1output -n $(Build.Repository.Name) displayName: Upload stage1 artifacts to source index env: BLOB_CONTAINER_URL: $(source-dot-net-stage1-blob-container-url) - DOTNET_ROLL_FORWARD_ON_NO_CANDIDATE_FX: 2 diff --git a/eng/common/tools.ps1 b/eng/common/tools.ps1 index 5d526c74..e607aa43 100644 --- a/eng/common/tools.ps1 +++ b/eng/common/tools.ps1 @@ -154,6 +154,9 @@ function InitializeDotNetCli([bool]$install, [bool]$createSdkLocationFile) { return $global:_DotNetInstallDir } + # In case of network error, try to log the current IP for reference + Try-LogClientIpAddress + # Don't resolve runtime, shared framework, or SDK from other locations to ensure build determinism $env:DOTNET_MULTILEVEL_LOOKUP=0 @@ -872,3 +875,21 @@ if (!$disableConfigureToolsetImport) { } } } + +function Try-LogClientIpAddress() +{ + Write-Host "Attempting to log this client's IP for Azure Package feed telemetry purposes" + try + { + $result = Invoke-WebRequest -Uri "http://co1.msedge.net/fdv2/diagnostics.aspx" -UseBasicParsing + $lines = $result.Content.Split([Environment]::NewLine) + $socketIp = $lines | Select-String -Pattern "^Socket IP:.*" + Write-Host $socketIp + $clientIp = $lines | Select-String -Pattern "^Client IP:.*" + Write-Host $clientIp + } + catch + { + Write-Host "Unable to get this machine's effective IP address for logging: $_" + } +} diff --git a/eng/common/tools.sh b/eng/common/tools.sh index 828119be..3c5f3a12 100755 --- a/eng/common/tools.sh +++ b/eng/common/tools.sh @@ -399,6 +399,13 @@ function StopProcesses { return 0 } +function TryLogClientIpAddress () { + echo 'Attempting to log this client''s IP for Azure Package feed telemetry purposes' + if command -v curl > /dev/null; then + curl -s 'http://co1.msedge.net/fdv2/diagnostics.aspx' | grep ' IP: ' + fi +} + function MSBuild { local args=$@ if [[ "$pipelines_log" == true ]]; then diff --git a/global.json b/global.json index 838d6a76..eae10d81 100644 --- a/global.json +++ b/global.json @@ -3,6 +3,6 @@ "dotnet": "6.0.100-rc.1.21379.2" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "6.0.0-beta.21412.1" + "Microsoft.DotNet.Arcade.Sdk": "6.0.0-beta.21420.4" } }