From 37ccc2ccd7c4a668531fe3b14833f084e578f30c Mon Sep 17 00:00:00 2001 From: Oleksandr <106967057+oleksandr-didyk@users.noreply.github.com> Date: Tue, 13 Sep 2022 13:58:47 +0200 Subject: [PATCH 1/2] feat: add tooling for generating text-only packages (#427) --- .gitattributes | 2 + generate.sh | 103 +++++++++------- src/common/RestoreProjects.proj | 16 +++ .../GenerateProjects/GenerateProjects.proj | 18 +++ .../GenerateSource/GenerateSource.proj | 16 +-- .../Tasks/GenerateProjects.cs | 6 - .../PackageGenerator.proj | 112 ++++++++++++++++++ .../textOnlyPackage.csproj.template} | 0 ...nalysis.collections.4.2.0-1.22102.8.csproj | 1 + .../microsoft.netcore.platforms.2.1.9.csproj | 1 + 10 files changed, 209 insertions(+), 66 deletions(-) mode change 100755 => 100644 generate.sh create mode 100644 src/common/RestoreProjects.proj create mode 100644 src/textOnlyPackageGenerator/PackageGenerator.proj rename src/{textOnlyPackages/src/microsoft.codeanalysis.collections/4.2.0-1.22102.8/microsoft.codeanalysis.collections-4.2.0-1.22102.8.csproj => textOnlyPackageGenerator/textOnlyPackage.csproj.template} (100%) create mode 100644 src/textOnlyPackages/src/microsoft.codeanalysis.collections/4.2.0-1.22102.8/microsoft.codeanalysis.collections.4.2.0-1.22102.8.csproj create mode 100644 src/textOnlyPackages/src/microsoft.netcore.platforms/2.1.9/microsoft.netcore.platforms.2.1.9.csproj diff --git a/.gitattributes b/.gitattributes index d2a0d72643..302053553a 100644 --- a/.gitattributes +++ b/.gitattributes @@ -2,3 +2,5 @@ # patch files need lf line-endings, even on Windows *.patch text eol=lf + +*.sh text eol=lf diff --git a/generate.sh b/generate.sh old mode 100755 new mode 100644 index 45e46703a7..ae96fc1f0f --- a/generate.sh +++ b/generate.sh @@ -4,33 +4,24 @@ # Stop script if command returns non-zero exit code. set -e -source="${BASH_SOURCE[0]}" - DEFAULT_TFM=net6.0 -# resolve $SOURCE until the file is no longer a symlink -while [[ -h $source ]]; do - scriptroot="$( cd -P "$( dirname "$source" )" && pwd )" - source="$(readlink "$source")" - - # if $source was a relative symlink, we need to resolve it relative to the path where the - # symlink file was located - [[ $source != /* ]] && source="$scriptroot/$source" -done - +source="$(readlink -f "${BASH_SOURCE[0]}")" scriptroot="$( cd -P "$( dirname "$source" )" && pwd )" -GREEN='\033[0;32m' RED='\033[0;31m' NC='\033[0m' usage() { echo "usage: $0 [options]" echo "" - echo " Restores reference package(s) and dependencies and generates cs files and accompanying projects" - echo " to build a reference package. The generated artifacts are added to the /src/ directory of this repo" - echo " unless a different destination is specified. If a csv file of packages is specified, multiple" - echo " packages and their dependencies are generated." + echo " Generates a reference package or a text-only package from the specified packages and versions. The" + echo " type of the generated package is controlled via the --type option." + echo "" + echo " Reference package generation will restore reference package(s) and dependencies and generate cs files" + echo " and with accompanying projects into the specified destination ('./src/referencePackages/' by default)." + echo " Text-only package generation will restore the specified package and copy the source-build-usable content" + echo " into the provided directory ('./src/textOnlyPackages/' by default)." echo "" echo " Either --pkg or --pkgCsv must be specified" echo "" @@ -42,21 +33,24 @@ usage() { echo " --pkgCsv A path to a csv file of packages to generate. Format is the same as the --pkg" echo " option above, one per line. If specified, the --pkg option is ignored." echo " --dest A path to the root of the repo to copy source into." + echo " --type Type of the package to generate. Accepted values: ref (default) | text." + echo " --feeds A semicolon-separated list of additional NuGet feeds to use during restore." echo "" } packageVersion= defaultPathToCSV="$scriptroot/artifacts/targetPackages.csv" pathToCSV= -pathToDestRepo="$scriptroot" -positional_args=() -requiredOptionSpecified=false -while :; do - if [ $# -le 0 ]; then - break - fi - lowerI="$(echo "$1" | awk '{print tolower($0)}')" - case $lowerI in +pathToDestRepo= +packageType="ref" + +if [[ $# -le 0 ]]; then + usage + exit 0 +fi + +while [ $# -gt 0 ]; do + case "$1" in "-?"|-h|--help) usage exit 0 @@ -67,7 +61,6 @@ while :; do exit 1 fi pathToCSV="$(cd "$(dirname "$2")"; pwd)/$(basename "$2")" - requiredOptionSpecified=true shift ;; --dest) @@ -78,9 +71,21 @@ while :; do pathToDestRepo="$(cd -P "$2" && pwd)" shift ;; + --type) + packageType="$2" + if [[ ! "$packageType" =~ ^(text|ref)$ ]]; then + echo -e "${RED}ERROR: unknown package type - $2${NC}" + usage + exit 1 + fi + shift + ;; --pkg) packageVersion="$2" - requiredOptionSpecified=true + shift + ;; + --feeds) + feeds="$2" shift ;; *) @@ -93,7 +98,8 @@ while :; do shift done -if [ "$requiredOptionSpecified" != "true" ]; then +if [ -z "$packageVersion" ] && [ -z "$pathToCSV" ]; then + echo -e "${RED}ERROR: Either --pkg or --pkgCsv must be specified!${NC}" usage exit 1 fi @@ -107,6 +113,14 @@ if [ "$pathToCSV" == "" ]; then pathToCSV="$defaultPathToCSV" fi +if [[ -z "$pathToDestRepo" ]]; then + if [ "$packageType" == "ref" ]; then + pathToDestRepo="$scriptroot/src/referencePackages/src" + else + pathToDestRepo="$scriptroot/src/textOnlyPackages/src/" + fi +fi + # Generate restore projects for each target package # Packges are restored in their own project to eliminate any conflicts # between different versions of the same package. @@ -120,8 +134,8 @@ if [ -d "$restoreProjectsDir" ]; then rm -rf "$restoreProjectsDir" fi mkdir -p "$restoreProjectsDir" -[ ! -f $INPUT ] && { echo "$INPUT file not found"; exit 99; } -while read pkgName pkgVersion tfm +[ ! -f "$INPUT" ] && { echo "$INPUT file not found"; exit 99; } +while read -r pkgName pkgVersion tfm do sed "s/##PackageName##/${pkgName}/g" "$targetPackageTemplate" > "$restoreProjectsDir/$pkgName.$pkgVersion.csproj" sed -i "s/##PackageVersion##/${pkgVersion}/g" "$restoreProjectsDir/$pkgName.$pkgVersion.csproj" @@ -130,21 +144,20 @@ do else sed -i "s/##Tfm##/${DEFAULT_TFM}/g" "$restoreProjectsDir/$pkgName.$pkgVersion.csproj" fi -done < $INPUT +done < "$INPUT" IFS=$OLDIFS -# Build the projects to generate source and projects -"$scriptroot/eng/common/build.sh" --build --restore -bl /p:GeneratePackageSource=true /p:GeneratorVersion=2 $@ +if [[ "$packageType" == "ref" ]]; then + pathToRepoSrc="$pathToDestRepo" -# Copy source to destination -pathToGeneratedSrc="$scriptroot/artifacts/generatedSrc" -pathToRepoSrc="$pathToDestRepo/src/referencePackages/src" + # Build the projects to generate source and projects + "$scriptroot/eng/common/build.sh" --build --restore -bl /p:GeneratePackageSource=true /p:GeneratorVersion=2 /p:PathToRepoSrc="$pathToRepoSrc" /p:RestoreAdditionalProjectSources="\"$feeds\"" "$@" +else + # -p:RestoreAdditionalProjectSources -> specifies additional Nuget package sources, including feeds: https://docs.microsoft.com/en-us/nuget/reference/msbuild-targets#restore-properties + # -p:RestoreUsingNuGetTargets -> switch that controls who will restore the project; if set to `false`, Arcade's Build.proj will execute the Restore target of the project itself. + # Otherwise, Build.proj will trigger its own instance of Nuget.targets.Restore and delegate the restore of all applicable projects to it. In this case, the Restore target is + # outside the project and we cannot hook our post-processing logic to it. For more info see: + # https://github.com/dotnet/arcade/blob/5b838a3ed7f8e53c3082724605e5237fa614a43c/src/Microsoft.DotNet.Arcade.Sdk/tools/Build.proj#L227 -pushd "$pathToGeneratedSrc" -chmod 755 ./CopyProjects.sh -./CopyProjects.sh "$pathToRepoSrc" -popd - -echo -echo -e " Copied generated projects from ${GREEN}$pathToGeneratedSrc${NC} to ${GREEN}"$pathToRepoSrc"${NC}" -echo + "$scriptroot/eng/common/build.sh" -bl --restore --projects "$scriptroot/src/textOnlyPackageGenerator/PackageGenerator.proj" /p:RestoreUsingNuGetTargets=false /p:TextOnlyPackageDestination="$pathToDestRepo" /p:PathToCsv="$pathToCSV" /p:RestoreAdditionalProjectSources="\"$feeds\"" +fi diff --git a/src/common/RestoreProjects.proj b/src/common/RestoreProjects.proj new file mode 100644 index 0000000000..abd51ebb02 --- /dev/null +++ b/src/common/RestoreProjects.proj @@ -0,0 +1,16 @@ + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/src/referencePackageSourceGenerator/GenerateProjects/GenerateProjects.proj b/src/referencePackageSourceGenerator/GenerateProjects/GenerateProjects.proj index 54280872f9..1643d3364f 100644 --- a/src/referencePackageSourceGenerator/GenerateProjects/GenerateProjects.proj +++ b/src/referencePackageSourceGenerator/GenerateProjects/GenerateProjects.proj @@ -29,6 +29,24 @@ GeneratorVersion="$(GeneratorVersion)" /> + + + + + + $([System.IO.Path]::GetFileName($([System.IO.Path]::TrimEndingDirectorySeparator(%(GeneratedSourcePackages.RelativeDir))))) + + + + + + + + diff --git a/src/referencePackageSourceGenerator/GenerateSource/GenerateSource.proj b/src/referencePackageSourceGenerator/GenerateSource/GenerateSource.proj index 825aaba60c..89620ecfcd 100644 --- a/src/referencePackageSourceGenerator/GenerateSource/GenerateSource.proj +++ b/src/referencePackageSourceGenerator/GenerateSource/GenerateSource.proj @@ -7,21 +7,7 @@ $(TargetPackagesPath) - - - - - - - - - - - - - + diff --git a/src/referencePackageSourceGenerator/Tasks/GenerateProjects.cs b/src/referencePackageSourceGenerator/Tasks/GenerateProjects.cs index 2b438addb7..a5e800799f 100644 --- a/src/referencePackageSourceGenerator/Tasks/GenerateProjects.cs +++ b/src/referencePackageSourceGenerator/Tasks/GenerateProjects.cs @@ -48,12 +48,6 @@ public override bool Execute() // Second, setup dependencies between all generated project data ProjectData.GenerateAllDependencies(); - // Create a script file to copy the newly added project source to the output directory - string projectSourceCopyCommands = PackageData.GetAll().Select(p => $"cp -r --parents ./{p.RelativePath}/* $1").Aggregate((a, b) => a + "\n" + b); - projectSourceCopyCommands += "\nfind . -type f -iname Directory.Build.props -exec cp --parents {} $1 \\;"; - - File.WriteAllText(Path.Combine(SrcPath, "CopyProjects.sh"), projectSourceCopyCommands); - foreach (var pkgData in PackageData.GetAll()) { string pkgProjectOutput = pkgProjectTemplate; diff --git a/src/textOnlyPackageGenerator/PackageGenerator.proj b/src/textOnlyPackageGenerator/PackageGenerator.proj new file mode 100644 index 0000000000..73c7a25174 --- /dev/null +++ b/src/textOnlyPackageGenerator/PackageGenerator.proj @@ -0,0 +1,112 @@ + + + + + net7.0 + $(ArtifactsDir)textOnlyPackages/ + ./textOnlyPackage.csproj.template + + + + + + + + + + + + + + $([System.String]::Copy('%(CsvLines.Identity)').Split(',')[0]) + $([System.String]::Copy('%(CsvLines.Identity)').Split(',')[1]) + + + + + + + + + + + + $(TargetPackagesPath)$(PackageName.ToLower()) + $(PackageName.ToLower()) + $(PackageName.ToLower()).$(PackageVersion).csproj + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/src/textOnlyPackages/src/microsoft.codeanalysis.collections/4.2.0-1.22102.8/microsoft.codeanalysis.collections-4.2.0-1.22102.8.csproj b/src/textOnlyPackageGenerator/textOnlyPackage.csproj.template similarity index 100% rename from src/textOnlyPackages/src/microsoft.codeanalysis.collections/4.2.0-1.22102.8/microsoft.codeanalysis.collections-4.2.0-1.22102.8.csproj rename to src/textOnlyPackageGenerator/textOnlyPackage.csproj.template diff --git a/src/textOnlyPackages/src/microsoft.codeanalysis.collections/4.2.0-1.22102.8/microsoft.codeanalysis.collections.4.2.0-1.22102.8.csproj b/src/textOnlyPackages/src/microsoft.codeanalysis.collections/4.2.0-1.22102.8/microsoft.codeanalysis.collections.4.2.0-1.22102.8.csproj new file mode 100644 index 0000000000..6b512ec924 --- /dev/null +++ b/src/textOnlyPackages/src/microsoft.codeanalysis.collections/4.2.0-1.22102.8/microsoft.codeanalysis.collections.4.2.0-1.22102.8.csproj @@ -0,0 +1 @@ + diff --git a/src/textOnlyPackages/src/microsoft.netcore.platforms/2.1.9/microsoft.netcore.platforms.2.1.9.csproj b/src/textOnlyPackages/src/microsoft.netcore.platforms/2.1.9/microsoft.netcore.platforms.2.1.9.csproj new file mode 100644 index 0000000000..6b512ec924 --- /dev/null +++ b/src/textOnlyPackages/src/microsoft.netcore.platforms/2.1.9/microsoft.netcore.platforms.2.1.9.csproj @@ -0,0 +1 @@ + From 1f8037df094f11e7ab0422cbe514726ccc920d78 Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" <42748379+dotnet-maestro[bot]@users.noreply.github.com> Date: Tue, 13 Sep 2022 14:34:35 -0500 Subject: [PATCH 2/2] [main] Update dependencies from dotnet/arcade (#435) * Update dependencies from https://github.com/dotnet/arcade build 20220902.1 Microsoft.DotNet.Arcade.Sdk From Version 7.0.0-beta.22411.2 -> To Version 8.0.0-beta.22452.1 Dependency coherency updates Microsoft.SourceLink.GitHub From Version 1.2.0-beta-22410-01 -> To Version 1.2.0-beta-22431-01 (parent: Microsoft.DotNet.Arcade.Sdk * Update dependencies from https://github.com/dotnet/arcade build 20220905.1 Microsoft.DotNet.Arcade.Sdk From Version 7.0.0-beta.22411.2 -> To Version 8.0.0-beta.22455.1 Dependency coherency updates Microsoft.SourceLink.GitHub From Version 1.2.0-beta-22410-01 -> To Version 1.2.0-beta-22453-01 (parent: Microsoft.DotNet.Arcade.Sdk * Update dependencies from https://github.com/dotnet/arcade build 20220906.4 Microsoft.DotNet.Arcade.Sdk From Version 7.0.0-beta.22411.2 -> To Version 8.0.0-beta.22456.4 Dependency coherency updates Microsoft.SourceLink.GitHub From Version 1.2.0-beta-22410-01 -> To Version 1.2.0-beta-22455-02 (parent: Microsoft.DotNet.Arcade.Sdk * Update dependencies from https://github.com/dotnet/arcade build 20220912.4 Microsoft.DotNet.Arcade.Sdk From Version 7.0.0-beta.22411.2 -> To Version 8.0.0-beta.22462.4 Dependency coherency updates Microsoft.SourceLink.GitHub From Version 1.2.0-beta-22410-01 -> To Version 1.2.0-beta-22461-01 (parent: Microsoft.DotNet.Arcade.Sdk Co-authored-by: dotnet-maestro[bot] --- eng/Version.Details.xml | 8 +-- eng/common/cross/arm/sources.list.focal | 11 ++++ eng/common/cross/arm/sources.list.jammy | 11 ++++ eng/common/cross/arm64/sources.list.focal | 11 ++++ eng/common/cross/arm64/sources.list.jammy | 11 ++++ eng/common/cross/build-rootfs.sh | 59 +++++++++++++--------- eng/common/generate-locproject.ps1 | 33 ++++++++++-- eng/common/sdk-task.ps1 | 2 +- eng/common/sdl/NuGet.config | 5 ++ eng/common/sdl/sdl.ps1 | 38 ++++++++++++++ eng/common/templates/steps/execute-sdl.yml | 39 +++++++------- eng/common/tools.ps1 | 4 +- global.json | 4 +- 13 files changed, 179 insertions(+), 57 deletions(-) create mode 100644 eng/common/cross/arm/sources.list.focal create mode 100644 eng/common/cross/arm/sources.list.jammy create mode 100644 eng/common/cross/arm64/sources.list.focal create mode 100644 eng/common/cross/arm64/sources.list.jammy create mode 100644 eng/common/sdl/sdl.ps1 diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 1552aa0475..1699a39986 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -13,14 +13,14 @@ c28368c0bc80f15c0a06b2242ad50c2895d4fb85 - + https://github.com/dotnet/arcade - 6a638cd0c13962ab2a1943cb1c878be5a41dd82e + 91db46836065516e87e05bbdb51c5aee0f90428c - + https://github.com/dotnet/sourcelink - 508c31bb736f3e6d69b8f47fa9ee38eedd9ce785 + 4d2cae3ec4edec7004c6e46cd6d1621966bf776d diff --git a/eng/common/cross/arm/sources.list.focal b/eng/common/cross/arm/sources.list.focal new file mode 100644 index 0000000000..4de2600c17 --- /dev/null +++ b/eng/common/cross/arm/sources.list.focal @@ -0,0 +1,11 @@ +deb http://ports.ubuntu.com/ubuntu-ports/ focal main restricted universe +deb-src http://ports.ubuntu.com/ubuntu-ports/ focal main restricted universe + +deb http://ports.ubuntu.com/ubuntu-ports/ focal-updates main restricted universe +deb-src http://ports.ubuntu.com/ubuntu-ports/ focal-updates main restricted universe + +deb http://ports.ubuntu.com/ubuntu-ports/ focal-backports main restricted +deb-src http://ports.ubuntu.com/ubuntu-ports/ focal-backports main restricted + +deb http://ports.ubuntu.com/ubuntu-ports/ focal-security main restricted universe multiverse +deb-src http://ports.ubuntu.com/ubuntu-ports/ focal-security main restricted universe multiverse diff --git a/eng/common/cross/arm/sources.list.jammy b/eng/common/cross/arm/sources.list.jammy new file mode 100644 index 0000000000..6bb0453029 --- /dev/null +++ b/eng/common/cross/arm/sources.list.jammy @@ -0,0 +1,11 @@ +deb http://ports.ubuntu.com/ubuntu-ports/ jammy main restricted universe +deb-src http://ports.ubuntu.com/ubuntu-ports/ jammy main restricted universe + +deb http://ports.ubuntu.com/ubuntu-ports/ jammy-updates main restricted universe +deb-src http://ports.ubuntu.com/ubuntu-ports/ jammy-updates main restricted universe + +deb http://ports.ubuntu.com/ubuntu-ports/ jammy-backports main restricted +deb-src http://ports.ubuntu.com/ubuntu-ports/ jammy-backports main restricted + +deb http://ports.ubuntu.com/ubuntu-ports/ jammy-security main restricted universe multiverse +deb-src http://ports.ubuntu.com/ubuntu-ports/ jammy-security main restricted universe multiverse diff --git a/eng/common/cross/arm64/sources.list.focal b/eng/common/cross/arm64/sources.list.focal new file mode 100644 index 0000000000..4de2600c17 --- /dev/null +++ b/eng/common/cross/arm64/sources.list.focal @@ -0,0 +1,11 @@ +deb http://ports.ubuntu.com/ubuntu-ports/ focal main restricted universe +deb-src http://ports.ubuntu.com/ubuntu-ports/ focal main restricted universe + +deb http://ports.ubuntu.com/ubuntu-ports/ focal-updates main restricted universe +deb-src http://ports.ubuntu.com/ubuntu-ports/ focal-updates main restricted universe + +deb http://ports.ubuntu.com/ubuntu-ports/ focal-backports main restricted +deb-src http://ports.ubuntu.com/ubuntu-ports/ focal-backports main restricted + +deb http://ports.ubuntu.com/ubuntu-ports/ focal-security main restricted universe multiverse +deb-src http://ports.ubuntu.com/ubuntu-ports/ focal-security main restricted universe multiverse diff --git a/eng/common/cross/arm64/sources.list.jammy b/eng/common/cross/arm64/sources.list.jammy new file mode 100644 index 0000000000..6bb0453029 --- /dev/null +++ b/eng/common/cross/arm64/sources.list.jammy @@ -0,0 +1,11 @@ +deb http://ports.ubuntu.com/ubuntu-ports/ jammy main restricted universe +deb-src http://ports.ubuntu.com/ubuntu-ports/ jammy main restricted universe + +deb http://ports.ubuntu.com/ubuntu-ports/ jammy-updates main restricted universe +deb-src http://ports.ubuntu.com/ubuntu-ports/ jammy-updates main restricted universe + +deb http://ports.ubuntu.com/ubuntu-ports/ jammy-backports main restricted +deb-src http://ports.ubuntu.com/ubuntu-ports/ jammy-backports main restricted + +deb http://ports.ubuntu.com/ubuntu-ports/ jammy-security main restricted universe multiverse +deb-src http://ports.ubuntu.com/ubuntu-ports/ jammy-security main restricted universe multiverse diff --git a/eng/common/cross/build-rootfs.sh b/eng/common/cross/build-rootfs.sh index d3b0ac3ba7..5680980fa2 100755 --- a/eng/common/cross/build-rootfs.sh +++ b/eng/common/cross/build-rootfs.sh @@ -76,10 +76,10 @@ __FreeBSDPackages+=" openssl" __FreeBSDPackages+=" krb5" __FreeBSDPackages+=" terminfo-db" -__IllumosPackages="icu-64.2nb2" -__IllumosPackages+=" mit-krb5-1.16.2nb4" -__IllumosPackages+=" openssl-1.1.1e" -__IllumosPackages+=" zlib-1.2.11" +__IllumosPackages="icu" +__IllumosPackages+=" mit-krb5" +__IllumosPackages+=" openssl" +__IllumosPackages+=" zlib" __HaikuPackages="gmp" __HaikuPackages+=" gmp_devel" @@ -186,32 +186,27 @@ while :; do __UbuntuArch=i386 __UbuntuRepo="http://archive.ubuntu.com/ubuntu/" ;; - lldb3.6) - __LLDB_Package="lldb-3.6-dev" - ;; - lldb3.8) - __LLDB_Package="lldb-3.8-dev" - ;; - lldb3.9) - __LLDB_Package="liblldb-3.9-dev" - ;; - lldb4.0) - __LLDB_Package="liblldb-4.0-dev" - ;; - lldb5.0) - __LLDB_Package="liblldb-5.0-dev" - ;; - lldb6.0) - __LLDB_Package="liblldb-6.0-dev" + lldb*) + version="${lowerI/lldb/}" + parts=(${version//./ }) + + # for versions > 6.0, lldb has dropped the minor version + if [[ "${parts[0]}" -gt 6 ]]; then + version="${parts[0]}" + fi + + __LLDB_Package="liblldb-${version}-dev" ;; no-lldb) unset __LLDB_Package ;; llvm*) - version="$(echo "$lowerI" | tr -d '[:alpha:]-=')" + version="${lowerI/llvm/}" parts=(${version//./ }) __LLVM_MajorVersion="${parts[0]}" __LLVM_MinorVersion="${parts[1]}" + + # for versions > 6.0, llvm has dropped the minor version if [[ -z "$__LLVM_MinorVersion" && "$__LLVM_MajorVersion" -le 6 ]]; then __LLVM_MinorVersion=0; fi @@ -231,6 +226,16 @@ while :; do __CodeName=bionic fi ;; + focal) # Ubuntu 20.04 + if [[ "$__CodeName" != "jessie" ]]; then + __CodeName=focal + fi + ;; + jammy) # Ubuntu 22.04 + if [[ "$__CodeName" != "jessie" ]]; then + __CodeName=jammy + fi + ;; jessie) # Debian 8 __CodeName=jessie __UbuntuRepo="http://ftp.debian.org/debian/" @@ -390,14 +395,18 @@ elif [[ "$__CodeName" == "illumos" ]]; then if [[ "$__UseMirror" == 1 ]]; then BaseUrl=http://pkgsrc.smartos.skylime.net fi - BaseUrl="$BaseUrl/packages/SmartOS/2020Q1/${__illumosArch}/All" + BaseUrl="$BaseUrl/packages/SmartOS/trunk/${__illumosArch}/All" + echo "Downloading manifest" + wget "$BaseUrl" echo "Downloading dependencies." read -ra array <<<"$__IllumosPackages" for package in "${array[@]}"; do - echo "Installing $package..." + echo "Installing '$package'" + package="$(grep ">$package-[0-9]" All | sed -En 's/.*href="(.*)\.tgz".*/\1/p')" + echo "Resolved name '$package'" wget "$BaseUrl"/"$package".tgz ar -x "$package".tgz - tar --skip-old-files -xzf "$package".tmp.tgz -C "$__RootfsDir" 2>/dev/null + tar --skip-old-files -xzf "$package".tmp.tg* -C "$__RootfsDir" 2>/dev/null done echo "Cleaning up temporary files." popd diff --git a/eng/common/generate-locproject.ps1 b/eng/common/generate-locproject.ps1 index afdd175029..dbf2ab4ee7 100644 --- a/eng/common/generate-locproject.ps1 +++ b/eng/common/generate-locproject.ps1 @@ -33,6 +33,8 @@ $jsonTemplateFiles | ForEach-Object { $jsonWinformsTemplateFiles = Get-ChildItem -Recurse -Path "$SourcesDirectory" | Where-Object { $_.FullName -Match "en\\strings\.json" } # current winforms pattern +$wxlFiles = Get-ChildItem -Recurse -Path "$SourcesDirectory" | Where-Object { $_.FullName -Match "\\.+\.wxl" -And -Not( $_.Directory.Name -Match "\d{4}" ) } # localized files live in four digit lang ID directories; this excludes them + $xlfFiles = @() $allXlfFiles = Get-ChildItem -Recurse -Path "$SourcesDirectory\*\*.xlf" @@ -60,7 +62,7 @@ $locJson = @{ $outputPath = "$(($_.DirectoryName | Resolve-Path -Relative) + "\")" $continue = $true foreach ($exclusion in $exclusions.Exclusions) { - if ($outputPath.Contains($exclusion)) + if ($_.FullName.Contains($exclusion)) { $continue = $false } @@ -77,8 +79,7 @@ $locJson = @{ CopyOption = "LangIDOnPath" OutputPath = "$($_.Directory.Parent.FullName | Resolve-Path -Relative)\" } - } - else { + } else { return @{ SourceFile = $sourceFile CopyOption = "LangIDOnName" @@ -88,6 +89,32 @@ $locJson = @{ } } ) + }, + @{ + LanguageSet = $LanguageSet + CloneLanguageSet = "WiX_CloneLanguages" + LssFiles = @( "wxl_loc.lss" ) + LocItems = @( + $wxlFiles | ForEach-Object { + $outputPath = "$($_.Directory.FullName | Resolve-Path -Relative)\" + $continue = $true + foreach ($exclusion in $exclusions.Exclusions) { + if ($_.FullName.Contains($exclusion)) + { + $continue = $false + } + } + $sourceFile = ($_.FullName | Resolve-Path -Relative) + if ($continue) + { + return @{ + SourceFile = $sourceFile + CopyOption = "LangIDOnPath" + OutputPath = $outputPath + } + } + } + ) } ) } diff --git a/eng/common/sdk-task.ps1 b/eng/common/sdk-task.ps1 index 119a6c660d..c35087a060 100644 --- a/eng/common/sdk-task.ps1 +++ b/eng/common/sdk-task.ps1 @@ -64,7 +64,7 @@ try { $GlobalJson.tools | Add-Member -Name "vs" -Value (ConvertFrom-Json "{ `"version`": `"16.5`" }") -MemberType NoteProperty } if( -not ($GlobalJson.tools.PSObject.Properties.Name -match "xcopy-msbuild" )) { - $GlobalJson.tools | Add-Member -Name "xcopy-msbuild" -Value "17.1.0" -MemberType NoteProperty + $GlobalJson.tools | Add-Member -Name "xcopy-msbuild" -Value "17.2.1" -MemberType NoteProperty } if ($GlobalJson.tools."xcopy-msbuild".Trim() -ine "none") { $xcopyMSBuildToolsFolder = InitializeXCopyMSBuild $GlobalJson.tools."xcopy-msbuild" -install $true diff --git a/eng/common/sdl/NuGet.config b/eng/common/sdl/NuGet.config index 0c5451c114..3849bdb3cf 100644 --- a/eng/common/sdl/NuGet.config +++ b/eng/common/sdl/NuGet.config @@ -7,6 +7,11 @@ + + + + + diff --git a/eng/common/sdl/sdl.ps1 b/eng/common/sdl/sdl.ps1 new file mode 100644 index 0000000000..648c5068d7 --- /dev/null +++ b/eng/common/sdl/sdl.ps1 @@ -0,0 +1,38 @@ + +function Install-Gdn { + param( + [Parameter(Mandatory=$true)] + [string]$Path, + + # If omitted, install the latest version of Guardian, otherwise install that specific version. + [string]$Version + ) + + $ErrorActionPreference = 'Stop' + Set-StrictMode -Version 2.0 + $disableConfigureToolsetImport = $true + $global:LASTEXITCODE = 0 + + # `tools.ps1` checks $ci to perform some actions. Since the SDL + # scripts don't necessarily execute in the same agent that run the + # build.ps1/sh script this variable isn't automatically set. + $ci = $true + . $PSScriptRoot\..\tools.ps1 + + $argumentList = @("install", "Microsoft.Guardian.Cli", "-Source https://securitytools.pkgs.visualstudio.com/_packaging/Guardian/nuget/v3/index.json", "-OutputDirectory $Path", "-NonInteractive", "-NoCache") + + if ($Version) { + $argumentList += "-Version $Version" + } + + Start-Process nuget -Verbose -ArgumentList $argumentList -NoNewWindow -Wait + + $gdnCliPath = Get-ChildItem -Filter guardian.cmd -Recurse -Path $Path + + if (!$gdnCliPath) + { + Write-PipelineTelemetryError -Category 'Sdl' -Message 'Failure installing Guardian' + } + + return $gdnCliPath.FullName +} \ No newline at end of file diff --git a/eng/common/templates/steps/execute-sdl.yml b/eng/common/templates/steps/execute-sdl.yml index 73245593ce..9dd5709f66 100644 --- a/eng/common/templates/steps/execute-sdl.yml +++ b/eng/common/templates/steps/execute-sdl.yml @@ -8,29 +8,28 @@ parameters: condition: '' steps: -- ${{ if ne(parameters.overrideGuardianVersion, '') }}: - - powershell: | - $content = Get-Content $(GuardianPackagesConfigFile) - - Write-Host "packages.config content was:`n$content" - - $content = $content.Replace('$(DefaultGuardianVersion)', '$(GuardianVersion)') - $content | Set-Content $(GuardianPackagesConfigFile) - - Write-Host "packages.config content updated to:`n$content" - displayName: Use overridden Guardian version ${{ parameters.overrideGuardianVersion }} +- task: NuGetAuthenticate@1 + inputs: + nuGetServiceConnections: GuardianConnect - task: NuGetToolInstaller@1 displayName: 'Install NuGet.exe' -- task: NuGetCommand@2 - displayName: 'Install Guardian' - inputs: - restoreSolution: $(Build.SourcesDirectory)\eng\common\sdl\packages.config - feedsToUse: config - nugetConfigPath: $(Build.SourcesDirectory)\eng\common\sdl\NuGet.config - externalFeedCredentials: GuardianConnect - restoreDirectory: $(Build.SourcesDirectory)\.packages +- ${{ if ne(parameters.overrideGuardianVersion, '') }}: + - pwsh: | + Set-Location -Path $(Build.SourcesDirectory)\eng\common\sdl + . .\sdl.ps1 + $guardianCliLocation = Install-Gdn -Path $(Build.SourcesDirectory)\.artifacts -Version ${{ parameters.overrideGuardianVersion }} + Write-Host "##vso[task.setvariable variable=GuardianCliLocation]$guardianCliLocation" + displayName: Install Guardian (Overridden) + +- ${{ if eq(parameters.overrideGuardianVersion, '') }}: + - pwsh: | + Set-Location -Path $(Build.SourcesDirectory)\eng\common\sdl + . .\sdl.ps1 + $guardianCliLocation = Install-Gdn -Path $(Build.SourcesDirectory)\.artifacts + Write-Host "##vso[task.setvariable variable=GuardianCliLocation]$guardianCliLocation" + displayName: Install Guardian - ${{ if ne(parameters.overrideParameters, '') }}: - powershell: ${{ parameters.executeAllSdlToolsScript }} ${{ parameters.overrideParameters }} @@ -40,7 +39,7 @@ steps: - ${{ if eq(parameters.overrideParameters, '') }}: - powershell: ${{ parameters.executeAllSdlToolsScript }} - -GuardianPackageName Microsoft.Guardian.Cli.$(GuardianVersion) + -GuardianCliLocation $(GuardianCliLocation) -NugetPackageDirectory $(Build.SourcesDirectory)\.packages -AzureDevOpsAccessToken $(dn-bot-dotnet-build-rw-code-rw) ${{ parameters.additionalParameters }} diff --git a/eng/common/tools.ps1 b/eng/common/tools.ps1 index f83a748c37..aba6308ad3 100644 --- a/eng/common/tools.ps1 +++ b/eng/common/tools.ps1 @@ -365,8 +365,8 @@ function InitializeVisualStudioMSBuild([bool]$install, [object]$vsRequirements = # If the version of msbuild is going to be xcopied, # use this version. Version matches a package here: - # https://dev.azure.com/dnceng/public/_packaging?_a=package&feed=dotnet-eng&package=RoslynTools.MSBuild&protocolType=NuGet&version=17.1.0&view=overview - $defaultXCopyMSBuildVersion = '17.1.0' + # https://dev.azure.com/dnceng/public/_packaging?_a=package&feed=dotnet-eng&package=RoslynTools.MSBuild&protocolType=NuGet&version=17.2.1&view=overview + $defaultXCopyMSBuildVersion = '17.2.1' if (!$vsRequirements) { if (Get-Member -InputObject $GlobalJson.tools -Name 'vs') { diff --git a/global.json b/global.json index 2407ce7583..0564fdc24e 100644 --- a/global.json +++ b/global.json @@ -1,8 +1,8 @@ { "tools": { - "dotnet": "7.0.100-preview.5.22307.18" + "dotnet": "7.0.100-preview.7.22377.5" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "7.0.0-beta.22411.2" + "Microsoft.DotNet.Arcade.Sdk": "8.0.0-beta.22462.4" } }