Skip to content

Commit 0b498d7

Browse files
oleksandr-didykoleksandr-didyk
committed
feat: add tooling for generating text-only packages
1 parent c1bc837 commit 0b498d7

File tree

10 files changed

+210
-66
lines changed

10 files changed

+210
-66
lines changed

‎.gitattributes‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,3 +2,5 @@
22

33
# patch files need lf line-endings, even on Windows
44
*.patch text eol=lf
5+
6+
*.sh text eol=lf

‎generate.sh‎

100755100644
Lines changed: 59 additions & 45 deletions
Original file line numberDiff line numberDiff line change
@@ -4,33 +4,24 @@
44
# Stop script if command returns non-zero exit code.
55
set -e
66

7-
source="${BASH_SOURCE[0]}"
8-
97
DEFAULT_TFM=net6.0
108

11-
# resolve $SOURCE until the file is no longer a symlink
12-
while [[ -h $source ]]; do
13-
scriptroot="$( cd -P "$( dirname "$source" )" && pwd )"
14-
source="$(readlink "$source")"
15-
16-
# if $source was a relative symlink, we need to resolve it relative to the path where the
17-
# symlink file was located
18-
[[ $source != /* ]] && source="$scriptroot/$source"
19-
done
20-
9+
source="$(readlink -f "${BASH_SOURCE[0]}")"
2110
scriptroot="$( cd -P "$( dirname "$source" )" && pwd )"
2211

23-
GREEN='\033[0;32m'
2412
RED='\033[0;31m'
2513
NC='\033[0m'
2614

2715
usage() {
2816
echo "usage: $0 [options]"
2917
echo ""
30-
echo " Restores reference package(s) and dependencies and generates cs files and accompanying projects"
31-
echo " to build a reference package. The generated artifacts are added to the /src/ directory of this repo"
32-
echo " unless a different destination is specified. If a csv file of packages is specified, multiple"
33-
echo " packages and their dependencies are generated."
18+
echo " Generates a reference package or a text-only package from the specified packages and versions. The"
19+
echo " type of the generated package is controlled via the --type option."
20+
echo ""
21+
echo " Reference package generation will restore reference package(s) and dependencies and generate cs files"
22+
echo " and with accompanying projects into the specified destination ('./src/referencePackages/' by default)."
23+
echo " Text-only package generation will restore the specified package and copy the source-build-usable content"
24+
echo " into the provided directory ('./src/textOnlyPackages/' by default)."
3425
echo ""
3526
echo " Either --pkg or --pkgCsv must be specified"
3627
echo ""
@@ -42,21 +33,25 @@ usage() {
4233
echo " --pkgCsv <pathToCSV> A path to a csv file of packages to generate. Format is the same as the --pkg"
4334
echo " option above, one per line. If specified, the --pkg option is ignored."
4435
echo " --dest <pathToDestRepo> A path to the root of the repo to copy source into."
36+
echo " --type <packageType> Type of the package to generate. Accepted values: text | ref."
37+
echo " Default: ref."
38+
echo " --feeds <nugetFeeds> A semicolon-separated list of additional NuGet feeds to use during restore."
4539
echo ""
4640
}
4741

4842
packageVersion=
4943
defaultPathToCSV="$scriptroot/artifacts/targetPackages.csv"
5044
pathToCSV=
51-
pathToDestRepo="$scriptroot"
52-
positional_args=()
53-
requiredOptionSpecified=false
54-
while :; do
55-
if [ $# -le 0 ]; then
56-
break
57-
fi
58-
lowerI="$(echo "$1" | awk '{print tolower($0)}')"
59-
case $lowerI in
45+
pathToDestRepo=
46+
packageType="ref"
47+
48+
if [[ $# -le 0 ]]; then
49+
usage
50+
exit 0
51+
fi
52+
53+
while [ $# -gt 0 ]; do
54+
case "$1" in
6055
"-?"|-h|--help)
6156
usage
6257
exit 0
@@ -67,7 +62,6 @@ while :; do
6762
exit 1
6863
fi
6964
pathToCSV="$(cd "$(dirname "$2")"; pwd)/$(basename "$2")"
70-
requiredOptionSpecified=true
7165
shift
7266
;;
7367
--dest)
@@ -78,9 +72,21 @@ while :; do
7872
pathToDestRepo="$(cd -P "$2" && pwd)"
7973
shift
8074
;;
75+
--type)
76+
packageType="$2"
77+
if [[ ! "$packageType" =~ ^(text|ref)$ ]]; then
78+
echo -e "${RED}ERROR: unknown package type - $2${NC}"
79+
usage
80+
exit 1
81+
fi
82+
shift
83+
;;
8184
--pkg)
8285
packageVersion="$2"
83-
requiredOptionSpecified=true
86+
shift
87+
;;
88+
--feeds)
89+
feeds="$2"
8490
shift
8591
;;
8692
*)
@@ -93,7 +99,8 @@ while :; do
9399
shift
94100
done
95101

96-
if [ "$requiredOptionSpecified" != "true" ]; then
102+
if [ -z "$packageVersion" ] && [ -z "$pathToCSV" ]; then
103+
echo -e "${RED}ERROR: Either --pkg or --pkgCsv must be specified!${NC}"
97104
usage
98105
exit 1
99106
fi
@@ -107,6 +114,14 @@ if [ "$pathToCSV" == "" ]; then
107114
pathToCSV="$defaultPathToCSV"
108115
fi
109116

117+
if [[ -z "$pathToDestRepo" ]]; then
118+
if [ "$packageType" == "ref" ]; then
119+
pathToDestRepo="$scriptroot/src/referencePackages/src"
120+
else
121+
pathToDestRepo="$scriptroot/src/textOnlyPackages/src/"
122+
fi
123+
fi
124+
110125
# Generate restore projects for each target package
111126
# Packges are restored in their own project to eliminate any conflicts
112127
# between different versions of the same package.
@@ -120,8 +135,8 @@ if [ -d "$restoreProjectsDir" ]; then
120135
rm -rf "$restoreProjectsDir"
121136
fi
122137
mkdir -p "$restoreProjectsDir"
123-
[ ! -f $INPUT ] && { echo "$INPUT file not found"; exit 99; }
124-
while read pkgName pkgVersion tfm
138+
[ ! -f "$INPUT" ] && { echo "$INPUT file not found"; exit 99; }
139+
while read -r pkgName pkgVersion tfm
125140
do
126141
sed "s/##PackageName##/${pkgName}/g" "$targetPackageTemplate" > "$restoreProjectsDir/$pkgName.$pkgVersion.csproj"
127142
sed -i "s/##PackageVersion##/${pkgVersion}/g" "$restoreProjectsDir/$pkgName.$pkgVersion.csproj"
@@ -130,21 +145,20 @@ do
130145
else
131146
sed -i "s/##Tfm##/${DEFAULT_TFM}/g" "$restoreProjectsDir/$pkgName.$pkgVersion.csproj"
132147
fi
133-
done < $INPUT
148+
done < "$INPUT"
134149
IFS=$OLDIFS
135150

136-
# Build the projects to generate source and projects
137-
"$scriptroot/eng/common/build.sh" --build --restore -bl /p:GeneratePackageSource=true /p:GeneratorVersion=2 $@
151+
if [[ "$packageType" == "ref" ]]; then
152+
pathToRepoSrc="$pathToDestRepo"
138153

139-
# Copy source to destination
140-
pathToGeneratedSrc="$scriptroot/artifacts/generatedSrc"
141-
pathToRepoSrc="$pathToDestRepo/src/referencePackages/src"
154+
# Build the projects to generate source and projects
155+
"$scriptroot/eng/common/build.sh" --build --restore -bl /p:GeneratePackageSource=true /p:GeneratorVersion=2 /p:PathToRepoSrc="$pathToRepoSrc" /p:RestoreAdditionalProjectSources="\"$feeds\"" "$@"
156+
else
157+
# -p:RestoreAdditionalProjectSources -> specifies additional Nuget package sources, including feeds: https://docs.microsoft.com/en-us/nuget/reference/msbuild-targets#restore-properties
158+
# -p:RestoreUsingNuGetTargets -> switch that controls who will restore the project; if set to `false`, Arcade's Build.proj will execute the Restore target of the project itself.
159+
# Otherwise, Build.proj will trigger its own instance of Nuget.targets.Restore and delegate the restore of all applicable projects to it. In this case, the Restore target is
160+
# outside the project and we cannot hook our post-processing logic to it. For more info see:
161+
# https://github.com/dotnet/arcade/blob/5b838a3ed7f8e53c3082724605e5237fa614a43c/src/Microsoft.DotNet.Arcade.Sdk/tools/Build.proj#L227
142162

143-
pushd "$pathToGeneratedSrc"
144-
chmod 755 ./CopyProjects.sh
145-
./CopyProjects.sh "$pathToRepoSrc"
146-
popd
147-
148-
echo
149-
echo -e " Copied generated projects from ${GREEN}$pathToGeneratedSrc${NC} to ${GREEN}"$pathToRepoSrc"${NC}"
150-
echo
163+
"$scriptroot/eng/common/build.sh" -bl --restore --projects "$scriptroot/src/textOnlyPackageGenerator/PackageGenerator.proj" /p:RestoreUsingNuGetTargets=false /p:TextOnlyPackageDestination="$pathToDestRepo" /p:PathToCsv="$pathToCSV" /p:RestoreAdditionalProjectSources="\"$feeds\""
164+
fi

‎src/common/RestoreProjects.proj‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
<?xml version="1.0" encoding="utf-8"?>
2+
<Project>
3+
<ItemGroup>
4+
<RestoreProjects Include="$(RepoRoot)artifacts/targetRestoreProjects/*.csproj" />
5+
</ItemGroup>
6+
7+
<Target Name="ErrorEmptyRestoreProjects" BeforeTargets="RestoreAllProjects" Condition=" '@(RestoreProjects)' == '' ">
8+
<Error Text="There are no projects to restore." />
9+
</Target>
10+
11+
<Target Name="RestoreAllProjects" BeforeTargets="_IsProjectRestoreSupported">
12+
<Message Importance="High" Text="==> Restoring all target packages to $(TargetPackagesPath)" />
13+
14+
<Exec Command="$(DOTNET_INSTALL_DIR)/dotnet restore %(RestoreProjects.Identity) /p:TargetPackagesPath=$(TargetPackagesPath) /p:RestoreAdditionalProjectSources=$(RestoreAdditionalProjectSources)" />
15+
</Target>
16+
</Project>

‎src/referencePackageSourceGenerator/GenerateProjects/GenerateProjects.proj‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,24 @@
2929
GeneratorVersion="$(GeneratorVersion)" />
3030
</Target>
3131

32+
<Target Name="CopyGeneratedSource" AfterTargets="CoreCompile">
33+
<ItemGroup>
34+
<GeneratedSourcePackages Include="$(GeneratedSourcePath)\**\Directory.Build.props" />
35+
<GeneratedSourceFiles Include="%(GeneratedSourcePackages.RelativeDir)\**\*.*" >
36+
<!-- Due to the usage of a glob pattern in 'Include', 'RecursiveDir' will only contain the path for the /**/ pattern
37+
https://docs.microsoft.com/en-us/visualstudio/msbuild/copy-task?view=vs-2022#parameters
38+
39+
As such, the parent directory containing the name of the package must be resolved manually.
40+
For 'GetFileName' to return the name of the directory, the trailing slash in 'RecursiveDir' has to be trimmed out -->
41+
<ParentDirectory>$([System.IO.Path]::GetFileName($([System.IO.Path]::TrimEndingDirectorySeparator(%(GeneratedSourcePackages.RelativeDir)))))</ParentDirectory>
42+
</GeneratedSourceFiles>
43+
</ItemGroup>
44+
45+
<Copy SourceFiles="@(GeneratedSourceFiles)" DestinationFiles="@(GeneratedSourceFiles->'$(PathToRepoSrc)/%(ParentDirectory)/%(RecursiveDir)%(Filename)%(Extension)')" />
46+
47+
<Message Importance="High" Text="==> Copied generated projects from $(GeneratedSourcePath) to $(PathToRepoSrc)" />
48+
</Target>
49+
3250
<Target Name="CreateManifestResourceNames" />
3351
<Import Project="$(ArtifactsDir)toolset/Common/Tools.proj.nuget.g.targets" />
3452
</Project>

‎src/referencePackageSourceGenerator/GenerateSource/GenerateSource.proj‎

Lines changed: 1 addition & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -7,21 +7,7 @@
77
<RestorePackagesPath>$(TargetPackagesPath)</RestorePackagesPath>
88
</PropertyGroup>
99

10-
<ItemGroup>
11-
<RestoreProjects Include="$(RepoRoot)artifacts/targetRestoreProjects/*.csproj"/>
12-
</ItemGroup>
13-
14-
<Target Name="ErrorEmptyRestoreProjects"
15-
BeforeTargets="RestoreAllProjects"
16-
Condition=" '@(RestoreProjects)' == '' ">
17-
<Error Text="There are no restore projects to process." />
18-
</Target>
19-
20-
<Target Name="RestoreAllProjects" BeforeTargets="_IsProjectRestoreSupported">
21-
<Message Importance="High" Text="==> Restoring all target packages to $(TargetPackagesPath)" />
22-
23-
<Exec Command="$(DOTNET_INSTALL_DIR)/dotnet restore %(RestoreProjects.Identity) /p:TargetPackagesPath=$(TargetPackagesPath)" />
24-
</Target>
10+
<Import Project="../../common/RestoreProjects.proj"/>
2511

2612
<Target Name="CopyNuspecFiles" BeforeTargets="CoreCompile">
2713
<Message Importance="High" Text="==> Copying all target package nuspec files to generated source" />

‎src/referencePackageSourceGenerator/Tasks/GenerateProjects.cs‎

Lines changed: 0 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -48,12 +48,6 @@ public override bool Execute()
4848
// Second, setup dependencies between all generated project data
4949
ProjectData.GenerateAllDependencies();
5050

51-
// Create a script file to copy the newly added project source to the output directory
52-
string projectSourceCopyCommands = PackageData.GetAll().Select(p => $"cp -r --parents ./{p.RelativePath}/* $1").Aggregate((a, b) => a + "\n" + b);
53-
projectSourceCopyCommands += "\nfind . -type f -iname Directory.Build.props -exec cp --parents {} $1 \\;";
54-
55-
File.WriteAllText(Path.Combine(SrcPath, "CopyProjects.sh"), projectSourceCopyCommands);
56-
5751
foreach (var pkgData in PackageData.GetAll())
5852
{
5953
string pkgProjectOutput = pkgProjectTemplate;

‎src/textOnlyPackageGenerator/PackageGenerator.proj‎

Lines changed: 112 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,112 @@
1+
<?xml version="1.0" encoding="utf-8"?>
2+
<Project Sdk="Microsoft.NET.Sdk">
3+
4+
<PropertyGroup>
5+
<TargetFramework>net7.0</TargetFramework>
6+
<TargetPackagesPath>$(ArtifactsDir)textOnlyPackages/</TargetPackagesPath>
7+
<CsProjTemplatePath>./textOnlyPackage.csproj.template</CsProjTemplatePath>
8+
</PropertyGroup>
9+
10+
<Import Project="../common/RestoreProjects.proj" />
11+
12+
<Target Name="GetPackageInfo" AfterTargets="Restore">
13+
<Error Text="'PathToCsv' property was not set!" Condition="$(PathToCsv) == ''" />
14+
15+
<ReadLinesFromFile File="$(PathToCsv)">
16+
<Output TaskParameter="Lines" ItemName="CsvLines" />
17+
</ReadLinesFromFile>
18+
19+
<ItemGroup>
20+
<PackageInfo Include="@(CsvLines)" Exclude="">
21+
<PackageName>$([System.String]::Copy('%(CsvLines.Identity)').Split(',')[0])</PackageName>
22+
<PackageVersion>$([System.String]::Copy('%(CsvLines.Identity)').Split(',')[1])</PackageVersion>
23+
</PackageInfo>
24+
</ItemGroup>
25+
</Target>
26+
27+
<!-- see ./generate.sh for 'AfterTargets' justification -->
28+
<Target Name="CopyTextOnlyPackages" AfterTargets="Restore">
29+
<MSBuild Projects="$(MSBuildProjectFile)" Targets="CopyTextOnlyPackage" Properties="PackageName=%(PackageInfo.PackageName);PackageVersion=%(PackageInfo.PackageVersion)" />
30+
</Target>
31+
32+
<Target Name="CopyTextOnlyPackage">
33+
<PropertyGroup>
34+
<RestoredPackagePath>$(TargetPackagesPath)$(PackageName.ToLower())</RestoredPackagePath>
35+
<TextOnlyPackageDirectory>$(PackageName.ToLower())</TextOnlyPackageDirectory>
36+
<CsProjFileName>$(PackageName.ToLower()).$(PackageVersion).csproj</CsProjFileName>
37+
</PropertyGroup>
38+
39+
<ItemGroup>
40+
<TextOnlyPackageContent Include="$(RestoredPackagePath)/**/*" Exclude="
41+
$(RestoredPackagePath)/**/.nupkg.metadata;
42+
$(RestoredPackagePath)/**/.signature.p7s;
43+
$(RestoredPackagePath)/**/*.nupkg;
44+
$(RestoredPackagePath)/**/*.nupkg.sha512;" />
45+
</ItemGroup>
46+
47+
<Error Text="Package $(PackageName) was not restored!" Condition=" '@(TextOnlyPackageContent)' == '' " />
48+
49+
<!-- Verify that invalid assets were not copied into text-only packs -->
50+
<ItemGroup>
51+
<!-- ttf, woff, woff2, eot are permissible font-related content -->
52+
<AllowedTextOnlyExtensions Include="
53+
.-;
54+
._;
55+
.bowerrc;
56+
.config;
57+
.cs;
58+
.cshtml;
59+
.csproj;
60+
.css;
61+
.db;
62+
.editorconfig;
63+
.env;
64+
.env.development;
65+
.eot;
66+
.fs;
67+
.fsproj;
68+
.gitignore;
69+
.gitkeep;
70+
.html;
71+
.ico;
72+
.js;
73+
.json;
74+
.map;
75+
.md;
76+
.nuspec;
77+
.otf;
78+
.png;
79+
.props;
80+
.proto;
81+
.razor;
82+
.resx;
83+
.rtf;
84+
.sln;
85+
.svg;
86+
.targets;
87+
.ts;
88+
.ttf;
89+
.tsx;
90+
.txt;
91+
.vb;
92+
.vbproj;
93+
.woff;
94+
.woff2;
95+
.xlf;
96+
browserslist;
97+
browserslistrc;
98+
LICENSE;" />
99+
<UnsupportedTextOnlyPackageContent Include="@(TextOnlyPackageContent)" />
100+
<UnsupportedTextOnlyPackageContent Remove="$(RestoredPackagePath)/**/*$([System.String]::Copy('%(AllowedTextOnlyExtensions.Identity)').ToLowerInvariant())" />
101+
<UnsupportedTextOnlyPackageContent Remove="$(RestoredPackagePath)/**/*$([System.String]::Copy('%(AllowedTextOnlyExtensions.Identity)').ToUpperInvariant())" />
102+
</ItemGroup>
103+
104+
<Error Text="Unsupported content found in text-only packages: @(UnsupportedTextOnlyPackageContent)" Condition=" '@(UnsupportedTextOnlyPackageContent)' != '' " />
105+
<Error Text="'TextOnlyPackageDestination' property was not set!" Condition="$(TextOnlyPackageDestination) == ''" />
106+
107+
<Copy SourceFiles="@(TextOnlyPackageContent)" DestinationFiles="@(TextOnlyPackageContent->'$(TextOnlyPackageDestination)/$(TextOnlyPackageDirectory)/%(RecursiveDir)%(Filename)%(Extension)')" />
108+
109+
<Copy SourceFiles="$(CsProjTemplatePath)" DestinationFiles="$(TextOnlyPackageDestination)/$(TextOnlyPackageDirectory)/$(PackageVersion)/$(CsProjFileName)" />
110+
111+
</Target>
112+
</Project>

src/textOnlyPackages/src/microsoft.codeanalysis.collections/4.2.0-1.22102.8/microsoft.codeanalysis.collections-4.2.0-1.22102.8.csproj renamed to src/textOnlyPackageGenerator/textOnlyPackage.csproj.template

File renamed without changes.

‎src/textOnlyPackages/src/microsoft.codeanalysis.collections/4.2.0-1.22102.8/microsoft.codeanalysis.collections.4.2.0-1.22102.8.csproj‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
<Project Sdk="Microsoft.NET.Sdk" />

‎src/textOnlyPackages/src/microsoft.netcore.platforms/2.1.9/microsoft.netcore.platforms.2.1.9.csproj‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
<Project Sdk="Microsoft.NET.Sdk" />

0 commit comments

Comments
 (0)