Skip to content

Commit f53bf43

Browse files
oleksandr-didykoleksandr-didyk
committed
feat: add tooling for generating text-only packages
1 parent c1bc837 commit f53bf43

File tree

8 files changed

+203
-58
lines changed

8 files changed

+203
-58
lines changed

‎.gitattributes‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,3 +2,5 @@
22

33
# patch files need lf line-endings, even on Windows
44
*.patch text eol=lf
5+
6+
*.sh text eol=lf

‎generate.sh‎

100755100644
Lines changed: 64 additions & 43 deletions
Original file line numberDiff line numberDiff line change
@@ -4,20 +4,9 @@
44
# Stop script if command returns non-zero exit code.
55
set -e
66

7-
source="${BASH_SOURCE[0]}"
8-
97
DEFAULT_TFM=net6.0
108

11-
# resolve $SOURCE until the file is no longer a symlink
12-
while [[ -h $source ]]; do
13-
scriptroot="$( cd -P "$( dirname "$source" )" && pwd )"
14-
source="$(readlink "$source")"
15-
16-
# if $source was a relative symlink, we need to resolve it relative to the path where the
17-
# symlink file was located
18-
[[ $source != /* ]] && source="$scriptroot/$source"
19-
done
20-
9+
source="$(readlink -f "${BASH_SOURCE[0]}")"
2110
scriptroot="$( cd -P "$( dirname "$source" )" && pwd )"
2211

2312
GREEN='\033[0;32m'
@@ -27,10 +16,13 @@ NC='\033[0m'
2716
usage() {
2817
echo "usage: $0 [options]"
2918
echo ""
30-
echo " Restores reference package(s) and dependencies and generates cs files and accompanying projects"
31-
echo " to build a reference package. The generated artifacts are added to the /src/ directory of this repo"
32-
echo " unless a different destination is specified. If a csv file of packages is specified, multiple"
33-
echo " packages and their dependencies are generated."
19+
echo " Generates a reference package or a text-only package from the specified packages and versions. The"
20+
echo " type of the generated package is controlled via the --textOnly option."
21+
echo ""
22+
echo " Reference package generation will restore reference package(s) and dependencies and generate cs files"
23+
echo " and with accompanying projects into the specified destination ('./src/referencePackages/' by default)."
24+
echo " Text-only package generation will restore the specified package and copy the source-build-usable content"
25+
echo " into the provided directory ('./src/textOnlyPackages/' by default)."
3426
echo ""
3527
echo " Either --pkg or --pkgCsv must be specified"
3628
echo ""
@@ -42,21 +34,24 @@ usage() {
4234
echo " --pkgCsv <pathToCSV> A path to a csv file of packages to generate. Format is the same as the --pkg"
4335
echo " option above, one per line. If specified, the --pkg option is ignored."
4436
echo " --dest <pathToDestRepo> A path to the root of the repo to copy source into."
37+
echo " --textOnly Generate text-only packages."
38+
echo " --feeds <nugetFeeds> A semicolon-separated list of additional NuGet feeds to use during restore."
4539
echo ""
4640
}
4741

4842
packageVersion=
4943
defaultPathToCSV="$scriptroot/artifacts/targetPackages.csv"
5044
pathToCSV=
51-
pathToDestRepo="$scriptroot"
52-
positional_args=()
53-
requiredOptionSpecified=false
54-
while :; do
55-
if [ $# -le 0 ]; then
56-
break
57-
fi
58-
lowerI="$(echo "$1" | awk '{print tolower($0)}')"
59-
case $lowerI in
45+
pathToDestRepo=
46+
generateTextOnly=false
47+
48+
if [[ $# -le 0 ]]; then
49+
usage
50+
exit 0
51+
fi
52+
53+
while [ $# -gt 0 ]; do
54+
case "$1" in
6055
"-?"|-h|--help)
6156
usage
6257
exit 0
@@ -67,7 +62,6 @@ while :; do
6762
exit 1
6863
fi
6964
pathToCSV="$(cd "$(dirname "$2")"; pwd)/$(basename "$2")"
70-
requiredOptionSpecified=true
7165
shift
7266
;;
7367
--dest)
@@ -78,9 +72,15 @@ while :; do
7872
pathToDestRepo="$(cd -P "$2" && pwd)"
7973
shift
8074
;;
75+
--textOnly)
76+
generateTextOnly=true
77+
;;
8178
--pkg)
8279
packageVersion="$2"
83-
requiredOptionSpecified=true
80+
shift
81+
;;
82+
--feeds)
83+
feeds="$2"
8484
shift
8585
;;
8686
*)
@@ -93,7 +93,8 @@ while :; do
9393
shift
9494
done
9595

96-
if [ "$requiredOptionSpecified" != "true" ]; then
96+
if [ -z "$packageVersion" ] && [ -z "$pathToCSV" ]; then
97+
echo -e "${RED}ERROR: Either --pkg or --pkgCsv must be specified!${NC}"
9798
usage
9899
exit 1
99100
fi
@@ -120,8 +121,8 @@ if [ -d "$restoreProjectsDir" ]; then
120121
rm -rf "$restoreProjectsDir"
121122
fi
122123
mkdir -p "$restoreProjectsDir"
123-
[ ! -f $INPUT ] && { echo "$INPUT file not found"; exit 99; }
124-
while read pkgName pkgVersion tfm
124+
[ ! -f "$INPUT" ] && { echo "$INPUT file not found"; exit 99; }
125+
while read -r pkgName pkgVersion tfm
125126
do
126127
sed "s/##PackageName##/${pkgName}/g" "$targetPackageTemplate" > "$restoreProjectsDir/$pkgName.$pkgVersion.csproj"
127128
sed -i "s/##PackageVersion##/${pkgVersion}/g" "$restoreProjectsDir/$pkgName.$pkgVersion.csproj"
@@ -130,21 +131,41 @@ do
130131
else
131132
sed -i "s/##Tfm##/${DEFAULT_TFM}/g" "$restoreProjectsDir/$pkgName.$pkgVersion.csproj"
132133
fi
133-
done < $INPUT
134+
done < "$INPUT"
134135
IFS=$OLDIFS
135136

136-
# Build the projects to generate source and projects
137-
"$scriptroot/eng/common/build.sh" --build --restore -bl /p:GeneratePackageSource=true /p:GeneratorVersion=2 $@
137+
if [[ "$generateTextOnly" != "true" ]]; then
138+
# Build the projects to generate source and projects
139+
"$scriptroot/eng/common/build.sh" --build --restore -bl /p:GeneratePackageSource=true /p:GeneratorVersion=2 -p:RestoreAdditionalProjectSources="\"$feeds\"" "$@"
138140

139-
# Copy source to destination
140-
pathToGeneratedSrc="$scriptroot/artifacts/generatedSrc"
141-
pathToRepoSrc="$pathToDestRepo/src/referencePackages/src"
141+
# Copy source to destination
142+
pathToGeneratedSrc="$scriptroot/artifacts/generatedSrc/"
143+
if [ -z "$pathToDestRepo" ]; then
144+
pathToRepoSrc="$scriptroot/src/referencePackages/src/"
145+
else
146+
pathToRepoSrc="$pathToDestRepo"
147+
fi
142148

143-
pushd "$pathToGeneratedSrc"
144-
chmod 755 ./CopyProjects.sh
145-
./CopyProjects.sh "$pathToRepoSrc"
146-
popd
149+
pushd "$pathToGeneratedSrc"
150+
chmod +x ./CopyProjects.sh
151+
./CopyProjects.sh "$pathToRepoSrc"
152+
popd
153+
154+
echo
155+
echo -e " Copied generated projects from ${GREEN}$pathToGeneratedSrc${NC} to ${GREEN}'$pathToRepoSrc'${NC}"
156+
echo
157+
else
158+
if [ -z "$pathToDestRepo" ]; then
159+
textOnlyPackageDestination="$scriptroot/src/textOnlyPackages/src/"
160+
else
161+
textOnlyPackageDestination="$pathToDestRepo"
162+
fi
147163

148-
echo
149-
echo -e " Copied generated projects from ${GREEN}$pathToGeneratedSrc${NC} to ${GREEN}"$pathToRepoSrc"${NC}"
150-
echo
164+
# -p:RestoreAdditionalProjectSources -> specifies additional Nuget package sources, including feeds: https://docs.microsoft.com/en-us/nuget/reference/msbuild-targets#restore-properties
165+
# -p:RestoreUsingNuGetTargets -> switch that controls who will restore the project; if set to `false`, Arcade's Build.proj will execute the Restore target of the project itself.
166+
# Otherwise, Build.proj will trigger its own instance of Nuget.targets.Restore and delegate the restore of all applicable projects to it. In this case, the Restore target is
167+
# outside the project and we cannot hook our post-processing logic to it. For more info see:
168+
# https://github.com/dotnet/arcade/blob/5b838a3ed7f8e53c3082724605e5237fa614a43c/src/Microsoft.DotNet.Arcade.Sdk/tools/Build.proj#L227
169+
170+
"$scriptroot/eng/common/build.sh" -bl --restore --projects "$scriptroot/src/textOnlyPackageGenerator/PackageGenerator.proj" -p:RestoreUsingNuGetTargets=false -p:TextOnlyPackageDestination="$textOnlyPackageDestination" -p:PathToCsv="$pathToCSV" -p:RestoreAdditionalProjectSources="\"$feeds\""
171+
fi

‎src/common/RestoreProjects.proj‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
<?xml version="1.0" encoding="utf-8"?>
2+
<Project>
3+
<ItemGroup>
4+
<RestoreProjects Include="$(RepoRoot)artifacts/targetRestoreProjects/*.csproj" />
5+
</ItemGroup>
6+
7+
<Target Name="ErrorEmptyRestoreProjects" BeforeTargets="RestoreAllProjects" Condition=" '@(RestoreProjects)' == '' ">
8+
<Error Text="There are no projects to restore." />
9+
</Target>
10+
11+
<Target Name="RestoreAllProjects" BeforeTargets="_IsProjectRestoreSupported">
12+
<Message Importance="High" Text="==> Restoring all target packages to $(TargetPackagesPath)" />
13+
14+
<Exec Command="$(DOTNET_INSTALL_DIR)/dotnet restore %(RestoreProjects.Identity) /p:TargetPackagesPath=$(TargetPackagesPath) -p:RestoreAdditionalProjectSources=$(RestoreAdditionalProjectSources)" />
15+
</Target>
16+
</Project>

‎src/referencePackageSourceGenerator/GenerateSource/GenerateSource.proj‎

Lines changed: 1 addition & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -7,21 +7,7 @@
77
<RestorePackagesPath>$(TargetPackagesPath)</RestorePackagesPath>
88
</PropertyGroup>
99

10-
<ItemGroup>
11-
<RestoreProjects Include="$(RepoRoot)artifacts/targetRestoreProjects/*.csproj"/>
12-
</ItemGroup>
13-
14-
<Target Name="ErrorEmptyRestoreProjects"
15-
BeforeTargets="RestoreAllProjects"
16-
Condition=" '@(RestoreProjects)' == '' ">
17-
<Error Text="There are no restore projects to process." />
18-
</Target>
19-
20-
<Target Name="RestoreAllProjects" BeforeTargets="_IsProjectRestoreSupported">
21-
<Message Importance="High" Text="==> Restoring all target packages to $(TargetPackagesPath)" />
22-
23-
<Exec Command="$(DOTNET_INSTALL_DIR)/dotnet restore %(RestoreProjects.Identity) /p:TargetPackagesPath=$(TargetPackagesPath)" />
24-
</Target>
10+
<Import Project="../../common/RestoreProjects.proj"/>
2511

2612
<Target Name="CopyNuspecFiles" BeforeTargets="CoreCompile">
2713
<Message Importance="High" Text="==> Copying all target package nuspec files to generated source" />

‎src/textOnlyPackageGenerator/PackageGenerator.proj‎

Lines changed: 118 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,118 @@
1+
<?xml version="1.0" encoding="utf-8"?>
2+
<Project Sdk="Microsoft.NET.Sdk">
3+
4+
<PropertyGroup>
5+
<TargetFramework>net7.0</TargetFramework>
6+
<TargetPackagesPath>$(ArtifactsDir)textOnlyPackages/</TargetPackagesPath>
7+
<CsProjTemplatePath>./textOnlyPackage.csproj.template</CsProjTemplatePath>
8+
</PropertyGroup>
9+
10+
<Import Project="../common/RestoreProjects.proj"/>
11+
12+
<Target Name="GetPackageInfo" AfterTargets="Restore">
13+
<ReadLinesFromFile File="$(PathToCsv)">
14+
<Output TaskParameter="Lines" ItemName="CsvLines"/>
15+
</ReadLinesFromFile>
16+
17+
<ItemGroup>
18+
<PackageInfo Include="@(CsvLines)" Exclude="">
19+
<PackageName>$([System.String]::Copy('%(CsvLines.Identity)').Split(',')[0])</PackageName>
20+
<PackageVersion>$([System.String]::Copy('%(CsvLines.Identity)').Split(',')[1])</PackageVersion>
21+
</PackageInfo>
22+
</ItemGroup>
23+
</Target>
24+
25+
<!-- see ./generate.sh for 'AfterTargets' justification -->
26+
<Target Name="CopyTextOnlyPackages" AfterTargets="Restore">
27+
<MSBuild
28+
Projects="$(MSBuildProjectFile)"
29+
Targets="CopyTextOnlyPackage"
30+
Properties="PackageName=%(PackageInfo.PackageName);PackageVersion=%(PackageInfo.PackageVersion)"/>
31+
</Target>
32+
33+
<Target Name="CopyTextOnlyPackage">
34+
<PropertyGroup>
35+
<RestoredPackagePath>$(TargetPackagesPath)$(PackageName.ToLower())</RestoredPackagePath>
36+
<TextOnlyPackageDirectory>$(PackageName.ToLower())</TextOnlyPackageDirectory>
37+
<CsProjFileName>$(PackageName.ToLower()).$(PackageVersion).csproj</CsProjFileName>
38+
</PropertyGroup>
39+
40+
<ItemGroup>
41+
<TextOnlyPackageContent
42+
Include="$(RestoredPackagePath)/**/*"
43+
Exclude="
44+
$(RestoredPackagePath)/**/.nupkg.metadata;
45+
$(RestoredPackagePath)/**/.signature.p7s;
46+
$(RestoredPackagePath)/**/*.nupkg;
47+
$(RestoredPackagePath)/**/*.nupkg.sha512;"/>
48+
</ItemGroup>
49+
50+
<Error Text="Package $(PackageName) was not restored!" Condition=" '@(TextOnlyPackageContent)' == '' " />
51+
52+
<!-- Verify that invalid assets were not copied into text-only packs -->
53+
<ItemGroup>
54+
<!-- ttf, woff, woff2, eot are permissible font-related content -->
55+
<AllowedTextOnlyExtensions Include="
56+
.-;
57+
._;
58+
.bowerrc;
59+
.config;
60+
.cs;
61+
.cshtml;
62+
.csproj;
63+
.css;
64+
.db;
65+
.editorconfig;
66+
.env;
67+
.env.development;
68+
.eot;
69+
.fs;
70+
.fsproj;
71+
.gitignore;
72+
.gitkeep;
73+
.html;
74+
.ico;
75+
.js;
76+
.json;
77+
.map;
78+
.md;
79+
.nuspec;
80+
.otf;
81+
.png;
82+
.props;
83+
.proto;
84+
.razor;
85+
.resx;
86+
.rtf;
87+
.sln;
88+
.svg;
89+
.targets;
90+
.ts;
91+
.ttf;
92+
.tsx;
93+
.txt;
94+
.vb;
95+
.vbproj;
96+
.woff;
97+
.woff2;
98+
.xlf;
99+
browserslist;
100+
browserslistrc;
101+
LICENSE;"/>
102+
<UnsupportedTextOnlyPackageContent Include="@(TextOnlyPackageContent)" />
103+
<UnsupportedTextOnlyPackageContent Remove="$(RestoredPackagePath)/**/*$([System.String]::Copy('%(AllowedTextOnlyExtensions.Identity)').ToLowerInvariant())" />
104+
<UnsupportedTextOnlyPackageContent Remove="$(RestoredPackagePath)/**/*$([System.String]::Copy('%(AllowedTextOnlyExtensions.Identity)').ToUpperInvariant())" />
105+
</ItemGroup>
106+
107+
<Error Text="Unsupported content found in text-only packages: @(UnsupportedTextOnlyPackageContent)" Condition=" '@(UnsupportedTextOnlyPackageContent)' != '' " />
108+
109+
<Copy
110+
SourceFiles="@(TextOnlyPackageContent)"
111+
DestinationFiles="@(TextOnlyPackageContent->'$(TextOnlyPackageDestination)/$(TextOnlyPackageDirectory)/%(RecursiveDir)%(Filename)%(Extension)')"/>
112+
113+
<Copy
114+
SourceFiles="$(CsProjTemplatePath)"
115+
DestinationFiles="$(TextOnlyPackageDestination)/$(TextOnlyPackageDirectory)/$(PackageVersion)/$(CsProjFileName)"/>
116+
117+
</Target>
118+
</Project>

src/textOnlyPackages/src/microsoft.codeanalysis.collections/4.2.0-1.22102.8/microsoft.codeanalysis.collections-4.2.0-1.22102.8.csproj renamed to src/textOnlyPackageGenerator/textOnlyPackage.csproj.template

File renamed without changes.

‎src/textOnlyPackages/src/microsoft.codeanalysis.collections/4.2.0-1.22102.8/microsoft.codeanalysis.collections.4.2.0-1.22102.8.csproj‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
<Project Sdk="Microsoft.NET.Sdk" />

‎src/textOnlyPackages/src/microsoft.netcore.platforms/2.1.9/microsoft.netcore.platforms.2.1.9.csproj‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
<Project Sdk="Microsoft.NET.Sdk" />

0 commit comments

Comments
 (0)