Infra for regeneration of third-party-notices file

eng/Subsets.props

@@ -146,6 +146,7 @@
     <!-- Utility -->
     <SubsetName Include="publish" OnDemand="true" Description="Generate asset manifests and prepare to publish to BAR." />
     <SubsetName Include="RegenerateDownloadTable" OnDemand="true" Description="Regenerates the nightly build download table" />
+    <SubsetName Include="RegenerateThirdPartyNotices" Category="" OnDemand="true" Description="Regenerates the THIRD-PARTY-NOTICES.TXT file based on other repos' TPN files." />
 
   </ItemGroup>
 
@@ -360,6 +361,10 @@
     <ProjectToBuild Include="$(RepositoryEngineeringDir)regenerate-download-table.proj" Pack="true" BuildInParallel="false" />
   </ItemGroup>
 
+  <ItemGroup Condition="$(_subset.Contains('regeneratethirdpartynotices'))">
+    <ProjectToBuild Include="$(RepositoryEngineeringDir)regenerate-third-party-notices.proj" Pack="false" BuildInParallel="false" />
+  </ItemGroup>
+
   <!-- Set default configurations. -->
   <ItemGroup>
     <ProjectToBuild Update="@(ProjectToBuild)">

eng/regenerate-third-party-notices.proj

@@ -0,0 +1,53 @@
+<Project>
+  <Import Project="$([MSBuild]::GetDirectoryNameOfFileAbove($(MSBuildThisFileDirectory).., Directory.Build.props))\Directory.Build.props" />
+
+  <UsingTask TaskName="RegenerateThirdPartyNotices" AssemblyFile="$(InstallerTasksAssemblyPath)" />
+
+  <Target Name="Build">
+    <PropertyGroup>
+      <TpnFile>$(RepoRoot)src/installer/pkg/THIRD-PARTY-NOTICES.TXT</TpnFile>
+    </PropertyGroup>
+
+    <!--
+      Repo configuration. Upstreams, but also more: the TPN in Core-Setup serves many repos outside
+      its graph, because Core-Setup produces the installer that ends up placing the single TPN file
+      in the dotnet home directory.
+    -->
+    <ItemGroup>
+      <TpnRepo Include="dotnet/runtime" />
+      <TpnRepo Include="dotnet/aspnetcore" />
+      <TpnRepo Include="dotnet/installer" />
+      <TpnRepo Include="dotnet/roslyn-analyzers" />
+      <TpnRepo Include="dotnet/templating" />
+      <TpnRepo Include="dotnet/winforms" />
+      <TpnRepo Include="dotnet/wpf" />
+
+      <!--
+        Additional repos that should be included but don't have any third-party-notices files:
+          dotnet/efcore
+          dotnet/extensions
+          dotnet/icu
+          dotnet/sdk
+          dotnet/windowsdesktop
+          mono/linker
+      -->
+
+      <TpnRepo Condition="'%(TpnRepo.Branch)' == ''" Branch="master" />
+
+      <PotentialTpnPath Include="THIRD-PARTY-NOTICES.TXT" />
+      <PotentialTpnPath Include="THIRD-PARTY-NOTICES.txt" />
+      <PotentialTpnPath Include="THIRD-PARTY-NOTICES" />
+      <PotentialTpnPath Include="THIRDPARTYNOTICES.TXT" />
+      <PotentialTpnPath Include="THIRDPARTYNOTICES.txt" />
+    </ItemGroup>
+
+    <RegenerateThirdPartyNotices
+      TpnFile="$(TpnFile)"
+      PotentialTpnPaths="@(PotentialTpnPath)"
+      TpnRepos="@(TpnRepo)" />
+
+    <Message Text="$(MSBuildProjectName) -> $(TpnFile)" Importance="High" />
+  </Target>
+
+  <Import Project="$([MSBuild]::GetDirectoryNameOfFileAbove($(MSBuildThisFileDirectory).., Directory.Build.targets))\Directory.Build.targets" />
+</Project>

src/tasks/installer.tasks/StaticFileRegeneration/EnumerableExtensions.cs

@@ -0,0 +1,15 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+
+namespace Microsoft.DotNet.Build.Tasks
+{
+    internal static class EnumerableExtensions
+    {
+        public static IEnumerable<T> NullAsEmpty<T>(this IEnumerable<T> source)
+        {
+            return source ?? Enumerable.Empty<T>();
+        }
+    }
+}

src/tasks/installer.tasks/StaticFileRegeneration/RegenerateThirdPartyNotices.cs

@@ -0,0 +1,190 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+using Microsoft.Build.Framework;
+using System;
+using System.IO;
+using System.Linq;
+using System.Net;
+using System.Net.Http;
+using System.Threading.Tasks;
+
+namespace Microsoft.DotNet.Build.Tasks
+{
+    public class RegenerateThirdPartyNotices : BuildTask
+    {
+        private const string GitHubRawContentBaseUrl = "https://raw.githubusercontent.com/";
+
+        private static readonly char[] NewlineChars = { '\n', '\r' };
+
+        /// <summary>
+        /// The Third Party Notices file (TPN file) to regenerate.
+        /// </summary>
+        [Required]
+        public string TpnFile { get; set; }
+
+        /// <summary>
+        /// Potential names for the file in various repositories. Each one is tried for each repo.
+        /// </summary>
+        [Required]
+        public string[] PotentialTpnPaths { get; set; }
+
+        /// <summary>
+        /// %(Identity): The "{organization}/{name}" of a repo to gather TPN info from.
+        /// %(Branch): The branch to pull from.
+        /// </summary>
+        [Required]
+        public ITaskItem[] TpnRepos { get; set; }
+
+        public override bool Execute()
+        {
+            using (var client = new HttpClient())
+            {
+                ExecuteAsync(client).Wait();
+            }
+
+            return !Log.HasLoggedErrors;
+        }
+
+        public async Task ExecuteAsync(HttpClient client)
+        {
+            var results = await Task.WhenAll(TpnRepos
+                .SelectMany(item =>
+                {
+                    string repo = item.ItemSpec;
+                    string branch = item.GetMetadata("Branch")
+                        ?? throw new ArgumentException($"{item.ItemSpec} specifies no Branch.");
+
+                    return PotentialTpnPaths.Select(path => new
+                    {
+                        Repo = repo,
+                        Branch = branch,
+                        PotentialPath = path,
+                        Url = $"{GitHubRawContentBaseUrl}{repo}/{branch}/{path}"
+                    });
+                })
+                .Select(async c =>
+                {
+                    TpnDocument content = null;
+
+                    Log.LogMessage(
+                        MessageImportance.High,
+                        $"Getting {c.Url}");
+
+                    HttpResponseMessage response = await client.GetAsync(c.Url);
+
+                    if (response.StatusCode != HttpStatusCode.NotFound)
+                    {
+                        response.EnsureSuccessStatusCode();
+
+                        string tpnContent = await response.Content.ReadAsStringAsync();
+
+                        try
+                        {
+                            content = TpnDocument.Parse(tpnContent.Split(NewlineChars));
+                        }
+                        catch
+                        {
+                            Log.LogError($"Failed to parse response from {c.Url}");
+                            throw;
+                        }
+
+                        Log.LogMessage($"Got content from URL: {c.Url}");
+                    }
+                    else
+                    {
+                        Log.LogMessage($"Checked for content, but does not exist: {c.Url}");
+                    }
+
+                    return new
+                    {
+                        c.Repo,
+                        c.Branch,
+                        c.PotentialPath,
+                        c.Url,
+                        Content = content
+                    };
+                }));
+
+            foreach (var r in results.Where(r => r.Content != null).OrderBy(r => r.Repo))
+            {
+                Log.LogMessage(
+                    MessageImportance.High,
+                    $"Found TPN: {r.Repo} [{r.Branch}] {r.PotentialPath}");
+            }
+
+            // Ensure we found one (and only one) TPN file for each repo.
+            foreach (var miscount in results
+                .GroupBy(r => r.Repo)
+                .Where(g => g.Count(r => r.Content != null) != 1))
+            {
+                Log.LogError($"Unable to find exactly one TPN for {miscount.Key}");
+            }
+
+            if (Log.HasLoggedErrors)
+            {
+                return;
+            }
+
+            TpnDocument existingTpn = TpnDocument.Parse(File.ReadAllLines(TpnFile));
+
+            Log.LogMessage(
+                MessageImportance.High,
+                $"Existing TPN file preamble: {existingTpn.Preamble.Substring(0, 10)}...");
+
+            foreach (var s in existingTpn.Sections.OrderBy(s => s.Header.SingleLineName))
+            {
+                Log.LogMessage(
+                    MessageImportance.High,
+                    $"{s.Header.StartLine + 1}:{s.Header.StartLine + s.Header.LineLength} {s.Header.Format} '{s.Header.SingleLineName}'");
+            }
+
+            TpnDocument[] otherTpns = results
+                .Select(r => r.Content)
+                .Where(r => r != null)
+                .ToArray();
+
+            TpnSection[] newSections = otherTpns
+                .SelectMany(o => o.Sections)
+                .Except(existingTpn.Sections, new TpnSection.ByHeaderNameComparer())
+                .OrderBy(s => s.Header.Name)
+                .ToArray();
+
+            foreach (TpnSection existing in results
+                .SelectMany(r => (r.Content?.Sections.Except(newSections)).NullAsEmpty())
+                .Where(s => !newSections.Contains(s))
+                .OrderBy(s => s.Header.Name))
+            {
+                Log.LogMessage(
+                    MessageImportance.High,
+                    $"Found already-imported section: '{existing.Header.SingleLineName}'");
+            }
+
+            foreach (var s in newSections)
+            {
+                Log.LogMessage(
+                    MessageImportance.High,
+                    $"New section to import: '{s.Header.SingleLineName}' of " +
+                    string.Join(
+                        ", ",
+                        results
+                            .Where(r => r.Content?.Sections.Contains(s) == true)
+                            .Select(r => r.Url)) +
+                    $" line {s.Header.StartLine}");
+            }
+
+            Log.LogMessage(MessageImportance.High, $"Importing {newSections.Length} sections...");
+
+            var newTpn = new TpnDocument
+            {
+                Preamble = existingTpn.Preamble,
+                Sections = existingTpn.Sections.Concat(newSections)
+            };
+
+            File.WriteAllText(TpnFile, newTpn.ToString());
+
+            Log.LogMessage(MessageImportance.High, $"Wrote new TPN contents to {TpnFile}.");
+        }
+    }
+}

src/tasks/installer.tasks/StaticFileRegeneration/TpnDocument.cs

@@ -0,0 +1,70 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+
+namespace Microsoft.DotNet.Build.Tasks
+{
+    public class TpnDocument
+    {
+        public static TpnDocument Parse(string[] lines)
+        {
+            var headers = TpnSectionHeader.ParseAll(lines).ToArray();
+
+            var sections = headers
+                .Select((h, i) =>
+                {
+                    int headerEndLine = h.StartLine + h.LineLength + 1;
+                    int linesUntilNext = lines.Length - headerEndLine;
+
+                    if (i + 1 < headers.Length)
+                    {
+                        linesUntilNext = headers[i + 1].StartLine - headerEndLine;
+                    }
+
+                    return new TpnSection
+                    {
+                        Header = h,
+                        Content = string.Join(
+                            Environment.NewLine,
+                            lines
+                                .Skip(headerEndLine)
+                                .Take(linesUntilNext)
+                                // Skip lines in the content that could be confused for separators.
+                                .Where(line => !TpnSectionHeader.IsSeparatorLine(line))
+                                // Trim empty line at the end of the section.
+                                .Reverse()
+                                .SkipWhile(line => string.IsNullOrWhiteSpace(line))
+                                .Reverse())
+                    };
+                })
+                .ToArray();
+
+            if (sections.Length == 0)
+            {
+                throw new ArgumentException($"No sections found.");
+            }
+
+            return new TpnDocument
+            {
+                Preamble = string.Join(
+                    Environment.NewLine,
+                    lines.Take(sections.First().Header.StartLine)),
+
+                Sections = sections
+            };
+        }
+
+        public string Preamble { get; set; }
+
+        public IEnumerable<TpnSection> Sections { get; set; }
+
+        public override string ToString() =>
+            Preamble + Environment.NewLine +
+            string.Join(Environment.NewLine + Environment.NewLine, Sections) +
+            Environment.NewLine;
+    }
+}

src/tasks/installer.tasks/StaticFileRegeneration/TpnSection.cs

@@ -0,0 +1,26 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+using System;
+using System.Collections.Generic;
+
+namespace Microsoft.DotNet.Build.Tasks
+{
+    public class TpnSection
+    {
+        public class ByHeaderNameComparer : EqualityComparer<TpnSection>
+        {
+            public override bool Equals(TpnSection x, TpnSection y) =>
+                string.Equals(x.Header.Name, y.Header.Name, StringComparison.OrdinalIgnoreCase);
+
+            public override int GetHashCode(TpnSection obj) => obj.Header.Name.GetHashCode();
+        }
+
+        public TpnSectionHeader Header { get; set; }
+        public string Content { get; set; }
+
+        public override string ToString() =>
+            Header + Environment.NewLine + Environment.NewLine + Content;
+    }
+}