Migrate four tools to commandline v2

NuGet.Config

@@ -5,6 +5,7 @@
     <clear />
     <add key="dotnet7" value="https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet7/nuget/v3/index.json" />
     <add key="dotnet-public" value="https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-public/nuget/v3/index.json" />
+    <add key="dotnet-libraries" value="https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-libraries/nuget/v3/index.json" />
     <add key="myget-legacy" value="https://pkgs.dev.azure.com/dnceng/public/_packaging/myget-legacy/nuget/v3/index.json" />
   </packageSources>
   <packageRestore>

build.cmd

@@ -1,5 +1,5 @@
 @echo off
-setlocal
+setlocal EnableDelayedExpansion
 
 REM Build and optionally publish sub projects
 REM
@@ -49,7 +49,12 @@ set projects=jit-diff jit-dasm jit-analyze jit-format pmi jit-dasm-pmi jit-decis
 REM Build each project
 for %%p in (%projects%) do (
     if %publish%==true (
-        dotnet publish -c %buildType% -o %appInstallDir% .\src\%%p
+        REM Publish src/pmi project without single-file, so it can be executed with a custom build of the runtime/JIT
+        if "%%p"=="pmi" (
+            dotnet publish -c %buildType% -o %appInstallDir% .\src\%%p
+        ) else (
+            dotnet publish -c %buildType% -o %appInstallDir% .\src\%%p -p:PublishSingleFile=true
+        )
         if errorlevel 1 echo ERROR: dotnet publish failed for .\src\%%p.&set __ExitCode=1
     ) else (
         dotnet build -c %buildType% .\src\%%p
@@ -62,7 +67,7 @@ exit /b %__ExitCode%
 
 :usage
 echo.
-echo  build.cmd [-b ^<BUILD TYPE^>] [-f] [-h] [-p]
+echo  build.cmd [-b ^<BUILD TYPE^>] [-h] [-p]
 echo.
 echo      -b ^<BUILD TYPE^>   : Build type, can be Debug or Release.
 echo      -h                : Show this message.

build.sh

@@ -9,7 +9,7 @@
 function usage
 {
     echo ""
-    echo "build.sh [-b <BUILD TYPE>] [-f] [-h] [-p]"
+    echo "build.sh [-b <BUILD TYPE>] [-h] [-p]"
     echo ""
     echo "    -b <BUILD TYPE> : Build type, can be Debug or Release."
     echo "    -h              : Show this message."
@@ -21,17 +21,13 @@ function usage
 __ErrMsgPrefix="ERROR: "
 final_exit_code=0
 buildType="Release"
-publish=false
-workingDir="$PWD"
-cd "`dirname \"$0\"`"
-scriptDir="$PWD"
-cd $workingDir
-platform="`dotnet --info | awk '/RID/ {print $2}'`"
+publish=0
+scriptDir="$(cd "$(dirname "$0")" || exit; pwd -P)"
 # default install in 'bin' dir at script location
 appInstallDir="$scriptDir/bin"
 
 # process for '-h', '-p', '-b <arg>'
-while getopts "hpfb:" opt; do
+while getopts "hpb:" opt; do
     case "$opt" in
     h)
         usage
@@ -41,7 +37,10 @@ while getopts "hpfb:" opt; do
         buildType=$OPTARG
         ;;
     p)  
-        publish=true
+        publish=1
+        ;;
+    *)  echo "ERROR: unknown argument %opt"
+        exit 1
         ;;
     esac
 done
@@ -52,15 +51,19 @@ declare -a projects=(jit-dasm jit-diff jit-analyze jit-format pmi jit-dasm-pmi j
 # for each project either build or publish
 for proj in "${projects[@]}"
 do
-    if [ "$publish" == true ]; then
-        dotnet publish -c $buildType -o $appInstallDir ./src/$proj
+    if [ "$publish" = 1 ]; then
+        case "$proj" in
+            # Publish src/pmi project without single-file, so it can be executed with a custom build of the runtime/JIT
+            pmi) dotnet publish -c "$buildType" -o "$appInstallDir" ./src/"$proj" ;;
+            *)   dotnet publish -c "$buildType" -o "$appInstallDir" ./src/"$proj" -p:PublishSingleFile=true ;;
+        esac
         exit_code=$?
         if [ $exit_code != 0 ]; then
             echo "${__ErrMsgPrefix}dotnet publish of ./src/${proj} failed."
             final_exit_code=1
         fi
     else
-        dotnet build -c $buildType ./src/$proj
+        dotnet build -c "$buildType" ./src/"$proj"
         exit_code=$?
         if [ $exit_code != 0 ]; then
             echo "${__ErrMsgPrefix}dotnet build  of ./src/${proj} failed."
@@ -69,4 +72,4 @@ do
     fi
 done
 
-exit $final_exit_code
+exit "$final_exit_code"

doc/diffs.md

@@ -66,11 +66,11 @@ build.sh [-b <BUILD TYPE>] [-f] [-h] [-p] [-t <TARGET>]
 By default, assembly code output (aka, "dasm") is generated by running crossgen
 with a specified JIT to compile a specified set of assemblies, by setting the
 following JIT environment variables to generate the output:
-* `DOTNET_NgenDisasm`
-* `DOTNET_NgenUnwindDump`
-* `DOTNET_NgenEHDump`
+* `DOTNET_JitDisasm`
+* `DOTNET_JitUnwindDump`
+* `DOTNET_JitEHDump`
 * `DOTNET_JitDiffableDasm`
-* optionally, `DOTNET_NgenGCDump`
+* optionally, `DOTNET_JitGCDump`
 
 Generating "diffs" involves generating assembly code output for both a baseline
 and a "diff" JIT, and comparing the results.

src/cijobs/CIClient.cs

@@ -0,0 +1,180 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using System.Net.Http;
+using System.Net.Http.Headers;
+using System.Text.Json;
+using System.Threading;
+using System.Threading.Tasks;
+
+namespace ManagedCodeGen
+{
+    // Wrap CI httpClient with focused APIs for product, job, and build.
+    // This logic is seperate from listing/copying and just extracts data.
+    internal sealed class CIClient
+    {
+        private HttpClient _client;
+
+        public CIClient(string server)
+        {
+            _client = new HttpClient();
+            _client.BaseAddress = new Uri(server);
+            _client.DefaultRequestHeaders.Accept.Clear();
+            _client.DefaultRequestHeaders.Accept.Add(new MediaTypeWithQualityHeaderValue("application/json"));
+            _client.Timeout = Timeout.InfiniteTimeSpan;
+        }
+
+        public async Task<bool> DownloadProduct(string messageString, string outputPath, string contentPath)
+        {
+                Console.WriteLine("Downloading: {0}", messageString);
+
+                HttpResponseMessage response = await _client.GetAsync(messageString);
+
+                bool downloaded = false;
+
+                if (response.IsSuccessStatusCode)
+                {
+                var zipPath = Path.Combine(outputPath, Path.GetFileName(contentPath));
+                using (var outputStream = System.IO.File.Create(zipPath))
+                {
+                    Stream inputStream = await response.Content.ReadAsStreamAsync();
+                    inputStream.CopyTo(outputStream);
+                }
+                downloaded = true;
+                }
+                else
+                {
+                    Console.Error.WriteLine("Zip not found!");
+                }
+
+                return downloaded;
+        }
+
+        public async Task<IEnumerable<Job>> GetProductJobs(string productName, string branchName)
+        {
+            string productString = $"job/{productName}/job/{branchName}/api/json?&tree=jobs[name,url]";
+
+            try
+            {
+                using HttpResponseMessage response = await _client.GetAsync(productString);
+
+                if (response.IsSuccessStatusCode)
+                {
+                    var json = await response.Content.ReadAsStringAsync();
+                    var productJobs = JsonSerializer.Deserialize<ProductJobs>(json);
+                    return productJobs.jobs;
+                }
+            }
+            catch (Exception ex)
+            {
+                Console.Error.WriteLine("Error enumerating jobs: {0} {1}", ex.Message, ex.InnerException.Message);
+            }
+
+            return Enumerable.Empty<Job>();
+        }
+
+        public async Task<IEnumerable<Build>> GetJobBuilds(string productName, string branchName,
+            string jobName, bool lastSuccessfulBuild, int number, string commit)
+        {
+            var jobString
+                = String.Format(@"job/{0}/job/{1}/job/{2}", productName, branchName, jobName);
+            var messageString
+                = String.Format("{0}/api/json?&tree=builds[number,url],lastSuccessfulBuild[number,url]",
+                    jobString);
+            HttpResponseMessage response = await _client.GetAsync(messageString);
+
+            if (response.IsSuccessStatusCode)
+            {
+                var json = await response.Content.ReadAsStringAsync();
+                var jobBuilds = JsonSerializer.Deserialize<JobBuilds>(json);
+
+                if (lastSuccessfulBuild)
+                {
+                    var lastSuccessfulNumber = jobBuilds.lastSuccessfulBuild.number;
+                    jobBuilds.lastSuccessfulBuild.info = await GetJobBuildInfo(productName, branchName, jobName, lastSuccessfulNumber);
+                    return Enumerable.Repeat(jobBuilds.lastSuccessfulBuild, 1);
+                }
+                else if (number != 0)
+                {
+                    var builds = jobBuilds.builds;
+
+                    var count = builds.Count();
+                    for (int i = 0; i < count; i++)
+                    {
+                        var build = builds[i];
+                        if (build.number == number)
+                        {
+                            build.info = await GetJobBuildInfo(productName, branchName, jobName, build.number);
+                            return Enumerable.Repeat(build, 1);
+                        }
+                    }
+                    return Enumerable.Empty<Build>();
+                }
+                else if (commit != null)
+                {
+                    var builds = jobBuilds.builds;
+
+                    var count = builds.Count();
+                    for (int i = 0; i < count; i++)
+                    {
+                        var build = builds[i];
+                        build.info = await GetJobBuildInfo(productName, branchName, jobName, build.number);
+                        var actions = build.info.actions.Where(x => x.lastBuiltRevision.SHA1 != null);
+                        foreach (var action in actions)
+                        {
+                            if (action.lastBuiltRevision.SHA1.Equals(commit, StringComparison.OrdinalIgnoreCase))
+                            {
+                                return Enumerable.Repeat(build, 1);
+                            }
+                        }
+                    }
+                    return Enumerable.Empty<Build>();
+                }
+                else
+                {
+                    var builds = jobBuilds.builds;
+
+                    var count = builds.Count();
+                    for (int i = 0; i < count; i++)
+                    {
+                        var build = builds[i];
+                        // fill in build info
+                        build.info = await GetJobBuildInfo(productName, branchName, jobName, build.number);
+                        builds[i] = build;
+                    }
+
+                    return jobBuilds.builds;
+                }
+            }
+            else
+            {
+                return Enumerable.Empty<Build>();
+            }
+        }
+
+        public async Task<BuildInfo> GetJobBuildInfo(string repoName, string branchName, string jobName, int number)
+        {
+            string buildString = String.Format("job/{0}/job/{1}/job/{2}/{3}",
+                repoName, branchName, jobName, number);
+            string buildMessage = String.Format("{0}/{1}", buildString,
+                "api/json?&tree=actions[lastBuiltRevision[SHA1]],artifacts[fileName,relativePath],result");
+            HttpResponseMessage response = await _client.GetAsync(buildMessage);
+
+            if (response.IsSuccessStatusCode)
+            {
+                var buildInfoJson = await response.Content.ReadAsStringAsync();
+                var info = JsonSerializer.Deserialize<BuildInfo>(buildInfoJson);
+                return info;
+            }
+            else
+            {
+                return null;
+            }
+        }
+    }
+}