Skip to content

Update JIT format job to pass new --cross argument on Linux #92751

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
BruceForstall merged 2 commits into from
Sep 28, 2023
Merged

Update JIT format job to pass new --cross argument on Linux #92751

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
4bcbbaf
Update JIT format job to pass new `--cross` argument on Linux
BruceForstall Sep 28, 2023
7d018d5
Fix copy/paste error
BruceForstall Sep 28, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 13 additions & 3 deletions eng/pipelines/coreclr/templates/format-job.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,13 +35,21 @@ jobs:
- ${{ if eq(parameters.osGroup, 'windows') }}:
- name: PythonScript
value: 'py -3'

- ${{ if ne(parameters.osGroup, 'windows') }}:
- name: PythonScript
value: 'python3'

- ${{ if eq(parameters.osGroup, 'linux') }}:
- name: LinuxCrossArg
value: '--cross'
- ${{ if ne(parameters.osGroup, 'linux') }}:
- name: LinuxCrossArg
value: ''

condition: ${{ parameters.condition }}

steps:

- task: UseDotNet@2
# This should match what jitutils YML uses to build.
displayName: 'Install .NET SDK'
Expand All @@ -50,12 +58,14 @@ jobs:
version: '6.x'
includePreviewVersions: true
installationPath: $(Agent.ToolsDirectory)/dotnet
- script: $(PythonScript) $(Build.SourcesDirectory)/src/coreclr/scripts/jitformat.py -c $(Build.SourcesDirectory)/src/coreclr -o $(osGroup) -a $(archType)

- script: $(PythonScript) $(Build.SourcesDirectory)/src/coreclr/scripts/jitformat.py -r $(Build.SourcesDirectory) -o $(osGroup) -a $(archType) $(LinuxCrossArg)
displayName: Run jitformat.py

- task: PublishBuildArtifacts@1
displayName: Publish format.patch
inputs:
PathtoPublish: '$(Build.SourcesDirectory)/src/coreclr/format.patch'
PathtoPublish: '$(Build.SourcesDirectory)/format.patch'
ArtifactName: format.$(osGroup).$(archType).patch
continueOnError: true
condition: failed()
227 changes: 100 additions & 127 deletions src/coreclr/scripts/jitformat.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,234 +10,207 @@
# Script to install and run jit-format over jit source for all configurations.
################################################################################


import argparse
import jitutil
import logging
import os
import shutil
import subprocess
import sys
import tarfile
import tempfile
import zipfile

class ChangeDir:
def __init__(self, dir):
self.dir = dir
self.cwd = None

def __enter__(self):
self.cwd = os.getcwd()
os.chdir(self.dir)

def __exit__(self, exc_type, exc_val, exc_tb):
os.chdir(self.cwd)

class TempDir:
def __init__(self, path=None):
self.dir = tempfile.mkdtemp() if path is None else path
self.cwd = None

def __enter__(self):
self.cwd = os.getcwd()
os.chdir(self.dir)

return self.dir

def __exit__(self, exc_type, exc_val, exc_tb):
os.chdir(self.cwd)

def expandPath(path):
return os.path.abspath(os.path.expanduser(path))

def del_rw(action, name, exc):
os.chmod(name, 0o651)
os.remove(name)

def cleanup(jitUtilsPath, bootstrapPath):
if os.path.isdir(jitUtilsPath):
logging.info("Deleting " + jitUtilsPath)
shutil.rmtree(jitUtilsPath, onerror=del_rw)

if os.path.isfile(bootstrapPath):
logging.info("Deleting " + bootstrapPath)
os.remove(bootstrapPath)

def main(argv):
logging.basicConfig(format="[%(asctime)s] %(message)s", datefmt="%H:%M:%S")
logger = logging.getLogger()
logger.setLevel(logging.INFO)

parser = argparse.ArgumentParser()

required = parser.add_argument_group('required arguments')
required.add_argument('-a', '--arch', type=str,
default=None, help='architecture to run jit-format on')
required.add_argument('-o', '--os', type=str,
default=None, help='operating system')
required.add_argument('-c', '--coreclr', type=str,
default=None, help='full path to coreclr')
required.add_argument('-r', '--runtime', type=str,
default=None, help='full path to runtime repo root')

optional = parser.add_argument_group('optional arguments')
optional.add_argument('--cross', action="store_true",
default=None, help='do cross builds on Linux')
optional.add_argument('-j', '--jitutils', type=str,
Copy link
Contributor

@kunalspathak kunalspathak Sep 28, 2023

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

i believe this is added for local usage?

Copy link
Contributor Author

@BruceForstall BruceForstall Sep 28, 2023

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes, this makes it easier to test locally

default=None, help='full path to built jitutils repo root. Uses this instead of downloading bootstrap.sh/cmd and cloning/building jitutils.')

args, unknown = parser.parse_known_args(argv)

if unknown:
logging.warning('Ignoring argument(s): {}'.format(','.join(unknown)))

if args.coreclr is None:
logging.error('Specify --coreclr')
if args.runtime is None:
logging.error('Specify --runtime')
return -1
if args.os is None:
logging.error('Specify --os')
return -1
if args.arch is None:
logging.error('Specify --arch')
return -1
if args.cross:
if args.os != "linux":
logging.error('--cross is only valid with -os linux')
return -1
if args.jitutils is not None:
jitutilsRoot = os.path.abspath(os.path.expanduser(args.jitutils))
if not os.path.isdir(jitutilsRoot):
logging.error('Bad path to jitutils')
return -1

if not os.path.isdir(expandPath(args.coreclr)):
logging.error('Bad path to coreclr')
runtime = os.path.abspath(os.path.expanduser(args.runtime)).replace('/', os.sep)
if not os.path.isdir(runtime):
logging.error('Bad runtime path')
return -1

coreclr = args.coreclr.replace('/', os.sep)

platform = args.os
arch = args.arch

my_env = os.environ

# Download formatting tools
repoRoot = os.path.dirname(os.path.dirname(coreclr))
formattingScriptFolder = os.path.join(repoRoot, "eng", "formatting")
formattingDownloadScriptCommand = []
if platform == 'linux' or platform == 'osx':
formattingDownloadScriptCommand = [os.path.join(formattingScriptFolder, "download-tools.sh")]
elif platform == 'windows':
# Download formatting tools clang-format and clang-tidy and add them to PATH
formattingScriptFolder = os.path.join(runtime, "eng", "formatting")
if not os.path.isdir(formattingScriptFolder):
logging.error('Bad runtime path: eng/formatting directory not found')
return -1

if platform == 'windows':
formattingDownloadScriptCommand = ["powershell", os.path.join(formattingScriptFolder, "download-tools.ps1")]
else:
formattingDownloadScriptCommand = [os.path.join(formattingScriptFolder, "download-tools.sh")]

proc = subprocess.Popen(formattingDownloadScriptCommand)

if proc.wait() != 0:
logging.error("Formatting tool download failed")
return -1

my_env["PATH"] = os.path.join(repoRoot, "artifacts", "tools") + os.pathsep + my_env["PATH"]
my_env["PATH"] = os.path.join(runtime, "artifacts", "tools") + os.pathsep + my_env["PATH"]

# Download bootstrap
with jitutil.TempDir() as temp_location:
assert len(os.listdir(temp_location)) == 0

bootstrapFilename = ""
if args.jitutils is not None:
logging.info('--jitutils passed: not downloading bootstrap.cmd/sh and cloning/building jitutils repo')

jitUtilsPath = os.path.join(coreclr, "jitutils")
else:

cleanup(jitUtilsPath, '')
# Download bootstrap
if platform == 'windows':
bootstrapFilename = "bootstrap.cmd"
else:
bootstrapFilename = "bootstrap.sh"

if platform == 'linux' or platform == 'osx':
bootstrapFilename = "bootstrap.sh"
elif platform == 'windows':
bootstrapFilename = "bootstrap.cmd"
bootstrapUrl = "https://raw.githubusercontent.com/dotnet/jitutils/main/" + bootstrapFilename
bootstrapPath = os.path.join(temp_location, bootstrapFilename)
if not jitutil.download_one_url(bootstrapUrl, bootstrapPath) or not os.path.isfile(bootstrapPath):
logging.error("Did not download bootstrap!")
return -1

bootstrapUrl = "https://raw.githubusercontent.com/dotnet/jitutils/main/" + bootstrapFilename
if platform == 'windows':
# Need to ensure we have Windows line endings on the downloaded script file,
# which is downloaded with Unix line endings.
logging.info('Convert {} to Windows line endings'.format(bootstrapPath))

with TempDir() as temp_location:
bootstrapPath = os.path.join(temp_location, bootstrapFilename)
content = None
with open(bootstrapPath, 'rb') as open_file:
content = open_file.read()

assert len(os.listdir(os.path.dirname(bootstrapPath))) == 0
content = content.replace(b'\n', b'\r\n')

if not jitutil.download_one_url(bootstrapUrl, bootstrapPath):
logging.error("Did not download bootstrap!")
return -1
with open(bootstrapPath, 'wb') as open_file:
open_file.write(content)

if platform == 'windows':
# Need to ensure we have Windows line endings on the downloaded script file,
# which is downloaded with Unix line endings.
logging.info('Convert {} to Windows line endings'.format(bootstrapPath))
# On *nix platforms, we need to make the bootstrap file executable

content = None
with open(bootstrapPath, 'rb') as open_file:
content = open_file.read()
if platform == 'linux' or platform == 'osx':
logging.info("Making bootstrap executable")
os.chmod(bootstrapPath, 0o751)

content = content.replace(b'\n', b'\r\n')
# Run bootstrap
if platform == 'windows':
command = [bootstrapPath]
else:
command = ['bash', bootstrapPath]

with open(bootstrapPath, 'wb') as open_file:
open_file.write(content)
command_string = " ".join(command)
logging.info('Running: {}'.format(command_string))
proc = subprocess.Popen(command, env=my_env)
output,error = proc.communicate()
if proc.returncode != 0:
logging.error("Bootstrap failed")
return -1

# On *nix platforms, we need to make the bootstrap file executable
jitutilsRoot = os.path.join(temp_location, "jitutils")

if platform == 'linux' or platform == 'osx':
logging.info("Making bootstrap executable")
os.chmod(bootstrapPath, 0o751)
# end of 'if args.jitutils is None'

# Run bootstrap
if platform == 'linux' or platform == 'osx':
logging.info('Running: bash {}'.format(bootstrapPath))
proc = subprocess.Popen(['bash', bootstrapPath], env=my_env)
output,error = proc.communicate()
elif platform == 'windows':
logging.info('Running: {}'.format(bootstrapPath))
proc = subprocess.Popen([bootstrapPath], env=my_env)
output,error = proc.communicate()
# Run jit-format

if proc.returncode != 0:
cleanup('', bootstrapPath)
logging.error("Bootstrap failed")
jitutilsBin = os.path.join(jitutilsRoot, "bin")
if not os.path.isdir(jitutilsBin):
logging.error("jitutils not built!")
return -1

# Run jit-format

returncode = 0
jitutilsBin = os.path.join(os.path.dirname(bootstrapPath), "jitutils", "bin")
my_env["PATH"] = jitutilsBin + os.pathsep + my_env["PATH"]

if not os.path.isdir(jitutilsBin):
logging.error("Jitutils not built!")
return -1
if platform == 'windows':
jitformat = os.path.join(jitutilsBin, "jit-format.exe")
else:
jitformat = os.path.join(jitutilsBin, "jit-format")

jitformat = jitutilsBin
if not os.path.isfile(jitformat):
logging.error("jit-format not found")
return -1

if platform == 'linux' or platform == 'osx':
jitformat = os.path.join(jitformat, "jit-format")
elif platform == 'windows':
jitformat = os.path.join(jitformat,"jit-format.exe")
errorMessage = ""

builds = ["Checked", "Debug", "Release"]
projects = ["dll", "standalone", "crossgen"]

returncode = 0
for build in builds:
for project in projects:
command = jitformat + " -a " + arch + " -b " + build + " -o " + platform + " -c " + coreclr + " --verbose --projects " + project
logging.info('Running: {}'.format(command))
proc = subprocess.Popen([jitformat, "-a", arch, "-b", build, "-o", platform, "-c", coreclr, "--verbose", "--projects", project], env=my_env)
command = [jitformat, "-a", arch, "-b", build, "-o", platform, "-r", runtime, "--verbose", "--projects", project]
if args.cross:
command += ["--cross"]

command_string = " ".join(command)
logging.info('Running: {}'.format(command_string))
proc = subprocess.Popen(command, env=my_env)
output,error = proc.communicate()
errorcode = proc.returncode

if errorcode != 0:
errorMessage += "\tjit-format -a " + arch + " -b " + build + " -o " + platform
errorMessage += " -c <absolute-path-to-coreclr> --verbose --fix --projects " + project +"\n"
errorMessage += " -r <absolute-path-to-runtime-root> --verbose --fix --projects " + project + "\n"
returncode = errorcode

# Fix mode doesn't return an error, so we have to run the build, then run with
# --fix to generate the patch. This means that it is likely only the first run
# of jit-format will return a formatting failure.
if errorcode == -2:
# If errorcode was -2, no need to run clang-tidy again
proc = subprocess.Popen([jitformat, "--fix", "--untidy", "-a", arch, "-b", build, "-o", platform, "-c", coreclr, "--verbose", "--projects", project], env=my_env)
proc = subprocess.Popen([jitformat, "--fix", "--untidy", "-a", arch, "-b", build, "-o", platform, "-r", runtime, "--verbose", "--projects", project], env=my_env)
output,error = proc.communicate()
else:
# Otherwise, must run both
proc = subprocess.Popen([jitformat, "--fix", "-a", arch, "-b", build, "-o", platform, "-c", coreclr, "--verbose", "--projects", project], env=my_env)
proc = subprocess.Popen([jitformat, "--fix", "-a", arch, "-b", build, "-o", platform, "-r", runtime, "--verbose", "--projects", project], env=my_env)
output,error = proc.communicate()

patchFilePath = os.path.join(coreclr, "format.patch")
patchFilePath = os.path.join(runtime, "format.patch")

if returncode != 0:
# Create a patch file
logging.info("Creating patch file {}".format(patchFilePath))
jitSrcPath = os.path.join(coreclr, "jit")
jitSrcPath = os.path.join(runtime, "src", "coreclr", "jit")
patchFile = open(patchFilePath, "w")
proc = subprocess.Popen(["git", "diff", "--patch", "-U20", "--", jitSrcPath], env=my_env, stdout=patchFile)
output,error = proc.communicate()

cleanup(jitUtilsPath, bootstrapPath)

if returncode != 0:
logging.info("There were errors in formatting. Please run jit-format locally with: \n")
logging.info(errorMessage)
Expand Down