diff --git a/.gitignore b/.gitignore index 0ddfd717d737f8..12be5f0a5520f0 100644 --- a/.gitignore +++ b/.gitignore @@ -57,6 +57,7 @@ Doc/env/ Doc/.env/ Include/pydtrace_probes.h Lib/distutils/command/*.pdb +Lib/idlelib/idle_test/coverage_venv/ Lib/lib2to3/*.pickle Lib/site-packages/* !Lib/site-packages/README.txt diff --git a/Lib/idlelib/idle_test/README.txt b/Lib/idlelib/idle_test/README.txt index 566bfd179fdf1b..8ccd3e85f03b2e 100644 --- a/Lib/idlelib/idle_test/README.txt +++ b/Lib/idlelib/idle_test/README.txt @@ -184,55 +184,14 @@ python -m idlelib.idle_test.htest 5. Test Coverage -Install the coverage package into your Python 3.6 site-packages -directory. (Its exact location depends on the OS). -> python3 -m pip install coverage -(On Windows, replace 'python3 with 'py -3.6' or perhaps just 'python'.) - -The problem with running coverage with repository python is that -coverage uses absolute imports for its submodules, hence it needs to be -in a directory in sys.path. One solution: copy the package to the -directory containing the cpython repository. Call it 'dev'. Then run -coverage either directly or from a script in that directory so that -'dev' is prepended to sys.path. - -Either edit or add dev/.coveragerc so it looks something like this. ---- -# .coveragerc sets coverage options. -[run] -branch = True - -[report] -# Regexes for lines to exclude from consideration -exclude_lines = - # Don't complain if non-runnable code isn't run: - if 0: - if __name__ == .__main__.: - - .*# htest # - if not _utest: - if _htest: ---- -The additions for IDLE are 'branch = True', to test coverage both ways, -and the last three exclude lines, to exclude things peculiar to IDLE -that are not executed during tests. - -A script like the following cover.bat (for Windows) is very handy. ---- -@echo off -rem Usage: cover filename [test_ suffix] # proper case required by coverage -rem filename without .py, 2nd parameter if test is not test_filename -setlocal -set py=f:\dev\3x\pcbuild\win32\python_d.exe -set src=idlelib.%1 -if "%2" EQU "" set tst=f:/dev/3x/Lib/idlelib/idle_test/test_%1.py -if "%2" NEQ "" set tst=f:/dev/ex/Lib/idlelib/idle_test/test_%2.py - -%py% -m coverage run --pylib --source=%src% %tst% -%py% -m coverage report --show-missing -%py% -m coverage html -start htmlcov\3x_Lib_idlelib_%1_py.html -rem Above opens new report; htmlcov\index.html displays report index ---- -The second parameter was added for tests of module x not named test_x. -(There were several before modules were renamed, now only one is left.) +To get a coverage report for a specific module's tests, run: + +python Lib/idlelib/idle_test/run_coverage.py + +Replace above with just the name of the module, +such as colorizer. + +To get a coverage report for IDLE's entire test suite, run the +above command with "all" instead of a module name. + +Note: this does not work with out-of-tree builds yet. diff --git a/Lib/idlelib/idle_test/run_coverage.py b/Lib/idlelib/idle_test/run_coverage.py new file mode 100755 index 00000000000000..7fa16cfa9549a9 --- /dev/null +++ b/Lib/idlelib/idle_test/run_coverage.py @@ -0,0 +1,169 @@ +#!/usr/bin/env python3 +import argparse +import contextlib +import os +import pathlib +import subprocess +import sys +import textwrap + + +def main(): + idlelib_path = find_idlelib_path() + sources_root_path = idlelib_path.parent.parent + + module_names = sorted(get_module_names(idlelib_path)) + + parser = argparse.ArgumentParser( + description='Run tests for an idlelib module ' + 'and collect coverage stats', + ) + parser.add_argument('module', choices=['all'] + module_names) + parser.add_argument( + '--no-html', action='store_true', + help='Do not create an HTML coverage report and open it in a browser') + args = parser.parse_args() + + venv_path, venv_python_path = ensure_venv(sources_root_path) + os.chdir(sources_root_path) + + with coveragerc_replacement(): + run_tests_with_coverage(module_name=args.module, + venv_python_path=venv_python_path) + generate_coverage_report(venv_python_path=venv_python_path, + no_html=args.no_html) + + if not args.no_html: + print('Coverage report available at htmlcov/index.html') + + +def listfiles(path): + return [ent.name for ent in os.scandir(path) if ent.is_file()] + + +def listdirs(path): + return [ent.name for ent in os.scandir(path) if ent.is_dir()] + + +def find_idlelib_path(): + curpath = pathlib.Path(os.getcwd()) + if curpath.name == 'idlelib': + idlelib_path = curpath + elif curpath.parent.name == 'idlelib': + idlelib_path = curpath.parent + elif 'idlelib' in listdirs(curpath): + idlelib_path = curpath.joinpath('idlelib') + elif ( + 'Lib' in listdirs(curpath) and + curpath.joinpath('Lib', 'idlelib').is_dir() + ): + idlelib_path = curpath.joinpath('Lib', 'idlelib') + else: + raise Exception('Failed to find idlelib path.\n' + 'Run in the base source dir or in Lib/idlelib.') + return idlelib_path + + +def get_module_names(idlelib_path): + test_module_names = { + filename[len('test_'):-len('.py')] + for filename in listfiles(idlelib_path.joinpath('idle_test')) + if filename.startswith('test_') and filename.endswith('.py') + } + module_names = { + filename[:-len('.py')] + for filename in listfiles(idlelib_path) + if filename.endswith('.py') + } + return set(test_module_names) & set(module_names) + + +def ensure_venv(sources_root_path): + venv_path = sources_root_path.joinpath( + 'Lib', 'idlelib', 'idle_test', 'coverage_venv') + venv_python_path = ( + venv_path.joinpath('Scripts', 'python.exe') + if sys.platform == 'win32' else + venv_path.joinpath('bin', 'python') + ) + built_python_path = sources_root_path.joinpath( + 'python.bat' if sys.platform == 'win32' else + 'python.exe' if sys.platform == 'darwin' else + 'python' + ) + if not venv_path.is_dir(): + subprocess.run([built_python_path, '-m', 'venv', venv_path]) + subprocess.run([venv_python_path, '-m', 'pip', 'install', 'coverage']) + return venv_path, venv_python_path + + +@contextlib.contextmanager +def coveragerc_replacement(): + coveragerc_path = pathlib.Path('.coveragerc') + orig_coverage_rc = None + if coveragerc_path.is_file(): + with coveragerc_path.open('r', encoding='utf-8') as f: + orig_coverage_rc = f.read() + try: + with coveragerc_path.open('w', encoding='utf-8') as f: + f.write(textwrap.dedent('''\ + [run] + branch = True + cover_pylib = True + + [report] + # Regexes for lines to exclude from consideration + exclude_lines = + # Don't complain if non-runnable code isn't run: + if 0: + if False: + if __name__ == .__main__.: + + .*# htest # + if not _utest: + if _htest: + show_missing = True + ''')) + yield + finally: + if orig_coverage_rc: + with coveragerc_path.open('w', encoding='utf-8') as f: + f.write(orig_coverage_rc) + else: + coveragerc_path.unlink() + + +def run_tests_with_coverage(*, module_name, venv_python_path): + if module_name == 'all': + subprocess.run([ + venv_python_path, '-m', 'coverage', 'run', + '--source=idlelib', + '-m', 'test', '-ugui', 'test_idle', + ]) + else: + subprocess.run([ + venv_python_path, '-m', 'coverage', 'run', + f'--source=idlelib.{module_name}', + f'./Lib/idlelib/idle_test/test_{module_name}.py', + ]) + if module_name in ['pyshell', 'run']: + # also run the tests in test_warning.py + subprocess.run([ + venv_python_path, '-m', 'coverage', 'run', '-a', + f'--source=idlelib.{module_name}', + './Lib/idlelib/idle_test/test_warning.py', + ]) + + +def generate_coverage_report(*, venv_python_path, no_html): + subprocess.run([venv_python_path, '-m', 'coverage', 'report']) + if not no_html: + subprocess.run([venv_python_path, '-m', 'coverage', 'html']) + + +if __name__ == '__main__': + if sys.version_info < (3,): + print('Running this script with Python 2 is not supported', + file=sys.stderr) + sys.exit(1) + main()