Add option --no-exit-first to disable pytest exit once failure detected

azdev/operations/tests/__init__.py

@@ -32,6 +32,7 @@
 # pylint: disable=too-many-statements
 def run_tests(tests, xml_path=None, discover=False, in_series=False,
               run_live=False, profile=None, last_failed=False, pytest_args=None,
+              no_exit_first=False,
               git_source=None, git_target=None, git_repo=None,
               cli_ci=False):
 
@@ -105,7 +106,10 @@ def _find_test(index, name):
 
     exit_code = 0
     with ProfileContext(profile):
-        runner = get_test_runner(parallel=not in_series, log_path=xml_path, last_failed=last_failed)
+        runner = get_test_runner(parallel=not in_series,
+                                 log_path=xml_path,
+                                 last_failed=last_failed,
+                                 no_exit_first=no_exit_first)
         exit_code = runner(test_paths=test_paths, pytest_args=pytest_args)
 
     sys.exit(0 if not exit_code else 1)

azdev/operations/tests/pytest_runner.py

@@ -11,7 +11,7 @@
 from azdev.utilities import call
 
 
-def get_test_runner(parallel, log_path, last_failed):
+def get_test_runner(parallel, log_path, last_failed, no_exit_first):
     """Create a pytest execution method"""
     def _run(test_paths, pytest_args):
 
@@ -22,6 +22,9 @@ def _run(test_paths, pytest_args):
         else:
             arguments = ['-x', '-v', '-p no:warnings', '--log-level=WARN', '--junit-xml', log_path]
 
+        if no_exit_first:
+            arguments.remove('-x')
+
         arguments.extend(test_paths)
         if parallel:
             arguments += ['-n', 'auto']

azdev/params.py

@@ -50,6 +50,7 @@ def load_arguments(self, _):
         c.argument('profile', options_list='--profile', help='Run automation against a specific profile. If omit, the tests will run against current profile.')
         c.argument('pytest_args', nargs=argparse.REMAINDER, options_list=['--pytest-args', '-a'], help='Denotes the remaining args will be passed to pytest.')
         c.argument('last_failed', options_list='--lf', action='store_true', help='Re-run the last tests that failed.')
+        c.argument('no_exit_first', options_list='--no-exitfirst', action='store_true', help='Do not exit on first error or failed test')
 
         # CI parameters
         c.argument('cli_ci',