xtask: Allow builds/runs of multiple tests at once (esp-rs#4678)

* Allow multiple test runs

* address reviews + protection against `--test ""`/ `--test " "`

* Update comment for value parser

* reimplement empty strings rejection, allow spaces between tests

Author: playfulFence

xtask/README.md

@@ -153,9 +153,9 @@ option, while the other will select `multiple-integrated`.
 You can specifiy a test or example by file, or by configuration. If the
 parameters match multiple files, they will be built or executed in sucession.
 
-For example, running `cargo xtask run-tests esp32 embassy_test` will run both
+For example, running `cargo xtask run tests esp32 embassy_test` will run both
 `embassy_test_single_integrated` and `embassy_test_multiple_integrated`, but you can also
-run `cargo xtask run-tests esp32 embassy_test_multiple_integrated` to select only one.
+run `cargo xtask run tests esp32 embassy_test_multiple_integrated` to select only one.
 
-In this example, running the `cargo xtask build-tests esp32 embassy_test` command creates an
+In this example, running the `cargo xtask build tests esp32 embassy_test` command creates an
 `embassy_test_single_integrated` and an `embassy_test_multiple_integrated` binary.

xtask/src/commands/mod.rs

@@ -70,9 +70,11 @@ pub struct TestsArgs {
     pub repeat: usize,
     /// Optional test to act on (all tests used if omitted).
     ///
-    /// The `test_suite::test_name` syntax allows running a single specific test.
-    #[arg(long, short = 't')]
-    pub test: Option<String>,
+    /// Multiple tests may be selected via a comma-separated list, e.g. `--test rmt,i2c,uart`.
+    /// The `test_suite::test_name` syntax allows selecting a specific test (and may be combined
+    /// with commas).
+    #[arg(long, short = 't', alias = "tests", value_delimiter = ',', num_args = 1..)]
+    pub test: Option<Vec<String>>,
 
     /// The toolchain used to build the tests
     #[arg(long)]
@@ -212,15 +214,6 @@ pub fn examples(workspace: &Path, mut args: ExamplesArgs, action: CargoAction) -
 
 /// Execute the given action on the specified HIL tests.
 pub fn tests(workspace: &Path, args: TestsArgs, action: CargoAction) -> Result<()> {
-    let (test_arg, filter) = if let Some(test_arg) = args.test.as_deref() {
-        match test_arg.split_once("::") {
-            Some((test, filter)) => (Some(test), Some(filter)),
-            None => (Some(test_arg), None),
-        }
-    } else {
-        (None, None)
-    };
-
     // Absolute path of the 'hil-test' package's root:
     let package_path = crate::windows_safe_path(&workspace.join("hil-test"));
 
@@ -236,11 +229,6 @@ pub fn tests(workspace: &Path, args: TestsArgs, action: CargoAction) -> Result<(
     // Sort all tests by name:
     tests.sort_by_key(|a| a.binary_name());
 
-    let run_test_extra_args = (action == CargoAction::Run)
-        .then(|| filter.as_ref().map(|f| std::slice::from_ref(f)))
-        .flatten()
-        .unwrap_or(&[]);
-
     if let CargoAction::Build(Some(out_dir)) = &action {
         // Make sure the tmp directory has no garbage for us.
         let tmp_dir = out_dir.join("tmp");
@@ -250,26 +238,59 @@ pub fn tests(workspace: &Path, args: TestsArgs, action: CargoAction) -> Result<(
 
     let mut commands = CargoCommandBatcher::new();
     // Execute the specified action:
-    if tests.iter().any(|test| test.matches(test_arg.as_deref())) {
-        for test in tests
-            .iter()
-            .filter(|test| test.matches(test_arg.as_deref()))
-        {
-            let command = crate::generate_build_command(
-                &package_path,
-                args.chip,
-                &target,
-                &test,
-                action.clone(),
-                false,
-                args.toolchain.as_deref(),
-                args.timings,
-                run_test_extra_args,
-            )?;
-            commands.push(command);
+    // If user sets some specific test(s)
+    if let Some(test_arg) = args.test.as_deref() {
+        let trimmed: Vec<&str> = test_arg.iter().map(|s| s.trim()).collect();
+
+        // Reject `--test ""` / `--test " "`
+        if trimmed.iter().all(|s| s.is_empty()) {
+            bail!("Empty test selector is not allowed.");
+        }
+
+        let mut selected_failed: Vec<String> = Vec::new();
+
+        for selected in trimmed.into_iter().filter(|s| !s.is_empty()) {
+            let (test_arg, filter) = match selected.split_once("::") {
+                Some((test, filter)) => (Some(test), Some(filter)),
+                None => (Some(selected), None),
+            };
+
+            let run_test_extra_args = (action == CargoAction::Run)
+                .then(|| filter.as_ref().map(|f| std::slice::from_ref(f)))
+                .flatten()
+                .unwrap_or(&[]);
+
+            let matched: Vec<_> = tests
+                .iter()
+                .filter(|t| t.matches(test_arg.as_deref()))
+                .collect();
+
+            if matched.is_empty() {
+                selected_failed.push(selected.to_string());
+            } else {
+                for test in matched {
+                    let command = crate::generate_build_command(
+                        &package_path,
+                        args.chip,
+                        &target,
+                        test,
+                        action.clone(),
+                        false,
+                        args.toolchain.as_deref(),
+                        args.timings,
+                        run_test_extra_args,
+                    )?;
+                    commands.push(command);
+                }
+            }
+        }
+
+        if !selected_failed.is_empty() {
+            bail!(
+                "Test not found or unsupported for the given chip: {}",
+                selected_failed.join(", ")
+            )
         }
-    } else if test_arg.is_some() {
-        bail!("Test not found or unsupported for the given chip")
     } else {
         for test in tests {
             let command = crate::generate_build_command(
@@ -281,11 +302,12 @@ pub fn tests(workspace: &Path, args: TestsArgs, action: CargoAction) -> Result<(
                 false,
                 args.toolchain.as_deref(),
                 args.timings,
-                run_test_extra_args,
+                &[],
             )?;
             commands.push(command);
         }
     }
+
     let mut failed = Vec::new();
 
     for c in commands.build(false) {