Improve running tests through ant

.github/workflows/ant.yml

@@ -24,6 +24,11 @@ jobs:
       run: sudo apt-get install -y xvfb
     - name: Run tests
       working-directory: ./app
-      run: xvfb-run --auto-servernum --server-args "-screen 0 1024x768x24" ant test
+      run: xvfb-run --auto-servernum --server-args "-screen 0 1024x768x24" ant test -Drunning-from-github-action=1
+    - name: Publish results
+      uses: actions/upload-artifact@v1
+      with:
+        name: html-results
+        path: app/test-bin/results/html/
     - name: Cleanup xvfb
       uses: bcomnes/cleanup-xvfb@v1

app/build.xml

@@ -1,5 +1,5 @@
 <?xml version="1.0"?>
-<project name="Arduino PDE" default="build">
+<project name="Arduino PDE" xmlns:if="ant:if" xmlns:unless="ant:unless" default="build">
 
   <path id="class.path">
     <fileset dir="lib">
@@ -109,7 +109,20 @@
       <fileset dir="test" includes="**/*.pac" />
     </copy>
 
-    <junit printsummary="yes" dir="${work.dir}" fork="true" showoutput="yes" failureproperty="test.failed">
+    <!-- XML and TXT reports will be written here -->
+    <property name="test.reportdir" value="test-bin/results"/>
+    <!-- Summary HTML report will be written here -->
+    <property name="test.htmldir" value="test-bin/results/html"/>
+
+    <!-- Remove the reportdir, so the HTML report only includes tests from this run -->
+    <delete dir="${test.reportdir}" />
+    <mkdir dir="${test.reportdir}"/>
+    <mkdir dir="${test.htmldir}"/>
+
+    <!-- Sanity check: when single-test-methods is set, but single-test-class is not, raise an error -->
+    <fail message="Need single-test-class if single-test-methods is set" if="single-test-methods" unless="single-test-class"/>
+
+    <junit printsummary="yes" dir="${work.dir}" fork="true" showoutput="no" failureproperty="test.failed">
       <jvmarg value="-Djava.library.path=${java.additional.library.path}"/>
       <jvmarg value="-DWORK_DIR=."/>
       <jvmarg value="-ea"/>
@@ -121,16 +134,45 @@
         <path refid="class.path.test"/>
       </classpath>
 
+      <!-- Write XML files (for report-generation) and TXT files (for manual review) for every test class -->
+      <formatter type="plain" />
       <formatter type="xml"/>
-
-      <batchtest fork="yes" todir="test-bin">
+      <!-- Print full details to stdout when running a single test -->
+      <formatter type="plain" usefile="false" if="single-test-class"/>
+      <!-- When running all tests, print details for failing tests and a summary for each test class (printsummary=yes above) -->
+      <formatter type="brief" usefile="false" unless="single-test-class"/>
+
+      <!-- When both single-test-class and single-test-methods are specified, pass both to unit -->
+      <test name="${single-test-class}" methods="${single-test-methods}" todir="${test.reportdir}" if="single-test-methods"/>
+      <!-- When just single-test-class is specified, omit methods to run the entire class -->
+      <test name="${single-test-class}" todir="${test.reportdir}" if="single-test-class" unless="single-test-methods"/>
+      <!-- When neither are specified, run *all* testcases -->
+      <batchtest fork="yes" todir="${test.reportdir}" unless="single-test-class">
         <fileset dir="test">
           <include name="**/*Test.java"/>
           <exclude name="**/Abstract*.java"/>
         </fileset>
       </batchtest>
     </junit>
 
+    <!-- Convert generated XML reports to browsable HTML -->
+    <junitreport todir="${test.reportdir}">
+      <fileset dir="${test.reportdir}">
+	<include name="TEST-*.xml" />
+      </fileset>
+      <!-- Create a browsable frame-based HTML -->
+      <report format="frames" todir="${test.htmldir}" />
+      <!-- Also create a single-page version, to use as CI test result -->
+      <report format="noframes" todir="${test.htmldir}" />
+    </junitreport>
+
+    <!-- Make these paths relative to user.dir, which is the current directory when invoking ant (so the resulting paths are relative to the environment of the user. -->
+    <property name="test.htmldir.display" location="${test.htmldir}" relative="true" basedir="${user.dir}"/>
+    <property name="test.reportdir.display" location="${test.reportdir}" relative="true" basedir="${user.dir}"/>
+    <echo unless:set="running-from-github-action" message="Detailed test results can be found in ${test.reportdir.display}"/>
+    <echo unless:set="running-from-github-action" message="A browsable HTML summary is generated in ${test.htmldir.display}/index.html"/>
+    <echo if:set="running-from-github-action" message="Detailed test results can be downloaded as an asset from this github action. Easiest to view is the single-file version in junit-noframes.html. Alternatively, the multi-file version can be viewed from index.html and includes full stdout/stderr output of tests."/>
+
     <fail if="test.failed"/>
   </target>
 

build/build.xml

@@ -166,7 +166,7 @@
     </java>
   </target>
 
-  <target name="test" depends="subprojects-test"/>
+  <target name="test" depends="subprojects-build, subprojects-test"/>
 
   <!-- - - - - - - - - - - - - - - - - - -->
   <!-- Subprojects: Core, App, Libraries -->
@@ -183,7 +183,14 @@
   </target>
 
   <target name="subprojects-test">
-    <subant buildpath="../app" target="test"/>
+    <subant buildpath="../app" target="test">
+      <propertyset>
+        <!-- Forward these to subant. Use propertyset/propertyref instead of a direct property so we do not need to specify a value -->
+        <propertyref name="single-test-class"/>
+        <propertyref name="single-test-methods"/>
+        <propertyref name="running-from-github-actions"/>
+      </propertyset>
+    </subant>
   </target>
 
   <!-- - - - - - - - - -->