CI: Generate unit tests' performance report

.github/workflows/build-test-linux-vcpkg.yml

@@ -179,6 +179,8 @@ jobs:
         run: xvfb-run -a ./build/${{ matrix.config }}/bin/MeshViewer -hidden -noEventLoop -unloadPluginsAtEnd
 
       - name: Unit Tests
+        env:
+          GTEST_OUTPUT: 'xml:unit_tests_report_gtest.xml'
         run: ./build/${{ matrix.config }}/bin/MRTest
 
       - name: C Unit Tests (old bindings)
@@ -192,19 +194,25 @@ jobs:
         if: ${{ inputs.mrbind }}
         timeout-minutes: 8
         working-directory: ./build/${{ matrix.config }}/bin
-        run: python3 ./../../../scripts/run_python_test_script.py -d '../test_python'
+        run: python3 ./../../../scripts/run_python_test_script.py -d '../test_python' -a ' --junit-xml=../unit_tests_report_pytest.xml'
 
       - name: Python Regression Tests
         if: ${{ inputs.internal_build && inputs.mrbind }}
         uses: ./.github/actions/python-regression-tests
         with:
           build_config: ${{ matrix.config }}
           mrbind: ${{ inputs.mrbind }}
-          pytest_args: "--run-cuda=negative"
+          pytest_args: "--run-cuda=negative --junit-xml=../unit_tests_report_regression.xml"
           smoke: ${{ !inputs.full_config_build && matrix.config == 'Debug' }}
           test_artifacts_path: linux-vcpkg-${{ matrix.arch }}/${{ matrix.config }}/${{ matrix.compiler }}
           upload_test_artifacts: ${{ inputs.upload_test_artifacts }}
 
+      - name: Generate Test Performance Report
+        continue-on-error: true
+        run: |
+          scripts/junit_to_csv.py unit_tests_report.csv unit_tests_report_*.xml
+          cat unit_tests_report.csv
+
       - name: Create Package
         if: ${{ matrix.config == 'Release' && matrix.compiler == 'Clang 20' }}
         run: |

.github/workflows/build-test-macos.yml

@@ -173,6 +173,8 @@ jobs:
         run: ./build/${{ matrix.config }}/bin/MeshViewer -tryHidden -noEventLoop -unloadPluginsAtEnd
 
       - name: Unit Tests
+        env:
+          GTEST_OUTPUT: 'xml:unit_tests_report_gtest.xml'
         run: ./build/${{ matrix.config }}/bin/MRTest
 
       - name: C Unit Tests (old bindings)
@@ -185,12 +187,12 @@ jobs:
       - name: Python Sanity Tests
         timeout-minutes: 8
         working-directory: ./build/${{ matrix.config }}/bin
-        run: python3 ./../../../scripts/run_python_test_script.py -d '../test_python'
+        run: python3 ./../../../scripts/run_python_test_script.py -d '../test_python' -a ' --junit-xml=../unit_tests_report_pytest.xml'
 
       - name: Python Regression Tests
         if: ${{ inputs.internal_build }}
         env:
-          RUN_CUDA_ARG: "--run-cuda=negative"
+          RUN_CUDA_ARG: "--run-cuda=negative --junit-xml=../unit_tests_report_regression.xml"
         uses: ./.github/actions/python-regression-tests
         with:
           build_config: ${{ matrix.config }}
@@ -199,6 +201,12 @@ jobs:
           test_artifacts_path: macos/${{ matrix.os }}
           upload_test_artifacts: ${{ inputs.upload_test_artifacts }}
 
+      - name: Generate Test Performance Report
+        continue-on-error: true
+        run: |
+          scripts/junit_to_csv.py unit_tests_report.csv unit_tests_report_*.xml
+          cat unit_tests_report.csv
+
       - name: Create Pkg
         if: ${{ matrix.config == 'Release' }}
         run: |

.github/workflows/build-test-ubuntu-arm64.yml

@@ -192,6 +192,8 @@ jobs:
         run: xvfb-run -a ./build/${{ matrix.config }}/bin/MeshViewer -hidden -noEventLoop -unloadPluginsAtEnd
 
       - name: Unit Tests
+        env:
+          GTEST_OUTPUT: 'xml:unit_tests_report_gtest.xml'
         run: ./build/${{ matrix.config }}/bin/MRTest
 
       - name: C Unit Tests (old bindings)
@@ -205,18 +207,24 @@ jobs:
         if: ${{ inputs.mrbind }}
         timeout-minutes: 8
         working-directory: ./build/${{ matrix.config }}/bin
-        run: python3 ./../../../scripts/run_python_test_script.py -d '../test_python'
+        run: python3 ./../../../scripts/run_python_test_script.py -d '../test_python' -a ' --junit-xml=../unit_tests_report_pytest.xml'
 
       - name: Python Regression Tests
         if: ${{ inputs.internal_build && inputs.mrbind }}
         uses: ./.github/actions/python-regression-tests
         with:
           build_config: ${{ matrix.config }}
-          pytest_args: "--run-cuda=negative"
+          pytest_args: "--run-cuda=negative --junit-xml=../unit_tests_report_regression.xml"
           smoke: ${{ !inputs.full_config_build && matrix.config == 'Debug' }}
           test_artifacts_path: ubuntu_arm64/${{ matrix.os }}
           upload_test_artifacts: ${{ inputs.upload_test_artifacts }}
 
+      - name: Generate Test Performance Report
+        continue-on-error: true
+        run: |
+          scripts/junit_to_csv.py unit_tests_report.csv unit_tests_report_*.xml
+          cat unit_tests_report.csv
+
       - name: Create Deb
         env:
           MESHLIB_BUILD_RELEASE: "ON"

.github/workflows/build-test-ubuntu-x64.yml

@@ -142,6 +142,8 @@ jobs:
         run: xvfb-run -a ./build/${{ matrix.config }}/bin/MeshViewer -hidden -noEventLoop -unloadPluginsAtEnd
 
       - name: Unit Tests
+        env:
+          GTEST_OUTPUT: 'xml:unit_tests_report_gtest.xml'
         run: ./build/${{ matrix.config }}/bin/MRTest
 
       - name: C Unit Tests (old bindings)
@@ -155,12 +157,12 @@ jobs:
         if: ${{ inputs.mrbind }}
         timeout-minutes: 8
         working-directory: ./build/${{ matrix.config }}/bin
-        run: python3 ./../../../scripts/run_python_test_script.py -d '../test_python'
+        run: python3 ./../../../scripts/run_python_test_script.py -d '../test_python' -a ' --junit-xml=../unit_tests_report_pytest.xml'
 
       - name: Python Regression Tests
         if: ${{ inputs.internal_build && inputs.mrbind }}
         env:
-          RUN_CUDA_ARG: "--run-cuda=negative"
+          RUN_CUDA_ARG: "--run-cuda=negative --junit-xml=../unit_tests_report_regression.xml"
         uses: ./.github/actions/python-regression-tests
         with:
           build_config: ${{ matrix.config }}
@@ -169,6 +171,12 @@ jobs:
           test_artifacts_path: ${{ matrix.os }}/${{matrix.config}}/${{matrix.compiler}}
           upload_test_artifacts: ${{ inputs.upload_test_artifacts }}
 
+      - name: Generate Test Performance Report
+        continue-on-error: true
+        run: |
+          scripts/junit_to_csv.py unit_tests_report.csv unit_tests_report_*.xml
+          cat unit_tests_report.csv
+
       - name: Create Deb
         if: ${{ matrix.compiler == 'GCC' && matrix.config == 'Release' }}
         env:

.github/workflows/build-test-windows.yml

@@ -238,6 +238,8 @@ jobs:
         shell: cmd
 
       - name: Unit Tests
+        env:
+          GTEST_OUTPUT: 'xml:unit_tests_report_gtest.xml'
         run: source\x64\${{ matrix.config }}\MRTest.exe
 
       - name: C Unit Tests (old bindings)
@@ -249,24 +251,37 @@ jobs:
 
       - name: C# Unit Tests
         if: ${{ env.BUILD_C_SHARP == 'true' }}
-        run: dotnet run --project source/MRDotNet2Test -c ${{ matrix.config }}
+        run: |
+          dotnet build source\MRDotNet2Test -c ${{ matrix.config }}
+          source\x64\${{ matrix.config }}\MRDotNet2Test.exe --result="_report_nunit.xml"
+          # convert report to JUnit format
+          [Environment]::CurrentDirectory = $pwd
+          $xslt = New-Object System.Xml.Xsl.XslCompiledTransform
+          $xslt.Load("scripts\devops\nunit3-junit.xslt")
+          $xslt.Transform("_report_nunit.xml", "unit_tests_report_nunit.xml")
 
       - name: Python Sanity Tests
         timeout-minutes: 8
         working-directory: source\x64\${{ matrix.config }}
-        run: py -3 ..\..\..\scripts\run_python_test_script.py -d '..\test_python'
+        run: py -3 ..\..\..\scripts\run_python_test_script.py -d '..\test_python' -a ' --junit-xml=../unit_tests_report_pytest.xml'
 
       - name: Python Regression Tests
         if: ${{ inputs.internal_build }}
         uses: ./.github/actions/python-regression-tests
         with:
           build_config: ${{ matrix.config }}
           mrbind: ${{ inputs.mrbind }}
-          pytest_args: "${{ env.BUILD_C_SHARP == 'true' && env.PYTEST_C_SHARP_ARGS || '' }} --run-cuda=negative"
+          pytest_args: "${{ env.BUILD_C_SHARP == 'true' && env.PYTEST_C_SHARP_ARGS || '' }} --run-cuda=negative --junit-xml=../unit_tests_report_regression.xml"
           smoke: ${{ !inputs.full_config_build && matrix.config == 'Debug' }}
           test_artifacts_path: ${{ matrix.test_artifacts_path }}
           upload_test_artifacts: ${{ inputs.upload_test_artifacts }}
 
+      - name: Generate Test Performance Report
+        continue-on-error: true
+        run: |
+          py -3 scripts/junit_to_csv.py unit_tests_report.csv (get-item unit_tests_report_*.xml)
+          cat unit_tests_report.csv
+
       - name: Delete unwanted files
         # Those files should not exist when using CMake.
         shell: cmd

requirements/ubuntu.txt

@@ -9,7 +9,6 @@ libfmt-dev
 libfreetype-dev
 libgdcm-dev
 libglfw3-dev
-libgtest-dev
 libgtk-3-dev
 libhidapi-dev
 libhpdf-dev

scripts/devops/nunit3-junit.xslt

@@ -0,0 +1,72 @@
+<?xml version="1.0" encoding="utf-8"?>
+<xsl:stylesheet version="2.0" xmlns:xsl="http://www.w3.org/1999/XSL/Transform">
+  <xsl:output method="xml" indent="yes"/>
+
+  <xsl:template match="/test-run">
+    <testsuites tests="{@testcasecount}" failures="{@failed}" disabled="{@skipped}" time="{@duration}">
+      <xsl:apply-templates/>
+    </testsuites>
+  </xsl:template>
+
+  <xsl:template match="test-suite">
+    <xsl:if test="test-case">
+      <testsuite tests="{@testcasecount}" time="{@duration}" errors="{@testcasecount - @passed - @skipped - @failed}" failures="{@failed}" skipped="{@skipped}" timestamp="{@start-time}">
+        <xsl:attribute name="name">
+          <xsl:for-each select="ancestor-or-self::test-suite[@type='TestSuite']/@name">
+            <xsl:value-of select="concat(., '.')"/>
+          </xsl:for-each>
+        </xsl:attribute>
+        <xsl:apply-templates select="test-case"/>
+      </testsuite>
+      <xsl:apply-templates select="test-suite"/>
+    </xsl:if>
+    <xsl:if test="not(test-case)">
+      <xsl:apply-templates/>
+    </xsl:if>
+  </xsl:template>
+
+  <xsl:template match="test-case">
+    <testcase name="{@name}" assertions="{@asserts}" time="{@duration}" status="{@result}" classname="{@classname}">
+      <xsl:if test="@runstate = 'Skipped' or @runstate = 'Ignored'">
+        <skipped/>
+      </xsl:if>
+
+      <xsl:apply-templates/>
+    </testcase>
+  </xsl:template>
+
+  <xsl:template match="command-line"/>
+  <xsl:template match="settings"/>
+  <xsl:template match="filter"/>
+
+  <xsl:template match="output">
+    <system-out>
+      <xsl:value-of select="."/>
+    </system-out>
+  </xsl:template>
+
+  <xsl:template match="stack-trace">
+  </xsl:template>
+
+  <xsl:template match="test-case/failure">
+    <failure message="{./message}">
+      <xsl:value-of select="./stack-trace"/>
+    </failure>
+  </xsl:template>
+
+  <xsl:template match="test-suite/failure"/>
+
+  <xsl:template match="test-case/reason">
+    <xsl:if test="./message != null">
+      <skipped message="{./message}"/>
+    </xsl:if>
+  </xsl:template>
+
+  <xsl:template match="test-case/assertions">
+  </xsl:template>
+
+  <xsl:template match="test-suite/reason"/>
+
+  <xsl:template match="properties"/>
+</xsl:stylesheet>
+

scripts/junit_to_csv.py

@@ -0,0 +1,25 @@
+#!/usr/bin/env python3
+import csv
+import sys
+import xml.etree.ElementTree as ET
+
+
+def gather_info(testsuite):
+    suite_name = testsuite.attrib['name']
+    for testcase in testsuite:
+        class_name = testcase.attrib['classname']
+        name = testcase.attrib['name']
+        time = testcase.attrib['time']
+        yield [suite_name, class_name, name, time]
+
+
+if __name__ == "__main__":
+    with open(sys.argv[1], 'w') as output:
+        writer = csv.writer(output, delimiter=',', quotechar='"')
+        writer.writerow(["suite_name", "class_name", "name", "time"])
+        for arg in sys.argv[2:]:
+            parser = ET.XMLParser(target=ET.TreeBuilder(insert_comments=True))
+            report = ET.parse(arg, parser)
+            for testsuite in report.getroot():
+                for info in gather_info(testsuite):
+                    writer.writerow(info)

thirdparty/CMakeLists.txt

@@ -38,8 +38,11 @@ IF(MR_EMSCRIPTEN)
   ENDIF()
 ENDIF()
 
-IF(APPLE)
-  set(INSTALL_GTEST ON)
+set(INSTALL_GTEST ON)
+IF(EMSCRIPTEN)
+  add_compile_definitions(GTEST_HAS_CXXABI_H_=0)
+  add_subdirectory(./googletest ./googletest)
+ELSE()
   # https://stackoverflow.com/a/42643260/7325599
   set(BUILD_SHARED_LIBS_OLD ${BUILD_SHARED_LIBS})
   set(BUILD_SHARED_LIBS ON CACHE BOOL "" FORCED)
@@ -94,10 +97,6 @@ IF(EMSCRIPTEN)
   set(GDCM_BUILD_DOCBOOK_MANPAGES OFF)
   add_subdirectory(./GDCM)
 
-  add_compile_definitions(GTEST_HAS_CXXABI_H_=0)
-  set(INSTALL_GTEST ON)
-  add_subdirectory(./googletest)
-
   set(BUILD_TOOLS OFF CACHE BOOL "")
   set(BUILD_REGRESS OFF CACHE BOOL "")
   set(BUILD_EXAMPLES OFF CACHE BOOL "")