diff --git a/.github/workflows/build-test-linux-vcpkg.yml b/.github/workflows/build-test-linux-vcpkg.yml index 2f0c91420784..f1abe10987a6 100644 --- a/.github/workflows/build-test-linux-vcpkg.yml +++ b/.github/workflows/build-test-linux-vcpkg.yml @@ -179,6 +179,8 @@ jobs: run: xvfb-run -a ./build/${{ matrix.config }}/bin/MeshViewer -hidden -noEventLoop -unloadPluginsAtEnd - name: Unit Tests + env: + GTEST_OUTPUT: 'xml:unit_tests_report_gtest.xml' run: ./build/${{ matrix.config }}/bin/MRTest - name: C Unit Tests (old bindings) @@ -192,7 +194,7 @@ jobs: if: ${{ inputs.mrbind }} timeout-minutes: 8 working-directory: ./build/${{ matrix.config }}/bin - run: python3 ./../../../scripts/run_python_test_script.py -d '../test_python' + run: python3 ./../../../scripts/run_python_test_script.py -d '../test_python' -a ' --junit-xml=../unit_tests_report_pytest.xml' - name: Python Regression Tests if: ${{ inputs.internal_build && inputs.mrbind }} @@ -200,11 +202,17 @@ jobs: with: build_config: ${{ matrix.config }} mrbind: ${{ inputs.mrbind }} - pytest_args: "--run-cuda=negative" + pytest_args: "--run-cuda=negative --junit-xml=../unit_tests_report_regression.xml" smoke: ${{ !inputs.full_config_build && matrix.config == 'Debug' }} test_artifacts_path: linux-vcpkg-${{ matrix.arch }}/${{ matrix.config }}/${{ matrix.compiler }} upload_test_artifacts: ${{ inputs.upload_test_artifacts }} + - name: Generate Test Performance Report + continue-on-error: true + run: | + scripts/junit_to_csv.py unit_tests_report.csv unit_tests_report_*.xml + cat unit_tests_report.csv + - name: Create Package if: ${{ matrix.config == 'Release' && matrix.compiler == 'Clang 20' }} run: | diff --git a/.github/workflows/build-test-macos.yml b/.github/workflows/build-test-macos.yml index 7bf1409e1a11..832c8784b30a 100644 --- a/.github/workflows/build-test-macos.yml +++ b/.github/workflows/build-test-macos.yml @@ -173,6 +173,8 @@ jobs: run: ./build/${{ matrix.config }}/bin/MeshViewer -tryHidden -noEventLoop -unloadPluginsAtEnd - name: Unit Tests + env: + GTEST_OUTPUT: 'xml:unit_tests_report_gtest.xml' run: ./build/${{ matrix.config }}/bin/MRTest - name: C Unit Tests (old bindings) @@ -185,12 +187,12 @@ jobs: - name: Python Sanity Tests timeout-minutes: 8 working-directory: ./build/${{ matrix.config }}/bin - run: python3 ./../../../scripts/run_python_test_script.py -d '../test_python' + run: python3 ./../../../scripts/run_python_test_script.py -d '../test_python' -a ' --junit-xml=../unit_tests_report_pytest.xml' - name: Python Regression Tests if: ${{ inputs.internal_build }} env: - RUN_CUDA_ARG: "--run-cuda=negative" + RUN_CUDA_ARG: "--run-cuda=negative --junit-xml=../unit_tests_report_regression.xml" uses: ./.github/actions/python-regression-tests with: build_config: ${{ matrix.config }} @@ -199,6 +201,12 @@ jobs: test_artifacts_path: macos/${{ matrix.os }} upload_test_artifacts: ${{ inputs.upload_test_artifacts }} + - name: Generate Test Performance Report + continue-on-error: true + run: | + scripts/junit_to_csv.py unit_tests_report.csv unit_tests_report_*.xml + cat unit_tests_report.csv + - name: Create Pkg if: ${{ matrix.config == 'Release' }} run: | diff --git a/.github/workflows/build-test-ubuntu-arm64.yml b/.github/workflows/build-test-ubuntu-arm64.yml index 9d20b71e591c..18b79b875dad 100644 --- a/.github/workflows/build-test-ubuntu-arm64.yml +++ b/.github/workflows/build-test-ubuntu-arm64.yml @@ -192,6 +192,8 @@ jobs: run: xvfb-run -a ./build/${{ matrix.config }}/bin/MeshViewer -hidden -noEventLoop -unloadPluginsAtEnd - name: Unit Tests + env: + GTEST_OUTPUT: 'xml:unit_tests_report_gtest.xml' run: ./build/${{ matrix.config }}/bin/MRTest - name: C Unit Tests (old bindings) @@ -205,18 +207,24 @@ jobs: if: ${{ inputs.mrbind }} timeout-minutes: 8 working-directory: ./build/${{ matrix.config }}/bin - run: python3 ./../../../scripts/run_python_test_script.py -d '../test_python' + run: python3 ./../../../scripts/run_python_test_script.py -d '../test_python' -a ' --junit-xml=../unit_tests_report_pytest.xml' - name: Python Regression Tests if: ${{ inputs.internal_build && inputs.mrbind }} uses: ./.github/actions/python-regression-tests with: build_config: ${{ matrix.config }} - pytest_args: "--run-cuda=negative" + pytest_args: "--run-cuda=negative --junit-xml=../unit_tests_report_regression.xml" smoke: ${{ !inputs.full_config_build && matrix.config == 'Debug' }} test_artifacts_path: ubuntu_arm64/${{ matrix.os }} upload_test_artifacts: ${{ inputs.upload_test_artifacts }} + - name: Generate Test Performance Report + continue-on-error: true + run: | + scripts/junit_to_csv.py unit_tests_report.csv unit_tests_report_*.xml + cat unit_tests_report.csv + - name: Create Deb env: MESHLIB_BUILD_RELEASE: "ON" diff --git a/.github/workflows/build-test-ubuntu-x64.yml b/.github/workflows/build-test-ubuntu-x64.yml index 84245bdad531..69e062e08108 100644 --- a/.github/workflows/build-test-ubuntu-x64.yml +++ b/.github/workflows/build-test-ubuntu-x64.yml @@ -142,6 +142,8 @@ jobs: run: xvfb-run -a ./build/${{ matrix.config }}/bin/MeshViewer -hidden -noEventLoop -unloadPluginsAtEnd - name: Unit Tests + env: + GTEST_OUTPUT: 'xml:unit_tests_report_gtest.xml' run: ./build/${{ matrix.config }}/bin/MRTest - name: C Unit Tests (old bindings) @@ -155,12 +157,12 @@ jobs: if: ${{ inputs.mrbind }} timeout-minutes: 8 working-directory: ./build/${{ matrix.config }}/bin - run: python3 ./../../../scripts/run_python_test_script.py -d '../test_python' + run: python3 ./../../../scripts/run_python_test_script.py -d '../test_python' -a ' --junit-xml=../unit_tests_report_pytest.xml' - name: Python Regression Tests if: ${{ inputs.internal_build && inputs.mrbind }} env: - RUN_CUDA_ARG: "--run-cuda=negative" + RUN_CUDA_ARG: "--run-cuda=negative --junit-xml=../unit_tests_report_regression.xml" uses: ./.github/actions/python-regression-tests with: build_config: ${{ matrix.config }} @@ -169,6 +171,12 @@ jobs: test_artifacts_path: ${{ matrix.os }}/${{matrix.config}}/${{matrix.compiler}} upload_test_artifacts: ${{ inputs.upload_test_artifacts }} + - name: Generate Test Performance Report + continue-on-error: true + run: | + scripts/junit_to_csv.py unit_tests_report.csv unit_tests_report_*.xml + cat unit_tests_report.csv + - name: Create Deb if: ${{ matrix.compiler == 'GCC' && matrix.config == 'Release' }} env: diff --git a/.github/workflows/build-test-windows.yml b/.github/workflows/build-test-windows.yml index d80f6ae89f21..4a32b1bb4bfe 100644 --- a/.github/workflows/build-test-windows.yml +++ b/.github/workflows/build-test-windows.yml @@ -238,6 +238,8 @@ jobs: shell: cmd - name: Unit Tests + env: + GTEST_OUTPUT: 'xml:unit_tests_report_gtest.xml' run: source\x64\${{ matrix.config }}\MRTest.exe - name: C Unit Tests (old bindings) @@ -249,12 +251,19 @@ jobs: - name: C# Unit Tests if: ${{ env.BUILD_C_SHARP == 'true' }} - run: dotnet run --project source/MRDotNet2Test -c ${{ matrix.config }} + run: | + dotnet build source\MRDotNet2Test -c ${{ matrix.config }} + source\x64\${{ matrix.config }}\MRDotNet2Test.exe --result="_report_nunit.xml" + # convert report to JUnit format + [Environment]::CurrentDirectory = $pwd + $xslt = New-Object System.Xml.Xsl.XslCompiledTransform + $xslt.Load("scripts\devops\nunit3-junit.xslt") + $xslt.Transform("_report_nunit.xml", "unit_tests_report_nunit.xml") - name: Python Sanity Tests timeout-minutes: 8 working-directory: source\x64\${{ matrix.config }} - run: py -3 ..\..\..\scripts\run_python_test_script.py -d '..\test_python' + run: py -3 ..\..\..\scripts\run_python_test_script.py -d '..\test_python' -a ' --junit-xml=../unit_tests_report_pytest.xml' - name: Python Regression Tests if: ${{ inputs.internal_build }} @@ -262,11 +271,17 @@ jobs: with: build_config: ${{ matrix.config }} mrbind: ${{ inputs.mrbind }} - pytest_args: "${{ env.BUILD_C_SHARP == 'true' && env.PYTEST_C_SHARP_ARGS || '' }} --run-cuda=negative" + pytest_args: "${{ env.BUILD_C_SHARP == 'true' && env.PYTEST_C_SHARP_ARGS || '' }} --run-cuda=negative --junit-xml=../unit_tests_report_regression.xml" smoke: ${{ !inputs.full_config_build && matrix.config == 'Debug' }} test_artifacts_path: ${{ matrix.test_artifacts_path }} upload_test_artifacts: ${{ inputs.upload_test_artifacts }} + - name: Generate Test Performance Report + continue-on-error: true + run: | + py -3 scripts/junit_to_csv.py unit_tests_report.csv (get-item unit_tests_report_*.xml) + cat unit_tests_report.csv + - name: Delete unwanted files # Those files should not exist when using CMake. shell: cmd diff --git a/requirements/ubuntu.txt b/requirements/ubuntu.txt index 9b3eccab08ee..4439a89aa550 100644 --- a/requirements/ubuntu.txt +++ b/requirements/ubuntu.txt @@ -9,7 +9,6 @@ libfmt-dev libfreetype-dev libgdcm-dev libglfw3-dev -libgtest-dev libgtk-3-dev libhidapi-dev libhpdf-dev diff --git a/scripts/devops/nunit3-junit.xslt b/scripts/devops/nunit3-junit.xslt new file mode 100644 index 000000000000..328d0b0238ce --- /dev/null +++ b/scripts/devops/nunit3-junit.xslt @@ -0,0 +1,72 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/scripts/junit_to_csv.py b/scripts/junit_to_csv.py new file mode 100755 index 000000000000..f8762842da8e --- /dev/null +++ b/scripts/junit_to_csv.py @@ -0,0 +1,25 @@ +#!/usr/bin/env python3 +import csv +import sys +import xml.etree.ElementTree as ET + + +def gather_info(testsuite): + suite_name = testsuite.attrib['name'] + for testcase in testsuite: + class_name = testcase.attrib['classname'] + name = testcase.attrib['name'] + time = testcase.attrib['time'] + yield [suite_name, class_name, name, time] + + +if __name__ == "__main__": + with open(sys.argv[1], 'w') as output: + writer = csv.writer(output, delimiter=',', quotechar='"') + writer.writerow(["suite_name", "class_name", "name", "time"]) + for arg in sys.argv[2:]: + parser = ET.XMLParser(target=ET.TreeBuilder(insert_comments=True)) + report = ET.parse(arg, parser) + for testsuite in report.getroot(): + for info in gather_info(testsuite): + writer.writerow(info) diff --git a/thirdparty/CMakeLists.txt b/thirdparty/CMakeLists.txt index bde01fd4b7c7..e4734cf29af3 100644 --- a/thirdparty/CMakeLists.txt +++ b/thirdparty/CMakeLists.txt @@ -38,8 +38,11 @@ IF(MR_EMSCRIPTEN) ENDIF() ENDIF() -IF(APPLE) - set(INSTALL_GTEST ON) +set(INSTALL_GTEST ON) +IF(EMSCRIPTEN) + add_compile_definitions(GTEST_HAS_CXXABI_H_=0) + add_subdirectory(./googletest ./googletest) +ELSE() # https://stackoverflow.com/a/42643260/7325599 set(BUILD_SHARED_LIBS_OLD ${BUILD_SHARED_LIBS}) set(BUILD_SHARED_LIBS ON CACHE BOOL "" FORCED) @@ -94,10 +97,6 @@ IF(EMSCRIPTEN) set(GDCM_BUILD_DOCBOOK_MANPAGES OFF) add_subdirectory(./GDCM) - add_compile_definitions(GTEST_HAS_CXXABI_H_=0) - set(INSTALL_GTEST ON) - add_subdirectory(./googletest) - set(BUILD_TOOLS OFF CACHE BOOL "") set(BUILD_REGRESS OFF CACHE BOOL "") set(BUILD_EXAMPLES OFF CACHE BOOL "")