Refactor testing to avoid shell scripts, make matching more robust

.github/actions/install-deps/action.yml

@@ -7,7 +7,7 @@ runs:
     - name: Setup `wasmtime` for tests
       uses: bytecodealliance/actions/wasmtime/setup@v1
       with:
-        version: "41.0.3"
+        version: "44.0.0"
     - name: Install ccache, ninja (macOS)
       run: brew install ccache ninja
       if: runner.os == 'macOS'

tests/CMakeLists.txt

@@ -28,12 +28,26 @@ set(opt_flags -O0 -O2 "-O2 -flto")
 
 add_custom_target(build-tests)
 
-# Executes a single `test` specified.
+# Registers `test` with CMake, compiling it with a number of flag combinations
+# and for all enabled targets. This will register up to many tests with CTest.
 #
-# This will compile `test` for all the various targets and with various
-# compiler options. If `runwasi` is non-empty then the test will be executed
-# in that runner as well.
-function(add_testcase runwasi test)
+# This function takes CMake-style arguments, specified as:
+#
+# * `EMULATED_CLOCKS` - enables `-lwasi-emulated-process-clocks` when compiling.
+# * `EMULATED_MMAN` - enables `-lwasi-emulated-mman` when compiling.
+# * `EMULATED_SIGNAL` - enables `-lwasi-emulated-signal` when compiling.
+# * `PRINTSCAN_LONG_DOUBLE` - enables `-lc-printscan-long-double` when compiling.
+# * `FSDIR` - requires `${test}.dir` to exist and mounts it when running the test
+# * `PASS_REGULAR_EXPRESSION` - same as the CTest property
+# * `ENV` - env vars (the `--env` flag in Wasmtime) to pass to the test.
+# * `COMPILE_ONLY` - does not actually execute this test, just compiles it.
+function(add_testcase test)
+  set(options EMULATED_CLOCKS EMULATED_MMAN EMULATED_SIGNAL
+    PRINTSCAN_LONG_DOUBLE FSDIR COMPILE_ONLY)
+  set(oneValueArgs PASS_REGULAR_EXPRESSION)
+  set(multiValueArgs ENV)
+  cmake_parse_arguments(PARSE_ARGV 1 arg "${options}" "${oneValueArgs}" "${multiValueArgs}")
+
   foreach(target IN LISTS WASI_SDK_TARGETS)
     foreach(compile_flags IN LISTS opt_flags)
       # Mangle the options into something appropriate for a CMake rule name
@@ -52,16 +66,19 @@ function(add_testcase runwasi test)
       endif()
 
       # Apply test-specific compile options and link flags.
-      if(test MATCHES "clocks.c$")
+      if(${arg_EMULATED_CLOCKS})
         target_compile_options(${target_name} PRIVATE -D_WASI_EMULATED_PROCESS_CLOCKS)
         target_link_options(${target_name} PRIVATE -lwasi-emulated-process-clocks)
-      elseif(test MATCHES "mmap.c$")
+      endif()
+      if(${arg_EMULATED_MMAN})
         target_compile_options(${target_name} PRIVATE -D_WASI_EMULATED_MMAN)
         target_link_options(${target_name} PRIVATE -lwasi-emulated-mman)
-      elseif(test MATCHES "(sigabrt|signals).c$")
+      endif()
+      if(${arg_EMULATED_SIGNAL})
         target_compile_options(${target_name} PRIVATE -D_WASI_EMULATED_SIGNAL)
         target_link_options(${target_name} PRIVATE -lwasi-emulated-signal)
-      elseif(test MATCHES "printf-long-double-enabled.c$")
+      endif()
+      if(${arg_PRINTSCAN_LONG_DOUBLE})
         target_link_options(${target_name} PRIVATE -lc-printscan-long-double)
       endif()
 
@@ -93,27 +110,42 @@ function(add_testcase runwasi test)
         target_link_options(${target_name} PRIVATE -Wno-deprecated)
       endif()
 
-      if(runwasi)
-        set(runner ${runwasi})
-        if(${runner} MATCHES wasmtime)
-          if(target MATCHES threads)
-            set(runner "${runner} -Wshared-memory")
-          endif()
-          if(WASI_SDK_EXCEPTIONS)
-            set(runner "${runner} -Wexceptions")
-          endif()
-          if(target MATCHES "wasip3")
-            set(runner "${runner} -Wcomponent-model-async -Sp3")
-          endif()
+      if(arg_COMPILE_ONLY)
+        continue()
+      endif()
+
+      set(runner ${WASI_SDK_RUNWASI})
+      set(args)
+      if(${runner} MATCHES wasmtime)
+        if(target MATCHES threads)
+          list(APPEND runner -Wshared-memory)
         endif()
-        add_test(
-          NAME test-${target_name}
-          COMMAND
-            bash ../testcase.sh
-            ${runner}
-            ${test}
-            $<TARGET_FILE:${target_name}>
-          WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR})
+        if(WASI_SDK_EXCEPTIONS)
+          list(APPEND runner -Wexceptions)
+        endif()
+        if(target MATCHES "wasip3")
+          list(APPEND runner -Wcomponent-model-async -Sp3)
+        endif()
+      endif()
+
+      foreach(env IN LISTS arg_ENV)
+        list(APPEND runner --env ${env})
+      endforeach()
+
+      if (${arg_FSDIR})
+        list(APPEND runner --dir ${CMAKE_CURRENT_SOURCE_DIR}/${test}.dir::${test}.dir)
+        list(APPEND args ${test}.dir)
+      endif()
+
+      add_test(
+        NAME test-${target_name}
+        COMMAND
+          ${runner}
+          $<TARGET_FILE:${target_name}>
+          ${args}
+      )
+      if (arg_PASS_REGULAR_EXPRESSION)
+        set_tests_properties(test-${target_name} PROPERTIES PASS_REGULAR_EXPRESSION ${arg_PASS_REGULAR_EXPRESSION})
       endif()
     endforeach()
   endforeach()

tests/compile-only/CMakeLists.txt

@@ -1,9 +1,3 @@
-file(GLOB c_compile_tests RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} "*.c")
-file(GLOB cxx_compile_tests RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} "*.cc")
-
-set(compile_tests ${c_compile_tests} ${cxx_compile_tests})
-
-foreach(test IN LISTS compile_tests)
-  add_testcase("" ${test})
-endforeach()
-
+add_testcase(addresses.c COMPILE_ONLY)
+add_testcase(printf-long-double.c COMPILE_ONLY)
+add_testcase(test.cc COMPILE_ONLY)

tests/general/CMakeLists.txt

@@ -1,8 +1,69 @@
-file(GLOB c_general_tests RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} CONFIGURE_DEPENDS "*.c")
-file(GLOB cxx_general_tests RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} CONFIGURE_DEPENDS "*.cc")
+add_testcase(abort.c PASS_REGULAR_EXPRESSION "wasm trap.*unreachable")
+add_testcase(argc_argv_main.c PASS_REGULAR_EXPRESSION "hello from argc argv main")
+add_testcase(argc_argv_main.cc PASS_REGULAR_EXPRESSION "hello from C\\+\\+ argc argv main")
+add_testcase(assert-fail.c PASS_REGULAR_EXPRESSION "Assertion failed: false.*wasm trap:.*unreachable")
+add_testcase(assert-pass.c)
+add_testcase(clocks.c EMULATED_CLOCKS)
+add_testcase(cpp_thread_local.cc)
+add_testcase(ctors_dtors.c PASS_REGULAR_EXPRESSION "\
+hello from_constructor101
+hello from_constructor
+hello from_constructor65535
+hello main
+goodbye main
+hello another_from_atexit
+hello from_atexit
+hello from_destructor65535
+hello from_destructor
+hello from_destructor101
+"
+)
+add_testcase(ctors_dtors.cc PASS_REGULAR_EXPRESSION "\
+hello from_constructor101
+hello from_constructor
+hello from_constructor65535
+hello StaticObject::StaticObject
+hello main
+goodbye main
+hello another_from_atexit
+hello from_atexit
+hello from_destructor65535
+hello from_destructor
+hello StaticObject::~StaticObject
+hello from_destructor101
+")
+add_testcase(empty.c)
+add_testcase(env-absent.c PASS_REGULAR_EXPRESSION "HELLO = \\(null\\)")
+add_testcase(env.c ENV HELLO=hello PASS_REGULAR_EXPRESSION "HELLO = hello")
+add_testcase(environ.c ENV HELLO=hello PASS_REGULAR_EXPRESSION "HELLO = hello")
+add_testcase(exceptions.cc)
+add_testcase(getentropy.c)
+add_testcase(iostream_main.cc PASS_REGULAR_EXPRESSION "hello from C\\+\\+ main with cout")
+add_testcase(main_errno.c PASS_REGULAR_EXPRESSION "initial errno is 0: Success")
+add_testcase(mmap.c EMULATED_MMAN FSDIR PASS_REGULAR_EXPRESSION "\
+“Would you tell me, please, which way I ought to go from here\\?”
 
-set(general_tests ${c_general_tests} ${cxx_general_tests})
-
-foreach(test IN LISTS general_tests)
-  add_testcase(${WASI_SDK_RUNWASI} ${test})
-endforeach()
+“That depends a good deal on where you want to get to,” said the Cat.
+"
+)
+add_testcase(no_arg_main.c PASS_REGULAR_EXPRESSION "hello from no-arg main")
+add_testcase(no_arg_main.cc PASS_REGULAR_EXPRESSION "hello from C\\+\\+ no-arg main")
+add_testcase(opendir.c FSDIR)
+add_testcase(printf-long-double-enabled.c PRINTSCAN_LONG_DOUBLE PASS_REGULAR_EXPRESSION "the answer is 42.000000")
+add_testcase(printf-no-float.c PASS_REGULAR_EXPRESSION "the answer is 42")
+add_testcase(printf-no-long-double.c PASS_REGULAR_EXPRESSION "the answer is 42.000000")
+add_testcase(sigabrt.c EMULATED_SIGNAL PASS_REGULAR_EXPRESSION "\
+raising SIGABRT...
+Program received fatal signal: Aborted
+.*failed to run main module.*
+")
+add_testcase(signals.c EMULATED_SIGNAL PASS_REGULAR_EXPRESSION "\
+psignal message for SIGINT: Interrupt
+strsignal for SIGHUP: 'Hangup'
+beginning handler test:
+handler for signal Window changed
+finished handler test
+")
+add_testcase(stat.c FSDIR)
+add_testcase(void_main.c PASS_REGULAR_EXPRESSION "hello from void main")
+add_testcase(void_main.cc PASS_REGULAR_EXPRESSION "hello from C\\+\\+ void main")