Fix RGENGC_CHECK_MODE without VM_CHECK_MODE #12823

Job	Run time
ruby-bench (--warmup=1 --bench=1 --excludes=shipit)	2m 51s
make (check, ubuntu-24.04, capi)	3m 1s
make (test-bundler-parallel, 50)	3m 19s
make (check, --disable-yjit)	3m 17s
make (check, i686)	3m 8s
make (check, --enable-shared --enable-load-relative)	3m 1s
make (check, cppflags=-DVM_CHECK_MODE)	3m 13s
make (test-bundled-gems)	3m 10s
make-ibm	0s
Ubuntu result	3s
	25m 3s

Provide feedback