CI: Automatically run recorded tests to validate PRs and commits

.github/workflows/tests.yml

@@ -0,0 +1,56 @@
+name: Run tests
+
+on: [push, pull_request]
+
+env:
+  DOTNET_CLI_TELEMETRY_OPTOUT: 1
+  DOTNET_SKIP_FIRST_TIME_EXPERIENCE: 1
+
+jobs:
+  windows:
+    if: ${{ (github.event_name == 'push' || github.event.pull_request.head.repo.full_name != github.repository) }}
+    strategy:
+      matrix:
+        platform: [
+          {netversion: 8.x, targetframework: net8.0, aot: false, singleFile: true}
+        ]
+      fail-fast: false
+    runs-on: windows-2025-vs2026
+
+    steps:
+      - name: Checkout repo
+        uses: actions/checkout@v6
+        with:
+          fetch-depth: 0
+
+      - name: Install .NET Core
+        uses: actions/setup-dotnet@v5
+        with:
+          dotnet-version: ${{ matrix.platform.netversion }}
+
+      - name: Setup MSBuild.exe
+        uses: microsoft/setup-msbuild@v3
+        with:
+          msbuild-architecture: x64
+
+      - name: Restore packages
+        run: dotnet restore -p:TargetFramework="${{ matrix.platform.targetframework }}" -r win-x64 -p:PublishAot="${{ matrix.platform.aot }}" -p:BuildWithNetFrameworkHostedCompiler=true
+
+      - name: Build Mesen
+        run: msbuild -nologo -m -p:Configuration=Release -p:Platform=x64 -t:Clean,PGOHelper -p:TargetFramework="${{ matrix.platform.targetframework }}"
+
+      - name: Checkout MesenTests repo
+        uses: actions/checkout@v6
+        with:
+          repository: nesdev-org/MesenTests
+          path: ./bin/win-x64/Release/Tests
+
+      - name: Run CI Tests
+        run: ./bin/win-x64/Release/PGOHelper.exe "./bin/win-x64/Release/Tests" citests
+
+      - name: Upload screenshots for failures
+        if: ${{ failure() }}
+        uses: actions/upload-artifact@v7
+        with:
+          name: Screenshots
+          path: ./bin/win-x64/Release/Tests/MesenHomeFolder/Screenshots/

Core/Shared/EmuSettings.cpp

@@ -149,7 +149,8 @@ void EmuSettings::Serialize(Serializer& s)
 			break;
 
 		case ConsoleType::Ws:
-			//TODOWS
+			SV(_ws.Model);
+			SV(_ws.UseBootRom);
 			break;
 
 		default:

Core/Shared/RecordedRomTest.cpp

@@ -240,7 +240,7 @@ RomTestResult RecordedRomTest::Run(string filename)
 			_emu->Resume();
 			_signal.Wait();
 			if(!_isLastFrameGood) {
-				_emu->GetVideoDecoder()->TakeScreenshot();
+				_emu->GetVideoDecoder()->TakeScreenshot(FolderUtilities::GetFilename(filename, false));
 			}
 			_emu->Stop(!_inBackground);
 			_runningTest = false;

Core/Shared/Video/VideoDecoder.cpp

@@ -242,10 +242,10 @@ bool VideoDecoder::IsRunning()
 	return _decodeThread != nullptr;
 }
 
-void VideoDecoder::TakeScreenshot()
+void VideoDecoder::TakeScreenshot(string romName)
 {
 	if(_videoFilter) {
-		_videoFilter->TakeScreenshot(_emu->GetRomInfo().RomFile.GetFileName(), _videoFilterType);
+		_videoFilter->TakeScreenshot(romName.empty() ? _emu->GetRomInfo().RomFile.GetFileName() : romName, _videoFilterType);
 	}
 }
 

Core/Shared/Video/VideoDecoder.h

@@ -50,7 +50,7 @@ class VideoDecoder
 	void Init();
 
 	void DecodeFrame(bool synchronous = false);
-	void TakeScreenshot();
+	void TakeScreenshot(string romName = "");
 	void TakeScreenshot(std::stringstream& stream);
 
 	void ForceFilterUpdate() { _forceFilterUpdate = true; }

Core/WS/WsConsole.h

@@ -75,6 +75,7 @@ class WsConsole final : public IConsole
 	bool IsColorModel();
 	bool IsPowerOff();
 	bool IsVerticalMode();
+	bool HasBootRom() { return _bootRom != nullptr; }
 	WsAudioMode GetAudioMode();
 	WsModel GetModel();
 

Core/WS/WsEeprom.cpp

@@ -19,7 +19,7 @@ WsEeprom::WsEeprom(Emulator* emu, WsConsole* console, WsEepromSize size, uint8_t
 	_state.WriteDisabled = true;
 
 	if(_isInternal) {
-		if(!emu->GetSettings()->GetWsConfig().UseBootRom) {
+		if(!_console->HasBootRom()) {
 			//Boot ROM leaves writes enabled
 			_state.WriteDisabled = false;
 		}

InteropDLL/TestApiWrapper.cpp

@@ -2,13 +2,15 @@
 #include "Core/Shared/RecordedRomTest.h"
 #include "Core/Shared/Emulator.h"
 #include "Core/Shared/EmuSettings.h"
+#include "Utilities/FolderUtilities.h"
+#include "Utilities/StringUtilities.h"
 
 extern unique_ptr<Emulator> _emu;
 shared_ptr<RecordedRomTest> _recordedRomTest;
 
 extern "C"
 {
-	DllExport RomTestResult __stdcall RunRecordedTest(char* filename, bool inBackground)
+	DllExport RomTestResult __stdcall RunRecordedTest(const char* filename, bool inBackground)
 	{
 		if(inBackground) {
 			unique_ptr<Emulator> emu(new Emulator());
@@ -73,4 +75,107 @@ extern "C"
 	{
 		return _recordedRomTest != nullptr;
 	}
+
+	DllExport bool __stdcall RunCiTests(string testFolder, bool enableDebugger)
+	{
+		std::replace(testFolder.begin(), testFolder.end(), '\\', '/');
+		if(!StringUtilities::EndsWith(testFolder, "/")) {
+			testFolder += "/";
+		}
+
+		//GBA tests can't run without the GBA BIOS, skip all of them
+		vector<string> foldersToSkip = { "GBA/" };
+
+		unordered_set<string> testsToSkip = {
+			//These GB tests fail because the test runner can't use the official GB boot rom
+			"GB/GBEmulatorShootout/hacktix/bully_gbc.mtp",
+			"GB/GBEmulatorShootout/hacktix/bully.mtp",
+			"GB/GBEmulatorShootout/mooneye/acceptance/boot_div-dmgABCmgb.mtp",
+			"GB/GBEmulatorShootout/mooneye/acceptance/boot_hwio-dmgABCmgb.mtp",
+			"GB/GBEmulatorShootout/mooneye/acceptance/serial/boot_sclk_align-dmgABCmgb.mtp",
+			"GB/GBEmulatorShootout/mooneye/misc/boot_div-cgbABCDE.mtp",
+
+			//SGB tests can't run properly without the SGB ROM
+			"SNES/sgb_packet_test.mtp"
+		};
+
+		vector<string> tests = FolderUtilities::GetFilesInFolder(testFolder, { ".mtp" }, true, 5);
+
+		vector<string> testsToRun;
+
+		for(string& test : tests) {
+			string testPath = test.substr(testFolder.size());
+			std::replace(testPath.begin(), testPath.end(), '\\', '/');
+
+			bool include = true;
+			for(string& folderToSkip : foldersToSkip) {
+				if(StringUtilities::StartsWith(testPath, folderToSkip.c_str())) {
+					include = false;
+					break;
+				}
+			}
+
+			if(testsToSkip.find(testPath) != testsToSkip.end()) {
+				include = false;
+			}
+
+			if(include) {
+				testsToRun.push_back(test);
+			}
+		}
+
+		FolderUtilities::SetHomeFolder(FolderUtilities::CombinePath(testFolder, "MesenHomeFolder"));
+
+		std::cout << "== CI test mode ==\n";
+		int threadCount = std::thread::hardware_concurrency();
+		int skipCount = (int)(tests.size() - testsToRun.size());
+		std::cout << "Running on " << threadCount << " threads\n";
+		std::cout << "Test count: " << tests.size() << "\n";
+		std::cout << "Tests to skip: " << skipCount << "\n\n";
+
+		std::atomic<bool> failed = false;
+		std::atomic<int> testNumber = 0;
+		std::atomic<int> failCount = 0;
+		std::atomic<int> passCount = 0;
+		std::atomic<int> progress = 0;
+
+		vector<thread> threads(threadCount);
+		for(int i = 0; i < threadCount; i++) {
+			threads[i] = std::thread([&]() {
+				while(true) {
+					int nextTest = testNumber++;
+					if(nextTest >= testsToRun.size()) {
+						break;
+					}
+
+					RomTestResult result = RunRecordedTest(testsToRun[nextTest].c_str(), true);
+					if(result.State == RomTestState::Failed) {
+						failCount++;
+						string testPath = testsToRun[nextTest].substr(testFolder.size());
+						std::cout << ("\n== FAILED: " + testPath + "\n");
+					} else {
+						passCount++;
+					}
+
+					int newProgress = (failCount + passCount) * 100 / (uint32_t)testsToRun.size();
+					if(newProgress > progress) {
+						std::cout << ("\rRunning... (" + std::to_string(newProgress) + "%)");
+						progress = newProgress;
+					}
+				}
+			});
+		}
+
+		for(int i = 0; i < threadCount; i++) {
+			threads[i].join();
+		}
+
+		std::cout << "\n\n===================\n";
+		std::cout << "Passed: " << passCount << "\n";
+		std::cout << "Failed: " << failCount << "\n";
+		std::cout << "Skipped: " << skipCount << "\n";
+		std::cout << "===================\n";
+
+		return failCount == 0;
+	}
 }

PGOHelper/PGOHelper.cpp

@@ -16,6 +16,7 @@ using std::vector;
 extern "C"
 {
 	void __stdcall PgoRunTest(vector<string> testRoms, bool enableDebugger);
+	bool __stdcall RunCiTests(string testFolder);
 }
 
 vector<string> GetFilesInFolder(string rootFolder, std::unordered_set<string> extensions)
@@ -44,12 +45,18 @@ vector<string> GetFilesInFolder(string rootFolder, std::unordered_set<string> ex
 int main(int argc, char* argv[])
 {
 	string romFolder = "../PGOGames";
+	bool ciTestMode = argc == 3 && strcmp(argv[2], "citests") == 0;
+
 	if(argc >= 2) {
 		romFolder = argv[1];
 	}
 
-	vector<string> testRoms = GetFilesInFolder(romFolder, { ".sfc", ".gb", ".gbc", ".gbx", ".nes", ".pce", ".cue", ".sms", ".gg", ".sg", ".gba", ".col", ".ws", ".wsc", ".pc2" });
-	PgoRunTest(testRoms, true);
+	if(ciTestMode) {
+		return RunCiTests(romFolder) ? 0 : -1;
+	} else {
+		vector<string> testRoms = GetFilesInFolder(romFolder, { ".sfc", ".gb", ".gbc", ".gbx", ".nes", ".pce", ".cue", ".sms", ".gg", ".sg", ".gba", ".col", ".ws", ".wsc", ".pc2" });
+		PgoRunTest(testRoms, true);
+	}
 	return 0;
 }
 

Utilities/FolderUtilities.cpp

@@ -175,7 +175,7 @@ vector<string> FolderUtilities::GetFolders(string rootFolder)
 	return folders;
 }
 
-vector<string> FolderUtilities::GetFilesInFolder(string rootFolder, std::unordered_set<string> extensions, bool recursive)
+vector<string> FolderUtilities::GetFilesInFolder(string rootFolder, std::unordered_set<string> extensions, bool recursive, int maxDepth)
 {
 	vector<string> files;
 	vector<string> folders = { { rootFolder } };
@@ -187,7 +187,7 @@ vector<string> FolderUtilities::GetFilesInFolder(string rootFolder, std::unorder
 
 	if(recursive) {
 		for(fs::recursive_directory_iterator i(fs::u8path(rootFolder)), end; i != end; i++) {
-			if(i.depth() > 1) {
+			if(i.depth() > maxDepth) {
 				//Prevent excessive recursion
 				i.disable_recursion_pending();
 			} else {

Utilities/FolderUtilities.h

@@ -31,7 +31,7 @@ class FolderUtilities
 	static string GetRecentGamesFolder();
 
 	static vector<string> GetFolders(string rootFolder);
-	static vector<string> GetFilesInFolder(string rootFolder, std::unordered_set<string> extensions, bool recursive);
+	static vector<string> GetFilesInFolder(string rootFolder, std::unordered_set<string> extensions, bool recursive, int maxDepth = 1);
 
 	static string GetFilename(string filepath, bool includeExtension);
 	static string GetExtension(string filename);

Utilities/ZipReader.cpp

@@ -48,15 +48,11 @@ bool ZipReader::ExtractFile(string filename, vector<uint8_t>& output)
 		size_t uncompSize;
 		void* p = mz_zip_reader_extract_file_to_heap(&_zipArchive, filename.c_str(), &uncompSize, 0);
 		if(!p) {
-#ifdef _DEBUG
-			std::cout << "mz_zip_reader_extract_file_to_heap() failed!" << std::endl;
-#endif
 			return false;
 		}
 
 		output = vector<uint8_t>((uint8_t*)p, (uint8_t*)p + uncompSize);
 
-		// We're done.
 		mz_free(p);
 
 		return true;