From 5f22ff7043ed6e6b81ebf84f4028f588ed743197 Mon Sep 17 00:00:00 2001 From: Mugdha Dhole Date: Wed, 1 Jul 2026 11:39:41 +0530 Subject: [PATCH] Adds CI job to detect missing py_test mappings to system tests - CI job created to run the system test bazel build mapping - Python checker script to detect tests_system test files not listed in local bazel py_test targets --- .github/workflows/ci.yml | 11 ++ tests_system/check_tests_and_build_mapping.py | 136 ++++++++++++++++++ tests_system/lobster_json/BUILD.bazel | 9 ++ 3 files changed, 156 insertions(+) create mode 100644 tests_system/check_tests_and_build_mapping.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f38c4379..da3e5cc9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -198,6 +198,16 @@ jobs: repository-cache: true - uses: actions/checkout@v4 - run: bazel test //tests_system/... //tests_unit/... + system-test-build-mapping: + name: System Test Build Mapping + needs: changes + if: ${{ always() && needs.changes.outputs.only_docu_modified == 'false' }} + runs-on: ubuntu-24.04 + steps: + - uses: actions/checkout@v4 + - name: Detect system tests missing in BUILD.bazel + run: | + python3 tests_system/check_tests_and_build_mapping.py failure: name: Check all jobs needs: @@ -207,6 +217,7 @@ jobs: - test - integration-tests - bazel-integration + - system-test-build-mapping if: ${{ failure() || cancelled() }} runs-on: ubuntu-24.04 steps: diff --git a/tests_system/check_tests_and_build_mapping.py b/tests_system/check_tests_and_build_mapping.py new file mode 100644 index 00000000..283f866b --- /dev/null +++ b/tests_system/check_tests_and_build_mapping.py @@ -0,0 +1,136 @@ +"""Detect tests_system test files not mapped to any py_test target.""" + +import re +import sys +from pathlib import Path + + +def _iter_py_test_blocks(build_text): + """Return each py_test(...) block from the given BUILD file text.""" + blocks = [] + lines = build_text.splitlines() + line_index = 0 + + while line_index < len(lines): + current_line = lines[line_index] + if not re.match(r"^\s*py_test\s*\(", current_line): + line_index += 1 + continue + + block_lines = [current_line] + open_parens = current_line.count("(") - current_line.count(")") + line_index += 1 + + while line_index < len(lines) and open_parens > 0: + current_line = lines[line_index] + block_lines.append(current_line) + open_parens += current_line.count("(") - current_line.count(")") + line_index += 1 + + blocks.append("\n".join(block_lines)) + + return blocks + + +def _extract_test_srcs(build_text): + """Return the test_*.py files referenced by py_test srcs lists.""" + test_names = set() + + for block in _iter_py_test_blocks(build_text): + match = re.search(r"srcs\s*=\s*\[(.*?)\]", block, flags=re.DOTALL) + if not match: + continue + + for src in re.findall(r'"([^"]+\.py)"', match.group(1)): + src_name = Path(src).name + if src_name.startswith("test_"): + test_names.add(src_name) + + return test_names + + +def _tool_directories(tests_system_dir): + """Return the immediate subdirectories in tests_system that may contain tests.""" + tool_dirs = [] + + for entry in sorted(tests_system_dir.iterdir()): + if not entry.is_dir(): + continue + if entry.name.startswith((".", "__")): + continue + tool_dirs.append(entry) + + return tool_dirs + + +def main(): + workspace_root = Path(__file__).resolve().parent.parent + tests_system_dir = workspace_root / "tests_system" + + missing_build_dirs = [] + missing_test_files = [] + stale_test_entries = [] + discovered_tests = set() + declared_tests = set() + + for tool_dir in _tool_directories(tests_system_dir): + test_files = set() + for test_file in tool_dir.glob("test_*.py"): + test_files.add(test_file.relative_to(workspace_root).as_posix()) + + if not test_files: + continue + + discovered_tests.update(test_files) + + build_file = tool_dir / "BUILD.bazel" + tool_name = tool_dir.relative_to(workspace_root).as_posix() + + if not build_file.is_file(): + missing_build_dirs.append(tool_name) + continue + + listed_tests = set() + for test_name in _extract_test_srcs(build_file.read_text(encoding="utf-8")): + listed_tests.add(f"{tool_name}/{test_name}") + + declared_tests.update(listed_tests) + + for test_path in sorted(test_files - listed_tests): + missing_test_files.append(test_path) + + for test_path in sorted(listed_tests - test_files): + stale_test_entries.append(test_path) + + has_errors = any([ + missing_build_dirs, + missing_test_files, + stale_test_entries, + len(discovered_tests) != len(declared_tests), + ]) + if not has_errors: + return 0 + + print("ERROR: tests_system BUILD mapping check failed.") + if len(discovered_tests) != len(declared_tests): + print( + "Mismatch between tests_system test file count " + "and bazel py_test target count." + ) + if missing_build_dirs: + print("Subfolders with tests but without BUILD.bazel:") + for tool_name in missing_build_dirs: + print(f" - {tool_name}") + if missing_test_files: + print("Test files missing in BUILD.bazel file:") + for test_path in missing_test_files: + print(f" - {test_path}") + if stale_test_entries: + print("py_test(srcs) entries whose test file is missing:") + for test_path in stale_test_entries: + print(f" - {test_path}") + return 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests_system/lobster_json/BUILD.bazel b/tests_system/lobster_json/BUILD.bazel index 0a512e63..e9126d2a 100644 --- a/tests_system/lobster_json/BUILD.bazel +++ b/tests_system/lobster_json/BUILD.bazel @@ -54,6 +54,15 @@ py_test( ], ) +py_test( + name = "test_invalid_json", + srcs = ["test_invalid_json.py"], + deps = [ + ":lobster_json", + "//tests_system", + ], +) + py_test( name = "test_justification_attribute", srcs = ["test_justification_attribute.py"],