From 0e3d7a8e8506ed3899198cc392c7e74a885b7bf6 Mon Sep 17 00:00:00 2001
From: ethanwee1 <ethanwee@amd.com>
Date: Mon, 11 May 2026 14:38:33 +0000
Subject: [PATCH 1/6] [CI] Fix parity commit run selection

Filter explicit-SHA parity lookups to main and carry commit labels through log parsing and running-time CSVs.
---
 .../detect_log_failures.py                    | 14 +++-
 .../download_testlogs                         | 46 +++++++----
 .../summarize_xml_testreports.py              | 81 ++++++++++++-------
 3 files changed, 96 insertions(+), 45 deletions(-)

diff --git a/.automation_scripts/pytorch-unit-test-scripts/detect_log_failures.py b/.automation_scripts/pytorch-unit-test-scripts/detect_log_failures.py
index b563fcf74bc1e..7b4a73a69508b 100755
--- a/.automation_scripts/pytorch-unit-test-scripts/detect_log_failures.py
+++ b/.automation_scripts/pytorch-unit-test-scripts/detect_log_failures.py
@@ -69,13 +69,23 @@
 
 
 def classify_log_file(filename):
-    """Return (platform, test_config, shard_num) from a log filename like rocm3.txt."""
+    """Return (platform, test_config, shard_num) from a log filename like rocm3.txt.
+
+    Commit-vs-commit parity prefixes log files with the short commit SHA
+    (for example, 09e0c59b_rocm3.txt). In that mode the SHA label is the
+    platform name used by generate_summary.py, so preserve it here.
+    """
     stem = Path(filename).stem
+    label = None
+    m = re.match(r"(?P<label>[0-9a-f]{8,40})_(?P<stem>.+)", stem)
+    if m:
+        label = m.group("label")[:8]
+        stem = m.group("stem")
     for prefix, (platform, test_config) in sorted(LOG_FILE_MAP.items(), key=lambda x: -len(x[0])):
         if stem.startswith(prefix):
             remainder = stem[len(prefix):]
             if remainder.isdigit():
-                return platform, test_config, int(remainder)
+                return label or platform, test_config, int(remainder)
     return None, None, None
 
 
diff --git a/.automation_scripts/pytorch-unit-test-scripts/download_testlogs b/.automation_scripts/pytorch-unit-test-scripts/download_testlogs
index cd75c602d8e99..559b8cc6dcc9a 100755
--- a/.automation_scripts/pytorch-unit-test-scripts/download_testlogs
+++ b/.automation_scripts/pytorch-unit-test-scripts/download_testlogs
@@ -336,7 +336,7 @@ def download_artifacts(wf, prefixes=[], test_folder=".", allowed_substrings=None
     )
     os.chdir("..")
 # for older runs, add 'created':'<=YYYY-MM-DD'. see https://docs.github.com/en/search-github/getting-started-with-searching-on-github/understanding-the-search-syntax#query-for-dates
-def download_workflow_run(created=None, max_pages=10, workflow=None, sha=None, ignore_status=False, status='success', error_msg='Error downloading workflow runs'):
+def download_workflow_run(created=None, max_pages=10, workflow=None, sha=None, branch=None, ignore_status=False, status='success', error_msg='Error downloading workflow runs'):
     if not workflow:
         raise Exception("Workflow must be specified")
     for page in range(max_pages):
@@ -348,8 +348,10 @@ def download_workflow_run(created=None, max_pages=10, workflow=None, sha=None, i
             params['created'] = created
         if sha:
             params['head_sha'] = sha
+            if branch:
+                params['branch'] = branch
         else:
-            params['branch'] = "main"
+            params['branch'] = branch or "main"
         print(".")
 
         # Uncomment below for additional debug info
@@ -365,6 +367,10 @@ def download_workflow_run(created=None, max_pages=10, workflow=None, sha=None, i
             raise Exception(response.text)
         if not workflow_runs:
             continue
+        if branch:
+            workflow_runs = [wf for wf in workflow_runs if wf.get('head_branch') == branch]
+            if not workflow_runs:
+                continue
         # Prefer completed runs over in-progress ones. When multiple
         # runs exist for the same SHA, the most recent may still be
         # running and have no artifacts yet.
@@ -545,6 +551,7 @@ def main():
         sha = args.sha1
         pr_id = None
     status = "success"
+    sha_branch = "main" if args.sha1 else None
     print(f"sha: {sha}")
 
     # When comparing two commits, prefix log filenames with short SHAs
@@ -565,7 +572,7 @@ def main():
         #https://docs.github.com/en/rest/actions/workflow-runs#list-workflow-runs-for-a-repository
         periodic_fallback_used = False
         try:
-            periodic_wf = download_workflow_run(created=args.created, max_pages=args.max_pages, workflow=ROCmWorkflowNames["distributed"], sha=periodic_sha, ignore_status=args.ignore_status, status=status, error_msg=error_msg)
+            periodic_wf = download_workflow_run(created=args.created, max_pages=args.max_pages, workflow=ROCmWorkflowNames["distributed"], sha=periodic_sha, branch=sha_branch, ignore_status=args.ignore_status, status=status, error_msg=error_msg)
         except (IndexError, Exception):
             periodic_wf = None
         periodic_fallbacks = {
@@ -575,7 +582,7 @@ def main():
         if periodic_wf is None and arch in periodic_fallbacks:
             fallback_wf, fallback_prefix = periodic_fallbacks[arch]
             print(f"Distributed not found in {ROCmWorkflowNames['distributed']}, falling back to {fallback_wf}")
-            periodic_wf = download_workflow_run(created=args.created, max_pages=args.max_pages, workflow=fallback_wf, sha=periodic_sha, ignore_status=args.ignore_status, status=status, error_msg=error_msg)
+            periodic_wf = download_workflow_run(created=args.created, max_pages=args.max_pages, workflow=fallback_wf, sha=periodic_sha, branch=sha_branch, ignore_status=args.ignore_status, status=status, error_msg=error_msg)
             periodic_fallback_used = True
         if periodic_wf is None:
             raise Exception(error_msg)
@@ -631,7 +638,7 @@ def main():
         error_msg="Error: rocm workflow not found in scanned workflow runs. Try increasing max_pages."
         default_fallback_used = False
         try:
-            rocm_wf = download_workflow_run(created=args.created, max_pages=args.max_pages, workflow=ROCmWorkflowNames["default"], sha=rocm_sha, ignore_status=args.ignore_status, status=status, error_msg=error_msg)
+            rocm_wf = download_workflow_run(created=args.created, max_pages=args.max_pages, workflow=ROCmWorkflowNames["default"], sha=rocm_sha, branch=sha_branch, ignore_status=args.ignore_status, status=status, error_msg=error_msg)
         except (IndexError, Exception):
             rocm_wf = None
         default_fallbacks = {
@@ -641,7 +648,7 @@ def main():
         if rocm_wf is None and arch in default_fallbacks:
             fallback_wf, fallback_prefix = default_fallbacks[arch]
             print(f"Default not found in {ROCmWorkflowNames['default']}, falling back to {fallback_wf}")
-            rocm_wf = download_workflow_run(created=args.created, max_pages=args.max_pages, workflow=fallback_wf, sha=rocm_sha, ignore_status=args.ignore_status, status=status, error_msg=error_msg)
+            rocm_wf = download_workflow_run(created=args.created, max_pages=args.max_pages, workflow=fallback_wf, sha=rocm_sha, branch=sha_branch, ignore_status=args.ignore_status, status=status, error_msg=error_msg)
             default_fallback_used = True
             rocm_job_prefix['default'] = fallback_prefix
         if rocm_wf is None:
@@ -692,7 +699,7 @@ def main():
         error_msg="Error: inductor workflow not found in scanned workflow runs. Try increasing max_pages."
         inductor_fallback_used = False
         try:
-            inductor_wf_rocm = download_workflow_run(created=args.created, max_pages=args.max_pages, workflow=ROCmWorkflowNames["inductor"], sha=inductor_rocm_sha, ignore_status=args.ignore_status, status=status, error_msg=error_msg)
+            inductor_wf_rocm = download_workflow_run(created=args.created, max_pages=args.max_pages, workflow=ROCmWorkflowNames["inductor"], sha=inductor_rocm_sha, branch=sha_branch, ignore_status=args.ignore_status, status=status, error_msg=error_msg)
         except (IndexError, Exception):
             inductor_wf_rocm = None
         inductor_fallbacks = {
@@ -701,7 +708,7 @@ def main():
         if inductor_wf_rocm is None and arch in inductor_fallbacks:
             fallback_wf, fallback_prefix = inductor_fallbacks[arch]
             print(f"Inductor not found in {ROCmWorkflowNames['inductor']}, falling back to {fallback_wf}")
-            inductor_wf_rocm = download_workflow_run(created=args.created, max_pages=args.max_pages, workflow=fallback_wf, sha=inductor_rocm_sha, ignore_status=args.ignore_status, status=status, error_msg=error_msg)
+            inductor_wf_rocm = download_workflow_run(created=args.created, max_pages=args.max_pages, workflow=fallback_wf, sha=inductor_rocm_sha, branch=sha_branch, ignore_status=args.ignore_status, status=status, error_msg=error_msg)
             inductor_fallback_used = True
         if inductor_wf_rocm is None:
             raise Exception(error_msg)
@@ -756,11 +763,15 @@ def main():
         if not args.ignore_status:
             params['status'] = status
         params['head_sha'] = sha
+        if sha_branch:
+            params['branch'] = sha_branch
         resp = requests.get(
             f"https://api.github.com/repos/pytorch/pytorch/actions/workflows/{CUDAWorkflowNames['default']}.yml/runs",
             headers=authentication_headers, params=params,
         )
         trunk_runs = resp.json().get('workflow_runs', [])
+        if sha_branch:
+            trunk_runs = [run for run in trunk_runs if run.get('head_branch') == sha_branch]
 
         for run in trunk_runs:
             jobs = get_workflow_jobs(run)
@@ -796,8 +807,13 @@ def main():
                             headers=authentication_headers,
                         )
                         trunk_wf = resp.json()
-                    print(f"CUDA test jobs are in trunk run {trunk_wf['id']} (found via check-runs)")
-                all_cuda_jobs = list(cuda_test_jobs)
+                    if sha_branch and trunk_wf.get('head_branch') != sha_branch:
+                        print(f"Skipping CUDA check-run from non-{sha_branch} branch run {trunk_wf.get('id')}")
+                        trunk_wf = None
+                        cuda_test_jobs = []
+                    else:
+                        print(f"CUDA test jobs are in trunk run {trunk_wf['id']} (found via check-runs)")
+                        all_cuda_jobs = list(cuda_test_jobs)
 
         if trunk_wf is None:
             trunk_wf = trunk_runs[0] if trunk_runs else None
@@ -873,7 +889,7 @@ def main():
             # find tests in inductor workflow with given sha and success status
             #https://docs.github.com/en/rest/actions/workflow-runs#list-workflow-runs-for-a-repository
             error_msg="Error: inductor workflow not found in scanned workflow runs. Try increasing max_pages."
-            inductor_wf_cuda = download_workflow_run(created=args.created, max_pages=args.max_pages, workflow=CUDAWorkflowNames["inductor"], sha=inductor_sha, ignore_status=args.ignore_status, status=status, error_msg=error_msg)
+            inductor_wf_cuda = download_workflow_run(created=args.created, max_pages=args.max_pages, workflow=CUDAWorkflowNames["inductor"], sha=inductor_sha, branch=sha_branch, ignore_status=args.ignore_status, status=status, error_msg=error_msg)
             print(f"Using workflow '{CUDAWorkflowNames['inductor']}' with id:{inductor_wf_cuda['id']} for CUDA inductor")
 
             inductor_cuda_jobs = get_workflow_jobs(inductor_wf_cuda)
@@ -930,7 +946,7 @@ def main():
             try:
                 baseline_default_wf = download_workflow_run(
                     created=args.created, max_pages=args.max_pages,
-                    workflow=ROCmWorkflowNames["default"], sha=baseline_sha,
+                    workflow=ROCmWorkflowNames["default"], sha=baseline_sha, branch="main",
                     ignore_status=args.ignore_status, status=status,
                     error_msg=f"Baseline default workflow not found for {baseline_sha}",
                 )
@@ -962,7 +978,7 @@ def main():
             try:
                 baseline_dist_wf = download_workflow_run(
                     created=args.created, max_pages=args.max_pages,
-                    workflow=ROCmWorkflowNames["distributed"], sha=baseline_sha,
+                    workflow=ROCmWorkflowNames["distributed"], sha=baseline_sha, branch="main",
                     ignore_status=args.ignore_status, status=status,
                     error_msg=f"Baseline distributed workflow not found for {baseline_sha}",
                 )
@@ -994,7 +1010,7 @@ def main():
             try:
                 baseline_inductor_wf = download_workflow_run(
                     created=args.created, max_pages=args.max_pages,
-                    workflow=ROCmWorkflowNames["inductor"], sha=baseline_sha,
+                    workflow=ROCmWorkflowNames["inductor"], sha=baseline_sha, branch="main",
                     ignore_status=args.ignore_status, status=status,
                     error_msg=f"Baseline inductor workflow not found for {baseline_sha}",
                 )
@@ -1033,7 +1049,7 @@ def main():
         try:
             inductor_periodic_wf = download_workflow_run(
                 created=args.created, max_pages=args.max_pages,
-                workflow="inductor-periodic", sha=sha,
+                workflow="inductor-periodic", sha=sha, branch=sha_branch,
                 ignore_status=args.ignore_status, status=status,
                 error_msg=error_msg,
             )
diff --git a/.automation_scripts/pytorch-unit-test-scripts/summarize_xml_testreports.py b/.automation_scripts/pytorch-unit-test-scripts/summarize_xml_testreports.py
index 97c4424d89b03..e513bd7abedba 100755
--- a/.automation_scripts/pytorch-unit-test-scripts/summarize_xml_testreports.py
+++ b/.automation_scripts/pytorch-unit-test-scripts/summarize_xml_testreports.py
@@ -315,10 +315,15 @@ def summarize_xml_files(args):
     # test file level running time: ROCm and CUDA
     test_file_level_ROCm: Dict[Tuple[str], float] = {}
     test_file_level_CUDA: Dict[Tuple[str], float] = {}
+    test_file_shards_ROCm: Dict[Tuple[str], set] = {}
+    test_file_shards_CUDA: Dict[Tuple[str], set] = {}
     for (k,v) in list(test_cases_set1_running_time.items()):
           test_file_name = k[0]
           test_config_name = k[2]
           tar_tup_rocm = (test_file_name, test_config_name,)
+          test_file_shards_ROCm.setdefault(tar_tup_rocm, set())
+          if v.get("shard"):
+              test_file_shards_ROCm[tar_tup_rocm].add(v["shard"])
           if test_file_level_ROCm.get(tar_tup_rocm) == None:
               test_file_level_ROCm[ ( test_file_name, test_config_name ) ] = v["running_time_xml"]
           else:
@@ -327,6 +332,9 @@ def summarize_xml_files(args):
           test_file_name = k[0]
           test_config_name = k[2]
           tar_tup_cuda = (test_file_name, test_config_name)
+          test_file_shards_CUDA.setdefault(tar_tup_cuda, set())
+          if v.get("shard"):
+              test_file_shards_CUDA[tar_tup_cuda].add(v["shard"])
           if test_file_level_CUDA.get(tar_tup_cuda) == None:
               test_file_level_CUDA[ ( test_file_name, test_config_name ) ] = v["running_time_xml"]
           else:
@@ -612,24 +620,35 @@ def sorting_key(e):
 
     # write test file running time to file
     test_file_running_time_for_csv = {}
+    set1_running_time_col = f"{set1_name}_running_time"
+    set2_running_time_col = f"{set2_name}_running_time"
+    set1_tests_run_col = f"{set1_name}_tests_run"
+    set2_tests_run_col = f"{set2_name}_tests_run"
+    set1_test_shards_col = f"{set1_name}_test_shards"
+    set2_test_shards_col = f"{set2_name}_test_shards"
+    set1_passed_col = f"{set1_name}_passed"
+    set1_skipped_col = f"{set1_name}_skipped"
+    set1_missed_col = f"{set1_name}_missed"
     for key_rocm in test_file_level_ROCm.keys():
         item_values = {}
         item_values["test_file"] = key_rocm[0]
         item_values["test_config"] = key_rocm[1]
-        item_values["rocm_running_time"] = test_file_level_ROCm[key_rocm]
-        item_values["cuda_running_time"] = 0.0
+        item_values[set1_running_time_col] = test_file_level_ROCm[key_rocm]
+        item_values[set2_running_time_col] = 0.0
         if key_rocm in test_file_level_CUDA.keys():
-            item_values["cuda_running_time"] = test_file_level_CUDA[key_rocm]
-        item_values["abs_time_diff"] = item_values["rocm_running_time"] - item_values["cuda_running_time"]
+            item_values[set2_running_time_col] = test_file_level_CUDA[key_rocm]
+        item_values["abs_time_diff"] = item_values[set1_running_time_col] - item_values[set2_running_time_col]
         item_values["relative_time_diff"] = 0.0
-        if item_values["cuda_running_time"] != 0.0:
-            item_values["relative_time_diff"] = 100 * (item_values["rocm_running_time"] - item_values["cuda_running_time"]) / item_values["cuda_running_time"]
+        if item_values[set2_running_time_col] != 0.0:
+            item_values["relative_time_diff"] = 100 * (item_values[set1_running_time_col] - item_values[set2_running_time_col]) / item_values[set2_running_time_col]
         # Add test counts
-        item_values["rocm_tests_run"] = test_file_counts_ROCm.get(key_rocm, {}).get('tests_run', 0)
-        item_values["cuda_tests_run"] = test_file_counts_CUDA.get(key_rocm, 0)
-        item_values["rocm_passed"] = test_file_counts_ROCm.get(key_rocm, {}).get('passed', 0)
-        item_values["rocm_skipped"] = test_file_counts_ROCm.get(key_rocm, {}).get('skipped', 0)
-        item_values["rocm_missed"] = test_file_counts_ROCm.get(key_rocm, {}).get('missed', 0)
+        item_values[set1_tests_run_col] = test_file_counts_ROCm.get(key_rocm, {}).get('tests_run', 0)
+        item_values[set2_tests_run_col] = test_file_counts_CUDA.get(key_rocm, 0)
+        item_values[set1_test_shards_col] = len(test_file_shards_ROCm.get(key_rocm, set()))
+        item_values[set2_test_shards_col] = len(test_file_shards_CUDA.get(key_rocm, set()))
+        item_values[set1_passed_col] = test_file_counts_ROCm.get(key_rocm, {}).get('passed', 0)
+        item_values[set1_skipped_col] = test_file_counts_ROCm.get(key_rocm, {}).get('skipped', 0)
+        item_values[set1_missed_col] = test_file_counts_ROCm.get(key_rocm, {}).get('missed', 0)
         test_file_running_time_for_csv[key_rocm] = item_values
 
     for key_cuda in test_file_level_CUDA.keys():
@@ -637,18 +656,20 @@ def sorting_key(e):
             item_values = {}
             item_values["test_file"] = key_cuda[0]
             item_values["test_config"] = key_cuda[1]
-            item_values["rocm_running_time"] = 0.0
-            item_values["cuda_running_time"] = test_file_level_CUDA[key_cuda]
-            item_values["abs_time_diff"] = item_values["rocm_running_time"] - item_values["cuda_running_time"]
+            item_values[set1_running_time_col] = 0.0
+            item_values[set2_running_time_col] = test_file_level_CUDA[key_cuda]
+            item_values["abs_time_diff"] = item_values[set1_running_time_col] - item_values[set2_running_time_col]
             item_values["relative_time_diff"] = 0.0
-            if item_values["cuda_running_time"] != 0.0:
-                item_values["relative_time_diff"] = 100 * (item_values["rocm_running_time"] - item_values["cuda_running_time"]) / item_values["cuda_running_time"]
+            if item_values[set2_running_time_col] != 0.0:
+                item_values["relative_time_diff"] = 100 * (item_values[set1_running_time_col] - item_values[set2_running_time_col]) / item_values[set2_running_time_col]
             # Add test counts
-            item_values["rocm_tests_run"] = test_file_counts_ROCm.get(key_cuda, {}).get('tests_run', 0)
-            item_values["cuda_tests_run"] = test_file_counts_CUDA.get(key_cuda, 0)
-            item_values["rocm_passed"] = test_file_counts_ROCm.get(key_cuda, {}).get('passed', 0)
-            item_values["rocm_skipped"] = test_file_counts_ROCm.get(key_cuda, {}).get('skipped', 0)
-            item_values["rocm_missed"] = test_file_counts_ROCm.get(key_cuda, {}).get('missed', 0)
+            item_values[set1_tests_run_col] = test_file_counts_ROCm.get(key_cuda, {}).get('tests_run', 0)
+            item_values[set2_tests_run_col] = test_file_counts_CUDA.get(key_cuda, 0)
+            item_values[set1_test_shards_col] = len(test_file_shards_ROCm.get(key_cuda, set()))
+            item_values[set2_test_shards_col] = len(test_file_shards_CUDA.get(key_cuda, set()))
+            item_values[set1_passed_col] = test_file_counts_ROCm.get(key_cuda, {}).get('passed', 0)
+            item_values[set1_skipped_col] = test_file_counts_ROCm.get(key_cuda, {}).get('skipped', 0)
+            item_values[set1_missed_col] = test_file_counts_ROCm.get(key_cuda, {}).get('missed', 0)
             test_file_running_time_for_csv[key_cuda] = item_values
 
     test_file_running_time_for_csv = dict(sorted(test_file_running_time_for_csv.items()))
@@ -658,24 +679,28 @@ def sorting_key_running_time(e):
           return 0
         elif e == "test_config":
           return 1
-        elif e == "rocm_running_time":
+        elif e == set1_running_time_col:
           return 2
-        elif e == "cuda_running_time":
+        elif e == set2_running_time_col:
           return 3
         elif e == "abs_time_diff":
           return 4
         elif e == "relative_time_diff":
           return 5
-        elif e == "rocm_tests_run":
+        elif e == set1_tests_run_col:
           return 6
-        elif e == "cuda_tests_run":
+        elif e == set2_tests_run_col:
           return 7
-        elif e == "rocm_passed":
+        elif e == set1_test_shards_col:
           return 8
-        elif e == "rocm_skipped":
+        elif e == set2_test_shards_col:
           return 9
-        elif e == "rocm_missed":
+        elif e == set1_passed_col:
           return 10
+        elif e == set1_skipped_col:
+          return 11
+        elif e == set1_missed_col:
+          return 12
         else:
           return 100
 

From 099dad51991da70e49946dc48174ac74282e2447 Mon Sep 17 00:00:00 2001
From: ethanwee1 <ethanwee@amd.com>
Date: Thu, 18 Jun 2026 14:21:34 +0000
Subject: [PATCH 2/6] parity summary: exclude inductor.test_cpu_repro from
 comparison

test_cpu_repro is blocklisted on ROCm upstream, so it never runs there.
The parity summary was therefore counting it as MISSED against the CUDA
baseline, which is misleading. Add it to EXCLUDED_TEST_SUITES so it is
dropped from the ROCm-vs-CUDA comparison.
---
 .../pytorch-unit-test-scripts/summarize_xml_testreports.py   | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/.automation_scripts/pytorch-unit-test-scripts/summarize_xml_testreports.py b/.automation_scripts/pytorch-unit-test-scripts/summarize_xml_testreports.py
index e513bd7abedba..f8db93ad67916 100755
--- a/.automation_scripts/pytorch-unit-test-scripts/summarize_xml_testreports.py
+++ b/.automation_scripts/pytorch-unit-test-scripts/summarize_xml_testreports.py
@@ -31,7 +31,10 @@
     "_nvfuser.test_torchscript",
     "test_jit_cuda_fuser",
     "test_nvfuser_dynamo",
-    "test_nvfuser_frontend"
+    "test_nvfuser_frontend",
+    # Blocklisted on ROCm upstream, so it never runs there and would otherwise
+    # show up as falsely MISSED against the CUDA baseline.
+    "inductor.test_cpu_repro"
 ]
 
 

From 441234590796695bef31f88d380a144950a49489 Mon Sep 17 00:00:00 2001
From: ethanwee1 <ethanwee@amd.com>
Date: Fri, 19 Jun 2026 21:18:41 +0000
Subject: [PATCH 3/6] parity summary: surface XML failure messages

Add the per-test error message (parsed from the test report XML, already
stored in the message_<set> CSV column) to the parity summary:

- CSV FAILED TESTS table gets a new 'Error Message' column so downstream
  dashboard parsing has it in structured form.
- Markdown summary gets a top-level collapsible 'Failure messages'
  <details> with one nested <details> per failed test (full traceback in
  an HTML-escaped <pre>), so the message is available without bloating
  the table.
---
 .../generate_summary.py                       | 53 +++++++++++++++++++
 1 file changed, 53 insertions(+)

diff --git a/.automation_scripts/pytorch-unit-test-scripts/generate_summary.py b/.automation_scripts/pytorch-unit-test-scripts/generate_summary.py
index b01d0ecf6c8de..8e1302e963217 100644
--- a/.automation_scripts/pytorch-unit-test-scripts/generate_summary.py
+++ b/.automation_scripts/pytorch-unit-test-scripts/generate_summary.py
@@ -1,6 +1,7 @@
 #!/usr/bin/env python3
 
 import argparse
+import ast
 import csv
 import os
 import re
@@ -266,6 +267,32 @@ def safe_float(v):
     return vals
 
 
+def _extract_message(raw):
+    """Turn a stored XML failure/error value into readable text.
+
+    summarize_xml_testreports.py writes the parsed XML failure node (a dict like
+    {'message': 'AssertionError: ...', 'text': '<full traceback>'}) into the
+    message_<set> CSV column via str(), so cells arrive as a dict repr. Prefer
+    the full traceback ('text'), fall back to the short 'message', and return
+    the raw string unchanged if it is not a dict repr.
+    """
+    if not raw:
+        return ''
+    raw = raw.strip()
+    if raw.startswith('{') and raw.endswith('}'):
+        try:
+            d = ast.literal_eval(raw)
+            if isinstance(d, dict):
+                return (d.get('text') or d.get('message') or '').strip()
+        except (ValueError, SyntaxError):
+            pass
+    return raw
+
+
+def _html_escape(s):
+    return (s or '').replace('&', '&amp;').replace('<', '&lt;').replace('>', '&gt;')
+
+
 def collect_failed_tests(arch_data, archs, s1_name, s2_name):
     """Return a list of failed test rows across all architectures.
 
@@ -278,6 +305,10 @@ def collect_failed_tests(arch_data, archs, s1_name, s2_name):
     for arch in archs:
         d = arch_data[arch]
         s1_col, s2_col, s1_time, _ = d['cols']
+        # message_<name> columns mirror the status_<name> naming (incl. the
+        # status_set1/set2 fallback), so derive them from the resolved cols.
+        s1_msg_col = s1_col.replace('status_', 'message_', 1)
+        s2_msg_col = s2_col.replace('status_', 'message_', 1)
         has_set2 = d.get('has_set2', True)
         for r in d['rows']:
             s1 = r[s1_col].strip()
@@ -293,11 +324,13 @@ def collect_failed_tests(arch_data, archs, s1_name, s2_name):
                     f'shard_{s1_name}': r.get(f'shard_{s1_name}', ''),
                     f'job_url_{s1_name}': r.get(f'job_url_{s1_name}', ''),
                     f'status_{s1_name}': s1,
+                    'error_message': _extract_message(r.get(s1_msg_col, '')),
                 }
                 if has_set2:
                     entry[f'shard_{s2_name}'] = r.get(f'shard_{s2_name}', '')
                     entry[f'job_url_{s2_name}'] = r.get(f'job_url_{s2_name}', '')
                     entry[f'status_{s2_name}'] = s2
+                    entry['error_message_set2'] = _extract_message(r.get(s2_msg_col, ''))
                 failed.append(entry)
 
     return failed
@@ -609,6 +642,7 @@ def _xml_test_shard(t, platform):
         if has_set2:
             header.append(f'Status ({s2_name})')
         header.append('Also Failing In')
+        header.append('Error Message')
         csv_rows.append(header)
         for t in s1_failed:
             row = [t['arch'], t['test_config'], t['test_file'],
@@ -623,6 +657,7 @@ def _xml_test_shard(t, platform):
             if has_set2:
                 row.append(t.get(f'status_{s2_name}', ''))
             row.append(t.get('also_failing_in', ''))
+            row.append(t.get('error_message', ''))
             csv_rows.append(row)
         csv_rows.append([])
 
@@ -758,6 +793,24 @@ def _job_id_link(url):
             line += ' |'
             lines.append(line)
         lines.append('')
+
+        with_messages = [t for t in s1_failed if t.get('error_message')]
+        if with_messages:
+            lines.append('<details>')
+            lines.append(f'<summary>Failure messages ({len(with_messages)})</summary>')
+            lines.append('')
+            for t in with_messages:
+                ident = (f"{t['arch']} \u00b7 {t['test_config']} \u00b7 "
+                         f"{t['test_file']}::{t['test_class']}::{t['test_name']}")
+                lines.append('<details>')
+                lines.append(f'<summary>{_html_escape(ident)}</summary>')
+                lines.append('')
+                lines.append(f'<pre>{_html_escape(t["error_message"])}</pre>')
+                lines.append('')
+                lines.append('</details>')
+                lines.append('')
+            lines.append('</details>')
+            lines.append('')
     else:
         lines.append('### FAILED TESTS')
         lines.append('')

From aef8a84a920acfd27cb113b7461d63cbd9704d83 Mon Sep 17 00:00:00 2001
From: ethanwee1 <ethanwee@amd.com>
Date: Mon, 22 Jun 2026 13:15:38 +0000
Subject: [PATCH 4/6] parity summary: cap failure-message size in markdown

The full .md is piped into $GITHUB_STEP_SUMMARY (1 MB cap); many full
tracebacks can push it over the limit and GitHub drops the whole summary.
Truncate each message to its tail (where the exception is) in the md;
the CSV 'Error Message' column keeps the full text for the dashboard.
---
 .../generate_summary.py                          | 16 +++++++++++++++-
 1 file changed, 15 insertions(+), 1 deletion(-)

diff --git a/.automation_scripts/pytorch-unit-test-scripts/generate_summary.py b/.automation_scripts/pytorch-unit-test-scripts/generate_summary.py
index 8e1302e963217..b2464f9b66afc 100644
--- a/.automation_scripts/pytorch-unit-test-scripts/generate_summary.py
+++ b/.automation_scripts/pytorch-unit-test-scripts/generate_summary.py
@@ -293,6 +293,20 @@ def _html_escape(s):
     return (s or '').replace('&', '&amp;').replace('<', '&lt;').replace('>', '&gt;')
 
 
+def _truncate_message(msg, limit=2000):
+    """Cap a failure message for the markdown summary.
+
+    The whole .md is piped into $GITHUB_STEP_SUMMARY, which GitHub rejects above
+    1 MB, so many large tracebacks would drop the entire summary. Keep the tail
+    (the exception/assertion lives at the end of a traceback) and point readers
+    to the CSV 'Error Message' column, which keeps the full text for dashboards.
+    """
+    if not msg or len(msg) <= limit:
+        return msg
+    return ('...(truncated; full message in the Error Message column of the CSV artifact)\n'
+            + msg[-limit:])
+
+
 def collect_failed_tests(arch_data, archs, s1_name, s2_name):
     """Return a list of failed test rows across all architectures.
 
@@ -805,7 +819,7 @@ def _job_id_link(url):
                 lines.append('<details>')
                 lines.append(f'<summary>{_html_escape(ident)}</summary>')
                 lines.append('')
-                lines.append(f'<pre>{_html_escape(t["error_message"])}</pre>')
+                lines.append(f'<pre>{_html_escape(_truncate_message(t["error_message"]))}</pre>')
                 lines.append('')
                 lines.append('</details>')
                 lines.append('')

From 6ed3a9d5b95bdbefd3971ae669a6be218b23bde8 Mon Sep 17 00:00:00 2001
From: ethanwee1 <ethanwee@amd.com>
Date: Mon, 22 Jun 2026 13:43:56 +0000
Subject: [PATCH 5/6] parity summary: move failure message into a per-row
 collapsible column

Replace the separate bottom 'Failure messages' section with an
'Error Message' column in the FAILED TESTS markdown table. Each cell is
a <details><summary>view</summary> toggle wrapping a <pre> (newlines as
&#10;, pipes escaped) so the table stays scannable and expands per row.
The CSV 'Error Message' column (full text) is unchanged.
---
 .../generate_summary.py                       | 39 ++++++++++---------
 1 file changed, 20 insertions(+), 19 deletions(-)

diff --git a/.automation_scripts/pytorch-unit-test-scripts/generate_summary.py b/.automation_scripts/pytorch-unit-test-scripts/generate_summary.py
index b2464f9b66afc..f1ba081eeb42d 100644
--- a/.automation_scripts/pytorch-unit-test-scripts/generate_summary.py
+++ b/.automation_scripts/pytorch-unit-test-scripts/generate_summary.py
@@ -307,6 +307,24 @@ def _truncate_message(msg, limit=2000):
             + msg[-limit:])
 
 
+def _message_cell(msg):
+    """Render a failure message as a collapsible cell for a markdown table.
+
+    Table cells can't contain raw newlines or unescaped pipes, so the message
+    is HTML-escaped, pipes are escaped, and newlines become &#10; inside a
+    <pre> (GitHub renders these as line breaks). The whole thing is wrapped in
+    a <details> so each row shows only a small 'view' toggle by default.
+    """
+    msg = _truncate_message(msg)
+    if not msg:
+        return ''
+    body = (_html_escape(msg)
+            .replace('\r', '')
+            .replace('\n', '&#10;')
+            .replace('|', '\\|'))
+    return f'<details><summary>view</summary><pre>{body}</pre></details>'
+
+
 def collect_failed_tests(arch_data, archs, s1_name, s2_name):
     """Return a list of failed test rows across all architectures.
 
@@ -782,6 +800,7 @@ def _job_id_link(url):
     cols.append(f'Job ID ({s1_name})')
     if has_set2:
         cols.append(f'Job ID ({s2_name})')
+    cols.append('Error Message')
 
     if s1_failed:
         lines.append(f'### FAILED TESTS ({len(s1_failed)})')
@@ -804,27 +823,9 @@ def _job_id_link(url):
             line += f" | {_job_id_link(t.get(f'job_url_{s1_name}', ''))}"
             if has_set2:
                 line += f" | {_job_id_link(t.get(f'job_url_{s2_name}', ''))}"
-            line += ' |'
+            line += f" | {_message_cell(t.get('error_message', ''))} |"
             lines.append(line)
         lines.append('')
-
-        with_messages = [t for t in s1_failed if t.get('error_message')]
-        if with_messages:
-            lines.append('<details>')
-            lines.append(f'<summary>Failure messages ({len(with_messages)})</summary>')
-            lines.append('')
-            for t in with_messages:
-                ident = (f"{t['arch']} \u00b7 {t['test_config']} \u00b7 "
-                         f"{t['test_file']}::{t['test_class']}::{t['test_name']}")
-                lines.append('<details>')
-                lines.append(f'<summary>{_html_escape(ident)}</summary>')
-                lines.append('')
-                lines.append(f'<pre>{_html_escape(_truncate_message(t["error_message"]))}</pre>')
-                lines.append('')
-                lines.append('</details>')
-                lines.append('')
-            lines.append('</details>')
-            lines.append('')
     else:
         lines.append('### FAILED TESTS')
         lines.append('')

From 9255fdf99f4026aa9ffd5bc0b8588c33c70d9491 Mon Sep 17 00:00:00 2001
From: ethanwee1 <ethanwee@amd.com>
Date: Mon, 22 Jun 2026 20:53:44 +0000
Subject: [PATCH 6/6] parity summary: budget failure messages against the 1 MiB
 step-summary cap

The per-row 'Error Message' column hard-truncated each message to 2000 chars so
the markdown couldn't exceed GitHub's per-step $GITHUB_STEP_SUMMARY limit
(1 MiB), which clipped useful tracebacks even on small runs with room to spare.

Raise the per-message cap to 50k and size messages against a global 950 KB
budget (1 MiB minus headroom for the surrounding tables): the rest of the
summary is measured first, then the largest uniform per-message cap that fits
the remaining budget is applied. Typical runs render full tracebacks; only when
a run would exceed the cap do the longest (outlier) messages get clipped. Full
text always remains in the CSV 'Error Message' column.

https://docs.github.com/en/actions/reference/workflows-and-actions/workflow-commands#step-isolation-and-limits
---
 .../generate_summary.py                       | 84 +++++++++++++++++--
 1 file changed, 75 insertions(+), 9 deletions(-)

diff --git a/.automation_scripts/pytorch-unit-test-scripts/generate_summary.py b/.automation_scripts/pytorch-unit-test-scripts/generate_summary.py
index f1ba081eeb42d..0eb4d010e5b0e 100644
--- a/.automation_scripts/pytorch-unit-test-scripts/generate_summary.py
+++ b/.automation_scripts/pytorch-unit-test-scripts/generate_summary.py
@@ -293,13 +293,28 @@ def _html_escape(s):
     return (s or '').replace('&', '&amp;').replace('<', '&lt;').replace('>', '&gt;')
 
 
-def _truncate_message(msg, limit=2000):
+# The whole .md is piped into $GITHUB_STEP_SUMMARY, and GitHub restricts each
+# step summary to 1 MiB - past that the upload fails (and content can be dropped
+# silently as it nears the limit), so a single run with many large tracebacks
+# would lose the entire summary. We keep the rendered summary under this budget
+# (1 MiB minus headroom for the surrounding tables/markdown) and only clip the
+# longest failure messages when a run would otherwise exceed it; the full text
+# always stays in the 'Error Message' column of the CSV artifact.
+# https://docs.github.com/en/actions/reference/workflows-and-actions/workflow-commands#step-isolation-and-limits
+STEP_SUMMARY_BYTE_BUDGET = 950_000
+
+# Per-message hard cap. Generous on purpose: real tracebacks fit well within it,
+# so they render in full; it only bounds pathological multi-MB messages. The
+# global budget above is what protects the summary when there are many failures.
+PER_MESSAGE_CHAR_LIMIT = 50_000
+
+
+def _truncate_message(msg, limit=PER_MESSAGE_CHAR_LIMIT):
     """Cap a failure message for the markdown summary.
 
-    The whole .md is piped into $GITHUB_STEP_SUMMARY, which GitHub rejects above
-    1 MB, so many large tracebacks would drop the entire summary. Keep the tail
-    (the exception/assertion lives at the end of a traceback) and point readers
-    to the CSV 'Error Message' column, which keeps the full text for dashboards.
+    Keeps the tail (the exception/assertion lives at the end of a traceback) and
+    points readers to the CSV 'Error Message' column, which keeps the full text
+    for dashboards.
     """
     if not msg or len(msg) <= limit:
         return msg
@@ -307,17 +322,21 @@ def _truncate_message(msg, limit=2000):
             + msg[-limit:])
 
 
-def _message_cell(msg):
+def _message_cell(msg, char_limit=PER_MESSAGE_CHAR_LIMIT):
     """Render a failure message as a collapsible cell for a markdown table.
 
     Table cells can't contain raw newlines or unescaped pipes, so the message
     is HTML-escaped, pipes are escaped, and newlines become &#10; inside a
     <pre> (GitHub renders these as line breaks). The whole thing is wrapped in
     a <details> so each row shows only a small 'view' toggle by default.
+    `char_limit` is set per-run by the budget pass below; 0 drops the body.
     """
-    msg = _truncate_message(msg)
     if not msg:
         return ''
+    if char_limit <= 0:
+        return ('<sub>message omitted to keep the summary under GitHub\u2019s 1 MiB '
+                'limit; see the Error Message column of the CSV artifact</sub>')
+    msg = _truncate_message(msg, char_limit)
     body = (_html_escape(msg)
             .replace('\r', '')
             .replace('\n', '&#10;')
@@ -325,6 +344,48 @@ def _message_cell(msg):
     return f'<details><summary>view</summary><pre>{body}</pre></details>'
 
 
+def _fit_message_cap(messages, budget):
+    """Largest uniform per-message char cap whose rendered cells fit `budget`
+    bytes. Messages shorter than the cap are untouched, so only the longest
+    (outlier) messages get clipped, and only when a run would exceed the budget.
+    """
+    if budget <= 0:
+        return 0
+
+    def total_bytes(cap):
+        return sum(len(_message_cell(m, cap).encode('utf-8')) for m in messages)
+
+    if total_bytes(PER_MESSAGE_CHAR_LIMIT) <= budget:
+        return PER_MESSAGE_CHAR_LIMIT
+    lo, hi = 0, PER_MESSAGE_CHAR_LIMIT
+    while lo < hi:
+        mid = (lo + hi + 1) // 2
+        if total_bytes(mid) <= budget:
+            lo = mid
+        else:
+            hi = mid - 1
+    return lo
+
+
+def _fill_failure_messages(lines, fail_rows):
+    """Fill the deferred 'Error Message' cells of the FAILED TESTS table.
+
+    `fail_rows` is a list of (line_index, row_prefix, raw_message). We first
+    measure every other byte in the summary (rendering these cells empty), then
+    spend whatever remains of STEP_SUMMARY_BYTE_BUDGET on the messages via a
+    single uniform cap, so typical runs show full tracebacks and only oversized
+    runs clip their longest messages.
+    """
+    if not fail_rows:
+        return
+    for idx, prefix, _ in fail_rows:
+        lines[idx] = f"{prefix} |  |"
+    base_bytes = len('\n'.join('' if l is None else l for l in lines).encode('utf-8'))
+    cap = _fit_message_cap([m for _, _, m in fail_rows], STEP_SUMMARY_BYTE_BUDGET - base_bytes)
+    for idx, prefix, msg in fail_rows:
+        lines[idx] = f"{prefix} | {_message_cell(msg, cap)} |"
+
+
 def collect_failed_tests(arch_data, archs, s1_name, s2_name):
     """Return a list of failed test rows across all architectures.
 
@@ -802,6 +863,9 @@ def _job_id_link(url):
         cols.append(f'Job ID ({s2_name})')
     cols.append('Error Message')
 
+    # Filled in by _fill_failure_messages once the rest of the summary is sized,
+    # so the message column can use whatever byte budget is left.
+    fail_rows = []
     if s1_failed:
         lines.append(f'### FAILED TESTS ({len(s1_failed)})')
         lines.append('')
@@ -823,8 +887,8 @@ def _job_id_link(url):
             line += f" | {_job_id_link(t.get(f'job_url_{s1_name}', ''))}"
             if has_set2:
                 line += f" | {_job_id_link(t.get(f'job_url_{s2_name}', ''))}"
-            line += f" | {_message_cell(t.get('error_message', ''))} |"
-            lines.append(line)
+            lines.append(None)  # placeholder; message cell filled in below
+            fail_rows.append((len(lines) - 1, line, t.get('error_message', '')))
         lines.append('')
     else:
         lines.append('### FAILED TESTS')
@@ -870,6 +934,8 @@ def _job_id_link(url):
                 )
             lines.append('')
 
+    _fill_failure_messages(lines, fail_rows)
+
     md = '\n'.join(lines)
     with open(output_path, 'w') as f:
         f.write(md)