From 5252d038e16c9cc2bc0df531f50128b0459d403d Mon Sep 17 00:00:00 2001 From: ethanwee1 Date: Fri, 19 Jun 2026 21:18:41 +0000 Subject: [PATCH 1/4] parity summary: surface XML failure messages Add the per-test error message (parsed from the test report XML, already stored in the message_ CSV column) to the parity summary: - CSV FAILED TESTS table gets a new 'Error Message' column so downstream dashboard parsing has it in structured form. - Markdown summary gets a top-level collapsible 'Failure messages'
with one nested
per failed test (full traceback in an HTML-escaped 
), so the message is available without bloating
  the table.
---
 .../generate_summary.py                       | 53 +++++++++++++++++++
 1 file changed, 53 insertions(+)

diff --git a/.automation_scripts/pytorch-unit-test-scripts/generate_summary.py b/.automation_scripts/pytorch-unit-test-scripts/generate_summary.py
index b01d0ecf6c8de..8e1302e963217 100644
--- a/.automation_scripts/pytorch-unit-test-scripts/generate_summary.py
+++ b/.automation_scripts/pytorch-unit-test-scripts/generate_summary.py
@@ -1,6 +1,7 @@
 #!/usr/bin/env python3
 
 import argparse
+import ast
 import csv
 import os
 import re
@@ -266,6 +267,32 @@ def safe_float(v):
     return vals
 
 
+def _extract_message(raw):
+    """Turn a stored XML failure/error value into readable text.
+
+    summarize_xml_testreports.py writes the parsed XML failure node (a dict like
+    {'message': 'AssertionError: ...', 'text': ''}) into the
+    message_ CSV column via str(), so cells arrive as a dict repr. Prefer
+    the full traceback ('text'), fall back to the short 'message', and return
+    the raw string unchanged if it is not a dict repr.
+    """
+    if not raw:
+        return ''
+    raw = raw.strip()
+    if raw.startswith('{') and raw.endswith('}'):
+        try:
+            d = ast.literal_eval(raw)
+            if isinstance(d, dict):
+                return (d.get('text') or d.get('message') or '').strip()
+        except (ValueError, SyntaxError):
+            pass
+    return raw
+
+
+def _html_escape(s):
+    return (s or '').replace('&', '&').replace('<', '<').replace('>', '>')
+
+
 def collect_failed_tests(arch_data, archs, s1_name, s2_name):
     """Return a list of failed test rows across all architectures.
 
@@ -278,6 +305,10 @@ def collect_failed_tests(arch_data, archs, s1_name, s2_name):
     for arch in archs:
         d = arch_data[arch]
         s1_col, s2_col, s1_time, _ = d['cols']
+        # message_ columns mirror the status_ naming (incl. the
+        # status_set1/set2 fallback), so derive them from the resolved cols.
+        s1_msg_col = s1_col.replace('status_', 'message_', 1)
+        s2_msg_col = s2_col.replace('status_', 'message_', 1)
         has_set2 = d.get('has_set2', True)
         for r in d['rows']:
             s1 = r[s1_col].strip()
@@ -293,11 +324,13 @@ def collect_failed_tests(arch_data, archs, s1_name, s2_name):
                     f'shard_{s1_name}': r.get(f'shard_{s1_name}', ''),
                     f'job_url_{s1_name}': r.get(f'job_url_{s1_name}', ''),
                     f'status_{s1_name}': s1,
+                    'error_message': _extract_message(r.get(s1_msg_col, '')),
                 }
                 if has_set2:
                     entry[f'shard_{s2_name}'] = r.get(f'shard_{s2_name}', '')
                     entry[f'job_url_{s2_name}'] = r.get(f'job_url_{s2_name}', '')
                     entry[f'status_{s2_name}'] = s2
+                    entry['error_message_set2'] = _extract_message(r.get(s2_msg_col, ''))
                 failed.append(entry)
 
     return failed
@@ -609,6 +642,7 @@ def _xml_test_shard(t, platform):
         if has_set2:
             header.append(f'Status ({s2_name})')
         header.append('Also Failing In')
+        header.append('Error Message')
         csv_rows.append(header)
         for t in s1_failed:
             row = [t['arch'], t['test_config'], t['test_file'],
@@ -623,6 +657,7 @@ def _xml_test_shard(t, platform):
             if has_set2:
                 row.append(t.get(f'status_{s2_name}', ''))
             row.append(t.get('also_failing_in', ''))
+            row.append(t.get('error_message', ''))
             csv_rows.append(row)
         csv_rows.append([])
 
@@ -758,6 +793,24 @@ def _job_id_link(url):
             line += ' |'
             lines.append(line)
         lines.append('')
+
+        with_messages = [t for t in s1_failed if t.get('error_message')]
+        if with_messages:
+            lines.append('
')
+            lines.append(f'Failure messages ({len(with_messages)})')
+            lines.append('')
+            for t in with_messages:
+                ident = (f"{t['arch']} \u00b7 {t['test_config']} \u00b7 "
+                         f"{t['test_file']}::{t['test_class']}::{t['test_name']}")
+                lines.append('
')
+                lines.append(f'{_html_escape(ident)}')
+                lines.append('')
+                lines.append(f'
{_html_escape(t["error_message"])}
')
+                lines.append('')
+                lines.append('
')
+                lines.append('')
+            lines.append('
')
+            lines.append('')
     else:
         lines.append('### FAILED TESTS')
         lines.append('')

From a0db185256589532ff3e74ac532dd40866347ca9 Mon Sep 17 00:00:00 2001
From: ethanwee1 
Date: Mon, 22 Jun 2026 13:15:38 +0000
Subject: [PATCH 2/4] parity summary: cap failure-message size in markdown

The full .md is piped into $GITHUB_STEP_SUMMARY (1 MB cap); many full
tracebacks can push it over the limit and GitHub drops the whole summary.
Truncate each message to its tail (where the exception is) in the md;
the CSV 'Error Message' column keeps the full text for the dashboard.
---
 .../generate_summary.py                          | 16 +++++++++++++++-
 1 file changed, 15 insertions(+), 1 deletion(-)

diff --git a/.automation_scripts/pytorch-unit-test-scripts/generate_summary.py b/.automation_scripts/pytorch-unit-test-scripts/generate_summary.py
index 8e1302e963217..b2464f9b66afc 100644
--- a/.automation_scripts/pytorch-unit-test-scripts/generate_summary.py
+++ b/.automation_scripts/pytorch-unit-test-scripts/generate_summary.py
@@ -293,6 +293,20 @@ def _html_escape(s):
     return (s or '').replace('&', '&').replace('<', '<').replace('>', '>')
 
 
+def _truncate_message(msg, limit=2000):
+    """Cap a failure message for the markdown summary.
+
+    The whole .md is piped into $GITHUB_STEP_SUMMARY, which GitHub rejects above
+    1 MB, so many large tracebacks would drop the entire summary. Keep the tail
+    (the exception/assertion lives at the end of a traceback) and point readers
+    to the CSV 'Error Message' column, which keeps the full text for dashboards.
+    """
+    if not msg or len(msg) <= limit:
+        return msg
+    return ('...(truncated; full message in the Error Message column of the CSV artifact)\n'
+            + msg[-limit:])
+
+
 def collect_failed_tests(arch_data, archs, s1_name, s2_name):
     """Return a list of failed test rows across all architectures.
 
@@ -805,7 +819,7 @@ def _job_id_link(url):
                 lines.append('
')
                 lines.append(f'{_html_escape(ident)}')
                 lines.append('')
-                lines.append(f'
{_html_escape(t["error_message"])}
')
+                lines.append(f'
{_html_escape(_truncate_message(t["error_message"]))}
')
                 lines.append('')
                 lines.append('
')
                 lines.append('')

From ff99551e62a87de74713af74289e8fc209f76867 Mon Sep 17 00:00:00 2001
From: ethanwee1 
Date: Mon, 22 Jun 2026 13:43:56 +0000
Subject: [PATCH 3/4] parity summary: move failure message into a per-row
 collapsible column

Replace the separate bottom 'Failure messages' section with an
'Error Message' column in the FAILED TESTS markdown table. Each cell is
a 
view toggle wrapping a 
 (newlines as

, pipes escaped) so the table stays scannable and expands per row.
The CSV 'Error Message' column (full text) is unchanged.
---
 .../generate_summary.py                       | 39 ++++++++++---------
 1 file changed, 20 insertions(+), 19 deletions(-)

diff --git a/.automation_scripts/pytorch-unit-test-scripts/generate_summary.py b/.automation_scripts/pytorch-unit-test-scripts/generate_summary.py
index b2464f9b66afc..f1ba081eeb42d 100644
--- a/.automation_scripts/pytorch-unit-test-scripts/generate_summary.py
+++ b/.automation_scripts/pytorch-unit-test-scripts/generate_summary.py
@@ -307,6 +307,24 @@ def _truncate_message(msg, limit=2000):
             + msg[-limit:])
 
 
+def _message_cell(msg):
+    """Render a failure message as a collapsible cell for a markdown table.
+
+    Table cells can't contain raw newlines or unescaped pipes, so the message
+    is HTML-escaped, pipes are escaped, and newlines become 
 inside a
+    
 (GitHub renders these as line breaks). The whole thing is wrapped in
+    a 
 so each row shows only a small 'view' toggle by default.
+    """
+    msg = _truncate_message(msg)
+    if not msg:
+        return ''
+    body = (_html_escape(msg)
+            .replace('\r', '')
+            .replace('\n', '
')
+            .replace('|', '\\|'))
+    return f'
view
{body}
'
+
+
 def collect_failed_tests(arch_data, archs, s1_name, s2_name):
     """Return a list of failed test rows across all architectures.
 
@@ -782,6 +800,7 @@ def _job_id_link(url):
     cols.append(f'Job ID ({s1_name})')
     if has_set2:
         cols.append(f'Job ID ({s2_name})')
+    cols.append('Error Message')
 
     if s1_failed:
         lines.append(f'### FAILED TESTS ({len(s1_failed)})')
@@ -804,27 +823,9 @@ def _job_id_link(url):
             line += f" | {_job_id_link(t.get(f'job_url_{s1_name}', ''))}"
             if has_set2:
                 line += f" | {_job_id_link(t.get(f'job_url_{s2_name}', ''))}"
-            line += ' |'
+            line += f" | {_message_cell(t.get('error_message', ''))} |"
             lines.append(line)
         lines.append('')
-
-        with_messages = [t for t in s1_failed if t.get('error_message')]
-        if with_messages:
-            lines.append('
')
-            lines.append(f'Failure messages ({len(with_messages)})')
-            lines.append('')
-            for t in with_messages:
-                ident = (f"{t['arch']} \u00b7 {t['test_config']} \u00b7 "
-                         f"{t['test_file']}::{t['test_class']}::{t['test_name']}")
-                lines.append('
')
-                lines.append(f'{_html_escape(ident)}')
-                lines.append('')
-                lines.append(f'
{_html_escape(_truncate_message(t["error_message"]))}
')
-                lines.append('')
-                lines.append('
')
-                lines.append('')
-            lines.append('
')
-            lines.append('')
     else:
         lines.append('### FAILED TESTS')
         lines.append('')

From 9401da6c7e37cc3b9308db24b906c1fba2166cb6 Mon Sep 17 00:00:00 2001
From: ethanwee1 
Date: Mon, 22 Jun 2026 20:53:44 +0000
Subject: [PATCH 4/4] parity summary: budget failure messages against the 1 MiB
 step-summary cap

The per-row 'Error Message' column hard-truncated each message to 2000 chars so
the markdown couldn't exceed GitHub's per-step $GITHUB_STEP_SUMMARY limit
(1 MiB), which clipped useful tracebacks even on small runs with room to spare.

Raise the per-message cap to 50k and size messages against a global 950 KB
budget (1 MiB minus headroom for the surrounding tables): the rest of the
summary is measured first, then the largest uniform per-message cap that fits
the remaining budget is applied. Typical runs render full tracebacks; only when
a run would exceed the cap do the longest (outlier) messages get clipped. Full
text always remains in the CSV 'Error Message' column.

https://docs.github.com/en/actions/reference/workflows-and-actions/workflow-commands#step-isolation-and-limits
---
 .../generate_summary.py                       | 84 +++++++++++++++++--
 1 file changed, 75 insertions(+), 9 deletions(-)

diff --git a/.automation_scripts/pytorch-unit-test-scripts/generate_summary.py b/.automation_scripts/pytorch-unit-test-scripts/generate_summary.py
index f1ba081eeb42d..0eb4d010e5b0e 100644
--- a/.automation_scripts/pytorch-unit-test-scripts/generate_summary.py
+++ b/.automation_scripts/pytorch-unit-test-scripts/generate_summary.py
@@ -293,13 +293,28 @@ def _html_escape(s):
     return (s or '').replace('&', '&').replace('<', '<').replace('>', '>')
 
 
-def _truncate_message(msg, limit=2000):
+# The whole .md is piped into $GITHUB_STEP_SUMMARY, and GitHub restricts each
+# step summary to 1 MiB - past that the upload fails (and content can be dropped
+# silently as it nears the limit), so a single run with many large tracebacks
+# would lose the entire summary. We keep the rendered summary under this budget
+# (1 MiB minus headroom for the surrounding tables/markdown) and only clip the
+# longest failure messages when a run would otherwise exceed it; the full text
+# always stays in the 'Error Message' column of the CSV artifact.
+# https://docs.github.com/en/actions/reference/workflows-and-actions/workflow-commands#step-isolation-and-limits
+STEP_SUMMARY_BYTE_BUDGET = 950_000
+
+# Per-message hard cap. Generous on purpose: real tracebacks fit well within it,
+# so they render in full; it only bounds pathological multi-MB messages. The
+# global budget above is what protects the summary when there are many failures.
+PER_MESSAGE_CHAR_LIMIT = 50_000
+
+
+def _truncate_message(msg, limit=PER_MESSAGE_CHAR_LIMIT):
     """Cap a failure message for the markdown summary.
 
-    The whole .md is piped into $GITHUB_STEP_SUMMARY, which GitHub rejects above
-    1 MB, so many large tracebacks would drop the entire summary. Keep the tail
-    (the exception/assertion lives at the end of a traceback) and point readers
-    to the CSV 'Error Message' column, which keeps the full text for dashboards.
+    Keeps the tail (the exception/assertion lives at the end of a traceback) and
+    points readers to the CSV 'Error Message' column, which keeps the full text
+    for dashboards.
     """
     if not msg or len(msg) <= limit:
         return msg
@@ -307,17 +322,21 @@ def _truncate_message(msg, limit=2000):
             + msg[-limit:])
 
 
-def _message_cell(msg):
+def _message_cell(msg, char_limit=PER_MESSAGE_CHAR_LIMIT):
     """Render a failure message as a collapsible cell for a markdown table.
 
     Table cells can't contain raw newlines or unescaped pipes, so the message
     is HTML-escaped, pipes are escaped, and newlines become 
 inside a
     
 (GitHub renders these as line breaks). The whole thing is wrapped in
     a 
 so each row shows only a small 'view' toggle by default.
+    `char_limit` is set per-run by the budget pass below; 0 drops the body.
     """
-    msg = _truncate_message(msg)
     if not msg:
         return ''
+    if char_limit <= 0:
+        return ('message omitted to keep the summary under GitHub\u2019s 1 MiB '
+                'limit; see the Error Message column of the CSV artifact')
+    msg = _truncate_message(msg, char_limit)
     body = (_html_escape(msg)
             .replace('\r', '')
             .replace('\n', '
')
@@ -325,6 +344,48 @@ def _message_cell(msg):
     return f'
view
{body}
'
 
 
+def _fit_message_cap(messages, budget):
+    """Largest uniform per-message char cap whose rendered cells fit `budget`
+    bytes. Messages shorter than the cap are untouched, so only the longest
+    (outlier) messages get clipped, and only when a run would exceed the budget.
+    """
+    if budget <= 0:
+        return 0
+
+    def total_bytes(cap):
+        return sum(len(_message_cell(m, cap).encode('utf-8')) for m in messages)
+
+    if total_bytes(PER_MESSAGE_CHAR_LIMIT) <= budget:
+        return PER_MESSAGE_CHAR_LIMIT
+    lo, hi = 0, PER_MESSAGE_CHAR_LIMIT
+    while lo < hi:
+        mid = (lo + hi + 1) // 2
+        if total_bytes(mid) <= budget:
+            lo = mid
+        else:
+            hi = mid - 1
+    return lo
+
+
+def _fill_failure_messages(lines, fail_rows):
+    """Fill the deferred 'Error Message' cells of the FAILED TESTS table.
+
+    `fail_rows` is a list of (line_index, row_prefix, raw_message). We first
+    measure every other byte in the summary (rendering these cells empty), then
+    spend whatever remains of STEP_SUMMARY_BYTE_BUDGET on the messages via a
+    single uniform cap, so typical runs show full tracebacks and only oversized
+    runs clip their longest messages.
+    """
+    if not fail_rows:
+        return
+    for idx, prefix, _ in fail_rows:
+        lines[idx] = f"{prefix} |  |"
+    base_bytes = len('\n'.join('' if l is None else l for l in lines).encode('utf-8'))
+    cap = _fit_message_cap([m for _, _, m in fail_rows], STEP_SUMMARY_BYTE_BUDGET - base_bytes)
+    for idx, prefix, msg in fail_rows:
+        lines[idx] = f"{prefix} | {_message_cell(msg, cap)} |"
+
+
 def collect_failed_tests(arch_data, archs, s1_name, s2_name):
     """Return a list of failed test rows across all architectures.
 
@@ -802,6 +863,9 @@ def _job_id_link(url):
         cols.append(f'Job ID ({s2_name})')
     cols.append('Error Message')
 
+    # Filled in by _fill_failure_messages once the rest of the summary is sized,
+    # so the message column can use whatever byte budget is left.
+    fail_rows = []
     if s1_failed:
         lines.append(f'### FAILED TESTS ({len(s1_failed)})')
         lines.append('')
@@ -823,8 +887,8 @@ def _job_id_link(url):
             line += f" | {_job_id_link(t.get(f'job_url_{s1_name}', ''))}"
             if has_set2:
                 line += f" | {_job_id_link(t.get(f'job_url_{s2_name}', ''))}"
-            line += f" | {_message_cell(t.get('error_message', ''))} |"
-            lines.append(line)
+            lines.append(None)  # placeholder; message cell filled in below
+            fail_rows.append((len(lines) - 1, line, t.get('error_message', '')))
         lines.append('')
     else:
         lines.append('### FAILED TESTS')
@@ -870,6 +934,8 @@ def _job_id_link(url):
                 )
             lines.append('')
 
+    _fill_failure_messages(lines, fail_rows)
+
     md = '\n'.join(lines)
     with open(output_path, 'w') as f:
         f.write(md)