[CI] Parity: surface XML failure messages in the summary

.automation_scripts/pytorch-unit-test-scripts/generate_summary.py

@@ -1,6 +1,7 @@
 #!/usr/bin/env python3
 
 import argparse
+import ast
 import csv
 import os
 import re
@@ -266,6 +267,125 @@ def safe_float(v):
     return vals
 
 
+def _extract_message(raw):
+    """Turn a stored XML failure/error value into readable text.
+
+    summarize_xml_testreports.py writes the parsed XML failure node (a dict like
+    {'message': 'AssertionError: ...', 'text': '<full traceback>'}) into the
+    message_<set> CSV column via str(), so cells arrive as a dict repr. Prefer
+    the full traceback ('text'), fall back to the short 'message', and return
+    the raw string unchanged if it is not a dict repr.
+    """
+    if not raw:
+        return ''
+    raw = raw.strip()
+    if raw.startswith('{') and raw.endswith('}'):
+        try:
+            d = ast.literal_eval(raw)
+            if isinstance(d, dict):
+                return (d.get('text') or d.get('message') or '').strip()
+        except (ValueError, SyntaxError):
+            pass
+    return raw
+
+
+def _html_escape(s):
+    return (s or '').replace('&', '&amp;').replace('<', '&lt;').replace('>', '&gt;')
+
+
+# The whole .md is piped into $GITHUB_STEP_SUMMARY, and GitHub restricts each
+# step summary to 1 MiB - past that the upload fails (and content can be dropped
+# silently as it nears the limit), so a single run with many large tracebacks
+# would lose the entire summary. We keep the rendered summary under this budget
+# (1 MiB minus headroom for the surrounding tables/markdown) and only clip the
+# longest failure messages when a run would otherwise exceed it; the full text
+# always stays in the 'Error Message' column of the CSV artifact.
+# https://docs.github.com/en/actions/reference/workflows-and-actions/workflow-commands#step-isolation-and-limits
+STEP_SUMMARY_BYTE_BUDGET = 950_000
+
+# Per-message hard cap. Generous on purpose: real tracebacks fit well within it,
+# so they render in full; it only bounds pathological multi-MB messages. The
+# global budget above is what protects the summary when there are many failures.
+PER_MESSAGE_CHAR_LIMIT = 50_000
+
+
+def _truncate_message(msg, limit=PER_MESSAGE_CHAR_LIMIT):
+    """Cap a failure message for the markdown summary.
+
+    Keeps the tail (the exception/assertion lives at the end of a traceback) and
+    points readers to the CSV 'Error Message' column, which keeps the full text
+    for dashboards.
+    """
+    if not msg or len(msg) <= limit:
+        return msg
+    return ('...(truncated; full message in the Error Message column of the CSV artifact)\n'
+            + msg[-limit:])
+
+
+def _message_cell(msg, char_limit=PER_MESSAGE_CHAR_LIMIT):
+    """Render a failure message as a collapsible cell for a markdown table.
+
+    Table cells can't contain raw newlines or unescaped pipes, so the message
+    is HTML-escaped, pipes are escaped, and newlines become &#10; inside a
+    <pre> (GitHub renders these as line breaks). The whole thing is wrapped in
+    a <details> so each row shows only a small 'view' toggle by default.
+    `char_limit` is set per-run by the budget pass below; 0 drops the body.
+    """
+    if not msg:
+        return ''
+    if char_limit <= 0:
+        return ('<sub>message omitted to keep the summary under GitHub\u2019s 1 MiB '
+                'limit; see the Error Message column of the CSV artifact</sub>')
+    msg = _truncate_message(msg, char_limit)
+    body = (_html_escape(msg)
+            .replace('\r', '')
+            .replace('\n', '&#10;')
+            .replace('|', '\\|'))
+    return f'<details><summary>view</summary><pre>{body}</pre></details>'
+
+
+def _fit_message_cap(messages, budget):
+    """Largest uniform per-message char cap whose rendered cells fit `budget`
+    bytes. Messages shorter than the cap are untouched, so only the longest
+    (outlier) messages get clipped, and only when a run would exceed the budget.
+    """
+    if budget <= 0:
+        return 0
+
+    def total_bytes(cap):
+        return sum(len(_message_cell(m, cap).encode('utf-8')) for m in messages)
+
+    if total_bytes(PER_MESSAGE_CHAR_LIMIT) <= budget:
+        return PER_MESSAGE_CHAR_LIMIT
+    lo, hi = 0, PER_MESSAGE_CHAR_LIMIT
+    while lo < hi:
+        mid = (lo + hi + 1) // 2
+        if total_bytes(mid) <= budget:
+            lo = mid
+        else:
+            hi = mid - 1
+    return lo
+
+
+def _fill_failure_messages(lines, fail_rows):
+    """Fill the deferred 'Error Message' cells of the FAILED TESTS table.
+
+    `fail_rows` is a list of (line_index, row_prefix, raw_message). We first
+    measure every other byte in the summary (rendering these cells empty), then
+    spend whatever remains of STEP_SUMMARY_BYTE_BUDGET on the messages via a
+    single uniform cap, so typical runs show full tracebacks and only oversized
+    runs clip their longest messages.
+    """
+    if not fail_rows:
+        return
+    for idx, prefix, _ in fail_rows:
+        lines[idx] = f"{prefix} |  |"
+    base_bytes = len('\n'.join('' if l is None else l for l in lines).encode('utf-8'))
+    cap = _fit_message_cap([m for _, _, m in fail_rows], STEP_SUMMARY_BYTE_BUDGET - base_bytes)
+    for idx, prefix, msg in fail_rows:
+        lines[idx] = f"{prefix} | {_message_cell(msg, cap)} |"
+
+
 def collect_failed_tests(arch_data, archs, s1_name, s2_name):
     """Return a list of failed test rows across all architectures.
 
@@ -278,6 +398,10 @@ def collect_failed_tests(arch_data, archs, s1_name, s2_name):
     for arch in archs:
         d = arch_data[arch]
         s1_col, s2_col, s1_time, _ = d['cols']
+        # message_<name> columns mirror the status_<name> naming (incl. the
+        # status_set1/set2 fallback), so derive them from the resolved cols.
+        s1_msg_col = s1_col.replace('status_', 'message_', 1)
+        s2_msg_col = s2_col.replace('status_', 'message_', 1)
         has_set2 = d.get('has_set2', True)
         for r in d['rows']:
             s1 = r[s1_col].strip()
@@ -293,11 +417,13 @@ def collect_failed_tests(arch_data, archs, s1_name, s2_name):
                     f'shard_{s1_name}': r.get(f'shard_{s1_name}', ''),
                     f'job_url_{s1_name}': r.get(f'job_url_{s1_name}', ''),
                     f'status_{s1_name}': s1,
+                    'error_message': _extract_message(r.get(s1_msg_col, '')),
                 }
                 if has_set2:
                     entry[f'shard_{s2_name}'] = r.get(f'shard_{s2_name}', '')
                     entry[f'job_url_{s2_name}'] = r.get(f'job_url_{s2_name}', '')
                     entry[f'status_{s2_name}'] = s2
+                    entry['error_message_set2'] = _extract_message(r.get(s2_msg_col, ''))
                 failed.append(entry)
 
     return failed
@@ -609,6 +735,7 @@ def _xml_test_shard(t, platform):
         if has_set2:
             header.append(f'Status ({s2_name})')
         header.append('Also Failing In')
+        header.append('Error Message')
         csv_rows.append(header)
         for t in s1_failed:
             row = [t['arch'], t['test_config'], t['test_file'],
@@ -623,6 +750,7 @@ def _xml_test_shard(t, platform):
             if has_set2:
                 row.append(t.get(f'status_{s2_name}', ''))
             row.append(t.get('also_failing_in', ''))
+            row.append(t.get('error_message', ''))
             csv_rows.append(row)
         csv_rows.append([])
 
@@ -733,7 +861,11 @@ def _job_id_link(url):
     cols.append(f'Job ID ({s1_name})')
     if has_set2:
         cols.append(f'Job ID ({s2_name})')
+    cols.append('Error Message')
 
+    # Filled in by _fill_failure_messages once the rest of the summary is sized,
+    # so the message column can use whatever byte budget is left.
+    fail_rows = []
     if s1_failed:
         lines.append(f'### FAILED TESTS ({len(s1_failed)})')
         lines.append('')
@@ -755,8 +887,8 @@ def _job_id_link(url):
             line += f" | {_job_id_link(t.get(f'job_url_{s1_name}', ''))}"
             if has_set2:
                 line += f" | {_job_id_link(t.get(f'job_url_{s2_name}', ''))}"
-            line += ' |'
-            lines.append(line)
+            lines.append(None)  # placeholder; message cell filled in below
+            fail_rows.append((len(lines) - 1, line, t.get('error_message', '')))
         lines.append('')
     else:
         lines.append('### FAILED TESTS')
@@ -802,6 +934,8 @@ def _job_id_link(url):
                 )
             lines.append('')
 
+    _fill_failure_messages(lines, fail_rows)
+
     md = '\n'.join(lines)
     with open(output_path, 'w') as f:
         f.write(md)