diff --git a/.automation_scripts/pytorch-unit-test-scripts/generate_summary.py b/.automation_scripts/pytorch-unit-test-scripts/generate_summary.py index b01d0ecf6c8de..0eb4d010e5b0e 100644 --- a/.automation_scripts/pytorch-unit-test-scripts/generate_summary.py +++ b/.automation_scripts/pytorch-unit-test-scripts/generate_summary.py @@ -1,6 +1,7 @@ #!/usr/bin/env python3 import argparse +import ast import csv import os import re @@ -266,6 +267,125 @@ def safe_float(v): return vals +def _extract_message(raw): + """Turn a stored XML failure/error value into readable text. + + summarize_xml_testreports.py writes the parsed XML failure node (a dict like + {'message': 'AssertionError: ...', 'text': ''}) into the + message_ CSV column via str(), so cells arrive as a dict repr. Prefer + the full traceback ('text'), fall back to the short 'message', and return + the raw string unchanged if it is not a dict repr. + """ + if not raw: + return '' + raw = raw.strip() + if raw.startswith('{') and raw.endswith('}'): + try: + d = ast.literal_eval(raw) + if isinstance(d, dict): + return (d.get('text') or d.get('message') or '').strip() + except (ValueError, SyntaxError): + pass + return raw + + +def _html_escape(s): + return (s or '').replace('&', '&').replace('<', '<').replace('>', '>') + + +# The whole .md is piped into $GITHUB_STEP_SUMMARY, and GitHub restricts each +# step summary to 1 MiB - past that the upload fails (and content can be dropped +# silently as it nears the limit), so a single run with many large tracebacks +# would lose the entire summary. We keep the rendered summary under this budget +# (1 MiB minus headroom for the surrounding tables/markdown) and only clip the +# longest failure messages when a run would otherwise exceed it; the full text +# always stays in the 'Error Message' column of the CSV artifact. +# https://docs.github.com/en/actions/reference/workflows-and-actions/workflow-commands#step-isolation-and-limits +STEP_SUMMARY_BYTE_BUDGET = 950_000 + +# Per-message hard cap. Generous on purpose: real tracebacks fit well within it, +# so they render in full; it only bounds pathological multi-MB messages. The +# global budget above is what protects the summary when there are many failures. +PER_MESSAGE_CHAR_LIMIT = 50_000 + + +def _truncate_message(msg, limit=PER_MESSAGE_CHAR_LIMIT): + """Cap a failure message for the markdown summary. + + Keeps the tail (the exception/assertion lives at the end of a traceback) and + points readers to the CSV 'Error Message' column, which keeps the full text + for dashboards. + """ + if not msg or len(msg) <= limit: + return msg + return ('...(truncated; full message in the Error Message column of the CSV artifact)\n' + + msg[-limit:]) + + +def _message_cell(msg, char_limit=PER_MESSAGE_CHAR_LIMIT): + """Render a failure message as a collapsible cell for a markdown table. + + Table cells can't contain raw newlines or unescaped pipes, so the message + is HTML-escaped, pipes are escaped, and newlines become inside a +
 (GitHub renders these as line breaks). The whole thing is wrapped in
+    a 
so each row shows only a small 'view' toggle by default. + `char_limit` is set per-run by the budget pass below; 0 drops the body. + """ + if not msg: + return '' + if char_limit <= 0: + return ('message omitted to keep the summary under GitHub\u2019s 1 MiB ' + 'limit; see the Error Message column of the CSV artifact') + msg = _truncate_message(msg, char_limit) + body = (_html_escape(msg) + .replace('\r', '') + .replace('\n', ' ') + .replace('|', '\\|')) + return f'
view
{body}
' + + +def _fit_message_cap(messages, budget): + """Largest uniform per-message char cap whose rendered cells fit `budget` + bytes. Messages shorter than the cap are untouched, so only the longest + (outlier) messages get clipped, and only when a run would exceed the budget. + """ + if budget <= 0: + return 0 + + def total_bytes(cap): + return sum(len(_message_cell(m, cap).encode('utf-8')) for m in messages) + + if total_bytes(PER_MESSAGE_CHAR_LIMIT) <= budget: + return PER_MESSAGE_CHAR_LIMIT + lo, hi = 0, PER_MESSAGE_CHAR_LIMIT + while lo < hi: + mid = (lo + hi + 1) // 2 + if total_bytes(mid) <= budget: + lo = mid + else: + hi = mid - 1 + return lo + + +def _fill_failure_messages(lines, fail_rows): + """Fill the deferred 'Error Message' cells of the FAILED TESTS table. + + `fail_rows` is a list of (line_index, row_prefix, raw_message). We first + measure every other byte in the summary (rendering these cells empty), then + spend whatever remains of STEP_SUMMARY_BYTE_BUDGET on the messages via a + single uniform cap, so typical runs show full tracebacks and only oversized + runs clip their longest messages. + """ + if not fail_rows: + return + for idx, prefix, _ in fail_rows: + lines[idx] = f"{prefix} | |" + base_bytes = len('\n'.join('' if l is None else l for l in lines).encode('utf-8')) + cap = _fit_message_cap([m for _, _, m in fail_rows], STEP_SUMMARY_BYTE_BUDGET - base_bytes) + for idx, prefix, msg in fail_rows: + lines[idx] = f"{prefix} | {_message_cell(msg, cap)} |" + + def collect_failed_tests(arch_data, archs, s1_name, s2_name): """Return a list of failed test rows across all architectures. @@ -278,6 +398,10 @@ def collect_failed_tests(arch_data, archs, s1_name, s2_name): for arch in archs: d = arch_data[arch] s1_col, s2_col, s1_time, _ = d['cols'] + # message_ columns mirror the status_ naming (incl. the + # status_set1/set2 fallback), so derive them from the resolved cols. + s1_msg_col = s1_col.replace('status_', 'message_', 1) + s2_msg_col = s2_col.replace('status_', 'message_', 1) has_set2 = d.get('has_set2', True) for r in d['rows']: s1 = r[s1_col].strip() @@ -293,11 +417,13 @@ def collect_failed_tests(arch_data, archs, s1_name, s2_name): f'shard_{s1_name}': r.get(f'shard_{s1_name}', ''), f'job_url_{s1_name}': r.get(f'job_url_{s1_name}', ''), f'status_{s1_name}': s1, + 'error_message': _extract_message(r.get(s1_msg_col, '')), } if has_set2: entry[f'shard_{s2_name}'] = r.get(f'shard_{s2_name}', '') entry[f'job_url_{s2_name}'] = r.get(f'job_url_{s2_name}', '') entry[f'status_{s2_name}'] = s2 + entry['error_message_set2'] = _extract_message(r.get(s2_msg_col, '')) failed.append(entry) return failed @@ -609,6 +735,7 @@ def _xml_test_shard(t, platform): if has_set2: header.append(f'Status ({s2_name})') header.append('Also Failing In') + header.append('Error Message') csv_rows.append(header) for t in s1_failed: row = [t['arch'], t['test_config'], t['test_file'], @@ -623,6 +750,7 @@ def _xml_test_shard(t, platform): if has_set2: row.append(t.get(f'status_{s2_name}', '')) row.append(t.get('also_failing_in', '')) + row.append(t.get('error_message', '')) csv_rows.append(row) csv_rows.append([]) @@ -733,7 +861,11 @@ def _job_id_link(url): cols.append(f'Job ID ({s1_name})') if has_set2: cols.append(f'Job ID ({s2_name})') + cols.append('Error Message') + # Filled in by _fill_failure_messages once the rest of the summary is sized, + # so the message column can use whatever byte budget is left. + fail_rows = [] if s1_failed: lines.append(f'### FAILED TESTS ({len(s1_failed)})') lines.append('') @@ -755,8 +887,8 @@ def _job_id_link(url): line += f" | {_job_id_link(t.get(f'job_url_{s1_name}', ''))}" if has_set2: line += f" | {_job_id_link(t.get(f'job_url_{s2_name}', ''))}" - line += ' |' - lines.append(line) + lines.append(None) # placeholder; message cell filled in below + fail_rows.append((len(lines) - 1, line, t.get('error_message', ''))) lines.append('') else: lines.append('### FAILED TESTS') @@ -802,6 +934,8 @@ def _job_id_link(url): ) lines.append('') + _fill_failure_messages(lines, fail_rows) + md = '\n'.join(lines) with open(output_path, 'w') as f: f.write(md)