From b19f834b6cbbe67ff5c9de3d1c878448b3f51f80 Mon Sep 17 00:00:00 2001 From: ethanwee1 Date: Thu, 25 Jun 2026 17:38:54 +0000 Subject: [PATCH] parity report: dedupe LOG-BASED FAILURES + fix empty Job ID links Two fixes to the parity report (both user-reported): - generate_summary.py: a test could be listed twice in LOG-BASED FAILURES - once as FAILED and once as CONSISTENT_FAILURE (a one-off FAILED line and a 'FAILED CONSISTENTLY' line describe the same test). Add a shared _select_rocm_log_failures() helper (used by both the CSV and markdown tables) that filters out XML-failed tests and dedupes per test, preferring CONSISTENT_FAILURE so a test is never shown as both. - download_testlogs: _shorten_unzipped_dirs extracted the upstream CI job id with re.search(r'_(\\d{6,})\\.zip$'), but the unzipped shard dirs end in '_' with no '.zip' (e.g. ...gfx950.2_83371682957). The id was dropped, dirs became 'test-default-1-8', and parse_xml_reports_as_dict couldn't recover it -> the FAILED TESTS 'Job ID' columns were always empty. Make '.zip' optional: r'_(\\d{6,})(?:\\.zip)?$'. --- .../download_testlogs | 11 +-- .../generate_summary.py | 69 ++++++++++--------- 2 files changed, 44 insertions(+), 36 deletions(-) diff --git a/.automation_scripts/pytorch-unit-test-scripts/download_testlogs b/.automation_scripts/pytorch-unit-test-scripts/download_testlogs index 59eb9fbba81e5..22371a1c9e9a1 100755 --- a/.automation_scripts/pytorch-unit-test-scripts/download_testlogs +++ b/.automation_scripts/pytorch-unit-test-scripts/download_testlogs @@ -338,11 +338,12 @@ def _shorten_unzipped_dirs(): m = re.search(r'test-(?:osdc-)?(default|distributed|inductor)-(\d+)-(\d+)', d.name) if m: short_name = f"test-{m.group(1)}-{m.group(2)}-{m.group(3)}" - # The original artifact name ends with "_.zip" where - # is the upstream pytorch CI job id (e.g. - # ..._68613413431.zip). Carry it onto short_name so - # summarize_xml_testreports.py can link to that job. - job_id_match = re.search(r'_(\d{6,})\.zip$', d.name) + # The artifact/dir name ends with "_" (optionally ".zip") + # where is the upstream pytorch CI job id (e.g. + # ..._68613413431 or ..._68613413431.zip). Carry it onto short_name + # so summarize_xml_testreports.py can link to that job. The unzipped + # directory has no .zip suffix, so it must be optional here. + job_id_match = re.search(r'_(\d{6,})(?:\.zip)?$', d.name) if job_id_match: short_name += f"_{job_id_match.group(1)}" if not Path(short_name).exists(): diff --git a/.automation_scripts/pytorch-unit-test-scripts/generate_summary.py b/.automation_scripts/pytorch-unit-test-scripts/generate_summary.py index 0eb4d010e5b0e..5c78317197b43 100644 --- a/.automation_scripts/pytorch-unit-test-scripts/generate_summary.py +++ b/.automation_scripts/pytorch-unit-test-scripts/generate_summary.py @@ -699,6 +699,42 @@ def _parse_log_failure_names(lf): return parts[0], parts[1] +def _select_rocm_log_failures(log_failures, failed_tests, s1_name): + """ROCm log-detected failures not already in the XML FAILED TESTS table, + deduplicated to one row per test. + + A single test can appear in log_failures more than once - e.g. a per-test + 'FAILED' line and a 'FAILED CONSISTENTLY' line (category CONSISTENT_FAILURE) + describe the same test. Collapse those to a single row, preferring + CONSISTENT_FAILURE over a one-off FAILED so a test is never listed as both. + """ + xml_failed_keys = { + (t['arch'], _norm_test_file(t['test_file']), t['test_class'], t['test_name']) + for t in (failed_tests or []) + } + best = {} + order = [] + for lf in (log_failures or []): + if lf.get('platform', '') != s1_name: + continue + test_class, test_name = _parse_log_failure_names(lf) + key = (lf.get('arch', ''), _norm_test_file(lf.get('test_file', '')), + test_class, test_name) + # Skip entries already present in the XML-based FAILED TESTS table to + # avoid double-counting the same failure, except FLAKY entries which + # represent an independent signal (a rerun passed). + if key in xml_failed_keys and lf.get('category', '') != 'FLAKY': + continue + existing = best.get(key) + if existing is None: + best[key] = lf + order.append(key) + elif (lf.get('category') == 'CONSISTENT_FAILURE' + and existing.get('category') != 'CONSISTENT_FAILURE'): + best[key] = lf + return [best[k] for k in order] + + def write_csv(rows, archs, output_path, failed_tests=None, s1_name='set1', s2_name='set2', has_set2=True, log_failures=None, shard_lookup=None): csv_rows = [] csv_rows.append([''] + list(archs)) @@ -755,23 +791,7 @@ def _xml_test_shard(t, platform): csv_rows.append([]) if log_failures: - xml_failed_keys = { - (t['arch'], _norm_test_file(t['test_file']), t['test_class'], t['test_name']) - for t in (failed_tests or []) - } - rocm_log_failures = [] - for lf in log_failures: - if lf.get('platform', '') != s1_name: - continue - test_class, test_name = _parse_log_failure_names(lf) - key = (lf.get('arch', ''), _norm_test_file(lf.get('test_file', '')), - test_class, test_name) - # Skip entries already present in the XML-based FAILED TESTS table - # to avoid double-counting the same failure, except for FLAKY - # entries which represent an independent signal (a rerun passed). - if key in xml_failed_keys and lf.get('category', '') != 'FLAKY': - continue - rocm_log_failures.append(lf) + rocm_log_failures = _select_rocm_log_failures(log_failures, failed_tests, s1_name) if rocm_log_failures: csv_rows.append(['LOG-BASED FAILURES (not in XML)']) csv_rows.append(['Arch', 'Platform', 'Test Config', 'Test File', 'Test Class', @@ -897,20 +917,7 @@ def _job_id_link(url): lines.append('') if log_failures: - xml_failed_keys = { - (t['arch'], _norm_test_file(t['test_file']), t['test_class'], t['test_name']) - for t in (failed_tests or []) - } - rocm_log_failures = [] - for lf in log_failures: - if lf.get('platform', '') != s1_name: - continue - test_class, test_name = _parse_log_failure_names(lf) - key = (lf.get('arch', ''), _norm_test_file(lf.get('test_file', '')), - test_class, test_name) - if key in xml_failed_keys and lf.get('category', '') != 'FLAKY': - continue - rocm_log_failures.append(lf) + rocm_log_failures = _select_rocm_log_failures(log_failures, failed_tests, s1_name) if rocm_log_failures: lines.append(f'### LOG-BASED FAILURES (not in XML) ({len(rocm_log_failures)})') lines.append('')