Skip to content

[CI] Fix parity commit run selection #3210

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
ethanwee1 wants to merge 6 commits into from
Closed

[CI] Fix parity commit run selection #3210

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
0e3d7a8
[CI] Fix parity commit run selection
ethanwee1 May 11, 2026
099dad5
parity summary: exclude inductor.test_cpu_repro from comparison
ethanwee1 Jun 18, 2026
4412345
parity summary: surface XML failure messages
ethanwee1 Jun 19, 2026
aef8a84
parity summary: cap failure-message size in markdown
ethanwee1 Jun 22, 2026
6ed3a9d
parity summary: move failure message into a per-row collapsible column
ethanwee1 Jun 22, 2026
9255fdf
parity summary: budget failure messages against the 1 MiB step-summar…
ethanwee1 Jun 22, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 12 additions & 2 deletions .automation_scripts/pytorch-unit-test-scripts/detect_log_failures.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -69,13 +69,23 @@


def classify_log_file(filename):
"""Return (platform, test_config, shard_num) from a log filename like rocm3.txt."""
"""Return (platform, test_config, shard_num) from a log filename like rocm3.txt.

Commit-vs-commit parity prefixes log files with the short commit SHA
(for example, 09e0c59b_rocm3.txt). In that mode the SHA label is the
platform name used by generate_summary.py, so preserve it here.
"""
stem = Path(filename).stem
label = None
m = re.match(r"(?P<label>[0-9a-f]{8,40})_(?P<stem>.+)", stem)
if m:
label = m.group("label")[:8]
stem = m.group("stem")
for prefix, (platform, test_config) in sorted(LOG_FILE_MAP.items(), key=lambda x: -len(x[0])):
if stem.startswith(prefix):
remainder = stem[len(prefix):]
if remainder.isdigit():
return platform, test_config, int(remainder)
return label or platform, test_config, int(remainder)
return None, None, None


Expand Down
46 changes: 31 additions & 15 deletions .automation_scripts/pytorch-unit-test-scripts/download_testlogs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -336,7 +336,7 @@ def download_artifacts(wf, prefixes=[], test_folder=".", allowed_substrings=None
)
os.chdir("..")
# for older runs, add 'created':'<=YYYY-MM-DD'. see https://docs.github.com/en/search-github/getting-started-with-searching-on-github/understanding-the-search-syntax#query-for-dates
def download_workflow_run(created=None, max_pages=10, workflow=None, sha=None, ignore_status=False, status='success', error_msg='Error downloading workflow runs'):
def download_workflow_run(created=None, max_pages=10, workflow=None, sha=None, branch=None, ignore_status=False, status='success', error_msg='Error downloading workflow runs'):
if not workflow:
raise Exception("Workflow must be specified")
for page in range(max_pages):
Expand All @@ -348,8 +348,10 @@ def download_workflow_run(created=None, max_pages=10, workflow=None, sha=None, i
params['created'] = created
if sha:
params['head_sha'] = sha
if branch:
params['branch'] = branch
else:
params['branch'] = "main"
params['branch'] = branch or "main"
print(".")

# Uncomment below for additional debug info
Expand All @@ -365,6 +367,10 @@ def download_workflow_run(created=None, max_pages=10, workflow=None, sha=None, i
raise Exception(response.text)
if not workflow_runs:
continue
if branch:
workflow_runs = [wf for wf in workflow_runs if wf.get('head_branch') == branch]
if not workflow_runs:
continue
# Prefer completed runs over in-progress ones. When multiple
# runs exist for the same SHA, the most recent may still be
# running and have no artifacts yet.
Expand Down Expand Up @@ -545,6 +551,7 @@ def main():
sha = args.sha1
pr_id = None
status = "success"
sha_branch = "main" if args.sha1 else None
print(f"sha: {sha}")

# When comparing two commits, prefix log filenames with short SHAs
Expand All @@ -565,7 +572,7 @@ def main():
#https://docs.github.com/en/rest/actions/workflow-runs#list-workflow-runs-for-a-repository
periodic_fallback_used = False
try:
periodic_wf = download_workflow_run(created=args.created, max_pages=args.max_pages, workflow=ROCmWorkflowNames["distributed"], sha=periodic_sha, ignore_status=args.ignore_status, status=status, error_msg=error_msg)
periodic_wf = download_workflow_run(created=args.created, max_pages=args.max_pages, workflow=ROCmWorkflowNames["distributed"], sha=periodic_sha, branch=sha_branch, ignore_status=args.ignore_status, status=status, error_msg=error_msg)
except (IndexError, Exception):
periodic_wf = None
periodic_fallbacks = {
Expand All @@ -575,7 +582,7 @@ def main():
if periodic_wf is None and arch in periodic_fallbacks:
fallback_wf, fallback_prefix = periodic_fallbacks[arch]
print(f"Distributed not found in {ROCmWorkflowNames['distributed']}, falling back to {fallback_wf}")
periodic_wf = download_workflow_run(created=args.created, max_pages=args.max_pages, workflow=fallback_wf, sha=periodic_sha, ignore_status=args.ignore_status, status=status, error_msg=error_msg)
periodic_wf = download_workflow_run(created=args.created, max_pages=args.max_pages, workflow=fallback_wf, sha=periodic_sha, branch=sha_branch, ignore_status=args.ignore_status, status=status, error_msg=error_msg)
periodic_fallback_used = True
if periodic_wf is None:
raise Exception(error_msg)
Expand Down Expand Up @@ -631,7 +638,7 @@ def main():
error_msg="Error: rocm workflow not found in scanned workflow runs. Try increasing max_pages."
default_fallback_used = False
try:
rocm_wf = download_workflow_run(created=args.created, max_pages=args.max_pages, workflow=ROCmWorkflowNames["default"], sha=rocm_sha, ignore_status=args.ignore_status, status=status, error_msg=error_msg)
rocm_wf = download_workflow_run(created=args.created, max_pages=args.max_pages, workflow=ROCmWorkflowNames["default"], sha=rocm_sha, branch=sha_branch, ignore_status=args.ignore_status, status=status, error_msg=error_msg)
except (IndexError, Exception):
rocm_wf = None
default_fallbacks = {
Expand All @@ -641,7 +648,7 @@ def main():
if rocm_wf is None and arch in default_fallbacks:
fallback_wf, fallback_prefix = default_fallbacks[arch]
print(f"Default not found in {ROCmWorkflowNames['default']}, falling back to {fallback_wf}")
rocm_wf = download_workflow_run(created=args.created, max_pages=args.max_pages, workflow=fallback_wf, sha=rocm_sha, ignore_status=args.ignore_status, status=status, error_msg=error_msg)
rocm_wf = download_workflow_run(created=args.created, max_pages=args.max_pages, workflow=fallback_wf, sha=rocm_sha, branch=sha_branch, ignore_status=args.ignore_status, status=status, error_msg=error_msg)
default_fallback_used = True
rocm_job_prefix['default'] = fallback_prefix
if rocm_wf is None:
Expand Down Expand Up @@ -692,7 +699,7 @@ def main():
error_msg="Error: inductor workflow not found in scanned workflow runs. Try increasing max_pages."
inductor_fallback_used = False
try:
inductor_wf_rocm = download_workflow_run(created=args.created, max_pages=args.max_pages, workflow=ROCmWorkflowNames["inductor"], sha=inductor_rocm_sha, ignore_status=args.ignore_status, status=status, error_msg=error_msg)
inductor_wf_rocm = download_workflow_run(created=args.created, max_pages=args.max_pages, workflow=ROCmWorkflowNames["inductor"], sha=inductor_rocm_sha, branch=sha_branch, ignore_status=args.ignore_status, status=status, error_msg=error_msg)
except (IndexError, Exception):
inductor_wf_rocm = None
inductor_fallbacks = {
Expand All @@ -701,7 +708,7 @@ def main():
if inductor_wf_rocm is None and arch in inductor_fallbacks:
fallback_wf, fallback_prefix = inductor_fallbacks[arch]
print(f"Inductor not found in {ROCmWorkflowNames['inductor']}, falling back to {fallback_wf}")
inductor_wf_rocm = download_workflow_run(created=args.created, max_pages=args.max_pages, workflow=fallback_wf, sha=inductor_rocm_sha, ignore_status=args.ignore_status, status=status, error_msg=error_msg)
inductor_wf_rocm = download_workflow_run(created=args.created, max_pages=args.max_pages, workflow=fallback_wf, sha=inductor_rocm_sha, branch=sha_branch, ignore_status=args.ignore_status, status=status, error_msg=error_msg)
inductor_fallback_used = True
if inductor_wf_rocm is None:
raise Exception(error_msg)
Expand Down Expand Up @@ -756,11 +763,15 @@ def main():
if not args.ignore_status:
params['status'] = status
params['head_sha'] = sha
if sha_branch:
params['branch'] = sha_branch
resp = requests.get(
f"https://api.github.com/repos/pytorch/pytorch/actions/workflows/{CUDAWorkflowNames['default']}.yml/runs",
headers=authentication_headers, params=params,
)
trunk_runs = resp.json().get('workflow_runs', [])
if sha_branch:
trunk_runs = [run for run in trunk_runs if run.get('head_branch') == sha_branch]

for run in trunk_runs:
jobs = get_workflow_jobs(run)
Expand Down Expand Up @@ -796,8 +807,13 @@ def main():
headers=authentication_headers,
)
trunk_wf = resp.json()
print(f"CUDA test jobs are in trunk run {trunk_wf['id']} (found via check-runs)")
all_cuda_jobs = list(cuda_test_jobs)
if sha_branch and trunk_wf.get('head_branch') != sha_branch:
print(f"Skipping CUDA check-run from non-{sha_branch} branch run {trunk_wf.get('id')}")
trunk_wf = None
cuda_test_jobs = []
else:
print(f"CUDA test jobs are in trunk run {trunk_wf['id']} (found via check-runs)")
all_cuda_jobs = list(cuda_test_jobs)

if trunk_wf is None:
trunk_wf = trunk_runs[0] if trunk_runs else None
Expand Down Expand Up @@ -873,7 +889,7 @@ def main():
# find tests in inductor workflow with given sha and success status
#https://docs.github.com/en/rest/actions/workflow-runs#list-workflow-runs-for-a-repository
error_msg="Error: inductor workflow not found in scanned workflow runs. Try increasing max_pages."
inductor_wf_cuda = download_workflow_run(created=args.created, max_pages=args.max_pages, workflow=CUDAWorkflowNames["inductor"], sha=inductor_sha, ignore_status=args.ignore_status, status=status, error_msg=error_msg)
inductor_wf_cuda = download_workflow_run(created=args.created, max_pages=args.max_pages, workflow=CUDAWorkflowNames["inductor"], sha=inductor_sha, branch=sha_branch, ignore_status=args.ignore_status, status=status, error_msg=error_msg)
print(f"Using workflow '{CUDAWorkflowNames['inductor']}' with id:{inductor_wf_cuda['id']} for CUDA inductor")

inductor_cuda_jobs = get_workflow_jobs(inductor_wf_cuda)
Expand Down Expand Up @@ -930,7 +946,7 @@ def main():
try:
baseline_default_wf = download_workflow_run(
created=args.created, max_pages=args.max_pages,
workflow=ROCmWorkflowNames["default"], sha=baseline_sha,
workflow=ROCmWorkflowNames["default"], sha=baseline_sha, branch="main",
ignore_status=args.ignore_status, status=status,
error_msg=f"Baseline default workflow not found for {baseline_sha}",
)
Expand Down Expand Up @@ -962,7 +978,7 @@ def main():
try:
baseline_dist_wf = download_workflow_run(
created=args.created, max_pages=args.max_pages,
workflow=ROCmWorkflowNames["distributed"], sha=baseline_sha,
workflow=ROCmWorkflowNames["distributed"], sha=baseline_sha, branch="main",
ignore_status=args.ignore_status, status=status,
error_msg=f"Baseline distributed workflow not found for {baseline_sha}",
)
Expand Down Expand Up @@ -994,7 +1010,7 @@ def main():
try:
baseline_inductor_wf = download_workflow_run(
created=args.created, max_pages=args.max_pages,
workflow=ROCmWorkflowNames["inductor"], sha=baseline_sha,
workflow=ROCmWorkflowNames["inductor"], sha=baseline_sha, branch="main",
ignore_status=args.ignore_status, status=status,
error_msg=f"Baseline inductor workflow not found for {baseline_sha}",
)
Expand Down Expand Up @@ -1033,7 +1049,7 @@ def main():
try:
inductor_periodic_wf = download_workflow_run(
created=args.created, max_pages=args.max_pages,
workflow="inductor-periodic", sha=sha,
workflow="inductor-periodic", sha=sha, branch=sha_branch,
ignore_status=args.ignore_status, status=status,
error_msg=error_msg,
)
Expand Down
Loading