From 1c4982902b72b9cef4e50884d41e149c7a831874 Mon Sep 17 00:00:00 2001 From: luoye-cpu <3412829159@qq.com> Date: Sat, 7 Nov 2026 22:35:20 +0800 Subject: [PATCH] test(cuda): add the #3320 retry-study per-run logs The per-run logs of the four-arm rerun of the #3320 test as landed (master b38635f63): arm A as-landed (24/24 fail at 'device memory not yet returned after the hog was freed'), the interleaved A/B that rules out probe perturbation, the bounded 60 s poll that hit the bound 6/6 in the slow window and returned instantly 6/6 after the mid-campaign window flip, and the 1/5/15/45/105 s silent ladder. SUMMARY.md carries the arms, results and the mechanism reading (the slow-window driver holds the freed hog memory to process exit). Committed at the maintainer's request as evidence for the retry-bound calibration of the implementation issue to follow. Signed-off-by: luoye-cpu <3412829159@qq.com> --- testdata/cuda-oom-retry-study/SUMMARY.md | 47 +++++++ .../cuda-oom-retry-study/armA/dev0/run-01.err | 5 + .../cuda-oom-retry-study/armA/dev0/run-01.out | 0 .../cuda-oom-retry-study/armA/dev0/run-02.err | 5 + .../cuda-oom-retry-study/armA/dev0/run-02.out | 0 .../cuda-oom-retry-study/armA/dev0/run-03.err | 5 + .../cuda-oom-retry-study/armA/dev0/run-03.out | 0 .../cuda-oom-retry-study/armA/dev0/run-04.err | 5 + .../cuda-oom-retry-study/armA/dev0/run-04.out | 0 .../cuda-oom-retry-study/armA/dev0/run-05.err | 5 + .../cuda-oom-retry-study/armA/dev0/run-05.out | 0 .../cuda-oom-retry-study/armA/dev0/run-06.err | 5 + .../cuda-oom-retry-study/armA/dev0/run-06.out | 0 .../cuda-oom-retry-study/armA/dev0/run-07.err | 5 + .../cuda-oom-retry-study/armA/dev0/run-07.out | 0 .../cuda-oom-retry-study/armA/dev0/run-08.err | 5 + .../cuda-oom-retry-study/armA/dev0/run-08.out | 0 .../cuda-oom-retry-study/armA/dev0/run-09.err | 5 + .../cuda-oom-retry-study/armA/dev0/run-09.out | 0 .../cuda-oom-retry-study/armA/dev0/run-10.err | 5 + .../cuda-oom-retry-study/armA/dev0/run-10.out | 0 .../cuda-oom-retry-study/armA/dev0/run-11.err | 5 + .../cuda-oom-retry-study/armA/dev0/run-11.out | 0 .../cuda-oom-retry-study/armA/dev0/run-12.err | 5 + .../cuda-oom-retry-study/armA/dev0/run-12.out | 0 .../cuda-oom-retry-study/armA/dev1/run-01.err | 6 + .../cuda-oom-retry-study/armA/dev1/run-01.out | 0 .../cuda-oom-retry-study/armA/dev1/run-02.err | 6 + .../cuda-oom-retry-study/armA/dev1/run-02.out | 0 .../cuda-oom-retry-study/armA/dev1/run-03.err | 6 + .../cuda-oom-retry-study/armA/dev1/run-03.out | 0 .../cuda-oom-retry-study/armA/dev1/run-04.err | 6 + .../cuda-oom-retry-study/armA/dev1/run-04.out | 0 .../cuda-oom-retry-study/armA/dev1/run-05.err | 6 + .../cuda-oom-retry-study/armA/dev1/run-05.out | 0 .../cuda-oom-retry-study/armA/dev1/run-06.err | 6 + .../cuda-oom-retry-study/armA/dev1/run-06.out | 0 .../cuda-oom-retry-study/armA/dev1/run-07.err | 6 + .../cuda-oom-retry-study/armA/dev1/run-07.out | 0 .../cuda-oom-retry-study/armA/dev1/run-08.err | 6 + .../cuda-oom-retry-study/armA/dev1/run-08.out | 0 .../cuda-oom-retry-study/armA/dev1/run-09.err | 6 + .../cuda-oom-retry-study/armA/dev1/run-09.out | 0 .../cuda-oom-retry-study/armA/dev1/run-10.err | 6 + .../cuda-oom-retry-study/armA/dev1/run-10.out | 0 .../cuda-oom-retry-study/armA/dev1/run-11.err | 6 + .../cuda-oom-retry-study/armA/dev1/run-11.out | 0 .../cuda-oom-retry-study/armA/dev1/run-12.err | 6 + .../cuda-oom-retry-study/armA/dev1/run-12.out | 0 .../cuda-oom-retry-study/armA/timeline.log | 97 ++++++++++++++ .../cuda-oom-retry-study/armB/dev0/run-01.err | 6 + .../cuda-oom-retry-study/armB/dev0/run-01.out | 0 .../cuda-oom-retry-study/armB/dev0/run-02.err | 6 + .../cuda-oom-retry-study/armB/dev0/run-02.out | 0 .../cuda-oom-retry-study/armB/dev0/run-03.err | 6 + .../cuda-oom-retry-study/armB/dev0/run-03.out | 0 .../cuda-oom-retry-study/armB/dev0/run-04.err | 6 + .../cuda-oom-retry-study/armB/dev0/run-04.out | 0 .../cuda-oom-retry-study/armB/dev0/run-05.err | 6 + .../cuda-oom-retry-study/armB/dev0/run-05.out | 0 .../cuda-oom-retry-study/armB/dev0/run-06.err | 6 + .../cuda-oom-retry-study/armB/dev0/run-06.out | 0 .../cuda-oom-retry-study/armB/dev0/run-07.err | 6 + .../cuda-oom-retry-study/armB/dev0/run-07.out | 0 .../cuda-oom-retry-study/armB/dev0/run-08.err | 6 + .../cuda-oom-retry-study/armB/dev0/run-08.out | 0 .../cuda-oom-retry-study/armB/dev0/run-09.err | 6 + .../cuda-oom-retry-study/armB/dev0/run-09.out | 0 .../cuda-oom-retry-study/armB/dev0/run-10.err | 6 + .../cuda-oom-retry-study/armB/dev0/run-10.out | 0 .../cuda-oom-retry-study/armB/dev0/run-11.err | 6 + .../cuda-oom-retry-study/armB/dev0/run-11.out | 0 .../cuda-oom-retry-study/armB/dev0/run-12.err | 6 + .../cuda-oom-retry-study/armB/dev0/run-12.out | 0 .../cuda-oom-retry-study/armB/dev1/run-01.err | 5 + .../cuda-oom-retry-study/armB/dev1/run-01.out | 0 .../cuda-oom-retry-study/armB/dev1/run-02.err | 5 + .../cuda-oom-retry-study/armB/dev1/run-02.out | 0 .../cuda-oom-retry-study/armB/dev1/run-03.err | 5 + .../cuda-oom-retry-study/armB/dev1/run-03.out | 0 .../cuda-oom-retry-study/armB/dev1/run-04.err | 5 + .../cuda-oom-retry-study/armB/dev1/run-04.out | 0 .../cuda-oom-retry-study/armB/dev1/run-05.err | 5 + .../cuda-oom-retry-study/armB/dev1/run-05.out | 0 .../cuda-oom-retry-study/armB/dev1/run-06.err | 5 + .../cuda-oom-retry-study/armB/dev1/run-06.out | 0 .../cuda-oom-retry-study/armB/dev1/run-07.err | 5 + .../cuda-oom-retry-study/armB/dev1/run-07.out | 0 .../cuda-oom-retry-study/armB/dev1/run-08.err | 5 + .../cuda-oom-retry-study/armB/dev1/run-08.out | 0 .../cuda-oom-retry-study/armB/dev1/run-09.err | 5 + .../cuda-oom-retry-study/armB/dev1/run-09.out | 0 .../cuda-oom-retry-study/armB/dev1/run-10.err | 5 + .../cuda-oom-retry-study/armB/dev1/run-10.out | 0 .../cuda-oom-retry-study/armB/dev1/run-11.err | 5 + .../cuda-oom-retry-study/armB/dev1/run-11.out | 0 .../cuda-oom-retry-study/armB/dev1/run-12.err | 5 + .../cuda-oom-retry-study/armB/dev1/run-12.out | 0 .../cuda-oom-retry-study/armB/timeline.log | 97 ++++++++++++++ testdata/cuda-oom-retry-study/armB_edit.py | 49 +++++++ .../cuda-oom-retry-study/armC/dev0/run-01.err | 5 + .../cuda-oom-retry-study/armC/dev0/run-01.out | 0 .../cuda-oom-retry-study/armC/dev0/run-02.err | 5 + .../cuda-oom-retry-study/armC/dev0/run-02.out | 0 .../cuda-oom-retry-study/armC/dev0/run-03.err | 5 + .../cuda-oom-retry-study/armC/dev0/run-03.out | 0 .../cuda-oom-retry-study/armC/dev0/run-04.err | 5 + .../cuda-oom-retry-study/armC/dev0/run-04.out | 0 .../cuda-oom-retry-study/armC/dev0/run-05.err | 5 + .../cuda-oom-retry-study/armC/dev0/run-05.out | 0 .../cuda-oom-retry-study/armC/dev0/run-06.err | 5 + .../cuda-oom-retry-study/armC/dev0/run-06.out | 0 .../cuda-oom-retry-study/armC/dev0/run-07.err | 5 + .../cuda-oom-retry-study/armC/dev0/run-07.out | 0 .../cuda-oom-retry-study/armC/dev0/run-08.err | 5 + .../cuda-oom-retry-study/armC/dev0/run-08.out | 0 .../cuda-oom-retry-study/armC/dev0/run-09.err | 5 + .../cuda-oom-retry-study/armC/dev0/run-09.out | 0 .../cuda-oom-retry-study/armC/dev0/run-10.err | 5 + .../cuda-oom-retry-study/armC/dev0/run-10.out | 0 .../cuda-oom-retry-study/armC/dev0/run-11.err | 5 + .../cuda-oom-retry-study/armC/dev0/run-11.out | 0 .../cuda-oom-retry-study/armC/dev0/run-12.err | 5 + .../cuda-oom-retry-study/armC/dev0/run-12.out | 0 .../cuda-oom-retry-study/armC/dev1/run-01.err | 5 + .../cuda-oom-retry-study/armC/dev1/run-01.out | 0 .../cuda-oom-retry-study/armC/dev1/run-02.err | 5 + .../cuda-oom-retry-study/armC/dev1/run-02.out | 0 .../cuda-oom-retry-study/armC/dev1/run-03.err | 5 + .../cuda-oom-retry-study/armC/dev1/run-03.out | 0 .../cuda-oom-retry-study/armC/dev1/run-04.err | 5 + .../cuda-oom-retry-study/armC/dev1/run-04.out | 0 .../cuda-oom-retry-study/armC/dev1/run-05.err | 5 + .../cuda-oom-retry-study/armC/dev1/run-05.out | 0 .../cuda-oom-retry-study/armC/dev1/run-06.err | 5 + .../cuda-oom-retry-study/armC/dev1/run-06.out | 0 .../cuda-oom-retry-study/armC/dev1/run-07.err | 5 + .../cuda-oom-retry-study/armC/dev1/run-07.out | 0 .../cuda-oom-retry-study/armC/dev1/run-08.err | 5 + .../cuda-oom-retry-study/armC/dev1/run-08.out | 0 .../cuda-oom-retry-study/armC/dev1/run-09.err | 5 + .../cuda-oom-retry-study/armC/dev1/run-09.out | 0 .../cuda-oom-retry-study/armC/dev1/run-10.err | 5 + .../cuda-oom-retry-study/armC/dev1/run-10.out | 0 .../cuda-oom-retry-study/armC/dev1/run-11.err | 5 + .../cuda-oom-retry-study/armC/dev1/run-11.out | 0 .../cuda-oom-retry-study/armC/dev1/run-12.err | 5 + .../cuda-oom-retry-study/armC/dev1/run-12.out | 0 .../cuda-oom-retry-study/armC/timeline.log | 97 ++++++++++++++ .../cuda-oom-retry-study/armD/dev0/run-1.err | 6 + .../cuda-oom-retry-study/armD/dev0/run-1.out | 0 .../cuda-oom-retry-study/armD/dev0/run-2.err | 6 + .../cuda-oom-retry-study/armD/dev0/run-2.out | 0 .../cuda-oom-retry-study/armD/dev0/run-3.err | 6 + .../cuda-oom-retry-study/armD/dev0/run-3.out | 0 .../cuda-oom-retry-study/armD/dev0/run-4.err | 6 + .../cuda-oom-retry-study/armD/dev0/run-4.out | 0 .../cuda-oom-retry-study/armD/dev0/run-5.err | 6 + .../cuda-oom-retry-study/armD/dev0/run-5.out | 0 .../cuda-oom-retry-study/armD/dev0/run-6.err | 6 + .../cuda-oom-retry-study/armD/dev0/run-6.out | 0 .../cuda-oom-retry-study/armD/dev1/run-1.err | 5 + .../cuda-oom-retry-study/armD/dev1/run-1.out | 0 .../cuda-oom-retry-study/armD/dev1/run-2.err | 5 + .../cuda-oom-retry-study/armD/dev1/run-2.out | 0 .../cuda-oom-retry-study/armD/dev1/run-3.err | 5 + .../cuda-oom-retry-study/armD/dev1/run-3.out | 0 .../cuda-oom-retry-study/armD/dev1/run-4.err | 5 + .../cuda-oom-retry-study/armD/dev1/run-4.out | 0 .../cuda-oom-retry-study/armD/dev1/run-5.err | 5 + .../cuda-oom-retry-study/armD/dev1/run-5.out | 0 .../cuda-oom-retry-study/armD/dev1/run-6.err | 5 + .../cuda-oom-retry-study/armD/dev1/run-6.out | 0 .../cuda-oom-retry-study/armD/sampler.log | 121 ++++++++++++++++++ .../cuda-oom-retry-study/armD/timeline.log | 25 ++++ .../ilv/dev0/run-1-aslanded.err | 5 + .../ilv/dev0/run-1-aslanded.out | 0 .../ilv/dev0/run-1-noprobe.err | 6 + .../ilv/dev0/run-1-noprobe.out | 0 .../ilv/dev0/run-2-aslanded.err | 5 + .../ilv/dev0/run-2-aslanded.out | 0 .../ilv/dev0/run-2-noprobe.err | 6 + .../ilv/dev0/run-2-noprobe.out | 0 .../ilv/dev0/run-3-aslanded.err | 5 + .../ilv/dev0/run-3-aslanded.out | 0 .../ilv/dev0/run-3-noprobe.err | 6 + .../ilv/dev0/run-3-noprobe.out | 0 .../ilv/dev0/run-4-aslanded.err | 5 + .../ilv/dev0/run-4-aslanded.out | 0 .../ilv/dev0/run-4-noprobe.err | 6 + .../ilv/dev0/run-4-noprobe.out | 0 .../ilv/dev0/run-5-aslanded.err | 5 + .../ilv/dev0/run-5-aslanded.out | 0 .../ilv/dev0/run-5-noprobe.err | 6 + .../ilv/dev0/run-5-noprobe.out | 0 .../ilv/dev0/run-6-aslanded.err | 5 + .../ilv/dev0/run-6-aslanded.out | 0 .../ilv/dev0/run-6-noprobe.err | 6 + .../ilv/dev0/run-6-noprobe.out | 0 .../ilv/dev1/run-01-aslanded.err | 5 + .../ilv/dev1/run-01-aslanded.out | 0 .../ilv/dev1/run-01-noprobe.err | 5 + .../ilv/dev1/run-01-noprobe.out | 0 .../ilv/dev1/run-02-aslanded.err | 5 + .../ilv/dev1/run-02-aslanded.out | 0 .../ilv/dev1/run-02-noprobe.err | 5 + .../ilv/dev1/run-02-noprobe.out | 0 .../ilv/dev1/run-03-aslanded.err | 5 + .../ilv/dev1/run-03-aslanded.out | 0 .../ilv/dev1/run-03-noprobe.err | 5 + .../ilv/dev1/run-03-noprobe.out | 0 .../ilv/dev1/run-04-aslanded.err | 5 + .../ilv/dev1/run-04-aslanded.out | 0 .../ilv/dev1/run-04-noprobe.err | 5 + .../ilv/dev1/run-04-noprobe.out | 0 .../ilv/dev1/run-05-aslanded.err | 5 + .../ilv/dev1/run-05-aslanded.out | 0 .../ilv/dev1/run-05-noprobe.err | 5 + .../ilv/dev1/run-05-noprobe.out | 0 .../ilv/dev1/run-06-aslanded.err | 5 + .../ilv/dev1/run-06-aslanded.out | 0 .../ilv/dev1/run-06-noprobe.err | 5 + .../ilv/dev1/run-06-noprobe.out | 0 .../ilv/dev1/run-07-aslanded.err | 5 + .../ilv/dev1/run-07-aslanded.out | 0 .../ilv/dev1/run-07-noprobe.err | 5 + .../ilv/dev1/run-07-noprobe.out | 0 .../ilv/dev1/run-08-aslanded.err | 5 + .../ilv/dev1/run-08-aslanded.out | 0 .../ilv/dev1/run-08-noprobe.err | 5 + .../ilv/dev1/run-08-noprobe.out | 0 .../ilv/dev1/run-09-aslanded.err | 5 + .../ilv/dev1/run-09-aslanded.out | 0 .../ilv/dev1/run-09-noprobe.err | 5 + .../ilv/dev1/run-09-noprobe.out | 0 .../ilv/dev1/run-10-aslanded.err | 6 + .../ilv/dev1/run-10-aslanded.out | 0 .../ilv/dev1/run-10-noprobe.err | 5 + .../ilv/dev1/run-10-noprobe.out | 0 .../ilv/dev1/run-11-aslanded.err | 5 + .../ilv/dev1/run-11-aslanded.out | 0 .../ilv/dev1/run-11-noprobe.err | 5 + .../ilv/dev1/run-11-noprobe.out | 0 .../ilv/dev1/run-12-aslanded.err | 5 + .../ilv/dev1/run-12-aslanded.out | 0 .../ilv/dev1/run-12-noprobe.err | 5 + .../ilv/dev1/run-12-noprobe.out | 0 .../cuda-oom-retry-study/ilv/timeline.log | 73 +++++++++++ testdata/cuda-oom-retry-study/run_armA.sh | 23 ++++ testdata/cuda-oom-retry-study/run_armB.sh | 23 ++++ testdata/cuda-oom-retry-study/run_armC.sh | 23 ++++ testdata/cuda-oom-retry-study/run_armD.sh | 31 +++++ 252 files changed, 1343 insertions(+) create mode 100644 testdata/cuda-oom-retry-study/SUMMARY.md create mode 100644 testdata/cuda-oom-retry-study/armA/dev0/run-01.err create mode 100644 testdata/cuda-oom-retry-study/armA/dev0/run-01.out create mode 100644 testdata/cuda-oom-retry-study/armA/dev0/run-02.err create mode 100644 testdata/cuda-oom-retry-study/armA/dev0/run-02.out create mode 100644 testdata/cuda-oom-retry-study/armA/dev0/run-03.err create mode 100644 testdata/cuda-oom-retry-study/armA/dev0/run-03.out create mode 100644 testdata/cuda-oom-retry-study/armA/dev0/run-04.err create mode 100644 testdata/cuda-oom-retry-study/armA/dev0/run-04.out create mode 100644 testdata/cuda-oom-retry-study/armA/dev0/run-05.err create mode 100644 testdata/cuda-oom-retry-study/armA/dev0/run-05.out create mode 100644 testdata/cuda-oom-retry-study/armA/dev0/run-06.err create mode 100644 testdata/cuda-oom-retry-study/armA/dev0/run-06.out create mode 100644 testdata/cuda-oom-retry-study/armA/dev0/run-07.err create mode 100644 testdata/cuda-oom-retry-study/armA/dev0/run-07.out create mode 100644 testdata/cuda-oom-retry-study/armA/dev0/run-08.err create mode 100644 testdata/cuda-oom-retry-study/armA/dev0/run-08.out create mode 100644 testdata/cuda-oom-retry-study/armA/dev0/run-09.err create mode 100644 testdata/cuda-oom-retry-study/armA/dev0/run-09.out create mode 100644 testdata/cuda-oom-retry-study/armA/dev0/run-10.err create mode 100644 testdata/cuda-oom-retry-study/armA/dev0/run-10.out create mode 100644 testdata/cuda-oom-retry-study/armA/dev0/run-11.err create mode 100644 testdata/cuda-oom-retry-study/armA/dev0/run-11.out create mode 100644 testdata/cuda-oom-retry-study/armA/dev0/run-12.err create mode 100644 testdata/cuda-oom-retry-study/armA/dev0/run-12.out create mode 100644 testdata/cuda-oom-retry-study/armA/dev1/run-01.err create mode 100644 testdata/cuda-oom-retry-study/armA/dev1/run-01.out create mode 100644 testdata/cuda-oom-retry-study/armA/dev1/run-02.err create mode 100644 testdata/cuda-oom-retry-study/armA/dev1/run-02.out create mode 100644 testdata/cuda-oom-retry-study/armA/dev1/run-03.err create mode 100644 testdata/cuda-oom-retry-study/armA/dev1/run-03.out create mode 100644 testdata/cuda-oom-retry-study/armA/dev1/run-04.err create mode 100644 testdata/cuda-oom-retry-study/armA/dev1/run-04.out create mode 100644 testdata/cuda-oom-retry-study/armA/dev1/run-05.err create mode 100644 testdata/cuda-oom-retry-study/armA/dev1/run-05.out create mode 100644 testdata/cuda-oom-retry-study/armA/dev1/run-06.err create mode 100644 testdata/cuda-oom-retry-study/armA/dev1/run-06.out create mode 100644 testdata/cuda-oom-retry-study/armA/dev1/run-07.err create mode 100644 testdata/cuda-oom-retry-study/armA/dev1/run-07.out create mode 100644 testdata/cuda-oom-retry-study/armA/dev1/run-08.err create mode 100644 testdata/cuda-oom-retry-study/armA/dev1/run-08.out create mode 100644 testdata/cuda-oom-retry-study/armA/dev1/run-09.err create mode 100644 testdata/cuda-oom-retry-study/armA/dev1/run-09.out create mode 100644 testdata/cuda-oom-retry-study/armA/dev1/run-10.err create mode 100644 testdata/cuda-oom-retry-study/armA/dev1/run-10.out create mode 100644 testdata/cuda-oom-retry-study/armA/dev1/run-11.err create mode 100644 testdata/cuda-oom-retry-study/armA/dev1/run-11.out create mode 100644 testdata/cuda-oom-retry-study/armA/dev1/run-12.err create mode 100644 testdata/cuda-oom-retry-study/armA/dev1/run-12.out create mode 100644 testdata/cuda-oom-retry-study/armA/timeline.log create mode 100644 testdata/cuda-oom-retry-study/armB/dev0/run-01.err create mode 100644 testdata/cuda-oom-retry-study/armB/dev0/run-01.out create mode 100644 testdata/cuda-oom-retry-study/armB/dev0/run-02.err create mode 100644 testdata/cuda-oom-retry-study/armB/dev0/run-02.out create mode 100644 testdata/cuda-oom-retry-study/armB/dev0/run-03.err create mode 100644 testdata/cuda-oom-retry-study/armB/dev0/run-03.out create mode 100644 testdata/cuda-oom-retry-study/armB/dev0/run-04.err create mode 100644 testdata/cuda-oom-retry-study/armB/dev0/run-04.out create mode 100644 testdata/cuda-oom-retry-study/armB/dev0/run-05.err create mode 100644 testdata/cuda-oom-retry-study/armB/dev0/run-05.out create mode 100644 testdata/cuda-oom-retry-study/armB/dev0/run-06.err create mode 100644 testdata/cuda-oom-retry-study/armB/dev0/run-06.out create mode 100644 testdata/cuda-oom-retry-study/armB/dev0/run-07.err create mode 100644 testdata/cuda-oom-retry-study/armB/dev0/run-07.out create mode 100644 testdata/cuda-oom-retry-study/armB/dev0/run-08.err create mode 100644 testdata/cuda-oom-retry-study/armB/dev0/run-08.out create mode 100644 testdata/cuda-oom-retry-study/armB/dev0/run-09.err create mode 100644 testdata/cuda-oom-retry-study/armB/dev0/run-09.out create mode 100644 testdata/cuda-oom-retry-study/armB/dev0/run-10.err create mode 100644 testdata/cuda-oom-retry-study/armB/dev0/run-10.out create mode 100644 testdata/cuda-oom-retry-study/armB/dev0/run-11.err create mode 100644 testdata/cuda-oom-retry-study/armB/dev0/run-11.out create mode 100644 testdata/cuda-oom-retry-study/armB/dev0/run-12.err create mode 100644 testdata/cuda-oom-retry-study/armB/dev0/run-12.out create mode 100644 testdata/cuda-oom-retry-study/armB/dev1/run-01.err create mode 100644 testdata/cuda-oom-retry-study/armB/dev1/run-01.out create mode 100644 testdata/cuda-oom-retry-study/armB/dev1/run-02.err create mode 100644 testdata/cuda-oom-retry-study/armB/dev1/run-02.out create mode 100644 testdata/cuda-oom-retry-study/armB/dev1/run-03.err create mode 100644 testdata/cuda-oom-retry-study/armB/dev1/run-03.out create mode 100644 testdata/cuda-oom-retry-study/armB/dev1/run-04.err create mode 100644 testdata/cuda-oom-retry-study/armB/dev1/run-04.out create mode 100644 testdata/cuda-oom-retry-study/armB/dev1/run-05.err create mode 100644 testdata/cuda-oom-retry-study/armB/dev1/run-05.out create mode 100644 testdata/cuda-oom-retry-study/armB/dev1/run-06.err create mode 100644 testdata/cuda-oom-retry-study/armB/dev1/run-06.out create mode 100644 testdata/cuda-oom-retry-study/armB/dev1/run-07.err create mode 100644 testdata/cuda-oom-retry-study/armB/dev1/run-07.out create mode 100644 testdata/cuda-oom-retry-study/armB/dev1/run-08.err create mode 100644 testdata/cuda-oom-retry-study/armB/dev1/run-08.out create mode 100644 testdata/cuda-oom-retry-study/armB/dev1/run-09.err create mode 100644 testdata/cuda-oom-retry-study/armB/dev1/run-09.out create mode 100644 testdata/cuda-oom-retry-study/armB/dev1/run-10.err create mode 100644 testdata/cuda-oom-retry-study/armB/dev1/run-10.out create mode 100644 testdata/cuda-oom-retry-study/armB/dev1/run-11.err create mode 100644 testdata/cuda-oom-retry-study/armB/dev1/run-11.out create mode 100644 testdata/cuda-oom-retry-study/armB/dev1/run-12.err create mode 100644 testdata/cuda-oom-retry-study/armB/dev1/run-12.out create mode 100644 testdata/cuda-oom-retry-study/armB/timeline.log create mode 100644 testdata/cuda-oom-retry-study/armB_edit.py create mode 100644 testdata/cuda-oom-retry-study/armC/dev0/run-01.err create mode 100644 testdata/cuda-oom-retry-study/armC/dev0/run-01.out create mode 100644 testdata/cuda-oom-retry-study/armC/dev0/run-02.err create mode 100644 testdata/cuda-oom-retry-study/armC/dev0/run-02.out create mode 100644 testdata/cuda-oom-retry-study/armC/dev0/run-03.err create mode 100644 testdata/cuda-oom-retry-study/armC/dev0/run-03.out create mode 100644 testdata/cuda-oom-retry-study/armC/dev0/run-04.err create mode 100644 testdata/cuda-oom-retry-study/armC/dev0/run-04.out create mode 100644 testdata/cuda-oom-retry-study/armC/dev0/run-05.err create mode 100644 testdata/cuda-oom-retry-study/armC/dev0/run-05.out create mode 100644 testdata/cuda-oom-retry-study/armC/dev0/run-06.err create mode 100644 testdata/cuda-oom-retry-study/armC/dev0/run-06.out create mode 100644 testdata/cuda-oom-retry-study/armC/dev0/run-07.err create mode 100644 testdata/cuda-oom-retry-study/armC/dev0/run-07.out create mode 100644 testdata/cuda-oom-retry-study/armC/dev0/run-08.err create mode 100644 testdata/cuda-oom-retry-study/armC/dev0/run-08.out create mode 100644 testdata/cuda-oom-retry-study/armC/dev0/run-09.err create mode 100644 testdata/cuda-oom-retry-study/armC/dev0/run-09.out create mode 100644 testdata/cuda-oom-retry-study/armC/dev0/run-10.err create mode 100644 testdata/cuda-oom-retry-study/armC/dev0/run-10.out create mode 100644 testdata/cuda-oom-retry-study/armC/dev0/run-11.err create mode 100644 testdata/cuda-oom-retry-study/armC/dev0/run-11.out create mode 100644 testdata/cuda-oom-retry-study/armC/dev0/run-12.err create mode 100644 testdata/cuda-oom-retry-study/armC/dev0/run-12.out create mode 100644 testdata/cuda-oom-retry-study/armC/dev1/run-01.err create mode 100644 testdata/cuda-oom-retry-study/armC/dev1/run-01.out create mode 100644 testdata/cuda-oom-retry-study/armC/dev1/run-02.err create mode 100644 testdata/cuda-oom-retry-study/armC/dev1/run-02.out create mode 100644 testdata/cuda-oom-retry-study/armC/dev1/run-03.err create mode 100644 testdata/cuda-oom-retry-study/armC/dev1/run-03.out create mode 100644 testdata/cuda-oom-retry-study/armC/dev1/run-04.err create mode 100644 testdata/cuda-oom-retry-study/armC/dev1/run-04.out create mode 100644 testdata/cuda-oom-retry-study/armC/dev1/run-05.err create mode 100644 testdata/cuda-oom-retry-study/armC/dev1/run-05.out create mode 100644 testdata/cuda-oom-retry-study/armC/dev1/run-06.err create mode 100644 testdata/cuda-oom-retry-study/armC/dev1/run-06.out create mode 100644 testdata/cuda-oom-retry-study/armC/dev1/run-07.err create mode 100644 testdata/cuda-oom-retry-study/armC/dev1/run-07.out create mode 100644 testdata/cuda-oom-retry-study/armC/dev1/run-08.err create mode 100644 testdata/cuda-oom-retry-study/armC/dev1/run-08.out create mode 100644 testdata/cuda-oom-retry-study/armC/dev1/run-09.err create mode 100644 testdata/cuda-oom-retry-study/armC/dev1/run-09.out create mode 100644 testdata/cuda-oom-retry-study/armC/dev1/run-10.err create mode 100644 testdata/cuda-oom-retry-study/armC/dev1/run-10.out create mode 100644 testdata/cuda-oom-retry-study/armC/dev1/run-11.err create mode 100644 testdata/cuda-oom-retry-study/armC/dev1/run-11.out create mode 100644 testdata/cuda-oom-retry-study/armC/dev1/run-12.err create mode 100644 testdata/cuda-oom-retry-study/armC/dev1/run-12.out create mode 100644 testdata/cuda-oom-retry-study/armC/timeline.log create mode 100644 testdata/cuda-oom-retry-study/armD/dev0/run-1.err create mode 100644 testdata/cuda-oom-retry-study/armD/dev0/run-1.out create mode 100644 testdata/cuda-oom-retry-study/armD/dev0/run-2.err create mode 100644 testdata/cuda-oom-retry-study/armD/dev0/run-2.out create mode 100644 testdata/cuda-oom-retry-study/armD/dev0/run-3.err create mode 100644 testdata/cuda-oom-retry-study/armD/dev0/run-3.out create mode 100644 testdata/cuda-oom-retry-study/armD/dev0/run-4.err create mode 100644 testdata/cuda-oom-retry-study/armD/dev0/run-4.out create mode 100644 testdata/cuda-oom-retry-study/armD/dev0/run-5.err create mode 100644 testdata/cuda-oom-retry-study/armD/dev0/run-5.out create mode 100644 testdata/cuda-oom-retry-study/armD/dev0/run-6.err create mode 100644 testdata/cuda-oom-retry-study/armD/dev0/run-6.out create mode 100644 testdata/cuda-oom-retry-study/armD/dev1/run-1.err create mode 100644 testdata/cuda-oom-retry-study/armD/dev1/run-1.out create mode 100644 testdata/cuda-oom-retry-study/armD/dev1/run-2.err create mode 100644 testdata/cuda-oom-retry-study/armD/dev1/run-2.out create mode 100644 testdata/cuda-oom-retry-study/armD/dev1/run-3.err create mode 100644 testdata/cuda-oom-retry-study/armD/dev1/run-3.out create mode 100644 testdata/cuda-oom-retry-study/armD/dev1/run-4.err create mode 100644 testdata/cuda-oom-retry-study/armD/dev1/run-4.out create mode 100644 testdata/cuda-oom-retry-study/armD/dev1/run-5.err create mode 100644 testdata/cuda-oom-retry-study/armD/dev1/run-5.out create mode 100644 testdata/cuda-oom-retry-study/armD/dev1/run-6.err create mode 100644 testdata/cuda-oom-retry-study/armD/dev1/run-6.out create mode 100644 testdata/cuda-oom-retry-study/armD/sampler.log create mode 100644 testdata/cuda-oom-retry-study/armD/timeline.log create mode 100644 testdata/cuda-oom-retry-study/ilv/dev0/run-1-aslanded.err create mode 100644 testdata/cuda-oom-retry-study/ilv/dev0/run-1-aslanded.out create mode 100644 testdata/cuda-oom-retry-study/ilv/dev0/run-1-noprobe.err create mode 100644 testdata/cuda-oom-retry-study/ilv/dev0/run-1-noprobe.out create mode 100644 testdata/cuda-oom-retry-study/ilv/dev0/run-2-aslanded.err create mode 100644 testdata/cuda-oom-retry-study/ilv/dev0/run-2-aslanded.out create mode 100644 testdata/cuda-oom-retry-study/ilv/dev0/run-2-noprobe.err create mode 100644 testdata/cuda-oom-retry-study/ilv/dev0/run-2-noprobe.out create mode 100644 testdata/cuda-oom-retry-study/ilv/dev0/run-3-aslanded.err create mode 100644 testdata/cuda-oom-retry-study/ilv/dev0/run-3-aslanded.out create mode 100644 testdata/cuda-oom-retry-study/ilv/dev0/run-3-noprobe.err create mode 100644 testdata/cuda-oom-retry-study/ilv/dev0/run-3-noprobe.out create mode 100644 testdata/cuda-oom-retry-study/ilv/dev0/run-4-aslanded.err create mode 100644 testdata/cuda-oom-retry-study/ilv/dev0/run-4-aslanded.out create mode 100644 testdata/cuda-oom-retry-study/ilv/dev0/run-4-noprobe.err create mode 100644 testdata/cuda-oom-retry-study/ilv/dev0/run-4-noprobe.out create mode 100644 testdata/cuda-oom-retry-study/ilv/dev0/run-5-aslanded.err create mode 100644 testdata/cuda-oom-retry-study/ilv/dev0/run-5-aslanded.out create mode 100644 testdata/cuda-oom-retry-study/ilv/dev0/run-5-noprobe.err create mode 100644 testdata/cuda-oom-retry-study/ilv/dev0/run-5-noprobe.out create mode 100644 testdata/cuda-oom-retry-study/ilv/dev0/run-6-aslanded.err create mode 100644 testdata/cuda-oom-retry-study/ilv/dev0/run-6-aslanded.out create mode 100644 testdata/cuda-oom-retry-study/ilv/dev0/run-6-noprobe.err create mode 100644 testdata/cuda-oom-retry-study/ilv/dev0/run-6-noprobe.out create mode 100644 testdata/cuda-oom-retry-study/ilv/dev1/run-01-aslanded.err create mode 100644 testdata/cuda-oom-retry-study/ilv/dev1/run-01-aslanded.out create mode 100644 testdata/cuda-oom-retry-study/ilv/dev1/run-01-noprobe.err create mode 100644 testdata/cuda-oom-retry-study/ilv/dev1/run-01-noprobe.out create mode 100644 testdata/cuda-oom-retry-study/ilv/dev1/run-02-aslanded.err create mode 100644 testdata/cuda-oom-retry-study/ilv/dev1/run-02-aslanded.out create mode 100644 testdata/cuda-oom-retry-study/ilv/dev1/run-02-noprobe.err create mode 100644 testdata/cuda-oom-retry-study/ilv/dev1/run-02-noprobe.out create mode 100644 testdata/cuda-oom-retry-study/ilv/dev1/run-03-aslanded.err create mode 100644 testdata/cuda-oom-retry-study/ilv/dev1/run-03-aslanded.out create mode 100644 testdata/cuda-oom-retry-study/ilv/dev1/run-03-noprobe.err create mode 100644 testdata/cuda-oom-retry-study/ilv/dev1/run-03-noprobe.out create mode 100644 testdata/cuda-oom-retry-study/ilv/dev1/run-04-aslanded.err create mode 100644 testdata/cuda-oom-retry-study/ilv/dev1/run-04-aslanded.out create mode 100644 testdata/cuda-oom-retry-study/ilv/dev1/run-04-noprobe.err create mode 100644 testdata/cuda-oom-retry-study/ilv/dev1/run-04-noprobe.out create mode 100644 testdata/cuda-oom-retry-study/ilv/dev1/run-05-aslanded.err create mode 100644 testdata/cuda-oom-retry-study/ilv/dev1/run-05-aslanded.out create mode 100644 testdata/cuda-oom-retry-study/ilv/dev1/run-05-noprobe.err create mode 100644 testdata/cuda-oom-retry-study/ilv/dev1/run-05-noprobe.out create mode 100644 testdata/cuda-oom-retry-study/ilv/dev1/run-06-aslanded.err create mode 100644 testdata/cuda-oom-retry-study/ilv/dev1/run-06-aslanded.out create mode 100644 testdata/cuda-oom-retry-study/ilv/dev1/run-06-noprobe.err create mode 100644 testdata/cuda-oom-retry-study/ilv/dev1/run-06-noprobe.out create mode 100644 testdata/cuda-oom-retry-study/ilv/dev1/run-07-aslanded.err create mode 100644 testdata/cuda-oom-retry-study/ilv/dev1/run-07-aslanded.out create mode 100644 testdata/cuda-oom-retry-study/ilv/dev1/run-07-noprobe.err create mode 100644 testdata/cuda-oom-retry-study/ilv/dev1/run-07-noprobe.out create mode 100644 testdata/cuda-oom-retry-study/ilv/dev1/run-08-aslanded.err create mode 100644 testdata/cuda-oom-retry-study/ilv/dev1/run-08-aslanded.out create mode 100644 testdata/cuda-oom-retry-study/ilv/dev1/run-08-noprobe.err create mode 100644 testdata/cuda-oom-retry-study/ilv/dev1/run-08-noprobe.out create mode 100644 testdata/cuda-oom-retry-study/ilv/dev1/run-09-aslanded.err create mode 100644 testdata/cuda-oom-retry-study/ilv/dev1/run-09-aslanded.out create mode 100644 testdata/cuda-oom-retry-study/ilv/dev1/run-09-noprobe.err create mode 100644 testdata/cuda-oom-retry-study/ilv/dev1/run-09-noprobe.out create mode 100644 testdata/cuda-oom-retry-study/ilv/dev1/run-10-aslanded.err create mode 100644 testdata/cuda-oom-retry-study/ilv/dev1/run-10-aslanded.out create mode 100644 testdata/cuda-oom-retry-study/ilv/dev1/run-10-noprobe.err create mode 100644 testdata/cuda-oom-retry-study/ilv/dev1/run-10-noprobe.out create mode 100644 testdata/cuda-oom-retry-study/ilv/dev1/run-11-aslanded.err create mode 100644 testdata/cuda-oom-retry-study/ilv/dev1/run-11-aslanded.out create mode 100644 testdata/cuda-oom-retry-study/ilv/dev1/run-11-noprobe.err create mode 100644 testdata/cuda-oom-retry-study/ilv/dev1/run-11-noprobe.out create mode 100644 testdata/cuda-oom-retry-study/ilv/dev1/run-12-aslanded.err create mode 100644 testdata/cuda-oom-retry-study/ilv/dev1/run-12-aslanded.out create mode 100644 testdata/cuda-oom-retry-study/ilv/dev1/run-12-noprobe.err create mode 100644 testdata/cuda-oom-retry-study/ilv/dev1/run-12-noprobe.out create mode 100644 testdata/cuda-oom-retry-study/ilv/timeline.log create mode 100644 testdata/cuda-oom-retry-study/run_armA.sh create mode 100644 testdata/cuda-oom-retry-study/run_armB.sh create mode 100644 testdata/cuda-oom-retry-study/run_armC.sh create mode 100644 testdata/cuda-oom-retry-study/run_armD.sh diff --git a/testdata/cuda-oom-retry-study/SUMMARY.md b/testdata/cuda-oom-retry-study/SUMMARY.md new file mode 100644 index 0000000000..8774db6ad5 --- /dev/null +++ b/testdata/cuda-oom-retry-study/SUMMARY.md @@ -0,0 +1,47 @@ +# oom-r20 — #3320 landed test: 12-round rerun + probe A/B + reclamation latency + +Date: 2026-11-07 (07:15-08:25 UTC). Build: master `b38635f63`, #3320 test as landed +(unchanged except the stated arm edits), MinGW64 gcc 16.2.0 / nvcc 13.4, +Windows 11, driver 621.01. Devices: CUDA_VISIBLE_DEVICES 0 = RTX 5080, +1 = RTX 4060 Laptop (nvidia-smi -i order is the opposite: 0=4060, 1=5080). +Raw logs: `armA/ armB/ armC/ armD/ ilv/` (per-run .err with rc appended, +timeline.log with nvidia-smi brackets, armD/sampler.log 5 s cadence). + +## Arms + +| arm | binary | post-release probe | rounds | +| --- | --- | --- | --- | +| A | as-landed | one-shot cuMemAlloc/cuMemFree 2 MiB | 12 x 2 devices | +| B | no-probe | none (only change) | 12 x 2 | +| C | bounded poll | 20 ms poll, 60 s bound, timed | 12 x 2 | +| D | silent ladder | single probes after silent 1/5/15/45/105 s | 6 x 2 | +| ilv | interleaved A/B | as-landed vs no-probe, pairs back-to-back | 12 pairs 4060, 6 pairs 5080 | + +## Results + +- **A**: 5080 12/12 and 4060 12/12 fail at "device memory not yet returned after + the hog was freed" (`memory back 0`); retry assertion never fired. Slow window. +- **ilv**: 5080 6/6 pairs both arms fail (as-landed @not-yet-returned, no-probe + @retry); 4060 12 pairs: as-landed 11/12 pass + 1 transient refusal, no-probe + 12/12 pass. **No systematic probe perturbation.** +- **C**: 4060 12/12 `memory back 1 after 0.0 ms (0 probes)`; 5080 6/6 bound hit + (`after 60000 ms (1921-1933 probes)`) then 6/6 instant after mid-campaign + window flip. +- **D**: 4060 6/6 probe OK at the 1 s checkpoint, retry rc 0; 5080 6/6 refused at + every checkpoint (1/5/15/45/105 s), retry rc -12. +- **armD/sampler.log** (5 s): during 5080 slow-window waits, used ~15.8 GiB / + free ~150 MiB stays flat -> the driver holds the freed hog until process exit. + Fresh processes re-take the same 30.8 GiB hog immediately (every run, all arms). + +## Conclusion + +Windows slow window = the driver holds the released hog's memory for the +lifetime of the freeing process (>= 105 s observed, returned at exit); fast +window = instant return. The in-process one-shot probe is not a perturbation +source; a bounded wait + skip-with-reason is the only design that maps to the +observed behavior. Hog sizes 30.8/22.9 GiB (WDDM oversubscription) vs 710 MiB +on the maintainer's Linux 4090. + +Arm edits used (local-only, reverted after): armB/C/D variants of +`check_oom_and_memory_return()` in core/test/test_cuda_oom_pictures_released.c; +binaries saved as build/test/{aslanded,noprobe}.exe. diff --git a/testdata/cuda-oom-retry-study/armA/dev0/run-01.err b/testdata/cuda-oom-retry-study/armA/dev0/run-01.err new file mode 100644 index 0000000000..be95ebdd46 --- /dev/null +++ b/testdata/cuda-oom-retry-study/armA/dev0/run-01.err @@ -0,0 +1,5 @@ +test_out_of_memory_returns_an_error_and_keeps_the_pool: CUDA error at ../core/src/libvmaf.c:650: CUDA_ERROR_OUT_OF_MEMORY (2) in cuEventSynchronize(priv->cuda.ready) +[hog 30720 MiB, luma probe under hog 0, first rc -12, memory back 0] [first frame with no memory left: -12] fail +device memory not yet returned after the hog was freed +1 tests run, 1 failed +rc=1 diff --git a/testdata/cuda-oom-retry-study/armA/dev0/run-01.out b/testdata/cuda-oom-retry-study/armA/dev0/run-01.out new file mode 100644 index 0000000000..e69de29bb2 diff --git a/testdata/cuda-oom-retry-study/armA/dev0/run-02.err b/testdata/cuda-oom-retry-study/armA/dev0/run-02.err new file mode 100644 index 0000000000..485fb06b1f --- /dev/null +++ b/testdata/cuda-oom-retry-study/armA/dev0/run-02.err @@ -0,0 +1,5 @@ +test_out_of_memory_returns_an_error_and_keeps_the_pool: CUDA error at ../core/src/libvmaf.c:650: CUDA_ERROR_OUT_OF_MEMORY (2) in cuEventSynchronize(priv->cuda.ready) +[hog 30836 MiB, luma probe under hog 0, first rc -12, memory back 0] [first frame with no memory left: -12] fail +device memory not yet returned after the hog was freed +1 tests run, 1 failed +rc=1 diff --git a/testdata/cuda-oom-retry-study/armA/dev0/run-02.out b/testdata/cuda-oom-retry-study/armA/dev0/run-02.out new file mode 100644 index 0000000000..e69de29bb2 diff --git a/testdata/cuda-oom-retry-study/armA/dev0/run-03.err b/testdata/cuda-oom-retry-study/armA/dev0/run-03.err new file mode 100644 index 0000000000..485fb06b1f --- /dev/null +++ b/testdata/cuda-oom-retry-study/armA/dev0/run-03.err @@ -0,0 +1,5 @@ +test_out_of_memory_returns_an_error_and_keeps_the_pool: CUDA error at ../core/src/libvmaf.c:650: CUDA_ERROR_OUT_OF_MEMORY (2) in cuEventSynchronize(priv->cuda.ready) +[hog 30836 MiB, luma probe under hog 0, first rc -12, memory back 0] [first frame with no memory left: -12] fail +device memory not yet returned after the hog was freed +1 tests run, 1 failed +rc=1 diff --git a/testdata/cuda-oom-retry-study/armA/dev0/run-03.out b/testdata/cuda-oom-retry-study/armA/dev0/run-03.out new file mode 100644 index 0000000000..e69de29bb2 diff --git a/testdata/cuda-oom-retry-study/armA/dev0/run-04.err b/testdata/cuda-oom-retry-study/armA/dev0/run-04.err new file mode 100644 index 0000000000..485fb06b1f --- /dev/null +++ b/testdata/cuda-oom-retry-study/armA/dev0/run-04.err @@ -0,0 +1,5 @@ +test_out_of_memory_returns_an_error_and_keeps_the_pool: CUDA error at ../core/src/libvmaf.c:650: CUDA_ERROR_OUT_OF_MEMORY (2) in cuEventSynchronize(priv->cuda.ready) +[hog 30836 MiB, luma probe under hog 0, first rc -12, memory back 0] [first frame with no memory left: -12] fail +device memory not yet returned after the hog was freed +1 tests run, 1 failed +rc=1 diff --git a/testdata/cuda-oom-retry-study/armA/dev0/run-04.out b/testdata/cuda-oom-retry-study/armA/dev0/run-04.out new file mode 100644 index 0000000000..e69de29bb2 diff --git a/testdata/cuda-oom-retry-study/armA/dev0/run-05.err b/testdata/cuda-oom-retry-study/armA/dev0/run-05.err new file mode 100644 index 0000000000..485fb06b1f --- /dev/null +++ b/testdata/cuda-oom-retry-study/armA/dev0/run-05.err @@ -0,0 +1,5 @@ +test_out_of_memory_returns_an_error_and_keeps_the_pool: CUDA error at ../core/src/libvmaf.c:650: CUDA_ERROR_OUT_OF_MEMORY (2) in cuEventSynchronize(priv->cuda.ready) +[hog 30836 MiB, luma probe under hog 0, first rc -12, memory back 0] [first frame with no memory left: -12] fail +device memory not yet returned after the hog was freed +1 tests run, 1 failed +rc=1 diff --git a/testdata/cuda-oom-retry-study/armA/dev0/run-05.out b/testdata/cuda-oom-retry-study/armA/dev0/run-05.out new file mode 100644 index 0000000000..e69de29bb2 diff --git a/testdata/cuda-oom-retry-study/armA/dev0/run-06.err b/testdata/cuda-oom-retry-study/armA/dev0/run-06.err new file mode 100644 index 0000000000..485fb06b1f --- /dev/null +++ b/testdata/cuda-oom-retry-study/armA/dev0/run-06.err @@ -0,0 +1,5 @@ +test_out_of_memory_returns_an_error_and_keeps_the_pool: CUDA error at ../core/src/libvmaf.c:650: CUDA_ERROR_OUT_OF_MEMORY (2) in cuEventSynchronize(priv->cuda.ready) +[hog 30836 MiB, luma probe under hog 0, first rc -12, memory back 0] [first frame with no memory left: -12] fail +device memory not yet returned after the hog was freed +1 tests run, 1 failed +rc=1 diff --git a/testdata/cuda-oom-retry-study/armA/dev0/run-06.out b/testdata/cuda-oom-retry-study/armA/dev0/run-06.out new file mode 100644 index 0000000000..e69de29bb2 diff --git a/testdata/cuda-oom-retry-study/armA/dev0/run-07.err b/testdata/cuda-oom-retry-study/armA/dev0/run-07.err new file mode 100644 index 0000000000..485fb06b1f --- /dev/null +++ b/testdata/cuda-oom-retry-study/armA/dev0/run-07.err @@ -0,0 +1,5 @@ +test_out_of_memory_returns_an_error_and_keeps_the_pool: CUDA error at ../core/src/libvmaf.c:650: CUDA_ERROR_OUT_OF_MEMORY (2) in cuEventSynchronize(priv->cuda.ready) +[hog 30836 MiB, luma probe under hog 0, first rc -12, memory back 0] [first frame with no memory left: -12] fail +device memory not yet returned after the hog was freed +1 tests run, 1 failed +rc=1 diff --git a/testdata/cuda-oom-retry-study/armA/dev0/run-07.out b/testdata/cuda-oom-retry-study/armA/dev0/run-07.out new file mode 100644 index 0000000000..e69de29bb2 diff --git a/testdata/cuda-oom-retry-study/armA/dev0/run-08.err b/testdata/cuda-oom-retry-study/armA/dev0/run-08.err new file mode 100644 index 0000000000..485fb06b1f --- /dev/null +++ b/testdata/cuda-oom-retry-study/armA/dev0/run-08.err @@ -0,0 +1,5 @@ +test_out_of_memory_returns_an_error_and_keeps_the_pool: CUDA error at ../core/src/libvmaf.c:650: CUDA_ERROR_OUT_OF_MEMORY (2) in cuEventSynchronize(priv->cuda.ready) +[hog 30836 MiB, luma probe under hog 0, first rc -12, memory back 0] [first frame with no memory left: -12] fail +device memory not yet returned after the hog was freed +1 tests run, 1 failed +rc=1 diff --git a/testdata/cuda-oom-retry-study/armA/dev0/run-08.out b/testdata/cuda-oom-retry-study/armA/dev0/run-08.out new file mode 100644 index 0000000000..e69de29bb2 diff --git a/testdata/cuda-oom-retry-study/armA/dev0/run-09.err b/testdata/cuda-oom-retry-study/armA/dev0/run-09.err new file mode 100644 index 0000000000..485fb06b1f --- /dev/null +++ b/testdata/cuda-oom-retry-study/armA/dev0/run-09.err @@ -0,0 +1,5 @@ +test_out_of_memory_returns_an_error_and_keeps_the_pool: CUDA error at ../core/src/libvmaf.c:650: CUDA_ERROR_OUT_OF_MEMORY (2) in cuEventSynchronize(priv->cuda.ready) +[hog 30836 MiB, luma probe under hog 0, first rc -12, memory back 0] [first frame with no memory left: -12] fail +device memory not yet returned after the hog was freed +1 tests run, 1 failed +rc=1 diff --git a/testdata/cuda-oom-retry-study/armA/dev0/run-09.out b/testdata/cuda-oom-retry-study/armA/dev0/run-09.out new file mode 100644 index 0000000000..e69de29bb2 diff --git a/testdata/cuda-oom-retry-study/armA/dev0/run-10.err b/testdata/cuda-oom-retry-study/armA/dev0/run-10.err new file mode 100644 index 0000000000..485fb06b1f --- /dev/null +++ b/testdata/cuda-oom-retry-study/armA/dev0/run-10.err @@ -0,0 +1,5 @@ +test_out_of_memory_returns_an_error_and_keeps_the_pool: CUDA error at ../core/src/libvmaf.c:650: CUDA_ERROR_OUT_OF_MEMORY (2) in cuEventSynchronize(priv->cuda.ready) +[hog 30836 MiB, luma probe under hog 0, first rc -12, memory back 0] [first frame with no memory left: -12] fail +device memory not yet returned after the hog was freed +1 tests run, 1 failed +rc=1 diff --git a/testdata/cuda-oom-retry-study/armA/dev0/run-10.out b/testdata/cuda-oom-retry-study/armA/dev0/run-10.out new file mode 100644 index 0000000000..e69de29bb2 diff --git a/testdata/cuda-oom-retry-study/armA/dev0/run-11.err b/testdata/cuda-oom-retry-study/armA/dev0/run-11.err new file mode 100644 index 0000000000..485fb06b1f --- /dev/null +++ b/testdata/cuda-oom-retry-study/armA/dev0/run-11.err @@ -0,0 +1,5 @@ +test_out_of_memory_returns_an_error_and_keeps_the_pool: CUDA error at ../core/src/libvmaf.c:650: CUDA_ERROR_OUT_OF_MEMORY (2) in cuEventSynchronize(priv->cuda.ready) +[hog 30836 MiB, luma probe under hog 0, first rc -12, memory back 0] [first frame with no memory left: -12] fail +device memory not yet returned after the hog was freed +1 tests run, 1 failed +rc=1 diff --git a/testdata/cuda-oom-retry-study/armA/dev0/run-11.out b/testdata/cuda-oom-retry-study/armA/dev0/run-11.out new file mode 100644 index 0000000000..e69de29bb2 diff --git a/testdata/cuda-oom-retry-study/armA/dev0/run-12.err b/testdata/cuda-oom-retry-study/armA/dev0/run-12.err new file mode 100644 index 0000000000..485fb06b1f --- /dev/null +++ b/testdata/cuda-oom-retry-study/armA/dev0/run-12.err @@ -0,0 +1,5 @@ +test_out_of_memory_returns_an_error_and_keeps_the_pool: CUDA error at ../core/src/libvmaf.c:650: CUDA_ERROR_OUT_OF_MEMORY (2) in cuEventSynchronize(priv->cuda.ready) +[hog 30836 MiB, luma probe under hog 0, first rc -12, memory back 0] [first frame with no memory left: -12] fail +device memory not yet returned after the hog was freed +1 tests run, 1 failed +rc=1 diff --git a/testdata/cuda-oom-retry-study/armA/dev0/run-12.out b/testdata/cuda-oom-retry-study/armA/dev0/run-12.out new file mode 100644 index 0000000000..e69de29bb2 diff --git a/testdata/cuda-oom-retry-study/armA/dev1/run-01.err b/testdata/cuda-oom-retry-study/armA/dev1/run-01.err new file mode 100644 index 0000000000..941721e7a5 --- /dev/null +++ b/testdata/cuda-oom-retry-study/armA/dev1/run-01.err @@ -0,0 +1,6 @@ +test_out_of_memory_returns_an_error_and_keeps_the_pool: CUDA error at ../core/src/cuda/picture_cuda.c:356: CUDA_ERROR_OUT_OF_MEMORY (2) in cuMemAllocPitch((CUdeviceptr *)&pic->data[i], (size_t *)&pic->stride[i], (size_t)pic->w[i] * ((pic->bpc + 7) / 8), pic->h[i], 8 << hbd) +libvmaf ERROR problem during prepare_ring_buffer +[hog 22954 MiB, luma probe under hog 0, first rc -12, memory back 0] [first frame with no memory left: -12] fail +device memory not yet returned after the hog was freed +1 tests run, 1 failed +rc=1 diff --git a/testdata/cuda-oom-retry-study/armA/dev1/run-01.out b/testdata/cuda-oom-retry-study/armA/dev1/run-01.out new file mode 100644 index 0000000000..e69de29bb2 diff --git a/testdata/cuda-oom-retry-study/armA/dev1/run-02.err b/testdata/cuda-oom-retry-study/armA/dev1/run-02.err new file mode 100644 index 0000000000..941721e7a5 --- /dev/null +++ b/testdata/cuda-oom-retry-study/armA/dev1/run-02.err @@ -0,0 +1,6 @@ +test_out_of_memory_returns_an_error_and_keeps_the_pool: CUDA error at ../core/src/cuda/picture_cuda.c:356: CUDA_ERROR_OUT_OF_MEMORY (2) in cuMemAllocPitch((CUdeviceptr *)&pic->data[i], (size_t *)&pic->stride[i], (size_t)pic->w[i] * ((pic->bpc + 7) / 8), pic->h[i], 8 << hbd) +libvmaf ERROR problem during prepare_ring_buffer +[hog 22954 MiB, luma probe under hog 0, first rc -12, memory back 0] [first frame with no memory left: -12] fail +device memory not yet returned after the hog was freed +1 tests run, 1 failed +rc=1 diff --git a/testdata/cuda-oom-retry-study/armA/dev1/run-02.out b/testdata/cuda-oom-retry-study/armA/dev1/run-02.out new file mode 100644 index 0000000000..e69de29bb2 diff --git a/testdata/cuda-oom-retry-study/armA/dev1/run-03.err b/testdata/cuda-oom-retry-study/armA/dev1/run-03.err new file mode 100644 index 0000000000..941721e7a5 --- /dev/null +++ b/testdata/cuda-oom-retry-study/armA/dev1/run-03.err @@ -0,0 +1,6 @@ +test_out_of_memory_returns_an_error_and_keeps_the_pool: CUDA error at ../core/src/cuda/picture_cuda.c:356: CUDA_ERROR_OUT_OF_MEMORY (2) in cuMemAllocPitch((CUdeviceptr *)&pic->data[i], (size_t *)&pic->stride[i], (size_t)pic->w[i] * ((pic->bpc + 7) / 8), pic->h[i], 8 << hbd) +libvmaf ERROR problem during prepare_ring_buffer +[hog 22954 MiB, luma probe under hog 0, first rc -12, memory back 0] [first frame with no memory left: -12] fail +device memory not yet returned after the hog was freed +1 tests run, 1 failed +rc=1 diff --git a/testdata/cuda-oom-retry-study/armA/dev1/run-03.out b/testdata/cuda-oom-retry-study/armA/dev1/run-03.out new file mode 100644 index 0000000000..e69de29bb2 diff --git a/testdata/cuda-oom-retry-study/armA/dev1/run-04.err b/testdata/cuda-oom-retry-study/armA/dev1/run-04.err new file mode 100644 index 0000000000..941721e7a5 --- /dev/null +++ b/testdata/cuda-oom-retry-study/armA/dev1/run-04.err @@ -0,0 +1,6 @@ +test_out_of_memory_returns_an_error_and_keeps_the_pool: CUDA error at ../core/src/cuda/picture_cuda.c:356: CUDA_ERROR_OUT_OF_MEMORY (2) in cuMemAllocPitch((CUdeviceptr *)&pic->data[i], (size_t *)&pic->stride[i], (size_t)pic->w[i] * ((pic->bpc + 7) / 8), pic->h[i], 8 << hbd) +libvmaf ERROR problem during prepare_ring_buffer +[hog 22954 MiB, luma probe under hog 0, first rc -12, memory back 0] [first frame with no memory left: -12] fail +device memory not yet returned after the hog was freed +1 tests run, 1 failed +rc=1 diff --git a/testdata/cuda-oom-retry-study/armA/dev1/run-04.out b/testdata/cuda-oom-retry-study/armA/dev1/run-04.out new file mode 100644 index 0000000000..e69de29bb2 diff --git a/testdata/cuda-oom-retry-study/armA/dev1/run-05.err b/testdata/cuda-oom-retry-study/armA/dev1/run-05.err new file mode 100644 index 0000000000..941721e7a5 --- /dev/null +++ b/testdata/cuda-oom-retry-study/armA/dev1/run-05.err @@ -0,0 +1,6 @@ +test_out_of_memory_returns_an_error_and_keeps_the_pool: CUDA error at ../core/src/cuda/picture_cuda.c:356: CUDA_ERROR_OUT_OF_MEMORY (2) in cuMemAllocPitch((CUdeviceptr *)&pic->data[i], (size_t *)&pic->stride[i], (size_t)pic->w[i] * ((pic->bpc + 7) / 8), pic->h[i], 8 << hbd) +libvmaf ERROR problem during prepare_ring_buffer +[hog 22954 MiB, luma probe under hog 0, first rc -12, memory back 0] [first frame with no memory left: -12] fail +device memory not yet returned after the hog was freed +1 tests run, 1 failed +rc=1 diff --git a/testdata/cuda-oom-retry-study/armA/dev1/run-05.out b/testdata/cuda-oom-retry-study/armA/dev1/run-05.out new file mode 100644 index 0000000000..e69de29bb2 diff --git a/testdata/cuda-oom-retry-study/armA/dev1/run-06.err b/testdata/cuda-oom-retry-study/armA/dev1/run-06.err new file mode 100644 index 0000000000..941721e7a5 --- /dev/null +++ b/testdata/cuda-oom-retry-study/armA/dev1/run-06.err @@ -0,0 +1,6 @@ +test_out_of_memory_returns_an_error_and_keeps_the_pool: CUDA error at ../core/src/cuda/picture_cuda.c:356: CUDA_ERROR_OUT_OF_MEMORY (2) in cuMemAllocPitch((CUdeviceptr *)&pic->data[i], (size_t *)&pic->stride[i], (size_t)pic->w[i] * ((pic->bpc + 7) / 8), pic->h[i], 8 << hbd) +libvmaf ERROR problem during prepare_ring_buffer +[hog 22954 MiB, luma probe under hog 0, first rc -12, memory back 0] [first frame with no memory left: -12] fail +device memory not yet returned after the hog was freed +1 tests run, 1 failed +rc=1 diff --git a/testdata/cuda-oom-retry-study/armA/dev1/run-06.out b/testdata/cuda-oom-retry-study/armA/dev1/run-06.out new file mode 100644 index 0000000000..e69de29bb2 diff --git a/testdata/cuda-oom-retry-study/armA/dev1/run-07.err b/testdata/cuda-oom-retry-study/armA/dev1/run-07.err new file mode 100644 index 0000000000..941721e7a5 --- /dev/null +++ b/testdata/cuda-oom-retry-study/armA/dev1/run-07.err @@ -0,0 +1,6 @@ +test_out_of_memory_returns_an_error_and_keeps_the_pool: CUDA error at ../core/src/cuda/picture_cuda.c:356: CUDA_ERROR_OUT_OF_MEMORY (2) in cuMemAllocPitch((CUdeviceptr *)&pic->data[i], (size_t *)&pic->stride[i], (size_t)pic->w[i] * ((pic->bpc + 7) / 8), pic->h[i], 8 << hbd) +libvmaf ERROR problem during prepare_ring_buffer +[hog 22954 MiB, luma probe under hog 0, first rc -12, memory back 0] [first frame with no memory left: -12] fail +device memory not yet returned after the hog was freed +1 tests run, 1 failed +rc=1 diff --git a/testdata/cuda-oom-retry-study/armA/dev1/run-07.out b/testdata/cuda-oom-retry-study/armA/dev1/run-07.out new file mode 100644 index 0000000000..e69de29bb2 diff --git a/testdata/cuda-oom-retry-study/armA/dev1/run-08.err b/testdata/cuda-oom-retry-study/armA/dev1/run-08.err new file mode 100644 index 0000000000..941721e7a5 --- /dev/null +++ b/testdata/cuda-oom-retry-study/armA/dev1/run-08.err @@ -0,0 +1,6 @@ +test_out_of_memory_returns_an_error_and_keeps_the_pool: CUDA error at ../core/src/cuda/picture_cuda.c:356: CUDA_ERROR_OUT_OF_MEMORY (2) in cuMemAllocPitch((CUdeviceptr *)&pic->data[i], (size_t *)&pic->stride[i], (size_t)pic->w[i] * ((pic->bpc + 7) / 8), pic->h[i], 8 << hbd) +libvmaf ERROR problem during prepare_ring_buffer +[hog 22954 MiB, luma probe under hog 0, first rc -12, memory back 0] [first frame with no memory left: -12] fail +device memory not yet returned after the hog was freed +1 tests run, 1 failed +rc=1 diff --git a/testdata/cuda-oom-retry-study/armA/dev1/run-08.out b/testdata/cuda-oom-retry-study/armA/dev1/run-08.out new file mode 100644 index 0000000000..e69de29bb2 diff --git a/testdata/cuda-oom-retry-study/armA/dev1/run-09.err b/testdata/cuda-oom-retry-study/armA/dev1/run-09.err new file mode 100644 index 0000000000..941721e7a5 --- /dev/null +++ b/testdata/cuda-oom-retry-study/armA/dev1/run-09.err @@ -0,0 +1,6 @@ +test_out_of_memory_returns_an_error_and_keeps_the_pool: CUDA error at ../core/src/cuda/picture_cuda.c:356: CUDA_ERROR_OUT_OF_MEMORY (2) in cuMemAllocPitch((CUdeviceptr *)&pic->data[i], (size_t *)&pic->stride[i], (size_t)pic->w[i] * ((pic->bpc + 7) / 8), pic->h[i], 8 << hbd) +libvmaf ERROR problem during prepare_ring_buffer +[hog 22954 MiB, luma probe under hog 0, first rc -12, memory back 0] [first frame with no memory left: -12] fail +device memory not yet returned after the hog was freed +1 tests run, 1 failed +rc=1 diff --git a/testdata/cuda-oom-retry-study/armA/dev1/run-09.out b/testdata/cuda-oom-retry-study/armA/dev1/run-09.out new file mode 100644 index 0000000000..e69de29bb2 diff --git a/testdata/cuda-oom-retry-study/armA/dev1/run-10.err b/testdata/cuda-oom-retry-study/armA/dev1/run-10.err new file mode 100644 index 0000000000..941721e7a5 --- /dev/null +++ b/testdata/cuda-oom-retry-study/armA/dev1/run-10.err @@ -0,0 +1,6 @@ +test_out_of_memory_returns_an_error_and_keeps_the_pool: CUDA error at ../core/src/cuda/picture_cuda.c:356: CUDA_ERROR_OUT_OF_MEMORY (2) in cuMemAllocPitch((CUdeviceptr *)&pic->data[i], (size_t *)&pic->stride[i], (size_t)pic->w[i] * ((pic->bpc + 7) / 8), pic->h[i], 8 << hbd) +libvmaf ERROR problem during prepare_ring_buffer +[hog 22954 MiB, luma probe under hog 0, first rc -12, memory back 0] [first frame with no memory left: -12] fail +device memory not yet returned after the hog was freed +1 tests run, 1 failed +rc=1 diff --git a/testdata/cuda-oom-retry-study/armA/dev1/run-10.out b/testdata/cuda-oom-retry-study/armA/dev1/run-10.out new file mode 100644 index 0000000000..e69de29bb2 diff --git a/testdata/cuda-oom-retry-study/armA/dev1/run-11.err b/testdata/cuda-oom-retry-study/armA/dev1/run-11.err new file mode 100644 index 0000000000..941721e7a5 --- /dev/null +++ b/testdata/cuda-oom-retry-study/armA/dev1/run-11.err @@ -0,0 +1,6 @@ +test_out_of_memory_returns_an_error_and_keeps_the_pool: CUDA error at ../core/src/cuda/picture_cuda.c:356: CUDA_ERROR_OUT_OF_MEMORY (2) in cuMemAllocPitch((CUdeviceptr *)&pic->data[i], (size_t *)&pic->stride[i], (size_t)pic->w[i] * ((pic->bpc + 7) / 8), pic->h[i], 8 << hbd) +libvmaf ERROR problem during prepare_ring_buffer +[hog 22954 MiB, luma probe under hog 0, first rc -12, memory back 0] [first frame with no memory left: -12] fail +device memory not yet returned after the hog was freed +1 tests run, 1 failed +rc=1 diff --git a/testdata/cuda-oom-retry-study/armA/dev1/run-11.out b/testdata/cuda-oom-retry-study/armA/dev1/run-11.out new file mode 100644 index 0000000000..e69de29bb2 diff --git a/testdata/cuda-oom-retry-study/armA/dev1/run-12.err b/testdata/cuda-oom-retry-study/armA/dev1/run-12.err new file mode 100644 index 0000000000..941721e7a5 --- /dev/null +++ b/testdata/cuda-oom-retry-study/armA/dev1/run-12.err @@ -0,0 +1,6 @@ +test_out_of_memory_returns_an_error_and_keeps_the_pool: CUDA error at ../core/src/cuda/picture_cuda.c:356: CUDA_ERROR_OUT_OF_MEMORY (2) in cuMemAllocPitch((CUdeviceptr *)&pic->data[i], (size_t *)&pic->stride[i], (size_t)pic->w[i] * ((pic->bpc + 7) / 8), pic->h[i], 8 << hbd) +libvmaf ERROR problem during prepare_ring_buffer +[hog 22954 MiB, luma probe under hog 0, first rc -12, memory back 0] [first frame with no memory left: -12] fail +device memory not yet returned after the hog was freed +1 tests run, 1 failed +rc=1 diff --git a/testdata/cuda-oom-retry-study/armA/dev1/run-12.out b/testdata/cuda-oom-retry-study/armA/dev1/run-12.out new file mode 100644 index 0000000000..e69de29bb2 diff --git a/testdata/cuda-oom-retry-study/armA/timeline.log b/testdata/cuda-oom-retry-study/armA/timeline.log new file mode 100644 index 0000000000..1c99a89dde --- /dev/null +++ b/testdata/cuda-oom-retry-study/armA/timeline.log @@ -0,0 +1,97 @@ +=== armA dev0 run01 start 2026-11-07T07:25:31Z +0 MiB, 0 % +=== armA dev0 run01 end 2026-11-07T07:25:41Z +0 MiB, 0 % +=== armA dev0 run02 start 2026-11-07T07:25:41Z +0 MiB, 0 % +=== armA dev0 run02 end 2026-11-07T07:25:48Z +0 MiB, 0 % +=== armA dev0 run03 start 2026-11-07T07:25:48Z +0 MiB, 0 % +=== armA dev0 run03 end 2026-11-07T07:25:52Z +0 MiB, 0 % +=== armA dev0 run04 start 2026-11-07T07:25:52Z +0 MiB, 0 % +=== armA dev0 run04 end 2026-11-07T07:25:57Z +0 MiB, 0 % +=== armA dev0 run05 start 2026-11-07T07:25:57Z +0 MiB, 0 % +=== armA dev0 run05 end 2026-11-07T07:26:02Z +0 MiB, 0 % +=== armA dev0 run06 start 2026-11-07T07:26:02Z +0 MiB, 0 % +=== armA dev0 run06 end 2026-11-07T07:26:07Z +0 MiB, 0 % +=== armA dev0 run07 start 2026-11-07T07:26:07Z +0 MiB, 0 % +=== armA dev0 run07 end 2026-11-07T07:26:13Z +0 MiB, 0 % +=== armA dev0 run08 start 2026-11-07T07:26:13Z +0 MiB, 0 % +=== armA dev0 run08 end 2026-11-07T07:26:18Z +0 MiB, 0 % +=== armA dev0 run09 start 2026-11-07T07:26:18Z +0 MiB, 0 % +=== armA dev0 run09 end 2026-11-07T07:26:23Z +0 MiB, 0 % +=== armA dev0 run10 start 2026-11-07T07:26:23Z +0 MiB, 0 % +=== armA dev0 run10 end 2026-11-07T07:26:27Z +0 MiB, 0 % +=== armA dev0 run11 start 2026-11-07T07:26:27Z +0 MiB, 0 % +=== armA dev0 run11 end 2026-11-07T07:26:32Z +0 MiB, 0 % +=== armA dev0 run12 start 2026-11-07T07:26:32Z +0 MiB, 0 % +=== armA dev0 run12 end 2026-11-07T07:26:37Z +0 MiB, 0 % +=== armA dev1 run01 start 2026-11-07T07:26:37Z +567 MiB, 24 % +=== armA dev1 run01 end 2026-11-07T07:26:41Z +644 MiB, 90 % +=== armA dev1 run02 start 2026-11-07T07:26:41Z +644 MiB, 63 % +=== armA dev1 run02 end 2026-11-07T07:26:44Z +802 MiB, 35 % +=== armA dev1 run03 start 2026-11-07T07:26:44Z +802 MiB, 35 % +=== armA dev1 run03 end 2026-11-07T07:26:48Z +789 MiB, 98 % +=== armA dev1 run04 start 2026-11-07T07:26:48Z +789 MiB, 98 % +=== armA dev1 run04 end 2026-11-07T07:26:51Z +789 MiB, 20 % +=== armA dev1 run05 start 2026-11-07T07:26:51Z +789 MiB, 20 % +=== armA dev1 run05 end 2026-11-07T07:26:55Z +789 MiB, 21 % +=== armA dev1 run06 start 2026-11-07T07:26:55Z +789 MiB, 21 % +=== armA dev1 run06 end 2026-11-07T07:26:58Z +789 MiB, 67 % +=== armA dev1 run07 start 2026-11-07T07:26:58Z +789 MiB, 67 % +=== armA dev1 run07 end 2026-11-07T07:27:01Z +789 MiB, 19 % +=== armA dev1 run08 start 2026-11-07T07:27:01Z +789 MiB, 19 % +=== armA dev1 run08 end 2026-11-07T07:27:04Z +789 MiB, 20 % +=== armA dev1 run09 start 2026-11-07T07:27:04Z +789 MiB, 20 % +=== armA dev1 run09 end 2026-11-07T07:27:07Z +789 MiB, 20 % +=== armA dev1 run10 start 2026-11-07T07:27:08Z +789 MiB, 91 % +=== armA dev1 run10 end 2026-11-07T07:27:11Z +789 MiB, 20 % +=== armA dev1 run11 start 2026-11-07T07:27:11Z +789 MiB, 20 % +=== armA dev1 run11 end 2026-11-07T07:27:14Z +789 MiB, 21 % +=== armA dev1 run12 start 2026-11-07T07:27:14Z +789 MiB, 21 % +=== armA dev1 run12 end 2026-11-07T07:27:17Z +794 MiB, 21 % +ARM-A-DONE 2026-11-07T07:27:17Z diff --git a/testdata/cuda-oom-retry-study/armB/dev0/run-01.err b/testdata/cuda-oom-retry-study/armB/dev0/run-01.err new file mode 100644 index 0000000000..41b5797468 --- /dev/null +++ b/testdata/cuda-oom-retry-study/armB/dev0/run-01.err @@ -0,0 +1,6 @@ +test_out_of_memory_returns_an_error_and_keeps_the_pool: CUDA error at ../core/src/libvmaf.c:650: CUDA_ERROR_OUT_OF_MEMORY (2) in cuEventSynchronize(priv->cuda.ready) +[hog 30792 MiB, luma probe under hog 0, first rc -12, memory back n/a] [first frame with no memory left: -12] CUDA error at ../core/src/libvmaf.c:650: CUDA_ERROR_OUT_OF_MEMORY (2) in cuEventSynchronize(priv->cuda.ready) +[retry rc -12] fail +the same index does not score once memory is back +1 tests run, 1 failed +rc=1 diff --git a/testdata/cuda-oom-retry-study/armB/dev0/run-01.out b/testdata/cuda-oom-retry-study/armB/dev0/run-01.out new file mode 100644 index 0000000000..e69de29bb2 diff --git a/testdata/cuda-oom-retry-study/armB/dev0/run-02.err b/testdata/cuda-oom-retry-study/armB/dev0/run-02.err new file mode 100644 index 0000000000..a67caafa5c --- /dev/null +++ b/testdata/cuda-oom-retry-study/armB/dev0/run-02.err @@ -0,0 +1,6 @@ +test_out_of_memory_returns_an_error_and_keeps_the_pool: CUDA error at ../core/src/libvmaf.c:650: CUDA_ERROR_OUT_OF_MEMORY (2) in cuEventSynchronize(priv->cuda.ready) +[hog 30836 MiB, luma probe under hog 0, first rc -12, memory back n/a] [first frame with no memory left: -12] CUDA error at ../core/src/libvmaf.c:650: CUDA_ERROR_OUT_OF_MEMORY (2) in cuEventSynchronize(priv->cuda.ready) +[retry rc -12] fail +the same index does not score once memory is back +1 tests run, 1 failed +rc=1 diff --git a/testdata/cuda-oom-retry-study/armB/dev0/run-02.out b/testdata/cuda-oom-retry-study/armB/dev0/run-02.out new file mode 100644 index 0000000000..e69de29bb2 diff --git a/testdata/cuda-oom-retry-study/armB/dev0/run-03.err b/testdata/cuda-oom-retry-study/armB/dev0/run-03.err new file mode 100644 index 0000000000..de6e70bc25 --- /dev/null +++ b/testdata/cuda-oom-retry-study/armB/dev0/run-03.err @@ -0,0 +1,6 @@ +test_out_of_memory_returns_an_error_and_keeps_the_pool: CUDA error at ../core/src/libvmaf.c:650: CUDA_ERROR_OUT_OF_MEMORY (2) in cuEventSynchronize(priv->cuda.ready) +[hog 30720 MiB, luma probe under hog 0, first rc -12, memory back n/a] [first frame with no memory left: -12] CUDA error at ../core/src/libvmaf.c:650: CUDA_ERROR_OUT_OF_MEMORY (2) in cuEventSynchronize(priv->cuda.ready) +[retry rc -12] fail +the same index does not score once memory is back +1 tests run, 1 failed +rc=1 diff --git a/testdata/cuda-oom-retry-study/armB/dev0/run-03.out b/testdata/cuda-oom-retry-study/armB/dev0/run-03.out new file mode 100644 index 0000000000..e69de29bb2 diff --git a/testdata/cuda-oom-retry-study/armB/dev0/run-04.err b/testdata/cuda-oom-retry-study/armB/dev0/run-04.err new file mode 100644 index 0000000000..a67caafa5c --- /dev/null +++ b/testdata/cuda-oom-retry-study/armB/dev0/run-04.err @@ -0,0 +1,6 @@ +test_out_of_memory_returns_an_error_and_keeps_the_pool: CUDA error at ../core/src/libvmaf.c:650: CUDA_ERROR_OUT_OF_MEMORY (2) in cuEventSynchronize(priv->cuda.ready) +[hog 30836 MiB, luma probe under hog 0, first rc -12, memory back n/a] [first frame with no memory left: -12] CUDA error at ../core/src/libvmaf.c:650: CUDA_ERROR_OUT_OF_MEMORY (2) in cuEventSynchronize(priv->cuda.ready) +[retry rc -12] fail +the same index does not score once memory is back +1 tests run, 1 failed +rc=1 diff --git a/testdata/cuda-oom-retry-study/armB/dev0/run-04.out b/testdata/cuda-oom-retry-study/armB/dev0/run-04.out new file mode 100644 index 0000000000..e69de29bb2 diff --git a/testdata/cuda-oom-retry-study/armB/dev0/run-05.err b/testdata/cuda-oom-retry-study/armB/dev0/run-05.err new file mode 100644 index 0000000000..a67caafa5c --- /dev/null +++ b/testdata/cuda-oom-retry-study/armB/dev0/run-05.err @@ -0,0 +1,6 @@ +test_out_of_memory_returns_an_error_and_keeps_the_pool: CUDA error at ../core/src/libvmaf.c:650: CUDA_ERROR_OUT_OF_MEMORY (2) in cuEventSynchronize(priv->cuda.ready) +[hog 30836 MiB, luma probe under hog 0, first rc -12, memory back n/a] [first frame with no memory left: -12] CUDA error at ../core/src/libvmaf.c:650: CUDA_ERROR_OUT_OF_MEMORY (2) in cuEventSynchronize(priv->cuda.ready) +[retry rc -12] fail +the same index does not score once memory is back +1 tests run, 1 failed +rc=1 diff --git a/testdata/cuda-oom-retry-study/armB/dev0/run-05.out b/testdata/cuda-oom-retry-study/armB/dev0/run-05.out new file mode 100644 index 0000000000..e69de29bb2 diff --git a/testdata/cuda-oom-retry-study/armB/dev0/run-06.err b/testdata/cuda-oom-retry-study/armB/dev0/run-06.err new file mode 100644 index 0000000000..a67caafa5c --- /dev/null +++ b/testdata/cuda-oom-retry-study/armB/dev0/run-06.err @@ -0,0 +1,6 @@ +test_out_of_memory_returns_an_error_and_keeps_the_pool: CUDA error at ../core/src/libvmaf.c:650: CUDA_ERROR_OUT_OF_MEMORY (2) in cuEventSynchronize(priv->cuda.ready) +[hog 30836 MiB, luma probe under hog 0, first rc -12, memory back n/a] [first frame with no memory left: -12] CUDA error at ../core/src/libvmaf.c:650: CUDA_ERROR_OUT_OF_MEMORY (2) in cuEventSynchronize(priv->cuda.ready) +[retry rc -12] fail +the same index does not score once memory is back +1 tests run, 1 failed +rc=1 diff --git a/testdata/cuda-oom-retry-study/armB/dev0/run-06.out b/testdata/cuda-oom-retry-study/armB/dev0/run-06.out new file mode 100644 index 0000000000..e69de29bb2 diff --git a/testdata/cuda-oom-retry-study/armB/dev0/run-07.err b/testdata/cuda-oom-retry-study/armB/dev0/run-07.err new file mode 100644 index 0000000000..de6e70bc25 --- /dev/null +++ b/testdata/cuda-oom-retry-study/armB/dev0/run-07.err @@ -0,0 +1,6 @@ +test_out_of_memory_returns_an_error_and_keeps_the_pool: CUDA error at ../core/src/libvmaf.c:650: CUDA_ERROR_OUT_OF_MEMORY (2) in cuEventSynchronize(priv->cuda.ready) +[hog 30720 MiB, luma probe under hog 0, first rc -12, memory back n/a] [first frame with no memory left: -12] CUDA error at ../core/src/libvmaf.c:650: CUDA_ERROR_OUT_OF_MEMORY (2) in cuEventSynchronize(priv->cuda.ready) +[retry rc -12] fail +the same index does not score once memory is back +1 tests run, 1 failed +rc=1 diff --git a/testdata/cuda-oom-retry-study/armB/dev0/run-07.out b/testdata/cuda-oom-retry-study/armB/dev0/run-07.out new file mode 100644 index 0000000000..e69de29bb2 diff --git a/testdata/cuda-oom-retry-study/armB/dev0/run-08.err b/testdata/cuda-oom-retry-study/armB/dev0/run-08.err new file mode 100644 index 0000000000..a67caafa5c --- /dev/null +++ b/testdata/cuda-oom-retry-study/armB/dev0/run-08.err @@ -0,0 +1,6 @@ +test_out_of_memory_returns_an_error_and_keeps_the_pool: CUDA error at ../core/src/libvmaf.c:650: CUDA_ERROR_OUT_OF_MEMORY (2) in cuEventSynchronize(priv->cuda.ready) +[hog 30836 MiB, luma probe under hog 0, first rc -12, memory back n/a] [first frame with no memory left: -12] CUDA error at ../core/src/libvmaf.c:650: CUDA_ERROR_OUT_OF_MEMORY (2) in cuEventSynchronize(priv->cuda.ready) +[retry rc -12] fail +the same index does not score once memory is back +1 tests run, 1 failed +rc=1 diff --git a/testdata/cuda-oom-retry-study/armB/dev0/run-08.out b/testdata/cuda-oom-retry-study/armB/dev0/run-08.out new file mode 100644 index 0000000000..e69de29bb2 diff --git a/testdata/cuda-oom-retry-study/armB/dev0/run-09.err b/testdata/cuda-oom-retry-study/armB/dev0/run-09.err new file mode 100644 index 0000000000..a67caafa5c --- /dev/null +++ b/testdata/cuda-oom-retry-study/armB/dev0/run-09.err @@ -0,0 +1,6 @@ +test_out_of_memory_returns_an_error_and_keeps_the_pool: CUDA error at ../core/src/libvmaf.c:650: CUDA_ERROR_OUT_OF_MEMORY (2) in cuEventSynchronize(priv->cuda.ready) +[hog 30836 MiB, luma probe under hog 0, first rc -12, memory back n/a] [first frame with no memory left: -12] CUDA error at ../core/src/libvmaf.c:650: CUDA_ERROR_OUT_OF_MEMORY (2) in cuEventSynchronize(priv->cuda.ready) +[retry rc -12] fail +the same index does not score once memory is back +1 tests run, 1 failed +rc=1 diff --git a/testdata/cuda-oom-retry-study/armB/dev0/run-09.out b/testdata/cuda-oom-retry-study/armB/dev0/run-09.out new file mode 100644 index 0000000000..e69de29bb2 diff --git a/testdata/cuda-oom-retry-study/armB/dev0/run-10.err b/testdata/cuda-oom-retry-study/armB/dev0/run-10.err new file mode 100644 index 0000000000..a67caafa5c --- /dev/null +++ b/testdata/cuda-oom-retry-study/armB/dev0/run-10.err @@ -0,0 +1,6 @@ +test_out_of_memory_returns_an_error_and_keeps_the_pool: CUDA error at ../core/src/libvmaf.c:650: CUDA_ERROR_OUT_OF_MEMORY (2) in cuEventSynchronize(priv->cuda.ready) +[hog 30836 MiB, luma probe under hog 0, first rc -12, memory back n/a] [first frame with no memory left: -12] CUDA error at ../core/src/libvmaf.c:650: CUDA_ERROR_OUT_OF_MEMORY (2) in cuEventSynchronize(priv->cuda.ready) +[retry rc -12] fail +the same index does not score once memory is back +1 tests run, 1 failed +rc=1 diff --git a/testdata/cuda-oom-retry-study/armB/dev0/run-10.out b/testdata/cuda-oom-retry-study/armB/dev0/run-10.out new file mode 100644 index 0000000000..e69de29bb2 diff --git a/testdata/cuda-oom-retry-study/armB/dev0/run-11.err b/testdata/cuda-oom-retry-study/armB/dev0/run-11.err new file mode 100644 index 0000000000..a67caafa5c --- /dev/null +++ b/testdata/cuda-oom-retry-study/armB/dev0/run-11.err @@ -0,0 +1,6 @@ +test_out_of_memory_returns_an_error_and_keeps_the_pool: CUDA error at ../core/src/libvmaf.c:650: CUDA_ERROR_OUT_OF_MEMORY (2) in cuEventSynchronize(priv->cuda.ready) +[hog 30836 MiB, luma probe under hog 0, first rc -12, memory back n/a] [first frame with no memory left: -12] CUDA error at ../core/src/libvmaf.c:650: CUDA_ERROR_OUT_OF_MEMORY (2) in cuEventSynchronize(priv->cuda.ready) +[retry rc -12] fail +the same index does not score once memory is back +1 tests run, 1 failed +rc=1 diff --git a/testdata/cuda-oom-retry-study/armB/dev0/run-11.out b/testdata/cuda-oom-retry-study/armB/dev0/run-11.out new file mode 100644 index 0000000000..e69de29bb2 diff --git a/testdata/cuda-oom-retry-study/armB/dev0/run-12.err b/testdata/cuda-oom-retry-study/armB/dev0/run-12.err new file mode 100644 index 0000000000..de6e70bc25 --- /dev/null +++ b/testdata/cuda-oom-retry-study/armB/dev0/run-12.err @@ -0,0 +1,6 @@ +test_out_of_memory_returns_an_error_and_keeps_the_pool: CUDA error at ../core/src/libvmaf.c:650: CUDA_ERROR_OUT_OF_MEMORY (2) in cuEventSynchronize(priv->cuda.ready) +[hog 30720 MiB, luma probe under hog 0, first rc -12, memory back n/a] [first frame with no memory left: -12] CUDA error at ../core/src/libvmaf.c:650: CUDA_ERROR_OUT_OF_MEMORY (2) in cuEventSynchronize(priv->cuda.ready) +[retry rc -12] fail +the same index does not score once memory is back +1 tests run, 1 failed +rc=1 diff --git a/testdata/cuda-oom-retry-study/armB/dev0/run-12.out b/testdata/cuda-oom-retry-study/armB/dev0/run-12.out new file mode 100644 index 0000000000..e69de29bb2 diff --git a/testdata/cuda-oom-retry-study/armB/dev1/run-01.err b/testdata/cuda-oom-retry-study/armB/dev1/run-01.err new file mode 100644 index 0000000000..2c03ab018a --- /dev/null +++ b/testdata/cuda-oom-retry-study/armB/dev1/run-01.err @@ -0,0 +1,5 @@ +test_out_of_memory_returns_an_error_and_keeps_the_pool: CUDA error at ../core/src/cuda/picture_cuda.c:356: CUDA_ERROR_OUT_OF_MEMORY (2) in cuMemAllocPitch((CUdeviceptr *)&pic->data[i], (size_t *)&pic->stride[i], (size_t)pic->w[i] * ((pic->bpc + 7) / 8), pic->h[i], 8 << hbd) +libvmaf ERROR problem during prepare_ring_buffer +[hog 22954 MiB, luma probe under hog 0, first rc -12, memory back n/a] [first frame with no memory left: -12] [retry rc 0] pass +1 tests run, 1 passed +rc=0 diff --git a/testdata/cuda-oom-retry-study/armB/dev1/run-01.out b/testdata/cuda-oom-retry-study/armB/dev1/run-01.out new file mode 100644 index 0000000000..e69de29bb2 diff --git a/testdata/cuda-oom-retry-study/armB/dev1/run-02.err b/testdata/cuda-oom-retry-study/armB/dev1/run-02.err new file mode 100644 index 0000000000..2c03ab018a --- /dev/null +++ b/testdata/cuda-oom-retry-study/armB/dev1/run-02.err @@ -0,0 +1,5 @@ +test_out_of_memory_returns_an_error_and_keeps_the_pool: CUDA error at ../core/src/cuda/picture_cuda.c:356: CUDA_ERROR_OUT_OF_MEMORY (2) in cuMemAllocPitch((CUdeviceptr *)&pic->data[i], (size_t *)&pic->stride[i], (size_t)pic->w[i] * ((pic->bpc + 7) / 8), pic->h[i], 8 << hbd) +libvmaf ERROR problem during prepare_ring_buffer +[hog 22954 MiB, luma probe under hog 0, first rc -12, memory back n/a] [first frame with no memory left: -12] [retry rc 0] pass +1 tests run, 1 passed +rc=0 diff --git a/testdata/cuda-oom-retry-study/armB/dev1/run-02.out b/testdata/cuda-oom-retry-study/armB/dev1/run-02.out new file mode 100644 index 0000000000..e69de29bb2 diff --git a/testdata/cuda-oom-retry-study/armB/dev1/run-03.err b/testdata/cuda-oom-retry-study/armB/dev1/run-03.err new file mode 100644 index 0000000000..2c03ab018a --- /dev/null +++ b/testdata/cuda-oom-retry-study/armB/dev1/run-03.err @@ -0,0 +1,5 @@ +test_out_of_memory_returns_an_error_and_keeps_the_pool: CUDA error at ../core/src/cuda/picture_cuda.c:356: CUDA_ERROR_OUT_OF_MEMORY (2) in cuMemAllocPitch((CUdeviceptr *)&pic->data[i], (size_t *)&pic->stride[i], (size_t)pic->w[i] * ((pic->bpc + 7) / 8), pic->h[i], 8 << hbd) +libvmaf ERROR problem during prepare_ring_buffer +[hog 22954 MiB, luma probe under hog 0, first rc -12, memory back n/a] [first frame with no memory left: -12] [retry rc 0] pass +1 tests run, 1 passed +rc=0 diff --git a/testdata/cuda-oom-retry-study/armB/dev1/run-03.out b/testdata/cuda-oom-retry-study/armB/dev1/run-03.out new file mode 100644 index 0000000000..e69de29bb2 diff --git a/testdata/cuda-oom-retry-study/armB/dev1/run-04.err b/testdata/cuda-oom-retry-study/armB/dev1/run-04.err new file mode 100644 index 0000000000..2c03ab018a --- /dev/null +++ b/testdata/cuda-oom-retry-study/armB/dev1/run-04.err @@ -0,0 +1,5 @@ +test_out_of_memory_returns_an_error_and_keeps_the_pool: CUDA error at ../core/src/cuda/picture_cuda.c:356: CUDA_ERROR_OUT_OF_MEMORY (2) in cuMemAllocPitch((CUdeviceptr *)&pic->data[i], (size_t *)&pic->stride[i], (size_t)pic->w[i] * ((pic->bpc + 7) / 8), pic->h[i], 8 << hbd) +libvmaf ERROR problem during prepare_ring_buffer +[hog 22954 MiB, luma probe under hog 0, first rc -12, memory back n/a] [first frame with no memory left: -12] [retry rc 0] pass +1 tests run, 1 passed +rc=0 diff --git a/testdata/cuda-oom-retry-study/armB/dev1/run-04.out b/testdata/cuda-oom-retry-study/armB/dev1/run-04.out new file mode 100644 index 0000000000..e69de29bb2 diff --git a/testdata/cuda-oom-retry-study/armB/dev1/run-05.err b/testdata/cuda-oom-retry-study/armB/dev1/run-05.err new file mode 100644 index 0000000000..2c03ab018a --- /dev/null +++ b/testdata/cuda-oom-retry-study/armB/dev1/run-05.err @@ -0,0 +1,5 @@ +test_out_of_memory_returns_an_error_and_keeps_the_pool: CUDA error at ../core/src/cuda/picture_cuda.c:356: CUDA_ERROR_OUT_OF_MEMORY (2) in cuMemAllocPitch((CUdeviceptr *)&pic->data[i], (size_t *)&pic->stride[i], (size_t)pic->w[i] * ((pic->bpc + 7) / 8), pic->h[i], 8 << hbd) +libvmaf ERROR problem during prepare_ring_buffer +[hog 22954 MiB, luma probe under hog 0, first rc -12, memory back n/a] [first frame with no memory left: -12] [retry rc 0] pass +1 tests run, 1 passed +rc=0 diff --git a/testdata/cuda-oom-retry-study/armB/dev1/run-05.out b/testdata/cuda-oom-retry-study/armB/dev1/run-05.out new file mode 100644 index 0000000000..e69de29bb2 diff --git a/testdata/cuda-oom-retry-study/armB/dev1/run-06.err b/testdata/cuda-oom-retry-study/armB/dev1/run-06.err new file mode 100644 index 0000000000..2c03ab018a --- /dev/null +++ b/testdata/cuda-oom-retry-study/armB/dev1/run-06.err @@ -0,0 +1,5 @@ +test_out_of_memory_returns_an_error_and_keeps_the_pool: CUDA error at ../core/src/cuda/picture_cuda.c:356: CUDA_ERROR_OUT_OF_MEMORY (2) in cuMemAllocPitch((CUdeviceptr *)&pic->data[i], (size_t *)&pic->stride[i], (size_t)pic->w[i] * ((pic->bpc + 7) / 8), pic->h[i], 8 << hbd) +libvmaf ERROR problem during prepare_ring_buffer +[hog 22954 MiB, luma probe under hog 0, first rc -12, memory back n/a] [first frame with no memory left: -12] [retry rc 0] pass +1 tests run, 1 passed +rc=0 diff --git a/testdata/cuda-oom-retry-study/armB/dev1/run-06.out b/testdata/cuda-oom-retry-study/armB/dev1/run-06.out new file mode 100644 index 0000000000..e69de29bb2 diff --git a/testdata/cuda-oom-retry-study/armB/dev1/run-07.err b/testdata/cuda-oom-retry-study/armB/dev1/run-07.err new file mode 100644 index 0000000000..2c03ab018a --- /dev/null +++ b/testdata/cuda-oom-retry-study/armB/dev1/run-07.err @@ -0,0 +1,5 @@ +test_out_of_memory_returns_an_error_and_keeps_the_pool: CUDA error at ../core/src/cuda/picture_cuda.c:356: CUDA_ERROR_OUT_OF_MEMORY (2) in cuMemAllocPitch((CUdeviceptr *)&pic->data[i], (size_t *)&pic->stride[i], (size_t)pic->w[i] * ((pic->bpc + 7) / 8), pic->h[i], 8 << hbd) +libvmaf ERROR problem during prepare_ring_buffer +[hog 22954 MiB, luma probe under hog 0, first rc -12, memory back n/a] [first frame with no memory left: -12] [retry rc 0] pass +1 tests run, 1 passed +rc=0 diff --git a/testdata/cuda-oom-retry-study/armB/dev1/run-07.out b/testdata/cuda-oom-retry-study/armB/dev1/run-07.out new file mode 100644 index 0000000000..e69de29bb2 diff --git a/testdata/cuda-oom-retry-study/armB/dev1/run-08.err b/testdata/cuda-oom-retry-study/armB/dev1/run-08.err new file mode 100644 index 0000000000..2c03ab018a --- /dev/null +++ b/testdata/cuda-oom-retry-study/armB/dev1/run-08.err @@ -0,0 +1,5 @@ +test_out_of_memory_returns_an_error_and_keeps_the_pool: CUDA error at ../core/src/cuda/picture_cuda.c:356: CUDA_ERROR_OUT_OF_MEMORY (2) in cuMemAllocPitch((CUdeviceptr *)&pic->data[i], (size_t *)&pic->stride[i], (size_t)pic->w[i] * ((pic->bpc + 7) / 8), pic->h[i], 8 << hbd) +libvmaf ERROR problem during prepare_ring_buffer +[hog 22954 MiB, luma probe under hog 0, first rc -12, memory back n/a] [first frame with no memory left: -12] [retry rc 0] pass +1 tests run, 1 passed +rc=0 diff --git a/testdata/cuda-oom-retry-study/armB/dev1/run-08.out b/testdata/cuda-oom-retry-study/armB/dev1/run-08.out new file mode 100644 index 0000000000..e69de29bb2 diff --git a/testdata/cuda-oom-retry-study/armB/dev1/run-09.err b/testdata/cuda-oom-retry-study/armB/dev1/run-09.err new file mode 100644 index 0000000000..2c03ab018a --- /dev/null +++ b/testdata/cuda-oom-retry-study/armB/dev1/run-09.err @@ -0,0 +1,5 @@ +test_out_of_memory_returns_an_error_and_keeps_the_pool: CUDA error at ../core/src/cuda/picture_cuda.c:356: CUDA_ERROR_OUT_OF_MEMORY (2) in cuMemAllocPitch((CUdeviceptr *)&pic->data[i], (size_t *)&pic->stride[i], (size_t)pic->w[i] * ((pic->bpc + 7) / 8), pic->h[i], 8 << hbd) +libvmaf ERROR problem during prepare_ring_buffer +[hog 22954 MiB, luma probe under hog 0, first rc -12, memory back n/a] [first frame with no memory left: -12] [retry rc 0] pass +1 tests run, 1 passed +rc=0 diff --git a/testdata/cuda-oom-retry-study/armB/dev1/run-09.out b/testdata/cuda-oom-retry-study/armB/dev1/run-09.out new file mode 100644 index 0000000000..e69de29bb2 diff --git a/testdata/cuda-oom-retry-study/armB/dev1/run-10.err b/testdata/cuda-oom-retry-study/armB/dev1/run-10.err new file mode 100644 index 0000000000..2c03ab018a --- /dev/null +++ b/testdata/cuda-oom-retry-study/armB/dev1/run-10.err @@ -0,0 +1,5 @@ +test_out_of_memory_returns_an_error_and_keeps_the_pool: CUDA error at ../core/src/cuda/picture_cuda.c:356: CUDA_ERROR_OUT_OF_MEMORY (2) in cuMemAllocPitch((CUdeviceptr *)&pic->data[i], (size_t *)&pic->stride[i], (size_t)pic->w[i] * ((pic->bpc + 7) / 8), pic->h[i], 8 << hbd) +libvmaf ERROR problem during prepare_ring_buffer +[hog 22954 MiB, luma probe under hog 0, first rc -12, memory back n/a] [first frame with no memory left: -12] [retry rc 0] pass +1 tests run, 1 passed +rc=0 diff --git a/testdata/cuda-oom-retry-study/armB/dev1/run-10.out b/testdata/cuda-oom-retry-study/armB/dev1/run-10.out new file mode 100644 index 0000000000..e69de29bb2 diff --git a/testdata/cuda-oom-retry-study/armB/dev1/run-11.err b/testdata/cuda-oom-retry-study/armB/dev1/run-11.err new file mode 100644 index 0000000000..2c03ab018a --- /dev/null +++ b/testdata/cuda-oom-retry-study/armB/dev1/run-11.err @@ -0,0 +1,5 @@ +test_out_of_memory_returns_an_error_and_keeps_the_pool: CUDA error at ../core/src/cuda/picture_cuda.c:356: CUDA_ERROR_OUT_OF_MEMORY (2) in cuMemAllocPitch((CUdeviceptr *)&pic->data[i], (size_t *)&pic->stride[i], (size_t)pic->w[i] * ((pic->bpc + 7) / 8), pic->h[i], 8 << hbd) +libvmaf ERROR problem during prepare_ring_buffer +[hog 22954 MiB, luma probe under hog 0, first rc -12, memory back n/a] [first frame with no memory left: -12] [retry rc 0] pass +1 tests run, 1 passed +rc=0 diff --git a/testdata/cuda-oom-retry-study/armB/dev1/run-11.out b/testdata/cuda-oom-retry-study/armB/dev1/run-11.out new file mode 100644 index 0000000000..e69de29bb2 diff --git a/testdata/cuda-oom-retry-study/armB/dev1/run-12.err b/testdata/cuda-oom-retry-study/armB/dev1/run-12.err new file mode 100644 index 0000000000..2c03ab018a --- /dev/null +++ b/testdata/cuda-oom-retry-study/armB/dev1/run-12.err @@ -0,0 +1,5 @@ +test_out_of_memory_returns_an_error_and_keeps_the_pool: CUDA error at ../core/src/cuda/picture_cuda.c:356: CUDA_ERROR_OUT_OF_MEMORY (2) in cuMemAllocPitch((CUdeviceptr *)&pic->data[i], (size_t *)&pic->stride[i], (size_t)pic->w[i] * ((pic->bpc + 7) / 8), pic->h[i], 8 << hbd) +libvmaf ERROR problem during prepare_ring_buffer +[hog 22954 MiB, luma probe under hog 0, first rc -12, memory back n/a] [first frame with no memory left: -12] [retry rc 0] pass +1 tests run, 1 passed +rc=0 diff --git a/testdata/cuda-oom-retry-study/armB/dev1/run-12.out b/testdata/cuda-oom-retry-study/armB/dev1/run-12.out new file mode 100644 index 0000000000..e69de29bb2 diff --git a/testdata/cuda-oom-retry-study/armB/timeline.log b/testdata/cuda-oom-retry-study/armB/timeline.log new file mode 100644 index 0000000000..8c0ecbb8bb --- /dev/null +++ b/testdata/cuda-oom-retry-study/armB/timeline.log @@ -0,0 +1,97 @@ +=== armB dev0 run01 start 2026-11-07T07:34:27Z +0 MiB, 0 % +=== armB dev0 run01 end 2026-11-07T07:34:34Z +0 MiB, 0 % +=== armB dev0 run02 start 2026-11-07T07:34:34Z +0 MiB, 0 % +=== armB dev0 run02 end 2026-11-07T07:34:39Z +0 MiB, 0 % +=== armB dev0 run03 start 2026-11-07T07:34:39Z +0 MiB, 0 % +=== armB dev0 run03 end 2026-11-07T07:34:45Z +0 MiB, 0 % +=== armB dev0 run04 start 2026-11-07T07:34:45Z +0 MiB, 0 % +=== armB dev0 run04 end 2026-11-07T07:34:50Z +0 MiB, 0 % +=== armB dev0 run05 start 2026-11-07T07:34:50Z +0 MiB, 0 % +=== armB dev0 run05 end 2026-11-07T07:34:56Z +0 MiB, 0 % +=== armB dev0 run06 start 2026-11-07T07:34:56Z +0 MiB, 0 % +=== armB dev0 run06 end 2026-11-07T07:35:01Z +0 MiB, 0 % +=== armB dev0 run07 start 2026-11-07T07:35:01Z +0 MiB, 0 % +=== armB dev0 run07 end 2026-11-07T07:35:06Z +0 MiB, 0 % +=== armB dev0 run08 start 2026-11-07T07:35:06Z +0 MiB, 0 % +=== armB dev0 run08 end 2026-11-07T07:35:10Z +0 MiB, 0 % +=== armB dev0 run09 start 2026-11-07T07:35:10Z +0 MiB, 0 % +=== armB dev0 run09 end 2026-11-07T07:35:14Z +0 MiB, 0 % +=== armB dev0 run10 start 2026-11-07T07:35:14Z +0 MiB, 0 % +=== armB dev0 run10 end 2026-11-07T07:35:19Z +0 MiB, 0 % +=== armB dev0 run11 start 2026-11-07T07:35:19Z +0 MiB, 0 % +=== armB dev0 run11 end 2026-11-07T07:35:24Z +0 MiB, 0 % +=== armB dev0 run12 start 2026-11-07T07:35:24Z +0 MiB, 0 % +=== armB dev0 run12 end 2026-11-07T07:35:29Z +0 MiB, 0 % +=== armB dev1 run01 start 2026-11-07T07:35:29Z +482 MiB, 70 % +=== armB dev1 run01 end 2026-11-07T07:35:32Z +502 MiB, 0 % +=== armB dev1 run02 start 2026-11-07T07:35:32Z +502 MiB, 0 % +=== armB dev1 run02 end 2026-11-07T07:35:36Z +557 MiB, 0 % +=== armB dev1 run03 start 2026-11-07T07:35:36Z +557 MiB, 0 % +=== armB dev1 run03 end 2026-11-07T07:35:40Z +557 MiB, 66 % +=== armB dev1 run04 start 2026-11-07T07:35:40Z +557 MiB, 49 % +=== armB dev1 run04 end 2026-11-07T07:35:43Z +557 MiB, 63 % +=== armB dev1 run05 start 2026-11-07T07:35:44Z +557 MiB, 63 % +=== armB dev1 run05 end 2026-11-07T07:35:47Z +557 MiB, 0 % +=== armB dev1 run06 start 2026-11-07T07:35:47Z +557 MiB, 0 % +=== armB dev1 run06 end 2026-11-07T07:35:51Z +557 MiB, 0 % +=== armB dev1 run07 start 2026-11-07T07:35:51Z +557 MiB, 0 % +=== armB dev1 run07 end 2026-11-07T07:35:55Z +557 MiB, 64 % +=== armB dev1 run08 start 2026-11-07T07:35:55Z +557 MiB, 64 % +=== armB dev1 run08 end 2026-11-07T07:35:58Z +557 MiB, 67 % +=== armB dev1 run09 start 2026-11-07T07:35:59Z +557 MiB, 67 % +=== armB dev1 run09 end 2026-11-07T07:36:02Z +578 MiB, 67 % +=== armB dev1 run10 start 2026-11-07T07:36:02Z +578 MiB, 67 % +=== armB dev1 run10 end 2026-11-07T07:36:06Z +578 MiB, 67 % +=== armB dev1 run11 start 2026-11-07T07:36:06Z +578 MiB, 13 % +=== armB dev1 run11 end 2026-11-07T07:36:09Z +578 MiB, 0 % +=== armB dev1 run12 start 2026-11-07T07:36:09Z +578 MiB, 0 % +=== armB dev1 run12 end 2026-11-07T07:36:13Z +578 MiB, 0 % +ARM-B-DONE 2026-11-07T07:36:13Z diff --git a/testdata/cuda-oom-retry-study/armB_edit.py b/testdata/cuda-oom-retry-study/armB_edit.py new file mode 100644 index 0000000000..6f282d5abc --- /dev/null +++ b/testdata/cuda-oom-retry-study/armB_edit.py @@ -0,0 +1,49 @@ +# -*- coding: utf-8 -*- +"""Apply the Arm B edit to test_cuda_oom_pictures_released.c (probe-removed arm). + +Local experiment edit, not committed. Run AFTER the Arm A campaign finishes. +Reversible with: git -C /c/PLAN/VMAF/worktrees/_oom-master checkout -- core/test/test_cuda_oom_pictures_released.c +""" +import io +import sys + +PATH = r"C:\PLAN\VMAF\worktrees\_oom-master\core\test\test_cuda_oom_pictures_released.c" + +OLD = """ hog_release(state, hog); + const int probe_back = probe_alloc(state, PROBE_BYTES); + (void)fprintf(stderr, "[hog %zu MiB, luma probe under hog %d, first rc %d, memory back %d] ", + hog->bytes >> 20, probe_hogged, oom_err, probe_back); + mu_assert("a frame was scored with the device's memory taken", oom_err != 0); + (void)fprintf(stderr, "[first frame with no memory left: %d] ", oom_err); + + /* The pair of the failed call is back in the pool, and the ring that could + * not be built is built now. */ + mu_assert("allocation probe failed", probe_back >= 0); + mu_assert("device memory not yet returned after the hog was freed", probe_back == 1); + return NULL;""" + +NEW = """ hog_release(state, hog); + /* #3277 probe-perturbation A/B, arm B: no in-process probe between the + * hog release and the retry (local experiment, not for commit). */ + (void)fprintf(stderr, "[hog %zu MiB, luma probe under hog %d, first rc %d, memory back n/a] ", + hog->bytes >> 20, probe_hogged, oom_err); + mu_assert("a frame was scored with the device's memory taken", oom_err != 0); + (void)fprintf(stderr, "[first frame with no memory left: %d] ", oom_err); + return NULL;""" + + +def main() -> int: + with io.open(PATH, "r", encoding="utf-8", newline="") as f: + text = f.read() + if OLD not in text: + print("PATTERN-NOT-FOUND (already applied or upstream changed)") + return 1 + text = text.replace(OLD, NEW, 1) + with io.open(PATH, "w", encoding="utf-8", newline="") as f: + f.write(text) + print("ARM-B-EDIT-APPLIED") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/testdata/cuda-oom-retry-study/armC/dev0/run-01.err b/testdata/cuda-oom-retry-study/armC/dev0/run-01.err new file mode 100644 index 0000000000..0f483151e4 --- /dev/null +++ b/testdata/cuda-oom-retry-study/armC/dev0/run-01.err @@ -0,0 +1,5 @@ +test_out_of_memory_returns_an_error_and_keeps_the_pool: CUDA error at ../core/src/libvmaf.c:650: CUDA_ERROR_OUT_OF_MEMORY (2) in cuEventSynchronize(priv->cuda.ready) +[hog 30836 MiB, luma probe under hog 0, first rc -12, memory back 0 after 60000.0 ms (1933 probes)] [first frame with no memory left: -12] fail +device memory not returned within the 60 s bound +1 tests run, 1 failed +rc=1 diff --git a/testdata/cuda-oom-retry-study/armC/dev0/run-01.out b/testdata/cuda-oom-retry-study/armC/dev0/run-01.out new file mode 100644 index 0000000000..e69de29bb2 diff --git a/testdata/cuda-oom-retry-study/armC/dev0/run-02.err b/testdata/cuda-oom-retry-study/armC/dev0/run-02.err new file mode 100644 index 0000000000..5e0062a5ed --- /dev/null +++ b/testdata/cuda-oom-retry-study/armC/dev0/run-02.err @@ -0,0 +1,5 @@ +test_out_of_memory_returns_an_error_and_keeps_the_pool: CUDA error at ../core/src/libvmaf.c:650: CUDA_ERROR_OUT_OF_MEMORY (2) in cuEventSynchronize(priv->cuda.ready) +[hog 30720 MiB, luma probe under hog 0, first rc -12, memory back 0 after 60000.0 ms (1921 probes)] [first frame with no memory left: -12] fail +device memory not returned within the 60 s bound +1 tests run, 1 failed +rc=1 diff --git a/testdata/cuda-oom-retry-study/armC/dev0/run-02.out b/testdata/cuda-oom-retry-study/armC/dev0/run-02.out new file mode 100644 index 0000000000..e69de29bb2 diff --git a/testdata/cuda-oom-retry-study/armC/dev0/run-03.err b/testdata/cuda-oom-retry-study/armC/dev0/run-03.err new file mode 100644 index 0000000000..5684141522 --- /dev/null +++ b/testdata/cuda-oom-retry-study/armC/dev0/run-03.err @@ -0,0 +1,5 @@ +test_out_of_memory_returns_an_error_and_keeps_the_pool: CUDA error at ../core/src/libvmaf.c:650: CUDA_ERROR_OUT_OF_MEMORY (2) in cuEventSynchronize(priv->cuda.ready) +[hog 30720 MiB, luma probe under hog 0, first rc -12, memory back 0 after 60000.0 ms (1931 probes)] [first frame with no memory left: -12] fail +device memory not returned within the 60 s bound +1 tests run, 1 failed +rc=1 diff --git a/testdata/cuda-oom-retry-study/armC/dev0/run-03.out b/testdata/cuda-oom-retry-study/armC/dev0/run-03.out new file mode 100644 index 0000000000..e69de29bb2 diff --git a/testdata/cuda-oom-retry-study/armC/dev0/run-04.err b/testdata/cuda-oom-retry-study/armC/dev0/run-04.err new file mode 100644 index 0000000000..fe815d98e8 --- /dev/null +++ b/testdata/cuda-oom-retry-study/armC/dev0/run-04.err @@ -0,0 +1,5 @@ +test_out_of_memory_returns_an_error_and_keeps_the_pool: CUDA error at ../core/src/libvmaf.c:650: CUDA_ERROR_OUT_OF_MEMORY (2) in cuEventSynchronize(priv->cuda.ready) +[hog 30720 MiB, luma probe under hog 0, first rc -12, memory back 0 after 60016.0 ms (1929 probes)] [first frame with no memory left: -12] fail +device memory not returned within the 60 s bound +1 tests run, 1 failed +rc=1 diff --git a/testdata/cuda-oom-retry-study/armC/dev0/run-04.out b/testdata/cuda-oom-retry-study/armC/dev0/run-04.out new file mode 100644 index 0000000000..e69de29bb2 diff --git a/testdata/cuda-oom-retry-study/armC/dev0/run-05.err b/testdata/cuda-oom-retry-study/armC/dev0/run-05.err new file mode 100644 index 0000000000..ffa13adef0 --- /dev/null +++ b/testdata/cuda-oom-retry-study/armC/dev0/run-05.err @@ -0,0 +1,5 @@ +test_out_of_memory_returns_an_error_and_keeps_the_pool: CUDA error at ../core/src/libvmaf.c:650: CUDA_ERROR_OUT_OF_MEMORY (2) in cuEventSynchronize(priv->cuda.ready) +[hog 30720 MiB, luma probe under hog 0, first rc -12, memory back 0 after 60015.0 ms (1926 probes)] [first frame with no memory left: -12] fail +device memory not returned within the 60 s bound +1 tests run, 1 failed +rc=1 diff --git a/testdata/cuda-oom-retry-study/armC/dev0/run-05.out b/testdata/cuda-oom-retry-study/armC/dev0/run-05.out new file mode 100644 index 0000000000..e69de29bb2 diff --git a/testdata/cuda-oom-retry-study/armC/dev0/run-06.err b/testdata/cuda-oom-retry-study/armC/dev0/run-06.err new file mode 100644 index 0000000000..314052e802 --- /dev/null +++ b/testdata/cuda-oom-retry-study/armC/dev0/run-06.err @@ -0,0 +1,5 @@ +test_out_of_memory_returns_an_error_and_keeps_the_pool: CUDA error at ../core/src/libvmaf.c:650: CUDA_ERROR_OUT_OF_MEMORY (2) in cuEventSynchronize(priv->cuda.ready) +[hog 30836 MiB, luma probe under hog 0, first rc -12, memory back 0 after 60000.0 ms (1931 probes)] [first frame with no memory left: -12] fail +device memory not returned within the 60 s bound +1 tests run, 1 failed +rc=1 diff --git a/testdata/cuda-oom-retry-study/armC/dev0/run-06.out b/testdata/cuda-oom-retry-study/armC/dev0/run-06.out new file mode 100644 index 0000000000..e69de29bb2 diff --git a/testdata/cuda-oom-retry-study/armC/dev0/run-07.err b/testdata/cuda-oom-retry-study/armC/dev0/run-07.err new file mode 100644 index 0000000000..7686533540 --- /dev/null +++ b/testdata/cuda-oom-retry-study/armC/dev0/run-07.err @@ -0,0 +1,5 @@ +test_out_of_memory_returns_an_error_and_keeps_the_pool: CUDA error at ../core/src/cuda/picture_cuda.c:356: CUDA_ERROR_OUT_OF_MEMORY (2) in cuMemAllocPitch((CUdeviceptr *)&pic->data[i], (size_t *)&pic->stride[i], (size_t)pic->w[i] * ((pic->bpc + 7) / 8), pic->h[i], 8 << hbd) +libvmaf ERROR problem during prepare_ring_buffer +[hog 30836 MiB, luma probe under hog 0, first rc -12, memory back 1 after 0.0 ms (0 probes)] [first frame with no memory left: -12] [retry rc 0] pass +1 tests run, 1 passed +rc=0 diff --git a/testdata/cuda-oom-retry-study/armC/dev0/run-07.out b/testdata/cuda-oom-retry-study/armC/dev0/run-07.out new file mode 100644 index 0000000000..e69de29bb2 diff --git a/testdata/cuda-oom-retry-study/armC/dev0/run-08.err b/testdata/cuda-oom-retry-study/armC/dev0/run-08.err new file mode 100644 index 0000000000..7686533540 --- /dev/null +++ b/testdata/cuda-oom-retry-study/armC/dev0/run-08.err @@ -0,0 +1,5 @@ +test_out_of_memory_returns_an_error_and_keeps_the_pool: CUDA error at ../core/src/cuda/picture_cuda.c:356: CUDA_ERROR_OUT_OF_MEMORY (2) in cuMemAllocPitch((CUdeviceptr *)&pic->data[i], (size_t *)&pic->stride[i], (size_t)pic->w[i] * ((pic->bpc + 7) / 8), pic->h[i], 8 << hbd) +libvmaf ERROR problem during prepare_ring_buffer +[hog 30836 MiB, luma probe under hog 0, first rc -12, memory back 1 after 0.0 ms (0 probes)] [first frame with no memory left: -12] [retry rc 0] pass +1 tests run, 1 passed +rc=0 diff --git a/testdata/cuda-oom-retry-study/armC/dev0/run-08.out b/testdata/cuda-oom-retry-study/armC/dev0/run-08.out new file mode 100644 index 0000000000..e69de29bb2 diff --git a/testdata/cuda-oom-retry-study/armC/dev0/run-09.err b/testdata/cuda-oom-retry-study/armC/dev0/run-09.err new file mode 100644 index 0000000000..7686533540 --- /dev/null +++ b/testdata/cuda-oom-retry-study/armC/dev0/run-09.err @@ -0,0 +1,5 @@ +test_out_of_memory_returns_an_error_and_keeps_the_pool: CUDA error at ../core/src/cuda/picture_cuda.c:356: CUDA_ERROR_OUT_OF_MEMORY (2) in cuMemAllocPitch((CUdeviceptr *)&pic->data[i], (size_t *)&pic->stride[i], (size_t)pic->w[i] * ((pic->bpc + 7) / 8), pic->h[i], 8 << hbd) +libvmaf ERROR problem during prepare_ring_buffer +[hog 30836 MiB, luma probe under hog 0, first rc -12, memory back 1 after 0.0 ms (0 probes)] [first frame with no memory left: -12] [retry rc 0] pass +1 tests run, 1 passed +rc=0 diff --git a/testdata/cuda-oom-retry-study/armC/dev0/run-09.out b/testdata/cuda-oom-retry-study/armC/dev0/run-09.out new file mode 100644 index 0000000000..e69de29bb2 diff --git a/testdata/cuda-oom-retry-study/armC/dev0/run-10.err b/testdata/cuda-oom-retry-study/armC/dev0/run-10.err new file mode 100644 index 0000000000..7686533540 --- /dev/null +++ b/testdata/cuda-oom-retry-study/armC/dev0/run-10.err @@ -0,0 +1,5 @@ +test_out_of_memory_returns_an_error_and_keeps_the_pool: CUDA error at ../core/src/cuda/picture_cuda.c:356: CUDA_ERROR_OUT_OF_MEMORY (2) in cuMemAllocPitch((CUdeviceptr *)&pic->data[i], (size_t *)&pic->stride[i], (size_t)pic->w[i] * ((pic->bpc + 7) / 8), pic->h[i], 8 << hbd) +libvmaf ERROR problem during prepare_ring_buffer +[hog 30836 MiB, luma probe under hog 0, first rc -12, memory back 1 after 0.0 ms (0 probes)] [first frame with no memory left: -12] [retry rc 0] pass +1 tests run, 1 passed +rc=0 diff --git a/testdata/cuda-oom-retry-study/armC/dev0/run-10.out b/testdata/cuda-oom-retry-study/armC/dev0/run-10.out new file mode 100644 index 0000000000..e69de29bb2 diff --git a/testdata/cuda-oom-retry-study/armC/dev0/run-11.err b/testdata/cuda-oom-retry-study/armC/dev0/run-11.err new file mode 100644 index 0000000000..7686533540 --- /dev/null +++ b/testdata/cuda-oom-retry-study/armC/dev0/run-11.err @@ -0,0 +1,5 @@ +test_out_of_memory_returns_an_error_and_keeps_the_pool: CUDA error at ../core/src/cuda/picture_cuda.c:356: CUDA_ERROR_OUT_OF_MEMORY (2) in cuMemAllocPitch((CUdeviceptr *)&pic->data[i], (size_t *)&pic->stride[i], (size_t)pic->w[i] * ((pic->bpc + 7) / 8), pic->h[i], 8 << hbd) +libvmaf ERROR problem during prepare_ring_buffer +[hog 30836 MiB, luma probe under hog 0, first rc -12, memory back 1 after 0.0 ms (0 probes)] [first frame with no memory left: -12] [retry rc 0] pass +1 tests run, 1 passed +rc=0 diff --git a/testdata/cuda-oom-retry-study/armC/dev0/run-11.out b/testdata/cuda-oom-retry-study/armC/dev0/run-11.out new file mode 100644 index 0000000000..e69de29bb2 diff --git a/testdata/cuda-oom-retry-study/armC/dev0/run-12.err b/testdata/cuda-oom-retry-study/armC/dev0/run-12.err new file mode 100644 index 0000000000..7686533540 --- /dev/null +++ b/testdata/cuda-oom-retry-study/armC/dev0/run-12.err @@ -0,0 +1,5 @@ +test_out_of_memory_returns_an_error_and_keeps_the_pool: CUDA error at ../core/src/cuda/picture_cuda.c:356: CUDA_ERROR_OUT_OF_MEMORY (2) in cuMemAllocPitch((CUdeviceptr *)&pic->data[i], (size_t *)&pic->stride[i], (size_t)pic->w[i] * ((pic->bpc + 7) / 8), pic->h[i], 8 << hbd) +libvmaf ERROR problem during prepare_ring_buffer +[hog 30836 MiB, luma probe under hog 0, first rc -12, memory back 1 after 0.0 ms (0 probes)] [first frame with no memory left: -12] [retry rc 0] pass +1 tests run, 1 passed +rc=0 diff --git a/testdata/cuda-oom-retry-study/armC/dev0/run-12.out b/testdata/cuda-oom-retry-study/armC/dev0/run-12.out new file mode 100644 index 0000000000..e69de29bb2 diff --git a/testdata/cuda-oom-retry-study/armC/dev1/run-01.err b/testdata/cuda-oom-retry-study/armC/dev1/run-01.err new file mode 100644 index 0000000000..7bf4307096 --- /dev/null +++ b/testdata/cuda-oom-retry-study/armC/dev1/run-01.err @@ -0,0 +1,5 @@ +test_out_of_memory_returns_an_error_and_keeps_the_pool: CUDA error at ../core/src/cuda/picture_cuda.c:356: CUDA_ERROR_OUT_OF_MEMORY (2) in cuMemAllocPitch((CUdeviceptr *)&pic->data[i], (size_t *)&pic->stride[i], (size_t)pic->w[i] * ((pic->bpc + 7) / 8), pic->h[i], 8 << hbd) +libvmaf ERROR problem during prepare_ring_buffer +[hog 22954 MiB, luma probe under hog 0, first rc -12, memory back 1 after 0.0 ms (0 probes)] [first frame with no memory left: -12] [retry rc 0] pass +1 tests run, 1 passed +rc=0 diff --git a/testdata/cuda-oom-retry-study/armC/dev1/run-01.out b/testdata/cuda-oom-retry-study/armC/dev1/run-01.out new file mode 100644 index 0000000000..e69de29bb2 diff --git a/testdata/cuda-oom-retry-study/armC/dev1/run-02.err b/testdata/cuda-oom-retry-study/armC/dev1/run-02.err new file mode 100644 index 0000000000..7bf4307096 --- /dev/null +++ b/testdata/cuda-oom-retry-study/armC/dev1/run-02.err @@ -0,0 +1,5 @@ +test_out_of_memory_returns_an_error_and_keeps_the_pool: CUDA error at ../core/src/cuda/picture_cuda.c:356: CUDA_ERROR_OUT_OF_MEMORY (2) in cuMemAllocPitch((CUdeviceptr *)&pic->data[i], (size_t *)&pic->stride[i], (size_t)pic->w[i] * ((pic->bpc + 7) / 8), pic->h[i], 8 << hbd) +libvmaf ERROR problem during prepare_ring_buffer +[hog 22954 MiB, luma probe under hog 0, first rc -12, memory back 1 after 0.0 ms (0 probes)] [first frame with no memory left: -12] [retry rc 0] pass +1 tests run, 1 passed +rc=0 diff --git a/testdata/cuda-oom-retry-study/armC/dev1/run-02.out b/testdata/cuda-oom-retry-study/armC/dev1/run-02.out new file mode 100644 index 0000000000..e69de29bb2 diff --git a/testdata/cuda-oom-retry-study/armC/dev1/run-03.err b/testdata/cuda-oom-retry-study/armC/dev1/run-03.err new file mode 100644 index 0000000000..7bf4307096 --- /dev/null +++ b/testdata/cuda-oom-retry-study/armC/dev1/run-03.err @@ -0,0 +1,5 @@ +test_out_of_memory_returns_an_error_and_keeps_the_pool: CUDA error at ../core/src/cuda/picture_cuda.c:356: CUDA_ERROR_OUT_OF_MEMORY (2) in cuMemAllocPitch((CUdeviceptr *)&pic->data[i], (size_t *)&pic->stride[i], (size_t)pic->w[i] * ((pic->bpc + 7) / 8), pic->h[i], 8 << hbd) +libvmaf ERROR problem during prepare_ring_buffer +[hog 22954 MiB, luma probe under hog 0, first rc -12, memory back 1 after 0.0 ms (0 probes)] [first frame with no memory left: -12] [retry rc 0] pass +1 tests run, 1 passed +rc=0 diff --git a/testdata/cuda-oom-retry-study/armC/dev1/run-03.out b/testdata/cuda-oom-retry-study/armC/dev1/run-03.out new file mode 100644 index 0000000000..e69de29bb2 diff --git a/testdata/cuda-oom-retry-study/armC/dev1/run-04.err b/testdata/cuda-oom-retry-study/armC/dev1/run-04.err new file mode 100644 index 0000000000..7bf4307096 --- /dev/null +++ b/testdata/cuda-oom-retry-study/armC/dev1/run-04.err @@ -0,0 +1,5 @@ +test_out_of_memory_returns_an_error_and_keeps_the_pool: CUDA error at ../core/src/cuda/picture_cuda.c:356: CUDA_ERROR_OUT_OF_MEMORY (2) in cuMemAllocPitch((CUdeviceptr *)&pic->data[i], (size_t *)&pic->stride[i], (size_t)pic->w[i] * ((pic->bpc + 7) / 8), pic->h[i], 8 << hbd) +libvmaf ERROR problem during prepare_ring_buffer +[hog 22954 MiB, luma probe under hog 0, first rc -12, memory back 1 after 0.0 ms (0 probes)] [first frame with no memory left: -12] [retry rc 0] pass +1 tests run, 1 passed +rc=0 diff --git a/testdata/cuda-oom-retry-study/armC/dev1/run-04.out b/testdata/cuda-oom-retry-study/armC/dev1/run-04.out new file mode 100644 index 0000000000..e69de29bb2 diff --git a/testdata/cuda-oom-retry-study/armC/dev1/run-05.err b/testdata/cuda-oom-retry-study/armC/dev1/run-05.err new file mode 100644 index 0000000000..7bf4307096 --- /dev/null +++ b/testdata/cuda-oom-retry-study/armC/dev1/run-05.err @@ -0,0 +1,5 @@ +test_out_of_memory_returns_an_error_and_keeps_the_pool: CUDA error at ../core/src/cuda/picture_cuda.c:356: CUDA_ERROR_OUT_OF_MEMORY (2) in cuMemAllocPitch((CUdeviceptr *)&pic->data[i], (size_t *)&pic->stride[i], (size_t)pic->w[i] * ((pic->bpc + 7) / 8), pic->h[i], 8 << hbd) +libvmaf ERROR problem during prepare_ring_buffer +[hog 22954 MiB, luma probe under hog 0, first rc -12, memory back 1 after 0.0 ms (0 probes)] [first frame with no memory left: -12] [retry rc 0] pass +1 tests run, 1 passed +rc=0 diff --git a/testdata/cuda-oom-retry-study/armC/dev1/run-05.out b/testdata/cuda-oom-retry-study/armC/dev1/run-05.out new file mode 100644 index 0000000000..e69de29bb2 diff --git a/testdata/cuda-oom-retry-study/armC/dev1/run-06.err b/testdata/cuda-oom-retry-study/armC/dev1/run-06.err new file mode 100644 index 0000000000..7bf4307096 --- /dev/null +++ b/testdata/cuda-oom-retry-study/armC/dev1/run-06.err @@ -0,0 +1,5 @@ +test_out_of_memory_returns_an_error_and_keeps_the_pool: CUDA error at ../core/src/cuda/picture_cuda.c:356: CUDA_ERROR_OUT_OF_MEMORY (2) in cuMemAllocPitch((CUdeviceptr *)&pic->data[i], (size_t *)&pic->stride[i], (size_t)pic->w[i] * ((pic->bpc + 7) / 8), pic->h[i], 8 << hbd) +libvmaf ERROR problem during prepare_ring_buffer +[hog 22954 MiB, luma probe under hog 0, first rc -12, memory back 1 after 0.0 ms (0 probes)] [first frame with no memory left: -12] [retry rc 0] pass +1 tests run, 1 passed +rc=0 diff --git a/testdata/cuda-oom-retry-study/armC/dev1/run-06.out b/testdata/cuda-oom-retry-study/armC/dev1/run-06.out new file mode 100644 index 0000000000..e69de29bb2 diff --git a/testdata/cuda-oom-retry-study/armC/dev1/run-07.err b/testdata/cuda-oom-retry-study/armC/dev1/run-07.err new file mode 100644 index 0000000000..7bf4307096 --- /dev/null +++ b/testdata/cuda-oom-retry-study/armC/dev1/run-07.err @@ -0,0 +1,5 @@ +test_out_of_memory_returns_an_error_and_keeps_the_pool: CUDA error at ../core/src/cuda/picture_cuda.c:356: CUDA_ERROR_OUT_OF_MEMORY (2) in cuMemAllocPitch((CUdeviceptr *)&pic->data[i], (size_t *)&pic->stride[i], (size_t)pic->w[i] * ((pic->bpc + 7) / 8), pic->h[i], 8 << hbd) +libvmaf ERROR problem during prepare_ring_buffer +[hog 22954 MiB, luma probe under hog 0, first rc -12, memory back 1 after 0.0 ms (0 probes)] [first frame with no memory left: -12] [retry rc 0] pass +1 tests run, 1 passed +rc=0 diff --git a/testdata/cuda-oom-retry-study/armC/dev1/run-07.out b/testdata/cuda-oom-retry-study/armC/dev1/run-07.out new file mode 100644 index 0000000000..e69de29bb2 diff --git a/testdata/cuda-oom-retry-study/armC/dev1/run-08.err b/testdata/cuda-oom-retry-study/armC/dev1/run-08.err new file mode 100644 index 0000000000..7bf4307096 --- /dev/null +++ b/testdata/cuda-oom-retry-study/armC/dev1/run-08.err @@ -0,0 +1,5 @@ +test_out_of_memory_returns_an_error_and_keeps_the_pool: CUDA error at ../core/src/cuda/picture_cuda.c:356: CUDA_ERROR_OUT_OF_MEMORY (2) in cuMemAllocPitch((CUdeviceptr *)&pic->data[i], (size_t *)&pic->stride[i], (size_t)pic->w[i] * ((pic->bpc + 7) / 8), pic->h[i], 8 << hbd) +libvmaf ERROR problem during prepare_ring_buffer +[hog 22954 MiB, luma probe under hog 0, first rc -12, memory back 1 after 0.0 ms (0 probes)] [first frame with no memory left: -12] [retry rc 0] pass +1 tests run, 1 passed +rc=0 diff --git a/testdata/cuda-oom-retry-study/armC/dev1/run-08.out b/testdata/cuda-oom-retry-study/armC/dev1/run-08.out new file mode 100644 index 0000000000..e69de29bb2 diff --git a/testdata/cuda-oom-retry-study/armC/dev1/run-09.err b/testdata/cuda-oom-retry-study/armC/dev1/run-09.err new file mode 100644 index 0000000000..7bf4307096 --- /dev/null +++ b/testdata/cuda-oom-retry-study/armC/dev1/run-09.err @@ -0,0 +1,5 @@ +test_out_of_memory_returns_an_error_and_keeps_the_pool: CUDA error at ../core/src/cuda/picture_cuda.c:356: CUDA_ERROR_OUT_OF_MEMORY (2) in cuMemAllocPitch((CUdeviceptr *)&pic->data[i], (size_t *)&pic->stride[i], (size_t)pic->w[i] * ((pic->bpc + 7) / 8), pic->h[i], 8 << hbd) +libvmaf ERROR problem during prepare_ring_buffer +[hog 22954 MiB, luma probe under hog 0, first rc -12, memory back 1 after 0.0 ms (0 probes)] [first frame with no memory left: -12] [retry rc 0] pass +1 tests run, 1 passed +rc=0 diff --git a/testdata/cuda-oom-retry-study/armC/dev1/run-09.out b/testdata/cuda-oom-retry-study/armC/dev1/run-09.out new file mode 100644 index 0000000000..e69de29bb2 diff --git a/testdata/cuda-oom-retry-study/armC/dev1/run-10.err b/testdata/cuda-oom-retry-study/armC/dev1/run-10.err new file mode 100644 index 0000000000..7bf4307096 --- /dev/null +++ b/testdata/cuda-oom-retry-study/armC/dev1/run-10.err @@ -0,0 +1,5 @@ +test_out_of_memory_returns_an_error_and_keeps_the_pool: CUDA error at ../core/src/cuda/picture_cuda.c:356: CUDA_ERROR_OUT_OF_MEMORY (2) in cuMemAllocPitch((CUdeviceptr *)&pic->data[i], (size_t *)&pic->stride[i], (size_t)pic->w[i] * ((pic->bpc + 7) / 8), pic->h[i], 8 << hbd) +libvmaf ERROR problem during prepare_ring_buffer +[hog 22954 MiB, luma probe under hog 0, first rc -12, memory back 1 after 0.0 ms (0 probes)] [first frame with no memory left: -12] [retry rc 0] pass +1 tests run, 1 passed +rc=0 diff --git a/testdata/cuda-oom-retry-study/armC/dev1/run-10.out b/testdata/cuda-oom-retry-study/armC/dev1/run-10.out new file mode 100644 index 0000000000..e69de29bb2 diff --git a/testdata/cuda-oom-retry-study/armC/dev1/run-11.err b/testdata/cuda-oom-retry-study/armC/dev1/run-11.err new file mode 100644 index 0000000000..7bf4307096 --- /dev/null +++ b/testdata/cuda-oom-retry-study/armC/dev1/run-11.err @@ -0,0 +1,5 @@ +test_out_of_memory_returns_an_error_and_keeps_the_pool: CUDA error at ../core/src/cuda/picture_cuda.c:356: CUDA_ERROR_OUT_OF_MEMORY (2) in cuMemAllocPitch((CUdeviceptr *)&pic->data[i], (size_t *)&pic->stride[i], (size_t)pic->w[i] * ((pic->bpc + 7) / 8), pic->h[i], 8 << hbd) +libvmaf ERROR problem during prepare_ring_buffer +[hog 22954 MiB, luma probe under hog 0, first rc -12, memory back 1 after 0.0 ms (0 probes)] [first frame with no memory left: -12] [retry rc 0] pass +1 tests run, 1 passed +rc=0 diff --git a/testdata/cuda-oom-retry-study/armC/dev1/run-11.out b/testdata/cuda-oom-retry-study/armC/dev1/run-11.out new file mode 100644 index 0000000000..e69de29bb2 diff --git a/testdata/cuda-oom-retry-study/armC/dev1/run-12.err b/testdata/cuda-oom-retry-study/armC/dev1/run-12.err new file mode 100644 index 0000000000..7bf4307096 --- /dev/null +++ b/testdata/cuda-oom-retry-study/armC/dev1/run-12.err @@ -0,0 +1,5 @@ +test_out_of_memory_returns_an_error_and_keeps_the_pool: CUDA error at ../core/src/cuda/picture_cuda.c:356: CUDA_ERROR_OUT_OF_MEMORY (2) in cuMemAllocPitch((CUdeviceptr *)&pic->data[i], (size_t *)&pic->stride[i], (size_t)pic->w[i] * ((pic->bpc + 7) / 8), pic->h[i], 8 << hbd) +libvmaf ERROR problem during prepare_ring_buffer +[hog 22954 MiB, luma probe under hog 0, first rc -12, memory back 1 after 0.0 ms (0 probes)] [first frame with no memory left: -12] [retry rc 0] pass +1 tests run, 1 passed +rc=0 diff --git a/testdata/cuda-oom-retry-study/armC/dev1/run-12.out b/testdata/cuda-oom-retry-study/armC/dev1/run-12.out new file mode 100644 index 0000000000..e69de29bb2 diff --git a/testdata/cuda-oom-retry-study/armC/timeline.log b/testdata/cuda-oom-retry-study/armC/timeline.log new file mode 100644 index 0000000000..c37a119276 --- /dev/null +++ b/testdata/cuda-oom-retry-study/armC/timeline.log @@ -0,0 +1,97 @@ +=== armC dev0 run01 start 2026-11-07T07:46:58Z +0 MiB, 0 % +=== armC dev0 run01 end 2026-11-07T07:48:06Z +0 MiB, 0 % +=== armC dev0 run02 start 2026-11-07T07:48:06Z +0 MiB, 0 % +=== armC dev0 run02 end 2026-11-07T07:49:16Z +0 MiB, 0 % +=== armC dev0 run03 start 2026-11-07T07:49:16Z +0 MiB, 0 % +=== armC dev0 run03 end 2026-11-07T07:50:22Z +0 MiB, 0 % +=== armC dev0 run04 start 2026-11-07T07:50:22Z +0 MiB, 0 % +=== armC dev0 run04 end 2026-11-07T07:51:31Z +0 MiB, 0 % +=== armC dev0 run05 start 2026-11-07T07:51:31Z +0 MiB, 0 % +=== armC dev0 run05 end 2026-11-07T07:52:39Z +0 MiB, 0 % +=== armC dev0 run06 start 2026-11-07T07:52:39Z +0 MiB, 0 % +=== armC dev0 run06 end 2026-11-07T07:53:44Z +0 MiB, 0 % +=== armC dev0 run07 start 2026-11-07T07:53:44Z +0 MiB, 0 % +=== armC dev0 run07 end 2026-11-07T07:53:50Z +0 MiB, 0 % +=== armC dev0 run08 start 2026-11-07T07:53:50Z +0 MiB, 0 % +=== armC dev0 run08 end 2026-11-07T07:53:57Z +0 MiB, 0 % +=== armC dev0 run09 start 2026-11-07T07:53:58Z +0 MiB, 0 % +=== armC dev0 run09 end 2026-11-07T07:54:03Z +0 MiB, 0 % +=== armC dev0 run10 start 2026-11-07T07:54:03Z +0 MiB, 0 % +=== armC dev0 run10 end 2026-11-07T07:54:08Z +0 MiB, 0 % +=== armC dev0 run11 start 2026-11-07T07:54:08Z +0 MiB, 0 % +=== armC dev0 run11 end 2026-11-07T07:54:13Z +0 MiB, 0 % +=== armC dev0 run12 start 2026-11-07T07:54:13Z +0 MiB, 0 % +=== armC dev0 run12 end 2026-11-07T07:54:18Z +0 MiB, 0 % +=== armC dev1 run01 start 2026-11-07T07:54:18Z +552 MiB, 4 % +=== armC dev1 run01 end 2026-11-07T07:54:23Z +528 MiB, 2 % +=== armC dev1 run02 start 2026-11-07T07:54:23Z +528 MiB, 2 % +=== armC dev1 run02 end 2026-11-07T07:54:28Z +528 MiB, 3 % +=== armC dev1 run03 start 2026-11-07T07:54:28Z +528 MiB, 3 % +=== armC dev1 run03 end 2026-11-07T07:54:33Z +528 MiB, 2 % +=== armC dev1 run04 start 2026-11-07T07:54:33Z +528 MiB, 2 % +=== armC dev1 run04 end 2026-11-07T07:54:39Z +541 MiB, 3 % +=== armC dev1 run05 start 2026-11-07T07:54:39Z +541 MiB, 3 % +=== armC dev1 run05 end 2026-11-07T07:54:45Z +538 MiB, 2 % +=== armC dev1 run06 start 2026-11-07T07:54:45Z +538 MiB, 40 % +=== armC dev1 run06 end 2026-11-07T07:54:49Z +564 MiB, 2 % +=== armC dev1 run07 start 2026-11-07T07:54:49Z +565 MiB, 2 % +=== armC dev1 run07 end 2026-11-07T07:54:53Z +580 MiB, 2 % +=== armC dev1 run08 start 2026-11-07T07:54:53Z +580 MiB, 2 % +=== armC dev1 run08 end 2026-11-07T07:54:57Z +556 MiB, 3 % +=== armC dev1 run09 start 2026-11-07T07:54:57Z +556 MiB, 2 % +=== armC dev1 run09 end 2026-11-07T07:55:01Z +565 MiB, 2 % +=== armC dev1 run10 start 2026-11-07T07:55:01Z +565 MiB, 3 % +=== armC dev1 run10 end 2026-11-07T07:55:05Z +565 MiB, 44 % +=== armC dev1 run11 start 2026-11-07T07:55:05Z +565 MiB, 44 % +=== armC dev1 run11 end 2026-11-07T07:55:10Z +563 MiB, 36 % +=== armC dev1 run12 start 2026-11-07T07:55:10Z +563 MiB, 36 % +=== armC dev1 run12 end 2026-11-07T07:55:15Z +563 MiB, 60 % +ARM-C-DONE 2026-11-07T07:55:15Z diff --git a/testdata/cuda-oom-retry-study/armD/dev0/run-1.err b/testdata/cuda-oom-retry-study/armD/dev0/run-1.err new file mode 100644 index 0000000000..149fdc3f24 --- /dev/null +++ b/testdata/cuda-oom-retry-study/armD/dev0/run-1.err @@ -0,0 +1,6 @@ +test_out_of_memory_returns_an_error_and_keeps_the_pool: CUDA error at ../core/src/libvmaf.c:650: CUDA_ERROR_OUT_OF_MEMORY (2) in cuEventSynchronize(priv->cuda.ready) +[silent-wait total 1 s -> probe 0] [silent-wait total 5 s -> probe 0] [silent-wait total 15 s -> probe 0] [silent-wait total 45 s -> probe 0] [silent-wait total 105 s -> probe 0] [hog 30720 MiB, luma probe under hog 0, first rc -12, memory back -1] [first frame with no memory left: -12] CUDA error at ../core/src/libvmaf.c:650: CUDA_ERROR_OUT_OF_MEMORY (2) in cuEventSynchronize(priv->cuda.ready) +[retry rc -12] fail +the same index does not score once memory is back +1 tests run, 1 failed +rc=1 diff --git a/testdata/cuda-oom-retry-study/armD/dev0/run-1.out b/testdata/cuda-oom-retry-study/armD/dev0/run-1.out new file mode 100644 index 0000000000..e69de29bb2 diff --git a/testdata/cuda-oom-retry-study/armD/dev0/run-2.err b/testdata/cuda-oom-retry-study/armD/dev0/run-2.err new file mode 100644 index 0000000000..8920abfc77 --- /dev/null +++ b/testdata/cuda-oom-retry-study/armD/dev0/run-2.err @@ -0,0 +1,6 @@ +test_out_of_memory_returns_an_error_and_keeps_the_pool: CUDA error at ../core/src/libvmaf.c:650: CUDA_ERROR_OUT_OF_MEMORY (2) in cuEventSynchronize(priv->cuda.ready) +[silent-wait total 1 s -> probe 0] [silent-wait total 5 s -> probe 0] [silent-wait total 15 s -> probe 0] [silent-wait total 45 s -> probe 0] [silent-wait total 105 s -> probe 0] [hog 30836 MiB, luma probe under hog 0, first rc -12, memory back -1] [first frame with no memory left: -12] CUDA error at ../core/src/libvmaf.c:650: CUDA_ERROR_OUT_OF_MEMORY (2) in cuEventSynchronize(priv->cuda.ready) +[retry rc -12] fail +the same index does not score once memory is back +1 tests run, 1 failed +rc=1 diff --git a/testdata/cuda-oom-retry-study/armD/dev0/run-2.out b/testdata/cuda-oom-retry-study/armD/dev0/run-2.out new file mode 100644 index 0000000000..e69de29bb2 diff --git a/testdata/cuda-oom-retry-study/armD/dev0/run-3.err b/testdata/cuda-oom-retry-study/armD/dev0/run-3.err new file mode 100644 index 0000000000..7b1c197fcc --- /dev/null +++ b/testdata/cuda-oom-retry-study/armD/dev0/run-3.err @@ -0,0 +1,6 @@ +test_out_of_memory_returns_an_error_and_keeps_the_pool: CUDA error at ../core/src/libvmaf.c:650: CUDA_ERROR_OUT_OF_MEMORY (2) in cuEventSynchronize(priv->cuda.ready) +[silent-wait total 1 s -> probe 0] [silent-wait total 5 s -> probe 0] [silent-wait total 15 s -> probe 0] [silent-wait total 45 s -> probe 0] [silent-wait total 105 s -> probe 0] [hog 30784 MiB, luma probe under hog 0, first rc -12, memory back -1] [first frame with no memory left: -12] CUDA error at ../core/src/libvmaf.c:650: CUDA_ERROR_OUT_OF_MEMORY (2) in cuEventSynchronize(priv->cuda.ready) +[retry rc -12] fail +the same index does not score once memory is back +1 tests run, 1 failed +rc=1 diff --git a/testdata/cuda-oom-retry-study/armD/dev0/run-3.out b/testdata/cuda-oom-retry-study/armD/dev0/run-3.out new file mode 100644 index 0000000000..e69de29bb2 diff --git a/testdata/cuda-oom-retry-study/armD/dev0/run-4.err b/testdata/cuda-oom-retry-study/armD/dev0/run-4.err new file mode 100644 index 0000000000..8920abfc77 --- /dev/null +++ b/testdata/cuda-oom-retry-study/armD/dev0/run-4.err @@ -0,0 +1,6 @@ +test_out_of_memory_returns_an_error_and_keeps_the_pool: CUDA error at ../core/src/libvmaf.c:650: CUDA_ERROR_OUT_OF_MEMORY (2) in cuEventSynchronize(priv->cuda.ready) +[silent-wait total 1 s -> probe 0] [silent-wait total 5 s -> probe 0] [silent-wait total 15 s -> probe 0] [silent-wait total 45 s -> probe 0] [silent-wait total 105 s -> probe 0] [hog 30836 MiB, luma probe under hog 0, first rc -12, memory back -1] [first frame with no memory left: -12] CUDA error at ../core/src/libvmaf.c:650: CUDA_ERROR_OUT_OF_MEMORY (2) in cuEventSynchronize(priv->cuda.ready) +[retry rc -12] fail +the same index does not score once memory is back +1 tests run, 1 failed +rc=1 diff --git a/testdata/cuda-oom-retry-study/armD/dev0/run-4.out b/testdata/cuda-oom-retry-study/armD/dev0/run-4.out new file mode 100644 index 0000000000..e69de29bb2 diff --git a/testdata/cuda-oom-retry-study/armD/dev0/run-5.err b/testdata/cuda-oom-retry-study/armD/dev0/run-5.err new file mode 100644 index 0000000000..149fdc3f24 --- /dev/null +++ b/testdata/cuda-oom-retry-study/armD/dev0/run-5.err @@ -0,0 +1,6 @@ +test_out_of_memory_returns_an_error_and_keeps_the_pool: CUDA error at ../core/src/libvmaf.c:650: CUDA_ERROR_OUT_OF_MEMORY (2) in cuEventSynchronize(priv->cuda.ready) +[silent-wait total 1 s -> probe 0] [silent-wait total 5 s -> probe 0] [silent-wait total 15 s -> probe 0] [silent-wait total 45 s -> probe 0] [silent-wait total 105 s -> probe 0] [hog 30720 MiB, luma probe under hog 0, first rc -12, memory back -1] [first frame with no memory left: -12] CUDA error at ../core/src/libvmaf.c:650: CUDA_ERROR_OUT_OF_MEMORY (2) in cuEventSynchronize(priv->cuda.ready) +[retry rc -12] fail +the same index does not score once memory is back +1 tests run, 1 failed +rc=1 diff --git a/testdata/cuda-oom-retry-study/armD/dev0/run-5.out b/testdata/cuda-oom-retry-study/armD/dev0/run-5.out new file mode 100644 index 0000000000..e69de29bb2 diff --git a/testdata/cuda-oom-retry-study/armD/dev0/run-6.err b/testdata/cuda-oom-retry-study/armD/dev0/run-6.err new file mode 100644 index 0000000000..149fdc3f24 --- /dev/null +++ b/testdata/cuda-oom-retry-study/armD/dev0/run-6.err @@ -0,0 +1,6 @@ +test_out_of_memory_returns_an_error_and_keeps_the_pool: CUDA error at ../core/src/libvmaf.c:650: CUDA_ERROR_OUT_OF_MEMORY (2) in cuEventSynchronize(priv->cuda.ready) +[silent-wait total 1 s -> probe 0] [silent-wait total 5 s -> probe 0] [silent-wait total 15 s -> probe 0] [silent-wait total 45 s -> probe 0] [silent-wait total 105 s -> probe 0] [hog 30720 MiB, luma probe under hog 0, first rc -12, memory back -1] [first frame with no memory left: -12] CUDA error at ../core/src/libvmaf.c:650: CUDA_ERROR_OUT_OF_MEMORY (2) in cuEventSynchronize(priv->cuda.ready) +[retry rc -12] fail +the same index does not score once memory is back +1 tests run, 1 failed +rc=1 diff --git a/testdata/cuda-oom-retry-study/armD/dev0/run-6.out b/testdata/cuda-oom-retry-study/armD/dev0/run-6.out new file mode 100644 index 0000000000..e69de29bb2 diff --git a/testdata/cuda-oom-retry-study/armD/dev1/run-1.err b/testdata/cuda-oom-retry-study/armD/dev1/run-1.err new file mode 100644 index 0000000000..d45be7c2fb --- /dev/null +++ b/testdata/cuda-oom-retry-study/armD/dev1/run-1.err @@ -0,0 +1,5 @@ +test_out_of_memory_returns_an_error_and_keeps_the_pool: CUDA error at ../core/src/cuda/picture_cuda.c:356: CUDA_ERROR_OUT_OF_MEMORY (2) in cuMemAllocPitch((CUdeviceptr *)&pic->data[i], (size_t *)&pic->stride[i], (size_t)pic->w[i] * ((pic->bpc + 7) / 8), pic->h[i], 8 << hbd) +libvmaf ERROR problem during prepare_ring_buffer +[silent-wait total 1 s -> probe 1] [hog 22954 MiB, luma probe under hog 0, first rc -12, memory back 1] [first frame with no memory left: -12] [retry rc 0] pass +1 tests run, 1 passed +rc=0 diff --git a/testdata/cuda-oom-retry-study/armD/dev1/run-1.out b/testdata/cuda-oom-retry-study/armD/dev1/run-1.out new file mode 100644 index 0000000000..e69de29bb2 diff --git a/testdata/cuda-oom-retry-study/armD/dev1/run-2.err b/testdata/cuda-oom-retry-study/armD/dev1/run-2.err new file mode 100644 index 0000000000..d45be7c2fb --- /dev/null +++ b/testdata/cuda-oom-retry-study/armD/dev1/run-2.err @@ -0,0 +1,5 @@ +test_out_of_memory_returns_an_error_and_keeps_the_pool: CUDA error at ../core/src/cuda/picture_cuda.c:356: CUDA_ERROR_OUT_OF_MEMORY (2) in cuMemAllocPitch((CUdeviceptr *)&pic->data[i], (size_t *)&pic->stride[i], (size_t)pic->w[i] * ((pic->bpc + 7) / 8), pic->h[i], 8 << hbd) +libvmaf ERROR problem during prepare_ring_buffer +[silent-wait total 1 s -> probe 1] [hog 22954 MiB, luma probe under hog 0, first rc -12, memory back 1] [first frame with no memory left: -12] [retry rc 0] pass +1 tests run, 1 passed +rc=0 diff --git a/testdata/cuda-oom-retry-study/armD/dev1/run-2.out b/testdata/cuda-oom-retry-study/armD/dev1/run-2.out new file mode 100644 index 0000000000..e69de29bb2 diff --git a/testdata/cuda-oom-retry-study/armD/dev1/run-3.err b/testdata/cuda-oom-retry-study/armD/dev1/run-3.err new file mode 100644 index 0000000000..d45be7c2fb --- /dev/null +++ b/testdata/cuda-oom-retry-study/armD/dev1/run-3.err @@ -0,0 +1,5 @@ +test_out_of_memory_returns_an_error_and_keeps_the_pool: CUDA error at ../core/src/cuda/picture_cuda.c:356: CUDA_ERROR_OUT_OF_MEMORY (2) in cuMemAllocPitch((CUdeviceptr *)&pic->data[i], (size_t *)&pic->stride[i], (size_t)pic->w[i] * ((pic->bpc + 7) / 8), pic->h[i], 8 << hbd) +libvmaf ERROR problem during prepare_ring_buffer +[silent-wait total 1 s -> probe 1] [hog 22954 MiB, luma probe under hog 0, first rc -12, memory back 1] [first frame with no memory left: -12] [retry rc 0] pass +1 tests run, 1 passed +rc=0 diff --git a/testdata/cuda-oom-retry-study/armD/dev1/run-3.out b/testdata/cuda-oom-retry-study/armD/dev1/run-3.out new file mode 100644 index 0000000000..e69de29bb2 diff --git a/testdata/cuda-oom-retry-study/armD/dev1/run-4.err b/testdata/cuda-oom-retry-study/armD/dev1/run-4.err new file mode 100644 index 0000000000..d45be7c2fb --- /dev/null +++ b/testdata/cuda-oom-retry-study/armD/dev1/run-4.err @@ -0,0 +1,5 @@ +test_out_of_memory_returns_an_error_and_keeps_the_pool: CUDA error at ../core/src/cuda/picture_cuda.c:356: CUDA_ERROR_OUT_OF_MEMORY (2) in cuMemAllocPitch((CUdeviceptr *)&pic->data[i], (size_t *)&pic->stride[i], (size_t)pic->w[i] * ((pic->bpc + 7) / 8), pic->h[i], 8 << hbd) +libvmaf ERROR problem during prepare_ring_buffer +[silent-wait total 1 s -> probe 1] [hog 22954 MiB, luma probe under hog 0, first rc -12, memory back 1] [first frame with no memory left: -12] [retry rc 0] pass +1 tests run, 1 passed +rc=0 diff --git a/testdata/cuda-oom-retry-study/armD/dev1/run-4.out b/testdata/cuda-oom-retry-study/armD/dev1/run-4.out new file mode 100644 index 0000000000..e69de29bb2 diff --git a/testdata/cuda-oom-retry-study/armD/dev1/run-5.err b/testdata/cuda-oom-retry-study/armD/dev1/run-5.err new file mode 100644 index 0000000000..d45be7c2fb --- /dev/null +++ b/testdata/cuda-oom-retry-study/armD/dev1/run-5.err @@ -0,0 +1,5 @@ +test_out_of_memory_returns_an_error_and_keeps_the_pool: CUDA error at ../core/src/cuda/picture_cuda.c:356: CUDA_ERROR_OUT_OF_MEMORY (2) in cuMemAllocPitch((CUdeviceptr *)&pic->data[i], (size_t *)&pic->stride[i], (size_t)pic->w[i] * ((pic->bpc + 7) / 8), pic->h[i], 8 << hbd) +libvmaf ERROR problem during prepare_ring_buffer +[silent-wait total 1 s -> probe 1] [hog 22954 MiB, luma probe under hog 0, first rc -12, memory back 1] [first frame with no memory left: -12] [retry rc 0] pass +1 tests run, 1 passed +rc=0 diff --git a/testdata/cuda-oom-retry-study/armD/dev1/run-5.out b/testdata/cuda-oom-retry-study/armD/dev1/run-5.out new file mode 100644 index 0000000000..e69de29bb2 diff --git a/testdata/cuda-oom-retry-study/armD/dev1/run-6.err b/testdata/cuda-oom-retry-study/armD/dev1/run-6.err new file mode 100644 index 0000000000..d45be7c2fb --- /dev/null +++ b/testdata/cuda-oom-retry-study/armD/dev1/run-6.err @@ -0,0 +1,5 @@ +test_out_of_memory_returns_an_error_and_keeps_the_pool: CUDA error at ../core/src/cuda/picture_cuda.c:356: CUDA_ERROR_OUT_OF_MEMORY (2) in cuMemAllocPitch((CUdeviceptr *)&pic->data[i], (size_t *)&pic->stride[i], (size_t)pic->w[i] * ((pic->bpc + 7) / 8), pic->h[i], 8 << hbd) +libvmaf ERROR problem during prepare_ring_buffer +[silent-wait total 1 s -> probe 1] [hog 22954 MiB, luma probe under hog 0, first rc -12, memory back 1] [first frame with no memory left: -12] [retry rc 0] pass +1 tests run, 1 passed +rc=0 diff --git a/testdata/cuda-oom-retry-study/armD/dev1/run-6.out b/testdata/cuda-oom-retry-study/armD/dev1/run-6.out new file mode 100644 index 0000000000..e69de29bb2 diff --git a/testdata/cuda-oom-retry-study/armD/sampler.log b/testdata/cuda-oom-retry-study/armD/sampler.log new file mode 100644 index 0000000000..e1fb72ff52 --- /dev/null +++ b/testdata/cuda-oom-retry-study/armD/sampler.log @@ -0,0 +1,121 @@ +08:11:26 | 4060: 0 MiB, 7956 MiB | 5080: 878 MiB, 15100 MiB +08:11:31 | 4060: 0 MiB, 7956 MiB | 5080: 15894 MiB, 84 MiB +08:11:37 | 4060: 0 MiB, 7956 MiB | 5080: 15903 MiB, 75 MiB +08:11:43 | 4060: 0 MiB, 7956 MiB | 5080: 15900 MiB, 78 MiB +08:11:48 | 4060: 0 MiB, 7956 MiB | 5080: 15909 MiB, 69 MiB +08:11:54 | 4060: 0 MiB, 7956 MiB | 5080: 15914 MiB, 64 MiB +08:12:00 | 4060: 0 MiB, 7956 MiB | 5080: 15838 MiB, 140 MiB +08:12:06 | 4060: 0 MiB, 7956 MiB | 5080: 15845 MiB, 133 MiB +08:12:11 | 4060: 0 MiB, 7956 MiB | 5080: 15870 MiB, 108 MiB +08:12:17 | 4060: 0 MiB, 7956 MiB | 5080: 15849 MiB, 129 MiB +08:12:23 | 4060: 0 MiB, 7956 MiB | 5080: 15792 MiB, 186 MiB +08:12:29 | 4060: 0 MiB, 7956 MiB | 5080: 15798 MiB, 180 MiB +08:12:34 | 4060: 0 MiB, 7956 MiB | 5080: 15800 MiB, 178 MiB +08:12:41 | 4060: 0 MiB, 7956 MiB | 5080: 15816 MiB, 162 MiB +08:12:46 | 4060: 0 MiB, 7956 MiB | 5080: 15808 MiB, 170 MiB +08:12:52 | 4060: 0 MiB, 7956 MiB | 5080: 15771 MiB, 207 MiB +08:12:57 | 4060: 0 MiB, 7956 MiB | 5080: 15734 MiB, 244 MiB +08:13:03 | 4060: 0 MiB, 7956 MiB | 5080: 15735 MiB, 243 MiB +08:13:08 | 4060: 0 MiB, 7956 MiB | 5080: 15743 MiB, 235 MiB +08:13:15 | 4060: 0 MiB, 7956 MiB | 5080: 15734 MiB, 244 MiB +08:13:20 | 4060: 0 MiB, 7956 MiB | 5080: 15936 MiB, 42 MiB +08:13:25 | 4060: 0 MiB, 7956 MiB | 5080: 15842 MiB, 136 MiB +08:13:30 | 4060: 0 MiB, 7956 MiB | 5080: 15863 MiB, 115 MiB +08:13:35 | 4060: 0 MiB, 7956 MiB | 5080: 15899 MiB, 79 MiB +08:13:41 | 4060: 0 MiB, 7956 MiB | 5080: 15885 MiB, 93 MiB +08:13:46 | 4060: 0 MiB, 7956 MiB | 5080: 15884 MiB, 94 MiB +08:13:51 | 4060: 0 MiB, 7956 MiB | 5080: 15884 MiB, 94 MiB +08:13:57 | 4060: 0 MiB, 7956 MiB | 5080: 15893 MiB, 85 MiB +08:14:02 | 4060: 0 MiB, 7956 MiB | 5080: 15834 MiB, 144 MiB +08:14:09 | 4060: 0 MiB, 7956 MiB | 5080: 15834 MiB, 144 MiB +08:14:14 | 4060: 0 MiB, 7956 MiB | 5080: 15834 MiB, 144 MiB +08:14:20 | 4060: 0 MiB, 7956 MiB | 5080: 15834 MiB, 144 MiB +08:14:25 | 4060: 0 MiB, 7956 MiB | 5080: 15838 MiB, 140 MiB +08:14:31 | 4060: 0 MiB, 7956 MiB | 5080: 15829 MiB, 149 MiB +08:14:36 | 4060: 0 MiB, 7956 MiB | 5080: 15830 MiB, 148 MiB +08:14:42 | 4060: 0 MiB, 7956 MiB | 5080: 15830 MiB, 148 MiB +08:14:47 | 4060: 0 MiB, 7956 MiB | 5080: 15829 MiB, 149 MiB +08:14:54 | 4060: 0 MiB, 7956 MiB | 5080: 15830 MiB, 148 MiB +08:14:59 | 4060: 0 MiB, 7956 MiB | 5080: 15829 MiB, 149 MiB +08:15:05 | 4060: 0 MiB, 7956 MiB | 5080: 15830 MiB, 148 MiB +08:15:10 | 4060: 0 MiB, 7956 MiB | 5080: 15928 MiB, 50 MiB +08:15:16 | 4060: 0 MiB, 7956 MiB | 5080: 15869 MiB, 109 MiB +08:15:21 | 4060: 0 MiB, 7956 MiB | 5080: 15883 MiB, 95 MiB +08:15:26 | 4060: 0 MiB, 7956 MiB | 5080: 15888 MiB, 90 MiB +08:15:32 | 4060: 0 MiB, 7956 MiB | 5080: 15888 MiB, 90 MiB +08:15:38 | 4060: 0 MiB, 7956 MiB | 5080: 15907 MiB, 71 MiB +08:15:46 | 4060: 0 MiB, 7956 MiB | 5080: 15908 MiB, 70 MiB +08:16:00 | 4060: 0 MiB, 7956 MiB | 5080: 15890 MiB, 88 MiB +08:16:10 | 4060: 0 MiB, 7956 MiB | 5080: 15902 MiB, 76 MiB +08:16:17 | 4060: 0 MiB, 7956 MiB | 5080: 15921 MiB, 57 MiB +08:16:22 | 4060: 0 MiB, 7956 MiB | 5080: 15882 MiB, 96 MiB +08:16:27 | 4060: 0 MiB, 7956 MiB | 5080: 15887 MiB, 91 MiB +08:16:33 | 4060: 0 MiB, 7956 MiB | 5080: 15902 MiB, 76 MiB +08:16:38 | 4060: 0 MiB, 7956 MiB | 5080: 15871 MiB, 107 MiB +08:16:44 | 4060: 0 MiB, 7956 MiB | 5080: 15871 MiB, 107 MiB +08:16:49 | 4060: 0 MiB, 7956 MiB | 5080: 15858 MiB, 120 MiB +08:16:55 | 4060: 0 MiB, 7956 MiB | 5080: 15847 MiB, 131 MiB +08:17:00 | 4060: 0 MiB, 7956 MiB | 5080: 936 MiB, 15042 MiB +08:17:06 | 4060: 0 MiB, 7956 MiB | 5080: 15920 MiB, 58 MiB +08:17:11 | 4060: 0 MiB, 7956 MiB | 5080: 15910 MiB, 68 MiB +08:17:18 | 4060: 0 MiB, 7956 MiB | 5080: 15873 MiB, 105 MiB +08:17:23 | 4060: 0 MiB, 7956 MiB | 5080: 15654 MiB, 324 MiB +08:17:28 | 4060: 0 MiB, 7956 MiB | 5080: 15643 MiB, 335 MiB +08:17:34 | 4060: 0 MiB, 7956 MiB | 5080: 15648 MiB, 330 MiB +08:17:40 | 4060: 0 MiB, 7956 MiB | 5080: 15649 MiB, 329 MiB +08:17:45 | 4060: 0 MiB, 7956 MiB | 5080: 15654 MiB, 324 MiB +08:17:51 | 4060: 0 MiB, 7956 MiB | 5080: 15657 MiB, 321 MiB +08:17:56 | 4060: 0 MiB, 7956 MiB | 5080: 15645 MiB, 333 MiB +08:18:01 | 4060: 0 MiB, 7956 MiB | 5080: 15651 MiB, 327 MiB +08:18:07 | 4060: 0 MiB, 7956 MiB | 5080: 15653 MiB, 325 MiB +08:18:12 | 4060: 0 MiB, 7956 MiB | 5080: 15650 MiB, 328 MiB +08:18:18 | 4060: 0 MiB, 7956 MiB | 5080: 15647 MiB, 331 MiB +08:18:24 | 4060: 0 MiB, 7956 MiB | 5080: 15650 MiB, 328 MiB +08:18:29 | 4060: 0 MiB, 7956 MiB | 5080: 15655 MiB, 323 MiB +08:18:35 | 4060: 0 MiB, 7956 MiB | 5080: 15658 MiB, 320 MiB +08:18:40 | 4060: 0 MiB, 7956 MiB | 5080: 15664 MiB, 314 MiB +08:18:46 | 4060: 0 MiB, 7956 MiB | 5080: 15665 MiB, 313 MiB +08:18:52 | 4060: 0 MiB, 7956 MiB | 5080: 15666 MiB, 312 MiB +08:18:58 | 4060: 0 MiB, 7956 MiB | 5080: 15917 MiB, 61 MiB +08:19:04 | 4060: 0 MiB, 7956 MiB | 5080: 15859 MiB, 119 MiB +08:19:10 | 4060: 0 MiB, 7956 MiB | 5080: 15892 MiB, 86 MiB +08:19:16 | 4060: 0 MiB, 7956 MiB | 5080: 15901 MiB, 77 MiB +08:19:21 | 4060: 0 MiB, 7956 MiB | 5080: 15903 MiB, 75 MiB +08:19:27 | 4060: 0 MiB, 7956 MiB | 5080: 15905 MiB, 73 MiB +08:19:32 | 4060: 0 MiB, 7956 MiB | 5080: 15824 MiB, 154 MiB +08:19:38 | 4060: 0 MiB, 7956 MiB | 5080: 15828 MiB, 150 MiB +08:19:43 | 4060: 0 MiB, 7956 MiB | 5080: 15834 MiB, 144 MiB +08:19:48 | 4060: 0 MiB, 7956 MiB | 5080: 15834 MiB, 144 MiB +08:19:54 | 4060: 0 MiB, 7956 MiB | 5080: 15841 MiB, 137 MiB +08:20:00 | 4060: 0 MiB, 7956 MiB | 5080: 15845 MiB, 133 MiB +08:20:05 | 4060: 0 MiB, 7956 MiB | 5080: 15848 MiB, 130 MiB +08:20:11 | 4060: 0 MiB, 7956 MiB | 5080: 15866 MiB, 112 MiB +08:20:17 | 4060: 0 MiB, 7956 MiB | 5080: 15903 MiB, 75 MiB +08:20:24 | 4060: 0 MiB, 7956 MiB | 5080: 15904 MiB, 74 MiB +08:20:33 | 4060: 0 MiB, 7956 MiB | 5080: 15906 MiB, 72 MiB +08:20:42 | 4060: 0 MiB, 7956 MiB | 5080: 15914 MiB, 64 MiB +08:20:51 | 4060: 0 MiB, 7956 MiB | 5080: 15852 MiB, 126 MiB +08:20:59 | 4060: 0 MiB, 7956 MiB | 5080: 15924 MiB, 54 MiB +08:21:06 | 4060: 0 MiB, 7956 MiB | 5080: 15858 MiB, 120 MiB +08:21:12 | 4060: 0 MiB, 7956 MiB | 5080: 15624 MiB, 354 MiB +08:21:19 | 4060: 0 MiB, 7956 MiB | 5080: 15429 MiB, 549 MiB +08:21:26 | 4060: 0 MiB, 7956 MiB | 5080: 15438 MiB, 540 MiB +08:21:32 | 4060: 0 MiB, 7956 MiB | 5080: 15444 MiB, 534 MiB +08:21:38 | 4060: 0 MiB, 7956 MiB | 5080: 15478 MiB, 500 MiB +08:21:46 | 4060: 0 MiB, 7956 MiB | 5080: 15486 MiB, 492 MiB +08:21:52 | 4060: 0 MiB, 7956 MiB | 5080: 15524 MiB, 454 MiB +08:21:59 | 4060: 0 MiB, 7956 MiB | 5080: 15552 MiB, 426 MiB +08:22:08 | 4060: 0 MiB, 7956 MiB | 5080: 15528 MiB, 450 MiB +08:22:16 | 4060: 0 MiB, 7956 MiB | 5080: 15545 MiB, 433 MiB +08:22:23 | 4060: 0 MiB, 7956 MiB | 5080: 15533 MiB, 445 MiB +08:22:32 | 4060: 115 MiB, 7842 MiB | 5080: 15704 MiB, 274 MiB +08:22:39 | 4060: 115 MiB, 7842 MiB | 5080: 15613 MiB, 365 MiB +08:22:45 | 4060: 115 MiB, 7842 MiB | 5080: 1488 MiB, 14490 MiB +08:22:51 | 4060: 176 MiB, 7781 MiB | 5080: 1715 MiB, 14263 MiB +08:22:57 | 4060: 7926 MiB, 31 MiB | 5080: 1751 MiB, 14227 MiB +08:23:06 | 4060: 7926 MiB, 31 MiB | 5080: 1439 MiB, 14539 MiB +08:23:14 | 4060: 115 MiB, 7842 MiB | 5080: 1317 MiB, 14661 MiB +08:23:21 | 4060: 188 MiB, 7769 MiB | 5080: 1335 MiB, 14643 MiB +08:23:28 | 4060: 7878 MiB, 79 MiB | 5080: 1317 MiB, 14661 MiB +08:23:34 | 4060: 198 MiB, 7759 MiB | 5080: 1274 MiB, 14704 MiB +08:23:40 | 4060: 7878 MiB, 79 MiB | 5080: 1275 MiB, 14703 MiB diff --git a/testdata/cuda-oom-retry-study/armD/timeline.log b/testdata/cuda-oom-retry-study/armD/timeline.log new file mode 100644 index 0000000000..e146c7811c --- /dev/null +++ b/testdata/cuda-oom-retry-study/armD/timeline.log @@ -0,0 +1,25 @@ +=== armD dev0 run1 start 08:11:26 +=== armD dev0 run1 end 08:13:17 +=== armD dev0 run2 start 08:13:17 +=== armD dev0 run2 end 08:15:08 +=== armD dev0 run3 start 08:15:08 +=== armD dev0 run3 end 08:17:01 +=== armD dev0 run4 start 08:17:01 +=== armD dev0 run4 end 08:18:56 +=== armD dev0 run5 start 08:18:56 +=== armD dev0 run5 end 08:20:51 +=== armD dev0 run6 start 08:20:51 +=== armD dev0 run6 end 08:22:45 +=== armD dev1 run1 start 08:22:46 +=== armD dev1 run1 end 08:22:55 +=== armD dev1 run2 start 08:22:55 +=== armD dev1 run2 end 08:23:03 +=== armD dev1 run3 start 08:23:04 +=== armD dev1 run3 end 08:23:14 +=== armD dev1 run4 start 08:23:14 +=== armD dev1 run4 end 08:23:26 +=== armD dev1 run5 start 08:23:26 +=== armD dev1 run5 end 08:23:36 +=== armD dev1 run6 start 08:23:36 +=== armD dev1 run6 end 08:23:46 +ARM-D-DONE 08:23:46 diff --git a/testdata/cuda-oom-retry-study/ilv/dev0/run-1-aslanded.err b/testdata/cuda-oom-retry-study/ilv/dev0/run-1-aslanded.err new file mode 100644 index 0000000000..485fb06b1f --- /dev/null +++ b/testdata/cuda-oom-retry-study/ilv/dev0/run-1-aslanded.err @@ -0,0 +1,5 @@ +test_out_of_memory_returns_an_error_and_keeps_the_pool: CUDA error at ../core/src/libvmaf.c:650: CUDA_ERROR_OUT_OF_MEMORY (2) in cuEventSynchronize(priv->cuda.ready) +[hog 30836 MiB, luma probe under hog 0, first rc -12, memory back 0] [first frame with no memory left: -12] fail +device memory not yet returned after the hog was freed +1 tests run, 1 failed +rc=1 diff --git a/testdata/cuda-oom-retry-study/ilv/dev0/run-1-aslanded.out b/testdata/cuda-oom-retry-study/ilv/dev0/run-1-aslanded.out new file mode 100644 index 0000000000..e69de29bb2 diff --git a/testdata/cuda-oom-retry-study/ilv/dev0/run-1-noprobe.err b/testdata/cuda-oom-retry-study/ilv/dev0/run-1-noprobe.err new file mode 100644 index 0000000000..a67caafa5c --- /dev/null +++ b/testdata/cuda-oom-retry-study/ilv/dev0/run-1-noprobe.err @@ -0,0 +1,6 @@ +test_out_of_memory_returns_an_error_and_keeps_the_pool: CUDA error at ../core/src/libvmaf.c:650: CUDA_ERROR_OUT_OF_MEMORY (2) in cuEventSynchronize(priv->cuda.ready) +[hog 30836 MiB, luma probe under hog 0, first rc -12, memory back n/a] [first frame with no memory left: -12] CUDA error at ../core/src/libvmaf.c:650: CUDA_ERROR_OUT_OF_MEMORY (2) in cuEventSynchronize(priv->cuda.ready) +[retry rc -12] fail +the same index does not score once memory is back +1 tests run, 1 failed +rc=1 diff --git a/testdata/cuda-oom-retry-study/ilv/dev0/run-1-noprobe.out b/testdata/cuda-oom-retry-study/ilv/dev0/run-1-noprobe.out new file mode 100644 index 0000000000..e69de29bb2 diff --git a/testdata/cuda-oom-retry-study/ilv/dev0/run-2-aslanded.err b/testdata/cuda-oom-retry-study/ilv/dev0/run-2-aslanded.err new file mode 100644 index 0000000000..be95ebdd46 --- /dev/null +++ b/testdata/cuda-oom-retry-study/ilv/dev0/run-2-aslanded.err @@ -0,0 +1,5 @@ +test_out_of_memory_returns_an_error_and_keeps_the_pool: CUDA error at ../core/src/libvmaf.c:650: CUDA_ERROR_OUT_OF_MEMORY (2) in cuEventSynchronize(priv->cuda.ready) +[hog 30720 MiB, luma probe under hog 0, first rc -12, memory back 0] [first frame with no memory left: -12] fail +device memory not yet returned after the hog was freed +1 tests run, 1 failed +rc=1 diff --git a/testdata/cuda-oom-retry-study/ilv/dev0/run-2-aslanded.out b/testdata/cuda-oom-retry-study/ilv/dev0/run-2-aslanded.out new file mode 100644 index 0000000000..e69de29bb2 diff --git a/testdata/cuda-oom-retry-study/ilv/dev0/run-2-noprobe.err b/testdata/cuda-oom-retry-study/ilv/dev0/run-2-noprobe.err new file mode 100644 index 0000000000..a67caafa5c --- /dev/null +++ b/testdata/cuda-oom-retry-study/ilv/dev0/run-2-noprobe.err @@ -0,0 +1,6 @@ +test_out_of_memory_returns_an_error_and_keeps_the_pool: CUDA error at ../core/src/libvmaf.c:650: CUDA_ERROR_OUT_OF_MEMORY (2) in cuEventSynchronize(priv->cuda.ready) +[hog 30836 MiB, luma probe under hog 0, first rc -12, memory back n/a] [first frame with no memory left: -12] CUDA error at ../core/src/libvmaf.c:650: CUDA_ERROR_OUT_OF_MEMORY (2) in cuEventSynchronize(priv->cuda.ready) +[retry rc -12] fail +the same index does not score once memory is back +1 tests run, 1 failed +rc=1 diff --git a/testdata/cuda-oom-retry-study/ilv/dev0/run-2-noprobe.out b/testdata/cuda-oom-retry-study/ilv/dev0/run-2-noprobe.out new file mode 100644 index 0000000000..e69de29bb2 diff --git a/testdata/cuda-oom-retry-study/ilv/dev0/run-3-aslanded.err b/testdata/cuda-oom-retry-study/ilv/dev0/run-3-aslanded.err new file mode 100644 index 0000000000..485fb06b1f --- /dev/null +++ b/testdata/cuda-oom-retry-study/ilv/dev0/run-3-aslanded.err @@ -0,0 +1,5 @@ +test_out_of_memory_returns_an_error_and_keeps_the_pool: CUDA error at ../core/src/libvmaf.c:650: CUDA_ERROR_OUT_OF_MEMORY (2) in cuEventSynchronize(priv->cuda.ready) +[hog 30836 MiB, luma probe under hog 0, first rc -12, memory back 0] [first frame with no memory left: -12] fail +device memory not yet returned after the hog was freed +1 tests run, 1 failed +rc=1 diff --git a/testdata/cuda-oom-retry-study/ilv/dev0/run-3-aslanded.out b/testdata/cuda-oom-retry-study/ilv/dev0/run-3-aslanded.out new file mode 100644 index 0000000000..e69de29bb2 diff --git a/testdata/cuda-oom-retry-study/ilv/dev0/run-3-noprobe.err b/testdata/cuda-oom-retry-study/ilv/dev0/run-3-noprobe.err new file mode 100644 index 0000000000..a67caafa5c --- /dev/null +++ b/testdata/cuda-oom-retry-study/ilv/dev0/run-3-noprobe.err @@ -0,0 +1,6 @@ +test_out_of_memory_returns_an_error_and_keeps_the_pool: CUDA error at ../core/src/libvmaf.c:650: CUDA_ERROR_OUT_OF_MEMORY (2) in cuEventSynchronize(priv->cuda.ready) +[hog 30836 MiB, luma probe under hog 0, first rc -12, memory back n/a] [first frame with no memory left: -12] CUDA error at ../core/src/libvmaf.c:650: CUDA_ERROR_OUT_OF_MEMORY (2) in cuEventSynchronize(priv->cuda.ready) +[retry rc -12] fail +the same index does not score once memory is back +1 tests run, 1 failed +rc=1 diff --git a/testdata/cuda-oom-retry-study/ilv/dev0/run-3-noprobe.out b/testdata/cuda-oom-retry-study/ilv/dev0/run-3-noprobe.out new file mode 100644 index 0000000000..e69de29bb2 diff --git a/testdata/cuda-oom-retry-study/ilv/dev0/run-4-aslanded.err b/testdata/cuda-oom-retry-study/ilv/dev0/run-4-aslanded.err new file mode 100644 index 0000000000..be95ebdd46 --- /dev/null +++ b/testdata/cuda-oom-retry-study/ilv/dev0/run-4-aslanded.err @@ -0,0 +1,5 @@ +test_out_of_memory_returns_an_error_and_keeps_the_pool: CUDA error at ../core/src/libvmaf.c:650: CUDA_ERROR_OUT_OF_MEMORY (2) in cuEventSynchronize(priv->cuda.ready) +[hog 30720 MiB, luma probe under hog 0, first rc -12, memory back 0] [first frame with no memory left: -12] fail +device memory not yet returned after the hog was freed +1 tests run, 1 failed +rc=1 diff --git a/testdata/cuda-oom-retry-study/ilv/dev0/run-4-aslanded.out b/testdata/cuda-oom-retry-study/ilv/dev0/run-4-aslanded.out new file mode 100644 index 0000000000..e69de29bb2 diff --git a/testdata/cuda-oom-retry-study/ilv/dev0/run-4-noprobe.err b/testdata/cuda-oom-retry-study/ilv/dev0/run-4-noprobe.err new file mode 100644 index 0000000000..a67caafa5c --- /dev/null +++ b/testdata/cuda-oom-retry-study/ilv/dev0/run-4-noprobe.err @@ -0,0 +1,6 @@ +test_out_of_memory_returns_an_error_and_keeps_the_pool: CUDA error at ../core/src/libvmaf.c:650: CUDA_ERROR_OUT_OF_MEMORY (2) in cuEventSynchronize(priv->cuda.ready) +[hog 30836 MiB, luma probe under hog 0, first rc -12, memory back n/a] [first frame with no memory left: -12] CUDA error at ../core/src/libvmaf.c:650: CUDA_ERROR_OUT_OF_MEMORY (2) in cuEventSynchronize(priv->cuda.ready) +[retry rc -12] fail +the same index does not score once memory is back +1 tests run, 1 failed +rc=1 diff --git a/testdata/cuda-oom-retry-study/ilv/dev0/run-4-noprobe.out b/testdata/cuda-oom-retry-study/ilv/dev0/run-4-noprobe.out new file mode 100644 index 0000000000..e69de29bb2 diff --git a/testdata/cuda-oom-retry-study/ilv/dev0/run-5-aslanded.err b/testdata/cuda-oom-retry-study/ilv/dev0/run-5-aslanded.err new file mode 100644 index 0000000000..be95ebdd46 --- /dev/null +++ b/testdata/cuda-oom-retry-study/ilv/dev0/run-5-aslanded.err @@ -0,0 +1,5 @@ +test_out_of_memory_returns_an_error_and_keeps_the_pool: CUDA error at ../core/src/libvmaf.c:650: CUDA_ERROR_OUT_OF_MEMORY (2) in cuEventSynchronize(priv->cuda.ready) +[hog 30720 MiB, luma probe under hog 0, first rc -12, memory back 0] [first frame with no memory left: -12] fail +device memory not yet returned after the hog was freed +1 tests run, 1 failed +rc=1 diff --git a/testdata/cuda-oom-retry-study/ilv/dev0/run-5-aslanded.out b/testdata/cuda-oom-retry-study/ilv/dev0/run-5-aslanded.out new file mode 100644 index 0000000000..e69de29bb2 diff --git a/testdata/cuda-oom-retry-study/ilv/dev0/run-5-noprobe.err b/testdata/cuda-oom-retry-study/ilv/dev0/run-5-noprobe.err new file mode 100644 index 0000000000..de6e70bc25 --- /dev/null +++ b/testdata/cuda-oom-retry-study/ilv/dev0/run-5-noprobe.err @@ -0,0 +1,6 @@ +test_out_of_memory_returns_an_error_and_keeps_the_pool: CUDA error at ../core/src/libvmaf.c:650: CUDA_ERROR_OUT_OF_MEMORY (2) in cuEventSynchronize(priv->cuda.ready) +[hog 30720 MiB, luma probe under hog 0, first rc -12, memory back n/a] [first frame with no memory left: -12] CUDA error at ../core/src/libvmaf.c:650: CUDA_ERROR_OUT_OF_MEMORY (2) in cuEventSynchronize(priv->cuda.ready) +[retry rc -12] fail +the same index does not score once memory is back +1 tests run, 1 failed +rc=1 diff --git a/testdata/cuda-oom-retry-study/ilv/dev0/run-5-noprobe.out b/testdata/cuda-oom-retry-study/ilv/dev0/run-5-noprobe.out new file mode 100644 index 0000000000..e69de29bb2 diff --git a/testdata/cuda-oom-retry-study/ilv/dev0/run-6-aslanded.err b/testdata/cuda-oom-retry-study/ilv/dev0/run-6-aslanded.err new file mode 100644 index 0000000000..485fb06b1f --- /dev/null +++ b/testdata/cuda-oom-retry-study/ilv/dev0/run-6-aslanded.err @@ -0,0 +1,5 @@ +test_out_of_memory_returns_an_error_and_keeps_the_pool: CUDA error at ../core/src/libvmaf.c:650: CUDA_ERROR_OUT_OF_MEMORY (2) in cuEventSynchronize(priv->cuda.ready) +[hog 30836 MiB, luma probe under hog 0, first rc -12, memory back 0] [first frame with no memory left: -12] fail +device memory not yet returned after the hog was freed +1 tests run, 1 failed +rc=1 diff --git a/testdata/cuda-oom-retry-study/ilv/dev0/run-6-aslanded.out b/testdata/cuda-oom-retry-study/ilv/dev0/run-6-aslanded.out new file mode 100644 index 0000000000..e69de29bb2 diff --git a/testdata/cuda-oom-retry-study/ilv/dev0/run-6-noprobe.err b/testdata/cuda-oom-retry-study/ilv/dev0/run-6-noprobe.err new file mode 100644 index 0000000000..a67caafa5c --- /dev/null +++ b/testdata/cuda-oom-retry-study/ilv/dev0/run-6-noprobe.err @@ -0,0 +1,6 @@ +test_out_of_memory_returns_an_error_and_keeps_the_pool: CUDA error at ../core/src/libvmaf.c:650: CUDA_ERROR_OUT_OF_MEMORY (2) in cuEventSynchronize(priv->cuda.ready) +[hog 30836 MiB, luma probe under hog 0, first rc -12, memory back n/a] [first frame with no memory left: -12] CUDA error at ../core/src/libvmaf.c:650: CUDA_ERROR_OUT_OF_MEMORY (2) in cuEventSynchronize(priv->cuda.ready) +[retry rc -12] fail +the same index does not score once memory is back +1 tests run, 1 failed +rc=1 diff --git a/testdata/cuda-oom-retry-study/ilv/dev0/run-6-noprobe.out b/testdata/cuda-oom-retry-study/ilv/dev0/run-6-noprobe.out new file mode 100644 index 0000000000..e69de29bb2 diff --git a/testdata/cuda-oom-retry-study/ilv/dev1/run-01-aslanded.err b/testdata/cuda-oom-retry-study/ilv/dev1/run-01-aslanded.err new file mode 100644 index 0000000000..d8cce2991b --- /dev/null +++ b/testdata/cuda-oom-retry-study/ilv/dev1/run-01-aslanded.err @@ -0,0 +1,5 @@ +test_out_of_memory_returns_an_error_and_keeps_the_pool: CUDA error at ../core/src/cuda/picture_cuda.c:356: CUDA_ERROR_OUT_OF_MEMORY (2) in cuMemAllocPitch((CUdeviceptr *)&pic->data[i], (size_t *)&pic->stride[i], (size_t)pic->w[i] * ((pic->bpc + 7) / 8), pic->h[i], 8 << hbd) +libvmaf ERROR problem during prepare_ring_buffer +[hog 22954 MiB, luma probe under hog 0, first rc -12, memory back 1] [first frame with no memory left: -12] [retry rc 0] pass +1 tests run, 1 passed +rc=0 diff --git a/testdata/cuda-oom-retry-study/ilv/dev1/run-01-aslanded.out b/testdata/cuda-oom-retry-study/ilv/dev1/run-01-aslanded.out new file mode 100644 index 0000000000..e69de29bb2 diff --git a/testdata/cuda-oom-retry-study/ilv/dev1/run-01-noprobe.err b/testdata/cuda-oom-retry-study/ilv/dev1/run-01-noprobe.err new file mode 100644 index 0000000000..2c03ab018a --- /dev/null +++ b/testdata/cuda-oom-retry-study/ilv/dev1/run-01-noprobe.err @@ -0,0 +1,5 @@ +test_out_of_memory_returns_an_error_and_keeps_the_pool: CUDA error at ../core/src/cuda/picture_cuda.c:356: CUDA_ERROR_OUT_OF_MEMORY (2) in cuMemAllocPitch((CUdeviceptr *)&pic->data[i], (size_t *)&pic->stride[i], (size_t)pic->w[i] * ((pic->bpc + 7) / 8), pic->h[i], 8 << hbd) +libvmaf ERROR problem during prepare_ring_buffer +[hog 22954 MiB, luma probe under hog 0, first rc -12, memory back n/a] [first frame with no memory left: -12] [retry rc 0] pass +1 tests run, 1 passed +rc=0 diff --git a/testdata/cuda-oom-retry-study/ilv/dev1/run-01-noprobe.out b/testdata/cuda-oom-retry-study/ilv/dev1/run-01-noprobe.out new file mode 100644 index 0000000000..e69de29bb2 diff --git a/testdata/cuda-oom-retry-study/ilv/dev1/run-02-aslanded.err b/testdata/cuda-oom-retry-study/ilv/dev1/run-02-aslanded.err new file mode 100644 index 0000000000..d8cce2991b --- /dev/null +++ b/testdata/cuda-oom-retry-study/ilv/dev1/run-02-aslanded.err @@ -0,0 +1,5 @@ +test_out_of_memory_returns_an_error_and_keeps_the_pool: CUDA error at ../core/src/cuda/picture_cuda.c:356: CUDA_ERROR_OUT_OF_MEMORY (2) in cuMemAllocPitch((CUdeviceptr *)&pic->data[i], (size_t *)&pic->stride[i], (size_t)pic->w[i] * ((pic->bpc + 7) / 8), pic->h[i], 8 << hbd) +libvmaf ERROR problem during prepare_ring_buffer +[hog 22954 MiB, luma probe under hog 0, first rc -12, memory back 1] [first frame with no memory left: -12] [retry rc 0] pass +1 tests run, 1 passed +rc=0 diff --git a/testdata/cuda-oom-retry-study/ilv/dev1/run-02-aslanded.out b/testdata/cuda-oom-retry-study/ilv/dev1/run-02-aslanded.out new file mode 100644 index 0000000000..e69de29bb2 diff --git a/testdata/cuda-oom-retry-study/ilv/dev1/run-02-noprobe.err b/testdata/cuda-oom-retry-study/ilv/dev1/run-02-noprobe.err new file mode 100644 index 0000000000..2c03ab018a --- /dev/null +++ b/testdata/cuda-oom-retry-study/ilv/dev1/run-02-noprobe.err @@ -0,0 +1,5 @@ +test_out_of_memory_returns_an_error_and_keeps_the_pool: CUDA error at ../core/src/cuda/picture_cuda.c:356: CUDA_ERROR_OUT_OF_MEMORY (2) in cuMemAllocPitch((CUdeviceptr *)&pic->data[i], (size_t *)&pic->stride[i], (size_t)pic->w[i] * ((pic->bpc + 7) / 8), pic->h[i], 8 << hbd) +libvmaf ERROR problem during prepare_ring_buffer +[hog 22954 MiB, luma probe under hog 0, first rc -12, memory back n/a] [first frame with no memory left: -12] [retry rc 0] pass +1 tests run, 1 passed +rc=0 diff --git a/testdata/cuda-oom-retry-study/ilv/dev1/run-02-noprobe.out b/testdata/cuda-oom-retry-study/ilv/dev1/run-02-noprobe.out new file mode 100644 index 0000000000..e69de29bb2 diff --git a/testdata/cuda-oom-retry-study/ilv/dev1/run-03-aslanded.err b/testdata/cuda-oom-retry-study/ilv/dev1/run-03-aslanded.err new file mode 100644 index 0000000000..d8cce2991b --- /dev/null +++ b/testdata/cuda-oom-retry-study/ilv/dev1/run-03-aslanded.err @@ -0,0 +1,5 @@ +test_out_of_memory_returns_an_error_and_keeps_the_pool: CUDA error at ../core/src/cuda/picture_cuda.c:356: CUDA_ERROR_OUT_OF_MEMORY (2) in cuMemAllocPitch((CUdeviceptr *)&pic->data[i], (size_t *)&pic->stride[i], (size_t)pic->w[i] * ((pic->bpc + 7) / 8), pic->h[i], 8 << hbd) +libvmaf ERROR problem during prepare_ring_buffer +[hog 22954 MiB, luma probe under hog 0, first rc -12, memory back 1] [first frame with no memory left: -12] [retry rc 0] pass +1 tests run, 1 passed +rc=0 diff --git a/testdata/cuda-oom-retry-study/ilv/dev1/run-03-aslanded.out b/testdata/cuda-oom-retry-study/ilv/dev1/run-03-aslanded.out new file mode 100644 index 0000000000..e69de29bb2 diff --git a/testdata/cuda-oom-retry-study/ilv/dev1/run-03-noprobe.err b/testdata/cuda-oom-retry-study/ilv/dev1/run-03-noprobe.err new file mode 100644 index 0000000000..2c03ab018a --- /dev/null +++ b/testdata/cuda-oom-retry-study/ilv/dev1/run-03-noprobe.err @@ -0,0 +1,5 @@ +test_out_of_memory_returns_an_error_and_keeps_the_pool: CUDA error at ../core/src/cuda/picture_cuda.c:356: CUDA_ERROR_OUT_OF_MEMORY (2) in cuMemAllocPitch((CUdeviceptr *)&pic->data[i], (size_t *)&pic->stride[i], (size_t)pic->w[i] * ((pic->bpc + 7) / 8), pic->h[i], 8 << hbd) +libvmaf ERROR problem during prepare_ring_buffer +[hog 22954 MiB, luma probe under hog 0, first rc -12, memory back n/a] [first frame with no memory left: -12] [retry rc 0] pass +1 tests run, 1 passed +rc=0 diff --git a/testdata/cuda-oom-retry-study/ilv/dev1/run-03-noprobe.out b/testdata/cuda-oom-retry-study/ilv/dev1/run-03-noprobe.out new file mode 100644 index 0000000000..e69de29bb2 diff --git a/testdata/cuda-oom-retry-study/ilv/dev1/run-04-aslanded.err b/testdata/cuda-oom-retry-study/ilv/dev1/run-04-aslanded.err new file mode 100644 index 0000000000..d8cce2991b --- /dev/null +++ b/testdata/cuda-oom-retry-study/ilv/dev1/run-04-aslanded.err @@ -0,0 +1,5 @@ +test_out_of_memory_returns_an_error_and_keeps_the_pool: CUDA error at ../core/src/cuda/picture_cuda.c:356: CUDA_ERROR_OUT_OF_MEMORY (2) in cuMemAllocPitch((CUdeviceptr *)&pic->data[i], (size_t *)&pic->stride[i], (size_t)pic->w[i] * ((pic->bpc + 7) / 8), pic->h[i], 8 << hbd) +libvmaf ERROR problem during prepare_ring_buffer +[hog 22954 MiB, luma probe under hog 0, first rc -12, memory back 1] [first frame with no memory left: -12] [retry rc 0] pass +1 tests run, 1 passed +rc=0 diff --git a/testdata/cuda-oom-retry-study/ilv/dev1/run-04-aslanded.out b/testdata/cuda-oom-retry-study/ilv/dev1/run-04-aslanded.out new file mode 100644 index 0000000000..e69de29bb2 diff --git a/testdata/cuda-oom-retry-study/ilv/dev1/run-04-noprobe.err b/testdata/cuda-oom-retry-study/ilv/dev1/run-04-noprobe.err new file mode 100644 index 0000000000..2c03ab018a --- /dev/null +++ b/testdata/cuda-oom-retry-study/ilv/dev1/run-04-noprobe.err @@ -0,0 +1,5 @@ +test_out_of_memory_returns_an_error_and_keeps_the_pool: CUDA error at ../core/src/cuda/picture_cuda.c:356: CUDA_ERROR_OUT_OF_MEMORY (2) in cuMemAllocPitch((CUdeviceptr *)&pic->data[i], (size_t *)&pic->stride[i], (size_t)pic->w[i] * ((pic->bpc + 7) / 8), pic->h[i], 8 << hbd) +libvmaf ERROR problem during prepare_ring_buffer +[hog 22954 MiB, luma probe under hog 0, first rc -12, memory back n/a] [first frame with no memory left: -12] [retry rc 0] pass +1 tests run, 1 passed +rc=0 diff --git a/testdata/cuda-oom-retry-study/ilv/dev1/run-04-noprobe.out b/testdata/cuda-oom-retry-study/ilv/dev1/run-04-noprobe.out new file mode 100644 index 0000000000..e69de29bb2 diff --git a/testdata/cuda-oom-retry-study/ilv/dev1/run-05-aslanded.err b/testdata/cuda-oom-retry-study/ilv/dev1/run-05-aslanded.err new file mode 100644 index 0000000000..d8cce2991b --- /dev/null +++ b/testdata/cuda-oom-retry-study/ilv/dev1/run-05-aslanded.err @@ -0,0 +1,5 @@ +test_out_of_memory_returns_an_error_and_keeps_the_pool: CUDA error at ../core/src/cuda/picture_cuda.c:356: CUDA_ERROR_OUT_OF_MEMORY (2) in cuMemAllocPitch((CUdeviceptr *)&pic->data[i], (size_t *)&pic->stride[i], (size_t)pic->w[i] * ((pic->bpc + 7) / 8), pic->h[i], 8 << hbd) +libvmaf ERROR problem during prepare_ring_buffer +[hog 22954 MiB, luma probe under hog 0, first rc -12, memory back 1] [first frame with no memory left: -12] [retry rc 0] pass +1 tests run, 1 passed +rc=0 diff --git a/testdata/cuda-oom-retry-study/ilv/dev1/run-05-aslanded.out b/testdata/cuda-oom-retry-study/ilv/dev1/run-05-aslanded.out new file mode 100644 index 0000000000..e69de29bb2 diff --git a/testdata/cuda-oom-retry-study/ilv/dev1/run-05-noprobe.err b/testdata/cuda-oom-retry-study/ilv/dev1/run-05-noprobe.err new file mode 100644 index 0000000000..2c03ab018a --- /dev/null +++ b/testdata/cuda-oom-retry-study/ilv/dev1/run-05-noprobe.err @@ -0,0 +1,5 @@ +test_out_of_memory_returns_an_error_and_keeps_the_pool: CUDA error at ../core/src/cuda/picture_cuda.c:356: CUDA_ERROR_OUT_OF_MEMORY (2) in cuMemAllocPitch((CUdeviceptr *)&pic->data[i], (size_t *)&pic->stride[i], (size_t)pic->w[i] * ((pic->bpc + 7) / 8), pic->h[i], 8 << hbd) +libvmaf ERROR problem during prepare_ring_buffer +[hog 22954 MiB, luma probe under hog 0, first rc -12, memory back n/a] [first frame with no memory left: -12] [retry rc 0] pass +1 tests run, 1 passed +rc=0 diff --git a/testdata/cuda-oom-retry-study/ilv/dev1/run-05-noprobe.out b/testdata/cuda-oom-retry-study/ilv/dev1/run-05-noprobe.out new file mode 100644 index 0000000000..e69de29bb2 diff --git a/testdata/cuda-oom-retry-study/ilv/dev1/run-06-aslanded.err b/testdata/cuda-oom-retry-study/ilv/dev1/run-06-aslanded.err new file mode 100644 index 0000000000..d8cce2991b --- /dev/null +++ b/testdata/cuda-oom-retry-study/ilv/dev1/run-06-aslanded.err @@ -0,0 +1,5 @@ +test_out_of_memory_returns_an_error_and_keeps_the_pool: CUDA error at ../core/src/cuda/picture_cuda.c:356: CUDA_ERROR_OUT_OF_MEMORY (2) in cuMemAllocPitch((CUdeviceptr *)&pic->data[i], (size_t *)&pic->stride[i], (size_t)pic->w[i] * ((pic->bpc + 7) / 8), pic->h[i], 8 << hbd) +libvmaf ERROR problem during prepare_ring_buffer +[hog 22954 MiB, luma probe under hog 0, first rc -12, memory back 1] [first frame with no memory left: -12] [retry rc 0] pass +1 tests run, 1 passed +rc=0 diff --git a/testdata/cuda-oom-retry-study/ilv/dev1/run-06-aslanded.out b/testdata/cuda-oom-retry-study/ilv/dev1/run-06-aslanded.out new file mode 100644 index 0000000000..e69de29bb2 diff --git a/testdata/cuda-oom-retry-study/ilv/dev1/run-06-noprobe.err b/testdata/cuda-oom-retry-study/ilv/dev1/run-06-noprobe.err new file mode 100644 index 0000000000..2c03ab018a --- /dev/null +++ b/testdata/cuda-oom-retry-study/ilv/dev1/run-06-noprobe.err @@ -0,0 +1,5 @@ +test_out_of_memory_returns_an_error_and_keeps_the_pool: CUDA error at ../core/src/cuda/picture_cuda.c:356: CUDA_ERROR_OUT_OF_MEMORY (2) in cuMemAllocPitch((CUdeviceptr *)&pic->data[i], (size_t *)&pic->stride[i], (size_t)pic->w[i] * ((pic->bpc + 7) / 8), pic->h[i], 8 << hbd) +libvmaf ERROR problem during prepare_ring_buffer +[hog 22954 MiB, luma probe under hog 0, first rc -12, memory back n/a] [first frame with no memory left: -12] [retry rc 0] pass +1 tests run, 1 passed +rc=0 diff --git a/testdata/cuda-oom-retry-study/ilv/dev1/run-06-noprobe.out b/testdata/cuda-oom-retry-study/ilv/dev1/run-06-noprobe.out new file mode 100644 index 0000000000..e69de29bb2 diff --git a/testdata/cuda-oom-retry-study/ilv/dev1/run-07-aslanded.err b/testdata/cuda-oom-retry-study/ilv/dev1/run-07-aslanded.err new file mode 100644 index 0000000000..d8cce2991b --- /dev/null +++ b/testdata/cuda-oom-retry-study/ilv/dev1/run-07-aslanded.err @@ -0,0 +1,5 @@ +test_out_of_memory_returns_an_error_and_keeps_the_pool: CUDA error at ../core/src/cuda/picture_cuda.c:356: CUDA_ERROR_OUT_OF_MEMORY (2) in cuMemAllocPitch((CUdeviceptr *)&pic->data[i], (size_t *)&pic->stride[i], (size_t)pic->w[i] * ((pic->bpc + 7) / 8), pic->h[i], 8 << hbd) +libvmaf ERROR problem during prepare_ring_buffer +[hog 22954 MiB, luma probe under hog 0, first rc -12, memory back 1] [first frame with no memory left: -12] [retry rc 0] pass +1 tests run, 1 passed +rc=0 diff --git a/testdata/cuda-oom-retry-study/ilv/dev1/run-07-aslanded.out b/testdata/cuda-oom-retry-study/ilv/dev1/run-07-aslanded.out new file mode 100644 index 0000000000..e69de29bb2 diff --git a/testdata/cuda-oom-retry-study/ilv/dev1/run-07-noprobe.err b/testdata/cuda-oom-retry-study/ilv/dev1/run-07-noprobe.err new file mode 100644 index 0000000000..2c03ab018a --- /dev/null +++ b/testdata/cuda-oom-retry-study/ilv/dev1/run-07-noprobe.err @@ -0,0 +1,5 @@ +test_out_of_memory_returns_an_error_and_keeps_the_pool: CUDA error at ../core/src/cuda/picture_cuda.c:356: CUDA_ERROR_OUT_OF_MEMORY (2) in cuMemAllocPitch((CUdeviceptr *)&pic->data[i], (size_t *)&pic->stride[i], (size_t)pic->w[i] * ((pic->bpc + 7) / 8), pic->h[i], 8 << hbd) +libvmaf ERROR problem during prepare_ring_buffer +[hog 22954 MiB, luma probe under hog 0, first rc -12, memory back n/a] [first frame with no memory left: -12] [retry rc 0] pass +1 tests run, 1 passed +rc=0 diff --git a/testdata/cuda-oom-retry-study/ilv/dev1/run-07-noprobe.out b/testdata/cuda-oom-retry-study/ilv/dev1/run-07-noprobe.out new file mode 100644 index 0000000000..e69de29bb2 diff --git a/testdata/cuda-oom-retry-study/ilv/dev1/run-08-aslanded.err b/testdata/cuda-oom-retry-study/ilv/dev1/run-08-aslanded.err new file mode 100644 index 0000000000..d8cce2991b --- /dev/null +++ b/testdata/cuda-oom-retry-study/ilv/dev1/run-08-aslanded.err @@ -0,0 +1,5 @@ +test_out_of_memory_returns_an_error_and_keeps_the_pool: CUDA error at ../core/src/cuda/picture_cuda.c:356: CUDA_ERROR_OUT_OF_MEMORY (2) in cuMemAllocPitch((CUdeviceptr *)&pic->data[i], (size_t *)&pic->stride[i], (size_t)pic->w[i] * ((pic->bpc + 7) / 8), pic->h[i], 8 << hbd) +libvmaf ERROR problem during prepare_ring_buffer +[hog 22954 MiB, luma probe under hog 0, first rc -12, memory back 1] [first frame with no memory left: -12] [retry rc 0] pass +1 tests run, 1 passed +rc=0 diff --git a/testdata/cuda-oom-retry-study/ilv/dev1/run-08-aslanded.out b/testdata/cuda-oom-retry-study/ilv/dev1/run-08-aslanded.out new file mode 100644 index 0000000000..e69de29bb2 diff --git a/testdata/cuda-oom-retry-study/ilv/dev1/run-08-noprobe.err b/testdata/cuda-oom-retry-study/ilv/dev1/run-08-noprobe.err new file mode 100644 index 0000000000..2c03ab018a --- /dev/null +++ b/testdata/cuda-oom-retry-study/ilv/dev1/run-08-noprobe.err @@ -0,0 +1,5 @@ +test_out_of_memory_returns_an_error_and_keeps_the_pool: CUDA error at ../core/src/cuda/picture_cuda.c:356: CUDA_ERROR_OUT_OF_MEMORY (2) in cuMemAllocPitch((CUdeviceptr *)&pic->data[i], (size_t *)&pic->stride[i], (size_t)pic->w[i] * ((pic->bpc + 7) / 8), pic->h[i], 8 << hbd) +libvmaf ERROR problem during prepare_ring_buffer +[hog 22954 MiB, luma probe under hog 0, first rc -12, memory back n/a] [first frame with no memory left: -12] [retry rc 0] pass +1 tests run, 1 passed +rc=0 diff --git a/testdata/cuda-oom-retry-study/ilv/dev1/run-08-noprobe.out b/testdata/cuda-oom-retry-study/ilv/dev1/run-08-noprobe.out new file mode 100644 index 0000000000..e69de29bb2 diff --git a/testdata/cuda-oom-retry-study/ilv/dev1/run-09-aslanded.err b/testdata/cuda-oom-retry-study/ilv/dev1/run-09-aslanded.err new file mode 100644 index 0000000000..d8cce2991b --- /dev/null +++ b/testdata/cuda-oom-retry-study/ilv/dev1/run-09-aslanded.err @@ -0,0 +1,5 @@ +test_out_of_memory_returns_an_error_and_keeps_the_pool: CUDA error at ../core/src/cuda/picture_cuda.c:356: CUDA_ERROR_OUT_OF_MEMORY (2) in cuMemAllocPitch((CUdeviceptr *)&pic->data[i], (size_t *)&pic->stride[i], (size_t)pic->w[i] * ((pic->bpc + 7) / 8), pic->h[i], 8 << hbd) +libvmaf ERROR problem during prepare_ring_buffer +[hog 22954 MiB, luma probe under hog 0, first rc -12, memory back 1] [first frame with no memory left: -12] [retry rc 0] pass +1 tests run, 1 passed +rc=0 diff --git a/testdata/cuda-oom-retry-study/ilv/dev1/run-09-aslanded.out b/testdata/cuda-oom-retry-study/ilv/dev1/run-09-aslanded.out new file mode 100644 index 0000000000..e69de29bb2 diff --git a/testdata/cuda-oom-retry-study/ilv/dev1/run-09-noprobe.err b/testdata/cuda-oom-retry-study/ilv/dev1/run-09-noprobe.err new file mode 100644 index 0000000000..2c03ab018a --- /dev/null +++ b/testdata/cuda-oom-retry-study/ilv/dev1/run-09-noprobe.err @@ -0,0 +1,5 @@ +test_out_of_memory_returns_an_error_and_keeps_the_pool: CUDA error at ../core/src/cuda/picture_cuda.c:356: CUDA_ERROR_OUT_OF_MEMORY (2) in cuMemAllocPitch((CUdeviceptr *)&pic->data[i], (size_t *)&pic->stride[i], (size_t)pic->w[i] * ((pic->bpc + 7) / 8), pic->h[i], 8 << hbd) +libvmaf ERROR problem during prepare_ring_buffer +[hog 22954 MiB, luma probe under hog 0, first rc -12, memory back n/a] [first frame with no memory left: -12] [retry rc 0] pass +1 tests run, 1 passed +rc=0 diff --git a/testdata/cuda-oom-retry-study/ilv/dev1/run-09-noprobe.out b/testdata/cuda-oom-retry-study/ilv/dev1/run-09-noprobe.out new file mode 100644 index 0000000000..e69de29bb2 diff --git a/testdata/cuda-oom-retry-study/ilv/dev1/run-10-aslanded.err b/testdata/cuda-oom-retry-study/ilv/dev1/run-10-aslanded.err new file mode 100644 index 0000000000..941721e7a5 --- /dev/null +++ b/testdata/cuda-oom-retry-study/ilv/dev1/run-10-aslanded.err @@ -0,0 +1,6 @@ +test_out_of_memory_returns_an_error_and_keeps_the_pool: CUDA error at ../core/src/cuda/picture_cuda.c:356: CUDA_ERROR_OUT_OF_MEMORY (2) in cuMemAllocPitch((CUdeviceptr *)&pic->data[i], (size_t *)&pic->stride[i], (size_t)pic->w[i] * ((pic->bpc + 7) / 8), pic->h[i], 8 << hbd) +libvmaf ERROR problem during prepare_ring_buffer +[hog 22954 MiB, luma probe under hog 0, first rc -12, memory back 0] [first frame with no memory left: -12] fail +device memory not yet returned after the hog was freed +1 tests run, 1 failed +rc=1 diff --git a/testdata/cuda-oom-retry-study/ilv/dev1/run-10-aslanded.out b/testdata/cuda-oom-retry-study/ilv/dev1/run-10-aslanded.out new file mode 100644 index 0000000000..e69de29bb2 diff --git a/testdata/cuda-oom-retry-study/ilv/dev1/run-10-noprobe.err b/testdata/cuda-oom-retry-study/ilv/dev1/run-10-noprobe.err new file mode 100644 index 0000000000..2c03ab018a --- /dev/null +++ b/testdata/cuda-oom-retry-study/ilv/dev1/run-10-noprobe.err @@ -0,0 +1,5 @@ +test_out_of_memory_returns_an_error_and_keeps_the_pool: CUDA error at ../core/src/cuda/picture_cuda.c:356: CUDA_ERROR_OUT_OF_MEMORY (2) in cuMemAllocPitch((CUdeviceptr *)&pic->data[i], (size_t *)&pic->stride[i], (size_t)pic->w[i] * ((pic->bpc + 7) / 8), pic->h[i], 8 << hbd) +libvmaf ERROR problem during prepare_ring_buffer +[hog 22954 MiB, luma probe under hog 0, first rc -12, memory back n/a] [first frame with no memory left: -12] [retry rc 0] pass +1 tests run, 1 passed +rc=0 diff --git a/testdata/cuda-oom-retry-study/ilv/dev1/run-10-noprobe.out b/testdata/cuda-oom-retry-study/ilv/dev1/run-10-noprobe.out new file mode 100644 index 0000000000..e69de29bb2 diff --git a/testdata/cuda-oom-retry-study/ilv/dev1/run-11-aslanded.err b/testdata/cuda-oom-retry-study/ilv/dev1/run-11-aslanded.err new file mode 100644 index 0000000000..d8cce2991b --- /dev/null +++ b/testdata/cuda-oom-retry-study/ilv/dev1/run-11-aslanded.err @@ -0,0 +1,5 @@ +test_out_of_memory_returns_an_error_and_keeps_the_pool: CUDA error at ../core/src/cuda/picture_cuda.c:356: CUDA_ERROR_OUT_OF_MEMORY (2) in cuMemAllocPitch((CUdeviceptr *)&pic->data[i], (size_t *)&pic->stride[i], (size_t)pic->w[i] * ((pic->bpc + 7) / 8), pic->h[i], 8 << hbd) +libvmaf ERROR problem during prepare_ring_buffer +[hog 22954 MiB, luma probe under hog 0, first rc -12, memory back 1] [first frame with no memory left: -12] [retry rc 0] pass +1 tests run, 1 passed +rc=0 diff --git a/testdata/cuda-oom-retry-study/ilv/dev1/run-11-aslanded.out b/testdata/cuda-oom-retry-study/ilv/dev1/run-11-aslanded.out new file mode 100644 index 0000000000..e69de29bb2 diff --git a/testdata/cuda-oom-retry-study/ilv/dev1/run-11-noprobe.err b/testdata/cuda-oom-retry-study/ilv/dev1/run-11-noprobe.err new file mode 100644 index 0000000000..2c03ab018a --- /dev/null +++ b/testdata/cuda-oom-retry-study/ilv/dev1/run-11-noprobe.err @@ -0,0 +1,5 @@ +test_out_of_memory_returns_an_error_and_keeps_the_pool: CUDA error at ../core/src/cuda/picture_cuda.c:356: CUDA_ERROR_OUT_OF_MEMORY (2) in cuMemAllocPitch((CUdeviceptr *)&pic->data[i], (size_t *)&pic->stride[i], (size_t)pic->w[i] * ((pic->bpc + 7) / 8), pic->h[i], 8 << hbd) +libvmaf ERROR problem during prepare_ring_buffer +[hog 22954 MiB, luma probe under hog 0, first rc -12, memory back n/a] [first frame with no memory left: -12] [retry rc 0] pass +1 tests run, 1 passed +rc=0 diff --git a/testdata/cuda-oom-retry-study/ilv/dev1/run-11-noprobe.out b/testdata/cuda-oom-retry-study/ilv/dev1/run-11-noprobe.out new file mode 100644 index 0000000000..e69de29bb2 diff --git a/testdata/cuda-oom-retry-study/ilv/dev1/run-12-aslanded.err b/testdata/cuda-oom-retry-study/ilv/dev1/run-12-aslanded.err new file mode 100644 index 0000000000..d8cce2991b --- /dev/null +++ b/testdata/cuda-oom-retry-study/ilv/dev1/run-12-aslanded.err @@ -0,0 +1,5 @@ +test_out_of_memory_returns_an_error_and_keeps_the_pool: CUDA error at ../core/src/cuda/picture_cuda.c:356: CUDA_ERROR_OUT_OF_MEMORY (2) in cuMemAllocPitch((CUdeviceptr *)&pic->data[i], (size_t *)&pic->stride[i], (size_t)pic->w[i] * ((pic->bpc + 7) / 8), pic->h[i], 8 << hbd) +libvmaf ERROR problem during prepare_ring_buffer +[hog 22954 MiB, luma probe under hog 0, first rc -12, memory back 1] [first frame with no memory left: -12] [retry rc 0] pass +1 tests run, 1 passed +rc=0 diff --git a/testdata/cuda-oom-retry-study/ilv/dev1/run-12-aslanded.out b/testdata/cuda-oom-retry-study/ilv/dev1/run-12-aslanded.out new file mode 100644 index 0000000000..e69de29bb2 diff --git a/testdata/cuda-oom-retry-study/ilv/dev1/run-12-noprobe.err b/testdata/cuda-oom-retry-study/ilv/dev1/run-12-noprobe.err new file mode 100644 index 0000000000..2c03ab018a --- /dev/null +++ b/testdata/cuda-oom-retry-study/ilv/dev1/run-12-noprobe.err @@ -0,0 +1,5 @@ +test_out_of_memory_returns_an_error_and_keeps_the_pool: CUDA error at ../core/src/cuda/picture_cuda.c:356: CUDA_ERROR_OUT_OF_MEMORY (2) in cuMemAllocPitch((CUdeviceptr *)&pic->data[i], (size_t *)&pic->stride[i], (size_t)pic->w[i] * ((pic->bpc + 7) / 8), pic->h[i], 8 << hbd) +libvmaf ERROR problem during prepare_ring_buffer +[hog 22954 MiB, luma probe under hog 0, first rc -12, memory back n/a] [first frame with no memory left: -12] [retry rc 0] pass +1 tests run, 1 passed +rc=0 diff --git a/testdata/cuda-oom-retry-study/ilv/dev1/run-12-noprobe.out b/testdata/cuda-oom-retry-study/ilv/dev1/run-12-noprobe.out new file mode 100644 index 0000000000..e69de29bb2 diff --git a/testdata/cuda-oom-retry-study/ilv/timeline.log b/testdata/cuda-oom-retry-study/ilv/timeline.log new file mode 100644 index 0000000000..4f277898b0 --- /dev/null +++ b/testdata/cuda-oom-retry-study/ilv/timeline.log @@ -0,0 +1,73 @@ +=== ilv dev1 run01 aslanded start 08:02:48 +775 MiB, 2 % +=== ilv dev1 run01 noprobe start 08:03:00 +869 MiB, 2 % +=== ilv dev1 run02 aslanded start 08:03:11 +860 MiB, 2 % +=== ilv dev1 run02 noprobe start 08:03:17 +860 MiB, 3 % +=== ilv dev1 run03 aslanded start 08:03:23 +864 MiB, 2 % +=== ilv dev1 run03 noprobe start 08:03:27 +873 MiB, 3 % +=== ilv dev1 run04 aslanded start 08:03:31 +872 MiB, 19 % +=== ilv dev1 run04 noprobe start 08:03:35 +867 MiB, 3 % +=== ilv dev1 run05 aslanded start 08:03:38 +867 MiB, 2 % +=== ilv dev1 run05 noprobe start 08:03:42 +867 MiB, 2 % +=== ilv dev1 run06 aslanded start 08:03:46 +867 MiB, 2 % +=== ilv dev1 run06 noprobe start 08:03:51 +867 MiB, 36 % +=== ilv dev1 run07 aslanded start 08:03:55 +867 MiB, 2 % +=== ilv dev1 run07 noprobe start 08:04:00 +867 MiB, 3 % +=== ilv dev1 run08 aslanded start 08:04:06 +867 MiB, 7 % +=== ilv dev1 run08 noprobe start 08:04:12 +867 MiB, 2 % +=== ilv dev1 run09 aslanded start 08:04:18 +867 MiB, 2 % +=== ilv dev1 run09 noprobe start 08:04:23 +876 MiB, 2 % +=== ilv dev1 run10 aslanded start 08:04:30 +899 MiB, 23 % +=== ilv dev1 run10 noprobe start 08:04:34 +889 MiB, 22 % +=== ilv dev1 run11 aslanded start 08:04:38 +885 MiB, 23 % +=== ilv dev1 run11 noprobe start 08:04:43 +884 MiB, 22 % +=== ilv dev1 run12 aslanded start 08:04:48 +894 MiB, 23 % +=== ilv dev1 run12 noprobe start 08:04:52 +892 MiB, 22 % +=== ilv dev0 run1 aslanded start 08:04:56 +0 MiB, 0 % +=== ilv dev0 run1 noprobe start 08:05:02 +0 MiB, 0 % +=== ilv dev0 run2 aslanded start 08:05:07 +0 MiB, 0 % +=== ilv dev0 run2 noprobe start 08:05:12 +0 MiB, 0 % +=== ilv dev0 run3 aslanded start 08:05:17 +0 MiB, 0 % +=== ilv dev0 run3 noprobe start 08:05:23 +0 MiB, 0 % +=== ilv dev0 run4 aslanded start 08:05:28 +0 MiB, 0 % +=== ilv dev0 run4 noprobe start 08:05:36 +0 MiB, 0 % +=== ilv dev0 run5 aslanded start 08:05:43 +0 MiB, 0 % +=== ilv dev0 run5 noprobe start 08:05:51 +0 MiB, 0 % +=== ilv dev0 run6 aslanded start 08:05:59 +0 MiB, 0 % +=== ilv dev0 run6 noprobe start 08:06:04 +0 MiB, 0 % +ILV-DONE 08:06:09 diff --git a/testdata/cuda-oom-retry-study/run_armA.sh b/testdata/cuda-oom-retry-study/run_armA.sh new file mode 100644 index 0000000000..b1551a26d6 --- /dev/null +++ b/testdata/cuda-oom-retry-study/run_armA.sh @@ -0,0 +1,23 @@ +#!/bin/bash +# Arm A campaign — as-landed #3320 test, 12 rounds per device, out-of-band memory logs. +export PATH="/c/msys64/mingw64/bin:/c/Program Files/NVIDIA GPU Computing Toolkit/CUDA/v13.4/bin:$PATH" +cd /c/PLAN/VMAF/worktrees/_oom-master || exit 2 +BASE=/c/PLAN/VMAF/verify/tmp/oom-r20 +mkdir -p "$BASE/armA/dev0" "$BASE/armA/dev1" +: > "$BASE/armA/timeline.log" +for DEV in 0 1; do + for i in $(seq -w 1 12); do + { + echo "=== armA dev$DEV run$i start $(date -u +%Y-%m-%dT%H:%M:%SZ)" + nvidia-smi -i "$DEV" --query-gpu=memory.used,utilization.gpu --format=csv,noheader + } >> "$BASE/armA/timeline.log" + CUDA_VISIBLE_DEVICES=$DEV ./build/test/test_cuda_oom_pictures_released.exe \ + > "$BASE/armA/dev$DEV/run-$i.out" 2> "$BASE/armA/dev$DEV/run-$i.err" + echo "rc=$?" >> "$BASE/armA/dev$DEV/run-$i.err" + { + echo "=== armA dev$DEV run$i end $(date -u +%Y-%m-%dT%H:%M:%SZ)" + nvidia-smi -i "$DEV" --query-gpu=memory.used,utilization.gpu --format=csv,noheader + } >> "$BASE/armA/timeline.log" + done +done +echo "ARM-A-DONE $(date -u +%Y-%m-%dT%H:%M:%SZ)" >> "$BASE/armA/timeline.log" diff --git a/testdata/cuda-oom-retry-study/run_armB.sh b/testdata/cuda-oom-retry-study/run_armB.sh new file mode 100644 index 0000000000..afb44efb5b --- /dev/null +++ b/testdata/cuda-oom-retry-study/run_armB.sh @@ -0,0 +1,23 @@ +#!/bin/bash +# Arm B campaign — probe-removed variant, 12 rounds per device, out-of-band memory logs. +export PATH="/c/msys64/mingw64/bin:/c/Program Files/NVIDIA GPU Computing Toolkit/CUDA/v13.4/bin:$PATH" +cd /c/PLAN/VMAF/worktrees/_oom-master || exit 2 +BASE=/c/PLAN/VMAF/verify/tmp/oom-r20 +mkdir -p "$BASE/armB/dev0" "$BASE/armB/dev1" +: > "$BASE/armB/timeline.log" +for DEV in 0 1; do + for i in $(seq -w 1 12); do + { + echo "=== armB dev$DEV run$i start $(date -u +%Y-%m-%dT%H:%M:%SZ)" + nvidia-smi -i "$DEV" --query-gpu=memory.used,utilization.gpu --format=csv,noheader + } >> "$BASE/armB/timeline.log" + CUDA_VISIBLE_DEVICES=$DEV ./build/test/test_cuda_oom_pictures_released.exe \ + > "$BASE/armB/dev$DEV/run-$i.out" 2> "$BASE/armB/dev$DEV/run-$i.err" + echo "rc=$?" >> "$BASE/armB/dev$DEV/run-$i.err" + { + echo "=== armB dev$DEV run$i end $(date -u +%Y-%m-%dT%H:%M:%SZ)" + nvidia-smi -i "$DEV" --query-gpu=memory.used,utilization.gpu --format=csv,noheader + } >> "$BASE/armB/timeline.log" + done +done +echo "ARM-B-DONE $(date -u +%Y-%m-%dT%H:%M:%SZ)" >> "$BASE/armB/timeline.log" diff --git a/testdata/cuda-oom-retry-study/run_armC.sh b/testdata/cuda-oom-retry-study/run_armC.sh new file mode 100644 index 0000000000..3fada57868 --- /dev/null +++ b/testdata/cuda-oom-retry-study/run_armC.sh @@ -0,0 +1,23 @@ +#!/bin/bash +# Arm C campaign — poll-until-back with timing (bounded 60 s), 12 rounds per device. +export PATH="/c/msys64/mingw64/bin:/c/Program Files/NVIDIA GPU Computing Toolkit/CUDA/v13.4/bin:$PATH" +cd /c/PLAN/VMAF/worktrees/_oom-master || exit 2 +BASE=/c/PLAN/VMAF/verify/tmp/oom-r20 +mkdir -p "$BASE/armC/dev0" "$BASE/armC/dev1" +: > "$BASE/armC/timeline.log" +for DEV in 0 1; do + for i in $(seq -w 1 12); do + { + echo "=== armC dev$DEV run$i start $(date -u +%Y-%m-%dT%H:%M:%SZ)" + nvidia-smi -i "$DEV" --query-gpu=memory.used,utilization.gpu --format=csv,noheader + } >> "$BASE/armC/timeline.log" + CUDA_VISIBLE_DEVICES=$DEV ./build/test/test_cuda_oom_pictures_released.exe \ + > "$BASE/armC/dev$DEV/run-$i.out" 2> "$BASE/armC/dev$DEV/run-$i.err" + echo "rc=$?" >> "$BASE/armC/dev$DEV/run-$i.err" + { + echo "=== armC dev$DEV run$i end $(date -u +%Y-%m-%dT%H:%M:%SZ)" + nvidia-smi -i "$DEV" --query-gpu=memory.used,utilization.gpu --format=csv,noheader + } >> "$BASE/armC/timeline.log" + done +done +echo "ARM-C-DONE $(date -u +%Y-%m-%dT%H:%M:%SZ)" >> "$BASE/armC/timeline.log" diff --git a/testdata/cuda-oom-retry-study/run_armD.sh b/testdata/cuda-oom-retry-study/run_armD.sh new file mode 100644 index 0000000000..feb27c1a19 --- /dev/null +++ b/testdata/cuda-oom-retry-study/run_armD.sh @@ -0,0 +1,31 @@ +#!/bin/bash +# Arm D campaign — silent-wait ladder, 6 rounds per device, with an external +# nvidia-smi sampler running alongside (out-of-band view of memory during waits). +export PATH="/c/msys64/mingw64/bin:/c/Program Files/NVIDIA GPU Computing Toolkit/CUDA/v13.4/bin:$PATH" +cd /c/PLAN/VMAF/worktrees/_oom-master || exit 2 +BASE=/c/PLAN/VMAF/verify/tmp/oom-r20 +mkdir -p "$BASE/armD/dev0" "$BASE/armD/dev1" + +# sampler: both GPUs every 5 s for the whole campaign (nvidia-smi -i: 0=4060, 1=5080) +( + for i in $(seq 1 400); do + echo "$(date -u +%H:%M:%S) | 4060: $(nvidia-smi -i 0 --query-gpu=memory.used,memory.free --format=csv,noheader) | 5080: $(nvidia-smi -i 1 --query-gpu=memory.used,memory.free --format=csv,noheader)" + sleep 5 + done +) > "$BASE/armD/sampler.log" 2>&1 & +SAMPLER_PID=$! +trap 'kill $SAMPLER_PID 2>/dev/null' EXIT + +: > "$BASE/armD/timeline.log" +for DEV in 0 1; do + for i in $(seq -w 1 6); do + { + echo "=== armD dev$DEV run$i start $(date -u +%H:%M:%S)" + } >> "$BASE/armD/timeline.log" + CUDA_VISIBLE_DEVICES=$DEV ./build/test/test_cuda_oom_pictures_released.exe \ + > "$BASE/armD/dev$DEV/run-$i.out" 2> "$BASE/armD/dev$DEV/run-$i.err" + echo "rc=$?" >> "$BASE/armD/dev$DEV/run-$i.err" + echo "=== armD dev$DEV run$i end $(date -u +%H:%M:%S)" >> "$BASE/armD/timeline.log" + done +done +echo "ARM-D-DONE $(date -u +%H:%M:%S)" >> "$BASE/armD/timeline.log"