diff --git a/benchmarks/results/native-evaluation-cpu/SHA256SUMS.json b/benchmarks/results/native-evaluation-cpu/SHA256SUMS.json new file mode 100644 index 000000000..bc7ca91d1 --- /dev/null +++ b/benchmarks/results/native-evaluation-cpu/SHA256SUMS.json @@ -0,0 +1,14 @@ +{ + "cycles.json": "42199247f1498902cc9943e1d0799f1b93371928fc3b36dad54f555096240524", + "native-eval-arguments.log": "b93ff96a29830c67255d1dc9aa98f42574529314c8d5a13a03576af5a9638dc7", + "native-eval-arguments.xml": "c43d9205ed339be17a7e04d0e9987461d8901fd8960ab315d224cf1abdaaa610", + "native-eval-first.log": "8116c8d7cc92a7a4c3fa2171b933039d1ee0059650e204d9d06e01182e2e3294", + "native-eval-first.xml": "d1f2c5cee8618c574e410657a3138955a57757da3cfd9ea14baf93a40c04d9a3", + "native-eval-learning.log": "1239ff375e8aca5e8529e523f70100957774c92320710d9466ce5d7d0ee1701e", + "native-eval-learning.xml": "e3f149ffb1729d12e73aae649986d0c0a89a685cc27276f11ce4f7f9b25b2e3d", + "native-eval-release.log": "a8c0105035c3d5ce38e925462f5d4d98326ddff34c87d79b33080e675b968639", + "native-eval-release.xml": "3428e0a286436526c21d370437e3a19caa2db6e9f9d087729b595879d235f977", + "native-eval-schedule.log": "c62b5c133928a5411a7dd79685d6a6b6d358ea0fb7a6321d5ca1a829caceed76", + "native-eval-schedule.xml": "c4f9fdd6af8157c218939bbb5d1c470332b9f48114227b520086b5f0f064cad5", + "validation.json": "3b0d517569c0c7f08a1e41877bca6db4e89c7c64b489fcdf32c48cd3c2d9189a" +} diff --git a/benchmarks/results/native-evaluation-cpu/cycles.json b/benchmarks/results/native-evaluation-cpu/cycles.json new file mode 100644 index 000000000..bbe2e2e13 --- /dev/null +++ b/benchmarks/results/native-evaluation-cpu/cycles.json @@ -0,0 +1,1763 @@ +[ + { + "case": "test_evaluation_preserves_updates_and_fresh_resume[ouro]", + "scope": "tiny CPU native/RLTT/Adam recovery; transport and tokenizer doubles; no quality claim", + "training": [ + { + "step": 0, + "tokens": [ + [ + 1, + 2, + 3, + 9 + ], + [ + 1, + 2, + 6, + 7 + ], + [ + 1, + 2, + 2, + 0 + ], + [ + 1, + 2, + 4, + 7 + ], + [ + 1, + 2, + 2, + 5 + ], + [ + 1, + 2, + 7, + 0 + ], + [ + 1, + 2, + 10, + 10 + ], + [ + 1, + 2, + 3, + 10 + ], + [ + 2, + 2, + 1, + 2 + ], + [ + 2, + 2, + 9, + 7 + ], + [ + 2, + 2, + 5, + 8 + ], + [ + 2, + 2, + 5, + 8 + ], + [ + 2, + 2, + 8, + 7 + ], + [ + 2, + 2, + 9, + 10 + ], + [ + 2, + 2, + 9, + 2 + ], + [ + 2, + 2, + 1, + 7 + ] + ], + "rewards": [ + 1.0, + 0.0, + 0.0, + 0.0, + 0.0, + 1.0, + 0.0, + 1.0, + 0.0, + 0.0, + 0.0, + 0.0, + 1.0, + 0.0, + 0.0, + 0.0 + ], + "mixed_groups": 2, + "seeds": [ + 42, + 43, + 44, + 45, + 46, + 47, + 48, + 49, + 50, + 51, + 52, + 53, + 54, + 55, + 56, + 57 + ], + "loss": -0.001913309097290039, + "grad_norm": 1.5851722144144906, + "parameter_delta": 0.02911689054705937, + "score_error": 2.384185791015625e-07, + "publication_digest": "68b4f2685697f2cb9e5af85e2a07a2ad683da96d1acf89a5143192cd82f4b8b3" + }, + { + "step": 1, + "tokens": [ + [ + 1, + 2, + 9, + 6 + ], + [ + 1, + 2, + 8, + 6 + ], + [ + 1, + 2, + 1, + 10 + ], + [ + 1, + 2, + 0, + 8 + ], + [ + 1, + 2, + 1, + 4 + ], + [ + 1, + 2, + 9, + 5 + ], + [ + 1, + 2, + 7, + 7 + ], + [ + 1, + 2, + 7, + 1 + ], + [ + 2, + 2, + 4, + 8 + ], + [ + 2, + 2, + 5, + 2 + ], + [ + 2, + 2, + 1, + 0 + ], + [ + 2, + 2, + 5, + 2 + ], + [ + 2, + 2, + 1, + 1 + ], + [ + 2, + 2, + 5, + 2 + ], + [ + 2, + 2, + 5, + 3 + ], + [ + 2, + 2, + 1, + 4 + ] + ], + "rewards": [ + 1.0, + 0.0, + 1.0, + 0.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + "mixed_groups": 2, + "seeds": [ + 1000061, + 1000062, + 1000063, + 1000064, + 1000065, + 1000066, + 1000067, + 1000068, + 1000069, + 1000070, + 1000071, + 1000072, + 1000073, + 1000074, + 1000075, + 1000076 + ], + "loss": -0.004699289798736572, + "grad_norm": 1.4602550737069868, + "parameter_delta": 0.022086046311152113, + "score_error": 2.384185791015625e-07, + "publication_digest": "7c5b15d00f5928b9eaf93a380e20e1371ab5786f7d135822bb9e9a40c259ae24" + }, + { + "step": 2, + "tokens": [ + [ + 1, + 2, + 10, + 1 + ], + [ + 1, + 2, + 5, + 7 + ], + [ + 1, + 2, + 7, + 7 + ], + [ + 1, + 2, + 6, + 8 + ], + [ + 1, + 2, + 7, + 7 + ], + [ + 1, + 2, + 9, + 9 + ], + [ + 1, + 2, + 9, + 5 + ], + [ + 1, + 2, + 9, + 3 + ], + [ + 2, + 2, + 4, + 1 + ], + [ + 2, + 2, + 3, + 4 + ], + [ + 2, + 2, + 5, + 2 + ], + [ + 2, + 2, + 5, + 1 + ], + [ + 2, + 2, + 7, + 4 + ], + [ + 2, + 2, + 8, + 5 + ], + [ + 2, + 2, + 6, + 8 + ], + [ + 2, + 2, + 6, + 4 + ] + ], + "rewards": [ + 0.0, + 1.0, + 1.0, + 0.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 0.0, + 0.0, + 0.0, + 0.0, + 1.0, + 1.0, + 1.0 + ], + "mixed_groups": 2, + "seeds": [ + 2000080, + 2000081, + 2000082, + 2000083, + 2000084, + 2000085, + 2000086, + 2000087, + 2000088, + 2000089, + 2000090, + 2000091, + 2000092, + 2000093, + 2000094, + 2000095 + ], + "loss": -0.022948265075683594, + "grad_norm": 1.056637973643531, + "parameter_delta": 0.016446933472595213, + "score_error": 4.76837158203125e-07, + "publication_digest": "14bb0557afd4426e82b5af8581cdab209728322e33dcdc1e36f319e90dc6f761" + } + ], + "heldout": [ + [ + { + "tokens": [ + 4, + 5, + 5, + 9 + ], + "reward": 0, + "seed": 5584658642194527637, + "version": 2, + "digest": "68b4f2685697f2cb9e5af85e2a07a2ad683da96d1acf89a5143192cd82f4b8b3" + }, + { + "tokens": [ + 4, + 5, + 3, + 9 + ], + "reward": 0, + "seed": 4511292036611990679, + "version": 2, + "digest": "68b4f2685697f2cb9e5af85e2a07a2ad683da96d1acf89a5143192cd82f4b8b3" + }, + { + "tokens": [ + 7, + 8, + 7, + 6 + ], + "reward": 0, + "seed": 1776775126518905334, + "version": 2, + "digest": "68b4f2685697f2cb9e5af85e2a07a2ad683da96d1acf89a5143192cd82f4b8b3" + }, + { + "tokens": [ + 7, + 8, + 6, + 3 + ], + "reward": 1, + "seed": 7856332686515200297, + "version": 2, + "digest": "68b4f2685697f2cb9e5af85e2a07a2ad683da96d1acf89a5143192cd82f4b8b3" + } + ], + [ + { + "tokens": [ + 4, + 5, + 5, + 9 + ], + "reward": 0, + "seed": 5584658642194527637, + "version": 3, + "digest": "7c5b15d00f5928b9eaf93a380e20e1371ab5786f7d135822bb9e9a40c259ae24" + }, + { + "tokens": [ + 4, + 5, + 3, + 9 + ], + "reward": 0, + "seed": 4511292036611990679, + "version": 3, + "digest": "7c5b15d00f5928b9eaf93a380e20e1371ab5786f7d135822bb9e9a40c259ae24" + }, + { + "tokens": [ + 7, + 8, + 7, + 6 + ], + "reward": 0, + "seed": 1776775126518905334, + "version": 3, + "digest": "7c5b15d00f5928b9eaf93a380e20e1371ab5786f7d135822bb9e9a40c259ae24" + }, + { + "tokens": [ + 7, + 8, + 3, + 3 + ], + "reward": 1, + "seed": 7856332686515200297, + "version": 3, + "digest": "7c5b15d00f5928b9eaf93a380e20e1371ab5786f7d135822bb9e9a40c259ae24" + } + ], + [ + { + "tokens": [ + 4, + 5, + 5, + 9 + ], + "reward": 0, + "seed": 5584658642194527637, + "version": 4, + "digest": "14bb0557afd4426e82b5af8581cdab209728322e33dcdc1e36f319e90dc6f761" + }, + { + "tokens": [ + 4, + 5, + 3, + 9 + ], + "reward": 0, + "seed": 4511292036611990679, + "version": 4, + "digest": "14bb0557afd4426e82b5af8581cdab209728322e33dcdc1e36f319e90dc6f761" + }, + { + "tokens": [ + 7, + 8, + 7, + 6 + ], + "reward": 0, + "seed": 1776775126518905334, + "version": 4, + "digest": "14bb0557afd4426e82b5af8581cdab209728322e33dcdc1e36f319e90dc6f761" + }, + { + "tokens": [ + 7, + 8, + 3, + 3 + ], + "reward": 1, + "seed": 7856332686515200297, + "version": 4, + "digest": "14bb0557afd4426e82b5af8581cdab209728322e33dcdc1e36f319e90dc6f761" + } + ] + ] + }, + { + "case": "test_evaluation_preserves_updates_and_fresh_resume[nanbeige]", + "scope": "tiny CPU native/RLTT/Adam recovery; transport and tokenizer doubles; no quality claim", + "training": [ + { + "step": 0, + "tokens": [ + [ + 1, + 2, + 1, + 9 + ], + [ + 1, + 2, + 1, + 7 + ], + [ + 1, + 2, + 2, + 0 + ], + [ + 1, + 2, + 1, + 8 + ], + [ + 1, + 2, + 2, + 1 + ], + [ + 1, + 2, + 4, + 1 + ], + [ + 1, + 2, + 10, + 10 + ], + [ + 1, + 2, + 3, + 4 + ], + [ + 2, + 2, + 1, + 2 + ], + [ + 2, + 2, + 9, + 6 + ], + [ + 2, + 2, + 5, + 8 + ], + [ + 2, + 2, + 1, + 8 + ], + [ + 2, + 2, + 5, + 7 + ], + [ + 2, + 2, + 10, + 10 + ], + [ + 2, + 2, + 3, + 8 + ], + [ + 2, + 2, + 1, + 7 + ] + ], + "rewards": [ + 1.0, + 1.0, + 0.0, + 1.0, + 0.0, + 0.0, + 0.0, + 1.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 1.0, + 0.0, + 0.0 + ], + "mixed_groups": 2, + "seeds": [ + 42, + 43, + 44, + 45, + 46, + 47, + 48, + 49, + 50, + 51, + 52, + 53, + 54, + 55, + 56, + 57 + ], + "loss": -0.06689101457595825, + "grad_norm": 0.6002090434887976, + "parameter_delta": 0.030593636317655583, + "score_error": 3.5762786865234375e-07, + "publication_digest": "53ea72dab366dec44a06bcf99bd3dd34fdf42335f610d92edb5ca63fbd2c2756" + }, + { + "step": 1, + "tokens": [ + [ + 1, + 2, + 9, + 6 + ], + [ + 1, + 2, + 0, + 6 + ], + [ + 1, + 2, + 1, + 10 + ], + [ + 1, + 2, + 0, + 8 + ], + [ + 1, + 2, + 1, + 4 + ], + [ + 1, + 2, + 1, + 8 + ], + [ + 1, + 2, + 10, + 7 + ], + [ + 1, + 2, + 7, + 1 + ], + [ + 2, + 2, + 4, + 8 + ], + [ + 2, + 2, + 5, + 2 + ], + [ + 2, + 2, + 1, + 0 + ], + [ + 2, + 2, + 1, + 2 + ], + [ + 2, + 2, + 1, + 2 + ], + [ + 2, + 2, + 5, + 2 + ], + [ + 2, + 2, + 5, + 3 + ], + [ + 2, + 2, + 1, + 4 + ] + ], + "rewards": [ + 1.0, + 0.0, + 1.0, + 0.0, + 1.0, + 1.0, + 0.0, + 1.0, + 1.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + "mixed_groups": 2, + "seeds": [ + 1000061, + 1000062, + 1000063, + 1000064, + 1000065, + 1000066, + 1000067, + 1000068, + 1000069, + 1000070, + 1000071, + 1000072, + 1000073, + 1000074, + 1000075, + 1000076 + ], + "loss": -0.053218454122543335, + "grad_norm": 0.8334220663167323, + "parameter_delta": 0.021697335907888384, + "score_error": 4.76837158203125e-07, + "publication_digest": "efe44ad43da976220d34b19a83816a575495335da194359208eb92b8694eae28" + }, + { + "step": 2, + "tokens": [ + [ + 1, + 2, + 10, + 1 + ], + [ + 1, + 2, + 5, + 7 + ], + [ + 1, + 2, + 7, + 7 + ], + [ + 1, + 2, + 6, + 8 + ], + [ + 1, + 2, + 5, + 7 + ], + [ + 1, + 2, + 1, + 9 + ], + [ + 1, + 2, + 1, + 5 + ], + [ + 1, + 2, + 9, + 3 + ], + [ + 2, + 2, + 4, + 1 + ], + [ + 2, + 2, + 3, + 4 + ], + [ + 2, + 2, + 5, + 2 + ], + [ + 2, + 2, + 5, + 1 + ], + [ + 2, + 2, + 10, + 4 + ], + [ + 2, + 2, + 1, + 5 + ], + [ + 2, + 2, + 6, + 8 + ], + [ + 2, + 2, + 6, + 4 + ] + ], + "rewards": [ + 0.0, + 1.0, + 1.0, + 0.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 0.0, + 0.0, + 0.0, + 1.0, + 0.0, + 1.0, + 1.0 + ], + "mixed_groups": 2, + "seeds": [ + 2000080, + 2000081, + 2000082, + 2000083, + 2000084, + 2000085, + 2000086, + 2000087, + 2000088, + 2000089, + 2000090, + 2000091, + 2000092, + 2000093, + 2000094, + 2000095 + ], + "loss": 0.08678185939788818, + "grad_norm": 0.5566346329899224, + "parameter_delta": 0.01893572354362318, + "score_error": 4.76837158203125e-07, + "publication_digest": "96ba4ae92ff04a568c900e896593738a1c44e7a0bbf195b843899655fba5fe45" + } + ], + "heldout": [ + [ + { + "tokens": [ + 4, + 5, + 9, + 4 + ], + "reward": 0, + "seed": 5584658642194527637, + "version": 2, + "digest": "53ea72dab366dec44a06bcf99bd3dd34fdf42335f610d92edb5ca63fbd2c2756" + }, + { + "tokens": [ + 4, + 5, + 9, + 9 + ], + "reward": 0, + "seed": 4511292036611990679, + "version": 2, + "digest": "53ea72dab366dec44a06bcf99bd3dd34fdf42335f610d92edb5ca63fbd2c2756" + }, + { + "tokens": [ + 7, + 8, + 5, + 9 + ], + "reward": 0, + "seed": 1776775126518905334, + "version": 2, + "digest": "53ea72dab366dec44a06bcf99bd3dd34fdf42335f610d92edb5ca63fbd2c2756" + }, + { + "tokens": [ + 7, + 8, + 5, + 8 + ], + "reward": 0, + "seed": 7856332686515200297, + "version": 2, + "digest": "53ea72dab366dec44a06bcf99bd3dd34fdf42335f610d92edb5ca63fbd2c2756" + } + ], + [ + { + "tokens": [ + 4, + 5, + 9, + 4 + ], + "reward": 0, + "seed": 5584658642194527637, + "version": 3, + "digest": "efe44ad43da976220d34b19a83816a575495335da194359208eb92b8694eae28" + }, + { + "tokens": [ + 4, + 5, + 9, + 9 + ], + "reward": 0, + "seed": 4511292036611990679, + "version": 3, + "digest": "efe44ad43da976220d34b19a83816a575495335da194359208eb92b8694eae28" + }, + { + "tokens": [ + 7, + 8, + 5, + 9 + ], + "reward": 0, + "seed": 1776775126518905334, + "version": 3, + "digest": "efe44ad43da976220d34b19a83816a575495335da194359208eb92b8694eae28" + }, + { + "tokens": [ + 7, + 8, + 5, + 8 + ], + "reward": 0, + "seed": 7856332686515200297, + "version": 3, + "digest": "efe44ad43da976220d34b19a83816a575495335da194359208eb92b8694eae28" + } + ], + [ + { + "tokens": [ + 4, + 5, + 9, + 4 + ], + "reward": 0, + "seed": 5584658642194527637, + "version": 4, + "digest": "96ba4ae92ff04a568c900e896593738a1c44e7a0bbf195b843899655fba5fe45" + }, + { + "tokens": [ + 4, + 5, + 9, + 9 + ], + "reward": 0, + "seed": 4511292036611990679, + "version": 4, + "digest": "96ba4ae92ff04a568c900e896593738a1c44e7a0bbf195b843899655fba5fe45" + }, + { + "tokens": [ + 7, + 8, + 5, + 9 + ], + "reward": 0, + "seed": 1776775126518905334, + "version": 4, + "digest": "96ba4ae92ff04a568c900e896593738a1c44e7a0bbf195b843899655fba5fe45" + }, + { + "tokens": [ + 7, + 8, + 5, + 8 + ], + "reward": 0, + "seed": 7856332686515200297, + "version": 4, + "digest": "96ba4ae92ff04a568c900e896593738a1c44e7a0bbf195b843899655fba5fe45" + } + ] + ] + }, + { + "case": "test_evaluation_preserves_updates_and_fresh_resume[huginn_raven]", + "scope": "tiny CPU native/RLTT/Adam recovery; transport and tokenizer doubles; no quality claim", + "training": [ + { + "step": 0, + "tokens": [ + [ + 1, + 2, + 9, + 9 + ], + [ + 1, + 2, + 2, + 10 + ], + [ + 1, + 2, + 2, + 0 + ], + [ + 1, + 2, + 10, + 9 + ], + [ + 1, + 2, + 2, + 2 + ], + [ + 1, + 2, + 10, + 9 + ], + [ + 1, + 2, + 10, + 10 + ], + [ + 1, + 2, + 3, + 10 + ], + [ + 2, + 2, + 2, + 2 + ], + [ + 2, + 2, + 9, + 9 + ], + [ + 2, + 2, + 9, + 9 + ], + [ + 2, + 2, + 9, + 4 + ], + [ + 2, + 2, + 9, + 4 + ], + [ + 2, + 2, + 9, + 9 + ], + [ + 2, + 2, + 9, + 9 + ], + [ + 2, + 2, + 10, + 9 + ] + ], + "rewards": [ + 1.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 1.0, + 1.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 1.0 + ], + "mixed_groups": 2, + "seeds": [ + 42, + 43, + 44, + 45, + 46, + 47, + 48, + 49, + 50, + 51, + 52, + 53, + 54, + 55, + 56, + 57 + ], + "loss": 0.046841442584991455, + "grad_norm": 3.1770623722174824, + "parameter_delta": 0.047832407945291686, + "score_error": 1.4156103134155273e-06, + "publication_digest": "a2b5ab09373fbb37acc2be96935c1da4af90bab56170196be6b16c1f43904fa6" + }, + { + "step": 1, + "tokens": [ + [ + 1, + 2, + 9, + 1 + ], + [ + 1, + 2, + 8, + 10 + ], + [ + 1, + 2, + 9, + 4 + ], + [ + 1, + 2, + 0, + 0 + ], + [ + 1, + 2, + 10, + 9 + ], + [ + 1, + 2, + 10, + 10 + ], + [ + 1, + 2, + 10, + 9 + ], + [ + 1, + 2, + 10, + 9 + ], + [ + 2, + 2, + 0, + 0 + ], + [ + 2, + 2, + 2, + 2 + ], + [ + 2, + 2, + 2, + 9 + ], + [ + 2, + 2, + 10, + 9 + ], + [ + 2, + 2, + 10, + 9 + ], + [ + 2, + 2, + 0, + 0 + ], + [ + 2, + 2, + 10, + 9 + ], + [ + 2, + 2, + 2, + 10 + ] + ], + "rewards": [ + 1.0, + 0.0, + 1.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0 + ], + "mixed_groups": 1, + "seeds": [ + 1000061, + 1000062, + 1000063, + 1000064, + 1000065, + 1000066, + 1000067, + 1000068, + 1000069, + 1000070, + 1000071, + 1000072, + 1000073, + 1000074, + 1000075, + 1000076 + ], + "loss": -0.1097540836508415, + "grad_norm": 2.4779377130636746, + "parameter_delta": 0.03340916235419829, + "score_error": 1.0728836059570312e-06, + "publication_digest": "cf7fde77ef0572b7afdbc30c0e4ff1cdc83cac66e1263610be18d33b72216d84" + }, + { + "step": 2, + "tokens": [ + [ + 1, + 2, + 10, + 1 + ], + [ + 1, + 2, + 10, + 9 + ], + [ + 1, + 2, + 10, + 9 + ], + [ + 1, + 2, + 10, + 9 + ], + [ + 1, + 2, + 2, + 2 + ], + [ + 1, + 2, + 8, + 9 + ], + [ + 1, + 2, + 9, + 4 + ], + [ + 1, + 2, + 9, + 9 + ], + [ + 2, + 2, + 8, + 1 + ], + [ + 2, + 2, + 2, + 2 + ], + [ + 2, + 2, + 9, + 4 + ], + [ + 2, + 2, + 4, + 5 + ], + [ + 2, + 2, + 10, + 9 + ], + [ + 2, + 2, + 9, + 4 + ], + [ + 2, + 2, + 0, + 0 + ], + [ + 2, + 2, + 10, + 10 + ] + ], + "rewards": [ + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 1.0, + 1.0, + 1.0, + 1.0, + 0.0, + 1.0, + 1.0, + 0.0, + 1.0, + 1.0 + ], + "mixed_groups": 2, + "seeds": [ + 2000080, + 2000081, + 2000082, + 2000083, + 2000084, + 2000085, + 2000086, + 2000087, + 2000088, + 2000089, + 2000090, + 2000091, + 2000092, + 2000093, + 2000094, + 2000095 + ], + "loss": 0.23252639919519424, + "grad_norm": 2.6597207128730966, + "parameter_delta": 0.028642841888614417, + "score_error": 1.1920928955078125e-06, + "publication_digest": "e88138914c2499e8e6ab67bc2dfa2bc69b5fd5df339f10575535941d5a9cd126" + } + ], + "heldout": [ + [ + { + "tokens": [ + 4, + 5, + 5, + 10 + ], + "reward": 0, + "seed": 5584658642194527637, + "version": 2, + "digest": "a2b5ab09373fbb37acc2be96935c1da4af90bab56170196be6b16c1f43904fa6" + }, + { + "tokens": [ + 4, + 5, + 7, + 8 + ], + "reward": 0, + "seed": 4511292036611990679, + "version": 2, + "digest": "a2b5ab09373fbb37acc2be96935c1da4af90bab56170196be6b16c1f43904fa6" + }, + { + "tokens": [ + 7, + 8, + 10, + 9 + ], + "reward": 0, + "seed": 1776775126518905334, + "version": 2, + "digest": "a2b5ab09373fbb37acc2be96935c1da4af90bab56170196be6b16c1f43904fa6" + }, + { + "tokens": [ + 7, + 8, + 9, + 9 + ], + "reward": 0, + "seed": 7856332686515200297, + "version": 2, + "digest": "a2b5ab09373fbb37acc2be96935c1da4af90bab56170196be6b16c1f43904fa6" + } + ], + [ + { + "tokens": [ + 4, + 5, + 5, + 10 + ], + "reward": 0, + "seed": 5584658642194527637, + "version": 3, + "digest": "cf7fde77ef0572b7afdbc30c0e4ff1cdc83cac66e1263610be18d33b72216d84" + }, + { + "tokens": [ + 4, + 5, + 7, + 8 + ], + "reward": 0, + "seed": 4511292036611990679, + "version": 3, + "digest": "cf7fde77ef0572b7afdbc30c0e4ff1cdc83cac66e1263610be18d33b72216d84" + }, + { + "tokens": [ + 7, + 8, + 10, + 9 + ], + "reward": 0, + "seed": 1776775126518905334, + "version": 3, + "digest": "cf7fde77ef0572b7afdbc30c0e4ff1cdc83cac66e1263610be18d33b72216d84" + }, + { + "tokens": [ + 7, + 8, + 9, + 9 + ], + "reward": 0, + "seed": 7856332686515200297, + "version": 3, + "digest": "cf7fde77ef0572b7afdbc30c0e4ff1cdc83cac66e1263610be18d33b72216d84" + } + ], + [ + { + "tokens": [ + 4, + 5, + 5, + 10 + ], + "reward": 0, + "seed": 5584658642194527637, + "version": 4, + "digest": "e88138914c2499e8e6ab67bc2dfa2bc69b5fd5df339f10575535941d5a9cd126" + }, + { + "tokens": [ + 4, + 5, + 7, + 8 + ], + "reward": 0, + "seed": 4511292036611990679, + "version": 4, + "digest": "e88138914c2499e8e6ab67bc2dfa2bc69b5fd5df339f10575535941d5a9cd126" + }, + { + "tokens": [ + 7, + 8, + 10, + 9 + ], + "reward": 0, + "seed": 1776775126518905334, + "version": 4, + "digest": "e88138914c2499e8e6ab67bc2dfa2bc69b5fd5df339f10575535941d5a9cd126" + }, + { + "tokens": [ + 7, + 8, + 9, + 9 + ], + "reward": 0, + "seed": 7856332686515200297, + "version": 4, + "digest": "e88138914c2499e8e6ab67bc2dfa2bc69b5fd5df339f10575535941d5a9cd126" + } + ] + ] + } +] diff --git a/benchmarks/results/native-evaluation-cpu/native-eval-arguments.log b/benchmarks/results/native-evaluation-cpu/native-eval-arguments.log new file mode 100644 index 000000000..3d5568268 --- /dev/null +++ b/benchmarks/results/native-evaluation-cpu/native-eval-arguments.log @@ -0,0 +1,17 @@ +torch=2.13.0 /Users/0z5a/Documents/infra/looped-grpo-integration-20261003/.venv/lib/python3.12/site-packages/torch/__init__.py +setuptools=84.0.0 /Users/0z5a/Documents/infra/looped-grpo-integration-20261003/.venv/lib/python3.12/site-packages/setuptools/__init__.py +============================= test session starts ============================== +platform darwin -- Python 3.12.14, pytest-9.1.1, pluggy-1.6.0 +rootdir: /Users/0z5a/Documents/infra/flashrlt-20261005/vime +configfile: pyproject.toml +plugins: platformdirs-4.12.2, anyio-4.15.1 +collected 2 items + +tests/rollout_backends/test_native_eval_arguments.py .. [100%] + +- generated xml file: /Users/0z5a/Documents/infra/flashrlt-20261005/evidence/native-eval-arguments.xml - +============================== slowest durations =============================== +7.25s call tests/rollout_backends/test_native_eval_arguments.py::test_eval_sampling_is_separate_from_training_validation[1.0] + +(5 durations < 0.005s hidden. Use -vv to show these durations.) +============================== 2 passed in 7.32s =============================== diff --git a/benchmarks/results/native-evaluation-cpu/native-eval-arguments.xml b/benchmarks/results/native-evaluation-cpu/native-eval-arguments.xml new file mode 100644 index 000000000..0ee0f993b --- /dev/null +++ b/benchmarks/results/native-evaluation-cpu/native-eval-arguments.xml @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/benchmarks/results/native-evaluation-cpu/native-eval-first.log b/benchmarks/results/native-evaluation-cpu/native-eval-first.log new file mode 100644 index 000000000..c90ae2ea2 --- /dev/null +++ b/benchmarks/results/native-evaluation-cpu/native-eval-first.log @@ -0,0 +1,52 @@ +torch=2.13.0 /Users/0z5a/Documents/infra/looped-grpo-integration-20261003/.venv/lib/python3.12/site-packages/torch/__init__.py +setuptools=84.0.0 /Users/0z5a/Documents/infra/looped-grpo-integration-20261003/.venv/lib/python3.12/site-packages/setuptools/__init__.py +============================= test session starts ============================== +platform darwin -- Python 3.12.14, pytest-9.1.1, pluggy-1.6.0 +rootdir: /Users/0z5a/Documents/infra/flashrlt-20261005/vime +configfile: pyproject.toml +plugins: platformdirs-4.12.2, anyio-4.15.1 +collected 25 items + +tests/rollout_backends/test_native_rlt_evaluation.py ............... [ 60%] +tests/rollout_backends/test_native_rlt_contract.py ..... [ 80%] +tests/rollout_backends/test_native_rlt_groups.py ..... [100%] + +- generated xml file: /Users/0z5a/Documents/infra/flashrlt-20261005/evidence/native-eval-first.xml - +============================== slowest durations =============================== +0.14s call tests/rollout_backends/test_native_rlt_evaluation.py::test_complete_datasets_fixed_seeds_and_unchanged_training_state +0.11s call tests/rollout_backends/test_native_rlt_contract.py::test_huginn_aliases_static_buffer_and_noise_profile +0.05s call tests/rollout_backends/test_native_rlt_contract.py::test_http_client_concurrency_matches_backend[vllm] +0.03s setup tests/rollout_backends/test_native_rlt_evaluation.py::test_complete_datasets_fixed_seeds_and_unchanged_training_state +0.03s call tests/rollout_backends/test_native_rlt_evaluation.py::test_policy_publication_between_batches_invalidates_evaluation +0.03s setup tests/rollout_backends/test_native_rlt_evaluation.py::test_long_heldout_prompts_are_not_silently_dropped[context] +0.02s call tests/rollout_backends/test_native_rlt_contract.py::test_incomplete_publication_cannot_resume_or_reuse_old_manifest +0.02s call tests/rollout_backends/test_native_rlt_evaluation.py::test_reward_key_and_metadata_override +0.02s call tests/rollout_backends/test_native_rlt_evaluation.py::test_sampling_and_chat_overrides_are_used +0.02s call tests/rollout_backends/test_native_rlt_contract.py::test_publication_cohort_and_serialization +0.02s call tests/rollout_backends/test_native_rlt_evaluation.py::test_reordered_or_missing_heldout_samples_are_rejected +0.01s setup tests/rollout_backends/test_native_rlt_evaluation.py::test_unsupported_eval_settings_reject_before_generation[custom_generate_function_path-custom.generate] +0.01s setup tests/rollout_backends/test_native_rlt_contract.py::test_huginn_aliases_static_buffer_and_noise_profile +0.01s setup tests/rollout_backends/test_native_rlt_evaluation.py::test_unsupported_eval_settings_reject_before_generation[app_service-http://example.invalid] +0.01s setup tests/rollout_backends/test_native_rlt_evaluation.py::test_unsupported_eval_settings_reject_before_generation[eval_task_timeout-1] +0.01s call tests/rollout_backends/test_native_rlt_contract.py::test_http_client_concurrency_matches_backend[vllm-rlt] +0.01s setup tests/rollout_backends/test_native_rlt_evaluation.py::test_unsupported_eval_settings_reject_before_generation[no_stop_trim-False] +0.01s setup tests/rollout_backends/test_native_rlt_evaluation.py::test_unsupported_eval_settings_reject_before_generation[stop-value0] +0.01s setup tests/rollout_backends/test_native_rlt_evaluation.py::test_unsupported_eval_settings_reject_before_generation[min_new_tokens-1] +0.01s setup tests/rollout_backends/test_native_rlt_evaluation.py::test_policy_publication_between_batches_invalidates_evaluation +0.01s setup tests/rollout_backends/test_native_rlt_evaluation.py::test_long_heldout_prompts_are_not_silently_dropped[prompt] +0.01s setup tests/rollout_backends/test_native_rlt_evaluation.py::test_unsupported_eval_settings_reject_before_generation[n_samples_per_eval_prompt-0] +0.01s setup tests/rollout_backends/test_native_rlt_contract.py::test_publication_cohort_and_serialization +0.01s setup tests/rollout_backends/test_native_rlt_evaluation.py::test_sampling_and_chat_overrides_are_used +0.01s setup tests/rollout_backends/test_native_rlt_evaluation.py::test_reward_key_and_metadata_override +0.01s call tests/rollout_backends/test_native_rlt_evaluation.py::test_long_heldout_prompts_are_not_silently_dropped[context] +0.01s setup tests/rollout_backends/test_native_rlt_evaluation.py::test_unsupported_eval_settings_reject_before_generation[repetition_penalty-1.2] +0.01s call tests/rollout_backends/test_native_rlt_evaluation.py::test_long_heldout_prompts_are_not_silently_dropped[prompt] +0.01s setup tests/rollout_backends/test_native_rlt_evaluation.py::test_reordered_or_missing_heldout_samples_are_rejected +0.01s call tests/rollout_backends/test_native_rlt_groups.py::test_native_groups_and_actual_math_rewards +0.01s call tests/rollout_backends/test_native_rlt_evaluation.py::test_unsupported_eval_settings_reject_before_generation[no_stop_trim-False] +0.01s call tests/rollout_backends/test_native_rlt_evaluation.py::test_unsupported_eval_settings_reject_before_generation[app_service-http://example.invalid] +0.01s setup tests/rollout_backends/test_native_rlt_contract.py::test_incomplete_publication_cannot_resume_or_reuse_old_manifest +0.01s call tests/rollout_backends/test_native_rlt_evaluation.py::test_unsupported_eval_settings_reject_before_generation[eval_task_timeout-1] + +(41 durations < 0.005s hidden. Use -vv to show these durations.) +============================== 25 passed in 8.98s ============================== diff --git a/benchmarks/results/native-evaluation-cpu/native-eval-first.xml b/benchmarks/results/native-evaluation-cpu/native-eval-first.xml new file mode 100644 index 000000000..ed987d6dc --- /dev/null +++ b/benchmarks/results/native-evaluation-cpu/native-eval-first.xml @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/benchmarks/results/native-evaluation-cpu/native-eval-learning.log b/benchmarks/results/native-evaluation-cpu/native-eval-learning.log new file mode 100644 index 000000000..adf698bc6 --- /dev/null +++ b/benchmarks/results/native-evaluation-cpu/native-eval-learning.log @@ -0,0 +1,56 @@ +torch=2.13.0 /Users/0z5a/Documents/infra/looped-grpo-integration-20261003/.venv/lib/python3.12/site-packages/torch/__init__.py +setuptools=84.0.0 /Users/0z5a/Documents/infra/looped-grpo-integration-20261003/.venv/lib/python3.12/site-packages/setuptools/__init__.py +============================= test session starts ============================== +platform darwin -- Python 3.12.14, pytest-9.1.1, pluggy-1.6.0 +rootdir: /Users/0z5a/Documents/infra/flashrlt-20261005/vime +configfile: pyproject.toml +plugins: platformdirs-4.12.2, anyio-4.15.1 +collected 3 items + +tests/plugins/test_native_heldout_training.py ... [100%] + +=============================== warnings summary =============================== +../../looped-grpo-integration-20261003/.venv/lib/python3.12/site-packages/torch/jit/_script.py:365: 14 warnings + /Users/0z5a/Documents/infra/looped-grpo-integration-20261003/.venv/lib/python3.12/site-packages/torch/jit/_script.py:365: DeprecationWarning: `torch.jit.script_method` is deprecated. Please switch to `torch.compile` or `torch.export`. + warnings.warn( + +../../looped-grpo-integration-20261003/runtime/megatron/megatron/core/distributed/fsdp/src/megatron_fsdp/mixed_precision.py:113 + /Users/0z5a/Documents/infra/looped-grpo-integration-20261003/runtime/megatron/megatron/core/distributed/fsdp/src/megatron_fsdp/mixed_precision.py:113: UserWarning: Transformer Engine and Apex are not installed. Falling back to local implementations of multi_tensor_applier and multi_tensor_scale + warnings.warn( + +../../looped-grpo-integration-20261003/runtime/megatron/megatron/core/inference/contexts/__init__.py:9 + /Users/0z5a/Documents/infra/looped-grpo-integration-20261003/runtime/megatron/megatron/core/inference/contexts/__init__.py:9: DeprecationWarning: The following imports from `dynamic_context.py` will be removed in this file in `megatron-core` 0.14. The imports here result in a cyclic import issue that causes rotary embeddings to import from Apex rather than Transformer Engine. + warnings.warn( + +../../looped-grpo-integration-20261003/runtime/megatron/megatron/core/optimizer/__init__.py:24 + /Users/0z5a/Documents/infra/looped-grpo-integration-20261003/runtime/megatron/megatron/core/optimizer/__init__.py:24: UserWarning: Transformer Engine and Apex are not installed. Falling back to Torch optimizers. + warnings.warn( + +../../looped-grpo-integration-20261003/runtime/megatron/megatron/core/optimizer/optimizer.py:28 + /Users/0z5a/Documents/infra/looped-grpo-integration-20261003/runtime/megatron/megatron/core/optimizer/optimizer.py:28: UserWarning: Transformer Engine and Apex are not installed. Falling back to local implementations of multi_tensor_applier and multi_tensor_scale + warnings.warn( + +../../looped-grpo-integration-20261003/runtime/megatron/megatron/core/optimizer/clip_grads.py:29 + /Users/0z5a/Documents/infra/looped-grpo-integration-20261003/runtime/megatron/megatron/core/optimizer/clip_grads.py:29: UserWarning: Transformer Engine and Apex are not installed. Falling back to local implementations of multi_tensor_applier, multi_tensor_l2norm, and multi_tensor_scale + warnings.warn( + +tests/plugins/test_native_heldout_training.py::test_evaluation_preserves_updates_and_fresh_resume[ouro] +tests/plugins/test_native_heldout_training.py::test_evaluation_preserves_updates_and_fresh_resume[nanbeige] +tests/plugins/test_native_heldout_training.py::test_evaluation_preserves_updates_and_fresh_resume[huginn_raven] + /Users/0z5a/Documents/infra/looped-grpo-integration-20261003/runtime/megatron/megatron/core/transformer/transformer_config.py:1734: UserWarning: full scope is deprecated. Use empty cuda_graph_scope to capture the whole layer. + warnings.warn( + +tests/plugins/test_native_heldout_training.py::test_evaluation_preserves_updates_and_fresh_resume[ouro] + /Users/0z5a/Documents/infra/looped-grpo-integration-20261003/runtime/megatron/megatron/core/models/backends.py:21: UserWarning: Apex is not installed. Falling back to Torch Norm + warnings.warn("Apex is not installed. Falling back to Torch Norm") + +-- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html +- generated xml file: /Users/0z5a/Documents/infra/flashrlt-20261005/evidence/native-eval-learning.xml - +============================== slowest durations =============================== +1.01s call tests/plugins/test_native_heldout_training.py::test_evaluation_preserves_updates_and_fresh_resume[ouro] +0.81s call tests/plugins/test_native_heldout_training.py::test_evaluation_preserves_updates_and_fresh_resume[huginn_raven] +0.53s call tests/plugins/test_native_heldout_training.py::test_evaluation_preserves_updates_and_fresh_resume[nanbeige] +0.01s setup tests/plugins/test_native_heldout_training.py::test_evaluation_preserves_updates_and_fresh_resume[ouro] + +(5 durations < 0.005s hidden. Use -vv to show these durations.) +======================= 3 passed, 23 warnings in 14.97s ======================== diff --git a/benchmarks/results/native-evaluation-cpu/native-eval-learning.xml b/benchmarks/results/native-evaluation-cpu/native-eval-learning.xml new file mode 100644 index 000000000..22e824831 --- /dev/null +++ b/benchmarks/results/native-evaluation-cpu/native-eval-learning.xml @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/benchmarks/results/native-evaluation-cpu/native-eval-release.log b/benchmarks/results/native-evaluation-cpu/native-eval-release.log new file mode 100644 index 000000000..a85a37963 --- /dev/null +++ b/benchmarks/results/native-evaluation-cpu/native-eval-release.log @@ -0,0 +1,519 @@ +torch=2.13.0 /Users/0z5a/Documents/infra/looped-grpo-integration-20261003/.venv/lib/python3.12/site-packages/torch/__init__.py +setuptools=84.0.0 /Users/0z5a/Documents/infra/looped-grpo-integration-20261003/.venv/lib/python3.12/site-packages/setuptools/__init__.py +============================= test session starts ============================== +platform darwin -- Python 3.12.14, pytest-9.1.1, pluggy-1.6.0 +rootdir: /Users/0z5a/Documents/infra/flashrlt-20261005/vime +configfile: pyproject.toml +plugins: platformdirs-4.12.2, anyio-4.15.1 +collected 515 items / 3 deselected / 512 selected + +tests/plugins/test_differentiable_prefix.py ............................ [ 5%] +.................................................................. [ 18%] +tests/plugins/test_huginn_ppo_provider.py .. [ 18%] +tests/plugins/test_looped_algorithms.py ........ [ 20%] +tests/plugins/test_looped_dppo.py . [ 20%] +tests/plugins/test_looped_grpo.py .... [ 21%] +tests/plugins/test_looped_response_readout.py .......................... [ 26%] +................ [ 29%] +tests/plugins/test_nanbeige_ppo_provider.py .. [ 29%] +tests/plugins/test_ouro_critic.py .. [ 30%] +tests/plugins/test_packed_recurrent_replay.py .........................s [ 35%] +sssss [ 36%] +tests/plugins/test_recurrent_rematerialization.py ...................... [ 40%] +.................... [ 44%] +tests/plugins/test_rltt_online_cycle.py ......... [ 46%] +tests/plugins/test_rltt_training.py .................................... [ 53%] +........... [ 55%] +tests/plugins/test_streamed_looped_readout.py .......................... [ 60%] +........................ [ 65%] +tests/plugins/test_native_heldout_training.py ... [ 65%] +tests/test_looped_grpo_recipe.py ....................................... [ 73%] +...................... [ 77%] +tests/test_megatron_argument_validation.py ............................. [ 83%] +................. [ 86%] +tests/test_placement_group.py ............. [ 89%] +tests/rollout_backends/test_native_rlt_evaluation.py ................. [ 92%] +tests/rollout_backends/test_native_rlt_contract.py ..... [ 93%] +tests/rollout_backends/test_native_rlt_groups.py ..... [ 94%] +tests/test_native_eval_schedule.py ..... [ 95%] +tests/test_rollout_data_utils.py ...... [ 96%] +tests/test_rollout_metrics.py ............ [ 99%] +tests/test_eval_config.py ..... [100%] + +=============================== warnings summary =============================== +../../looped-grpo-integration-20261003/.venv/lib/python3.12/site-packages/torch/jit/_script.py:365: 14 warnings + /Users/0z5a/Documents/infra/looped-grpo-integration-20261003/.venv/lib/python3.12/site-packages/torch/jit/_script.py:365: DeprecationWarning: `torch.jit.script_method` is deprecated. Please switch to `torch.compile` or `torch.export`. + warnings.warn( + +../../looped-grpo-integration-20261003/runtime/megatron/megatron/core/distributed/fsdp/src/megatron_fsdp/mixed_precision.py:113 + /Users/0z5a/Documents/infra/looped-grpo-integration-20261003/runtime/megatron/megatron/core/distributed/fsdp/src/megatron_fsdp/mixed_precision.py:113: UserWarning: Transformer Engine and Apex are not installed. Falling back to local implementations of multi_tensor_applier and multi_tensor_scale + warnings.warn( + +../../looped-grpo-integration-20261003/runtime/megatron/megatron/core/inference/contexts/__init__.py:9 + /Users/0z5a/Documents/infra/looped-grpo-integration-20261003/runtime/megatron/megatron/core/inference/contexts/__init__.py:9: DeprecationWarning: The following imports from `dynamic_context.py` will be removed in this file in `megatron-core` 0.14. The imports here result in a cyclic import issue that causes rotary embeddings to import from Apex rather than Transformer Engine. + warnings.warn( + +../../looped-grpo-integration-20261003/runtime/megatron/megatron/core/optimizer/__init__.py:24 + /Users/0z5a/Documents/infra/looped-grpo-integration-20261003/runtime/megatron/megatron/core/optimizer/__init__.py:24: UserWarning: Transformer Engine and Apex are not installed. Falling back to Torch optimizers. + warnings.warn( + +../../looped-grpo-integration-20261003/runtime/megatron/megatron/core/optimizer/optimizer.py:28 + /Users/0z5a/Documents/infra/looped-grpo-integration-20261003/runtime/megatron/megatron/core/optimizer/optimizer.py:28: UserWarning: Transformer Engine and Apex are not installed. Falling back to local implementations of multi_tensor_applier and multi_tensor_scale + warnings.warn( + +../../looped-grpo-integration-20261003/runtime/megatron/megatron/core/optimizer/clip_grads.py:29 + /Users/0z5a/Documents/infra/looped-grpo-integration-20261003/runtime/megatron/megatron/core/optimizer/clip_grads.py:29: UserWarning: Transformer Engine and Apex are not installed. Falling back to local implementations of multi_tensor_applier, multi_tensor_l2norm, and multi_tensor_scale + warnings.warn( + +tests/plugins/test_differentiable_prefix.py: 90 warnings +tests/plugins/test_huginn_ppo_provider.py: 2 warnings +tests/plugins/test_looped_algorithms.py: 9 warnings +tests/plugins/test_looped_grpo.py: 4 warnings +tests/plugins/test_looped_response_readout.py: 42 warnings +tests/plugins/test_nanbeige_ppo_provider.py: 2 warnings +tests/plugins/test_ouro_critic.py: 2 warnings +tests/plugins/test_packed_recurrent_replay.py: 24 warnings +tests/plugins/test_recurrent_rematerialization.py: 39 warnings +tests/plugins/test_rltt_online_cycle.py: 9 warnings +tests/plugins/test_rltt_training.py: 36 warnings +tests/plugins/test_native_heldout_training.py: 3 warnings + /Users/0z5a/Documents/infra/looped-grpo-integration-20261003/runtime/megatron/megatron/core/transformer/transformer_config.py:1734: UserWarning: full scope is deprecated. Use empty cuda_graph_scope to capture the whole layer. + warnings.warn( + +tests/plugins/test_huginn_ppo_provider.py::test_packed_policy_values_gradients_and_hf_export[False] + /Users/0z5a/Documents/infra/looped-grpo-integration-20261003/runtime/megatron/megatron/core/models/backends.py:21: UserWarning: Apex is not installed. Falling back to Torch Norm + warnings.warn("Apex is not installed. Falling back to Torch Norm") + +tests/plugins/test_huginn_ppo_provider.py::test_packed_policy_values_gradients_and_hf_export[False] + /Users/0z5a/Documents/infra/looped-grpo-integration-20261003/runtime/megatron/megatron/core/transformer/multi_latent_attention.py:36: DeprecationWarning: The 'megatron.core.transformer.custom_layers.transformer_engine' + module is deprecated and will be removed in 0.10.0. Please use + 'megatron.core.extensions.transformer_engine' instead. + from megatron.core.transformer.custom_layers.transformer_engine import ( + +tests/plugins/test_huginn_ppo_provider.py::test_packed_policy_values_gradients_and_hf_export[False] + /Users/0z5a/Documents/infra/looped-grpo-integration-20261003/runtime/megatron/megatron/core/models/gpt/gpt_layer_specs.py:70: UserWarning: Apex is not installed. Falling back to Torch Norm + warnings.warn("Apex is not installed. Falling back to Torch Norm") + +-- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html +- generated xml file: /Users/0z5a/Documents/infra/flashrlt-20261005/evidence/native-eval-release.xml - +============================== slowest durations =============================== +3.84s call tests/plugins/test_looped_algorithms.py::test_three_updates_and_fresh_resume[ppo-nanbeige] +1.81s call tests/plugins/test_native_heldout_training.py::test_evaluation_preserves_updates_and_fresh_resume[huginn_raven] +1.34s call tests/plugins/test_rltt_online_cycle.py::test_fresh_reward_updates_and_reference_adam_resume[serial-huginn_raven] +1.16s call tests/plugins/test_rltt_online_cycle.py::test_fresh_reward_updates_and_reference_adam_resume[remat-huginn_raven] +1.00s call tests/plugins/test_rltt_online_cycle.py::test_fresh_reward_updates_and_reference_adam_resume[remat-nanbeige] +0.97s call tests/plugins/test_rltt_training.py::test_packed_reference_and_logical_microbatch_gradient[serial-True-token_mean-ouro] +0.94s call tests/plugins/test_rltt_online_cycle.py::test_fresh_reward_updates_and_reference_adam_resume[serial-ouro] +0.93s call tests/plugins/test_rltt_training.py::test_packed_reference_and_logical_microbatch_gradient[remat-False-response_mean-ouro] +0.88s call tests/plugins/test_recurrent_rematerialization.py::test_full_parameter_gradient_and_nonzero_update[plan5-False-huginn] +0.82s call tests/plugins/test_native_heldout_training.py::test_evaluation_preserves_updates_and_fresh_resume[ouro] +0.73s call tests/plugins/test_looped_algorithms.py::test_three_updates_and_fresh_resume[flow-dppo-huginn_raven] +0.69s call tests/plugins/test_rltt_online_cycle.py::test_fresh_reward_updates_and_reference_adam_resume[sdpa-reference-huginn_raven] +0.62s call tests/plugins/test_rltt_online_cycle.py::test_fresh_reward_updates_and_reference_adam_resume[remat-ouro] +0.61s call tests/plugins/test_rltt_online_cycle.py::test_fresh_reward_updates_and_reference_adam_resume[sdpa-reference-ouro] +0.61s call tests/plugins/test_looped_response_readout.py::test_response_boundary_loops_and_shared_parameter_gradients[streamed-1-True-ouro] +0.59s call tests/plugins/test_rltt_online_cycle.py::test_fresh_reward_updates_and_reference_adam_resume[serial-nanbeige] +0.57s call tests/plugins/test_packed_recurrent_replay.py::test_ragged_scores_gradients_and_optimizer_delta[16-True-ouro] +0.57s call tests/plugins/test_native_heldout_training.py::test_evaluation_preserves_updates_and_fresh_resume[nanbeige] +0.49s call tests/plugins/test_rltt_training.py::test_packed_reference_and_logical_microbatch_gradient[remat-False-response_mean-huginn] +0.48s call tests/plugins/test_packed_recurrent_replay.py::test_ragged_scores_gradients_and_optimizer_delta[16-True-huginn] +0.46s call tests/plugins/test_rltt_training.py::test_packed_reference_and_logical_microbatch_gradient[serial-True-response_mean-ouro] +0.45s call tests/plugins/test_rltt_training.py::test_packed_reference_and_logical_microbatch_gradient[remat-True-token_mean-ouro] +0.43s call tests/plugins/test_rltt_training.py::test_packed_reference_and_logical_microbatch_gradient[remat-False-token_mean-ouro] +0.42s call tests/plugins/test_looped_algorithms.py::test_three_updates_and_fresh_resume[dppo-huginn_raven] +0.40s call tests/plugins/test_rltt_training.py::test_packed_reference_and_logical_microbatch_gradient[remat-True-response_mean-ouro] +0.39s call tests/plugins/test_looped_algorithms.py::test_three_updates_and_fresh_resume[ppo-huginn_raven] +0.36s call tests/plugins/test_huginn_ppo_provider.py::test_packed_policy_values_gradients_and_hf_export[False] +0.34s call tests/plugins/test_rltt_training.py::test_packed_reference_and_logical_microbatch_gradient[sdpa-reference-True-response_mean-ouro] +0.33s call tests/plugins/test_looped_algorithms.py::test_three_updates_and_fresh_resume[flow-dppo-nanbeige] +0.30s call tests/plugins/test_rltt_training.py::test_packed_reference_and_logical_microbatch_gradient[remat-True-token_mean-huginn] +0.29s call tests/plugins/test_looped_algorithms.py::test_three_updates_and_fresh_resume[dppo-nanbeige] +0.28s call tests/plugins/test_rltt_training.py::test_packed_reference_and_logical_microbatch_gradient[sdpa-reference-True-token_mean-ouro] +0.28s call tests/plugins/test_rltt_online_cycle.py::test_fresh_reward_updates_and_reference_adam_resume[sdpa-reference-nanbeige] +0.27s call tests/plugins/test_recurrent_rematerialization.py::test_full_parameter_gradient_and_nonzero_update[plan5-True-ouro] +0.26s call tests/plugins/test_looped_algorithms.py::test_three_updates_and_fresh_resume[grpo-huginn_raven] +0.24s call tests/plugins/test_recurrent_rematerialization.py::test_full_parameter_gradient_and_nonzero_update[plan1-False-ouro] +0.24s call tests/plugins/test_rltt_training.py::test_packed_reference_and_logical_microbatch_gradient[remat-True-response_mean-huginn] +0.24s call tests/plugins/test_packed_recurrent_replay.py::test_ragged_scores_gradients_and_optimizer_delta[16-True-nanbeige] +0.24s call tests/plugins/test_rltt_training.py::test_packed_reference_and_logical_microbatch_gradient[remat-False-token_mean-huginn] +0.23s call tests/plugins/test_rltt_training.py::test_packed_reference_and_logical_microbatch_gradient[serial-True-token_mean-huginn] +0.23s call tests/plugins/test_packed_recurrent_replay.py::test_ragged_scores_gradients_and_optimizer_delta[16-False-huginn] +0.23s call tests/plugins/test_rltt_training.py::test_packed_reference_and_logical_microbatch_gradient[serial-False-response_mean-ouro] +0.22s call tests/plugins/test_rltt_training.py::test_packed_reference_and_logical_microbatch_gradient[serial-False-token_mean-ouro] +0.22s call tests/plugins/test_differentiable_prefix.py::test_full_gradient_and_update_across_suffix_microbatches[False-dtype0-groups1-False-huginn] +0.22s call tests/plugins/test_packed_recurrent_replay.py::test_ragged_scores_gradients_and_optimizer_delta[16-False-ouro] +0.21s call tests/test_rollout_metrics.py::test_decode_int32_meta_array_decodes_base64_to_tensor +0.21s call tests/plugins/test_rltt_training.py::test_packed_reference_and_logical_microbatch_gradient[serial-False-token_mean-huginn] +0.20s call tests/plugins/test_rltt_training.py::test_packed_reference_and_logical_microbatch_gradient[remat-True-token_mean-nanbeige] +0.19s call tests/plugins/test_rltt_training.py::test_packed_reference_and_logical_microbatch_gradient[serial-False-response_mean-huginn] +0.19s call tests/plugins/test_rltt_training.py::test_packed_reference_and_logical_microbatch_gradient[remat-False-response_mean-nanbeige] +0.19s call tests/plugins/test_differentiable_prefix.py::test_full_gradient_and_update_across_suffix_microbatches[False-dtype0-groups0-False-ouro] +0.19s call tests/plugins/test_rltt_training.py::test_packed_reference_and_logical_microbatch_gradient[sdpa-reference-True-response_mean-huginn] +0.17s call tests/plugins/test_rltt_training.py::test_packed_reference_and_logical_microbatch_gradient[serial-True-response_mean-huginn] +0.16s call tests/plugins/test_packed_recurrent_replay.py::test_ragged_scores_gradients_and_optimizer_delta[4-True-ouro] +0.15s call tests/plugins/test_rltt_training.py::test_packed_reference_and_logical_microbatch_gradient[sdpa-reference-True-token_mean-huginn] +0.15s call tests/plugins/test_rltt_training.py::test_packed_reference_and_logical_microbatch_gradient[remat-True-response_mean-nanbeige] +0.15s call tests/plugins/test_recurrent_rematerialization.py::test_full_parameter_gradient_and_nonzero_update[plan4-False-ouro] +0.15s call tests/plugins/test_rltt_training.py::test_packed_reference_and_logical_microbatch_gradient[sdpa-reference-False-token_mean-ouro] +0.15s call tests/plugins/test_looped_grpo.py::test_grouped_signal_packed_replay_publication_and_cpu_restore[True-huginn_raven] +0.15s call tests/plugins/test_rltt_training.py::test_packed_reference_and_logical_microbatch_gradient[sdpa-reference-False-response_mean-huginn] +0.14s call tests/plugins/test_looped_response_readout.py::test_response_boundary_loops_and_shared_parameter_gradients[streamed-0-True-ouro] +0.14s call tests/plugins/test_recurrent_rematerialization.py::test_full_parameter_gradient_and_nonzero_update[plan4-True-ouro] +0.14s call tests/plugins/test_looped_algorithms.py::test_three_updates_and_fresh_resume[grpo-nanbeige] +0.13s call tests/plugins/test_differentiable_prefix.py::test_full_gradient_and_update_across_suffix_microbatches[True-dtype1-groups1-True-ouro] +0.12s call tests/plugins/test_differentiable_prefix.py::test_full_gradient_and_update_across_suffix_microbatches[False-dtype1-groups1-True-ouro] +0.12s call tests/plugins/test_differentiable_prefix.py::test_full_gradient_and_update_across_suffix_microbatches[False-dtype1-groups0-True-ouro] +0.12s call tests/plugins/test_recurrent_rematerialization.py::test_full_parameter_gradient_and_nonzero_update[plan5-False-ouro] +0.12s call tests/plugins/test_rltt_training.py::test_packed_reference_and_logical_microbatch_gradient[serial-False-token_mean-nanbeige] +0.12s call tests/plugins/test_differentiable_prefix.py::test_full_gradient_and_update_across_suffix_microbatches[False-dtype0-groups0-True-huginn] +0.12s call tests/plugins/test_rltt_training.py::test_packed_reference_and_logical_microbatch_gradient[serial-True-response_mean-nanbeige] +0.12s call tests/plugins/test_rltt_training.py::test_packed_reference_and_logical_microbatch_gradient[remat-False-token_mean-nanbeige] +0.12s call tests/plugins/test_differentiable_prefix.py::test_full_gradient_and_update_across_suffix_microbatches[True-dtype0-groups2-True-huginn] +0.12s call tests/plugins/test_differentiable_prefix.py::test_full_gradient_and_update_across_suffix_microbatches[True-dtype1-groups1-False-ouro] +0.11s call tests/plugins/test_differentiable_prefix.py::test_full_gradient_and_update_across_suffix_microbatches[True-dtype1-groups0-True-huginn] +0.11s call tests/plugins/test_recurrent_rematerialization.py::test_full_parameter_gradient_and_nonzero_update[plan5-True-huginn] +0.11s call tests/plugins/test_differentiable_prefix.py::test_full_gradient_and_update_across_suffix_microbatches[True-dtype1-groups0-False-ouro] +0.11s call tests/plugins/test_recurrent_rematerialization.py::test_full_parameter_gradient_and_nonzero_update[plan3-True-huginn] +0.11s call tests/plugins/test_rltt_training.py::test_packed_reference_and_logical_microbatch_gradient[sdpa-reference-False-token_mean-huginn] +0.11s call tests/plugins/test_differentiable_prefix.py::test_full_gradient_and_update_across_suffix_microbatches[False-dtype1-groups2-True-huginn] +0.11s call tests/plugins/test_recurrent_rematerialization.py::test_full_parameter_gradient_and_nonzero_update[plan3-True-ouro] +0.11s call tests/plugins/test_differentiable_prefix.py::test_full_gradient_and_update_across_suffix_microbatches[False-dtype1-groups1-False-huginn] +0.11s call tests/plugins/test_rltt_training.py::test_packed_reference_and_logical_microbatch_gradient[serial-True-token_mean-nanbeige] +0.11s call tests/plugins/test_differentiable_prefix.py::test_full_gradient_and_update_across_suffix_microbatches[False-dtype1-groups2-True-ouro] +0.11s call tests/plugins/test_differentiable_prefix.py::test_full_gradient_and_update_across_suffix_microbatches[False-dtype0-groups1-False-ouro] +0.11s call tests/plugins/test_huginn_ppo_provider.py::test_packed_policy_values_gradients_and_hf_export[True] +0.11s call tests/plugins/test_rltt_training.py::test_packed_reference_and_logical_microbatch_gradient[sdpa-reference-False-response_mean-ouro] +0.11s call tests/plugins/test_packed_recurrent_replay.py::test_ragged_scores_gradients_and_optimizer_delta[4-True-huginn] +0.11s call tests/plugins/test_packed_recurrent_replay.py::test_ragged_scores_gradients_and_optimizer_delta[4-False-huginn] +0.11s call tests/plugins/test_differentiable_prefix.py::test_full_gradient_and_update_across_suffix_microbatches[True-dtype1-groups1-True-huginn] +0.10s call tests/plugins/test_differentiable_prefix.py::test_full_gradient_and_update_across_suffix_microbatches[False-dtype1-groups1-True-huginn] +0.10s call tests/plugins/test_differentiable_prefix.py::test_full_gradient_and_update_across_suffix_microbatches[True-dtype0-groups0-True-huginn] +0.10s call tests/plugins/test_packed_recurrent_replay.py::test_ragged_scores_gradients_and_optimizer_delta[1-True-ouro] +0.10s call tests/plugins/test_differentiable_prefix.py::test_full_gradient_and_update_across_suffix_microbatches[True-dtype0-groups1-True-huginn] +0.10s call tests/plugins/test_differentiable_prefix.py::test_full_gradient_and_update_across_suffix_microbatches[True-dtype0-groups0-True-ouro] +0.10s call tests/plugins/test_differentiable_prefix.py::test_full_gradient_and_update_across_suffix_microbatches[True-dtype1-groups2-True-huginn] +0.10s call tests/plugins/test_packed_recurrent_replay.py::test_ragged_scores_gradients_and_optimizer_delta[16-False-nanbeige] +0.10s call tests/plugins/test_differentiable_prefix.py::test_full_gradient_and_update_across_suffix_microbatches[True-dtype1-groups0-False-huginn] +0.10s call tests/plugins/test_recurrent_rematerialization.py::test_full_parameter_gradient_and_nonzero_update[plan4-True-huginn] +0.10s call tests/plugins/test_looped_response_readout.py::test_response_boundary_loops_and_shared_parameter_gradients[streamed-0-False-ouro] +0.10s call tests/plugins/test_differentiable_prefix.py::test_full_gradient_and_update_across_suffix_microbatches[True-dtype1-groups2-False-ouro] +0.10s call tests/plugins/test_differentiable_prefix.py::test_full_gradient_and_update_across_suffix_microbatches[True-dtype1-groups2-True-ouro] +0.10s call tests/plugins/test_differentiable_prefix.py::test_full_gradient_and_update_across_suffix_microbatches[True-dtype0-groups1-False-ouro] +0.10s call tests/plugins/test_looped_grpo.py::test_grouped_signal_packed_replay_publication_and_cpu_restore[False-huginn_raven] +0.09s call tests/plugins/test_differentiable_prefix.py::test_full_gradient_and_update_across_suffix_microbatches[False-dtype1-groups2-False-ouro] +0.09s call tests/plugins/test_differentiable_prefix.py::test_full_gradient_and_update_across_suffix_microbatches[False-dtype0-groups0-True-ouro] +0.09s call tests/plugins/test_differentiable_prefix.py::test_full_gradient_and_update_across_suffix_microbatches[True-dtype1-groups1-False-huginn] +0.09s call tests/plugins/test_differentiable_prefix.py::test_full_gradient_and_update_across_suffix_microbatches[False-dtype1-groups0-True-huginn] +0.09s call tests/plugins/test_recurrent_rematerialization.py::test_full_parameter_gradient_and_nonzero_update[plan0-True-huginn] +0.09s call tests/plugins/test_differentiable_prefix.py::test_full_gradient_and_update_across_suffix_microbatches[True-dtype0-groups2-True-ouro] +0.09s call tests/plugins/test_recurrent_rematerialization.py::test_full_parameter_gradient_and_nonzero_update[plan4-False-huginn] +0.09s call tests/plugins/test_differentiable_prefix.py::test_full_gradient_and_update_across_suffix_microbatches[True-dtype1-groups0-True-ouro] +0.09s call tests/plugins/test_recurrent_rematerialization.py::test_full_parameter_gradient_and_nonzero_update[plan4-False-nanbeige] +0.09s call tests/plugins/test_recurrent_rematerialization.py::test_full_parameter_gradient_and_nonzero_update[plan1-False-huginn] +0.09s call tests/plugins/test_rltt_training.py::test_packed_reference_and_logical_microbatch_gradient[serial-False-response_mean-nanbeige] +0.09s call tests/plugins/test_recurrent_rematerialization.py::test_full_parameter_gradient_and_nonzero_update[plan2-True-ouro] +0.09s call tests/plugins/test_differentiable_prefix.py::test_full_gradient_and_update_across_suffix_microbatches[False-dtype1-groups1-False-ouro] +0.09s call tests/rollout_backends/test_native_rlt_contract.py::test_http_client_concurrency_matches_backend[vllm] +0.09s call tests/plugins/test_differentiable_prefix.py::test_full_gradient_and_update_across_suffix_microbatches[True-dtype0-groups1-True-ouro] +0.09s call tests/plugins/test_looped_response_readout.py::test_response_boundary_loops_and_shared_parameter_gradients[dense-4-True-ouro] +0.09s call tests/plugins/test_differentiable_prefix.py::test_full_gradient_and_update_across_suffix_microbatches[True-dtype0-groups0-False-ouro] +0.08s call tests/plugins/test_rltt_training.py::test_packed_reference_and_logical_microbatch_gradient[sdpa-reference-True-response_mean-nanbeige] +0.08s call tests/plugins/test_differentiable_prefix.py::test_full_gradient_and_update_across_suffix_microbatches[False-dtype1-groups2-False-huginn] +0.08s call tests/plugins/test_differentiable_prefix.py::test_packed_suffix_empty_single_token_and_ragged_gradients[lengths2-huginn] +0.08s call tests/plugins/test_recurrent_rematerialization.py::test_full_parameter_gradient_and_nonzero_update[plan3-False-huginn] +0.08s call tests/rollout_backends/test_native_rlt_evaluation.py::test_complete_datasets_fixed_seeds_and_unchanged_training_state[huginn] +0.08s call tests/plugins/test_differentiable_prefix.py::test_full_gradient_and_update_across_suffix_microbatches[False-dtype0-groups1-True-ouro] +0.08s call tests/plugins/test_looped_response_readout.py::test_response_boundary_loops_and_shared_parameter_gradients[dense-1-True-ouro] +0.08s call tests/plugins/test_recurrent_rematerialization.py::test_full_parameter_gradient_and_nonzero_update[plan2-True-huginn] +0.08s call tests/plugins/test_differentiable_prefix.py::test_full_gradient_and_update_across_suffix_microbatches[True-dtype0-groups2-False-ouro] +0.08s call tests/plugins/test_differentiable_prefix.py::test_full_gradient_and_update_across_suffix_microbatches[False-dtype0-groups1-False-nanbeige] +0.08s call tests/plugins/test_looped_response_readout.py::test_response_boundary_loops_and_shared_parameter_gradients[streamed-4-True-ouro] +0.08s call tests/plugins/test_recurrent_rematerialization.py::test_full_parameter_gradient_and_nonzero_update[plan0-True-ouro] +0.08s call tests/plugins/test_recurrent_rematerialization.py::test_full_parameter_gradient_and_nonzero_update[plan5-False-nanbeige] +0.07s call tests/plugins/test_differentiable_prefix.py::test_full_gradient_and_update_across_suffix_microbatches[False-dtype0-groups0-False-huginn] +0.07s call tests/plugins/test_rltt_training.py::test_packed_reference_and_logical_microbatch_gradient[sdpa-reference-True-token_mean-nanbeige] +0.07s call tests/plugins/test_differentiable_prefix.py::test_full_gradient_and_update_across_suffix_microbatches[False-dtype1-groups0-False-huginn] +0.07s call tests/plugins/test_differentiable_prefix.py::test_full_gradient_and_update_across_suffix_microbatches[False-dtype1-groups1-True-nanbeige] +0.07s call tests/plugins/test_differentiable_prefix.py::test_full_gradient_and_update_across_suffix_microbatches[True-dtype1-groups2-False-huginn] +0.07s call tests/plugins/test_differentiable_prefix.py::test_full_gradient_and_update_across_suffix_microbatches[True-dtype0-groups0-False-huginn] +0.07s call tests/rollout_backends/test_native_rlt_evaluation.py::test_complete_datasets_fixed_seeds_and_unchanged_training_state[ouro] +0.07s call tests/plugins/test_recurrent_rematerialization.py::test_full_parameter_gradient_and_nonzero_update[plan5-True-nanbeige] +0.07s call tests/plugins/test_recurrent_rematerialization.py::test_full_parameter_gradient_and_nonzero_update[plan3-False-ouro] +0.07s call tests/plugins/test_differentiable_prefix.py::test_full_gradient_and_update_across_suffix_microbatches[False-dtype0-groups2-True-ouro] +0.07s call tests/plugins/test_differentiable_prefix.py::test_full_gradient_and_update_across_suffix_microbatches[True-dtype1-groups0-False-nanbeige] +0.07s call tests/plugins/test_differentiable_prefix.py::test_full_gradient_and_update_across_suffix_microbatches[False-dtype0-groups2-True-huginn] +0.07s call tests/plugins/test_recurrent_rematerialization.py::test_full_parameter_gradient_and_nonzero_update[plan0-False-huginn] +0.07s call tests/plugins/test_packed_recurrent_replay.py::test_ragged_scores_gradients_and_optimizer_delta[4-False-ouro] +0.07s call tests/plugins/test_differentiable_prefix.py::test_full_gradient_and_update_across_suffix_microbatches[True-dtype0-groups1-False-huginn] +0.07s call tests/plugins/test_differentiable_prefix.py::test_full_gradient_and_update_across_suffix_microbatches[False-dtype0-groups2-False-huginn] +0.07s call tests/plugins/test_differentiable_prefix.py::test_full_gradient_and_update_across_suffix_microbatches[False-dtype1-groups0-False-ouro] +0.07s call tests/plugins/test_differentiable_prefix.py::test_full_gradient_and_update_across_suffix_microbatches[True-dtype1-groups1-True-nanbeige] +0.06s call tests/plugins/test_recurrent_rematerialization.py::test_full_parameter_gradient_and_nonzero_update[plan0-True-nanbeige] +0.06s call tests/plugins/test_looped_grpo.py::test_grouped_signal_packed_replay_publication_and_cpu_restore[False-nanbeige] +0.06s call tests/plugins/test_recurrent_rematerialization.py::test_full_parameter_gradient_and_nonzero_update[plan1-False-nanbeige] +0.06s call tests/plugins/test_differentiable_prefix.py::test_full_gradient_and_update_across_suffix_microbatches[True-dtype0-groups1-True-nanbeige] +0.06s call tests/plugins/test_differentiable_prefix.py::test_full_gradient_and_update_across_suffix_microbatches[True-dtype1-groups2-True-nanbeige] +0.06s call tests/plugins/test_differentiable_prefix.py::test_full_gradient_and_update_across_suffix_microbatches[True-dtype0-groups2-False-huginn] +0.06s call tests/plugins/test_differentiable_prefix.py::test_full_gradient_and_update_across_suffix_microbatches[False-dtype1-groups2-True-nanbeige] +0.06s call tests/plugins/test_looped_response_readout.py::test_response_boundary_loops_and_shared_parameter_gradients[streamed-4-False-ouro] +0.06s call tests/plugins/test_looped_grpo.py::test_grouped_signal_packed_replay_publication_and_cpu_restore[True-nanbeige] +0.06s call tests/plugins/test_recurrent_rematerialization.py::test_full_parameter_gradient_and_nonzero_update[plan4-True-nanbeige] +0.06s call tests/plugins/test_differentiable_prefix.py::test_packed_suffix_empty_single_token_and_ragged_gradients[lengths2-ouro] +0.06s call tests/plugins/test_differentiable_prefix.py::test_full_gradient_and_update_across_suffix_microbatches[True-dtype1-groups1-False-nanbeige] +0.06s call tests/plugins/test_recurrent_rematerialization.py::test_full_parameter_gradient_and_nonzero_update[plan0-False-ouro] +0.06s call tests/plugins/test_rltt_training.py::test_packed_reference_and_logical_microbatch_gradient[sdpa-reference-False-token_mean-nanbeige] +0.06s call tests/plugins/test_looped_response_readout.py::test_response_boundary_loops_and_shared_parameter_gradients[dense-0-True-ouro] +0.06s call tests/plugins/test_looped_response_readout.py::test_response_boundary_loops_and_shared_parameter_gradients[streamed-1-True-huginn] +0.06s call tests/plugins/test_differentiable_prefix.py::test_full_gradient_and_update_across_suffix_microbatches[False-dtype0-groups2-False-ouro] +0.06s call tests/plugins/test_recurrent_rematerialization.py::test_full_parameter_gradient_and_nonzero_update[plan2-True-nanbeige] +0.06s call tests/plugins/test_recurrent_rematerialization.py::test_full_parameter_gradient_and_nonzero_update[plan3-False-nanbeige] +0.06s call tests/plugins/test_differentiable_prefix.py::test_full_gradient_and_update_across_suffix_microbatches[False-dtype0-groups0-True-nanbeige] +0.06s call tests/test_looped_grpo_recipe.py::test_recipe_roles_groups_precision_and_resume[3-dppo-True-ouro] +0.05s call tests/plugins/test_differentiable_prefix.py::test_full_gradient_and_update_across_suffix_microbatches[False-dtype1-groups0-True-nanbeige] +0.05s call tests/plugins/test_differentiable_prefix.py::test_full_gradient_and_update_across_suffix_microbatches[False-dtype0-groups0-False-nanbeige] +0.05s call tests/plugins/test_recurrent_rematerialization.py::test_full_parameter_gradient_and_nonzero_update[plan3-True-nanbeige] +0.05s call tests/test_looped_grpo_recipe.py::test_recipe_roles_groups_precision_and_resume[3-dppo-True-nanbeige] +0.05s call tests/plugins/test_looped_response_readout.py::test_response_boundary_loops_and_shared_parameter_gradients[streamed-0-False-huginn] +0.05s call tests/plugins/test_differentiable_prefix.py::test_full_gradient_and_update_across_suffix_microbatches[False-dtype0-groups1-True-huginn] +0.05s call tests/plugins/test_differentiable_prefix.py::test_full_gradient_and_update_across_suffix_microbatches[True-dtype1-groups2-False-nanbeige] +0.05s call tests/plugins/test_differentiable_prefix.py::test_full_gradient_and_update_across_suffix_microbatches[False-dtype1-groups2-False-nanbeige] +0.05s call tests/plugins/test_rltt_training.py::test_packed_reference_and_logical_microbatch_gradient[sdpa-reference-False-response_mean-nanbeige] +0.05s call tests/plugins/test_packed_recurrent_replay.py::test_ragged_scores_gradients_and_optimizer_delta[4-True-nanbeige] +0.05s call tests/plugins/test_nanbeige_ppo_provider.py::test_two_loop_logits_values_and_gradients[False] +0.05s call tests/plugins/test_differentiable_prefix.py::test_full_gradient_and_update_across_suffix_microbatches[True-dtype1-groups0-True-nanbeige] +0.05s teardown tests/plugins/test_recurrent_rematerialization.py::test_full_parameter_gradient_and_nonzero_update[plan1-False-ouro] +0.05s call tests/plugins/test_differentiable_prefix.py::test_full_gradient_and_update_across_suffix_microbatches[True-dtype0-groups0-False-nanbeige] +0.05s call tests/plugins/test_packed_recurrent_replay.py::test_ragged_scores_gradients_and_optimizer_delta[1-False-ouro] +0.05s call tests/plugins/test_differentiable_prefix.py::test_full_gradient_and_update_across_suffix_microbatches[True-dtype0-groups1-False-nanbeige] +0.04s call tests/plugins/test_recurrent_rematerialization.py::test_full_parameter_gradient_and_nonzero_update[plan2-False-huginn] +0.04s call tests/plugins/test_differentiable_prefix.py::test_full_gradient_and_update_across_suffix_microbatches[False-dtype0-groups2-False-nanbeige] +0.04s call tests/plugins/test_differentiable_prefix.py::test_full_gradient_and_update_across_suffix_microbatches[True-dtype0-groups0-True-nanbeige] +0.04s call tests/test_looped_grpo_recipe.py::test_recipe_roles_groups_precision_and_resume[None-grpo-False-ouro] +0.04s call tests/plugins/test_recurrent_rematerialization.py::test_full_parameter_gradient_and_nonzero_update[plan2-False-ouro] +0.04s call tests/plugins/test_looped_response_readout.py::test_response_boundary_loops_and_shared_parameter_gradients[dense-1-True-huginn] +0.04s call tests/plugins/test_recurrent_rematerialization.py::test_full_parameter_gradient_and_nonzero_update[plan1-True-huginn] +0.04s call tests/plugins/test_differentiable_prefix.py::test_full_gradient_and_update_across_suffix_microbatches[True-dtype0-groups2-False-nanbeige] +0.04s call tests/plugins/test_differentiable_prefix.py::test_full_gradient_and_update_across_suffix_microbatches[False-dtype1-groups1-False-nanbeige] +0.04s call tests/plugins/test_looped_response_readout.py::test_response_boundary_loops_and_shared_parameter_gradients[dense-4-True-huginn] +0.04s call tests/plugins/test_recurrent_rematerialization.py::test_full_parameter_gradient_and_nonzero_update[plan1-True-ouro] +0.04s call tests/rollout_backends/test_native_rlt_evaluation.py::test_complete_datasets_fixed_seeds_and_unchanged_training_state[nanbeige] +0.04s call tests/plugins/test_looped_response_readout.py::test_response_boundary_loops_and_shared_parameter_gradients[streamed-0-True-huginn] +0.04s setup tests/plugins/test_streamed_looped_readout.py::test_loss_and_all_readout_gradients[0.7-32-dtype0] +0.04s call tests/test_looped_grpo_recipe.py::test_recipe_roles_groups_precision_and_resume[3-dppo-False-huginn_raven] +0.04s call tests/plugins/test_packed_recurrent_replay.py::test_ragged_scores_gradients_and_optimizer_delta[4-False-nanbeige] +0.04s call tests/plugins/test_differentiable_prefix.py::test_packed_suffix_empty_single_token_and_ragged_gradients[lengths1-ouro] +0.04s call tests/plugins/test_differentiable_prefix.py::test_packed_suffix_empty_single_token_and_ragged_gradients[lengths1-huginn] +0.04s call tests/plugins/test_recurrent_rematerialization.py::test_serial_empty_response_retains_zero_gradient[ouro] +0.04s call tests/test_looped_grpo_recipe.py::test_recipe_roles_groups_precision_and_resume[3-ppo-True-huginn_raven] +0.04s call tests/plugins/test_packed_recurrent_replay.py::test_ragged_scores_gradients_and_optimizer_delta[1-True-huginn] +0.04s call tests/plugins/test_differentiable_prefix.py::test_packed_suffix_empty_single_token_and_ragged_gradients[lengths0-huginn] +0.04s call tests/plugins/test_differentiable_prefix.py::test_first_response_readout_is_a_live_boundary[ouro] +0.04s call tests/plugins/test_recurrent_rematerialization.py::test_serial_empty_response_retains_zero_gradient[huginn] +0.04s call tests/plugins/test_packed_recurrent_replay.py::test_ragged_scores_gradients_and_optimizer_delta[1-False-huginn] +0.04s call tests/plugins/test_looped_response_readout.py::test_response_boundary_loops_and_shared_parameter_gradients[streamed-0-True-nanbeige] +0.04s call tests/test_looped_grpo_recipe.py::test_recipe_roles_groups_precision_and_resume[None-grpo-False-nanbeige] +0.04s call tests/plugins/test_looped_response_readout.py::test_response_boundary_loops_and_shared_parameter_gradients[streamed-4-False-huginn] +0.04s call tests/plugins/test_differentiable_prefix.py::test_full_gradient_and_update_across_suffix_microbatches[True-dtype0-groups2-True-nanbeige] +0.04s call tests/plugins/test_looped_response_readout.py::test_response_boundary_loops_and_shared_parameter_gradients[streamed-1-False-ouro] +0.04s call tests/plugins/test_differentiable_prefix.py::test_packed_suffix_empty_single_token_and_ragged_gradients[lengths2-nanbeige] +0.04s call tests/plugins/test_recurrent_rematerialization.py::test_full_parameter_gradient_and_nonzero_update[plan2-False-nanbeige] +0.04s call tests/plugins/test_differentiable_prefix.py::test_full_gradient_and_update_across_suffix_microbatches[False-dtype1-groups0-False-nanbeige] +0.04s call tests/plugins/test_nanbeige_ppo_provider.py::test_two_loop_logits_values_and_gradients[True] +0.03s call tests/plugins/test_looped_response_readout.py::test_response_boundary_loops_and_shared_parameter_gradients[streamed-4-True-huginn] +0.03s call tests/plugins/test_differentiable_prefix.py::test_full_gradient_and_update_across_suffix_microbatches[False-dtype0-groups2-True-nanbeige] +0.03s call tests/plugins/test_looped_response_readout.py::test_response_boundary_loops_and_shared_parameter_gradients[dense-4-False-huginn] +0.03s call tests/plugins/test_looped_response_readout.py::test_response_boundary_loops_and_shared_parameter_gradients[dense-4-False-ouro] +0.03s call tests/plugins/test_recurrent_rematerialization.py::test_full_parameter_gradient_and_nonzero_update[plan0-False-nanbeige] +0.03s call tests/plugins/test_differentiable_prefix.py::test_packed_suffix_empty_single_token_and_ragged_gradients[lengths0-ouro] +0.03s call tests/plugins/test_looped_response_readout.py::test_response_boundary_loops_and_shared_parameter_gradients[dense-0-True-huginn] +0.03s call tests/plugins/test_packed_recurrent_replay.py::test_sequence_isolation_and_reordered_packs[True-ouro] +0.03s call tests/plugins/test_packed_recurrent_replay.py::test_ragged_scores_gradients_and_optimizer_delta[1-True-nanbeige] +0.03s call tests/test_looped_grpo_recipe.py::test_recipe_roles_groups_precision_and_resume[None-grpo-True-ouro] +0.03s call tests/plugins/test_differentiable_prefix.py::test_packed_suffix_empty_single_token_and_ragged_gradients[lengths1-nanbeige] +0.03s call tests/plugins/test_looped_response_readout.py::test_response_boundary_loops_and_shared_parameter_gradients[streamed-1-True-nanbeige] +0.03s call tests/plugins/test_packed_recurrent_replay.py::test_sequence_isolation_and_reordered_packs[False-ouro] +0.03s call tests/plugins/test_looped_response_readout.py::test_response_boundary_loops_and_shared_parameter_gradients[dense-0-False-huginn] +0.03s call tests/rollout_backends/test_native_rlt_contract.py::test_huginn_aliases_static_buffer_and_noise_profile +0.03s call tests/plugins/test_packed_recurrent_replay.py::test_ragged_scores_gradients_and_optimizer_delta[1-False-nanbeige] +0.03s call tests/plugins/test_packed_recurrent_replay.py::test_sequence_isolation_and_reordered_packs[True-huginn] +0.03s call tests/plugins/test_looped_response_readout.py::test_response_boundary_loops_and_shared_parameter_gradients[dense-1-False-ouro] +0.03s call tests/plugins/test_differentiable_prefix.py::test_full_gradient_and_update_across_suffix_microbatches[False-dtype0-groups1-True-nanbeige] +0.03s call tests/test_looped_grpo_recipe.py::test_recipe_roles_groups_precision_and_resume[3-grpo-False-huginn_raven] +0.03s call tests/plugins/test_streamed_looped_readout.py::test_finite_difference_gradcheck +0.03s call tests/plugins/test_looped_response_readout.py::test_response_boundary_loops_and_shared_parameter_gradients[dense-1-False-huginn] +0.03s call tests/plugins/test_looped_response_readout.py::test_response_boundary_loops_and_shared_parameter_gradients[streamed-1-False-huginn] +0.03s call tests/plugins/test_looped_response_readout.py::test_response_boundary_loops_and_shared_parameter_gradients[dense-0-False-ouro] +0.03s call tests/plugins/test_streamed_looped_readout.py::test_recurrent_parameters_credit_and_partition_invariance[1.5-False-response_mean] +0.03s call tests/rollout_backends/test_native_rlt_evaluation.py::test_policy_publication_between_batches_invalidates_evaluation +0.03s call tests/plugins/test_looped_response_readout.py::test_one_token_prompt_empty_response[ouro] +0.03s call tests/test_looped_grpo_recipe.py::test_recipe_roles_groups_precision_and_resume[3-dppo-True-huginn_raven] +0.03s call tests/test_looped_grpo_recipe.py::test_recipe_roles_groups_precision_and_resume[3-ppo-True-ouro] +0.03s call tests/plugins/test_streamed_looped_readout.py::test_recurrent_parameters_credit_and_partition_invariance[0.0-True-token_mean] +0.03s call tests/plugins/test_looped_response_readout.py::test_response_boundary_loops_and_shared_parameter_gradients[dense-1-True-nanbeige] +0.03s call tests/plugins/test_looped_response_readout.py::test_response_boundary_loops_and_shared_parameter_gradients[streamed-0-False-nanbeige] +0.03s call tests/plugins/test_recurrent_rematerialization.py::test_full_parameter_gradient_and_nonzero_update[plan1-True-nanbeige] +0.03s call tests/plugins/test_streamed_looped_readout.py::test_loss_and_all_readout_gradients[0.7-1-dtype0] +0.03s call tests/test_looped_grpo_recipe.py::test_recipe_roles_groups_precision_and_resume[3-grpo-True-ouro] +0.03s call tests/plugins/test_differentiable_prefix.py::test_huginn_prefix_does_not_alias_different_latent_identities +0.02s call tests/plugins/test_packed_recurrent_replay.py::test_sequence_isolation_and_reordered_packs[False-huginn] +0.02s call tests/plugins/test_streamed_looped_readout.py::test_recurrent_parameters_credit_and_partition_invariance[0.0-False-token_mean] +0.02s call tests/plugins/test_differentiable_prefix.py::test_empty_response_and_replaced_parameter[ouro] +0.02s call tests/test_looped_grpo_recipe.py::test_recipe_roles_groups_precision_and_resume[3-dppo-False-ouro] +0.02s call tests/plugins/test_differentiable_prefix.py::test_first_response_readout_is_a_live_boundary[nanbeige] +0.02s call tests/plugins/test_ouro_critic.py::test_critic_head_and_core_gradients[True] +0.02s call tests/test_looped_grpo_recipe.py::test_recipe_roles_groups_precision_and_resume[3-flow-dppo-True-ouro] +0.02s call tests/plugins/test_streamed_looped_readout.py::test_recurrent_parameters_credit_and_partition_invariance[0.0-False-response_mean] +0.02s call tests/rollout_backends/test_native_rlt_groups.py::test_native_groups_and_actual_math_rewards +0.02s call tests/plugins/test_looped_response_readout.py::test_response_boundary_loops_and_shared_parameter_gradients[dense-4-True-nanbeige] +0.02s call tests/test_megatron_argument_validation.py::test_update_weight_disk_dir_required_for_disk_transport +0.02s setup tests/rollout_backends/test_native_rlt_contract.py::test_huginn_aliases_static_buffer_and_noise_profile +0.02s call tests/plugins/test_recurrent_rematerialization.py::test_serial_empty_response_retains_zero_gradient[nanbeige] +0.02s call tests/plugins/test_looped_response_readout.py::test_response_boundary_loops_and_shared_parameter_gradients[dense-0-True-nanbeige] +0.02s call tests/plugins/test_looped_response_readout.py::test_response_boundary_loops_and_shared_parameter_gradients[streamed-1-False-nanbeige] +0.02s call tests/plugins/test_differentiable_prefix.py::test_packed_suffix_empty_single_token_and_ragged_gradients[lengths0-nanbeige] +0.02s call tests/plugins/test_looped_response_readout.py::test_response_boundary_loops_and_shared_parameter_gradients[streamed-4-True-nanbeige] +0.02s call tests/plugins/test_differentiable_prefix.py::test_prefix_backward_is_once_per_logical_group +0.02s call tests/test_looped_grpo_recipe.py::test_recipe_roles_groups_precision_and_resume[3-ppo-False-huginn_raven] +0.02s call tests/plugins/test_looped_response_readout.py::test_response_boundary_loops_and_shared_parameter_gradients[dense-1-False-nanbeige] +0.02s call tests/plugins/test_differentiable_prefix.py::test_empty_response_and_replaced_parameter[huginn] +0.02s call tests/test_looped_grpo_recipe.py::test_recipe_roles_groups_precision_and_resume[3-grpo-True-nanbeige] +0.02s call tests/test_looped_grpo_recipe.py::test_recipe_roles_groups_precision_and_resume[3-ppo-True-nanbeige] +0.02s call tests/test_looped_grpo_recipe.py::test_recipe_roles_groups_precision_and_resume[3-grpo-False-nanbeige] +0.02s call tests/plugins/test_streamed_looped_readout.py::test_recurrent_parameters_credit_and_partition_invariance[1.5-True-token_mean] +0.02s call tests/plugins/test_streamed_looped_readout.py::test_recurrent_parameters_credit_and_partition_invariance[1.5-False-token_mean] +0.02s call tests/test_native_eval_schedule.py::test_completed_update_indices_preserve_initial_and_resumed_eval[vllm-rlt-0-3-expected0] +0.02s call tests/plugins/test_streamed_looped_readout.py::test_recurrent_parameters_credit_and_partition_invariance[0.0-True-response_mean] +0.02s call tests/rollout_backends/test_native_rlt_contract.py::test_incomplete_publication_cannot_resume_or_reuse_old_manifest +0.02s call tests/plugins/test_looped_response_readout.py::test_response_boundary_loops_and_shared_parameter_gradients[streamed-4-False-nanbeige] +0.02s call tests/test_megatron_argument_validation.py::test_force_fp8_ue8m0_scale_argument +0.02s call tests/rollout_backends/test_native_rlt_evaluation.py::test_sampling_and_chat_overrides_are_used +0.02s call tests/rollout_backends/test_native_rlt_evaluation.py::test_reward_key_and_metadata_override +0.02s call tests/plugins/test_looped_response_readout.py::test_response_boundary_loops_and_shared_parameter_gradients[dense-4-False-nanbeige] +0.02s call tests/test_looped_grpo_recipe.py::test_recipe_roles_groups_precision_and_resume[None-grpo-False-huginn_raven] +0.02s call tests/rollout_backends/test_native_rlt_contract.py::test_publication_cohort_and_serialization +0.02s call tests/plugins/test_looped_response_readout.py::test_terminal_readout_keeps_only_final_loop[ouro] +0.02s call tests/plugins/test_ouro_critic.py::test_critic_head_and_core_gradients[False] +0.02s setup tests/plugins/test_looped_algorithms.py::test_three_updates_and_fresh_resume[dppo-huginn_raven] +0.02s call tests/plugins/test_looped_response_readout.py::test_one_token_prompt_empty_response[huginn] +0.02s teardown tests/plugins/test_rltt_online_cycle.py::test_fresh_reward_updates_and_reference_adam_resume[serial-ouro] +0.02s call tests/plugins/test_packed_recurrent_replay.py::test_sequence_isolation_and_reordered_packs[True-nanbeige] +0.01s call tests/plugins/test_streamed_looped_readout.py::test_recurrent_parameters_credit_and_partition_invariance[1.5-True-response_mean] +0.01s call tests/plugins/test_looped_response_readout.py::test_terminal_readout_keeps_only_final_loop[huginn] +0.01s call tests/plugins/test_differentiable_prefix.py::test_policy_prompt_depth_and_parameter_mutation_rejected +0.01s call tests/plugins/test_packed_recurrent_replay.py::test_sequence_isolation_and_reordered_packs[False-nanbeige] +0.01s call tests/plugins/test_differentiable_prefix.py::test_group_partition_rejects_missing_and_duplicate_suffixes +0.01s call tests/plugins/test_recurrent_rematerialization.py::test_query_chunks_keep_full_prefix_and_key_value_gradients[1] +0.01s call tests/test_megatron_argument_validation.py::test_update_weight_delta_requires_local_checkpoint_dir +0.01s call tests/plugins/test_differentiable_prefix.py::test_empty_response_and_replaced_parameter[nanbeige] +0.01s setup tests/rollout_backends/test_native_rlt_evaluation.py::test_complete_datasets_fixed_seeds_and_unchanged_training_state[ouro] +0.01s call tests/plugins/test_looped_response_readout.py::test_response_boundary_loops_and_shared_parameter_gradients[dense-0-False-nanbeige] +0.01s call tests/test_looped_grpo_recipe.py::test_recipe_roles_groups_precision_and_resume[3-dppo-False-nanbeige] +0.01s call tests/rollout_backends/test_native_rlt_groups.py::test_native_rejects_corrupt_groups_before_rewards[reorder] +0.01s call tests/test_megatron_argument_validation.py::test_update_weight_delta_requires_disk_transport +0.01s call tests/test_megatron_argument_validation.py::test_update_weight_delta_rejects_colocate +0.01s call tests/plugins/test_streamed_looped_readout.py::test_loss_and_all_readout_gradients[0.7-1-dtype2] +0.01s call tests/test_megatron_argument_validation.py::test_vime_validate_args_rejects_non_positive_rollout_temperature[0.0] +0.01s call tests/test_megatron_argument_validation.py::test_colocate_offload_resolution[False-False-None-False] +0.01s call tests/test_native_eval_schedule.py::test_pass_rate_uses_each_eval_datasets_completion_count +0.01s call tests/test_megatron_argument_validation.py::test_vime_validate_args_rejects_non_positive_rollout_temperature[-0.1] +0.01s call tests/test_megatron_argument_validation.py::test_vime_validate_args_preserves_explicit_start_rollout_id +0.01s teardown tests/plugins/test_packed_recurrent_replay.py::test_ragged_scores_gradients_and_optimizer_delta[16-True-ouro] +0.01s call tests/test_megatron_argument_validation.py::test_colocate_offload_resolution[None-False-True-True] +0.01s call tests/test_megatron_argument_validation.py::test_vime_validate_args_rejects_equal_debug_data_paths +0.01s call tests/test_megatron_argument_validation.py::test_vime_validate_args_defaults_start_rollout_id_to_zero +0.01s call tests/test_megatron_argument_validation.py::test_colocate_offload_resolution[None-True-True-False] +0.01s call tests/test_megatron_argument_validation.py::test_colocate_offload_resolution[None-False-False-True] +0.01s call tests/test_megatron_argument_validation.py::test_colocate_offload_resolution[None-True-False-True] +0.01s call tests/rollout_backends/test_native_rlt_evaluation.py::test_reordered_or_missing_heldout_samples_are_rejected +0.01s call tests/test_megatron_argument_validation.py::test_vime_validate_args_derives_dspark_from_speculative_method +0.01s call tests/test_megatron_argument_validation.py::test_colocate_offload_resolution[None-True-None-False] +0.01s call tests/test_megatron_argument_validation.py::test_colocate_offload_resolution[None-False-True-False] +0.01s call tests/test_megatron_argument_validation.py::test_colocate_offload_resolution[None-False-None-True] +0.01s call tests/test_megatron_argument_validation.py::test_vime_validate_args_preserves_larger_rollout_gpus_under_colocate +0.01s call tests/plugins/test_looped_response_readout.py::test_one_token_prompt_empty_response[nanbeige] +0.01s call tests/test_megatron_argument_validation.py::test_colocate_offload_resolution[None-True-None-True] +0.01s call tests/test_megatron_argument_validation.py::test_colocate_offload_resolution[None-False-None-False] +0.01s teardown tests/plugins/test_native_heldout_training.py::test_evaluation_preserves_updates_and_fresh_resume[ouro] +0.01s call tests/test_megatron_argument_validation.py::test_vime_validate_args_preserves_zero_rollout_gpus_without_colocate +0.01s call tests/test_megatron_argument_validation.py::test_colocate_offload_resolution[False-True-True-True] +0.01s call tests/test_megatron_argument_validation.py::test_colocate_offload_resolution[None-False-False-False] +0.01s call tests/test_megatron_argument_validation.py::test_vime_validate_args_preserves_zero_rollout_gpus_under_colocate +0.01s call tests/test_megatron_argument_validation.py::test_colocate_offload_resolution[None-True-False-False] +0.01s call tests/rollout_backends/test_native_rlt_contract.py::test_http_client_concurrency_matches_backend[vllm-rlt] +0.01s call tests/test_megatron_argument_validation.py::test_colocate_offload_resolution[False-False-None-True] +0.01s teardown tests/plugins/test_rltt_training.py::test_packed_reference_and_logical_microbatch_gradient[remat-False-response_mean-huginn] +0.01s call tests/plugins/test_streamed_looped_readout.py::test_loss_and_all_readout_gradients[0.7-32-dtype0] +0.01s call tests/test_megatron_argument_validation.py::test_colocate_offload_resolution[None-True-True-True] +0.01s call tests/test_megatron_argument_validation.py::test_colocate_offload_resolution[False-True-False-True] +0.01s call tests/test_megatron_argument_validation.py::test_update_weight_delta_disk_is_valid +0.01s call tests/test_looped_grpo_recipe.py::test_recipe_roles_groups_precision_and_resume[None-rltt-True-ouro] +0.01s call tests/plugins/test_looped_dppo.py::test_decoupled_loss_matches_three_policy_reference +0.01s call tests/test_looped_grpo_recipe.py::test_recipe_roles_groups_precision_and_resume[None-grpo-True-nanbeige] +0.01s call tests/test_megatron_argument_validation.py::test_colocate_offload_resolution[False-True-True-False] +0.01s teardown tests/plugins/test_rltt_online_cycle.py::test_fresh_reward_updates_and_reference_adam_resume[serial-huginn_raven] +0.01s call tests/test_megatron_argument_validation.py::test_colocate_offload_resolution[False-False-True-False] +0.01s call tests/plugins/test_streamed_looped_readout.py::test_loss_and_all_readout_gradients[0.7-4-dtype1] +0.01s call tests/test_megatron_argument_validation.py::test_colocate_offload_resolution[False-True-None-False] +0.01s call tests/test_rollout_metrics.py::test_top_p_kept_vocab_metric_uses_loss_mask +0.01s call tests/test_megatron_argument_validation.py::test_colocate_offload_resolution[False-True-False-False] +0.01s call tests/test_megatron_argument_validation.py::test_colocate_offload_resolution[False-False-True-True] +0.01s call tests/test_megatron_argument_validation.py::test_colocate_offload_resolution[False-True-None-True] +0.01s call tests/test_megatron_argument_validation.py::test_colocate_offload_resolution[False-False-False-True] +0.01s setup tests/rollout_backends/test_native_rlt_evaluation.py::test_sampling_and_chat_overrides_are_used +0.01s call tests/test_megatron_argument_validation.py::test_colocate_offload_resolution[False-False-False-False] +0.01s call tests/test_looped_grpo_recipe.py::test_recipe_roles_groups_precision_and_resume[None-flow-dppo-False-nanbeige] +0.01s setup tests/plugins/test_native_heldout_training.py::test_evaluation_preserves_updates_and_fresh_resume[ouro] +0.01s setup tests/plugins/test_huginn_ppo_provider.py::test_packed_policy_values_gradients_and_hf_export[False] +0.01s call tests/test_looped_grpo_recipe.py::test_recipe_roles_groups_precision_and_resume[3-ppo-False-ouro] +0.01s call tests/test_looped_grpo_recipe.py::test_grpo_rejects_single_completion_before_starting_ray +0.01s call tests/test_looped_grpo_recipe.py::test_recipe_roles_groups_precision_and_resume[None-rltt-False-nanbeige] +0.01s call tests/test_looped_grpo_recipe.py::test_recipe_roles_groups_precision_and_resume[None-dppo-True-nanbeige] +0.01s call tests/test_looped_grpo_recipe.py::test_recipe_roles_groups_precision_and_resume[None-flow-dppo-True-huginn_raven] +0.01s setup tests/rollout_backends/test_native_rlt_evaluation.py::test_unsupported_eval_settings_reject_before_generation[no_stop_trim-False] +0.01s call tests/test_looped_grpo_recipe.py::test_recipe_roles_groups_precision_and_resume[None-flow-dppo-True-nanbeige] +0.01s call tests/test_looped_grpo_recipe.py::test_recipe_roles_groups_precision_and_resume[None-ppo-True-nanbeige] +0.01s call tests/plugins/test_looped_response_readout.py::test_terminal_readout_keeps_only_final_loop[nanbeige] +0.01s call tests/plugins/test_differentiable_prefix.py::test_strict_fp64_attention_and_all_boundary_adjoints[1-1] +0.01s setup tests/rollout_backends/test_native_rlt_evaluation.py::test_complete_datasets_fixed_seeds_and_unchanged_training_state[huginn] +0.01s call tests/plugins/test_streamed_looped_readout.py::test_loss_and_all_readout_gradients[0.7-1-dtype1] +0.01s setup tests/rollout_backends/test_native_rlt_contract.py::test_publication_cohort_and_serialization +0.01s setup tests/rollout_backends/test_native_rlt_evaluation.py::test_policy_publication_between_batches_invalidates_evaluation +0.01s call tests/test_looped_grpo_recipe.py::test_recipe_roles_groups_precision_and_resume[None-grpo-True-huginn_raven] +0.01s call tests/test_looped_grpo_recipe.py::test_recipe_roles_groups_precision_and_resume[None-rltt-True-nanbeige] +0.01s call tests/test_looped_grpo_recipe.py::test_recipe_roles_groups_precision_and_resume[None-dppo-True-ouro] +0.01s call tests/test_looped_grpo_recipe.py::test_recipe_roles_groups_precision_and_resume[3-grpo-True-huginn_raven] +0.01s call tests/plugins/test_recurrent_rematerialization.py::test_query_chunks_keep_full_prefix_and_key_value_gradients[3] +0.01s call tests/rollout_backends/test_native_rlt_evaluation.py::test_unsupported_eval_settings_reject_before_generation[no_stop_trim-False] +0.01s call tests/test_looped_grpo_recipe.py::test_recipe_roles_groups_precision_and_resume[None-rltt-True-huginn_raven] +0.01s call tests/test_looped_grpo_recipe.py::test_recipe_roles_groups_precision_and_resume[3-grpo-False-ouro] +0.01s call tests/test_looped_grpo_recipe.py::test_recipe_roles_groups_precision_and_resume[None-flow-dppo-True-ouro] +0.01s call tests/test_rollout_metrics.py::test_append_response_tokens_merges_top_p_tensors +0.01s call tests/plugins/test_streamed_looped_readout.py::test_loss_and_all_readout_gradients[1.0-1-dtype1] +0.01s setup tests/rollout_backends/test_native_rlt_contract.py::test_incomplete_publication_cannot_resume_or_reuse_old_manifest +0.01s call tests/test_looped_grpo_recipe.py::test_recipe_roles_groups_precision_and_resume[None-rltt-False-huginn_raven] +0.01s call tests/rollout_backends/test_native_rlt_evaluation.py::test_unsupported_eval_settings_reject_before_generation[stop-value0] +0.01s call tests/test_looped_grpo_recipe.py::test_recipe_roles_groups_precision_and_resume[None-ppo-False-ouro] +0.01s call tests/rollout_backends/test_native_rlt_evaluation.py::test_unsupported_eval_settings_reject_before_generation[min_new_tokens-1] +0.01s call tests/plugins/test_differentiable_prefix.py::test_strict_fp64_attention_and_all_boundary_adjoints[3-5] +0.01s setup tests/rollout_backends/test_native_rlt_evaluation.py::test_unsupported_eval_settings_reject_before_generation[eval_task_timeout-1] +0.01s call tests/test_looped_grpo_recipe.py::test_recipe_roles_groups_precision_and_resume[3-rltt-True-huginn_raven] +0.01s call tests/test_looped_grpo_recipe.py::test_recipe_roles_groups_precision_and_resume[None-flow-dppo-False-huginn_raven] +0.01s setup tests/rollout_backends/test_native_rlt_evaluation.py::test_unsupported_eval_settings_reject_before_generation[stop-value0] +0.01s call tests/test_rollout_data_utils.py::test_save_and_load_debug_rollout_data_round_trip +0.01s call tests/test_looped_grpo_recipe.py::test_recipe_roles_groups_precision_and_resume[None-dppo-False-huginn_raven] +0.01s setup tests/rollout_backends/test_native_rlt_evaluation.py::test_reordered_or_missing_heldout_samples_are_rejected +0.01s call tests/test_looped_grpo_recipe.py::test_recipe_roles_groups_precision_and_resume[None-dppo-True-huginn_raven] +0.01s setup tests/rollout_backends/test_native_rlt_evaluation.py::test_long_heldout_prompts_are_not_silently_dropped[context] +0.01s call tests/test_looped_grpo_recipe.py::test_recipe_roles_groups_precision_and_resume[None-ppo-True-ouro] +0.01s call tests/test_looped_grpo_recipe.py::test_recipe_roles_groups_precision_and_resume[None-dppo-False-nanbeige] +0.01s call tests/plugins/test_streamed_looped_readout.py::test_loss_and_all_readout_gradients[1.0-1-dtype0] +0.01s call tests/test_megatron_argument_validation.py::test_hf_validate_all_moe_skips_dense_intermediate_size +0.01s call tests/test_looped_grpo_recipe.py::test_recipe_roles_groups_precision_and_resume[None-flow-dppo-False-ouro] +0.01s setup tests/plugins/test_rltt_online_cycle.py::test_fresh_reward_updates_and_reference_adam_resume[serial-nanbeige] +0.01s setup tests/rollout_backends/test_native_rlt_evaluation.py::test_unsupported_eval_settings_reject_before_generation[min_new_tokens-1] +0.01s teardown tests/plugins/test_rltt_training.py::test_packed_reference_and_logical_microbatch_gradient[serial-True-response_mean-ouro] +0.01s call tests/test_looped_grpo_recipe.py::test_recipe_roles_groups_precision_and_resume[None-rltt-False-ouro] +0.01s setup tests/test_looped_grpo_recipe.py::test_recipe_roles_groups_precision_and_resume[None-ppo-True-nanbeige] + +(1113 durations < 0.005s hidden. Use -vv to show these durations.) +==== 506 passed, 6 skipped, 3 deselected, 284 warnings in 60.43s (0:01:00) ===== diff --git a/benchmarks/results/native-evaluation-cpu/native-eval-release.xml b/benchmarks/results/native-evaluation-cpu/native-eval-release.xml new file mode 100644 index 000000000..9fa0eb77f --- /dev/null +++ b/benchmarks/results/native-evaluation-cpu/native-eval-release.xml @@ -0,0 +1 @@ +/Users/0z5a/Documents/infra/flashrlt-20261005/vime/tests/plugins/test_packed_recurrent_replay.py:91: requires an existing CUDA runtime/Users/0z5a/Documents/infra/flashrlt-20261005/vime/tests/plugins/test_packed_recurrent_replay.py:91: requires an existing CUDA runtime/Users/0z5a/Documents/infra/flashrlt-20261005/vime/tests/plugins/test_packed_recurrent_replay.py:91: requires an existing CUDA runtime/Users/0z5a/Documents/infra/flashrlt-20261005/vime/tests/plugins/test_packed_recurrent_replay.py:91: requires an existing CUDA runtime/Users/0z5a/Documents/infra/flashrlt-20261005/vime/tests/plugins/test_packed_recurrent_replay.py:91: requires an existing CUDA runtime/Users/0z5a/Documents/infra/flashrlt-20261005/vime/tests/plugins/test_packed_recurrent_replay.py:91: requires an existing CUDA runtime \ No newline at end of file diff --git a/benchmarks/results/native-evaluation-cpu/native-eval-schedule.log b/benchmarks/results/native-evaluation-cpu/native-eval-schedule.log new file mode 100644 index 000000000..c8f6a502c --- /dev/null +++ b/benchmarks/results/native-evaluation-cpu/native-eval-schedule.log @@ -0,0 +1,60 @@ +torch=2.13.0 /Users/0z5a/Documents/infra/looped-grpo-integration-20261003/.venv/lib/python3.12/site-packages/torch/__init__.py +setuptools=84.0.0 /Users/0z5a/Documents/infra/looped-grpo-integration-20261003/.venv/lib/python3.12/site-packages/setuptools/__init__.py +============================= test session starts ============================== +platform darwin -- Python 3.12.14, pytest-9.1.1, pluggy-1.6.0 +rootdir: /Users/0z5a/Documents/infra/flashrlt-20261005/vime +configfile: pyproject.toml +plugins: platformdirs-4.12.2, anyio-4.15.1 +collected 50 items + +tests/rollout_backends/test_native_rlt_evaluation.py ................. [ 34%] +tests/rollout_backends/test_native_rlt_contract.py ..... [ 44%] +tests/rollout_backends/test_native_rlt_groups.py ..... [ 54%] +tests/test_native_eval_schedule.py ..... [ 64%] +tests/test_rollout_data_utils.py ...... [ 76%] +tests/test_rollout_metrics.py ............ [100%] + +- generated xml file: /Users/0z5a/Documents/infra/flashrlt-20261005/evidence/native-eval-schedule.xml - +============================== slowest durations =============================== +0.62s call tests/test_rollout_metrics.py::test_decode_int32_meta_array_decodes_base64_to_tensor +0.45s call tests/rollout_backends/test_native_rlt_evaluation.py::test_complete_datasets_fixed_seeds_and_unchanged_training_state[ouro] +0.32s call tests/rollout_backends/test_native_rlt_evaluation.py::test_complete_datasets_fixed_seeds_and_unchanged_training_state[huginn] +0.17s call tests/rollout_backends/test_native_rlt_contract.py::test_http_client_concurrency_matches_backend[vllm] +0.08s call tests/rollout_backends/test_native_rlt_evaluation.py::test_reward_key_and_metadata_override +0.08s setup tests/rollout_backends/test_native_rlt_evaluation.py::test_complete_datasets_fixed_seeds_and_unchanged_training_state[nanbeige] +0.07s call tests/rollout_backends/test_native_rlt_evaluation.py::test_complete_datasets_fixed_seeds_and_unchanged_training_state[nanbeige] +0.04s setup tests/rollout_backends/test_native_rlt_evaluation.py::test_complete_datasets_fixed_seeds_and_unchanged_training_state[ouro] +0.03s call tests/rollout_backends/test_native_rlt_evaluation.py::test_policy_publication_between_batches_invalidates_evaluation +0.03s call tests/rollout_backends/test_native_rlt_contract.py::test_huginn_aliases_static_buffer_and_noise_profile +0.03s setup tests/rollout_backends/test_native_rlt_evaluation.py::test_reward_key_and_metadata_override +0.03s call tests/rollout_backends/test_native_rlt_evaluation.py::test_sampling_and_chat_overrides_are_used +0.02s call tests/rollout_backends/test_native_rlt_contract.py::test_publication_cohort_and_serialization +0.02s call tests/rollout_backends/test_native_rlt_contract.py::test_incomplete_publication_cannot_resume_or_reuse_old_manifest +0.02s call tests/rollout_backends/test_native_rlt_evaluation.py::test_reordered_or_missing_heldout_samples_are_rejected +0.01s call tests/rollout_backends/test_native_rlt_contract.py::test_http_client_concurrency_matches_backend[vllm-rlt] +0.01s setup tests/rollout_backends/test_native_rlt_contract.py::test_huginn_aliases_static_buffer_and_noise_profile +0.01s setup tests/rollout_backends/test_native_rlt_contract.py::test_incomplete_publication_cannot_resume_or_reuse_old_manifest +0.01s call tests/test_native_eval_schedule.py::test_completed_update_indices_preserve_initial_and_resumed_eval[vllm-rlt-0-3-expected0] +0.01s setup tests/rollout_backends/test_native_rlt_evaluation.py::test_sampling_and_chat_overrides_are_used +0.01s setup tests/rollout_backends/test_native_rlt_evaluation.py::test_unsupported_eval_settings_reject_before_generation[app_service-http://example.invalid] +0.01s call tests/rollout_backends/test_native_rlt_groups.py::test_native_groups_and_actual_math_rewards +0.01s setup tests/rollout_backends/test_native_rlt_contract.py::test_publication_cohort_and_serialization +0.01s setup tests/rollout_backends/test_native_rlt_evaluation.py::test_unsupported_eval_settings_reject_before_generation[no_stop_trim-False] +0.01s setup tests/rollout_backends/test_native_rlt_evaluation.py::test_reordered_or_missing_heldout_samples_are_rejected +0.01s setup tests/rollout_backends/test_native_rlt_evaluation.py::test_unsupported_eval_settings_reject_before_generation[eval_task_timeout-1] +0.01s setup tests/rollout_backends/test_native_rlt_evaluation.py::test_policy_publication_between_batches_invalidates_evaluation +0.01s setup tests/rollout_backends/test_native_rlt_evaluation.py::test_complete_datasets_fixed_seeds_and_unchanged_training_state[huginn] +0.01s setup tests/rollout_backends/test_native_rlt_evaluation.py::test_unsupported_eval_settings_reject_before_generation[stop-value0] +0.01s setup tests/rollout_backends/test_native_rlt_evaluation.py::test_unsupported_eval_settings_reject_before_generation[min_new_tokens-1] +0.01s setup tests/rollout_backends/test_native_rlt_evaluation.py::test_unsupported_eval_settings_reject_before_generation[custom_generate_function_path-custom.generate] +0.01s setup tests/rollout_backends/test_native_rlt_evaluation.py::test_unsupported_eval_settings_reject_before_generation[repetition_penalty-1.2] +0.01s setup tests/rollout_backends/test_native_rlt_evaluation.py::test_unsupported_eval_settings_reject_before_generation[n_samples_per_eval_prompt-0] +0.01s setup tests/rollout_backends/test_native_rlt_evaluation.py::test_long_heldout_prompts_are_not_silently_dropped[prompt] +0.01s call tests/rollout_backends/test_native_rlt_evaluation.py::test_unsupported_eval_settings_reject_before_generation[app_service-http://example.invalid] +0.01s setup tests/rollout_backends/test_native_rlt_evaluation.py::test_long_heldout_prompts_are_not_silently_dropped[context] +0.01s call tests/rollout_backends/test_native_rlt_evaluation.py::test_unsupported_eval_settings_reject_before_generation[repetition_penalty-1.2] +0.01s call tests/rollout_backends/test_native_rlt_evaluation.py::test_unsupported_eval_settings_reject_before_generation[stop-value0] +0.01s call tests/rollout_backends/test_native_rlt_evaluation.py::test_unsupported_eval_settings_reject_before_generation[custom_generate_function_path-custom.generate] + +(111 durations < 0.005s hidden. Use -vv to show these durations.) +============================= 50 passed in 15.50s ============================== diff --git a/benchmarks/results/native-evaluation-cpu/native-eval-schedule.xml b/benchmarks/results/native-evaluation-cpu/native-eval-schedule.xml new file mode 100644 index 000000000..54e622144 --- /dev/null +++ b/benchmarks/results/native-evaluation-cpu/native-eval-schedule.xml @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/benchmarks/results/native-evaluation-cpu/validation.json b/benchmarks/results/native-evaluation-cpu/validation.json new file mode 100644 index 000000000..ba93d19ef --- /dev/null +++ b/benchmarks/results/native-evaluation-cpu/validation.json @@ -0,0 +1,38 @@ +{ + "parent": "61903fbef8a06eff4a0f7ce9daf42f5da792aa0b", + "python": "3.12.14", + "torch": "2.13.0", + "scope": "CPU components; tiny actual native engines/MCore loss/Adam; transport/tokenizer doubles; no official GPU or Ray lifecycle", + "suites": [ + { + "name": "native-eval-release", + "passed": 506, + "skipped": 6 + }, + { + "name": "native-eval-arguments", + "passed": 2, + "skipped": 0 + } + ], + "unique_passed": 508, + "known_factory_deselected": 3, + "sources": { + "examples/looped_ppo/README.md": "ecd51d2d635279350a0e154d4bf09736f851eb665a92e5099b9041eb7db0bc68", + "train.py": "5d6912878b7490aee786d16e0359fc4f90573c8d0d2f7e28a228f730901a86d0", + "vime/backends/vllm_rlt_utils/README.md": "7a5ae34bfa1c64cc77fe2a37e9a6b3db3cd5038f18b03ca2a6897ca7f7b8274b", + "vime/backends/vllm_rlt_utils/arguments.py": "1418acaa0a4b99b78409b5c0043b98759f3124b4b3064be4fb020d7dca400410", + "vime/backends/vllm_rlt_utils/engine.py": "bb5f804e8d202efba0860c7b24f68041178594e2e9a2d10b3c15da598afffa49", + "vime/observability/rollout_metrics.py": "d85918ee581bcce4dd4a69b4482b2459950619b55b739880cd03a54298c18b79", + "vime/rollout/vllm_rlt_rollout.py": "b60f2a8583a12a01310d0b2c9b8690bde4a9923379ac3ef0ab66ffeed8712855", + "vime/rollout/native_eval.py": "06a7aab463bd888f4d7946544f8f390d1d462504fe88120305e0fa5f4ca4a363", + "tests/rollout_backends/test_native_rlt_evaluation.py": "e0b150f8fd49b9bdabca3322b21f0f683efe01959010f5fb31c1d6de4810bef0", + "tests/rollout_backends/test_native_eval_arguments.py": "3c954479254632a8def3385b8f9d2e3c14e6bf09df0bee5562e1c9aafe33cefa", + "tests/plugins/test_native_heldout_training.py": "9019b5914fb8cbb57c20df072f023ce4fb448b9714d34d07e0a7708418da2c43", + "tests/test_native_eval_schedule.py": "60e9432c1d36b0512125b4f9405ddeb8c7337fb548f81748776ba2b62de5104b", + "docs/native_evaluation_results.md": "475da4c4117ebd8ab6f0808bd7c339b61e4184b4cd9fcfd29a1166d1a8d38482", + "docs/rltt_training_validation.md": "b27d9036758dd84641727814f7b018a38cf2cfacb34c75877a016a90f86a5ea7" + }, + "suite_identity_audit": "508 distinct (classname, name) identities across the two final suites; zero failures/errors", + "recovery_scope": "Fresh objects in the same pytest process; not fresh worker processes or distributed checkpoint recovery" +} diff --git a/docs/native_evaluation_results.md b/docs/native_evaluation_results.md new file mode 100644 index 000000000..2b8715124 --- /dev/null +++ b/docs/native_evaluation_results.md @@ -0,0 +1,118 @@ +# Native held-out evaluation + +The native `vllm-rlt` backend can now evaluate complete held-out datasets between +policy updates. The parent (`61903fbef8a06eff4a0f7ce9daf42f5da792aa0b`, draft #12) +rejects `--eval-interval`. This change removes that entry barrier; it does not +establish reward convergence or a performance improvement. + +## Evaluation contract + +The evaluator uses VIME's JSONL/Parquet dataset loader, chat-template settings, +math reward and custom per-sample reward hooks. Every prompt and requested +completion is retained in its original order. Empty datasets, invalid lengths +and prompts outside the declared context budget fail the evaluation instead of +silently reducing the held-out population. Generation uses bounded batches of +`--rlt-max-num-seqs`; all batches must report the same runtime epoch, publication +version and physical-weight digest. + +Each completion has a stable seed derived from the experiment seed, dataset +name, prompt index and completion index. It does not depend on training rollout +IDs, request IDs or evaluation time. Huginn records and replays the actual latent +seed. Evaluation accepts separate temperature, top-p, top-k, response budget and +token stops; the full-vocabulary training restriction is unchanged. Generated +stop tokens remain in the trace. String stops, repetition penalties, minimum +length, group rewards, multimodal/tool/server execution and partial-dataset early +stopping are rejected by this initial text-only path. + +Native evaluation uses completed-update indices: `eval_0.pt` is the initial +policy, and `eval_1.pt` follows the first update. Evaluation-only resume uses the +restored cursor. The publication version remains a separate field: an initial +weight publication may already be version one. Per-dataset completion counts +are passed to pass-rate logging rather than replaced by the global default. + +The native sampler uses request-local seeds. Tests with the built-in math reward +also preserve Python and Torch global RNG state. An arbitrary custom reward hook +must manage its own external state and randomness; the evaluator cannot make +that guarantee on its behalf. + +Usage and the supported argument profile are documented in the +[backend README](../vime/backends/vllm_rlt_utils/README.md#held-out-evaluation). +The short example remains a lifecycle probe, not a frozen convergence recipe. + +## Validation + +The two final, disjoint suites pass **508 CPU checks**, with six existing CUDA +checks skipped and three known `megatron.training` factory checks deselected +because Triton is unavailable in this existing CPU environment. The 506-case +release and two argument checks have distinct test identities. Earlier subset +runs overlap and are not added to that total. + +| Gate | Result | Actual coverage | +| --- | --- | --- | +| Complete native evaluation | 17 passed | Real tiny Ouro, Nanbeige and Huginn engines; multiple datasets, stable seeds, overrides, rewards, order and policy-cohort checks | +| Update index and metric grouping | 5 passed | Actual `train.train` control flow and Torch debug files; actor/transport doubles; initial evaluation is not overwritten | +| Evaluation interleaved with learning | 3 passed | Real tiny native sampling, MCore RLTT loss, AdamW, weight publications and restored objects | +| Resolved evaluation arguments | 2 passed | Actual dataset config builder and native validator; separate evaluation top-p/top-k accepted, training restriction retained | +| Complete final CPU regression | 508 passed | The above cases and inherited tests; six CUDA skips and three factory deselections are not passes | + +Each learning case performs three nonzero AdamW updates using fresh native +completions and output token-parity training rewards. A paired run inserts math +held-out evaluation after every update. Training tokens, rewards, request seeds, +losses, gradients, parameter deltas, physical publication digests, final tensors +and Adam state match the no-evaluation run exactly. + +After update two, a real Torch checkpoint is restored into newly constructed +actor, frozen reference, optimizer and native engine objects. Held-out outputs +match the saved policy; update three and the final held-out evaluation match the +uninterrupted run exactly. These are fresh objects in the same test process, +not independent worker-process or distributed-checkpoint recovery. + +| Tiny family | Nonzero updates | Training with/without evaluation | Restored next update and held-out samples | +| --- | ---: | --- | --- | +| Ouro | 3 | Exact | Exact | +| Nanbeige | 3 | Exact | Exact | +| Huginn | 3 | Exact | Exact, including latent seeds | + +The engine and loss are real; Ray calls and tokenization are test doubles. +Fixtures use random tiny weights, numeric-token prompts and a test decoder. +Their held-out reward values are validation data, not reasoning quality. They +cannot support a learning or convergence claim. Full raw training/evaluation +records are in [cycles.json](../benchmarks/results/native-evaluation-cpu/cycles.json). + +## Performance comparison + +| Scope | Unchanged parent #12 | This change | Parent/current speed ratio | +| --- | --- | --- | --- | +| Native periodic held-out evaluation | Rejected at argument validation | CPU component path passes | Undefined: parent has no executable evaluation path | +| Official-model full RL | NOT_RUN | NOT_RUN | — | +| Peak CUDA memory under matched load | NOT_RUN | NOT_RUN | — | +| Independent-seed reward time-to-quality | NOT_RUN | NOT_RUN | — | + +No timing comparison is inferred from pytest duration. Actual evaluation adds +work to a training run and must be included in the formal wall-time and GPU-hour +budget. A matched numerical oracle, complete Ray/MCore/CUDA lifecycle, official +tokenizer/checkpoint evaluation and at least three independent training seeds +remain required before the main quality comparison. + +## Reproduction and provenance + +Runtime: existing Python 3.12.14, Torch 2.13.0 and pytest 9.1.1, one OpenMP thread. +Native RLT is `7f367ee02abb5b8c700a705062fbe98842338d21` (runtime unchanged from +`7d6e08e`). MCore is `1dcf0dafa884ad52ffb243625717a3471643e087` with the standard +VIME patch SHA256 `6fa39fdfdac8dae6b9bb3e44014df6766d774e0968ac07d6025a6b40f211aa98`. +No dependency was installed, updated or replaced. + +Focused commands in an existing compatible environment: + +```sh +PYTHONPATH=.:../rlt OMP_NUM_THREADS=1 pytest -q \ + tests/rollout_backends/test_native_rlt_evaluation.py \ + tests/rollout_backends/test_native_eval_arguments.py \ + tests/test_native_eval_schedule.py \ + tests/plugins/test_native_heldout_training.py +``` + +The retained [validation manifest](../benchmarks/results/native-evaluation-cpu/validation.json) +records tested source hashes and disjoint-suite totals; `SHA256SUMS.json` covers +the raw XML/log files and records. Local validation used the existing resident +Python harness to avoid evicted filesystem modules without changing packages. diff --git a/docs/rltt_training_validation.md b/docs/rltt_training_validation.md index 79c8a990c..8ebc2e00f 100644 --- a/docs/rltt_training_validation.md +++ b/docs/rltt_training_validation.md @@ -29,9 +29,11 @@ official-model training result. The existing recipe defaults remain a short lifecycle probe: FP32 for RLTT, SGD, three updates, short responses, and a single seed. These are not the original -RLTT optimizer or a convergence recipe. Native held-out evaluation is still -unavailable through VIME's dataset evaluator. Do not use this probe as the main -quality experiment. +RLTT optimizer or a convergence recipe. The subsequent +[native held-out evaluator](native_evaluation_results.md) has CPU component +coverage for all three families, including evaluation between updates and +restored objects. Official Ray/CUDA quality runs remain pending. Do not use this +probe as the main quality experiment. ## Parent training-bridge validation result @@ -103,6 +105,6 @@ patched-source manifest, with no additional private patches. This is distinct from the separately frozen upstream Megatron framework-baseline archive. Before performance or quality conclusions: qualify official-checkpoint G0 and -BF16 arithmetic, run the actual Ray/MCore CUDA lifecycle and typed resume, expose -held-out evaluation, freeze the optimizer/budget/reward and source-matched baseline +BF16 arithmetic, run the actual Ray/MCore CUDA lifecycle and typed resume, qualify +official held-out evaluation, freeze the optimizer/budget/reward and source-matched baseline profiles, then run matched systems measurements and independent training seeds. diff --git a/examples/looped_ppo/README.md b/examples/looped_ppo/README.md index 3e3723b72..9f804ebe5 100644 --- a/examples/looped_ppo/README.md +++ b/examples/looped_ppo/README.md @@ -6,12 +6,14 @@ The shared launcher runs VIME's standard Ray entry with native vLLM-RLT rollout. Ouro also uses this entry with `--model /models/Ouro-1.4B` and checkpoint revision `574fa66cb8bf5abdc979642d01cf2b79b16bfab1`. -Use `--algorithm ppo`, `grpo`, `dppo`, or `flow-dppo`. PPO and GRPO reuse the existing +Use `--algorithm ppo`, `grpo`, `dppo`, `flow-dppo`, or `rltt`. PPO and GRPO reuse the existing clipped policy loss. DPPO keeps rollout behavior scores separate from proximal scores recomputed by the training provider, and applies VIME's existing bounded importance correction. Flow-DPPO freezes full-vocabulary old-policy scores and applies the categorical divergence gate. Actor/critic roles use GAE for PPO, DPPO and Flow-DPPO; GRPO uses an actor and prompt-group reward normalization. +RLTT uses the explicit recurrent weighted-logprob objective, logical-batch +reduction and frozen initial reference described in [its contract](../../docs/looped_readout_contract.md). Install the [pinned RLT engine](../../vime/backends/vllm_rlt_utils/README.md) and Megatron `1dcf0dafa884ad52ffb243625717a3471643e087` with @@ -48,3 +50,10 @@ The launcher enables deterministic training and passes the cuBLAS workspace and NCCL algorithm settings to Ray workers for this comparison. Report end-to-end timing including rollout, rewards, optimization, full publications and checkpoints, with source pins, dtype and resource contention. + +Periodic held-out evaluation uses the same committed native policy through +`--eval-interval` and `--eval-prompt-data` or `--eval-config`. It does not consume +the training dataset cursor. Initial and post-update results retain distinct +completed-update indices, and the output samples retain their policy and seed +traces. See the [native evaluation contract](../../vime/backends/vllm_rlt_utils/README.md#held-out-evaluation). +Existing short update/resume checks do not establish reward convergence. diff --git a/tests/plugins/test_native_heldout_training.py b/tests/plugins/test_native_heldout_training.py new file mode 100644 index 000000000..79369499e --- /dev/null +++ b/tests/plugins/test_native_heldout_training.py @@ -0,0 +1,138 @@ +"""Held-out calls preserve real tiny RLTT updates and fresh Adam recovery. + +This is CPU component integration, with token-parity training rewards and test +tokenization. It does not qualify Ray/CUDA training or reasoning convergence. +""" + +import copy +import json +from argparse import Namespace +from types import SimpleNamespace + +import pytest +import torch +from test_looped_grpo import cpu_engine, publish +from test_rltt_online_cycle import pair, update + +from vime.rollout import native_eval +from vime.utils.eval_config import EvalDatasetConfig + + +def evaluate(engine, data, monkeypatch): + tokenizer = SimpleNamespace( + encode=lambda prompt, **kwargs: [int(value) for value in prompt.split()], + decode=lambda tokens, **kwargs: r"\boxed{" + str(tokens[-1]) + "}", + ) + monkeypatch.setattr(native_eval, "_tokenizer", lambda checkpoint: tokenizer) + monkeypatch.setattr(native_eval.ray, "get", lambda result: result) + config = EvalDatasetConfig( + "heldout", + str(data), + input_key="prompt", + label_key="label", + metadata_key="metadata", + n_samples_per_eval_prompt=2, + temperature=0.8, + top_p=1.0, + top_k=-1, + max_response_len=2, + ) + args = Namespace( + **( + vars(engine.args) + | { + "hf_checkpoint": "tiny-fixture", + "eval_datasets": [config], + "group_rm": False, + "multimodal_keys": None, + "apply_chat_template": False, + "apply_chat_template_kwargs": {}, + "eval_max_prompt_len": 8, + "eval_max_context_len": 16, + "eval_reward_key": None, + "reward_key": None, + "custom_rm_path": None, + "rm_type": "math", + "rlt_max_num_seqs": 2, + "rlt_engine": SimpleNamespace(evaluate=SimpleNamespace(remote=engine.evaluate)), + } + ) + ) + output = native_eval.evaluate(args) + return [ + { + "tokens": sample.tokens, + "reward": sample.reward, + "seed": sample.recurrent_trace.seed, + "version": sample.recurrent_trace.policy_version, + "digest": sample.recurrent_trace.publication_digest, + } + for sample in output.data["heldout"]["samples"] + ] + + +def assert_optimizer_equal(left, right): + assert left["param_groups"] == right["param_groups"] + for index, state in left["state"].items(): + for key, value in state.items(): + torch.testing.assert_close(value, right["state"][index][key], rtol=0, atol=0) + + +@pytest.mark.parametrize("family", ["ouro", "nanbeige", "huginn_raven"]) +def test_evaluation_preserves_updates_and_fresh_resume(family, tmp_path, monkeypatch, record_property): + data = tmp_path / "heldout.jsonl" + data.write_text('{"prompt":"4 5","label":"2"}\n{"prompt":"7 8","label":"3"}\n') + evaluations, reference_runs, final_states = [], [], [] + for enabled in (False, True): + directory = tmp_path / str(enabled) + directory.mkdir() + native, actor = pair(family) + reference = copy.deepcopy(actor).requires_grad_(False) + optimizer = torch.optim.AdamW(actor.parameters(), lr=1e-3, betas=(0.9, 0.99), weight_decay=0) + engine = cpu_engine(native, family) + records = [] + for step in range(3): + records.append(update(actor, reference, optimizer, engine, family, directory, step, "sdpa-reference")) + if enabled: + evaluations.append(evaluate(engine, data, monkeypatch)) + if enabled and step == 1: + torch.save( + { + "actor": actor.state_dict(), + "reference": reference.state_dict(), + "optimizer": optimizer.state_dict(), + "rng": torch.get_rng_state(), + }, + tmp_path / "resume.pt", + ) + reference_runs.append(records) + final_states.append((copy.deepcopy(actor.state_dict()), copy.deepcopy(optimizer.state_dict()))) + engine.close() + assert reference_runs[0] == reference_runs[1] + for name, expected in final_states[0][0].items(): + assert torch.equal(expected, final_states[1][0][name]), name + assert_optimizer_equal(final_states[0][1], final_states[1][1]) + + checkpoint = torch.load(tmp_path / "resume.pt", weights_only=True) + native, resumed = pair(family) + resumed.load_state_dict(checkpoint["actor"]) + reference = copy.deepcopy(resumed).requires_grad_(False) + reference.load_state_dict(checkpoint["reference"]) + optimizer = torch.optim.AdamW(resumed.parameters(), lr=1e-3, betas=(0.9, 0.99), weight_decay=0) + optimizer.load_state_dict(checkpoint["optimizer"]) + engine = cpu_engine(native, family) + publish(resumed, engine, family, tmp_path / "restored-policy", 3) + torch.set_rng_state(checkpoint["rng"]) + assert evaluate(engine, data, monkeypatch) == evaluations[1] + directory = tmp_path / "fresh" + directory.mkdir() + actual = update(resumed, reference, optimizer, engine, family, directory, 2, "sdpa-reference") + assert actual == reference_runs[1][2] + assert evaluate(engine, data, monkeypatch) == evaluations[2] + for name, expected in final_states[1][0].items(): + assert torch.equal(expected, resumed.state_dict()[name]), name + assert_optimizer_equal(final_states[1][1], optimizer.state_dict()) + engine.close() + record_property("scope", "tiny CPU native/RLTT/Adam recovery; transport and tokenizer doubles; no quality claim") + record_property("training", json.dumps(reference_runs[1])) + record_property("heldout", json.dumps(evaluations)) diff --git a/tests/rollout_backends/test_native_eval_arguments.py b/tests/rollout_backends/test_native_eval_arguments.py new file mode 100644 index 000000000..827cf51ea --- /dev/null +++ b/tests/rollout_backends/test_native_eval_arguments.py @@ -0,0 +1,55 @@ +"""The actual backend validator admits evaluation without relaxing training sampling.""" + +import json +from argparse import Namespace + +import pytest + +from vime.backends.vllm_rlt_utils.arguments import validate_args +from vime.utils.eval_config import build_eval_dataset_configs + + +@pytest.mark.parametrize("training_top_p", [1.0, 0.9]) +def test_eval_sampling_is_separate_from_training_validation(tmp_path, training_top_p): + (tmp_path / "config.json").write_text(json.dumps({"model_type": "ouro", "total_ut_steps": 4})) + args = Namespace( + hf_checkpoint=str(tmp_path), + rlt_depth=None, + rlt_kv_blocks=128, + rlt_max_num_seqs=2, + rlt_start_version=0, + rollout_num_gpus=1, + rollout_num_gpus_per_engine=1, + colocate=False, + offload_rollout=False, + release_train=False, + use_fault_tolerance=False, + rollout_external=False, + use_opd=False, + use_rollout_routing_replay=False, + check_weight_update_equal=False, + update_weight_mode="full", + update_weight_transport="disk", + update_weight_local_checkpoint_dir=None, + rollout_top_p=training_top_p, + rollout_top_k=-1, + rollout_function_path="vime.rollout.vllm_rlt_rollout.generate_rollout", + eval_interval=1, + group_rm=False, + multimodal_keys=None, + eval_temperature=0.0, + eval_top_p=0.7, + eval_top_k=3, + eval_max_response_len=16, + n_samples_per_eval_prompt=2, + input_key="prompt", + label_key="label", + ) + args.eval_datasets = build_eval_dataset_configs(args, [{"name": "heldout", "path": "heldout.jsonl"}], {}) + if training_top_p == 1.0: + validate_args(args) + assert args.rlt_depth == 4 + assert args.eval_datasets[0].temperature == 0 and args.eval_datasets[0].top_p == 0.7 + else: + with pytest.raises(ValueError, match="training requires full-vocabulary"): + validate_args(args) diff --git a/tests/rollout_backends/test_native_rlt_evaluation.py b/tests/rollout_backends/test_native_rlt_evaluation.py new file mode 100644 index 000000000..517a00991 --- /dev/null +++ b/tests/rollout_backends/test_native_rlt_evaluation.py @@ -0,0 +1,260 @@ +"""Real tiny native generation/math rewards; Ray transport and tokenizer are test seams.""" + +import json +import random +from argparse import Namespace +from dataclasses import replace +from types import SimpleNamespace + +import pytest +import torch +from test_native_rlt_contract import engine as engine +from test_native_rlt_contract import publish + +from vime.rollout import native_eval, vllm_rlt_rollout +from vime.utils.eval_config import EvalDatasetConfig +from vime.utils.types import Sample + + +class Tokenizer: + def encode(self, prompt, **kwargs): + return [int(token) for token in prompt.split()] + + def decode(self, tokens, **kwargs): + return r"\boxed{" + str(tokens[-1]) + "}" + + def apply_chat_template(self, messages, **kwargs): + return messages[0]["content"] + " 3" + + +def setup_eval(engine, tmp_path, monkeypatch): + path = tmp_path / "heldout.jsonl" + path.write_text( + "\n".join( + json.dumps({"prompt": prompt, "label": "2", "metadata": {"source": "heldout"}}) + for prompt in ("1 2", "3 4") + ) + + "\n" + ) + config = EvalDatasetConfig( + "heldout", + str(path), + input_key="prompt", + label_key="label", + metadata_key="metadata", + n_samples_per_eval_prompt=2, + temperature=0.8, + top_p=1.0, + top_k=-1, + max_response_len=2, + ) + args = Namespace( + **( + vars(engine.args) + | { + "hf_checkpoint": "tiny-fixture", + "eval_datasets": [config], + "group_rm": False, + "multimodal_keys": None, + "apply_chat_template": False, + "apply_chat_template_kwargs": {}, + "eval_max_prompt_len": 8, + "eval_max_context_len": 16, + "eval_reward_key": None, + "reward_key": None, + "custom_rm_path": None, + "rm_type": "math", + "rlt_max_num_seqs": 2, + "rlt_engine": SimpleNamespace(evaluate=SimpleNamespace(remote=engine.evaluate)), + } + ) + ) + engine.args = args + monkeypatch.setattr(native_eval, "_tokenizer", lambda checkpoint: Tokenizer()) + monkeypatch.setattr(native_eval.ray, "get", lambda result: result) + publish(engine, tmp_path / "weights", 1) + engine.continue_generation() + return args, config + + +def observed(output): + return { + name: [ + ( + s.tokens, + s.reward, + s.rollout_log_probs, + s.status, + s.recurrent_trace.seed, + s.recurrent_trace.policy_version, + ) + for s in data["samples"] + ] + for name, data in output.data.items() + } + + +@pytest.mark.parametrize("family", ["ouro", "nanbeige", "huginn"]) +def test_complete_datasets_fixed_seeds_and_unchanged_training_state(engine, tmp_path, monkeypatch, family): + if family != "ouro": + from vllm_rlt import LLM, CacheConfig + from vllm_rlt.models.huginn import HuginnConfig, HuginnForCausalLM + from vllm_rlt.models.nanbeige import NanbeigeConfig, NanbeigeForCausalLM + + engine.llm.close() + if family == "nanbeige": + native = NanbeigeForCausalLM( + NanbeigeConfig( + vocab_size=11, + hidden_size=8, + intermediate_size=16, + num_hidden_layers=1, + num_attention_heads=2, + num_key_value_heads=1, + head_dim=4, + num_loops=2, + max_position_embeddings=32, + bos_token_id=None, + eos_token_id=None, + pad_token_id=None, + ) + ) + else: + native = HuginnForCausalLM( + HuginnConfig( + vocab_size=11, + padded_vocab_size=11, + n_embd=8, + n_heads=2, + n_layers=3, + n_layers_in_prelude=1, + n_layers_in_recurrent_block=1, + n_layers_in_coda=1, + intermediate_size=16, + mean_recurrence=2, + block_size=32, + bos_token_id=None, + eos_token_id=None, + pad_token_id=None, + ) + ) + engine.llm = LLM(native, cache_config=CacheConfig(num_blocks=16)) + engine.args.rlt_model_family = "huginn_raven" if family == "huginn" else family + engine.args.rlt_depth = 2 + args, config = setup_eval(engine, tmp_path, monkeypatch) + args.eval_datasets.append(replace(config, name="second", n_samples_per_eval_prompt=1)) + before = engine.generate([Sample(tokens=[1, 2], index=7, group_index=2)], 9)[0] + python_rng, torch_rng = random.getstate(), torch.get_rng_state().clone() + train_data = SimpleNamespace(get_samples=lambda count: pytest.fail("Evaluation consumed training data")) + first = vllm_rlt_rollout.generate_rollout(args, 0, train_data, evaluation=True) + second = vllm_rlt_rollout.generate_rollout(args, 15, train_data, evaluation=True) + after = engine.generate([Sample(tokens=[1, 2], index=7, group_index=2)], 9)[0] + assert observed(first) == observed(second) + assert (before.tokens, before.rollout_log_probs, before.recurrent_trace.seed) == ( + after.tokens, + after.rollout_log_probs, + after.recurrent_trace.seed, + ) + assert random.getstate() == python_rng + assert torch.equal(torch.get_rng_state(), torch_rng) + assert first.metrics["eval/native_samples"] == 6 + assert first.metrics["eval/native_policy_version"] == 1 + assert first.data["heldout"]["n_samples_per_prompt"] == 2 + assert first.data["second"]["n_samples_per_prompt"] == 1 + seeds = [s.recurrent_trace.seed for data in first.data.values() for s in data["samples"]] + assert len(set(seeds)) == 6 + for data in first.data.values(): + for sample, reward in zip(data["samples"], data["rewards"], strict=True): + assert reward == float(sample.tokens[-1] == 2) + assert sample.metadata["source"] == "heldout" + assert sample.recurrent_trace.publication_digest == engine.committed_digest + if family == "huginn": + assert sample.recurrent_trace.latent_seed == sample.recurrent_trace.seed + assert sample.recurrent_trace.latent_profile == "like-init-cpu-f32-v1" + + +def test_sampling_and_chat_overrides_are_used(engine, tmp_path, monkeypatch): + args, config = setup_eval(engine, tmp_path, monkeypatch) + args.eval_datasets = [replace(config, top_k=1, temperature=0, max_response_len=1, apply_chat_template=True)] + output = native_eval.evaluate(args) + for sample in output.data["heldout"]["samples"]: + assert sample.response_length == 1 and sample.tokens[2] == 3 + assert sample.recurrent_trace.temperature == 0 + assert sample.metadata["native_eval"]["top_k"] == 1 + assert engine.args.rollout_temperature == 0.8 and engine.args.rollout_max_response_len == 2 + + +@pytest.mark.parametrize( + "field,value", + [ + ("stop", ["END"]), + ("min_new_tokens", 1), + ("repetition_penalty", 1.2), + ("no_stop_trim", False), + ("custom_generate_function_path", "custom.generate"), + ("app_service", "http://example.invalid"), + ("eval_task_timeout", 1), + ("n_samples_per_eval_prompt", 0), + ], +) +def test_unsupported_eval_settings_reject_before_generation(engine, tmp_path, monkeypatch, field, value): + args, config = setup_eval(engine, tmp_path, monkeypatch) + args.eval_datasets = [replace(config, **{field: value})] + args.rlt_engine.evaluate.remote = lambda *args: pytest.fail("Invalid evaluation generated tokens") + with pytest.raises(ValueError): + native_eval.evaluate(args) + + +@pytest.mark.parametrize("limit", ["prompt", "context"]) +def test_long_heldout_prompts_are_not_silently_dropped(engine, tmp_path, monkeypatch, limit): + args, _ = setup_eval(engine, tmp_path, monkeypatch) + if limit == "prompt": + args.eval_max_prompt_len = 1 + else: + args.eval_max_context_len = 3 + args.rlt_engine.evaluate.remote = lambda *args: pytest.fail("Invalid evaluation generated tokens") + with pytest.raises(ValueError, match="limit"): + native_eval.evaluate(args) + + +def test_policy_publication_between_batches_invalidates_evaluation(engine, tmp_path, monkeypatch): + args, _ = setup_eval(engine, tmp_path, monkeypatch) + calls = 0 + + def evaluate(samples, sampling, seeds): + nonlocal calls + if calls: + engine.pause_generation() + publish(engine, tmp_path / "next-policy", 2) + engine.continue_generation() + calls += 1 + return engine.evaluate(samples, sampling, seeds) + + args.rlt_engine.evaluate.remote = evaluate + with pytest.raises(RuntimeError, match="policy cohorts"): + native_eval.evaluate(args) + + +def test_reordered_or_missing_heldout_samples_are_rejected(engine, tmp_path, monkeypatch): + args, _ = setup_eval(engine, tmp_path, monkeypatch) + + def evaluate(samples, sampling, seeds): + return list(reversed(engine.evaluate(samples, sampling, seeds))) + + args.rlt_engine.evaluate.remote = evaluate + with pytest.raises(ValueError, match="every held-out sample"): + native_eval.evaluate(args) + + +def test_reward_key_and_metadata_override(engine, tmp_path, monkeypatch): + args, config = setup_eval(engine, tmp_path, monkeypatch) + args.eval_reward_key = "accuracy" + args.eval_datasets = [replace(config, metadata_overrides={"rm_type": "custom"})] + + async def reward(args, sample): + assert sample.metadata["rm_type"] == "custom" + return {"accuracy": int(sample.tokens[-1] == 2), "other": -1} + + monkeypatch.setattr(vllm_rlt_rollout, "async_rm", reward) + output = native_eval.evaluate(args) + assert all(value in (0.0, 1.0) for value in output.data["heldout"]["rewards"]) diff --git a/tests/test_native_eval_schedule.py b/tests/test_native_eval_schedule.py new file mode 100644 index 000000000..d02250478 --- /dev/null +++ b/tests/test_native_eval_schedule.py @@ -0,0 +1,114 @@ +"""Execute the real train loop with transport doubles and actual evaluation dumps.""" + +from argparse import Namespace +from types import SimpleNamespace + +import pytest +import torch + +import train +from vime.observability.rollout_data_utils import save_debug_rollout_data +from vime.observability import rollout_metrics +from vime.utils.types import Sample + + +@pytest.mark.parametrize( + "backend,start,updates,expected", + [ + ("vllm-rlt", 0, 3, [0, 1, 2, 3]), + ("vllm-rlt", 2, 3, [3]), + ("vllm-rlt", 2, 0, [2]), + ("vllm", 0, 2, [0, 0, 1]), + ], +) +def test_completed_update_indices_preserve_initial_and_resumed_eval( + monkeypatch, tmp_path, backend, start, updates, expected +): + evaluations = [] + policy = start - 1 + + def publish(): + nonlocal policy + policy += 1 + + def evaluate(rollout_id): + evaluations.append(rollout_id) + sample = Sample(tokens=[policy + 1], response_length=1, reward=float(policy)) + save_debug_rollout_data( + str(tmp_path / "{rollout_id}.pt"), + {"heldout": {"samples": [sample]}}, + rollout_id=rollout_id, + evaluation=True, + ) + + def remote(function): + return SimpleNamespace(remote=function) + + manager = SimpleNamespace( + eval=remote(evaluate), + generate=remote(lambda step: step), + save=remote(lambda step: None), + dispose=remote(lambda: None), + shutdown=remote(lambda: None), + ) + actor = SimpleNamespace( + update_weights=publish, + async_train=lambda *args: None, + save_model=lambda *args, **kwargs: None, + clear_memory=lambda: None, + close=lambda: None, + ) + args = Namespace( + release_train=False, + offload_rollout=False, + offload_train=False, + check_weight_update_equal=False, + num_rollout=updates, + eval_interval=1, + start_rollout_id=start, + stop_after_rollout=None, + skip_eval_before_train=False, + use_critic=False, + save_interval=None, + rollout_global_dataset=True, + rollout_backend=backend, + ) + monkeypatch.setattr(train, "configure_logger", lambda: None) + monkeypatch.setattr(train, "init_tracking", lambda args: None) + monkeypatch.setattr(train, "finish_tracking", lambda args: None) + monkeypatch.setattr(train, "create_placement_groups", lambda args: {"rollout": None}) + monkeypatch.setattr(train, "create_rollout_manager", lambda *args: (manager, 1)) + monkeypatch.setattr(train, "create_training_models", lambda *args: (actor, None)) + monkeypatch.setattr(train.ray, "get", lambda value: value) + train.train(args) + assert evaluations == expected + if backend == "vllm-rlt": + assert len(list(tmp_path.glob("eval_*.pt"))) == len(expected) + for step in expected: + saved = torch.load(tmp_path / f"eval_{step}.pt", weights_only=False) + assert saved["rollout_id"] == step + assert saved["samples"][0]["reward"] == step + + +def test_pass_rate_uses_each_eval_datasets_completion_count(monkeypatch): + group_sizes = [] + + def pass_rate(flat_rewards, group_size): + group_sizes.append(group_size) + return {"pass@1": sum(flat_rewards) / len(flat_rewards)} + + monkeypatch.setattr(rollout_metrics, "compute_pass_rate", pass_rate) + monkeypatch.setattr(rollout_metrics, "compute_rollout_step", lambda args, step: step) + monkeypatch.setattr(rollout_metrics.logging_utils, "log", lambda *args, **kwargs: None) + args = Namespace(custom_eval_rollout_log_function_path=None, log_passrate=True, n_samples_per_eval_prompt=7) + result = rollout_metrics.log_eval_rollout_data( + 2, + args, + { + "first": {"rewards": [1, 0, 0, 1], "n_samples_per_prompt": 2}, + "second": {"rewards": [1, 0], "n_samples_per_prompt": 1}, + "legacy": {"rewards": [1] * 7}, + }, + ) + assert group_sizes == [2, 1, 7] + assert result["eval/first"] == result["eval/second"] == 0.5 diff --git a/train.py b/train.py index 2e165532c..8b06505d9 100644 --- a/train.py +++ b/train.py @@ -34,7 +34,8 @@ def train(args): # special case for eval-only if args.num_rollout == 0 and args.eval_interval is not None: - ray.get(rollout_manager.eval.remote(rollout_id=0)) + evaluation_step = args.start_rollout_id if args.rollout_backend == "vllm-rlt" else 0 + ray.get(rollout_manager.eval.remote(rollout_id=evaluation_step)) def offload_train(actor_trains_this_step): # Each model auto-offloads after train() when offload_train is set, @@ -91,7 +92,10 @@ def offload_train(actor_trains_this_step): ray.get(rollout_manager.onload_kv.remote()) if should_run_periodic_action(rollout_id, args.eval_interval, num_rollout_per_epoch): - ray.get(rollout_manager.eval.remote(rollout_id)) + # Native held-out curves use completed updates: preserve eval_0 as + # the initial policy instead of overwriting it after update one. + evaluation_step = rollout_id + 1 if args.rollout_backend == "vllm-rlt" else rollout_id + ray.get(rollout_manager.eval.remote(evaluation_step)) ray.get(rollout_manager.dispose.remote()) if args.rollout_backend == "vllm-rlt": diff --git a/vime/backends/vllm_rlt_utils/README.md b/vime/backends/vllm_rlt_utils/README.md index 775538137..b87b29cbb 100644 --- a/vime/backends/vllm_rlt_utils/README.md +++ b/vime/backends/vllm_rlt_utils/README.md @@ -8,7 +8,7 @@ use; backend imports stay lazy for other rollout engines. The initial implementation supports Ouro, Nanbeige and Huginn at the checkpoint's fixed full depth, one dedicated rollout GPU, full-vocabulary sampling and full physical-weight publication through disk. Configure training GPUs separately. -Colocation, rollout offload/recovery, external HTTP, evaluation and truncated +Colocation, rollout offload/recovery, external HTTP and truncated training sampling require additional backend contracts and are rejected by validation. Install the pinned engine in VIME's isolated environment: @@ -41,3 +41,43 @@ Defaults use BF16 training dtype, Triton attention and eager execution. `--rlt-attention-backend torch` selects the Torch reference path; `--rlt-cuda-graphs` opts into graphs. Set cache capacity with `--rlt-kv-blocks` and admission concurrency with `--rlt-max-num-seqs`. + +## Held-out evaluation + +The standard `--eval-interval` and `--eval-prompt-data`/`--eval-config` paths now +support native text evaluation. Datasets use VIME's existing JSONL/Parquet loader, +chat-template settings and per-sample reward hooks. Evaluation supports its own +temperature, top-p/top-k, response length and token stops; training still uses +full-vocabulary sampling. String stops, minimum generation lengths, repetition +penalties and custom/server generation paths are explicitly unsupported. + +Every configured prompt is evaluated. Empty or over-limit prompts fail instead +of silently reducing the held-out denominator. Generation is batched through the +same committed engine, and crossing a policy epoch/version/digest fails the entire +evaluation. Evaluation reads its own datasets without consuming the training data +source. Seeds derive from the run seed, dataset name, prompt position and completion +position under `native-heldout-v1`; they remain fixed across updates and resume. +Huginn records its actual seed and latent profile for every completion. + +Native evaluation filenames and metric steps count completed updates: `eval_0` +is the initial actor, `eval_1` follows the first update, and so on. The committed +publication version is a separate trace field; it need not equal that step index. +Evaluation-only resume uses the restored training cursor. Per-dataset completion +counts are used when computing pass rates. + +Add these flags to the existing qualified model training command: + +```bash +--eval-interval 1 \ +--eval-prompt-data heldout /data/heldout.jsonl \ +--eval-temperature 0 \ +--eval-max-response-len 48 \ +--n-samples-per-eval-prompt 1 \ +--save-debug-rollout-data '/runs/experiment/rollouts/{rollout_id}.pt' +``` + +Use a held-out dataset distinct from training, freeze both inputs and the source +revisions, and size the native cache for the declared prompt/response budget. +The example only selects the evaluation interface; it is not a validated +convergence configuration. CPU numerical and recovery evidence is in +[native_evaluation_results.md](../../../docs/native_evaluation_results.md). diff --git a/vime/backends/vllm_rlt_utils/arguments.py b/vime/backends/vllm_rlt_utils/arguments.py index 01d7cbf2a..c668cd932 100644 --- a/vime/backends/vllm_rlt_utils/arguments.py +++ b/vime/backends/vllm_rlt_utils/arguments.py @@ -57,4 +57,6 @@ def validate_args(args): if args.rollout_function_path != "vime.rollout.vllm_rlt_rollout.generate_rollout": raise ValueError("Use the native RLT rollout function with --rollout-backend=vllm-rlt") if args.eval_interval is not None: - raise ValueError("Native RLT evaluation is not yet exposed through the dataset evaluator") + from vime.rollout.native_eval import validate_evaluation + + validate_evaluation(args) diff --git a/vime/backends/vllm_rlt_utils/engine.py b/vime/backends/vllm_rlt_utils/engine.py index 8aaed632e..85c33dcba 100644 --- a/vime/backends/vllm_rlt_utils/engine.py +++ b/vime/backends/vllm_rlt_utils/engine.py @@ -1,13 +1,18 @@ """Serial engine ownership and fail-closed full physical-weight publications.""" import hashlib +from dataclasses import replace from pathlib import Path +from typing import TYPE_CHECKING from uuid import uuid4 import torch from vime.utils.types import RecurrentTrace, Sample +if TYPE_CHECKING: + from vllm_rlt import SamplingParams + def weight_digest(path: Path) -> tuple[str, list[Path]]: files = sorted(path.glob("*.safetensors")) @@ -44,10 +49,6 @@ def init(self, args): def generate(self, samples: list[Sample], rollout_id: int) -> list[Sample]: from vllm_rlt import SamplingParams - if self.paused or not self.ready: - raise RuntimeError("Publish a complete policy and resume generation first") - assert self.committed_digest is not None - version = self.llm.get_weight_version() if any(sample.index is None for sample in samples): raise ValueError("Native rollout requires stable sample indices") params = [ @@ -62,9 +63,29 @@ def generate(self, samples: list[Sample], rollout_id: int) -> list[Sample]: ) for sample in samples ] - if self.args.rlt_model_family == "huginn_raven": - from dataclasses import replace + return self._generate(samples, params) + def evaluate(self, samples: list[Sample], sampling: "SamplingParams", seeds: tuple[int, ...]) -> list[Sample]: + """Held-out seeds are independent of rollout counters and request IDs.""" + params = [ + replace( + sampling, + seed=seed, + min_loops=self.args.rlt_depth, + max_loops=self.args.rlt_depth, + logprobs=0, + logprobs_mode="processed", + ) + for _, seed in zip(samples, seeds, strict=True) + ] + return self._generate(samples, params) + + def _generate(self, samples: list[Sample], params: list["SamplingParams"]) -> list[Sample]: + if self.paused or not self.ready: + raise RuntimeError("Publish a complete policy and resume generation first") + assert self.committed_digest is not None + version = self.llm.get_weight_version() + if self.args.rlt_model_family == "huginn_raven": params = [replace(sampling, latent_seed=sampling.seed) for sampling in params] outputs = self.llm.generate([sample.tokens for sample in samples], params) for sample, output, sampling in zip(samples, outputs, params, strict=True): diff --git a/vime/observability/rollout_metrics.py b/vime/observability/rollout_metrics.py index baf91f83c..ff2110e58 100644 --- a/vime/observability/rollout_metrics.py +++ b/vime/observability/rollout_metrics.py @@ -246,7 +246,7 @@ def log_eval_rollout_data(rollout_id, args, data, extra_metrics: dict[str, Any] log_dict |= dict_add_prefix( compute_pass_rate( flat_rewards=rewards, - group_size=args.n_samples_per_eval_prompt, + group_size=data[key].get("n_samples_per_prompt", args.n_samples_per_eval_prompt), ), f"eval/{key}-", ) diff --git a/vime/rollout/native_eval.py b/vime/rollout/native_eval.py new file mode 100644 index 000000000..f5ee3a005 --- /dev/null +++ b/vime/rollout/native_eval.py @@ -0,0 +1,179 @@ +"""Complete held-out datasets through the committed native recurrent policy.""" + +import copy +import hashlib +import json +import time +from argparse import Namespace +from typing import TYPE_CHECKING + +import ray + +from vime.rollout.base_types import RolloutFnEvalOutput +from vime.rollout.vllm_rlt_rollout import _tokenizer, decode_and_reward +from vime.utils.data import Dataset +from vime.utils.eval_config import EvalDatasetConfig +from vime.utils.types import Sample + +if TYPE_CHECKING: + from vllm_rlt import SamplingParams + + +def evaluation_seed(seed: int, dataset: str, prompt: int, completion: int) -> int: + identity = json.dumps(("native-heldout-v1", seed, dataset, prompt, completion), separators=(",", ":")) + return int.from_bytes(hashlib.sha256(identity.encode()).digest()[:8], "big") % (2**63) + + +def evaluation_sampling(config: EvalDatasetConfig) -> "SamplingParams": + from vllm_rlt import SamplingParams + + if config.stop or config.min_new_tokens not in (None, 0) or config.repetition_penalty not in (None, 1): + raise ValueError("Native evaluation supports token stops, without minimum length or repetition penalties") + if config.no_stop_trim is False: + raise ValueError("Native evaluation retains generated stop tokens") + if config.max_response_len is None or config.temperature is None or config.top_p is None or config.top_k is None: + raise ValueError("Native evaluation requires resolved length, temperature, top-p and top-k settings") + return SamplingParams( + max_tokens=config.max_response_len, + temperature=config.temperature, + top_p=config.top_p, + top_k=config.top_k, + stop_token_ids=tuple(config.stop_token_ids or ()), + ) + + +def validate_evaluation(args: Namespace) -> None: + if not args.eval_datasets or args.group_rm: + raise ValueError("Native evaluation requires held-out datasets and per-sample rewards") + if len({config.name for config in args.eval_datasets}) != len(args.eval_datasets): + raise ValueError("Native evaluation dataset names must be unique") + for config in args.eval_datasets: + if config.n_samples_per_eval_prompt is None or config.n_samples_per_eval_prompt < 1: + raise ValueError("Native evaluation requires a positive completion count") + if ( + config.multimodal_keys + or args.multimodal_keys + or config.tool_key + or config.custom_generate_function_path + or config.app_service + or config.message_processor + or config.reward_model + or config.remote_environment + ): + raise ValueError("Native evaluation uses text-only native generation and per-sample reward hooks") + if ( + config.eval_task_timeout is not None + or config.eval_early_stop_remaining is not None + or config.eval_early_stop_idle_timeout is not None + ): + raise ValueError("Native evaluation completes the entire held-out dataset without early stopping") + evaluation_sampling(config) + + +def _reward(sample: Sample, key: str | None) -> float: + value = sample.reward + if key: + if not isinstance(value, dict): + raise ValueError("Evaluation reward key requires a reward mapping") + value = value[key] + if not isinstance(value, (int, float)): + raise ValueError("Native evaluation requires a scalar reward or an explicit evaluation reward key") + return float(value) + + +def evaluate(args: Namespace) -> RolloutFnEvalOutput: + """Do not read, shuffle or advance the training data source or its RNG.""" + validate_evaluation(args) + started = time.perf_counter() + tokenizer = _tokenizer(args.hf_checkpoint) + results = {} + cohort: tuple[str, int, str] | None = None + total_samples = 0 + for config in args.eval_datasets: + chat = args.apply_chat_template if config.apply_chat_template is None else config.apply_chat_template + chat_kwargs = ( + args.apply_chat_template_kwargs + if config.apply_chat_template_kwargs is None + else config.apply_chat_template_kwargs + ) + dataset = Dataset( + config.path, + tokenizer, + None, + None, + prompt_key=config.input_key, + label_key=config.label_key, + metadata_key=config.metadata_key, + apply_chat_template=chat, + apply_chat_template_kwargs=chat_kwargs, + ) + if not dataset.samples: + raise ValueError(f"Native evaluation dataset {config.name!r} is empty") + sampling = evaluation_sampling(config) + samples, seeds = [], [] + for prompt_index, original in enumerate(dataset.samples): + tokens = tokenizer.encode(original.prompt, add_special_tokens=False) + if not tokens or (args.eval_max_prompt_len is not None and len(tokens) > args.eval_max_prompt_len): + raise ValueError("Held-out prompts must be nonempty and fit the declared limit; none are dropped") + if args.eval_max_context_len is not None and len(tokens) + sampling.max_tokens > args.eval_max_context_len: + raise ValueError("Held-out prompt plus response budget exceeds the evaluation context limit") + for completion in range(config.n_samples_per_eval_prompt): + sample = copy.deepcopy(original) + sample.index, sample.group_index = len(samples), prompt_index + sample.tokens = list(tokens) + sample.metadata = config.inject_metadata(sample.metadata) + sample.metadata["native_eval"] = { + "dataset": config.name, + "prompt_index": prompt_index, + "completion": completion, + "seed_profile": "native-heldout-v1", + "top_p": sampling.top_p, + "top_k": sampling.top_k, + "max_tokens": sampling.max_tokens, + "stop_token_ids": list(sampling.stop_token_ids), + } + sample.custom_rm_path = config.custom_rm_path + samples.append(sample) + seeds.append(evaluation_seed(args.seed, config.name, prompt_index, completion)) + if config.min_eval_samples is not None and len(samples) < config.min_eval_samples: + raise ValueError("Held-out dataset has fewer completions than min_eval_samples") + generated = [] + for begin in range(0, len(samples), args.rlt_max_num_seqs): + batch = samples[begin : begin + args.rlt_max_num_seqs] + identities = [(sample.group_index, sample.index) for sample in batch] + outputs = ray.get( + args.rlt_engine.evaluate.remote(batch, sampling, tuple(seeds[begin : begin + len(batch)])) + ) + if [(sample.group_index, sample.index) for sample in outputs] != identities: + raise ValueError("Native evaluation must preserve every held-out sample in order") + for sample in outputs: + trace = sample.recurrent_trace + if trace is None: + raise RuntimeError("Native evaluation requires a committed policy trace") + identity = (trace.runtime_epoch, trace.policy_version, trace.publication_digest) + if cohort is not None and identity != cohort: + raise RuntimeError("Held-out evaluation crossed committed policy cohorts") + cohort = identity + decode_and_reward( + args, + outputs, + tokenizer, + skip_special_tokens=True if config.skip_special_tokens is None else config.skip_special_tokens, + ) + generated.extend(outputs) + results[config.name] = { + "rewards": [_reward(sample, args.eval_reward_key or args.reward_key) for sample in generated], + "truncated": [sample.status == Sample.Status.TRUNCATED for sample in generated], + "samples": generated, + "n_samples_per_prompt": config.n_samples_per_eval_prompt, + } + total_samples += len(generated) + assert cohort is not None + return RolloutFnEvalOutput( + data=results, + metrics={ + "eval/native_seconds": time.perf_counter() - started, + "eval/native_policy_version": cohort[1], + "eval/native_samples": total_samples, + }, + ) diff --git a/vime/rollout/vllm_rlt_rollout.py b/vime/rollout/vllm_rlt_rollout.py index 9847579ea..15228fac2 100644 --- a/vime/rollout/vllm_rlt_rollout.py +++ b/vime/rollout/vllm_rlt_rollout.py @@ -8,6 +8,7 @@ from vime.rollout.base_types import RolloutFnTrainOutput from vime.rollout.rm_hub import async_rm +from vime.utils.types import Sample @lru_cache(maxsize=1) @@ -17,7 +18,9 @@ def _tokenizer(checkpoint: str): def generate_rollout(args, rollout_id, data_source, evaluation=False): if evaluation: - raise ValueError("Native RLT evaluation requires an explicit evaluation recipe") + from vime.rollout.native_eval import evaluate + + return evaluate(args) groups = data_source.get_samples(args.rollout_batch_size) count = args.n_samples_per_prompt if len(groups) != args.rollout_batch_size or any(len(group) != count for group in groups): @@ -41,12 +44,18 @@ def generate_rollout(args, rollout_id, data_source, evaluation=False): generated = ray.get(args.rlt_engine.generate.remote(samples, rollout_id)) if [(sample.group_index, sample.index) for sample in generated] != identities: raise ValueError("Native rollout must preserve every prompt group and sample index in order") + decode_and_reward(args, generated, tokenizer) + return RolloutFnTrainOutput(samples=[generated[i : i + count] for i in range(0, len(generated), count)]) + + +def decode_and_reward(args, generated: list[Sample], tokenizer, *, skip_special_tokens: bool = True) -> None: for sample in generated: - sample.response = tokenizer.decode(sample.tokens[-sample.response_length :], skip_special_tokens=True) + sample.response = tokenizer.decode( + sample.tokens[-sample.response_length :], skip_special_tokens=skip_special_tokens + ) async def reward(): return await asyncio.gather(*(async_rm(args, sample) for sample in generated)) for sample, value in zip(generated, asyncio.run(reward()), strict=True): sample.reward = value - return RolloutFnTrainOutput(samples=[generated[i : i + count] for i in range(0, len(generated), count)])