From 8ee8ac67b3e11a1e1fb2c5b65d6f34bd619fa266 Mon Sep 17 00:00:00 2001 From: Antoine Zambelli Date: Thu, 20 Aug 2026 07:14:47 -0500 Subject: [PATCH 1/3] eval: add Ornith 1.5 9B sweep --- docs/MODEL_REGISTRY.md | 1 + eval_results_ornith.jsonl | 3 +++ src/forge/clients/sampling_defaults.py | 4 ++++ tests/eval/batch_eval.py | 1 + tests/unit/test_batch_eval_recipes.py | 2 ++ tests/unit/test_sampling_defaults.py | 13 +++++++++++++ 6 files changed, 24 insertions(+) create mode 100644 eval_results_ornith.jsonl diff --git a/docs/MODEL_REGISTRY.md b/docs/MODEL_REGISTRY.md index fb7a21a..0f9b8e8 100644 --- a/docs/MODEL_REGISTRY.md +++ b/docs/MODEL_REGISTRY.md @@ -68,6 +68,7 @@ Sampling params staged, no eval data published. Forge supports these — perform |---|---|---|---|---|---|---|---| | Qwen3 4B Instruct 2507 | Q4_K_M | 0.7 | 0.8 | 20 | 0.0 | — | [HF](https://huggingface.co/Qwen/Qwen3-4B-Instruct-2507) | | Qwen3 4B Thinking 2507 | Q4_K_M | 0.6 | 0.95 | 20 | 0.0 | — | [HF](https://huggingface.co/Qwen/Qwen3-4B-Thinking-2507) | +| Ornith 1.5 9B | Q4_K_M | 1.0 | 0.95 | 20 | 0.0 | general-tasks profile; repeat_penalty=1.0, presence_penalty=1.5 | [HF](https://huggingface.co/ornith-ai/Ornith-1.5-9B-GGUF) | | Qwen3.5 122B-A10B | Q4_K_M | 0.7 | 0.8 | 20 | — | balanced (instruct) preset; thinking mode is separate⁸ | [HF](https://huggingface.co/Qwen/Qwen3.5-122B-A10B) | | DeepSeek-V4-Flash-0731 UD | Q4_K_XL | 1.0 | 0.95 | — | — | agentic profile; registry-selected `reasoning_effort` (`low`/`high`/`max`)¹² | [HF](https://huggingface.co/deepseek-ai/DeepSeek-V4-Flash-0731) | | Inkling Small UD | IQ4_XS | 1.0 | 1.0 | — | 0.0 | `reasoning_effort="high"` stress profile¹³ | [Unsloth](https://unsloth.ai/docs/models/inkling) | diff --git a/eval_results_ornith.jsonl b/eval_results_ornith.jsonl new file mode 100644 index 0000000..1d14071 --- /dev/null +++ b/eval_results_ornith.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fda2580972710fe3fc5c6085d9e8fa657837e132a4b164bdda35aeb1b71751c7 +size 1845575 diff --git a/src/forge/clients/sampling_defaults.py b/src/forge/clients/sampling_defaults.py index 3426bfa..5b45612 100644 --- a/src/forge/clients/sampling_defaults.py +++ b/src/forge/clients/sampling_defaults.py @@ -72,6 +72,10 @@ # presence_penalty (0.0 vs 1.5) for thinking-mode general tasks. "qwen3.6:27b-q4_K_M": {"temperature": 1.0, "top_p": 0.95, "top_k": 20, "min_p": 0.0, "presence_penalty": 0.0}, # https://huggingface.co/Qwen/Qwen3.6-27B "Qwen3.6-27B-Q4_K_M": {"temperature": 1.0, "top_p": 0.95, "top_k": 20, "min_p": 0.0, "presence_penalty": 0.0}, # https://huggingface.co/Qwen/Qwen3.6-27B + # Ornith 1.5 9B — publisher's general-tasks profile. The precise-coding + # alternative changes temperature to 0.6 and presence_penalty to 0.0. + "ornith-1.5:9b": {"temperature": 1.0, "top_p": 0.95, "top_k": 20, "min_p": 0.0, "presence_penalty": 1.5, "repeat_penalty": 1.0}, # https://huggingface.co/ornith-ai/Ornith-1.5-9B-GGUF + "Ornith-1.5-9B-Q4_K_M": {"temperature": 1.0, "top_p": 0.95, "top_k": 20, "min_p": 0.0, "presence_penalty": 1.5, "repeat_penalty": 1.0}, # https://huggingface.co/ornith-ai/Ornith-1.5-9B-GGUF # Qwen3.8-27B — Unsloth's thinking-mode profile. xhigh is the model's # documented default reasoning effort for complex coding and agentic work. "Qwen3.8-27B-UD-Q4_K_XL": {"temperature": 1.0, "top_p": 0.95, "top_k": 20, "min_p": 0.0, "presence_penalty": 0.0, "repeat_penalty": 1.0, "chat_template_kwargs": {"reasoning_effort": "xhigh"}}, # https://unsloth.ai/docs/models/qwen3.8 diff --git a/tests/eval/batch_eval.py b/tests/eval/batch_eval.py index a41b27f..e384cc3 100644 --- a/tests/eval/batch_eval.py +++ b/tests/eval/batch_eval.py @@ -173,6 +173,7 @@ def _glimmer_server_recipe(reasoning_strength: str) -> _BatchServerRecipe: ("Qwen3.5-35B-A3B-Q4_K_M.gguf", _REASONING_SERVER_RECIPE), ("Qwen3.6-27B-Q4_K_M.gguf", _REASONING_SERVER_RECIPE), ("Qwen3.6-35B-A3B-UD-Q4_K_M.gguf", _REASONING_SERVER_RECIPE), + ("Ornith-1.5-9B-Q4_K_M.gguf", _REASONING_SERVER_RECIPE), ("Qwen3.8-27B-UD-Q4_K_XL.gguf", _QWEN38_SERVER_RECIPE), ("Nemotron-3-Nano-30B-A3B-Q4_K_M.gguf", _REASONING_SERVER_RECIPE), ("Muse-Glimmer-30B-UD-Q4_K_XL.gguf", _GLIMMER_SERVER_RECIPE), diff --git a/tests/unit/test_batch_eval_recipes.py b/tests/unit/test_batch_eval_recipes.py index af081b3..cfbde81 100644 --- a/tests/unit/test_batch_eval_recipes.py +++ b/tests/unit/test_batch_eval_recipes.py @@ -29,6 +29,7 @@ ("Qwen3.5-35B-A3B-Q4_K_M", "Qwen3.5-35B-A3B-Q4_K_M.gguf"), ("Qwen3.6-27B-Q4_K_M", "Qwen3.6-27B-Q4_K_M.gguf"), ("Qwen3.6-35B-A3B-UD-Q4_K_M", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), + ("Ornith-1.5-9B-Q4_K_M", "Ornith-1.5-9B-Q4_K_M.gguf"), ("Qwen3.8-27B-UD-Q4_K_XL", "Qwen3.8-27B-UD-Q4_K_XL.gguf"), ("Nemotron-3-Nano-30B-A3B-Q4_K_M", "Nemotron-3-Nano-30B-A3B-Q4_K_M.gguf"), ("Muse-Glimmer-30B-UD-Q4_K_XL", "Muse-Glimmer-30B-UD-Q4_K_XL.gguf"), @@ -113,6 +114,7 @@ def _glimmer_flags(reasoning_strength: str) -> tuple[str, ...]: "Qwen3.5-35B-A3B-Q4_K_M": _REASONING_FLAGS, "Qwen3.6-27B-Q4_K_M": _REASONING_FLAGS, "Qwen3.6-35B-A3B-UD-Q4_K_M": _REASONING_FLAGS, + "Ornith-1.5-9B-Q4_K_M": _REASONING_FLAGS, "Qwen3.8-27B-UD-Q4_K_XL": _QWEN38_FLAGS, "Nemotron-3-Nano-30B-A3B-Q4_K_M": _REASONING_FLAGS, "Muse-Glimmer-30B-UD-Q4_K_XL": _GLIMMER_FLAGS, diff --git a/tests/unit/test_sampling_defaults.py b/tests/unit/test_sampling_defaults.py index b35684b..a1cb732 100644 --- a/tests/unit/test_sampling_defaults.py +++ b/tests/unit/test_sampling_defaults.py @@ -56,6 +56,19 @@ def test_qwen3_8_uses_thinking_profile_at_xhigh(self) -> None: "chat_template_kwargs": {"reasoning_effort": "xhigh"}, } + def test_ornith_1_5_9b_uses_general_tasks_profile(self) -> None: + """Ornith's general profile includes both published penalties.""" + expected = { + "temperature": 1.0, + "top_p": 0.95, + "top_k": 20, + "min_p": 0.0, + "presence_penalty": 1.5, + "repeat_penalty": 1.0, + } + assert get_sampling_defaults("ornith-1.5:9b") == expected + assert get_sampling_defaults("Ornith-1.5-9B-Q4_K_M") == expected + def test_qwen3_coder_uses_repeat_penalty(self) -> None: """Qwen3-Coder card specifies repeat_penalty=1.05, no min_p.""" result = get_sampling_defaults("qwen3-coder:30b-a3b-instruct-q4_K_M") From a967700914e53fbdac728b8ac3c6df4dcfeb7cc1 Mon Sep 17 00:00:00 2001 From: Antoine Zambelli Date: Thu, 20 Aug 2026 07:26:52 -0500 Subject: [PATCH 2/3] eval: publish Ornith 1.5 35B sweep Add the Ornith 35B sampling and batch configuration, fold the generation-3 9B and 35B rows into the v0.9 dataset, and remove the standalone 9B intermediate file. --- eval_results_ornith.jsonl | 3 --- eval_results_v0.9.0.jsonl | 4 ++-- src/forge/clients/sampling_defaults.py | 2 ++ tests/eval/batch_eval.py | 1 + 4 files changed, 5 insertions(+), 5 deletions(-) delete mode 100644 eval_results_ornith.jsonl diff --git a/eval_results_ornith.jsonl b/eval_results_ornith.jsonl deleted file mode 100644 index 1d14071..0000000 --- a/eval_results_ornith.jsonl +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:fda2580972710fe3fc5c6085d9e8fa657837e132a4b164bdda35aeb1b71751c7 -size 1845575 diff --git a/eval_results_v0.9.0.jsonl b/eval_results_v0.9.0.jsonl index dac388d..32ae489 100644 --- a/eval_results_v0.9.0.jsonl +++ b/eval_results_v0.9.0.jsonl @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:50b2ae49a8b9161711289c60dc060a3403dbf2534ec1cbb6d5834e3f384fa8c3 -size 19761100 +oid sha256:e36079168689159362b1aac23901ac121615f38380ccbed5f9a1e90f324a9c8e +size 23456457 diff --git a/src/forge/clients/sampling_defaults.py b/src/forge/clients/sampling_defaults.py index 5b45612..7e173dd 100644 --- a/src/forge/clients/sampling_defaults.py +++ b/src/forge/clients/sampling_defaults.py @@ -79,6 +79,8 @@ # Qwen3.8-27B — Unsloth's thinking-mode profile. xhigh is the model's # documented default reasoning effort for complex coding and agentic work. "Qwen3.8-27B-UD-Q4_K_XL": {"temperature": 1.0, "top_p": 0.95, "top_k": 20, "min_p": 0.0, "presence_penalty": 0.0, "repeat_penalty": 1.0, "chat_template_kwargs": {"reasoning_effort": "xhigh"}}, # https://unsloth.ai/docs/models/qwen3.8 + # Ornith 1.5 35B-A3B — model-card general-task profile. + "Ornith-1.5-35B-Q4_K_M": {"temperature": 0.6, "top_p": 0.95, "top_k": 20}, # https://huggingface.co/ornith-ai/Ornith-1.5-35B-A3B-GGUF # Qwen3-Coder — non-thinking instruct; card does not mention min_p or presence_penalty "qwen3-coder:30b-a3b-instruct-q4_K_M": {"temperature": 0.7, "top_p": 0.8, "top_k": 20, "repeat_penalty": 1.05}, # https://huggingface.co/Qwen/Qwen3-Coder-30B-A3B-Instruct "Qwen3-Coder-30B-A3B-Instruct-Q4_K_M": {"temperature": 0.7, "top_p": 0.8, "top_k": 20, "repeat_penalty": 1.05}, # https://huggingface.co/Qwen/Qwen3-Coder-30B-A3B-Instruct diff --git a/tests/eval/batch_eval.py b/tests/eval/batch_eval.py index e384cc3..b0c2df0 100644 --- a/tests/eval/batch_eval.py +++ b/tests/eval/batch_eval.py @@ -175,6 +175,7 @@ def _glimmer_server_recipe(reasoning_strength: str) -> _BatchServerRecipe: ("Qwen3.6-35B-A3B-UD-Q4_K_M.gguf", _REASONING_SERVER_RECIPE), ("Ornith-1.5-9B-Q4_K_M.gguf", _REASONING_SERVER_RECIPE), ("Qwen3.8-27B-UD-Q4_K_XL.gguf", _QWEN38_SERVER_RECIPE), + ("Ornith-1.5-35B-Q4_K_M.gguf", _REASONING_SERVER_RECIPE), ("Nemotron-3-Nano-30B-A3B-Q4_K_M.gguf", _REASONING_SERVER_RECIPE), ("Muse-Glimmer-30B-UD-Q4_K_XL.gguf", _GLIMMER_SERVER_RECIPE), # Gemma-4 large (rig-04, az/eval-large): 26B-A4B MoE + 31B dense. Native FC; From 1e5ebd66c112b94ace7dc4231e6274ee3483f5ed Mon Sep 17 00:00:00 2001 From: Antoine Zambelli Date: Thu, 20 Aug 2026 07:49:12 -0500 Subject: [PATCH 3/3] eval: finalize Ornith publication surfaces --- docs/MODEL_REGISTRY.md | 3 ++- docs/results/dashboard.html | 3 ++- docs/results/index.md | 2 +- docs/results/raw/native-vs-prompt.md | 2 +- docs/results/raw/reasoning-replay.md | 2 +- docs/results/raw/reforged-vs-bare.md | 24 +++++++++++++++++++++++- docs/results/raw/reforged/all.md | 4 +++- docs/results/raw/reforged/by-backend.md | 2 +- docs/results/raw/reforged/by-family.md | 22 +++++++++++++++++++++- tests/eval/publication.py | 20 ++++++++++---------- tests/eval/report.py | 3 +++ tests/unit/test_batch_eval_recipes.py | 2 ++ 12 files changed, 70 insertions(+), 19 deletions(-) diff --git a/docs/MODEL_REGISTRY.md b/docs/MODEL_REGISTRY.md index 0f9b8e8..50ab5f5 100644 --- a/docs/MODEL_REGISTRY.md +++ b/docs/MODEL_REGISTRY.md @@ -30,6 +30,8 @@ Sampling values are sourced from the model's HuggingFace card unless noted. Valu | Qwen3.5 35B-A3B | Q4_K_M | 1.0 | 0.95 | 20 | 0.0 | — | 1.5 | [HF](https://huggingface.co/Qwen/Qwen3.5-35B-A3B) | | Qwen3.6 27B | Q4_K_M | 1.0 | 0.95 | 20 | 0.0 | — | 0.0⁶ | [HF](https://huggingface.co/Qwen/Qwen3.6-27B) | | Qwen3.6 35B-A3B UD | Q4_K_M | 1.0 | 0.95 | 20 | 0.0 | — | 1.5 | [HF](https://huggingface.co/Qwen/Qwen3.6-35B-A3B) | +| Ornith 1.5 9B | Q4_K_M | 1.0 | 0.95 | 20 | 0.0 | 1.0 | 1.5 | [HF](https://huggingface.co/ornith-ai/Ornith-1.5-9B-GGUF) | +| Ornith 1.5 35B-A3B | Q4_K_M | 0.6 | 0.95 | 20 | — | — | — | [HF](https://huggingface.co/ornith-ai/Ornith-1.5-35B-A3B-GGUF) | | Nemotron-3 Nano 30B-A3B | Q4_K_M | 0.6 | 0.95 | — | — | — | —⁷ | [HF](https://huggingface.co/nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16) | | Claude Haiku 4.5⁵ | — | — | — | — | — | — | — | (SDK-managed) | | Claude Sonnet 4.6⁵ | — | — | — | — | — | — | — | (SDK-managed) | @@ -68,7 +70,6 @@ Sampling params staged, no eval data published. Forge supports these — perform |---|---|---|---|---|---|---|---| | Qwen3 4B Instruct 2507 | Q4_K_M | 0.7 | 0.8 | 20 | 0.0 | — | [HF](https://huggingface.co/Qwen/Qwen3-4B-Instruct-2507) | | Qwen3 4B Thinking 2507 | Q4_K_M | 0.6 | 0.95 | 20 | 0.0 | — | [HF](https://huggingface.co/Qwen/Qwen3-4B-Thinking-2507) | -| Ornith 1.5 9B | Q4_K_M | 1.0 | 0.95 | 20 | 0.0 | general-tasks profile; repeat_penalty=1.0, presence_penalty=1.5 | [HF](https://huggingface.co/ornith-ai/Ornith-1.5-9B-GGUF) | | Qwen3.5 122B-A10B | Q4_K_M | 0.7 | 0.8 | 20 | — | balanced (instruct) preset; thinking mode is separate⁸ | [HF](https://huggingface.co/Qwen/Qwen3.5-122B-A10B) | | DeepSeek-V4-Flash-0731 UD | Q4_K_XL | 1.0 | 0.95 | — | — | agentic profile; registry-selected `reasoning_effort` (`low`/`high`/`max`)¹² | [HF](https://huggingface.co/deepseek-ai/DeepSeek-V4-Flash-0731) | | Inkling Small UD | IQ4_XS | 1.0 | 1.0 | — | 0.0 | `reasoning_effort="high"` stress profile¹³ | [Unsloth](https://unsloth.ai/docs/models/inkling) | diff --git a/docs/results/dashboard.html b/docs/results/dashboard.html index 2649807..d9aef56 100644 --- a/docs/results/dashboard.html +++ b/docs/results/dashboard.html @@ -14,9 +14,10 @@ `+a.stack}}var Jn=Object.prototype.hasOwnProperty,wn=h.unstable_scheduleCallback,Wn=h.unstable_cancelCallback,qo=h.unstable_shouldYield,Yo=h.unstable_requestPaint,ut=h.unstable_now,Go=h.unstable_getCurrentPriorityLevel,zf=h.unstable_ImmediatePriority,Af=h.unstable_UserBlockingPriority,Eu=h.unstable_NormalPriority,Qo=h.unstable_LowPriority,pf=h.unstable_IdlePriority,Xo=h.log,Zo=h.unstable_setDisableYieldValue,Ma=null,nt=null;function It(l){if(typeof Xo=="function"&&Zo(l),nt&&typeof nt.setStrictMode=="function")try{nt.setStrictMode(Ma,l)}catch{}}var ct=Math.clz32?Math.clz32:Ko,Vo=Math.log,Lo=Math.LN2;function Ko(l){return l>>>=0,l===0?32:31-(Vo(l)/Lo|0)|0}var Tu=256,_u=262144,Ou=4194304;function Te(l){var t=l&42;if(t!==0)return t;switch(l&-l){case 1:return 1;case 2:return 2;case 4:return 4;case 8:return 8;case 16:return 16;case 32:return 32;case 64:return 64;case 128:return 128;case 256:case 512:case 1024:case 2048:case 4096:case 8192:case 16384:case 32768:case 65536:case 131072:return l&261888;case 262144:case 524288:case 1048576:case 2097152:return l&3932160;case 4194304:case 8388608:case 16777216:case 33554432:return l&62914560;case 67108864:return 67108864;case 134217728:return 134217728;case 268435456:return 268435456;case 536870912:return 536870912;case 1073741824:return 0;default:return l}}function Mu(l,t,e){var a=l.pendingLanes;if(a===0)return 0;var u=0,n=l.suspendedLanes,c=l.pingedLanes;l=l.warmLanes;var i=a&134217727;return i!==0?(a=i&~n,a!==0?u=Te(a):(c&=i,c!==0?u=Te(c):e||(e=i&~l,e!==0&&(u=Te(e))))):(i=a&~n,i!==0?u=Te(i):c!==0?u=Te(c):e||(e=a&~l,e!==0&&(u=Te(e)))),u===0?0:t!==0&&t!==u&&(t&n)===0&&(n=u&-u,e=t&-t,n>=e||n===32&&(e&4194048)!==0)?t:u}function xa(l,t){return(l.pendingLanes&~(l.suspendedLanes&~l.pingedLanes)&t)===0}function Jo(l,t){switch(l){case 1:case 2:case 4:case 8:case 64:return t+250;case 16:case 32:case 128:case 256:case 512:case 1024:case 2048:case 4096:case 8192:case 16384:case 32768:case 65536:case 131072:case 262144:case 524288:case 1048576:case 2097152:return t+5e3;case 4194304:case 8388608:case 16777216:case 33554432:return-1;case 67108864:case 134217728:case 268435456:case 536870912:case 1073741824:return-1;default:return-1}}function Ef(){var l=Ou;return Ou<<=1,(Ou&62914560)===0&&(Ou=4194304),l}function $n(l){for(var t=[],e=0;31>e;e++)t.push(l);return t}function Na(l,t){l.pendingLanes|=t,t!==268435456&&(l.suspendedLanes=0,l.pingedLanes=0,l.warmLanes=0)}function wo(l,t,e,a,u,n){var c=l.pendingLanes;l.pendingLanes=e,l.suspendedLanes=0,l.pingedLanes=0,l.warmLanes=0,l.expiredLanes&=e,l.entangledLanes&=e,l.errorRecoveryDisabledLanes&=e,l.shellSuspendCounter=0;var i=l.entanglements,f=l.expirationTimes,y=l.hiddenUpdates;for(e=c&~e;0"u")return null;try{return l.activeElement||l.body}catch{return l.body}}var Po=/[\n"\\]/g;function ht(l){return l.replace(Po,function(t){return"\\"+t.charCodeAt(0).toString(16)+" "})}function tc(l,t,e,a,u,n,c,i){l.name="",c!=null&&typeof c!="function"&&typeof c!="symbol"&&typeof c!="boolean"?l.type=c:l.removeAttribute("type"),t!=null?c==="number"?(t===0&&l.value===""||l.value!=t)&&(l.value=""+yt(t)):l.value!==""+yt(t)&&(l.value=""+yt(t)):c!=="submit"&&c!=="reset"||l.removeAttribute("value"),t!=null?ec(l,c,yt(t)):e!=null?ec(l,c,yt(e)):a!=null&&l.removeAttribute("value"),u==null&&n!=null&&(l.defaultChecked=!!n),u!=null&&(l.checked=u&&typeof u!="function"&&typeof u!="symbol"),i!=null&&typeof i!="function"&&typeof i!="symbol"&&typeof i!="boolean"?l.name=""+yt(i):l.removeAttribute("name")}function Bf(l,t,e,a,u,n,c,i){if(n!=null&&typeof n!="function"&&typeof n!="symbol"&&typeof n!="boolean"&&(l.type=n),t!=null||e!=null){if(!(n!=="submit"&&n!=="reset"||t!=null)){lc(l);return}e=e!=null?""+yt(e):"",t=t!=null?""+yt(t):e,i||t===l.value||(l.value=t),l.defaultValue=t}a=a??u,a=typeof a!="function"&&typeof a!="symbol"&&!!a,l.checked=i?l.checked:!!a,l.defaultChecked=!!a,c!=null&&typeof c!="function"&&typeof c!="symbol"&&typeof c!="boolean"&&(l.name=c),lc(l)}function ec(l,t,e){t==="number"&&Du(l.ownerDocument)===l||l.defaultValue===""+e||(l.defaultValue=""+e)}function we(l,t,e,a){if(l=l.options,t){t={};for(var u=0;u"u"||typeof window.document>"u"||typeof window.document.createElement>"u"),ic=!1;if(Bt)try{var Ra={};Object.defineProperty(Ra,"passive",{get:function(){ic=!0}}),window.addEventListener("test",Ra,Ra),window.removeEventListener("test",Ra,Ra)}catch{ic=!1}var le=null,fc=null,Cu=null;function Vf(){if(Cu)return Cu;var l,t=fc,e=t.length,a,u="value"in le?le.value:le.textContent,n=u.length;for(l=0;l=Ba),$f=" ",kf=!1;function Ff(l,t){switch(l){case"keyup":return Mm.indexOf(t.keyCode)!==-1;case"keydown":return t.keyCode!==229;case"keypress":case"mousedown":case"focusout":return!0;default:return!1}}function If(l){return l=l.detail,typeof l=="object"&&"data"in l?l.data:null}var Fe=!1;function Nm(l,t){switch(l){case"compositionend":return If(t);case"keypress":return t.which!==32?null:(kf=!0,$f);case"textInput":return l=t.data,l===$f&&kf?null:l;default:return null}}function Dm(l,t){if(Fe)return l==="compositionend"||!rc&&Ff(l,t)?(l=Vf(),Cu=fc=le=null,Fe=!1,l):null;switch(l){case"paste":return null;case"keypress":if(!(t.ctrlKey||t.altKey||t.metaKey)||t.ctrlKey&&t.altKey){if(t.char&&1=t)return{node:e,offset:t-l};l=a}l:{for(;e;){if(e.nextSibling){e=e.nextSibling;break l}e=e.parentNode}e=void 0}e=cs(e)}}function fs(l,t){return l&&t?l===t?!0:l&&l.nodeType===3?!1:t&&t.nodeType===3?fs(l,t.parentNode):"contains"in l?l.contains(t):l.compareDocumentPosition?!!(l.compareDocumentPosition(t)&16):!1:!1}function ss(l){l=l!=null&&l.ownerDocument!=null&&l.ownerDocument.defaultView!=null?l.ownerDocument.defaultView:window;for(var t=Du(l.document);t instanceof l.HTMLIFrameElement;){try{var e=typeof t.contentWindow.location.href=="string"}catch{e=!1}if(e)l=t.contentWindow;else break;t=Du(l.document)}return t}function vc(l){var t=l&&l.nodeName&&l.nodeName.toLowerCase();return t&&(t==="input"&&(l.type==="text"||l.type==="search"||l.type==="tel"||l.type==="url"||l.type==="password")||t==="textarea"||l.contentEditable==="true")}var Ym=Bt&&"documentMode"in document&&11>=document.documentMode,Ie=null,gc=null,Qa=null,Sc=!1;function ds(l,t,e){var a=e.window===e?e.document:e.nodeType===9?e:e.ownerDocument;Sc||Ie==null||Ie!==Du(a)||(a=Ie,"selectionStart"in a&&vc(a)?a={start:a.selectionStart,end:a.selectionEnd}:(a=(a.ownerDocument&&a.ownerDocument.defaultView||window).getSelection(),a={anchorNode:a.anchorNode,anchorOffset:a.anchorOffset,focusNode:a.focusNode,focusOffset:a.focusOffset}),Qa&&Ga(Qa,a)||(Qa=a,a=On(gc,"onSelect"),0>=c,u-=c,Nt=1<<32-ct(t)+u|e<w?(el=B,B=null):el=B.sibling;var cl=v(m,B,r[w],A);if(cl===null){B===null&&(B=el);break}l&&B&&cl.alternate===null&&t(m,B),s=n(cl,s,w),nl===null?Q=cl:nl.sibling=cl,nl=cl,B=el}if(w===r.length)return e(m,B),al&&Yt(m,w),Q;if(B===null){for(;ww?(el=B,B=null):el=B.sibling;var pe=v(m,B,cl.value,A);if(pe===null){B===null&&(B=el);break}l&&B&&pe.alternate===null&&t(m,B),s=n(pe,s,w),nl===null?Q=pe:nl.sibling=pe,nl=pe,B=el}if(cl.done)return e(m,B),al&&Yt(m,w),Q;if(B===null){for(;!cl.done;w++,cl=r.next())cl=p(m,cl.value,A),cl!==null&&(s=n(cl,s,w),nl===null?Q=cl:nl.sibling=cl,nl=cl);return al&&Yt(m,w),Q}for(B=a(B);!cl.done;w++,cl=r.next())cl=S(B,m,w,cl.value,A),cl!==null&&(l&&cl.alternate!==null&&B.delete(cl.key===null?w:cl.key),s=n(cl,s,w),nl===null?Q=cl:nl.sibling=cl,nl=cl);return l&&B.forEach(function(uy){return t(m,uy)}),al&&Yt(m,w),Q}function zl(m,s,r,A){if(typeof r=="object"&&r!==null&&r.type===rl&&r.key===null&&(r=r.props.children),typeof r=="object"&&r!==null){switch(r.$$typeof){case $:l:{for(var Q=r.key;s!==null;){if(s.key===Q){if(Q=r.type,Q===rl){if(s.tag===7){e(m,s.sibling),A=u(s,r.props.children),A.return=m,m=A;break l}}else if(s.elementType===Q||typeof Q=="object"&&Q!==null&&Q.$$typeof===Bl&&He(Q)===s.type){e(m,s.sibling),A=u(s,r.props),Ja(A,r),A.return=m,m=A;break l}e(m,s);break}else t(m,s);s=s.sibling}r.type===rl?(A=De(r.props.children,m.mode,A,r.key),A.return=m,m=A):(A=Zu(r.type,r.key,r.props,null,m.mode,A),Ja(A,r),A.return=m,m=A)}return c(m);case ll:l:{for(Q=r.key;s!==null;){if(s.key===Q)if(s.tag===4&&s.stateNode.containerInfo===r.containerInfo&&s.stateNode.implementation===r.implementation){e(m,s.sibling),A=u(s,r.children||[]),A.return=m,m=A;break l}else{e(m,s);break}else t(m,s);s=s.sibling}A=_c(r,m.mode,A),A.return=m,m=A}return c(m);case Bl:return r=He(r),zl(m,s,r,A)}if(at(r))return j(m,s,r,A);if(il(r)){if(Q=il(r),typeof Q!="function")throw Error(o(150));return r=Q.call(r),Z(m,s,r,A)}if(typeof r.then=="function")return zl(m,s,$u(r),A);if(r.$$typeof===sl)return zl(m,s,Ku(m,r),A);ku(m,r)}return typeof r=="string"&&r!==""||typeof r=="number"||typeof r=="bigint"?(r=""+r,s!==null&&s.tag===6?(e(m,s.sibling),A=u(s,r),A.return=m,m=A):(e(m,s),A=Tc(r,m.mode,A),A.return=m,m=A),c(m)):e(m,s)}return function(m,s,r,A){try{Ka=0;var Q=zl(m,s,r,A);return sa=null,Q}catch(B){if(B===fa||B===wu)throw B;var nl=ft(29,B,null,m.mode);return nl.lanes=A,nl.return=m,nl}}}var qe=Rs(!0),js=Rs(!1),ne=!1;function qc(l){l.updateQueue={baseState:l.memoizedState,firstBaseUpdate:null,lastBaseUpdate:null,shared:{pending:null,lanes:0,hiddenCallbacks:null},callbacks:null}}function Yc(l,t){l=l.updateQueue,t.updateQueue===l&&(t.updateQueue={baseState:l.baseState,firstBaseUpdate:l.firstBaseUpdate,lastBaseUpdate:l.lastBaseUpdate,shared:l.shared,callbacks:null})}function ce(l){return{lane:l,tag:0,payload:null,callback:null,next:null}}function ie(l,t,e){var a=l.updateQueue;if(a===null)return null;if(a=a.shared,(fl&2)!==0){var u=a.pending;return u===null?t.next=t:(t.next=u.next,u.next=t),a.pending=t,t=Xu(l),gs(l,null,e),t}return Qu(l,a,t,e),Xu(l)}function wa(l,t,e){if(t=t.updateQueue,t!==null&&(t=t.shared,(e&4194048)!==0)){var a=t.lanes;a&=l.pendingLanes,e|=a,t.lanes=e,_f(l,e)}}function Gc(l,t){var e=l.updateQueue,a=l.alternate;if(a!==null&&(a=a.updateQueue,e===a)){var u=null,n=null;if(e=e.firstBaseUpdate,e!==null){do{var c={lane:e.lane,tag:e.tag,payload:e.payload,callback:null,next:null};n===null?u=n=c:n=n.next=c,e=e.next}while(e!==null);n===null?u=n=t:n=n.next=t}else u=n=t;e={baseState:a.baseState,firstBaseUpdate:u,lastBaseUpdate:n,shared:a.shared,callbacks:a.callbacks},l.updateQueue=e;return}l=e.lastBaseUpdate,l===null?e.firstBaseUpdate=t:l.next=t,e.lastBaseUpdate=t}var Qc=!1;function Wa(){if(Qc){var l=ia;if(l!==null)throw l}}function $a(l,t,e,a){Qc=!1;var u=l.updateQueue;ne=!1;var n=u.firstBaseUpdate,c=u.lastBaseUpdate,i=u.shared.pending;if(i!==null){u.shared.pending=null;var f=i,y=f.next;f.next=null,c===null?n=y:c.next=y,c=f;var b=l.alternate;b!==null&&(b=b.updateQueue,i=b.lastBaseUpdate,i!==c&&(i===null?b.firstBaseUpdate=y:i.next=y,b.lastBaseUpdate=f))}if(n!==null){var p=u.baseState;c=0,b=y=f=null,i=n;do{var v=i.lane&-536870913,S=v!==i.lane;if(S?(tl&v)===v:(a&v)===v){v!==0&&v===ca&&(Qc=!0),b!==null&&(b=b.next={lane:0,tag:i.tag,payload:i.payload,callback:null,next:null});l:{var j=l,Z=i;v=t;var zl=e;switch(Z.tag){case 1:if(j=Z.payload,typeof j=="function"){p=j.call(zl,p,v);break l}p=j;break l;case 3:j.flags=j.flags&-65537|128;case 0:if(j=Z.payload,v=typeof j=="function"?j.call(zl,p,v):j,v==null)break l;p=U({},p,v);break l;case 2:ne=!0}}v=i.callback,v!==null&&(l.flags|=64,S&&(l.flags|=8192),S=u.callbacks,S===null?u.callbacks=[v]:S.push(v))}else S={lane:v,tag:i.tag,payload:i.payload,callback:i.callback,next:null},b===null?(y=b=S,f=p):b=b.next=S,c|=v;if(i=i.next,i===null){if(i=u.shared.pending,i===null)break;S=i,i=S.next,S.next=null,u.lastBaseUpdate=S,u.shared.pending=null}}while(!0);b===null&&(f=p),u.baseState=f,u.firstBaseUpdate=y,u.lastBaseUpdate=b,n===null&&(u.shared.lanes=0),me|=c,l.lanes=c,l.memoizedState=p}}function Hs(l,t){if(typeof l!="function")throw Error(o(191,l));l.call(t)}function Bs(l,t){var e=l.callbacks;if(e!==null)for(l.callbacks=null,l=0;ln?n:8;var c=z.T,i={};z.T=i,ni(l,!1,t,e);try{var f=u(),y=z.S;if(y!==null&&y(i,f),f!==null&&typeof f=="object"&&typeof f.then=="function"){var b=wm(f,a);Ia(l,t,b,rt(l))}else Ia(l,t,a,rt(l))}catch(p){Ia(l,t,{then:function(){},status:"rejected",reason:p},rt())}finally{N.p=n,c!==null&&i.types!==null&&(c.types=i.types),z.T=c}}function Pm(){}function ai(l,t,e,a){if(l.tag!==5)throw Error(o(476));var u=y0(l).queue;r0(l,u,t,V,e===null?Pm:function(){return h0(l),e(a)})}function y0(l){var t=l.memoizedState;if(t!==null)return t;t={memoizedState:V,baseState:V,baseQueue:null,queue:{pending:null,lanes:0,dispatch:null,lastRenderedReducer:Zt,lastRenderedState:V},next:null};var e={};return t.next={memoizedState:e,baseState:e,baseQueue:null,queue:{pending:null,lanes:0,dispatch:null,lastRenderedReducer:Zt,lastRenderedState:e},next:null},l.memoizedState=t,l=l.alternate,l!==null&&(l.memoizedState=t),t}function h0(l){var t=y0(l);t.next===null&&(t=l.alternate.memoizedState),Ia(l,t.next.queue,{},rt())}function ui(){return Zl(hu)}function v0(){return Ul().memoizedState}function g0(){return Ul().memoizedState}function lr(l){for(var t=l.return;t!==null;){switch(t.tag){case 24:case 3:var e=rt();l=ce(e);var a=ie(t,l,e);a!==null&&(et(a,t,e),wa(a,t,e)),t={cache:Rc()},l.payload=t;return}t=t.return}}function tr(l,t,e){var a=rt();e={lane:a,revertLane:0,gesture:null,action:e,hasEagerState:!1,eagerState:null,next:null},cn(l)?b0(t,e):(e=pc(l,t,e,a),e!==null&&(et(e,l,a),z0(e,t,a)))}function S0(l,t,e){var a=rt();Ia(l,t,e,a)}function Ia(l,t,e,a){var u={lane:a,revertLane:0,gesture:null,action:e,hasEagerState:!1,eagerState:null,next:null};if(cn(l))b0(t,u);else{var n=l.alternate;if(l.lanes===0&&(n===null||n.lanes===0)&&(n=t.lastRenderedReducer,n!==null))try{var c=t.lastRenderedState,i=n(c,e);if(u.hasEagerState=!0,u.eagerState=i,it(i,c))return Qu(l,t,u,0),Al===null&&Gu(),!1}catch{}if(e=pc(l,t,u,a),e!==null)return et(e,l,a),z0(e,t,a),!0}return!1}function ni(l,t,e,a){if(a={lane:2,revertLane:qi(),gesture:null,action:a,hasEagerState:!1,eagerState:null,next:null},cn(l)){if(t)throw Error(o(479))}else t=pc(l,e,a,2),t!==null&&et(t,l,2)}function cn(l){var t=l.alternate;return l===J||t!==null&&t===J}function b0(l,t){oa=Pu=!0;var e=l.pending;e===null?t.next=t:(t.next=e.next,e.next=t),l.pending=t}function z0(l,t,e){if((e&4194048)!==0){var a=t.lanes;a&=l.pendingLanes,e|=a,t.lanes=e,_f(l,e)}}var Pa={readContext:Zl,use:en,useCallback:Ol,useContext:Ol,useEffect:Ol,useImperativeHandle:Ol,useLayoutEffect:Ol,useInsertionEffect:Ol,useMemo:Ol,useReducer:Ol,useRef:Ol,useState:Ol,useDebugValue:Ol,useDeferredValue:Ol,useTransition:Ol,useSyncExternalStore:Ol,useId:Ol,useHostTransitionStatus:Ol,useFormState:Ol,useActionState:Ol,useOptimistic:Ol,useMemoCache:Ol,useCacheRefresh:Ol};Pa.useEffectEvent=Ol;var A0={readContext:Zl,use:en,useCallback:function(l,t){return Wl().memoizedState=[l,t===void 0?null:t],l},useContext:Zl,useEffect:u0,useImperativeHandle:function(l,t,e){e=e!=null?e.concat([l]):null,un(4194308,4,f0.bind(null,t,l),e)},useLayoutEffect:function(l,t){return un(4194308,4,l,t)},useInsertionEffect:function(l,t){un(4,2,l,t)},useMemo:function(l,t){var e=Wl();t=t===void 0?null:t;var a=l();if(Ye){It(!0);try{l()}finally{It(!1)}}return e.memoizedState=[a,t],a},useReducer:function(l,t,e){var a=Wl();if(e!==void 0){var u=e(t);if(Ye){It(!0);try{e(t)}finally{It(!1)}}}else u=t;return a.memoizedState=a.baseState=u,l={pending:null,lanes:0,dispatch:null,lastRenderedReducer:l,lastRenderedState:u},a.queue=l,l=l.dispatch=tr.bind(null,J,l),[a.memoizedState,l]},useRef:function(l){var t=Wl();return l={current:l},t.memoizedState=l},useState:function(l){l=Ic(l);var t=l.queue,e=S0.bind(null,J,t);return t.dispatch=e,[l.memoizedState,e]},useDebugValue:ti,useDeferredValue:function(l,t){var e=Wl();return ei(e,l,t)},useTransition:function(){var l=Ic(!1);return l=r0.bind(null,J,l.queue,!0,!1),Wl().memoizedState=l,[!1,l]},useSyncExternalStore:function(l,t,e){var a=J,u=Wl();if(al){if(e===void 0)throw Error(o(407));e=e()}else{if(e=t(),Al===null)throw Error(o(349));(tl&127)!==0||Zs(a,t,e)}u.memoizedState=e;var n={value:e,getSnapshot:t};return u.queue=n,u0(Ls.bind(null,a,n,l),[l]),a.flags|=2048,ra(9,{destroy:void 0},Vs.bind(null,a,n,e,t),null),e},useId:function(){var l=Wl(),t=Al.identifierPrefix;if(al){var e=Dt,a=Nt;e=(a&~(1<<32-ct(a)-1)).toString(32)+e,t="_"+t+"R_"+e,e=ln++,0<\/script>",n=n.removeChild(n.firstChild);break;case"select":n=typeof a.is=="string"?c.createElement("select",{is:a.is}):c.createElement("select"),a.multiple?n.multiple=!0:a.size&&(n.size=a.size);break;default:n=typeof a.is=="string"?c.createElement(u,{is:a.is}):c.createElement(u)}}n[Ql]=t,n[kl]=a;l:for(c=t.child;c!==null;){if(c.tag===5||c.tag===6)n.appendChild(c.stateNode);else if(c.tag!==4&&c.tag!==27&&c.child!==null){c.child.return=c,c=c.child;continue}if(c===t)break l;for(;c.sibling===null;){if(c.return===null||c.return===t)break l;c=c.return}c.sibling.return=c.return,c=c.sibling}t.stateNode=n;l:switch(Ll(n,u,a),u){case"button":case"input":case"select":case"textarea":a=!!a.autoFocus;break l;case"img":a=!0;break l;default:a=!1}a&&Lt(t)}}return Tl(t),bi(t,t.type,l===null?null:l.memoizedProps,t.pendingProps,e),null;case 6:if(l&&t.stateNode!=null)l.memoizedProps!==a&&Lt(t);else{if(typeof a!="string"&&t.stateNode===null)throw Error(o(166));if(l=K.current,ua(t)){if(l=t.stateNode,e=t.memoizedProps,a=null,u=Xl,u!==null)switch(u.tag){case 27:case 5:a=u.memoizedProps}l[Ql]=t,l=!!(l.nodeValue===e||a!==null&&a.suppressHydrationWarning===!0||Qd(l.nodeValue,e)),l||ae(t,!0)}else l=Mn(l).createTextNode(a),l[Ql]=t,t.stateNode=l}return Tl(t),null;case 31:if(e=t.memoizedState,l===null||l.memoizedState!==null){if(a=ua(t),e!==null){if(l===null){if(!a)throw Error(o(318));if(l=t.memoizedState,l=l!==null?l.dehydrated:null,!l)throw Error(o(557));l[Ql]=t}else Ue(),(t.flags&128)===0&&(t.memoizedState=null),t.flags|=4;Tl(t),l=!1}else e=Nc(),l!==null&&l.memoizedState!==null&&(l.memoizedState.hydrationErrors=e),l=!0;if(!l)return t.flags&256?(dt(t),t):(dt(t),null);if((t.flags&128)!==0)throw Error(o(558))}return Tl(t),null;case 13:if(a=t.memoizedState,l===null||l.memoizedState!==null&&l.memoizedState.dehydrated!==null){if(u=ua(t),a!==null&&a.dehydrated!==null){if(l===null){if(!u)throw Error(o(318));if(u=t.memoizedState,u=u!==null?u.dehydrated:null,!u)throw Error(o(317));u[Ql]=t}else Ue(),(t.flags&128)===0&&(t.memoizedState=null),t.flags|=4;Tl(t),u=!1}else u=Nc(),l!==null&&l.memoizedState!==null&&(l.memoizedState.hydrationErrors=u),u=!0;if(!u)return t.flags&256?(dt(t),t):(dt(t),null)}return dt(t),(t.flags&128)!==0?(t.lanes=e,t):(e=a!==null,l=l!==null&&l.memoizedState!==null,e&&(a=t.child,u=null,a.alternate!==null&&a.alternate.memoizedState!==null&&a.alternate.memoizedState.cachePool!==null&&(u=a.alternate.memoizedState.cachePool.pool),n=null,a.memoizedState!==null&&a.memoizedState.cachePool!==null&&(n=a.memoizedState.cachePool.pool),n!==u&&(a.flags|=2048)),e!==l&&e&&(t.child.flags|=8192),mn(t,t.updateQueue),Tl(t),null);case 4:return Nl(),l===null&&Xi(t.stateNode.containerInfo),Tl(t),null;case 10:return Qt(t.type),Tl(t),null;case 19:if(g(Dl),a=t.memoizedState,a===null)return Tl(t),null;if(u=(t.flags&128)!==0,n=a.rendering,n===null)if(u)tu(a,!1);else{if(Ml!==0||l!==null&&(l.flags&128)!==0)for(l=t.child;l!==null;){if(n=Iu(l),n!==null){for(t.flags|=128,tu(a,!1),l=n.updateQueue,t.updateQueue=l,mn(t,l),t.subtreeFlags=0,l=e,e=t.child;e!==null;)Ss(e,l),e=e.sibling;return O(Dl,Dl.current&1|2),al&&Yt(t,a.treeForkCount),t.child}l=l.sibling}a.tail!==null&&ut()>gn&&(t.flags|=128,u=!0,tu(a,!1),t.lanes=4194304)}else{if(!u)if(l=Iu(n),l!==null){if(t.flags|=128,u=!0,l=l.updateQueue,t.updateQueue=l,mn(t,l),tu(a,!0),a.tail===null&&a.tailMode==="hidden"&&!n.alternate&&!al)return Tl(t),null}else 2*ut()-a.renderingStartTime>gn&&e!==536870912&&(t.flags|=128,u=!0,tu(a,!1),t.lanes=4194304);a.isBackwards?(n.sibling=t.child,t.child=n):(l=a.last,l!==null?l.sibling=n:t.child=n,a.last=n)}return a.tail!==null?(l=a.tail,a.rendering=l,a.tail=l.sibling,a.renderingStartTime=ut(),l.sibling=null,e=Dl.current,O(Dl,u?e&1|2:e&1),al&&Yt(t,a.treeForkCount),l):(Tl(t),null);case 22:case 23:return dt(t),Zc(),a=t.memoizedState!==null,l!==null?l.memoizedState!==null!==a&&(t.flags|=8192):a&&(t.flags|=8192),a?(e&536870912)!==0&&(t.flags&128)===0&&(Tl(t),t.subtreeFlags&6&&(t.flags|=8192)):Tl(t),e=t.updateQueue,e!==null&&mn(t,e.retryQueue),e=null,l!==null&&l.memoizedState!==null&&l.memoizedState.cachePool!==null&&(e=l.memoizedState.cachePool.pool),a=null,t.memoizedState!==null&&t.memoizedState.cachePool!==null&&(a=t.memoizedState.cachePool.pool),a!==e&&(t.flags|=2048),l!==null&&g(je),null;case 24:return e=null,l!==null&&(e=l.memoizedState.cache),t.memoizedState.cache!==e&&(t.flags|=2048),Qt(Cl),Tl(t),null;case 25:return null;case 30:return null}throw Error(o(156,t.tag))}function cr(l,t){switch(Mc(t),t.tag){case 1:return l=t.flags,l&65536?(t.flags=l&-65537|128,t):null;case 3:return Qt(Cl),Nl(),l=t.flags,(l&65536)!==0&&(l&128)===0?(t.flags=l&-65537|128,t):null;case 26:case 27:case 5:return pu(t),null;case 31:if(t.memoizedState!==null){if(dt(t),t.alternate===null)throw Error(o(340));Ue()}return l=t.flags,l&65536?(t.flags=l&-65537|128,t):null;case 13:if(dt(t),l=t.memoizedState,l!==null&&l.dehydrated!==null){if(t.alternate===null)throw Error(o(340));Ue()}return l=t.flags,l&65536?(t.flags=l&-65537|128,t):null;case 19:return g(Dl),null;case 4:return Nl(),null;case 10:return Qt(t.type),null;case 22:case 23:return dt(t),Zc(),l!==null&&g(je),l=t.flags,l&65536?(t.flags=l&-65537|128,t):null;case 24:return Qt(Cl),null;case 25:return null;default:return null}}function K0(l,t){switch(Mc(t),t.tag){case 3:Qt(Cl),Nl();break;case 26:case 27:case 5:pu(t);break;case 4:Nl();break;case 31:t.memoizedState!==null&&dt(t);break;case 13:dt(t);break;case 19:g(Dl);break;case 10:Qt(t.type);break;case 22:case 23:dt(t),Zc(),l!==null&&g(je);break;case 24:Qt(Cl)}}function eu(l,t){try{var e=t.updateQueue,a=e!==null?e.lastEffect:null;if(a!==null){var u=a.next;e=u;do{if((e.tag&l)===l){a=void 0;var n=e.create,c=e.inst;a=n(),c.destroy=a}e=e.next}while(e!==u)}}catch(i){vl(t,t.return,i)}}function de(l,t,e){try{var a=t.updateQueue,u=a!==null?a.lastEffect:null;if(u!==null){var n=u.next;a=n;do{if((a.tag&l)===l){var c=a.inst,i=c.destroy;if(i!==void 0){c.destroy=void 0,u=t;var f=e,y=i;try{y()}catch(b){vl(u,f,b)}}}a=a.next}while(a!==n)}}catch(b){vl(t,t.return,b)}}function J0(l){var t=l.updateQueue;if(t!==null){var e=l.stateNode;try{Bs(t,e)}catch(a){vl(l,l.return,a)}}}function w0(l,t,e){e.props=Ge(l.type,l.memoizedProps),e.state=l.memoizedState;try{e.componentWillUnmount()}catch(a){vl(l,t,a)}}function au(l,t){try{var e=l.ref;if(e!==null){switch(l.tag){case 26:case 27:case 5:var a=l.stateNode;break;case 30:a=l.stateNode;break;default:a=l.stateNode}typeof e=="function"?l.refCleanup=e(a):e.current=a}}catch(u){vl(l,t,u)}}function Ut(l,t){var e=l.ref,a=l.refCleanup;if(e!==null)if(typeof a=="function")try{a()}catch(u){vl(l,t,u)}finally{l.refCleanup=null,l=l.alternate,l!=null&&(l.refCleanup=null)}else if(typeof e=="function")try{e(null)}catch(u){vl(l,t,u)}else e.current=null}function W0(l){var t=l.type,e=l.memoizedProps,a=l.stateNode;try{l:switch(t){case"button":case"input":case"select":case"textarea":e.autoFocus&&a.focus();break l;case"img":e.src?a.src=e.src:e.srcSet&&(a.srcset=e.srcSet)}}catch(u){vl(l,l.return,u)}}function zi(l,t,e){try{var a=l.stateNode;xr(a,l.type,e,t),a[kl]=t}catch(u){vl(l,l.return,u)}}function $0(l){return l.tag===5||l.tag===3||l.tag===26||l.tag===27&&ge(l.type)||l.tag===4}function Ai(l){l:for(;;){for(;l.sibling===null;){if(l.return===null||$0(l.return))return null;l=l.return}for(l.sibling.return=l.return,l=l.sibling;l.tag!==5&&l.tag!==6&&l.tag!==18;){if(l.tag===27&&ge(l.type)||l.flags&2||l.child===null||l.tag===4)continue l;l.child.return=l,l=l.child}if(!(l.flags&2))return l.stateNode}}function pi(l,t,e){var a=l.tag;if(a===5||a===6)l=l.stateNode,t?(e.nodeType===9?e.body:e.nodeName==="HTML"?e.ownerDocument.body:e).insertBefore(l,t):(t=e.nodeType===9?e.body:e.nodeName==="HTML"?e.ownerDocument.body:e,t.appendChild(l),e=e._reactRootContainer,e!=null||t.onclick!==null||(t.onclick=Ht));else if(a!==4&&(a===27&&ge(l.type)&&(e=l.stateNode,t=null),l=l.child,l!==null))for(pi(l,t,e),l=l.sibling;l!==null;)pi(l,t,e),l=l.sibling}function rn(l,t,e){var a=l.tag;if(a===5||a===6)l=l.stateNode,t?e.insertBefore(l,t):e.appendChild(l);else if(a!==4&&(a===27&&ge(l.type)&&(e=l.stateNode),l=l.child,l!==null))for(rn(l,t,e),l=l.sibling;l!==null;)rn(l,t,e),l=l.sibling}function k0(l){var t=l.stateNode,e=l.memoizedProps;try{for(var a=l.type,u=t.attributes;u.length;)t.removeAttributeNode(u[0]);Ll(t,a,e),t[Ql]=l,t[kl]=e}catch(n){vl(l,l.return,n)}}var Kt=!1,Hl=!1,Ei=!1,F0=typeof WeakSet=="function"?WeakSet:Set,Gl=null;function ir(l,t){if(l=l.containerInfo,Li=jn,l=ss(l),vc(l)){if("selectionStart"in l)var e={start:l.selectionStart,end:l.selectionEnd};else l:{e=(e=l.ownerDocument)&&e.defaultView||window;var a=e.getSelection&&e.getSelection();if(a&&a.rangeCount!==0){e=a.anchorNode;var u=a.anchorOffset,n=a.focusNode;a=a.focusOffset;try{e.nodeType,n.nodeType}catch{e=null;break l}var c=0,i=-1,f=-1,y=0,b=0,p=l,v=null;t:for(;;){for(var S;p!==e||u!==0&&p.nodeType!==3||(i=c+u),p!==n||a!==0&&p.nodeType!==3||(f=c+a),p.nodeType===3&&(c+=p.nodeValue.length),(S=p.firstChild)!==null;)v=p,p=S;for(;;){if(p===l)break t;if(v===e&&++y===u&&(i=c),v===n&&++b===a&&(f=c),(S=p.nextSibling)!==null)break;p=v,v=p.parentNode}p=S}e=i===-1||f===-1?null:{start:i,end:f}}else e=null}e=e||{start:0,end:0}}else e=null;for(Ki={focusedElem:l,selectionRange:e},jn=!1,Gl=t;Gl!==null;)if(t=Gl,l=t.child,(t.subtreeFlags&1028)!==0&&l!==null)l.return=t,Gl=l;else for(;Gl!==null;){switch(t=Gl,n=t.alternate,l=t.flags,t.tag){case 0:if((l&4)!==0&&(l=t.updateQueue,l=l!==null?l.events:null,l!==null))for(e=0;e title"))),Ll(n,a,e),n[Ql]=l,Yl(n),a=n;break l;case"link":var c=ao("link","href",u).get(a+(e.href||""));if(c){for(var i=0;izl&&(c=zl,zl=Z,Z=c);var m=is(i,Z),s=is(i,zl);if(m&&s&&(S.rangeCount!==1||S.anchorNode!==m.node||S.anchorOffset!==m.offset||S.focusNode!==s.node||S.focusOffset!==s.offset)){var r=p.createRange();r.setStart(m.node,m.offset),S.removeAllRanges(),Z>zl?(S.addRange(r),S.extend(s.node,s.offset)):(r.setEnd(s.node,s.offset),S.addRange(r))}}}}for(p=[],S=i;S=S.parentNode;)S.nodeType===1&&p.push({element:S,left:S.scrollLeft,top:S.scrollTop});for(typeof i.focus=="function"&&i.focus(),i=0;ie?32:e,z.T=null,e=Di,Di=null;var n=ye,c=kt;if(ql=0,Sa=ye=null,kt=0,(fl&6)!==0)throw Error(o(331));var i=fl;if(fl|=4,fd(n.current),nd(n,n.current,c,e),fl=i,su(0,!1),nt&&typeof nt.onPostCommitFiberRoot=="function")try{nt.onPostCommitFiberRoot(Ma,n)}catch{}return!0}finally{N.p=u,z.T=a,Od(l,t)}}function xd(l,t,e){t=gt(e,t),t=si(l.stateNode,t,2),l=ie(l,t,2),l!==null&&(Na(l,2),Ct(l))}function vl(l,t,e){if(l.tag===3)xd(l,l,e);else for(;t!==null;){if(t.tag===3){xd(t,l,e);break}else if(t.tag===1){var a=t.stateNode;if(typeof t.type.getDerivedStateFromError=="function"||typeof a.componentDidCatch=="function"&&(re===null||!re.has(a))){l=gt(e,l),e=N0(2),a=ie(t,e,2),a!==null&&(D0(e,a,t,l),Na(a,2),Ct(a));break}}t=t.return}}function ji(l,t,e){var a=l.pingCache;if(a===null){a=l.pingCache=new dr;var u=new Set;a.set(t,u)}else u=a.get(t),u===void 0&&(u=new Set,a.set(t,u));u.has(e)||(Oi=!0,u.add(e),l=hr.bind(null,l,t,e),t.then(l,l))}function hr(l,t,e){var a=l.pingCache;a!==null&&a.delete(t),l.pingedLanes|=l.suspendedLanes&e,l.warmLanes&=~e,Al===l&&(tl&e)===e&&(Ml===4||Ml===3&&(tl&62914560)===tl&&300>ut()-vn?(fl&2)===0&&ba(l,0):Mi|=e,ga===tl&&(ga=0)),Ct(l)}function Nd(l,t){t===0&&(t=Ef()),l=Ne(l,t),l!==null&&(Na(l,t),Ct(l))}function vr(l){var t=l.memoizedState,e=0;t!==null&&(e=t.retryLane),Nd(l,e)}function gr(l,t){var e=0;switch(l.tag){case 31:case 13:var a=l.stateNode,u=l.memoizedState;u!==null&&(e=u.retryLane);break;case 19:a=l.stateNode;break;case 22:a=l.stateNode._retryCache;break;default:throw Error(o(314))}a!==null&&a.delete(t),Nd(l,e)}function Sr(l,t){return wn(l,t)}var En=null,Aa=null,Hi=!1,Tn=!1,Bi=!1,ve=0;function Ct(l){l!==Aa&&l.next===null&&(Aa===null?En=Aa=l:Aa=Aa.next=l),Tn=!0,Hi||(Hi=!0,zr())}function su(l,t){if(!Bi&&Tn){Bi=!0;do for(var e=!1,a=En;a!==null;){if(l!==0){var u=a.pendingLanes;if(u===0)var n=0;else{var c=a.suspendedLanes,i=a.pingedLanes;n=(1<<31-ct(42|l)+1)-1,n&=u&~(c&~i),n=n&201326741?n&201326741|1:n?n|2:0}n!==0&&(e=!0,Rd(a,n))}else n=tl,n=Mu(a,a===Al?n:0,a.cancelPendingCommit!==null||a.timeoutHandle!==-1),(n&3)===0||xa(a,n)||(e=!0,Rd(a,n));a=a.next}while(e);Bi=!1}}function br(){Dd()}function Dd(){Tn=Hi=!1;var l=0;ve!==0&&Dr()&&(l=ve);for(var t=ut(),e=null,a=En;a!==null;){var u=a.next,n=Ud(a,t);n===0?(a.next=null,e===null?En=u:e.next=u,u===null&&(Aa=e)):(e=a,(l!==0||(n&3)!==0)&&(Tn=!0)),a=u}ql!==0&&ql!==5||su(l),ve!==0&&(ve=0)}function Ud(l,t){for(var e=l.suspendedLanes,a=l.pingedLanes,u=l.expirationTimes,n=l.pendingLanes&-62914561;0i)break;var b=f.transferSize,p=f.initiatorType;b&&Xd(p)&&(f=f.responseEnd,c+=b*(f"u"?null:document;function Pd(l,t,e){var a=pa;if(a&&typeof t=="string"&&t){var u=ht(t);u='link[rel="'+l+'"][href="'+u+'"]',typeof e=="string"&&(u+='[crossorigin="'+e+'"]'),Id.has(u)||(Id.add(u),l={rel:l,crossOrigin:e,href:t},a.querySelector(u)===null&&(t=a.createElement("link"),Ll(t,"link",l),Yl(t),a.head.appendChild(t)))}}function Gr(l){Ft.D(l),Pd("dns-prefetch",l,null)}function Qr(l,t){Ft.C(l,t),Pd("preconnect",l,t)}function Xr(l,t,e){Ft.L(l,t,e);var a=pa;if(a&&l&&t){var u='link[rel="preload"][as="'+ht(t)+'"]';t==="image"&&e&&e.imageSrcSet?(u+='[imagesrcset="'+ht(e.imageSrcSet)+'"]',typeof e.imageSizes=="string"&&(u+='[imagesizes="'+ht(e.imageSizes)+'"]')):u+='[href="'+ht(l)+'"]';var n=u;switch(t){case"style":n=Ea(l);break;case"script":n=Ta(l)}Et.has(n)||(l=U({rel:"preload",href:t==="image"&&e&&e.imageSrcSet?void 0:l,as:t},e),Et.set(n,l),a.querySelector(u)!==null||t==="style"&&a.querySelector(ru(n))||t==="script"&&a.querySelector(yu(n))||(t=a.createElement("link"),Ll(t,"link",l),Yl(t),a.head.appendChild(t)))}}function Zr(l,t){Ft.m(l,t);var e=pa;if(e&&l){var a=t&&typeof t.as=="string"?t.as:"script",u='link[rel="modulepreload"][as="'+ht(a)+'"][href="'+ht(l)+'"]',n=u;switch(a){case"audioworklet":case"paintworklet":case"serviceworker":case"sharedworker":case"worker":case"script":n=Ta(l)}if(!Et.has(n)&&(l=U({rel:"modulepreload",href:l},t),Et.set(n,l),e.querySelector(u)===null)){switch(a){case"audioworklet":case"paintworklet":case"serviceworker":case"sharedworker":case"worker":case"script":if(e.querySelector(yu(n)))return}a=e.createElement("link"),Ll(a,"link",l),Yl(a),e.head.appendChild(a)}}}function Vr(l,t,e){Ft.S(l,t,e);var a=pa;if(a&&l){var u=Ke(a).hoistableStyles,n=Ea(l);t=t||"default";var c=u.get(n);if(!c){var i={loading:0,preload:null};if(c=a.querySelector(ru(n)))i.loading=5;else{l=U({rel:"stylesheet",href:l,"data-precedence":t},e),(e=Et.get(n))&&Ii(l,e);var f=c=a.createElement("link");Yl(f),Ll(f,"link",l),f._p=new Promise(function(y,b){f.onload=y,f.onerror=b}),f.addEventListener("load",function(){i.loading|=1}),f.addEventListener("error",function(){i.loading|=2}),i.loading|=4,Nn(c,t,a)}c={type:"stylesheet",instance:c,count:1,state:i},u.set(n,c)}}}function Lr(l,t){Ft.X(l,t);var e=pa;if(e&&l){var a=Ke(e).hoistableScripts,u=Ta(l),n=a.get(u);n||(n=e.querySelector(yu(u)),n||(l=U({src:l,async:!0},t),(t=Et.get(u))&&Pi(l,t),n=e.createElement("script"),Yl(n),Ll(n,"link",l),e.head.appendChild(n)),n={type:"script",instance:n,count:1,state:null},a.set(u,n))}}function Kr(l,t){Ft.M(l,t);var e=pa;if(e&&l){var a=Ke(e).hoistableScripts,u=Ta(l),n=a.get(u);n||(n=e.querySelector(yu(u)),n||(l=U({src:l,async:!0,type:"module"},t),(t=Et.get(u))&&Pi(l,t),n=e.createElement("script"),Yl(n),Ll(n,"link",l),e.head.appendChild(n)),n={type:"script",instance:n,count:1,state:null},a.set(u,n))}}function lo(l,t,e,a){var u=(u=K.current)?xn(u):null;if(!u)throw Error(o(446));switch(l){case"meta":case"title":return null;case"style":return typeof e.precedence=="string"&&typeof e.href=="string"?(t=Ea(e.href),e=Ke(u).hoistableStyles,a=e.get(t),a||(a={type:"style",instance:null,count:0,state:null},e.set(t,a)),a):{type:"void",instance:null,count:0,state:null};case"link":if(e.rel==="stylesheet"&&typeof e.href=="string"&&typeof e.precedence=="string"){l=Ea(e.href);var n=Ke(u).hoistableStyles,c=n.get(l);if(c||(u=u.ownerDocument||u,c={type:"stylesheet",instance:null,count:0,state:{loading:0,preload:null}},n.set(l,c),(n=u.querySelector(ru(l)))&&!n._p&&(c.instance=n,c.state.loading=5),Et.has(l)||(e={rel:"preload",as:"style",href:e.href,crossOrigin:e.crossOrigin,integrity:e.integrity,media:e.media,hrefLang:e.hrefLang,referrerPolicy:e.referrerPolicy},Et.set(l,e),n||Jr(u,l,e,c.state))),t&&a===null)throw Error(o(528,""));return c}if(t&&a!==null)throw Error(o(529,""));return null;case"script":return t=e.async,e=e.src,typeof e=="string"&&t&&typeof t!="function"&&typeof t!="symbol"?(t=Ta(e),e=Ke(u).hoistableScripts,a=e.get(t),a||(a={type:"script",instance:null,count:0,state:null},e.set(t,a)),a):{type:"void",instance:null,count:0,state:null};default:throw Error(o(444,l))}}function Ea(l){return'href="'+ht(l)+'"'}function ru(l){return'link[rel="stylesheet"]['+l+"]"}function to(l){return U({},l,{"data-precedence":l.precedence,precedence:null})}function Jr(l,t,e,a){l.querySelector('link[rel="preload"][as="style"]['+t+"]")?a.loading=1:(t=l.createElement("link"),a.preload=t,t.addEventListener("load",function(){return a.loading|=1}),t.addEventListener("error",function(){return a.loading|=2}),Ll(t,"link",e),Yl(t),l.head.appendChild(t))}function Ta(l){return'[src="'+ht(l)+'"]'}function yu(l){return"script[async]"+l}function eo(l,t,e){if(t.count++,t.instance===null)switch(t.type){case"style":var a=l.querySelector('style[data-href~="'+ht(e.href)+'"]');if(a)return t.instance=a,Yl(a),a;var u=U({},e,{"data-href":e.href,"data-precedence":e.precedence,href:null,precedence:null});return a=(l.ownerDocument||l).createElement("style"),Yl(a),Ll(a,"style",u),Nn(a,e.precedence,l),t.instance=a;case"stylesheet":u=Ea(e.href);var n=l.querySelector(ru(u));if(n)return t.state.loading|=4,t.instance=n,Yl(n),n;a=to(e),(u=Et.get(u))&&Ii(a,u),n=(l.ownerDocument||l).createElement("link"),Yl(n);var c=n;return c._p=new Promise(function(i,f){c.onload=i,c.onerror=f}),Ll(n,"link",a),t.state.loading|=4,Nn(n,e.precedence,l),t.instance=n;case"script":return n=Ta(e.src),(u=l.querySelector(yu(n)))?(t.instance=u,Yl(u),u):(a=e,(u=Et.get(n))&&(a=U({},e),Pi(a,u)),l=l.ownerDocument||l,u=l.createElement("script"),Yl(u),Ll(u,"link",a),l.head.appendChild(u),t.instance=u);case"void":return null;default:throw Error(o(443,t.type))}else t.type==="stylesheet"&&(t.state.loading&4)===0&&(a=t.instance,t.state.loading|=4,Nn(a,e.precedence,l));return t.instance}function Nn(l,t,e){for(var a=e.querySelectorAll('link[rel="stylesheet"][data-precedence],style[data-precedence]'),u=a.length?a[a.length-1]:null,n=u,c=0;c title"):null)}function wr(l,t,e){if(e===1||t.itemProp!=null)return!1;switch(l){case"meta":case"title":return!0;case"style":if(typeof t.precedence!="string"||typeof t.href!="string"||t.href==="")break;return!0;case"link":if(typeof t.rel!="string"||typeof t.href!="string"||t.href===""||t.onLoad||t.onError)break;return t.rel==="stylesheet"?(l=t.disabled,typeof t.precedence=="string"&&l==null):!0;case"script":if(t.async&&typeof t.async!="function"&&typeof t.async!="symbol"&&!t.onLoad&&!t.onError&&t.src&&typeof t.src=="string")return!0}return!1}function no(l){return!(l.type==="stylesheet"&&(l.state.loading&3)===0)}function Wr(l,t,e,a){if(e.type==="stylesheet"&&(typeof a.media!="string"||matchMedia(a.media).matches!==!1)&&(e.state.loading&4)===0){if(e.instance===null){var u=Ea(a.href),n=t.querySelector(ru(u));if(n){t=n._p,t!==null&&typeof t=="object"&&typeof t.then=="function"&&(l.count++,l=Un.bind(l),t.then(l,l)),e.state.loading|=4,e.instance=n,Yl(n);return}n=t.ownerDocument||t,a=to(a),(u=Et.get(u))&&Ii(a,u),n=n.createElement("link"),Yl(n);var c=n;c._p=new Promise(function(i,f){c.onload=i,c.onerror=f}),Ll(n,"link",a),e.instance=n}l.stylesheets===null&&(l.stylesheets=new Map),l.stylesheets.set(e,t),(t=e.state.preload)&&(e.state.loading&3)===0&&(l.count++,e=Un.bind(l),t.addEventListener("load",e),t.addEventListener("error",e))}}var lf=0;function $r(l,t){return l.stylesheets&&l.count===0&&Rn(l,l.stylesheets),0lf?50:800)+t);return l.unsuspend=e,function(){l.unsuspend=null,clearTimeout(a),clearTimeout(u)}}:null}function Un(){if(this.count--,this.count===0&&(this.imgCount===0||!this.waitingForImages)){if(this.stylesheets)Rn(this,this.stylesheets);else if(this.unsuspend){var l=this.unsuspend;this.unsuspend=null,l()}}}var Cn=null;function Rn(l,t){l.stylesheets=null,l.unsuspend!==null&&(l.count++,Cn=new Map,t.forEach(kr,l),Cn=null,Un.call(l))}function kr(l,t){if(!(t.state.loading&4)){var e=Cn.get(l);if(e)var a=e.get(null);else{e=new Map,Cn.set(l,e);for(var u=l.querySelectorAll("link[data-precedence],style[data-precedence]"),n=0;n"u"||typeof __REACT_DEVTOOLS_GLOBAL_HOOK__.checkDCE!="function"))try{__REACT_DEVTOOLS_GLOBAL_HOOK__.checkDCE(h)}catch(M){console.error(M)}}return h(),df.exports=my(),df.exports}var yy=ry();const hy=[{id:"all",label:"All"},{id:"lambda",label:"Lambda"},{id:"stateful",label:"Stateful"}],vy=[{id:"all",label:"All"},{id:"og18",label:"OG-18"},{id:"advanced_reasoning",label:"Advanced Reasoning"}],gf=["backend","mode","family","quant","replay"],gy=[{id:"reforged",label:"Reforged"},{id:"bare-vs-reforged",label:"Reforged vs Bare"},{id:"ablation",label:"Full Ablation"}],Do=["reforged","bare","no_rescue","no_nudge","no_steps","no_recovery","no_compact"],Uo=["none","keep-last","full"],hf=[{id:"all",label:"All",groupBy:[]},{id:"by-backend",label:"By Backend",groupBy:["model","quant","ablation","replay"],intraSort:"backend"},{id:"by-family",label:"By Family",groupBy:["family"]}];async function Sy(){const h=window;if(!h.__FORGE_DATA__)throw new Error("window.__FORGE_DATA__ not injected — build via `python -m tests.eval.report --html `");return h.__FORGE_DATA__}function Co(h,M){if(M==="reforged")return h.filter(o=>o.ablation==="reforged");if(M==="bare-vs-reforged")return h.filter(o=>o.ablation==="reforged"||o.ablation==="bare");const C=new Set;for(const o of h)o.ablation.startsWith("no_")&&C.add(`${o.model}\0${o.backend}\0${o.mode}`);return h.filter(o=>C.has(`${o.model}\0${o.backend}\0${o.mode}`))}function Ro(h){const M=Do.indexOf(h);return M===-1?Do.length:M}function Zn(h){const M=Uo.indexOf(h);return M===-1?Uo.length:M}function Au(h){return h==null?"":h>=95?"text-emerald-400":h>=90?"text-emerald-500/80":h>=70?"text-amber-400":h>=50?"text-orange-400":"text-red-400"}function Xn(h,M=0){return h==null?"—":`${h.toFixed(M)}%`}const by={0:"⁰",1:"¹",2:"²",3:"³",4:"⁴",5:"⁵",6:"⁶",7:"⁷",8:"⁸",9:"⁹"};function zy(h){return String(h).split("").map(M=>by[M]??M).join("")}function Ay(h,M,C,o,q){const G=T=>T.endsWith("_stateful");let H=M;return C==="lambda"?H=H.filter(T=>!G(T)):C==="stateful"&&(H=H.filter(G)),o==="og18"?H=H.filter(T=>q[T]==="og18"):o==="advanced_reasoning"&&(H=H.filter(T=>q[T]==="advanced_reasoning")),H.length===0?{rows:h,scenarios:M}:H.length===M.length?{rows:h,scenarios:M}:{rows:h.map(T=>{if(!(T.scenarioAttempted!==void 0&&T.scenarioCorrect!==void 0&&T.scenarioCompleted!==void 0&&T.scenarioValidated!==void 0&&T.scenarioIdealCalls!==void 0&&T.scenarioActualCalls!==void 0&&T.scenarioWastedSum!==void 0&&T.scenarioWastedN!==void 0&&T.scenarioSpeedSum!==void 0&&T.scenarioSpeedN!==void 0))return T;let Y=0,U=0,x=0,$=0,ll=0,rl=0,ul=0,gl=0,k=0,sl=0;for(const il of H)Y+=T.scenarioAttempted?.[il]??0,U+=T.scenarioCorrect?.[il]??0,x+=T.scenarioCompleted?.[il]??0,$+=T.scenarioValidated?.[il]??0,ll+=T.scenarioIdealCalls?.[il]??0,rl+=T.scenarioActualCalls?.[il]??0,ul+=T.scenarioWastedSum?.[il]??0,gl+=T.scenarioWastedN?.[il]??0,k+=T.scenarioSpeedSum?.[il]??0,sl+=T.scenarioSpeedN?.[il]??0;const F=il=>Math.round(il*10)/10,xl=Y>0?F(U/Y*100):0,pl=$>0?F(U/$*100):null,W=Y>0?F(x/Y*100):0,Bl=rl>0?F(Math.min(ll/rl,1)*100):0,Kl=gl>0?F(ul/gl):0,xt=sl>0?F(k/sl):0,$l=Math.max(0,...H.map(il=>T.scenarioAttempted?.[il]??0));return{...T,score:xl,validatedAccuracy:pl,completionRate:W,attemptedCount:Y,correctCount:U,validatedCount:$,completedCount:x,efficiency:Bl,wasted:Kl,speed:xt,n:$l}}),scenarios:H}}function py(h,M,C){const o=[...h];return o.sort((q,G)=>{let H,R;return M.col==="label"?(H=q.label,R=G.label):C.includes(M.col)?(H=q.scenarios[M.col]??-1,R=G.scenarios[M.col]??-1):(H=q[M.col]??-1,R=G[M.col]??-1),typeof H=="string"&&typeof R=="string"?M.asc?H.localeCompare(R):R.localeCompare(H):M.asc?H-R:R-H}),o}function Ey(h,M){return M.map(C=>String(h[C])).join("\0")}function Ty(h,M,C,o,q){const G=q==="reforged"?M:{id:M.id,label:M.label,groupBy:["model","backend","mode"]},H=q!=="reforged";if(G.groupBy.length===0)return{sorted:py(h,C,o),groups:[]};const R=new Map;for(const Y of h){const U=Ey(Y,G.groupBy);R.has(U)||R.set(U,[]),R.get(U).push(Y)}const T=[];for(const[Y,U]of R){U.sort(($,ll)=>{if(H){const ul=Ro($.ablation)-Ro(ll.ablation);if(ul!==0)return ul;const gl=Zn($.replay)-Zn(ll.replay);return gl!==0?gl:ll.score-$.score}const rl=ll.score-$.score;return rl!==0?rl:G.intraSort?String($[G.intraSort]).localeCompare(String(ll[G.intraSort])):0});const x=G.groupBy.map($=>U[0][$]).join(" / ");T.push({key:Y,label:x,rows:U})}return T.sort((Y,U)=>{const x=Math.max(...Y.rows.map(ll=>ll.score));return Math.max(...U.rows.map(ll=>ll.score))-x}),{sorted:T.flatMap(Y=>Y.rows),groups:T}}function _y({active:h,onChange:M}){return _.jsxs("fieldset",{className:"mb-4",children:[_.jsx("legend",{className:"text-[0.65rem] font-semibold uppercase tracking-wider text-zinc-400 px-1 mb-1",children:"Screen"}),_.jsx("div",{className:"flex flex-col rounded border border-zinc-700 overflow-hidden",children:gy.map((C,o)=>_.jsx("button",{onClick:()=>M(C.id),className:`text-xs px-2 py-1.5 text-left transition-colors ${o>0?"border-t border-zinc-700":""} ${h===C.id?"bg-emerald-500/20 text-emerald-300 font-medium":"bg-zinc-900/40 text-zinc-400 hover:bg-zinc-900/70 hover:text-zinc-200"}`,children:C.label},C.id))})]})}function Oy({active:h,onChange:M}){return _.jsxs("fieldset",{className:"mb-3 border border-zinc-800 rounded p-2",children:[_.jsx("legend",{className:"text-[0.65rem] font-semibold uppercase tracking-wider text-zinc-400 px-1",children:"View"}),_.jsx("div",{className:"flex flex-wrap gap-1",children:hf.map(C=>_.jsx("button",{onClick:()=>M(C.id),className:`text-[0.65rem] px-2 py-0.5 rounded-full border transition-colors ${h===C.id?"border-emerald-500 bg-emerald-500/15 text-emerald-400":"border-zinc-700 text-zinc-500 hover:border-zinc-500 hover:text-zinc-300"}`,children:C.label},C.id))})]})}const My={backend:"Backend",mode:"Mode",family:"Family",quant:"Quant",replay:"Reasoning Replay"};function xy({rows:h,filters:M,onFilterChange:C,activeScreen:o,onScreenChange:q,activeView:G,onViewChange:H,scenarioScope:R,onScopeChange:T,suiteScope:E,onSuiteChange:Y,showRetired:U,onShowRetiredChange:x,hasRetired:$,filteredCount:ll,totalCount:rl,totalAttempts:ul,timestamp:gl}){return _.jsxs("nav",{className:"w-52 min-w-52 shrink-0 border-r border-zinc-800 p-4 sticky top-0 h-screen overflow-y-auto bg-zinc-950/80",children:[_.jsx("h1",{className:"text-lg font-semibold mb-0.5",children:"Forge Eval"}),_.jsxs("p",{className:"text-xs text-zinc-500 mb-3",children:[ll,"/",rl," configs ·"," ",ul.toLocaleString()," attempts"]}),_.jsx(_y,{active:o,onChange:q}),_.jsxs("fieldset",{className:"mb-3 border border-zinc-800 rounded p-2",children:[_.jsx("legend",{className:"text-[0.65rem] font-semibold uppercase tracking-wider text-zinc-400 px-1",children:"Suite"}),_.jsx("div",{className:"flex flex-wrap gap-1",children:vy.map(k=>_.jsx("button",{onClick:()=>Y(k.id),className:`text-[0.65rem] px-2 py-0.5 rounded-full border transition-colors ${E===k.id?"border-emerald-500 bg-emerald-500/15 text-emerald-400":"border-zinc-700 text-zinc-500 hover:border-zinc-500 hover:text-zinc-300"}`,children:k.label},k.id))})]}),_.jsxs("fieldset",{className:"mb-3 border border-zinc-800 rounded p-2",children:[_.jsx("legend",{className:"text-[0.65rem] font-semibold uppercase tracking-wider text-zinc-400 px-1",children:"Scenarios"}),_.jsx("div",{className:"flex flex-wrap gap-1",children:hy.map(k=>_.jsx("button",{onClick:()=>T(k.id),className:`text-[0.65rem] px-2 py-0.5 rounded-full border transition-colors ${R===k.id?"border-emerald-500 bg-emerald-500/15 text-emerald-400":"border-zinc-700 text-zinc-500 hover:border-zinc-500 hover:text-zinc-300"}`,children:k.label},k.id))})]}),o==="reforged"&&_.jsx(Oy,{active:G,onChange:H}),gf.map(k=>{const sl=[...new Set(h.map(F=>F[k]))].sort(k==="replay"?(F,xl)=>Zn(F)-Zn(xl):void 0);return sl.length<2?null:_.jsxs("fieldset",{className:"mb-3 border border-zinc-800 rounded p-2",children:[_.jsx("legend",{className:"text-[0.65rem] font-semibold uppercase tracking-wider text-zinc-400 px-1",children:My[k]}),sl.map(F=>_.jsxs("label",{className:"flex items-center gap-1.5 text-xs py-0.5 cursor-pointer hover:text-zinc-200",children:[_.jsx("input",{type:"checkbox",checked:M[k]?.has(F)??!0,onChange:xl=>C(k,F,xl.target.checked),className:"w-3.5 h-3.5 rounded border-zinc-600 bg-zinc-800 accent-emerald-500"}),_.jsx("span",{children:F})]},F))]},k)}),$&&_.jsxs("label",{className:"flex items-center gap-1.5 text-xs py-0.5 mt-2 cursor-pointer text-zinc-500 hover:text-zinc-300",children:[_.jsx("input",{type:"checkbox",checked:U,onChange:k=>x(k.target.checked),className:"w-3.5 h-3.5 rounded border-zinc-600 bg-zinc-800 accent-emerald-500"}),_.jsx("span",{children:"Show retired models"})]}),_.jsx("p",{className:"text-[0.6rem] text-zinc-600 mt-4",children:gl})]})}const jo=[{key:"score",label:"Score%",title:"Score: correct / attempted"},{key:"validatedAccuracy",label:"VAcc%",title:"Validated accuracy: correct / validated"},{key:"completionRate",label:"Comp%",title:"Completion rate: completed / attempted"},{key:"efficiency",label:"Eff%",title:"Efficiency: ideal / actual calls on completed correct runs"},{key:"wasted",label:"Wst",title:"Average wasted calls on completed runs"},{key:"speed",label:"Spd",title:"Average elapsed seconds, excluding timing-outlier scenarios"},{key:"n",label:"N",title:"Maximum recorded attempts per included scenario"}];function yf({col:h,sort:M}){return M.col!==h?null:_.jsx("span",{className:"ml-0.5 text-emerald-400",children:M.asc?"▲":"▼"})}function Ny({rows:h,scenarios:M,scenarioAbbrev:C,sort:o,onSort:q,checked:G,onCompareToggle:H,groups:R,maxGen:T,genInfo:E}){const Y=new Map;if(R.length>0){let x=0;for(const $ of R)Y.set(x,$.label),x+=$.rows.length}const U=2+jo.length+M.length;return _.jsx("div",{className:"w-full overflow-x-auto",children:_.jsxs("table",{className:"text-xs whitespace-nowrap border-collapse",children:[_.jsx("thead",{children:_.jsxs("tr",{className:"border-b border-zinc-800",children:[_.jsx("th",{className:"p-1.5 w-8"}),_.jsxs("th",{className:"p-1.5 text-left cursor-pointer select-none hover:text-emerald-400 sticky left-0 bg-zinc-950 z-10",onClick:()=>q("label"),children:["Model/Backend",_.jsx(yf,{col:"label",sort:o})]}),jo.map(x=>_.jsxs("th",{className:"p-1.5 text-right cursor-pointer select-none hover:text-emerald-400",onClick:()=>q(x.key),title:x.title,children:[x.label,_.jsx(yf,{col:x.key,sort:o})]},x.key)),M.map(x=>_.jsxs("th",{className:"p-1.5 text-right cursor-pointer select-none hover:text-emerald-400",onClick:()=>q(x),title:x,children:[C[x]||x.slice(0,3),_.jsx(yf,{col:x,sort:o})]},x))]})}),_.jsx("tbody",{children:h.map((x,$)=>{const ll=G.includes($),rl=Y.get($);return _.jsxs(ml.Fragment,{children:[rl!=null&&_.jsx("tr",{className:"bg-zinc-900/30",children:_.jsx("td",{colSpan:U,className:"px-2 py-1 text-[0.6rem] font-semibold text-zinc-400 uppercase tracking-wider border-t border-zinc-700/50",children:rl})}),_.jsxs("tr",{className:`border-b border-zinc-900 hover:bg-zinc-900/50 transition-colors ${ll?"bg-zinc-800/40":""} ${x.retired?"opacity-60":""}`,children:[_.jsx("td",{className:"p-1.5 text-center",children:_.jsx("input",{type:"checkbox",checked:ll,onChange:ul=>H($,ul.target.checked),className:"w-3.5 h-3.5 rounded border-zinc-600 bg-zinc-800 accent-emerald-500 cursor-pointer"})}),_.jsxs("td",{className:"p-1.5 font-mono sticky left-0 bg-zinc-950 z-10",children:[x.label,T>0&&x.gen{const ul=E?.[String(x.gen)];return ul?`gen ${x.gen}: ${ul.note} (commit ${ul.commit}, ${ul.date})`:`gen ${x.gen}`})(),children:zy(x.gen)}),x.retired&&_.jsx("span",{className:"ml-1.5 align-middle text-[0.55rem] uppercase tracking-wider text-zinc-500 border border-zinc-700 rounded px-1",children:"retired"})]}),_.jsx("td",{className:`p-1.5 text-right tabular-nums ${Au(x.score)}`,children:Xn(x.score,1)}),_.jsx("td",{className:`p-1.5 text-right tabular-nums ${Au(x.validatedAccuracy)}`,children:Xn(x.validatedAccuracy,1)}),_.jsx("td",{className:`p-1.5 text-right tabular-nums ${Au(x.completionRate)}`,children:Xn(x.completionRate,1)}),_.jsx("td",{className:`p-1.5 text-right tabular-nums ${Au(x.efficiency)}`,children:Xn(x.efficiency)}),_.jsx("td",{className:"p-1.5 text-right tabular-nums text-zinc-400",children:x.wasted.toFixed(1)}),_.jsxs("td",{className:"p-1.5 text-right tabular-nums text-zinc-400",children:[x.speed.toFixed(1),"s"]}),_.jsx("td",{className:"p-1.5 text-right tabular-nums text-zinc-500",children:x.n}),M.map(ul=>{const gl=x.scenarios[ul],k=x.scenarioAttempted?.[ul]??0;let sl,F;return gl!=null?(sl=String(gl),F=Au(gl)):k===0?(sl="I",F="text-zinc-700"):(sl="—",F="text-zinc-600"),_.jsx("td",{className:`p-1.5 text-right tabular-nums ${F}`,children:sl},ul)})]})]},x.label)})})]})})}const Dy=[{key:"score",label:"Score",fmt:h=>h==null?"—":`${h.toFixed(1)}%`,higherBetter:!0},{key:"validatedAccuracy",label:"Validated Accuracy",fmt:h=>h==null?"—":`${h.toFixed(1)}%`,higherBetter:!0},{key:"completionRate",label:"Completion Rate",fmt:h=>h==null?"—":`${h.toFixed(1)}%`,higherBetter:!0},{key:"efficiency",label:"Efficiency",fmt:h=>h==null?"—":`${h.toFixed(1)}%`,higherBetter:!0},{key:"wasted",label:"Avg Wasted",fmt:h=>h==null?"—":h.toFixed(1),higherBetter:!1},{key:"speed",label:"Speed",fmt:h=>h==null?"—":`${h.toFixed(1)}s`,higherBetter:!1}];function Ho({va:h,vb:M,higherBetter:C}){if(h==null||M==null)return _.jsx("td",{className:"p-1.5 text-right text-zinc-600",children:"—"});const o=M-h,G=(o>0?"+":"")+(Number.isInteger(o)?o:o.toFixed(1));let H="text-zinc-500";return o!==0&&(H=o>0===C?"text-emerald-400":"text-red-400"),_.jsx("td",{className:`p-1.5 text-right tabular-nums font-medium ${H}`,children:G})}function Uy({a:h,b:M,scenarios:C,scenarioAbbrev:o,onSwap:q,onClear:G}){const H=(R,T)=>T in R.scenarios?R.scenarios[T]:R[T]??null;return _.jsxs("div",{className:"mt-6 border border-zinc-800 rounded-lg p-4 max-w-2xl",children:[_.jsxs("div",{className:"flex items-center justify-between mb-3",children:[_.jsx("h3",{className:"text-sm font-semibold",children:"Compare"}),_.jsxs("div",{className:"flex gap-2",children:[_.jsx("button",{onClick:q,className:"text-xs px-2.5 py-1 rounded border border-zinc-700 hover:border-zinc-500 transition-colors",children:"Swap A↔B"}),_.jsx("button",{onClick:G,className:"text-xs px-2.5 py-1 rounded border border-zinc-700 hover:border-red-500/50 hover:text-red-400 transition-colors",children:"Clear"})]})]}),_.jsxs("table",{className:"text-xs w-full border-collapse",children:[_.jsx("thead",{children:_.jsxs("tr",{className:"border-b border-zinc-800",children:[_.jsx("th",{className:"p-1.5 text-left text-zinc-500",children:"Metric"}),_.jsx("th",{className:"p-1.5 text-right text-zinc-400 max-w-48 truncate",title:h.label,children:h.label}),_.jsx("th",{className:"p-1.5 text-right text-zinc-500 w-16",children:"Delta"}),_.jsx("th",{className:"p-1.5 text-right text-zinc-400 max-w-48 truncate",title:M.label,children:M.label})]})}),_.jsxs("tbody",{children:[Dy.map(R=>{const T=H(h,R.key),E=H(M,R.key);return _.jsxs("tr",{className:"border-b border-zinc-900/50",children:[_.jsx("td",{className:"p-1.5 text-zinc-400",children:R.label}),_.jsx("td",{className:"p-1.5 text-right tabular-nums",children:R.fmt(T)}),_.jsx(Ho,{va:T,vb:E,higherBetter:R.higherBetter}),_.jsx("td",{className:"p-1.5 text-right tabular-nums",children:R.fmt(E)})]},R.key)}),_.jsx("tr",{children:_.jsx("td",{colSpan:4,className:"py-1",children:_.jsx("div",{className:"border-t border-zinc-800"})})}),C.map(R=>{const T=h.scenarios[R],E=M.scenarios[R],Y=(U,x)=>U!=null?`${U}%`:(x.scenarioAttempted?.[R]??0)===0?"I":"—";return _.jsxs("tr",{className:"border-b border-zinc-900/50",children:[_.jsx("td",{className:"p-1.5 text-zinc-500",children:o[R]||R}),_.jsx("td",{className:"p-1.5 text-right tabular-nums",children:Y(T,h)}),_.jsx(Ho,{va:T,vb:E,higherBetter:!0}),_.jsx("td",{className:"p-1.5 text-right tabular-nums",children:Y(E,M)})]},R)})]})]})]})}function Cy(h){const M={};for(const C of gf)M[C]=new Set(h.map(o=>o[C]));return M}function Ry(){const[h,M]=ml.useState(null),[C,o]=ml.useState(null),[q,G]=ml.useState({col:"score",asc:!1}),[H,R]=ml.useState([]),[T,E]=ml.useState("reforged"),[Y,U]=ml.useState("all"),[x,$]=ml.useState("all"),[ll,rl]=ml.useState("all"),[ul,gl]=ml.useState(!1);ml.useEffect(()=>{Sy().then(g=>{M(g),o(Cy(g.rows))})},[]);const k=ml.useMemo(()=>h?ul?h.rows:h.rows.filter(g=>!g.retired):[],[h,ul]),sl=ml.useMemo(()=>h?h.rows.some(g=>g.retired):!1,[h]),F=ml.useMemo(()=>!h||!C?[]:Co(k,T).filter(O=>gf.every(D=>!C[D]||C[D].has(O[D]))),[h,C,T,k]),{rows:xl,scenarios:pl}=ml.useMemo(()=>Ay(F,h?.scenarios??[],x,ll,h?.scenarioSuite??{}),[F,h,x,ll]),W=ml.useMemo(()=>{if(!h)return{};const g=h.scenarioAbbrev,O=new Set(pl),D={};for(const[X,K]of Object.entries(g))O.has(X)&&(D[X]=K);return D},[h,pl]),Bl=ml.useMemo(()=>hf.find(g=>g.id===Y)??hf[0],[Y]),{sorted:Kl,groups:xt}=ml.useMemo(()=>Ty(xl,Bl,q,pl,T),[xl,Bl,q,pl,T]),$l=ml.useMemo(()=>xl.reduce((g,O)=>g+(O.attemptedCount??O.n*pl.length),0),[xl,pl]),il=ml.useCallback((g,O,D)=>{o(X=>{if(!X)return X;const K={...X,[g]:new Set(X[g])};return D?K[g].add(O):K[g].delete(O),K}),R([])},[]),Rt=ml.useCallback(g=>{E(g),R([])},[]),Tt=ml.useCallback(g=>{U(g),R([])},[]),at=ml.useCallback(g=>{$(g),R([])},[]),z=ml.useCallback(g=>{rl(g),R([])},[]),N=ml.useCallback(g=>{gl(g),R([])},[]),V=ml.useCallback(g=>{G(O=>O.col===g?{col:g,asc:!O.asc}:{col:g,asc:g==="label"})},[]),dl=ml.useCallback((g,O)=>{R(D=>O?D.length>=2?[D[1],g]:[...D,g]:D.filter(X=>X!==g))},[]),yl=ml.useCallback(()=>{R(g=>[...g].reverse())},[]),d=ml.useCallback(()=>{R([])},[]);return!h||!C?_.jsx("div",{className:"flex items-center justify-center min-h-screen text-zinc-500",children:"Loading..."}):_.jsxs("div",{className:"flex min-h-screen",children:[_.jsx(xy,{rows:k,filters:C,onFilterChange:il,activeScreen:T,onScreenChange:Rt,activeView:Y,onViewChange:Tt,scenarioScope:x,onScopeChange:at,suiteScope:ll,onSuiteChange:z,showRetired:ul,onShowRetiredChange:N,hasRetired:sl,filteredCount:F.length,totalCount:Co(k,T).length,totalAttempts:$l,timestamp:h.timestamp}),_.jsxs("main",{className:"flex-1 min-w-0 p-4 flex flex-col",children:[_.jsx(Ny,{rows:Kl,scenarios:pl,scenarioAbbrev:W,sort:q,onSort:V,checked:H,onCompareToggle:dl,groups:xt,maxGen:h.maxGen??0,genInfo:h.genInfo}),H.length===2&&_.jsx(Uy,{a:Kl[H[0]],b:Kl[H[1]],scenarios:pl,scenarioAbbrev:W,onSwap:yl,onClear:d}),_.jsxs("p",{className:"text-[0.6rem] text-zinc-600 mt-6",children:["Generated ",h.timestamp]})]})]})}yy.createRoot(document.getElementById("root")).render(_.jsx(ml.StrictMode,{children:_.jsx(Ry,{})})); - +
+ diff --git a/docs/results/index.md b/docs/results/index.md index b9935dd..31e7248 100644 --- a/docs/results/index.md +++ b/docs/results/index.md @@ -17,4 +17,4 @@ For model and backend recommendations, see [Model Guide](../MODEL_GUIDE.md). - [native-vs-prompt.md](raw/native-vs-prompt.md) — llama-server native FC vs prompt-injected, reforged only - [reasoning-replay.md](raw/reasoning-replay.md) — reasoning_replay policy comparison (none / keep-last / full) per config -*Generated 2026-08-17 17:14* +*Generated 2026-08-20 07:41* diff --git a/docs/results/raw/native-vs-prompt.md b/docs/results/raw/native-vs-prompt.md index 38c23c1..d55de7d 100644 --- a/docs/results/raw/native-vs-prompt.md +++ b/docs/results/raw/native-vs-prompt.md @@ -324,4 +324,4 @@ Eval generations (older runs carried forward, superscript-tagged): ¹ gen 1 — v0.6.0 suite — incl. Anthropic ablation (commit 2b05dc4, 2026-05-08) ² gen 2 — v0.7.0 lineup refresh (8–14B) + 32GB tier debut (v0.7.4) (commit 655e1f6, 2026-05-22) -*Generated 2026-08-17 17:14* +*Generated 2026-08-20 07:41* diff --git a/docs/results/raw/reasoning-replay.md b/docs/results/raw/reasoning-replay.md index 4e39516..fc70f6e 100644 --- a/docs/results/raw/reasoning-replay.md +++ b/docs/results/raw/reasoning-replay.md @@ -693,4 +693,4 @@ Eval generations (older runs carried forward, superscript-tagged): ¹ gen 1 — v0.6.0 suite — incl. Anthropic ablation (commit 2b05dc4, 2026-05-08) ² gen 2 — v0.7.0 lineup refresh (8–14B) + 32GB tier debut (v0.7.4) (commit 655e1f6, 2026-05-22) -*Generated 2026-08-17 17:14* +*Generated 2026-08-20 07:41* diff --git a/docs/results/raw/reforged-vs-bare.md b/docs/results/raw/reforged-vs-bare.md index 0751b3b..5bf8bab 100644 --- a/docs/results/raw/reforged-vs-bare.md +++ b/docs/results/raw/reforged-vs-bare.md @@ -83,6 +83,17 @@ Muse-Glimmer-30B-UD-Q4_K_XL LS/N [bare:full@xhigh] 86.5% 95.7% ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- ``` +## Ornith-1.5-35B-Q4_K_M (llamaserver/native) + +``` +------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +Model/Backend Scr VAcc Cmp Eff Wst Spd N rel arg tsl b2s s3s crt srn err dgr dge art grs iar rel_s arg_s tsl_s b2s_s s3s_s crt_s srn_s err_s dgr_s dge_s art_s grs_s iar_s +------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +Ornith-1.5-35B-Q4_K_M LS/N [reforged] 97.8% 97.8% 100.0% 100% 0.4 7.1s 50 100 100 100 100 100 100 100 100 100 94 98 86 98 100 100 100 100 100 100 100 96 100 94 100 82 94 +Ornith-1.5-35B-Q4_K_M LS/N [bare] 60.3% 90.8% 66.4% 100% 0.0 9.0s 50 4 98 0 0 100 100 100 0 98 62 100 76 86 4 98 0 0 100 100 100 0 94 68 100 80 0 +------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +``` + ## Qwen3.6-35B-A3B-UD-Q4_K_M (llamaserver/native) ``` @@ -125,6 +136,17 @@ claude-haiku-4-5-20251001 AN/N [bare+any] 74.0% 80.2% 92.3% 100% 0. ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- ``` +## Ornith-1.5-9B-Q4_K_M (llamaserver/native) + +``` +------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ +Model/Backend Scr VAcc Cmp Eff Wst Spd N rel arg tsl b2s s3s crt srn err dgr dge art grs iar rel_s arg_s tsl_s b2s_s s3s_s crt_s srn_s err_s dgr_s dge_s art_s grs_s iar_s +------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ +Ornith-1.5-9B-Q4_K_M LS/N [reforged] 94.2% 94.2% 99.9% 100% 0.3 11.1s 50 100 100 100 98 100 100 100 62 98 78 94 90 94 100 100 100 98 100 100 100 58 100 88 100 96 94 +Ornith-1.5-9B-Q4_K_M LS/N [bare] 62.2% 91.2% 68.2% 100% 0.0 12.5s 50 2 100 2 84 100 86 100 0 72 32 96 84 96 4 100 0 84 100 96 98 0 68 40 94 78 2 +------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ +``` + ## Qwen3.5-27B-Q4_K_M (llamaserver/native) ``` @@ -902,4 +924,4 @@ Eval generations (older runs carried forward, superscript-tagged): ¹ gen 1 — v0.6.0 suite — incl. Anthropic ablation (commit 2b05dc4, 2026-05-08) ² gen 2 — v0.7.0 lineup refresh (8–14B) + 32GB tier debut (v0.7.4) (commit 655e1f6, 2026-05-22) -*Generated 2026-08-17 17:14* +*Generated 2026-08-20 07:41* diff --git a/docs/results/raw/reforged/all.md b/docs/results/raw/reforged/all.md index 806af19..eadd687 100644 --- a/docs/results/raw/reforged/all.md +++ b/docs/results/raw/reforged/all.md @@ -12,6 +12,7 @@ Qwen3.8-27B-UD-Q4_K_XL LS/N [reforged@medium] 98.5% Qwen3.8-27B-UD-Q4_K_XL LS/N [reforged@low] 98.8% 98.8% 100.0% 100% 0.1 11.8s 50 100 100 100 100 100 100 100 100 100 96 100 92 100 100 100 100 100 100 100 100 100 100 100 88 94 100 claude-opus-4-6 AN/N [reforged:full]¹ 99.2% 99.8% 99.4% 100% 0.0 15.6s 50 100 100 100 100 100 98 100 100 100 100 98 94 98 100 100 100 100 100 100 100 96 100 100 98 100 98 Muse-Glimmer-30B-UD-Q4_K_XL LS/N [reforged@high] 98.8% 99.8% 98.9% 83% 1.1 8.4s 50 100 100 100 100 100 100 100 100 100 98 86 100 100 100 100 100 100 100 100 100 98 100 100 86 100 100 +Ornith-1.5-35B-Q4_K_M LS/N [reforged] 97.8% 97.8% 100.0% 100% 0.4 7.1s 50 100 100 100 100 100 100 100 100 100 94 98 86 98 100 100 100 100 100 100 100 96 100 94 100 82 94 Muse-Glimmer-30B-UD-Q4_K_XL LS/N [reforged@low] 97.8% 98.5% 99.2% 91% 0.8 4.2s 50 100 100 100 100 100 100 100 100 100 82 88 100 100 100 100 100 100 100 100 100 98 100 84 90 100 100 Muse-Glimmer-30B-UD-Q4_K_XL LS/N [reforged@medium] 98.5% 99.3% 99.2% 86% 1.0 6.1s 50 100 100 100 100 100 100 100 100 100 96 86 100 100 100 100 100 100 100 100 100 94 100 94 90 100 100 Muse-Glimmer-30B-UD-Q4_K_XL LS/N [reforged:full@xhigh] 97.9% 99.3% 98.6% 82% 1.2 9.0s 50 100 100 100 100 100 100 100 100 100 100 84 100 100 100 100 100 100 98 100 100 86 100 100 80 100 98 @@ -19,6 +20,7 @@ Muse-Glimmer-30B-UD-Q4_K_XL LS/N [reforged@xhigh] 97.5% Muse-Glimmer-30B-UD-Q4_K_XL LS/N [reforged:keep-last@xhigh] 97.5% 99.4% 98.2% 83% 1.1 8.8s 50 100 100 100 100 100 100 100 100 100 98 74 100 100 100 100 100 100 100 100 100 88 100 98 78 100 100 Qwen3.6-35B-A3B-UD-Q4_K_M LS/N [reforged:full] 95.2% 95.2% 99.9% 100% 0.5 7.2s 50 100 100 100 100 100 98 96 100 100 60 76 94 100 100 100 100 100 100 100 98 100 100 74 82 98 98 claude-haiku-4-5-20251001 AN/N [reforged] 94.2% 94.2% 99.9% 100% 0.3 6.6s 50 100 100 100 100 100 100 98 100 100 74 74 98 100 100 100 100 100 100 100 100 100 100 72 38 94 100 +Ornith-1.5-9B-Q4_K_M LS/N [reforged] 94.2% 94.2% 99.9% 100% 0.3 11.1s 50 100 100 100 98 100 100 100 62 98 78 94 90 94 100 100 100 98 100 100 100 58 100 88 100 96 94 Qwen3.5-122B-A10B-Q4_K_M LS/N [reforged:full] 94.5% 94.5% 100.0% 100% 0.2 52.8s 50 100 100 100 100 100 100 100 100 100 94 36 100 100 100 100 100 100 100 100 100 100 100 90 40 96 100 Qwen3.6-35B-A3B-UD-Q4_K_M LS/N [reforged:keep-last] 92.7% 92.7% 100.0% 100% 0.4 6.6s 50 100 100 100 100 100 100 98 100 100 46 78 78 100 100 100 100 98 100 98 98 100 98 52 86 80 100 Qwen3.6-27B-Q4_K_M LS/N [reforged:keep-last] 92.5% 93.0% 99.5% 100% 0.1 17.6s 50 100 100 100 100 100 100 100 96 98 44 74 98 100 100 100 100 100 100 100 100 92 98 40 68 98 100 @@ -150,4 +152,4 @@ Eval generations (older runs carried forward, superscript-tagged): ¹ gen 1 — v0.6.0 suite — incl. Anthropic ablation (commit 2b05dc4, 2026-05-08) ² gen 2 — v0.7.0 lineup refresh (8–14B) + 32GB tier debut (v0.7.4) (commit 655e1f6, 2026-05-22) -*Generated 2026-08-17 17:14* +*Generated 2026-08-20 07:41* diff --git a/docs/results/raw/reforged/by-backend.md b/docs/results/raw/reforged/by-backend.md index a8fe822..590166f 100644 --- a/docs/results/raw/reforged/by-backend.md +++ b/docs/results/raw/reforged/by-backend.md @@ -151,4 +151,4 @@ Eval generations (older runs carried forward, superscript-tagged): ¹ gen 1 — v0.6.0 suite — incl. Anthropic ablation (commit 2b05dc4, 2026-05-08) ² gen 2 — v0.7.0 lineup refresh (8–14B) + 32GB tier debut (v0.7.4) (commit 655e1f6, 2026-05-22) -*Generated 2026-08-17 17:14* +*Generated 2026-08-20 07:41* diff --git a/docs/results/raw/reforged/by-family.md b/docs/results/raw/reforged/by-family.md index 685e1b5..3bda0db 100644 --- a/docs/results/raw/reforged/by-family.md +++ b/docs/results/raw/reforged/by-family.md @@ -50,6 +50,16 @@ Muse-Glimmer-30B-UD-Q4_K_XL LS/N [reforged:keep-last@xhigh] 97.5% 99.4% ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- ``` +## ornith-1.5-35b + +``` +------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +Model/Backend Scr VAcc Cmp Eff Wst Spd N rel arg tsl b2s s3s crt srn err dgr dge art grs iar rel_s arg_s tsl_s b2s_s s3s_s crt_s srn_s err_s dgr_s dge_s art_s grs_s iar_s +------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +Ornith-1.5-35B-Q4_K_M LS/N [reforged] 97.8% 97.8% 100.0% 100% 0.4 7.1s 50 100 100 100 100 100 100 100 100 100 94 98 86 98 100 100 100 100 100 100 100 96 100 94 100 82 94 +------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +``` + ## qwen3.6-35b-a3b ``` @@ -76,6 +86,16 @@ Qwen3.5-122B-A10B-Q4_K_M LS/N [reforged] 91.3% 91.4% 99.9% -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- ``` +## ornith-1.5-9b + +``` +------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ +Model/Backend Scr VAcc Cmp Eff Wst Spd N rel arg tsl b2s s3s crt srn err dgr dge art grs iar rel_s arg_s tsl_s b2s_s s3s_s crt_s srn_s err_s dgr_s dge_s art_s grs_s iar_s +------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ +Ornith-1.5-9B-Q4_K_M LS/N [reforged] 94.2% 94.2% 99.9% 100% 0.3 11.1s 50 100 100 100 98 100 100 100 62 98 78 94 90 94 100 100 100 98 100 100 100 58 100 88 100 96 94 +------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ +``` + ## qwen3.5-27b ``` @@ -368,4 +388,4 @@ Eval generations (older runs carried forward, superscript-tagged): ¹ gen 1 — v0.6.0 suite — incl. Anthropic ablation (commit 2b05dc4, 2026-05-08) ² gen 2 — v0.7.0 lineup refresh (8–14B) + 32GB tier debut (v0.7.4) (commit 655e1f6, 2026-05-22) -*Generated 2026-08-17 17:14* +*Generated 2026-08-20 07:41* diff --git a/tests/eval/publication.py b/tests/eval/publication.py index 28c4c98..82946f1 100644 --- a/tests/eval/publication.py +++ b/tests/eval/publication.py @@ -57,20 +57,20 @@ VIEW_NAMES = ("history", "snapshot", "latest") PINNED_VIEW_COUNTS = { - "history": 547_300, - "snapshot": 406_900, - "latest": 288_600, + "history": 552_500, + "snapshot": 412_100, + "latest": 293_800, } PINNED_VIEW_METRIC_COUNTS = { - "history": (341_192, 456_242, 456_242), - "snapshot": (252_245, 339_699, 339_699), - "latest": (201_092, 255_014, 255_014), + "history": (345_280, 460_591, 460_591), + "snapshot": (256_333, 344_048, 344_048), + "latest": (205_180, 259_363, 259_363), } PINNED_SNAPSHOT_SOURCE_IDENTITY_SHA256 = ( - "f19679e627ff3ca5e339c38f6564dfc9f215f95adbe824036dad58667f0bfe39" + "405a6d71e092d2af847f0bbd049a0b60e21ee307649b960e48fd99ff22fbacee" ) PINNED_SNAPSHOT_SOURCE_PAYLOAD_SHA256 = ( - "6b0a9e85b6d4b863c19117b549f92e7127b41e564945d722f0c6e5c037717c98" + "8508da22521cdb821f4245e9538c36019cae905e006745a46300ee17a572ddc8" ) @@ -337,8 +337,8 @@ class SourceSpec: "v0.9.0", 3, CANONICAL_DIALECT, - 28_600, - "50b2ae49a8b9161711289c60dc060a3403dbf2534ec1cbb6d5834e3f384fa8c3", + 33_800, + "e36079168689159362b1aac23901ac121615f38380ccbed5f9a1e90f324a9c8e", ), ) diff --git a/tests/eval/report.py b/tests/eval/report.py index 58a6170..6328068 100644 --- a/tests/eval/report.py +++ b/tests/eval/report.py @@ -583,6 +583,9 @@ def compute_config_metrics( "Muse-Glimmer-30B-UD-Q4_K_XL": {"family": "muse-glimmer-30b", "cross_backend_key": "muse-glimmer-30b-q4_K_XL"}, # DeepSeek V4 Flash 0731 (managed llama.cpp RPC; UD is a quant variant) "DeepSeek-V4-Flash-0731-UD-Q4_K_XL": {"family": "deepseek-v4-flash", "cross_backend_key": "deepseek-v4-flash-0731-q4_K_XL"}, + # Ornith 1.5 (llama-server only) + "Ornith-1.5-9B-Q4_K_M": {"family": "ornith-1.5-9b", "cross_backend_key": "ornith-1.5-9b-q4_K_M"}, + "Ornith-1.5-35B-Q4_K_M": {"family": "ornith-1.5-35b", "cross_backend_key": "ornith-1.5-35b-q4_K_M"}, # lfm2.5 8b-a1b (llama-server only) "LFM2.5-8B-A1B-Q4_K_M": {"family": "lfm2.5-8b-a1b", "cross_backend_key": "lfm2.5-8b-a1b-q4_K_M"}, # mellum2 12b-a2.5b (llama-server only; instruct + thinking share a family) diff --git a/tests/unit/test_batch_eval_recipes.py b/tests/unit/test_batch_eval_recipes.py index cfbde81..75525c9 100644 --- a/tests/unit/test_batch_eval_recipes.py +++ b/tests/unit/test_batch_eval_recipes.py @@ -31,6 +31,7 @@ ("Qwen3.6-35B-A3B-UD-Q4_K_M", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), ("Ornith-1.5-9B-Q4_K_M", "Ornith-1.5-9B-Q4_K_M.gguf"), ("Qwen3.8-27B-UD-Q4_K_XL", "Qwen3.8-27B-UD-Q4_K_XL.gguf"), + ("Ornith-1.5-35B-Q4_K_M", "Ornith-1.5-35B-Q4_K_M.gguf"), ("Nemotron-3-Nano-30B-A3B-Q4_K_M", "Nemotron-3-Nano-30B-A3B-Q4_K_M.gguf"), ("Muse-Glimmer-30B-UD-Q4_K_XL", "Muse-Glimmer-30B-UD-Q4_K_XL.gguf"), ("gemma-4-26B-A4B-it-UD-Q4_K_M", "gemma-4-26B-A4B-it-UD-Q4_K_M.gguf"), @@ -115,6 +116,7 @@ def _glimmer_flags(reasoning_strength: str) -> tuple[str, ...]: "Qwen3.6-27B-Q4_K_M": _REASONING_FLAGS, "Qwen3.6-35B-A3B-UD-Q4_K_M": _REASONING_FLAGS, "Ornith-1.5-9B-Q4_K_M": _REASONING_FLAGS, + "Ornith-1.5-35B-Q4_K_M": _REASONING_FLAGS, "Qwen3.8-27B-UD-Q4_K_XL": _QWEN38_FLAGS, "Nemotron-3-Nano-30B-A3B-Q4_K_M": _REASONING_FLAGS, "Muse-Glimmer-30B-UD-Q4_K_XL": _GLIMMER_FLAGS,