From d3668baeb4b371a603963cc7216fffbcd0a80464 Mon Sep 17 00:00:00 2001 From: 370025263 <370025263@qq.com> Date: Wed, 1 Jul 2026 17:12:00 +0800 Subject: [PATCH] Add Xarena spreadsheet benchmark package --- benchmark/spreadsheet_xarena/README.md | 69 + .../spreadsheet_xarena/algo_app/.gitignore | 2 + .../spreadsheet_xarena/algo_app/Dockerfile | 54 + .../spreadsheet_xarena/algo_app/README.md | 31 + .../spreadsheet_xarena/algo_app/build.sh | 78 + .../spreadsheet_xarena/algo_app/config.yaml | 94 + .../algo_app/entrypoint_train.sh | 946 ++++++ .../algo_app/multi_turn_rollout.py | 779 +++++ .../algo_app/requirements.txt | 22 + .../algo_app/sync_skills_to.sh | 50 + .../algo_app/train_split/test/items.json | 1 + .../algo_app/train_split/train/items.json | 155 + .../algo_app/train_split/val/items.json | 80 + .../spreadsheet_xarena/dataset/.gitignore | 2 + .../spreadsheet_xarena/dataset/README.md | 26 + .../dataset/prepare_data_root.sh | 39 + .../dataset/train_split/test/items.json | 1 + .../dataset/train_split/train/items.json | 155 + .../dataset/train_split/val/items.json | 80 + .../third_party/SkillOpt/.env.example | 34 + .../third_party/SkillOpt/.gitignore | 56 + .../third_party/SkillOpt/CONTRIBUTING.md | 43 + .../third_party/SkillOpt/LICENSE | 21 + .../third_party/SkillOpt/README.md | 395 +++ .../third_party/SkillOpt/SECURITY.md | 14 + .../third_party/SkillOpt/ckpt/README.md | 79 + .../SkillOpt/ckpt/alfworld/gpt5.5_skill.md | 113 + .../SkillOpt/ckpt/docvqa/gpt5.5_skill.md | 26 + .../SkillOpt/ckpt/livemath/gpt5.5_skill.md | 35 + .../SkillOpt/ckpt/officeqa/gpt5.5_skill.md | 50 + .../SkillOpt/ckpt/searchqa/gpt5.5_skill.md | 71 + .../ckpt/spreadsheetbench/gpt5.5_skill.md | 133 + .../SkillOpt/configs/_base_/default.yaml | 100 + .../SkillOpt/configs/alfworld/default.yaml | 29 + .../SkillOpt/configs/docvqa/default.yaml | 28 + .../SkillOpt/configs/features/soft_gate.yaml | 47 + .../livemathematicianbench/default.yaml | 22 + .../SkillOpt/configs/officeqa/default.yaml | 34 + .../SkillOpt/configs/searchqa/default.yaml | 32 + .../configs/spreadsheetbench/default.yaml | 34 + .../third_party/SkillOpt/docs/contributing.md | 69 + .../SkillOpt/docs/guide/configuration.md | 109 + .../SkillOpt/docs/guide/dl-analogy.md | 51 + .../SkillOpt/docs/guide/first-experiment.md | 110 + .../SkillOpt/docs/guide/installation.md | 89 + .../SkillOpt/docs/guide/local-env-smoke.md | 143 + .../SkillOpt/docs/guide/new-backend.md | 130 + .../SkillOpt/docs/guide/new-benchmark.md | 393 +++ .../SkillOpt/docs/guide/skill-document.md | 78 + .../SkillOpt/docs/guide/training-loop.md | 92 + .../third_party/SkillOpt/docs/index.md | 170 + .../SkillOpt/docs/reference/api.md | 195 ++ .../SkillOpt/docs/reference/cli.md | 71 + .../SkillOpt/docs/reference/config.md | 85 + .../third_party/SkillOpt/index.html | 2739 +++++++++++++++++ .../third_party/SkillOpt/mkdocs.yml | 78 + .../third_party/SkillOpt/pyproject.toml | 75 + .../third_party/SkillOpt/requirements.txt | 25 + .../third_party/SkillOpt/scripts/__init__.py | 0 .../third_party/SkillOpt/scripts/eval_only.py | 451 +++ .../SkillOpt/scripts/run_alfworld.sh | 60 + .../SkillOpt/scripts/run_searchqa.sh | 40 + .../SkillOpt/scripts/run_spreadsheetbench.sh | 39 + .../third_party/SkillOpt/scripts/train.py | 548 ++++ .../skillopt-assets/arxiv-logomark-small.svg | 1 + .../third_party/SkillOpt/skillopt.html | 2739 +++++++++++++++++ .../third_party/SkillOpt/skillopt/__init__.py | 28 + .../third_party/SkillOpt/skillopt/config.py | 286 ++ .../SkillOpt/skillopt/datasets/__init__.py | 7 + .../SkillOpt/skillopt/datasets/base.py | 512 +++ .../SkillOpt/skillopt/engine/__init__.py | 9 + .../SkillOpt/skillopt/engine/trainer.py | 2083 +++++++++++++ .../SkillOpt/skillopt/envs/__init__.py | 1 + .../skillopt/envs/_template/README.md | 43 + .../envs/_template/config_template.yaml | 55 + .../skillopt/envs/_template/env_template.py | 196 ++ .../envs/_template/loader_template.py | 87 + .../skillopt/envs/alfworld/__init__.py | 5 + .../skillopt/envs/alfworld/adapter.py | 459 +++ .../skillopt/envs/alfworld/dataloader.py | 123 + .../envs/alfworld/prompts/analyst_error.md | 55 + .../envs/alfworld/prompts/analyst_success.md | 33 + .../alfworld/prompts/rollout_no_history.md | 8 + .../alfworld/prompts/rollout_with_history.md | 9 + .../alfworld/prompts/rollout_with_memory.md | 16 + .../skillopt/envs/alfworld/reflect.py | 4 + .../skillopt/envs/alfworld/rollout.py | 347 +++ .../skillopt/envs/alfworld/skills/initial.md | 45 + .../skillopt/envs/alfworld/vendor/__init__.py | 9 + .../envs/alfworld/vendor/alfworld_envs.py | 221 ++ .../alfworld/vendor/alfworld_projection.py | 60 + .../envs/alfworld/vendor/alfworld_prompts.py | 8 + .../envs/alfworld/vendor/config_tw.yaml | 145 + .../skillopt/envs/alfworld/vendor/env_base.py | 84 + .../envs/alfworld/vendor/env_manager.py | 139 + .../skillopt/envs/alfworld/vendor/memory.py | 87 + .../SkillOpt/skillopt/envs/base.py | 309 ++ .../SkillOpt/skillopt/envs/docvqa/__init__.py | 1 + .../SkillOpt/skillopt/envs/docvqa/adapter.py | 115 + .../skillopt/envs/docvqa/dataloader.py | 61 + .../skillopt/envs/docvqa/evaluator.py | 113 + .../envs/docvqa/prompts/analyst_error.md | 35 + .../envs/docvqa/prompts/analyst_success.md | 24 + .../envs/docvqa/prompts/rollout_system.md | 12 + .../SkillOpt/skillopt/envs/docvqa/rollout.py | 391 +++ .../skillopt/envs/docvqa/skills/initial.md | 11 + .../envs/livemathematicianbench/__init__.py | 1 + .../envs/livemathematicianbench/adapter.py | 162 + .../envs/livemathematicianbench/dataloader.py | 308 ++ .../envs/livemathematicianbench/evaluator.py | 62 + .../prompts/analyst_error.md | 37 + .../prompts/analyst_success.md | 25 + .../prompts/rollout_system.md | 12 + .../envs/livemathematicianbench/reflect.py | 4 + .../envs/livemathematicianbench/rollout.py | 434 +++ .../livemathematicianbench/skills/initial.md | 16 + .../skillopt/envs/officeqa/__init__.py | 1 + .../skillopt/envs/officeqa/adapter.py | 135 + .../skillopt/envs/officeqa/dataloader.py | 71 + .../skillopt/envs/officeqa/evaluator.py | 46 + .../envs/officeqa/prompts/analyst_error.md | 37 + .../envs/officeqa/prompts/analyst_success.md | 25 + .../envs/officeqa/prompts/rollout_system.md | 15 + .../skillopt/envs/officeqa/rollout.py | 799 +++++ .../skillopt/envs/officeqa/skills/initial.md | 15 + .../skillopt/envs/officeqa/tool_runtime.py | 552 ++++ .../skillopt/envs/searchqa/__init__.py | 1 + .../skillopt/envs/searchqa/adapter.py | 129 + .../skillopt/envs/searchqa/dataloader.py | 42 + .../skillopt/envs/searchqa/evaluator.py | 100 + .../envs/searchqa/prompts/analyst_error.md | 46 + .../envs/searchqa/prompts/analyst_success.md | 32 + .../envs/searchqa/prompts/rollout_system.md | 13 + .../skillopt/envs/searchqa/reflect.py | 4 + .../skillopt/envs/searchqa/rollout.py | 481 +++ .../skillopt/envs/searchqa/skills/initial.md | 3 + .../envs/spreadsheetbench/__init__.py | 5 + .../skillopt/envs/spreadsheetbench/adapter.py | 192 ++ .../envs/spreadsheetbench/codegen_agent.py | 748 +++++ .../envs/spreadsheetbench/dataloader.py | 37 + .../envs/spreadsheetbench/evaluator.py | 158 + .../envs/spreadsheetbench/executor.py | 67 + .../spreadsheetbench/prompts/analyst_error.md | 46 + .../prompts/analyst_success.md | 32 + .../prompts/codegen_system.md | 1 + .../prompts/critical_rules.md | 9 + .../spreadsheetbench/prompts/react_system.md | 21 + .../envs/spreadsheetbench/react_agent.py | 395 +++ .../skillopt/envs/spreadsheetbench/reflect.py | 4 + .../skillopt/envs/spreadsheetbench/rollout.py | 934 ++++++ .../envs/spreadsheetbench/skills/initial.md | 56 + .../SkillOpt/skillopt/evaluation/__init__.py | 13 + .../SkillOpt/skillopt/evaluation/gate.py | 148 + .../SkillOpt/skillopt/gradient/__init__.py | 15 + .../SkillOpt/skillopt/gradient/aggregate.py | 253 ++ .../SkillOpt/skillopt/gradient/reflect.py | 588 ++++ .../SkillOpt/skillopt/model/__init__.py | 512 +++ .../SkillOpt/skillopt/model/azure_openai.py | 915 ++++++ .../SkillOpt/skillopt/model/backend_config.py | 185 ++ .../SkillOpt/skillopt/model/claude_backend.py | 365 +++ .../skillopt/model/claude_backend.py.orig | 359 +++ .../SkillOpt/skillopt/model/codex_backend.py | 664 ++++ .../SkillOpt/skillopt/model/codex_harness.py | 1160 +++++++ .../SkillOpt/skillopt/model/common.py | 229 ++ .../skillopt/model/minimax_backend.py | 277 ++ .../SkillOpt/skillopt/model/qwen_backend.py | 455 +++ .../SkillOpt/skillopt/model/router.py | 236 ++ .../SkillOpt/skillopt/optimizer/__init__.py | 15 + .../SkillOpt/skillopt/optimizer/clip.py | 109 + .../skillopt/optimizer/lr_autonomous.py | 108 + .../SkillOpt/skillopt/optimizer/meta_skill.py | 79 + .../SkillOpt/skillopt/optimizer/rewrite.py | 59 + .../SkillOpt/skillopt/optimizer/scheduler.py | 127 + .../SkillOpt/skillopt/optimizer/select.py | 4 + .../SkillOpt/skillopt/optimizer/skill.py | 164 + .../skillopt/optimizer/slow_update.py | 396 +++ .../skillopt/optimizer/update_modes.py | 136 + .../SkillOpt/skillopt/prompts/__init__.py | 63 + .../skillopt/prompts/analyst_error.md | 41 + .../prompts/analyst_error_full_rewrite.md | 32 + .../skillopt/prompts/analyst_error_rewrite.md | 44 + .../skillopt/prompts/analyst_success.md | 36 + .../prompts/analyst_success_full_rewrite.md | 30 + .../prompts/analyst_success_rewrite.md | 33 + .../skillopt/prompts/lr_autonomous.md | 20 + .../skillopt/prompts/merge_failure.md | 30 + .../prompts/merge_failure_full_rewrite.md | 28 + .../skillopt/prompts/merge_failure_rewrite.md | 26 + .../SkillOpt/skillopt/prompts/merge_final.md | 33 + .../prompts/merge_final_full_rewrite.md | 28 + .../skillopt/prompts/merge_final_rewrite.md | 25 + .../skillopt/prompts/merge_success.md | 28 + .../prompts/merge_success_full_rewrite.md | 28 + .../skillopt/prompts/merge_success_rewrite.md | 25 + .../SkillOpt/skillopt/prompts/meta_skill.md | 40 + .../SkillOpt/skillopt/prompts/ranking.md | 20 + .../skillopt/prompts/ranking_rewrite.md | 15 + .../skillopt/prompts/rewrite_skill.md | 25 + .../SkillOpt/skillopt/prompts/slow_update.md | 60 + .../SkillOpt/skillopt/scheduler/__init__.py | 8 + .../third_party/SkillOpt/skillopt/types.py | 306 ++ .../SkillOpt/skillopt/utils/__init__.py | 4 + .../SkillOpt/skillopt/utils/json_utils.py | 42 + .../SkillOpt/skillopt/utils/scoring.py | 28 + .../SkillOpt/skillopt_webui/__init__.py | 0 .../SkillOpt/skillopt_webui/__main__.py | 3 + .../SkillOpt/skillopt_webui/app.py | 550 ++++ .../third_party/SkillOpt/tests/__init__.py | 0 .../SkillOpt/tests/test_json_utils.py | 112 + .../SkillOpt/tests/test_scoring.py | 106 + .../third_party/SkillOpt/tests/test_types.py | 249 ++ 211 files changed, 34878 insertions(+) create mode 100644 benchmark/spreadsheet_xarena/README.md create mode 100644 benchmark/spreadsheet_xarena/algo_app/.gitignore create mode 100644 benchmark/spreadsheet_xarena/algo_app/Dockerfile create mode 100644 benchmark/spreadsheet_xarena/algo_app/README.md create mode 100755 benchmark/spreadsheet_xarena/algo_app/build.sh create mode 100644 benchmark/spreadsheet_xarena/algo_app/config.yaml create mode 100755 benchmark/spreadsheet_xarena/algo_app/entrypoint_train.sh create mode 100644 benchmark/spreadsheet_xarena/algo_app/multi_turn_rollout.py create mode 100644 benchmark/spreadsheet_xarena/algo_app/requirements.txt create mode 100755 benchmark/spreadsheet_xarena/algo_app/sync_skills_to.sh create mode 100644 benchmark/spreadsheet_xarena/algo_app/train_split/test/items.json create mode 100644 benchmark/spreadsheet_xarena/algo_app/train_split/train/items.json create mode 100644 benchmark/spreadsheet_xarena/algo_app/train_split/val/items.json create mode 100644 benchmark/spreadsheet_xarena/dataset/.gitignore create mode 100644 benchmark/spreadsheet_xarena/dataset/README.md create mode 100755 benchmark/spreadsheet_xarena/dataset/prepare_data_root.sh create mode 100644 benchmark/spreadsheet_xarena/dataset/train_split/test/items.json create mode 100644 benchmark/spreadsheet_xarena/dataset/train_split/train/items.json create mode 100644 benchmark/spreadsheet_xarena/dataset/train_split/val/items.json create mode 100644 benchmark/spreadsheet_xarena/third_party/SkillOpt/.env.example create mode 100644 benchmark/spreadsheet_xarena/third_party/SkillOpt/.gitignore create mode 100644 benchmark/spreadsheet_xarena/third_party/SkillOpt/CONTRIBUTING.md create mode 100644 benchmark/spreadsheet_xarena/third_party/SkillOpt/LICENSE create mode 100644 benchmark/spreadsheet_xarena/third_party/SkillOpt/README.md create mode 100644 benchmark/spreadsheet_xarena/third_party/SkillOpt/SECURITY.md create mode 100644 benchmark/spreadsheet_xarena/third_party/SkillOpt/ckpt/README.md create mode 100644 benchmark/spreadsheet_xarena/third_party/SkillOpt/ckpt/alfworld/gpt5.5_skill.md create mode 100644 benchmark/spreadsheet_xarena/third_party/SkillOpt/ckpt/docvqa/gpt5.5_skill.md create mode 100644 benchmark/spreadsheet_xarena/third_party/SkillOpt/ckpt/livemath/gpt5.5_skill.md create mode 100644 benchmark/spreadsheet_xarena/third_party/SkillOpt/ckpt/officeqa/gpt5.5_skill.md create mode 100644 benchmark/spreadsheet_xarena/third_party/SkillOpt/ckpt/searchqa/gpt5.5_skill.md create mode 100644 benchmark/spreadsheet_xarena/third_party/SkillOpt/ckpt/spreadsheetbench/gpt5.5_skill.md create mode 100644 benchmark/spreadsheet_xarena/third_party/SkillOpt/configs/_base_/default.yaml create mode 100644 benchmark/spreadsheet_xarena/third_party/SkillOpt/configs/alfworld/default.yaml create mode 100644 benchmark/spreadsheet_xarena/third_party/SkillOpt/configs/docvqa/default.yaml create mode 100644 benchmark/spreadsheet_xarena/third_party/SkillOpt/configs/features/soft_gate.yaml create mode 100644 benchmark/spreadsheet_xarena/third_party/SkillOpt/configs/livemathematicianbench/default.yaml create mode 100644 benchmark/spreadsheet_xarena/third_party/SkillOpt/configs/officeqa/default.yaml create mode 100644 benchmark/spreadsheet_xarena/third_party/SkillOpt/configs/searchqa/default.yaml create mode 100644 benchmark/spreadsheet_xarena/third_party/SkillOpt/configs/spreadsheetbench/default.yaml create mode 100644 benchmark/spreadsheet_xarena/third_party/SkillOpt/docs/contributing.md create mode 100644 benchmark/spreadsheet_xarena/third_party/SkillOpt/docs/guide/configuration.md create mode 100644 benchmark/spreadsheet_xarena/third_party/SkillOpt/docs/guide/dl-analogy.md create mode 100644 benchmark/spreadsheet_xarena/third_party/SkillOpt/docs/guide/first-experiment.md create mode 100644 benchmark/spreadsheet_xarena/third_party/SkillOpt/docs/guide/installation.md create mode 100644 benchmark/spreadsheet_xarena/third_party/SkillOpt/docs/guide/local-env-smoke.md create mode 100644 benchmark/spreadsheet_xarena/third_party/SkillOpt/docs/guide/new-backend.md create mode 100644 benchmark/spreadsheet_xarena/third_party/SkillOpt/docs/guide/new-benchmark.md create mode 100644 benchmark/spreadsheet_xarena/third_party/SkillOpt/docs/guide/skill-document.md create mode 100644 benchmark/spreadsheet_xarena/third_party/SkillOpt/docs/guide/training-loop.md create mode 100644 benchmark/spreadsheet_xarena/third_party/SkillOpt/docs/index.md create mode 100644 benchmark/spreadsheet_xarena/third_party/SkillOpt/docs/reference/api.md create mode 100644 benchmark/spreadsheet_xarena/third_party/SkillOpt/docs/reference/cli.md create mode 100644 benchmark/spreadsheet_xarena/third_party/SkillOpt/docs/reference/config.md create mode 100644 benchmark/spreadsheet_xarena/third_party/SkillOpt/index.html create mode 100644 benchmark/spreadsheet_xarena/third_party/SkillOpt/mkdocs.yml create mode 100644 benchmark/spreadsheet_xarena/third_party/SkillOpt/pyproject.toml create mode 100644 benchmark/spreadsheet_xarena/third_party/SkillOpt/requirements.txt create mode 100644 benchmark/spreadsheet_xarena/third_party/SkillOpt/scripts/__init__.py create mode 100644 benchmark/spreadsheet_xarena/third_party/SkillOpt/scripts/eval_only.py create mode 100755 benchmark/spreadsheet_xarena/third_party/SkillOpt/scripts/run_alfworld.sh create mode 100755 benchmark/spreadsheet_xarena/third_party/SkillOpt/scripts/run_searchqa.sh create mode 100755 benchmark/spreadsheet_xarena/third_party/SkillOpt/scripts/run_spreadsheetbench.sh create mode 100644 benchmark/spreadsheet_xarena/third_party/SkillOpt/scripts/train.py create mode 100644 benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt-assets/arxiv-logomark-small.svg create mode 100644 benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt.html create mode 100644 benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/__init__.py create mode 100644 benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/config.py create mode 100644 benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/datasets/__init__.py create mode 100644 benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/datasets/base.py create mode 100644 benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/engine/__init__.py create mode 100644 benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/engine/trainer.py create mode 100644 benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/__init__.py create mode 100644 benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/_template/README.md create mode 100644 benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/_template/config_template.yaml create mode 100644 benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/_template/env_template.py create mode 100644 benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/_template/loader_template.py create mode 100644 benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/alfworld/__init__.py create mode 100644 benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/alfworld/adapter.py create mode 100644 benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/alfworld/dataloader.py create mode 100644 benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/alfworld/prompts/analyst_error.md create mode 100644 benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/alfworld/prompts/analyst_success.md create mode 100644 benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/alfworld/prompts/rollout_no_history.md create mode 100644 benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/alfworld/prompts/rollout_with_history.md create mode 100644 benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/alfworld/prompts/rollout_with_memory.md create mode 100644 benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/alfworld/reflect.py create mode 100644 benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/alfworld/rollout.py create mode 100644 benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/alfworld/skills/initial.md create mode 100644 benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/alfworld/vendor/__init__.py create mode 100644 benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/alfworld/vendor/alfworld_envs.py create mode 100644 benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/alfworld/vendor/alfworld_projection.py create mode 100644 benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/alfworld/vendor/alfworld_prompts.py create mode 100644 benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/alfworld/vendor/config_tw.yaml create mode 100644 benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/alfworld/vendor/env_base.py create mode 100644 benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/alfworld/vendor/env_manager.py create mode 100644 benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/alfworld/vendor/memory.py create mode 100644 benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/base.py create mode 100644 benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/docvqa/__init__.py create mode 100644 benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/docvqa/adapter.py create mode 100644 benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/docvqa/dataloader.py create mode 100644 benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/docvqa/evaluator.py create mode 100644 benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/docvqa/prompts/analyst_error.md create mode 100644 benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/docvqa/prompts/analyst_success.md create mode 100644 benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/docvqa/prompts/rollout_system.md create mode 100644 benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/docvqa/rollout.py create mode 100644 benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/docvqa/skills/initial.md create mode 100644 benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/livemathematicianbench/__init__.py create mode 100644 benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/livemathematicianbench/adapter.py create mode 100644 benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/livemathematicianbench/dataloader.py create mode 100644 benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/livemathematicianbench/evaluator.py create mode 100644 benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/livemathematicianbench/prompts/analyst_error.md create mode 100644 benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/livemathematicianbench/prompts/analyst_success.md create mode 100644 benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/livemathematicianbench/prompts/rollout_system.md create mode 100644 benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/livemathematicianbench/reflect.py create mode 100644 benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/livemathematicianbench/rollout.py create mode 100644 benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/livemathematicianbench/skills/initial.md create mode 100644 benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/officeqa/__init__.py create mode 100644 benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/officeqa/adapter.py create mode 100644 benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/officeqa/dataloader.py create mode 100644 benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/officeqa/evaluator.py create mode 100644 benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/officeqa/prompts/analyst_error.md create mode 100644 benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/officeqa/prompts/analyst_success.md create mode 100644 benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/officeqa/prompts/rollout_system.md create mode 100644 benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/officeqa/rollout.py create mode 100644 benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/officeqa/skills/initial.md create mode 100644 benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/officeqa/tool_runtime.py create mode 100644 benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/searchqa/__init__.py create mode 100644 benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/searchqa/adapter.py create mode 100644 benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/searchqa/dataloader.py create mode 100644 benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/searchqa/evaluator.py create mode 100644 benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/searchqa/prompts/analyst_error.md create mode 100644 benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/searchqa/prompts/analyst_success.md create mode 100644 benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/searchqa/prompts/rollout_system.md create mode 100644 benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/searchqa/reflect.py create mode 100644 benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/searchqa/rollout.py create mode 100644 benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/searchqa/skills/initial.md create mode 100644 benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/spreadsheetbench/__init__.py create mode 100644 benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/spreadsheetbench/adapter.py create mode 100644 benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/spreadsheetbench/codegen_agent.py create mode 100644 benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/spreadsheetbench/dataloader.py create mode 100644 benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/spreadsheetbench/evaluator.py create mode 100644 benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/spreadsheetbench/executor.py create mode 100644 benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/spreadsheetbench/prompts/analyst_error.md create mode 100644 benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/spreadsheetbench/prompts/analyst_success.md create mode 100644 benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/spreadsheetbench/prompts/codegen_system.md create mode 100644 benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/spreadsheetbench/prompts/critical_rules.md create mode 100644 benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/spreadsheetbench/prompts/react_system.md create mode 100644 benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/spreadsheetbench/react_agent.py create mode 100644 benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/spreadsheetbench/reflect.py create mode 100644 benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/spreadsheetbench/rollout.py create mode 100644 benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/spreadsheetbench/skills/initial.md create mode 100644 benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/evaluation/__init__.py create mode 100644 benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/evaluation/gate.py create mode 100644 benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/gradient/__init__.py create mode 100644 benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/gradient/aggregate.py create mode 100644 benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/gradient/reflect.py create mode 100644 benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/model/__init__.py create mode 100644 benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/model/azure_openai.py create mode 100644 benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/model/backend_config.py create mode 100644 benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/model/claude_backend.py create mode 100644 benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/model/claude_backend.py.orig create mode 100644 benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/model/codex_backend.py create mode 100644 benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/model/codex_harness.py create mode 100644 benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/model/common.py create mode 100644 benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/model/minimax_backend.py create mode 100644 benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/model/qwen_backend.py create mode 100644 benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/model/router.py create mode 100644 benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/optimizer/__init__.py create mode 100644 benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/optimizer/clip.py create mode 100644 benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/optimizer/lr_autonomous.py create mode 100644 benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/optimizer/meta_skill.py create mode 100644 benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/optimizer/rewrite.py create mode 100644 benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/optimizer/scheduler.py create mode 100644 benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/optimizer/select.py create mode 100644 benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/optimizer/skill.py create mode 100644 benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/optimizer/slow_update.py create mode 100644 benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/optimizer/update_modes.py create mode 100644 benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/prompts/__init__.py create mode 100644 benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/prompts/analyst_error.md create mode 100644 benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/prompts/analyst_error_full_rewrite.md create mode 100644 benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/prompts/analyst_error_rewrite.md create mode 100644 benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/prompts/analyst_success.md create mode 100644 benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/prompts/analyst_success_full_rewrite.md create mode 100644 benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/prompts/analyst_success_rewrite.md create mode 100644 benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/prompts/lr_autonomous.md create mode 100644 benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/prompts/merge_failure.md create mode 100644 benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/prompts/merge_failure_full_rewrite.md create mode 100644 benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/prompts/merge_failure_rewrite.md create mode 100644 benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/prompts/merge_final.md create mode 100644 benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/prompts/merge_final_full_rewrite.md create mode 100644 benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/prompts/merge_final_rewrite.md create mode 100644 benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/prompts/merge_success.md create mode 100644 benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/prompts/merge_success_full_rewrite.md create mode 100644 benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/prompts/merge_success_rewrite.md create mode 100644 benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/prompts/meta_skill.md create mode 100644 benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/prompts/ranking.md create mode 100644 benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/prompts/ranking_rewrite.md create mode 100644 benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/prompts/rewrite_skill.md create mode 100644 benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/prompts/slow_update.md create mode 100644 benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/scheduler/__init__.py create mode 100644 benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/types.py create mode 100644 benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/utils/__init__.py create mode 100644 benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/utils/json_utils.py create mode 100644 benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/utils/scoring.py create mode 100644 benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt_webui/__init__.py create mode 100644 benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt_webui/__main__.py create mode 100644 benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt_webui/app.py create mode 100644 benchmark/spreadsheet_xarena/third_party/SkillOpt/tests/__init__.py create mode 100644 benchmark/spreadsheet_xarena/third_party/SkillOpt/tests/test_json_utils.py create mode 100644 benchmark/spreadsheet_xarena/third_party/SkillOpt/tests/test_scoring.py create mode 100644 benchmark/spreadsheet_xarena/third_party/SkillOpt/tests/test_types.py diff --git a/benchmark/spreadsheet_xarena/README.md b/benchmark/spreadsheet_xarena/README.md new file mode 100644 index 00000000..e182b688 --- /dev/null +++ b/benchmark/spreadsheet_xarena/README.md @@ -0,0 +1,69 @@ +# Spreadsheet Xarena Benchmark + +This directory packages the xskill SpreadsheetBench submission for Xarena. + +## Layout + +```text +benchmark/spreadsheet_xarena/ + algo_app/ # submitter-container image source + dataset/train_split/ # lightweight train/val/test task split files + dataset/prepare_data_root.sh # downloads or copies workbook data before build + third_party/SkillOpt/ # rollout harness used during training +``` + +`algo_app` is the algorithm image. It trains xskill inside the Xarena job and writes the skill package to the shared volume: + +```text +/shared/skill/ALGO +/shared/skill/DONE +/shared/skill/skills//SKILL.md +``` + +The evaluator image is still owned by the leaderboard board. `third_party/SkillOpt` is only used as the rollout harness for training trajectories; it is not the evaluator container. + +## Build + +From this directory: + +```bash +cd algo_app +TAG=main-xarena PUSH=1 LOAD_KIND=lb bash build.sh +``` + +By default the image name is: + +```text +localhost:5000/p_user1/algo-xskill: +``` + +`build.sh` copies the current repository checkout into the Docker build context, so changes on this MR branch are included in the image. It stages `third_party/SkillOpt` and a prepared `dataset/data_root` into `_ctx`. + +The workbook data is not committed. If `dataset/data_root` is missing, `build.sh` calls `dataset/prepare_data_root.sh`, which downloads `https://xskill.wiki/zip/xskill-compete.zip` and extracts the 100-task SpreadsheetBench package. To use an existing local dataset instead: + +```bash +DATA_ROOT=/path/to/data_root TAG=main-xarena bash algo_app/build.sh +``` + +## Submit + +Submit the built image to the Spreadsheet leaderboard board with Xarena env vars similar to: + +```text +EVAL_MODEL=deepseek-v4-flash +XSKILL_WORKERS=3 +XSKILL_MAX_TURNS=5 +XSKILL_EPOCHS=4 +XSKILL_VAL_BLOCK=true +XSKILL_VAL_BLOCK_TIMEOUT=1800 +OUTPUT_DIR=/shared/out +``` + +The API keys should come from the leaderboard Kubernetes secret: + +```text +DEEPSEEK_API_KEY +DASHSCOPE_API_KEY +``` + +Do not put API keys into this repository or into `env_text`. diff --git a/benchmark/spreadsheet_xarena/algo_app/.gitignore b/benchmark/spreadsheet_xarena/algo_app/.gitignore new file mode 100644 index 00000000..bba8f1fb --- /dev/null +++ b/benchmark/spreadsheet_xarena/algo_app/.gitignore @@ -0,0 +1,2 @@ +/_ctx/ +!config.yaml diff --git a/benchmark/spreadsheet_xarena/algo_app/Dockerfile b/benchmark/spreadsheet_xarena/algo_app/Dockerfile new file mode 100644 index 00000000..4b72d06b --- /dev/null +++ b/benchmark/spreadsheet_xarena/algo_app/Dockerfile @@ -0,0 +1,54 @@ +FROM python:3.11-slim +ENV PYTHONUNBUFFERED=1 \ + PIP_INDEX_URL=https://pypi.tuna.tsinghua.edu.cn/simple \ + PIP_TRUSTED_HOST=pypi.tuna.tsinghua.edu.cn + +# ── 系统依赖 + Node 20（claude CLI 运行时）──────────────────────────────── +# git/bash 给脚本与 setuptools-scm 用；dulwich 让 xskill 不依赖系统 git，但留着无害。 +RUN apt-get update && apt-get install -y --no-install-recommends \ + git bash curl ca-certificates \ + && curl -fsSL https://deb.nodesource.com/setup_20.x | bash - \ + && apt-get install -y --no-install-recommends nodejs \ + && rm -rf /var/lib/apt/lists/* + +# ── claude CLI（绝对路径在 entrypoint 用 $(command -v claude)）───────────── +RUN npm install -g @anthropic-ai/claude-code \ + && claude --version + +# ── Python 第三方依赖 ───────────────────────────────────────────────────── +COPY requirements.txt /app/requirements.txt +RUN pip install --no-cache-dir -r /app/requirements.txt + +# ── xskill 本体（从烘焙源码 editable 装）────────────────────────────────── +# 源码烘焙后无 .git，setuptools-scm 无法推断版本 -> 用 PRETEND_VERSION 兜底。 +COPY _ctx/xskill /app/xskill +ENV SETUPTOOLS_SCM_PRETEND_VERSION=0.6.1 +RUN pip install --no-cache-dir -e /app/xskill \ + && xskill serve --help >/dev/null && echo "xskill serve OK" + +# ── SkillOpt 本体（rollout 用 eval_only.py）─────────────────────────────── +COPY _ctx/SkillOpt /app/SkillOpt +RUN pip install --no-cache-dir -e /app/SkillOpt || true +ENV PYTHONPATH=/app/SkillOpt + +# ── 数据（26M）、配置、训练 split、sync 脚本、空 skill ──────────────────── +COPY _ctx/data_root /data +WORKDIR /app +COPY config.yaml /app/config.yaml +COPY train_split/ /app/train_split/ +COPY _ctx/sync_skills_to.sh /app/sync_skills_to.sh +COPY entrypoint_train.sh /app/entrypoint_train.sh +COPY multi_turn_rollout.py /app/multi_turn_rollout.py +RUN chmod +x /app/entrypoint_train.sh /app/sync_skills_to.sh \ + && : > /app/empty_skill.md + +# IS_SANDBOX=1：容器以 root 运行，claude CLI 默认拒绝 root 下的 +# --dangerously-skip-permissions；设此变量让其放行（容器本身即沙箱）。 +# rollout 子壳里也会再设一次，这里兜底任何镜像内 claude 调用。 +ENV SKILL_DIR=/shared/skill \ + EVAL_MODEL=deepseek-v4-flash \ + DATA_ROOT=/data \ + TRAIN_SPLIT=/app/train_split \ + XSKILL_PROMO_THRESHOLD=5 \ + IS_SANDBOX=1 +CMD ["bash","/app/entrypoint_train.sh"] diff --git a/benchmark/spreadsheet_xarena/algo_app/README.md b/benchmark/spreadsheet_xarena/algo_app/README.md new file mode 100644 index 00000000..7be41f54 --- /dev/null +++ b/benchmark/spreadsheet_xarena/algo_app/README.md @@ -0,0 +1,31 @@ +# xskill Xarena Algorithm Image + +This is the submitter-container image for the SpreadsheetBench Xarena board. + +It starts xskill, runs SpreadsheetBench training rollouts, collects graduated skills, and writes the result to the Xarena shared volume: + +```text +/shared/skill/ALGO +/shared/skill/DONE +/shared/skill/skills//SKILL.md +``` + +Build from this directory: + +```bash +TAG=main-xarena PUSH=1 LOAD_KIND=lb bash build.sh +``` + +Useful runtime variables: + +```text +EVAL_MODEL=deepseek-v4-flash +XSKILL_WORKERS=3 +XSKILL_MAX_TURNS=5 +XSKILL_EPOCHS=4 +XSKILL_VAL_BLOCK=true +XSKILL_VAL_BLOCK_TIMEOUT=1800 +OUTPUT_DIR=/shared/out +``` + +`DEEPSEEK_API_KEY` and `DASHSCOPE_API_KEY` must be injected by the leaderboard job secret. diff --git a/benchmark/spreadsheet_xarena/algo_app/build.sh b/benchmark/spreadsheet_xarena/algo_app/build.sh new file mode 100755 index 00000000..d708359f --- /dev/null +++ b/benchmark/spreadsheet_xarena/algo_app/build.sh @@ -0,0 +1,78 @@ +#!/usr/bin/env bash +# Build the Xarena-compatible xskill SpreadsheetBench algorithm image. +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +BENCH_DIR="$(cd "$SCRIPT_DIR/.." && pwd)" +REPO_ROOT="$(cd "$SCRIPT_DIR/../../.." && pwd)" + +REG="${REG:-localhost:5000}" +IMAGE_REPO="${IMAGE_REPO:-p_user1/algo-xskill}" +TAG="${TAG:-main-xarena}" +IMG="$REG/$IMAGE_REPO:$TAG" + +DATA_ROOT="${DATA_ROOT:-$BENCH_DIR/dataset/data_root}" +SKILLOPT_SRC="${SKILLOPT_SRC:-$BENCH_DIR/third_party/SkillOpt}" +PUSH="${PUSH:-0}" +LOAD_KIND="${LOAD_KIND:-}" +PREPARE_DATA="${PREPARE_DATA:-1}" + +require_dir() { + local path=$1 + local label=$2 + if [ ! -d "$path" ]; then + echo "missing $label: $path" >&2 + exit 1 + fi +} + +if [ ! -d "$DATA_ROOT" ] && [ "$PREPARE_DATA" = "1" ]; then + bash "$BENCH_DIR/dataset/prepare_data_root.sh" +fi + +require_dir "$DATA_ROOT" "SpreadsheetBench dataset" +require_dir "$SKILLOPT_SRC" "SkillOpt rollout harness" + +cd "$SCRIPT_DIR" +rm -rf _ctx +mkdir -p _ctx + +rsync -a --delete \ + --exclude .git \ + --exclude .venv \ + --exclude __pycache__ \ + --exclude '*.pyc' \ + --exclude .pytest_cache \ + --exclude .mypy_cache \ + --exclude .ruff_cache \ + --exclude .key \ + --exclude '.env' \ + --exclude '*.pem' \ + --exclude '*.key' \ + --exclude 'benchmark/spreadsheet_xarena/algo_app/_ctx' \ + --exclude 'benchmark/spreadsheet_xarena/dataset' \ + --exclude 'benchmark/spreadsheet_xarena/third_party' \ + "$REPO_ROOT/" _ctx/xskill/ + +rsync -a --delete \ + --exclude .git \ + --exclude __pycache__ \ + --exclude '*.pyc' \ + --exclude .pytest_cache \ + --exclude '.env' \ + "$SKILLOPT_SRC/" _ctx/SkillOpt/ + +rsync -a --delete "$DATA_ROOT/" _ctx/data_root/ +cp "$SCRIPT_DIR/sync_skills_to.sh" _ctx/sync_skills_to.sh + +docker build -t "$IMG" . + +if [ "$PUSH" = "1" ]; then + docker push "$IMG" +fi + +if [ -n "$LOAD_KIND" ]; then + kind load docker-image "$IMG" --name "$LOAD_KIND" +fi + +echo "built $IMG" diff --git a/benchmark/spreadsheet_xarena/algo_app/config.yaml b/benchmark/spreadsheet_xarena/algo_app/config.yaml new file mode 100644 index 00000000..0b572ff9 --- /dev/null +++ b/benchmark/spreadsheet_xarena/algo_app/config.yaml @@ -0,0 +1,94 @@ +# xskill config TEMPLATE — 打榜镜像·xskill (reduced-scale real training). +# xskill 不读环境变量/key 文件，所以 api_key 用占位符；entrypoint_train.sh 在 +# 运行时把 __DEEPSEEK_API_KEY__ / __DASHSCOPE_API_KEY__ 替换成真实 key 后落到 +# $XSKILL_HOME/config.yaml。 +# +# 关键：xskill 的 baby->main "毕业"门槛是 candidates.py 里硬编码的 +# ATOM_PROMOTION_THRESHOLD（runner 构造 SkillEditAgent 时不传 threshold，所以 +# config 的 candidates.threshold 改不动它）。reduced 规模下必须在 entrypoint 里 +# 直接 patch 该常量调低，否则 4 条轨迹攒不满默认 10 分 -> 零毕业 -> 全是 baby +# stub -> 没有真正蒸馏出的 SKILL.md。详见 entrypoint_train.sh。 + +# ===== Skill repository (candidate + main 都落这里) ===== +skill_dir: __XSKILL_HOME__/skill + +# ===== LLM (generation / scoring / cluster / SkillEdit) ===== +llm: + base_url: https://api.deepseek.com + model: deepseek-v4-flash + api_key: __DEEPSEEK_API_KEY__ + max_tokens: 10000 + request_timeout: 120 + connect_timeout: 15 + +# ===== Embedding (atom 入库 + AtomTaskSearch 向量检索) ===== +# DeepSeek 无 embeddings API；用 DashScope text-embedding-v4。 +embedding: + base_url: https://dashscope.aliyuncs.com/compatible-mode/v1 + model: text-embedding-v4 + api_key: __DASHSCOPE_API_KEY__ + dim: 0 + +# ===== 候选攒分阈值 (v1 路径 + stale 归档用；v2 毕业门槛见 entrypoint patch) ===== +candidates: + threshold: 2 # v1 ready_for_promotion / stale 用；调低无副作用 + stale_days: 3650 # 别在短跑里把候选归档成 stale + min_source_trajs: 1 + +# ===== SkillEditAgent（把 candidates 整理成正文 SKILL.md 的写作 agent）===== +skill_edit_agent: + tool_call_limit: 24 + timeout_seconds: 600 + read_file_max_bytes: 15000 + +# ===== Canary（main->staging 灰度）：team-CS 在线进化模式开启并调低门槛。 ===== +# epoch1.. team-CS：worker 作为 distinct client 上传解题轨迹，server 端 CS 归因 +# 给 atom 按 side 打 ux_score；main 有分后 SkillEdit 把已有技能的更新路由到 +# commit_to_staging 开灰度分支，check_and_decide 在样本够时 promote/discard。 +# 小数据下必须调低（否则永远凑不够样本）：probability=0.5 让 ~50% 流量进 staging， +# min_samples/total_samples=2 让 2 条样本就能决策，scope_top_n=1。 +# 注：entrypoint_train.sh 渲染时还会再覆盖这段（双保险），见 1a) 段。 +canary: + enabled: true + probability: 0.5 + min_samples: 2 + max_days_hold: 14 + rotate_interval: 60 + scope_top_n: 1 + total_samples: 2 + +# ===== description 触发优化：commit 时跑一次 hill-climb 调 frontmatter。短跑 +# 可保留（失败不阻塞 commit），但把预算压小省时间/省 token。 ===== +skill_opt: + enabled: true + n_cases: 6 + runs_per_case: 1 + max_iters: 2 + max_llm_calls: 60 + train_frac: 0.6 + seed: 42 + catalog_max_skills: 12 + catalog_desc_cap: 256 + probe_case_timeout: 45 + rerun_enabled: false + +# ===== Watcher（serve 里的目录轮询）===== +watcher: + poll_interval: 10 + max_concurrent: 2 + +# ===== Ingest（claude_code session jsonl -> traj_*.md 桥接）===== +ingest: + # 评测场景：脚本批量产 session、写完即定稿，settle 调小到 5s 便于尽快入库。 + settle_seconds: 5 + mask_patterns: + # 剥掉 SkillOpt codex harness 每题固定的 turn-0 提示词外壳，防聚类被任务外壳吸住。 + - '(?s)Use the workspace files to solve the task\..*?summarize the approach\.' + +# ===== Dashboard：关掉（打榜镜像无需 web 控制台）===== +dashboard: + enabled: false + public: false + password: "" + default_harness: claude_code + default_model: deepseek-v4-flash diff --git a/benchmark/spreadsheet_xarena/algo_app/entrypoint_train.sh b/benchmark/spreadsheet_xarena/algo_app/entrypoint_train.sh new file mode 100755 index 00000000..24bd4027 --- /dev/null +++ b/benchmark/spreadsheet_xarena/algo_app/entrypoint_train.sh @@ -0,0 +1,946 @@ +#!/usr/bin/env bash +# ============================================================================ +# 打榜镜像·xskill —— TEAM-CS ONLINE-EVOLUTION 训练入口 (epoch0 cold_flush 冷启动 +# + epoch1..N team-CS 多用户灰度在线进化)。 +# +# ── 训练范式（两段式）───────────────────────────────────────────────────── +# epoch0 = 冷启动：一个 daemon 既是 HTTP team server 又是 server_mode +# DirectoryWatcher。本机直接用 multi_turn_rollout 写轨迹到 server 的 +# 被 watch home（$HOME_ROOT/.claude/projects），跑完落 $BARRIER_FILE， +# daemon 用 cold_flush=1 把每个有候选的 baby skill 批量毕业 baby->main +# 出 v1（绕开小数据下永远到不了的 weightscore 阈值）。 +# epoch1..N = team-CS 灰度在线进化（核心）：起 N 个 worker，每个独立 +# HOME=/root/whome_$k（独立 .claude/projects+skills+settings.json）， +# cg_exec 隔离，作为 distinct `xskill connect --label worker$k` client +# 连本机 server。每个 worker 用 multi_turn_rollout（--chome 指各自 +# home/.claude）对分到的题解题——解题轨迹落各自 $WHOME/.claude/projects， +# connect client 的 ingester 镜像成 traj_*.md 并上传 server。 +# server 端对每条上传轨迹做 CS 归因（score_atom 给 atom 按 side 打 +# ux_score）：epoch1 起 main 攒到 ux_score 后，正常在线 SkillEdit 把 +# "已有 main 技能的更新"路由到 commit_to_staging（开灰度分支）；后续 +# 上传按 pick_side_scoped 分流 main/staging，check_and_decide 在样本 +# 够时 staging_avg>=main_avg 则 promote v->v+1，否则 discard。 +# +# **关键决策**：epoch1..N **不落屏障**（cold_start.epochs=1 只让 epoch0 +# 走 cold_flush）；epoch1 起 cold_flush=False，SkillEdit 走正常在线 +# 增量路径 -> commit_to_staging 产生灰度对象 -> canary 决策真跑起来。 +# +# ── 已确认的源码事实（file:line 由调研核实）────────────────────────────── +# * serve --server: cli.py cmd_serve -> serve(server_mode=True)，一进程 = +# FastAPI team server + server_mode watcher。 +# * join token: $XSKILL_HOME/team_server.json 的 **join_token** 字段 +# （team/server/state.py ensure_join_token；不是 .token）。 +# * connect --label: 成 client 指纹一部分，distinct label 避免同 hostname 塌缩 +# 成一个 client_id（cli.py cmd_connect + client_registry.py）。 +# * client run_forever: collect_and_upload($HOME/.claude/projects 经 ingester +# 镜像成 ~/.xskill/cc_sessions/traj_*.md)->sync->reconcile_skill_sides(按 +# manifest 的 side 装 main/staging 到 $HOME/.claude/skills)。 +# * collector 去抖 quiet_seconds=180 / min_change_interval=600 **硬编码**在 +# TeamClient.__init__（connect 无 flag、不读 config）-> 训练里 runtime patch +# 调低（同 ATOM_PROMOTION_THRESHOLD 既有 patch 套路）。 +# * staging 前置：SkillEditAgent 守门 3——main 上开 staging 要求 main 已有真实 +# ux_score（_main_has_ux_score）。epoch1 worker 必须真"用到"已毕业技能（native +# 模式全量挂载），上传轨迹经 CS 归因给 main 打分后，下一轮 SkillEdit 才会开 +# staging。故每个在线 epoch 后留足 CANARY_SETTLE 让多轮 watcher tick 跑完。 +# +# sidecar restartPolicy:Always —— 任何情况下都不退出（退出会无限重训），所有 +# 路径（成功 / fatal）末尾都 sleep infinity。 +# ============================================================================ +set -uo pipefail + +# ── 必需 key（运行时注入；xskill 不读环境变量，需写进 config.yaml）────────── +DEEPSEEK_API_KEY="${DEEPSEEK_API_KEY:?DEEPSEEK_API_KEY required}" +DASHSCOPE_API_KEY="${DASHSCOPE_API_KEY:?DASHSCOPE_API_KEY required}" + +# ── 产出目录 ──────────────────────────────────────────────────────────── +SKILL_OUT="${SKILL_DIR:-/shared/skill}" +SKILLS_OUT="$SKILL_OUT/skills" # Anthropic skills/ 多 skill 目录 +mkdir -p "$SKILLS_OUT" + +# ── 可调参数 ──────────────────────────────────────────────────────────── +PROMO_THRESHOLD="${XSKILL_PROMO_THRESHOLD:-5}" # patch 进 candidates.py 的毕业门槛 +DAEMON_PORT="${XSKILL_DAEMON_PORT:-8791}" +FINAL_SETTLE="${FINAL_SETTLE:-150}" # 每个 epoch 跑完后等一轮入库/聚类 +ITEM_TIMEOUT="${ITEM_TIMEOUT:-420}" # 单 rollout exec 超时 +TRAIN_SPLIT="${TRAIN_SPLIT:-/app/train_split}" +DATA_ROOT="${DATA_ROOT:-/data}" +EVAL_MODEL="${EVAL_MODEL:-deepseek-v4-flash}" + +# ── 并行 worker + 多轮 + 多 epoch 参数 ────────────────────────────────── +# WORKERS : team-CS 在线 epoch 的并行 worker 数（= distinct client 数） +# MAX_TURNS : 每题多轮纠错的最大轮数（透传 multi_turn_rollout.py --max-turns） +# EPOCHS : 训练 epoch 数（epoch0=冷启动落屏障批量毕业；1..N-1=team-CS 灰度） +# CANARY_SETTLE : 每个在线 epoch 后等待秒数，让"上传->蒸馏/打分->canary 决策"闭环跑完 +# WORKER_CPU_MAX : cgroup v2 cpu.max 值（" "，默认 80000 100000 = 0.8 核） +# WORKER_MEM_MAX : cgroup v2 memory.max 值（默认 2G） +WORKERS="${XSKILL_WORKERS:-3}" +MAX_TURNS="${XSKILL_MAX_TURNS:-5}" +EPOCHS="${XSKILL_EPOCHS:-4}" +CANARY_SETTLE="${XSKILL_CANARY_SETTLE:-600}" +# 决策触发式 val：canary 缺 val 分时挂起等后台 loop 补分（默认开）。关掉则退回 +# "缺 val 即纯 ux"旧行为。val_block_timeout 兜底防死等（默认 30min）。 +VAL_BLOCK="${XSKILL_VAL_BLOCK:-true}" +VAL_BLOCK_TIMEOUT="${XSKILL_VAL_BLOCK_TIMEOUT:-1800}" +# 后台 val 监听 loop 扫 .val_request.json 的间隔（秒） +VAL_LOOP_INTERVAL="${XSKILL_VAL_LOOP_INTERVAL:-25}" +WORKER_CPU_MAX="${XSKILL_WORKER_CPU_MAX:-80000 100000}" +WORKER_MEM_MAX="${XSKILL_WORKER_MEM_MAX:-2G}" + +# ── team client 去抖 patch 值（runtime patch TeamClient/__init__ 默认）────── +# connect 不读 config、无 flag，故把 collector 去抖硬编码默认直接 patch 调低， +# 否则训练里 worker 每题轨迹要静默 600s 才上传 -> 单 epoch 等不起。 +CLIENT_QUIET="${XSKILL_CLIENT_QUIET:-20}" # mtime 静默窗口（默认 20s） +CLIENT_MIN_CHANGE="${XSKILL_CLIENT_MIN_CHANGE:-30}" # hash 去抖窗口（默认 30s） +CLIENT_POLL="${XSKILL_CLIENT_POLL:-10}" # client run_forever poll_interval +# JsonlIngester 入库完成屏障（settle barrier）默认 120s——源 jsonl mtime 距今 < +# settle 秒就不桥接。client 不读 config.yaml 故吃 INGEST_SETTLE_SECONDS_DEFAULT +# （config.py=120.0），训练里每条轨迹白等 120s 才进 cc_sessions。调低到 ~15s。 +INGEST_SETTLE="${XSKILL_INGEST_SETTLE:-15}" # client 端 JsonlIngester settle 默认 + +# ── xskill / claude home 路径 ────────────────────────────────────────── +# server 端：HOME=$XHOME -> XSKILL_HOME=$XHOME/.xskill（config + skill repo + +# team_server.json 在此）。daemon serve --server --home $HOME_ROOT -> epoch0 +# 冷启动本机轨迹落 $HOME_ROOT/.claude/projects（被 watch）。 +# server 的 skill 仓 = $XSKILL_SKILL_DIR；client reconcile 后把 main/staging 装 +# 各 worker 自己 home/.claude/skills。 +export XHOME=/root/xhome +export XSKILL_HOME="$XHOME/.xskill" +HOME_ROOT=/app/cchome +CHOME="$HOME_ROOT/.claude" +XSKILL_SKILL_DIR="$XSKILL_HOME/skill" # 候选 + 毕业 + 灰度 skill 仓根（config.skill_dir） +TEAM_SERVER_JSON="$XSKILL_HOME/team_server.json" +mkdir -p "$XSKILL_HOME" "$CHOME/projects" "$CHOME/skills" + +CLAUDE_ABS="$(command -v claude)" +[ -n "$CLAUDE_ABS" ] || { echo "FATAL: claude CLI not found on PATH"; sleep infinity; } + +# log 写到 stderr：这样 collect_skills 等被 $(...) 捕获 stdout 的函数里调用 log +# 不会污染捕获值（N=$(collect_skills) 只拿到纯数字）；stderr 仍随容器 2>&1 入日志。 +log() { echo "[$(date '+%F %T')] $*" >&2; } + +# ── 0b) cgroup v2 检测 + worker 隔离 helper ────────────────────────────── +CG_ROOT=/sys/fs/cgroup +CG_OK=0 +if grep -q 'cgroup2' /proc/mounts 2>/dev/null && [ -w "$CG_ROOT/cgroup.subtree_control" ]; then + if echo "+cpu +memory" > "$CG_ROOT/cgroup.subtree_control" 2>/dev/null; then + CG_OK=1 + log "cgroup v2 可用：已在 $CG_ROOT/cgroup.subtree_control 委派 +cpu +memory；worker 走 cgroup 硬隔离" + else + log "WARN: cgroup v2 已挂载但写 subtree_control 失败（控制器未委派？），worker 退化用 nice 软隔离" + fi +else + log "WARN: cgroup v2 不可用（容器非特权？无 cgroup2 挂载或 subtree_control 不可写），worker 退化用 nice 软隔离" +fi + +# cg_exec —— 在隔离环境里 exec 一条命令（用于子 shell 里）。 +cg_exec() { + local name=$1; shift + if [ "$CG_OK" = "1" ]; then + local cg="$CG_ROOT/$name" + mkdir -p "$cg" 2>/dev/null || log "WARN: mkdir cgroup $cg 失败，worker 不隔离直跑" + if [ -d "$cg" ]; then + echo "$WORKER_CPU_MAX" > "$cg/cpu.max" 2>/dev/null || log "WARN: 写 $cg/cpu.max 失败（已忽略）" + echo "$WORKER_MEM_MAX" > "$cg/memory.max" 2>/dev/null || log "WARN: 写 $cg/memory.max 失败（已忽略）" + echo "$BASHPID" > "$cg/cgroup.procs" 2>/dev/null || log "WARN: 迁 $BASHPID 进 $cg/cgroup.procs 失败（worker 不隔离直跑）" + fi + exec "$@" + else + exec nice -n 10 "$@" + fi +} + +# ── 1) 写 config.yaml（占位符替换真实 key）────────────────────────────── +log "writing xskill config -> $XSKILL_HOME/config.yaml" +sed -e "s#__XSKILL_HOME__#$XSKILL_HOME#g" \ + -e "s#__DEEPSEEK_API_KEY__#$DEEPSEEK_API_KEY#g" \ + -e "s#__DASHSCOPE_API_KEY__#$DASHSCOPE_API_KEY#g" \ + /app/config.yaml > "$XSKILL_HOME/config.yaml" + +# ── 1a) canary 调低 + cold_start epoch 屏障 注入 config ────────────────── +# canary 调低（否则小数据永远凑不够样本）：probability=0.5 让 50% 流量进 staging， +# min_samples/total_samples=2 让 2 条样本就能决策，scope_top_n=1。 +# cold_start：enabled+flush_threshold=1+**epochs=1**（只 epoch0 走 cold_flush 屏障 +# 毕业 baby->main；epoch1.. cold_flush 自然失效 -> SkillEdit 走在线 staging 路径）。 +BARRIER_FILE="${XSKILL_BARRIER_FILE:-$HOME_ROOT/EPOCH_FLUSH}" +# 冷启动期默认关 description 触发优化（hill-climb）：批量毕业前跑 6 cases 探针拖慢 +# flush；native 评测全量挂载不依赖 description 触发，对分数无影响。 +COLD_DISABLE_DESC_OPT="${XSKILL_COLD_DISABLE_DESC_OPT:-true}" +python - "$XSKILL_HOME/config.yaml" "$BARRIER_FILE" "$COLD_DISABLE_DESC_OPT" \ + "$VAL_BLOCK" "$VAL_BLOCK_TIMEOUT" <<'PY' +import sys, yaml +path, barrier, dis = sys.argv[1], sys.argv[2], sys.argv[3] == "true" +val_block = sys.argv[4] == "true" +val_block_timeout = float(sys.argv[5]) +c = yaml.safe_load(open(path, encoding="utf-8")) or {} +# canary 调低：让小数据灰度链路真能凑够样本并出决策 +c["canary"] = { + "enabled": True, + "probability": 0.5, + "min_samples": 2, + "total_samples": 2, + "scope_top_n": 1, + "rotate_interval": 60, + "max_days_hold": 14, + # 综合分：0.5*val 正确率(归一 0-10) + 0.5*ux；.val_scores.json 由后台 val + # 监听 loop（见 entrypoint val_request_loop）按需写入。 + "val_weight": 0.5, + # 决策触发式 val：canary 攒够 ux、本应裁决但缺 val 分时，写 .val_request.json + # 挂起（waiting_val），等后台 loop 补分后下一轮综合裁决。XSKILL_VAL_BLOCK + # 控制（默认 true）。val_block_timeout 兜底：等太久仍缺 val 退回纯 ux，不死等。 + "val_block": val_block, + "val_block_timeout": val_block_timeout, +} +# cold_start：epochs=1 -> 只 epoch0 落屏障 cold_flush 毕业 baby->main；epoch1.. 不再 +# cold_flush，SkillEdit 走正常在线增量 -> commit_to_staging 开灰度。 +c["cold_start"] = {"enabled": True, "flush_threshold": 1, "epochs": 1, + "barrier_path": barrier} +if dis: + c.setdefault("skill_opt", {})["enabled"] = False +yaml.safe_dump(c, open(path, "w", encoding="utf-8"), allow_unicode=True, sort_keys=False) +print("[config] canary lowered (prob=0.5 min/total=2 top_n=1); " + "val_block =", val_block, "val_block_timeout =", val_block_timeout, + "; cold_start epochs=1 barrier=", barrier, + "; skill_opt.enabled =", c.get("skill_opt", {}).get("enabled")) +PY +rm -f "$BARRIER_FILE" +log "config injected: canary(lowered) + cold_start(epochs=1) barrier=$BARRIER_FILE desc_opt_disabled=$COLD_DISABLE_DESC_OPT" + +# ── 1b) atom 拆分模式（消融开关）────────────────────────────────────────── +# split_mode_config（xskill.config）读 env XSKILL_SPLIT_MODE 覆盖 config.atom.split_mode： +# agentic（默认）= TaskAgent 按用户意图拆 1..N 个 atom； +# whole = 消融，整条新增轨迹成 1 个 atom（不调 LLM 拆分）。 +# xskill 这一项是直接读 os.environ（不像别的配置写进 config.yaml），故必须把它 +# **export** 出去让 daemon（serve --server 进程及 watchdog 重启）继承到。默认 +# agentic（entrypoint 行为零改变）；提交时用 `docker run -e XSKILL_SPLIT_MODE=whole` +# 开启消融。 +export XSKILL_SPLIT_MODE="${XSKILL_SPLIT_MODE:-agentic}" +log "atom split_mode = $XSKILL_SPLIT_MODE (env XSKILL_SPLIT_MODE; daemon will inherit)" + +# ── 2) PATCH 毕业门槛：把 ATOM_PROMOTION_THRESHOLD 调到 $PROMO_THRESHOLD ── +CAND_PY="$(HOME=$XHOME python -c 'import xskill.skill.candidates as c; print(c.__file__)' 2>/dev/null)" +if [ -n "$CAND_PY" ] && [ -f "$CAND_PY" ]; then + python - "$CAND_PY" "$PROMO_THRESHOLD" <<'PYEOF' +import re, sys +path, thr = sys.argv[1], int(sys.argv[2]) +src = open(path, encoding="utf-8").read() +new, n = re.subn(r'^ATOM_PROMOTION_THRESHOLD\s*=\s*\d+', + f'ATOM_PROMOTION_THRESHOLD = {thr}', src, flags=re.M) +if n != 1: + print(f"WARN: ATOM_PROMOTION_THRESHOLD not patched (matched {n}) in {path}") + sys.exit(0) +open(path, "w", encoding="utf-8").write(new) +print(f"patched ATOM_PROMOTION_THRESHOLD -> {thr} in {path}") +PYEOF + HOME=$XHOME python -c "import xskill.skill.candidates as c; print('[verify] ATOM_PROMOTION_THRESHOLD =', c.ATOM_PROMOTION_THRESHOLD)" 2>/dev/null \ + || log "WARN: could not verify patched threshold" +else + log "WARN: could not locate candidates.py to patch promotion threshold" +fi + +# ── 2b) PATCH team client 去抖默认（connect 不读 config/无 flag）────────── +# 把 TeamClient.__init__ 与 TeamCollector.__init__ 的 quiet_seconds / +# min_change_interval / poll_interval 默认值改小，让 worker 解题轨迹尽快上传。 +DAEMON_FILE="$(HOME=$XHOME python -c 'import xskill.team.client.daemon as d; print(d.__file__)' 2>/dev/null)" +COLLECTOR_FILE="$(HOME=$XHOME python -c 'import xskill.team.client.collector as d; print(d.__file__)' 2>/dev/null)" +for f in "$DAEMON_FILE" "$COLLECTOR_FILE"; do + [ -n "$f" ] && [ -f "$f" ] || { log "WARN: 找不到要 patch 的 team client 文件: $f"; continue; } + python - "$f" "$CLIENT_QUIET" "$CLIENT_MIN_CHANGE" "$CLIENT_POLL" <<'PYEOF' +import re, sys +path, q, m, poll = sys.argv[1], sys.argv[2], sys.argv[3], sys.argv[4] +src = open(path, encoding="utf-8").read() +n_total = 0 +for key, val in (("quiet_seconds", q), ("min_change_interval", m), ("poll_interval", poll)): + # 形如 `quiet_seconds: int = 180,` 或 `poll_interval: float = 30.0,` + src, n = re.subn(rf'({key}\s*:\s*(?:int|float)\s*=\s*)[0-9.]+', + rf'\g<1>{val}', src) + n_total += n +open(path, "w", encoding="utf-8").write(src) +print(f"patched {n_total} debounce default(s) in {path} " + f"(quiet={q} min_change={m} poll={poll})") +PYEOF +done + +# ── 2c) PATCH JsonlIngester settle barrier 默认（client 不读 config）───────── +# INGEST_SETTLE_SECONDS_DEFAULT 在 config.py，client 端 ingester 缺省吃它（120s）。 +CONFIG_FILE="$(HOME=$XHOME python -c 'import xskill.config as c; print(c.__file__)' 2>/dev/null)" +if [ -n "$CONFIG_FILE" ] && [ -f "$CONFIG_FILE" ]; then + python - "$CONFIG_FILE" "$INGEST_SETTLE" <<'PYEOF' +import re, sys +path, val = sys.argv[1], sys.argv[2] +src = open(path, encoding="utf-8").read() +src, n = re.subn(r'(INGEST_SETTLE_SECONDS_DEFAULT\s*=\s*)[0-9.]+', + rf'\g<1>{float(val)}', src) +open(path, "w", encoding="utf-8").write(src) +print(f"patched INGEST_SETTLE_SECONDS_DEFAULT -> {val} (matched {n}) in {path}") +PYEOF +else + log "WARN: 找不到 config.py 来 patch INGEST_SETTLE_SECONDS_DEFAULT" +fi + +# ── 3) seed server-side claude home settings.json（bypass perms + skill 预算） +cat > "$CHOME/settings.json" <<'JSON' +{"skillListingBudgetFraction":0.1,"permissions":{"defaultMode":"bypassPermissions"}} +JSON + +# ── 3b) 预置宽 description 种子技能（稳定聚类粒度 → 单技能）────────────────── +# 问题：同样 20 题，聚类（TaskClusterAgent）有时聚成 1 个技能、有时 7 个。规则是 +# "atom 的 description 精准匹配 catalog 里已有技能就复用、否则新建"。第一批 catalog +# 为空时，第一个技能 description 写宽（罩全部）还是写窄（各起炉灶）随机决定结局。 +# 对策（与 temperature=0 双管齐下）：训练前在 server skill 仓预置一个 **baby 分支** +# 种子技能，name=openpyxl-excel-automation，description 写得足够宽，罩住 +# SpreadsheetBench 全部 openpyxl 操作 → 第一批 catalog 即有这条宽 desc，所有 atom +# 走"精准匹配→复用"被吸进它 → 稳定单技能。冷启动 flush 时它作为 baby（攒了 +# candidates）正常 graduate baby→main，把 atom 蒸进去。 +# env 开关 XSKILL_SEED_SKILL（默认 on），对照实验可设 false 关掉。 +SEED_SKILL="${XSKILL_SEED_SKILL:-true}" +# 种子粒度 profile： +# wide1 = 单个宽 description 种子（强压单技能；sub-33 证明 test 仅 62.86，看着全但解题 agent 抓不住重点） +# focused2 = 两个聚焦种子（cell-level-manipulation + cross-sheet-data-aggregation，复刻 organic 81.43 那套粒度） +SEED_PROFILE="${XSKILL_SEED_PROFILE:-wide1}" +if [ "$SEED_SKILL" = "true" ]; then + log "seeding baby skill(s) profile='$SEED_PROFILE' into $XSKILL_SKILL_DIR" + HOME=$XHOME python - "$XSKILL_SKILL_DIR" "$SEED_PROFILE" <<'PY' +import sys +from pathlib import Path +from xskill.skill.git import init_skill_repo_on_baby, current_branch + +skill_root, profile = sys.argv[1], sys.argv[2] + +WIDE = ( + "Automate Excel/.xlsx spreadsheet tasks programmatically with Python openpyxl: " + "read and write cell values, modify and create worksheets, aggregate and summarize " + "data (SUM/COUNT/AVERAGE/SUMIFS including OR-criteria), format cells (number formats, " + "fonts, fills, borders, alignment, conditional formatting), look up and replace values " + "across sheets, insert or delete rows and columns, convert formulas to computed values, " + "handle datetime parsing and formatting, merge and unmerge cells, copy ranges, sort and " + "filter tables, and apply per-cell edits. Use whenever a task involves reading, " + "transforming, computing over, or formatting tabular data in an Excel workbook." +) +# 两个聚焦种子：复刻 organic 81.43 跑法的技能切分（一个管单元格级操作，一个管跨表聚合）。 +CELL = ( + "Per-cell and per-column Excel edits with openpyxl: replace/convert individual cell " + "values, write computed values instead of formula strings (evaluate SUMIFS/IF in Python " + "then write the number), read formula-bearing input with data_only=True, format single " + "cells (datetime->time with number_format, conditional decimal precision, zero-as-dash), " + "type-safe cell access (ws.cell(row,col), data_type checks, merged-cell safety), and " + "ordered row deletion. Use for tasks editing specific cells/columns within one worksheet." +) +CROSS = ( + "Cross-sheet and table-level Excel aggregation with openpyxl: aggregate/summarize data " + "across multiple worksheets by key (sum/count by month and category), look up and replace " + "values by matching a reference column or header (vertical and horizontal lookup), adaptive " + "column detection instead of hardcoded indices, range-match lookup tables, blank/None row " + "handling, and ITEM re-numbering. Use for tasks that read from or join several sheets/ranges " + "to compute a summary." +) + +if profile == "focused2": + seeds = [("cell-level-manipulation", CELL), ("cross-sheet-data-aggregation", CROSS)] +elif profile == "wide1": + seeds = [("openpyxl-excel-automation", WIDE)] +else: + raise SystemExit(f"unknown XSKILL_SEED_PROFILE={profile!r}") + +for name, description in seeds: + skill_dir = str(Path(skill_root) / name) + if Path(skill_dir, ".git").exists(): + print(f"[seed] skill already exists at {skill_dir}; skip") + else: + init_skill_repo_on_baby(skill_dir, name, description) + print(f"[seed] init baby skill {name} @ {skill_dir} branch={current_branch(skill_dir)}") +PY + log "seed skill done (branch reported by python above)" +else + log "XSKILL_SEED_SKILL=$SEED_SKILL → skipping seed skill (organic control run)" +fi + +# ── 3c) I/O 约定 SkillEdit guidance（修 input.xlsx harness bug）────────────── +# 根因：eval/训练用 SkillOpt executor 跑 solution.py——它把顶层 INPUT_PATH=/OUTPUT_PATH= +# 赋值正则剥掉、再把真实路径注入 exec globals。若生成代码用裸 INPUT_PATH 名 → 命中； +# 若写成 os.environ.get('INPUT_PATH','input.xlsx') 或 def solve()+sys.argv → 注入到不了 +# → 抓字面量 'input.xlsx' → FileNotFoundError（sub-35 中招 3 次，占了和 SkillOpt 的全部差距）。 +# 对策：给 SkillEdit 写作指导段追加一条强制 I/O 约定（XSKILL_SKILLEDIT_GUIDANCE_FILE +# 整体替换 → 从 vendored 源码读默认两段免手抄 + 追加约定），让每个蒸馏出的技能都带这条， +# 解题 agent 照着写裸顶层变量。env 开关 XSKILL_IO_RULE（默认 on）。 +IO_RULE="${XSKILL_IO_RULE:-true}" +if [ "$IO_RULE" = "true" ]; then + # 用**追加**模式（XSKILL_SKILLEDIT_GUIDANCE_APPEND_FILE）：只把 I/O 约定追加到默认 + # 写作纪律 block_2 末尾，保留默认两段在原位。**别用整体替换**——替换会把 block_2 + # 挪到 SKILL.md schema 之前、清空 schema 后的槽，损坏 prompt 结构（sub-38 实测 65.71 回退）。 + GUIDANCE_OUT="$XSKILL_HOME/skilledit_io_rule.md" + cat > "$GUIDANCE_OUT" <<'EOF' +# 强制 I/O 约定（SpreadsheetBench 执行台契约，必须作为一条独立规则写进每个技能正文） + +解题脚本（solution.py / 可复用脚本）读写工作簿的路径**必须**遵守： +- 把 `INPUT_PATH` 和 `OUTPUT_PATH` 定义为**裸顶层字符串变量**（`INPUT_PATH = "input.xlsx"` 形式）， + 正文中**直接用这两个名字** `openpyxl.load_workbook(INPUT_PATH)` / `wb.save(OUTPUT_PATH)`。 +- **绝不**用 `os.environ.get('INPUT_PATH', ...)` / `os.getenv` / `sys.argv` 取路径。 +- **绝不**写任何 `'input.xlsx'` / `'output.xlsx'` 兜底回退（执行台会剥掉顶层赋值并注入真实路径， + 裸名命中注入；任何 environ/argv/兜底都会拿到错误的字面量路径 → FileNotFoundError）。 +EOF + export XSKILL_SKILLEDIT_GUIDANCE_APPEND_FILE="$GUIDANCE_OUT" + log "I/O 约定 guidance 追加注入完成: XSKILL_SKILLEDIT_GUIDANCE_APPEND_FILE=$GUIDANCE_OUT" +else + log "XSKILL_IO_RULE=$IO_RULE → 不注入 I/O 约定 guidance（对照）" +fi + +# ── 4) 起 xskill daemon（serve --server：HTTP team server + server_mode watcher） +DAEMON_LOG=/tmp/xskill_daemon.log +log "starting xskill TEAM SERVER (HOME=$XHOME serve --server --home=$HOME_ROOT port=$DAEMON_PORT)" +HOME=$XHOME "$(command -v xskill)" --debug serve --server \ + --host 127.0.0.1 --port "$DAEMON_PORT" --home "$HOME_ROOT" \ + > "$DAEMON_LOG" 2>&1 & +DAEMON_PID=$! +log "daemon pid=$DAEMON_PID" +sleep 12 +if ! kill -0 "$DAEMON_PID" 2>/dev/null; then + log "FATAL: daemon exited within 12s — tail of $DAEMON_LOG:" + tail -40 "$DAEMON_LOG" + echo xskill > "$SKILL_OUT/ALGO" + log "daemon dead at startup; sleeping (sidecar must not exit)" + sleep infinity +fi +ensure_daemon() { + if ! kill -0 "$DAEMON_PID" 2>/dev/null; then + log "WATCHDOG: daemon died — restarting" + HOME=$XHOME "$(command -v xskill)" --debug serve --server \ + --host 127.0.0.1 --port "$DAEMON_PORT" --home "$HOME_ROOT" \ + >> "$DAEMON_LOG" 2>&1 & + DAEMON_PID=$! + log "daemon restarted pid=$DAEMON_PID"; sleep 10 + fi +} + +# ── 4b) 取 join token（team_server.json 的 join_token 字段）──────────────── +TOKEN="" +for _ in $(seq 1 30); do + if [ -f "$TEAM_SERVER_JSON" ]; then + TOKEN="$(python -c "import json,sys;print(json.load(open('$TEAM_SERVER_JSON')).get('join_token',''))" 2>/dev/null)" + [ -n "$TOKEN" ] && break + fi + sleep 1 +done +if [ -n "$TOKEN" ]; then + log "team server join_token acquired (len=${#TOKEN}) from $TEAM_SERVER_JSON" +else + log "WARN: could not read join_token from $TEAM_SERVER_JSON after 30s — connect 将失败（team-CS epoch 无法上传）" +fi + +# ── collect helper：把蒸馏出的 SKILL.md（main 分支）收集到 /shared/skill/skills/ ─ +GIT="$(command -v git)" +is_stub() { # $1=SKILL.md path; 0=stub, 1=real + grep -q '(placeholder —' "$1" 2>/dev/null && return 0 + return 1 +} +collect_skills() { + local mode_count=0 + rm -rf "$SKILLS_OUT"; mkdir -p "$SKILLS_OUT" + [ -d "$XSKILL_SKILL_DIR" ] || { echo 0; return; } + + # ---- pass A: main 分支 ---- + for d in "$XSKILL_SKILL_DIR"/*/; do + [ -d "$d" ] || continue + local name; name=$(basename "$d") + [ "${name#.}" != "$name" ] && continue + [ -f "$d/SKILL.md" ] || continue + local br="" + [ -d "$d/.git" ] && [ -n "$GIT" ] && br=$("$GIT" -C "$d" rev-parse --abbrev-ref HEAD 2>/dev/null) + [ "$br" = "main" ] || continue + mkdir -p "$SKILLS_OUT/$name" + cp "$d/SKILL.md" "$SKILLS_OUT/$name/SKILL.md" + for ex in references scripts assets templates; do + [ -d "$d/$ex" ] && cp -r "$d/$ex" "$SKILLS_OUT/$name/$ex" 2>/dev/null + done + mode_count=$((mode_count+1)) + done + if [ "$mode_count" -gt 0 ]; then + log "collect: shipped $mode_count MAIN-branch (graduated) skill(s)" + echo "$mode_count"; return + fi + + # ---- pass B: 任何非 stub 正文（不论分支）---- + for d in "$XSKILL_SKILL_DIR"/*/; do + [ -d "$d" ] || continue + local name; name=$(basename "$d") + [ "${name#.}" != "$name" ] && continue + [ -f "$d/SKILL.md" ] || continue + is_stub "$d/SKILL.md" && continue + mkdir -p "$SKILLS_OUT/$name" + cp "$d/SKILL.md" "$SKILLS_OUT/$name/SKILL.md" + for ex in references scripts assets templates; do + [ -d "$d/$ex" ] && cp -r "$d/$ex" "$SKILLS_OUT/$name/$ex" 2>/dev/null + done + mode_count=$((mode_count+1)) + done + if [ "$mode_count" -gt 0 ]; then + log "collect: no MAIN skills; shipped $mode_count non-stub SKILL.md (real distilled body, branch!=main)" + echo "$mode_count"; return + fi + + # ---- pass C: 最后兜底（baby stub）---- + for d in "$XSKILL_SKILL_DIR"/*/; do + [ -d "$d" ] || continue + local name; name=$(basename "$d") + [ "${name#.}" != "$name" ] && continue + [ -f "$d/SKILL.md" ] || continue + mkdir -p "$SKILLS_OUT/$name" + cp "$d/SKILL.md" "$SKILLS_OUT/$name/SKILL.md" + mode_count=$((mode_count+1)) + done + [ "$mode_count" -gt 0 ] && log "collect: FALLBACK shipped $mode_count baby STUB skill(s) (no real distillation graduated)" + echo "$mode_count" +} + +snapshot() { # 打印当前 skill 仓状态（含 staging 分支与灰度物化目录，调试） + [ -d "$XSKILL_SKILL_DIR" ] || { log " (skill dir not yet created)"; return; } + local any=0 + for d in "$XSKILL_SKILL_DIR"/*/; do + [ -d "$d" ] || continue + local name; name=$(basename "$d") + [ "${name#.}" != "$name" ] && continue + local br="-"; [ -d "$d/.git" ] && [ -n "$GIT" ] && br=$("$GIT" -C "$d" rev-parse --abbrev-ref HEAD 2>/dev/null) + local has_st="no" + [ -d "$d/.git" ] && [ -n "$GIT" ] && "$GIT" -C "$d" rev-parse --verify staging >/dev/null 2>&1 && has_st="YES" + local kind="stub"; [ -f "$d/SKILL.md" ] && ! is_stub "$d/SKILL.md" && kind="REAL" + log " skill: $name [branch=$br staging=$has_st body=$kind]" + any=1 + done + [ "$any" = 0 ] && log " (no skills yet)" +} + +# ── 5) team-CS worker（**所有 epoch 含 epoch0**）：独立 home + connect client + 上传 ─ +# 重要架构事实：serve --server（server_mode）下 watcher **只消费 client 经 +# /api/v1/team/upload 上传的轨迹**（落 team_trajectories/clients//sessions +# 并注册成 watch_dir），**不 ingest server 本机 --home 的 .claude/projects** +# （api/app.py 的 _ensure_ingesters_for_detected_ecosystems 开头 `if team_server: +# return`）。故 epoch0 冷启动也必须走 team client 上传——否则 server 永远看不到 +# epoch0 轨迹、零 atom、零毕业。epoch0 与 epoch1.. 的唯一区别是：epoch0 末落屏障 +# 触发 cold_flush 批量毕业 baby->main；epoch1.. 不落屏障走在线 staging 灰度。 +# 每个 worker_k: +# 每个 worker_k: +# * 独立 HOME=/root/whome_$k（.claude/projects+skills+settings.json）。 +# * 启动一个常驻 `xskill connect --label worker$k` client（HOME=$WHOME）， +# 它持续把 $WHOME/.claude/projects 的轨迹镜像+上传 server、并把 server 分给 +# 该 client 的 skill side(main/staging) 装到 $WHOME/.claude/skills。 +# * 用 multi_turn_rollout（--chome $WHOME/.claude）让该 worker 的 claude 解题 +# ——claude 加载 $WHOME/.claude/skills 里 client 装好的技能（native 全量挂载）， +# 解题轨迹落 $WHOME/.claude/projects -> client 上传。 +declare -A WORKER_CLIENT_PID # worker_k -> connect client pid + +whome_for() { echo "/root/whome_$1"; } + +setup_worker_home() { # $1=worker_k —— 建独立 home + settings，幂等 + local k=$1 wh; wh="$(whome_for "$k")" + mkdir -p "$wh/.claude/projects" "$wh/.claude/skills" "$wh/.xskill" + cat > "$wh/.claude/settings.json" <<'JSON' +{"skillListingBudgetFraction":0.1,"permissions":{"defaultMode":"bypassPermissions"}} +JSON +} + +start_worker_client() { # $1=worker_k —— 起常驻 connect client（若未起） + local k=$1 wh; wh="$(whome_for "$k")" + [ -n "$TOKEN" ] || { log "WARN: 无 join_token，worker$k connect 跳过（无法上传）"; return; } + # 已在跑则不重起 + local pid="${WORKER_CLIENT_PID[$k]:-}" + if [ -n "$pid" ] && kill -0 "$pid" 2>/dev/null; then return; fi + local clog="/tmp/client_w${k}.log" + # 子壳：HOME=$WHOME 让 client 的 ingester/install 都锚在该 worker home； + # 首次带 address+token 注册（distinct --label worker$k -> distinct client_id）。 + ( + export HOME="$wh" + cg_exec "xskill_cli${k}" \ + "$(command -v xskill)" --debug connect "127.0.0.1:$DAEMON_PORT" \ + --token "$TOKEN" --label "worker$k" + ) > "$clog" 2>&1 & + WORKER_CLIENT_PID[$k]=$! + log " [client] worker$k connect started pid=${WORKER_CLIENT_PID[$k]} HOME=$wh log=$clog" +} + +run_item_team() { # $1=idx $2=id $3=worker_k —— team-CS 解题（写 worker home） + local idx=$1 id=$2 k=$3 wh; wh="$(whome_for "$k")" + local ws="/tmp/ws_e${ep}_w${k}_${id}" + local out="/tmp/mtres_e${ep}_${id}.json" + local logf="/tmp/mtres_e${ep}_${id}.log" + rm -rf "$ws"; mkdir -p "$ws" + log " [ep$ep w$k] TEAM rollout item $idx id=$id (chome=$wh/.claude max_turns=$MAX_TURNS timeout=${ITEM_TIMEOUT}s)" + ( + export CLAUDE_CONFIG_DIR="$wh/.claude" + export ANTHROPIC_BASE_URL=https://api.deepseek.com/anthropic + export ANTHROPIC_AUTH_TOKEN="$DEEPSEEK_API_KEY" + export CLAUDE_CODE_EXEC_USE_SDK=cli + export IS_SANDBOX=1 + export XSKILL_SKILL_MODE=native + cg_exec "xskill_w${k}" \ + python /app/multi_turn_rollout.py \ + --item-id "$id" --data-root "$DATA_ROOT" --chome "$wh/.claude" \ + --claude-path "$CLAUDE_ABS" --model "$EVAL_MODEL" \ + --max-turns "$MAX_TURNS" \ + --workspace "$ws" --out "$out" --timeout "$ITEM_TIMEOUT" + ) > "$logf" 2>&1 + local rc=$? + if [ -f "$out" ]; then + local succ turns; succ=$(python -c "import json;print(json.load(open('$out')).get('success'))" 2>/dev/null || echo NA) + turns=$(python -c "import json;print(json.load(open('$out')).get('turns_used'))" 2>/dev/null || echo NA) + log " [ep$ep w$k] item $idx id=$id -> done (success=$succ turns=$turns rc=$rc)" + else + log " [ep$ep w$k] item $idx id=$id -> no result json (rc=$rc); tail:" + tail -8 "$logf" 2>/dev/null | while IFS= read -r l; do log " mt| $l"; done + fi +} + +# ── 5c) 一个 epoch：并行跑全部 TRAIN_IDS（分批 worker，显式 PID wait）───────── +# 关键：只 wait 显式 worker rollout PID（绝不裸 wait——会等永不退出的 daemon + +# connect client -> 死锁）。worker slot k 在 0..WORKERS-1 轮转。 +run_epoch() { # 所有 epoch 都走 team client 上传路径 + local idx=0 slot=0 + local pids=() + for id in "${TRAIN_IDS[@]}"; do + idx=$((idx+1)) + run_item_team "$idx" "$id" "$slot" & + pids+=("$!") + slot=$(( (slot+1) % WORKERS )) + if [ "${#pids[@]}" -ge "$WORKERS" ]; then + wait "${pids[@]}" + pids=() + fi + done + [ "${#pids[@]}" -gt 0 ] && wait "${pids[@]}" + return 0 +} + +# poll daemon 日志里的 canary / staging / 毕业关键行（在线 epoch settle 时观测） +poll_canary_log() { # $1=label + local label=$1 + log " [$label] daemon canary/staging/graduation lines (tail):" + grep -aiE 'commit_to_staging|staging|canary|promot|merge_staging|discard_staging|🎓|graduat|baby.*main|CS (attribution|score)' \ + "$DAEMON_LOG" 2>/dev/null | tail -25 | while IFS= read -r l; do log " cs| $l"; done +} + +# ── 5e) 决策触发式 val 集评测：后台监听 .val_request.json loop ──────────────── +# 设计（why）：canary 晋升比较的不再是纯 ux_score，而是综合分 = +# 0.5*(val 集解题正确率, 归一化 0-10) + 0.5*(ux_score)。 +# 旧做法在每个 epoch settle **固定跑一次** val——时机和 canary 决策错位：早期 +# canary 攒够 ux 想裁决时 .val_scores.json 还没该 sha 的分，回退纯 ux，val 白测。 +# 新做法（决策触发式）：xskill canary 攒够 ux 但缺 val 分时，写一个 +# /.val_request.json {main_sha,staging_sha,requested_ts} 并挂起 +# （action=waiting_val，不动 staging）。本 entrypoint 起一个**后台常驻 loop**， +# 每隔 VAL_LOOP_INTERVAL 秒扫 server 技能仓所有 .val_request.json；发现一个就对 +# 它记的 main_sha / staging_sha **各跑 val** 写 .val_scores.json[sha]，删掉该 +# .val_request.json。下一轮 canary tick 两 sha 的分都齐了 → 走综合分裁决。 +# 这样"canary 一挂起 → 后台很快补分 → 下轮裁决"闭环对齐，每次晋升都真用上 val。 +# +# 怎么"用某个版本技能解 val"（最简方案）：技能是 git 分支。对每个 sha： +# 1. `git show :SKILL.md` 物化到一个**隔离 val HOME**的 .claude/skills// +# （该 HOME 不在 server watch 范围、也不是任何 connect client 的 home，故 val +# 解题轨迹绝不被当训练数据入库 / 污染 A/B）。 +# 2. multi_turn_rollout.py --max-turns 1（single turn：测正确率不产训练轨迹）对 +# val 10 题各解一遍，读 result json 的 success 算 acc=对数/10。 +# 3. 写 .val_scores.json[]={acc,n}。 +# val 解题带与 rollout 同一套 deepseek env（ANTHROPIC_BASE_URL / AUTH_TOKEN）。 +# 可终止性：loop 是 background job，写一个 $VAL_LOOP_STOP 文件即令其下一拍退出； +# 主流程全 epoch 跑完后 touch 该文件 + wait loop PID，绝不留僵尸。 +VAL_ITEMS_JSON="${VAL_ITEMS_JSON:-$TRAIN_SPLIT/val/items.json}" +VAL_TIMEOUT="${XSKILL_VAL_TIMEOUT:-300}" # 单 val item single-turn 超时 +VAL_HOME_ROOT="${XSKILL_VAL_HOME_ROOT:-/root/valhome}" # 隔离、未注册 watch 的 HOME 根 +VAL_ENABLE="${XSKILL_VAL_ENABLE:-true}" +VAL_LOOP_STOP="${XSKILL_VAL_LOOP_STOP:-$HOME_ROOT/VAL_LOOP_STOP}" # 触此文件令 loop 退出 +VAL_LOOP_PID="" + +mapfile -t VAL_IDS < <(python -c "import json;[print(i['id']) for i in json.load(open('$VAL_ITEMS_JSON'))]" 2>/dev/null) + +# 把某技能子仓某分支的 SKILL.md(+references/scripts/assets/templates) 物化到 dst// +materialize_side_skill() { # $1=skill_subrepo $2=branch $3=dst_skills_dir $4=name + local d=$1 br=$2 dst=$3 name=$4 + rm -rf "$dst/$name"; mkdir -p "$dst/$name" + if ! "$GIT" -C "$d" show "$br:SKILL.md" > "$dst/$name/SKILL.md" 2>/dev/null; then + "$GIT" -C "$d" show "$br:skill.md" > "$dst/$name/SKILL.md" 2>/dev/null || return 1 + fi + # 附带目录（best-effort：用 git archive 抽该分支的子树） + for ex in references scripts assets templates; do + "$GIT" -C "$d" archive "$br" "$ex" 2>/dev/null | tar -x -C "$dst/$name" 2>/dev/null || true + done + return 0 +} + +# 在隔离 HOME 里对 val 全集单轮解题，回显 acc（对数/总数） +eval_val_acc() { # $1=val_home（其 .claude/skills 已物化好该 side 技能） + local vh=$1 correct=0 total=0 id ws out + for id in "${VAL_IDS[@]}"; do + total=$((total+1)) + ws="/tmp/valws_$$_${id}"; out="/tmp/valres_$$_${id}.json" + rm -rf "$ws"; mkdir -p "$ws" + ( + export CLAUDE_CONFIG_DIR="$vh/.claude" + export ANTHROPIC_BASE_URL=https://api.deepseek.com/anthropic + export ANTHROPIC_AUTH_TOKEN="$DEEPSEEK_API_KEY" + export CLAUDE_CODE_EXEC_USE_SDK=cli + export IS_SANDBOX=1 + export XSKILL_SKILL_MODE=native + python /app/multi_turn_rollout.py \ + --item-id "$id" --data-root "$DATA_ROOT" --chome "$vh/.claude" \ + --claude-path "$CLAUDE_ABS" --model "$EVAL_MODEL" \ + --max-turns 1 \ + --workspace "$ws" --out "$out" --timeout "$VAL_TIMEOUT" + ) >/dev/null 2>&1 + if [ -f "$out" ]; then + local succ; succ=$(python -c "import json;print(1 if json.load(open('$out')).get('success') else 0)" 2>/dev/null || echo 0) + correct=$((correct+succ)) + fi + rm -rf "$ws" "$out" + done + python -c "print(round($correct/$total,4) if $total else 0.0)" +} + +# 对某 sha 跑一遍 val，幂等写 .val_scores.json[sha]={acc,n}。已有该 sha 条目则跳过。 +# $1=skill_subrepo $2=skill_name $3=sha $4=tag(供日志区分 main/staging) +eval_val_for_sha() { + local d=$1 name=$2 sha=$3 tag=$4 + [ -n "$sha" ] || return 0 + # 已测过该 sha → 跳过（幂等，省算力） + if [ -f "$d/.val_scores.json" ] && python - "$d/.val_scores.json" "$sha" <<'PY' 2>/dev/null +import json, sys +try: + d = json.load(open(sys.argv[1], encoding="utf-8")) or {} +except Exception: + d = {} +e = d.get(sys.argv[2]) +sys.exit(0 if isinstance(e, dict) and e.get("acc") is not None else 1) +PY + then + log " [valloop] $name/$tag ${sha:0:8} already scored, skip" + return 0 + fi + local vh="$VAL_HOME_ROOT/${name}_${tag}_${sha:0:8}" + rm -rf "$vh"; mkdir -p "$vh/.claude/projects" "$vh/.claude/skills" + cat > "$vh/.claude/settings.json" <<'JSON' +{"skillListingBudgetFraction":0.1,"permissions":{"defaultMode":"bypassPermissions"}} +JSON + # materialize_side_skill 第 2 参吃 git ref —— sha 也是合法 ref（git show :..）。 + if ! materialize_side_skill "$d" "$sha" "$vh/.claude/skills" "$name"; then + log " [valloop] $name/$tag ${sha:0:8}: no SKILL.md at sha, skip"; rm -rf "$vh"; return 0 + fi + local acc; acc=$(eval_val_acc "$vh") + python - "$d/.val_scores.json" "$sha" "$acc" "${#VAL_IDS[@]}" <<'PY' +import json, os, sys +path, sha, acc, n = sys.argv[1], sys.argv[2], float(sys.argv[3]), int(sys.argv[4]) +data = {} +if os.path.exists(path): + try: data = json.load(open(path, encoding="utf-8")) or {} + except Exception: data = {} +data[sha] = {"acc": acc, "n": n} +json.dump(data, open(path, "w", encoding="utf-8"), ensure_ascii=False, indent=2) +print(f"[val] {os.path.basename(os.path.dirname(path))} {sha[:8]} acc={acc} n={n}") +PY + log " [valloop] $name/$tag ${sha:0:8} acc=$acc → wrote .val_scores.json" + rm -rf "$vh" +} + +# 处理一个技能子仓的 .val_request.json：读 main_sha/staging_sha，各跑 val 写分， +# 然后删掉该 .val_request.json（loop 服完即清；canary 下轮 tick 见两 sha 分齐 → +# 综合裁决，clear_val_request 幂等再清一次也无副作用）。 +# 注意只在两 sha 都成功写入分后才删——任一失败（如该 sha 无 SKILL.md）则保留请求， +# 下一拍重试，避免分没补齐就把请求删了让 canary 永远 waiting_val。 +process_one_val_request() { # $1=skill_subrepo + local d=$1 name; name=$(basename "$d") + local req="$d/.val_request.json" + [ -f "$req" ] || return 0 + local m_sha s_sha + m_sha=$(python -c "import json;print(json.load(open('$req')).get('main_sha',''))" 2>/dev/null) + s_sha=$(python -c "import json;print(json.load(open('$req')).get('staging_sha',''))" 2>/dev/null) + [ -n "$m_sha$s_sha" ] || { log " [valloop] $name: empty/bad .val_request.json, removing"; rm -f "$req"; return 0; } + log " [valloop] serving val request $name main=${m_sha:0:8} staging=${s_sha:0:8}" + eval_val_for_sha "$d" "$name" "$m_sha" main + eval_val_for_sha "$d" "$name" "$s_sha" staging + # 确认两 sha 分都已落盘才删请求（eval_val_for_sha 幂等，已测过会跳过仍算齐） + if python - "$d/.val_scores.json" "$m_sha" "$s_sha" <<'PY' 2>/dev/null +import json, sys +try: + d = json.load(open(sys.argv[1], encoding="utf-8")) or {} +except Exception: + sys.exit(1) +def ok(sha): + e = d.get(sha) + return isinstance(e, dict) and e.get("acc") is not None +sys.exit(0 if (ok(sys.argv[2]) and ok(sys.argv[3])) else 1) +PY + then + rm -f "$req" + log " [valloop] $name: both sha scored → removed .val_request.json" + else + log " [valloop] $name: not all sha scored yet → keep .val_request.json for retry" + fi +} + +# 后台常驻 loop：每 VAL_LOOP_INTERVAL 秒扫所有 .val_request.json 补 val 分。 +# 见 $VAL_LOOP_STOP 文件即退出（主流程结束时 touch 它 + wait，保证不留僵尸）。 +val_request_loop() { + log " [valloop] background val-request loop started (interval=${VAL_LOOP_INTERVAL}s, stop=$VAL_LOOP_STOP)" + while [ ! -f "$VAL_LOOP_STOP" ]; do + if [ -d "$XSKILL_SKILL_DIR" ] && [ -n "$GIT" ] && [ "${#VAL_IDS[@]}" -gt 0 ]; then + for d in "$XSKILL_SKILL_DIR"/*/; do + [ -f "$VAL_LOOP_STOP" ] && break + [ -d "$d/.git" ] || continue + local nm; nm=$(basename "$d"); [ "${nm#.}" != "$nm" ] && continue + [ -f "$d/.val_request.json" ] || continue + process_one_val_request "$d" + done + fi + [ -f "$VAL_LOOP_STOP" ] && break + sleep "$VAL_LOOP_INTERVAL" + done + log " [valloop] stop file seen → background val-request loop exiting" +} + +start_val_loop() { # 幂等：已起则跳过；val 关 / 无 val 题则不起 + [ "$VAL_ENABLE" = "true" ] || { log " [valloop] disabled (XSKILL_VAL_ENABLE!=true)"; return; } + [ "${#VAL_IDS[@]}" -gt 0 ] || { log " [valloop] no val items in $VAL_ITEMS_JSON, loop not started"; return; } + [ -n "$VAL_LOOP_PID" ] && kill -0 "$VAL_LOOP_PID" 2>/dev/null && return + rm -f "$VAL_LOOP_STOP" + val_request_loop & + VAL_LOOP_PID=$! + log " [valloop] started pid=$VAL_LOOP_PID" +} + +stop_val_loop() { # 令 loop 退出并回收，避免僵尸（sidecar sleep infinity 前调用） + [ -n "$VAL_LOOP_PID" ] || return + touch "$VAL_LOOP_STOP" + log " [valloop] stop requested (pid=$VAL_LOOP_PID); waiting up to ${VAL_LOOP_INTERVAL}s to drain" + for _ in $(seq 1 "$VAL_LOOP_INTERVAL"); do + kill -0 "$VAL_LOOP_PID" 2>/dev/null || break + sleep 1 + done + kill -0 "$VAL_LOOP_PID" 2>/dev/null && kill "$VAL_LOOP_PID" 2>/dev/null || true + wait "$VAL_LOOP_PID" 2>/dev/null || true + log " [valloop] stopped" +} + +# ── 6) 多 epoch 训练循环 ──────────────────────────────────────────────── +mapfile -t TRAIN_IDS < <(python -c "import json;[print(i['id']) for i in json.load(open('$TRAIN_SPLIT/train/items.json'))]") +log "train items = ${#TRAIN_IDS[@]} :: ${TRAIN_IDS[*]}; EPOCHS=$EPOCHS WORKERS=$WORKERS MAX_TURNS=$MAX_TURNS CANARY_SETTLE=$CANARY_SETTLE" + +# epoch0 屏障 flush + poll 等 daemon 消费（消费=baby->main 批量毕业完成信号） +FLUSH_WAIT="${XSKILL_FLUSH_WAIT:-600}" +flush_barrier() { + touch "$BARRIER_FILE" + log "epoch0 done → barrier dropped $BARRIER_FILE; waiting up to ${FLUSH_WAIT}s for cold flush (baby→main)" + local consumed=0 _ + for _ in $(seq 1 "$FLUSH_WAIT"); do + ensure_daemon + if [ ! -f "$BARRIER_FILE" ]; then consumed=1; log "barrier consumed → epoch0 cold flush done"; break; fi + sleep 1 + done + [ "$consumed" = 1 ] || log "WARN: barrier still present after ${FLUSH_WAIT}s (cold flush may be incomplete)" + sleep 10; ensure_daemon +} + +# 所有 epoch 起头都先拉起 worker 独立 home + 常驻 connect client（幂等：已跑则跳过）。 +# epoch0 也必须有 client 在跑——server_mode 只 ingest client 上传的轨迹。 +start_all_clients() { + for k in $(seq 0 $((WORKERS-1))); do + setup_worker_home "$k" + start_worker_client "$k" + done +} + +# 分段 poll settle：每 ~settle/6 打一次 canary/staging 日志，给闭环足够 watcher tick。 +settle_with_polling() { # $1=secs $2=label + local total=$1 label=$2 seg elapsed=0 + seg=$(( total / 6 )); [ "$seg" -lt 20 ] && seg=20 + while [ "$elapsed" -lt "$total" ]; do + sleep "$seg"; elapsed=$((elapsed+seg)); ensure_daemon + poll_canary_log "$label +${elapsed}s" + done +} + +# 决策触发式 val 后台 loop：整个在线阶段常驻，按 .val_request.json 按需补 val 分。 +# 在 epoch0 之前就起（幂等）——epoch1+ canary 一挂起请求它立刻能跟上。 +start_val_loop + +for ep in $(seq 0 $((EPOCHS-1))); do + ensure_daemon + start_all_clients + sleep 8 # 给 client 一拍完成 register + 首次 sync（装好已毕业 main 技能） + if [ "$ep" -eq 0 ]; then + # ── epoch0：冷启动（client 上传 -> server cold_flush 毕业 baby→main）── + log "=== epoch 0 / $((EPOCHS-1)) :: COLD-START via team clients (upload → baby→main graduation via barrier) ===" + run_epoch + log "epoch0 rollout done; settle ${FINAL_SETTLE}s for upload(debounce)→ingest→cluster before barrier" + settle_with_polling "$FINAL_SETTLE" "ep0-settle" + flush_barrier + snapshot + log "=== epoch0 graduation snapshot above ===" + else + # ── epoch1..N-1：team-CS 灰度在线进化（不落屏障，cold_flush 已在 epoch0 耗尽）── + log "=== epoch $ep / $((EPOCHS-1)) :: TEAM-CS canary online evolution (WORKERS=$WORKERS clients) ===" + run_epoch + # settle：让 upload→distill→score→SkillEdit 把"已有 main 技能的更新"路由到 + # staging 并攒 ux；canary 攒够 ux 缺 val 时写 .val_request.json 挂起 + # （waiting_val），后台 val loop 按需补分，下一轮 canary tick 综合裁决。 + # 不再固定跑一次 val——决策触发式由 loop 闭环，settle 只需给足 watcher tick + + # loop 补分往返的时间。start_val_loop 幂等：loop 若被异常杀掉会在此重起。 + start_val_loop + log "epoch $ep rollout done; settle ${CANARY_SETTLE}s (staging open → ux → waiting_val → val loop fills → composite decide)" + settle_with_polling "$CANARY_SETTLE" "ep$ep" + snapshot + fi +done + +# 训练全 epoch 跑完 → 停后台 val loop，避免僵尸常驻。 +stop_val_loop + +log "=== final skill-repo snapshot ===" +snapshot +log "=== daemon log tail (cluster / SkillEdit / staging / canary / graduation) ===" +grep -aiE 'cluster|skilledit|skill_edit|graduat|baby|promot|atom|staging|canary|commit_to_staging|merge_staging|discard_staging|🎓|🌱' \ + "$DAEMON_LOG" 2>/dev/null | tail -40 \ + || tail -40 "$DAEMON_LOG" 2>/dev/null + +# git log per skill（看 staging 分支 + 晋升痕迹） +log "=== per-skill git log (branches + recent commits) ===" +if [ -d "$XSKILL_SKILL_DIR" ] && [ -n "$GIT" ]; then + for d in "$XSKILL_SKILL_DIR"/*/; do + [ -d "$d/.git" ] || continue + name=$(basename "$d") + log " --- $name branches ---" + "$GIT" -C "$d" branch -a 2>/dev/null | while IFS= read -r l; do log " br| $l"; done + "$GIT" -C "$d" log --oneline --all -8 2>/dev/null | while IFS= read -r l; do log " lg| $l"; done + done +fi + +N=$(collect_skills) +log "collected $N skill folder(s) into $SKILLS_OUT" + +echo xskill > "$SKILL_OUT/ALGO" + +if [ "${N:-0}" -gt 0 ] && find "$SKILLS_OUT" -name SKILL.md | grep -q .; then + touch "$SKILL_OUT/DONE" + log "DONE: $(find "$SKILLS_OUT" -name SKILL.md | wc -l) SKILL.md under $SKILLS_OUT" + find "$SKILLS_OUT" -name SKILL.md | while IFS= read -r f; do log " -> $f"; done +else + log "FATAL: collected zero SKILL.md — not writing DONE. Sleeping (sidecar must not exit)." +fi + +# ── /output 交付 ──────────────────────────────────────────────────────── +OUTPUT_DIR="${OUTPUT_DIR:-/shared/out}" +mkdir -p "$OUTPUT_DIR/algo" "$OUTPUT_DIR/trajectories" "$OUTPUT_DIR/rollout_results" 2>/dev/null || true +[ -d "$XSKILL_SKILL_DIR" ] && cp -a "$XSKILL_SKILL_DIR" "$OUTPUT_DIR/algo/skill_repo" 2>/dev/null || true +cp -a "$DAEMON_LOG" "$OUTPUT_DIR/algo/" 2>/dev/null || true +cp -a /tmp/mtres_*.log "$OUTPUT_DIR/algo/" 2>/dev/null || true +cp -a /tmp/client_w*.log "$OUTPUT_DIR/algo/" 2>/dev/null || true +cp -a "$SKILLS_OUT" "$OUTPUT_DIR/algo/skills_shipped" 2>/dev/null || true +cp -a /tmp/mtres_*.json "$OUTPUT_DIR/rollout_results/" 2>/dev/null || true +# 所有 worker home + server home 产出的轨迹 jsonl（递归收集，扁平化） +for proj in "$CHOME/projects" /root/whome_*/.claude/projects; do + [ -d "$proj" ] || continue + find "$proj" -name '*.jsonl' 2>/dev/null | while IFS= read -r jf; do + rel=$(echo "$jf" | sed "s#^/##; s#/#__#g") + cp -a "$jf" "$OUTPUT_DIR/trajectories/$rel" 2>/dev/null || true + done +done +NTRAJ=$(find "$OUTPUT_DIR/trajectories" -name '*.jsonl' 2>/dev/null | wc -l) +NRES=$(find "$OUTPUT_DIR/rollout_results" -name '*.json' 2>/dev/null | wc -l) +log "copied algo artifacts -> $OUTPUT_DIR/algo; trajectories=$NTRAJ jsonl; rollout_results=$NRES json" + +# sidecar restartPolicy:Always —— 永不退出 +log "entrypoint reached steady state; sleep infinity" +sleep infinity diff --git a/benchmark/spreadsheet_xarena/algo_app/multi_turn_rollout.py b/benchmark/spreadsheet_xarena/algo_app/multi_turn_rollout.py new file mode 100644 index 00000000..4b37e63e --- /dev/null +++ b/benchmark/spreadsheet_xarena/algo_app/multi_turn_rollout.py @@ -0,0 +1,779 @@ +#!/usr/bin/env python3 +"""多轮纠错 rollout —— 针对单个 SpreadsheetBench 题目，驱动 claude CLI 多轮对话。 + +目的（why） +========== +当前 xskill 的训练轨迹是"单轮"的：solver 一次写出 solution.py 就结束，无论对错。 +单轮轨迹里没有"我错了 → 我怎么定位 → 我怎么改对"的修正过程，蒸馏出来的技能 +往往是净负面的（教坏 solver）。本脚本产出"错误 → 修正 → 成功"的多轮高质量轨迹： + + turn0 solver 读题 + 用 Skill 工具匹配技能 → 把解法写到 solution.py → 执行 + 判错 用（复制自官方 evaluator 的）cell 对比逻辑判对错 + 反馈 若错，由"人味反馈"（隐藏 golden 真实数值，只点出哪些 cell 错了 + 怎么想）引导 + 续会话 claude 用 `--resume ` 续上**同一会话**修正 + 循环 直到做对 或 到 max_turns + +claude CLI 的多轮会话会被它自己写到 `$CLAUDE_CONFIG_DIR/projects/*.jsonl`， +xskill daemon 监听该目录即可把整段多轮对话入库、蒸馏成技能。 + +自包含约束（重要） +================== +- 只依赖 openpyxl + 标准库。**不 import** skillopt 的任何模块（怕拖重依赖如 torch）。 + cell 对比 / 反馈 / prompt 全部在本文件内本地实现（逻辑复制自下列参考）： + * cell 对比口径 <- skillopt/envs/spreadsheetbench/evaluator.py + * 隐藏 golden 反馈 <- skillopt/envs/spreadsheetbench/codegen_agent._build_eval_feedback + * workspace / 路径 <- skillopt/envs/spreadsheetbench/rollout.py + * claude CLI cmd 构造 + env 隔离 <- skillopt/model/codex_harness._run_claude_code_cli_exec +- fail-loud 但不崩循环：claude 没输出 / 没写 solution.py / session_id 解析失败 / + 执行报错——都记录到 result 并合理处理，不静默吞错、也不让整脚本 crash。 + +claude CLI `-p` 非交互 + `--resume` 多轮的不确定点（需容器实测） +=============================================================== +见文件末 `RESUME_CAVEAT` 常量与 README 报告。简言之：`-p`(print/非交互)模式下 +`--resume -- ` 是否把新 prompt 当作"同一会话的下一条 user 消息" +续上、并把整段对话追加进**同一个** projects/*.jsonl，需要在有 claude CLI 的容器里 +实测验证。本脚本按"会续上同一会话、首轮 json 输出里能解析到 session_id"的假设实现。 +""" +from __future__ import annotations + +import argparse +import datetime +import json +import os +import subprocess +import sys +import traceback + +import openpyxl + + +# ════════════════════════════════════════════════════════════════════════════ +# 1) cell 对比逻辑 —— 逐字复制自 skillopt/envs/spreadsheetbench/evaluator.py +# 口径：数值 round 2 位；datetime.time 去微秒；datetime 转 Excel 序列号取整； +# ""与None视为相等；类型不同即 FAIL。保证训练判对错与官方评测一致。 +# ════════════════════════════════════════════════════════════════════════════ + +def _datetime_to_float(dt: datetime.datetime) -> float: + excel_start_date = datetime.datetime(1899, 12, 30) + delta = dt - excel_start_date + return delta.days + delta.seconds / 86400.0 + + +def _transform_value(v): + if isinstance(v, bool): + # openpyxl 可能返回 Python bool；官方把 bool 当数值处理(round(float(True))==1.0) + return round(float(v), 2) + if isinstance(v, (int, float)): + return round(float(v), 2) + if isinstance(v, datetime.time): + return str(v)[:-3] + if isinstance(v, datetime.datetime): + return round(_datetime_to_float(v), 0) + if isinstance(v, str): + try: + return round(float(v), 2) + except ValueError: + return v + return v + + +def _compare_cell_value(v1, v2) -> bool: + v1 = _transform_value(v1) + v2 = _transform_value(v2) + if (v1 == "" and v2 is None) or (v1 is None and v2 == ""): + return True + if (v1 == "" and v2 == "") or (v1 is None and v2 is None): + return True + if type(v1) is not type(v2): + return False + return v1 == v2 + + +def _col_num2name(n: int) -> str: + name = "" + while n > 0: + n, r = divmod(n - 1, 26) + name = chr(65 + r) + name + return name + + +def _col_name2num(name: str) -> int: + num = 0 + for c in name: + num = num * 26 + (ord(c) - ord("A") + 1) + return num + + +def _parse_range(range_str: str): + start_cell, end_cell = range_str.split(":") + sc = "".join(ch for ch in start_cell if ch.isalpha()) + sr = "".join(ch for ch in start_cell if ch.isdigit()) + ec = "".join(ch for ch in end_cell if ch.isalpha()) + er = "".join(ch for ch in end_cell if ch.isdigit()) + return (_col_name2num(sc), int(sr)), (_col_name2num(ec), int(er)) + + +def _generate_cell_names(range_str: str): + if ":" not in range_str: + return [range_str] + (sc, sr), (ec, er) = _parse_range(range_str) + cols = [_col_num2name(i) for i in range(sc, ec + 1)] + return [f"{c}{r}" for c in cols for r in range(sr, er + 1)] + + +def _iter_answer_targets(answer_position: str, default_sheet: str): + """把 answer_position 串拆成 [(sheet_name, [cell_name, ...]), ...]。 + + answer_position 形如 "I12:I13" 或 "Sheet2!A1:B3" 或逗号分隔的多段； + 不含 '!' 时落到 default_sheet（gt 的第一个 sheet，与官方一致）。 + """ + targets = [] + for scr in (answer_position or "").split(","): + scr = scr.strip() + if not scr: + continue + if "!" in scr: + sheet_name, cell_range = scr.split("!", 1) + sheet_name = sheet_name.strip().strip("'\"") + else: + sheet_name = default_sheet + cell_range = scr + cell_range = cell_range.strip().strip("'\"") + targets.append((sheet_name, _generate_cell_names(cell_range))) + return targets + + +def evaluate_output(pred_path: str, gold_path: str, answer_position: str) -> dict: + """对比 pred 与 gold 在 answer_position 处的 cell。 + + 返回 {"ok": bool, "reason": str, "wrong_cells": [{"cell","got"}...]}。 + wrong_cells 只含 pred 自己的值(got)，**不含 expected 真值**——供反馈生成时使用， + 天然避免 golden 泄漏。fail-loud：文件缺失/打不开都如实返回 ok=False + reason。 + """ + if not os.path.exists(pred_path): + return {"ok": False, "reason": "output file does not exist", "wrong_cells": []} + try: + wb_gt = openpyxl.load_workbook(filename=gold_path, data_only=True) + wb_proc = openpyxl.load_workbook(filename=pred_path, data_only=True) + except Exception as e: # noqa: BLE001 + return {"ok": False, "reason": f"load error: {e}", "wrong_cells": []} + + try: + default_sheet = wb_gt.sheetnames[0] + wrong_cells: list[dict] = [] + reason = "" + for sheet_name, cell_names in _iter_answer_targets(answer_position, default_sheet): + if sheet_name not in wb_proc.sheetnames: + if not reason: + reason = f"worksheet not found in output: {sheet_name}" + # 整个 sheet 缺失：把该 sheet 的目标 cell 全部记为错(got 缺失) + for cn in cell_names: + wrong_cells.append({"cell": f"{sheet_name}!{cn}", "got": None}) + continue + ws_gt = wb_gt[sheet_name] + ws_proc = wb_proc[sheet_name] + for cn in cell_names: + cg = ws_gt[cn].value + cp = ws_proc[cn].value + if not _compare_cell_value(cg, cp): + wrong_cells.append({"cell": f"{sheet_name}!{cn}", "got": cp}) + if not reason: + reason = f"value mismatch @ {sheet_name}!{cn}" + ok = len(wrong_cells) == 0 + return {"ok": ok, "reason": "" if ok else reason, "wrong_cells": wrong_cells} + finally: + wb_gt.close() + wb_proc.close() + + +# ════════════════════════════════════════════════════════════════════════════ +# 2) workbook 预览 —— 复制自 codegen_agent._preview_workbook +# ════════════════════════════════════════════════════════════════════════════ + +def preview_workbook(path: str, max_rows: int = 5, max_cols: int = 20) -> str: + """生成 workbook 前几行的文本预览（用于 turn0 的 input 预览）。""" + wb = openpyxl.load_workbook(path, data_only=False) + chunks: list[str] = [] + try: + for sheet_name in wb.sheetnames: + ws = wb[sheet_name] + chunks.append( + f"## Sheet: {sheet_name} " + f"(dim={ws.dimensions}, max_row={ws.max_row}, max_col={ws.max_column})" + ) + for row in ws.iter_rows( + min_row=1, + max_row=min(ws.max_row, max_rows), + max_col=min(ws.max_column, max_cols), + values_only=False, + ): + cells = [] + for cell in row: + v = cell.value + if v is None: + cells.append(f"{cell.coordinate}=") + else: + s = str(v) + if len(s) > 40: + s = s[:37] + "..." + cells.append(f"{cell.coordinate}={s}") + chunks.append(" | ".join(cells)) + if ws.max_row > max_rows: + chunks.append(f"... ({ws.max_row - max_rows} more rows)") + chunks.append("") + finally: + wb.close() + return "\n".join(chunks) + + +# ════════════════════════════════════════════════════════════════════════════ +# 3) 数据定位 —— 复制 rollout._find_test_cases 的命名约定（简化为单 case） +# ════════════════════════════════════════════════════════════════════════════ + +def load_dataset(data_root: str) -> list[dict]: + """读 /dataset.json（官方为 list[dict]）。""" + path = os.path.join(data_root, "dataset.json") + if not os.path.exists(path): + raise FileNotFoundError(f"dataset.json not found at {path}") + with open(path, encoding="utf-8") as f: + data = json.load(f) + if isinstance(data, dict): + data = data.get("data") or list(data.values()) + return list(data) + + +def find_item(items: list[dict], item_id: str) -> dict: + """按 str(id) 匹配（id 可能是 int 或 str）。找不到 fail-loud。""" + want = str(item_id) + for it in items: + if str(it.get("id")) == want: + return it + raise KeyError(f"item id={item_id!r} not found in dataset.json") + + +def find_input_and_golden(task_dir: str) -> tuple[str, str]: + """在 spreadsheet// 找 input 和 golden。 + + 优先 *_init.xlsx + *_golden.xlsx（verified_400 的 1__init.xlsx 命名）； + 回退 initial.xlsx + golden.xlsx。fail-loud：找不到就抛错。 + """ + import glob as _glob + inits = sorted(_glob.glob(os.path.join(task_dir, "*_init.xlsx"))) + for ip in inits: + gp = ip.replace("_init.xlsx", "_golden.xlsx") + if os.path.exists(gp): + return ip, gp + bare_init = os.path.join(task_dir, "initial.xlsx") + bare_gold = os.path.join(task_dir, "golden.xlsx") + if os.path.exists(bare_init) and os.path.exists(bare_gold): + return bare_init, bare_gold + raise FileNotFoundError( + f"no (input, golden) pair found in {task_dir} " + f"(looked for *_init.xlsx+*_golden.xlsx, initial.xlsx+golden.xlsx)" + ) + + +# ════════════════════════════════════════════════════════════════════════════ +# 4) workspace 准备：run_solution.py 模板（注入 INPUT_PATH / OUTPUT_PATH） +# ════════════════════════════════════════════════════════════════════════════ + +def _build_run_solution_driver(input_path: str, output_path: str) -> str: + """run_solution.py：定义 INPUT_PATH/OUTPUT_PATH，再 exec 同目录 solution.py。 + + 复制 codegen_agent._build_codex_driver 的思路：剥掉 solution.py 里用户自己写的 + INPUT_PATH/OUTPUT_PATH 赋值，强制用我们注入的路径，避免硬编码工作簿。 + """ + return ( + "import pathlib\n" + "import re\n" + "import sys\n" + "import traceback\n\n" + f"INPUT_PATH = {input_path!r}\n" + f"OUTPUT_PATH = {output_path!r}\n" + "code = pathlib.Path(__file__).with_name('solution.py').read_text(encoding='utf-8')\n" + "# 剥掉用户在 solution.py 里自己写的 INPUT_PATH/OUTPUT_PATH 赋值\n" + "code = re.sub(r'^\\s*(INPUT_PATH|OUTPUT_PATH)\\s*=\\s*.+$', '', code, flags=re.MULTILINE)\n" + "g = {'__name__': '__main__', 'INPUT_PATH': INPUT_PATH, 'OUTPUT_PATH': OUTPUT_PATH}\n" + "try:\n" + " exec(compile(code, 'solution.py', 'exec'), g, g)\n" + "except Exception:\n" + " traceback.print_exc()\n" + " sys.exit(2)\n" + ) + + +def prepare_workspace(workspace: str, input_src: str) -> tuple[str, str]: + """建 workspace：拷 input.xlsx、写 run_solution.py。返回 (input_path, output_path)。""" + import shutil + os.makedirs(workspace, exist_ok=True) + input_path = os.path.join(workspace, "input.xlsx") + output_path = os.path.join(workspace, "output.xlsx") + shutil.copy2(input_src, input_path) + driver = _build_run_solution_driver(input_path, output_path) + with open(os.path.join(workspace, "run_solution.py"), "w", encoding="utf-8") as f: + f.write(driver) + return input_path, output_path + + +# ════════════════════════════════════════════════════════════════════════════ +# 5) prompt 构造 +# ════════════════════════════════════════════════════════════════════════════ + +def build_turn0_prompt( + instruction: str, + input_path: str, + instruction_type: str, + answer_position: str, +) -> str: + """turn0：完整任务描述 + input 预览 + 用 Skill 工具匹配技能 + 写 solution.py 的指示。""" + try: + preview = preview_workbook(input_path) + except Exception as e: # noqa: BLE001 + preview = f"(failed to preview workbook: {e})" + extra = "" + if instruction_type: + extra += f"\nInstruction type: {instruction_type}" + if answer_position: + extra += f"\nExpected answer position: {answer_position}" + return ( + f"# Instruction\n{instruction}\n{extra}\n\n" + f"# Input spreadsheet preview\n{preview}\n\n" + "# Task\n" + "- First, use the Skill tool to find and invoke any available skill whose " + "description matches this spreadsheet task, then follow its guidance.\n" + "- Inspect `input.xlsx` in this workspace if useful.\n" + "- Write the final Python solution to `solution.py`. The script must read the " + "workbook from the `INPUT_PATH` variable and write the modified workbook to " + "`OUTPUT_PATH`. Preserve all other cells unchanged.\n" + "- The preview may be truncated — do not hardcode row counts or assume the data " + "ends at the last previewed row; iterate over all actual rows instead.\n" + "- You may run `python run_solution.py` to validate locally.\n" + "- In your final message, confirm `solution.py` was written and summarize the approach." + ) + + +def build_human_feedback(wrong_cells: list[dict], instruction: str) -> str: + """第一阶段（--human-sim off）的"人味反馈"：golden-diff 直接生成。 + + 复制 codegen_agent._build_eval_feedback 的核心约束——**列出预测错的 cell + 你的值， + 但绝不写出 expected 真值**——但语气改成同事口吻而非机器报错。 + + 例（同事味）： + "你 I12:I13 这几格的结果好像不太对。你算出来是 3 / 5 …… + 看看是不是 ……？另外检查一下 ……" + 而不是机器味的 "cell I12 got=3 expected=5"。 + """ + if not wrong_cells: + # 不应发生（有反馈才进这里），但 fail-soft 给个通用提示 + return ( + "运行起来了，但结果还差点意思。再核对一下题目要求的那几个单元格，" + "看看计算口径有没有偏差？" + ) + + # 把错的 cell 按 sheet!列 归类，方便用"这几格"的口语化指代 + cell_bits = [] + for wc in wrong_cells: + got = wc.get("got") + # got=None 用"是空的"表述，更像人话 + if got is None: + cell_bits.append(f"{wc['cell']} 还是空的") + else: + cell_bits.append(f"{wc['cell']} 你填的是 {got!r}") + + listed = "；".join(cell_bits) + # 给几个"同事会顺口提醒"的排查方向，引导但不给答案、不泄漏 expected + hints = ( + "我猜可能的坑：(1) 日期/时间被当成文本处理了？(2) 公式没被求值——" + "openpyxl 不会算 Excel 公式，得在 Python 里把结果算出来再写值；" + "(3) 空格的填充/跳过逻辑、或者数值的四舍五入位数对不上；" + "(4) 行数是不是只覆盖了预览里那几行、漏了后面的数据。" + ) + return ( + f"嘿，跑通了但结果对不上。具体是这几格：{listed}。\n" + f"对照下题目要求（“{instruction[:120]}{'…' if len(instruction) > 120 else ''}”），" + f"{hints}\n" + "你按这些方向再排查一下，把 `solution.py` 改对，然后我再帮你看。" + ) + + +def build_exec_error_feedback(err: str) -> str: + """执行报错时的反馈（同样同事口吻，把报错贴给它让它改）。""" + return ( + "诶，你这版 `solution.py` 跑的时候直接报错了：\n\n" + f"```\n{err[:2500]}\n```\n\n" + "先把这个错修掉吧，记得继续用 `INPUT_PATH` / `OUTPUT_PATH` 变量，别硬编码路径。" + ) + + +def build_no_solution_feedback() -> str: + """没写出 solution.py 时的反馈。""" + return ( + "我没在 workspace 里找到 `solution.py`。请把完整解法写进 `solution.py`，" + "用 `INPUT_PATH` 读、写到 `OUTPUT_PATH`，可以先 `python run_solution.py` 自测。" + ) + + +# 第二阶段占位：用 LLM 把 golden-diff 翻译成更自然的人味反馈 +def human_feedback_via_llm(wrong_cells, instruction, **kwargs): # noqa: D401 + """[第二阶段实现] 用 LLM 生成更自然的人味反馈。 + + 设计：把 wrong_cells（只含 got，无 expected）+ instruction 喂给一个小模型， + 让它扮演"看了你输出但没看答案的同事"，产出引导性反馈。当前未实现， + --human-sim on 时先回退到 golden-diff 版的 build_human_feedback，避免阻塞第一阶段。 + """ + raise NotImplementedError( + "human_feedback_via_llm 是第二阶段特性，尚未实现；" + "当前 --human-sim 仅作占位，运行时会回退到 build_human_feedback。" + ) + + +# ════════════════════════════════════════════════════════════════════════════ +# 6) claude CLI 调用 —— 复制自 codex_harness._run_claude_code_cli_exec 的 cmd/env +# ════════════════════════════════════════════════════════════════════════════ + +# tools 必须含 Skill，否则 Claude Code 会把 skill_listing 整个剥掉 => 技能永远进不了预算。 +_DEFAULT_TOOLS = "Read,Write,Edit,Bash,Skill" + + +def call_claude( + *, + claude_path: str, + work_dir: str, + chome: str, + model: str, + prompt: str, + timeout: int, + output_format: str, + resume_session_id: str | None = None, + tools: str = _DEFAULT_TOOLS, +) -> tuple[str, str, int]: + """调一次 claude CLI。返回 (stdout, raw含stderr, returncode)。 + + cmd 构造照 codex_harness._run_claude_code_cli_exec： + claude -p --output-format --permission-mode bypassPermissions + --add-dir --tools --allowedTools + --model --setting-sources user,project + [--resume ] -- + env 继承 os.environ 并设 CLAUDE_CONFIG_DIR=（隔离到实验 config， + 实验技能在 /skills 下，--setting-sources user,project 才能加载到）。 + cwd=work_dir。 + + fail-loud：超时/非零返回都如实带回 raw + returncode，由调用方判定，不在此吞错。 + """ + cmd = [ + claude_path, + "-p", + "--output-format", output_format, + "--permission-mode", "bypassPermissions", + "--add-dir", work_dir, + "--tools", tools, + "--allowedTools", tools, + "--setting-sources", "user,project", + ] + if model: + cmd += ["--model", model] + if resume_session_id: + # 续接同一会话（详见文件头与 RESUME_CAVEAT：-p 模式下的确切行为需容器实测） + cmd += ["--resume", resume_session_id] + cmd += ["--", prompt] + + run_env = dict(os.environ) + run_env["CLAUDE_CONFIG_DIR"] = chome + + try: + proc = subprocess.run( + cmd, + cwd=work_dir, + capture_output=True, + text=True, + timeout=timeout, + env=run_env, + ) + except subprocess.TimeoutExpired as exc: + stdout = exc.stdout or "" + stderr = exc.stderr or "" + raw = stdout + if stderr: + raw = f"{raw}\n[stderr]\n{stderr}" if raw else stderr + return "", (raw or f"timeout after {timeout}s"), 124 + + stdout = proc.stdout or "" + stderr = proc.stderr or "" + raw = stdout + if stderr: + raw = f"{raw}\n[stderr]\n{stderr}" if raw else stderr + return stdout, raw, proc.returncode + + +def parse_session_id(stdout: str) -> str: + """从 `--output-format json` 的 stdout 解析 session_id。 + + claude CLI 的 -p json 输出是一个 JSON 对象（含 session_id/result 等字段）。 + fail-loud：解析不到返回空串，由调用方记录"session_id 解析失败"。 + """ + s = (stdout or "").strip() + if not s: + return "" + # 先试整体 JSON + try: + obj = json.loads(s) + if isinstance(obj, dict): + sid = obj.get("session_id") or obj.get("sessionId") or "" + if sid: + return str(sid) + except (json.JSONDecodeError, ValueError): + pass + # 回退：逐行找可解析的 JSON 对象（stream-json / 多行场景） + for line in s.splitlines(): + line = line.strip() + if not line.startswith("{"): + continue + try: + obj = json.loads(line) + except (json.JSONDecodeError, ValueError): + continue + if isinstance(obj, dict): + sid = obj.get("session_id") or obj.get("sessionId") or "" + if sid: + return str(sid) + return "" + + +# ════════════════════════════════════════════════════════════════════════════ +# 7) 主流程：多轮纠错循环 +# ════════════════════════════════════════════════════════════════════════════ + +def run_multi_turn(args) -> dict: + """对单个 item 跑多轮纠错 rollout，返回 result dict。""" + result = { + "item_id": str(args.item_id), + "success": False, + "turns_used": 0, + "session_id": "", + "per_turn": [], + "fail_reason": "", + } + + # ── setup：定位 item / 文件 / 准备 workspace ────────────────────────── + items = load_dataset(args.data_root) + item = find_item(items, args.item_id) + instruction = item["instruction"] + instruction_type = item.get("instruction_type", "") + answer_position = item.get("answer_position", "") + answer_sheet = item.get("answer_sheet", "") + # answer_position 不带 sheet 但题目给了 answer_sheet 时，拼成 Sheet!Range（与官方一致） + if answer_position and answer_sheet and "!" not in answer_position: + answer_position_eval = f"{answer_sheet}!{answer_position}" + else: + answer_position_eval = answer_position + result["answer_position"] = answer_position_eval + + sp = item.get("spreadsheet_path", f"spreadsheet/{args.item_id}") + task_dir = sp if os.path.isabs(sp) else os.path.join(args.data_root, sp) + input_src, golden_path = find_input_and_golden(task_dir) + + input_path, output_path = prepare_workspace(args.workspace, input_src) + result["workspace"] = args.workspace + + session_id = "" + + # ── 多轮循环 ────────────────────────────────────────────────────────── + for turn in range(args.max_turns): + per = {"turn": turn, "exec_ok": False, "eval_ok": False, + "n_wrong_cells": None, "claude_ok": False, "note": ""} + + # a) 组装本轮 prompt + if turn == 0: + prompt = build_turn0_prompt( + instruction, input_path, instruction_type, answer_position_eval + ) + else: + prompt = feedback # 上一轮末尾生成的人味反馈（见循环末） + + # b) 调 claude CLI。turn0 用 json 拿 session_id；turn>0 用 --resume 续会话。 + # 先删旧 output.xlsx，确保拿到的是本轮真实产物（不是上一轮残留）。 + if os.path.exists(output_path): + try: + os.remove(output_path) + except OSError: + pass + + if turn == 0: + stdout, raw, rc = call_claude( + claude_path=args.claude_path, work_dir=args.workspace, + chome=args.chome, model=args.model, prompt=prompt, + timeout=args.timeout, output_format="json", + ) + session_id = parse_session_id(stdout) + result["session_id"] = session_id + if not session_id: + # session_id 解析失败：后续轮无法 --resume 续会话。 + # fail-loud 记录；本轮仍尝试执行（solution.py 可能已写出）。 + per["note"] = "session_id parse failed (cannot --resume in later turns)" + else: + if not session_id: + # 没有 session_id，无法续会话——终止循环（fail-loud，不静默重开新会话） + per["note"] = "no session_id; cannot resume — aborting multi-turn loop" + result["per_turn"].append(per) + if not result["fail_reason"]: + result["fail_reason"] = "no-session-id-to-resume" + break + stdout, raw, rc = call_claude( + claude_path=args.claude_path, work_dir=args.workspace, + chome=args.chome, model=args.model, prompt=prompt, + timeout=args.timeout, output_format="text", + resume_session_id=session_id, + ) + + per["returncode"] = rc + per["claude_ok"] = bool((stdout or "").strip()) and rc == 0 + if not per["claude_ok"] and not per["note"]: + per["note"] = f"claude produced no usable output (rc={rc})" + + # c) 检查 solution.py 是否写出 + solution_path = os.path.join(args.workspace, "solution.py") + if not os.path.exists(solution_path): + # 没写出 solution.py：本轮失败。还有后续轮则给反馈续上，否则终止。 + per["note"] = (per["note"] + "; " if per["note"] else "") + "no solution.py written" + result["per_turn"].append(per) + result["turns_used"] = turn + 1 + if turn + 1 >= args.max_turns: + if not result["fail_reason"]: + result["fail_reason"] = "no-solution-py" + break + feedback = build_no_solution_feedback() + continue + + # d) 执行 run_solution.py 产出 output.xlsx + exec_ok, exec_err = _exec_run_solution(args.workspace, args.timeout) + per["exec_ok"] = exec_ok + + if not exec_ok: + # 执行报错：把报错反馈给它，进下一轮（或终止） + result["per_turn"].append(per) + result["turns_used"] = turn + 1 + if turn + 1 >= args.max_turns: + if not result["fail_reason"]: + result["fail_reason"] = f"exec-error: {exec_err[:200]}" + break + feedback = build_exec_error_feedback(exec_err) + continue + + # e) 判对错（复制来的 cell 对比逻辑） + ev = evaluate_output(output_path, golden_path, answer_position_eval) + per["eval_ok"] = ev["ok"] + per["n_wrong_cells"] = len(ev["wrong_cells"]) + result["per_turn"].append(per) + result["turns_used"] = turn + 1 + + if ev["ok"]: + # 做对了 —— 记 success、break + result["success"] = True + result["fail_reason"] = "" + break + + # 做错了 —— 生成人味反馈进下一轮（若还有轮次） + if turn + 1 >= args.max_turns: + if not result["fail_reason"]: + result["fail_reason"] = f"eval-mismatch: {ev['reason'][:200]}" + break + + if args.human_sim: + # 第二阶段：尝试 LLM 人味反馈；未实现则回退 golden-diff（注明） + try: + feedback = human_feedback_via_llm(ev["wrong_cells"], instruction) + except NotImplementedError: + feedback = build_human_feedback(ev["wrong_cells"], instruction) + else: + feedback = build_human_feedback(ev["wrong_cells"], instruction) + + return result + + +def _exec_run_solution(workspace: str, timeout: int) -> tuple[bool, str]: + """执行 workspace/run_solution.py，产出 output.xlsx。返回 (ok, err)。 + + 用子进程跑（隔离 solver 代码的副作用/崩溃），fail-loud 把 stdout+stderr 带回。 + """ + run_solution = os.path.join(workspace, "run_solution.py") + output_path = os.path.join(workspace, "output.xlsx") + try: + proc = subprocess.run( + [sys.executable, run_solution], + cwd=workspace, + capture_output=True, + text=True, + timeout=timeout, + ) + except subprocess.TimeoutExpired: + return False, f"run_solution.py timeout after {timeout}s" + if proc.returncode != 0: + return False, (proc.stdout + "\n" + proc.stderr).strip() + if not os.path.exists(output_path): + return False, "run_solution.py finished but output.xlsx was not created" + return True, "" + + +# claude CLI -p 模式 --resume 多轮的最大不确定点（需容器实测） +RESUME_CAVEAT = ( + "claude CLI 在 `-p`(print/非交互) 模式下 `--resume -- ` 的确切行为需实测：\n" + " (1) 它是否把新 prompt 当作同一会话的下一条 user 消息续上、并把整段多轮对话\n" + " 追加进同一个 $CLAUDE_CONFIG_DIR/projects/*.jsonl（而非新开一个 session 文件）？\n" + " (2) `--output-format json` 首轮输出里 session_id 字段的确切键名（session_id / sessionId）；\n" + " (3) --resume 是否要求与首轮相同的 --add-dir / cwd / model 才能成功续接；\n" + " (4) bypassPermissions 下 Bash 自测(python run_solution.py)是否真能在该 workspace 跑通。\n" + "本脚本按 (1) 成立、(2) 取 session_id 的假设实现；不成立时需改用 --continue 或落地会话目录。" +) + + +def main() -> int: + parser = argparse.ArgumentParser( + description="单题 SpreadsheetBench 多轮纠错 rollout（驱动 claude CLI）" + ) + parser.add_argument("--item-id", required=True, help="dataset.json 里的题目 id（int/str 皆可）") + parser.add_argument("--data-root", default="/data", help="含 dataset.json 与 spreadsheet/ 的根目录") + parser.add_argument("--chome", required=True, help="CLAUDE_CONFIG_DIR（隔离的 claude config 目录）") + parser.add_argument("--claude-path", default="claude", help="claude CLI 可执行文件路径") + parser.add_argument("--model", default="deepseek-v4-flash", help="模型名") + parser.add_argument("--max-turns", type=int, default=5, help="最大轮数") + parser.add_argument("--workspace", default=None, help="工作目录（默认 /tmp/mt_）") + parser.add_argument("--out", default=None, help="result json 落盘路径（可选）") + parser.add_argument("--timeout", type=int, default=420, help="每轮 claude / 执行的超时秒数") + parser.add_argument("--human-sim", action="store_true", + help="[第二阶段占位] 用 LLM 生成人味反馈；当前未实现，会回退 golden-diff") + args = parser.parse_args() + + if not args.workspace: + args.workspace = f"/tmp/mt_{args.item_id}" + + try: + result = run_multi_turn(args) + except Exception as e: # noqa: BLE001 + # setup 阶段的硬错误（找不到 item / 文件等）也写成 result，整脚本不 crash + result = { + "item_id": str(args.item_id), + "success": False, + "turns_used": 0, + "session_id": "", + "per_turn": [], + "fail_reason": f"setup-error: {type(e).__name__}: {e}", + "error": traceback.format_exc(), + } + + if args.out: + os.makedirs(os.path.dirname(os.path.abspath(args.out)), exist_ok=True) + with open(args.out, "w", encoding="utf-8") as f: + json.dump(result, f, ensure_ascii=False, indent=2) + + # stdout 打印一行简洁结果 + print( + f"[mt-rollout] id={result['item_id']} " + f"success={result['success']} turns={result['turns_used']} " + f"session={result.get('session_id') or '-'} " + f"reason={result.get('fail_reason') or 'ok'}" + ) + return 0 if result["success"] else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/benchmark/spreadsheet_xarena/algo_app/requirements.txt b/benchmark/spreadsheet_xarena/algo_app/requirements.txt new file mode 100644 index 00000000..9d1194c7 --- /dev/null +++ b/benchmark/spreadsheet_xarena/algo_app/requirements.txt @@ -0,0 +1,22 @@ +# xskill + SkillOpt(rollout) 运行所需 Python 依赖。 +# xskill 与 SkillOpt 本体用 `pip install -e` 从烘焙源码装（见 Dockerfile）， +# 这里装它们的第三方依赖。agno 固定到经验运行验证过的版本，避免大版本 API 漂移。 +# ── xskill deps ── +agno==2.6.12 +dulwich>=0.21 +rank-bm25>=0.2 +detect-secrets>=1.5 +fastapi +uvicorn +sse-starlette +tqdm +# ── 公共 / SkillOpt deps ── +openai>=1.30.0 +pyyaml>=6.0 +numpy>=1.24.0 +openpyxl>=3.1.0 +pandas>=2.0.0,<3 +httpx>=0.27.0 +azure-identity>=1.15.0 +azure-core>=1.30.0 +tenacity>=8.0.0 diff --git a/benchmark/spreadsheet_xarena/algo_app/sync_skills_to.sh b/benchmark/spreadsheet_xarena/algo_app/sync_skills_to.sh new file mode 100755 index 00000000..168042f0 --- /dev/null +++ b/benchmark/spreadsheet_xarena/algo_app/sync_skills_to.sh @@ -0,0 +1,50 @@ +#!/usr/bin/env bash +# Sync xskill MAIN-branch skills into a target .claude/skills dir. +# usage: sync_skills_to.sh [--mirror] +# Behaviour matches Phase-3a sync_skills.sh: +# - copies ONLY skills whose git branch == main (real promotion); baby stubs skipped +# - skips skills the daemon already symlinked into DEST (avoids copy/symlink collision) +# - prunes the daemon's `..replaced-by-symlink` cruft +# --mirror (used for the val home): DEST is a plain dir not touched by the daemon, +# so we do a clean full materialise (remove stale, copy all main skills). +set -uo pipefail +# Vendored reference script. The xskill algo entrypoint uses its own collect_skills; +# this is kept for compatibility/attribution. No hardcoded host path: the xskill skill +# dir is taken from env — set XSKILL_SKILL_DIR, or XSKILL_HOME (defaults to $HOME/.xskill). +XSKILL_SKILL_DIR=${XSKILL_SKILL_DIR:-${XSKILL_HOME:-$HOME/.xskill}/skill} +DEST=${1:?need dest skills dir} +MODE=${2:-} +GIT=$(command -v git) + +mkdir -p "$DEST" +# prune daemon-collision cruft so the listing stays clean +find "$DEST" -maxdepth 1 -name '.*.replaced-by-symlink' -exec rm -rf {} + 2>/dev/null + +if [ "$MODE" = "--mirror" ]; then + # fresh materialise: clear non-hidden entries first + find "$DEST" -maxdepth 1 -mindepth 1 -exec rm -rf {} + 2>/dev/null +fi + +synced=0; linked=0; skipped=0 +if [ -d "$XSKILL_SKILL_DIR" ]; then + for d in "$XSKILL_SKILL_DIR"/*/; do + [ -d "$d" ] || continue + name=$(basename "$d") + [ "${name#.}" != "$name" ] && continue + [ -f "$d/SKILL.md" ] || continue + if [ -d "$d/.git" ] && [ -n "$GIT" ]; then + br=$("$GIT" -C "$d" rev-parse --abbrev-ref HEAD 2>/dev/null) + [ "$br" = "main" ] || { skipped=$((skipped+1)); continue; } + fi + if [ "$MODE" != "--mirror" ] && [ -L "$DEST/$name" ]; then linked=$((linked+1)); continue; fi + rm -rf "$DEST/$name" + mkdir -p "$DEST/$name" + cp "$d/SKILL.md" "$DEST/$name/SKILL.md" + for extra in references scripts assets; do + [ -d "$d/$extra" ] && cp -r "$d/$extra" "$DEST/$name/$extra" + done + synced=$((synced+1)) + done +fi +echo "[sync->$DEST] copied=$synced daemon-linked=$linked skipped(non-main)=$skipped" +ls -1 "$DEST" 2>/dev/null | grep -v '^\.' | sed 's/^/[sync] /' diff --git a/benchmark/spreadsheet_xarena/algo_app/train_split/test/items.json b/benchmark/spreadsheet_xarena/algo_app/train_split/test/items.json new file mode 100644 index 00000000..0637a088 --- /dev/null +++ b/benchmark/spreadsheet_xarena/algo_app/train_split/test/items.json @@ -0,0 +1 @@ +[] \ No newline at end of file diff --git a/benchmark/spreadsheet_xarena/algo_app/train_split/train/items.json b/benchmark/spreadsheet_xarena/algo_app/train_split/train/items.json new file mode 100644 index 00000000..9b3e731f --- /dev/null +++ b/benchmark/spreadsheet_xarena/algo_app/train_split/train/items.json @@ -0,0 +1,155 @@ +[ + { + "id": 32438, + "instruction": "How can I format the cells in column J to only display the standard time in 00:00:00 and AM/PM from I2, which contains both the date and the time in the format 'DD/MM/YYYY 00:00:00'? Using the =RIGHT(I2,8) function returns the time as a string of 8 numbers without the desired time formatting (e.g., 00:00:00), even after attempting to apply time formatting to the cell.", + "spreadsheet_path": "spreadsheet/32438", + "instruction_type": "Cell-Level Manipulation", + "answer_position": "J2:J4" + }, + { + "id": "398-14", + "instruction": "I have data across multiple sheets where I need to match and sum values from columns that are repeated across these sheets. To do this, I need to insert a column called 'BALANCE' which subtracts the 'RET' value from the 'SALE' columns. Whenever I add new sheets that have the same structure, the macro should be able to adapt to include them. The results, along with headers, should be compiled in the 'collection' sheet. Reference the values and logic of the existing rows, then complete the table for any missing information from the other sheets by adding it to the bottom of the current table, but do not alter the order of the original rows. Make sure that rows are aggregated across all sheets where “TY” and “OR” are the same. Additionally, instead of showing a formula in column G, it should display the calculated value, and any empty cells should show a hyphen instead of a zero, and any negatives should have a text color hex code of #FF0000, and column headers should be un-bolded and left-side aligned. All other formatting should remain unchanged, and any new data added should match. Columns A, E, F, and G now align right, while columns B, C, and D align left. Make all text unbold, and font Calibri 11pts.", + "spreadsheet_path": "spreadsheet/398-14", + "instruction_type": "Sheet-Level Manipulation", + "answer_position": "'COLLECTION'!A2:G9", + "answer_sheet": "COLLECTION", + "data_position": "A1:G9" + }, + { + "id": 47766, + "instruction": "I am attempting to calculate my agent's yearly production which includes yearly bonuses that reset annually. In the Excel spreadsheet I attached, I have used SUMIF formulas to calculate their total production based on certain criteria, such as including \"*PE*\" in my searches: =SUMIF($H$8:$H$37,\"*PE*\",$C$8:$C$37), =SUMIF($H$41:$H$58,\"*PE*\",$C$41:$C$58), =SUMIF($H$62:$H$74,\"*PE*\",$C$62:$C$74). However, I am having trouble adapting these formulas to work with a date range for the start and end of the year, using closing dates that are found in Column F. I would like to find out how to modify these formulas or use a different approach to consider the date range for accurately calculating annual production. Please change the values in table J39:O53 to reflect the correct yearly totals.", + "spreadsheet_path": "spreadsheet/47766", + "instruction_type": "Cell-Level Manipulation", + "answer_position": "K40" + }, + { + "id": 48365, + "instruction": "How can I sum amounts from a data sheet using SUMIFS when a user can select up to 3 regions for a product, with an 'All' option that sums all amounts? I need a formula that can handle multiple region choices in one criteria using an 'OR' logic, or a more elegant alternative.SUMIFS(Data!$C:$C,Data!$A:$A,$C$2,Data!$B:$B,$C11)", + "spreadsheet_path": "spreadsheet/48365", + "instruction_type": "Cell-Level Manipulation", + "answer_position": "'Dashboard'!C4" + }, + { + "id": 32255, + "instruction": "I'm not an expert with spreadsheets, but I have a specific problem I can't figure out. I'm working on a spreadsheet and have run into an issue regarding how to ignore empty or blank cells in a formula I'm using. The formula needs to perform a certain action only on cells that contain data, and I need help adjusting it accordingly. An example of my problem should be evident in the attached spreadsheet I've provided. How can I modify my formula to ignore these empty or blank cells?\nA10 through A13 has no data entered, I need column D to ignore these cells (i.e., empty), until there is data entered.", + "spreadsheet_path": "spreadsheet/32255", + "instruction_type": "Cell-Level Manipulation", + "answer_position": "D2:D13" + }, + { + "id": 10747, + "instruction": "I've attached a file containing a basic table and another table showing only a portion of that basic data. I've also included a formula that is supposed to extract the desired part, but it isn't working, and I can't identify the mistake. Could you please help me understand where I've gone wrong with the formula?\nIF((@$A$3:$A$8=$I3)*(@$B$3:$B$8=$J3),C3,0). The correct formula should sum the net profit from the first table based on the year and share no values in cells I3 and J3. It should be placed into cell K6.", + "spreadsheet_path": "spreadsheet/10747", + "instruction_type": "Cell-Level Manipulation", + "answer_position": "K6" + }, + { + "id": 50916, + "instruction": "How can I create an Excel school calendar where typing a cycle day number into a column automatically populates the classes for that day across each row? My children's school operates on a seven-day cycle, and there's a double period that could complicate matters. I want to avoid manual copy/paste because of irregular 'off days' that require adjusting the cycle day and associated classes. I managed a formula that works in one cell but am struggling to copy it across other cells as shown by the problems in cells C12, E12:H12, and D13:D14. IF(A$12=A$2,C$2,IF(A$12=A$3,C$3,IF(A$12=A$4,C$4,IF(A$12=A$5,C$5,IF(A$12=A$6,C$6,IF(A$12=A$7,C$7,IF(A$12=A$8,C$8)))))))", + "spreadsheet_path": "spreadsheet/50916", + "instruction_type": "Cell-Level Manipulation", + "answer_position": "C12:H14" + }, + { + "id": "577-40", + "instruction": "How can I delete entire rows in an Excel worksheet if both Column J and Column K are either blank or contain any of the following specific values: -, 0, $, $0, or $0.0? I need solution other than formula or a VBA macro that performs this task without deleting rows where only one of the cells in Column J or K meets these criteria, as I need to retain rows where at least one of these columns contains actual data.", + "spreadsheet_path": "spreadsheet/577-40", + "instruction_type": "Sheet-Level Manipulation", + "answer_position": "'Sheet2'!A1:K54", + "answer_sheet": "Sheet2", + "data_position": "A1:K54" + }, + { + "id": 35742, + "instruction": "I've created a spreadsheet to record the results of an upcoming horse show, which involves multiple sections and aggregate awards that need to be calculated. My IF functions seem to be working correctly, but I'm struggling to sum the scores at the end of the row. I've tried using the =VALUE, =SUM, and =SUMPRODUCT functions but haven't been successful. In my spreadsheet, I'm trying to add the results shown in the blue cells across to a green cell where I've manually entered the total. How can I correctly sum these values?", + "spreadsheet_path": "spreadsheet/35742", + "instruction_type": "Cell-Level Manipulation", + "answer_position": "O4:O7" + }, + { + "id": 46121, + "instruction": "How do I configure Excel to read a date (month) on column A of the 'Transactions' sheet and then place the total amount spent in that month under the corresponding month and category on the '2022' sheet? Specifically, I have set up an income and expense tracker for my farm business where I input transactions on one sheet and categorize them. While I can tally the transactions based on selected categories, I'm struggling to get Excel to correctly allocate these tallied sums into the appropriate monthly column on a second sheet. As an example, a payment made in April is incorrectly being included in the March column on the second sheet.\n \nMy current formula is:\nIF(Transactions!D3=Transactions!D2,\"\",SUMIF(Transactions!D:D,Transactions!D3,Transactions!C:C))\n\nWrite a formula in the income section of the '2022' sheet. Apply Currency formatting.", + "spreadsheet_path": "spreadsheet/46121", + "instruction_type": "Cell-Level Manipulation", + "answer_position": "'2022'!B5:M8" + }, + { + "id": 51090, + "instruction": "How can I calculate the difference between the total cartons received as indicated in Column M and the errors present in Columns N to R, while filtering for a specific warehouse (e.g., warehouse 27), error codes (specifically for each of II, IR, IT, OV, PI with code II in column N), date, and users (e.g., CHROGIL1, CHDSPOLJ, CHSJEFFE, CHBTHOMA for each error code) across a data set? Additionally, for the IR error code listed in Column O, I only want to include positive numbers from the Errors Table. The goal is to simplify a formula that can achieve this, as my current one is too complex and not providing the correct value, such as the expected 699 for Q2.", + "spreadsheet_path": "spreadsheet/51090", + "instruction_type": "Cell-Level Manipulation", + "answer_position": "Daily Numbers'!Q3:Q24" + }, + { + "id": 51249, + "instruction": "How do I create an Excel formula that outputs different results based on the text values of two different cells. For example, in D1, if B1 contains 'Description A' and B2 is blank, then D1 should display 'Single A'; if B1 contains 'Description B' and B2 is blank, then D1 should display 'Single B'; and if B1 contains 'Description A' and B2 contains 'Description B', then D1 should display 'Multiple'? I have also provided a sample spreadsheet showing the input and desired result. Do this output for each cell in column D that next to a cell that says \"Result:\". Fill any cells in column D with content with RGB: 226-239-218", + "spreadsheet_path": "spreadsheet/51249", + "instruction_type": "Cell-Level Manipulation", + "answer_position": "D1,D5,D9" + }, + { + "id": "82-30", + "instruction": "How can I modify my VBA code to only copy and paste whole numbers from a range of columns (A to F) in the 'Raw Data' sheet, starting from row 1 and continuing across columns A to F in the 'Numbers' sheet, without including numbers with decimal points and ensuring a maximum of 6 numbers per row? You can refer to Manual Result sheet for outcome, and it should be in the exact same sorting order.", + "spreadsheet_path": "spreadsheet/82-30", + "instruction_type": "Sheet-Level Manipulation", + "answer_position": "'Numbers'!A1:F9", + "answer_sheet": "Numbers", + "data_position": "Raw Data!'A1:F18, 'Manual Result!'A1:F9" + }, + { + "id": 56274, + "instruction": "I need an Excel formula that will automatically populate cells D9, D10, D11, and D12 with the Opening Balance, Debits, Credits, and Closing Balance respectively, based on the Fiscal Month in cell D7. The details and expected results are outlined in the attached excel sheet.", + "spreadsheet_path": "spreadsheet/56274", + "instruction_type": "Cell-Level Manipulation", + "answer_position": "D9:D12", + "exclude": "golden has data validation dropdown set to a completely different value" + }, + { + "id": 57445, + "instruction": "How can I create a formula in Column E on my Pricing Worksheet that will reference manually inputted data in Columns C and D and return the corresponding Cost data from a different worksheet named 'Package & Weight Data'? The table and necessary data are in an attached Excel file named 'Cost-Calculator.xlsx'.", + "spreadsheet_path": "spreadsheet/57445", + "instruction_type": "Cell-Level Manipulation", + "answer_position": "'Pricing'!E2:E5" + }, + { + "id": 46646, + "instruction": "How can I calculate the percentage of night shifts for a given month, considering a variable year range from 2010 to 2022. Take into account leap years and the specific days of the month. Please perform this task and fill out row 11 using the year value shown. Output the values in decimal format, such that a) a zero value has no decimal places, b) a value that only contains a single, non-zero decimal place has one decimal place, and c) all other values have nine decimal places.", + "spreadsheet_path": "spreadsheet/46646", + "instruction_type": "Cell-Level Manipulation", + "answer_position": "B11:M11" + }, + { + "id": "105-24", + "instruction": "In my Excel file, the 'Open' sheet contains IDs in Columns C and K. Find all these IDs and replace them with their respective Names. The corresponding Names for these IDs are located in Columns A and B of the 'Productivity' sheet. I require assistance with creating a macro or to automate this replacement process and also automatically consider any new additions to Columns A and B in the future for replacements. Do not create instructions, explanations, or a ID_Lookup_Helper tab. Do not create add data to any other columns other than C or K in the Open tab. Do not hide any data.", + "spreadsheet_path": "spreadsheet/105-24", + "instruction_type": "Sheet-Level Manipulation", + "answer_position": "'Open'!C2:C23", + "answer_sheet": "Open", + "data_position": "Open!'B1:P23,'Productivity!'A1:E21'" + }, + { + "id": 6239, + "instruction": "I'm creating an incentive program to reward my employees based on their performance and need help with setting it up in Excel. Employees have individual goals that range from any number between 0% to 100%. The 'Variance to Goal' is calculated as 1 - (Achieved/Goal), varying from - infinity to +infinity. Another parameter, 'Metric2', also ranges from - infinity to +infinity without a fixed goal. The incentive percentage is determined based on two filters: 'Goal' is divided into four buckets based on values ranging from 65% to 99%, and 'Variance to Goal' is noted within a range of 0% to more than 7% under each 'Goal' range. Then, 'Metric2' is divided into five buckets with values from -20% to +20%, and in each, the final incentive percentage an employee should receive is specified. For instance, if an employee has a 'Goal' between 65% to 74%, a 'Variance to Goal' between 0% to 3%, and a 'Metric2' between -20% to -10%, they should get a 3% incentive. If 'Metric2' was more than 20% under the same 'Goal' and 'Variance', they should get a 7% incentive. How can I set up Excel to calculate and output these incentive percentages based on the provided criteria and ranges?", + "spreadsheet_path": "spreadsheet/6239", + "instruction_type": "Cell-Level Manipulation", + "answer_position": "G2:G19" + }, + { + "id": "414-20", + "instruction": "Delete all rows above the first occurrence of the text “Invoice No.” in column A. The macro should search for the phrase (case-insensitive) and delete everything above it, leaving the “Invoice No.” row and all rows below untouched.", + "spreadsheet_path": "spreadsheet/414-20", + "instruction_type": "Sheet-Level Manipulation", + "answer_position": "'Sheet1'!A4:A18", + "answer_sheet": "Sheet1", + "data_position": "A4:A14" + }, + { + "id": "165-23", + "instruction": "Here's what I need: For Sheet 1, using VBA or alternative, delete all rows where column C is not the #N/A error value (Do not change any formatting and let row 1 remain blank). Do not delete anything else\n\nI need assistance with a macro for Sheet1 in Excel. The macro, named delrow2, should delete rows where the cell in column C does not contain the #N/A error value. I've attempted to write these macros, but both are returning errors. Additionally, after receiving advice to use the IsError function or smarter methods like filter or Range.SpecialCells, I created two more macros, delrow3 and delrow4, which also failed to work correctly. At one point, I tried replacing .Value with .Text in the If codeline as suggested, which seemed to resolve the issue. However, I'm unsure if it is the most efficient solution. I have limited knowledge of macro coding, and my attempts are attached in a file for review.", + "spreadsheet_path": "spreadsheet/165-23", + "instruction_type": "Sheet-Level Manipulation", + "answer_position": "'Sheet1'!A2:N73", + "answer_sheet": "Sheet1", + "data_position": "Sheet1!'A2:N73,'Desired Sheet1 after delrow1!'A2:N69,'Desired Sheet1 after delrow2!'A2:N5'" + } +] \ No newline at end of file diff --git a/benchmark/spreadsheet_xarena/algo_app/train_split/val/items.json b/benchmark/spreadsheet_xarena/algo_app/train_split/val/items.json new file mode 100644 index 00000000..f6bc5d4f --- /dev/null +++ b/benchmark/spreadsheet_xarena/algo_app/train_split/val/items.json @@ -0,0 +1,80 @@ +[ + { + "id": 40892, + "instruction": "I need a formula to extract the color mentioned in each cell of column 1 and display that same color in column 2. For example, if a cell in column 1 says 'Dress with red details,' I want the corresponding cell in column 2 to show 'Red'. I have uploaded the file for reference. If no answer is found, output an empty string. Only add the colors mentioned in the list located in Column D.", + "spreadsheet_path": "spreadsheet/40892", + "instruction_type": "Cell-Level Manipulation", + "answer_position": "B2:B17" + }, + { + "id": 48745, + "instruction": "How can I look up multiple product codes that are located within the same cell, separated by semicolons, in an Excel table where these codes span columns 'C' and the lookup table covers columns 'G' and 'H'? Specifically, I need to identify which group each product code belongs to, with 'Group 1', 'Group 2', or 'BOTH' as potential outcomes for each cell if it contains multiple product codes that map to different groups. Additionally, if a cell contains only one product code, the semicolon is still present. Example scenarios include a single product code look-up resulting in 'Group 1', multiple product codes from the same group resulting in the respective group name, and multiple product codes from different groups resulting in 'BOTH'.I have a table where column 'c' can contain multiple values separated with a semicolon.\nI want to lookup each separated value in that cell against the lookup table in columns G and H.\nRow 5 would be PRD1 lookups and finds it is in Group 1\nRow 6 would be PRD2;PRD1 lookups and finds they are in Group 1\nRow 7 would be PRD4 lookups and finds it is in Group 2\nRow 8 would be PRD5;PRD6 lookups and finds they are in Group 2\nRow 9 would be PRD1;PRD4;PRD6 lookups and finds they are in Group 1 and Group 2 and would return Group 1, Group 2\nRow 10 would be PRD1;PRD6 lookups and finds they are in Group 1 and Group 2 and would return Group 1, Group 2\nThe semicolon is present in the same cell.\nIf multiple Groups in single cell, please range the index in asending order.", + "spreadsheet_path": "spreadsheet/48745", + "instruction_type": "Cell-Level Manipulation", + "answer_position": "D5:D10" + }, + { + "id": 32612, + "instruction": "How can I create a workday column in an Excel file that identifies each date as the day of the week (e.g., Mon, Tue, Wed, etc.)? Additionally, when the date is a holiday, you do not need to fill in the day of the week, instead you need to marks the date as a public holiday, specifying if it's a working public holiday (PH(working)) or a non-working public holiday (PH(non-working)), using a provided list of public holidays.", + "spreadsheet_path": "spreadsheet/32612", + "instruction_type": "Cell-Level Manipulation", + "answer_position": "E2:E75" + }, + { + "id": "325-44", + "instruction": "I am working with data at a wtype_id, name, and status level, and I need to split the filter data from a column into specific columns of data. I've attached the input data tab and the output data tab for reference. Leave the cell which does not have any data defined blank and leading zeros can be omitted. Retain the 'Input' data tab and update the 'Output' data tab with the result.", + "spreadsheet_path": "spreadsheet/325-44", + "instruction_type": "Sheet-Level Manipulation", + "answer_position": "'Output'!A1:G10", + "answer_sheet": "Output", + "data_position": "Input!'A1:D4,'Output!'A1:G6'" + }, + { + "id": "262-17", + "instruction": "How can I create ranges for header values 'Task' and 'Responsibility' without hard coding the range definitions, and then apply a multiple column sort using VBA in Excel? The macro should identify the positions of these headers dynamically and perform the sorting based on the data provided in the first worksheet, with the expected outcome detailed in the second worksheet attached to the thread. The first level sorts the \"Tasks\" column from small to large, and the second level sorts the \"Responsibilities\" column from small to large", + "spreadsheet_path": "spreadsheet/262-17", + "instruction_type": "Sheet-Level Manipulation", + "answer_position": "Sheet1'!A1:F14", + "answer_sheet": "'Sheet1','Sheet2'", + "data_position": "Sheet1!A1:F14','Sheet2!A1:F14'" + }, + { + "id": "141-20", + "instruction": "How can I create a macro that finds and deletes rows in two Excel sheets named 'PL Recon Items' and 'Statement Recon Items', where the invoice number in column C of 'PL Recon Items' and the value in column D match the reference number in column F and value in column I of 'Statement Recon Items'? The matching rows, which I have highlighted in yellow, should be deleted from both sheets. Could someone provide the code to accomplish this?", + "spreadsheet_path": "spreadsheet/141-20", + "instruction_type": "Sheet-Level Manipulation", + "answer_position": "'PL Recon Items!'A1:D2,'Statement Recon Items!'A1:J3", + "answer_sheet": "PL Recon Items,Statement Recon Items", + "data_position": "PL Recon Items!'A1: D2, 'Statement Recon Items!'A1: J3'" + }, + { + "id": 52216, + "instruction": "I'm having some trouble with getting my INDEX-MATCH formula to return values from subsequent rows beneath the first row (the first row returns the correct values, as desired).\nI'm not sure if a third MATCH argument is needed in the formula, but, if it is, I'm struggling with determining the correct lookup_value & lookup_array syntax.\nI've attached a workbook with some mock data on there to assist - on the 'INPUTS' tab, you'll see the dollar amounts in C15:G15 are displaying correctly which is being pulled from the 'Sheet2\" tab.\nHow can I adjust my INDEX-MATCH formula in the 'INPUTS' tab to retrieve values from rows below the first row, which currently returns the correct data, specifically for a dataset in which the output is contingent on a dropdown selection that alternates between 'Metro' and 'Regional' and affects the displayed dollar amounts?\nSo, I just need some help with getting this formula to display the correct amounts for the rest of the expenses underneath Advertising & Marketing (C15).\nI figure that if just the electricity line can be solved, then I can adopt it for the remainder (hence the lack of numbers for most of the other expenses).\nWrite a formula that will work when copied across columns C through G, and apply it to the first 5 years of Electricity expenses. No need to apply the new formula to the expense rows below Electricity.", + "spreadsheet_path": "spreadsheet/52216", + "instruction_type": "Cell-Level Manipulation", + "answer_position": "'INPUTS'!C15:G16" + }, + { + "id": "22-47", + "instruction": "I need to sort data in column B based on a helper column (J) so that the first entries written in column J are shown first in the output range F:H. Names listed in column J, include all matching rows and keep their original order from the source and do not sort within the group. Names not listed in column J should be arranged as they are in the original data (same order). The sort should skip empty cells, headers, and duplicate items, where duplicates are defined by identical entries in both column B and C. If the helper column J is empty, only then sort alphabetically A–Z and still skip empty cells, headers, and duplicate items. Additionally, I need this sorting process to handle new ranges that contain headers and some names in range A:C, as well as changes in names listed in column J. The final answer should be output in columns G and H, and sort only column H sorted lowest to highest.", + "spreadsheet_path": "spreadsheet/22-47", + "instruction_type": "Sheet-Level Manipulation", + "answer_position": "F2:H10", + "answer_sheet": "sheet1", + "data_position": "sheet1!A1:j24',',ورق1!B1:B11'" + }, + { + "id": 55421, + "instruction": "How can I create a formula in Excel that will check the numbers in Column A and return specific text in Column F based on the following conditions: when a number in Column A is accompanied exclusively by the status 'SCH' in Column D, the text 'FUTURE' should be returned in Column F; if it appears with both statuses 'NO SHOW' and 'SCH', then 'NS/SCHED' should be shown in Column F; if the number comes with the 'NO SHOW' status and a date is present in Column E, 'NO ACTION NEEDED' should be the output; and finally, if there is a 'NO SHOW' status with an empty date field in Column E, 'CALL PT' should appear in Column F?", + "spreadsheet_path": "spreadsheet/55421", + "instruction_type": "Cell-Level Manipulation", + "answer_position": "F2:F20" + }, + { + "id": 56427, + "instruction": "How can I transpose the values in column G where column B has the value 1 and pull sequential values from consecutive rows in column G, stopping after the number of runners specified in column C? The formula should preserve blanks as blanks (not zeros). An example dataset has been provided where the transposition was done partially, but I'm seeking a formulaic solution that will provide the full result. Once complete, shade the range H2:S28 in #E2EFD, and remove decimals for any whole values that range. Then center align all cells to the right of column G.", + "spreadsheet_path": "spreadsheet/56427", + "instruction_type": "Cell-Level Manipulation", + "answer_position": "H2:S28" + } +] \ No newline at end of file diff --git a/benchmark/spreadsheet_xarena/dataset/.gitignore b/benchmark/spreadsheet_xarena/dataset/.gitignore new file mode 100644 index 00000000..f501c027 --- /dev/null +++ b/benchmark/spreadsheet_xarena/dataset/.gitignore @@ -0,0 +1,2 @@ +/data_root/ +/.cache/ diff --git a/benchmark/spreadsheet_xarena/dataset/README.md b/benchmark/spreadsheet_xarena/dataset/README.md new file mode 100644 index 00000000..7d744b82 --- /dev/null +++ b/benchmark/spreadsheet_xarena/dataset/README.md @@ -0,0 +1,26 @@ +# SpreadsheetBench Dataset + +This directory stores the lightweight task split files used by the Xarena leaderboard benchmark. The workbook files are intentionally not committed. + +Committed files: + +```text +train_split/ + train/items.json + val/items.json + test/items.json +``` + +Prepare the workbook data before building: + +```bash +bash prepare_data_root.sh +``` + +By default the script downloads `https://xskill.wiki/zip/xskill-compete.zip` and extracts its `data_root` into this directory. If you already have a local data root, use: + +```bash +SOURCE_DATA_ROOT=/path/to/data_root bash prepare_data_root.sh +``` + +The algorithm image copies the prepared `data_root` to `/data` during the Docker build. diff --git a/benchmark/spreadsheet_xarena/dataset/prepare_data_root.sh b/benchmark/spreadsheet_xarena/dataset/prepare_data_root.sh new file mode 100755 index 00000000..fe4c56ca --- /dev/null +++ b/benchmark/spreadsheet_xarena/dataset/prepare_data_root.sh @@ -0,0 +1,39 @@ +#!/usr/bin/env bash +# Prepare SpreadsheetBench workbook data for local Docker builds. +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +DEST="${DEST:-$SCRIPT_DIR/data_root}" +SOURCE_DATA_ROOT="${SOURCE_DATA_ROOT:-}" +ZIP_URL="${ZIP_URL:-https://xskill.wiki/zip/xskill-compete.zip}" +CACHE_DIR="${CACHE_DIR:-$SCRIPT_DIR/.cache}" +ZIP_PATH="${ZIP_PATH:-$CACHE_DIR/xskill-compete.zip}" + +if [ -n "$SOURCE_DATA_ROOT" ]; then + if [ ! -d "$SOURCE_DATA_ROOT" ]; then + echo "SOURCE_DATA_ROOT does not exist: $SOURCE_DATA_ROOT" >&2 + exit 1 + fi + rm -rf "$DEST" + mkdir -p "$DEST" + rsync -a --delete "$SOURCE_DATA_ROOT/" "$DEST/" + echo "prepared data_root from $SOURCE_DATA_ROOT -> $DEST" + exit 0 +fi + +mkdir -p "$CACHE_DIR" +if [ ! -f "$ZIP_PATH" ]; then + curl -L "$ZIP_URL" -o "$ZIP_PATH" +fi + +TMP_DIR="$(mktemp -d)" +cleanup() { + rm -rf "$TMP_DIR" +} +trap cleanup EXIT + +unzip -q "$ZIP_PATH" 'xskill-compete-pkg/data_root/*' -d "$TMP_DIR" +rm -rf "$DEST" +mkdir -p "$DEST" +rsync -a --delete "$TMP_DIR/xskill-compete-pkg/data_root/" "$DEST/" +echo "prepared data_root from $ZIP_PATH -> $DEST" diff --git a/benchmark/spreadsheet_xarena/dataset/train_split/test/items.json b/benchmark/spreadsheet_xarena/dataset/train_split/test/items.json new file mode 100644 index 00000000..0637a088 --- /dev/null +++ b/benchmark/spreadsheet_xarena/dataset/train_split/test/items.json @@ -0,0 +1 @@ +[] \ No newline at end of file diff --git a/benchmark/spreadsheet_xarena/dataset/train_split/train/items.json b/benchmark/spreadsheet_xarena/dataset/train_split/train/items.json new file mode 100644 index 00000000..9b3e731f --- /dev/null +++ b/benchmark/spreadsheet_xarena/dataset/train_split/train/items.json @@ -0,0 +1,155 @@ +[ + { + "id": 32438, + "instruction": "How can I format the cells in column J to only display the standard time in 00:00:00 and AM/PM from I2, which contains both the date and the time in the format 'DD/MM/YYYY 00:00:00'? Using the =RIGHT(I2,8) function returns the time as a string of 8 numbers without the desired time formatting (e.g., 00:00:00), even after attempting to apply time formatting to the cell.", + "spreadsheet_path": "spreadsheet/32438", + "instruction_type": "Cell-Level Manipulation", + "answer_position": "J2:J4" + }, + { + "id": "398-14", + "instruction": "I have data across multiple sheets where I need to match and sum values from columns that are repeated across these sheets. To do this, I need to insert a column called 'BALANCE' which subtracts the 'RET' value from the 'SALE' columns. Whenever I add new sheets that have the same structure, the macro should be able to adapt to include them. The results, along with headers, should be compiled in the 'collection' sheet. Reference the values and logic of the existing rows, then complete the table for any missing information from the other sheets by adding it to the bottom of the current table, but do not alter the order of the original rows. Make sure that rows are aggregated across all sheets where “TY” and “OR” are the same. Additionally, instead of showing a formula in column G, it should display the calculated value, and any empty cells should show a hyphen instead of a zero, and any negatives should have a text color hex code of #FF0000, and column headers should be un-bolded and left-side aligned. All other formatting should remain unchanged, and any new data added should match. Columns A, E, F, and G now align right, while columns B, C, and D align left. Make all text unbold, and font Calibri 11pts.", + "spreadsheet_path": "spreadsheet/398-14", + "instruction_type": "Sheet-Level Manipulation", + "answer_position": "'COLLECTION'!A2:G9", + "answer_sheet": "COLLECTION", + "data_position": "A1:G9" + }, + { + "id": 47766, + "instruction": "I am attempting to calculate my agent's yearly production which includes yearly bonuses that reset annually. In the Excel spreadsheet I attached, I have used SUMIF formulas to calculate their total production based on certain criteria, such as including \"*PE*\" in my searches: =SUMIF($H$8:$H$37,\"*PE*\",$C$8:$C$37), =SUMIF($H$41:$H$58,\"*PE*\",$C$41:$C$58), =SUMIF($H$62:$H$74,\"*PE*\",$C$62:$C$74). However, I am having trouble adapting these formulas to work with a date range for the start and end of the year, using closing dates that are found in Column F. I would like to find out how to modify these formulas or use a different approach to consider the date range for accurately calculating annual production. Please change the values in table J39:O53 to reflect the correct yearly totals.", + "spreadsheet_path": "spreadsheet/47766", + "instruction_type": "Cell-Level Manipulation", + "answer_position": "K40" + }, + { + "id": 48365, + "instruction": "How can I sum amounts from a data sheet using SUMIFS when a user can select up to 3 regions for a product, with an 'All' option that sums all amounts? I need a formula that can handle multiple region choices in one criteria using an 'OR' logic, or a more elegant alternative.SUMIFS(Data!$C:$C,Data!$A:$A,$C$2,Data!$B:$B,$C11)", + "spreadsheet_path": "spreadsheet/48365", + "instruction_type": "Cell-Level Manipulation", + "answer_position": "'Dashboard'!C4" + }, + { + "id": 32255, + "instruction": "I'm not an expert with spreadsheets, but I have a specific problem I can't figure out. I'm working on a spreadsheet and have run into an issue regarding how to ignore empty or blank cells in a formula I'm using. The formula needs to perform a certain action only on cells that contain data, and I need help adjusting it accordingly. An example of my problem should be evident in the attached spreadsheet I've provided. How can I modify my formula to ignore these empty or blank cells?\nA10 through A13 has no data entered, I need column D to ignore these cells (i.e., empty), until there is data entered.", + "spreadsheet_path": "spreadsheet/32255", + "instruction_type": "Cell-Level Manipulation", + "answer_position": "D2:D13" + }, + { + "id": 10747, + "instruction": "I've attached a file containing a basic table and another table showing only a portion of that basic data. I've also included a formula that is supposed to extract the desired part, but it isn't working, and I can't identify the mistake. Could you please help me understand where I've gone wrong with the formula?\nIF((@$A$3:$A$8=$I3)*(@$B$3:$B$8=$J3),C3,0). The correct formula should sum the net profit from the first table based on the year and share no values in cells I3 and J3. It should be placed into cell K6.", + "spreadsheet_path": "spreadsheet/10747", + "instruction_type": "Cell-Level Manipulation", + "answer_position": "K6" + }, + { + "id": 50916, + "instruction": "How can I create an Excel school calendar where typing a cycle day number into a column automatically populates the classes for that day across each row? My children's school operates on a seven-day cycle, and there's a double period that could complicate matters. I want to avoid manual copy/paste because of irregular 'off days' that require adjusting the cycle day and associated classes. I managed a formula that works in one cell but am struggling to copy it across other cells as shown by the problems in cells C12, E12:H12, and D13:D14. IF(A$12=A$2,C$2,IF(A$12=A$3,C$3,IF(A$12=A$4,C$4,IF(A$12=A$5,C$5,IF(A$12=A$6,C$6,IF(A$12=A$7,C$7,IF(A$12=A$8,C$8)))))))", + "spreadsheet_path": "spreadsheet/50916", + "instruction_type": "Cell-Level Manipulation", + "answer_position": "C12:H14" + }, + { + "id": "577-40", + "instruction": "How can I delete entire rows in an Excel worksheet if both Column J and Column K are either blank or contain any of the following specific values: -, 0, $, $0, or $0.0? I need solution other than formula or a VBA macro that performs this task without deleting rows where only one of the cells in Column J or K meets these criteria, as I need to retain rows where at least one of these columns contains actual data.", + "spreadsheet_path": "spreadsheet/577-40", + "instruction_type": "Sheet-Level Manipulation", + "answer_position": "'Sheet2'!A1:K54", + "answer_sheet": "Sheet2", + "data_position": "A1:K54" + }, + { + "id": 35742, + "instruction": "I've created a spreadsheet to record the results of an upcoming horse show, which involves multiple sections and aggregate awards that need to be calculated. My IF functions seem to be working correctly, but I'm struggling to sum the scores at the end of the row. I've tried using the =VALUE, =SUM, and =SUMPRODUCT functions but haven't been successful. In my spreadsheet, I'm trying to add the results shown in the blue cells across to a green cell where I've manually entered the total. How can I correctly sum these values?", + "spreadsheet_path": "spreadsheet/35742", + "instruction_type": "Cell-Level Manipulation", + "answer_position": "O4:O7" + }, + { + "id": 46121, + "instruction": "How do I configure Excel to read a date (month) on column A of the 'Transactions' sheet and then place the total amount spent in that month under the corresponding month and category on the '2022' sheet? Specifically, I have set up an income and expense tracker for my farm business where I input transactions on one sheet and categorize them. While I can tally the transactions based on selected categories, I'm struggling to get Excel to correctly allocate these tallied sums into the appropriate monthly column on a second sheet. As an example, a payment made in April is incorrectly being included in the March column on the second sheet.\n \nMy current formula is:\nIF(Transactions!D3=Transactions!D2,\"\",SUMIF(Transactions!D:D,Transactions!D3,Transactions!C:C))\n\nWrite a formula in the income section of the '2022' sheet. Apply Currency formatting.", + "spreadsheet_path": "spreadsheet/46121", + "instruction_type": "Cell-Level Manipulation", + "answer_position": "'2022'!B5:M8" + }, + { + "id": 51090, + "instruction": "How can I calculate the difference between the total cartons received as indicated in Column M and the errors present in Columns N to R, while filtering for a specific warehouse (e.g., warehouse 27), error codes (specifically for each of II, IR, IT, OV, PI with code II in column N), date, and users (e.g., CHROGIL1, CHDSPOLJ, CHSJEFFE, CHBTHOMA for each error code) across a data set? Additionally, for the IR error code listed in Column O, I only want to include positive numbers from the Errors Table. The goal is to simplify a formula that can achieve this, as my current one is too complex and not providing the correct value, such as the expected 699 for Q2.", + "spreadsheet_path": "spreadsheet/51090", + "instruction_type": "Cell-Level Manipulation", + "answer_position": "Daily Numbers'!Q3:Q24" + }, + { + "id": 51249, + "instruction": "How do I create an Excel formula that outputs different results based on the text values of two different cells. For example, in D1, if B1 contains 'Description A' and B2 is blank, then D1 should display 'Single A'; if B1 contains 'Description B' and B2 is blank, then D1 should display 'Single B'; and if B1 contains 'Description A' and B2 contains 'Description B', then D1 should display 'Multiple'? I have also provided a sample spreadsheet showing the input and desired result. Do this output for each cell in column D that next to a cell that says \"Result:\". Fill any cells in column D with content with RGB: 226-239-218", + "spreadsheet_path": "spreadsheet/51249", + "instruction_type": "Cell-Level Manipulation", + "answer_position": "D1,D5,D9" + }, + { + "id": "82-30", + "instruction": "How can I modify my VBA code to only copy and paste whole numbers from a range of columns (A to F) in the 'Raw Data' sheet, starting from row 1 and continuing across columns A to F in the 'Numbers' sheet, without including numbers with decimal points and ensuring a maximum of 6 numbers per row? You can refer to Manual Result sheet for outcome, and it should be in the exact same sorting order.", + "spreadsheet_path": "spreadsheet/82-30", + "instruction_type": "Sheet-Level Manipulation", + "answer_position": "'Numbers'!A1:F9", + "answer_sheet": "Numbers", + "data_position": "Raw Data!'A1:F18, 'Manual Result!'A1:F9" + }, + { + "id": 56274, + "instruction": "I need an Excel formula that will automatically populate cells D9, D10, D11, and D12 with the Opening Balance, Debits, Credits, and Closing Balance respectively, based on the Fiscal Month in cell D7. The details and expected results are outlined in the attached excel sheet.", + "spreadsheet_path": "spreadsheet/56274", + "instruction_type": "Cell-Level Manipulation", + "answer_position": "D9:D12", + "exclude": "golden has data validation dropdown set to a completely different value" + }, + { + "id": 57445, + "instruction": "How can I create a formula in Column E on my Pricing Worksheet that will reference manually inputted data in Columns C and D and return the corresponding Cost data from a different worksheet named 'Package & Weight Data'? The table and necessary data are in an attached Excel file named 'Cost-Calculator.xlsx'.", + "spreadsheet_path": "spreadsheet/57445", + "instruction_type": "Cell-Level Manipulation", + "answer_position": "'Pricing'!E2:E5" + }, + { + "id": 46646, + "instruction": "How can I calculate the percentage of night shifts for a given month, considering a variable year range from 2010 to 2022. Take into account leap years and the specific days of the month. Please perform this task and fill out row 11 using the year value shown. Output the values in decimal format, such that a) a zero value has no decimal places, b) a value that only contains a single, non-zero decimal place has one decimal place, and c) all other values have nine decimal places.", + "spreadsheet_path": "spreadsheet/46646", + "instruction_type": "Cell-Level Manipulation", + "answer_position": "B11:M11" + }, + { + "id": "105-24", + "instruction": "In my Excel file, the 'Open' sheet contains IDs in Columns C and K. Find all these IDs and replace them with their respective Names. The corresponding Names for these IDs are located in Columns A and B of the 'Productivity' sheet. I require assistance with creating a macro or to automate this replacement process and also automatically consider any new additions to Columns A and B in the future for replacements. Do not create instructions, explanations, or a ID_Lookup_Helper tab. Do not create add data to any other columns other than C or K in the Open tab. Do not hide any data.", + "spreadsheet_path": "spreadsheet/105-24", + "instruction_type": "Sheet-Level Manipulation", + "answer_position": "'Open'!C2:C23", + "answer_sheet": "Open", + "data_position": "Open!'B1:P23,'Productivity!'A1:E21'" + }, + { + "id": 6239, + "instruction": "I'm creating an incentive program to reward my employees based on their performance and need help with setting it up in Excel. Employees have individual goals that range from any number between 0% to 100%. The 'Variance to Goal' is calculated as 1 - (Achieved/Goal), varying from - infinity to +infinity. Another parameter, 'Metric2', also ranges from - infinity to +infinity without a fixed goal. The incentive percentage is determined based on two filters: 'Goal' is divided into four buckets based on values ranging from 65% to 99%, and 'Variance to Goal' is noted within a range of 0% to more than 7% under each 'Goal' range. Then, 'Metric2' is divided into five buckets with values from -20% to +20%, and in each, the final incentive percentage an employee should receive is specified. For instance, if an employee has a 'Goal' between 65% to 74%, a 'Variance to Goal' between 0% to 3%, and a 'Metric2' between -20% to -10%, they should get a 3% incentive. If 'Metric2' was more than 20% under the same 'Goal' and 'Variance', they should get a 7% incentive. How can I set up Excel to calculate and output these incentive percentages based on the provided criteria and ranges?", + "spreadsheet_path": "spreadsheet/6239", + "instruction_type": "Cell-Level Manipulation", + "answer_position": "G2:G19" + }, + { + "id": "414-20", + "instruction": "Delete all rows above the first occurrence of the text “Invoice No.” in column A. The macro should search for the phrase (case-insensitive) and delete everything above it, leaving the “Invoice No.” row and all rows below untouched.", + "spreadsheet_path": "spreadsheet/414-20", + "instruction_type": "Sheet-Level Manipulation", + "answer_position": "'Sheet1'!A4:A18", + "answer_sheet": "Sheet1", + "data_position": "A4:A14" + }, + { + "id": "165-23", + "instruction": "Here's what I need: For Sheet 1, using VBA or alternative, delete all rows where column C is not the #N/A error value (Do not change any formatting and let row 1 remain blank). Do not delete anything else\n\nI need assistance with a macro for Sheet1 in Excel. The macro, named delrow2, should delete rows where the cell in column C does not contain the #N/A error value. I've attempted to write these macros, but both are returning errors. Additionally, after receiving advice to use the IsError function or smarter methods like filter or Range.SpecialCells, I created two more macros, delrow3 and delrow4, which also failed to work correctly. At one point, I tried replacing .Value with .Text in the If codeline as suggested, which seemed to resolve the issue. However, I'm unsure if it is the most efficient solution. I have limited knowledge of macro coding, and my attempts are attached in a file for review.", + "spreadsheet_path": "spreadsheet/165-23", + "instruction_type": "Sheet-Level Manipulation", + "answer_position": "'Sheet1'!A2:N73", + "answer_sheet": "Sheet1", + "data_position": "Sheet1!'A2:N73,'Desired Sheet1 after delrow1!'A2:N69,'Desired Sheet1 after delrow2!'A2:N5'" + } +] \ No newline at end of file diff --git a/benchmark/spreadsheet_xarena/dataset/train_split/val/items.json b/benchmark/spreadsheet_xarena/dataset/train_split/val/items.json new file mode 100644 index 00000000..f6bc5d4f --- /dev/null +++ b/benchmark/spreadsheet_xarena/dataset/train_split/val/items.json @@ -0,0 +1,80 @@ +[ + { + "id": 40892, + "instruction": "I need a formula to extract the color mentioned in each cell of column 1 and display that same color in column 2. For example, if a cell in column 1 says 'Dress with red details,' I want the corresponding cell in column 2 to show 'Red'. I have uploaded the file for reference. If no answer is found, output an empty string. Only add the colors mentioned in the list located in Column D.", + "spreadsheet_path": "spreadsheet/40892", + "instruction_type": "Cell-Level Manipulation", + "answer_position": "B2:B17" + }, + { + "id": 48745, + "instruction": "How can I look up multiple product codes that are located within the same cell, separated by semicolons, in an Excel table where these codes span columns 'C' and the lookup table covers columns 'G' and 'H'? Specifically, I need to identify which group each product code belongs to, with 'Group 1', 'Group 2', or 'BOTH' as potential outcomes for each cell if it contains multiple product codes that map to different groups. Additionally, if a cell contains only one product code, the semicolon is still present. Example scenarios include a single product code look-up resulting in 'Group 1', multiple product codes from the same group resulting in the respective group name, and multiple product codes from different groups resulting in 'BOTH'.I have a table where column 'c' can contain multiple values separated with a semicolon.\nI want to lookup each separated value in that cell against the lookup table in columns G and H.\nRow 5 would be PRD1 lookups and finds it is in Group 1\nRow 6 would be PRD2;PRD1 lookups and finds they are in Group 1\nRow 7 would be PRD4 lookups and finds it is in Group 2\nRow 8 would be PRD5;PRD6 lookups and finds they are in Group 2\nRow 9 would be PRD1;PRD4;PRD6 lookups and finds they are in Group 1 and Group 2 and would return Group 1, Group 2\nRow 10 would be PRD1;PRD6 lookups and finds they are in Group 1 and Group 2 and would return Group 1, Group 2\nThe semicolon is present in the same cell.\nIf multiple Groups in single cell, please range the index in asending order.", + "spreadsheet_path": "spreadsheet/48745", + "instruction_type": "Cell-Level Manipulation", + "answer_position": "D5:D10" + }, + { + "id": 32612, + "instruction": "How can I create a workday column in an Excel file that identifies each date as the day of the week (e.g., Mon, Tue, Wed, etc.)? Additionally, when the date is a holiday, you do not need to fill in the day of the week, instead you need to marks the date as a public holiday, specifying if it's a working public holiday (PH(working)) or a non-working public holiday (PH(non-working)), using a provided list of public holidays.", + "spreadsheet_path": "spreadsheet/32612", + "instruction_type": "Cell-Level Manipulation", + "answer_position": "E2:E75" + }, + { + "id": "325-44", + "instruction": "I am working with data at a wtype_id, name, and status level, and I need to split the filter data from a column into specific columns of data. I've attached the input data tab and the output data tab for reference. Leave the cell which does not have any data defined blank and leading zeros can be omitted. Retain the 'Input' data tab and update the 'Output' data tab with the result.", + "spreadsheet_path": "spreadsheet/325-44", + "instruction_type": "Sheet-Level Manipulation", + "answer_position": "'Output'!A1:G10", + "answer_sheet": "Output", + "data_position": "Input!'A1:D4,'Output!'A1:G6'" + }, + { + "id": "262-17", + "instruction": "How can I create ranges for header values 'Task' and 'Responsibility' without hard coding the range definitions, and then apply a multiple column sort using VBA in Excel? The macro should identify the positions of these headers dynamically and perform the sorting based on the data provided in the first worksheet, with the expected outcome detailed in the second worksheet attached to the thread. The first level sorts the \"Tasks\" column from small to large, and the second level sorts the \"Responsibilities\" column from small to large", + "spreadsheet_path": "spreadsheet/262-17", + "instruction_type": "Sheet-Level Manipulation", + "answer_position": "Sheet1'!A1:F14", + "answer_sheet": "'Sheet1','Sheet2'", + "data_position": "Sheet1!A1:F14','Sheet2!A1:F14'" + }, + { + "id": "141-20", + "instruction": "How can I create a macro that finds and deletes rows in two Excel sheets named 'PL Recon Items' and 'Statement Recon Items', where the invoice number in column C of 'PL Recon Items' and the value in column D match the reference number in column F and value in column I of 'Statement Recon Items'? The matching rows, which I have highlighted in yellow, should be deleted from both sheets. Could someone provide the code to accomplish this?", + "spreadsheet_path": "spreadsheet/141-20", + "instruction_type": "Sheet-Level Manipulation", + "answer_position": "'PL Recon Items!'A1:D2,'Statement Recon Items!'A1:J3", + "answer_sheet": "PL Recon Items,Statement Recon Items", + "data_position": "PL Recon Items!'A1: D2, 'Statement Recon Items!'A1: J3'" + }, + { + "id": 52216, + "instruction": "I'm having some trouble with getting my INDEX-MATCH formula to return values from subsequent rows beneath the first row (the first row returns the correct values, as desired).\nI'm not sure if a third MATCH argument is needed in the formula, but, if it is, I'm struggling with determining the correct lookup_value & lookup_array syntax.\nI've attached a workbook with some mock data on there to assist - on the 'INPUTS' tab, you'll see the dollar amounts in C15:G15 are displaying correctly which is being pulled from the 'Sheet2\" tab.\nHow can I adjust my INDEX-MATCH formula in the 'INPUTS' tab to retrieve values from rows below the first row, which currently returns the correct data, specifically for a dataset in which the output is contingent on a dropdown selection that alternates between 'Metro' and 'Regional' and affects the displayed dollar amounts?\nSo, I just need some help with getting this formula to display the correct amounts for the rest of the expenses underneath Advertising & Marketing (C15).\nI figure that if just the electricity line can be solved, then I can adopt it for the remainder (hence the lack of numbers for most of the other expenses).\nWrite a formula that will work when copied across columns C through G, and apply it to the first 5 years of Electricity expenses. No need to apply the new formula to the expense rows below Electricity.", + "spreadsheet_path": "spreadsheet/52216", + "instruction_type": "Cell-Level Manipulation", + "answer_position": "'INPUTS'!C15:G16" + }, + { + "id": "22-47", + "instruction": "I need to sort data in column B based on a helper column (J) so that the first entries written in column J are shown first in the output range F:H. Names listed in column J, include all matching rows and keep their original order from the source and do not sort within the group. Names not listed in column J should be arranged as they are in the original data (same order). The sort should skip empty cells, headers, and duplicate items, where duplicates are defined by identical entries in both column B and C. If the helper column J is empty, only then sort alphabetically A–Z and still skip empty cells, headers, and duplicate items. Additionally, I need this sorting process to handle new ranges that contain headers and some names in range A:C, as well as changes in names listed in column J. The final answer should be output in columns G and H, and sort only column H sorted lowest to highest.", + "spreadsheet_path": "spreadsheet/22-47", + "instruction_type": "Sheet-Level Manipulation", + "answer_position": "F2:H10", + "answer_sheet": "sheet1", + "data_position": "sheet1!A1:j24',',ورق1!B1:B11'" + }, + { + "id": 55421, + "instruction": "How can I create a formula in Excel that will check the numbers in Column A and return specific text in Column F based on the following conditions: when a number in Column A is accompanied exclusively by the status 'SCH' in Column D, the text 'FUTURE' should be returned in Column F; if it appears with both statuses 'NO SHOW' and 'SCH', then 'NS/SCHED' should be shown in Column F; if the number comes with the 'NO SHOW' status and a date is present in Column E, 'NO ACTION NEEDED' should be the output; and finally, if there is a 'NO SHOW' status with an empty date field in Column E, 'CALL PT' should appear in Column F?", + "spreadsheet_path": "spreadsheet/55421", + "instruction_type": "Cell-Level Manipulation", + "answer_position": "F2:F20" + }, + { + "id": 56427, + "instruction": "How can I transpose the values in column G where column B has the value 1 and pull sequential values from consecutive rows in column G, stopping after the number of runners specified in column C? The formula should preserve blanks as blanks (not zeros). An example dataset has been provided where the transposition was done partially, but I'm seeking a formulaic solution that will provide the full result. Once complete, shade the range H2:S28 in #E2EFD, and remove decimals for any whole values that range. Then center align all cells to the right of column G.", + "spreadsheet_path": "spreadsheet/56427", + "instruction_type": "Cell-Level Manipulation", + "answer_position": "H2:S28" + } +] \ No newline at end of file diff --git a/benchmark/spreadsheet_xarena/third_party/SkillOpt/.env.example b/benchmark/spreadsheet_xarena/third_party/SkillOpt/.env.example new file mode 100644 index 00000000..7060b868 --- /dev/null +++ b/benchmark/spreadsheet_xarena/third_party/SkillOpt/.env.example @@ -0,0 +1,34 @@ +# SkillOpt Environment Variables +# Copy this file to .env and fill in your values. +# Usage: set -a; source .env; set +a + +# ── Azure OpenAI (required for openai_chat backend) ────────────────── +export AZURE_OPENAI_ENDPOINT=https://your-resource.openai.azure.com/ +export AZURE_OPENAI_API_VERSION=2024-12-01-preview +# Authentication: choose one method +# Option 1: API Key +export AZURE_OPENAI_API_KEY= +# Option 2: Azure CLI (no API key needed, recommended on Azure VMs) +# export AZURE_OPENAI_AUTH_MODE=azure_cli +# Option 3: Managed Identity +# export AZURE_OPENAI_AUTH_MODE=managed_identity +# export AZURE_OPENAI_MANAGED_IDENTITY_CLIENT_ID=your-client-id + +# ── OpenAI-compatible endpoints ────────────────────────────────────── +# Set AUTH_MODE to openai_compatible and reuse AZURE_OPENAI_ENDPOINT / _API_KEY. +# The plain OpenAI client is used; no Azure auth, no api-version header. +# export AZURE_OPENAI_ENDPOINT=https://api.openai.com/v1 +# export AZURE_OPENAI_API_KEY=sk-... +# export AZURE_OPENAI_AUTH_MODE=openai_compatible + +# ── Anthropic / Claude (for claude_chat backend) ───────────────────── +# export ANTHROPIC_API_KEY=sk-ant-... + +# ── Qwen Local Model (for qwen_chat backend) ──────────────────────── +# export QWEN_CHAT_BASE_URL=http://localhost:8000/v1 +# export QWEN_CHAT_MODEL=Qwen/Qwen3.5-4B + +# ── MiniMax (for minimax_chat backend) ────────────────────────────── +# export MINIMAX_BASE_URL=https://api.minimax.io/v1 +# export MINIMAX_API_KEY=... +# export MINIMAX_MODEL=MiniMax-M2.7 diff --git a/benchmark/spreadsheet_xarena/third_party/SkillOpt/.gitignore b/benchmark/spreadsheet_xarena/third_party/SkillOpt/.gitignore new file mode 100644 index 00000000..3d94e846 --- /dev/null +++ b/benchmark/spreadsheet_xarena/third_party/SkillOpt/.gitignore @@ -0,0 +1,56 @@ +__pycache__/ +*.pyc +*.egg-info/ +build/ +dist/ +site/ + +data/* +!data/README.md +!data/searchqa_id_split/ +!data/searchqa_id_split/** +!data/livemathematicianbench_id_split/ +!data/livemathematicianbench_id_split/** +!data/docvqa_id_split/ +!data/docvqa_id_split/** +!data/officeqa_id_split/ +!data/officeqa_id_split/** +!data/spreadsheetbench_id_split/ +!data/spreadsheetbench_id_split/** +!data/alfworld_path_split/ +!data/alfworld_path_split/** +outputs/ +logs/ +external/ + +/BabyVision/ +/MMRB/ +/SpreadsheetBench/ +/dl4ir-searchQA/ + +configs/local/ +configs/**/*.local.yaml +*.local.md +*.secret.md +*.bak + +.env +.secrets/ +.codex_azure*/ + +# Internal docs (not for open-source release) +docs/ablation_plan.md +docs/ablation_paper_tables.md +docs/ablation_paper_tables.html +docs/experiment_commands.md +docs/slow_update_flowchart.md +docs/session_memory.md +docs/harness_fresh_machine_handoff.md +docs/harness_monitoring_memory.md +docs/harness_reproduction_secrets.secret.md +docs/reflact_conda_env_export.yml +docs/reflact_overview.html +docs/render_ablation_paper_tables.py +docs/让* +.gradio/ +.venv diff --git a/benchmark/spreadsheet_xarena/third_party/SkillOpt/CONTRIBUTING.md b/benchmark/spreadsheet_xarena/third_party/SkillOpt/CONTRIBUTING.md new file mode 100644 index 00000000..5a3f3c9d --- /dev/null +++ b/benchmark/spreadsheet_xarena/third_party/SkillOpt/CONTRIBUTING.md @@ -0,0 +1,43 @@ +# Contributing to SkillOpt + +Thank you for your interest in contributing! SkillOpt welcomes contributions of all kinds. + +## Getting Started + +```bash +git clone https://github.com/microsoft/SkillOpt.git +cd SkillOpt +pip install -e ".[dev]" +``` + +## How to Contribute + +### 🐛 Bug Reports +Open a GitHub issue with reproduction steps, expected/actual behavior, and your config file (remove API keys). + +### 🔧 Add a Benchmark +See the [guide](docs/guide/new-benchmark.md) and use the scaffold at `skillopt/envs/_template/`. + +### 🤖 Add a Model Backend +See the [guide](docs/guide/new-backend.md). + +### 📝 Improve Documentation +```bash +pip install -e ".[docs]" +mkdocs serve # Preview at http://localhost:8000 +``` + +## Pull Request Process + +1. Fork the repo and create a feature branch +2. Make changes and test with an existing benchmark +3. Submit a PR with a clear description +4. Ensure CI passes + +## Code Style +- Follow existing patterns in the codebase +- Use type hints for function signatures +- Keep docstrings concise + +## License +By contributing, you agree your contributions are licensed under the [MIT License](LICENSE). diff --git a/benchmark/spreadsheet_xarena/third_party/SkillOpt/LICENSE b/benchmark/spreadsheet_xarena/third_party/SkillOpt/LICENSE new file mode 100644 index 00000000..cf7bcb2b --- /dev/null +++ b/benchmark/spreadsheet_xarena/third_party/SkillOpt/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2026 Microsoft Corporation + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/benchmark/spreadsheet_xarena/third_party/SkillOpt/README.md b/benchmark/spreadsheet_xarena/third_party/SkillOpt/README.md new file mode 100644 index 00000000..395c70b4 --- /dev/null +++ b/benchmark/spreadsheet_xarena/third_party/SkillOpt/README.md @@ -0,0 +1,395 @@ +# SkillOpt: Executive Strategy for Self-Evolving Agent Skills + +*Train agent skills like you train neural networks — with epochs, (mini-)batchsize, learning rates, and validation gates — but without touching model weights.* + +[![Project Page](https://img.shields.io/badge/Project%20Page-SkillOpt-8dbb3c)](https://microsoft.github.io/SkillOpt/) [![Paper](https://img.shields.io/badge/Paper-arXiv-b31b1b)](https://arxiv.org/abs/2605.23904) [![Project Video](https://img.shields.io/badge/Project%20Video-Watch%20Demo-ff0000)](https://youtu.be/JUBMDTCiM0M) [![Python 3.10+](https://img.shields.io/badge/Python-3.10%2B-blue.svg)](https://www.python.org/) [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](LICENSE) + +--- + +## Overview + +Modern agent skills are usually hand-crafted, generated one-shot by a strong +LLM, or evolved through loosely controlled self-revision — none of which +behaves like a deep-learning optimizer for the skill itself, and none of +which reliably improves over its starting point under feedback. + +**SkillOpt treats the skill document as the trainable state of a frozen +agent**, and trains it with the discipline that makes weight-space +optimization reproducible. A separate optimizer model turns scored rollouts +into bounded add / delete / replace edits on a single skill document; a +candidate edit is accepted only when it strictly improves a held-out +validation score. A textual learning-rate budget, a rejected-edit buffer, +and an epoch-wise slow / meta update make skill training stable while +adding **zero inference-time model calls** at deployment. + +The deployed artifact is a compact `best_skill.md` (typically 300–2,000 +tokens) that runs against the unchanged target model. Across **six +benchmarks, seven target models, and three execution harnesses** (direct +chat, Codex CLI, Claude Code CLI), SkillOpt is best or tied-best on **all +52 evaluated (model, benchmark, harness) cells** and on GPT-5.5 lifts the +average no-skill accuracy by **+23.5 points in direct chat, +24.8 inside +the Codex agentic loop, and +19.1 inside Claude Code**. Optimized skill +artifacts transfer across model scales, between Codex and Claude Code +harnesses, and to nearby benchmarks without further optimization. + +For the full method, ablations, and per-cell results see the [paper](https://arxiv.org/abs/2605.23904); for a visual walkthrough of the loop see the [project page](https://microsoft.github.io/SkillOpt/); for deeper API / backend / benchmark docs see [`docs/`](docs/). + +## 🎬 Demo Video + +https://github.com/user-attachments/assets/eb12d3bc-371c-467f-904d-91b61f339ed7 + +

+ ▶ Watch the full demo on YouTube +

+ +--- + +## Install + +### Requirements + +- Python 3.10+ + +```bash +git clone https://github.com/microsoft/SkillOpt.git +cd SkillOpt +pip install -e . + +# For the ALFWorld benchmark (optional): +pip install -e ".[alfworld]" +alfworld-download +``` + +### Configure API Credentials + +```bash +cp .env.example .env +# Edit .env with your API credentials, then: +source .env +``` + +#### Azure OpenAI *(recommended)* + +```bash +export AZURE_OPENAI_ENDPOINT="https://your-resource.openai.azure.com/" +# Option 1: API key auth +export AZURE_OPENAI_API_KEY="your-key" +# Option 2: Azure CLI auth (no API key needed) +export AZURE_OPENAI_AUTH_MODE="azure_cli" +``` + +> **Note:** `AZURE_OPENAI_ENDPOINT` is required for all three modes (`api_key`, `azure_cli`, `openai_compatible`). Without it, all LLM calls will fail. + +#### OpenAI-compatible endpoints + +```bash +export AZURE_OPENAI_ENDPOINT="https://api.openai.com/v1" +export AZURE_OPENAI_API_KEY="sk-..." +export AZURE_OPENAI_AUTH_MODE="openai_compatible" +``` + +This routes all calls through the plain OpenAI Python client (no Azure auth, no `api-version` header). + +> **Note:** SkillOpt reuses the `AZURE_OPENAI_*` env var names even in this mode — there is no separate `OPENAI_API_KEY` knob. + +#### Anthropic Claude + +```bash +export ANTHROPIC_API_KEY="sk-ant-..." +``` + +#### Qwen *(local vLLM)* + +```bash +export QWEN_CHAT_BASE_URL="http://localhost:8000/v1" +export QWEN_CHAT_MODEL="Qwen/Qwen3.5-4B" +``` + +`qwen_chat` can also be used as the optimizer backend. When optimizer and +target should point to different local vLLM services, use the role-specific +settings: + +```bash +python scripts/train.py \ + --config configs/searchqa/default.yaml \ + --optimizer_backend qwen_chat \ + --target_backend qwen_chat \ + --optimizer_model Qwen/Qwen3.5-4B \ + --target_model Qwen/Qwen3.5-4B \ + --optimizer_qwen_chat_base_url http://localhost:8001/v1 \ + --target_qwen_chat_base_url http://localhost:8000/v1 +``` + +#### MiniMax + +```bash +export MINIMAX_BASE_URL="https://api.minimax.io/v1" +export MINIMAX_API_KEY="..." +export MINIMAX_MODEL="MiniMax-M2.7" +``` + +--- + +## Quick Start + +### Training + +```bash +# Minimal example — train on SearchQA: +python scripts/train.py \ + --config configs/searchqa/default.yaml \ + --split_dir /path/to/your/searchqa_split \ + --azure_openai_endpoint https://your-resource.openai.azure.com/ \ + --optimizer_model gpt-5.5 \ + --target_model gpt-5.5 + +# Train on LiveMathematicianBench: +python scripts/train.py \ + --config configs/livemathematicianbench/default.yaml \ + --split_dir /path/to/your/livemath_split \ + --azure_openai_endpoint https://your-resource.openai.azure.com/ \ + --optimizer_model gpt-5.5 \ + --target_model gpt-5.5 + +# Train on ALFWorld: +python scripts/train.py \ + --config configs/alfworld/default.yaml \ + --split_dir data/alfworld_path_split \ + --azure_openai_endpoint https://your-resource.openai.azure.com/ \ + --optimizer_model gpt-5.5 \ + --target_model gpt-5.5 +``` + +Key CLI arguments: + +| Argument | Description | Example | +|---|---|---| +| `--config` | Benchmark config YAML | `configs/searchqa/default.yaml` | +| `--split_dir` | Path to data split directory | `/path/to/split` | +| `--azure_openai_endpoint` | Azure OpenAI endpoint URL | `https://your-resource.openai.azure.com/` | +| `--optimizer_model` | Optimizer model deployment name | `gpt-5.5` | +| `--target_model` | Target model deployment name | `gpt-5.5` | +| `--num_epochs` | Number of training epochs | `4` | +| `--batch_size` | Batch size per step | `40` | +| `--workers` | Parallel rollout workers | `8` | +| `--out_root` | Output directory | `outputs/my_run` | + +### Eval Only + +Evaluate a trained skill on specific data splits without training: + +```bash +# Evaluate the packaged GPT-5.5 SearchQA skill on the test split: +python scripts/eval_only.py \ + --config configs/searchqa/default.yaml \ + --skill ckpt/searchqa/gpt5.5_skill.md \ + --split valid_unseen \ + --split_dir /path/to/searchqa_split \ + --azure_openai_endpoint https://your-resource.openai.azure.com/ + +# Evaluate on all splits (train + val + test): +python scripts/eval_only.py \ + --config configs/searchqa/default.yaml \ + --skill ckpt/searchqa/gpt5.5_skill.md \ + --split all \ + --split_dir /path/to/searchqa_split \ + --azure_openai_endpoint https://your-resource.openai.azure.com/ +``` + +To evaluate a skill produced by your own training run, replace `--skill` with that run's best-skill path, for example `outputs/my_run/best_skill.md`. + +| Split | Description | +|---|---| +| `valid_unseen` | Test set | +| `valid_seen` | Validation set | +| `train` | Training set | +| `all` | All splits combined (default) | + +### Output Structure + +Each training run writes to a structured output directory: + +``` +outputs// +├── config.json # Flattened runtime config +├── history.json # Per-step training history +├── runtime_state.json # Resume checkpoint +├── best_skill.md # Best validated skill document +├── skills/skill_vXXXX.md # Skill snapshot per step +├── steps/step_XXXX/ # Per-step artifacts (patches, evals) +├── slow_update/epoch_XX/ # Slow update logs +└── meta_skill/epoch_XX/ # Meta skill logs +``` + +Re-running the same command auto-resumes from the last completed step. + +### Pretrained Skill Artifacts + +We provide a subset of the paper's main Table 1 GPT-5.5 optimized skills in +[`ckpt/`](ckpt/) as reference artifacts. Use them with `scripts/eval_only.py` +to evaluate the provided skills on a matching data split without re-running +training. See [`ckpt/README.md`](ckpt/README.md) for the full per-benchmark +command. This is the first artifact batch; we plan to continue uploading +the remaining optimized skills and benchmark split manifests as they are +cleaned and verified. + +--- + +## Data Preparation + +### Directory layout + +SkillOpt expects data in a **split directory** with `train/`, `val/`, `test/` subdirectories, each containing a JSON file (e.g., `items.json`): + +``` +data/my_split/ +├── train/items.json +├── val/items.json +└── test/items.json +``` + +Each JSON file is an array of task items. The required fields depend on the benchmark. For example, SearchQA items look like: + +```json +[ + { + "id": "unique_item_id", + "question": "Who wrote the novel ...", + "context": "[DOC] relevant passage text ...", + "answers": ["expected answer"] + } +] +``` + +See `skillopt/envs//dataloader.py` for the exact format each benchmark expects. + +> **Note:** Most benchmark datasets are not included in this repository. Prepare your own data following the format above. The exact SearchQA split used in the paper is provided at [`data/searchqa_id_split/`](data/searchqa_id_split) (400 train / 200 val / 1400 test). We are preparing the remaining benchmark split manifests for upload. + +### Supported Benchmarks + +| Benchmark | Type | Config | +|---|---|---| +| SearchQA | QA | `configs/searchqa/default.yaml` | +| ALFWorld | Embodied agent | `configs/alfworld/default.yaml` | +| DocVQA | Document QA | `configs/docvqa/default.yaml` | +| LiveMathematicianBench | Math | `configs/livemathematicianbench/default.yaml` | +| SpreadsheetBench | Code generation | `configs/spreadsheetbench/default.yaml` | +| OfficeQA | Tool-augmented QA | `configs/officeqa/default.yaml` | + +--- + +## Configuration + +### Default settings and paper-reproduction knobs + +`configs/_base_/default.yaml` is the single source of truth for SkillOpt's +runtime knobs. Out of the box, every included benchmark config inherits +from it and keeps the paper protocol visible: 4 epochs, rollout batch 40, +reflection minibatch 8, textual learning rate 4 with cosine decay, strict +hard validation gating, and slow-update + meta-skill enabled. One detail to +watch is slow-update acceptance: the current `main` default is the newer +post-submission force-accept mode, while the paper protocol and the +paper-aligned skills under `ckpt/` use the gated semantics described in +paper Section 3.6. + +### Slow-update acceptance mode + +The epoch-boundary slow / meta update can be applied two ways, controlled +by `optimizer.slow_update_gate_with_selection`: + +```yaml +optimizer: + slow_update_gate_with_selection: false # current main default +``` + +- **`false`** *(current `main` default)*: force-accept. The + slow-update guidance is injected into both `current_skill` and + `best_skill` unconditionally at the epoch boundary. This is the newer + post-submission behavior on `main`. +- **`true`** *(paper / ckpt-skill reproduction)*: gated, matching paper + Section 3.6 verbatim. The slow-update candidate is evaluated on the + selection split and accepted only if it passes the same validation gate + as a step-level edit. Use this setting when re-running optimization to + match the paper protocol and the provenance of the provided `ckpt/` skills. + +The trainer prints which mode is active at startup +(`[slow update] acceptance=...`). See issue #22 for the discussion that +led to the flag. + +### Gate metric (`hard` / `soft` / `mixed`) + +The validation gate compares candidate vs. current skills on the selection +split using `gate_metric`: + +- **`hard`** *(default, paper)*: exact-match accuracy, strictly greater + than the current score is required. +- **`soft`**: per-item soft / partial-credit score. Useful when the + selection split is small (e.g. ≤10 items) and the reward is continuous, + where the discrete hard gate often rejects every candidate. +- **`mixed`**: weighted average, `(1 - w) * hard + w * soft`, with `w` + set by `gate_mixed_weight` (default `0.5`). + +Default is `hard`. Use the optional feature config below to switch. + +### Optional feature configs + +These are **not** default SkillOpt settings — they are optional feature configs +contributed by users for specific scenarios. The paper-reported numbers +were obtained with the default settings, not these. + +- **[`configs/features/soft_gate.yaml`](configs/features/soft_gate.yaml)** + *(PR #25, contributed by [@lvbaocheng](https://github.com/lvbaocheng))* — + switches `gate_metric` to `soft` (or `mixed`). See the comment at the + top of the file for when to use and when not to. + +--- + +## Extensibility & WebUI + +### Adding a new backend + +A backend = a chat / exec target (e.g. `openai_chat`, `claude_chat`, +`qwen_chat`, `minimax_chat`, `codex_exec`, `claude_code_exec`). See +[`docs/guide/new-backend.md`](docs/guide/new-backend.md) for the full +contract; in short you add a `skillopt/model/_backend.py` module, +register it in `skillopt/model/common.py` + `backend_config.py`, and wire +it through the router in `skillopt/model/__init__.py`. `qwen_backend.py` +and `minimax_backend.py` are good templates. + +### Adding a new benchmark + +A benchmark = a `skillopt/envs//` package with a `dataloader.py`, a +`rollout.py`, and an `initial.md` seed skill. See +[`docs/guide/new-benchmark.md`](docs/guide/new-benchmark.md) for the full +contract; the simplest reference is `skillopt/envs/searchqa/`. + +### WebUI + +Launch the monitoring dashboard (optional): + +```bash +pip install -e ".[webui]" +python -m skillopt_webui.app +``` + +| Flag | Default | Description | +|---|---|---| +| `--port` | 7860 | Server port | +| `--host` | `0.0.0.0` | Bind address | +| `--share` | off | Create a public Gradio share link | + +--- + +## Citation + +```bibtex +@misc{yang2026skilloptexecutivestrategyselfevolving, + title={SkillOpt: Executive Strategy for Self-Evolving Agent Skills}, + author={Yifan Yang and Ziyang Gong and Weiquan Huang and Qihao Yang and Ziwei Zhou and Zisu Huang and Yan Li and Xuemei Gao and Qi Dai and Bei Liu and Kai Qiu and Yuqing Yang and Dongdong Chen and Xue Yang and Chong Luo}, + year={2026}, + eprint={2605.23904}, + archivePrefix={arXiv}, + primaryClass={cs.AI}, + url={https://arxiv.org/abs/2605.23904} +} +``` diff --git a/benchmark/spreadsheet_xarena/third_party/SkillOpt/SECURITY.md b/benchmark/spreadsheet_xarena/third_party/SkillOpt/SECURITY.md new file mode 100644 index 00000000..e751608f --- /dev/null +++ b/benchmark/spreadsheet_xarena/third_party/SkillOpt/SECURITY.md @@ -0,0 +1,14 @@ + + +## Security + +Microsoft takes the security of our software products and services seriously, which +includes all source code repositories in our GitHub organizations. + +**Please do not report security vulnerabilities through public GitHub issues.** + +For security reporting information, locations, contact information, and policies, +please review the latest guidance for Microsoft repositories at +[https://aka.ms/SECURITY.md](https://aka.ms/SECURITY.md). + + \ No newline at end of file diff --git a/benchmark/spreadsheet_xarena/third_party/SkillOpt/ckpt/README.md b/benchmark/spreadsheet_xarena/third_party/SkillOpt/ckpt/README.md new file mode 100644 index 00000000..b79f7666 --- /dev/null +++ b/benchmark/spreadsheet_xarena/third_party/SkillOpt/ckpt/README.md @@ -0,0 +1,79 @@ +# Paper-aligned SkillOpt reference skills (GPT-5.5) + +This folder provides a subset of the paper's main Table 1 GPT-5.5 optimized +skills as reference artifacts — one `gpt5.5_skill.md` per currently included +benchmark. You can plug them into `scripts/eval_only.py` to evaluate the +provided skills on a given split without re-running the training loop. + +> These are checkpoints associated with the paper, not a general-purpose +> tool. They're here so you can verify the reported numbers and use the +> skills as portable artifacts. If you want to *train* your own skill, +> use `scripts/train.py` per the top-level README. +> +> This is the first artifact batch. We plan to continue uploading the +> remaining optimized skills and benchmark split manifests as they are +> cleaned and verified. + +## What's here + +| Benchmark | Skill artifact | Matching config | +|---|---|---| +| SearchQA | `ckpt/searchqa/gpt5.5_skill.md` | `configs/searchqa/default.yaml` | +| ALFWorld | `ckpt/alfworld/gpt5.5_skill.md` | `configs/alfworld/default.yaml` | +| DocVQA | `ckpt/docvqa/gpt5.5_skill.md` | `configs/docvqa/default.yaml` | +| LiveMathematicianBench | `ckpt/livemath/gpt5.5_skill.md` | `configs/livemathematicianbench/default.yaml` | +| OfficeQA | `ckpt/officeqa/gpt5.5_skill.md` | `configs/officeqa/default.yaml` | +| SpreadsheetBench | `ckpt/spreadsheetbench/gpt5.5_skill.md` | `configs/spreadsheetbench/default.yaml` | + +Each file is a plain Markdown skill document (~2k–13k chars). It contains a +protected `SLOW_UPDATE` section at the end that holds epoch-wise +longitudinal guidance — that's expected, not a formatting issue. + +## How to evaluate a provided skill + +`scripts/eval_only.py` runs a single skill against a data split without +invoking the optimizer. Example for SearchQA against the test split: + +```bash +python scripts/eval_only.py \ + --config configs/searchqa/default.yaml \ + --skill ckpt/searchqa/gpt5.5_skill.md \ + --split valid_unseen \ + --split_dir data/searchqa_id_split \ + --azure_openai_endpoint https://your-resource.openai.azure.com/ \ + --target_model gpt-5.5 +``` + +Substitute the benchmark, config, skill path, and `--split_dir` to evaluate +any of the other five. `--split valid_unseen` is the test split, `valid_seen` +is the selection / validation split, `train` is the training split, and +`all` runs all three. + +## On comparing to the paper numbers + +To compare against the paper-reported cells, use the same dataset split and +scorer. SearchQA's split is checked in at `data/searchqa_id_split/` (400 +train / 200 selection / 1400 test). For the other benchmarks, point +`--split_dir` at your own materialized split; the loader is deterministic +from `split_seed` (default `42`) + `split_ratio` (default `2:1:7`) when +`split_mode: ratio` is used, so a given `data_path` + seed reproduces +across machines. Explicit per-benchmark split manifests are being prepared +for upload — see issues #14 and #21. + +## Why force-accept vs. gated slow-update matters + +These `ckpt/` skills were produced with the gated slow-update semantics +described in paper Section 3.6: + +```yaml +optimizer: + slow_update_gate_with_selection: true +``` + +Current `main` defaults to `false` (force-accept mode), a newer +post-submission behavior where the slow-update guidance is written into +`current_skill` and `best_skill` unconditionally at the epoch boundary. If +you re-train with the current default, you may produce a *different* +`best_skill.md` than the one checked in here. Both modes are supported; +see the top-level README's "Configuration -> Slow-update acceptance mode" +section. diff --git a/benchmark/spreadsheet_xarena/third_party/SkillOpt/ckpt/alfworld/gpt5.5_skill.md b/benchmark/spreadsheet_xarena/third_party/SkillOpt/ckpt/alfworld/gpt5.5_skill.md new file mode 100644 index 00000000..fb309431 --- /dev/null +++ b/benchmark/spreadsheet_xarena/third_party/SkillOpt/ckpt/alfworld/gpt5.5_skill.md @@ -0,0 +1,113 @@ +# ALFWorld Embodied Agent Skill + +## Overview +This skill guides agents operating in the ALFWorld text-based embodied environment. +The agent must complete household tasks by navigating rooms, interacting with objects, +and using appliances. Actions must be chosen from the admissible action list provided +at each step. + +**Output format**: Always output `...` for reasoning, then `...` for the chosen action. + +--- + +## Task Types + +| Type | Goal | Key Steps | +|------|------|-----------| +| Pick & Place | Put object X in/on receptacle Y | Find X -> take X -> go to Y -> put X in/on Y | +| Pick Two & Place | Put two instances of X in/on Y | Find X1 -> take -> place -> find X2 -> take -> place | + +### Pick Two Object Bookkeeping +For `pick_two_obj_and_place`, choose one destination receptacle instance once it is opened/usable, and remember it as the target. Both object instances should be placed into that same remembered receptacle. After placing the first object, do not remove it again; if the second object was already seen, return directly to its remembered location rather than searching randomly. If the two objects are accidentally split across different receptacles, consolidate them into the chosen target receptacle. +| Examine in Light | Examine object X under desklamp | Find X -> take X -> find desklamp -> use desklamp | + +| Examine in Light detail | Final interaction | While holding X where a desklamp is visible, use the desklamp; do not try to place X on the lamp first. | +| Clean & Place | Clean object X and put in/on Y | Find X -> take X -> go to sink -> clean X -> go to Y -> put X | +| Heat & Place | Heat object X and put in/on Y | Find X -> take X -> go to microwave -> heat X -> go to Y -> put X | +| Cool & Place | Cool object X and put in/on Y | Find X -> take X -> go to fridge -> cool X -> go to Y -> put X | + +--- + +## General Principles + +1. **Decompose the task**: Parse the goal into ordered sub-goals (locate, acquire, transform, deliver). Complete each before moving to the next. +2. **Systematic exploration**: Search each surface and container exactly once before revisiting. Open closed containers (drawers, cabinets, fridge) before judging them empty. + +- Prioritize semantically likely locations first, then broaden systematically: food in fridges/on countertops or dining tables; dishes/utensils/cookware on countertops, dining tables, stoveburners, cabinets, or drawers; office/bedroom items on desks, shelves, dressers, sidetables, or in drawers; newspapers on coffeetables, sidetables, sofas, or tvstands; toiletries/cleaning items near sinks, bathroom counters, shelves, carts, or cabinets. + +- For portable kitchen targets such as bread, mugs, cups, plates, bowls, and utensils, check broad exposed surfaces early: after one or two empty countertops, try dining tables or other open surfaces before opening many cabinets/drawers. For small office/bedroom targets, alternate drawers with exposed desks, shelves, sidetables, and dressers rather than exhausting drawers first. + +- Keep a persistent **searched set** of receptacle instances, e.g. `drawer 1`, `shelf 3`, `countertop 2`. Once an observation shows no needed target object there, mark it searched and do not call it “unexplored” later. + - If all locations in the current preferred class are searched, **broaden to any unvisited admissible `go to ...` location** instead of restarting the same sequence. Search broadly across surfaces, furniture, containers, and appliances when relevant. + - If a visible object is itself an openable/container-like object, such as a box, and opening/examining it is admissible, inspect it before leaving the area. +3. **Grab immediately**: When a required object is visible and reachable, take it right away before moving elsewhere. + +- Pick up only the exact requested object type. Similar or related objects, such as a cup when the task asks for a mug, a spoon when it asks for a knife, or a pot when it asks for a pan, are distractors; leave them in place and mark that location searched for the target. +4. **Transform before placing**: If the task requires cleaning, heating, or cooling, perform the state change at the appropriate appliance before heading to the final destination. + +- Do not repeatedly revisit the sink, microwave, fridge, or final destination before holding the target object. If you find the appliance early, remember its location, then resume searching unvisited object locations until the target object is acquired. + +- Use direct admissible appliance/tool commands immediately when available, such as `clean X with sinkbasin`, `heat X with microwave`, `cool X with fridge`, or `use desklamp`. Do not waste steps opening, closing, toggling, or examining the appliance unless the needed action is unavailable or opening is required for searching/placing. +5. **Direct delivery**: Once holding the transformed (or untransformed) goal object, navigate straight to the target receptacle and place it. + +- Remember known destination receptacles and return directly to the same instance after pickup/transformation. If the destination is also a semantically likely source location, check/open it early rather than only after exhaustive search: food may already be in the fridge, utensils may be on the diningtable, newspapers may be on/near the sofa, and a target drawer can be opened early for pick-two tasks. If the object starts at the destination but needs cleaning/heating/cooling, take it out, transform it, then return to that same instance and place it back. +6. **Track progress**: Maintain an internal count of how many objects still need to be found and placed. Only stop searching when the count reaches zero. +7. **Avoid loops**: Never repeat the same action more than twice in a row. If stuck, move to a different unexplored location. +8. **Only choose admissible actions**: Always pick an action from the admissible action list. Do not invent actions. + +--- + +## Common Mistakes to Avoid + +- **Revisiting searched locations**: Keep track of which surfaces/containers have been checked; do not re-examine them. +- **Ignoring visible objects**: If the target object appears in the observation, pick it up immediately. +- **Skipping state changes**: Do not place an object at the destination without first cleaning/heating/cooling it when required. +- **Premature termination**: Do not stop the episode until all goal conditions are verified as met. +- **Action loops**: Repeatedly toggling or examining the same object wastes steps. Move on to new locations instead. + +### Hard Search-Loop Recovery + +- **Exact-instance lockout before pickup**: once a receptacle/surface instance has been observed and does not contain the target object, do not go back to that exact instance while still searching for the object. A phase change, such as holding the object or needing final delivery, is the only reason to return. +- **Fast broadening threshold**: after 3-4 misses in the same receptacle class, switch to a different likely class or any unvisited admissible location instead of continuing or restarting that class, unless the target has already been seen there. +- **No search reset by recency**: do not say a location is "unsearched" merely because it was not in the last few observations. The searched set is global for the whole episode. +- **Finite-class exhaustion**: if all visible instances of a small class have been checked once, such as all stoveburners, diningtables, countertops, or shelves, mark that class exhausted for object search and do not start a second pass. Remember a usable destination instance, then search different receptacle classes. +- **Unvisited beats likely-but-searched**: after several misses, prefer any admissible unvisited `go to`, `open`, or `examine` target over revisiting a semantically likely but already-searched location. +- **Destination surfaces before pickup**: if the destination receptacle is also a likely object location, inspect each instance at most once before pickup. If it lacks the object, remember it as the final destination but stop using it as a search target until the object has been transformed and is ready to place. +- **Kitchen item fallback**: for cookware and dishware, after checking obvious burners/tables/counters once, broaden to unsearched cabinets, drawers, shelves, sinkbasins, and other kitchen storage/surfaces rather than cycling among the obvious locations. + +### Strict Search Ledger Action Filter + +Before every empty-handed search action, apply this hard filter: + +1. If a required target object is visible, take it immediately. +2. Otherwise choose an exact receptacle/surface/container instance whose contents have not yet been observed in the current object-search phase. +3. Reject any `go to`, `examine`, or `open` action for an exact instance already observed to lack the target, even if it is semantically likely, nearby, recently mentioned, or the final destination type. +4. If all likely instances are rejected by the ledger, broaden to any unvisited admissible location/class instead of restarting from instance 1 of a searched class. + +The searched ledger survives inventory checks, appliance visits, placing the first object in a pick-two task, and putting down an irrelevant inspected object/container. These events are not permission to rescan shelves, drawers, cabinets, tables, counters, or destination receptacles from the beginning. + +### Destination-as-Source Lockout + +When the final receptacle type is also a plausible source location, inspect each visible destination instance at most once before pickup. After it lacks the target, remember a usable destination instance and lock that exact instance out of object search until you are holding the required object ready for delivery. Do not alternate between destination instances and other searched source instances while still empty-handed. + +### Pick-Two Phase Memory + +After placing the first object in a pick-two task, do not begin a fresh room/class search. If another required instance was previously seen, return directly to that remembered source location for the second pickup. If no second instance is remembered, continue from the existing unsearched-location ledger rather than revisiting locations already checked before the first placement. + + +Preserve the successful pattern: when the exact requested object is visible, take it immediately; perform the required clean/heat/cool/use action as soon as the correct command is admissible; then deliver directly to the remembered destination. + +Treat tool locations as tools, not repeated search targets. If a sinkbasin, fridge, microwave, desklamp, or destination receptacle has already been checked and does not contain the target while you are empty-handed, remember it for later but do not revisit it until you are holding the required object or ready to place/use it. + +Use a next-unsearched-instance pointer for every numbered class. If you leave cabinets, drawers, shelves, countertops, or stoveburners and later return to that class, resume at the lowest exact instance not yet observed; never restart at instance 1 and never revisit an instance already observed to lack the target. + +For pan-to-stoveburner tasks, search in a step-efficient order: make one quick pass over stoveburners only to find a pan or remember an empty destination, then leave stoveburners until delivery. Next check countertops/islands and sinkbasins. Then prioritize cabinets in numeric order, opening each closed cabinet and observing its contents, before low-yield drawers. Do not abandon cabinet search to revisit searched stoveburners, countertops, or drawers. + +For kettle/teapot clean-and-place tasks, after checking obvious countertops/islands, check stoveburners and sinkbasins once, then cabinets in numeric order. If several cabinets are empty, continue to the next unsearched cabinet or broaden to unvisited shelves/carts/dining tables; do not return to already searched countertops. Remember one open/empty cabinet as the final destination, but do not keep using searched cabinets as search targets. + +For dishsponge clean-and-place tasks, check sinkbasin and nearby countertops once, then search unvisited cabinets, drawers, shelves, carts, and other storage/surfaces. Because the sink is needed for cleaning, remember it after the first visit; do not go back to the sink while empty-handed just because the sponge is likely near it. Because shelf is the destination, remember a usable shelf after inspecting it once; after a shelf lacks the sponge, search only unvisited shelves or other unvisited locations until the sponge is found. + +When the step budget is running and you are still empty-handed, prefer any unvisited admissible location over any searched likely location. A location being semantically likely, useful later, or recently mentioned is never a reason to rescan it before acquisition. + +Do not let the broadening threshold cause class restarts. Broadening means move to a different unvisited class or continue at the next unsearched instance of a promising storage class; it never means cycling back through exact instances already observed. + diff --git a/benchmark/spreadsheet_xarena/third_party/SkillOpt/ckpt/docvqa/gpt5.5_skill.md b/benchmark/spreadsheet_xarena/third_party/SkillOpt/ckpt/docvqa/gpt5.5_skill.md new file mode 100644 index 00000000..476992fc --- /dev/null +++ b/benchmark/spreadsheet_xarena/third_party/SkillOpt/ckpt/docvqa/gpt5.5_skill.md @@ -0,0 +1,26 @@ +# DocVQA Skill + +## Visual Evidence Discipline +- Read the document carefully before answering. +- Prefer the smallest exact text span that answers the question. + +- For questions asking for a value, count, page number, date, or graph reading, return only the requested value span; omit nearby labels, category names, units, or explanatory words unless the question explicitly asks for them. +- When several nearby strings look similar, choose the one whose surrounding labels or layout best match the question. + +## Exact Answer Discipline +- Copy names, numbers, and dates exactly from the document whenever possible. + +- Preserve the document's exact spelling and punctuation for names and quoted phrases; do not substitute similar letters or change straight/curly quotes, spacing, or parentheses when the visible text provides them. +- Prefer direct extraction over paraphrase. +- Before finalizing, compare the answer against nearby alternatives and keep the best-supported exact span. + +## Structured Layout Lookup +- For tables, first find the row or entry named in the question, then read the value under the requested column, header, date, or category; answer with that cell only. +- For forms, receipts, or labeled fields, locate the exact role, party, or field label mentioned in the question, then copy the filled-in value from the same line, box, block, or immediately adjacent field. +- For table-of-contents, indexed, numbered, or bulleted lists, match the requested title, entry, or point number, then follow the same line or list item to the associated value; do not take a nearby value from another item. + +## Anchored Handwriting / Nearby Text +- For handwritten or list/table questions with an anchor term, first locate the anchor, then inspect the immediately adjacent text in the same row, column, or nearby margin. If legible, provide the best-supported nearby span rather than leaving the answer blank. + + + \ No newline at end of file diff --git a/benchmark/spreadsheet_xarena/third_party/SkillOpt/ckpt/livemath/gpt5.5_skill.md b/benchmark/spreadsheet_xarena/third_party/SkillOpt/ckpt/livemath/gpt5.5_skill.md new file mode 100644 index 00000000..2d1ac842 --- /dev/null +++ b/benchmark/spreadsheet_xarena/third_party/SkillOpt/ckpt/livemath/gpt5.5_skill.md @@ -0,0 +1,35 @@ +# Live Mathematical MCQ Heuristics + +## Option Comparison + +### Meta-Options About Stronger Results +- Treat options of the form “one of the remaining options is correct, but a stronger result can be proven” as serious candidates, especially when the question asks for the strongest statement. +- If a concrete option is true but your theorem or derivation gives a strictly stronger conclusion not exactly listed, choose the meta-option rather than the weaker concrete statement. +- When options are nested by strength, rank them explicitly before answering: e.g. finite-time blowup is stronger than merely “not globally bounded”; positive stable growth is stronger than ordinary unboundedness; sharper constants, rates, exceptional-set bounds, endpoint inclusion, or full equivalences are stronger than weaker asymptotic versions. +- Compare all options before committing. The correct choice is often the strongest statement justified by the question, while nearby distractors are weaker, overstrong, or miss an equality case. +- Track exact quantifiers such as "there exists", "for every", "if and only if", and "exactly when". + +## Theorem-Level Precision + +- Do not add converse, realization, or classification claims unless the theorem explicitly proves them. Phrases such as “conversely,” “every such parameter occurs,” “if and only if,” or “exactly all” add strength beyond a one-way implication. +- Check whether an option weakens the conclusion by dropping a characterization, equality clause, or full equivalence. +- Check whether an option overstates the theorem by upgrading regularity, removing scale restrictions, or changing an existential statement into a universal one. + +## Hypotheses + +### Exact Conditions and Thresholds +- For biconditional/equivalence questions, reject conditions that are merely necessary or merely sufficient. A broader condition, such as congruence modulo a divisor instead of modulo the full modulus, is usually weaker and not equivalent unless the domain collapses the extra cases. +- For threshold conditions, verify the exact sign and endpoint: distinguish \(\mu_0\) from \(-\mu_0\), \(<\) from \(\le\), and whether the equality case belongs to the positive, zero, or negative parameter regime. +- When options differ by “for every” vs “for sufficiently large,” local vs global domains, strict vs non-strict inequalities, or dependence of constants, rank them by logical strength and match the sharpest justified version. +- Verify the hypotheses and domain carefully. Distractors often keep the theorem shape but alter the required assumptions. +- Pay close attention to equality cases, extremal conditions, and whether a result applies to the full family or only a restricted subfamily. + +## Final Answer +- Output the final answer as the single option label only. + +## Exact Scope and Quantitative Wording +- Distinguish global conclusions from localized or completed ones. Equivalence after localization, completion, or at each prime/scale is usually weaker than an unqualified equivalence. +- In estimate-heavy options, compare every quantitative detail: exponent, derivative index range, constants and their parameter dependence, log factors, additive terms, one-sided vs two-sided notation, and pointwise vs uniform convergence. + + + \ No newline at end of file diff --git a/benchmark/spreadsheet_xarena/third_party/SkillOpt/ckpt/officeqa/gpt5.5_skill.md b/benchmark/spreadsheet_xarena/third_party/SkillOpt/ckpt/officeqa/gpt5.5_skill.md new file mode 100644 index 00000000..869c1d48 --- /dev/null +++ b/benchmark/spreadsheet_xarena/third_party/SkillOpt/ckpt/officeqa/gpt5.5_skill.md @@ -0,0 +1,50 @@ +# OfficeQA Skill + +## Retrieval Discipline + +- When an external official time-series observation is needed, prefer the source's series/data-download/table page once identified. If exact-date or guessed-value searches return empty results, stop repeating them; broaden to the official series name/code plus `data` or `download` and use the table values. + +- Treat provided/oracle parsed pages as primary evidence: if they contain the relevant table and period, extract directly from them before searching elsewhere; search only for missing continuation pages, missing periods, or an official actual value not present. +- Start by narrowing to the most likely candidate file before reading long passages. +- Prefer targeted search terms that name the exact entity, period, measure, or table concept from the question. +- After a promising match, read only a small surrounding span and verify it matches the requested year, basis, and unit. + +- If the requested date range extends beyond the provided/oracle page, first enumerate the required periods and verify that every period is present in evidence. Do not compute from a partial ledger or fill missing periods from memory; retrieve continuation pages, adjacent issues, or a later issue of the same table that contains the missing dates/revisions. + +## Evidence Discipline +- Extract the exact value from the retrieved text before doing any arithmetic. +- Keep track of each operand's period, unit, and semantic role so nearby proxy values are not mixed in. + +- For Treasury financing narratives, label each amount by transaction role before calculating: offered amount, tenders/subscriptions received, tenders accepted, competitive/noncompetitive accepted, foreign or Government-account exchange tenders, refunding, and **new cash** are not interchangeable. +- When converting currencies or scales, make a direction ledger first: source table unit, source currency, exchange-rate orientation (foreign currency per U.S. dollar means divide by the rate; U.S. dollars per foreign unit means multiply), and requested final unit. + +- For tables, align values by row label and exact column header, not proximity alone; watch for continued or unlabeled columns, footnotes, adjacent amount-versus-percent columns, fiscal-year versus calendar-year sections, and repeated month rows under different year blocks. +- If the question asks for a transformed or derived quantity, compute only after confirming every operand. + +- For derived comparisons, preserve the direction and sign implied by the wording: “change from A to B” means B minus A; “former than latter” means former minus latter; “share accounted for by X” means X divided by the stated total; paired “gap” questions require computing each within-row difference before ranking. +- For statistical, regression, correlation, and growth-rate questions, write a formula ledger before calculating: confirm the exact series/endpoints, ordered vector, elapsed intervals, and requested convention such as continuously compounded rate, CAGR, Pearson correlation, or OLS index/year choice. +- For multi-stage questions where one table determines the period/entity used in another lookup, freeze that derived key with evidence first, then retrieve the second measure only for that exact month/year/reporting date/entity. + +- For inclusive time-series ranges, make a period-by-period ledger covering every requested month/year exactly once, preserving calendar versus fiscal basis, end-of-month or end-of-fiscal-month status, source units, and any specified adjustments. + +- For statistical transforms over time-series windows, confirm endpoint inclusion/exclusion exactly as worded, use consecutive time indices for trend regressions when appropriate, sort values before medians, and for logarithmic growth use ln(final/initial) before converting to the requested percentage format. + +## Final Answer Discipline + +- Before finalizing, enforce the requested unit and format: convert thousands/millions/billions or full nominal dollars as needed, then apply no-comma, fixed-decimal, whole-number, or nearest-tenth/thousandth formatting exactly as asked. +- Return the final answer only after one last consistency check against the retrieved evidence. +- Copy the final answer from a checked value, not from an unverified intermediate guess. + +## Statistical and Time-Series Calculation Checks + +- Before computing any statistic, write the intended formula and denominator convention. If the prompt explicitly says **population standard deviation**, divide by `n`; if it says **sample**, divide by `n-1`; for a z-score comparing one observation against a small set of comparison months/periods and no population convention is stated, estimate dispersion with the **sample** standard deviation of the comparison set. Do not round intermediate operands, weighted averages, logs, exchange-rate conversions, or standard deviations before the final requested rounding. +- For long inclusive ranges, first enumerate the expected count of observations and the first/last period, then verify the ledger has exactly that count. Exclude totals, cumulative-to-date columns, comparable-period columns, estimates, and extra latest-month columns outside the requested calendar or fiscal range. +- When a page contains multiple nearby sections with similar labels, use only the section whose title and row label match the requested measure exactly; do not compute from the first visible table if the requested measure/table title is absent or only partially shown. +- For Treasury security quotations, obey the table's quote basis. If the table states that price decimals are 32nds, convert quotes such as `99.27` as `99 + 27/32`, not as decimal `99.27`. If a task asks for smoothing, averaging, or forecasting in a target currency using period-specific exchange rates, convert each period's observation to the target currency first unless the prompt explicitly says to compute in the source currency and convert only the final result. + +## Stricter Final Formatting + +- Match any requested output template exactly. Unless the prompt explicitly asks for unit words or explanatory text, return only the numeric value or requested list; do not append words such as `million`, `dollars`, `percent`, or `percentage points`. Include symbols/commas only when the prompt requests currency-formatted output or the answer format clearly requires them. + + + diff --git a/benchmark/spreadsheet_xarena/third_party/SkillOpt/ckpt/searchqa/gpt5.5_skill.md b/benchmark/spreadsheet_xarena/third_party/SkillOpt/ckpt/searchqa/gpt5.5_skill.md new file mode 100644 index 00000000..b58d10ac --- /dev/null +++ b/benchmark/spreadsheet_xarena/third_party/SkillOpt/ckpt/searchqa/gpt5.5_skill.md @@ -0,0 +1,71 @@ +# Question Answering Skill + +(No learned rules yet. Rules will be added through the reflection process.) + +## Concise Answer Normalization +- Prefer the shortest unambiguous answer that directly satisfies the question. Do not include generic descriptors, legal suffixes, or expanded formal names unless the question specifically asks for the full official name or the descriptor is necessary to identify the entity. + +- If the answer appears inside a longer descriptive phrase, strip words that merely repeat the clue's requested type or modifiers already stated in the clue. For short-answer trivia, return the distinctive core entity or headword rather than role titles, product flavor adjectives, or place/facility designators, even when those words are part of a fuller official phrase, unless the full official name is explicitly requested. +- For place/name-etymology questions asking for “the name” or “the word” that means something, answer the distinctive name/word itself rather than a larger phrase with a generic type label. + +- For natural geographic features, preserve conventional feature designators such as “Lake,” “River,” “Bay,” “Gorge,” “Mount,” or “Island” when they are part of the proper name or match the requested feature type. Do not shorten “Lake Okeechobee,” “Tampa Bay,” or “Olduvai Gorge” to an ambiguous base name merely to be concise. +- For companies, brands, and organizations, answer the common distinctive name when sufficient; omit additions such as “Company,” “Corporation,” “Inc.,” etc. unless explicitly required. + +- Preserve the answer surface form supported by the strongest evidence when exact variants differ: spelling, capitalization, punctuation, and word order can matter. Do not substitute an equivalent official/common variant such as an alternate spelling or inverted institution name if a direct title/snippet/answer field gives the expected form. + +- When copying titles or quoted names, preserve ordinary ASCII punctuation from the evidence, especially straight apostrophes (`'`). Do not replace them with typographic curly quotes/apostrophes unless that exact stylized form is explicitly shown as the supported answer. + +- For nicknames, epithets, saints, and quoted titles, copy the supported surface form exactly, including spacing, capitalization, and conventional abbreviations such as “St.” Do not normalize a stylized or quoted form into a lowercase dictionary word or an expanded spelling when the clue/evidence points to the stylized answer. + +- For person answers in trivia or crossword-style clues, prefer the conventional supported name. Use just a surname, first name, or saint/regnal name only when the clue/source clearly expects that short form; otherwise use the canonical full personal name from the strongest evidence or answer field, especially when a lone given name would be ambiguous. +- Return the grammatical base form expected by the clue. Do not add a plural `s` merely because a crossword source pluralizes a shared name or category; if the clue lists people sharing a first name, answer the singular given name. + +- For common-noun category answers, default to the singular dictionary headword in trivia/crossword-style clues, even if the clue uses plural words like “these,” “those,” “places,” or “items” for grammar. Use a plural only when the term is inherently plural or an answer field/source clearly gives a plural phrase. + +- For common-noun clues about things being replaced, used in place of, or substituted by another system/item, answer the broad headword for the thing replaced unless a narrowing modifier is required by the clue or answer field. Do not add adjectives such as “letter,” “regular,” or “standard” merely because they appear in explanatory context. +- For fill-in-the-blank or definitional clues using words like “this” or “that,” provide a standalone noun phrase. Avoid context-dependent pronouns or possessives from the source text; use a natural article such as “the” when needed (e.g., answer “the highest point,” not “its highest point”). + +## Context-Grounded Evidence Matching +- Start by identifying the most distinctive terms in the question: proper names, dates, titles, quoted phrases, unusual words, roles, relationships, and category descriptors. +- Prioritize passages or document titles where several distinctive clue terms occur together, especially if the wording directly repeats or closely paraphrases the question. +- Treat document titles as useful evidence: the answer is often named in a title while the snippet confirms the clue facts. + +- Do not assume the document title itself is the answer. If the requested type differs from the title entity, use the title as context and extract the matching typed entity from the snippet or clue relationship. + +- For “known as,” “called,” “defined as,” or category/type clues, choose the canonical term explicitly used in the strongest matching title/snippet or scraped answer field rather than inventing a related derivative or near-synonym from the clue wording. When multiple plausible candidates appear, prefer the candidate whose evidence directly states the requested relationship and repeats the most distinctive clue facts. +- Ignore noisy results that only match generic words; prefer evidence that directly connects the clue facts to one specific entity. + +## Clue Interpretation and Answer Type +- For Jeopardy-style wording such as “this man,” “this group,” “this film,” “this country,” “this system,” “he,” or “his wife,” infer the expected answer type before choosing the answer. +- Use that expected type to validate candidates: answer with the concise person, place, title, organization, object, term, or phrase requested by the clue. + +- Treat modifiers attached to the requested type as hard filters, not background flavor: constraints like dates, “largest,” “2-letter-named,” “1978 remake,” “hot dog brand,” “dual throne,” or “on this company’s board” must all fit the candidate before you answer. +- For clues centered on creative works such as books, films, plays, songs, poems, or other media, first determine whether the clue asks for the work itself, its creator, a performer or cast member, a character, a quotation source, or a setting. Verbs such as “wrote,” “directed,” “stars,” “played,” and “set in,” plus pronouns like “he” or “her,” usually determine the target. + +- For fill-in-style clues with placeholders such as “this,” “these,” or “one of these,” substitute each candidate back into the clue and choose the concise answer that makes the full phrase, title, or fact read correctly. +- For terse clues that are just examples or names separated by commas, slashes, or “or,” infer the shared category, class, or synonym that links them, then answer with that concise common term. + +- For crossword-style clues, treat parenthetical numbers or stated letter counts as hard constraints on the answer length, and omit generic labels that would violate them. In dual-definition clues using wording like “X, or what Y does,” choose the single word that satisfies both senses and preserve the required inflected form. +- If the clue references an unavailable image or link with wording like “seen here,” “pictured,” or parenthetical visual hints, rely on the textual clues and context to infer the answer; do not treat the missing image as necessary evidence. +- If multiple snippets support the same entity, use that corroboration to choose the canonical/common form of the answer. + +## Trivia / Jeopardy Snippet Formats +- Retrieved trivia snippets may contain the clue and answer in scraped formats such as `CATEGORY | clue | answer`, `clue. ANSWER`, or labels like `right:`. +- When the question text matches the clue in such a snippet, extract the answer field or adjacent answer name, not the category or the whole clue sentence. + +## Common Clue Traps +- Watch for inverse relationships: if the clue says “His third wife was Jiang Qing,” the requested answer is the husband, not Jiang Qing. + +- More generally, preserve relation direction in clues: “A is evidence of this B,” “A is related to this language,” or “home to these characters” asks for the target of the relationship, not the entity already named in the clue. + +- When a clue says examples, models, breeds, members, or items “include,” “like,” or “such as” named entities, treat those names as evidence for the requested parent class or entity. Answer the encompassing brand, animal, category, place, or term requested by “this,” not one of the examples already given. +- If the question gives the start of a quotation or phrase, answer with the exact missing continuation from the context. + +- For song, poem, nursery-rhyme, or quotation clues, first decide whether the question asks for a missing word or phrase from the quote or for the associated creator, performer, or work; use pronouns and answer-type signals to choose the right target. +- When a clue asks for a constrained form such as a first name, abbreviation, acronym, or lyric word, return that exact form rather than the fuller person, title, or explanation; preserve conventional punctuation or spelling when it is part of the requested form. +- If the clue contains wordplay, quotation marks, or puns, treat them as hints, but answer with the real entity supported by the evidence. + +- If a clue includes a quoted title, quoted narration or lyric, named event, slogan, or other distinctive phrase but asks for an associated “this” entity, treat the quote or name as evidence to identify the requested person, work, place, group, category, source, or term; do not return the quoted anchor unless the clue explicitly asks for it. + + + diff --git a/benchmark/spreadsheet_xarena/third_party/SkillOpt/ckpt/spreadsheetbench/gpt5.5_skill.md b/benchmark/spreadsheet_xarena/third_party/SkillOpt/ckpt/spreadsheetbench/gpt5.5_skill.md new file mode 100644 index 00000000..9584d2df --- /dev/null +++ b/benchmark/spreadsheet_xarena/third_party/SkillOpt/ckpt/spreadsheetbench/gpt5.5_skill.md @@ -0,0 +1,133 @@ +# Spreadsheet Manipulation Skill (xlsx) + +## Overview +This skill guides agents in manipulating Excel (.xlsx) spreadsheets using Python. + +**Primary libraries**: `openpyxl` (structure-preserving read/write), `pandas` (data transformation). +Never use any other third-party libraries. + +--- + +## Common Workflow + +1. **Explore** the input file: list sheets, inspect headers, check dimensions. + +- Inspect actual workbook data beyond the preview, including nearby rows/columns, sample outputs, formulas, labels, headers, and any reference/example sheets such as `Output`, `Manual Result`, or `Desired...` tabs. + +- Treat existing filled cells in the requested output area or adjacent example tables as semantic examples for edge cases and expected formats, but still recompute and write the complete requested target range. + - Scan the used range for complete header groups, not just row 1. Tables may start in later rows/columns, have title rows above them, or have multiple source/result tables on the same sheet; use nearby labels and the requested output range to distinguish sources from destinations. + - Locate tables, fields, and target ranges by header text, nearby labels, and surrounding nonblank structure rather than fixed coordinates. Build header maps from actual cells when useful, e.g. `{str(cell.value).strip(): cell.column}`. +2. **Write `solution.py`** with `INPUT_PATH` and `OUTPUT_PATH` defined at the top. +3. **Execute** `python solution.py` and verify the output file was created. +4. **Confirm** the target cells/range contain the expected values. + +--- + +## Library Selection + +| Use case | Library | +|----------|---------| +| Preserve formulas, formatting, named ranges | `openpyxl` | +| Bulk data transformation, aggregation, sorting | `pandas` → write back with `openpyxl` | +| Simple cell read/write | `openpyxl` | + +**Warning**: `pandas.to_excel()` silently destroys existing formulas and named ranges. +When writing back to a spreadsheet that contains formulas, always use `openpyxl.save()`. + +**Formula evaluation caution**: `openpyxl` can write formulas but does **not** calculate them or update cached results. If the requested output will be checked as cell values, compute the result in Python and write literal values unless the user explicitly requires live formulas. When existing formulas are inputs to your logic, load a second workbook with `data_only=True` to read cached values while saving changes through the normal workbook: + +```python +wb = openpyxl.load_workbook(INPUT_PATH) +wb_values = openpyxl.load_workbook(INPUT_PATH, data_only=True) +ws = wb["Sheet1"] +ws_values = wb_values["Sheet1"] +``` + +Treat wording such as “write/fix a formula,” “SUMIFS/COUNTIFS,” “VBA,” or “macro” as a description of the spreadsheet logic unless the deliverable explicitly requires live formula text, an `.xlsm`, or a preserved VBA project. For normal `.xlsx` outputs, implement the equivalent logic in Python/openpyxl and write the computed final values to the requested cells so verification does not depend on Excel recalculation or macros. + +When the user provides an existing or broken formula, use it as a semantic specification: honor its referenced lookup ranges, criteria ranges, return ranges, aggregation intent, and error-handling behavior, then write the resulting values rather than guessing different source columns or leaving unevaluated formulas. + +--- + +## solution.py Template + +```python +import openpyxl +import pandas as pd + +INPUT_PATH = "..." # set to the actual input path +OUTPUT_PATH = "..." # set to the actual output path + +wb = openpyxl.load_workbook(INPUT_PATH) +ws = wb.active # or wb["SheetName"] + +# --- perform manipulation --- + +wb.save(OUTPUT_PATH) +``` + +--- + +## Output Requirements + +- Save the result to `OUTPUT_PATH`. +- Do not hardcode row counts or column letters — iterate over actual rows in the workbook. +- Preserve sheets and cells not mentioned in the instruction. + +## Matching and Target Range Hygiene + +- Choose the comparison operator from the instruction and examples: use `startswith` for “begins with”, substring search for “contains/search/occurrence”, and exact normalized equality only when a whole-cell match is implied. +- Create small helper functions for comparisons and numeric parsing. Normalize text by trimming, collapsing repeated spaces/NBSPs, and casefolding; when names or labels have punctuation/spacing inconsistencies, consider punctuation-insensitive keys. Parse numeric text after removing commas/currency symbols while preserving signs and decimal points; skip `None`/blank and booleans for numeric tests, and handle placeholders such as `"-"`, `"$"`, `"$0"`, blanks, and numeric zero deliberately. +- Normalize date keys deliberately: handle `datetime`/`date` objects, Excel serial numbers, and date-like strings, then compare at the granularity implied by the task, such as exact date, month, month/year, fiscal period, or year. For workday/date-window logic, compute the range in Python and exclude weekends/holidays as specified. + +- For monthly or period summary grids, canonicalize period labels from all sources: sheet names, title text, row/column headers, text months such as `March`, and actual date cells. Match summaries by normalized period plus the other stated criteria rather than by fixed month offsets or existing formulas. +- For date ranges and rolling windows, infer endpoint inclusivity from wording and examples. Phrases like `X to Y`, `through`, and `up to`, or examples such as `2 to 5` meaning `4 days`, usually require inclusive boundary handling. +- For time extraction or time-threshold logic, parse `datetime`, `time`, Excel serial/fractional times, and time-like strings into real Python `time`/`datetime` values. Write real time values with an Excel `number_format` such as `hh:mm:ss AM/PM`; do not write text substrings when the result should behave as a time. +- For joins, deduplication, grouping, interval lookups, lookup grids, and ordered outputs, build explicit normalized keys, including composite keys when the task refers to multiple fields. Preserve original source order within each group unless sorting is explicitly requested. +- For outputs that depend on other rows or lookup grids, make a first pass to build normalized dictionaries/groups/range structures, then a second pass to write results. Avoid nested full-sheet scans per row; split delimited tokens and ignore empty tokens, and treat error literals such as `#N/A` as meaningful sentinel values when the task refers to them. + +- For lookups, filters, joins, and label/header matching, normalize comparison keys consistently: trim whitespace, skip blanks explicitly, use case-insensitive text matching when appropriate, and treat numeric-looking IDs consistently (`330`, `330.0`, and `"330"`). Keep numeric outputs numeric; use `number_format` for display formatting instead of converting numbers to strings unless text is explicitly required. +- When replacing a generated output area, clear only the instructed target range before writing new results so stale values/formulas do not remain. Preserve formatting, column widths, borders, formulas, and unrelated cells unless the instruction explicitly asks to change them. + +- If the instruction includes formatting changes, apply them exactly after writing values and only to the requested cells/range. Use `openpyxl` styles for fills, alignment, fonts, borders, and number formats; convert hex colors to ARGB when needed, for example `#FFC000` → `FFFFC000`. For “format as text,” set `number_format = '@'` and write string values when the expected cell values are text. + +- When the instruction names a destination range or columns, write derived results directly there. Do not insert rows/columns, relocate the source table, or sort/delete source records unless that structural change is explicitly requested. +- For filtered lists, summaries, and aggregations, first collect all source records/results in memory, preserving the required order, then write from the first output row and clear leftover cells below the new results in the target columns. When adding rows, copy style/alignment/number format from an existing template row when appropriate; when deleting rows, delete from bottom to top to avoid row-index shifts. +- Preserve intended blanks as empty cells (`None`) rather than placeholder text or `0` unless the task specifies otherwise. + +- For numeric aggregation, crosstab, SUMIFS-like, and INDEX/MATCH-style summary outputs, infer missing-match behavior from table semantics and examples: numeric summary grids usually require literal `0` for no matching records, while filtered lists or “show only once” outputs usually require blanks (`None`). +- For blank-sensitive logic such as “if input is blank, output blank,” evaluate the driving input with `data_only=True` when it may itself be a formula, and write `None` for truly blank outputs rather than relying on a new formula returning `""`. + +## Robustness for Simple Fill Tasks + +- Prefer simple, auditable row/column loops over complex workbook XML parsing unless the task truly requires unsupported workbook internals. Before returning, run the script once to catch syntax/indentation errors and verify that representative target rows were actually written. + + +When the user asks for a formula, macro, VBA code, or a fix to an Excel formula, still deliver the completed workbook state: compute the intended results in Python and write literal final values into the requested cells. Do not write formula strings unless the task explicitly says the output must contain live formulas. + +After writing, reload or inspect the saved workbook and verify that every requested/evaluated target cell contains a non-formula literal where a value is expected. If a target cell is still `None` unexpectedly, fix the script before finishing. + +Use existing formulas in the workbook as examples/specifications, not as output. If a cell contains a reference formula such as `=A25` or an INDEX/MATCH/SUMIFS pattern, parse what source cells/ranges/criteria it refers to, compute those results yourself, and overwrite the destination with the referenced or calculated value. + +For blank-sensitive formula tasks, compute the branch explicitly: if the driving source cell is truly blank, write `None`; otherwise write the actual result such as `0`, `1`, a category label, or a lookup value. Never rely on `IF(...,"",...)` formulas to be recalculated later. + +For lookup/category tasks, locate both the input rows and the lookup table by headers and nearby labels. Support exact keys, numeric-looking keys, and interval/range tables; then fill every destination row that has a driving input, not just the first visible example. + +For “every nth row” or OFFSET-style tasks, infer the source column, first source row, and step from the provided examples or formulas, then copy the actual source values into the requested output range as literals. + +For schedule/calendar fill tasks, build a cycle-day-to-periods mapping from the schedule/template area first, then fill the daily rows across all requested class columns based on each row’s cycle day. Preserve repeated/double periods exactly as shown by the template; do not leave formulas in the schedule cells. + +For INDEX/MATCH problems where the first row works but subsequent rows fail, treat row labels, column/year headers, region/type criteria, and expense/category labels as a multi-key lookup. Fill the whole result matrix with values from the source data table, using cached `data_only` values when source cells are formulas. + +For multi-step macro/VBA-style requests, implement every stated operation in the workbook, not just the first deletion/filtering step. Re-read the numbered requirements before saving and verify later computed columns, totals, and derived fields as well as the obvious filtered rows. + +When a target range includes special rows such as `Total`, `Grand Total`, `min`, `max`, constraints, headers, or blank separators, do not apply ordinary row logic blindly to those rows. Compute totals as aggregates when indicated, and leave constraint/header/blank cells untouched unless explicitly requested. + +For residual-balancing tasks, identify data rows separately from min/max constraint rows. Add positive residuals from unit 1 toward unit 5 without exceeding max values; subtract negative residuals from unit 5 toward unit 1 without going below min values; update only the unit cells in actual data rows. + +For time-threshold rows, decide per row whether it is a normal data row or a summary row. Normal rows use the before/after threshold rule; summary rows should aggregate the computed normal-row results if the workbook labels or examples indicate a total. + +Keep scripts simple enough to run cleanly. Avoid unnecessary dynamic code generation and fragile f-strings with regex expressions inside them. Always execute the final `solution.py`; fix any syntax, indentation, or runtime error, then verify representative target cells. + +If workbook cells contain arbitrary sample text that could be sensitive or trigger content filters, do not quote large raw cell contents in your response. Process them locally in Python with neutral variable names and output only the completed script/workbook changes. + diff --git a/benchmark/spreadsheet_xarena/third_party/SkillOpt/configs/_base_/default.yaml b/benchmark/spreadsheet_xarena/third_party/SkillOpt/configs/_base_/default.yaml new file mode 100644 index 00000000..eb2d58da --- /dev/null +++ b/benchmark/spreadsheet_xarena/third_party/SkillOpt/configs/_base_/default.yaml @@ -0,0 +1,100 @@ +# SkillOpt default configuration — base for all environments. +# Environment configs should inherit via: _base_: default.yaml + +model: + backend: azure_openai + optimizer: gpt-5.5 + target: gpt-5.5 + optimizer_backend: openai_chat + target_backend: openai_chat + reasoning_effort: medium + rewrite_reasoning_effort: "" + rewrite_max_completion_tokens: 64000 + codex_exec_path: codex + codex_exec_sandbox: workspace-write + codex_exec_profile: "" + codex_exec_full_auto: false + codex_exec_reasoning_effort: none + codex_exec_use_sdk: auto + codex_exec_network_access: false + codex_exec_web_search: false + codex_exec_approval_policy: never + claude_code_exec_path: claude + claude_code_exec_profile: "" + claude_code_exec_use_sdk: auto + claude_code_exec_effort: medium + claude_code_exec_max_thinking_tokens: 16384 + codex_trace_to_optimizer: true + azure_openai_endpoint: "" # e.g. "https://your-resource.openai.azure.com/" + azure_openai_api_version: "2024-12-01-preview" + azure_openai_api_key: "" # Fill locally if you do not export AZURE_OPENAI_API_KEY + azure_openai_auth_mode: "" # empty → fall back to AZURE_OPENAI_AUTH_MODE env (default "azure_cli") + azure_openai_ad_scope: "https://cognitiveservices.azure.com/.default" + azure_openai_managed_identity_client_id: "" + optimizer_azure_openai_endpoint: "" # e.g. "https://your-resource.openai.azure.com/" + optimizer_azure_openai_api_version: "2024-12-01-preview" + optimizer_azure_openai_api_key: "" + optimizer_azure_openai_auth_mode: "" # empty → fall back to OPTIMIZER_AZURE_OPENAI_AUTH_MODE env, then shared + optimizer_azure_openai_ad_scope: "https://cognitiveservices.azure.com/.default" + optimizer_azure_openai_managed_identity_client_id: "" + target_azure_openai_endpoint: "" # e.g. "https://your-resource.openai.azure.com/" + target_azure_openai_api_version: "2024-12-01-preview" + target_azure_openai_api_key: "" + target_azure_openai_auth_mode: "" # empty → fall back to TARGET_AZURE_OPENAI_AUTH_MODE env, then shared + target_azure_openai_ad_scope: "https://cognitiveservices.azure.com/.default" + target_azure_openai_managed_identity_client_id: "" + + # MiniMax backend settings (minimax_chat target) + minimax_base_url: "" # https://api.minimax.io/v1 if blank + minimax_api_key: "" + minimax_model: "MiniMax-M2.7" + minimax_temperature: "0.7" + minimax_max_tokens: "8000" + minimax_enable_thinking: "false" + optimizer_minimax_base_url: "" # per-role override + target_minimax_base_url: "" # per-role override + optimizer_minimax_api_key: "" + target_minimax_api_key: "" + +train: + num_epochs: 4 + train_size: 0 # 0 = derive from dataset split when available + batch_size: 40 + accumulation: 1 + seed: 42 + +gradient: + minibatch_size: 8 + merge_batch_size: 8 + analyst_workers: 16 + max_analyst_rounds: 3 + failure_only: false + +optimizer: + learning_rate: 4 # max edits per step (edit_budget) + min_learning_rate: 2 # min edits for decay schedulers + lr_scheduler: cosine # constant / linear / cosine / autonomous + lr_control_mode: fixed # fixed / autonomous / none + skill_update_mode: patch # patch / rewrite_from_suggestions / full_rewrite_minibatch + use_slow_update: true + slow_update_samples: 20 + slow_update_gate_with_selection: false + longitudinal_pair_policy: mixed # mixed / changed / unchanged + use_meta_skill: true + +evaluation: + use_gate: true + sel_env_num: 0 + test_env_num: 0 + eval_test: true + +env: + name: "" + skill_init: "" + split_mode: ratio # ratio = build deterministic split from data_path; split_dir = use pre-split train/val/test + split_seed: 42 + split_dir: "" + data_path: "" + split_output_dir: "" + exec_timeout: 120 # per target model/code-agent call timeout in seconds + out_root: "" diff --git a/benchmark/spreadsheet_xarena/third_party/SkillOpt/configs/alfworld/default.yaml b/benchmark/spreadsheet_xarena/third_party/SkillOpt/configs/alfworld/default.yaml new file mode 100644 index 00000000..95041405 --- /dev/null +++ b/benchmark/spreadsheet_xarena/third_party/SkillOpt/configs/alfworld/default.yaml @@ -0,0 +1,29 @@ +_base_: ../_base_/default.yaml + +train: + train_size: 0 + accumulation: 1 + +gradient: + minibatch_size: 8 + merge_batch_size: 8 + +optimizer: + learning_rate: 4 + +evaluation: + sel_env_num: 0 + test_env_num: 0 + +env: + name: alfworld + skill_init: skillopt/envs/alfworld/skills/initial.md + split_mode: split_dir + split_dir: data/alfworld_path_split + data_path: "" + split_output_dir: "" + max_steps: 50 + max_completion_tokens: 16384 + workers: 8 + max_api_workers: 8 + limit: 0 diff --git a/benchmark/spreadsheet_xarena/third_party/SkillOpt/configs/docvqa/default.yaml b/benchmark/spreadsheet_xarena/third_party/SkillOpt/configs/docvqa/default.yaml new file mode 100644 index 00000000..c3e8ce05 --- /dev/null +++ b/benchmark/spreadsheet_xarena/third_party/SkillOpt/configs/docvqa/default.yaml @@ -0,0 +1,28 @@ +_base_: ../_base_/default.yaml + +model: + reasoning_effort: medium + +train: + batch_size: 40 + accumulation: 1 + +gradient: + minibatch_size: 8 + merge_batch_size: 8 + +optimizer: + learning_rate: 4 + +env: + name: docvqa + skill_init: skillopt/envs/docvqa/skills/initial.md + split_mode: split_dir + split_dir: data/docvqa/splits + data_path: "" + split_output_dir: "" + max_turns: 1 + max_completion_tokens: 16384 + workers: 16 + image_detail: auto + limit: 0 diff --git a/benchmark/spreadsheet_xarena/third_party/SkillOpt/configs/features/soft_gate.yaml b/benchmark/spreadsheet_xarena/third_party/SkillOpt/configs/features/soft_gate.yaml new file mode 100644 index 00000000..7b622d3a --- /dev/null +++ b/benchmark/spreadsheet_xarena/third_party/SkillOpt/configs/features/soft_gate.yaml @@ -0,0 +1,47 @@ +# ───────────────────────────────────────────────────────────────────────────── +# Feature: soft / mixed validation-gate metric (community-contributed, PR #25) +# ───────────────────────────────────────────────────────────────────────────── +# +# This is NOT a default SkillOpt setting and was NOT used to produce the +# numbers reported in the paper. It is provided as a reference for users +# who encounter a specific scenario where the default `hard` gate is too +# coarse to drive training. +# +# When to consider this: +# - You are running on a custom environment. +# - Your held-out *selection* split has very few items (e.g. ≤ ~10). +# - Your reward function is continuous / partial-credit (e.g. F1, BLEU, +# soft match) rather than purely binary 0/1. +# +# Symptom this addresses: +# With a small selection split + continuous rewards, candidate skills +# often improve per-item soft scores (e.g. 0.06 → 0.26 on one item) but +# never flip the discrete hard outcome. The default `hard` gate then +# rejects every candidate and training stalls. Switching the gate to +# `soft` or `mixed` lets these partial improvements be accepted. +# +# When NOT to use this: +# - When reproducing the paper. The paper-reported numbers were obtained +# under the default `hard` gate. +# - When your selection split is large (dozens+ items) and / or your +# reward is already binary — `hard` is the more conservative choice +# and matches the design described in the paper. +# +# To use: inherit your env config from this file, e.g. +# _base_: ../features/soft_gate.yaml +# or copy the `evaluation:` block below into your config. +# ───────────────────────────────────────────────────────────────────────────── + +_base_: ../_base_/default.yaml + +evaluation: + # Three options: + # 'hard' — default; exact-match accuracy. Use this to reproduce the paper. + # 'soft' — per-item soft / partial-credit score (recommended for the + # small-split + continuous-reward scenario described above). + # 'mixed' — weighted average: (1 - w) * hard + w * soft, with `w` set by + # `gate_mixed_weight` below. + gate_metric: soft + + # Only used when gate_metric == 'mixed'. Ignored otherwise. + gate_mixed_weight: 0.5 diff --git a/benchmark/spreadsheet_xarena/third_party/SkillOpt/configs/livemathematicianbench/default.yaml b/benchmark/spreadsheet_xarena/third_party/SkillOpt/configs/livemathematicianbench/default.yaml new file mode 100644 index 00000000..19401abc --- /dev/null +++ b/benchmark/spreadsheet_xarena/third_party/SkillOpt/configs/livemathematicianbench/default.yaml @@ -0,0 +1,22 @@ +_base_: ../_base_/default.yaml + +train: + train_size: 0 + batch_size: 40 + accumulation: 1 + +env: + name: livemathematicianbench + skill_init: skillopt/envs/livemathematicianbench/skills/initial.md + split_mode: split_dir + split_dir: data/livemathematicianbench_split + data_path: "" + split_output_dir: "" + max_turns: 1 + max_completion_tokens: 16384 + exec_timeout: 300 + workers: 64 + limit: 0 + shuffle_choices: true + use_theorem: false + use_sketch: false diff --git a/benchmark/spreadsheet_xarena/third_party/SkillOpt/configs/officeqa/default.yaml b/benchmark/spreadsheet_xarena/third_party/SkillOpt/configs/officeqa/default.yaml new file mode 100644 index 00000000..7b72f1a4 --- /dev/null +++ b/benchmark/spreadsheet_xarena/third_party/SkillOpt/configs/officeqa/default.yaml @@ -0,0 +1,34 @@ +_base_: ../_base_/default.yaml + +model: + reasoning_effort: medium + +train: + batch_size: 40 + accumulation: 1 + +gradient: + minibatch_size: 8 + merge_batch_size: 8 + +optimizer: + learning_rate: 4 + +env: + name: officeqa + skill_init: skillopt/envs/officeqa/skills/initial.md + split_mode: split_dir + split_dir: data/officeqa_split + data_dirs: + - data/officeqa_docs_official + workers: 4 + max_tool_turns: 24 + max_completion_tokens: 16384 + search_mode: offline + max_queries_per_turn: 4 + search_api_url: http://apisix.westus2.cloudapp.azure.com/search_tool/search + search_auth_env: OFFICEQA_CUSTOM_SEARCH_AUTH + search_provider: duckduckgo + search_max_num_results: 4 + search_timeout_seconds: 20 + limit: 0 diff --git a/benchmark/spreadsheet_xarena/third_party/SkillOpt/configs/searchqa/default.yaml b/benchmark/spreadsheet_xarena/third_party/SkillOpt/configs/searchqa/default.yaml new file mode 100644 index 00000000..a1177ab4 --- /dev/null +++ b/benchmark/spreadsheet_xarena/third_party/SkillOpt/configs/searchqa/default.yaml @@ -0,0 +1,32 @@ +_base_: ../_base_/default.yaml + +model: + reasoning_effort: medium + +train: + train_size: 400 + batch_size: 40 + accumulation: 1 + +gradient: + minibatch_size: 8 + merge_batch_size: 8 + +optimizer: + learning_rate: 4 + +evaluation: + sel_env_num: 0 + test_env_num: 0 + +env: + name: searchqa + skill_init: skillopt/envs/searchqa/skills/initial.md + split_mode: split_dir + split_dir: data/searchqa_split + data_path: "" + split_output_dir: "" + max_turns: 1 + max_completion_tokens: 16384 + workers: 24 + limit: 0 diff --git a/benchmark/spreadsheet_xarena/third_party/SkillOpt/configs/spreadsheetbench/default.yaml b/benchmark/spreadsheet_xarena/third_party/SkillOpt/configs/spreadsheetbench/default.yaml new file mode 100644 index 00000000..e93c3a3b --- /dev/null +++ b/benchmark/spreadsheet_xarena/third_party/SkillOpt/configs/spreadsheetbench/default.yaml @@ -0,0 +1,34 @@ +_base_: ../_base_/default.yaml + +model: + reasoning_effort: medium + +train: + train_size: 80 + batch_size: 40 + accumulation: 1 + +gradient: + minibatch_size: 8 + merge_batch_size: 8 + +optimizer: + learning_rate: 4 + +evaluation: + sel_env_num: 0 + test_env_num: 0 + +env: + name: spreadsheetbench + skill_init: skillopt/envs/spreadsheetbench/skills/initial.md + split_mode: split_dir + split_dir: data/spreadsheetbench_split + data_path: "" + split_output_dir: "" + data_root: data/spreadsheetbench_verified_400 + mode: multi + max_turns: 30 + max_completion_tokens: 16384 + exec_timeout: 600 + workers: 24 diff --git a/benchmark/spreadsheet_xarena/third_party/SkillOpt/docs/contributing.md b/benchmark/spreadsheet_xarena/third_party/SkillOpt/docs/contributing.md new file mode 100644 index 00000000..818a67e4 --- /dev/null +++ b/benchmark/spreadsheet_xarena/third_party/SkillOpt/docs/contributing.md @@ -0,0 +1,69 @@ +# Contributing to SkillOpt + +Thank you for your interest in contributing to SkillOpt! This guide covers how to get started. + +## Development Setup + +```bash +git clone https://github.com/microsoft/SkillOpt.git +cd SkillOpt +pip install -e ".[dev]" +``` + +## Ways to Contribute + +### 🐛 Bug Reports + +Open an issue with: +- Steps to reproduce +- Expected vs actual behavior +- Config file used (sanitize API keys) +- Python version and OS + +### 🔧 New Benchmark + +See [Add a New Benchmark](guide/new-benchmark.md) for the implementation guide. + +**Checklist:** +- [ ] Data loader in `skillopt/envs//loader.py` +- [ ] Environment adapter in `skillopt/envs//env.py` +- [ ] Config file in `configs//default.yaml` +- [ ] Registration in `skillopt/envs/__init__.py` +- [ ] Documentation page in `docs/` + +### 🤖 New Model Backend + +See [Add a New Model Backend](guide/new-backend.md) for the implementation guide. + +**Checklist:** +- [ ] Backend in `skillopt/model/.py` +- [ ] Registration in `skillopt/model/__init__.py` +- [ ] API key entry in `.env.example` +- [ ] Documentation update + +### 📝 Documentation + +Documentation is built with MkDocs Material: + +```bash +pip install -e ".[docs]" +mkdocs serve # Preview at http://localhost:8000 +``` + +## Code Style + +- Follow existing patterns in the codebase +- Use type hints for function signatures +- Keep docstrings concise + +## Pull Request Process + +1. Fork the repository +2. Create a feature branch: `git checkout -b feature/my-benchmark` +3. Make your changes +4. Test with an existing benchmark config +5. Submit a PR with a clear description + +## License + +By contributing, you agree that your contributions will be licensed under the MIT License. diff --git a/benchmark/spreadsheet_xarena/third_party/SkillOpt/docs/guide/configuration.md b/benchmark/spreadsheet_xarena/third_party/SkillOpt/docs/guide/configuration.md new file mode 100644 index 00000000..55a3a86f --- /dev/null +++ b/benchmark/spreadsheet_xarena/third_party/SkillOpt/docs/guide/configuration.md @@ -0,0 +1,109 @@ +# Configuration Guide + +SkillOpt uses YAML configuration files with a hierarchical override system. + +## Config Structure + +``` +configs/ +├── _base_/ +│ └── default.yaml # Global defaults +├── searchqa/ +│ └── default.yaml # SearchQA overrides +├── docvqa/ +│ └── default.yaml # DocVQA overrides +└── alfworld/ + └── default.yaml # ALFWorld overrides +``` + +Benchmark configs inherit from `_base_/default.yaml` and override specific values. + +## Key Parameters + +### Model + +```yaml +model: + backend: azure_openai # azure_openai | openai_chat | claude_code_exec | qwen + optimizer: gpt-5.5 # Optimizer model (for reflection) + target: gpt-5.5 # Target model (for rollout) +``` + +### Training + +```yaml +train: + num_epochs: 4 # Number of training epochs + batch_size: 40 # Tasks per step (batch size) + accumulation: 1 # Gradient accumulation + seed: 42 +``` + +### Gradient (Reflection) + +```yaml +gradient: + minibatch_size: 8 # Reflect minibatch size + analyst_workers: 16 # Parallel reflection workers + max_analyst_rounds: 3 # Max rounds of analyst reflection + failure_only: false # Only reflect on failures +``` + +### Optimizer + +```yaml +optimizer: + learning_rate: 4 # Max edits per step (edit budget) + min_learning_rate: 2 # Min edits for decay schedulers + lr_scheduler: cosine # constant | linear | cosine | autonomous + use_slow_update: true # Momentum-like blending at epoch boundary + slow_update_samples: 20 # Samples for slow update evaluation + use_meta_skill: true # Cross-epoch strategy memory +``` + +### Evaluation + +```yaml +evaluation: + use_gate: true # Validation gating (accept/reject updates) + eval_test: true # Run test evaluation after training +``` + +### Environment (Data) + +```yaml +env: + name: searchqa # Benchmark name + split_mode: ratio # ratio | split_dir + split_ratio: "2:1:7" # train:val:test ratio + data_path: "" # Path to dataset + exec_timeout: 120 # Per-task timeout (seconds) +``` + +## CLI Overrides + +Override any config value from the command line: + +```bash +python scripts/train.py \ + --config configs/searchqa/default.yaml \ + optimizer.learning_rate=16 \ + optimizer.lr_scheduler=linear \ + gradient.analyst_workers=8 +``` + +## Environment Variables + +Model credentials are loaded from environment variables: + +| Variable | Backend | Description | +|---|---|---| +| `AZURE_OPENAI_ENDPOINT` | azure_openai | Azure resource endpoint | +| `AZURE_OPENAI_API_KEY` | azure_openai | Azure API key | +| `OPENAI_API_KEY` | openai | OpenAI API key | +| `ANTHROPIC_API_KEY` | claude | Anthropic API key | +| `QWEN_API_BASE` | qwen | Local Qwen vLLM endpoint | + +## Full Reference + +See [Configuration Reference](../reference/config.md) for the complete parameter list. diff --git a/benchmark/spreadsheet_xarena/third_party/SkillOpt/docs/guide/dl-analogy.md b/benchmark/spreadsheet_xarena/third_party/SkillOpt/docs/guide/dl-analogy.md new file mode 100644 index 00000000..758566ff --- /dev/null +++ b/benchmark/spreadsheet_xarena/third_party/SkillOpt/docs/guide/dl-analogy.md @@ -0,0 +1,51 @@ +# Deep Learning ↔ SkillOpt Analogy + +SkillOpt is designed around a core insight: **optimizing natural-language prompts follows the same structure as training neural networks**. This page maps every DL concept to its SkillOpt counterpart. + +## Complete Mapping + +| Deep Learning | SkillOpt | Description | +|---|---|---| +| **Model weights** | Skill document (Markdown) | The thing being optimized | +| **Forward pass** | Rollout | Target executes tasks using current skill | +| **Loss function** | Task evaluator | Scores task execution quality | +| **Backpropagation** | Reflect | Optimizer analyzes failures → edit patches | +| **Gradients** | Edit patches | Proposed changes to the skill | +| **Gradient aggregation** | Patch aggregation | Merge similar edits | +| **Gradient clipping** | Edit selection | Cap max edits per step | +| **Learning rate** | `learning_rate` | Max number of edits applied per step | +| **LR scheduler** | `lr_scheduler` | Decay schedule: cosine, linear, constant | +| **SGD step** | Skill update | Apply selected patches to document | +| **Validation set** | Selection split | Gate checks improvement before accepting | +| **Early stopping** | Gate patience | Reject updates that don't improve | +| **Training step** | Step | One rollout → reflect → update cycle | +| **Epoch** | Epoch | Full pass with slow update + meta memory | +| **Momentum** | Slow update | Longitudinal comparison at epoch boundary | +| **Meta-learning** | Meta skill | Cross-epoch optimizer strategy memory | +| **Batch size** | `batch_size` | Tasks sampled per rollout | +| **Data parallelism** | `analyst_workers` | Parallel reflection workers | +| **Training set** | Train split | Items used for rollout | +| **Test set** | Test split | Held-out final evaluation | +| **Warm-up** | (implicit) | High LR early steps explore broadly | +| **Checkpointing** | Skill snapshots | Saved after each accepted step | +| **Transfer learning** | Seed skill / cross-benchmark init | Start from pre-trained skill | + +## Why This Analogy Matters + +1. **Familiar mental model**: ML practitioners immediately understand how to tune SkillOpt +2. **Principled hyperparameter search**: Grid search over `learning_rate` × `lr_scheduler` works just like in DL +3. **Proven mechanisms**: Gating ≈ validation-based selection, patience ≈ early stopping, slow update ≈ momentum — all with strong theoretical motivation + +## Hyperparameter Transfer Rules + +From our experiments, these DL intuitions transfer well: + +!!! success "What transfers" + - **Cosine schedule > constant** — same as in DL, cosine annealing helps convergence + - **Moderate LR (4-16) > very high/low** — too few edits = slow learning, too many = noisy + - **Slow update helps** — longitudinal comparison prevents catastrophic forgetting across epochs + - **Meta skill memory improves reflection** — optimizer benefits from cross-epoch strategy notes + +!!! warning "What doesn't transfer" + - **Batch size ≠ better** — larger rollout batches have diminishing returns due to API costs + - **More epochs ≠ better** — skills converge faster than neural networks (2-4 epochs usually enough) diff --git a/benchmark/spreadsheet_xarena/third_party/SkillOpt/docs/guide/first-experiment.md b/benchmark/spreadsheet_xarena/third_party/SkillOpt/docs/guide/first-experiment.md new file mode 100644 index 00000000..2a655898 --- /dev/null +++ b/benchmark/spreadsheet_xarena/third_party/SkillOpt/docs/guide/first-experiment.md @@ -0,0 +1,110 @@ +# Your First Experiment + +This guide walks through running a complete SkillOpt training on SearchQA. + +## 1. Choose a Benchmark + +SkillOpt includes ready-to-use configs for several benchmarks: + +| Benchmark | Difficulty | Typical Runtime | +|---|---|---| +| SearchQA | ⭐ Easy | ~30 min | +| DocVQA | ⭐⭐ Medium | ~2 hours | +| ALFWorld | ⭐⭐⭐ Hard | ~3 hours | + +We'll use **SearchQA** as it's the fastest to complete. + +## 2. Configure + +Review the config file: + +```bash +cat configs/searchqa/default.yaml +``` + +Key parameters (deep learning analogy in parentheses): + +```yaml +train: + num_epochs: 4 # (epochs) + batch_size: 40 # (batch size) + +optimizer: + learning_rate: 4 # (max edits per step) + lr_scheduler: cosine # (learning rate schedule) + use_slow_update: true # (momentum at epoch boundary) + use_meta_skill: true # (cross-epoch optimizer memory) + +gradient: + analyst_workers: 16 # (parallel reflection workers) + +evaluation: + use_gate: true # (validation gating) +``` + +## 3. Train + +```bash +python scripts/train.py --config configs/searchqa/default.yaml +``` + +You'll see output like: + +``` +[Step 1/8] Rollout: 20 items, 4 workers... +[Step 1/8] Score: 0.65 → Reflect... +[Step 1/8] 6 edit patches generated +[Step 1/8] Selected 4 edits (lr=8, cosine → 7.7) +[Step 1/8] Gate: val score 0.68 > 0.65 ✓ ACCEPT +[Step 2/8] ... +``` + +## 4. Monitor + +Training outputs are saved to `outputs///`: + +``` +outputs/searchqa/2024-01-15_10-30-00/ +├── steps/ +│ ├── step_0001/ +│ │ ├── candidate_skill.md +│ │ ├── step_record.json +│ │ └── trajectory_digest.json +│ └── step_0002/ +├── slow_update/ +│ └── epoch_02/ +├── meta_skill/ +│ └── epoch_02/ +├── skills/ +│ └── step_0001.md +├── best_skill.md +├── history.json +└── config.yaml +``` + +## 5. Evaluate + +Evaluate the best skill on the test split: + +```bash +python scripts/eval_only.py \ + --config configs/searchqa/default.yaml \ + --skill outputs/searchqa//skills/best_skill.md +``` + +## WebUI + +Prefer a graphical interface? Launch the WebUI: + +```bash +pip install -e ".[webui]" +python -m skillopt_webui.app +``` + +Then open `http://localhost:7860` in your browser to configure parameters and launch training. + +## Next Steps + +- [Understand the training loop](training-loop.md) +- [Configuration reference](../reference/config.md) +- [Add a new benchmark](new-benchmark.md) diff --git a/benchmark/spreadsheet_xarena/third_party/SkillOpt/docs/guide/installation.md b/benchmark/spreadsheet_xarena/third_party/SkillOpt/docs/guide/installation.md new file mode 100644 index 00000000..0fd390e5 --- /dev/null +++ b/benchmark/spreadsheet_xarena/third_party/SkillOpt/docs/guide/installation.md @@ -0,0 +1,89 @@ +# Installation + +## Requirements + +- Python ≥ 3.10 +- At least one model API key (Azure OpenAI, OpenAI, Anthropic, or local Qwen) + +## Quick Install + +```bash +git clone https://github.com/microsoft/SkillOpt.git +cd SkillOpt +pip install -e . +``` + +## Optional Dependencies + +Install extras for specific benchmarks or backends: + +=== "ALFWorld" + + ```bash + pip install -e ".[alfworld]" + ``` + +=== "Claude Backend" + + ```bash + pip install -e ".[claude]" + ``` + +=== "Qwen (Local)" + + ```bash + pip install -e ".[qwen]" + ``` + +=== "WebUI" + + ```bash + pip install -e ".[webui]" + ``` + +=== "Development" + + ```bash + pip install -e ".[dev]" + ``` + +=== "All" + + ```bash + pip install -e ".[alfworld,claude,qwen,webui,dev]" + ``` + +## Environment Variables + +Copy the example `.env` file and fill in your credentials: + +```bash +cp .env.example .env +``` + +Edit `.env` with your API keys: + +```ini +# Azure OpenAI (default backend) +AZURE_OPENAI_ENDPOINT=https://your-resource.openai.azure.com/ +AZURE_OPENAI_API_KEY=your-key + +# Or use OpenAI directly +OPENAI_API_KEY=sk-... + +# Or Anthropic Claude +ANTHROPIC_API_KEY=sk-ant-... +``` + +!!! tip + You only need credentials for the backend you plan to use. Azure OpenAI is the default. + +## Verify Installation + +```bash +python -c "import skillopt; print('SkillOpt ready!')" +``` + +## Next Steps + +→ [Run your first experiment](first-experiment.md) diff --git a/benchmark/spreadsheet_xarena/third_party/SkillOpt/docs/guide/local-env-smoke.md b/benchmark/spreadsheet_xarena/third_party/SkillOpt/docs/guide/local-env-smoke.md new file mode 100644 index 00000000..f1af13ed --- /dev/null +++ b/benchmark/spreadsheet_xarena/third_party/SkillOpt/docs/guide/local-env-smoke.md @@ -0,0 +1,143 @@ +# Local Environment Smoke Tests + +This guide describes a lightweight pattern for testing a custom SkillOpt environment before connecting it to expensive model calls or a full benchmark dataset. + +The goal is to validate the training loop plumbing first: + +- config loading +- adapter construction +- dataloader splits +- rollout output shape +- reflection patch shape +- merge/rank/update control flow +- artifact creation under `out_root` + +Once those are stable, you can switch the same environment to real model calls and larger evaluation splits. + +## 1. Add a tiny fixture split + +Start with a handful of deterministic examples that cover the expected pass/fail cases for your environment. Keep them small enough that a single training step can run locally. + +A minimal fixture item usually needs: + +```json +{ + "id": "example-1", + "split": "train", + "question": "...", + "expected": "..." +} +``` + +Use the split names your adapter maps to SkillOpt phases: + +- `train` for optimization rollouts +- `val` or `valid_seen` for selection/gating +- `test` or `valid_unseen` for final evaluation + +## 2. Support an offline mock mode + +Add a configuration flag such as `mock: true` to your adapter. In mock mode, `rollout()` should return deterministic responses without calling external model APIs. + +This lets you verify the SkillOpt loop with a fast command such as: + +```bash +python scripts/train.py \ + --config configs/myenv/tiny_mock.yaml +``` + +Mock mode should still write the same artifacts as a real run, for example: + +- `responses.json` +- `rollout_results.json` +- `ranked_edits.json` +- `candidate_skill.md` +- `summary.json` + +## 3. Keep the smoke config tiny + +A CI-friendly smoke config should run a single small step: + +```yaml +train: + num_epochs: 1 + train_size: 3 + batch_size: 3 + +gradient: + minibatch_size: 1 + merge_batch_size: 2 + analyst_workers: 1 + max_analyst_rounds: 1 + +optimizer: + learning_rate: 1 + min_learning_rate: 1 + lr_scheduler: constant + skill_update_mode: patch + use_slow_update: false + +evaluation: + use_gate: true + sel_env_num: 2 + test_env_num: 2 + eval_test: false + +env: + name: myenv + out_root: outputs/myenv_tiny_mock + mock: true +``` + +Prefer a mock config that runs without credentials. That makes it useful for contributors and CI. + +## 4. Validate optimizer JSON before returning it + +If your environment or extension asks an LLM to merge or rank skill edits, validate the returned JSON before passing it back into SkillOpt. This avoids silent fallbacks from empty, malformed, or out-of-range responses. + +Useful checks for edit payloads: + +- response is a JSON object +- `edits` is a non-empty list +- every edit is an object +- every edit has an allowed operation +- required fields such as `content` or `target` are present for that operation + +Useful checks for ranking payloads: + +- `selected_indices` exists +- indices are integers +- indices are unique +- indices are within the candidate edit range +- selected count does not exceed the edit budget + +On failure, retry with a compact prompt that includes the schema error. If retries fail, raise an explicit error instead of silently accepting malformed output. + +## 5. Run progressively stronger checks + +A good development sequence is: + +```bash +python -m py_compile scripts/train.py skillopt/envs/myenv/adapter.py +python scripts/train.py --config configs/myenv/tiny_mock.yaml +python scripts/train.py --config configs/myenv/tiny.yaml +``` + +For the real tiny run, verify that: + +- the run completes +- `summary.json` is written +- `ranked_edits.json` contains the expected ranking metadata +- any optimizer bridge log marks the response schema as valid +- no generated files are written outside `out_root` + +## 6. Keep custom environments isolated + +When adding a custom environment to the registry, avoid side effects for existing benchmarks: + +- lazy-import optional dependencies +- install environment-specific hooks only when `cfg["env"]` matches your environment +- keep mock behavior behind an explicit config flag +- write generated artifacts only under `out_root` + +This makes it easier to review and test a custom integration without affecting the built-in benchmarks. diff --git a/benchmark/spreadsheet_xarena/third_party/SkillOpt/docs/guide/new-backend.md b/benchmark/spreadsheet_xarena/third_party/SkillOpt/docs/guide/new-backend.md new file mode 100644 index 00000000..03fca9e4 --- /dev/null +++ b/benchmark/spreadsheet_xarena/third_party/SkillOpt/docs/guide/new-backend.md @@ -0,0 +1,130 @@ +# Add a New Model Backend + +SkillOpt supports multiple LLM backends. This guide shows how to add your own. + +## Backend Architecture + +``` +skillopt/model/ +├── base.py # Abstract base class +├── azure_openai.py # Azure OpenAI backend +├── openai_model.py # Direct OpenAI backend +├── claude.py # Anthropic Claude backend +├── qwen.py # Local Qwen (vLLM) backend +└── your_backend.py # Your new backend +``` + +## Step 1: Create the Backend + +Create `skillopt/model/your_backend.py`: + +```python +from skillopt.model.base import ModelBackend, ModelResponse + +class YourBackend(ModelBackend): + """Your custom model backend.""" + + def __init__(self, cfg: dict): + super().__init__(cfg) + self.model_name = cfg.get('model_name', 'your-default-model') + self.api_key = os.environ.get('YOUR_API_KEY', '') + self.client = self._init_client() + + def _init_client(self): + """Initialize API client.""" + # TODO: Set up your API client + pass + + async def generate( + self, + messages: list[dict], + temperature: float = 0.7, + max_tokens: int = 4096, + **kwargs + ) -> ModelResponse: + """ + Generate a completion. + + Args: + messages: Chat messages [{"role": "...", "content": "..."}] + temperature: Sampling temperature + max_tokens: Maximum tokens in response + + Returns: + ModelResponse with content, usage, and metadata + """ + response = await self.client.chat( + model=self.model_name, + messages=messages, + temperature=temperature, + max_tokens=max_tokens, + ) + + return ModelResponse( + content=response.text, + usage={ + 'prompt_tokens': response.usage.input, + 'completion_tokens': response.usage.output, + }, + model=self.model_name, + ) + + async def generate_with_tools( + self, + messages: list[dict], + tools: list[dict], + **kwargs + ) -> ModelResponse: + """Generate with tool/function calling support.""" + # Optional: implement if your model supports tool use + raise NotImplementedError("Tool use not supported") +``` + +## Step 2: Register the Backend + +Add to `skillopt/model/__init__.py`: + +```python +from .your_backend import YourBackend + +BACKEND_REGISTRY = { + # ... existing backends ... + 'your_backend': YourBackend, +} +``` + +## Step 3: Configure + +Use your backend in any config: + +```yaml +model: + backend: your_backend + model_name: your-model-id + temperature: 0.7 + max_tokens: 4096 +``` + +Set credentials via environment variable: + +```bash +export YOUR_API_KEY="your-key" +``` + +## Required Interface + +Your backend must implement these methods: + +| Method | Required | Description | +|---|---|---| +| `generate()` | ✅ | Basic text generation | +| `generate_with_tools()` | Optional | Tool/function calling | +| `count_tokens()` | Optional | Token counting for context management | + +## Tips + +!!! tip + - Test your backend with `python -c "from skillopt.model.your_backend import YourBackend"` first + - Use `async` methods for all API calls — SkillOpt uses asyncio throughout + - Implement retry logic with exponential backoff for production use + - Add your API key to `.env.example` when submitting a PR diff --git a/benchmark/spreadsheet_xarena/third_party/SkillOpt/docs/guide/new-benchmark.md b/benchmark/spreadsheet_xarena/third_party/SkillOpt/docs/guide/new-benchmark.md new file mode 100644 index 00000000..6d2f009a --- /dev/null +++ b/benchmark/spreadsheet_xarena/third_party/SkillOpt/docs/guide/new-benchmark.md @@ -0,0 +1,393 @@ +# Add a New Benchmark + +Extend SkillOpt with your own benchmark in ~200 lines of code. We will use +a tiny worked example, `docfaithful`, that scores a target model on +how faithfully it answers questions grounded in a small reference doc. + +> **Working reference.** The easiest way to copy-cargo-cult a new env is +> to read [`skillopt/envs/officeqa/`](https://github.com/microsoft/SkillOpt/tree/main/skillopt/envs/officeqa). +> Everything below is the same shape, simplified. + +## What you need to build + +To add a benchmark you implement four things: + +1. **A `SplitDataLoader` subclass** — knows how to load train / val / test + item dicts from disk. +2. **A rollout helper** — runs the target model on a batch of items + under the current skill and scores each prediction. +3. **An `EnvAdapter` subclass** — wires the loader + rollout helper into + SkillOpt's lifecycle (`build_*_env`, `rollout`, `reflect`, + `get_task_types`). +4. **A YAML config** — references your env name plus the standard + train / optimizer / gradient knobs. + +Then one line in `scripts/train.py`'s `_register_builtins()` makes it +discoverable. + +--- + +## Step 1 — Create the package + +```bash +mkdir -p skillopt/envs/docfaithful +touch skillopt/envs/docfaithful/__init__.py +``` + +## Step 2 — Implement the data loader + +`skillopt/envs/docfaithful/loader.py`: + +```python +from __future__ import annotations + +import json +from pathlib import Path + +from skillopt.datasets.base import SplitDataLoader + + +def _normalize(raw: dict) -> dict: + """Make sure every item has an ``id``. Other keys are env-specific.""" + return { + "id": str(raw["uid"]), + "question": raw["question"], + "ground_truth": raw["answer"], + "reference_text": raw.get("reference", ""), + "task_type": raw.get("category", "docfaithful"), + } + + +class DocFaithfulDataLoader(SplitDataLoader): + """Load DocFaithful items from JSON files inside each split dir.""" + + def load_split_items(self, split_path: str) -> list[dict]: + # split_path is e.g. data/docfaithful_split/train/ + json_files = sorted(Path(split_path).glob("*.json")) + if not json_files: + raise FileNotFoundError(f"No .json file found in {split_path}") + with json_files[0].open(encoding="utf-8") as f: + raw = json.load(f) + return [_normalize(item) for item in raw] +``` + +Only `load_split_items()` is mandatory. If you also want to support +`split_mode="ratio"` (auto-split a single raw file into train/val/test), +override `load_raw_items(data_path)` as well — see +`skillopt/datasets/base.py` docstrings. + +## Step 3 — Write the rollout helper + +`skillopt/envs/docfaithful/rollout.py`: + +```python +from __future__ import annotations + +import json +import os +from pathlib import Path + +from skillopt.model import chat_target + + +def _score(prediction: str, ground_truth: str) -> tuple[int, float]: + """Trivial exact-match scorer. Replace with F1 / ROUGE / LLM-judge.""" + p = (prediction or "").strip().lower() + g = (ground_truth or "").strip().lower() + hard = int(p == g and bool(g)) + soft = 1.0 if hard else 0.0 + return hard, soft + + +def _rollout_one(item: dict, skill_content: str, + *, max_completion_tokens: int) -> dict: + system = skill_content + user = ( + f"Question: {item['question']}\n\n" + f"Reference:\n{item.get('reference_text', '')}\n\n" + "Answer:" + ) + prediction, _usage = chat_target( + system=system, + user=user, + max_completion_tokens=max_completion_tokens, + ) + hard, soft = _score(prediction, item.get("ground_truth", "")) + return { + "id": str(item["id"]), + "hard": hard, + "soft": soft, + "predicted_answer": prediction, + "question": item.get("question", ""), + "reference_text": item.get("reference_text", ""), + "task_type": item.get("task_type", "docfaithful"), + } + + +def run_batch(*, items: list[dict], skill_content: str, out_root: str, + workers: int = 4, max_completion_tokens: int = 4096) -> list[dict]: + """Run a batch of episodes sequentially or with a thread pool.""" + os.makedirs(out_root, exist_ok=True) + # For brevity we go sequentially — swap in concurrent.futures.ThreadPoolExecutor + # when network / model latency dominates. + results = [ + _rollout_one(item, skill_content, + max_completion_tokens=max_completion_tokens) + for item in items + ] + Path(out_root, "rollouts.json").write_text( + json.dumps(results, ensure_ascii=False, indent=2) + ) + return results +``` + +Two design points worth flagging: + +- **Scoring lives here, not in `EnvAdapter`.** There is no `evaluate()` + method on the ABC. Whatever signal you put in `hard` (0/1, or a float + in [0, 1] for smoothed reward) and `soft` (float in [0, 1]) is what + the optimizer reads. +- **Use `skillopt.model.chat_target`**, not raw OpenAI/Claude calls. + That routes through whichever **chat** target backend the user + configured (`openai_chat` / `claude_chat` / `qwen_chat` / + `minimax_chat`) without your adapter caring. Exec-style backends + (`codex_exec`, `claude_code_exec`) need env-specific rollout code — + see `skillopt/envs/swebench/` for an example. + +## Step 4 — Implement the environment adapter + +`skillopt/envs/docfaithful/adapter.py`: + +```python +from __future__ import annotations + +import os + +from skillopt.datasets.base import BatchSpec +from skillopt.envs.base import EnvAdapter +from skillopt.envs.docfaithful.loader import DocFaithfulDataLoader +from skillopt.envs.docfaithful.rollout import run_batch +from skillopt.gradient.reflect import run_minibatch_reflect + + +class DocFaithfulAdapter(EnvAdapter): + """SkillOpt adapter for the DocFaithful benchmark.""" + + def __init__( + self, + split_dir: str = "", + data_path: str = "", + split_mode: str = "split_dir", + split_ratio: str = "2:1:7", + split_seed: int = 42, + split_output_dir: str = "", + workers: int = 4, + analyst_workers: int = 4, + failure_only: bool = False, + minibatch_size: int = 8, + edit_budget: int = 4, + seed: int = 42, + limit: int = 0, + max_completion_tokens: int = 4096, + ) -> None: + self.workers = workers + self.analyst_workers = analyst_workers + self.failure_only = failure_only + self.minibatch_size = minibatch_size + self.edit_budget = edit_budget + self.max_completion_tokens = int(max_completion_tokens) + self.dataloader = DocFaithfulDataLoader( + split_dir=split_dir, + data_path=data_path, + split_mode=split_mode, + split_ratio=split_ratio, + split_seed=split_seed, + split_output_dir=split_output_dir, + seed=seed, + limit=limit, + ) + + # ── Lifecycle ─────────────────────────────────────────────────────── + + def setup(self, cfg: dict) -> None: + super().setup(cfg) + self.dataloader.setup(cfg) + + def get_dataloader(self): + return self.dataloader + + # ── Env construction ──────────────────────────────────────────────── + + def build_env_from_batch(self, batch: BatchSpec, **kwargs): + # For dataset-backed envs the "manager" is just the items list. + return list(batch.payload or []) + + def build_train_env(self, batch_size: int, seed: int, **kwargs): + batch = self.dataloader.build_train_batch( + batch_size=batch_size, seed=seed, **kwargs + ) + return self.build_env_from_batch(batch, **kwargs) + + def build_eval_env(self, env_num: int, split: str, seed: int, **kwargs): + batch = self.dataloader.build_eval_batch( + env_num=env_num, split=split, seed=seed, **kwargs + ) + return self.build_env_from_batch(batch, **kwargs) + + # ── The two real action methods ───────────────────────────────────── + + def rollout(self, env_manager, skill_content: str, + out_dir: str, **kwargs) -> list[dict]: + items: list[dict] = env_manager + return run_batch( + items=items, + skill_content=skill_content, + out_root=out_dir, + workers=self.workers, + max_completion_tokens=self.max_completion_tokens, + ) + + def reflect(self, results: list[dict], skill_content: str, + out_dir: str, **kwargs) -> list[dict | None]: + return run_minibatch_reflect( + results=results, + skill_content=skill_content, + prediction_dir=kwargs.get( + "prediction_dir", os.path.join(out_dir, "predictions") + ), + patches_dir=kwargs.get( + "patches_dir", os.path.join(out_dir, "patches") + ), + workers=self.analyst_workers, + failure_only=self.failure_only, + minibatch_size=self.minibatch_size, + edit_budget=self.edit_budget, + random_seed=kwargs.get("random_seed"), + error_system=self.get_error_minibatch_prompt(), + success_system=self.get_success_minibatch_prompt(), + step_buffer_context=kwargs.get("step_buffer_context", ""), + update_mode=getattr(self, "_cfg", {}).get("skill_update_mode", "patch"), + ) + + def get_task_types(self) -> list[str]: + seen: list[str] = [] + for item in ( + self.dataloader.train_items + + self.dataloader.val_items + + self.dataloader.test_items + ): + tt = str(item.get("task_type") or "docfaithful") + if tt not in seen: + seen.append(tt) + return seen or ["docfaithful"] +``` + +### What the rollout actually does + +Look back at `run_batch` from Step 3 — it sends each `item["question"]` +to the target model with `skill_content` as the system prompt, scores +the answer against `item["ground_truth"]`, and returns a list of dicts: + +```python +[ + {"id": "ex_001", "hard": 1, "soft": 0.92, + "predicted_answer": "...", "question": "...", + "reference_text": item["reference_text"]}, + {"id": "ex_002", "hard": 0, "soft": 0.13, "fail_reason": "...", ...}, + ... +] +``` + +The trainer only requires `id`, `hard`, `soft`. The rest is preserved on +`RolloutResult.extras` (see `skillopt/types.py`) and is what your +`reflect()` consumes via `run_minibatch_reflect`. + +## Step 5 — Register the adapter + +Edit [`scripts/train.py`](https://github.com/microsoft/SkillOpt/blob/main/scripts/train.py) +and add to `_register_builtins()`: + +```python + try: + from skillopt.envs.docfaithful.adapter import DocFaithfulAdapter + _ENV_REGISTRY["docfaithful"] = DocFaithfulAdapter + except ImportError: + pass # docfaithful deps not installed — skip +``` + +There is **no `BENCHMARK_REGISTRY` dict in `skillopt/envs/__init__.py`** — +the registry lives in `scripts/train.py` and is populated lazily so that +optional deps don't break `--help`. + +## Step 6 — Create the YAML config + +`configs/docfaithful/default.yaml`: + +```yaml +_base_: ../_base_/default.yaml # NOTE: string, not list + +model: + reasoning_effort: medium + +train: + batch_size: 16 + accumulation: 1 + num_epochs: 4 + +gradient: + minibatch_size: 8 + merge_batch_size: 8 + +optimizer: + learning_rate: 4 + +env: + name: docfaithful + # Optional: a seed skill document. Create this file (or any markdown + # file) yourself before the first run, or omit the key to let SkillOpt + # start from an empty skill. + skill_init: skillopt/envs/docfaithful/skills/initial.md + split_mode: split_dir + split_dir: data/docfaithful_split + workers: 4 + max_completion_tokens: 4096 + limit: 0 +``` + +> ⚠️ `_base_` is currently parsed as a **string path**, not a list. Write +> `_base_: ../_base_/default.yaml`, not `_base_: ['../_base_/default.yaml']`. +> See [`skillopt/config.py`](https://github.com/microsoft/SkillOpt/blob/main/skillopt/config.py) +> if you want to add list-form inheritance. + +## Step 7 — Run + +```bash +# If you set skill_init above, create the seed skill first: +# mkdir -p skillopt/envs/docfaithful/skills +# echo "# DocFaithful initial skill" > skillopt/envs/docfaithful/skills/initial.md + +python scripts/train.py --config configs/docfaithful/default.yaml +``` + +If you get `ValueError: Unknown environment 'docfaithful'. Available: [...]`, +you forgot Step 5. + +If you get `TypeError: Can't instantiate abstract class DocFaithfulAdapter`, +you forgot to implement one of the five abstract methods on `EnvAdapter`: +`build_train_env`, `build_eval_env`, `rollout`, `reflect`, +`get_task_types`. + +## Tips + +- Start with `train.batch_size: 4` and `limit: 10` while debugging. +- The `evaluate` half lives **inside your `rollout`**, not as a separate + method — there is no `evaluate()` in the `EnvAdapter` ABC. Score the + prediction in `run_batch` and put the score on each result dict's + `hard` / `soft`. +- Noisy scoring kills the optimizer. Spend time on `run_batch`'s scoring + before you spend time on prompts. +- If your benchmark needs heavy optional deps (selenium, vllm, ...), + wrap the registration block with `try / except ImportError` (Step 5) + so people without those deps can still `--help`. +- Copy `skillopt/envs/_template/` as a starting skeleton — it now + implements the real abstract methods. diff --git a/benchmark/spreadsheet_xarena/third_party/SkillOpt/docs/guide/skill-document.md b/benchmark/spreadsheet_xarena/third_party/SkillOpt/docs/guide/skill-document.md new file mode 100644 index 00000000..62d1a345 --- /dev/null +++ b/benchmark/spreadsheet_xarena/third_party/SkillOpt/docs/guide/skill-document.md @@ -0,0 +1,78 @@ +# Skill Document + +A **skill document** is a Markdown file that serves as the "prompt weights" of your agent. SkillOpt trains this document through iterative optimization. + +## What is a Skill Document? + +A skill document is a structured set of instructions that tells a language model **how** to approach a specific type of task. It's analogous to learned weights in a neural network — encoding task-specific knowledge in natural language rather than floating-point parameters. + +## Structure + +A typical skill document contains: + +```markdown +# Task Strategy + +## General Approach +- Break complex problems into sub-steps +- Always verify intermediate results + +## Common Patterns +- When you see X, try approach Y +- Avoid Z because it leads to errors + +## Edge Cases +- If the input contains A, handle it specially by... +- Watch out for B — it requires C + +## Output Format +- Always include reasoning before the answer +- Format numbers with proper units +``` + +## How It Evolves + +During training, the skill document is modified by **edit patches**: + +1. **Additions**: New rules or strategies discovered from failed trajectories +2. **Modifications**: Refining existing rules that are partially correct +3. **Deletions**: Removing rules that consistently lead to errors + +Each edit is validated through the **gate** mechanism before being permanently accepted. + +## Initial Skill + +You can start training with: + +- **Empty skill**: The system learns everything from scratch +- **Seed skill**: Provide initial instructions to bootstrap training +- **Pre-trained skill**: Transfer a skill from a related benchmark + +Configure the initial skill in your YAML: + +```yaml +train: + init_skill: "path/to/initial_skill.md" # or omit for empty +``` + +## Skill Quality Metrics + +Track your skill's evolution through: + +- **Validation score**: Primary metric on the selection split +- **Test score**: Final metric on held-out test data +- **Skill length**: Total tokens in the document +- **Edit acceptance rate**: Fraction of proposed edits that pass gating + +## Best Practices + +!!! tip "Tips for better skills" + 1. **Start with a seed skill** (`env.skill_init`) if you have domain knowledge — it converges faster + 2. **Use cosine LR schedule** — aggressive early exploration + careful late refinement + 3. **Enable slow update** (`use_slow_update: true`) to prevent forgetting across epochs + 4. **Enable meta skill** (`use_meta_skill: true`) so the optimizer accumulates strategy memory + +## Next Steps + +- [Deep Learning Analogy](dl-analogy.md) +- [Configuration Reference](../reference/config.md) diff --git a/benchmark/spreadsheet_xarena/third_party/SkillOpt/docs/guide/training-loop.md b/benchmark/spreadsheet_xarena/third_party/SkillOpt/docs/guide/training-loop.md new file mode 100644 index 00000000..7922305e --- /dev/null +++ b/benchmark/spreadsheet_xarena/third_party/SkillOpt/docs/guide/training-loop.md @@ -0,0 +1,92 @@ +# The Training Loop + +SkillOpt's core insight: **optimizing natural-language skill documents follows the same structure as training neural networks**. + +## Overview + +``` +┌─────────────────────────────────────────────────────────┐ +│ Training Loop │ +│ │ +│ for epoch in epochs: │ +│ for step in steps: │ +│ 1. Rollout — Target executes tasks │ +│ 2. Reflect — Optimizer analyzes trajectories │ +│ 3. Aggregate — Hierarchical merge of patches │ +│ 4. Select — Rank & clip edits (learning rate) │ +│ 5. Update — Apply patches to skill doc │ +│ 6. Gate — Validate & accept/reject │ +│ │ +│ Epoch Boundary: │ +│ • Slow Update (longitudinal comparison & guidance) │ +│ • Meta Skill (cross-epoch strategy memory) │ +└─────────────────────────────────────────────────────────┘ +``` + +## Stage Details + +### 1. Rollout (Forward Pass) + +The **target** model executes tasks using the current skill document as its prompt. Each task produces a trajectory and a score. + +```python +# Analogy: forward pass through the network +predictions = model(input, skill_document) +scores = evaluate(predictions, ground_truth) +``` + +### 2. Reflect (Backward Pass) + +The **optimizer** model analyzes failed trajectories and produces **edit patches** — structured suggestions for improving the skill document. + +Two modes: + +- **Shallow**: Analyze each trajectory independently +- **Deep**: Cross-reference multiple failures to find systemic issues + +```python +# Analogy: computing gradients +gradients = loss.backward() # → edit patches +``` + +### 3. Aggregate + +Semantically similar edit patches are merged to avoid redundant edits. + +### 4. Select (Gradient Clipping) + +Edits are ranked by relevance score. The `learning_rate` parameter caps how many edits are applied per step — just like gradient clipping prevents overshooting. + +```python +# Analogy: gradient clipping + optimizer step size +selected = top_k(edits, k=learning_rate) +``` + +The `lr_scheduler` adjusts this over training: + +- **cosine**: Start aggressive, taper smoothly +- **linear**: Linear decay +- **constant**: Fixed rate + +### 5. Update (Parameter Update) + +Selected edits are applied to the skill document, producing a new version. + +### 6. Gate (Validation) + +The updated skill is evaluated on a **selection split** (analogous to a validation set). The update is only accepted if performance improves. + +## Epoch Boundary Mechanisms + +### Slow Update + +At the end of each epoch (starting from epoch 2), the system performs a **longitudinal comparison**: it rolls out both the previous epoch's skill and the current skill on the same samples, categorizes items as improved/regressed/persistent_fail/stable_success, then generates high-level **guidance** that is injected into the skill document. This prevents catastrophic forgetting of earlier improvements. + +### Meta Skill + +A **meta-skill memory** accumulates high-level strategy notes across the entire training run. At the end of each epoch, the optimizer reflects on what changed between epochs and produces a compact memory that is provided as additional context during future reflection steps. + +## Next Steps + +- [Understand Skill Documents](skill-document.md) +- [DL ↔ SkillOpt analogy table](dl-analogy.md) diff --git a/benchmark/spreadsheet_xarena/third_party/SkillOpt/docs/index.md b/benchmark/spreadsheet_xarena/third_party/SkillOpt/docs/index.md new file mode 100644 index 00000000..2dfe6da7 --- /dev/null +++ b/benchmark/spreadsheet_xarena/third_party/SkillOpt/docs/index.md @@ -0,0 +1,170 @@ +--- +hide: + - navigation +--- + +
+ +# SkillOpt + +### Train Agent Skills Like Neural Networks + +*Optimize natural-language skill documents through iterative rollout, reflection, and gated validation — with epochs, learning rates, and validation gates — without touching model weights.* + +[Get Started :material-rocket-launch:](guide/installation.md){ .md-button .md-button--primary } +[View on GitHub :material-github:](https://github.com/microsoft/SkillOpt){ .md-button } + +
+ +--- + +## How It Works + +
+
+ +
+
🎯
+
Rollout
+
Target executes tasks
+
+ +
+ +
+
🔍
+
Reflect
+
Optimizer analyzes trajectories
+
+ +
+ +
+
🔗
+
Aggregate
+
Merge edit patches
+
+ +
+ +
+
✂️
+
Select
+
Rank & clip edits
+
+ +
+ +
+
📝
+
Update
+
Apply to skill doc
+
+ +
+ +
+
🚦
+
Gate
+
Validate & accept
+
+ +
+ +
+
🔄 Slow Update
+
🧠 Meta Skill
+
Epoch Boundary
+
+ +
+ +--- + +## Deep Learning Analogy + +SkillOpt brings the familiar deep-learning training paradigm to agentic prompt optimization: + +| Deep Learning | SkillOpt | +|---|---| +| Model weights | Skill document (Markdown) | +| Forward pass | Rollout (target executes tasks) | +| Loss / gradient | Reflect (optimizer produces edit patches) | +| Gradient clipping | Edit selection (`learning_rate` = max edits) | +| SGD step | Patch application to skill | +| Validation set | Gated evaluation on selection split | +| LR schedule | `lr_scheduler`: cosine, linear, constant | +| Epochs | Multi-epoch with slow update & meta skill memory | + +--- + +## Supported Benchmarks + +| Benchmark | Type | Config | +|---|---|---| +| **DocVQA** | Document QA | `configs/docvqa/` | +| **ALFWorld** | Embodied AI | `configs/alfworld/` | +| **OfficeQA** | Enterprise QA | `configs/officeqa/` | +| **SearchQA** | Open-domain QA | `configs/searchqa/` | +| **LiveMathBench** | Math reasoning | `configs/livemathematicianbench/` | +| **SWEBench** | Software Engineering | `configs/swebench/` | +| + 5 more | Various | See [docs](guide/first-experiment.md) | + +--- + +## Quick Example + +```bash +# Install +pip install -e . + +# Configure credentials +export AZURE_OPENAI_ENDPOINT="https://your-resource.openai.azure.com/" +export AZURE_OPENAI_API_KEY="your-key" + +# Train on SearchQA +python scripts/train.py --config configs/searchqa/default.yaml + +# Evaluate best skill +python scripts/eval_only.py \ + --config configs/searchqa/default.yaml \ + --skill outputs/best_skill.md +``` + +--- + +
+ +- :material-book-open-variant:{ .lg .middle } **Getting Started** + + --- + + Install SkillOpt, configure your API keys, and run your first experiment in 5 minutes. + + [:octicons-arrow-right-24: Installation](guide/installation.md) + +- :material-puzzle:{ .lg .middle } **Add a Benchmark** + + --- + + Extend SkillOpt with your own benchmark in ~100 lines of code. + + [:octicons-arrow-right-24: Extension Guide](guide/new-benchmark.md) + +- :material-cog:{ .lg .middle } **Configuration** + + --- + + Full reference for all hyperparameters with deep learning analogies. + + [:octicons-arrow-right-24: Config Reference](reference/config.md) + +- :material-monitor-dashboard:{ .lg .middle } **WebUI** + + --- + + Configure, launch, and monitor training from your browser. + + [:octicons-arrow-right-24: WebUI Guide](guide/first-experiment.md#webui) + +
diff --git a/benchmark/spreadsheet_xarena/third_party/SkillOpt/docs/reference/api.md b/benchmark/spreadsheet_xarena/third_party/SkillOpt/docs/reference/api.md new file mode 100644 index 00000000..8e364c7a --- /dev/null +++ b/benchmark/spreadsheet_xarena/third_party/SkillOpt/docs/reference/api.md @@ -0,0 +1,195 @@ +# API Reference + +This page documents the public Python API SkillOpt exposes for **extending the +framework** with new environments / benchmarks. For ready-made adapters, +browse [`skillopt/envs/`](https://github.com/microsoft/SkillOpt/tree/main/skillopt/envs). + +> **Source of truth.** The classes below are real Python ABCs defined in +> `skillopt/envs/base.py`, `skillopt/datasets/base.py`, `skillopt/types.py`, +> and `skillopt/evaluation/gate.py`. If this page ever drifts, the code +> wins — please open an issue. + +--- + +## Core Classes + +### `EnvAdapter` + +`skillopt/envs/base.py` — abstract adapter that connects the SkillOpt +trainer to an environment (benchmark, simulator, REST API, ...). +Subclasses **must** implement the five abstract methods below. + +```python +from abc import ABC, abstractmethod +from skillopt.datasets.base import BaseDataLoader, BatchSpec + +class EnvAdapter(ABC): + + # ── Lifecycle hooks (have defaults; override only if needed) ──────── + + def setup(self, cfg: dict) -> None: ... + def get_dataloader(self) -> BaseDataLoader | None: ... + def requires_ray(self) -> bool: ... # default False + + # ── Abstract methods (subclasses MUST implement) ──────────────────── + + @abstractmethod + def build_train_env(self, batch_size: int, seed: int, **kwargs): + """Return an environment-manager object to be passed to rollout().""" + + @abstractmethod + def build_eval_env(self, env_num: int, split: str, seed: int, **kwargs): + """Like build_train_env() but for a fixed eval split.""" + + @abstractmethod + def rollout(self, env_manager, skill_content: str, + out_dir: str, **kwargs) -> list[dict]: + """Run a batch of episodes with the current skill. + + Each returned dict MUST contain: + - "id": str episode/task identifier + - "hard": int (0|1) pass/fail (may be float 0.0-1.0 if smoothed) + - "soft": float partial-credit score in [0.0, 1.0] + It MAY contain env-specific extra keys (parsed into RolloutResult.extras). + """ + + @abstractmethod + def reflect(self, results: list[dict], skill_content: str, + out_dir: str, **kwargs) -> list[dict | None]: + """Turn rollout results into a list of raw patch dicts. + + Each dict (or None to drop the slot) MUST contain: + - "patch": {"edits": [...]} a Patch.to_dict() payload + - "source_type": "failure" | "success" + """ + + @abstractmethod + def get_task_types(self) -> list[str]: + """Distinct task-type strings used for stratified sampling.""" +``` + +The trainer also calls a few default-implemented helpers on every adapter: +`build_reference_text`, `get_reference_metadata`, `attach_reference_context`, +`select_representative_items`, and `build_env_from_batch`. Read the docstrings +in `skillopt/envs/base.py` if you need to override any of these — most +benchmarks don't. + +### `BaseDataLoader` / `SplitDataLoader` + +`skillopt/datasets/base.py` — episode-planning loaders. + +```python +class BaseDataLoader(ABC): + def setup(self, cfg: dict) -> None: ... + @abstractmethod + def build_train_batch(self, batch_size: int, seed: int, **kwargs) -> BatchSpec: ... + @abstractmethod + def build_eval_batch(self, env_num: int, split: str, seed: int, **kwargs) -> BatchSpec: ... + +class SplitDataLoader(BaseDataLoader): + """Concrete base for dataset-backed envs with on-disk train/val/test splits. + + Subclasses only need to implement load_split_items() (and optionally + load_raw_items() if you also want ``split_mode='ratio'``). + """ + def load_split_items(self, split_path: str) -> list[dict]: ... + def load_raw_items(self, data_path: str) -> list[dict]: ... # optional +``` + +`SplitDataLoader` handles two layout modes: + +| `split_mode` | What it expects | +|---|---| +| `"split_dir"` | A directory with `train/`, `val/`, `test/` subdirs already split. | +| `"ratio"` | A raw dataset path + `split_ratio: "2:1:7"` style string. | + +In either case the items returned by `load_split_items()` are plain +`dict` objects with at minimum an `"id"` key. + +### `BatchSpec` + +`skillopt/datasets/base.py` — a slotted dataclass describing one batch +request the trainer hands to the adapter. + +```python +@dataclass(slots=True) +class BatchSpec: + phase: str # "train" | "eval" + split: str # "train" | "val" | "test" | "valid_seen" | ... + seed: int + batch_size: int + payload: object | None = None # what the loader produced (e.g. list[dict]) + metadata: dict = field(default_factory=dict) +``` + +### `Edit` / `Patch` + +`skillopt/types.py` — the I/O types Reflect / Aggregate / Update produce +and consume. + +```python +EditOp = Literal["append", "insert_after", "replace", "delete"] + +@dataclass +class Edit: + op: EditOp + content: str = "" + target: str = "" + support_count: int | None = None + source_type: Literal["failure", "success"] | None = None + merge_level: int | None = None + update_origin: str = "" + update_target: str = "" + +@dataclass +class Patch: + edits: list[Edit] = field(default_factory=list) + reasoning: str = "" + ranking_details: dict[str, Any] | None = None +``` + +Both types support `to_dict()` / `from_dict()` for serialization. + +### `RolloutResult` + +`skillopt/types.py` — the normalised rollout return type. The trainer +calls `RolloutResult.from_dict(...)` on each dict returned from +`EnvAdapter.rollout()`, so the only **hard** requirement on those dicts is +the three keys above (`id`, `hard`, `soft`). Extra fields are preserved +into `RolloutResult.extras`. + +### `GateResult` / `GateAction` + +`skillopt/evaluation/gate.py` — the validation-gate decision types +returned each epoch. + +--- + +## Registering an environment + +Environments are not registered via decorators or a `BENCHMARK_REGISTRY` +dict. The trainer keeps a lazy registry inside `scripts/train.py` — +`_ENV_REGISTRY` — populated by `_register_builtins()`. To add a new env +you append a `try / except ImportError` block there. See +[Add a New Benchmark](../guide/new-benchmark.md) for the full step-by-step. + +--- + +## Backends (model layer) + +The model layer lives under `skillopt.model.*`. Backends are selected +via `model.optimizer_backend` and `model.target_backend` in the config — +not via a base class subclass. Supported values (as of this writing): + +| Backend | Optimizer? | Target? | +|---|---|---| +| `openai_chat` | ✓ | ✓ | +| `claude_chat` | ✓ | ✓ | +| `qwen_chat` | ✓ | ✓ | +| `minimax_chat` | ✓ | ✓ | +| `codex_exec` | — | ✓ | +| `claude_code_exec` | — | ✓ | + +See `skillopt/model/backend_config.py` for the live whitelist and +[`docs/reference/config.md`](./config.md) for the per-backend +configuration keys. diff --git a/benchmark/spreadsheet_xarena/third_party/SkillOpt/docs/reference/cli.md b/benchmark/spreadsheet_xarena/third_party/SkillOpt/docs/reference/cli.md new file mode 100644 index 00000000..24b53251 --- /dev/null +++ b/benchmark/spreadsheet_xarena/third_party/SkillOpt/docs/reference/cli.md @@ -0,0 +1,71 @@ +# CLI Reference + +## Training + +```bash +python scripts/train.py --config [overrides...] +``` + +### Arguments + +| Argument | Description | +|---|---| +| `--config` | Path to YAML config file (required) | +| `key=value` | Override any config parameter | + +### Examples + +```bash +# Basic training +python scripts/train.py --config configs/searchqa/default.yaml + +# With overrides +python scripts/train.py \ + --config configs/searchqa/default.yaml \ + --cfg-options optimizer.learning_rate=16 optimizer.lr_scheduler=linear + +# With custom initial skill +python scripts/train.py \ + --config configs/searchqa/default.yaml \ + --cfg-options env.skill_init=skills/my_seed.md +``` + +## Evaluation + +```bash +python scripts/eval_only.py --config --skill +``` + +### Arguments + +| Argument | Description | +|---|---| +| `--config` | Path to YAML config file (required) | +| `--skill` | Path to skill document to evaluate (required) | +| `--split` | Evaluation split: `test` (default), `valid`, `train` | + +### Examples + +```bash +# Evaluate best skill on test set +python scripts/eval_only.py \ + --config configs/searchqa/default.yaml \ + --skill outputs/searchqa/run_001/skills/best_skill.md + +# Evaluate on validation set +python scripts/eval_only.py \ + --config configs/searchqa/default.yaml \ + --skill outputs/searchqa/run_001/skills/best_skill.md \ + --split valid +``` + +## WebUI + +```bash +python -m skillopt_webui.app [--port PORT] [--share] +``` + +| Argument | Default | Description | +|---|---|---| +| `--port` | 7860 | Port number | +| `--share` | false | Create public Gradio link | diff --git a/benchmark/spreadsheet_xarena/third_party/SkillOpt/docs/reference/config.md b/benchmark/spreadsheet_xarena/third_party/SkillOpt/docs/reference/config.md new file mode 100644 index 00000000..0b39bd0a --- /dev/null +++ b/benchmark/spreadsheet_xarena/third_party/SkillOpt/docs/reference/config.md @@ -0,0 +1,85 @@ +# Configuration Reference + +Complete reference for all SkillOpt configuration parameters. + +## Model + +| Parameter | Type | Default | Description | +|---|---|---|---| +| `model.backend` | str | `azure_openai` | Backend: `azure_openai` / `openai_chat` / `claude_code_exec` / `qwen` | +| `model.optimizer` | str | `gpt-5.5` | Optimizer model (for reflection & slow update) | +| `model.target` | str | `gpt-5.5` | Target model (for rollout execution) | +| `model.reasoning_effort` | str | `medium` | Reasoning effort level | +| `model.optimizer_backend` | str | `openai_chat` | Optimizer backend: `openai_chat` / `claude_chat` / `qwen_chat` / `minimax_chat` | +| `model.target_backend` | str | `openai_chat` | Target backend: chat backends plus execution harnesses | +| `model.qwen_chat_base_url` | str | `http://localhost:8000/v1` | Shared Qwen/vLLM OpenAI-compatible endpoint | +| `model.qwen_chat_enable_thinking` | bool | `false` | Shared Qwen thinking flag | +| `model.optimizer_qwen_chat_base_url` | str | — | Optimizer-specific Qwen/vLLM endpoint; overrides shared `qwen_chat_base_url` | +| `model.target_qwen_chat_base_url` | str | — | Target-specific Qwen/vLLM endpoint; overrides shared `qwen_chat_base_url` | + +## Training (`train`) + +| Parameter | Type | Default | DL Analogy | Description | +|---|---|---|---|---| +| `train.num_epochs` | int | 4 | Epochs | Number of training epochs | +| `train.batch_size` | int | 40 | Batch size | Tasks sampled per step | +| `train.accumulation` | int | 1 | Gradient accumulation | Accumulation rounds per step | +| `train.seed` | int | 42 | Random seed | Reproducibility seed | + +## Gradient / Reflection (`gradient`) + +| Parameter | Type | Default | Description | +|---|---|---|---| +| `gradient.minibatch_size` | int | 8 | Reflect minibatch size | +| `gradient.merge_batch_size` | int | 8 | Patch merge batch size | +| `gradient.analyst_workers` | int | 16 | Parallel reflection workers | +| `gradient.max_analyst_rounds` | int | 3 | Max rounds of analyst reflection | +| `gradient.failure_only` | bool | `false` | Only reflect on failures | + +## Optimizer (`optimizer`) + +| Parameter | Type | Default | DL Analogy | Description | +|---|---|---|---|---| +| `optimizer.learning_rate` | int | 4 | Learning rate | Max edit patches per step (edit budget) | +| `optimizer.min_learning_rate` | int | 2 | Min LR | Min edits for decay schedulers | +| `optimizer.lr_scheduler` | str | `cosine` | LR schedule | `constant` / `linear` / `cosine` / `autonomous` | +| `optimizer.skill_update_mode` | str | `patch` | — | `patch` / `rewrite_from_suggestions` / `full_rewrite_minibatch` | +| `optimizer.use_slow_update` | bool | `true` | Momentum | Epoch-boundary longitudinal comparison & guidance | +| `optimizer.slow_update_samples` | int | 20 | — | Samples for slow update evaluation | +| `optimizer.use_meta_skill` | bool | `true` | Meta-learning | Cross-epoch optimizer-side strategy memory | +| `optimizer.longitudinal_pair_policy` | str | `mixed` | — | `mixed` / `changed` / `unchanged` | + +## Evaluation (`evaluation`) + +| Parameter | Type | Default | Description | +|---|---|---|---| +| `evaluation.use_gate` | bool | `true` | Enable validation gating (accept/reject updates) | +| `evaluation.eval_test` | bool | `true` | Run test evaluation after training | + +## Environment (`env`) + +| Parameter | Type | Default | Description | +|---|---|---|---| +| `env.name` | str | — | Benchmark name (e.g., `searchqa`, `docvqa`) | +| `env.data_path` | str | — | Path to dataset | +| `env.skill_init` | str | — | Path to initial seed skill (optional) | +| `env.split_mode` | str | `ratio` | `ratio` or `split_dir` | +| `env.split_ratio` | str | `2:1:7` | Train:val:test ratio | +| `env.exec_timeout` | int | 120 | Per-task timeout in seconds | +| `env.out_root` | str | — | Output directory | + +## Azure OpenAI Credentials + +| Variable | Description | +|---|---| +| `AZURE_OPENAI_ENDPOINT` / `model.azure_openai_endpoint` | Azure resource endpoint | +| `AZURE_OPENAI_API_KEY` / `model.azure_openai_api_key` | Azure API key | +| `OPENAI_API_KEY` | OpenAI API key (for `openai_chat` backend) | +| `ANTHROPIC_API_KEY` | Anthropic API key (for `claude_code_exec` backend) | +| `QWEN_CHAT_BASE_URL` | Shared local vLLM endpoint for `qwen_chat` | +| `QWEN_CHAT_MODEL` | Shared served model name for `qwen_chat` | +| `QWEN_CHAT_API_KEY` | Optional API key for the shared Qwen endpoint | +| `OPTIMIZER_QWEN_CHAT_BASE_URL` | Optimizer-specific local vLLM endpoint | +| `OPTIMIZER_QWEN_CHAT_MODEL` | Optimizer-specific served model name | +| `TARGET_QWEN_CHAT_BASE_URL` | Target-specific local vLLM endpoint | +| `TARGET_QWEN_CHAT_MODEL` | Target-specific served model name | diff --git a/benchmark/spreadsheet_xarena/third_party/SkillOpt/index.html b/benchmark/spreadsheet_xarena/third_party/SkillOpt/index.html new file mode 100644 index 00000000..53114013 --- /dev/null +++ b/benchmark/spreadsheet_xarena/third_party/SkillOpt/index.html @@ -0,0 +1,2739 @@ + + + + + + SkillOpt | Executive Strategy for Self-Evolving Agent Skills + + + + + + + +
+
+
+ Text-space optimization for frozen agents +

SkillOpt

+

+ Executive Strategy for Self-Evolving Agent Skills. SkillOpt treats a compact + natural-language skill document as the trainable state of a frozen language + agent, then learns that document through rollouts, reflection, bounded edits, + and held-out validation gates. +

+
+ Core Idea + Method + View Results + + + Code Repo + + + + Paper + + + + Video + +
+ + + + Related project + SkillLens studies model-generated agent skills. + A companion project page from Microsoft Research. + + + +
+ + +
+
+ +
+
+
+ Project Video +
+

SkillOpt in motion.

+

+ A short visual overview of how SkillOpt treats natural-language skills + as trainable artifacts: roll out, reflect, edit, validate, and export. +

+
+
+
+ +
+

+ Promotional video for the SkillOpt project page. The static paper teaser is shown below for high-resolution inspection. +

+
+ +
+
+ Paper Teaser +
+

The core loop at a glance.

+

+ The teaser summarizes the SkillOpt training loop: rollout evidence, + optimizer-side reflection, bounded skill edits, validation gating, + and the exported reusable skill. +

+
+
+
+ SkillOpt teaser figure showing the target model, optimizer model, bounded edits, validation gate, and exported best skill. +
+

+ Figure from the SkillOpt paper. On small screens, the figure area scrolls horizontally to preserve the original details. +

+
+ +
+
+
01 / Core Idea
+
+

Train the procedure, not the weights.

+

+ SkillOpt makes the skill document itself the optimization target. The + target model, backend, and harness stay fixed; the procedure that guides + evidence gathering, tool use, verification, and output formatting evolves. +

+
+
+ +
+
+

A skill is external state for an agent.

+

+ Instead of fine-tuning a model or hand-maintaining prompts, SkillOpt runs + the frozen agent on scored batches, asks a separate optimizer model to + propose structured edits, and accepts a candidate only when validation + performance improves. +

+
+ Frozen target model + Optimizer model + Add / delete / replace edits + Held-out gate +
+
+ +
+
+ Rollout +

The target model executes tasks with the current skill and records scored trajectories.

+
+
+ Reflect +

The optimizer analyzes success and failure minibatches to find reusable procedures.

+
+
+ Edit +

Candidate add, delete, and replace operations are merged and ranked under a budget.

+
+
+ Gate +

The candidate skill is kept only if it improves held-out selection performance.

+
+
+
+
+ +
+
+
02 / Method
+
+

A training loop for natural-language skills.

+

+ The loop deliberately mirrors a learning algorithm: rollout evidence acts + like a forward pass, reflection acts like a language-level backward pass, + and the textual learning rate bounds how far the skill can move. +

+
+
+ +
+
+

Evidence

+

Rollout batches capture messages, tool calls, verifier feedback, task metadata, and final scores.

+
+
+

Minibatches

+

Failures and successes are reflected separately so edits correct recurring errors while preserving working behavior.

+
+
+

Bounded Edits

+

An edit budget functions as a textual learning rate, preventing useful rules from being overwritten by broad rewrites.

+
+
+

Memory

+

Rejected edits, slow update, and optimizer-side meta skill provide longer-horizon feedback without bloating deployment.

+
+
+ +
+ SkillOpt pipeline showing rollout, reflection, bounded edits, validation gate, slow update, and meta skill. +
+ SkillOpt pipeline from the paper. The frozen target model executes with the current skill; the optimizer model proposes bounded edits; held-out validation decides whether the candidate becomes the new current skill. +
+
+
+ +
+
+
03 / Main Results
+
+

SkillOpt improves GPT and Qwen target models.

+

+ The table reports main-result gains across target models and + execution harnesses, comparing no-skill execution with the final + SkillOpt skill on held-out test splits. +

+
+
+ +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Target modelHarnessSearchQASheetOfficeDocVQALiveMathALFWorldAvg gain
OpenAI logoGPT-5.5Direct chat+9.6+38.9+39.0+12.4+29.3+11.9+23.5
OpenAI logoGPT-5.4Direct chat+6.2+21.1+12.8+13.6+7.2+15.6+12.8
OpenAI logoGPT-5.4-miniDirect chat+4.3+11.4+26.7+16.5+4.8+12.7+12.7
OpenAI logoGPT-5.4-nanoDirect chat+19.0+8.2+33.7+49.4+4.0+35.1+24.9
OpenAI logoGPT-5.2Direct chat+11.2+18.9+21.5+16.5+15.2+16.4+16.6
Qwen logoQwen3.5-4BDirect chat+3.1+14.6+15.2+2.1+29.6+50.7+19.2
Qwen logoQwen3.6-35B-A3BDirect chat+7.6+9.3+1.2+3.8+10.4+22.4+9.1
OpenAI logoGPT-5.5Codex+5.5+57.5+12.8+5.0+28.0N/A+21.8
OpenAI logoGPT-5.5Claude Code+4.0+58.3+13.9+3.5+13.3N/A+18.6
+
+ +
+
+
+ Method comparison +

SkillOpt clears the strongest baseline on every benchmark.

+
+
+
+
+
+ +
+ +
+
+
04 / Ablations
+
+

The controls are doing real work.

+

+ The paper isolates the optimizer components that keep skill learning stable: + enough evidence, bounded textual updates, rejected-edit feedback, slow + update, and optimizer-side memory. +

+
+
+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
ComponentSettingSearchQASpreadsheetLiveMath
Learning ratelr=4 default87.177.561.3
Learning ratewithout lr84.675.757.3
Rejected bufferwith buffer87.177.561.3
Rejected bufferwithout buffer85.572.958.9
Update memorymeta skill + slow update87.177.561.3
Update memorywithout both86.355.059.7
+
+ +
+

What the ablations say

+
+
+ Bounded + Textual learning rates prevent destructive rewrites while keeping enough plasticity to learn new procedures. +
+
+ Gated + Held-out selection turns reflection into propose-and-test optimization rather than unconditional self-editing. +
+
+ Buffered + Rejected edits become negative feedback, helping the optimizer avoid repeating harmful directions. +
+
+
+
+ +
+ Epoch checkpoint trends for SpreadsheetBench, SearchQA, and LiveMath. +
+ Epoch checkpoint trends from the paper. Selection-best checkpoints are compared with train rollout score and unseen test performance. +
+
+
+ +
+
+
05 / Skill Evolution
+
+

A typical run turns failures into concrete operating rules.

+

+ This ALFWorld run uses GPT-5.4-mini as the frozen target model and + GPT-5.5 as the optimizer model. The plot tracks train rollout and + held-out selection scores; hover or focus a point to inspect the + skill edit proposed at that stage. +

+
+
+ +
+
+
+ ALFWorld / train-sel evolution +
+ Train rollout + Selection gate +
+
+
+ + ALFWorld skill evolution scores + Selection score rises from 68.6 percent to 81.4 percent, while rejected edits are visible as downward candidate points. + + + + + + + + 85% + 80% + 75% + 70% + 65% + base + step 1 + step 2 + step 3 + slow + step 4 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+
+ Accepted edits become the current skill only after held-out selection improves. + Step 3 is rescued by a slow update; Step 4 trains higher but fails selection. +
+
+ + +
+ +
+
+ Run setup + Target model: GPT-5.4-mini. Optimizer model: GPT-5.5. The skill starts from a compact ALFWorld instruction file and is edited in text space. +
+
+ Selection rule + Candidate edits are accepted only when held-out selection improves the current best score. +
+
+ Outcome + The selected skill improves final ALFWorld test hard score from 70.9% to 85.8%. +
+
+
+ +
+
+
06 / Transfer
+
+

The exported skill behaves like a reusable artifact.

+

+ SkillOpt exports a compact best_skill.md. The paper tests + whether that artifact transfers across model sizes, execution harnesses, + and nearby benchmarks without further target-side optimization. +

+
+
+ +
+
+ Cross-model + +15.2 +

GPT-5.4 LiveMath skill transferred to GPT-5.4-nano on LiveMathBench.

+
+
+ Cross-harness + +31.8 +

Codex-trained SpreadsheetBench skill transferred into Claude Code.

+
+
+ Self-optimizer + +10.4 +

GPT-5.4-nano used as its own optimizer improved SpreadsheetBench over baseline.

+
+
+ Deployment + 1 file +

The target model consumes only the final skill, not optimizer memory.

+
+
+ +
+ A stronger optimizer model gives the largest gains, but the loop is not merely + distillation from a stronger model. Even matched target-as-optimizer settings + can discover useful edits when the update is constrained, buffered, and + validated. +
+
+ +
+
+
07 / BibTeX
+
+

Citation.

+

+ If you find SkillOpt useful, please cite the arXiv preprint below. +

+
+
+ +
+ + 
@misc{yang2026skilloptexecutivestrategyselfevolving,
+      title={SkillOpt: Executive Strategy for Self-Evolving Agent Skills}, 
+      author={Yifan Yang and Ziyang Gong and Weiquan Huang and Qihao Yang and Ziwei Zhou and Zisu Huang and Yan Li and Xuemei Gao and Qi Dai and Bei Liu and Kai Qiu and Yuqing Yang and Dongdong Chen and Xue Yang and Chong Luo},
+      year={2026},
+      eprint={2605.23904},
+      archivePrefix={arXiv},
+      primaryClass={cs.AI},
+      url={https://arxiv.org/abs/2605.23904}, 
+}
+
+
+ +
+ SkillOpt: Executive Strategy for Self-Evolving Agent Skills + Code / Citation +
+
+ + + diff --git a/benchmark/spreadsheet_xarena/third_party/SkillOpt/mkdocs.yml b/benchmark/spreadsheet_xarena/third_party/SkillOpt/mkdocs.yml new file mode 100644 index 00000000..7fc32db7 --- /dev/null +++ b/benchmark/spreadsheet_xarena/third_party/SkillOpt/mkdocs.yml @@ -0,0 +1,78 @@ +site_name: SkillOpt Documentation +site_url: https://microsoft.github.io/SkillOpt +site_description: "SkillOpt: Agentic Skill Optimization via Reflective Training Loops" +repo_url: https://github.com/microsoft/SkillOpt +repo_name: microsoft/SkillOpt + +theme: + name: material + palette: + - scheme: default + primary: indigo + accent: deep purple + toggle: + icon: material/brightness-7 + name: Switch to dark mode + - scheme: slate + primary: indigo + accent: deep purple + toggle: + icon: material/brightness-4 + name: Switch to light mode + features: + - navigation.instant + - navigation.tracking + - navigation.sections + - navigation.expand + - navigation.top + - content.code.copy + - content.tabs.link + - search.suggest + - search.highlight + icon: + repo: fontawesome/brands/github + font: + text: Inter + code: JetBrains Mono + + + +nav: + - Home: index.md + - Getting Started: + - Installation: guide/installation.md + - First Experiment: guide/first-experiment.md + - Configuration: guide/configuration.md + - Core Concepts: + - Training Loop: guide/training-loop.md + - Skill Document: guide/skill-document.md + - Deep Learning Analogy: guide/dl-analogy.md + - Extension Guides: + - Add a New Benchmark: guide/new-benchmark.md + - Local Environment Smoke Tests: guide/local-env-smoke.md + - Add a New Model Backend: guide/new-backend.md + - Reference: + - Configuration Reference: reference/config.md + - CLI Reference: reference/cli.md + - API Reference: reference/api.md + - Contributing: contributing.md + +markdown_extensions: + - admonition + - pymdownx.details + - pymdownx.superfences + - pymdownx.tabbed: + alternate_style: true + - pymdownx.highlight: + anchor_linenums: true + - pymdownx.inlinehilite + - pymdownx.emoji: + emoji_index: !!python/name:material.extensions.emoji.twemoji + emoji_generator: !!python/name:material.extensions.emoji.to_svg + - attr_list + - md_in_html + - toc: + permalink: true + +plugins: + - search diff --git a/benchmark/spreadsheet_xarena/third_party/SkillOpt/pyproject.toml b/benchmark/spreadsheet_xarena/third_party/SkillOpt/pyproject.toml new file mode 100644 index 00000000..a45fe5b5 --- /dev/null +++ b/benchmark/spreadsheet_xarena/third_party/SkillOpt/pyproject.toml @@ -0,0 +1,75 @@ +[build-system] +requires = ["setuptools>=68.0", "wheel"] +build-backend = "setuptools.build_meta" + +[project] +name = "skillopt" +version = "0.1.0" +description = "SkillOpt: Agentic Skill Optimization via Reflective Training Loops" +readme = "README.md" +license = {text = "MIT"} +requires-python = ">=3.10" +authors = [ + {name = "SkillOpt Team"}, +] +keywords = ["agent", "prompt-optimization", "skill-learning", "LLM", "agentic"] +classifiers = [ + "Development Status :: 3 - Alpha", + "Intended Audience :: Science/Research", + "License :: OSI Approved :: MIT License", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Topic :: Scientific/Engineering :: Artificial Intelligence", +] +dependencies = [ + "openai>=1.30.0", + "pyyaml>=6.0", + "numpy>=1.24.0", + "openpyxl>=3.1.0", + "azure-identity>=1.15.0", + "azure-core>=1.30.0", + "httpx>=0.27.0", +] + +[project.optional-dependencies] +# Benchmark-specific dependencies +alfworld = ["alfworld>=0.4.0", "gymnasium>=0.29.0"] +# Claude model backend +claude = ["claude-agent-sdk>=0.1.0"] +# Qwen local model backend (via vLLM) +qwen = ["vllm>=0.4.0"] +# Documentation site +docs = ["mkdocs-material>=9.5.0", "mkdocstrings[python]>=0.24.0"] +# WebUI dashboard +webui = ["gradio>=4.0.0"] +# Development tools +dev = ["ruff>=0.4.0", "pytest>=8.0.0"] +# All optional dependencies (except docs/dev/webui) +all = [ + "alfworld>=0.4.0", + "gymnasium>=0.29.0", + "claude-agent-sdk>=0.1.0", +] + +[project.scripts] +skillopt-train = "scripts.train:main" +skillopt-eval = "scripts.eval_only:main" + +[project.urls] +Homepage = "https://github.com/microsoft/SkillOpt" +Documentation = "https://microsoft.github.io/SkillOpt" +Repository = "https://github.com/microsoft/SkillOpt" +Issues = "https://github.com/microsoft/SkillOpt/issues" + +[tool.setuptools.packages.find] +include = ["skillopt*", "scripts*"] + +[tool.ruff] +line-length = 120 +target-version = "py310" + +[tool.ruff.lint] +select = ["E", "F", "I", "W"] +ignore = ["E501"] diff --git a/benchmark/spreadsheet_xarena/third_party/SkillOpt/requirements.txt b/benchmark/spreadsheet_xarena/third_party/SkillOpt/requirements.txt new file mode 100644 index 00000000..29d1eb77 --- /dev/null +++ b/benchmark/spreadsheet_xarena/third_party/SkillOpt/requirements.txt @@ -0,0 +1,25 @@ +# ── Core ────────────────────────────────────────── +openai>=1.30.0 +pyyaml>=6.0 +numpy>=1.24.0 +openpyxl>=3.1.0 +azure-identity>=1.15.0 +azure-core>=1.30.0 +httpx>=0.27.0 + +# ── Optional: ALFWorld benchmark ────────────────── +# alfworld>=0.4.0 +# gymnasium>=0.29.0 + +# ── Optional: Claude model backend ──────────────── +# claude-agent-sdk>=0.1.0 + +# ── Optional: Qwen local model (via vLLM) ──────── +# vllm>=0.4.0 + +# ── Optional: WebUI dashboard ──────────────────── +# gradio>=4.0.0 + +# ── Optional: Documentation site ───────────────── +# mkdocs-material>=9.5.0 +# mkdocstrings[python]>=0.24.0 diff --git a/benchmark/spreadsheet_xarena/third_party/SkillOpt/scripts/__init__.py b/benchmark/spreadsheet_xarena/third_party/SkillOpt/scripts/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/benchmark/spreadsheet_xarena/third_party/SkillOpt/scripts/eval_only.py b/benchmark/spreadsheet_xarena/third_party/SkillOpt/scripts/eval_only.py new file mode 100644 index 00000000..ec6cd375 --- /dev/null +++ b/benchmark/spreadsheet_xarena/third_party/SkillOpt/scripts/eval_only.py @@ -0,0 +1,451 @@ +#!/usr/bin/env python3 +"""SkillOpt eval-only: run a single skill on a dataset without training. + +Usage +----- + python scripts/eval_only.py \ + --config configs/spreadsheetbench/default.yaml \ + --skill skillopt/envs/spreadsheetbench/skills/initial.md \ + --split_dir /path/to/split \ + --out_root outputs/eval_skill0 + +All YAML keys can be overridden from the CLI, same as train.py. +""" +from __future__ import annotations + +import argparse +import datetime +import json +import os +import sys + +_SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) +_PROJECT_ROOT = os.path.dirname(_SCRIPT_DIR) +if _PROJECT_ROOT not in sys.path: + sys.path.insert(0, _PROJECT_ROOT) + +from skillopt.model import ( + configure_azure_openai, + configure_claude_code_exec, + configure_codex_exec, + set_reasoning_effort, + set_target_backend, + set_target_deployment, + set_optimizer_backend, + set_optimizer_deployment, +) +from skillopt.model.common import default_model_for_backend, normalize_backend_name + +_OPENAI_DEFAULT_MODEL_SENTINELS = {"gpt-5.4", "gpt-5.5"} +from skillopt.utils import compute_score + + +# ── Reuse registry from train.py ─────────────────────────────────────────── + +_ENV_REGISTRY: dict[str, type] = {} + + +def _register_builtins() -> None: + try: + from skillopt.envs.alfworld.adapter import ALFWorldAdapter + _ENV_REGISTRY["alfworld"] = ALFWorldAdapter + except ImportError: + pass + try: + from skillopt.envs.searchqa.adapter import SearchQAAdapter + _ENV_REGISTRY["searchqa"] = SearchQAAdapter + except ImportError: + pass + try: + from skillopt.envs.livemathematicianbench.adapter import LiveMathematicianBenchAdapter + _ENV_REGISTRY["livemathematicianbench"] = LiveMathematicianBenchAdapter + except ImportError: + pass + try: + from skillopt.envs.babyvision.adapter import BabyVisionAdapter + _ENV_REGISTRY["babyvision"] = BabyVisionAdapter + except ImportError: + pass + try: + from skillopt.envs.spreadsheetbench.adapter import SpreadsheetBenchAdapter + _ENV_REGISTRY["spreadsheetbench"] = SpreadsheetBenchAdapter + except ImportError: + pass + try: + from skillopt.envs.mmrb.adapter import MMRBAdapter + _ENV_REGISTRY["mmrb"] = MMRBAdapter + except ImportError: + pass + try: + from skillopt.envs.docvqa.adapter import DocVQAAdapter + _ENV_REGISTRY["docvqa"] = DocVQAAdapter + except ImportError: + pass + try: + from skillopt.envs.mathverse.adapter import MathVerseAdapter + _ENV_REGISTRY["mathverse"] = MathVerseAdapter + except ImportError: + pass + try: + from skillopt.envs.officeqa.adapter import OfficeQAAdapter + _ENV_REGISTRY["officeqa"] = OfficeQAAdapter + except ImportError: + pass + try: + from skillopt.envs.sealqa.adapter import SealQAAdapter + _ENV_REGISTRY["sealqa"] = SealQAAdapter + except ImportError: + pass + try: + from skillopt.envs.swebench.adapter import SWEBenchAdapter + _ENV_REGISTRY["swebench"] = SWEBenchAdapter + except ImportError: + pass + + +def get_adapter(cfg: dict): + _register_builtins() + env_name = cfg.get("env", "alfworld") + if env_name not in _ENV_REGISTRY: + raise ValueError( + f"Unknown environment '{env_name}'. " + f"Available: {list(_ENV_REGISTRY.keys())}" + ) + adapter_cls = _ENV_REGISTRY[env_name] + + import inspect + sig = inspect.signature(adapter_cls.__init__) + accepted = set(sig.parameters.keys()) - {"self"} + adapter_kwargs = {k: cfg[k] for k in accepted if k in cfg} + return adapter_cls(**adapter_kwargs) + + +# ── CLI ──────────────────────────────────────────────────────────────────── + +_BOOL = lambda x: str(x).lower() in ("true", "1", "yes") # noqa: E731 + + +def parse_args() -> argparse.Namespace: + p = argparse.ArgumentParser(description="SkillOpt eval-only") + p.add_argument("--config", type=str, required=True) + p.add_argument("--skill", type=str, required=True, + help="Path to skill .md file to evaluate") + p.add_argument("--split", type=str, default="all", + help="Which split to eval: train/valid_seen/valid_unseen/all (default: all)") + p.add_argument("--cfg-options", nargs="+", default=[], + help="Override config: section.key=value") + # Legacy flat overrides + p.add_argument("--env", type=str) + p.add_argument("--backend", type=str, + choices=["azure_openai", "codex", "codex_exec", "claude", "claude_chat", "claude_code_exec"]) + p.add_argument("--optimizer_model", type=str) + p.add_argument("--target_model", type=str) + p.add_argument("--optimizer_backend", type=str) + p.add_argument("--target_backend", type=str) + p.add_argument("--reasoning_effort", type=str, + choices=["", "low", "medium", "high", "xhigh", "max"]) + p.add_argument("--azure_endpoint", type=str) + p.add_argument("--azure_api_version", type=str) + p.add_argument("--azure_api_key", type=str) + p.add_argument("--azure_openai_endpoint", type=str) + p.add_argument("--azure_openai_api_version", type=str) + p.add_argument("--azure_openai_api_key", type=str) + p.add_argument("--azure_openai_auth_mode", type=str) + p.add_argument("--azure_openai_ad_scope", type=str) + p.add_argument("--azure_openai_managed_identity_client_id", type=str) + p.add_argument("--optimizer_azure_openai_endpoint", type=str) + p.add_argument("--optimizer_azure_openai_api_version", type=str) + p.add_argument("--optimizer_azure_openai_api_key", type=str) + p.add_argument("--optimizer_azure_openai_auth_mode", type=str) + p.add_argument("--optimizer_azure_openai_ad_scope", type=str) + p.add_argument("--optimizer_azure_openai_managed_identity_client_id", type=str) + p.add_argument("--target_azure_openai_endpoint", type=str) + p.add_argument("--target_azure_openai_api_version", type=str) + p.add_argument("--target_azure_openai_api_key", type=str) + p.add_argument("--target_azure_openai_auth_mode", type=str) + p.add_argument("--target_azure_openai_ad_scope", type=str) + p.add_argument("--target_azure_openai_managed_identity_client_id", type=str) + p.add_argument("--codex_exec_path", type=str) + p.add_argument("--codex_exec_sandbox", type=str) + p.add_argument("--codex_exec_profile", type=str) + p.add_argument("--codex_exec_full_auto", type=_BOOL) + p.add_argument("--codex_exec_reasoning_effort", type=str) + p.add_argument("--codex_exec_use_sdk", type=str) + p.add_argument("--codex_exec_network_access", type=_BOOL) + p.add_argument("--codex_exec_web_search", type=_BOOL) + p.add_argument("--codex_exec_approval_policy", type=str) + p.add_argument("--claude_code_exec_path", type=str) + p.add_argument("--claude_code_exec_profile", type=str) + p.add_argument("--claude_code_exec_use_sdk", type=str) + p.add_argument("--claude_code_exec_effort", type=str) + p.add_argument("--claude_code_exec_max_thinking_tokens", type=int) + p.add_argument("--out_root", type=str) + p.add_argument("--data_path", type=str) + p.add_argument("--split_mode", type=str, + choices=["ratio", "split_dir"]) + p.add_argument("--split_ratio", type=str) + p.add_argument("--split_seed", type=int) + p.add_argument("--split_dir", type=str) + p.add_argument("--split_output_dir", type=str) + p.add_argument("--data_root", type=str) + p.add_argument("--max_turns", type=int) + p.add_argument("--workers", type=int) + p.add_argument("--max_api_workers", type=int) + p.add_argument("--seed", type=int) + p.add_argument("--test_env_num", type=int) + p.add_argument("--mode", type=str, + help="SpreadsheetBench: single/multi/react (default comes from config)") + return p.parse_args() + + +def main() -> None: + args = parse_args() + + from skillopt.config import load_config as _load, flatten_config, is_structured + + cfg = _load(args.config, overrides=args.cfg_options) + structured = is_structured(cfg) + + # Apply legacy --key value overrides + cli = {k: v for k, v in vars(args).items() + if v is not None and k not in ("config", "skill", "split", "cfg_options")} + if cli: + if structured: + from skillopt.config import apply_overrides + _MAP = { + "backend": "model.backend", + "optimizer_model": "model.optimizer", + "target_model": "model.target", + "optimizer_backend": "model.optimizer_backend", + "target_backend": "model.target_backend", + "reasoning_effort": "model.reasoning_effort", + "azure_endpoint": "model.azure_endpoint", + "azure_api_version": "model.azure_api_version", + "azure_api_key": "model.azure_api_key", + "azure_openai_endpoint": "model.azure_openai_endpoint", + "azure_openai_api_version": "model.azure_openai_api_version", + "azure_openai_api_key": "model.azure_openai_api_key", + "azure_openai_auth_mode": "model.azure_openai_auth_mode", + "azure_openai_ad_scope": "model.azure_openai_ad_scope", + "azure_openai_managed_identity_client_id": "model.azure_openai_managed_identity_client_id", + "optimizer_azure_openai_endpoint": "model.optimizer_azure_openai_endpoint", + "optimizer_azure_openai_api_version": "model.optimizer_azure_openai_api_version", + "optimizer_azure_openai_api_key": "model.optimizer_azure_openai_api_key", + "optimizer_azure_openai_auth_mode": "model.optimizer_azure_openai_auth_mode", + "optimizer_azure_openai_ad_scope": "model.optimizer_azure_openai_ad_scope", + "optimizer_azure_openai_managed_identity_client_id": "model.optimizer_azure_openai_managed_identity_client_id", + "target_azure_openai_endpoint": "model.target_azure_openai_endpoint", + "target_azure_openai_api_version": "model.target_azure_openai_api_version", + "target_azure_openai_api_key": "model.target_azure_openai_api_key", + "target_azure_openai_auth_mode": "model.target_azure_openai_auth_mode", + "target_azure_openai_ad_scope": "model.target_azure_openai_ad_scope", + "target_azure_openai_managed_identity_client_id": "model.target_azure_openai_managed_identity_client_id", + "codex_exec_path": "model.codex_exec_path", + "codex_exec_sandbox": "model.codex_exec_sandbox", + "codex_exec_profile": "model.codex_exec_profile", + "codex_exec_full_auto": "model.codex_exec_full_auto", + "codex_exec_reasoning_effort": "model.codex_exec_reasoning_effort", + "codex_exec_use_sdk": "model.codex_exec_use_sdk", + "codex_exec_network_access": "model.codex_exec_network_access", + "codex_exec_web_search": "model.codex_exec_web_search", + "codex_exec_approval_policy": "model.codex_exec_approval_policy", + "claude_code_exec_path": "model.claude_code_exec_path", + "claude_code_exec_profile": "model.claude_code_exec_profile", + "claude_code_exec_use_sdk": "model.claude_code_exec_use_sdk", + "claude_code_exec_effort": "model.claude_code_exec_effort", + "claude_code_exec_max_thinking_tokens": "model.claude_code_exec_max_thinking_tokens", + "seed": "train.seed", + "test_env_num": "evaluation.test_env_num", + "env": "env.name", + "out_root": "env.out_root", + } + mapped = [] + for k, v in cli.items(): + dotted = _MAP.get(k) + if dotted: + mapped.append(f"{dotted}={v}") + else: + mapped.append(f"env.{k}={v}") + apply_overrides(cfg, mapped) + else: + cfg.update(cli) + + cfg = flatten_config(cfg) if structured else cfg + + for new_key, old_key in ( + ("azure_openai_endpoint", "azure_endpoint"), + ("azure_openai_api_version", "azure_api_version"), + ("azure_openai_api_key", "azure_api_key"), + ): + if cfg.get(new_key) in (None, "") and cfg.get(old_key) not in (None, ""): + cfg[new_key] = cfg[old_key] + + explicit_backend = getattr(args, "backend", None) + if explicit_backend is None: + for option in args.cfg_options or []: + key = str(option).split("=", 1)[0].strip() + if key == "model.backend": + explicit_backend = str(option).split("=", 1)[1].strip() + break + + backend = normalize_backend_name(cfg.get("model_backend") or cfg.get("target_backend") or "azure_openai") + + def _has_model_override(dotted_key: str, legacy_key: str) -> bool: + if getattr(args, legacy_key, None) is not None: + return True + for option in args.cfg_options or []: + key = str(option).split("=", 1)[0].strip() + if key == dotted_key: + return True + return False + + if explicit_backend is not None: + backend = normalize_backend_name(explicit_backend) + cfg["model_backend"] = backend + if backend in {"claude", "claude_chat"}: + cfg.setdefault("optimizer_backend", "claude_chat") + cfg.setdefault("target_backend", "claude_chat") + elif backend in {"codex", "codex_exec"}: + cfg.setdefault("optimizer_backend", "openai_chat") + cfg.setdefault("target_backend", "codex_exec") + elif backend == "claude_code_exec": + cfg.setdefault("optimizer_backend", "openai_chat") + cfg.setdefault("target_backend", "claude_code_exec") + else: + cfg.setdefault("optimizer_backend", "openai_chat") + cfg.setdefault("target_backend", "openai_chat") + else: + cfg.setdefault("optimizer_backend", "openai_chat") + cfg.setdefault("target_backend", "openai_chat") + + if cfg.get("optimizer_backend") == "claude_chat": + if ( + str(cfg.get("optimizer_model", "") or "").strip() in _OPENAI_DEFAULT_MODEL_SENTINELS + and not _has_model_override("model.optimizer", "optimizer_model") + ): + cfg["optimizer_model"] = default_model_for_backend("claude_chat") + if cfg.get("target_backend") == "claude_chat": + if ( + str(cfg.get("target_model", "") or "").strip() in _OPENAI_DEFAULT_MODEL_SENTINELS + and not _has_model_override("model.target", "target_model") + ): + cfg["target_model"] = default_model_for_backend("claude_chat") + if cfg.get("target_backend") == "claude_code_exec": + if ( + str(cfg.get("target_model", "") or "").strip() in _OPENAI_DEFAULT_MODEL_SENTINELS + and not _has_model_override("model.target", "target_model") + ): + cfg["target_model"] = default_model_for_backend("claude_chat") + + if not cfg.get("out_root"): + env = cfg.get("env", "unknown") + model = cfg.get("target_model", "unknown").replace("/", "-") + ts = datetime.datetime.now().strftime("%Y%m%d_%H%M%S") + cfg["out_root"] = os.path.join("outputs", f"eval_{env}_{model}_{ts}") + + cfg["out_root"] = os.path.abspath(cfg["out_root"]) + + out_root = cfg["out_root"] + os.makedirs(out_root, exist_ok=True) + + # Load skill + skill_path = os.path.abspath(args.skill) + with open(skill_path) as f: + skill_content = f.read() + print(f" [skill] {skill_path} ({len(skill_content)} chars)") + + # Configure models + configure_azure_openai( + endpoint=(cfg.get("azure_openai_endpoint") or cfg.get("azure_endpoint") or None), + api_version=(cfg.get("azure_openai_api_version") or cfg.get("azure_api_version") or None), + api_key=(cfg.get("azure_openai_api_key") or cfg.get("azure_api_key") or None), + auth_mode=cfg.get("azure_openai_auth_mode") or None, + ad_scope=cfg.get("azure_openai_ad_scope") or None, + managed_identity_client_id=cfg.get("azure_openai_managed_identity_client_id") or None, + optimizer_endpoint=cfg.get("optimizer_azure_openai_endpoint") or None, + optimizer_api_version=cfg.get("optimizer_azure_openai_api_version") or None, + optimizer_api_key=cfg.get("optimizer_azure_openai_api_key") or None, + optimizer_auth_mode=cfg.get("optimizer_azure_openai_auth_mode") or None, + optimizer_ad_scope=cfg.get("optimizer_azure_openai_ad_scope") or None, + optimizer_managed_identity_client_id=( + cfg.get("optimizer_azure_openai_managed_identity_client_id") or None + ), + target_endpoint=cfg.get("target_azure_openai_endpoint") or None, + target_api_version=cfg.get("target_azure_openai_api_version") or None, + target_api_key=cfg.get("target_azure_openai_api_key") or None, + target_auth_mode=cfg.get("target_azure_openai_auth_mode") or None, + target_ad_scope=cfg.get("target_azure_openai_ad_scope") or None, + target_managed_identity_client_id=( + cfg.get("target_azure_openai_managed_identity_client_id") or None + ), + ) + set_optimizer_backend(cfg.get("optimizer_backend", "openai_chat")) + set_target_backend(cfg.get("target_backend", "openai_chat")) + set_optimizer_deployment(cfg.get("optimizer_model", default_model_for_backend(backend))) + set_target_deployment(cfg.get("target_model", default_model_for_backend(backend))) + configure_codex_exec( + path=cfg.get("codex_exec_path", "codex"), + sandbox=cfg.get("codex_exec_sandbox", "workspace-write"), + profile=cfg.get("codex_exec_profile", ""), + full_auto=cfg.get("codex_exec_full_auto", False), + reasoning_effort=cfg.get("codex_exec_reasoning_effort", "none"), + use_sdk=cfg.get("codex_exec_use_sdk", None), + network_access=cfg.get("codex_exec_network_access", False), + web_search=cfg.get("codex_exec_web_search", False), + approval_policy=cfg.get("codex_exec_approval_policy", "never"), + ) + configure_claude_code_exec( + path=cfg.get("claude_code_exec_path", "claude"), + profile=cfg.get("claude_code_exec_profile", ""), + use_sdk=cfg.get("claude_code_exec_use_sdk", None), + effort=cfg.get("claude_code_exec_effort", cfg.get("reasoning_effort", "medium")), + max_thinking_tokens=cfg.get("claude_code_exec_max_thinking_tokens", 16384), + ) + set_reasoning_effort(cfg.get("reasoning_effort", "") or None) + + # Build adapter + adapter = get_adapter(cfg) + adapter.setup(cfg) + + seed = cfg.get("seed", 42) + split = args.split or "all" + + if split == "all": + items = ( + adapter.build_eval_env(0, "train", seed) + + adapter.build_eval_env(0, "valid_seen", seed) + + adapter.build_eval_env(0, "valid_unseen", seed) + ) + else: + env_num = cfg.get("test_env_num", 0) + items = adapter.build_eval_env(env_num, split, seed) + + print(f"\n [eval] split={split} items={len(items)}") + print(f" [eval] out_root={out_root}") + print(f"{'='*60}") + + # Run rollout + results = adapter.rollout(items, skill_content, out_root) + + # Score + hard, soft = compute_score(results) + print(f"\n{'='*60}") + print(f" Results: hard={hard:.4f} soft={soft:.4f} (n={len(results)})") + print(f"{'='*60}") + + # Save summary + summary = { + "skill": skill_path, + "split": split, + "n_items": len(results), + "hard": hard, + "soft": soft, + } + with open(os.path.join(out_root, "eval_summary.json"), "w") as f: + json.dump(summary, f, indent=2, ensure_ascii=False) + + print(f" Saved to: {out_root}") + + +if __name__ == "__main__": + main() diff --git a/benchmark/spreadsheet_xarena/third_party/SkillOpt/scripts/run_alfworld.sh b/benchmark/spreadsheet_xarena/third_party/SkillOpt/scripts/run_alfworld.sh new file mode 100755 index 00000000..05c5c936 --- /dev/null +++ b/benchmark/spreadsheet_xarena/third_party/SkillOpt/scripts/run_alfworld.sh @@ -0,0 +1,60 @@ +#!/usr/bin/env bash +# ────────────────────────────────────────────────────────────────────────────── +# SkillOpt — ALFWorld training launch script +# +# Prerequisites: +# pip install -e ".[alfworld]" +# pip install alfworld[full] && alfworld-download +# +# Usage: +# bash scripts/run_alfworld.sh +# bash scripts/run_alfworld.sh --num_epochs 2 --edit_budget 6 +# bash scripts/run_alfworld.sh --split_dir /path/to/alfworld_split +# ────────────────────────────────────────────────────────────────────────────── +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +PROJECT_ROOT="$(dirname "${SCRIPT_DIR}")" + +export PYTHONPATH="${PROJECT_ROOT}:${PYTHONPATH:-}" + +# ALFWorld data — uses ~/.cache/alfworld by default +export ALFWORLD_DATA="${ALFWORLD_DATA:-${HOME}/.cache/alfworld}" + +if [ ! -d "${ALFWORLD_DATA}/json_2.1.1" ]; then + echo "ERROR: ALFWorld data not found at ${ALFWORLD_DATA}/json_2.1.1" + echo "" + echo "To download ALFWorld data, run:" + echo " pip install alfworld[full]" + echo " alfworld-download" + echo "" + echo "Or set ALFWORLD_DATA to the directory containing json_2.1.1/" + exit 1 +fi + +OPTIMIZER_MODEL="${OPTIMIZER_MODEL:-gpt-5.5}" +TARGET_MODEL="${TARGET_MODEL:-gpt-5.5}" + +TIMESTAMP=$(date +%Y%m%d_%H%M%S) +DEFAULT_OUT_ROOT="${PROJECT_ROOT}/outputs/skillopt_alfworld_${TARGET_MODEL}_${TIMESTAMP}" + +echo "============================================================" +echo " SkillOpt — ALFWorld Training" +echo "============================================================" +echo " Optimizer: ${OPTIMIZER_MODEL}" +echo " Target: ${TARGET_MODEL}" +echo " ALFWORLD_DATA: ${ALFWORLD_DATA}" +echo " Output: ${DEFAULT_OUT_ROOT}" +echo "============================================================" + +cd "${PROJECT_ROOT}" + +python scripts/train.py \ + --config configs/alfworld/default.yaml \ + --optimizer_model "${OPTIMIZER_MODEL}" \ + --target_model "${TARGET_MODEL}" \ + --out_root "${DEFAULT_OUT_ROOT}" \ + "$@" + +echo "" +echo "Done! Results saved to: ${DEFAULT_OUT_ROOT}" diff --git a/benchmark/spreadsheet_xarena/third_party/SkillOpt/scripts/run_searchqa.sh b/benchmark/spreadsheet_xarena/third_party/SkillOpt/scripts/run_searchqa.sh new file mode 100755 index 00000000..0f7a7cb3 --- /dev/null +++ b/benchmark/spreadsheet_xarena/third_party/SkillOpt/scripts/run_searchqa.sh @@ -0,0 +1,40 @@ +#!/usr/bin/env bash +# ────────────────────────────────────────────────────────────────────────────── +# SkillOpt — SearchQA training launch script +# +# Usage: +# bash scripts/run_searchqa.sh +# bash scripts/run_searchqa.sh --num_epochs 2 --edit_budget 6 +# bash scripts/run_searchqa.sh --split_dir /path/to/searchqa_split +# ────────────────────────────────────────────────────────────────────────────── +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +PROJECT_ROOT="$(dirname "${SCRIPT_DIR}")" + +export PYTHONPATH="${PROJECT_ROOT}:${PYTHONPATH:-}" + +OPTIMIZER_MODEL="${OPTIMIZER_MODEL:-gpt-5.5}" +TARGET_MODEL="${TARGET_MODEL:-gpt-5.5}" + +TIMESTAMP=$(date +%Y%m%d_%H%M%S) +DEFAULT_OUT_ROOT="${PROJECT_ROOT}/outputs/skillopt_searchqa_${TARGET_MODEL}_${TIMESTAMP}" + +echo "============================================================" +echo " SkillOpt — SearchQA Training" +echo "============================================================" +echo " Optimizer: ${OPTIMIZER_MODEL}" +echo " Target: ${TARGET_MODEL}" +echo "============================================================" + +cd "${PROJECT_ROOT}" + +python scripts/train.py \ + --config configs/searchqa/default.yaml \ + --optimizer_model "${OPTIMIZER_MODEL}" \ + --target_model "${TARGET_MODEL}" \ + --out_root "${DEFAULT_OUT_ROOT}" \ + "$@" + +echo "" +echo "Done! Results saved to: ${DEFAULT_OUT_ROOT}" diff --git a/benchmark/spreadsheet_xarena/third_party/SkillOpt/scripts/run_spreadsheetbench.sh b/benchmark/spreadsheet_xarena/third_party/SkillOpt/scripts/run_spreadsheetbench.sh new file mode 100755 index 00000000..bcbb32c2 --- /dev/null +++ b/benchmark/spreadsheet_xarena/third_party/SkillOpt/scripts/run_spreadsheetbench.sh @@ -0,0 +1,39 @@ +#!/usr/bin/env bash +# ────────────────────────────────────────────────────────────────────────────── +# SkillOpt — SpreadsheetBench training launch script +# +# Usage: +# bash scripts/run_spreadsheetbench.sh --split_dir /path/to/split --data_root /path/to/data +# bash scripts/run_spreadsheetbench.sh --num_epochs 2 --edit_budget 6 +# ────────────────────────────────────────────────────────────────────────────── +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +PROJECT_ROOT="$(dirname "${SCRIPT_DIR}")" + +export PYTHONPATH="${PROJECT_ROOT}:${PYTHONPATH:-}" + +OPTIMIZER_MODEL="${OPTIMIZER_MODEL:-gpt-5.5}" +TARGET_MODEL="${TARGET_MODEL:-gpt-5.5}" + +TIMESTAMP=$(date +%Y%m%d_%H%M%S) +DEFAULT_OUT_ROOT="${PROJECT_ROOT}/outputs/skillopt_spreadsheetbench_${TARGET_MODEL}_${TIMESTAMP}" + +echo "============================================================" +echo " SkillOpt — SpreadsheetBench Training" +echo "============================================================" +echo " Optimizer: ${OPTIMIZER_MODEL}" +echo " Target: ${TARGET_MODEL}" +echo "============================================================" + +cd "${PROJECT_ROOT}" + +python scripts/train.py \ + --config configs/spreadsheetbench/default.yaml \ + --optimizer_model "${OPTIMIZER_MODEL}" \ + --target_model "${TARGET_MODEL}" \ + --out_root "${DEFAULT_OUT_ROOT}" \ + "$@" + +echo "" +echo "Done! Results saved to: ${DEFAULT_OUT_ROOT}" diff --git a/benchmark/spreadsheet_xarena/third_party/SkillOpt/scripts/train.py b/benchmark/spreadsheet_xarena/third_party/SkillOpt/scripts/train.py new file mode 100644 index 00000000..c16474b7 --- /dev/null +++ b/benchmark/spreadsheet_xarena/third_party/SkillOpt/scripts/train.py @@ -0,0 +1,548 @@ +#!/usr/bin/env python3 +"""SkillOpt unified training entry point. + +Usage +----- + python scripts/train.py --config configs/alfworld/default.yaml + +Any YAML key can be overridden from the command line:: + + python scripts/train.py --config configs/alfworld/default.yaml \\ + --batch_size 40 --num_epochs 2 --seed 123 + +Run ``python scripts/train.py --help`` for a full list of options. +""" +from __future__ import annotations + +import argparse +import datetime +import os +import sys + +# Ensure the project root is on sys.path so ``import skillopt`` works +# regardless of where the script is invoked from. +_SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) +_PROJECT_ROOT = os.path.dirname(_SCRIPT_DIR) +if _PROJECT_ROOT not in sys.path: + sys.path.insert(0, _PROJECT_ROOT) + +from skillopt.model.common import default_model_for_backend, normalize_backend_name + +_OPENAI_DEFAULT_MODEL_SENTINELS = {"gpt-5.4", "gpt-5.5"} + + +# ── Environment registry ──────────────────────────────────────────────────── + +_ENV_REGISTRY: dict[str, type] = {} + + +def _register_builtins() -> None: + """Lazy-import built-in adapters so we don't pull heavy deps at CLI parse time.""" + try: + from skillopt.envs.alfworld.adapter import ALFWorldAdapter + _ENV_REGISTRY["alfworld"] = ALFWorldAdapter + except ImportError: + pass # ALFWorld deps not installed — skip + try: + from skillopt.envs.searchqa.adapter import SearchQAAdapter + _ENV_REGISTRY["searchqa"] = SearchQAAdapter + except ImportError: + pass + try: + from skillopt.envs.livemathematicianbench.adapter import LiveMathematicianBenchAdapter + _ENV_REGISTRY["livemathematicianbench"] = LiveMathematicianBenchAdapter + except ImportError: + pass + try: + from skillopt.envs.babyvision.adapter import BabyVisionAdapter + _ENV_REGISTRY["babyvision"] = BabyVisionAdapter + except ImportError: + pass + try: + from skillopt.envs.spreadsheetbench.adapter import SpreadsheetBenchAdapter + _ENV_REGISTRY["spreadsheetbench"] = SpreadsheetBenchAdapter + except ImportError: + pass + try: + from skillopt.envs.mmrb.adapter import MMRBAdapter + _ENV_REGISTRY["mmrb"] = MMRBAdapter + except ImportError: + pass + try: + from skillopt.envs.docvqa.adapter import DocVQAAdapter + _ENV_REGISTRY["docvqa"] = DocVQAAdapter + except ImportError: + pass + try: + from skillopt.envs.mathverse.adapter import MathVerseAdapter + _ENV_REGISTRY["mathverse"] = MathVerseAdapter + except ImportError: + pass + try: + from skillopt.envs.officeqa.adapter import OfficeQAAdapter + _ENV_REGISTRY["officeqa"] = OfficeQAAdapter + except ImportError: + pass + try: + from skillopt.envs.sealqa.adapter import SealQAAdapter + _ENV_REGISTRY["sealqa"] = SealQAAdapter + except ImportError: + pass + try: + from skillopt.envs.swebench.adapter import SWEBenchAdapter + _ENV_REGISTRY["swebench"] = SWEBenchAdapter + except ImportError: + pass + + +def get_adapter(cfg: dict): + """Instantiate the environment adapter specified in ``cfg["env"]``.""" + _register_builtins() + env_name = cfg.get("env", "alfworld") + if env_name not in _ENV_REGISTRY: + raise ValueError( + f"Unknown environment '{env_name}'. " + f"Available: {list(_ENV_REGISTRY.keys())}" + ) + adapter_cls = _ENV_REGISTRY[env_name] + + # Inspect adapter __init__ signature and only pass accepted kwargs + import inspect + sig = inspect.signature(adapter_cls.__init__) + accepted = set(sig.parameters.keys()) - {"self"} + adapter_kwargs: dict = {} + for key in accepted: + if key in cfg: + adapter_kwargs[key] = cfg[key] + + return adapter_cls(**adapter_kwargs) + + +# ── CLI ────────────────────────────────────────────────────────────────────── + +_BOOL = lambda x: x.lower() in ("true", "1", "yes") # noqa: E731 + + +def parse_args() -> argparse.Namespace: + p = argparse.ArgumentParser( + description="SkillOpt: Executive Strategy for Self-Evolving Agent Skills", + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog=__doc__, + ) + p.add_argument("--config", type=str, required=True, + help="Path to YAML config file") + p.add_argument("--cfg-options", nargs="+", default=[], + help="Override config: section.key=value (e.g. train.batch_size=40)") + + # Legacy flat CLI overrides (still work, prefer --cfg-options for new usage) + p.add_argument("--env", type=str) + p.add_argument("--backend", type=str, + choices=["azure_openai", "codex", "codex_exec", "claude", "claude_chat", "claude_code_exec", "qwen", "qwen_chat", "minimax", "minimax_chat"]) + p.add_argument("--optimizer_model", type=str) + p.add_argument("--target_model", type=str) + p.add_argument("--optimizer_backend", type=str) + p.add_argument("--target_backend", type=str) + p.add_argument("--reasoning_effort", type=str, + choices=["", "low", "medium", "high", "xhigh", "max"]) + p.add_argument("--rewrite_reasoning_effort", type=str) + p.add_argument("--rewrite_max_completion_tokens", type=int) + p.add_argument("--azure_endpoint", type=str) + p.add_argument("--azure_api_version", type=str) + p.add_argument("--azure_api_key", type=str) + p.add_argument("--azure_openai_endpoint", type=str) + p.add_argument("--azure_openai_api_version", type=str) + p.add_argument("--azure_openai_api_key", type=str) + p.add_argument("--azure_openai_auth_mode", type=str) + p.add_argument("--azure_openai_ad_scope", type=str) + p.add_argument("--azure_openai_managed_identity_client_id", type=str) + p.add_argument("--optimizer_azure_openai_endpoint", type=str) + p.add_argument("--optimizer_azure_openai_api_version", type=str) + p.add_argument("--optimizer_azure_openai_api_key", type=str) + p.add_argument("--optimizer_azure_openai_auth_mode", type=str) + p.add_argument("--optimizer_azure_openai_ad_scope", type=str) + p.add_argument("--optimizer_azure_openai_managed_identity_client_id", type=str) + p.add_argument("--target_azure_openai_endpoint", type=str) + p.add_argument("--target_azure_openai_api_version", type=str) + p.add_argument("--target_azure_openai_api_key", type=str) + p.add_argument("--target_azure_openai_auth_mode", type=str) + p.add_argument("--target_azure_openai_ad_scope", type=str) + p.add_argument("--target_azure_openai_managed_identity_client_id", type=str) + p.add_argument("--qwen_chat_base_url", type=str) + p.add_argument("--qwen_chat_api_key", type=str) + p.add_argument("--qwen_chat_temperature", type=float) + p.add_argument("--qwen_chat_timeout_seconds", type=float) + p.add_argument("--qwen_chat_max_tokens", type=int) + p.add_argument("--qwen_chat_enable_thinking", type=_BOOL) + p.add_argument("--optimizer_qwen_chat_base_url", type=str) + p.add_argument("--optimizer_qwen_chat_api_key", type=str) + p.add_argument("--optimizer_qwen_chat_temperature", type=float) + p.add_argument("--optimizer_qwen_chat_timeout_seconds", type=float) + p.add_argument("--optimizer_qwen_chat_max_tokens", type=int) + p.add_argument("--optimizer_qwen_chat_enable_thinking", type=_BOOL) + p.add_argument("--target_qwen_chat_base_url", type=str) + p.add_argument("--target_qwen_chat_api_key", type=str) + p.add_argument("--target_qwen_chat_temperature", type=float) + p.add_argument("--target_qwen_chat_timeout_seconds", type=float) + p.add_argument("--target_qwen_chat_max_tokens", type=int) + p.add_argument("--target_qwen_chat_enable_thinking", type=_BOOL) + p.add_argument("--minimax_base_url", type=str) + p.add_argument("--minimax_api_key", type=str) + p.add_argument("--minimax_model", type=str) + p.add_argument("--minimax_temperature", type=float) + p.add_argument("--minimax_max_tokens", type=int) + p.add_argument("--minimax_enable_thinking", type=_BOOL) + p.add_argument("--codex_exec_path", type=str) + p.add_argument("--codex_exec_sandbox", type=str) + p.add_argument("--codex_exec_profile", type=str) + p.add_argument("--codex_exec_full_auto", type=_BOOL) + p.add_argument("--codex_exec_reasoning_effort", type=str) + p.add_argument("--codex_exec_use_sdk", type=str) + p.add_argument("--codex_exec_network_access", type=_BOOL) + p.add_argument("--codex_exec_web_search", type=_BOOL) + p.add_argument("--codex_exec_approval_policy", type=str) + p.add_argument("--claude_code_exec_path", type=str) + p.add_argument("--claude_code_exec_profile", type=str) + p.add_argument("--claude_code_exec_use_sdk", type=str) + p.add_argument("--claude_code_exec_effort", type=str) + p.add_argument("--claude_code_exec_max_thinking_tokens", type=int) + p.add_argument("--codex_trace_to_optimizer", type=_BOOL) + p.add_argument("--skill_init", type=str) + p.add_argument("--num_epochs", type=int) + p.add_argument("--train_size", type=int) + p.add_argument("--steps_per_epoch", type=int) + p.add_argument("--batch_size", type=int) + p.add_argument("--accumulation", type=int) + p.add_argument("--seed", type=int) + p.add_argument("--edit_budget", type=int) + p.add_argument("--min_edit_budget", type=int) + p.add_argument("--lr_scheduler", type=str, + choices=["constant", "linear", "cosine", "autonomous"]) + p.add_argument("--lr_control_mode", type=str, + choices=["fixed", "autonomous", "none"]) + p.add_argument("--merge_batch_size", type=int) + p.add_argument("--max_analyst_rounds", type=int) + p.add_argument("--sel_env_num", type=int) + p.add_argument("--test_env_num", type=int) + p.add_argument("--eval_test", type=_BOOL) + p.add_argument("--use_gate", type=_BOOL) + p.add_argument("--max_steps", type=int) + p.add_argument("--max_api_workers", type=int) + p.add_argument("--analyst_workers", type=int) + p.add_argument("--failure_only", type=_BOOL) + p.add_argument("--minibatch_size", type=int) + p.add_argument("--skill_update_mode", type=str, + choices=[ + "patch", + "rewrite_from_suggestions", + "rewrite", + "suggestions", + "full_rewrite", + "full_rewrite_minibatch", + "minibatch_full_rewrite", + ]) + p.add_argument("--use_slow_update", type=_BOOL) + p.add_argument("--slow_update_samples", type=int) + p.add_argument("--longitudinal_pair_policy", type=str, + choices=["mixed", "changed", "unchanged"]) + p.add_argument("--use_meta_skill", type=_BOOL) + p.add_argument("--data_path", type=str) + p.add_argument("--split_mode", type=str, + choices=["ratio", "split_dir"]) + p.add_argument("--split_ratio", type=str) + p.add_argument("--split_seed", type=int) + p.add_argument("--split_dir", type=str) + p.add_argument("--split_output_dir", type=str) + p.add_argument("--data_root", type=str) + p.add_argument("--max_turns", type=int) + p.add_argument("--workers", type=int) + p.add_argument("--limit", type=int) + p.add_argument("--shuffle_choices", type=_BOOL) + p.add_argument("--use_theorem", type=_BOOL) + p.add_argument("--use_sketch", type=_BOOL) + p.add_argument("--image_detail", type=str) + p.add_argument("--judge_model", type=str) + p.add_argument("--judge_max_completion_tokens", type=int) + p.add_argument("--judge_retries", type=int) + p.add_argument("--out_root", type=str) + p.add_argument("--mode", type=str) + + return p.parse_args() + + +# ── Flat key → structured path mapping (for legacy CLI → structured config) ── + +_LEGACY_TO_STRUCTURED: dict[str, str] = { + "backend": "model.backend", + "optimizer_model": "model.optimizer", + "target_model": "model.target", + "optimizer_backend": "model.optimizer_backend", + "target_backend": "model.target_backend", + "reasoning_effort": "model.reasoning_effort", + "rewrite_reasoning_effort": "model.rewrite_reasoning_effort", + "rewrite_max_completion_tokens": "model.rewrite_max_completion_tokens", + "azure_endpoint": "model.azure_endpoint", + "azure_api_version": "model.azure_api_version", + "azure_api_key": "model.azure_api_key", + "azure_openai_endpoint": "model.azure_openai_endpoint", + "azure_openai_api_version": "model.azure_openai_api_version", + "azure_openai_api_key": "model.azure_openai_api_key", + "azure_openai_auth_mode": "model.azure_openai_auth_mode", + "azure_openai_ad_scope": "model.azure_openai_ad_scope", + "azure_openai_managed_identity_client_id": "model.azure_openai_managed_identity_client_id", + "optimizer_azure_openai_endpoint": "model.optimizer_azure_openai_endpoint", + "optimizer_azure_openai_api_version": "model.optimizer_azure_openai_api_version", + "optimizer_azure_openai_api_key": "model.optimizer_azure_openai_api_key", + "optimizer_azure_openai_auth_mode": "model.optimizer_azure_openai_auth_mode", + "optimizer_azure_openai_ad_scope": "model.optimizer_azure_openai_ad_scope", + "optimizer_azure_openai_managed_identity_client_id": "model.optimizer_azure_openai_managed_identity_client_id", + "target_azure_openai_endpoint": "model.target_azure_openai_endpoint", + "target_azure_openai_api_version": "model.target_azure_openai_api_version", + "target_azure_openai_api_key": "model.target_azure_openai_api_key", + "target_azure_openai_auth_mode": "model.target_azure_openai_auth_mode", + "target_azure_openai_ad_scope": "model.target_azure_openai_ad_scope", + "target_azure_openai_managed_identity_client_id": "model.target_azure_openai_managed_identity_client_id", + "qwen_chat_base_url": "model.qwen_chat_base_url", + "qwen_chat_api_key": "model.qwen_chat_api_key", + "qwen_chat_temperature": "model.qwen_chat_temperature", + "qwen_chat_timeout_seconds": "model.qwen_chat_timeout_seconds", + "qwen_chat_max_tokens": "model.qwen_chat_max_tokens", + "qwen_chat_enable_thinking": "model.qwen_chat_enable_thinking", + "optimizer_qwen_chat_base_url": "model.optimizer_qwen_chat_base_url", + "optimizer_qwen_chat_api_key": "model.optimizer_qwen_chat_api_key", + "optimizer_qwen_chat_temperature": "model.optimizer_qwen_chat_temperature", + "optimizer_qwen_chat_timeout_seconds": "model.optimizer_qwen_chat_timeout_seconds", + "optimizer_qwen_chat_max_tokens": "model.optimizer_qwen_chat_max_tokens", + "optimizer_qwen_chat_enable_thinking": "model.optimizer_qwen_chat_enable_thinking", + "target_qwen_chat_base_url": "model.target_qwen_chat_base_url", + "target_qwen_chat_api_key": "model.target_qwen_chat_api_key", + "target_qwen_chat_temperature": "model.target_qwen_chat_temperature", + "target_qwen_chat_timeout_seconds": "model.target_qwen_chat_timeout_seconds", + "target_qwen_chat_max_tokens": "model.target_qwen_chat_max_tokens", + "target_qwen_chat_enable_thinking": "model.target_qwen_chat_enable_thinking", + "minimax_base_url": "model.minimax_base_url", + "minimax_api_key": "model.minimax_api_key", + "minimax_model": "model.minimax_model", + "minimax_temperature": "model.minimax_temperature", + "minimax_max_tokens": "model.minimax_max_tokens", + "minimax_enable_thinking": "model.minimax_enable_thinking", + "codex_exec_path": "model.codex_exec_path", + "codex_exec_sandbox": "model.codex_exec_sandbox", + "codex_exec_profile": "model.codex_exec_profile", + "codex_exec_full_auto": "model.codex_exec_full_auto", + "codex_exec_reasoning_effort": "model.codex_exec_reasoning_effort", + "codex_exec_use_sdk": "model.codex_exec_use_sdk", + "codex_exec_network_access": "model.codex_exec_network_access", + "codex_exec_web_search": "model.codex_exec_web_search", + "codex_exec_approval_policy": "model.codex_exec_approval_policy", + "claude_code_exec_path": "model.claude_code_exec_path", + "claude_code_exec_profile": "model.claude_code_exec_profile", + "claude_code_exec_use_sdk": "model.claude_code_exec_use_sdk", + "claude_code_exec_effort": "model.claude_code_exec_effort", + "claude_code_exec_max_thinking_tokens": "model.claude_code_exec_max_thinking_tokens", + "codex_trace_to_optimizer": "model.codex_trace_to_optimizer", + "num_epochs": "train.num_epochs", + "train_size": "train.train_size", + "steps_per_epoch": "train.steps_per_epoch", + "batch_size": "train.batch_size", + "accumulation": "train.accumulation", + "seed": "train.seed", + "minibatch_size": "gradient.minibatch_size", + "merge_batch_size": "gradient.merge_batch_size", + "analyst_workers": "gradient.analyst_workers", + "max_analyst_rounds": "gradient.max_analyst_rounds", + "failure_only": "gradient.failure_only", + "edit_budget": "optimizer.learning_rate", + "min_edit_budget": "optimizer.min_learning_rate", + "lr_scheduler": "optimizer.lr_scheduler", + "lr_control_mode": "optimizer.lr_control_mode", + "skill_update_mode": "optimizer.skill_update_mode", + "use_slow_update": "optimizer.use_slow_update", + "slow_update_samples": "optimizer.slow_update_samples", + "longitudinal_pair_policy": "optimizer.longitudinal_pair_policy", + "use_meta_skill": "optimizer.use_meta_skill", + "use_gate": "evaluation.use_gate", + "sel_env_num": "evaluation.sel_env_num", + "test_env_num": "evaluation.test_env_num", + "eval_test": "evaluation.eval_test", + "env": "env.name", + "skill_init": "env.skill_init", + "out_root": "env.out_root", +} + + +def load_config(args: argparse.Namespace) -> dict: + """Load config with _base_ inheritance, then apply CLI overrides.""" + from skillopt.config import load_config as _load, flatten_config, is_structured + + cfg = _load(args.config, overrides=args.cfg_options) + structured = is_structured(cfg) + + # Apply legacy --key value overrides + cli = {k: v for k, v in vars(args).items() + if v is not None and k not in ("config", "cfg_options")} + if cli: + if structured: + from skillopt.config import apply_overrides + mapped = [] + for k, v in cli.items(): + dotted = _LEGACY_TO_STRUCTURED.get(k) + if dotted: + mapped.append(f"{dotted}={v}") + else: + mapped.append(f"env.{k}={v}") + apply_overrides(cfg, mapped) + else: + cfg.update(cli) + + # Flatten structured config → flat dict for trainer/adapter + flat = flatten_config(cfg) if structured else cfg + + for new_key, old_key in ( + ("azure_openai_endpoint", "azure_endpoint"), + ("azure_openai_api_version", "azure_api_version"), + ("azure_openai_api_key", "azure_api_key"), + ): + if flat.get(new_key) in (None, "") and flat.get(old_key) not in (None, ""): + flat[new_key] = flat[old_key] + + explicit_backend = getattr(args, "backend", None) + if explicit_backend is None: + for option in args.cfg_options or []: + key = str(option).split("=", 1)[0].strip() + if key == "model.backend": + explicit_backend = str(option).split("=", 1)[1].strip() + break + + backend = normalize_backend_name(flat.get("model_backend") or flat.get("target_backend") or "azure_openai") + + def _has_model_override(dotted_key: str, legacy_key: str) -> bool: + if getattr(args, legacy_key, None) is not None: + return True + for option in args.cfg_options or []: + key = str(option).split("=", 1)[0].strip() + if key == dotted_key: + return True + return False + + if explicit_backend is not None: + backend = normalize_backend_name(explicit_backend) + flat["model_backend"] = backend + if backend in {"claude", "claude_chat"}: + flat.setdefault("optimizer_backend", "claude_chat") + flat.setdefault("target_backend", "claude_chat") + elif backend in {"codex", "codex_exec"}: + flat.setdefault("optimizer_backend", "openai_chat") + flat.setdefault("target_backend", "codex_exec") + elif backend == "claude_code_exec": + flat.setdefault("optimizer_backend", "openai_chat") + flat.setdefault("target_backend", "claude_code_exec") + elif backend in {"qwen", "qwen_chat"}: + flat.setdefault("optimizer_backend", "openai_chat") + flat.setdefault("target_backend", "qwen_chat") + elif backend in {"minimax", "minimax_chat"}: + flat.setdefault("optimizer_backend", "openai_chat") + flat.setdefault("target_backend", "minimax_chat") + else: + flat.setdefault("optimizer_backend", "openai_chat") + flat.setdefault("target_backend", "openai_chat") + else: + flat.setdefault("optimizer_backend", "openai_chat") + flat.setdefault("target_backend", "openai_chat") + + if flat.get("optimizer_backend") == "claude_chat": + if ( + str(flat.get("optimizer_model", "") or "").strip() in _OPENAI_DEFAULT_MODEL_SENTINELS + and not _has_model_override("model.optimizer", "optimizer_model") + ): + flat["optimizer_model"] = default_model_for_backend("claude_chat") + if flat.get("optimizer_backend") == "qwen_chat": + if ( + str(flat.get("optimizer_model", "") or "").strip() in _OPENAI_DEFAULT_MODEL_SENTINELS + and not _has_model_override("model.optimizer", "optimizer_model") + ): + flat["optimizer_model"] = default_model_for_backend("qwen_chat") + if flat.get("target_backend") == "claude_chat": + if ( + str(flat.get("target_model", "") or "").strip() in _OPENAI_DEFAULT_MODEL_SENTINELS + and not _has_model_override("model.target", "target_model") + ): + flat["target_model"] = default_model_for_backend("claude_chat") + if flat.get("target_backend") == "claude_code_exec": + if ( + str(flat.get("target_model", "") or "").strip() in _OPENAI_DEFAULT_MODEL_SENTINELS + and not _has_model_override("model.target", "target_model") + ): + flat["target_model"] = default_model_for_backend("claude_chat") + if flat.get("target_backend") == "qwen_chat": + if ( + str(flat.get("target_model", "") or "").strip() in _OPENAI_DEFAULT_MODEL_SENTINELS + and not _has_model_override("model.target", "target_model") + ): + flat["target_model"] = default_model_for_backend("qwen_chat") + if flat.get("target_backend") == "minimax_chat": + if ( + str(flat.get("target_model", "") or "").strip() in _OPENAI_DEFAULT_MODEL_SENTINELS + and not _has_model_override("model.target", "target_model") + ): + flat["target_model"] = ( + flat.get("minimax_model") + or default_model_for_backend("minimax_chat") + ) + + # Auto-generate output root + if not flat.get("out_root"): + env = flat.get("env", "unknown") + model = flat.get("optimizer_model", "unknown").replace("/", "-") + ts = datetime.datetime.now().strftime("%Y%m%d_%H%M%S") + flat["out_root"] = os.path.join("outputs", f"skillopt_{env}_{model}_{ts}") + + flat["out_root"] = os.path.abspath(flat["out_root"]) + return flat + + +# ── Main ───────────────────────────────────────────────────────────────────── + +def main() -> None: + args = parse_args() + cfg = load_config(args) + + print(f"\n{'='*60}") + print(f" SkillOpt — Executive Strategy for Self-Evolving Agent Skills") + print(f"{'='*60}") + print(f" env: {cfg.get('env')}") + print(f" optimizer_model: {cfg.get('optimizer_model')}") + print(f" target_model: {cfg.get('target_model')}") + print(f" optimizer_backend:{cfg.get('optimizer_backend', 'openai_chat')}") + print(f" target_backend:{cfg.get('target_backend', 'openai_chat')}") + print(f" reasoning: {cfg.get('reasoning_effort') or 'off'}") + print(f" rewrite_effort: {cfg.get('rewrite_reasoning_effort') or 'off'}") + print(f" epochs: {cfg.get('num_epochs')}") + print(f" train_size: {cfg.get('train_size') or 'from dataset'}") + print(f" steps/epoch: auto") + print(f" batch_size: {cfg.get('batch_size')}") + print(f" edit_budget: {cfg.get('edit_budget')}") + print(f" lr_scheduler: {cfg.get('lr_scheduler', 'constant')}") + print(f" update_mode: {cfg.get('skill_update_mode', 'patch')}") + print(f" min_edit_budget:{cfg.get('min_edit_budget', 2)}") + print(f" minibatch_size: {cfg.get('minibatch_size')}") + print(f" seed: {cfg.get('seed')}") + print(f" meta_skill: {cfg.get('use_meta_skill', False)}") + print(f" slow_update: {cfg.get('use_slow_update', False)}") + print(f" out_root: {cfg.get('out_root')}") + print(f"{'='*60}\n") + + # Build adapter + adapter = get_adapter(cfg) + + # Build trainer and run + from skillopt.engine.trainer import ReflACTTrainer + trainer = ReflACTTrainer(cfg, adapter) + summary = trainer.train() + + print(f"\n Output saved to: {cfg['out_root']}") + if summary.get("test_hard") is not None: + print(f" Final test: {summary['test_hard']:.4f}") + + +if __name__ == "__main__": + main() diff --git a/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt-assets/arxiv-logomark-small.svg b/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt-assets/arxiv-logomark-small.svg new file mode 100644 index 00000000..91e027c3 --- /dev/null +++ b/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt-assets/arxiv-logomark-small.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt.html b/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt.html new file mode 100644 index 00000000..53114013 --- /dev/null +++ b/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt.html @@ -0,0 +1,2739 @@ + + + + + + SkillOpt | Executive Strategy for Self-Evolving Agent Skills + + + + + + + +
+
+
+ Text-space optimization for frozen agents +

SkillOpt

+

+ Executive Strategy for Self-Evolving Agent Skills. SkillOpt treats a compact + natural-language skill document as the trainable state of a frozen language + agent, then learns that document through rollouts, reflection, bounded edits, + and held-out validation gates. +

+
+ Core Idea + Method + View Results + + + Code Repo + + + + Paper + + + + Video + +
+ + + + Related project + SkillLens studies model-generated agent skills. + A companion project page from Microsoft Research. + + + +
+ + +
+
+ +
+
+
+ Project Video +
+

SkillOpt in motion.

+

+ A short visual overview of how SkillOpt treats natural-language skills + as trainable artifacts: roll out, reflect, edit, validate, and export. +

+
+
+
+ +
+

+ Promotional video for the SkillOpt project page. The static paper teaser is shown below for high-resolution inspection. +

+
+ +
+
+ Paper Teaser +
+

The core loop at a glance.

+

+ The teaser summarizes the SkillOpt training loop: rollout evidence, + optimizer-side reflection, bounded skill edits, validation gating, + and the exported reusable skill. +

+
+
+
+ SkillOpt teaser figure showing the target model, optimizer model, bounded edits, validation gate, and exported best skill. +
+

+ Figure from the SkillOpt paper. On small screens, the figure area scrolls horizontally to preserve the original details. +

+
+ +
+
+
01 / Core Idea
+
+

Train the procedure, not the weights.

+

+ SkillOpt makes the skill document itself the optimization target. The + target model, backend, and harness stay fixed; the procedure that guides + evidence gathering, tool use, verification, and output formatting evolves. +

+
+
+ +
+
+

A skill is external state for an agent.

+

+ Instead of fine-tuning a model or hand-maintaining prompts, SkillOpt runs + the frozen agent on scored batches, asks a separate optimizer model to + propose structured edits, and accepts a candidate only when validation + performance improves. +

+
+ Frozen target model + Optimizer model + Add / delete / replace edits + Held-out gate +
+
+ +
+
+ Rollout +

The target model executes tasks with the current skill and records scored trajectories.

+
+
+ Reflect +

The optimizer analyzes success and failure minibatches to find reusable procedures.

+
+
+ Edit +

Candidate add, delete, and replace operations are merged and ranked under a budget.

+
+
+ Gate +

The candidate skill is kept only if it improves held-out selection performance.

+
+
+
+
+ +
+
+
02 / Method
+
+

A training loop for natural-language skills.

+

+ The loop deliberately mirrors a learning algorithm: rollout evidence acts + like a forward pass, reflection acts like a language-level backward pass, + and the textual learning rate bounds how far the skill can move. +

+
+
+ +
+
+

Evidence

+

Rollout batches capture messages, tool calls, verifier feedback, task metadata, and final scores.

+
+
+

Minibatches

+

Failures and successes are reflected separately so edits correct recurring errors while preserving working behavior.

+
+
+

Bounded Edits

+

An edit budget functions as a textual learning rate, preventing useful rules from being overwritten by broad rewrites.

+
+
+

Memory

+

Rejected edits, slow update, and optimizer-side meta skill provide longer-horizon feedback without bloating deployment.

+
+
+ +
+ SkillOpt pipeline showing rollout, reflection, bounded edits, validation gate, slow update, and meta skill. +
+ SkillOpt pipeline from the paper. The frozen target model executes with the current skill; the optimizer model proposes bounded edits; held-out validation decides whether the candidate becomes the new current skill. +
+
+
+ +
+
+
03 / Main Results
+
+

SkillOpt improves GPT and Qwen target models.

+

+ The table reports main-result gains across target models and + execution harnesses, comparing no-skill execution with the final + SkillOpt skill on held-out test splits. +

+
+
+ +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Target modelHarnessSearchQASheetOfficeDocVQALiveMathALFWorldAvg gain
OpenAI logoGPT-5.5Direct chat+9.6+38.9+39.0+12.4+29.3+11.9+23.5
OpenAI logoGPT-5.4Direct chat+6.2+21.1+12.8+13.6+7.2+15.6+12.8
OpenAI logoGPT-5.4-miniDirect chat+4.3+11.4+26.7+16.5+4.8+12.7+12.7
OpenAI logoGPT-5.4-nanoDirect chat+19.0+8.2+33.7+49.4+4.0+35.1+24.9
OpenAI logoGPT-5.2Direct chat+11.2+18.9+21.5+16.5+15.2+16.4+16.6
Qwen logoQwen3.5-4BDirect chat+3.1+14.6+15.2+2.1+29.6+50.7+19.2
Qwen logoQwen3.6-35B-A3BDirect chat+7.6+9.3+1.2+3.8+10.4+22.4+9.1
OpenAI logoGPT-5.5Codex+5.5+57.5+12.8+5.0+28.0N/A+21.8
OpenAI logoGPT-5.5Claude Code+4.0+58.3+13.9+3.5+13.3N/A+18.6
+
+ +
+
+
+ Method comparison +

SkillOpt clears the strongest baseline on every benchmark.

+
+
+
+
+
+ +
+ +
+
+
04 / Ablations
+
+

The controls are doing real work.

+

+ The paper isolates the optimizer components that keep skill learning stable: + enough evidence, bounded textual updates, rejected-edit feedback, slow + update, and optimizer-side memory. +

+
+
+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
ComponentSettingSearchQASpreadsheetLiveMath
Learning ratelr=4 default87.177.561.3
Learning ratewithout lr84.675.757.3
Rejected bufferwith buffer87.177.561.3
Rejected bufferwithout buffer85.572.958.9
Update memorymeta skill + slow update87.177.561.3
Update memorywithout both86.355.059.7
+
+ +
+

What the ablations say

+
+
+ Bounded + Textual learning rates prevent destructive rewrites while keeping enough plasticity to learn new procedures. +
+
+ Gated + Held-out selection turns reflection into propose-and-test optimization rather than unconditional self-editing. +
+
+ Buffered + Rejected edits become negative feedback, helping the optimizer avoid repeating harmful directions. +
+
+
+
+ +
+ Epoch checkpoint trends for SpreadsheetBench, SearchQA, and LiveMath. +
+ Epoch checkpoint trends from the paper. Selection-best checkpoints are compared with train rollout score and unseen test performance. +
+
+
+ +
+
+
05 / Skill Evolution
+
+

A typical run turns failures into concrete operating rules.

+

+ This ALFWorld run uses GPT-5.4-mini as the frozen target model and + GPT-5.5 as the optimizer model. The plot tracks train rollout and + held-out selection scores; hover or focus a point to inspect the + skill edit proposed at that stage. +

+
+
+ +
+
+
+ ALFWorld / train-sel evolution +
+ Train rollout + Selection gate +
+
+
+ + ALFWorld skill evolution scores + Selection score rises from 68.6 percent to 81.4 percent, while rejected edits are visible as downward candidate points. + + + + + + + + 85% + 80% + 75% + 70% + 65% + base + step 1 + step 2 + step 3 + slow + step 4 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+
+ Accepted edits become the current skill only after held-out selection improves. + Step 3 is rescued by a slow update; Step 4 trains higher but fails selection. +
+
+ + +
+ +
+
+ Run setup + Target model: GPT-5.4-mini. Optimizer model: GPT-5.5. The skill starts from a compact ALFWorld instruction file and is edited in text space. +
+
+ Selection rule + Candidate edits are accepted only when held-out selection improves the current best score. +
+
+ Outcome + The selected skill improves final ALFWorld test hard score from 70.9% to 85.8%. +
+
+
+ +
+
+
06 / Transfer
+
+

The exported skill behaves like a reusable artifact.

+

+ SkillOpt exports a compact best_skill.md. The paper tests + whether that artifact transfers across model sizes, execution harnesses, + and nearby benchmarks without further target-side optimization. +

+
+
+ +
+
+ Cross-model + +15.2 +

GPT-5.4 LiveMath skill transferred to GPT-5.4-nano on LiveMathBench.

+
+
+ Cross-harness + +31.8 +

Codex-trained SpreadsheetBench skill transferred into Claude Code.

+
+
+ Self-optimizer + +10.4 +

GPT-5.4-nano used as its own optimizer improved SpreadsheetBench over baseline.

+
+
+ Deployment + 1 file +

The target model consumes only the final skill, not optimizer memory.

+
+
+ +
+ A stronger optimizer model gives the largest gains, but the loop is not merely + distillation from a stronger model. Even matched target-as-optimizer settings + can discover useful edits when the update is constrained, buffered, and + validated. +
+
+ +
+
+
07 / BibTeX
+
+

Citation.

+

+ If you find SkillOpt useful, please cite the arXiv preprint below. +

+
+
+ +
+ + 
@misc{yang2026skilloptexecutivestrategyselfevolving,
+      title={SkillOpt: Executive Strategy for Self-Evolving Agent Skills}, 
+      author={Yifan Yang and Ziyang Gong and Weiquan Huang and Qihao Yang and Ziwei Zhou and Zisu Huang and Yan Li and Xuemei Gao and Qi Dai and Bei Liu and Kai Qiu and Yuqing Yang and Dongdong Chen and Xue Yang and Chong Luo},
+      year={2026},
+      eprint={2605.23904},
+      archivePrefix={arXiv},
+      primaryClass={cs.AI},
+      url={https://arxiv.org/abs/2605.23904}, 
+}
+
+
+ +
+ SkillOpt: Executive Strategy for Self-Evolving Agent Skills + Code / Citation +
+
+ + + diff --git a/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/__init__.py b/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/__init__.py new file mode 100644 index 00000000..a41cfaae --- /dev/null +++ b/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/__init__.py @@ -0,0 +1,28 @@ +"""ReflACT: Reflective Agent Tuning. + +A general-purpose framework for iteratively optimizing LLM agent skills +through structured reflection and self-improvement. + +Pipeline stages: + 1. Rollout — execute episodes with current skill + 2. Reflect — analyze trajectories, generate patches + 3. Aggregate — hierarchical merge of patches + 4. Select — rank and select top edits + 5. Update — apply edits to skill document + 6. Evaluate — validate candidate skill, accept/reject +""" + +__version__ = "0.1.0" + +from skillopt.types import ( # noqa: F401 + BatchSpec, + Edit, + EditOp, + FailureSummaryEntry, + GateAction, + GateResult, + Patch, + RawPatch, + RolloutResult, + SlowUpdateResult, +) diff --git a/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/config.py b/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/config.py new file mode 100644 index 00000000..5962a05e --- /dev/null +++ b/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/config.py @@ -0,0 +1,286 @@ +"""ReflACT config loading engine — structured YAML with inheritance. + +Supports two config formats: + 1. **Structured** (new): sections like ``model``, ``train``, ``gradient``, + ``optimizer``, ``evaluation``, ``env`` — with ``_base_`` inheritance. + 2. **Flat** (legacy): all keys at top level — fully backward compatible. + +Usage:: + + from skillopt.config import load_config, flatten_config + + cfg = load_config("configs/searchqa_default.yaml") + flat = flatten_config(cfg) # always returns flat dict for trainer +""" +from __future__ import annotations + +import copy +import os +from typing import Any + +import yaml + +# ── Section names that indicate a structured config ────────────────────── + +_STRUCTURED_SECTIONS = frozenset({ + "model", "train", "gradient", "optimizer", "evaluation", "env", +}) + +# ── Structured → flat key mapping ──────────────────────────────────────── + +_FLATTEN_MAP: dict[str, str] = { + "model.backend": "model_backend", + "model.optimizer": "optimizer_model", + "model.target": "target_model", + "model.optimizer_backend": "optimizer_backend", + "model.target_backend": "target_backend", + "model.reasoning_effort": "reasoning_effort", + "model.rewrite_reasoning_effort": "rewrite_reasoning_effort", + "model.rewrite_max_completion_tokens": "rewrite_max_completion_tokens", + "model.codex_exec_path": "codex_exec_path", + "model.codex_exec_sandbox": "codex_exec_sandbox", + "model.codex_exec_profile": "codex_exec_profile", + "model.codex_exec_full_auto": "codex_exec_full_auto", + "model.codex_exec_reasoning_effort": "codex_exec_reasoning_effort", + "model.codex_exec_use_sdk": "codex_exec_use_sdk", + "model.codex_exec_network_access": "codex_exec_network_access", + "model.codex_exec_web_search": "codex_exec_web_search", + "model.codex_exec_approval_policy": "codex_exec_approval_policy", + "model.claude_code_exec_path": "claude_code_exec_path", + "model.claude_code_exec_profile": "claude_code_exec_profile", + "model.claude_code_exec_use_sdk": "claude_code_exec_use_sdk", + "model.claude_code_exec_effort": "claude_code_exec_effort", + "model.claude_code_exec_max_thinking_tokens": "claude_code_exec_max_thinking_tokens", + "model.codex_trace_to_optimizer": "codex_trace_to_optimizer", + "model.azure_endpoint": "azure_endpoint", + "model.azure_api_version": "azure_api_version", + "model.azure_api_key": "azure_api_key", + "model.azure_openai_endpoint": "azure_openai_endpoint", + "model.azure_openai_api_version": "azure_openai_api_version", + "model.azure_openai_api_key": "azure_openai_api_key", + "model.azure_openai_auth_mode": "azure_openai_auth_mode", + "model.azure_openai_ad_scope": "azure_openai_ad_scope", + "model.azure_openai_managed_identity_client_id": "azure_openai_managed_identity_client_id", + "model.optimizer_azure_openai_endpoint": "optimizer_azure_openai_endpoint", + "model.optimizer_azure_openai_api_version": "optimizer_azure_openai_api_version", + "model.optimizer_azure_openai_api_key": "optimizer_azure_openai_api_key", + "model.optimizer_azure_openai_auth_mode": "optimizer_azure_openai_auth_mode", + "model.optimizer_azure_openai_ad_scope": "optimizer_azure_openai_ad_scope", + "model.optimizer_azure_openai_managed_identity_client_id": "optimizer_azure_openai_managed_identity_client_id", + "model.target_azure_openai_endpoint": "target_azure_openai_endpoint", + "model.target_azure_openai_api_version": "target_azure_openai_api_version", + "model.target_azure_openai_api_key": "target_azure_openai_api_key", + "model.target_azure_openai_auth_mode": "target_azure_openai_auth_mode", + "model.target_azure_openai_ad_scope": "target_azure_openai_ad_scope", + "model.target_azure_openai_managed_identity_client_id": "target_azure_openai_managed_identity_client_id", + "model.qwen_chat_base_url": "qwen_chat_base_url", + "model.qwen_chat_api_key": "qwen_chat_api_key", + "model.qwen_chat_temperature": "qwen_chat_temperature", + "model.qwen_chat_timeout_seconds": "qwen_chat_timeout_seconds", + "model.qwen_chat_max_tokens": "qwen_chat_max_tokens", + "model.qwen_chat_enable_thinking": "qwen_chat_enable_thinking", + "model.optimizer_qwen_chat_base_url": "optimizer_qwen_chat_base_url", + "model.optimizer_qwen_chat_api_key": "optimizer_qwen_chat_api_key", + "model.optimizer_qwen_chat_temperature": "optimizer_qwen_chat_temperature", + "model.optimizer_qwen_chat_timeout_seconds": "optimizer_qwen_chat_timeout_seconds", + "model.optimizer_qwen_chat_max_tokens": "optimizer_qwen_chat_max_tokens", + "model.optimizer_qwen_chat_enable_thinking": "optimizer_qwen_chat_enable_thinking", + "model.target_qwen_chat_base_url": "target_qwen_chat_base_url", + "model.target_qwen_chat_api_key": "target_qwen_chat_api_key", + "model.target_qwen_chat_temperature": "target_qwen_chat_temperature", + "model.target_qwen_chat_timeout_seconds": "target_qwen_chat_timeout_seconds", + "model.target_qwen_chat_max_tokens": "target_qwen_chat_max_tokens", + "model.target_qwen_chat_enable_thinking": "target_qwen_chat_enable_thinking", + "model.minimax_base_url": "minimax_base_url", + "model.minimax_api_key": "minimax_api_key", + "model.minimax_model": "minimax_model", + "model.minimax_temperature": "minimax_temperature", + "model.minimax_max_tokens": "minimax_max_tokens", + "model.minimax_enable_thinking": "minimax_enable_thinking", + "train.num_epochs": "num_epochs", + "train.train_size": "train_size", + "train.steps_per_epoch": "steps_per_epoch", + "train.batch_size": "batch_size", + "train.accumulation": "accumulation", + "train.seed": "seed", + "gradient.minibatch_size": "minibatch_size", + "gradient.merge_batch_size": "merge_batch_size", + "gradient.analyst_workers": "analyst_workers", + "gradient.failure_only": "failure_only", + "gradient.max_analyst_rounds": "max_analyst_rounds", + "optimizer.learning_rate": "edit_budget", + "optimizer.min_learning_rate": "min_edit_budget", + "optimizer.lr_scheduler": "lr_scheduler", + "optimizer.lr_control_mode": "lr_control_mode", + "optimizer.skill_update_mode": "skill_update_mode", + "optimizer.meta_learning_rate": "meta_edit_budget", + "optimizer.use_slow_update": "use_slow_update", + "optimizer.slow_update_samples": "slow_update_samples", + "optimizer.slow_update_gate_with_selection": "slow_update_gate_with_selection", + "optimizer.longitudinal_pair_policy": "longitudinal_pair_policy", + "optimizer.use_meta_skill": "use_meta_skill", + "evaluation.use_gate": "use_gate", + "evaluation.gate_metric": "gate_metric", + "evaluation.gate_mixed_weight": "gate_mixed_weight", + "evaluation.sel_env_num": "sel_env_num", + "evaluation.test_env_num": "test_env_num", + "evaluation.eval_test": "eval_test", + "env.name": "env", + "env.skill_init": "skill_init", + "env.out_root": "out_root", +} + + +# ── Deep merge ─────────────────────────────────────────────────────────── + +def _deep_merge(base: dict, override: dict) -> dict: + """Recursively merge *override* into *base* (returns new dict).""" + result = copy.deepcopy(base) + for key, val in override.items(): + if key in result and isinstance(result[key], dict) and isinstance(val, dict): + result[key] = _deep_merge(result[key], val) + else: + result[key] = copy.deepcopy(val) + return result + + +# ── YAML loading with _base_ inheritance ───────────────────────────────── + +def _load_yaml(path: str, _visited: set[str] | None = None) -> dict: + """Load a YAML file, resolving ``_base_`` inheritance recursively.""" + abs_path = os.path.abspath(path) + if _visited is None: + _visited = set() + if abs_path in _visited: + raise ValueError(f"Circular _base_ inheritance: {abs_path}") + _visited.add(abs_path) + + with open(abs_path) as f: + cfg = yaml.safe_load(f) or {} + + base_ref = cfg.pop("_base_", None) + if base_ref: + base_path = os.path.join(os.path.dirname(abs_path), base_ref) + base_cfg = _load_yaml(base_path, _visited) + cfg = _deep_merge(base_cfg, cfg) + + return cfg + + +# ── Format detection ───────────────────────────────────────────────────── + +def is_structured(cfg: dict) -> bool: + """Return True if *cfg* uses the new structured section format.""" + return any( + key in _STRUCTURED_SECTIONS and isinstance(cfg.get(key), dict) + for key in cfg + ) + + +# ── Flatten ────────────────────────────────────────────────────────────── + +def flatten_config(cfg: dict) -> dict: + """Convert a structured config to the flat dict expected by the trainer. + + If *cfg* is already flat, returns a shallow copy unchanged. + """ + if not is_structured(cfg): + return dict(cfg) + + flat: dict[str, Any] = {} + + evaluation_section = cfg.get("evaluation", {}) + if isinstance(evaluation_section, dict) and evaluation_section.get("use_gate") is False: + raise ValueError( + "Gate validation is mandatory in this branch. Remove " + "`evaluation.use_gate: false` from the config." + ) + + # Apply the explicit mapping + for dotted, flat_key in _FLATTEN_MAP.items(): + section, key = dotted.split(".", 1) + section_dict = cfg.get(section, {}) + if isinstance(section_dict, dict) and key in section_dict: + flat[flat_key] = section_dict[key] + + # Pass through env-specific keys not in the explicit mapping + env_section = cfg.get("env", {}) + if isinstance(env_section, dict): + mapped_env_keys = { + k.split(".", 1)[1] + for k in _FLATTEN_MAP + if k.startswith("env.") + } + for key, val in env_section.items(): + if key not in mapped_env_keys: + flat[key] = val + + return flat + + +# ── Override application ───────────────────────────────────────────────── + +def _cast_value(val_str: str) -> Any: + """Auto-cast a CLI string value to int / float / bool / str.""" + if val_str.lower() in ("true", "yes"): + return True + if val_str.lower() in ("false", "no"): + return False + try: + return int(val_str) + except ValueError: + pass + try: + return float(val_str) + except ValueError: + pass + return val_str + + +def apply_overrides(cfg: dict, overrides: list[str]) -> None: + """Apply ``key=value`` overrides to a structured config (in place). + + Supports both ``section.key=value`` (for structured configs) and + ``key=value`` (for flat configs or flat keys in env section). + """ + for item in overrides: + if "=" not in item: + raise ValueError(f"Invalid override (expected key=value): {item!r}") + key, val_str = item.split("=", 1) + val = _cast_value(val_str) + + if "." in key: + section, subkey = key.split(".", 1) + if section in cfg and isinstance(cfg[section], dict): + cfg[section][subkey] = val + else: + cfg.setdefault(section, {})[subkey] = val + else: + # Flat key — apply to top level (for legacy compat) + cfg[key] = val + + +# ── Public API ─────────────────────────────────────────────────────────── + +def load_config( + path: str, + overrides: list[str] | None = None, +) -> dict: + """Load a config file with ``_base_`` inheritance and optional overrides. + + Parameters + ---------- + path : str + Path to the YAML config file. + overrides : list[str] | None + ``key=value`` strings from ``--cfg-options``. + + Returns + ------- + dict + The merged config (structured or flat depending on the YAML). + """ + cfg = _load_yaml(path) + if overrides: + apply_overrides(cfg, overrides) + return cfg diff --git a/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/datasets/__init__.py b/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/datasets/__init__.py new file mode 100644 index 00000000..3aa2eb81 --- /dev/null +++ b/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/datasets/__init__.py @@ -0,0 +1,7 @@ +"""ReflACT Datasets -- task batch planning and data loading. + +Analogous to the datasets and dataloaders in neural network training: +provides batch sampling, epoch planning, and data management for the +ReflACT training pipeline. +""" +from skillopt.datasets.base import BaseDataLoader, BatchSpec, SplitDataLoader # noqa: F401 diff --git a/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/datasets/base.py b/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/datasets/base.py new file mode 100644 index 00000000..668f201b --- /dev/null +++ b/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/datasets/base.py @@ -0,0 +1,512 @@ +"""Generic task dataloader abstractions for ReflACT. + +ReflACT does not train model parameters directly. Instead, it iterates over +task batches, rolls out the current skill, reflects on failures/successes, +and updates the skill document. Because of that, the "dataloader" abstraction +here is closer to a batch sampler / episode planner than a tensor loader. + +Class hierarchy:: + + BaseDataLoader # abstract — simulator-backed envs (e.g. ALFWorld) + └── SplitDataLoader # abstract — dataset-backed envs with split_dir + +SplitDataLoader supports two dataset entry modes: + +1. ``split_mode="split_dir"``: consume an existing split directory. +2. ``split_mode="ratio"``: build a deterministic split directory from a raw + dataset path using an explicit train:val:test ratio. + +In either case, the standardised split layout is: + + split_dir/ + ├── train/ # training items + ├── val/ # validation / selection items (gate) + └── test/ # held-out test items + +Each subdirectory's contents are benchmark-specific. Subclasses only need +to implement ``load_split_items(split_path)`` to teach the loader how to +read items from one of those directories. +""" +from __future__ import annotations + +import glob +import json +import os +import random +from abc import ABC, abstractmethod +from dataclasses import dataclass, field +from typing import Any + + +@dataclass(slots=True) +class BatchSpec: + """A concrete batch request consumed by the training loop. + + Parameters + ---------- + phase : str + ``"train"`` or ``"eval"``. + split : str + Dataset split name, typically ``"train"`` or an eval split. + seed : int + Random seed used to construct the batch deterministically. + batch_size : int + Requested number of items / episodes in this batch. + payload : object | None + Environment-specific batch payload. For dataset-backed environments + this is often a list of sampled items; for simulator-backed + environments this may be ``None`` and the seed alone can define the + batch. + metadata : dict[str, Any] + Optional structured metadata for logging, resume, or curriculum logic. + """ + + phase: str + split: str + seed: int + batch_size: int + payload: object | None = None + metadata: dict[str, Any] = field(default_factory=dict) + + +class BaseDataLoader(ABC): + """Abstract base class for task batch planning in ReflACT. + + Subclasses are responsible for defining how a train or eval batch is + sampled. The default implementation here provides deterministic epoch seed + planning so all loaders share the same reproducibility behavior. + """ + + def setup(self, cfg: dict) -> None: + """Optional one-time initialization with the full trainer config.""" + + def set_out_root(self, out_root: str) -> None: + """Optional hook for loaders that persist split files or state.""" + + def state_dict(self) -> dict[str, Any]: + """Return serializable loader state for resume support.""" + return {} + + def load_state_dict(self, state: dict[str, Any]) -> None: + """Restore loader state from :meth:`state_dict` output.""" + + def get_train_size(self) -> int | None: + """Return the size of the training pool when known.""" + return None + + @staticmethod + def make_base_seeds(steps_per_epoch: int, accumulation: int, seed: int) -> list[int]: + """Return the deterministic seed pool used to define train batches.""" + batches_per_epoch = steps_per_epoch * accumulation + return [seed + i + 1 for i in range(batches_per_epoch)] + + @staticmethod + def shuffle_epoch_seeds(base_seeds: list[int], epoch: int, seed: int) -> list[int]: + """Return the per-epoch deterministic shuffle of *base_seeds*.""" + epoch_rng = random.Random(seed + epoch * 1000) + shuffled = list(base_seeds) + epoch_rng.shuffle(shuffled) + return shuffled + + def plan_train_epoch( + self, + *, + epoch: int, + steps_per_epoch: int, + accumulation: int, + batch_size: int, + seed: int, + **kwargs, + ) -> list[BatchSpec]: + """Build the full list of training batches for one epoch.""" + base_seeds = self.make_base_seeds( + steps_per_epoch=steps_per_epoch, + accumulation=accumulation, + seed=seed, + ) + shuffled_seeds = self.shuffle_epoch_seeds(base_seeds, epoch=epoch, seed=seed) + return [ + self.build_train_batch(batch_size=batch_size, seed=batch_seed, **kwargs) + for batch_seed in shuffled_seeds + ] + + @abstractmethod + def build_train_batch(self, batch_size: int, seed: int, **kwargs) -> BatchSpec: + """Construct one training batch specification.""" + + @abstractmethod + def build_eval_batch( + self, + env_num: int, + split: str, + seed: int, + **kwargs, + ) -> BatchSpec: + """Construct one evaluation batch specification.""" + + +# ── Split-based dataloader for dataset-backed environments ────────────── + +# Canonical split names expected under split_dir/ +SPLIT_NAMES = ("train", "val", "test") + +# Maps legacy / trainer split names → canonical directory names +_SPLIT_ALIAS: dict[str, str] = { + "train": "train", + "valid_seen": "val", + "selection": "val", + "val": "val", + "valid_unseen": "test", + "test": "test", +} + + +def _load_json_or_jsonl(path: str) -> list[dict]: + """Load a list of items from a JSON or JSONL file.""" + with open(path, encoding="utf-8") as f: + content = f.read().strip() + if not content: + return [] + + try: + data = json.loads(content) + except json.JSONDecodeError: + data = None + + if isinstance(data, list): + return data + if isinstance(data, dict): + nested = data.get("data") + if isinstance(nested, list): + return nested + return list(data.values()) + + items: list[dict] = [] + for line in content.splitlines(): + line = line.strip() + if line: + items.append(json.loads(line)) + return items + + +def _parse_split_ratio(text: str) -> tuple[int, int, int]: + parts = [part.strip() for part in str(text or "").split(":") if part.strip()] + if len(parts) != 3: + raise ValueError( + f"split_ratio must be in train:val:test form, got {text!r}" + ) + try: + train, val, test = (int(part) for part in parts) + except ValueError as exc: + raise ValueError( + f"split_ratio must contain integers, got {text!r}" + ) from exc + if min(train, val, test) <= 0: + raise ValueError(f"split_ratio parts must be positive, got {text!r}") + return train, val, test + + +def _compute_split_counts(total: int, ratio: tuple[int, int, int]) -> tuple[int, int, int]: + weights = list(ratio) + denom = sum(weights) + raw = [total * weight / denom for weight in weights] + counts = [int(value) for value in raw] + remaining = total - sum(counts) + order = sorted( + range(len(raw)), + key=lambda idx: (raw[idx] - counts[idx], weights[idx]), + reverse=True, + ) + for idx in order[:remaining]: + counts[idx] += 1 + return counts[0], counts[1], counts[2] + + +class SplitDataLoader(BaseDataLoader): + """Base class for dataset-backed environments. + + Supported modes: + + - ``split_mode="split_dir"``: load an existing ``train/``, ``val/``, + ``test/`` directory tree. + - ``split_mode="ratio"``: load raw items from ``data_path`` and materialize + a deterministic split directory with the requested ratio. + """ + + def __init__( + self, + split_dir: str = "", + data_path: str = "", + split_mode: str = "ratio", + split_ratio: str = "2:1:7", + split_seed: int = 42, + split_output_dir: str = "", + seed: int = 42, + limit: int = 0, + **kwargs, + ) -> None: + self.split_dir = split_dir + self.data_path = data_path + self.split_mode = split_mode + self.split_ratio = split_ratio + self.split_seed = int(split_seed) + self.split_output_dir = split_output_dir + self.seed = seed + self.limit = limit + self._splits: dict[str, list[dict]] = {} + + # ── Setup ──────────────────────────────────────────────────────────── + + def setup(self, cfg: dict) -> None: + if not self.split_mode: + self.split_mode = str(cfg.get("split_mode", "ratio") or "ratio") + if not self.split_dir: + self.split_dir = cfg.get("split_dir", "") + if not self.data_path: + self.data_path = cfg.get("data_path", "") + if not self.split_output_dir: + self.split_output_dir = cfg.get("split_output_dir", "") + if "split_seed" in cfg and not self.split_seed: + self.split_seed = int(cfg.get("split_seed", 0) or 0) + if not self.split_seed: + self.split_seed = self.seed + if not self.split_ratio: + self.split_ratio = str(cfg.get("split_ratio", "2:1:7") or "2:1:7") + + mode = str(self.split_mode or "ratio").strip().lower() + if mode not in {"ratio", "split_dir"}: + raise ValueError( + f"{type(self).__name__} split_mode must be 'ratio' or 'split_dir', " + f"got {self.split_mode!r}" + ) + self.split_mode = mode + + if self.split_mode == "ratio": + self.split_dir = self._materialize_ratio_split(cfg) + if not self.split_dir: + raise ValueError( + f"{type(self).__name__} requires either " + "`split_mode=ratio` with `data_path`, or `split_mode=split_dir` " + f"with `split_dir` pointing to {'/'.join(SPLIT_NAMES)}/." + ) + self._load_all_splits() + + def _resolve_split_output_dir(self, cfg: dict) -> str: + if self.split_output_dir: + return os.path.abspath(self.split_output_dir) + out_root = os.path.abspath(str(cfg.get("out_root") or os.getcwd())) + env_name = str(cfg.get("env") or type(self).__name__.replace("DataLoader", "").lower()) + ratio_tag = str(self.split_ratio or "2:1:7").replace(":", "-") + return os.path.join(out_root, "_generated_splits", f"{env_name}_{ratio_tag}_seed{self.split_seed}") + + def load_raw_items(self, data_path: str) -> list[dict]: + """Load raw items from a dataset path before ratio splitting. + + Subclasses can override when the raw dataset is not a single JSON/JSONL + file or when directory layouts require custom normalization. + """ + if os.path.isdir(data_path): + if any(os.path.isdir(os.path.join(data_path, name)) for name in SPLIT_NAMES): + raise ValueError( + f"{type(self).__name__} got a split directory as data_path. " + "Use split_mode=split_dir and pass it as split_dir instead." + ) + candidates = sorted(glob.glob(os.path.join(data_path, "*.json"))) + candidates += sorted(glob.glob(os.path.join(data_path, "*.jsonl"))) + if len(candidates) != 1: + raise ValueError( + f"{type(self).__name__} expected data_path to be one JSON/JSONL file " + f"or a directory containing exactly one such file, got: {data_path}" + ) + return _load_json_or_jsonl(candidates[0]) + return _load_json_or_jsonl(data_path) + + def write_split_items(self, split_path: str, items: list[dict]) -> None: + os.makedirs(split_path, exist_ok=True) + out_path = os.path.join(split_path, "items.json") + with open(out_path, "w", encoding="utf-8") as f: + json.dump(items, f, ensure_ascii=False, indent=2) + + def _materialize_ratio_split(self, cfg: dict) -> str: + data_path = os.path.abspath(str(self.data_path or "").strip()) + if not data_path: + raise ValueError( + f"{type(self).__name__} requires data_path when split_mode=ratio." + ) + + ratio = _parse_split_ratio(self.split_ratio) + items = self.load_raw_items(data_path) + if not isinstance(items, list) or not items: + raise ValueError(f"No raw items available for ratio split from {data_path}") + + shuffled = list(items) + rng = random.Random(self.split_seed) + rng.shuffle(shuffled) + + train_n, val_n, test_n = _compute_split_counts(len(shuffled), ratio) + train_items = shuffled[:train_n] + val_items = shuffled[train_n: train_n + val_n] + test_items = shuffled[train_n + val_n: train_n + val_n + test_n] + + split_dir = self._resolve_split_output_dir(cfg) + manifest = { + "source_data_path": data_path, + "split_mode": "ratio", + "split_ratio": self.split_ratio, + "split_seed": self.split_seed, + "counts": { + "train": len(train_items), + "val": len(val_items), + "test": len(test_items), + }, + } + os.makedirs(split_dir, exist_ok=True) + self.write_split_items(os.path.join(split_dir, "train"), train_items) + self.write_split_items(os.path.join(split_dir, "val"), val_items) + self.write_split_items(os.path.join(split_dir, "test"), test_items) + with open(os.path.join(split_dir, "split_manifest.json"), "w", encoding="utf-8") as f: + json.dump(manifest, f, ensure_ascii=False, indent=2) + print( + f" [{type(self).__name__}] generated ratio split {self.split_ratio} " + f"at {split_dir} from {data_path}" + ) + return split_dir + + def _load_all_splits(self) -> None: + for name in SPLIT_NAMES: + split_path = os.path.join(self.split_dir, name) + if not os.path.isdir(split_path): + raise ValueError( + f"Missing '{name}/' subdirectory in split_dir: {self.split_dir}" + ) + items = self.load_split_items(split_path) + if self.limit: + items = items[: self.limit] + self._splits[name] = items + + counts = " ".join(f"{k}={len(v)}" for k, v in self._splits.items()) + print(f" [{type(self).__name__}] {counts} (from {self.split_dir})") + + def load_split_items(self, split_path: str) -> list[dict]: + """Load items from one split directory (e.g. ``split_dir/train/``). + + Default: finds the first ``.json`` file in the directory and loads it + as a JSON array. Subclasses can override for custom formats. + """ + json_files = sorted(glob.glob(os.path.join(split_path, "*.json"))) + if not json_files: + raise FileNotFoundError( + f"No .json file found in {split_path}" + ) + with open(json_files[0], encoding="utf-8") as f: + items = json.load(f) + if not isinstance(items, list): + raise ValueError( + f"Expected JSON array in {json_files[0]}, got {type(items).__name__}" + ) + return items + + # ── Accessors ──────────────────────────────────────────────────────── + + @property + def train_items(self) -> list[dict]: + return self._splits.get("train", []) + + @property + def val_items(self) -> list[dict]: + return self._splits.get("val", []) + + @property + def test_items(self) -> list[dict]: + return self._splits.get("test", []) + + def get_split_items(self, split: str) -> list[dict]: + """Resolve a split name (including legacy aliases) to its item list.""" + canonical = _SPLIT_ALIAS.get(split, split) + return list(self._splits.get(canonical, self.val_items)) + + def get_train_size(self) -> int: + return len(self.train_items) + + def plan_train_epoch( + self, + *, + epoch: int, + steps_per_epoch: int, + accumulation: int, + batch_size: int, + seed: int, + **kwargs, + ) -> list[BatchSpec]: + """Build one full epoch that covers the train split in shuffled order. + + For split-backed datasets, an epoch should correspond to one pass over + the available training items rather than repeated independent sampling. + """ + epoch_rng = random.Random(seed + epoch * 1000) + items = list(self.train_items) + epoch_rng.shuffle(items) + + total_batches = steps_per_epoch * accumulation + if total_batches <= 0: + return [] + + batches: list[BatchSpec] = [] + cursor = 0 + for batch_idx in range(total_batches): + batch_items = items[cursor: cursor + batch_size] + cursor += len(batch_items) + + # Extremely small datasets can leave trailing empty microbatches + # when accumulation > 1. Reuse the shuffled prefix in that case so + # the trainer still receives the expected batch count. + if not batch_items and items: + refill_rng = random.Random(seed + epoch * 1000 + batch_idx + 1) + batch_items = list(items) + refill_rng.shuffle(batch_items) + batch_items = batch_items[:batch_size] + + batches.append( + BatchSpec( + phase="train", + split="train", + seed=seed + epoch * 1000 + batch_idx + 1, + batch_size=len(batch_items), + payload=batch_items, + ) + ) + + return batches + + # ── Batch construction ─────────────────────────────────────────────── + + def build_train_batch(self, batch_size: int, seed: int, **kwargs) -> BatchSpec: + rng = random.Random(seed) + items = list(self.train_items) + rng.shuffle(items) + items = items[:batch_size] + return BatchSpec( + phase="train", + split="train", + seed=seed, + batch_size=len(items), + payload=items, + ) + + def build_eval_batch( + self, + env_num: int, + split: str, + seed: int, + **kwargs, + ) -> BatchSpec: + items = self.get_split_items(split) + if env_num and env_num < len(items): + items = items[:env_num] + return BatchSpec( + phase="eval", + split=split, + seed=seed, + batch_size=len(items), + payload=items, + ) diff --git a/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/engine/__init__.py b/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/engine/__init__.py new file mode 100644 index 00000000..b876e704 --- /dev/null +++ b/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/engine/__init__.py @@ -0,0 +1,9 @@ +"""ReflACT Engine -- the training runner. + +Analogous to the Runner in mmengine: orchestrates the full training pipeline +including rollout, gradient computation, aggregation, optimization, and +evaluation. +""" +from skillopt.engine.trainer import ReflACTTrainer # noqa: F401 + +__all__ = ["ReflACTTrainer"] diff --git a/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/engine/trainer.py b/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/engine/trainer.py new file mode 100644 index 00000000..a0de43be --- /dev/null +++ b/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/engine/trainer.py @@ -0,0 +1,2083 @@ +"""ReflACT Trainer — the main training loop. + +Orchestrates the 6-stage ReflACT pipeline: + 1. Rollout — execute episodes with current skill + 2. Reflect — analyze trajectories, generate patches + 3. Aggregate — hierarchical merge of patches + 4. Select — rank and select top edits + 5. Update — apply edits to skill document + 6. Evaluate — validate candidate skill, accept/reject + +The trainer is environment-agnostic; all environment-specific logic is +delegated to an :class:`~skillopt.envs.base.EnvAdapter` instance. +""" +from __future__ import annotations + +import glob +import json +import math +import os +import random +import re +import time +from collections import defaultdict + +from skillopt.datasets.base import BatchSpec +from skillopt.envs.base import EnvAdapter +from skillopt.evaluation.gate import evaluate_gate, select_gate_score +from skillopt.gradient.aggregate import merge_patches +from skillopt.optimizer.meta_skill import run_meta_skill +from skillopt.optimizer.clip import rank_and_select +from skillopt.optimizer.lr_autonomous import decide_autonomous_learning_rate +from skillopt.optimizer.rewrite import rewrite_skill_from_suggestions +from skillopt.optimizer.scheduler import build_scheduler +from skillopt.optimizer.skill import apply_patch_with_report +from skillopt.optimizer.slow_update import ( + build_comparison_pairs, + extract_slow_update_field, + inject_empty_slow_update_field, + replace_slow_update_field, + run_slow_update, + save_comparison_pairs, +) +from skillopt.optimizer.update_modes import ( + get_payload_items, + is_full_rewrite_minibatch_mode, + normalize_update_mode, + payload_label, + short_item_summary, +) +from skillopt.model import ( + configure_azure_openai, + configure_claude_code_exec, + configure_codex_exec, + configure_minimax_chat, + configure_qwen_chat, + get_token_summary, + reset_token_tracker, + set_reasoning_effort, + set_target_backend, + set_target_deployment, + set_optimizer_backend, + set_optimizer_deployment, +) +from skillopt.utils import compute_score, skill_hash + + +# ── Patch normalization ─────────────────────────────────────────────────────── + +def _normalise_patches( + raw_patches: list[dict | None], + update_mode: str = "patch", +) -> tuple[list[dict], list[dict]]: + """Extract inner 'patch' sub-dict, split into failure/success lists. + + Each element is expected to conform to :class:`~skillopt.types.RawPatch`. + """ + mode = normalize_update_mode(update_mode) + failure: list[dict] = [] + success: list[dict] = [] + for p in raw_patches: + if not isinstance(p, dict): + continue + inner = p.get("patch", p) + if not isinstance(inner, dict): + continue + items = get_payload_items(inner, mode) + if not items: + continue + support = max(int(p.get("batch_size", 0) or 0), 1) + for item in items: + if isinstance(item, dict): + item.setdefault("source_type", p.get("source_type", "failure")) + item.setdefault("support_count", support) + if p.get("source_type", "failure") == "success": + success.append(inner) + else: + failure.append(inner) + return failure, success + + +def _normalise_longitudinal_pair_policy(policy: str | None) -> str: + raw = str(policy or "mixed").strip().lower() + aliases = { + "mixed": "mixed", + "default": "mixed", + "random": "mixed", + "all": "mixed", + "changed": "changed", + "change": "changed", + "delta": "changed", + "10_01": "changed", + "01_10": "changed", + "unchanged": "unchanged", + "stable": "unchanged", + "same": "unchanged", + "00_11": "unchanged", + } + if raw not in aliases: + raise ValueError( + "optimizer.longitudinal_pair_policy must be one of " + "mixed, changed, unchanged" + ) + return aliases[raw] + + +def _normalise_lr_control_mode(mode: str | None) -> str: + raw = str(mode or "fixed").strip().lower() + aliases = { + "fixed": "fixed", + "manual": "fixed", + "scheduler": "fixed", + "scheduled": "fixed", + "autonomous": "autonomous", + "auto": "autonomous", + "optimizer": "autonomous", + "none": "none", + "off": "none", + "no_lr": "none", + } + if raw not in aliases: + raise ValueError("optimizer.lr_control_mode must be one of fixed, autonomous, none") + return aliases[raw] + + +def _filter_longitudinal_pairs(pairs: list[dict], policy: str) -> list[dict]: + if policy == "mixed": + return pairs + if policy == "changed": + keep = {"improved", "regressed"} + elif policy == "unchanged": + keep = {"persistent_fail", "stable_success"} + else: + raise ValueError(f"Unknown longitudinal pair policy: {policy}") + return [p for p in pairs if p.get("category") in keep] + + +def _pair_category_counts(pairs: list[dict]) -> dict[str, int]: + counts = { + "improved": 0, + "regressed": 0, + "persistent_fail": 0, + "stable_success": 0, + } + for pair in pairs: + cat = str(pair.get("category", "")) + counts[cat] = counts.get(cat, 0) + 1 + return counts + + +def _safe_pair_id(value: str) -> str: + safe = re.sub(r"[^A-Za-z0-9_.-]+", "_", str(value)).strip("_") + return safe[:80] or "item" + + +def _build_longitudinal_pairs( + *, + adapter: EnvAdapter, + dataloader, + prev_skill: str, + curr_skill: str, + initial_items: list[dict], + initial_prev_results: list[dict], + initial_curr_results: list[dict], + prev_rollout_dir: str, + curr_rollout_dir: str, + policy: str, + target_n: int, + seed: int, + out_root: str, +) -> tuple[list[dict], list[dict]]: + """Build longitudinal pairs, optionally filtering by change category. + + ``mixed`` preserves the legacy behavior exactly. ``changed`` keeps only + 10/01 pairs and attempts to top up to ``target_n`` by scanning the train + split once. ``unchanged`` keeps only 00/11 pairs and does not top up. + """ + all_pairs = build_comparison_pairs( + initial_prev_results, + initial_curr_results, + initial_items, + prev_rollout_dir=prev_rollout_dir, + curr_rollout_dir=curr_rollout_dir, + ) + selected_pairs = _filter_longitudinal_pairs(all_pairs, policy) + if policy != "changed" or len(selected_pairs) >= target_n or dataloader is None: + return selected_pairs, all_pairs + + train_items = list(getattr(dataloader, "train_items", []) or []) + if not train_items: + return selected_pairs, all_pairs + + seen_ids = {str(p.get("id", "")) for p in all_pairs} + rng = random.Random(seed) + candidates = list(train_items) + rng.shuffle(candidates) + candidates = [item for item in candidates if str(item.get("id", "")) not in seen_ids] + + for idx, item in enumerate(candidates): + if len(selected_pairs) >= target_n: + break + item_id = _safe_pair_id(str(item.get("id", f"item_{idx}"))) + batch = BatchSpec( + phase="train", + split="train", + seed=seed + idx + 1, + batch_size=1, + payload=[item], + ) + env = adapter.build_env_from_batch(batch, out_root=out_root) + prev_dir = os.path.join(prev_rollout_dir, "topup", item_id) + curr_dir = os.path.join(curr_rollout_dir, "topup", item_id) + prev_results = adapter.rollout(env, prev_skill, prev_dir) + curr_results = adapter.rollout(env, curr_skill, curr_dir) + pair = build_comparison_pairs( + prev_results, + curr_results, + [item], + prev_rollout_dir=prev_dir, + curr_rollout_dir=curr_dir, + ) + all_pairs.extend(pair) + selected_pairs.extend(_filter_longitudinal_pairs(pair, policy)) + + return selected_pairs[:target_n], all_pairs + + +# ── History / persistence helpers ───────────────────────────────────────────── + +_SECRET_KEYS = { + "azure_api_key", + "api_key", + "openai_api_key", +} + + +def _redact_value(val: str) -> str: + if len(val) <= 8: + return "*" * len(val) + return f"{val[:4]}...{val[-4:]}" + + +def _redact_cfg(cfg: dict) -> dict: + redacted = dict(cfg) + for key in list(redacted): + if key.lower() in _SECRET_KEYS and redacted.get(key): + redacted[key] = _redact_value(str(redacted[key])) + return redacted + +def _load_history(out_root: str) -> list[dict]: + path = os.path.join(out_root, "history.json") + if os.path.exists(path): + with open(path) as f: + return json.load(f) + return [] + + +def _save_history(out_root: str, history: list[dict]) -> None: + path = os.path.join(out_root, "history.json") + with open(path, "w") as f: + json.dump(history, f, ensure_ascii=False, indent=2) + + +def _save_skill(out_root: str, step: int, content: str) -> None: + skills_dir = os.path.join(out_root, "skills") + os.makedirs(skills_dir, exist_ok=True) + with open(os.path.join(skills_dir, f"skill_v{step:04d}.md"), "w") as f: + f.write(content) + + +def _load_skill(out_root: str, step: int) -> str: + path = os.path.join(out_root, "skills", f"skill_v{step:04d}.md") + with open(path) as f: + return f.read() + + +def _load_meta_skill_content(out_root: str, epoch: int) -> str: + if epoch <= 0: + return "" + path = os.path.join( + out_root, "meta_skill", f"epoch_{epoch:02d}", "meta_skill_result.json", + ) + if not os.path.exists(path): + return "" + try: + with open(path) as f: + result = json.load(f) + return str(result.get("meta_skill_content", "")).strip() + except Exception: + return "" + + +def _load_runtime_state(out_root: str) -> dict | None: + path = os.path.join(out_root, "runtime_state.json") + if not os.path.exists(path): + return None + try: + with open(path) as f: + state = json.load(f) + return state if isinstance(state, dict) else None + except Exception: + return None + + +def _save_runtime_state(out_root: str, state: dict) -> None: + path = os.path.join(out_root, "runtime_state.json") + with open(path, "w") as f: + json.dump(state, f, ensure_ascii=False, indent=2) + + +def _resolve_train_size(cfg: dict, dataloader) -> int: + configured = int(cfg.get("train_size", 0) or 0) + inferred: int | None = None + + if dataloader is not None: + getter = getattr(dataloader, "get_train_size", None) + if callable(getter): + try: + value = getter() + except Exception: + value = None + if value is not None: + inferred = int(value) + elif hasattr(dataloader, "train_items"): + try: + inferred = len(getattr(dataloader, "train_items")) + except Exception: + inferred = None + + if inferred is not None and inferred <= 0: + inferred = None + + if configured > 0 and inferred is not None and configured != inferred: + raise ValueError( + f"Configured train_size={configured} does not match loaded train split " + f"size={inferred}. Fix the config or the dataset split." + ) + + train_size = configured if configured > 0 else inferred + if train_size is None or train_size <= 0: + raise ValueError( + "Unable to determine train_size automatically. " + "Provide train.train_size in the config for this environment." + ) + return int(train_size) + + +def _compute_task_type_buckets(results: list[dict], task_types: list[str]) -> dict[str, dict]: + """Compute per-task-type success rates.""" + buckets: dict[str, dict] = {} + for task in task_types + ["overall"]: + buckets[task] = {"total": 0, "hard": 0, "soft": 0.0} + for r in results: + tt = r.get("task_type", "other") + for key in [tt, "overall"]: + if key not in buckets: + buckets[key] = {"total": 0, "hard": 0, "soft": 0.0} + buckets[key]["total"] += 1 + buckets[key]["hard"] += float(r.get("hard", 0)) + buckets[key]["soft"] += float(r.get("soft", 0.0)) + return buckets + + +def _format_rejection_buffer(buffer: list[dict]) -> str: + """**DEPRECATED** — kept for backward compat; use _format_step_buffer.""" + return _format_step_buffer(buffer) + + +def _extract_failure_patterns( + rollout_results: list[dict], + step_dir: str, +) -> list[dict]: + """Extract compact failure patterns from rollout results. + + Uses analyst ``failure_summary`` from minibatch patches when available, + otherwise falls back to ``fail_reason`` prefix grouping. + """ + failures = [r for r in rollout_results if not r.get("hard") or float(r.get("hard", 0)) < 1e-9] + if not failures: + return [] + + # Group by fail_reason prefix + groups: dict[str, list[dict]] = defaultdict(list) + for r in failures: + reason = r.get("fail_reason", "unknown") + prefix = reason.split(":")[0].strip() if ":" in reason else reason + groups[prefix].append(r) + + # Try richer descriptions from analyst patches + analyst_descs: list[str] = [] + patch_globs = [ + os.path.join(step_dir, "patches", "minibatch_fail_*.json"), + os.path.join(step_dir, "batch_*", "patches", "minibatch_fail_*.json"), + ] + seen_patch_files: set[str] = set() + for pattern in patch_globs: + for fname in sorted(glob.glob(pattern)): + if fname in seen_patch_files: + continue + seen_patch_files.add(fname) + try: + with open(fname) as f: + patch = json.load(f) + for fs in patch.get("failure_summary", []): + ft = fs.get("failure_type", "") + sd = fs.get("description", "") + analyst_descs.append(f"{ft}: {sd}" if sd else ft) + except Exception: + pass + + patterns = [] + desc_iter = iter(analyst_descs) + for prefix, items in groups.items(): + desc = next(desc_iter, None) or prefix + patterns.append({ + "pattern": desc, + "count": len(items), + "task_ids": [str(r.get("id", "?")) for r in items], + }) + return patterns + + +def _format_step_buffer(buffer: list[dict]) -> str: + """Format the unified step buffer into a single context block. + + Each entry captures what happened at a previous step: failure patterns + observed during rollout, and — when the step was rejected — the specific + edits that were tried and the resulting score drop. + + Returns empty string when *buffer* is empty. + """ + if not buffer: + return "" + + parts = [ + "Below is a summary of previous steps in this epoch. " + "Use it to avoid repeating ineffective edits and to prioritise " + "failure patterns that remain unsolved.\n" + ] + + for entry in buffer: + step = entry["step"] + action = entry["action"] + n_fail = entry.get("n_fail", 0) + n_total = entry.get("n_total", "?") + + parts.append(f"### Step {step} — {action.upper()} ({n_fail}/{n_total} failed)") + + # Failure patterns + for p in entry.get("failure_patterns", []): + ids = ", ".join(p["task_ids"][:3]) + parts.append(f' - "{p["pattern"]}" (×{p["count"]}, tasks: {ids})') + + # Rejected edits (only present on reject) + rejected = entry.get("rejected_edits", []) + if rejected: + score_before = entry.get("score_before", "?") + score_after = entry.get("score_after", "?") + parts.append( + f" Rejected edits (score {score_before} → {score_after}):" + ) + for i, e in enumerate(rejected, 1): + if e.get("op") is not None: + op = e.get("op", "?") + content = e.get("content", "") + target = e.get("target", "") + if target: + parts.append(f' {i}. [{op}] target="{target[:80]}" → "{content}"') + else: + parts.append(f' {i}. [{op}] "{content}"') + else: + kind = e.get("type", "?") + title = e.get("title", "") + instruction = e.get("instruction", "") + parts.append(f' {i}. [{kind}] "{title}" → "{instruction}"') + + return "\n".join(parts) + + +# ── Trainer ────────────────────────────────────────────────────────────────── + +class ReflACTTrainer: + """Main ReflACT training loop. + + Parameters + ---------- + cfg : dict + Configuration dictionary. See ``configs/alfworld_default.yaml`` + for the full list of keys. + adapter : EnvAdapter + Environment adapter instance. + """ + + def __init__(self, cfg: dict, adapter: EnvAdapter) -> None: + self.cfg = cfg + self.adapter = adapter + + def train(self) -> dict: + """Execute the full ReflACT training loop. Returns summary dict.""" + cfg = self.cfg + adapter = self.adapter + out_root = cfg["out_root"] + os.makedirs(out_root, exist_ok=True) + + # ── Adapter setup (one-time init) ──────────────────────────── + adapter.setup(cfg) + dataloader = adapter.get_dataloader() + + def _build_train_env(batch: BatchSpec): + env_manager = adapter.build_env_from_batch(batch, out_root=out_root) + return env_manager, batch.batch_size, batch.seed + + def _build_eval_env(split: str, env_num: int, seed: int): + if dataloader is None: + env_manager = adapter.build_eval_env( + env_num=env_num, + split=split, + seed=seed, + out_root=out_root, + ) + actual_n = len(env_manager) if hasattr(env_manager, "__len__") else env_num + return env_manager, actual_n + + batch = dataloader.build_eval_batch( + env_num=env_num, + split=split, + seed=seed, + out_root=out_root, + ) + env_manager = adapter.build_env_from_batch(batch, out_root=out_root) + return env_manager, batch.batch_size + + # ── Configure models ───────────────────────────────────────────── + backend = cfg.get("model_backend", "azure_openai") + configure_azure_openai( + endpoint=( + cfg.get("azure_openai_endpoint") + or cfg.get("azure_endpoint") + or None + ), + api_version=( + cfg.get("azure_openai_api_version") + or cfg.get("azure_api_version") + or None + ), + api_key=( + cfg.get("azure_openai_api_key") + or cfg.get("azure_api_key") + or None + ), + auth_mode=cfg.get("azure_openai_auth_mode") or None, + ad_scope=cfg.get("azure_openai_ad_scope") or None, + managed_identity_client_id=cfg.get("azure_openai_managed_identity_client_id") or None, + optimizer_endpoint=cfg.get("optimizer_azure_openai_endpoint") or None, + optimizer_api_version=cfg.get("optimizer_azure_openai_api_version") or None, + optimizer_api_key=cfg.get("optimizer_azure_openai_api_key") or None, + optimizer_auth_mode=cfg.get("optimizer_azure_openai_auth_mode") or None, + optimizer_ad_scope=cfg.get("optimizer_azure_openai_ad_scope") or None, + optimizer_managed_identity_client_id=( + cfg.get("optimizer_azure_openai_managed_identity_client_id") or None + ), + target_endpoint=cfg.get("target_azure_openai_endpoint") or None, + target_api_version=cfg.get("target_azure_openai_api_version") or None, + target_api_key=cfg.get("target_azure_openai_api_key") or None, + target_auth_mode=cfg.get("target_azure_openai_auth_mode") or None, + target_ad_scope=cfg.get("target_azure_openai_ad_scope") or None, + target_managed_identity_client_id=( + cfg.get("target_azure_openai_managed_identity_client_id") or None + ), + ) + optimizer_backend = cfg.get("optimizer_backend") + target_backend = cfg.get("target_backend") + if not optimizer_backend or not target_backend: + if backend in {"claude", "claude_chat"}: + optimizer_backend = optimizer_backend or "claude_chat" + target_backend = target_backend or "claude_chat" + elif backend in {"codex", "codex_exec"}: + optimizer_backend = optimizer_backend or "openai_chat" + target_backend = target_backend or "codex_exec" + elif backend == "claude_code_exec": + optimizer_backend = optimizer_backend or "openai_chat" + target_backend = target_backend or "claude_code_exec" + elif backend in {"qwen", "qwen_chat"}: + optimizer_backend = optimizer_backend or "openai_chat" + target_backend = target_backend or "qwen_chat" + else: + optimizer_backend = optimizer_backend or "openai_chat" + target_backend = target_backend or "openai_chat" + cfg["optimizer_backend"] = optimizer_backend + cfg["target_backend"] = target_backend + set_optimizer_backend(optimizer_backend) + set_target_backend(target_backend) + set_optimizer_deployment(cfg["optimizer_model"]) + set_target_deployment(cfg["target_model"]) + configure_codex_exec( + path=cfg.get("codex_exec_path", "codex"), + sandbox=cfg.get("codex_exec_sandbox", "workspace-write"), + profile=cfg.get("codex_exec_profile", ""), + full_auto=cfg.get("codex_exec_full_auto", False), + reasoning_effort=cfg.get("codex_exec_reasoning_effort", "none"), + use_sdk=cfg.get("codex_exec_use_sdk", None), + network_access=cfg.get("codex_exec_network_access", False), + web_search=cfg.get("codex_exec_web_search", False), + approval_policy=cfg.get("codex_exec_approval_policy", "never"), + ) + configure_claude_code_exec( + path=cfg.get("claude_code_exec_path", "claude"), + profile=cfg.get("claude_code_exec_profile", ""), + use_sdk=cfg.get("claude_code_exec_use_sdk", None), + effort=cfg.get("claude_code_exec_effort", cfg.get("reasoning_effort", "medium")), + max_thinking_tokens=cfg.get("claude_code_exec_max_thinking_tokens", 16384), + ) + configure_qwen_chat( + base_url=cfg.get("qwen_chat_base_url") or None, + api_key=cfg.get("qwen_chat_api_key") or None, + temperature=cfg.get("qwen_chat_temperature"), + timeout_seconds=cfg.get("qwen_chat_timeout_seconds"), + max_tokens=cfg.get("qwen_chat_max_tokens"), + enable_thinking=cfg.get("qwen_chat_enable_thinking"), + optimizer_base_url=cfg.get("optimizer_qwen_chat_base_url") or None, + optimizer_api_key=cfg.get("optimizer_qwen_chat_api_key") or None, + optimizer_temperature=cfg.get("optimizer_qwen_chat_temperature"), + optimizer_timeout_seconds=cfg.get("optimizer_qwen_chat_timeout_seconds"), + optimizer_max_tokens=cfg.get("optimizer_qwen_chat_max_tokens"), + optimizer_enable_thinking=cfg.get("optimizer_qwen_chat_enable_thinking"), + target_base_url=cfg.get("target_qwen_chat_base_url") or None, + target_api_key=cfg.get("target_qwen_chat_api_key") or None, + target_temperature=cfg.get("target_qwen_chat_temperature"), + target_timeout_seconds=cfg.get("target_qwen_chat_timeout_seconds"), + target_max_tokens=cfg.get("target_qwen_chat_max_tokens"), + target_enable_thinking=cfg.get("target_qwen_chat_enable_thinking"), + ) + configure_minimax_chat( + base_url=cfg.get("minimax_base_url") or None, + api_key=cfg.get("minimax_api_key") or None, + temperature=cfg.get("minimax_temperature"), + max_tokens=cfg.get("minimax_max_tokens"), + enable_thinking=cfg.get("minimax_enable_thinking"), + ) + minimax_model_cfg = cfg.get("minimax_model") + if minimax_model_cfg and cfg.get("target_backend") == "minimax_chat": + set_target_deployment(str(minimax_model_cfg)) + os.environ["REFLACT_CODEX_TRACE_TO_OPTIMIZER"] = ( + "1" + if target_backend == "codex_exec" and cfg.get("codex_trace_to_optimizer", False) + else "0" + ) + reasoning = cfg.get("reasoning_effort", "") or None + set_reasoning_effort(reasoning) + print( + f" [model config] backend={backend} " + f"optimizer={cfg['optimizer_model']} ({optimizer_backend}) " + f"target={cfg['target_model']} ({target_backend}) " + f"reasoning={reasoning or 'off'}" + ) + + # ── Initialize Ray ─────────────────────────────────────────────── + if adapter.requires_ray(): + try: + import ray + except ImportError as e: + raise ImportError( + "This environment requires ray, but ray is not installed." + ) from e + + if not ray.is_initialized(): + ray.init(num_gpus=0) + + # ── Load initial skill ─────────────────────────────────────────── + skill_init_path = os.path.abspath(cfg["skill_init"]) + if os.path.exists(skill_init_path): + with open(skill_init_path) as f: + skill_init = f.read() + print(f" [initial skill] {skill_init_path} ({len(skill_init)} chars)") + else: + skill_init = "" + print(" [initial skill] no initial skill file — starting from blank") + + # ── Training parameters ────────────────────────────────────────── + batch_size = cfg["batch_size"] + num_epochs = cfg["num_epochs"] + accumulation = cfg["accumulation"] + seed = cfg["seed"] + merge_bs = cfg["merge_batch_size"] + max_analyst_rounds = int(cfg.get("max_analyst_rounds", 3) or 3) + update_mode = normalize_update_mode(cfg.get("skill_update_mode", "patch")) + lr_control_mode = _normalise_lr_control_mode(cfg.get("lr_control_mode", "fixed")) + if is_full_rewrite_minibatch_mode(update_mode): + lr_control_mode = "none" + longitudinal_pair_policy = _normalise_longitudinal_pair_policy( + cfg.get("longitudinal_pair_policy", "mixed") + ) + rewrite_reasoning_effort = cfg.get("rewrite_reasoning_effort", "high") + if rewrite_reasoning_effort == "": + rewrite_reasoning_effort = None + rewrite_max_completion_tokens = int(cfg.get("rewrite_max_completion_tokens", 64000)) + if batch_size <= 0: + raise ValueError(f"batch_size must be positive, got {batch_size}") + if accumulation <= 0: + raise ValueError(f"accumulation must be positive, got {accumulation}") + + train_size = _resolve_train_size(cfg, dataloader) + steps_per_epoch = math.ceil(train_size / (batch_size * accumulation)) + batches_per_epoch = steps_per_epoch * accumulation + total_steps = num_epochs * steps_per_epoch + + # Persist resolved derived fields so config.json / summary.json match + # the actual runtime recipe. + cfg["train_size"] = train_size + cfg["steps_per_epoch"] = steps_per_epoch + cfg["batches_per_epoch"] = batches_per_epoch + cfg["samples_per_epoch"] = train_size + cfg["skill_update_mode"] = update_mode + cfg["lr_control_mode"] = lr_control_mode + + # Save config after deriving runtime values. + with open(os.path.join(out_root, "config.json"), "w") as f: + json.dump(_redact_cfg(cfg), f, indent=2, ensure_ascii=False) + + train_pool_size = train_size + + scheduler = build_scheduler( + mode=cfg.get("lr_scheduler", "constant"), + max_lr=cfg["edit_budget"], + min_lr=cfg.get("min_edit_budget", 2), + total_steps=total_steps, + ) + + # Fixed training pool: base seeds (each seed = one deterministic batch) + if dataloader is not None: + base_seeds = dataloader.make_base_seeds( + steps_per_epoch=steps_per_epoch, + accumulation=accumulation, + seed=seed, + ) + else: + base_seeds = [seed + i + 1 for i in range(batches_per_epoch)] + + print(f"\n [config] epochs={num_epochs} steps/epoch={steps_per_epoch} " + f"(auto) accum={accumulation} batch_size={batch_size}") + print(f" [config] train_size={train_size}") + print(f" [config] batches/epoch={batches_per_epoch} " + f"total_steps={total_steps} " + f"games/epoch={train_pool_size}") + print(f" [config] lr_scheduler={cfg.get('lr_scheduler', 'constant')} " + f"edit_budget={cfg['edit_budget']} " + f"min_edit_budget={cfg.get('min_edit_budget', 2)}") + print(f" [config] skill_update_mode={update_mode} " + f"lr_control_mode={lr_control_mode} " + f"rewrite_reasoning_effort={rewrite_reasoning_effort or 'off'} " + f"rewrite_max_completion_tokens={rewrite_max_completion_tokens} " + f"max_analyst_rounds={max_analyst_rounds}") + print(f" [config] longitudinal_pair_policy={longitudinal_pair_policy}") + print(f" [config] base_seeds={base_seeds}") + + # ── Resume check ───────────────────────────────────────────────── + history = _load_history(out_root) + runtime_state = _load_runtime_state(out_root) + if runtime_state: + last_step = int(runtime_state.get("last_completed_step", 0) or 0) + current_skill_path = runtime_state.get("current_skill_path") or os.path.join( + out_root, "skills", f"skill_v{last_step:04d}.md", + ) + with open(current_skill_path) as f: + current_skill = f.read() + best_skill_path = runtime_state.get("best_skill_path") or os.path.join( + out_root, "best_skill.md", + ) + if os.path.exists(best_skill_path): + with open(best_skill_path) as f: + best_skill = f.read() + else: + best_skill = current_skill + current_score = float(runtime_state.get("current_score", -1.0) or -1.0) + best_score = float(runtime_state.get("best_score", current_score) or current_score) + best_step = runtime_state.get("best_step", last_step) + current_origin = str( + runtime_state.get("current_origin") + or (f"step_{last_step:04d}" if last_step > 0 else "initial_skill") + ) + best_origin = str(runtime_state.get("best_origin") or current_origin) + resume_from = last_step + 1 + scheduler.load_state_dict({"current_step": last_step}) + print( + f" [resume] from step {resume_from} " + f"current={current_score:.4f} best={best_score:.4f} " + f"(origin={current_origin})" + ) + elif history: + last_step = history[-1]["step"] + current_skill = _load_skill(out_root, last_step) + best_rec = max(history, key=lambda h: h.get("best_score", 0.0)) + best_score = best_rec["best_score"] + best_step = best_rec["best_step"] + best_skill_path = os.path.join(out_root, "best_skill.md") + if os.path.exists(best_skill_path): + with open(best_skill_path) as f: + best_skill = f.read() + else: + best_skill = _load_skill(out_root, best_step) + current_score = history[-1].get("current_score", best_score) + current_origin = f"step_{last_step:04d}" + best_origin = f"step_{int(best_step):04d}" if isinstance(best_step, int) else str(best_step) + resume_from = last_step + 1 + scheduler.load_state_dict({"current_step": last_step}) + print( + f" [resume] from step {resume_from} " + f"current={current_score:.4f} best={best_score:.4f}" + ) + else: + current_skill = skill_init + best_skill = skill_init + best_score = -1.0 + current_score = -1.0 + best_step = 0 + current_origin = "initial_skill" + best_origin = "initial_skill" + resume_from = 1 + + _save_skill(out_root, 0, skill_init) + + def _persist_runtime_state(last_completed_step: int) -> None: + _save_runtime_state( + out_root, + { + "last_completed_step": last_completed_step, + "current_skill_path": os.path.join( + out_root, "skills", f"skill_v{last_completed_step:04d}.md", + ), + "current_score": current_score, + "current_origin": current_origin, + "best_skill_path": os.path.join(out_root, "best_skill.md"), + "best_score": best_score, + "best_step": best_step, + "best_origin": best_origin, + }, + ) + + # ── Selection cache ────────────────────────────────────────────── + sel_cache: dict[str, tuple[float, float]] = {} + for rec in history: + sh = rec.get("candidate_hash", "") + if sh and rec.get("selection_hard") is not None: + sel_cache[sh] = (rec["selection_hard"], rec["selection_soft"]) + + # ── Baseline evaluation on selection set ───────────────────────── + if cfg.get("use_gate") is False: + raise ValueError( + "Gate validation is mandatory in this branch. Remove " + "`evaluation.use_gate=false` from the config." + ) + gate_metric = str(cfg.get("gate_metric", "hard")).strip().lower() + if gate_metric not in {"hard", "soft", "mixed"}: + raise ValueError( + f"evaluation.gate_metric must be 'hard' | 'soft' | 'mixed', " + f"got {gate_metric!r}" + ) + gate_mixed_weight = float(cfg.get("gate_mixed_weight", 0.5)) + if not 0.0 <= gate_mixed_weight <= 1.0: + raise ValueError( + f"evaluation.gate_mixed_weight must be in [0, 1], " + f"got {gate_mixed_weight}" + ) + print( + f" [gate] metric={gate_metric}" + + ( + f" mixed_weight={gate_mixed_weight}" + if gate_metric == "mixed" + else "" + ) + ) + slow_gate_with_selection = bool( + cfg.get("slow_update_gate_with_selection", False) + ) + print( + " [slow update] acceptance=" + + ("gated (selection-set validation)" + if slow_gate_with_selection + else "force-accept (unconditional)") + ) + if current_score < 0: + print(f"\n{'='*60}") + print(" BASELINE — evaluate initial skill on Selection set (valid_seen)") + print(f"{'='*60}") + sel_env, sel_n = _build_eval_env( + split="valid_seen", + env_num=cfg["sel_env_num"], + seed=seed, + ) + print(f" Selection items: {sel_n}") + baseline_dir = os.path.join(out_root, "selection_eval_baseline") + baseline_results = adapter.rollout(sel_env, skill_init, baseline_dir) + baseline_hard, baseline_soft = compute_score(baseline_results) + current_score = select_gate_score( + baseline_hard, baseline_soft, gate_metric, gate_mixed_weight, + ) + best_score = current_score + sh = skill_hash(skill_init) + sel_cache[sh] = (baseline_hard, baseline_soft) + current_origin = "initial_skill" + best_origin = "initial_skill" + _persist_runtime_state(0) + print( + f" [baseline result] selection hard={baseline_hard:.4f} " + f"soft={baseline_soft:.4f} " + f"gate[{gate_metric}]={current_score:.4f}" + ) + + # ── Training loop ──────────────────────────────────────────────── + t_loop_start = time.time() + + if resume_from > total_steps: + print(f"\n [skip] all {total_steps} steps complete — jumping to evaluation") + + global_step = 0 + for epoch in range(1, num_epochs + 1): + if dataloader is not None: + epoch_batches = dataloader.plan_train_epoch( + epoch=epoch, + steps_per_epoch=steps_per_epoch, + accumulation=accumulation, + batch_size=batch_size, + seed=seed, + out_root=out_root, + ) + shuffled_seeds = [batch.seed for batch in epoch_batches] + else: + epoch_batches = [] + epoch_rng = random.Random(seed + epoch * 1000) + shuffled_seeds = base_seeds.copy() + epoch_rng.shuffle(shuffled_seeds) + + # Step buffer: accumulates per-step context (failure patterns + + # rejected edits) within this epoch so optimizers see full history. + step_buffer: list[dict] = [] + active_meta_skill = ( + _load_meta_skill_content(out_root, epoch - 1) + if cfg.get("use_meta_skill", False) + else "" + ) + + print( + f"\n [EPOCH {epoch}/{num_epochs}] " + f"shuffled_seeds={shuffled_seeds}" + ) + if active_meta_skill: + print( + f" [meta skill] loaded from epoch {epoch - 1} " + f"({len(active_meta_skill)} chars)" + ) + + for step_in_epoch in range(steps_per_epoch): + global_step += 1 + if global_step < resume_from: + continue + + step_t0 = time.time() + step_dir = os.path.join(out_root, "steps", f"step_{global_step:04d}") + os.makedirs(step_dir, exist_ok=True) + + tokens_before = get_token_summary() + + print( + f"\n [STEP {global_step}/{total_steps}] " + f"epoch={epoch} step_in_epoch={step_in_epoch} " + f"{'='*30}" + ) + + step_rec: dict = { + "step": global_step, + "epoch": epoch, + "step_in_epoch": step_in_epoch, + "timing": {}, + "tokens": {}, + } + + # ── Accumulation: Rollout + Reflect ────────────────────── + all_failure_patches: list[dict] = [] + all_success_patches: list[dict] = [] + all_raw_patches: list[dict | None] = [] + all_rollout_results: list[dict] = [] + accum_rollout_stats: list[dict] = [] + total_rollout_time = 0.0 + total_reflect_time = 0.0 + + for a in range(accumulation): + batch_idx = step_in_epoch * accumulation + a + if dataloader is not None: + batch_spec = epoch_batches[batch_idx] + train_env, train_n, batch_seed = _build_train_env(batch_spec) + else: + batch_seed = shuffled_seeds[batch_idx] + train_env = adapter.build_train_env( + batch_size=batch_size, + seed=batch_seed, + out_root=out_root, + ) + train_n = len(train_env) if hasattr(train_env, "__len__") else batch_size + + # Directory routing + if accumulation > 1: + batch_dir = os.path.join(step_dir, f"batch_{a}") + else: + batch_dir = step_dir + + rollout_dir = os.path.join(batch_dir, "rollout") + patches_dir = os.path.join(batch_dir, "patches") + + # ① ROLLOUT ──────────────────────────────────────────── + t_phase = time.time() + print(f" [1/6 ROLLOUT] train items={train_n} (from pool, batch_seed={batch_seed})") + rollout_results = adapter.rollout( + train_env, current_skill, rollout_dir, + use_eval_feedback=True, + ) + r_hard, r_soft = compute_score(rollout_results) + total_rollout_time += time.time() - t_phase + all_rollout_results.extend(rollout_results) + print(f" [1/6 done] hard={r_hard:.4f} soft={r_soft:.4f}") + + # ② REFLECT ──────────────────────────────────────────── + t_phase = time.time() + pred_dir = os.path.join(rollout_dir, "predictions") + + # Build step context from buffer + step_buffer_context = _format_step_buffer(step_buffer) + + raw_patches = adapter.reflect( + rollout_results, current_skill, batch_dir, + prediction_dir=pred_dir, patches_dir=patches_dir, + random_seed=batch_seed, + step_buffer_context=step_buffer_context, + meta_skill_context=active_meta_skill, + ) + failure_patches, success_patches = _normalise_patches( + raw_patches, + update_mode=update_mode, + ) + all_failure_patches.extend(failure_patches) + all_success_patches.extend(success_patches) + all_raw_patches.extend(raw_patches) + total_reflect_time += time.time() - t_phase + + print( + f" [2/6 done] failure_patches={len(failure_patches)} " + f"success_patches={len(success_patches)}" + ) + + # Track per-batch stats + accum_rollout_stats.append({ + "batch_idx": a, + "batch_seed": batch_seed, + "n_envs": len(rollout_results), + "hard": r_hard, + "soft": r_soft, + "n_failure_patches": len(failure_patches), + "n_success_patches": len(success_patches), + }) + + # ── End of accumulation loop ───────────────────────────── + + # Aggregate rollout stats across batches + total_n = sum(b["n_envs"] for b in accum_rollout_stats) + agg_hard = sum(b["hard"] * b["n_envs"] for b in accum_rollout_stats) / max(total_n, 1) + agg_soft = sum(b["soft"] * b["n_envs"] for b in accum_rollout_stats) / max(total_n, 1) + + step_rec["rollout_hard"] = round(agg_hard, 6) + step_rec["rollout_soft"] = round(agg_soft, 6) + step_rec["rollout_n"] = total_n + step_rec["accumulation_batches"] = accum_rollout_stats + step_rec["timing"]["rollout_s"] = round(total_rollout_time, 1) + step_rec["timing"]["reflect_s"] = round(total_reflect_time, 1) + + n_total_patches = len(all_failure_patches) + len(all_success_patches) + step_rec["n_patches"] = n_total_patches + step_rec["n_failure_patches"] = len(all_failure_patches) + step_rec["n_success_patches"] = len(all_success_patches) + + if accumulation > 1: + print( + f" [accum done] total: failure={len(all_failure_patches)} " + f"success={len(all_success_patches)} " + f"from {accumulation} batches" + ) + + # ── No patches? Skip ───────────────────────────────────── + if not all_failure_patches and not all_success_patches: + step_rec["action"] = "skip_no_patches" + step_rec["current_score"] = current_score + step_rec["best_score"] = best_score + step_rec["best_step"] = best_step + step_rec["skill_len"] = len(current_skill) + step_rec["wall_time_s"] = round(time.time() - step_t0, 1) + history.append(step_rec) + _save_history(out_root, history) + _save_skill(out_root, global_step, current_skill) + _persist_runtime_state(global_step) + with open(os.path.join(step_dir, "step_record.json"), "w") as f: + json.dump(step_rec, f, indent=2, ensure_ascii=False) + print(" [skip] no usable patches — skill unchanged") + continue + + # ③ AGGREGATE ────────────────────────────────────────────── + t_phase = time.time() + merged_patch = merge_patches( + current_skill, all_failure_patches, all_success_patches, + batch_size=merge_bs, verbose=True, + workers=cfg["analyst_workers"], + update_mode=update_mode, + meta_skill_context=active_meta_skill, + ) + with open(os.path.join(step_dir, "merged_patch.json"), "w") as f: + json.dump(merged_patch, f, ensure_ascii=False, indent=2) + + merged_items = get_payload_items(merged_patch, update_mode) + n_edits_merged = len(merged_items) + step_rec["n_edits_merged"] = n_edits_merged + step_rec["timing"]["aggregate_s"] = round(time.time() - t_phase, 1) + print(f" [3/6 done] merged {n_edits_merged} {payload_label(update_mode)}") + + # ④ SELECT ───────────────────────────────────────────────── + t_phase = time.time() + lr_decision = None + if is_full_rewrite_minibatch_mode(update_mode): + edit_budget = None + ranked_patch = merged_patch + ranked_items = merged_items + n_edits_ranked = len(ranked_items) + step_rec["n_edits_ranked"] = n_edits_ranked + step_rec["edit_budget"] = None + step_rec["lr_control_mode"] = "none" + with open(os.path.join(step_dir, "ranked_edits.json"), "w") as f: + json.dump(ranked_patch, f, ensure_ascii=False, indent=2) + else: + if lr_control_mode == "autonomous": + lr_decision = decide_autonomous_learning_rate( + skill_content=current_skill, + merged_patch=merged_patch, + update_mode=update_mode, + rollout_hard=agg_hard, + rollout_soft=agg_soft, + rollout_n=total_n, + step_buffer_context=step_buffer_context, + meta_skill_context=active_meta_skill, + ) + edit_budget = int(lr_decision["learning_rate"]) + with open(os.path.join(step_dir, "lr_decision.json"), "w") as f: + json.dump(lr_decision, f, ensure_ascii=False, indent=2) + with open(os.path.join(out_root, "lr_history.jsonl"), "a") as f: + f.write(json.dumps({ + "step": global_step, + "epoch": epoch, + **lr_decision, + }, ensure_ascii=False) + "\n") + else: + edit_budget = scheduler.step() + ranked_patch = rank_and_select( + current_skill, merged_patch, + max_edits=edit_budget, + update_mode=update_mode, + meta_skill_context=active_meta_skill, + ) + with open(os.path.join(step_dir, "ranked_edits.json"), "w") as f: + json.dump(ranked_patch, f, ensure_ascii=False, indent=2) + + ranked_items = get_payload_items(ranked_patch, update_mode) + n_edits_ranked = len(ranked_items) + step_rec["n_edits_ranked"] = n_edits_ranked + step_rec["edit_budget"] = edit_budget + step_rec["lr_control_mode"] = lr_control_mode + if lr_decision is not None: + step_rec["lr_decision"] = lr_decision + step_rec["timing"]["select_s"] = round(time.time() - t_phase, 1) + + support_counts = [ + item.get("support_count", 0) for item in ranked_items if isinstance(item, dict) + ] + step_rec["support_counts"] = support_counts + if is_full_rewrite_minibatch_mode(update_mode): + print( + f" [4/6 SELECT] skipped LR/select; " + f"using {n_edits_ranked} merged {payload_label(update_mode)}" + ) + else: + print( + f" [4/6 SELECT] " + f"{n_edits_merged} -> {n_edits_ranked} {payload_label(update_mode)} " + f"(budget={edit_budget}, lr_control={lr_control_mode})" + ) + + # ⑤ UPDATE ───────────────────────────────────────────────── + t_phase = time.time() + rewrite_result = None + if update_mode == "rewrite_from_suggestions": + rewrite_result = rewrite_skill_from_suggestions( + current_skill, + ranked_patch, + step_buffer_context=step_buffer_context, + env=cfg.get("env"), + reasoning_effort=rewrite_reasoning_effort, + max_completion_tokens=rewrite_max_completion_tokens, + ) + if rewrite_result and rewrite_result.get("new_skill"): + candidate_skill = rewrite_result["new_skill"] + apply_report = [] + with open(os.path.join(step_dir, "rewrite_result.json"), "w") as f: + json.dump(rewrite_result, f, ensure_ascii=False, indent=2) + else: + candidate_skill = current_skill + apply_report = [] + elif is_full_rewrite_minibatch_mode(update_mode): + skill_candidates = get_payload_items(ranked_patch, update_mode) + selected_candidate = next( + ( + item for item in skill_candidates + if isinstance(item, dict) and str(item.get("new_skill", "")).strip() + ), + None, + ) + if selected_candidate: + candidate_skill = str(selected_candidate["new_skill"]).rstrip() + "\n" + apply_report = [] + rewrite_result = { + "reasoning": ranked_patch.get("reasoning", ""), + "change_summary": selected_candidate.get("change_summary", []), + "title": selected_candidate.get("title", ""), + "source_type": selected_candidate.get("source_type", ""), + } + with open(os.path.join(step_dir, "full_rewrite_result.json"), "w") as f: + json.dump( + { + "selected_candidate": selected_candidate, + "merged_patch": ranked_patch, + }, + f, + ensure_ascii=False, + indent=2, + ) + else: + candidate_skill = current_skill + apply_report = [] + else: + candidate_skill, apply_report = apply_patch_with_report(current_skill, ranked_patch) + with open(os.path.join(step_dir, "candidate_skill.md"), "w") as f: + f.write(candidate_skill) + if apply_report: + with open(os.path.join(step_dir, "edit_apply_report.json"), "w") as f: + json.dump(apply_report, f, indent=2, ensure_ascii=False) + + cand_hash = skill_hash(candidate_skill) + step_rec["candidate_hash"] = cand_hash + step_rec["candidate_skill_len"] = len(candidate_skill) + if rewrite_result: + step_rec["rewrite_change_summary"] = rewrite_result.get("change_summary", []) + if apply_report: + step_rec["edit_apply_summary"] = { + "total": len(apply_report), + "applied": sum( + 1 for row in apply_report if str(row.get("status", "")).startswith("applied") + ), + "skipped": sum( + 1 for row in apply_report if str(row.get("status", "")).startswith("skipped") + ), + "errors": sum( + 1 for row in apply_report if row.get("status") == "error" + ), + } + step_rec["timing"]["update_s"] = round(time.time() - t_phase, 1) + if ( + update_mode == "rewrite_from_suggestions" + and rewrite_result is None + ) or ( + is_full_rewrite_minibatch_mode(update_mode) + and rewrite_result is None + ): + step_rec["action"] = "skip_no_rewrite" + step_rec["current_score"] = current_score + step_rec["best_score"] = best_score + step_rec["best_step"] = best_step + step_rec["skill_len"] = len(current_skill) + step_rec["wall_time_s"] = round(time.time() - step_t0, 1) + history.append(step_rec) + _save_history(out_root, history) + _save_skill(out_root, global_step, current_skill) + _persist_runtime_state(global_step) + with open(os.path.join(step_dir, "step_record.json"), "w") as f: + json.dump(step_rec, f, indent=2, ensure_ascii=False) + print(" [skip] no usable rewrite generated — skill unchanged") + continue + print( + f" [5/6 UPDATE] " + f"skill_len {len(current_skill)} -> {len(candidate_skill)}" + ) + + # ⑥ EVALUATE ─────────────────────────────────────────────── + t_phase = time.time() + if cand_hash in sel_cache: + cand_hard, cand_soft = sel_cache[cand_hash] + print( + f" [6/6 EVALUATE] " + f"cache hit {cand_hash}: hard={cand_hard:.4f}" + ) + else: + sel_env, sel_n = _build_eval_env( + split="valid_seen", + env_num=cfg["sel_env_num"], + seed=seed, + ) + print(f" [6/6 EVALUATE] selection items={sel_n}") + sel_eval_dir = os.path.join(step_dir, "selection_eval") + sel_results = adapter.rollout(sel_env, candidate_skill, sel_eval_dir) + cand_hard, cand_soft = compute_score(sel_results) + sel_cache[cand_hash] = (cand_hard, cand_soft) + + step_rec["selection_hard"] = cand_hard + step_rec["selection_soft"] = cand_soft + + gate = evaluate_gate( + candidate_skill=candidate_skill, + cand_hard=cand_hard, + current_skill=current_skill, + current_score=current_score, + best_skill=best_skill, + best_score=best_score, + best_step=best_step, + global_step=global_step, + cand_soft=cand_soft, + metric=gate_metric, + mixed_weight=gate_mixed_weight, + ) + cand_gate_score = select_gate_score( + cand_hard, cand_soft, gate_metric, gate_mixed_weight, + ) + step_rec["gate_metric"] = gate_metric + step_rec["candidate_gate_score"] = cand_gate_score + step_rec["action"] = gate.action + prev_current = current_score + prev_best = best_score + current_skill = gate.current_skill + current_score = gate.current_score + best_skill = gate.best_skill + best_score = gate.best_score + best_step = gate.best_step + if gate.action in {"accept", "accept_new_best"}: + current_origin = f"step_{global_step:04d}" + if gate.action == "accept_new_best": + best_origin = current_origin + + if gate_metric == "hard": + score_label = f"hard={cand_hard:.4f}" + elif gate_metric == "soft": + score_label = f"soft={cand_soft:.4f}" + else: + score_label = ( + f"mixed[w={gate_mixed_weight}]={cand_gate_score:.4f} " + f"(hard={cand_hard:.4f} soft={cand_soft:.4f})" + ) + if gate.action == "accept_new_best": + print( + f" [6/6 EVALUATE] ACCEPT (new best) " + f"{score_label} > prev best {prev_best:.4f}" + ) + elif gate.action == "accept": + print( + f" [6/6 EVALUATE] ACCEPT " + f"{score_label} > current={prev_current:.4f}" + ) + else: + print( + f" [6/6 EVALUATE] REJECT " + f"{score_label} <= current={current_score:.4f}" + ) + + step_rec["timing"]["evaluate_s"] = round(time.time() - t_phase, 1) + + # ── Step buffer: unified failure patterns + rejected edits ─ + action = step_rec.get("action", "unknown") + n_total = len(all_rollout_results) or 1 + n_fail = sum(1 for r in all_rollout_results if not r.get("hard") or float(r.get("hard", 0)) < 1e-9) + failure_patterns = _extract_failure_patterns( + all_rollout_results, step_dir, + ) + + buf_entry: dict = { + "step": global_step, + "action": action, + "n_total": n_total, + "n_fail": n_fail, + "failure_patterns": failure_patterns, + } + + # Attach rejected edits when the step was rejected + if "reject" in action and ranked_patch: + rejected_edits = [ + short_item_summary(item, update_mode) + for item in ranked_items + if isinstance(item, dict) + ] + buf_entry["score_before"] = current_score + buf_entry["score_after"] = cand_gate_score + buf_entry["rejected_edits"] = rejected_edits + + step_buffer.append(buf_entry) + + # Persist step digest for step buffer context + digest_path = os.path.join(step_dir, "trajectory_digest.json") + with open(digest_path, "w") as f: + json.dump(buf_entry, f, indent=2, ensure_ascii=False) + + # ── Token snapshot ─────────────────────────────────────── + tokens_after = get_token_summary() + step_tokens: dict = {} + for stage in tokens_after: + if stage == "_total": + continue + after = tokens_after[stage] + before = tokens_before.get(stage, {}) + step_tokens[stage] = { + "calls": after.get("calls", 0) - before.get("calls", 0), + "prompt_tokens": after.get("prompt_tokens", 0) + - before.get("prompt_tokens", 0), + "completion_tokens": after.get("completion_tokens", 0) + - before.get("completion_tokens", 0), + } + step_rec["tokens"] = step_tokens + + # ── Save state ─────────────────────────────────────────── + step_rec["current_score"] = current_score + step_rec["best_score"] = best_score + step_rec["best_step"] = best_step + step_rec["current_origin"] = current_origin + step_rec["best_origin"] = best_origin + step_rec["skill_len"] = len(current_skill) + step_rec["wall_time_s"] = round(time.time() - step_t0, 1) + + _save_skill(out_root, global_step, current_skill) + with open(os.path.join(out_root, "best_skill.md"), "w") as f: + f.write(best_skill) + history.append(step_rec) + _save_history(out_root, history) + _persist_runtime_state(global_step) + with open(os.path.join(step_dir, "step_record.json"), "w") as f: + json.dump(step_rec, f, indent=2, ensure_ascii=False) + + timing = step_rec["timing"] + print( + f"\n [STEP {global_step} done] " + f"epoch={epoch} action={step_rec['action']} " + f"current={current_score:.4f} best={best_score:.4f} " + f"dt={step_rec['wall_time_s']}s\n" + f" timing: rollout={timing.get('rollout_s',0)}s " + f"reflect={timing.get('reflect_s',0)}s " + f"aggregate={timing.get('aggregate_s',0)}s " + f"select={timing.get('select_s',0)}s " + f"evaluate={timing.get('evaluate_s',0)}s" + ) + + epoch_last_step_skill = current_skill + epoch_comparison_pairs: list[dict] | None = None + + # ── SLOW UPDATE (end of epoch) ────────────────────────────── + use_slow = cfg.get("use_slow_update", False) + if use_slow: + slow_dir = os.path.join(out_root, "slow_update", f"epoch_{epoch:02d}") + slow_done_path = os.path.join(slow_dir, "slow_result.json") + + if os.path.exists(slow_done_path): + # Resume support + print( + f"\n [SLOW UPDATE epoch {epoch}] " + f"resumed — already done" + ) + with open(slow_done_path) as f: + slow_saved = json.load(f) + comparison_path = os.path.join(slow_dir, "comparison_pairs.json") + if os.path.exists(comparison_path): + try: + with open(comparison_path) as f: + epoch_comparison_pairs = json.load(f) + except Exception: + epoch_comparison_pairs = None + if ( + slow_saved.get("slow_update_content") + and epoch >= 2 + ): + action = slow_saved.get("action") + if slow_gate_with_selection: + # Gated mode (follow SkillReflection): re-apply the + # guidance to current_skill only when it was accepted. + if action in {"accept", "accept_new_best"}: + current_skill = replace_slow_update_field( + current_skill, + slow_saved["slow_update_content"], + ) + elif action in { + "accept", "accept_new_best", "force_accept", + }: + # Force-accept mode: re-apply to both current & best. + current_skill = replace_slow_update_field( + current_skill, slow_saved["slow_update_content"], + ) + best_skill = replace_slow_update_field( + best_skill, slow_saved["slow_update_content"], + ) + elif epoch == 1: + # Epoch 1: inject empty placeholder + os.makedirs(slow_dir, exist_ok=True) + current_skill = inject_empty_slow_update_field(current_skill) + current_origin = f"slow_update_placeholder_epoch_{epoch:02d}" + _save_skill(out_root, global_step, current_skill) + with open(os.path.join(out_root, "best_skill.md"), "w") as f: + f.write(best_skill if best_score > current_score else current_skill) + with open(slow_done_path, "w") as f: + json.dump({"action": "inject_placeholder", "epoch": epoch}, f, indent=2) + _persist_runtime_state(global_step) + print( + f"\n [SLOW UPDATE epoch {epoch}] " + f"injected empty placeholder" + ) + else: + # Epoch 2+: longitudinal comparison + os.makedirs(slow_dir, exist_ok=True) + print( + f"\n {'='*60}\n" + f" SLOW UPDATE — Epoch {epoch} " + f"(comparing epoch {epoch-1} vs {epoch})\n" + f" {'='*60}" + ) + + # 1. Get skill from last step of previous epoch + prev_epoch_records = [ + h for h in history if h.get("epoch") == epoch - 1 + ] + prev_epoch_last_step = prev_epoch_records[-1]["step"] + prev_skill = _load_skill(out_root, prev_epoch_last_step) + + # 2. Sample items from train set + slow_n = cfg.get("slow_update_samples", 20) + slow_seed = seed + epoch * 2000 + if dataloader is not None: + slow_batch = dataloader.build_train_batch( + batch_size=slow_n, + seed=slow_seed, + out_root=out_root, + ) + slow_env = adapter.build_env_from_batch( + slow_batch, out_root=out_root, + ) + else: + slow_env = adapter.build_train_env( + batch_size=slow_n, + seed=slow_seed, + out_root=out_root, + ) + slow_items = list(slow_env) if hasattr(slow_env, "__iter__") else slow_env + print(f" [slow update] sampled {len(slow_items)} train items (seed={slow_seed})") + + # 3. Rollout with both skills + t_slow = time.time() + prev_rollout_dir = os.path.join(slow_dir, "rollout_prev") + curr_rollout_dir = os.path.join(slow_dir, "rollout_curr") + results_prev = adapter.rollout(slow_env, prev_skill, prev_rollout_dir) + results_curr = adapter.rollout(slow_env, current_skill, curr_rollout_dir) + + prev_hard, _ = compute_score(results_prev) + curr_hard, _ = compute_score(results_curr) + print( + f" [slow update] prev epoch hard={prev_hard:.4f} " + f"curr epoch hard={curr_hard:.4f}" + ) + + # 4. Build and save structured comparison pairs + comparison_pairs, all_comparison_pairs = _build_longitudinal_pairs( + adapter=adapter, + dataloader=dataloader, + prev_skill=prev_skill, + curr_skill=current_skill, + initial_items=slow_items, + initial_prev_results=results_prev, + initial_curr_results=results_curr, + prev_rollout_dir=prev_rollout_dir, + curr_rollout_dir=curr_rollout_dir, + policy=longitudinal_pair_policy, + target_n=slow_n, + seed=slow_seed, + out_root=out_root, + ) + epoch_comparison_pairs = comparison_pairs + if all_comparison_pairs is not comparison_pairs: + save_comparison_pairs( + all_comparison_pairs, + os.path.join(slow_dir, "comparison_pairs_all.json"), + ) + save_comparison_pairs( + comparison_pairs, + os.path.join(slow_dir, "comparison_pairs.json"), + ) + n_regressed = sum(1 for p in comparison_pairs if p["category"] == "regressed") + n_improved = sum(1 for p in comparison_pairs if p["category"] == "improved") + n_persist = sum(1 for p in comparison_pairs if p["category"] == "persistent_fail") + n_stable = sum(1 for p in comparison_pairs if p["category"] == "stable_success") + print( + f" [slow update] comparison: " + f"regressed={n_regressed} improved={n_improved} " + f"persistent_fail={n_persist} stable_success={n_stable} " + f"policy={longitudinal_pair_policy} " + f"kept={len(comparison_pairs)}/{len(all_comparison_pairs)}" + ) + + # 5. Extract previous slow update guidance for reflection + existing_guidance = extract_slow_update_field(current_skill) + + # 6. Optimizer analysis (with reflection on previous guidance) + slow_result = run_slow_update( + current_skill, + results_prev, + results_curr, + slow_items, + prev_skill=prev_skill, + prev_slow_update_content=existing_guidance, + prev_rollout_dir=prev_rollout_dir, + curr_rollout_dir=curr_rollout_dir, + comparison_pairs=comparison_pairs, + ) + slow_time = round(time.time() - t_slow, 1) + + if slow_result and slow_result.get("slow_update_content"): + slow_candidate = replace_slow_update_field( + current_skill, slow_result["slow_update_content"], + ) + slow_candidate_hash = skill_hash(slow_candidate) + with open(os.path.join(slow_dir, "candidate_skill.md"), "w") as f: + f.write(slow_candidate) + slow_result["time_s"] = slow_time + slow_result["prev_hard"] = prev_hard + slow_result["curr_hard"] = curr_hard + slow_result["candidate_hash"] = slow_candidate_hash + slow_result["update_origin"] = "slow_update_momentum" + slow_result["update_target"] = ( + "Address longitudinal regressions and persistent failures " + "observed across adjacent epochs." + ) + + # Slow update acceptance — two modes selected via + # `optimizer.slow_update_gate_with_selection`. + if slow_gate_with_selection: + # ── Gated mode (follow SkillReflection) ────────── + # Evaluate the slow-update candidate on the + # selection set and accept/reject via the same + # validation gate used for step-level updates. + if slow_candidate_hash in sel_cache: + slow_sel_hard, slow_sel_soft = sel_cache[ + slow_candidate_hash + ] + print( + f" [slow gate] cache hit: " + f"hard={slow_sel_hard:.4f}" + ) + else: + sel_env, sel_n = _build_eval_env( + split="valid_seen", + env_num=cfg["sel_env_num"], + seed=seed, + ) + print(f" [slow gate] selection items={sel_n}") + slow_eval_dir = os.path.join( + slow_dir, "selection_eval", + ) + slow_eval_results = adapter.rollout( + sel_env, slow_candidate, slow_eval_dir, + ) + slow_sel_hard, slow_sel_soft = compute_score( + slow_eval_results + ) + sel_cache[slow_candidate_hash] = ( + slow_sel_hard, slow_sel_soft, + ) + + slow_gate = evaluate_gate( + candidate_skill=slow_candidate, + cand_hard=slow_sel_hard, + current_skill=current_skill, + current_score=current_score, + best_skill=best_skill, + best_score=best_score, + best_step=best_step, + global_step=global_step, + cand_soft=slow_sel_soft, + metric=gate_metric, + mixed_weight=gate_mixed_weight, + ) + slow_result["selection_hard"] = slow_sel_hard + slow_result["selection_soft"] = slow_sel_soft + slow_result["action"] = slow_gate.action + prev_current = current_score + prev_best = best_score + current_skill = slow_gate.current_skill + current_score = slow_gate.current_score + best_skill = slow_gate.best_skill + best_score = slow_gate.best_score + best_step = slow_gate.best_step + if slow_gate.action in {"accept", "accept_new_best"}: + current_origin = ( + f"slow_update_epoch_{epoch:02d}" + ) + if slow_gate.action == "accept_new_best": + best_origin = current_origin + print( + f" [slow gate] ACCEPT (new best) " + f"hard={slow_sel_hard:.4f} > " + f"prev best {prev_best:.4f}" + ) + elif slow_gate.action == "accept": + print( + f" [slow gate] ACCEPT " + f"hard={slow_sel_hard:.4f} > " + f"current={prev_current:.4f}" + ) + else: + print( + f" [slow gate] REJECT " + f"hard={slow_sel_hard:.4f} <= " + f"current={current_score:.4f}" + ) + print( + f" [slow update] guidance written " + f"({len(slow_result['slow_update_content'])} " + f"chars), {slow_time}s" + ) + else: + # ── Force-accept mode (default) ────────────────── + # The epoch-level longitudinal guidance is injected + # into both current_skill and best_skill + # unconditionally — it must not be gated by + # step-level selection scores. + slow_content = slow_result["slow_update_content"] + current_skill = replace_slow_update_field( + current_skill, slow_content, + ) + best_skill = replace_slow_update_field( + best_skill, slow_content, + ) + # Update caches so downstream steps use the + # slow-update-injected skill for hashing. + slow_candidate_hash = skill_hash(current_skill) + sel_cache[slow_candidate_hash] = (current_score, 0.0) + + slow_result["action"] = "force_accept" + current_origin = f"slow_update_epoch_{epoch:02d}" + + print( + f" [slow update] force-injected into " + f"current & best " + f"({len(slow_content)} chars), " + f"{slow_time}s" + ) + else: + slow_result = slow_result or {} + slow_result["action"] = "no_content" + slow_result["time_s"] = slow_time + print( + f" [slow update] no guidance produced, " + f"{slow_time}s" + ) + + # 5. Save + with open(slow_done_path, "w") as f: + json.dump(slow_result, f, indent=2, ensure_ascii=False) + _save_skill(out_root, global_step, current_skill) + with open(os.path.join(out_root, "best_skill.md"), "w") as f: + f.write(best_skill) + _persist_runtime_state(global_step) + + print( + f"\n [SLOW UPDATE epoch {epoch} done] " + f"current={current_score:.4f} best={best_score:.4f}" + ) + + # ── META SKILL (end of epoch, optimizer-side memory) ───────── + use_meta_skill = cfg.get("use_meta_skill", False) + if use_meta_skill: + meta_skill_dir = os.path.join(out_root, "meta_skill", f"epoch_{epoch:02d}") + meta_skill_done_path = os.path.join(meta_skill_dir, "meta_skill_result.json") + os.makedirs(meta_skill_dir, exist_ok=True) + + if os.path.exists(meta_skill_done_path): + print(f"\n [META SKILL epoch {epoch}] resumed — already done") + elif epoch == 1: + with open(meta_skill_done_path, "w") as f: + json.dump( + {"action": "skip_first_epoch", "epoch": epoch}, + f, indent=2, ensure_ascii=False, + ) + print(f"\n [META SKILL epoch {epoch}] skipped — first epoch") + else: + print( + f"\n {'='*60}\n" + f" META SKILL — Epoch {epoch} " + f"(optimizer memory from epoch {epoch-1} vs {epoch})\n" + f" {'='*60}" + ) + + prev_epoch_records = [h for h in history if h.get("epoch") == epoch - 1] + prev_epoch_last_step = prev_epoch_records[-1]["step"] + prev_skill = _load_skill(out_root, prev_epoch_last_step) + prev_meta_skill = _load_meta_skill_content(out_root, epoch - 1) + + if epoch_comparison_pairs is None: + meta_n = cfg.get("slow_update_samples", 20) + meta_seed = seed + epoch * 2000 + if dataloader is not None: + meta_batch = dataloader.build_train_batch( + batch_size=meta_n, + seed=meta_seed, + out_root=out_root, + ) + meta_env = adapter.build_env_from_batch( + meta_batch, out_root=out_root, + ) + else: + meta_env = adapter.build_train_env( + batch_size=meta_n, + seed=meta_seed, + out_root=out_root, + ) + meta_items = list(meta_env) if hasattr(meta_env, "__iter__") else meta_env + prev_rollout_dir = os.path.join(meta_skill_dir, "rollout_prev") + curr_rollout_dir = os.path.join(meta_skill_dir, "rollout_curr") + results_prev = adapter.rollout(meta_env, prev_skill, prev_rollout_dir) + results_curr = adapter.rollout(meta_env, epoch_last_step_skill, curr_rollout_dir) + epoch_comparison_pairs, all_meta_comparison_pairs = _build_longitudinal_pairs( + adapter=adapter, + dataloader=dataloader, + prev_skill=prev_skill, + curr_skill=epoch_last_step_skill, + initial_items=meta_items, + initial_prev_results=results_prev, + initial_curr_results=results_curr, + prev_rollout_dir=prev_rollout_dir, + curr_rollout_dir=curr_rollout_dir, + policy=longitudinal_pair_policy, + target_n=meta_n, + seed=meta_seed, + out_root=out_root, + ) + if all_meta_comparison_pairs is not epoch_comparison_pairs: + save_comparison_pairs( + all_meta_comparison_pairs, + os.path.join(meta_skill_dir, "comparison_pairs_all.json"), + ) + save_comparison_pairs( + epoch_comparison_pairs, + os.path.join(meta_skill_dir, "comparison_pairs.json"), + ) + meta_counts = _pair_category_counts(epoch_comparison_pairs) + print( + f" [meta skill] comparison: " + f"regressed={meta_counts.get('regressed', 0)} " + f"improved={meta_counts.get('improved', 0)} " + f"persistent_fail={meta_counts.get('persistent_fail', 0)} " + f"stable_success={meta_counts.get('stable_success', 0)} " + f"policy={longitudinal_pair_policy} " + f"kept={len(epoch_comparison_pairs)}/{len(all_meta_comparison_pairs)}" + ) + + t_meta_skill = time.time() + meta_skill_result = run_meta_skill( + prev_skill=prev_skill, + curr_skill=epoch_last_step_skill, + comparison_pairs=epoch_comparison_pairs or [], + prev_meta_skill_content=prev_meta_skill, + ) + meta_skill_time = round(time.time() - t_meta_skill, 1) + + if meta_skill_result and meta_skill_result.get("meta_skill_content"): + meta_skill_result["time_s"] = meta_skill_time + meta_skill_result["action"] = "write_meta_skill" + print( + f" [meta skill] memory written " + f"({len(meta_skill_result['meta_skill_content'])} chars), " + f"{meta_skill_time}s" + ) + else: + meta_skill_result = meta_skill_result or {} + meta_skill_result["time_s"] = meta_skill_time + meta_skill_result["action"] = "no_content" + print(f" [meta skill] no memory produced, {meta_skill_time}s") + + with open(meta_skill_done_path, "w") as f: + json.dump(meta_skill_result, f, indent=2, ensure_ascii=False) + + # ── Save best skill ────────────────────────────────────────────── + with open(os.path.join(out_root, "best_skill.md"), "w") as f: + f.write(best_skill) + _persist_runtime_state(global_step) + print( + f"\n [done] best skill from step {best_step}, " + f"score={best_score:.4f}" + ) + + # ── Final test evaluation (valid_unseen) ───────────────────────── + baseline_test_hard = None + baseline_test_soft = None + test_hard = None + test_soft = None + + if cfg["eval_test"]: + task_types = adapter.get_task_types() + + # Baseline: S_0 on test set (valid_unseen) + print(f"\n{'='*60}") + print(" BASELINE TEST — evaluate initial skill on Test set (valid_unseen)") + print(f"{'='*60}") + test_env, test_n = _build_eval_env( + split="valid_unseen", + env_num=cfg["test_env_num"], + seed=seed, + ) + print(f" Test items: {test_n}") + baseline_test_dir = os.path.join(out_root, "test_eval_baseline") + baseline_test_results = adapter.rollout(test_env, skill_init, baseline_test_dir) + baseline_test_hard, baseline_test_soft = compute_score(baseline_test_results) + baseline_buckets = _compute_task_type_buckets(baseline_test_results, task_types) + print("\n === Baseline Test Results (S_0) ===") + for task_type in task_types + ["overall"]: + b = baseline_buckets.get(task_type, {"total": 0, "hard": 0}) + t = max(b["total"], 1) + print( + f" {task_type:<40s}: " + f"hard={b['hard']}/{b['total']}={b['hard']/t:.4f}" + ) + with open(os.path.join(baseline_test_dir, "summary.json"), "w") as f: + json.dump( + { + k: { + "total": b["total"], + "hard_acc": b["hard"] / max(b["total"], 1), + } + for k, b in baseline_buckets.items() + }, + f, indent=2, ensure_ascii=False, + ) + + # Best skill on test set + print(f"\n{'='*60}") + print(" BEST SKILL TEST — evaluate best skill on Test set (valid_unseen)") + print(f"{'='*60}") + test_env2, test_n2 = _build_eval_env( + split="valid_unseen", + env_num=cfg["test_env_num"], + seed=seed, + ) + print(f" Test items: {test_n2}") + test_dir = os.path.join(out_root, "test_eval") + test_results = adapter.rollout(test_env2, best_skill, test_dir) + test_hard, test_soft = compute_score(test_results) + best_buckets = _compute_task_type_buckets(test_results, task_types) + print("\n === Best Skill Test Results ===") + for task_type in task_types + ["overall"]: + b = best_buckets.get(task_type, {"total": 0, "hard": 0}) + t = max(b["total"], 1) + print( + f" {task_type:<40s}: " + f"hard={b['hard']}/{b['total']}={b['hard']/t:.4f}" + ) + with open(os.path.join(test_dir, "summary.json"), "w") as f: + json.dump( + { + k: { + "total": b["total"], + "hard_acc": b["hard"] / max(b["total"], 1), + } + for k, b in best_buckets.items() + }, + f, indent=2, ensure_ascii=False, + ) + + # Comparison + delta_hard = (test_hard or 0) - (baseline_test_hard or 0) + print(f"\n === Improvement (best vs baseline) ===") + print( + f" hard: {baseline_test_hard:.4f} -> {test_hard:.4f} " + f"(delta={delta_hard:+.4f})" + ) + + # ── Global summary ─────────────────────────────────────────────── + total_wall = time.time() - t_loop_start + n_accept = sum(1 for h in history if "accept" in h.get("action", "")) + n_reject = sum(1 for h in history if h.get("action") == "reject") + n_skip = sum(1 for h in history if h.get("action") == "skip_no_patches") + + token_summary = get_token_summary() + + # Epoch-level statistics + epoch_stats = [] + for e in range(1, num_epochs + 1): + epoch_records = [h for h in history if h.get("epoch") == e] + if epoch_records: + epoch_stats.append({ + "epoch": e, + "steps": [h["step"] for h in epoch_records], + "accepts": sum(1 for h in epoch_records if "accept" in h.get("action", "")), + "rejects": sum(1 for h in epoch_records if h.get("action") == "reject"), + "skips": sum(1 for h in epoch_records if h.get("action") == "skip_no_patches"), + "best_score_at_epoch_end": epoch_records[-1].get("best_score", 0.0), + "current_score_at_epoch_end": epoch_records[-1].get("current_score", 0.0), + }) + + summary = { + "version": "skillopt-0.1.0", + "config": _redact_cfg(cfg), + "baseline_selection_hard": sel_cache.get( + skill_hash(skill_init), (None, None), + )[0], + "best_selection_hard": best_score, + "best_step": best_step, + "current_origin": current_origin, + "best_origin": best_origin, + "total_steps": len(history), + "total_accepts": n_accept, + "total_rejects": n_reject, + "total_skips": n_skip, + "epoch_stats": epoch_stats, + "baseline_test_hard": baseline_test_hard, + "baseline_test_soft": baseline_test_soft, + "test_hard": test_hard, + "test_soft": test_soft, + "test_delta_hard": ( + (test_hard or 0) - (baseline_test_hard or 0) + if test_hard is not None + else None + ), + "total_wall_time_s": round(total_wall, 1), + "token_summary": token_summary, + } + with open(os.path.join(out_root, "summary.json"), "w") as f: + json.dump(summary, f, indent=2, ensure_ascii=False) + + print(f"\n{'='*60}") + print(" Final Summary") + print(f"{'='*60}") + print( + f" steps={len(history)} accept={n_accept} " + f"reject={n_reject} skip={n_skip}" + ) + print(f" best_score={best_score:.4f} (step {best_step}) wall={total_wall:.0f}s") + if epoch_stats: + for es in epoch_stats: + print( + f" epoch {es['epoch']}: accept={es['accepts']} reject={es['rejects']} " + f"best={es['best_score_at_epoch_end']:.4f}" + ) + if test_hard is not None: + print(f" test_hard={test_hard:.4f} test_soft={test_soft:.4f}") + if token_summary.get("_total"): + t = token_summary["_total"] + print( + f" total tokens: {t['total_tokens']:,} " + f"(prompt={t['prompt_tokens']:,} " + f"completion={t['completion_tokens']:,} " + f"calls={t['calls']})" + ) + + return summary diff --git a/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/__init__.py b/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/__init__.py new file mode 100644 index 00000000..ecd0aaa0 --- /dev/null +++ b/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/__init__.py @@ -0,0 +1 @@ +"""ReflACT environment adapters.""" diff --git a/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/_template/README.md b/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/_template/README.md new file mode 100644 index 00000000..787efe24 --- /dev/null +++ b/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/_template/README.md @@ -0,0 +1,43 @@ +# Benchmark Template + +This directory provides scaffold files for adding a new benchmark to SkillOpt. + +## Files + +- `env_template.py` — Environment adapter template (subclasses + `EnvAdapter`; implements the 5 abstract methods so the file is + instantiable out of the box). +- `loader_template.py` — Data loader template (subclasses + `SplitDataLoader`; implements `load_split_items` for `.json`/`.jsonl`). +- `config_template.yaml` — Config file template. + +## Usage + +1. **Copy the directory:** + ```bash + cp -r skillopt/envs/_template skillopt/envs/your_benchmark + ``` +2. **Rename the files** (drop the `_template` suffix): + ```bash + cd skillopt/envs/your_benchmark + mv env_template.py adapter.py + mv loader_template.py loader.py + ``` + …and inside each file rename the classes + (`TemplateBenchmarkEnv → YourBenchmarkAdapter`, + `TemplateBenchmarkLoader → YourBenchmarkLoader`) + and fix the cross-import in `adapter.py`. +3. **Implement the TODO blocks** inside `adapter.py:rollout` and the + `_normalize_item` helper in `loader.py`. If you want real reflection, + uncomment the `run_minibatch_reflect` block in `adapter.py:reflect`. +4. **Register** the adapter — add a `try / except ImportError` block in + `scripts/train.py`'s `_register_builtins()` mapping the registry key + to your `YourBenchmarkAdapter` class. There is no + `BENCHMARK_REGISTRY` dict in `skillopt/envs/__init__.py`; the live + registry is `_ENV_REGISTRY` in `scripts/train.py`. +5. **Create the config** at `configs/your_benchmark/default.yaml` + (start from `config_template.yaml`). `_base_` is a **string path**, + not a list. + +See the [Add a New Benchmark guide](../../../docs/guide/new-benchmark.md) +for the full step-by-step with a worked `docfaithful` example. diff --git a/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/_template/config_template.yaml b/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/_template/config_template.yaml new file mode 100644 index 00000000..b482cc71 --- /dev/null +++ b/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/_template/config_template.yaml @@ -0,0 +1,55 @@ +# ────────────────────────────────────────────────── +# SkillOpt Config Template — +# ────────────────────────────────────────────────── +# Copy this file to configs//default.yaml +# and customize the values below. + +# Inherit global defaults. +# NOTE: `_base_` is a string path, not a list. +_base_: ../_base_/default.yaml + +# ── Environment ────────────────────────────────── +env: + name: your_benchmark # Must match the key registered in scripts/train.py + # Optional: a seed skill document. Create this file yourself before the + # first run, or omit the key to start from an empty skill. + # skill_init: skillopt/envs/your_benchmark/skills/initial.md + data_path: data/your_benchmark # Path to your data (for split_mode: ratio) + split_dir: "" # Set this and use split_mode: split_dir for pre-split data + split_mode: ratio # "ratio" or "split_dir" + split_ratio: "2:1:7" # train:val:test (used when split_mode: ratio) + workers: 4 # Parallel rollout workers + max_completion_tokens: 4096 # Cap per target-model call + limit: 0 # 0 = no limit; small int = debug sample + +# ── Training ───────────────────────────────────── +train: + num_epochs: 4 + batch_size: 40 + accumulation: 1 + seed: 42 + +# ── Gradient (Reflection) ─────────────────────── +gradient: + analyst_workers: 16 # Parallel reflection workers + minibatch_size: 8 + merge_batch_size: 8 + +# ── Optimizer ──────────────────────────────────── +optimizer: + learning_rate: 4 # Max edits per step (edit budget) + lr_scheduler: cosine # cosine | linear | constant | autonomous + use_slow_update: true # Epoch-boundary momentum + use_meta_skill: true # Cross-epoch optimizer memory + +# ── Evaluation ─────────────────────────────────── +evaluation: + use_gate: true # Validation gating + eval_test: true # Run test eval after training + +# ── Model ──────────────────────────────────────── +# Override only what differs from the inherited defaults. +model: + optimizer_backend: openai_chat # openai_chat | claude_chat | qwen_chat | minimax_chat + target_backend: openai_chat # … plus codex_exec / claude_code_exec for target only + reasoning_effort: medium diff --git a/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/_template/env_template.py b/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/_template/env_template.py new file mode 100644 index 00000000..63a70b19 --- /dev/null +++ b/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/_template/env_template.py @@ -0,0 +1,196 @@ +""" +Benchmark Environment Template +=============================== +Copy this file and implement the TODO sections to add a new benchmark. + +The EnvAdapter is responsible for: + 1. Building per-batch environment managers (train and eval splits). + 2. Running rollouts under the current skill document. + 3. Reflecting on those rollouts into raw patch dicts. + 4. Reporting the distinct task types in your data (for stratified + sampling). + +For a fully worked example see ``skillopt/envs/officeqa/``. +""" +from __future__ import annotations + +import os + +from skillopt.datasets.base import BatchSpec +from skillopt.envs.base import EnvAdapter +from skillopt.envs._template.loader_template import TemplateBenchmarkLoader +# When you wire in real reflection, also import: +# from skillopt.gradient.reflect import run_minibatch_reflect + + +class TemplateBenchmarkEnv(EnvAdapter): + """ + Environment adapter for . + + Rename this class. Each abstract method below is required by + :class:`skillopt.envs.base.EnvAdapter`. The template implementations + are minimal so this file is importable and instantiable; replace the + TODOs with real logic. + """ + + def __init__( + self, + split_dir: str = "", + data_path: str = "", + split_mode: str = "split_dir", + split_ratio: str = "2:1:7", + split_seed: int = 42, + split_output_dir: str = "", + workers: int = 4, + analyst_workers: int = 4, + failure_only: bool = False, + minibatch_size: int = 8, + edit_budget: int = 4, + seed: int = 42, + limit: int = 0, + max_completion_tokens: int = 4096, + ) -> None: + self.workers = workers + self.analyst_workers = analyst_workers + self.failure_only = failure_only + self.minibatch_size = minibatch_size + self.edit_budget = edit_budget + self.max_completion_tokens = int(max_completion_tokens) + self.dataloader = TemplateBenchmarkLoader( + split_dir=split_dir, + data_path=data_path, + split_mode=split_mode, + split_ratio=split_ratio, + split_seed=split_seed, + split_output_dir=split_output_dir, + seed=seed, + limit=limit, + ) + + # ── Lifecycle hooks ──────────────────────────────────────────────── + + def setup(self, cfg: dict) -> None: + super().setup(cfg) + self.dataloader.setup(cfg) + + def get_dataloader(self): + return self.dataloader + + # ── Batch → env manager ──────────────────────────────────────────── + + def build_env_from_batch(self, batch: BatchSpec, **kwargs): + # Dataset-backed envs typically just pass items straight through. + return list(batch.payload or []) + + def build_train_env(self, batch_size: int, seed: int, **kwargs): + batch = self.dataloader.build_train_batch( + batch_size=batch_size, seed=seed, **kwargs + ) + return self.build_env_from_batch(batch, **kwargs) + + def build_eval_env(self, env_num: int, split: str, seed: int, **kwargs): + batch = self.dataloader.build_eval_batch( + env_num=env_num, split=split, seed=seed, **kwargs + ) + return self.build_env_from_batch(batch, **kwargs) + + # ── Rollout: run episodes under current skill ────────────────────── + + def rollout( + self, + env_manager, + skill_content: str, + out_dir: str, + **kwargs, + ) -> list[dict]: + """ + Run a batch of episodes under the current skill. + + TODO: replace this loop with your real rollout. For each item: + 1. Build the prompt using `skill_content` as the system message. + 2. Call your target model. + 3. Score the prediction. + 4. Return a dict with at minimum: ``id`` (str), ``hard`` (0|1), + ``soft`` (float in [0, 1]). Add any env-specific extras you + need for reflect() — they will be preserved on + ``RolloutResult.extras``. + """ + items: list[dict] = env_manager + results: list[dict] = [] + for item in items: + # ── REPLACE THIS BLOCK WITH YOUR REAL ROLLOUT ── + results.append( + { + "id": str(item.get("id", "")), + "hard": 0, + "soft": 0.0, + "predicted_answer": "", + "question": item.get("question", ""), + "fail_reason": "template rollout — not implemented", + } + ) + return results + + # ── Reflect: turn rollout results into patch dicts ───────────────── + + def reflect( + self, + results: list[dict], + skill_content: str, + out_dir: str, + **kwargs, + ) -> list[dict | None]: + """ + Turn rollouts into a list of raw patch dicts (or None to drop). + + Each non-None dict MUST have: + - "patch": {"edits": [...]} a Patch.to_dict() payload + - "source_type": "failure" | "success" + + Most benchmarks delegate to + :func:`skillopt.gradient.reflect.run_minibatch_reflect` which + will call the optimizer model with the + ``analyst_error_*`` / ``analyst_success_*`` prompts. To enable it, + uncomment the import above and call: + + from skillopt.gradient.reflect import run_minibatch_reflect + return run_minibatch_reflect( + results=results, + skill_content=skill_content, + prediction_dir=kwargs.get( + "prediction_dir", os.path.join(out_dir, "predictions") + ), + patches_dir=kwargs.get( + "patches_dir", os.path.join(out_dir, "patches") + ), + workers=self.analyst_workers, + failure_only=self.failure_only, + minibatch_size=self.minibatch_size, + edit_budget=self.edit_budget, + random_seed=kwargs.get("random_seed"), + error_system=self.get_error_minibatch_prompt(), + success_system=self.get_success_minibatch_prompt(), + step_buffer_context=kwargs.get("step_buffer_context", ""), + update_mode=getattr(self, "_cfg", {}).get( + "skill_update_mode", "patch" + ), + ) + """ + # Template default: produce no patches (no-op trainer step). + return [None for _ in results] + + # ── Stratification hint ──────────────────────────────────────────── + + def get_task_types(self) -> list[str]: + """Distinct task-type strings used for stratified sampling.""" + seen: list[str] = [] + all_items = ( + self.dataloader.train_items + + self.dataloader.val_items + + self.dataloader.test_items + ) + for item in all_items: + tt = str(item.get("task_type") or "template") + if tt not in seen: + seen.append(tt) + return seen or ["template"] diff --git a/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/_template/loader_template.py b/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/_template/loader_template.py new file mode 100644 index 00000000..fa8bd44c --- /dev/null +++ b/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/_template/loader_template.py @@ -0,0 +1,87 @@ +""" +Benchmark Data Loader Template +================================ +Copy this file and implement ``load_split_items`` to load your benchmark +data. The loader is a :class:`skillopt.datasets.base.SplitDataLoader` +subclass — the base class handles both ``split_mode="split_dir"`` (read +an existing train/val/test layout) and ``split_mode="ratio"`` (build the +splits from a single raw file deterministically). + +For a fully worked example see +``skillopt/envs/officeqa/dataloader.py``. +""" +from __future__ import annotations + +import json +from pathlib import Path + +from skillopt.datasets.base import SplitDataLoader + + +def _normalize_item(raw: dict) -> dict: + """ + Normalise one raw entry into the dict shape SkillOpt expects. + + The only **hard** requirement is ``"id"`` (str). Add whatever extra + fields your :class:`TemplateBenchmarkEnv.rollout` needs. + """ + return { + "id": str(raw.get("uid") or raw.get("id") or ""), + "question": str(raw.get("question") or raw.get("prompt") or ""), + "ground_truth": str(raw.get("ground_truth") or raw.get("answer") or ""), + "task_type": str(raw.get("category") or raw.get("task_type") or "template"), + # ── add benchmark-specific keys here ── + } + + +class TemplateBenchmarkLoader(SplitDataLoader): + """ + Data loader for . + + Subclass note: you usually only need to implement + :meth:`load_split_items`. The base class drives ``setup(cfg)``, + materialises ratio-mode splits, exposes ``train_items``, + ``val_items``, ``test_items``, and builds ``BatchSpec`` objects on + demand. + + If you want to support ``split_mode="ratio"`` (auto-split a single + file into train/val/test), also implement + :meth:`load_raw_items(data_path)` returning the full list of items. + """ + + def load_split_items(self, split_path: str) -> list[dict]: + """Load all items for one split directory. + + ``split_path`` is e.g. ``data/your_benchmark/train/``. Return a + list of dicts, each shaped like :func:`_normalize_item`'s output. + """ + path = Path(split_path) + + json_files = sorted(path.glob("*.json")) + if json_files: + with json_files[0].open(encoding="utf-8") as f: + payload = json.load(f) + if not isinstance(payload, list): + raise ValueError( + f"Expected JSON array at top level of {json_files[0]}" + ) + return [_normalize_item(row) for row in payload] + + jsonl_files = sorted(path.glob("*.jsonl")) + if jsonl_files: + items: list[dict] = [] + with jsonl_files[0].open(encoding="utf-8") as f: + for line in f: + line = line.strip() + if not line: + continue + items.append(_normalize_item(json.loads(line))) + return items + + raise FileNotFoundError( + f"No .json or .jsonl file found in {split_path}" + ) + + # Optional — only needed if you intend to use ``split_mode='ratio'``. + # def load_raw_items(self, data_path: str) -> list[dict]: + # ... diff --git a/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/alfworld/__init__.py b/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/alfworld/__init__.py new file mode 100644 index 00000000..e9a28ff4 --- /dev/null +++ b/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/alfworld/__init__.py @@ -0,0 +1,5 @@ +"""ALFWorld environment adapter for ReflACT.""" + +from skillopt.envs.alfworld.adapter import ALFWorldAdapter + +__all__ = ["ALFWorldAdapter"] diff --git a/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/alfworld/adapter.py b/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/alfworld/adapter.py new file mode 100644 index 00000000..e6891692 --- /dev/null +++ b/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/alfworld/adapter.py @@ -0,0 +1,459 @@ +"""ALFWorld environment adapter for ReflACT. + +Connects the ReflACT training loop to ALFWorld by implementing +:class:`~skillopt.envs.base.EnvAdapter`. +""" +from __future__ import annotations + +from dataclasses import dataclass +import json +import os + +from skillopt.datasets.base import BatchSpec +from skillopt.envs.base import EnvAdapter +from skillopt.envs.alfworld.dataloader import ALFWorldDataLoader +from skillopt.envs.alfworld.rollout import ( + build_alfworld_env, + run_alfworld_batch, + TASKS, +) +from skillopt.gradient.reflect import run_minibatch_reflect +from skillopt.utils import compute_score + + +@dataclass(frozen=True) +class ALFWorldBatchRun: + """Lazy ALFWorld batch description. + + The adapter materializes this in rollout chunks so a large evaluation set + does not keep every ALFWorld simulator open at once. + """ + + env_num: int + eval_dataset: str + seed: int + is_train: bool + workers: int + specific_gamefiles: list[str] | None = None + result_ids: list[str] | None = None + items: list[dict] | None = None + + def __iter__(self): + return iter(self.items or []) + + def __len__(self) -> int: + return int(self.env_num or 0) + + +class ALFWorldAdapter(EnvAdapter): + """ALFWorld environment adapter. + + Parameters + ---------- + max_steps : int + Maximum steps per ALFWorld episode (default 50). + max_api_workers : int + Maximum concurrent API calls during rollout (default 8). + analyst_workers : int + Parallel workers for analyst stage (default 16). + failure_only : bool + If True, only run error analyst (skip success analyst). + minibatch_size : int + Trajectories per analyst group, M (default 8). + edit_budget : int + Maximum edits per minibatch, L (default 4). + """ + + def __init__( + self, + split_dir: str = "", + data_path: str = "", + split_mode: str = "split_dir", + split_ratio: str = "2:1:7", + split_seed: int = 42, + split_output_dir: str = "", + seed: int = 42, + limit: int = 0, + train_size: int = 0, + max_steps: int = 50, + workers: int = 8, + max_api_workers: int = 8, + analyst_workers: int = 16, + failure_only: bool = False, + minibatch_size: int = 8, + edit_budget: int = 4, + max_completion_tokens: int = 16384, + ) -> None: + self.max_steps = max_steps + self.workers = max(int(workers or 1), 1) + self.max_api_workers = max_api_workers + self.max_completion_tokens = int(max_completion_tokens) + self.analyst_workers = analyst_workers + self.failure_only = failure_only + self.minibatch_size = minibatch_size + self.edit_budget = edit_budget + self.dataloader = ALFWorldDataLoader( + split_dir=split_dir, + data_path=data_path, + split_mode=split_mode, + split_ratio=split_ratio, + split_seed=split_seed, + split_output_dir=split_output_dir, + seed=seed, + limit=limit, + train_size=train_size, + ) + self._traj_cache: dict[str, dict | None] = {} + + def setup(self, cfg: dict) -> None: + super().setup(cfg) + self.dataloader.setup(cfg) + + def _load_traj_data(self, item: dict) -> dict | None: + gamefile = str(item.get("gamefile") or "").strip() + if not gamefile: + return None + if gamefile in self._traj_cache: + return self._traj_cache[gamefile] + + traj_path = os.path.join(os.path.dirname(gamefile), "traj_data.json") + try: + with open(traj_path, encoding="utf-8") as f: + data = json.load(f) + except Exception: + data = None + self._traj_cache[gamefile] = data + return data + + @staticmethod + def _unique_lines(values: list[str], *, limit: int = 0) -> list[str]: + lines: list[str] = [] + seen: set[str] = set() + for raw in values: + line = str(raw or "").strip() + if not line or line in seen: + continue + seen.add(line) + lines.append(line) + if limit > 0 and len(lines) >= limit: + break + return lines + + @staticmethod + def _format_high_pddl(high_pddl: list[dict]) -> list[str]: + steps: list[str] = [] + for idx, step in enumerate(high_pddl or [], start=1): + discrete = step.get("discrete_action") or {} + action = str(discrete.get("action") or "").strip() + args = [str(arg).strip() for arg in (discrete.get("args") or []) if str(arg).strip()] + if action and args: + text = f"{action}({', '.join(args)})" + elif action: + text = action + else: + planner_action = step.get("planner_action") or {} + text = str(planner_action.get("action") or "").strip() + if text: + steps.append(f"{idx}. {text}") + return steps + + def _build_reference_bundle(self, item: dict) -> dict: + data = self._load_traj_data(item) + if not data: + return {} + + anns = ((data.get("turk_annotations") or {}).get("anns") or []) + task_descs = self._unique_lines( + [ann.get("task_desc", "") for ann in anns], + limit=3, + ) + high_descs = self._unique_lines( + [step for ann in anns for step in (ann.get("high_descs") or [])], + limit=12, + ) + pddl_params = { + key: value + for key, value in (data.get("pddl_params") or {}).items() + if value not in ("", None, [], {}) + } + scene = data.get("scene") or {} + scene_summary = { + key: scene.get(key) + for key in ("floor_plan", "scene_num", "dirty_and_empty") + if scene.get(key) not in ("", None, [], {}) + } + high_pddl = self._format_high_pddl((data.get("plan") or {}).get("high_pddl") or []) + task_type = str(data.get("task_type") or item.get("task_type") or "").strip() + return { + "task_type": task_type, + "task_descs": task_descs, + "high_descs": high_descs, + "pddl_params": pddl_params, + "high_pddl": high_pddl, + "scene_summary": scene_summary, + } + + def build_reference_text(self, item: dict) -> str: + bundle = self._build_reference_bundle(item) + if not bundle: + return "" + + parts: list[str] = [] + if bundle["task_type"]: + parts.append(f"## Reference Task Type\n{bundle['task_type']}") + if bundle["task_descs"]: + parts.append( + "## Reference Human Task Descriptions\n" + + "\n".join(f"- {line}" for line in bundle["task_descs"]) + ) + if bundle["high_descs"]: + parts.append( + "## Reference Human High-Level Steps\n" + + "\n".join(f"{idx}. {line}" for idx, line in enumerate(bundle["high_descs"], start=1)) + ) + if bundle["pddl_params"]: + parts.append( + "## Reference PDDL Params\n" + + "\n".join(f"- {key}: {value}" for key, value in bundle["pddl_params"].items()) + ) + if bundle["high_pddl"]: + parts.append( + "## Reference Planner High-Level Plan\n" + "\n".join(bundle["high_pddl"]) + ) + if bundle["scene_summary"]: + parts.append( + "## Reference Scene Summary\n" + + "\n".join(f"- {key}: {value}" for key, value in bundle["scene_summary"].items()) + ) + return "\n\n".join(parts) + + def get_reference_metadata(self, item: dict) -> dict: + bundle = self._build_reference_bundle(item) + if not bundle: + return {"fields": [], "preview": ""} + + fields: list[str] = [] + previews: list[str] = [] + if bundle["task_type"]: + fields.append("task_type") + previews.append(f"[task_type] {bundle['task_type']}") + if bundle["task_descs"]: + fields.append("task_desc") + previews.append("[task_desc]\n" + "\n".join(bundle["task_descs"][:2])) + if bundle["high_descs"]: + fields.append("high_descs") + previews.append("[high_descs]\n" + "\n".join(bundle["high_descs"][:3])) + if bundle["pddl_params"]: + fields.append("pddl_params") + previews.append( + "[pddl_params]\n" + + "\n".join( + f"{key}: {value}" for key, value in list(bundle["pddl_params"].items())[:4] + ) + ) + if bundle["high_pddl"]: + fields.append("plan.high_pddl") + previews.append("[plan.high_pddl]\n" + "\n".join(bundle["high_pddl"][:3])) + if bundle["scene_summary"]: + fields.append("scene") + previews.append( + "[scene]\n" + + "\n".join( + f"{key}: {value}" for key, value in bundle["scene_summary"].items() + ) + ) + return { + "fields": fields, + "preview": "\n\n".join(previews)[:600], + } + + @staticmethod + def _infer_dataset_from_gamefile(gamefile: str) -> tuple[str, bool]: + path = str(gamefile or "") + if "/valid_seen/" in path: + return "eval_in_distribution", False + if "/valid_unseen/" in path: + return "eval_out_of_distribution", False + return "train", True + + def get_dataloader(self): + return self.dataloader + + def _comparison_items(self, items: list[dict]) -> list[dict]: + enriched: list[dict] = [] + for item in items: + row = dict(item) + bundle = self._build_reference_bundle(row) + if bundle.get("task_descs"): + row["task_description"] = bundle["task_descs"][0] + elif bundle.get("task_type"): + row["task_description"] = bundle["task_type"] + enriched.append(row) + return enriched + + def requires_ray(self) -> bool: + return False + + def build_env_from_batch(self, batch: BatchSpec, **kwargs): + gamefiles = list(batch.metadata.get("gamefiles") or []) + result_ids = list(batch.metadata.get("result_ids") or []) + items = self._comparison_items(list(batch.payload or [])) + return ALFWorldBatchRun( + env_num=batch.batch_size, + eval_dataset=batch.metadata.get("eval_dataset", batch.split), + seed=batch.seed, + is_train=batch.metadata.get("is_train", batch.phase == "train"), + specific_gamefiles=gamefiles or None, + result_ids=result_ids or None, + items=items, + workers=self.workers, + ) + + def build_train_env(self, batch_size: int, seed: int, **kwargs): + batch = self.dataloader.build_train_batch(batch_size=batch_size, seed=seed, **kwargs) + return self.build_env_from_batch(batch, **kwargs) + + def build_eval_env(self, env_num: int, split: str, seed: int, **kwargs): + batch = self.dataloader.build_eval_batch(env_num=env_num, split=split, seed=seed, **kwargs) + return self.build_env_from_batch(batch, **kwargs) + + def rollout( + self, + env_manager, + skill_content: str, + out_dir: str, + **kwargs, + ) -> list[dict]: + results_path = os.path.join(out_dir, "results.jsonl") + os.makedirs(out_dir, exist_ok=True) + + # Resume support + if os.path.exists(results_path): + existing: list[dict] = [] + with open(results_path) as f: + for line in f: + try: + existing.append(json.loads(line)) + except Exception: + pass + if existing: + return existing + + if isinstance(env_manager, ALFWorldBatchRun): + results = self._run_batch( + env_manager, + skill_content=skill_content, + out_dir=out_dir, + ) + else: + results = run_alfworld_batch( + env_manager=env_manager, + skill_content=skill_content, + max_steps=self.max_steps, + out_root=out_dir, + max_api_workers=self.max_api_workers, + max_completion_tokens=self.max_completion_tokens, + result_ids=getattr(env_manager, "_skillopt_result_ids", None), + ) + + with open(results_path, "w") as f: + for r in results: + f.write(json.dumps(r, ensure_ascii=False) + "\n") + + return results + + @staticmethod + def _close_env(env_manager) -> None: + close = getattr(env_manager, "close", None) + if callable(close): + close() + + def _run_batch( + self, + batch: ALFWorldBatchRun, + skill_content: str, + out_dir: str, + *, + diagnostic_mode: bool = False, + diagnostic_instruction: str = "", + ) -> list[dict]: + total = int(batch.env_num or 0) + if total <= 0: + return [] + + workers = max(1, min(int(batch.workers or self.workers), total)) + if total > workers: + print( + f" [alfworld rollout] episodes={total} " + f"env_workers={workers} chunks={(total + workers - 1) // workers}" + ) + + all_results: list[dict] = [] + for start in range(0, total, workers): + chunk_size = min(workers, total - start) + chunk_gamefiles = ( + batch.specific_gamefiles[start:start + chunk_size] + if batch.specific_gamefiles + else None + ) + chunk_ids = ( + batch.result_ids[start:start + chunk_size] + if batch.result_ids + else [f"env_{idx:03d}" for idx in range(start, start + chunk_size)] + ) + chunk_env = build_alfworld_env( + env_num=chunk_size, + eval_dataset=batch.eval_dataset, + seed=batch.seed + start, + is_train=batch.is_train, + specific_gamefiles=chunk_gamefiles, + ) + try: + chunk_results = run_alfworld_batch( + env_manager=chunk_env, + skill_content=skill_content, + max_steps=self.max_steps, + out_root=out_dir, + max_api_workers=min(self.max_api_workers, chunk_size), + max_completion_tokens=self.max_completion_tokens, + diagnostic_mode=diagnostic_mode, + diagnostic_instruction=diagnostic_instruction, + result_ids=chunk_ids, + ) + finally: + self._close_env(chunk_env) + all_results.extend(chunk_results) + return all_results + + def reflect( + self, + results: list[dict], + skill_content: str, + out_dir: str, + **kwargs, + ) -> list[dict | None]: + prediction_dir = kwargs.get("prediction_dir", os.path.join(out_dir, "predictions")) + patches_dir = kwargs.get("patches_dir", os.path.join(out_dir, "patches")) + random_seed = kwargs.get("random_seed") + step_buffer_context = kwargs.get("step_buffer_context", "") + meta_skill_context = kwargs.get("meta_skill_context", "") + + return run_minibatch_reflect( + results=results, + skill_content=skill_content, + prediction_dir=prediction_dir, + patches_dir=patches_dir, + workers=self.analyst_workers, + failure_only=self.failure_only, + minibatch_size=self.minibatch_size, + edit_budget=self.edit_budget, + random_seed=random_seed, + error_system=self.get_error_minibatch_prompt(), + success_system=self.get_success_minibatch_prompt(), + step_buffer_context=step_buffer_context, + meta_skill_context=meta_skill_context, + ) + + + def get_task_types(self) -> list[str]: + return list(TASKS) diff --git a/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/alfworld/dataloader.py b/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/alfworld/dataloader.py new file mode 100644 index 00000000..80fcd709 --- /dev/null +++ b/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/alfworld/dataloader.py @@ -0,0 +1,123 @@ +"""ALFWorld task dataloader.""" +from __future__ import annotations + +from skillopt.datasets.base import BatchSpec, SplitDataLoader + + +class ALFWorldDataLoader(SplitDataLoader): + """ALFWorld batch planner. + + In split_dir mode, batches are fixed gamefile items so ablations differ + only in how the same training set is batched. + """ + + def __init__( + self, + split_dir: str = "", + data_path: str = "", + split_mode: str = "split_dir", + split_ratio: str = "2:1:7", + split_seed: int = 42, + split_output_dir: str = "", + seed: int = 42, + limit: int = 0, + train_size: int = 0, + **kwargs, + ) -> None: + super().__init__( + split_dir=split_dir, + data_path=data_path, + split_mode=split_mode, + split_ratio=split_ratio, + split_seed=split_seed, + split_output_dir=split_output_dir, + seed=seed, + limit=limit, + ) + self.train_size_override = int(train_size or 0) + + @staticmethod + def _metadata_for_items(items: list[dict], split: str, phase: str) -> dict: + gamefiles = [str(item.get("gamefile") or "") for item in items] + if any(not gamefile for gamefile in gamefiles): + raise ValueError("ALFWorld split items must contain non-empty gamefile paths.") + eval_dataset = "train" + is_train = phase == "train" + first = gamefiles[0] if gamefiles else "" + if "/valid_seen/" in first: + eval_dataset = "eval_in_distribution" + is_train = False + elif "/valid_unseen/" in first: + eval_dataset = "eval_out_of_distribution" + is_train = False + return { + "eval_dataset": eval_dataset, + "is_train": is_train, + "gamefiles": gamefiles, + "result_ids": [str(item.get("id") or idx) for idx, item in enumerate(items)], + } + + def get_train_size(self) -> int: + if self.train_size_override > 0: + return self.train_size_override + return super().get_train_size() + + def build_train_batch(self, batch_size: int, seed: int, **kwargs) -> BatchSpec: + batch = super().build_train_batch(batch_size=batch_size, seed=seed, **kwargs) + items = list(batch.payload or []) + batch.metadata.update(self._metadata_for_items(items, "train", "train")) + return BatchSpec( + phase="train", + split="train", + seed=seed, + batch_size=len(items), + payload=items, + metadata=batch.metadata, + ) + + def plan_train_epoch( + self, + *, + epoch: int, + steps_per_epoch: int, + accumulation: int, + batch_size: int, + seed: int, + **kwargs, + ) -> list[BatchSpec]: + batches = super().plan_train_epoch( + epoch=epoch, + steps_per_epoch=steps_per_epoch, + accumulation=accumulation, + batch_size=batch_size, + seed=seed, + **kwargs, + ) + for batch in batches: + items = list(batch.payload or []) + batch.metadata.update(self._metadata_for_items(items, "train", "train")) + return batches + + def build_eval_batch( + self, + env_num: int, + split: str, + seed: int, + **kwargs, + ) -> BatchSpec: + batch = super().build_eval_batch( + env_num=env_num, + split=split, + seed=seed, + **kwargs, + ) + items = list(batch.payload or []) + batch.metadata.update(self._metadata_for_items(items, split, "eval")) + return BatchSpec( + phase="eval", + split=split, + seed=seed, + batch_size=len(items), + payload=items, + metadata=batch.metadata, + ) diff --git a/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/alfworld/prompts/analyst_error.md b/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/alfworld/prompts/analyst_error.md new file mode 100644 index 00000000..f4647160 --- /dev/null +++ b/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/alfworld/prompts/analyst_error.md @@ -0,0 +1,55 @@ +You are an expert failure-analysis agent for ALFWorld embodied household tasks. + +You will be given MULTIPLE failed agent trajectories from a single minibatch +and the current skill document. +Your job is to identify the most important COMMON failure patterns across +the batch and propose a concise set of skill edits. + +## ALFWorld Task Types +- pick_and_place: Put object in/on a receptacle +- pick_two_obj_and_place: Put two instances of an object in/on a receptacle +- look_at_obj_in_light: Examine an object under a desklamp +- pick_heat_then_place_in_recep: Heat an object and put it in/on a receptacle +- pick_cool_then_place_in_recep: Cool an object and put it in/on a receptacle +- pick_clean_then_place_in_recep: Clean an object and put it in/on a receptacle + +## Failure Type Categories +- **navigation_loop**: the agent revisits the same locations repeatedly without progress +- **missed_object**: the agent fails to pick up a visible/reachable goal object +- **wrong_sequence**: the agent performs actions in the wrong order (e.g., placing before transforming) +- **premature_stop**: the agent stops or gets stuck before completing all goal conditions +- **action_loop**: the agent repeats the same action without advancing +- **appliance_error**: the agent misuses or skips an appliance (microwave, fridge, sink) +- **rule_missing**: the skill lacks a relevant rule for this situation +- **rule_wrong**: an existing skill rule is misleading or incorrect +- **rule_ignored**: the skill has the right rule but the agent did not follow it +- **other**: none of the above + +## Analysis Process +1. Read ALL trajectories in the minibatch. +2. Identify the most prevalent, systematic failure patterns across them. +3. For each pattern, classify its failure type. +4. Propose skill edits that address the COMMON patterns — not individual edge cases. +5. Edits must be generalizable; do not hardcode task-specific values. +6. Only patch gaps in the skill — do not duplicate existing content. + +You will be told the maximum number of edits (the budget L). Produce AT MOST L edits, +focusing on the highest-impact patterns. You may produce fewer if warranted. + +Respond ONLY with a valid JSON object (no markdown fences, no extra text): +{ + "batch_size": , + "failure_summary": [ + {"failure_type": "", "count": , "description": ""} + ], + "patch": { + "reasoning": "", + "edits": [ + {"op": "append", "content": ""}, + {"op": "insert_after", "target": "", "content": ""}, + {"op": "replace", "target": "", "content": ""}, + {"op": "delete", "target": ""} + ] + } +} +Only include edits that are needed. "edits" can be an empty list if no patch is warranted. diff --git a/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/alfworld/prompts/analyst_success.md b/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/alfworld/prompts/analyst_success.md new file mode 100644 index 00000000..957d3a8e --- /dev/null +++ b/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/alfworld/prompts/analyst_success.md @@ -0,0 +1,33 @@ +You are an expert success-pattern analyst for AI agents operating in ALFWorld, +a text-based embodied household environment. + +You will be given MULTIPLE successful agent trajectories from a single minibatch +and the current skill document. Your job is to identify generalizable behavior +patterns that are COMMON across the batch and worth encoding in the skill. + +## Rules +- Only propose patches for patterns NOT already covered in the skill. +- Focus on patterns that appear across MULTIPLE trajectories in the batch. +- Be concise. Patterns must generalize beyond specific tasks. +- Prefer reinforcing existing sections over adding new top-level sections. +- If the agents' success involved efficient exploration or smart appliance usage, + consider reinforcing that in the patch. + +You will be told the maximum number of edits (the budget L). Produce AT MOST L edits, +focusing on the most broadly applicable patterns. You may produce fewer if warranted. + +Respond ONLY with a valid JSON object: +{ + "batch_size": , + "success_patterns": ["", ""], + "patch": { + "reasoning": "", + "edits": [ + {"op": "append", "content": ""}, + {"op": "insert_after", "target": "", "content": ""}, + {"op": "replace", "target": "", "content": ""}, + {"op": "delete", "target": ""} + ] + } +} +"edits" may be empty if the skill already covers all observed patterns. diff --git a/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/alfworld/prompts/rollout_no_history.md b/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/alfworld/prompts/rollout_no_history.md new file mode 100644 index 00000000..d1d605bf --- /dev/null +++ b/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/alfworld/prompts/rollout_no_history.md @@ -0,0 +1,8 @@ + +You are an expert agent operating in the ALFRED Embodied Environment. +Your current observation is: {current_observation} +Your admissible actions of the current situation are: [{admissible_actions}]. + +Now it's your turn to take an action. +You should first reason step-by-step about the current situation. This reasoning process MUST be enclosed within tags. +Once you've finished your reasoning, you should choose an admissible action for current step and present it within tags. diff --git a/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/alfworld/prompts/rollout_with_history.md b/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/alfworld/prompts/rollout_with_history.md new file mode 100644 index 00000000..f0a635db --- /dev/null +++ b/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/alfworld/prompts/rollout_with_history.md @@ -0,0 +1,9 @@ + +You are an expert agent operating in the ALFRED Embodied Environment. Your task is to: {task_description} +Prior to this step, you have already taken {step_count} step(s). Below are the most recent {history_length} observations and the corresponding actions you took: {action_history} +You are now at step {current_step} and your current observation is: {current_observation} +Your admissible actions of the current situation are: [{admissible_actions}]. + +Now it's your turn to take an action. +You should first reason step-by-step about the current situation. This reasoning process MUST be enclosed within tags. +Once you've finished your reasoning, you should choose an admissible action for current step and present it within tags. diff --git a/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/alfworld/prompts/rollout_with_memory.md b/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/alfworld/prompts/rollout_with_memory.md new file mode 100644 index 00000000..c90dc7f4 --- /dev/null +++ b/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/alfworld/prompts/rollout_with_memory.md @@ -0,0 +1,16 @@ + +You are an expert agent operating in the ALFRED Embodied Environment. Your task is to: {task_description} + +## Retrieved Relevant Experience + +{retrieved_memories} + +## Current Progress + +Prior to this step, you have already taken {step_count} step(s). Below are the most recent {history_length} observations and the corresponding actions you took: {action_history} +You are now at step {current_step} and your current observation is: {current_observation} +Your admissible actions of the current situation are: [{admissible_actions}]. + +Now it's your turn to take an action. +You should first reason step-by-step about the current situation. This reasoning process MUST be enclosed within tags. +Once you've finished your reasoning, you should choose an admissible action for current step and present it within tags. diff --git a/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/alfworld/reflect.py b/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/alfworld/reflect.py new file mode 100644 index 00000000..a32d9897 --- /dev/null +++ b/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/alfworld/reflect.py @@ -0,0 +1,4 @@ +"""ALFWorld Reflect stage. + +Prompts are now loaded from .md files by the base adapter. +""" diff --git a/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/alfworld/rollout.py b/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/alfworld/rollout.py new file mode 100644 index 00000000..8c3b4ac9 --- /dev/null +++ b/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/alfworld/rollout.py @@ -0,0 +1,347 @@ +"""ALFWorld rollout module for ReflACT. + +Provides: + - build_alfworld_env(): build ALFWorld environment (wraps vendored SkillRL env) + - run_alfworld_batch(): run a batch of ALFWorld episodes in parallel + - TASKS: list of ALFWorld task types +""" +from __future__ import annotations + +import json +import os +import re +import sys +import concurrent.futures +import numpy as np + +from skillopt.model import chat_target + +# ── Constants ───────────────────────────────────────────────────────────────── + +TASKS = [ + "pick_and_place", + "pick_two_obj_and_place", + "look_at_obj_in_light", + "pick_heat_then_place_in_recep", + "pick_cool_then_place_in_recep", + "pick_clean_then_place_in_recep", +] + +# ── Helpers ─────────────────────────────────────────────────────────────────── + + +def _get_task_type(gamefile: str) -> str: + for task in TASKS: + if task in gamefile: + return task + return "other" + + +def _extract_action(model_response: str) -> str | None: + match = re.search(r"(.*?)", model_response, re.DOTALL) + return match.group(1).strip() if match else None + + +def _extract_think(model_response: str) -> str | None: + match = re.search(r"(.*?)", model_response, re.DOTALL) + return match.group(1).strip() if match else None + + +def _build_skill_prompt(skill_content: str) -> str: + """Build the skill section to inject into the agent's system prompt.""" + if not skill_content or not skill_content.strip(): + return "" + return ( + "\n\n## Skill Knowledge\n" + "Below is a skill document with learned strategies. " + "Use these guidelines to inform your decisions:\n\n" + f"{skill_content}\n" + ) + + +def _append_diagnostic_instruction(prompt: str, diagnostic_instruction: str) -> str: + if not diagnostic_instruction or not diagnostic_instruction.strip(): + return prompt + return f"{prompt}\n\n## Training Readout\n{diagnostic_instruction.strip()}\n" + + +# ── Environment builder ────────────────────────────────────────────────────── + + +def build_alfworld_env( + env_num: int, + eval_dataset: str = "eval_out_of_distribution", + seed: int = 42, + is_train: bool = False, + specific_gamefiles: list[str] | None = None, +): + """Build ALFWorld environment manager. + + Args: + env_num: number of parallel environments + eval_dataset: 'eval_in_distribution' or 'eval_out_of_distribution' or train + seed: random seed + is_train: whether to use training set + + Returns: + env_manager: AlfWorldEnvironmentManager instance + """ + from omegaconf import OmegaConf + from functools import partial + + from skillopt.envs.alfworld.vendor.alfworld_envs import build_alfworld_envs + from skillopt.envs.alfworld.vendor.alfworld_projection import alfworld_projection + from skillopt.envs.alfworld.vendor.env_manager import AlfWorldEnvironmentManager + + HERE = os.path.dirname(os.path.abspath(__file__)) + + alf_config_path = os.path.join(HERE, "vendor", "config_tw.yaml") + env_kwargs = {"eval_dataset": eval_dataset} + + envs = build_alfworld_envs( + alf_config_path, + seed=seed, + env_num=env_num, + group_n=1, + is_train=is_train, + env_kwargs=env_kwargs, + resources_per_worker=None, + gamefiles=specific_gamefiles, + ) + + config = OmegaConf.create( + { + "env": { + "history_length": 2, + "env_name": "alfworld/AlfredTWEnv", + } + } + ) + + projection_f = partial(alfworld_projection) + env_manager = AlfWorldEnvironmentManager(envs, projection_f, config) + return env_manager + + +# ── Batch rollout ───────────────────────────────────────────────────────────── + + +def run_alfworld_batch( + env_manager, + skill_content: str, + max_steps: int = 50, + out_root: str = "", + max_api_workers: int = 8, + temperature: float = 0.4, + max_completion_tokens: int = 16384, + diagnostic_mode: bool = False, + diagnostic_instruction: str = "", + result_ids: list[str] | None = None, +) -> list[dict]: + """Run a batch of ALFWorld episodes. + + Returns a list of result dicts compatible with SkillOpt pipeline: + [ + { + "id": "_", + "hard": 0 or 1, + "soft": 0.0 or 1.0, + "n_turns": , + "fail_reason": "", + "agent_ok": True, + "task_type": "", + "gamefile": "", + "task_description": "", + }, + ... + ] + + Also saves conversation.json per environment in out_root/predictions// + """ + skill_prompt = _build_skill_prompt(skill_content) + + obs, infos = env_manager.reset({}) + env_num = len(obs["text"]) + env_dones = [False] * env_num + overall_success = [False] * env_num + + # Build per-env metadata + env_meta: list[dict] = [] + for i in range(env_num): + gamefile = infos[i].get("extra.gamefile", "") if isinstance(infos[i], dict) else "" + task_type = _get_task_type(gamefile) + # Extract task description from initial observation + task_desc = "" + anchor_text = obs["anchor"][i] if "anchor" in obs else "" + task_start = anchor_text.find("Your task is to: ") + if task_start != -1: + task_desc = anchor_text[task_start + len("Your task is to: "):].strip() + + env_meta.append({ + "gamefile": gamefile, + "task_type": task_type, + "task_description": task_desc, + }) + + # Per-env conversation records + conversations: list[list[dict]] = [[] for _ in range(env_num)] + + for step_idx in range(max_steps): + if all(env_dones): + break + + active_indices = [i for i in range(env_num) if not env_dones[i]] + + # Build prompts with skill injection + prompts: dict[int, str] = {} + for i in active_indices: + prompt = obs["text"][i] + if skill_prompt: + # Inject skill before the action instruction + prompt = skill_prompt + "\n" + prompt + if diagnostic_mode and diagnostic_instruction.strip(): + prompt = _append_diagnostic_instruction(prompt, diagnostic_instruction) + prompts[i] = prompt + + # Call API in parallel + actions = ["None"] * env_num + + def call_api(idx): + try: + response, _ = chat_target( + system="You are an expert agent operating in the ALFRED Embodied Environment.", + user=prompts[idx], + max_completion_tokens=max_completion_tokens, + retries=5, + stage="rollout", + timeout=None, + ) + response = (response or "").strip() + if not response: + return idx, "empty model responselook" + if _extract_action(response) is None: + return idx, "missing action taglook" + return idx, response + except Exception as e: + return idx, "errorlook" + + executor = concurrent.futures.ThreadPoolExecutor(max_workers=max_api_workers) + try: + futures = {executor.submit(call_api, i): i for i in active_indices} + pending_futs = set(futures) + while pending_futs: + done, _ = concurrent.futures.wait( + pending_futs, + timeout=5, + return_when=concurrent.futures.FIRST_COMPLETED, + ) + for future in done: + pending_futs.remove(future) + try: + idx, response = future.result() + except Exception: # noqa: BLE001 + idx = futures[future] + response = "errorlook" + actions[idx] = response + finally: + executor.shutdown(wait=False, cancel_futures=True) + + # Save model responses before stepping + model_responses = {i: actions[i] for i in active_indices} + + # Step environment + obs, rewards, dones, infos = env_manager.step(actions) + + # Record trajectory + for i in active_indices: + step_record = { + "step": step_idx, + "action": _extract_action(model_responses[i]), + "reasoning": _extract_think(model_responses[i]), + "model_response": model_responses[i], + "env_feedback": obs["anchor"][i] if "anchor" in obs else "", + "reward": float(rewards[i]), + "done": bool(dones[i]), + } + conversations[i].append(step_record) + + # Update done status + for i in range(env_num): + if env_dones[i]: + continue + if dones[i]: + env_dones[i] = True + won = bool(infos[i].get("won", False)) + overall_success[i] = won + + # Build results and save conversations + results: list[dict] = [] + pred_dir = os.path.join(out_root, "predictions") if out_root else "" + + for i in range(env_num): + gamefile = env_meta[i]["gamefile"] + task_type = env_meta[i]["task_type"] + task_desc = env_meta[i]["task_description"] + n_turns = len(conversations[i]) + won = overall_success[i] + + # Generate stable task ID from env index and gamefile + task_id = str(result_ids[i]) if result_ids and i < len(result_ids) else f"env_{i:03d}" + + fail_reason = "" + if not won: + if not env_dones[i]: + fail_reason = f"Timeout after {max_steps} steps" + else: + fail_reason = "Episode ended without completing the task" + + result = { + "id": task_id, + "hard": 1 if won else 0, + "soft": 1.0 if won else 0.0, + "n_turns": n_turns, + "fail_reason": fail_reason, + "agent_ok": True, # ALFWorld agent always runs OK (no crash) + "task_type": task_type, + "gamefile": gamefile, + "task_description": task_desc, + "instruction_type": task_type, # for compatibility with v2 pipeline + } + results.append(result) + + # Save conversation + if pred_dir: + conv_dir = os.path.join(pred_dir, task_id) + os.makedirs(conv_dir, exist_ok=True) + with open(os.path.join(conv_dir, "conversation.json"), "w") as f: + json.dump(conversations[i], f, ensure_ascii=False, indent=2) + + return results + + +# ── Item loading (for compatibility with split_three_way) ──────────────────── + + +def load_alfworld_items( + eval_dataset: str, + env_num: int, + seed: int = 42, + is_train: bool = False, +) -> list[dict]: + """Create pseudo-item dicts for ALFWorld environments. + + Since ALFWorld doesn't have a static JSON dataset like SpreadsheetBench, + we create lightweight item dicts that carry enough metadata for the pipeline. + The actual environment is built dynamically. + + Returns: + List of dicts with "id" keys, one per environment slot. + """ + items = [] + for i in range(env_num): + items.append({ + "id": f"env_{i:03d}", + "eval_dataset": eval_dataset, + "env_index": i, + }) + return items diff --git a/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/alfworld/skills/initial.md b/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/alfworld/skills/initial.md new file mode 100644 index 00000000..d19ad023 --- /dev/null +++ b/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/alfworld/skills/initial.md @@ -0,0 +1,45 @@ +# ALFWorld Embodied Agent Skill + +## Overview +This skill guides agents operating in the ALFWorld text-based embodied environment. +The agent must complete household tasks by navigating rooms, interacting with objects, +and using appliances. Actions must be chosen from the admissible action list provided +at each step. + +**Output format**: Always output `...` for reasoning, then `...` for the chosen action. + +--- + +## Task Types + +| Type | Goal | Key Steps | +|------|------|-----------| +| Pick & Place | Put object X in/on receptacle Y | Find X -> take X -> go to Y -> put X in/on Y | +| Pick Two & Place | Put two instances of X in/on Y | Find X1 -> take -> place -> find X2 -> take -> place | +| Examine in Light | Examine object X under desklamp | Find X -> take X -> find desklamp -> use desklamp | +| Clean & Place | Clean object X and put in/on Y | Find X -> take X -> go to sink -> clean X -> go to Y -> put X | +| Heat & Place | Heat object X and put in/on Y | Find X -> take X -> go to microwave -> heat X -> go to Y -> put X | +| Cool & Place | Cool object X and put in/on Y | Find X -> take X -> go to fridge -> cool X -> go to Y -> put X | + +--- + +## General Principles + +1. **Decompose the task**: Parse the goal into ordered sub-goals (locate, acquire, transform, deliver). Complete each before moving to the next. +2. **Systematic exploration**: Search each surface and container exactly once before revisiting. Open closed containers (drawers, cabinets, fridge) before judging them empty. +3. **Grab immediately**: When a required object is visible and reachable, take it right away before moving elsewhere. +4. **Transform before placing**: If the task requires cleaning, heating, or cooling, perform the state change at the appropriate appliance before heading to the final destination. +5. **Direct delivery**: Once holding the transformed (or untransformed) goal object, navigate straight to the target receptacle and place it. +6. **Track progress**: Maintain an internal count of how many objects still need to be found and placed. Only stop searching when the count reaches zero. +7. **Avoid loops**: Never repeat the same action more than twice in a row. If stuck, move to a different unexplored location. +8. **Only choose admissible actions**: Always pick an action from the admissible action list. Do not invent actions. + +--- + +## Common Mistakes to Avoid + +- **Revisiting searched locations**: Keep track of which surfaces/containers have been checked; do not re-examine them. +- **Ignoring visible objects**: If the target object appears in the observation, pick it up immediately. +- **Skipping state changes**: Do not place an object at the destination without first cleaning/heating/cooling it when required. +- **Premature termination**: Do not stop the episode until all goal conditions are verified as met. +- **Action loops**: Repeatedly toggling or examining the same object wastes steps. Move on to new locations instead. diff --git a/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/alfworld/vendor/__init__.py b/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/alfworld/vendor/__init__.py new file mode 100644 index 00000000..93dd8cb0 --- /dev/null +++ b/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/alfworld/vendor/__init__.py @@ -0,0 +1,9 @@ +"""Vendored ALFWorld environment runtime. + +Minimal subset of SkillRL's agent_system package needed to run +ALFWorld environments with ReflACT. Original source: +https://github.com/NTU-LANTERN/SkillRL (Apache-2.0 License) +""" +from .alfworld_envs import AlfworldEnvs, build_alfworld_envs +from .alfworld_projection import alfworld_projection +from .env_manager import AlfWorldEnvironmentManager diff --git a/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/alfworld/vendor/alfworld_envs.py b/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/alfworld/vendor/alfworld_envs.py new file mode 100644 index 00000000..06b97164 --- /dev/null +++ b/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/alfworld/vendor/alfworld_envs.py @@ -0,0 +1,221 @@ +# Vendored from SkillRL (Apache-2.0 License) +# Original: agent_system/environments/env_package/alfworld/envs.py +# Modified: imports use pip-installed alfworld package instead of vendored copy. + +import os +import multiprocessing as mp +import traceback +import yaml +import gymnasium as gym +import numpy as np + +from alfworld.agents.environment import get_environment + + +def load_config_file(path): + assert os.path.exists(path), f"Invalid config file: {path}" + with open(path) as reader: + config = yaml.safe_load(reader) + return config + + +def compute_reward(info, multi_modal=False): + if multi_modal: + reward = 10.0 * float(info['won']) + float(info['goal_condition_success_rate']) + else: + reward = 10.0 * float(info['won']) + return reward + + +class AlfworldWorker: + """Stateful worker that holds one ALFWorld sub-environment.""" + + def __init__(self, config, seed, base_env, gamefile=None): + if gamefile: + base_env.game_files = [gamefile] + if hasattr(base_env, "num_games"): + base_env.num_games = 1 + self.env = base_env.init_env(batch_size=1) + self.env.seed(seed) + + def step(self, action): + actions = [action] + obs, scores, dones, infos = self.env.step(actions) + infos['observation_text'] = obs + return obs, scores, dones, infos + + def reset(self): + obs, infos = self.env.reset() + infos['observation_text'] = obs + return obs, infos + + +def _worker_loop(cmd_q, result_q, config, seed, is_train, eval_dataset, gamefile): + """Run one ALFWorld environment in a child process.""" + try: + env_type = config['env']['type'] + base_env = get_environment(env_type)( + config, + train_eval='train' if is_train else eval_dataset, + ) + worker = AlfworldWorker(config, seed, base_env, gamefile) + result_q.put((True, "ready")) + except BaseException: + result_q.put((False, traceback.format_exc())) + return + + while True: + cmd, payload = cmd_q.get() + if cmd == "close": + result_q.put((True, None)) + return + try: + if cmd == "reset": + result = worker.reset() + elif cmd == "step": + result = worker.step(payload) + else: + raise ValueError(f"Unknown ALFWorld worker command: {cmd}") + result_q.put((True, result)) + except BaseException: + result_q.put((False, traceback.format_exc())) + + +class _ProcessWorker: + """Small stdlib actor wrapper for one environment process.""" + + def __init__(self, ctx, config, seed, is_train, eval_dataset, gamefile=None): + self.cmd_q = ctx.Queue(maxsize=1) + self.result_q = ctx.Queue(maxsize=1) + self.process = ctx.Process( + target=_worker_loop, + args=(self.cmd_q, self.result_q, config, seed, is_train, eval_dataset, gamefile), + ) + self.process.start() + ok, payload = self.result_q.get() + if not ok: + self.close(kill=True) + raise RuntimeError(f"Failed to start ALFWorld worker:\n{payload}") + + def send(self, cmd, payload=None): + self.cmd_q.put((cmd, payload)) + + def recv(self): + ok, payload = self.result_q.get() + if not ok: + raise RuntimeError(f"ALFWorld worker failed:\n{payload}") + return payload + + def close(self, kill=False): + if self.process.is_alive() and not kill: + try: + self.send("close") + self.recv() + except Exception: + kill = True + if kill and self.process.is_alive(): + self.process.terminate() + self.process.join(timeout=5) + if self.process.is_alive(): + self.process.kill() + self.process.join(timeout=1) + self.cmd_q.close() + self.result_q.close() + + +class AlfworldEnvs(gym.Env): + """Vectorized ALFWorld environment using local process workers.""" + + def __init__(self, alf_config_path, seed, env_num, group_n, + resources_per_worker, is_train=True, env_kwargs=None, gamefiles=None): + super().__init__() + if env_kwargs is None: + env_kwargs = {} + + eval_dataset = env_kwargs.get('eval_dataset', 'eval_in_distribution') + config = load_config_file(alf_config_path) + env_type = config['env']['type'] + self.multi_modal = (env_type == 'AlfredThorEnv') + self.num_processes = env_num * group_n + self.group_n = group_n + self.gamefiles = list(gamefiles or []) + if self.gamefiles and len(self.gamefiles) != self.num_processes: + raise ValueError( + f"Expected {self.num_processes} gamefiles, got {len(self.gamefiles)}" + ) + + start_method = os.environ.get("ALFWORLD_WORKER_START_METHOD") or None + ctx = mp.get_context(start_method) if start_method else mp.get_context() + self.workers = [] + for i in range(self.num_processes): + worker_gamefile = self.gamefiles[i] if self.gamefiles else None + worker = _ProcessWorker( + ctx, + config, + seed + (i // self.group_n), + is_train, + eval_dataset, + worker_gamefile, + ) + self.workers.append(worker) + + self.prev_admissible_commands = [None for _ in range(self.num_processes)] + + def step(self, actions): + assert len(actions) == self.num_processes + + for i, worker in enumerate(self.workers): + worker.send("step", actions[i]) + results = [worker.recv() for worker in self.workers] + + text_obs_list = [] + rewards_list = [] + dones_list = [] + info_list = [] + + for i, (obs, scores, dones, info) in enumerate(results): + for k in info.keys(): + info[k] = info[k][0] + text_obs_list.append(obs[0]) + dones_list.append(dones[0]) + info_list.append(info) + self.prev_admissible_commands[i] = info['admissible_commands'] + rewards_list.append(compute_reward(info, self.multi_modal)) + + image_obs_list = None + return text_obs_list, image_obs_list, rewards_list, dones_list, info_list + + def reset(self): + for worker in self.workers: + worker.send("reset") + results = [worker.recv() for worker in self.workers] + + text_obs_list = [] + info_list = [] + + for i, (obs, info) in enumerate(results): + for k in info.keys(): + info[k] = info[k][0] + text_obs_list.append(obs[0]) + self.prev_admissible_commands[i] = info['admissible_commands'] + info_list.append(info) + + image_obs_list = None + return text_obs_list, image_obs_list, info_list + + @property + def get_admissible_commands(self): + return self.prev_admissible_commands + + def close(self): + for worker in self.workers: + worker.close() + + +def build_alfworld_envs(alf_config_path, seed, env_num, group_n, + resources_per_worker, is_train=True, env_kwargs=None, gamefiles=None): + """Build vectorized ALFWorld environments.""" + return AlfworldEnvs( + alf_config_path, seed, env_num, group_n, + resources_per_worker, is_train, env_kwargs, gamefiles, + ) diff --git a/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/alfworld/vendor/alfworld_projection.py b/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/alfworld/vendor/alfworld_projection.py new file mode 100644 index 00000000..8c499ff4 --- /dev/null +++ b/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/alfworld/vendor/alfworld_projection.py @@ -0,0 +1,60 @@ +# Vendored from SkillRL (Apache-2.0 License) +# Original: agent_system/environments/env_package/alfworld/projection.py + +from typing import List +import re + + +def alfworld_projection(actions: List[str], action_pools: List[List[str]]): + """Process raw model outputs into valid ALFWorld actions. + + Extracts text from ``...`` tags and validates that + the response also contains ``...`` tags. + + Parameters + ---------- + actions : list[str] + Raw model outputs, one per environment. + action_pools : list[list[str]] + Admissible action lists per environment (unused but kept for API compat). + + Returns + ------- + actions : list[str] + Cleaned action strings. + valids : list[int] + 1 if the action was successfully parsed, 0 otherwise. + """ + valids = [0] * len(actions) + + for i in range(len(actions)): + original_str = actions[i] + actions[i] = actions[i].lower() + + start_tag = "" + end_tag = "" + start_idx = actions[i].find(start_tag) + end_idx = actions[i].find(end_tag) + try: + if start_idx == -1 or end_idx == -1: + actions[i] = actions[i][-30:] + continue + + extracted_action = actions[i][start_idx + len(start_tag):end_idx].strip().lower() + actions[i] = extracted_action + valids[i] = 1 + + except Exception: + actions[i] = actions[i][-30:] + + # Require ... + think_start_idx = original_str.find("") + think_end_idx = original_str.find("") + if think_start_idx == -1 or think_end_idx == -1: + valids[i] = 0 + + # Reject responses containing Chinese characters + if re.search(r'[\u4e00-\u9fff]', original_str): + valids[i] = 0 + + return actions, valids diff --git a/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/alfworld/vendor/alfworld_prompts.py b/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/alfworld/vendor/alfworld_prompts.py new file mode 100644 index 00000000..bb7ec49c --- /dev/null +++ b/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/alfworld/vendor/alfworld_prompts.py @@ -0,0 +1,8 @@ +# Vendored from SkillRL (Apache-2.0 License) +# Original: agent_system/environments/prompts/alfworld.py + +from skillopt.prompts import load_prompt + +ALFWORLD_TEMPLATE_NO_HIS = load_prompt("rollout_no_history", env="alfworld") +ALFWORLD_TEMPLATE = load_prompt("rollout_with_history", env="alfworld") +ALFWORLD_TEMPLATE_WITH_MEMORY = load_prompt("rollout_with_memory", env="alfworld") diff --git a/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/alfworld/vendor/config_tw.yaml b/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/alfworld/vendor/config_tw.yaml new file mode 100644 index 00000000..e9bf169b --- /dev/null +++ b/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/alfworld/vendor/config_tw.yaml @@ -0,0 +1,145 @@ +dataset: + data_path: '$ALFWORLD_DATA/json_2.1.1/train' + eval_id_data_path: '$ALFWORLD_DATA/json_2.1.1/valid_seen' # null/None to disable + eval_ood_data_path: '$ALFWORLD_DATA/json_2.1.1/valid_unseen' # null/None to disable + num_train_games: -1 # max training games (<=0 indicates full dataset) + num_eval_games: -1 # max evaluation games (<=0 indicates full dataset) + +logic: + domain: '$ALFWORLD_DATA/logic/alfred.pddl' # PDDL domain file that defines the world dynamics + grammar: '$ALFWORLD_DATA/logic/alfred.twl2' # Grammar file that defines the text feedbacks + +env: + type: 'AlfredTWEnv' # 'AlfredTWEnv' or 'AlfredThorEnv' or 'AlfredHybrid' + # regen_game_files: False # check if game is solvable by expert and save to game.tw-pddl file + domain_randomization: False # shuffle Textworld print order and object id nums + task_types: [1, 2, 3, 4, 5, 6] # task-type ids: 1 - Pick & Place, 2 - Examine in Light, 3 - Clean & Place, 4 - Heat & Place, 5 - Cool & Place, 6 - Pick Two & Place + expert_timeout_steps: 150 # max steps before timeout for expert to solve the task + expert_type: "handcoded" # 'handcoded' or 'planner'. Note: the planner is very slow for real-time use + goal_desc_human_anns_prob: 0.0 # prob of using human-annotated goal language instead of templated goals (1.0 indicates all human annotations from ALFRED) + + hybrid: + start_eps: 100000 # starting episode of hybrid training, tw-only training upto this point + thor_prob: 0.5 # prob of AlfredThorEnv during hybrid training + eval_mode: "tw" # 'tw' or 'thor' - env used for evaluation during hybrid training + + thor: + screen_width: 300 # width of THOR window + screen_height: 300 # height of THOR window + smooth_nav: False # smooth rotations, looks, and translations during navigation (very slow) + save_frames_to_disk: False # save frame PNGs to disk (useful for making videos) + save_frames_path: './videos/' # path to save frame PNGs + +controller: + type: 'oracle' # 'oracle' or 'oracle_astar' or 'mrcnn' or 'mrcnn_astar' (aka BUTLER) + debug: False + load_receps: True # load receptacle locations from precomputed dict (if available) + +mask_rcnn: + pretrained_model_path: '$ALFWORLD_DATA/detectors/mrcnn.pth' + +general: + random_seed: 42 + use_cuda: True # disable this when running on machine without cuda + visdom: False # plot training/eval curves, run with visdom server + task: 'alfred' + training_method: 'dagger' # 'dqn' or 'dagger' + save_path: './training/' # path to save pytorch models + observation_pool_capacity: 3 # k-size queue, 0 indicates no observation + hide_init_receptacles: False # remove initial observation containing navigable receptacles + + training: + batch_size: 10 + max_episode: 50000 + smoothing_eps: 0.1 + optimizer: + learning_rate: 0.001 + clip_grad_norm: 5 + + evaluate: + run_eval: True + batch_size: 10 + env: + type: "AlfredTWEnv" + + checkpoint: + report_frequency: 1000 # report every N episode + experiment_tag: 'test' # name of experiment + load_pretrained: False # during test, enable this so that the agent load your pretrained model + load_from_tag: 'not loading anything' # name of pre-trained model to load in save_path + + model: + encoder_layers: 1 + decoder_layers: 1 + encoder_conv_num: 5 + block_hidden_dim: 64 + n_heads: 1 + dropout: 0.1 + block_dropout: 0.1 + recurrent: True + +rl: + action_space: "admissible" # 'admissible' (candidates from text engine) or 'generation' (seq2seq-style generation) or 'beam_search_choice' or 'exhaustive' (not working) + max_target_length: 20 # max token length for seq2seq generation + beam_width: 10 # 1 means greedy + generate_top_k: 3 + + training: + max_nb_steps_per_episode: 50 # terminate after this many steps + learn_start_from_this_episode: 0 # delay updates until this epsiode + target_net_update_frequency: 500 # sync target net with online net per this many epochs + + replay: + accumulate_reward_from_final: True + count_reward_lambda: 0.0 # 0 to disable + novel_object_reward_lambda: 0.0 # 0 to disable + discount_gamma_game_reward: 0.9 + discount_gamma_count_reward: 0.5 + discount_gamma_novel_object_reward: 0.5 + replay_memory_capacity: 500000 # adjust this depending on your RAM size + replay_memory_priority_fraction: 0.5 + update_per_k_game_steps: 5 + replay_batch_size: 64 + multi_step: 3 + replay_sample_history_length: 4 + replay_sample_update_from: 2 + + epsilon_greedy: + noisy_net: False # if this is true, then epsilon greedy is disabled + epsilon_anneal_episodes: 1000 # -1 if not annealing + epsilon_anneal_from: 0.3 + epsilon_anneal_to: 0.1 + +dagger: + action_space: "generation" # 'admissible' (candidates from text engine) or 'generation' (seq2seq-style generation) or 'exhaustive' (not working) + max_target_length: 20 # max token length for seq2seq generation + beam_width: 10 # 1 means greedy + generate_top_k: 5 + unstick_by_beam_search: False # use beam-search for failed actions, set True during evaluation + + training: + max_nb_steps_per_episode: 50 # terminate after this many steps + + fraction_assist: + fraction_assist_anneal_episodes: 50000 + fraction_assist_anneal_from: 1.0 + fraction_assist_anneal_to: 0.01 + + fraction_random: + fraction_random_anneal_episodes: 0 + fraction_random_anneal_from: 0.0 + fraction_random_anneal_to: 0.0 + + replay: + replay_memory_capacity: 500000 + update_per_k_game_steps: 5 + replay_batch_size: 64 + replay_sample_history_length: 4 + replay_sample_update_from: 2 + +vision_dagger: + model_type: "resnet" # 'resnet' (whole image features) or 'maskrcnn_whole' (whole image MaskRCNN feats) or 'maskrcnn' (top k MaskRCNN detection feats) or 'no_vision' (zero vision input) + resnet_fc_dim: 64 + maskrcnn_top_k_boxes: 10 # top k box features + use_exploration_frame_feats: False # append feats from initial exploration (memory intensive!) + sequence_aggregation_method: "average" # 'sum' or 'average' or 'rnn' diff --git a/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/alfworld/vendor/env_base.py b/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/alfworld/vendor/env_base.py new file mode 100644 index 00000000..00affa72 --- /dev/null +++ b/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/alfworld/vendor/env_base.py @@ -0,0 +1,84 @@ +# Vendored from SkillRL (Apache-2.0 License) +# Original: agent_system/environments/base.py +# Trimmed to only include what ALFWorld needs. + +from typing import List, Tuple, Dict, Any +import numpy as np +from collections import defaultdict + + +def to_numpy(data): + """Convert data to numpy array.""" + # Lazy-check for torch.Tensor to avoid hard dependency on torch + _torch_tensor = None + try: + import torch + _torch_tensor = torch.Tensor + except ImportError: + pass + + if _torch_tensor is not None and isinstance(data, _torch_tensor): + data = data.detach().cpu().numpy() + elif isinstance(data, np.ndarray): + pass + elif isinstance(data, (int, float, bool, Tuple, List)): + data = np.array(data) + else: + raise ValueError(f"Unsupported type: {type(data)})") + return data + + +class EnvironmentManagerBase: + """Base class for vectorized environment managers. + + Manages a set of parallel environments, handles action projection, + observation post-processing, and history tracking. + """ + + def __init__(self, envs, projection_f, config): + self.envs = envs + self.projection_f = projection_f + self.config = config + + def reset(self, kwargs) -> Dict[str, Any]: + obs, infos = self.envs.reset() + return {'text': None, 'image': obs, 'anchor': None}, infos + + def step(self, text_actions: List[str]): + actions, valids = self.projection_f(text_actions) + next_obs, rewards, dones, infos = self.envs.step(actions) + + next_observations = { + 'text': None, + 'image': next_obs, + 'anchor': None, + } + for i, info in enumerate(infos): + info['is_action_valid'] = to_numpy(valids[i]) + + rewards = to_numpy(rewards) + dones = to_numpy(dones) + return next_observations, rewards, dones, infos + + def close(self) -> None: + self.envs.close() + + def success_evaluator(self, *args, **kwargs) -> Dict[str, np.ndarray]: + total_infos = kwargs['total_infos'] + total_batch_list = kwargs['total_batch_list'] + batch_size = len(total_batch_list) + + success = defaultdict(list) + for bs in range(batch_size): + self._process_batch(bs, total_batch_list, total_infos, success) + assert len(success['success_rate']) == batch_size + return {key: np.array(value) for key, value in success.items()} + + def _process_batch(self, batch_idx, total_batch_list, total_infos, success): + for i in reversed(range(len(total_batch_list[batch_idx]))): + batch_item = total_batch_list[batch_idx][i] + if batch_item['active_masks']: + info = total_infos[batch_idx][i] + won_value = float(info['won']) + success['success_rate'].append(won_value) + return diff --git a/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/alfworld/vendor/env_manager.py b/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/alfworld/vendor/env_manager.py new file mode 100644 index 00000000..d937e4d3 --- /dev/null +++ b/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/alfworld/vendor/env_manager.py @@ -0,0 +1,139 @@ +# Vendored from SkillRL (Apache-2.0 License) +# Original: agent_system/environments/env_manager.py +# Trimmed to only include AlfWorldEnvironmentManager and its helpers. + +from typing import List, Dict, Any +from collections import defaultdict +import numpy as np + +from skillopt.envs.alfworld.vendor.env_base import EnvironmentManagerBase, to_numpy +from skillopt.envs.alfworld.vendor.alfworld_prompts import ( + ALFWORLD_TEMPLATE, + ALFWORLD_TEMPLATE_NO_HIS, + ALFWORLD_TEMPLATE_WITH_MEMORY, +) +from skillopt.envs.alfworld.vendor.memory import SimpleMemory + + +def parse_gamefile(infos): + gamefile = [] + for info in infos: + if 'extra.gamefile' in info: + gamefile.append(info['extra.gamefile']) + else: + gamefile.append(None) + return gamefile + + +def set_gamefile(infos, gamefile): + for i in range(len(infos)): + if 'extra.gamefile' in infos[i]: + infos[i]['extra.gamefile'] = gamefile[i] + else: + infos[i]['extra.gamefile'] = None + return infos + + +class AlfWorldEnvironmentManager(EnvironmentManagerBase): + """Manages parallel ALFWorld environments with observation templating.""" + + def __init__(self, envs, projection_f, config): + self.memory = SimpleMemory() + self.retrieval_memory = None + super().__init__(envs, projection_f, config) + + def reset(self, kwargs): + text_obs, image_obs, infos = self.envs.reset() + self.gamefile = parse_gamefile(infos) + self.memory.reset(batch_size=len(text_obs)) + self.tasks = [] + self.pre_text_obs = text_obs + self.extract_task(text_obs) + + full_text_obs = self.build_text_obs(text_obs, self.envs.get_admissible_commands, init=True) + return {'text': full_text_obs, 'image': image_obs, 'anchor': text_obs}, infos + + def step(self, text_actions: List[str]): + actions, valids = self.projection_f(text_actions, self.envs.get_admissible_commands) + text_obs, image_obs, rewards, dones, infos = self.envs.step(actions) + self.memory.store({'text_obs': self.pre_text_obs, 'action': actions}) + self.pre_text_obs = text_obs + + full_text_obs = self.build_text_obs(text_obs, self.envs.get_admissible_commands) + if infos[0].get("extra.gamefile") is None: + infos = set_gamefile(infos, self.gamefile) + + for i, info in enumerate(infos): + info['is_action_valid'] = to_numpy(valids[i]) + + next_observations = {'text': full_text_obs, 'image': image_obs, 'anchor': text_obs} + rewards = to_numpy(rewards) + dones = to_numpy(dones) + return next_observations, rewards, dones, infos + + def extract_task(self, text_obs: List[str]): + for obs in text_obs: + task_start = obs.find('Your task is to: ') + if task_start != -1: + self.tasks.append(obs[task_start + len('Your task is to: '):].strip()) + else: + raise ValueError("Task description not found in text observation.") + + def build_text_obs(self, text_obs: List[str], admissible_actions: List[List[str]], init: bool = False) -> List[str]: + postprocess_text_obs = [] + if not init and self.config.env.history_length > 0: + memory_contexts, valid_lens = self.memory.fetch( + self.config.env.history_length, + obs_key="text_obs", + action_key="action", + ) + + for i in range(len(text_obs)): + reformatted_admissible_actions = "\n ".join( + f"'{s}'" for s in admissible_actions[i] if s != 'help' + ) + + if init or self.config.env.history_length <= 0: + obs = ALFWORLD_TEMPLATE_NO_HIS.format( + current_observation=text_obs[i], + admissible_actions=reformatted_admissible_actions, + ) + else: + obs = ALFWORLD_TEMPLATE.format( + task_description=self.tasks[i], + step_count=len(self.memory[i]), + history_length=valid_lens[i], + action_history=memory_contexts[i], + current_step=len(self.memory[i]) + 1, + current_observation=text_obs[i], + admissible_actions=reformatted_admissible_actions, + ) + postprocess_text_obs.append(obs) + return postprocess_text_obs + + def _process_batch(self, batch_idx, total_batch_list, total_infos, success): + for i in reversed(range(len(total_batch_list[batch_idx]))): + batch_item = total_batch_list[batch_idx][i] + if batch_item['active_masks']: + info = total_infos[batch_idx][i] + won_value = float(info['won']) + success['success_rate'].append(won_value) + + gamefile = info.get("extra.gamefile") + if gamefile: + self._process_gamefile(gamefile, won_value, success) + return + + def _process_gamefile(self, gamefile, won_value, success): + tasks = [ + "pick_and_place", + "pick_two_obj_and_place", + "look_at_obj_in_light", + "pick_heat_then_place_in_recep", + "pick_cool_then_place_in_recep", + "pick_clean_then_place_in_recep", + ] + for task in tasks: + if task in gamefile: + success[f"{task}_success_rate"].append(won_value) + break diff --git a/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/alfworld/vendor/memory.py b/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/alfworld/vendor/memory.py new file mode 100644 index 00000000..045f3064 --- /dev/null +++ b/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/alfworld/vendor/memory.py @@ -0,0 +1,87 @@ +# Vendored from SkillRL (Apache-2.0 License) +# Original: agent_system/memory/base.py + agent_system/memory/memory.py +# Merged into a single file for simplicity. + +from abc import ABC, abstractmethod +from typing import List, Dict, Any, Tuple + + +class BaseMemory(ABC): + """Base class for memory management.""" + + @abstractmethod + def __len__(self): + pass + + @abstractmethod + def __getitem__(self, idx: int): + pass + + @abstractmethod + def reset(self, batch_size: int): + pass + + @abstractmethod + def store(self, record: Dict[str, List[Any]]): + pass + + @abstractmethod + def fetch(self, step: int): + pass + + +class SimpleMemory(BaseMemory): + """Per-environment history buffer for storing observations and actions.""" + + def __init__(self): + self._data = None + self.keys = None + self.batch_size = 0 + + def __len__(self): + return len(self._data) + + def __getitem__(self, idx): + return self._data[idx] + + def reset(self, batch_size: int): + if self._data is not None: + self._data.clear() + self._data = [[] for _ in range(batch_size)] + self.batch_size = batch_size + self.keys = None + + def store(self, record: Dict[str, List[Any]]): + if self.keys is None: + self.keys = list(record.keys()) + assert self.keys == list(record.keys()) + + for env_idx in range(self.batch_size): + self._data[env_idx].append({k: record[k][env_idx] for k in self.keys}) + + def fetch( + self, + history_length: int, + obs_key: str = "text_obs", + action_key: str = "action", + ) -> Tuple[List[str], List[int]]: + memory_contexts, valid_lengths = [], [] + + for env_idx in range(self.batch_size): + recent = self._data[env_idx][-history_length:] + valid_len = len(recent) + start_idx = len(self._data[env_idx]) - valid_len + + lines = [] + for j, rec in enumerate(recent): + step_num = start_idx + j + 1 + act = rec[action_key] + obs = rec[obs_key] + lines.append( + f"[Observation {step_num}: '{obs}', Action {step_num}: '{act}']" + ) + + memory_contexts.append("\n".join(lines)) + valid_lengths.append(valid_len) + + return memory_contexts, valid_lengths diff --git a/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/base.py b/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/base.py new file mode 100644 index 00000000..c2e57eaa --- /dev/null +++ b/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/base.py @@ -0,0 +1,309 @@ +"""ReflACT environment adapter — abstract interface. + +To connect ReflACT to a new environment (benchmark, simulator, etc.), +implement a subclass of :class:`EnvAdapter` with environment-specific +rollout and reflection logic. + +Example:: + + class MyBenchAdapter(EnvAdapter): + def build_train_env(self, batch_size, seed, **kw): + return MyEnvManager(split="train", n=batch_size, seed=seed) + + def build_eval_env(self, env_num, split, seed, **kw): + return MyEnvManager(split=split, n=env_num, seed=seed) + + def rollout(self, env_manager, skill_content, out_dir, **kw): + # Run episodes, return [{"id": ..., "hard": 0/1, "soft": 0.0-1.0, ...}] + ... + + def reflect(self, results, skill_content, out_dir, **kw): + # Analyze trajectories, return list of patch dicts + ... + + def get_task_types(self): + return ["task_a", "task_b"] +""" +from __future__ import annotations + +from abc import ABC, abstractmethod +import os +import random + +from skillopt.datasets.base import BaseDataLoader, BatchSpec +from skillopt.prompts import load_prompt + + +class EnvAdapter(ABC): + """Abstract adapter for connecting ReflACT to any environment. + + Subclasses must implement all abstract methods. The ReflACT trainer + calls these methods at the appropriate pipeline stages. + """ + + # ── Lifecycle hooks ──────────────────────────────────────────────────── + + def setup(self, cfg: dict) -> None: + """Called once by the trainer before the training loop begins. + + Override to perform one-time initialization that requires the full + config (e.g., data loading, split creation). Default is a no-op. + """ + self._cfg = dict(cfg) + + def get_dataloader(self) -> BaseDataLoader | None: + """Return the task dataloader used by this adapter, if any.""" + return None + + def requires_ray(self) -> bool: + """Return whether this adapter requires Ray runtime initialization.""" + return False + + def build_reference_text(self, item: dict) -> str: + """Return hidden reference material for reflection, if any.""" + return str(item.get("reference_text") or "").strip() + + def get_reference_metadata(self, item: dict) -> dict: + """Return structured metadata about hidden reference material.""" + reference_text = self.build_reference_text(item) + if not reference_text: + return {"fields": [], "preview": ""} + return { + "fields": ["reference_text"], + "preview": reference_text[:400], + } + + def attach_reference_context( + self, + results: list[dict], + items: list[dict] | None, + ) -> list[dict]: + """Attach environment-specific hidden reference text to result dicts.""" + if not results or not items: + return list(results) + + item_by_id = { + str(item.get("id")): item + for item in items + if isinstance(item, dict) and item.get("id") is not None + } + enriched: list[dict] = [] + for row in results: + merged = dict(row) + item = item_by_id.get(str(row.get("id"))) + if item: + reference_text = self.build_reference_text(item) + if reference_text: + merged["reference_text"] = reference_text + enriched.append(merged) + return enriched + + def select_representative_items( + self, + results: list[dict], + items: list[dict] | None, + *, + n_failures: int, + n_successes: int, + seed: int | None = None, + ) -> list[dict]: + """Select a small diverse subset of current-batch items by outcome.""" + if not items: + return [] + + item_by_id = { + str(item.get("id")): item + for item in items + if isinstance(item, dict) and item.get("id") is not None + } + failures = [ + (result, item_by_id[str(result.get("id"))]) + for result in results + if not result.get("hard") and str(result.get("id")) in item_by_id + ] + successes = [ + (result, item_by_id[str(result.get("id"))]) + for result in results + if result.get("hard") and str(result.get("id")) in item_by_id + ] + + rng = random.Random(seed) + + def _pick(pool: list[tuple[dict, dict]], quota: int) -> list[dict]: + if quota <= 0 or not pool: + return [] + shuffled = list(pool) + rng.shuffle(shuffled) + + picked_ids: set[str] = set() + picked: list[dict] = [] + seen_types: set[str] = set() + + for result, item in shuffled: + task_type = str(result.get("task_type") or item.get("task_type") or item.get("subtype") or "unknown") + item_id = str(item["id"]) + if task_type in seen_types or item_id in picked_ids: + continue + picked.append(item) + picked_ids.add(item_id) + seen_types.add(task_type) + if len(picked) >= quota: + return picked + + for _, item in shuffled: + item_id = str(item["id"]) + if item_id in picked_ids: + continue + picked.append(item) + picked_ids.add(item_id) + if len(picked) >= quota: + break + return picked + + selected = _pick(failures, n_failures) + selected_ids = {str(item["id"]) for item in selected} + selected.extend( + item for item in _pick(successes, n_successes) + if str(item["id"]) not in selected_ids + ) + return selected + + def build_env_from_batch(self, batch: BatchSpec, **kwargs): + """Build an environment manager or item list from a :class:`BatchSpec`. + + Default behavior preserves the legacy adapter API by routing training + batches through :meth:`build_train_env` and evaluation batches through + :meth:`build_eval_env`. + """ + if batch.phase == "train": + return self.build_train_env(batch_size=batch.batch_size, seed=batch.seed, **kwargs) + return self.build_eval_env( + env_num=batch.batch_size, + split=batch.split, + seed=batch.seed, + **kwargs, + ) + + @abstractmethod + def build_train_env(self, batch_size: int, seed: int, **kwargs): + """Build a training environment manager. + + Returns + ------- + object + An environment manager that can be passed to :meth:`rollout`. + """ + + @abstractmethod + def build_eval_env(self, env_num: int, split: str, seed: int, **kwargs): + """Build an evaluation environment manager. + + Parameters + ---------- + env_num : int + Number of evaluation environments. + split : str + Dataset split (e.g. ``"valid_seen"``, ``"valid_unseen"``). + seed : int + Random seed for reproducibility. + + Returns + ------- + object + An environment manager that can be passed to :meth:`rollout`. + """ + + @abstractmethod + def rollout( + self, + env_manager, + skill_content: str, + out_dir: str, + **kwargs, + ) -> list[dict]: + """Run a batch of episodes using the current skill. + + Returns + ------- + list[dict] + Each dict conforms to :class:`~skillopt.types.RolloutResult`: + must have ``"id"`` (str), ``"hard"`` (0/1), ``"soft"`` + (float 0-1). May include env-specific fields. + """ + + @abstractmethod + def reflect( + self, + results: list[dict], + skill_content: str, + out_dir: str, + **kwargs, + ) -> list[dict | None]: + """Analyze rollout results and produce patches. + + Each returned dict conforms to :class:`~skillopt.types.RawPatch`: + ``"patch"`` (with ``"edits"`` list) + ``"source_type"`` + (``"failure"`` or ``"success"``). + + Returns + ------- + list[dict | None] + Raw analyst outputs; ``None`` entries are filtered out. + """ + + @abstractmethod + def get_task_types(self) -> list[str]: + """Return the list of task type names for this environment.""" + + # ── Prompt configuration (two-level priority) ──────────────────────── + # + # Priority: env-specific prompt file > generic default prompt file. + # + # Prompts are loaded from ``.md`` files via ``load_prompt(name, env)``: + # 1. ``skillopt/envs//prompts/.md`` (env-specific) + # 2. ``skillopt/prompts/.md`` (generic fallback) + # + # Subclasses can still override ``get_*_prompt()`` for full control. + + @property + def _env_name(self) -> str: + """Derive the env directory name from this adapter's module path.""" + # e.g. "skillopt.envs.searchqa.adapter" → "searchqa" + module = type(self).__module__ + parts = module.split(".") + if len(parts) >= 3 and parts[-3] == "envs": + return parts[-2] + return "" + + def _load_env_prompt(self, name: str) -> str | None: + """Load a prompt with env-specific override. Returns None if not found.""" + try: + return load_prompt(name, env=self._env_name) + except FileNotFoundError: + return None + + def get_error_minibatch_prompt(self) -> str | None: + update_mode = getattr(self, "_cfg", {}).get("skill_update_mode", "patch") + raw_mode = str(update_mode).strip().lower() + if raw_mode in {"full_rewrite", "full_rewrite_minibatch", "minibatch_full_rewrite", "skill_rewrite_minibatch"}: + prompt = self._load_env_prompt("analyst_error_full_rewrite") + if prompt is not None: + return prompt + if raw_mode in {"rewrite", "rewrite_from_suggestions", "suggestions", "rewrite_suggestions"}: + prompt = self._load_env_prompt("analyst_error_rewrite") + if prompt is not None: + return prompt + return self._load_env_prompt("analyst_error") + + def get_success_minibatch_prompt(self) -> str | None: + update_mode = getattr(self, "_cfg", {}).get("skill_update_mode", "patch") + raw_mode = str(update_mode).strip().lower() + if raw_mode in {"full_rewrite", "full_rewrite_minibatch", "minibatch_full_rewrite", "skill_rewrite_minibatch"}: + prompt = self._load_env_prompt("analyst_success_full_rewrite") + if prompt is not None: + return prompt + if raw_mode in {"rewrite", "rewrite_from_suggestions", "suggestions", "rewrite_suggestions"}: + prompt = self._load_env_prompt("analyst_success_rewrite") + if prompt is not None: + return prompt + return self._load_env_prompt("analyst_success") diff --git a/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/docvqa/__init__.py b/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/docvqa/__init__.py new file mode 100644 index 00000000..38c999da --- /dev/null +++ b/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/docvqa/__init__.py @@ -0,0 +1 @@ +"""DocVQA environment package for ReflACT.""" diff --git a/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/docvqa/adapter.py b/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/docvqa/adapter.py new file mode 100644 index 00000000..91849061 --- /dev/null +++ b/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/docvqa/adapter.py @@ -0,0 +1,115 @@ +from __future__ import annotations + +import os + +from skillopt.datasets.base import BatchSpec +from skillopt.envs.base import EnvAdapter +from skillopt.envs.docvqa.dataloader import DocVQADataLoader +from skillopt.envs.docvqa.rollout import run_batch +from skillopt.gradient.reflect import run_minibatch_reflect + + +class DocVQAAdapter(EnvAdapter): + def __init__( + self, + split_dir: str = "", + data_path: str = "", + split_mode: str = "split_dir", + split_ratio: str = "2:1:7", + split_seed: int = 42, + split_output_dir: str = "", + max_turns: int = 1, + exec_timeout: int = 120, + workers: int = 16, + analyst_workers: int = 16, + failure_only: bool = False, + minibatch_size: int = 8, + edit_budget: int = 4, + seed: int = 42, + limit: int = 0, + image_detail: str = "auto", + max_completion_tokens: int = 16384, + ) -> None: + self.max_turns = max_turns + self.exec_timeout = exec_timeout + self.workers = workers + self.max_completion_tokens = int(max_completion_tokens) + self.analyst_workers = analyst_workers + self.failure_only = failure_only + self.minibatch_size = minibatch_size + self.edit_budget = edit_budget + self.image_detail = image_detail + self.dataloader = DocVQADataLoader( + split_dir=split_dir, + data_path=data_path, + split_mode=split_mode, + split_ratio=split_ratio, + split_seed=split_seed, + split_output_dir=split_output_dir, + seed=seed, + limit=limit, + ) + + def setup(self, cfg: dict) -> None: + super().setup(cfg) + self.dataloader.setup(cfg) + + def get_dataloader(self): + return self.dataloader + + def build_env_from_batch(self, batch: BatchSpec, **kwargs): + return list(batch.payload or []) + + def build_train_env(self, batch_size: int, seed: int, **kwargs): + batch = self.dataloader.build_train_batch(batch_size=batch_size, seed=seed, **kwargs) + return self.build_env_from_batch(batch, **kwargs) + + def build_eval_env(self, env_num: int, split: str, seed: int, **kwargs): + batch = self.dataloader.build_eval_batch(env_num=env_num, split=split, seed=seed, **kwargs) + return self.build_env_from_batch(batch, **kwargs) + + def rollout(self, env_manager, skill_content: str, out_dir: str, **kwargs) -> list[dict]: + items: list[dict] = env_manager + return run_batch( + items=items, + out_root=out_dir, + skill_content=skill_content, + max_turns=self.max_turns, + exec_timeout=self.exec_timeout, + workers=self.workers, + image_detail=self.image_detail, + max_completion_tokens=self.max_completion_tokens, + diagnostic_mode=kwargs.get("diagnostic_mode", False), + diagnostic_instruction=kwargs.get("diagnostic_instruction", ""), + task_timeout=self.exec_timeout, + ) + + def reflect(self, results: list[dict], skill_content: str, out_dir: str, **kwargs) -> list[dict | None]: + prediction_dir = kwargs.get("prediction_dir", os.path.join(out_dir, "predictions")) + patches_dir = kwargs.get("patches_dir", os.path.join(out_dir, "patches")) + random_seed = kwargs.get("random_seed") + step_buffer_context = kwargs.get("step_buffer_context", "") + return run_minibatch_reflect( + results=results, + skill_content=skill_content, + prediction_dir=prediction_dir, + patches_dir=patches_dir, + workers=self.analyst_workers, + failure_only=self.failure_only, + minibatch_size=self.minibatch_size, + edit_budget=self.edit_budget, + random_seed=random_seed, + error_system=self.get_error_minibatch_prompt(), + success_system=self.get_success_minibatch_prompt(), + step_buffer_context=step_buffer_context, + update_mode=getattr(self, "_cfg", {}).get("skill_update_mode", "patch"), + ) + + + def get_task_types(self) -> list[str]: + seen: list[str] = [] + for item in self.dataloader.train_items + self.dataloader.val_items + self.dataloader.test_items: + task_type = str(item.get("task_type") or "docvqa") + if task_type not in seen: + seen.append(task_type) + return seen or ["docvqa"] diff --git a/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/docvqa/dataloader.py b/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/docvqa/dataloader.py new file mode 100644 index 00000000..212f0ef0 --- /dev/null +++ b/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/docvqa/dataloader.py @@ -0,0 +1,61 @@ +from __future__ import annotations + +import ast +import csv +from pathlib import Path + +from skillopt.datasets.base import SplitDataLoader + + +def _parse_answers(raw: str) -> list[str]: + text = str(raw or "").strip() + if not text: + return [] + try: + parsed = ast.literal_eval(text) + except Exception: + return [text] + if isinstance(parsed, list): + return [str(item).strip() for item in parsed if str(item).strip()] + return [str(parsed).strip()] + + +def _extract_document_path(question: str) -> tuple[str, str]: + marker = "document_path:" + if marker not in question: + return question.strip(), "" + main, tail = question.split(marker, 1) + return main.strip(), tail.strip() + + +def _normalize_row(row: dict[str, str]) -> dict: + question_text, document_path = _extract_document_path(str(row.get("question") or "")) + answers = _parse_answers(row.get("answer") or row.get("ground_truth") or "") + image_path = str(row.get("image_path") or document_path or "").strip() + task_type = str(row.get("topic") or row.get("category") or "docvqa").strip() or "docvqa" + return { + "id": str(row.get("questionId") or row.get("id") or "").strip(), + "question": question_text, + "answer": answers[0] if answers else "", + "answers": answers, + "task_type": task_type, + "subtask": task_type, + "image_paths": [image_path] if image_path else [], + "image_path": image_path, + "questionId": str(row.get("questionId") or "").strip(), + "docId": str(row.get("docId") or "").strip(), + "ucsf_document_id": str(row.get("ucsf_document_id") or "").strip(), + "ucsf_document_page_no": str(row.get("ucsf_document_page_no") or "").strip(), + "source_split": str(row.get("source_split") or "").strip(), + } + + +class DocVQADataLoader(SplitDataLoader): + def load_split_items(self, split_path: str) -> list[dict]: + path = Path(split_path) + csv_files = sorted(path.glob("*.csv")) + if not csv_files: + raise FileNotFoundError(f"No .csv file found in {split_path}") + with csv_files[0].open(encoding="utf-8", newline="") as f: + reader = csv.DictReader(f) + return [_normalize_row(row) for row in reader] diff --git a/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/docvqa/evaluator.py b/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/docvqa/evaluator.py new file mode 100644 index 00000000..85c09efa --- /dev/null +++ b/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/docvqa/evaluator.py @@ -0,0 +1,113 @@ +from __future__ import annotations + +import ast +import json +from collections.abc import Iterable +from typing import Any + +DEFAULT_ANLS_THRESHOLD = 0.5 + + +def _normalize_text(value: Any) -> str: + if value is None: + return "" + text = str(value).strip().lower() + return " ".join(text.split()) + + +def _levenshtein_distance(a: str, b: str) -> int: + if a == b: + return 0 + if not a: + return len(b) + if not b: + return len(a) + if len(a) > len(b): + a, b = b, a + previous = list(range(len(b) + 1)) + for i, char_a in enumerate(a, start=1): + current = [i] + for j, char_b in enumerate(b, start=1): + insert_cost = current[j - 1] + 1 + delete_cost = previous[j] + 1 + replace_cost = previous[j - 1] + (char_a != char_b) + current.append(min(insert_cost, delete_cost, replace_cost)) + previous = current + return previous[-1] + + +def _score_single_answer(predicted: Any, target: Any, threshold: float) -> float: + predicted_norm = _normalize_text(predicted) + target_norm = _normalize_text(target) + if not predicted_norm and not target_norm: + return 1.0 + if not predicted_norm or not target_norm: + return 0.0 + distance = _levenshtein_distance(predicted_norm, target_norm) + normalized_distance = distance / max(len(predicted_norm), len(target_norm)) + if normalized_distance >= threshold: + return 0.0 + return 1.0 - normalized_distance + + +def _extract_answer_strings(raw: Any) -> list[str]: + if raw is None: + return [""] + if isinstance(raw, str): + text = raw.strip() + if not text: + return [""] + parsed = None + if text[0] in "[{": + try: + parsed = json.loads(text) + except json.JSONDecodeError: + try: + parsed = ast.literal_eval(text) + except (ValueError, SyntaxError): + parsed = None + if parsed is None: + return [text] + return _extract_answer_strings(parsed) + if isinstance(raw, dict): + for key in ("answers", "ground_truth", "answer"): + if key in raw: + return _extract_answer_strings(raw[key]) + return [str(raw)] + if isinstance(raw, Iterable) and not isinstance(raw, (bytes, bytearray)): + answers: list[str] = [] + for item in raw: + if isinstance(item, dict): + for key in ("text", "answer", "value"): + if key in item: + answers.extend(_extract_answer_strings(item[key])) + break + else: + answers.append(str(item)) + continue + answers.append(str(item)) + return answers or [""] + return [str(raw)] + + +def extract_answer(text: str) -> str: + lower = text.lower() + start = lower.rfind("") + end = lower.rfind("") + if start != -1 and end != -1 and end > start: + return text[start + len(""):end].strip() + lines = [line.strip() for line in text.splitlines() if line.strip()] + return lines[-1] if lines else text.strip() + + +def evaluate(prediction_text: str, gold_answers: Any) -> dict: + answer = extract_answer(prediction_text) + answers = _extract_answer_strings(gold_answers) + score = 0.0 + for target in answers: + score = max(score, _score_single_answer(answer, target, DEFAULT_ANLS_THRESHOLD)) + return { + "anls": score, + "predicted_answer": answer, + "gold_answers": answers, + } diff --git a/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/docvqa/prompts/analyst_error.md b/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/docvqa/prompts/analyst_error.md new file mode 100644 index 00000000..9f6c3672 --- /dev/null +++ b/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/docvqa/prompts/analyst_error.md @@ -0,0 +1,35 @@ +You are an expert failure-analysis agent for visual document question answering tasks. + +You will be given MULTIPLE failed DocVQA trajectories from a single minibatch and the current skill document. Each trajectory includes the model response and an evaluation result scored with ANLS against one or more acceptable answers. + +Your job is to identify the most important COMMON failure patterns across the batch and propose concise skill edits. + +## Failure Type Categories +- evidence_miss: the model overlooked the relevant visible region or line +- near_match_confusion: the model selected a nearby but incorrect text span +- normalization_error: the answer differed mainly in formatting, spacing, punctuation, or minor text normalization +- reading_error: the model misread the document content +- other: none of the above + +## Rules +- Focus on common, reusable reading and extraction behaviors. +- Do not hardcode image-specific answers. +- Prefer concise edits that improve evidence selection and exact span extraction. + +Respond ONLY with a valid JSON object (no markdown fences, no extra text): +{ + "batch_size": , + "failure_summary": [ + {"failure_type": "", "count": , "description": ""} + ], + "patch": { + "reasoning": "", + "edits": [ + {"op": "append", "content": ""}, + {"op": "insert_after", "target": "", "content": ""}, + {"op": "replace", "target": "", "content": ""}, + {"op": "delete", "target": ""} + ] + } +} +Only include edits that are needed. "edits" can be an empty list if no patch is warranted. diff --git a/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/docvqa/prompts/analyst_success.md b/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/docvqa/prompts/analyst_success.md new file mode 100644 index 00000000..2ce71d83 --- /dev/null +++ b/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/docvqa/prompts/analyst_success.md @@ -0,0 +1,24 @@ +You are an expert success-pattern analyst for visual document question answering tasks. + +You will be given MULTIPLE successful DocVQA trajectories from a single minibatch and the current skill document. Your job is to identify common visual reading and exact-answer extraction behaviors worth encoding in the skill. + +## Rules +- Focus on patterns shared across multiple successful trajectories. +- Reinforce reusable behaviors like locating the right region, copying exact spans, and preferring the shortest exact answer over paraphrase. +- Only propose patches for patterns not already captured by the current skill. + +Respond ONLY with a valid JSON object: +{ + "batch_size": , + "success_patterns": ["", ""], + "patch": { + "reasoning": "", + "edits": [ + {"op": "append", "content": ""}, + {"op": "insert_after", "target": "", "content": ""}, + {"op": "replace", "target": "", "content": ""}, + {"op": "delete", "target": ""} + ] + } +} +"edits" may be empty if the skill already covers all observed patterns. diff --git a/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/docvqa/prompts/rollout_system.md b/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/docvqa/prompts/rollout_system.md new file mode 100644 index 00000000..e859c027 --- /dev/null +++ b/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/docvqa/prompts/rollout_system.md @@ -0,0 +1,12 @@ +You are an expert visual document question answering agent. + +{skill_section}You will receive a document image and a question about the document. +Read the visual evidence carefully and answer concisely. + +Rules: +- Ground the answer in the visible document content. +- Prefer exact spans, numbers, dates, and names from the document. +- Do not invent content that is not visible. +- If multiple near-matches exist, choose the one best supported by the document. + +Return the final answer inside .... diff --git a/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/docvqa/rollout.py b/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/docvqa/rollout.py new file mode 100644 index 00000000..6396163f --- /dev/null +++ b/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/docvqa/rollout.py @@ -0,0 +1,391 @@ +from __future__ import annotations + +import json +import os +import time +from concurrent.futures import FIRST_COMPLETED, ThreadPoolExecutor, wait + +from skillopt.envs.docvqa.evaluator import evaluate +from skillopt.model import chat_target_messages, get_target_backend, is_target_exec_backend +from skillopt.model.codex_harness import prepare_workspace, render_skill_md, run_target_exec +from skillopt.prompts import load_prompt + + +def _build_system(skill_content: str) -> str: + if skill_content.strip(): + skill_section = f"## Skill\n{skill_content.strip()}\n\n" + else: + skill_section = "" + return load_prompt("rollout_system", env="docvqa").format(skill_section=skill_section) + + +def _image_to_data_uri(path: str) -> str: + import base64 + import mimetypes + + mime = mimetypes.guess_type(path)[0] or "image/png" + with open(path, "rb") as f: + encoded = base64.b64encode(f.read()).decode("ascii") + return f"data:{mime};base64,{encoded}" + + +def _build_messages( + item: dict, + skill_content: str, + image_detail: str, + *, + diagnostic_mode: bool = False, + diagnostic_instruction: str = "", +) -> tuple[list[dict], str, str]: + system = _build_system(skill_content) + user_text = item["question"] + "\n\nReturn the final answer inside ...." + if diagnostic_mode and diagnostic_instruction.strip(): + user_text += f"\n\n## Training Readout\n{diagnostic_instruction.strip()}" + image_url = {"url": _image_to_data_uri(item["image_path"])} + if image_detail and image_detail != "auto": + image_url["detail"] = image_detail + messages = [ + {"role": "system", "content": system}, + { + "role": "user", + "content": [ + {"type": "text", "text": user_text}, + {"type": "image_url", "image_url": image_url}, + ], + }, + ] + return messages, system, user_text + + +def _build_codex_skill(skill_content: str) -> str: + return render_skill_md( + skill_content, + description="Dynamic ReflACT skill for solving the current DocVQA document-image question.", + preamble=( + "Use this skill when answering the current DocVQA question.\n" + "Inspect the attached document image carefully and return the final answer inside ...." + ), + ) + + +def _run_codex_once( + *, + pred_dir: str, + item: dict, + skill_content: str, + model: str, + timeout: int, + image_detail: str, + diagnostic_mode: bool = False, + diagnostic_instruction: str = "", + previous_response: str = "", +) -> tuple[str, str, str, str]: + _ = image_detail + _messages, _system, user_text = _build_messages( + item, + skill_content, + image_detail, + diagnostic_mode=diagnostic_mode, + diagnostic_instruction=diagnostic_instruction, + ) + task_parts = [user_text] + image_abs = os.path.abspath(item["image_path"]) + task_parts.append( + "## Document Image\n" + "The document image is available in this workspace via `ATTACHMENTS.md`.\n" + f"Original image path: `{image_abs}`\n" + "Open or inspect that image before answering; do not answer from memory." + ) + if previous_response: + task_parts.append( + "## Previous Attempt\n" + f"{previous_response}\n\n" + "Review the same document image carefully and correct the answer if needed." + ) + task_text = "\n\n".join(task_parts) + skill_md = _build_codex_skill(skill_content) + work_dir = os.path.join(pred_dir, "codex_exec") + prepare_workspace( + work_dir=work_dir, + skill_md=skill_md, + task_text=task_text, + images=[item["image_path"]], + ) + prompt = ( + "Use the `skillopt-target` skill available in this workspace.\n" + "Read `task.md`, inspect the attached document image, and answer the DocVQA question.\n" + "Return the final answer inside ...." + ) + final_message, raw = run_target_exec( + work_dir=work_dir, + prompt=prompt, + model=model, + timeout=timeout, + images=[item["image_path"]], + ) + return final_message or raw, raw, skill_md, task_text + + +def process_one( + item: dict, + out_root: str, + skill_content: str, + *, + max_turns: int = 1, + exec_timeout: int = 120, + image_detail: str = "auto", + max_completion_tokens: int = 16384, + diagnostic_mode: bool = False, + diagnostic_instruction: str = "", +) -> dict: + item_id = str(item["id"]) + result = { + "id": item_id, + "question": item["question"], + "task_type": item.get("subtask") or item.get("task_type") or "docvqa", + "task_description": item["question"], + "hard": 0, + "soft": 0.0, + "predicted_answer": "", + "response": "", + "fail_reason": "", + "agent_ok": False, + "n_turns": 0, + "image_paths": item.get("image_paths", []), + "gold_answer": item.get("answers", []), + } + try: + response = "" + system_prompt = "" + user_text = "" + conversation: list[dict] = [] + if is_target_exec_backend(): + from skillopt.model import azure_openai as _llm + + conversation = [ + { + "role": "user", + "content": item["question"] + "\n\n" + f"[image] {os.path.basename(item['image_path'])}", + } + ] + for turn in range(max_turns): + response, _raw, system_prompt, user_text = _run_codex_once( + pred_dir=os.path.join(out_root, "predictions", item_id), + item=item, + skill_content=skill_content, + model=_llm.TARGET_DEPLOYMENT, + timeout=exec_timeout, + image_detail=image_detail, + diagnostic_mode=diagnostic_mode if turn == 0 else False, + diagnostic_instruction=diagnostic_instruction if turn == 0 else "", + previous_response=response if turn > 0 else "", + ) + conversation.append({"type": "message", "turn": turn + 1, "content": response}) + if "" in response.lower(): + break + else: + messages, system_prompt, user_text = _build_messages( + item, + skill_content, + image_detail, + diagnostic_mode=diagnostic_mode, + diagnostic_instruction=diagnostic_instruction, + ) + conversation = [ + { + "role": "user", + "content": user_text + "\n\n" + f"[image] {os.path.basename(item['image_path'])}", + } + ] + for turn in range(max_turns): + if turn == 0: + resp_text, _ = chat_target_messages( + messages=messages, + max_completion_tokens=max_completion_tokens, + retries=5, + stage="rollout", + timeout=exec_timeout, + ) + else: + refinement_messages = [ + messages[0], + messages[1], + {"role": "assistant", "content": response}, + {"role": "user", "content": "Review the same image carefully and answer again. Keep the final answer inside ...."}, + ] + resp_text, _ = chat_target_messages( + messages=refinement_messages, + max_completion_tokens=max_completion_tokens, + retries=5, + stage="rollout", + timeout=exec_timeout, + ) + response = resp_text + conversation.append({"type": "message", "turn": turn + 1, "content": resp_text}) + if "" in resp_text.lower(): + break + + result["response"] = response + result["agent_ok"] = True + result["n_turns"] = len(conversation) - 1 + + pred_dir = os.path.join(out_root, "predictions", item_id) + os.makedirs(pred_dir, exist_ok=True) + with open(os.path.join(pred_dir, "target_system_prompt.txt"), "w", encoding="utf-8") as f: + f.write(system_prompt) + with open(os.path.join(pred_dir, "target_user_prompt.txt"), "w", encoding="utf-8") as f: + f.write(user_text) + + eval_result = evaluate(response, item.get("answers", [])) + result["predicted_answer"] = eval_result["predicted_answer"] + result["hard"] = int(eval_result["anls"] >= 0.999) + result["soft"] = eval_result["anls"] + if result["soft"] <= 0.0: + result["fail_reason"] = f"predicted '{eval_result['predicted_answer']}' but expected one of {item.get('answers', [])}" + + eval_detail = ( + "[EVALUATION RESULT]\n" + f"Question: {item['question']}\n" + f"Predicted answer: {eval_result['predicted_answer']!r}\n" + f"Gold answers: {item.get('answers', [])!r}\n" + f"ANLS: {eval_result['anls']:.4f}" + ) + conversation.append({"role": "system", "content": eval_detail}) + with open(os.path.join(pred_dir, "conversation.json"), "w", encoding="utf-8") as f: + json.dump(conversation, f, ensure_ascii=False, indent=2) + except Exception as e: # noqa: BLE001 + result["fail_reason"] = f"error: {e}" + return result + + +def run_batch( + items: list[dict], + out_root: str, + skill_content: str, + *, + max_turns: int = 1, + exec_timeout: int = 120, + workers: int = 16, + image_detail: str = "auto", + max_completion_tokens: int = 16384, + diagnostic_mode: bool = False, + diagnostic_instruction: str = "", + task_timeout: int = 600, +) -> list[dict]: + task_timeout = max(int(task_timeout), int(exec_timeout) + 60) + results_path = os.path.join(out_root, "results.jsonl") + os.makedirs(out_root, exist_ok=True) + + done_ids: set[str] = set() + existing: list[dict] = [] + if os.path.exists(results_path): + with open(results_path, encoding="utf-8") as f: + for line in f: + try: + row = json.loads(line) + except Exception: + continue + done_ids.add(str(row["id"])) + existing.append(row) + + pending = [item for item in items if str(item["id"]) not in done_ids] + if not pending: + return existing + + def _timeout_result(item: dict) -> dict: + return { + "id": str(item["id"]), + "question": item.get("question", ""), + "task_type": item.get("subtask") or item.get("task_type") or "docvqa", + "task_description": item.get("question", ""), + "hard": 0, + "soft": 0.0, + "predicted_answer": "", + "response": "", + "fail_reason": f"task-timeout-{task_timeout}s", + "agent_ok": False, + "n_turns": 0, + "image_paths": item.get("image_paths", []), + "gold_answer": item.get("answers", []), + "phase": "timeout", + } + + def _error_result(item: dict, exc: Exception) -> dict: + row = _timeout_result(item) + row["phase"] = "error" + row["fail_reason"] = f"unexpected: {type(exc).__name__}: {exc}" + return row + + started_at: dict[str, float] = {} + + def _run_one(item: dict) -> dict: + started_at[str(item["id"])] = time.time() + return process_one( + item, + out_root, + skill_content, + max_turns=max_turns, + exec_timeout=exec_timeout, + image_detail=image_detail, + max_completion_tokens=max_completion_tokens, + diagnostic_mode=diagnostic_mode, + diagnostic_instruction=diagnostic_instruction, + ) + + total = len(existing) + len(pending) + completed = len(existing) + correct = sum(1 for r in existing if r.get("hard", 0)) + if existing: + print(f" [rollout] resuming: {completed}/{total} already done", flush=True) + + results = list(existing) + with open(results_path, "a", encoding="utf-8") as outf: + ex = ThreadPoolExecutor(max_workers=workers) + try: + futs = {ex.submit(_run_one, item): item for item in pending} + pending_futs = set(futs) + while pending_futs: + done, _ = wait(pending_futs, timeout=5, return_when=FIRST_COMPLETED) + now = time.time() + timed_out = [ + fut for fut in pending_futs - done + if str(futs[fut]["id"]) in started_at + and now - started_at[str(futs[fut]["id"])] >= task_timeout + ] + for fut in done: + pending_futs.remove(fut) + item = futs[fut] + try: + res = fut.result() + except Exception as exc: # noqa: BLE001 + res = _error_result(item, exc) + results.append(res) + completed += 1 + if res.get("hard", 0): + correct += 1 + acc = correct / completed if completed else 0 + print( + f" [rollout] {completed}/{total} " + f"(acc={acc:.3f}) id={res['id']} " + f"hard={res.get('hard', '?')}", + flush=True, + ) + outf.write(json.dumps(res, ensure_ascii=False) + "\n") + outf.flush() + for fut in timed_out: + pending_futs.remove(fut) + fut.cancel() + res = _timeout_result(futs[fut]) + results.append(res) + completed += 1 + acc = correct / completed if completed else 0 + print( + f" [rollout] {completed}/{total} " + f"(acc={acc:.3f}) id={res['id']} TIMEOUT", + flush=True, + ) + outf.write(json.dumps(res, ensure_ascii=False) + "\n") + outf.flush() + finally: + ex.shutdown(wait=False, cancel_futures=True) + return results diff --git a/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/docvqa/skills/initial.md b/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/docvqa/skills/initial.md new file mode 100644 index 00000000..806fbe67 --- /dev/null +++ b/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/docvqa/skills/initial.md @@ -0,0 +1,11 @@ +# DocVQA Skill + +## Visual Evidence Discipline +- Read the document carefully before answering. +- Prefer the smallest exact text span that answers the question. +- When several nearby strings look similar, choose the one whose surrounding labels or layout best match the question. + +## Exact Answer Discipline +- Copy names, numbers, and dates exactly from the document whenever possible. +- Prefer direct extraction over paraphrase. +- Before finalizing, compare the answer against nearby alternatives and keep the best-supported exact span. diff --git a/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/livemathematicianbench/__init__.py b/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/livemathematicianbench/__init__.py new file mode 100644 index 00000000..bcc21386 --- /dev/null +++ b/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/livemathematicianbench/__init__.py @@ -0,0 +1 @@ +"""LiveMathematicianBench environment package for ReflACT.""" diff --git a/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/livemathematicianbench/adapter.py b/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/livemathematicianbench/adapter.py new file mode 100644 index 00000000..554b0675 --- /dev/null +++ b/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/livemathematicianbench/adapter.py @@ -0,0 +1,162 @@ +"""LiveMathematicianBench environment adapter for ReflACT.""" +from __future__ import annotations + +import json +import os + +from skillopt.datasets.base import BatchSpec +from skillopt.gradient.reflect import run_minibatch_reflect +from skillopt.envs.base import EnvAdapter +from skillopt.envs.livemathematicianbench.dataloader import LiveMathematicianBenchDataLoader +from skillopt.envs.livemathematicianbench.rollout import run_batch +from skillopt.model import get_target_backend + + +class LiveMathematicianBenchAdapter(EnvAdapter): + """LiveMathematicianBench adapter.""" + + def build_reference_text(self, item: dict) -> str: + parts: list[str] = [] + theorem = str(item.get("theorem") or "").strip() + sketch = str(item.get("sketch") or "").strip() + if theorem: + parts.append(f"## Reference Theorem\n{theorem}") + if sketch: + parts.append(f"## Reference Sketch\n{sketch}") + return "\n\n".join(parts) + + def get_reference_metadata(self, item: dict) -> dict: + fields: list[str] = [] + previews: list[str] = [] + theorem = str(item.get("theorem") or "").strip() + sketch = str(item.get("sketch") or "").strip() + if theorem: + fields.append("theorem") + previews.append(f"[theorem]\n{theorem[:220]}") + if sketch: + fields.append("sketch") + previews.append(f"[sketch]\n{sketch[:220]}") + return { + "fields": fields, + "preview": "\n\n".join(previews)[:500], + } + + def __init__( + self, + split_dir: str = "", + data_path: str = "", + split_mode: str = "ratio", + split_ratio: str = "2:1:7", + split_seed: int = 42, + split_output_dir: str = "", + max_turns: int = 1, + exec_timeout: int = 600, + workers: int = 64, + analyst_workers: int = 16, + failure_only: bool = False, + minibatch_size: int = 8, + edit_budget: int = 4, + seed: int = 42, + limit: int = 0, + shuffle_choices: bool = True, + use_theorem: bool = False, + use_sketch: bool = False, + max_completion_tokens: int = 16384, + ) -> None: + self.max_turns = max_turns + self.exec_timeout = exec_timeout + self.workers = workers + self.max_completion_tokens = int(max_completion_tokens) + self.analyst_workers = analyst_workers + self.failure_only = failure_only + self.minibatch_size = minibatch_size + self.edit_budget = edit_budget + self.use_theorem = use_theorem + self.use_sketch = use_sketch + self.dataloader = LiveMathematicianBenchDataLoader( + split_dir=split_dir, + data_path=data_path, + split_mode=split_mode, + split_ratio=split_ratio, + split_seed=split_seed, + split_output_dir=split_output_dir, + seed=seed, + limit=limit, + shuffle_choices=shuffle_choices, + ) + + def setup(self, cfg: dict) -> None: + super().setup(cfg) + self.dataloader.setup(cfg) + + def get_dataloader(self): + return self.dataloader + + def build_env_from_batch(self, batch: BatchSpec, **kwargs): + return list(batch.payload or []) + + def build_train_env(self, batch_size: int, seed: int, **kwargs): + batch = self.dataloader.build_train_batch(batch_size=batch_size, seed=seed, **kwargs) + return self.build_env_from_batch(batch, **kwargs) + + def build_eval_env(self, env_num: int, split: str, seed: int, **kwargs): + batch = self.dataloader.build_eval_batch(env_num=env_num, split=split, seed=seed, **kwargs) + return self.build_env_from_batch(batch, **kwargs) + + def rollout( + self, + env_manager, + skill_content: str, + out_dir: str, + **kwargs, + ) -> list[dict]: + items: list[dict] = env_manager + return run_batch( + items=items, + out_root=out_dir, + skill_content=skill_content, + max_turns=self.max_turns, + exec_timeout=self.exec_timeout, + workers=self.workers, + max_completion_tokens=self.max_completion_tokens, + use_theorem=self.use_theorem, + use_sketch=self.use_sketch, + diagnostic_mode=kwargs.get("diagnostic_mode", False), + diagnostic_instruction=kwargs.get("diagnostic_instruction", ""), + diagnostic_trace_context_by_id=kwargs.get("diagnostic_trace_context_by_id"), + task_timeout=self.exec_timeout, + ) + + def reflect( + self, + results: list[dict], + skill_content: str, + out_dir: str, + **kwargs, + ) -> list[dict | None]: + prediction_dir = kwargs.get("prediction_dir", os.path.join(out_dir, "predictions")) + patches_dir = kwargs.get("patches_dir", os.path.join(out_dir, "patches")) + random_seed = kwargs.get("random_seed") + step_buffer_context = kwargs.get("step_buffer_context", "") + meta_skill_context = kwargs.get("meta_skill_context", "") + + return run_minibatch_reflect( + results=results, + skill_content=skill_content, + prediction_dir=prediction_dir, + patches_dir=patches_dir, + workers=self.analyst_workers, + failure_only=self.failure_only, + minibatch_size=self.minibatch_size, + edit_budget=self.edit_budget, + random_seed=random_seed, + error_system=self.get_error_minibatch_prompt(), + success_system=self.get_success_minibatch_prompt(), + step_buffer_context=step_buffer_context, + meta_skill_context=meta_skill_context, + update_mode=getattr(self, "_cfg", {}).get("skill_update_mode", "patch"), + ) + + + def get_task_types(self) -> list[str]: + return self.dataloader.get_task_types() diff --git a/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/livemathematicianbench/dataloader.py b/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/livemathematicianbench/dataloader.py new file mode 100644 index 00000000..3ab53f58 --- /dev/null +++ b/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/livemathematicianbench/dataloader.py @@ -0,0 +1,308 @@ +"""LiveMathematicianBench task dataloader.""" +from __future__ import annotations + +import glob +import hashlib +import json +import os +import random +from typing import Any + +from skillopt.datasets.base import BatchSpec, SplitDataLoader + + +# ── Raw data loading utilities (for preprocessing / standalone eval) ───── + +_CHOICE_LABELS = ["A", "B", "C", "D", "E", "F", "G"] + + +def _load_json(path: str) -> Any: + with open(path) as f: + return json.load(f) + + +def _iter_monthly_files(data_path: str) -> list[str]: + if not data_path: + return [] + if os.path.isfile(data_path): + return [data_path] + if os.path.isdir(data_path): + nested = glob.glob( + os.path.join(data_path, "**", "qa_*_final.json"), + recursive=True, + ) + flat = glob.glob(os.path.join(data_path, "qa_*_final.json")) + return sorted(set(nested + flat)) + return [] + + +def _coerce_choices(raw_choices: Any) -> list[dict]: + if isinstance(raw_choices, list): + choices: list[dict] = [] + for idx, item in enumerate(raw_choices): + if isinstance(item, dict): + label = str(item.get("label") or _CHOICE_LABELS[idx]).strip() + text = str(item.get("text") or item.get("content") or "").strip() + else: + label = _CHOICE_LABELS[idx] + text = str(item).strip() + if text: + choices.append({"label": label, "text": text}) + return choices + + if isinstance(raw_choices, dict): + labels = sorted(raw_choices.keys()) + return [ + {"label": str(label).strip(), "text": str(raw_choices[label]).strip()} + for label in labels + if str(raw_choices[label]).strip() + ] + + return [] + + +def _coerce_theorem_types(raw: Any) -> list[str]: + if isinstance(raw, list): + return [str(x).strip() for x in raw if str(x).strip()] + if raw is None: + return [] + text = str(raw).strip() + return [text] if text else [] + + +def _normalize_label(text: str) -> str: + return str(text).strip().upper().rstrip(".):") + + +def _normalize_item(item: dict, row_idx: int, source_path: str) -> dict: + mcq = item.get("mcq", {}) if isinstance(item.get("mcq"), dict) else {} + question = str(mcq.get("question") or item.get("question") or "").strip() + choices = _coerce_choices(mcq.get("choices") or item.get("choices") or []) + correct = mcq.get("correct_choice") or item.get("correct_choice") or {} + + if isinstance(correct, dict): + correct_label = _normalize_label(correct.get("label", "")) + correct_text = str(correct.get("text") or "").strip() + else: + correct_label = _normalize_label(correct) + correct_text = "" + + choice_by_label = { + _normalize_label(choice["label"]): choice["text"] + for choice in choices + } + if correct_label and not correct_text: + correct_text = choice_by_label.get(correct_label, "") + if correct_label and correct_text and correct_label not in choice_by_label: + choices.append({"label": correct_label, "text": correct_text}) + choices.sort(key=lambda choice: _CHOICE_LABELS.index(choice["label"]) if choice["label"] in _CHOICE_LABELS else len(_CHOICE_LABELS)) + choice_by_label[correct_label] = correct_text + + month = str(item.get("month") or "").strip() + item_no = item.get("no", row_idx + 1) + item_id = f"{month}:{item_no}" if month else str(item_no) + + return { + "id": item_id, + "month": month, + "no": item_no, + "paper_link": str(item.get("paper_link") or "").strip(), + "theorem": str(item.get("theorem") or "").strip(), + "sketch": str(item.get("sketch") or "").strip(), + "theorem_type": _coerce_theorem_types(item.get("theorem_type")), + "question": question, + "choices": choices, + "correct_choice": { + "label": correct_label, + "text": correct_text, + }, + "source_path": source_path, + } + + +def load_items(data_path: str) -> list[dict]: + """Load and normalise LiveMathematicianBench items from JSON files.""" + files = _iter_monthly_files(data_path) + if not files: + raise ValueError( + "LiveMathematicianBench requires data_path to be a qa_*_final.json file " + "or a directory containing monthly qa_*_final.json files." + ) + + items: list[dict] = [] + for path in files: + raw = _load_json(path) + if not isinstance(raw, list): + raise ValueError(f"Expected JSON array in {path}, got {type(raw).__name__}") + for row_idx, item in enumerate(raw): + norm = _normalize_item(item, row_idx=row_idx, source_path=path) + if norm["question"] and norm["choices"] and norm["correct_choice"]["label"]: + items.append(norm) + if not items: + raise ValueError(f"No valid LiveMathematicianBench items loaded from {data_path}") + return items + + +# ── Dataloader ─────────────────────────────────────────────────────────── + +class LiveMathematicianBenchDataLoader(SplitDataLoader): + """LiveMathematicianBench dataloader with per-seed choice shuffling.""" + + def __init__( + self, + split_dir: str = "", + data_path: str = "", + split_mode: str = "ratio", + split_ratio: str = "2:1:7", + split_seed: int = 42, + split_output_dir: str = "", + seed: int = 42, + limit: int = 0, + shuffle_choices: bool = True, + **kwargs, + ) -> None: + super().__init__( + split_dir=split_dir, + data_path=data_path, + split_mode=split_mode, + split_ratio=split_ratio, + split_seed=split_seed, + split_output_dir=split_output_dir, + seed=seed, + limit=limit, + ) + self.shuffle_choices = shuffle_choices + self._task_types: list[str] = [] + + def load_raw_items(self, data_path: str) -> list[dict]: + return load_items(data_path) + + def setup(self, cfg: dict) -> None: + super().setup(cfg) + all_items = self.train_items + self.val_items + self.test_items + task_types: set[str] = set() + for item in all_items: + for name in item.get("theorem_type", []): + if name: + task_types.add(name) + self._task_types = sorted(task_types) + + def get_task_types(self) -> list[str]: + return list(self._task_types) + + # ── Choice shuffling ───────────────────────────────────────────────── + + @staticmethod + def _item_shuffle_seed(item_id: str, seed: int) -> int: + digest = hashlib.sha256(f"{seed}:{item_id}".encode("utf-8")).hexdigest() + return int(digest[:16], 16) + + def _shuffle_item_choices(self, item: dict, seed: int) -> dict: + if not self.shuffle_choices: + return { + **item, + "choices": [dict(c) for c in item["choices"]], + "correct_choice": dict(item["correct_choice"]), + } + + shuffled_choices = [dict(c) for c in item["choices"]] + rng = random.Random(self._item_shuffle_seed(str(item["id"]), seed)) + rng.shuffle(shuffled_choices) + + original_correct = _normalize_label(item["correct_choice"]["label"]) + remapped_choices: list[dict] = [] + new_correct_choice = dict(item["correct_choice"]) + + for idx, choice in enumerate(shuffled_choices): + new_label = _CHOICE_LABELS[idx] + old_label = _normalize_label(choice["label"]) + remapped_choices.append({"label": new_label, "text": choice["text"]}) + if old_label == original_correct: + new_correct_choice = {"label": new_label, "text": choice["text"]} + + transformed = dict(item) + transformed["choices"] = remapped_choices + transformed["correct_choice"] = new_correct_choice + return transformed + + def _materialize_batch(self, items: list[dict], seed: int) -> list[dict]: + return [self._shuffle_item_choices(item, seed) for item in items] + + # ── Batch construction (override for choice shuffling) ─────────────── + + def plan_train_epoch( + self, + *, + epoch: int, + steps_per_epoch: int, + accumulation: int, + batch_size: int, + seed: int, + **kwargs, + ) -> list[BatchSpec]: + """Build a shuffled epoch while preserving per-batch choice shuffling.""" + epoch_rng = random.Random(seed + epoch * 1000) + items = list(self.train_items) + epoch_rng.shuffle(items) + + total_batches = steps_per_epoch * accumulation + if total_batches <= 0: + return [] + + batches: list[BatchSpec] = [] + cursor = 0 + for batch_idx in range(total_batches): + batch_seed = seed + epoch * 1000 + batch_idx + 1 + batch_items = items[cursor: cursor + batch_size] + cursor += len(batch_items) + + if not batch_items and items: + refill_rng = random.Random(batch_seed) + batch_items = list(items) + refill_rng.shuffle(batch_items) + batch_items = batch_items[:batch_size] + + batch_items = self._materialize_batch(batch_items, batch_seed) + batches.append( + BatchSpec( + phase="train", + split="train", + seed=batch_seed, + batch_size=len(batch_items), + payload=batch_items, + ) + ) + + return batches + + def build_train_batch(self, batch_size: int, seed: int, **kwargs) -> BatchSpec: + rng = random.Random(seed) + items = list(self.train_items) + rng.shuffle(items) + items = self._materialize_batch(items[:batch_size], seed) + return BatchSpec( + phase="train", + split="train", + seed=seed, + batch_size=len(items), + payload=items, + ) + + def build_eval_batch( + self, + env_num: int, + split: str, + seed: int, + **kwargs, + ) -> BatchSpec: + items = self.get_split_items(split) + if env_num and env_num < len(items): + items = items[:env_num] + items = self._materialize_batch(items, seed) + return BatchSpec( + phase="eval", + split=split, + seed=seed, + batch_size=len(items), + payload=items, + ) diff --git a/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/livemathematicianbench/evaluator.py b/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/livemathematicianbench/evaluator.py new file mode 100644 index 00000000..d15db3e3 --- /dev/null +++ b/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/livemathematicianbench/evaluator.py @@ -0,0 +1,62 @@ +"""LiveMathematicianBench evaluation helpers.""" +from __future__ import annotations + +import re + + +def extract_answer(text: str) -> str: + matches = re.findall(r"(.*?)", text, re.DOTALL | re.IGNORECASE) + if matches: + return matches[-1].strip() + lines = [ln.strip() for ln in text.strip().splitlines() if ln.strip()] + if lines: + return lines[-1] + return text.strip() + + +def normalize_label(text: str) -> str: + return str(text).strip().upper().rstrip(".):") + + +def parse_choice_label(prediction_text: str, choices: list[dict]) -> str: + answer = extract_answer(prediction_text) + label = normalize_label(answer) + valid_labels = {normalize_label(choice.get("label", "")) for choice in choices} + if label in valid_labels: + return label + + answer_lower = answer.lower() + for choice in choices: + choice_label = normalize_label(choice.get("label", "")) + choice_text = str(choice.get("text", "")).strip() + if choice_text and choice_text.lower() == answer_lower: + return choice_label + + first_token = normalize_label(answer.split()[0]) if answer.split() else "" + if first_token in valid_labels: + return first_token + return label + + +def evaluate(prediction_text: str, correct_choice: dict, choices: list[dict]) -> dict: + predicted_label = parse_choice_label(prediction_text, choices) + correct_label = normalize_label(correct_choice.get("label", "")) + predicted_text = "" + correct_text = str(correct_choice.get("text", "")).strip() + + for choice in choices: + if normalize_label(choice.get("label", "")) == predicted_label: + predicted_text = str(choice.get("text", "")).strip() + break + + is_correct = float(predicted_label == correct_label) + return { + "em": is_correct, + "f1": is_correct, + "sub_em": is_correct, + "predicted_answer": predicted_label or extract_answer(prediction_text), + "predicted_label": predicted_label, + "predicted_text": predicted_text, + "correct_label": correct_label, + "correct_text": correct_text, + } diff --git a/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/livemathematicianbench/prompts/analyst_error.md b/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/livemathematicianbench/prompts/analyst_error.md new file mode 100644 index 00000000..dac1d049 --- /dev/null +++ b/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/livemathematicianbench/prompts/analyst_error.md @@ -0,0 +1,37 @@ +You are an expert failure-analysis agent for theorem-grounded mathematical multiple-choice questions. + +You will be given MULTIPLE failed trajectories from a single minibatch and the current skill document. +Each trajectory includes the target's response and an evaluation result showing the predicted option +versus the correct option. + +Your job is to identify COMMON reasoning failures across the batch and propose concise skill edits. + +## Failure Type Categories +- **quantifier_miss**: the agent missed exact quantifiers, scope, or existence/uniqueness conditions +- **strength_mismatch**: the agent preferred a weaker or stronger statement than what was proved +- **condition_miss**: the agent ignored hypotheses, equality cases, or domain restrictions +- **option_confusion**: the agent confused similar answer choices or failed to compare them exactly +- **other**: none of the above + +## Rules +1. Focus on patterns that recur across the minibatch. +2. Prefer edits that improve exact choice discrimination, not theorem-specific memorization. +3. Do not hardcode paper-specific content. +4. Only patch gaps not already covered by the skill. + +Respond ONLY with a valid JSON object: +{ + "batch_size": , + "failure_summary": [ + {"failure_type": "", "count": , "description": ""} + ], + "patch": { + "reasoning": "", + "edits": [ + {"op": "append", "content": ""}, + {"op": "insert_after", "target": "", "content": ""}, + {"op": "replace", "target": "", "content": ""}, + {"op": "delete", "target": ""} + ] + } +} diff --git a/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/livemathematicianbench/prompts/analyst_success.md b/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/livemathematicianbench/prompts/analyst_success.md new file mode 100644 index 00000000..7ff47d1d --- /dev/null +++ b/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/livemathematicianbench/prompts/analyst_success.md @@ -0,0 +1,25 @@ +You are an expert success-pattern analyst for theorem-grounded mathematical multiple-choice questions. + +You will be given MULTIPLE successful trajectories from a minibatch and the current skill document. +Identify generalizable behavior patterns that are genuinely helping the agent choose the exact correct option. + +## Rules +- Focus on broadly useful reasoning behaviors. +- Prefer patterns about exact comparison of options, quantifiers, and equality conditions. +- Do not add theorem-specific facts. +- "edits" may be empty if the skill already captures the useful patterns. + +Respond ONLY with a valid JSON object: +{ + "batch_size": , + "success_patterns": ["", ""], + "patch": { + "reasoning": "", + "edits": [ + {"op": "append", "content": ""}, + {"op": "insert_after", "target": "", "content": ""}, + {"op": "replace", "target": "", "content": ""}, + {"op": "delete", "target": ""} + ] + } +} diff --git a/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/livemathematicianbench/prompts/rollout_system.md b/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/livemathematicianbench/prompts/rollout_system.md new file mode 100644 index 00000000..607153d7 --- /dev/null +++ b/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/livemathematicianbench/prompts/rollout_system.md @@ -0,0 +1,12 @@ +You are an expert mathematical reasoning agent solving multiple-choice questions. + +{skill_section}## Task Format +You will receive one mathematics multiple-choice question and its answer choices. +Reason carefully about quantifiers, hypotheses, extremal wording, and exact equality conditions. + +## Answer Format +Think step by step, then provide your final answer inside ... tags. +Inside the tags, output only the single choice label, such as A or C. + +Example: +B diff --git a/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/livemathematicianbench/reflect.py b/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/livemathematicianbench/reflect.py new file mode 100644 index 00000000..b738481b --- /dev/null +++ b/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/livemathematicianbench/reflect.py @@ -0,0 +1,4 @@ +"""LiveMathematicianBench Reflect stage. + +Prompts are now loaded from .md files by the base adapter. +""" diff --git a/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/livemathematicianbench/rollout.py b/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/livemathematicianbench/rollout.py new file mode 100644 index 00000000..01de404f --- /dev/null +++ b/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/livemathematicianbench/rollout.py @@ -0,0 +1,434 @@ +"""LiveMathematicianBench rollout — theorem-grounded math MCQ agent.""" +from __future__ import annotations + +import json +import os +import time +from concurrent.futures import FIRST_COMPLETED, ThreadPoolExecutor, wait + +from skillopt.envs.livemathematicianbench.evaluator import evaluate +from skillopt.model import chat_target, get_target_backend, is_target_exec_backend +from skillopt.model.codex_harness import prepare_workspace, render_skill_md, run_target_exec +from skillopt.prompts import load_prompt + +def _build_system(skill_content: str) -> str: + if skill_content.strip(): + skill_section = f"## Skill\n{skill_content.strip()}\n\n" + else: + skill_section = "" + return load_prompt("rollout_system", env="livemathematicianbench").format(skill_section=skill_section) + + +def _format_choices(choices: list[dict]) -> str: + return "\n".join( + f"{choice['label']}. {choice['text']}" + for choice in choices + ) + + +def _build_user( + item: dict, + *, + use_theorem: bool = False, + use_sketch: bool = False, + diagnostic_mode: bool = False, + diagnostic_instruction: str = "", + diagnostic_trace_context: str = "", +) -> str: + parts = [f"## Question\n{item['question']}", f"## Choices\n{_format_choices(item['choices'])}"] + if use_theorem and item.get("theorem"): + parts.append(f"## Theorem\n{item['theorem']}") + if use_sketch and item.get("sketch"): + parts.append(f"## Proof Sketch\n{item['sketch']}") + if diagnostic_trace_context.strip(): + parts.append( + "## Previous Codex Trace Snapshot\n" + "This is a partial transcript from an earlier attempt. Use it as your current reasoning context.\n\n" + f"{diagnostic_trace_context.strip()}" + ) + if diagnostic_mode and diagnostic_instruction.strip(): + parts.append(f"## Training Readout\n{diagnostic_instruction.strip()}") + return "\n\n".join(parts) + + +def _build_codex_skill(skill_content: str) -> str: + return render_skill_md( + skill_content, + description="Dynamic ReflACT skill for solving the current LiveMathematicianBench multiple-choice question.", + preamble=( + "Use this skill when solving the current math multiple-choice question.\n" + "Inspect the option wording carefully and output only the final choice label inside ...." + ), + ) + +def _run_codex_once( + *, + pred_dir: str, + skill_content: str, + item: dict, + model: str, + timeout: int, + use_theorem: bool = False, + use_sketch: bool = False, + diagnostic_mode: bool = False, + diagnostic_instruction: str = "", + diagnostic_trace_context: str = "", + previous_response: str = "", +) -> tuple[str, str, str, str]: + user = _build_user( + item, + use_theorem=use_theorem, + use_sketch=use_sketch, + diagnostic_mode=diagnostic_mode, + diagnostic_instruction=diagnostic_instruction, + diagnostic_trace_context=diagnostic_trace_context, + ) + task_parts = [user] + if previous_response: + task_parts.append( + "## Previous Attempt\n" + f"{previous_response}\n\n" + "Re-evaluate the exact option wording. If needed, correct it." + ) + task_text = "\n\n".join(task_parts) + skill_md = _build_codex_skill(skill_content) + work_dir = os.path.join(pred_dir, "codex_exec") + prepare_workspace(work_dir=work_dir, skill_md=skill_md, task_text=task_text) + prompt = ( + "Use the `skillopt-target` skill available in this workspace.\n" + "Read `task.md` and solve the multiple-choice problem.\n" + "Output only the final choice label inside ...." + ) + final_message, raw = run_target_exec( + work_dir=work_dir, + prompt=prompt, + model=model, + timeout=timeout, + ) + return final_message or raw, raw, skill_md, task_text + + +def process_one( + item: dict, + out_root: str, + skill_content: str, + *, + max_turns: int = 1, + use_theorem: bool = False, + use_sketch: bool = False, + diagnostic_mode: bool = False, + diagnostic_instruction: str = "", + diagnostic_trace_context: str = "", + exec_timeout: int | None = 300, + max_completion_tokens: int = 16384, +) -> dict: + item_id = str(item["id"]) + result = { + "id": item_id, + "question": item["question"], + "task_type": item.get("theorem_type", ["math_mcq"])[0] if item.get("theorem_type") else "math_mcq", + "hard": 0, + "soft": 0.0, + "predicted_answer": "", + "predicted_label": "", + "predicted_text": "", + "correct_label": item["correct_choice"]["label"], + "correct_text": item["correct_choice"]["text"], + "response": "", + "fail_reason": "", + "agent_ok": False, + "n_turns": 0, + } + + try: + pred_dir = os.path.join(out_root, "predictions", item_id) + os.makedirs(pred_dir, exist_ok=True) + llm_timeout = int(exec_timeout) if exec_timeout and int(exec_timeout) > 0 else None + + if is_target_exec_backend(): + from skillopt.model import azure_openai as _llm + + conversation: list[dict] = [] + response = "" + system = "" + user = "" + for turn in range(max_turns): + response, raw, system, user = _run_codex_once( + pred_dir=pred_dir, + skill_content=skill_content, + item=item, + model=_llm.TARGET_DEPLOYMENT, + timeout=llm_timeout, + use_theorem=use_theorem, + use_sketch=use_sketch, + diagnostic_mode=diagnostic_mode if turn == 0 else False, + diagnostic_instruction=diagnostic_instruction if turn == 0 else "", + diagnostic_trace_context=diagnostic_trace_context if turn == 0 else "", + previous_response=response if turn > 0 else "", + ) + conversation.append({"type": "message", "turn": turn + 1, "content": response}) + if "" in response.lower(): + break + + result["response"] = response + result["agent_ok"] = True + result["n_turns"] = len(conversation) + + with open(os.path.join(pred_dir, "target_system_prompt.txt"), "w", encoding="utf-8") as f: + f.write(system) + with open(os.path.join(pred_dir, "target_user_prompt.txt"), "w", encoding="utf-8") as f: + f.write(user) + + eval_result = evaluate(response, item["correct_choice"], item["choices"]) + result["hard"] = int(eval_result["em"]) + result["soft"] = eval_result["f1"] + result["predicted_answer"] = eval_result["predicted_answer"] + result["predicted_label"] = eval_result["predicted_label"] + result["predicted_text"] = eval_result["predicted_text"] + if not result["hard"]: + result["fail_reason"] = ( + f"MCQ=0: predicted '{eval_result['predicted_label'] or eval_result['predicted_answer']}' " + f"but expected '{eval_result['correct_label']}'" + ) + eval_detail = ( + f"[EVALUATION RESULT]\n" + f"Question: {item['question']}\n" + f"Predicted label: {eval_result['predicted_label']!r}\n" + f"Predicted text: {eval_result['predicted_text']!r}\n" + f"Correct label: {eval_result['correct_label']!r}\n" + f"Correct text: {eval_result['correct_text']!r}\n" + f"Exact Match: {eval_result['em']}" + ) + conversation.append({"role": "system", "content": eval_detail}) + with open(os.path.join(pred_dir, "conversation.json"), "w") as f: + json.dump(conversation, f, ensure_ascii=False, indent=2) + return result + + system = _build_system(skill_content) + user = _build_user( + item, + use_theorem=use_theorem, + use_sketch=use_sketch, + diagnostic_mode=diagnostic_mode, + diagnostic_instruction=diagnostic_instruction, + diagnostic_trace_context=diagnostic_trace_context, + ) + conversation: list[dict] = [] + response = "" + + for turn in range(max_turns): + if turn == 0: + resp_text, _ = chat_target( + system=system, + user=user, + max_completion_tokens=max_completion_tokens, + retries=5, + stage="rollout", + timeout=llm_timeout, + ) + else: + refinement = ( + f"Your previous answer was:\n{response}\n\n" + "Re-evaluate the exact option wording. If needed, correct it. " + "Output only the final choice label inside ...." + ) + resp_text, _ = chat_target( + system=system, + user=refinement, + max_completion_tokens=max_completion_tokens, + retries=5, + stage="rollout", + timeout=llm_timeout, + ) + response = resp_text + conversation.append({"type": "message", "turn": turn + 1, "content": resp_text}) + if "" in resp_text.lower(): + break + + result["response"] = response + result["agent_ok"] = True + result["n_turns"] = len(conversation) + + with open(os.path.join(pred_dir, "target_system_prompt.txt"), "w", encoding="utf-8") as f: + f.write(system) + with open(os.path.join(pred_dir, "target_user_prompt.txt"), "w", encoding="utf-8") as f: + f.write(user) + + eval_result = evaluate(response, item["correct_choice"], item["choices"]) + result["hard"] = int(eval_result["em"]) + result["soft"] = eval_result["f1"] + result["predicted_answer"] = eval_result["predicted_answer"] + result["predicted_label"] = eval_result["predicted_label"] + result["predicted_text"] = eval_result["predicted_text"] + + if not result["hard"]: + result["fail_reason"] = ( + f"MCQ=0: predicted '{eval_result['predicted_label'] or eval_result['predicted_answer']}' " + f"but expected '{eval_result['correct_label']}'" + ) + + eval_detail = ( + f"[EVALUATION RESULT]\n" + f"Question: {item['question']}\n" + f"Predicted label: {eval_result['predicted_label']!r}\n" + f"Predicted text: {eval_result['predicted_text']!r}\n" + f"Correct label: {eval_result['correct_label']!r}\n" + f"Correct text: {eval_result['correct_text']!r}\n" + f"Exact Match: {eval_result['em']}" + ) + conversation.append({"role": "system", "content": eval_detail}) + + with open(os.path.join(pred_dir, "conversation.json"), "w") as f: + json.dump(conversation, f, ensure_ascii=False, indent=2) + + except Exception as e: # noqa: BLE001 + result["fail_reason"] = f"error: {e}" + + return result + + +def run_batch( + items: list[dict], + out_root: str, + skill_content: str, + *, + max_turns: int = 1, + exec_timeout: int | None = 300, + workers: int = 64, + max_completion_tokens: int = 16384, + use_theorem: bool = False, + use_sketch: bool = False, + diagnostic_mode: bool = False, + diagnostic_instruction: str = "", + diagnostic_trace_context_by_id: dict[str, str] | None = None, + task_timeout: int | None = 600, +) -> list[dict]: + exec_timeout_value = int(exec_timeout) if exec_timeout and int(exec_timeout) > 0 else 0 + task_timeout_value = int(task_timeout) if task_timeout and int(task_timeout) > 0 else 0 + if exec_timeout_value <= 0 or task_timeout_value <= 0: + task_timeout = None + else: + task_timeout = max(task_timeout_value, exec_timeout_value + 60) + results_path = os.path.join(out_root, "results.jsonl") + os.makedirs(out_root, exist_ok=True) + + done_ids: set[str] = set() + existing: list[dict] = [] + if os.path.exists(results_path): + with open(results_path) as f: + for line in f: + try: + r = json.loads(line) + done_ids.add(str(r["id"])) + existing.append(r) + except Exception: + pass + + pending = [it for it in items if str(it["id"]) not in done_ids] + if not pending: + return existing + + total = len(existing) + len(pending) + completed = len(existing) + correct_count = sum(1 for r in existing if r.get("hard", 0)) + if existing: + print(f" [rollout] resuming: {completed}/{total} already done", flush=True) + + results = list(existing) + + started_at: dict[str, float] = {} + + def _run_one(it: dict) -> dict: + started_at[str(it["id"])] = time.time() + return process_one( + it, + out_root, + skill_content, + max_turns=max_turns, + exec_timeout=exec_timeout, + max_completion_tokens=max_completion_tokens, + use_theorem=use_theorem, + use_sketch=use_sketch, + diagnostic_mode=diagnostic_mode, + diagnostic_instruction=diagnostic_instruction, + diagnostic_trace_context=(diagnostic_trace_context_by_id or {}).get(str(it["id"]), ""), + ) + + def _timeout_result(it: dict) -> dict: + correct = it.get("correct_choice") or {} + return { + "id": str(it["id"]), + "question": it.get("question", ""), + "task_type": it.get("theorem_type", ["math_mcq"])[0] if it.get("theorem_type") else "math_mcq", + "hard": 0, + "soft": 0.0, + "predicted_answer": "", + "predicted_label": "", + "predicted_text": "", + "correct_label": correct.get("label", ""), + "correct_text": correct.get("text", ""), + "response": "", + "fail_reason": f"task-timeout-{task_timeout}s", + "agent_ok": False, + "n_turns": 0, + } + + def _error_result(it: dict, exc: Exception) -> dict: + res = _timeout_result(it) + res["fail_reason"] = f"error: {type(exc).__name__}: {exc}" + return res + + with open(results_path, "a") as outf: + ex = ThreadPoolExecutor(max_workers=workers) + try: + futs = { + ex.submit(_run_one, it): it + for it in pending + } + pending_futs = set(futs) + while pending_futs: + done, _ = wait(pending_futs, timeout=5, return_when=FIRST_COMPLETED) + now = time.time() + timed_out = [ + fut for fut in pending_futs - done + if task_timeout is not None + if str(futs[fut]["id"]) in started_at + and now - started_at[str(futs[fut]["id"])] >= task_timeout + ] + for fut in done: + pending_futs.remove(fut) + item = futs[fut] + try: + res = fut.result() + except Exception as e: # noqa: BLE001 + res = _error_result(item, e) + results.append(res) + completed += 1 + if res.get("hard", 0): + correct_count += 1 + acc = correct_count / completed if completed else 0 + print( + f" [rollout] {completed}/{total} " + f"(acc={acc:.3f}) id={res['id']} " + f"hard={res.get('hard', '?')}", + flush=True, + ) + outf.write(json.dumps(res, ensure_ascii=False) + "\n") + outf.flush() + for fut in timed_out: + pending_futs.remove(fut) + res = _timeout_result(futs[fut]) + results.append(res) + completed += 1 + acc = correct_count / completed if completed else 0 + print( + f" [rollout] {completed}/{total} " + f"(acc={acc:.3f}) id={res['id']} TIMEOUT", + flush=True, + ) + outf.write(json.dumps(res, ensure_ascii=False) + "\n") + outf.flush() + finally: + ex.shutdown(wait=False, cancel_futures=True) + + return results diff --git a/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/livemathematicianbench/skills/initial.md b/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/livemathematicianbench/skills/initial.md new file mode 100644 index 00000000..d34f603b --- /dev/null +++ b/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/livemathematicianbench/skills/initial.md @@ -0,0 +1,16 @@ +# Live Mathematical MCQ Heuristics + +## Option Comparison +- Compare all options before committing. The correct choice is often the strongest statement justified by the question, while nearby distractors are weaker, overstrong, or miss an equality case. +- Track exact quantifiers such as "there exists", "for every", "if and only if", and "exactly when". + +## Theorem-Level Precision +- Check whether an option weakens the conclusion by dropping a characterization, equality clause, or full equivalence. +- Check whether an option overstates the theorem by upgrading regularity, removing scale restrictions, or changing an existential statement into a universal one. + +## Hypotheses +- Verify the hypotheses and domain carefully. Distractors often keep the theorem shape but alter the required assumptions. +- Pay close attention to equality cases, extremal conditions, and whether a result applies to the full family or only a restricted subfamily. + +## Final Answer +- Output the final answer as the single option label only. diff --git a/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/officeqa/__init__.py b/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/officeqa/__init__.py new file mode 100644 index 00000000..5316aaff --- /dev/null +++ b/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/officeqa/__init__.py @@ -0,0 +1 @@ +"""OfficeQA environment package for ReflACT.""" diff --git a/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/officeqa/adapter.py b/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/officeqa/adapter.py new file mode 100644 index 00000000..ba2e6f1c --- /dev/null +++ b/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/officeqa/adapter.py @@ -0,0 +1,135 @@ +from __future__ import annotations + +import os + +from skillopt.datasets.base import BatchSpec +from skillopt.envs.base import EnvAdapter +from skillopt.envs.officeqa.dataloader import OfficeQADataLoader +from skillopt.envs.officeqa.rollout import run_batch +from skillopt.gradient.reflect import run_minibatch_reflect + + +class OfficeQAAdapter(EnvAdapter): + def __init__( + self, + split_dir: str = "", + data_path: str = "", + split_mode: str = "split_dir", + split_ratio: str = "2:1:7", + split_seed: int = 42, + split_output_dir: str = "", + workers: int = 8, + analyst_workers: int = 8, + failure_only: bool = False, + minibatch_size: int = 8, + edit_budget: int = 4, + seed: int = 42, + limit: int = 0, + max_tool_turns: int = 12, + max_completion_tokens: int = 16384, + search_mode: str = "offline", + max_queries_per_turn: int = 4, + search_api_url: str = os.environ.get("OFFICEQA_SEARCH_API_URL", "http://localhost:8080/search_tool/search"), + search_auth_env: str = "OFFICEQA_CUSTOM_SEARCH_AUTH", + search_provider: str = "duckduckgo", + search_max_num_results: int = 4, + search_timeout_seconds: int = 20, + use_local_tools: bool = True, + data_dirs: list[str] | str | None = None, + docs_dirs: list[str] | str | None = None, ) -> None: + self.workers = workers + self.analyst_workers = analyst_workers + self.failure_only = failure_only + self.minibatch_size = minibatch_size + self.edit_budget = edit_budget + self.max_tool_turns = max_tool_turns + self.max_completion_tokens = int(max_completion_tokens) + self.search_mode = str(search_mode or "offline") + self.max_queries_per_turn = int(max_queries_per_turn) + self.search_api_url = str(search_api_url or "").strip() + self.search_auth_env = str(search_auth_env or "OFFICEQA_CUSTOM_SEARCH_AUTH").strip() + self.search_provider = str(search_provider or "duckduckgo").strip() + self.search_max_num_results = int(search_max_num_results) + self.search_timeout_seconds = int(search_timeout_seconds) + self.use_local_tools = bool(use_local_tools) + self.data_dirs = data_dirs if data_dirs is not None else docs_dirs + self.dataloader = OfficeQADataLoader( + split_dir=split_dir, + data_path=data_path, + split_mode=split_mode, + split_ratio=split_ratio, + split_seed=split_seed, + split_output_dir=split_output_dir, + seed=seed, + limit=limit, + ) + + def setup(self, cfg: dict) -> None: + super().setup(cfg) + self.dataloader.setup(cfg) + + def get_dataloader(self): + return self.dataloader + + def build_env_from_batch(self, batch: BatchSpec, **kwargs): + return list(batch.payload or []) + + def build_train_env(self, batch_size: int, seed: int, **kwargs): + batch = self.dataloader.build_train_batch(batch_size=batch_size, seed=seed, **kwargs) + return self.build_env_from_batch(batch, **kwargs) + + def build_eval_env(self, env_num: int, split: str, seed: int, **kwargs): + batch = self.dataloader.build_eval_batch(env_num=env_num, split=split, seed=seed, **kwargs) + return self.build_env_from_batch(batch, **kwargs) + + def rollout(self, env_manager, skill_content: str, out_dir: str, **kwargs) -> list[dict]: + items: list[dict] = env_manager + return run_batch( + items=items, + out_root=out_dir, + skill_content=skill_content, + workers=self.workers, + max_tool_turns=self.max_tool_turns, + max_completion_tokens=self.max_completion_tokens, + search_mode=self.search_mode, + max_queries_per_turn=self.max_queries_per_turn, + search_api_url=self.search_api_url, + search_auth_env=self.search_auth_env, + search_provider=self.search_provider, + search_max_num_results=self.search_max_num_results, + search_timeout_seconds=self.search_timeout_seconds, + use_local_tools=self.use_local_tools, + data_dirs=self.data_dirs, + diagnostic_mode=kwargs.get("diagnostic_mode", False), + diagnostic_instruction=kwargs.get("diagnostic_instruction", ""), + ) + + def reflect(self, results: list[dict], skill_content: str, out_dir: str, **kwargs) -> list[dict | None]: + prediction_dir = kwargs.get("prediction_dir", os.path.join(out_dir, "predictions")) + patches_dir = kwargs.get("patches_dir", os.path.join(out_dir, "patches")) + random_seed = kwargs.get("random_seed") + step_buffer_context = kwargs.get("step_buffer_context", "") + return run_minibatch_reflect( + results=results, + skill_content=skill_content, + prediction_dir=prediction_dir, + patches_dir=patches_dir, + workers=self.analyst_workers, + failure_only=self.failure_only, + minibatch_size=self.minibatch_size, + edit_budget=self.edit_budget, + random_seed=random_seed, + error_system=self.get_error_minibatch_prompt(), + success_system=self.get_success_minibatch_prompt(), + step_buffer_context=step_buffer_context, + update_mode=getattr(self, "_cfg", {}).get("skill_update_mode", "patch"), + ) + + + def get_task_types(self) -> list[str]: + seen: list[str] = [] + for item in self.dataloader.train_items + self.dataloader.val_items + self.dataloader.test_items: + task_type = str(item.get("task_type") or "officeqa") + if task_type not in seen: + seen.append(task_type) + return seen or ["officeqa"] diff --git a/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/officeqa/dataloader.py b/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/officeqa/dataloader.py new file mode 100644 index 00000000..a9c22b46 --- /dev/null +++ b/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/officeqa/dataloader.py @@ -0,0 +1,71 @@ +from __future__ import annotations + +import csv +import json +import os +from pathlib import Path + +from skillopt.datasets.base import SplitDataLoader + + +def _parse_list_field(value: str | list[str] | None) -> list[str]: + if value is None: + return [] + if isinstance(value, list): + return [str(item).strip() for item in value if str(item).strip()] + text = str(value).strip() + if not text: + return [] + try: + loaded = json.loads(text) + except json.JSONDecodeError: + loaded = None + if isinstance(loaded, list): + return [str(item).strip() for item in loaded if str(item).strip()] + if "\n" in text: + return [part.strip() for part in text.splitlines() if part.strip()] + if "," in text and not text.lower().endswith(".txt"): + return [part.strip() for part in text.split(",") if part.strip()] + return [text] + + +def _normalize_row(row: dict[str, str]) -> dict: + item_id = str(row.get("uid") or row.get("id") or "").strip() + question = str(row.get("question") or "").strip() + ground_truth = str(row.get("ground_truth") or row.get("answer") or "").strip() + task_type = str(row.get("category") or row.get("difficulty") or "officeqa").strip() or "officeqa" + source_files = _parse_list_field(row.get("source_files")) + source_docs = _parse_list_field(row.get("source_docs")) + split = str(row.get("split") or "").strip() + return { + "id": item_id, + "uid": item_id, + "question": question, + "ground_truth": ground_truth, + "answers": [ground_truth] if ground_truth else [], + "task_type": task_type, + "category": task_type, + "source_files": source_files, + "source_docs": source_docs, + "split": split, + } + + +class OfficeQADataLoader(SplitDataLoader): + def load_split_items(self, split_path: str) -> list[dict]: + path = Path(split_path) + csv_files = sorted(path.glob("*.csv")) + if csv_files: + with csv_files[0].open(encoding="utf-8", newline="") as f: + reader = csv.DictReader(f) + return [_normalize_row(row) for row in reader] + + json_files = sorted(path.glob("*.json")) + if json_files: + with json_files[0].open(encoding="utf-8") as f: + data = json.load(f) + if not isinstance(data, list): + raise ValueError(f"Expected JSON array in {json_files[0]}") + return [_normalize_row(item) for item in data] + + raise FileNotFoundError(f"No .csv or .json file found in {split_path}") diff --git a/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/officeqa/evaluator.py b/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/officeqa/evaluator.py new file mode 100644 index 00000000..124d25d5 --- /dev/null +++ b/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/officeqa/evaluator.py @@ -0,0 +1,46 @@ +from __future__ import annotations + +import re +import string +from collections import Counter + + +_NUMERIC_CHARS = set("0123456789.-") + + +def normalize_answer(text: str) -> str: + text = text.lower().strip() + text = text.replace(",", "") + text = "".join(ch for ch in text if ch not in string.punctuation or ch in _NUMERIC_CHARS or ch == "%") + text = re.sub(r"\b(million|millions|billion|billions|dollars|dollar|nominal)\b", " ", text) + text = " ".join(text.split()) + return text + + +def exact_match(prediction: str, gold: str) -> float: + return 1.0 if normalize_answer(prediction) == normalize_answer(gold) else 0.0 + + +def token_f1(prediction: str, gold: str) -> float: + pred_tokens = normalize_answer(prediction).split() + gold_tokens = normalize_answer(gold).split() + if not pred_tokens or not gold_tokens: + return 1.0 if pred_tokens == gold_tokens else 0.0 + common = Counter(pred_tokens) & Counter(gold_tokens) + n_common = sum(common.values()) + if n_common == 0: + return 0.0 + precision = n_common / len(pred_tokens) + recall = n_common / len(gold_tokens) + return 2 * precision * recall / (precision + recall) + + +def evaluate(prediction: str, gold: str) -> dict: + em = exact_match(prediction, gold) + f1 = token_f1(prediction, gold) + return { + "em": em, + "f1": f1, + "predicted_answer": prediction.strip(), + "gold_answer": gold, + } diff --git a/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/officeqa/prompts/analyst_error.md b/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/officeqa/prompts/analyst_error.md new file mode 100644 index 00000000..ec9a87e2 --- /dev/null +++ b/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/officeqa/prompts/analyst_error.md @@ -0,0 +1,37 @@ +You are an expert failure-analysis agent for OfficeQA document-retrieval question answering tasks. + +You will be given MULTIPLE failed OfficeQA trajectories from a single minibatch and the current skill document. The trajectories may include local document tool calls such as file search, grep, and partial reads. + +Your job is to identify COMMON failure patterns across the batch and propose concise skill edits. + +## Failure Type Categories +- retrieval_miss: the agent searched the wrong file or failed to narrow to the right file +- evidence_miss: the agent read documents but missed the decisive evidence span +- operand_error: the agent extracted the wrong value or the wrong operands +- calculation_error: the agent identified the right evidence but computed the result incorrectly +- answer_format: the agent reached the right result but formatted it wrong +- other: none of the above + +## Rules +- Focus on patterns common across multiple trajectories. +- Prefer general retrieval and evidence-grounding rules over task-specific hacks. +- Only patch gaps in the skill; do not duplicate rules already present. +- Do not hardcode file names, years, or question-specific constants unless the pattern truly requires a reusable retrieval heuristic. + +Respond ONLY with a valid JSON object (no markdown fences, no extra text): +{ + "batch_size": , + "failure_summary": [ + {"failure_type": "", "count": , "description": ""} + ], + "patch": { + "reasoning": "", + "edits": [ + {"op": "append", "content": ""}, + {"op": "insert_after", "target": "", "content": ""}, + {"op": "replace", "target": "", "content": ""}, + {"op": "delete", "target": ""} + ] + } +} +Only include edits that are needed. "edits" can be an empty list if no patch is warranted. diff --git a/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/officeqa/prompts/analyst_success.md b/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/officeqa/prompts/analyst_success.md new file mode 100644 index 00000000..4ce3da5e --- /dev/null +++ b/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/officeqa/prompts/analyst_success.md @@ -0,0 +1,25 @@ +You are an expert success-pattern analyst for OfficeQA document-retrieval question answering tasks. + +You will be given MULTIPLE successful OfficeQA trajectories from a single minibatch and the current skill document. Your job is to identify common retrieval, evidence-selection, and numeric-grounding behaviors worth encoding in the skill. + +## Rules +- Focus on patterns shared across multiple successful trajectories. +- Prefer reusable retrieval and extraction discipline over question-specific tips. +- Reinforce compact, high-value behaviors such as narrowing files early, reading only the relevant span, building a clean operand ledger, and copying the final answer from checked evidence. +- Only propose patches for patterns not already captured in the current skill. + +Respond ONLY with a valid JSON object: +{ + "batch_size": , + "success_patterns": ["", ""], + "patch": { + "reasoning": "", + "edits": [ + {"op": "append", "content": ""}, + {"op": "insert_after", "target": "", "content": ""}, + {"op": "replace", "target": "", "content": ""}, + {"op": "delete", "target": ""} + ] + } +} +"edits" may be empty if the skill already covers all observed patterns. diff --git a/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/officeqa/prompts/rollout_system.md b/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/officeqa/prompts/rollout_system.md new file mode 100644 index 00000000..db229312 --- /dev/null +++ b/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/officeqa/prompts/rollout_system.md @@ -0,0 +1,15 @@ +You are an expert OfficeQA agent working over local Treasury bulletin text files. + +{skill_section}## Rules +1. Use only the provided local document tools to inspect candidate files. +2. Narrow to the most relevant file before reading long passages. +3. Prefer short targeted searches, then small reads around matching evidence. +4. Do not invent values that are not grounded in the retrieved text. +5. When the question requires arithmetic, compute only after extracting the exact operands. +6. If you have enough evidence, return the final answer inside .... + +## Tool Use +Use the provided function tools directly when you need them. Prefer searching and small reads before answering. Do not ask the user for permission to use tools; just call the tools. + +## Final Answer Format +When you are ready to answer, emit the final answer inside ... and do not request another tool. diff --git a/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/officeqa/rollout.py b/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/officeqa/rollout.py new file mode 100644 index 00000000..01afe8b5 --- /dev/null +++ b/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/officeqa/rollout.py @@ -0,0 +1,799 @@ +from __future__ import annotations +import json +import os +import re +from concurrent.futures import ThreadPoolExecutor, as_completed +from skillopt.envs.officeqa.evaluator import evaluate +from skillopt.envs.officeqa.tool_runtime import ( + build_oracle_parsed_pages_context, + custom_search, + resolve_candidate_files, + resolve_docs_roots, + run_tool, +) +from skillopt.model import chat_target_messages, get_target_backend, is_target_exec_backend +from skillopt.model.codex_harness import prepare_workspace, render_skill_md, run_target_exec +from skillopt.prompts import load_prompt +_TOOL_SCHEMAS = [ + { + "type": "function", + "function": { + "name": "glob", + "description": "Find candidate local document files by filename or relative-path glob pattern.", + "parameters": { + "type": "object", + "properties": {"pattern": {"type": "string"}}, + "required": ["pattern"], + }, + }, + }, + { + "type": "function", + "function": { + "name": "read", + "description": "Read a local text document excerpt by path and line window.", + "parameters": { + "type": "object", + "properties": { + "path": {"type": "string"}, + "start": {"type": "integer"}, + "limit": {"type": "integer"}, + }, + "required": ["path"], + }, + }, + }, + { + "type": "function", + "function": { + "name": "grep", + "description": "Search a local text document for a literal pattern and return matching lines.", + "parameters": { + "type": "object", + "properties": { + "pattern": {"type": "string"}, + "path": {"type": "string"}, + }, + "required": ["pattern", "path"], + }, + }, + }, +] +_FINAL_RE = re.compile(r"(.*?)", re.IGNORECASE | re.DOTALL) +_SEARCH_RE = re.compile(r"(.*?)", re.IGNORECASE | re.DOTALL) +_DEFAULT_SEARCH_MODE = "offline" +_CUSTOM_SEARCH_MODE = "custom_search" +_AZURE_SEARCH_MODE = "azure_search" +def _normalize_search_mode(search_mode: str | None) -> str: + normalized = str(search_mode or _DEFAULT_SEARCH_MODE).strip().lower() + if normalized in {"custom", _CUSTOM_SEARCH_MODE}: + return _CUSTOM_SEARCH_MODE + if normalized in {"azure", _AZURE_SEARCH_MODE}: + return _AZURE_SEARCH_MODE + return _DEFAULT_SEARCH_MODE +def _build_system( + skill_content: str, + *, + search_mode: str = _DEFAULT_SEARCH_MODE, + use_local_tools: bool = True, + max_tool_turns: int = 12, + max_queries_per_turn: int = 4, +) -> str: + if skill_content.strip(): + skill_section = f"## Skill\n{skill_content.strip()}\n\n" + else: + skill_section = "" + normalized_search_mode = _normalize_search_mode(search_mode) + if normalized_search_mode == _AZURE_SEARCH_MODE: + return ( + "You are an expert OfficeQA research assistant. Solve the question using the model's built-in web " + "search tool when needed, keep the answer grounded in authoritative evidence, and return the final " + "answer inside ....\n\n" + + skill_section + ).rstrip() + if normalized_search_mode == _CUSTOM_SEARCH_MODE: + protocol = ( + "You are an expert OfficeQA research assistant. Solve the question using the provided oracle parsed " + "OfficeQA page(s) and evidence returned by the controller-managed custom search loop.\n\n" + "Search protocol:\n" + f"- You have at most {max_tool_turns} model rounds total.\n" + f"- On any non-final round, you may either return `[\"query 1\", \"query 2\"]` " + f"with up to {max_queries_per_turn} queries, or return `...` if you are ready.\n" + "- If you request search, do not include an answer in the same response.\n" + "- On the final round, you must return `...` and must not request more search.\n" + "- Base your answer on the returned evidence, reconcile conflicting snippets carefully, and stay concise.\n\n" + ) + return protocol + skill_section + "Return the final answer inside ... when you are ready." + if not use_local_tools: + return ( + "You are an expert OfficeQA research assistant. Solve the question using the provided oracle parsed " + "OfficeQA page(s) and source hints. Do not request or assume access to any external search or local " + "function tools. Return the final answer inside ....\n\n" + + skill_section + ).rstrip() + return load_prompt("rollout_system", env="officeqa").format(skill_section=skill_section) +def _build_round_instruction( + *, + turn: int, + max_tool_turns: int, + max_queries_per_turn: int, +) -> str: + if turn >= max_tool_turns: + return ( + "## Round Policy\n" + f"This is the final round ({turn}/{max_tool_turns}). You must return `...` now. " + "Do not output ``." + ) + remaining_rounds = max_tool_turns - turn + return ( + "## Round Policy\n" + f"This is round {turn}/{max_tool_turns}. " + f"You may either return `...` now, or request up to {max_queries_per_turn} search queries " + f"inside `...`. " + f"After this response, at most {remaining_rounds} model rounds remain." + ) +def _message_debug_metadata(message: object) -> dict: + metadata = getattr(message, "metadata", None) + if isinstance(metadata, dict): + return metadata + return {} +def _build_user( + item: dict, + candidate_files: list[str] | None = None, + *, + diagnostic_mode: bool = False, + diagnostic_instruction: str = "", + corpus_note: str = "", + search_mode: str = _DEFAULT_SEARCH_MODE, + turn: int = 1, + max_tool_turns: int = 12, + max_queries_per_turn: int = 4, + oracle_context: str = "", +) -> str: + normalized_search_mode = _normalize_search_mode(search_mode) + parts = [f"## Question\n{item['question']}"] + if oracle_context.strip(): + parts.append(f"## Oracle Parsed Pages\n{oracle_context.strip()}") + if normalized_search_mode == _DEFAULT_SEARCH_MODE: + file_block = "\n".join(f"- {path}" for path in (candidate_files or [])[:20]) or "- none resolved" + if corpus_note.strip(): + parts.append(f"## Document Corpus\n{corpus_note.strip()}") + parts.append(f"## Candidate Files\n{file_block}") + if item.get("source_docs"): + parts.append("## Source Hints\n" + "\n".join(f"- {hint}" for hint in item["source_docs"])) + if normalized_search_mode != _DEFAULT_SEARCH_MODE and item.get("source_files"): + parts.append("## File Hints\n" + "\n".join(f"- {hint}" for hint in item["source_files"])) + if diagnostic_mode and diagnostic_instruction.strip(): + parts.append(f"## Training Readout\n{diagnostic_instruction.strip()}") + if normalized_search_mode == _CUSTOM_SEARCH_MODE: + parts.append( + _build_round_instruction( + turn=turn, + max_tool_turns=max_tool_turns, + max_queries_per_turn=max_queries_per_turn, + ) + ) + parts.append( + "## Output Format\n" + "If you need more evidence, return only `[...]`.\n" + "If you are ready to answer, return only `...`." + ) + parts.append( + "Use only the provided oracle parsed pages and controller-provided custom search evidence. " + "Do not rely on any built-in web search capability." + ) + elif normalized_search_mode == _AZURE_SEARCH_MODE: + parts.append("Use the model's built-in web search tool when needed. Return the final answer inside ....") + return "\n\n".join(parts) +def _extract_answer(text: str) -> str: + match = _FINAL_RE.search(text) + if match: + return match.group(1).strip() + lines = [line.strip() for line in text.splitlines() if line.strip()] + return lines[-1] if lines else text.strip() +def _extract_search_queries(text: str) -> list[str]: + match = _SEARCH_RE.search(text or "") + if not match: + return [] + raw = match.group(1).strip() + if not raw: + return [] + parsed_queries: list[str] = [] + try: + parsed = json.loads(raw) + except json.JSONDecodeError: + parsed = None + if isinstance(parsed, dict): + for key in ("queries", "search_queries", "query"): + value = parsed.get(key) + if isinstance(value, str) and value.strip(): + parsed_queries = [value.strip()] + break + if isinstance(value, list): + parsed_queries = [str(item).strip() for item in value if str(item).strip()] + break + elif isinstance(parsed, list): + parsed_queries = [str(item).strip() for item in parsed if str(item).strip()] + elif isinstance(parsed, str) and parsed.strip(): + parsed_queries = [parsed.strip()] + if not parsed_queries: + raw_lines = [line.strip(" -*\t\r\n\"'") for line in raw.splitlines()] + parsed_queries = [line for line in raw_lines if line] + if len(parsed_queries) <= 1 and parsed_queries: + multi = [part.strip(" \"'") for part in re.split(r"[;,]", parsed_queries[0]) if part.strip(" \"'")] + if len(multi) > 1: + parsed_queries = multi + deduped: list[str] = [] + seen: set[str] = set() + for query in parsed_queries: + normalized = query.strip() + if not normalized or normalized in seen: + continue + seen.add(normalized) + deduped.append(normalized) + return deduped +def _docs_link_targets(docs_roots: list[str]) -> list[tuple[str, str]]: + return [(root, os.path.join("docs", f"root_{idx}")) for idx, root in enumerate(docs_roots, start=1)] +def _workspace_doc_path(path: str, docs_roots: list[str]) -> str: + resolved_path = os.path.realpath(path) + for idx, root in enumerate(docs_roots, start=1): + resolved_root = os.path.realpath(root) + if resolved_path == resolved_root or resolved_path.startswith(resolved_root + os.sep): + rel_path = os.path.relpath(resolved_path, resolved_root) + return os.path.join("docs", f"root_{idx}", rel_path) + return path +def _build_codex_skill(skill_content: str) -> str: + return render_skill_md( + skill_content, + description="Dynamic ReflACT skill for solving the current OfficeQA local-document question.", + preamble=( + "Use this skill when answering the current OfficeQA question.\n" + "Inspect the provided local document excerpts or files, ground the answer in the evidence,\n" + "and return the final answer inside ...." + ), + ) +def _run_codex_once( + *, + pred_dir: str, + item: dict, + skill_content: str, + candidate_files: list[str], + docs_roots: list[str], + model: str, + timeout: int, + diagnostic_mode: bool = False, + diagnostic_instruction: str = "", + previous_response: str = "", + oracle_context: str = "", +) -> tuple[str, str, str, str]: + rel_files = [_workspace_doc_path(path, docs_roots) for path in candidate_files[:20]] + corpus_note = ( + "The full OfficeQA document corpus is available under `docs/`. " + "The candidate files below are source hints or likely starting points; search the full corpus if needed." + ) + user = _build_user( + item, + rel_files, + diagnostic_mode=diagnostic_mode, + diagnostic_instruction=diagnostic_instruction, + corpus_note=corpus_note, + oracle_context=oracle_context, + ) + task_parts = [user] + if previous_response: + task_parts.append( + "## Previous Attempt\n" + f"{previous_response}\n\n" + "Review the local documents again and correct the answer if needed." + ) + task_text = "\n\n".join(task_parts) + skill_md = _build_codex_skill(skill_content) + work_dir = os.path.join(pred_dir, "codex_exec") + prepare_workspace( + work_dir=work_dir, + skill_md=skill_md, + task_text=task_text, + link_dirs=_docs_link_targets(docs_roots), + ) + prompt = ( + "Use the `skillopt-target` skill available in this workspace.\n" + "Read `task.md`, inspect or search the full OfficeQA corpus under `docs/`, and answer the question.\n" + "Treat candidate files in `task.md` as hints, not an access limit.\n" + "Return the final answer inside ...." + ) + final_message, raw = run_target_exec( + work_dir=work_dir, + prompt=prompt, + model=model, + timeout=timeout, + data_dirs=docs_roots, + ) + return final_message or raw, raw, skill_md, task_text +def _execute_custom_search_round( + queries: list[str], + *, + api_url: str, + auth_env: str, + provider: str, + max_num_results: int, + timeout: int, +) -> str: + blocks = [] + for index, query in enumerate(queries, start=1): + try: + result = custom_search( + query, + api_url=api_url, + auth_env=auth_env, + provider=provider, + max_num_results=max_num_results, + timeout=timeout, + ) + except Exception as search_error: # noqa: BLE001 + result = f"Query: {query}\n\n[search error: {search_error}]" + blocks.append(f"## Query {index}\n{result}") + return "\n\n".join(blocks) +def _run_custom_search_process( + item: dict, + skill_content: str, + *, + max_tool_turns: int, + max_completion_tokens: int, + max_queries_per_turn: int, + diagnostic_mode: bool, + diagnostic_instruction: str, + search_api_url: str, + search_auth_env: str, + search_provider: str, + search_max_num_results: int, + search_timeout_seconds: int, + oracle_context: str = "", +) -> tuple[str, str, str, str, list[dict], str, dict]: + if not str(search_api_url or "").strip(): + raise ValueError("custom_search mode requires a non-empty search_api_url") + if not os.environ.get(search_auth_env, "").strip(): + raise ValueError(f"custom_search mode requires auth token env var {search_auth_env}") + if get_target_backend() not in {"openai_chat", "qwen_chat"}: + raise ValueError("custom_search mode is only supported with target_backend='openai_chat' or 'qwen_chat'") + system = _build_system( + skill_content, + search_mode=_CUSTOM_SEARCH_MODE, + max_tool_turns=max_tool_turns, + max_queries_per_turn=max_queries_per_turn, + ) + initial_user = _build_user( + item, + diagnostic_mode=diagnostic_mode, + diagnostic_instruction=diagnostic_instruction, + search_mode=_CUSTOM_SEARCH_MODE, + turn=1, + max_tool_turns=max_tool_turns, + max_queries_per_turn=max_queries_per_turn, + oracle_context=oracle_context, + ) + latest_user = initial_user + messages: list[dict] = [ + {"role": "system", "content": system}, + {"role": "user", "content": initial_user}, + ] + conversation: list[dict] = [{"role": "user", "content": initial_user}] + final_response = "" + final_answer = "" + fail_reason = "" + last_response_metadata: dict = {} + for turn in range(1, max_tool_turns + 1): + message, _ = chat_target_messages( + messages=messages, + max_completion_tokens=max_completion_tokens, + retries=5, + stage="rollout", + return_message=True, + ) + response = message.content or "" + final_response = response + last_response_metadata = _message_debug_metadata(message) + messages.append({"role": "assistant", "content": response}) + message_event = {"type": "message", "turn": turn, "content": response} + if last_response_metadata: + message_event["response_metadata"] = last_response_metadata + conversation.append(message_event) + if "" in response.lower(): + final_answer = _extract_answer(response) + return system, latest_user, final_response, final_answer, conversation, "", last_response_metadata + if turn == max_tool_turns: + fail_reason = f"Final round ({max_tool_turns}) ended without ..." + break + queries = _extract_search_queries(response)[:max_queries_per_turn] + if not queries: + fail_reason = "Model neither produced search queries nor a final answer" + break + results_text = _execute_custom_search_round( + queries, + api_url=search_api_url, + auth_env=search_auth_env, + provider=search_provider, + max_num_results=search_max_num_results, + timeout=search_timeout_seconds, + ) + conversation.append({"type": "tool_call", "turn": turn, "cmd": f"custom_search({queries!r})", "obs": results_text}) + latest_user = ( + f"## Search Results Round {turn}\n{results_text}\n\n" + + _build_round_instruction( + turn=turn + 1, + max_tool_turns=max_tool_turns, + max_queries_per_turn=max_queries_per_turn, + ) + + "\n\nFollow the round policy above exactly." + ) + messages.append({"role": "user", "content": latest_user}) + conversation.append({"role": "user", "turn": turn + 1, "content": latest_user}) + return system, latest_user, final_response, final_answer, conversation, fail_reason, last_response_metadata +def _run_azure_search_process( + item: dict, + skill_content: str, + *, + max_completion_tokens: int, + diagnostic_mode: bool, + diagnostic_instruction: str, +) -> tuple[str, str, str, str, list[dict], str, dict]: + if get_target_backend() != "openai_chat": + raise ValueError("azure_search mode is only supported with target_backend='openai_chat'") + system = _build_system(skill_content, search_mode=_AZURE_SEARCH_MODE) + user = _build_user( + item, + diagnostic_mode=diagnostic_mode, + diagnostic_instruction=diagnostic_instruction, + search_mode=_AZURE_SEARCH_MODE, + ) + messages = [ + {"role": "system", "content": system}, + {"role": "user", "content": user}, + ] + conversation: list[dict] = [{"role": "user", "content": user}] + message, _ = chat_target_messages( + messages=messages, + max_completion_tokens=max_completion_tokens, + retries=5, + stage="rollout", + return_message=True, + tools=[{"type": "web_search"}], + ) + response = message.content or "" + last_response_metadata = _message_debug_metadata(message) + message_event = {"type": "message", "content": response} + if last_response_metadata: + message_event["response_metadata"] = last_response_metadata + conversation.append(message_event) + if "" in response.lower(): + return system, user, response, _extract_answer(response), conversation, "", last_response_metadata + return system, user, response, "", conversation, "Model did not produce a final answer", last_response_metadata +def _run_offline_no_tools_process( + item: dict, + skill_content: str, + *, + max_completion_tokens: int, + diagnostic_mode: bool, + diagnostic_instruction: str, + candidate_files: list[str], + oracle_context: str = "", +) -> tuple[str, str, str, str, list[dict], str, dict]: + system = _build_system(skill_content, search_mode=_DEFAULT_SEARCH_MODE, use_local_tools=False) + user = _build_user( + item, + candidate_files, + diagnostic_mode=diagnostic_mode, + diagnostic_instruction=diagnostic_instruction, + search_mode=_DEFAULT_SEARCH_MODE, + oracle_context=oracle_context, + ) + messages = [ + {"role": "system", "content": system}, + {"role": "user", "content": user}, + ] + conversation: list[dict] = [{"role": "user", "content": user}] + message, _ = chat_target_messages( + messages=messages, + max_completion_tokens=max_completion_tokens, + retries=5, + stage="rollout", + return_message=True, + ) + response = message.content or "" + last_response_metadata = _message_debug_metadata(message) + message_event = {"type": "message", "content": response} + if last_response_metadata: + message_event["response_metadata"] = last_response_metadata + conversation.append(message_event) + if "" in response.lower(): + return system, user, response, _extract_answer(response), conversation, "", last_response_metadata + return system, user, response, "", conversation, "Model did not produce a final answer", last_response_metadata +def process_one( + item: dict, + out_root: str, + skill_content: str, + *, + max_tool_turns: int = 12, + max_completion_tokens: int = 16384, + search_mode: str = _DEFAULT_SEARCH_MODE, + max_queries_per_turn: int = 4, + search_api_url: str = "", + search_auth_env: str = "OFFICEQA_CUSTOM_SEARCH_AUTH", + search_provider: str = "duckduckgo", + search_max_num_results: int = 4, + search_timeout_seconds: int = 20, + use_local_tools: bool = True, + data_dirs: list[str] | str | None = None, + diagnostic_mode: bool = False, + diagnostic_instruction: str = "", +) -> dict: + item_id = str(item["id"]) + pred_dir = os.path.join(out_root, "predictions", item_id) + os.makedirs(pred_dir, exist_ok=True) + normalized_search_mode = _normalize_search_mode(search_mode) + docs_roots: list[str] = [] + candidate_files: list[str] = [] + oracle_context = "" + if normalized_search_mode == _DEFAULT_SEARCH_MODE: + docs_roots = resolve_docs_roots(data_dirs) + candidate_files = resolve_candidate_files(item.get("source_files", []), docs_roots) + oracle_context = build_oracle_parsed_pages_context( + item.get("source_files", []), + item.get("source_docs", []), + docs_roots, + evidence_note=( + "Treat it as primary document evidence and combine it with local document tool evidence when useful." + if use_local_tools + else "Treat it as primary document evidence for answering the question." + ), + ) + elif normalized_search_mode == _CUSTOM_SEARCH_MODE: + docs_roots = resolve_docs_roots(data_dirs) + if item.get("source_files"): + candidate_files = resolve_candidate_files(item.get("source_files", []), docs_roots) + oracle_context = build_oracle_parsed_pages_context( + item.get("source_files", []), + item.get("source_docs", []), + docs_roots, + ) + system = _build_system( + skill_content, + search_mode=normalized_search_mode, + use_local_tools=use_local_tools, + max_tool_turns=max_tool_turns, + max_queries_per_turn=max_queries_per_turn, + ) + user = _build_user( + item, + candidate_files if normalized_search_mode == _DEFAULT_SEARCH_MODE else None, + diagnostic_mode=diagnostic_mode, + diagnostic_instruction=diagnostic_instruction, + search_mode=normalized_search_mode, + max_tool_turns=max_tool_turns, + max_queries_per_turn=max_queries_per_turn, + oracle_context=oracle_context, + ) + conversation: list[dict] = [{"role": "user", "content": user}] + final_response = "" + final_answer = "" + fail_reason = "" + last_response_metadata: dict = {} + allowed_files = [os.path.basename(path) for path in candidate_files] + try: + if normalized_search_mode == _CUSTOM_SEARCH_MODE: + system, user, final_response, final_answer, conversation, fail_reason, last_response_metadata = _run_custom_search_process( + item, + skill_content, + max_tool_turns=max_tool_turns, + max_completion_tokens=max_completion_tokens, + max_queries_per_turn=max_queries_per_turn, + diagnostic_mode=diagnostic_mode, + diagnostic_instruction=diagnostic_instruction, + search_api_url=search_api_url, + search_auth_env=search_auth_env, + search_provider=search_provider, + search_max_num_results=search_max_num_results, + search_timeout_seconds=search_timeout_seconds, + oracle_context=oracle_context, + ) + elif normalized_search_mode == _AZURE_SEARCH_MODE: + system, user, final_response, final_answer, conversation, fail_reason, last_response_metadata = _run_azure_search_process( + item, + skill_content, + max_completion_tokens=max_completion_tokens, + diagnostic_mode=diagnostic_mode, + diagnostic_instruction=diagnostic_instruction, + ) + elif not use_local_tools: + system, user, final_response, final_answer, conversation, fail_reason, last_response_metadata = _run_offline_no_tools_process( + item, + skill_content, + max_completion_tokens=max_completion_tokens, + diagnostic_mode=diagnostic_mode, + diagnostic_instruction=diagnostic_instruction, + candidate_files=candidate_files, + oracle_context=oracle_context, + ) + elif is_target_exec_backend(): + from skillopt.model import azure_openai as _llm + response = "" + system = "" + user = "" + for turn in range(1, max_tool_turns + 1): + response, _raw, system, user = _run_codex_once( + pred_dir=pred_dir, + item=item, + skill_content=skill_content, + candidate_files=candidate_files, + docs_roots=docs_roots, + model=_llm.TARGET_DEPLOYMENT, + timeout=180, + diagnostic_mode=diagnostic_mode if turn == 1 else False, + diagnostic_instruction=diagnostic_instruction if turn == 1 else "", + previous_response=response if turn > 1 else "", + oracle_context=oracle_context, + ) + final_response = response + conversation.append({"type": "message", "turn": turn, "content": response}) + if "" in response.lower(): + final_answer = _extract_answer(response) + break + if not final_answer: + fail_reason = f"Exceeded codex turn budget ({max_tool_turns})" + system = system or _build_codex_skill(skill_content) + user = user or _build_user(item, [_workspace_doc_path(path, docs_roots) for path in candidate_files]) + else: + messages: list[dict] = [ + {"role": "system", "content": system}, + {"role": "user", "content": user}, + ] + for turn in range(1, max_tool_turns + 1): + message, _ = chat_target_messages( + messages=messages, + max_completion_tokens=max_completion_tokens, + retries=5, + stage="rollout", + tools=_TOOL_SCHEMAS, + tool_choice="auto", + return_message=True, + ) + response = message.content or "" + final_response = response + assistant_message = {"role": "assistant", "content": response} + if getattr(message, "tool_calls", None): + assistant_message["tool_calls"] = [tool_call.model_dump(mode="json") for tool_call in message.tool_calls] + messages.append(assistant_message) + conversation.append({"type": "message", "content": response}) + if getattr(message, "tool_calls", None): + for tool_call in message.tool_calls: + tool_name = tool_call.function.name + arguments = json.loads(tool_call.function.arguments) if tool_call.function.arguments else {} + cmd, obs = run_tool(tool_name, arguments, allowed_roots=docs_roots, allowed_files=allowed_files) + conversation.append({"type": "tool_call", "cmd": cmd, "obs": obs}) + messages.append({ + "role": "tool", + "tool_call_id": tool_call.id, + "content": obs, + }) + continue + if "" in response.lower(): + final_answer = _extract_answer(response) + break + if turn == max_tool_turns: + fail_reason = f"Exceeded tool-turn budget ({max_tool_turns})" + else: + fail_reason = "Model neither produced a tool request nor a final answer" + break + except Exception as e: # noqa: BLE001 + fail_reason = f"error: {e}" + with open(os.path.join(pred_dir, "target_system_prompt.txt"), "w", encoding="utf-8") as f: + f.write(system) + with open(os.path.join(pred_dir, "target_user_prompt.txt"), "w", encoding="utf-8") as f: + f.write(user) + with open(os.path.join(pred_dir, "conversation.json"), "w", encoding="utf-8") as f: + json.dump(conversation, f, ensure_ascii=False, indent=2) + eval_result = evaluate(final_answer, item.get("ground_truth", "")) if final_answer else {"em": 0.0, "f1": 0.0, "predicted_answer": "", "gold_answer": item.get("ground_truth", "")} + result = { + "id": item_id, + "question": item.get("question", ""), + "task_type": item.get("task_type", "officeqa"), + "task_description": item.get("question", ""), + "predicted_answer": eval_result["predicted_answer"], + "response": final_response, + "ground_truth": item.get("ground_truth", ""), + "source_files": item.get("source_files", []), + "resolved_source_paths": candidate_files, + "oracle_parsed_pages_included": bool(oracle_context), + "oracle_parsed_pages_chars": len(oracle_context), + "use_local_tools": bool(use_local_tools), + "hard": int(eval_result["em"]), + "soft": eval_result["f1"], + "fail_reason": fail_reason or ("" if eval_result["em"] else f"predicted '{eval_result['predicted_answer']}' but expected '{item.get('ground_truth', '')}'"), + "agent_ok": not fail_reason, + "n_turns": len(conversation), + "last_finish_reason": last_response_metadata.get("finish_reason", ""), + "target_system_prompt": system, + "target_user_prompt": user, + } + return result +def run_batch( + items: list[dict], + out_root: str, + skill_content: str, + *, + workers: int = 8, + max_tool_turns: int = 12, + max_completion_tokens: int = 16384, + search_mode: str = _DEFAULT_SEARCH_MODE, + max_queries_per_turn: int = 4, + search_api_url: str = "", + search_auth_env: str = "OFFICEQA_CUSTOM_SEARCH_AUTH", + search_provider: str = "duckduckgo", + search_max_num_results: int = 4, + search_timeout_seconds: int = 20, + use_local_tools: bool = True, + data_dirs: list[str] | str | None = None, + diagnostic_mode: bool = False, + diagnostic_instruction: str = "", +) -> list[dict]: + results_path = os.path.join(out_root, "results.jsonl") + os.makedirs(out_root, exist_ok=True) + done_ids: set[str] = set() + existing: list[dict] = [] + if os.path.exists(results_path): + with open(results_path, encoding="utf-8") as f: + for line in f: + try: + row = json.loads(line) + except json.JSONDecodeError: + continue + done_ids.add(str(row.get("id"))) + existing.append(row) + pending = [item for item in items if str(item["id"]) not in done_ids] + if not pending: + return existing + total = len(existing) + len(pending) + completed = len(existing) + correct_count = sum(1 for r in existing if r.get("hard", 0)) + if existing: + print(f" [rollout] resuming: {completed}/{total} already done", flush=True) + + results = list(existing) + with open(results_path, "a", encoding="utf-8") as outf, ThreadPoolExecutor(max_workers=workers) as ex: + futs = { + ex.submit( + process_one, + item, + out_root, + skill_content, + max_tool_turns=max_tool_turns, + max_completion_tokens=max_completion_tokens, + search_mode=search_mode, + max_queries_per_turn=max_queries_per_turn, + search_api_url=search_api_url, + search_auth_env=search_auth_env, + search_provider=search_provider, + search_max_num_results=search_max_num_results, + search_timeout_seconds=search_timeout_seconds, + use_local_tools=use_local_tools, + data_dirs=data_dirs, + diagnostic_mode=diagnostic_mode, + diagnostic_instruction=diagnostic_instruction, + ): item + for item in pending + } + for fut in as_completed(futs): + res = fut.result() + results.append(res) + completed += 1 + if res.get("hard", 0): + correct_count += 1 + acc = correct_count / completed if completed else 0 + print( + f" [rollout] {completed}/{total} " + f"(acc={acc:.3f}) id={res.get('id', '?')} " + f"hard={res.get('hard', '?')}", + flush=True, + ) + outf.write(json.dumps(res, ensure_ascii=False) + "\n") + outf.flush() + return results diff --git a/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/officeqa/skills/initial.md b/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/officeqa/skills/initial.md new file mode 100644 index 00000000..530b7538 --- /dev/null +++ b/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/officeqa/skills/initial.md @@ -0,0 +1,15 @@ +# OfficeQA Skill + +## Retrieval Discipline +- Start by narrowing to the most likely candidate file before reading long passages. +- Prefer targeted search terms that name the exact entity, period, measure, or table concept from the question. +- After a promising match, read only a small surrounding span and verify it matches the requested year, basis, and unit. + +## Evidence Discipline +- Extract the exact value from the retrieved text before doing any arithmetic. +- Keep track of each operand's period, unit, and semantic role so nearby proxy values are not mixed in. +- If the question asks for a transformed or derived quantity, compute only after confirming every operand. + +## Final Answer Discipline +- Return the final answer only after one last consistency check against the retrieved evidence. +- Copy the final answer from a checked value, not from an unverified intermediate guess. diff --git a/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/officeqa/tool_runtime.py b/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/officeqa/tool_runtime.py new file mode 100644 index 00000000..89be327d --- /dev/null +++ b/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/officeqa/tool_runtime.py @@ -0,0 +1,552 @@ +from __future__ import annotations + +import fnmatch +import html +import json +import os +import re +import socket +import time +from functools import lru_cache +from html.parser import HTMLParser +from pathlib import Path +from urllib.error import HTTPError, URLError +from urllib.parse import parse_qs, urlparse +from urllib.request import Request, urlopen + +_MAX_READ_CHARS = 4000 +_MAX_GREP_MATCHES = 20 +_MAX_GLOB_MATCHES = 50 +_MAX_ORACLE_PAGE_CHARS = 24000 +_MAX_ORACLE_CONTEXT_CHARS = 80000 +DEFAULT_USER_AGENT = ( + "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 " + "(KHTML, like Gecko) Chrome/135.0 Safari/537.36" +) +DEFAULT_CUSTOM_SEARCH_URL = "http://apisix.westus2.cloudapp.azure.com/search_tool/search" +DEFAULT_CUSTOM_SEARCH_AUTH_ENV = "OFFICEQA_CUSTOM_SEARCH_AUTH" +DEFAULT_CUSTOM_SEARCH_PROVIDER = "duckduckgo" +DEFAULT_CUSTOM_SEARCH_MAX_RESULTS = 4 +DEFAULT_CUSTOM_SEARCH_TIMEOUT = 20 +DEFAULT_CUSTOM_SEARCH_MAX_RETRIES = 4 +DEFAULT_CUSTOM_SEARCH_INITIAL_BACKOFF_SECONDS = 1.0 + + +def _normalize_data_dirs(data_dirs: list[str] | tuple[str, ...] | str | None, project_root: Path) -> list[str]: + if data_dirs is None: + return [] + if isinstance(data_dirs, str): + items = [part.strip() for chunk in data_dirs.split(os.pathsep) for part in chunk.split(",")] + else: + items = [str(item).strip() for item in data_dirs] + resolved: list[str] = [] + for item in items: + if not item: + continue + path = Path(item).expanduser() + if not path.is_absolute(): + path = project_root / path + resolved.append(str(path)) + return resolved + + +def resolve_docs_roots(data_dirs: list[str] | tuple[str, ...] | str | None = None) -> list[str]: + project_root = Path(__file__).resolve().parents[3] + env_value = os.environ.get("OFFICEQA_DOCS_DIR", "").strip() + candidates = _normalize_data_dirs(data_dirs, project_root) + candidates.extend(_normalize_data_dirs(env_value, project_root)) + candidates.extend([ + str(project_root / "data" / "officeqa_docs_official"), + str(project_root / "data" / "officeqa_smoke_docs"), + os.path.expanduser("~/officeqa-sparse/treasury_bulletins_parsed"), + os.path.expanduser("~/officeqa/treasury_bulletins_parsed"), + ]) + roots: list[str] = [] + seen: set[str] = set() + for candidate in candidates: + path = Path(candidate).expanduser() + if not path.is_dir(): + continue + transformed = path / "transformed" + resolved = str((transformed if transformed.is_dir() else path).resolve()) + if resolved in seen: + continue + seen.add(resolved) + roots.append(resolved) + if not roots: + raise FileNotFoundError("OfficeQA docs directory not found. Set OFFICEQA_DOCS_DIR or env.data_dirs.") + return roots + + +def _is_allowed(path: str, allowed_roots: list[str], allowed_files: list[str]) -> bool: + try: + resolved = str(Path(path).resolve()) + except FileNotFoundError: + return False + if not any(resolved.startswith(root + os.sep) or resolved == root for root in allowed_roots): + return False + if not allowed_files: + return True + base = os.path.basename(resolved) + return base in allowed_files + + +def resolve_candidate_files(source_files: list[str], allowed_roots: list[str]) -> list[str]: + resolved: list[str] = [] + seen: set[str] = set() + for root in allowed_roots: + for dirpath, _, filenames in os.walk(root): + for filename in filenames: + if source_files and filename not in source_files: + continue + full = str(Path(dirpath, filename).resolve()) + if full in seen: + continue + seen.add(full) + resolved.append(full) + return resolved + + +def _as_list(value: object) -> list[str]: + if value is None: + return [] + if isinstance(value, list): + return [str(item).strip() for item in value if str(item).strip()] + text = str(value).strip() + if not text: + return [] + try: + loaded = json.loads(text) + except json.JSONDecodeError: + loaded = None + if isinstance(loaded, list): + return [str(item).strip() for item in loaded if str(item).strip()] + if "\n" in text: + return [part.strip() for part in text.splitlines() if part.strip()] + return [text] + + +def _extract_page_number(source_doc: str) -> int | None: + text = str(source_doc or "").strip() + if not text: + return None + parsed = urlparse(text) + query = parse_qs(parsed.query) + for key in ("page", "pagenum", "page_id"): + for raw_value in query.get(key, []): + try: + return int(str(raw_value).strip()) + except ValueError: + continue + match = re.search(r"(?:[?&]|^)page=(\d+)", text) + if match: + return int(match.group(1)) + return None + + +def _iter_oracle_refs(source_files: object, source_docs: object) -> list[tuple[str, int, str]]: + files = _as_list(source_files) + docs = _as_list(source_docs) + refs: list[tuple[str, int, str]] = [] + seen: set[tuple[str, int, str]] = set() + if not files or not docs: + return refs + for index, source_doc in enumerate(docs): + page_number = _extract_page_number(source_doc) + if page_number is None: + continue + if index < len(files): + source_file = files[index] + elif len(files) == 1: + source_file = files[0] + else: + continue + key = (source_file, page_number, source_doc) + if key in seen: + continue + seen.add(key) + refs.append(key) + return refs + + +def _parsed_root_candidates(docs_roots: list[str]) -> list[Path]: + candidates: list[Path] = [] + seen: set[str] = set() + for root in docs_roots: + path = Path(root).expanduser() + for candidate in ( + path, + path.parent, + path / "treasury_bulletins_parsed", + path.parent / "treasury_bulletins_parsed", + ): + resolved = str(candidate.resolve()) if candidate.exists() else str(candidate) + if resolved in seen: + continue + seen.add(resolved) + candidates.append(candidate) + return candidates + + +def _locate_parsed_json(source_file: str, docs_roots: list[str]) -> Path | None: + source_path = Path(str(source_file).strip()) + stem = source_path.stem if source_path.suffix else source_path.name + if not stem: + return None + candidate_names = [stem + ".json"] + if source_path.suffix == ".json": + candidate_names.insert(0, source_path.name) + for root in _parsed_root_candidates(docs_roots): + for name in candidate_names: + path = root / "jsons" / name + if path.is_file(): + return path + return None + + +class _TableMarkdownParser(HTMLParser): + def __init__(self) -> None: + super().__init__(convert_charrefs=True) + self.rows: list[list[str]] = [] + self._row: list[str] | None = None + self._cell: list[str] | None = None + + def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None: + if tag.lower() == "tr": + self._row = [] + elif tag.lower() in {"td", "th"} and self._row is not None: + self._cell = [] + + def handle_data(self, data: str) -> None: + if self._cell is not None: + self._cell.append(data) + + def handle_endtag(self, tag: str) -> None: + normalized_tag = tag.lower() + if normalized_tag in {"td", "th"} and self._cell is not None and self._row is not None: + cell = re.sub(r"\s+", " ", "".join(self._cell)).strip() + self._row.append(cell) + self._cell = None + elif normalized_tag == "tr" and self._row is not None: + if any(cell for cell in self._row): + self.rows.append(self._row) + self._row = None + self._cell = None + + +def _escape_markdown_cell(value: str) -> str: + return str(value).replace("\n", " ").replace("|", "\\|").strip() + + +def _html_table_to_markdown(raw_html: str) -> str: + parser = _TableMarkdownParser() + try: + parser.feed(raw_html) + except Exception: # noqa: BLE001 + parser.rows = [] + rows = parser.rows + if not rows: + text = re.sub(r"(?is)<[^>]+>", " ", raw_html) + return re.sub(r"\s+", " ", html.unescape(text)).strip() + width = max(len(row) for row in rows) + normalized_rows = [row + [""] * (width - len(row)) for row in rows] + header = normalized_rows[0] + body = normalized_rows[1:] + lines = [ + "| " + " | ".join(_escape_markdown_cell(cell) for cell in header) + " |", + "| " + " | ".join(["---"] * width) + " |", + ] + lines.extend("| " + " | ".join(_escape_markdown_cell(cell) for cell in row) + " |" for row in body) + return "\n".join(lines) + + +def _render_parsed_content(content: str) -> str: + text = content.strip() + if not text: + return "" + if " set[int]: + page_ids: set[int] = set() + bbox = element.get("bbox") + if not isinstance(bbox, list): + return page_ids + for box in bbox: + if not isinstance(box, dict): + continue + raw_page_id = box.get("page_id") + try: + page_ids.add(int(raw_page_id)) + except (TypeError, ValueError): + continue + return page_ids + + +@lru_cache(maxsize=256) +def _load_parsed_elements(json_path: str) -> tuple[dict, ...]: + with open(json_path, encoding="utf-8") as f: + payload = json.load(f) + document = payload.get("document") if isinstance(payload, dict) else {} + elements = document.get("elements") if isinstance(document, dict) else [] + if not isinstance(elements, list): + return () + return tuple(element for element in elements if isinstance(element, dict)) + + +@lru_cache(maxsize=2048) +def _render_parsed_page(json_path: str, page_number: int) -> str: + rendered: list[str] = [] + for element in _load_parsed_elements(json_path): + if page_number not in _element_page_ids(element): + continue + content = element.get("content") + if not isinstance(content, str) or not content.strip(): + continue + section = _render_parsed_content(content) + if section: + rendered.append(section) + return "\n\n".join(rendered).strip() + + +def build_oracle_parsed_pages_context( + source_files: object, + source_docs: object, + docs_roots: list[str], + *, + max_page_chars: int = _MAX_ORACLE_PAGE_CHARS, + max_total_chars: int = _MAX_ORACLE_CONTEXT_CHARS, + evidence_note: str = "Treat it as primary document evidence and combine it with custom web search results when useful.", +) -> str: + """Render oracle parsed OfficeQA pages referenced by source_docs/source_files.""" + refs = _iter_oracle_refs(source_files, source_docs) + if not refs: + return "" + + blocks: list[str] = [] + total_chars = 0 + seen_pages: set[tuple[str, int]] = set() + for source_file, page_number, source_doc in refs: + json_path = _locate_parsed_json(source_file, docs_roots) + if json_path is None: + continue + page_key = (str(json_path), page_number) + if page_key in seen_pages: + continue + seen_pages.add(page_key) + page_text = _render_parsed_page(str(json_path), page_number) + if not page_text: + continue + if len(page_text) > max_page_chars: + omitted = len(page_text) - max_page_chars + page_text = page_text[:max_page_chars].rstrip() + f"\n\n[... {omitted} characters omitted from this parsed page ...]" + block = ( + f"### {source_file} page {page_number}\n" + f"Source URL: {source_doc}\n\n" + f"{page_text}" + ) + if total_chars + len(block) > max_total_chars: + remaining = max_total_chars - total_chars + if remaining <= 0: + break + block = block[:remaining].rstrip() + "\n\n[... oracle parsed page context truncated ...]" + blocks.append(block) + break + blocks.append(block) + total_chars += len(block) + if not blocks: + return "" + return ( + "The following content is pre-parsed from the oracle OfficeQA source page(s). " + f"{evidence_note.strip()}\n\n" + + "\n\n".join(blocks) + ) + + +def _extract_search_items(payload: object) -> list[dict]: + if isinstance(payload, list): + return [item for item in payload if isinstance(item, dict)] + if not isinstance(payload, dict): + return [] + candidate_keys = ( + "results", + "items", + "data", + "organic", + "organic_results", + "search_results", + "webPages", + "value", + ) + for key in candidate_keys: + value = payload.get(key) + if isinstance(value, list): + return [item for item in value if isinstance(item, dict)] + if isinstance(value, dict): + nested = _extract_search_items(value) + if nested: + return nested + return [] + + +def _normalize_search_item(item: dict, index: int) -> str: + title = str( + item.get("title") + or item.get("name") + or item.get("headline") + or item.get("source") + or f"Result {index}" + ).strip() + url = str( + item.get("url") + or item.get("link") + or item.get("href") + or item.get("display_url") + or "" + ).strip() + snippet = str( + item.get("snippet") + or item.get("description") + or item.get("body") + or item.get("text") + or item.get("content") + or "" + ).strip() + lines = [f"[{index}] {title}"] + if url: + lines.append(f"URL: {url}") + if snippet: + lines.append(f"Snippet: {snippet}") + return "\n".join(lines) + + +def _format_search_payload(query: str, payload: object) -> str: + items = _extract_search_items(payload) + header = f"Query: {query}" + if not items: + body = json.dumps(payload, ensure_ascii=False) if payload else "[no results]" + return f"{header}\n{body}" + rendered = [_normalize_search_item(item, index) for index, item in enumerate(items, start=1)] + return f"{header}\n\n" + "\n\n".join(rendered) + + +def _is_retryable_search_http_error(status_code: int) -> bool: + return status_code in {408, 429} or status_code >= 500 + + +def custom_search( + query: str, + *, + api_url: str = DEFAULT_CUSTOM_SEARCH_URL, + auth_token: str | None = None, + auth_env: str = DEFAULT_CUSTOM_SEARCH_AUTH_ENV, + provider: str = DEFAULT_CUSTOM_SEARCH_PROVIDER, + max_num_results: int = DEFAULT_CUSTOM_SEARCH_MAX_RESULTS, + timeout: int = DEFAULT_CUSTOM_SEARCH_TIMEOUT, + max_retries: int = DEFAULT_CUSTOM_SEARCH_MAX_RETRIES, + initial_backoff_seconds: float = DEFAULT_CUSTOM_SEARCH_INITIAL_BACKOFF_SECONDS, +) -> str: + query = str(query or "").strip() + if not query: + raise ValueError("custom_search query must be non-empty") + token = str(auth_token or os.environ.get(auth_env, "")).strip() + if not token: + raise ValueError(f"custom_search auth token missing; set {auth_env}") + payload = json.dumps( + { + "query": query, + "max_num_results": int(max_num_results), + "provider": provider, + }, + ensure_ascii=False, + ).encode("utf-8") + req = Request( + api_url, + data=payload, + headers={ + "Authorization": token, + "Content-Type": "application/json", + "User-Agent": DEFAULT_USER_AGENT, + }, + method="POST", + ) + attempts = max(1, int(max_retries) + 1) + last_error: RuntimeError | None = None + for attempt in range(1, attempts + 1): + try: + with urlopen(req, timeout=timeout) as response: + raw_body = response.read().decode("utf-8", errors="ignore") + break + except HTTPError as exc: + detail = exc.read().decode("utf-8", errors="ignore") + last_error = RuntimeError(f"custom_search HTTP {exc.code}: {detail[:1000]}") + if attempt >= attempts or not _is_retryable_search_http_error(exc.code): + raise last_error from exc + except (URLError, TimeoutError, socket.timeout) as exc: + last_error = RuntimeError(f"custom_search connection error: {exc}") + if attempt >= attempts: + raise last_error from exc + backoff_seconds = max(0.0, float(initial_backoff_seconds)) * (2 ** (attempt - 1)) + if backoff_seconds > 0: + time.sleep(backoff_seconds) + else: + raise last_error or RuntimeError("custom_search failed without a captured error") + try: + parsed = json.loads(raw_body) + except json.JSONDecodeError: + return f"Query: {query}\n\n{raw_body.strip() or '[empty response]'}" + return _format_search_payload(query, parsed) + + +def run_tool(name: str, arguments: dict, *, allowed_roots: list[str], allowed_files: list[str]) -> tuple[str, str]: + if name == "glob": + pattern = str(arguments.get("pattern") or "*") + matches: list[str] = [] + for root in allowed_roots: + for dirpath, _, filenames in os.walk(root): + for filename in filenames: + if allowed_files and filename not in allowed_files: + continue + rel = os.path.relpath(os.path.join(dirpath, filename), root) + if fnmatch.fnmatch(rel, pattern) or fnmatch.fnmatch(filename, pattern): + matches.append(os.path.join(dirpath, filename)) + if len(matches) >= _MAX_GLOB_MATCHES: + break + if len(matches) >= _MAX_GLOB_MATCHES: + break + return f"glob(pattern={pattern!r})", "\n".join(matches) if matches else "[no matches]" + + if name == "read": + path = str(arguments.get("path") or "") + if not path: + return "read(path='')", "[read error: missing path]" + if not _is_allowed(path, allowed_roots, allowed_files): + return f"read(path={path!r})", "[read error: path not allowed]" + start = max(int(arguments.get("start") or 1), 1) + limit = max(int(arguments.get("limit") or 80), 1) + with open(path, encoding="utf-8") as f: + lines = f.readlines() + excerpt = "".join(lines[start - 1:start - 1 + limit]) + return f"read(path={path!r}, start={start}, limit={limit})", excerpt[:_MAX_READ_CHARS] or "[empty file]" + + if name == "grep": + pattern = str(arguments.get("pattern") or "").lower() + path = str(arguments.get("path") or "") + if not pattern or not path: + return f"grep(pattern={pattern!r}, path={path!r})", "[grep error: missing pattern or path]" + if not _is_allowed(path, allowed_roots, allowed_files): + return f"grep(pattern={pattern!r}, path={path!r})", "[grep error: path not allowed]" + matches: list[str] = [] + with open(path, encoding="utf-8") as f: + for idx, line in enumerate(f, start=1): + if pattern in line.lower(): + matches.append(f"{idx}: {line.rstrip()}") + if len(matches) >= _MAX_GREP_MATCHES: + break + return f"grep(pattern={pattern!r}, path={path!r})", "\n".join(matches) if matches else "[no matches]" + + return name, f"[tool error: unknown tool {name}]" diff --git a/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/searchqa/__init__.py b/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/searchqa/__init__.py new file mode 100644 index 00000000..d60fc5f6 --- /dev/null +++ b/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/searchqa/__init__.py @@ -0,0 +1 @@ +"""SearchQA environment package for ReflACT.""" diff --git a/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/searchqa/adapter.py b/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/searchqa/adapter.py new file mode 100644 index 00000000..2253ebe5 --- /dev/null +++ b/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/searchqa/adapter.py @@ -0,0 +1,129 @@ +"""SearchQA environment adapter for ReflACT.""" +from __future__ import annotations + +import json +import os + +from skillopt.datasets.base import BatchSpec +from skillopt.envs.base import EnvAdapter +from skillopt.envs.searchqa.dataloader import SearchQADataLoader +from skillopt.envs.searchqa.rollout import run_batch +from skillopt.gradient.reflect import run_minibatch_reflect +from skillopt.model import get_target_backend + + +class SearchQAAdapter(EnvAdapter): + """SearchQA environment adapter.""" + + def __init__( + self, + split_dir: str = "", + data_path: str = "", + split_mode: str = "ratio", + split_ratio: str = "2:1:7", + split_seed: int = 42, + split_output_dir: str = "", + max_turns: int = 1, + exec_timeout: int = 120, + workers: int = 64, + analyst_workers: int = 16, + failure_only: bool = False, + minibatch_size: int = 8, + edit_budget: int = 4, + seed: int = 42, + limit: int = 0, + max_completion_tokens: int = 16384, + ) -> None: + self.max_turns = max_turns + self.exec_timeout = exec_timeout + self.workers = workers + self.max_completion_tokens = int(max_completion_tokens) + self.analyst_workers = analyst_workers + self.failure_only = failure_only + self.minibatch_size = minibatch_size + self.edit_budget = edit_budget + self.dataloader = SearchQADataLoader( + split_dir=split_dir, + data_path=data_path, + split_mode=split_mode, + split_ratio=split_ratio, + split_seed=split_seed, + split_output_dir=split_output_dir, + seed=seed, + limit=limit, + ) + + def setup(self, cfg: dict) -> None: + super().setup(cfg) + self.dataloader.setup(cfg) + + def get_dataloader(self): + return self.dataloader + + def build_env_from_batch(self, batch: BatchSpec, **kwargs): + return list(batch.payload or []) + + def build_train_env(self, batch_size: int, seed: int, **kwargs): + batch = self.dataloader.build_train_batch(batch_size=batch_size, seed=seed, **kwargs) + return self.build_env_from_batch(batch, **kwargs) + + def build_eval_env(self, env_num: int, split: str, seed: int, **kwargs): + batch = self.dataloader.build_eval_batch(env_num=env_num, split=split, seed=seed, **kwargs) + return self.build_env_from_batch(batch, **kwargs) + + def rollout( + self, + env_manager, # actually list[dict] for SearchQA + skill_content: str, + out_dir: str, + **kwargs, + ) -> list[dict]: + """Run QA agent on items. Resume-aware.""" + items: list[dict] = env_manager # type alias for clarity + return run_batch( + items=items, + out_root=out_dir, + skill_content=skill_content, + max_turns=self.max_turns, + exec_timeout=self.exec_timeout, + workers=self.workers, + max_completion_tokens=self.max_completion_tokens, + diagnostic_mode=kwargs.get("diagnostic_mode", False), + diagnostic_instruction=kwargs.get("diagnostic_instruction", ""), + diagnostic_trace_context_by_id=kwargs.get("diagnostic_trace_context_by_id"), + task_timeout=self.exec_timeout, + ) + + def reflect( + self, + results: list[dict], + skill_content: str, + out_dir: str, + **kwargs, + ) -> list[dict | None]: + prediction_dir = kwargs.get("prediction_dir", os.path.join(out_dir, "predictions")) + patches_dir = kwargs.get("patches_dir", os.path.join(out_dir, "patches")) + random_seed = kwargs.get("random_seed") + step_buffer_context = kwargs.get("step_buffer_context", "") + meta_skill_context = kwargs.get("meta_skill_context", "") + + return run_minibatch_reflect( + results=results, + skill_content=skill_content, + prediction_dir=prediction_dir, + patches_dir=patches_dir, + workers=self.analyst_workers, + failure_only=self.failure_only, + minibatch_size=self.minibatch_size, + edit_budget=self.edit_budget, + random_seed=random_seed, + error_system=self.get_error_minibatch_prompt(), + success_system=self.get_success_minibatch_prompt(), + step_buffer_context=step_buffer_context, + meta_skill_context=meta_skill_context, + update_mode=getattr(self, "_cfg", {}).get("skill_update_mode", "patch"), + ) + + + def get_task_types(self) -> list[str]: + return ["qa"] diff --git a/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/searchqa/dataloader.py b/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/searchqa/dataloader.py new file mode 100644 index 00000000..2dc1c1e0 --- /dev/null +++ b/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/searchqa/dataloader.py @@ -0,0 +1,42 @@ +"""SearchQA task dataloader.""" +from __future__ import annotations + +import json + +from skillopt.datasets.base import SplitDataLoader + + +# ── Raw data loading utilities (for preprocessing / standalone eval) ───── + +def _load_items(path: str) -> list[dict]: + """Load items from JSON or JSONL file.""" + with open(path) as f: + content = f.read().strip() + try: + data = json.loads(content) + if isinstance(data, list): + return data + if isinstance(data, dict): + return data.get("data") or list(data.values()) + except json.JSONDecodeError: + pass + + items = [] + for line in content.splitlines(): + line = line.strip() + if line: + items.append(json.loads(line)) + return items + + +# ── Dataloader ─────────────────────────────────────────────────────────── + +class SearchQADataLoader(SplitDataLoader): + """SearchQA dataloader. + + Each split directory (train/, val/, test/) contains a .json file — + a JSON array of question items. + """ + + def load_raw_items(self, data_path: str) -> list[dict]: + return _load_items(data_path) diff --git a/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/searchqa/evaluator.py b/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/searchqa/evaluator.py new file mode 100644 index 00000000..8c6c488f --- /dev/null +++ b/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/searchqa/evaluator.py @@ -0,0 +1,100 @@ +"""SearchQA evaluation metrics: Exact Match, F1, and Substring Match. + +Normalization follows the SQuAD convention: + - lowercase + - remove punctuation + - remove articles (a, an, the) + - collapse whitespace + +Answer extraction looks for ... XML tags, +falling back to the last non-empty line of the response. +""" +from __future__ import annotations + +import re +import string +from collections import Counter + + +def normalize_answer(s: str) -> str: + """Normalize answer string (SQuAD convention).""" + s = s.lower() + s = "".join(ch for ch in s if ch not in string.punctuation) + s = re.sub(r"\b(a|an|the)\b", " ", s) + s = " ".join(s.split()) + return s.strip() + + +def extract_answer(text: str) -> str: + """Extract answer from ... tags. + + Fallback: last non-empty line, then full response stripped. + """ + matches = re.findall(r"(.*?)", text, re.DOTALL | re.IGNORECASE) + if matches: + return matches[-1].strip() + lines = [ln.strip() for ln in text.strip().splitlines() if ln.strip()] + if lines: + return lines[-1] + return text.strip() + + +def exact_match(prediction: str, gold_answers: list[str]) -> float: + norm_pred = normalize_answer(prediction) + for gold in gold_answers: + if normalize_answer(gold) == norm_pred: + return 1.0 + return 0.0 + + +def f1_score(prediction: str, gold_answers: list[str]) -> float: + """Token-level F1 (SQuAD-style), max across all gold answers.""" + norm_pred = normalize_answer(prediction) + pred_tokens = norm_pred.split() + + if not pred_tokens: + for gold in gold_answers: + if not normalize_answer(gold).split(): + return 1.0 + return 0.0 + + best_f1 = 0.0 + for gold in gold_answers: + gold_tokens = normalize_answer(gold).split() + if not gold_tokens: + continue + common = Counter(pred_tokens) & Counter(gold_tokens) + n_common = sum(common.values()) + if n_common == 0: + continue + precision = n_common / len(pred_tokens) + recall = n_common / len(gold_tokens) + f1 = 2 * precision * recall / (precision + recall) + best_f1 = max(best_f1, f1) + + return best_f1 + + +def sub_em(prediction: str, gold_answers: list[str]) -> float: + """1.0 if any normalized gold is a substring of prediction, or vice versa.""" + norm_pred = normalize_answer(prediction) + for gold in gold_answers: + norm_gold = normalize_answer(gold) + if norm_gold in norm_pred or norm_pred in norm_gold: + return 1.0 + return 0.0 + + +def evaluate(prediction_text: str, gold_answers: list[str]) -> dict: + """Evaluate a single QA prediction against gold answers. + + Returns dict with: em, f1, sub_em, predicted_answer, gold_answers. + """ + answer = extract_answer(prediction_text) + return { + "em": exact_match(answer, gold_answers), + "f1": f1_score(answer, gold_answers), + "sub_em": sub_em(answer, gold_answers), + "predicted_answer": answer, + "gold_answers": gold_answers, + } diff --git a/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/searchqa/prompts/analyst_error.md b/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/searchqa/prompts/analyst_error.md new file mode 100644 index 00000000..a60e73dd --- /dev/null +++ b/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/searchqa/prompts/analyst_error.md @@ -0,0 +1,46 @@ +You are an expert failure-analysis agent for question answering tasks. + +You will be given MULTIPLE failed QA agent responses from a single minibatch +and the current skill document. Each trajectory includes the agent's response +and an evaluation result showing the predicted answer vs. the gold answer(s). + +Your job is to identify the most important COMMON failure patterns across +the batch and propose a concise set of skill edits. + +## Failure Type Categories +- **rule_missing**: the skill lacks a relevant rule for this type of question +- **rule_wrong**: an existing skill rule is misleading or incorrect +- **rule_ignored**: the skill has the right rule but the agent did not follow it +- **answer_format**: the agent found the right information but formatted it incorrectly +- **other**: none of the above + +## Analysis Process +1. Read ALL failed trajectories in the minibatch. +2. Carefully compare each predicted answer against the gold answer(s) — + understand exactly WHY the Exact Match failed. +3. Identify the most prevalent, systematic failure patterns across them. +4. For each pattern, classify its failure type. +5. Propose skill edits that address the COMMON patterns — not individual edge cases. +6. Edits must be generalizable; do not hardcode question-specific values. +7. Only patch gaps in the skill — do not duplicate existing content. + +You will be told the maximum number of edits (the budget L). Produce AT MOST L edits, +focusing on the highest-impact patterns. You may produce fewer if warranted. + +Respond ONLY with a valid JSON object (no markdown fences, no extra text): +{ + "batch_size": , + "failure_summary": [ + {"failure_type": "", "count": , "description": ""} + ], + "patch": { + "reasoning": "", + "edits": [ + {"op": "append", "content": ""}, + {"op": "insert_after", "target": "", "content": ""}, + {"op": "replace", "target": "", "content": ""}, + {"op": "delete", "target": ""} + ] + } +} +Only include edits that are needed. "edits" can be an empty list if no patch is warranted. diff --git a/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/searchqa/prompts/analyst_success.md b/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/searchqa/prompts/analyst_success.md new file mode 100644 index 00000000..6476d946 --- /dev/null +++ b/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/searchqa/prompts/analyst_success.md @@ -0,0 +1,32 @@ +You are an expert success-pattern analyst for AI question answering agents. + +You will be given MULTIPLE successful QA agent responses from a single minibatch +and the current skill document. Your job is to identify generalizable behavior +patterns that are COMMON across the batch and worth encoding in the skill. + +## Rules +- Only propose patches for patterns NOT already covered in the skill. +- Focus on patterns that appear across MULTIPLE trajectories in the batch. +- Be concise. Patterns must generalize beyond specific questions. +- Prefer reinforcing existing sections over adding new top-level sections. +- If the agents' success involved a smart reading strategy or disambiguation + approach, consider reinforcing that in the patch. + +You will be told the maximum number of edits (the budget L). Produce AT MOST L edits, +focusing on the most broadly applicable patterns. You may produce fewer if warranted. + +Respond ONLY with a valid JSON object: +{ + "batch_size": , + "success_patterns": ["", ""], + "patch": { + "reasoning": "", + "edits": [ + {"op": "append", "content": ""}, + {"op": "insert_after", "target": "", "content": ""}, + {"op": "replace", "target": "", "content": ""}, + {"op": "delete", "target": ""} + ] + } +} +"edits" may be empty if the skill already covers all observed patterns. diff --git a/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/searchqa/prompts/rollout_system.md b/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/searchqa/prompts/rollout_system.md new file mode 100644 index 00000000..1befe4e0 --- /dev/null +++ b/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/searchqa/prompts/rollout_system.md @@ -0,0 +1,13 @@ +You are an expert question answering agent. + +{skill_section}## Task Format +You will receive a CONTEXT containing document passages and a QUESTION. +Read the context carefully and answer the question based on the information provided. + +## Answer Format +Think step by step, then provide your final answer inside ... tags. +Keep your answer concise — typically a few words or a short phrase. +Do not repeat the question. Do not include unnecessary explanation in the answer tags. + +Example: +Abraham Lincoln diff --git a/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/searchqa/reflect.py b/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/searchqa/reflect.py new file mode 100644 index 00000000..7a99207b --- /dev/null +++ b/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/searchqa/reflect.py @@ -0,0 +1,4 @@ +"""SearchQA Reflect stage. + +Prompts are now loaded from .md files by the base adapter. +""" diff --git a/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/searchqa/rollout.py b/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/searchqa/rollout.py new file mode 100644 index 00000000..ab7215db --- /dev/null +++ b/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/searchqa/rollout.py @@ -0,0 +1,481 @@ +"""SearchQA rollout — single-turn QA agent + batch execution. + +The QA agent receives a skill document, question, and context passages, +then produces an answer in ... tags. + +Public API +---------- +- :func:`process_one` — run + evaluate one QA item +- :func:`run_batch` — parallel execution of a list of items +""" +from __future__ import annotations + +import json +import os +import time +import traceback +from concurrent.futures import FIRST_COMPLETED, ThreadPoolExecutor, wait + +from skillopt.model import chat_target, get_target_backend, is_target_exec_backend +from skillopt.model.codex_harness import prepare_workspace, render_skill_md, run_target_exec +from skillopt.prompts import load_prompt +from skillopt.envs.searchqa.evaluator import evaluate + + +# ── Prompt templates ───────────────────────────────────────────────────────── + +_MAX_CONTEXT_CHARS = 6000 + + +def _truncate_context(context: str, max_chars: int = _MAX_CONTEXT_CHARS) -> str: + """Truncate context at [DOC] boundaries to stay within budget.""" + if len(context) <= max_chars: + return context + docs = context.split("[DOC]") + result = "" + for doc in docs: + candidate = result + "[DOC]" + doc if result else doc + if len(candidate) > max_chars: + break + result = candidate + if not result: + result = context[:max_chars] + "\n...[truncated]" + return result + + +def _build_system(skill_content: str) -> str: + if skill_content.strip(): + skill_section = f"## Skill\n{skill_content.strip()}\n\n" + else: + skill_section = "" + return load_prompt("rollout_system", env="searchqa").format(skill_section=skill_section) + + +def _build_user( + question: str, + context: str, + *, + diagnostic_mode: bool = False, + diagnostic_instruction: str = "", + diagnostic_trace_context: str = "", +) -> str: + context = _truncate_context(context) + parts = [ + f"## Context\n{context}", + f"## Question\n{question}", + ] + if diagnostic_trace_context.strip(): + parts.append( + "## Previous Codex Trace Snapshot\n" + "This is a partial transcript from an earlier attempt. Use it as your current reasoning context.\n\n" + f"{diagnostic_trace_context.strip()}" + ) + if diagnostic_mode and diagnostic_instruction.strip(): + parts.append(f"## Training Readout\n{diagnostic_instruction.strip()}") + return "\n\n".join(parts) + + +def _build_codex_skill(skill_content: str) -> str: + return render_skill_md( + skill_content, + description="Dynamic ReflACT skill for solving the current SearchQA example.", + preamble=( + "Use this skill when solving the current SearchQA task.\n" + "Read the provided context carefully, ground the answer in that context,\n" + "and return the final answer inside ...." + ), + ) + + +def _run_codex_once( + *, + pred_dir: str, + skill_content: str, + question: str, + context: str, + model: str, + timeout: int, + diagnostic_mode: bool = False, + diagnostic_instruction: str = "", + diagnostic_trace_context: str = "", + previous_response: str = "", +) -> tuple[str, str, str, str]: + user = _build_user( + question, + context, + diagnostic_mode=diagnostic_mode, + diagnostic_instruction=diagnostic_instruction, + diagnostic_trace_context=diagnostic_trace_context, + ) + task_parts = [user] + if previous_response: + task_parts.append( + "## Previous Attempt\n" + f"{previous_response}\n\n" + "Review it against the same context and question. If needed, correct it." + ) + task_text = "\n\n".join(task_parts) + skill_md = _build_codex_skill(skill_content) + work_dir = os.path.join(pred_dir, "codex_exec") + prepare_workspace( + work_dir=work_dir, + skill_md=skill_md, + task_text=task_text, + ) + prompt = ( + "Use the `skillopt-target` skill available in this workspace.\n" + "Read `task.md` and answer the SearchQA question.\n" + "Return the final answer inside ...." + ) + final_message, raw = run_target_exec( + work_dir=work_dir, + prompt=prompt, + model=model, + timeout=timeout, + ) + return final_message or raw, raw, skill_md, task_text + + +# ── Single-item execution ─────────────────────────────────────────────────── + + +def process_one( + item: dict, + out_root: str, + skill_content: str, + max_turns: int = 1, + diagnostic_mode: bool = False, + diagnostic_instruction: str = "", + diagnostic_trace_context: str = "", + exec_timeout: int = 120, + max_completion_tokens: int = 16384, +) -> dict: + """Process a single QA item: run agent + evaluate. + + Parameters + ---------- + item : dict + Must have keys: ``id``, ``question``, ``context``, ``answers``. + out_root : str + Output directory (predictions saved under ``predictions//``). + skill_content : str + Current skill document text. + max_turns : int + Max reasoning turns (1 = single-turn QA). + + Returns + ------- + dict + Result with ``hard`` (EM as int), ``soft`` (F1), etc. + """ + item_id = str(item["id"]) + question = item["question"] + context = item.get("context", "") + gold_answers = item.get("answers", []) + + result = { + "id": item_id, + "question": question, + "em": 0.0, + "f1": 0.0, + "sub_em": 0.0, + "hard": 0, + "soft": 0.0, + "predicted_answer": "", + "gold_answers": gold_answers, + "response": "", + "fail_reason": "", + "agent_ok": False, + "n_turns": 0, + } + + try: + pred_dir = os.path.join(out_root, "predictions", item_id) + os.makedirs(pred_dir, exist_ok=True) + + if is_target_exec_backend(): + from skillopt.model import azure_openai as _llm + + conversation: list[dict] = [] + response = "" + system = "" + user = "" + for turn in range(max_turns): + response, raw, system, user = _run_codex_once( + pred_dir=pred_dir, + skill_content=skill_content, + question=question, + context=context, + model=_llm.TARGET_DEPLOYMENT, + timeout=exec_timeout, + diagnostic_mode=diagnostic_mode if turn == 0 else False, + diagnostic_instruction=diagnostic_instruction if turn == 0 else "", + diagnostic_trace_context=diagnostic_trace_context if turn == 0 else "", + previous_response=response if turn > 0 else "", + ) + conversation.append({"type": "message", "turn": turn + 1, "content": response}) + if turn > 0 and "" in response.lower(): + break + + result["response"] = response + result["agent_ok"] = True + result["n_turns"] = len(conversation) + + with open(os.path.join(pred_dir, "target_system_prompt.txt"), "w") as f: + f.write(system) + with open(os.path.join(pred_dir, "target_user_prompt.txt"), "w") as f: + f.write(user) + with open(os.path.join(pred_dir, "conversation.json"), "w") as f: + json.dump(conversation, f, ensure_ascii=False, indent=2) + + eval_result = evaluate(response, gold_answers) + result["em"] = eval_result["em"] + result["f1"] = eval_result["f1"] + result["sub_em"] = eval_result["sub_em"] + result["predicted_answer"] = eval_result["predicted_answer"] + result["hard"] = int(eval_result["em"]) + result["soft"] = eval_result["f1"] + if eval_result["em"] < 1.0: + result["fail_reason"] = ( + f"EM=0: predicted '{eval_result['predicted_answer']}' " + f"but expected {gold_answers}" + ) + eval_detail = ( + f"[EVALUATION RESULT]\n" + f"Question: {question}\n" + f"Predicted answer: {eval_result['predicted_answer']!r}\n" + f"Gold answers: {gold_answers!r}\n" + f"Exact Match: {eval_result['em']}\n" + f"F1: {eval_result['f1']:.4f}" + ) + conversation.append({"role": "system", "content": eval_detail}) + with open(os.path.join(pred_dir, "conversation.json"), "w") as f: + json.dump(conversation, f, ensure_ascii=False, indent=2) + return result + + system = _build_system(skill_content) + user = _build_user( + question, + context, + diagnostic_mode=diagnostic_mode, + diagnostic_instruction=diagnostic_instruction, + diagnostic_trace_context=diagnostic_trace_context, + ) + + conversation: list[dict] = [] + response = "" + + for turn in range(max_turns): + if turn == 0: + resp_text, _ = chat_target( + system=system, user=user, + max_completion_tokens=max_completion_tokens, + retries=5, stage="rollout", + timeout=exec_timeout, + ) + else: + refinement = ( + f"Your previous answer was:\n{response}\n\n" + f"Review it against the context and question. " + f"If correct, repeat it. If wrong, provide a corrected answer.\n" + f"Use ... tags for your final answer." + ) + resp_text, _ = chat_target( + system=system, user=refinement, + max_completion_tokens=max_completion_tokens, + retries=5, stage="rollout", + timeout=exec_timeout, + ) + + response = resp_text + conversation.append({"type": "message", "turn": turn + 1, "content": resp_text}) + + if turn > 0 and "" in resp_text.lower(): + break + + result["response"] = response + result["agent_ok"] = True + result["n_turns"] = len(conversation) + + # Save conversation + with open(os.path.join(pred_dir, "target_system_prompt.txt"), "w") as f: + f.write(system) + with open(os.path.join(pred_dir, "target_user_prompt.txt"), "w") as f: + f.write(user) + with open(os.path.join(pred_dir, "conversation.json"), "w") as f: + json.dump(conversation, f, ensure_ascii=False, indent=2) + + # Evaluate + eval_result = evaluate(response, gold_answers) + result["em"] = eval_result["em"] + result["f1"] = eval_result["f1"] + result["sub_em"] = eval_result["sub_em"] + result["predicted_answer"] = eval_result["predicted_answer"] + result["hard"] = int(eval_result["em"]) + result["soft"] = eval_result["f1"] + + if eval_result["em"] < 1.0: + result["fail_reason"] = ( + f"EM=0: predicted '{eval_result['predicted_answer']}' " + f"but expected {gold_answers}" + ) + + # Append eval details to conversation for the analyst + eval_detail = ( + f"[EVALUATION RESULT]\n" + f"Question: {question}\n" + f"Predicted answer: {eval_result['predicted_answer']!r}\n" + f"Gold answers: {gold_answers!r}\n" + f"Exact Match: {eval_result['em']}\n" + f"F1: {eval_result['f1']:.4f}" + ) + conversation.append({ + "role": "system", + "content": eval_detail, + }) + # Re-save enriched conversation + with open(os.path.join(pred_dir, "conversation.json"), "w") as f: + json.dump(conversation, f, ensure_ascii=False, indent=2) + + except Exception as e: # noqa: BLE001 + result["fail_reason"] = f"error: {e}" + + return result + + +# ── Batch execution ────────────────────────────────────────────────────────── + + +def run_batch( + items: list[dict], + out_root: str, + skill_content: str, + max_turns: int = 1, + exec_timeout: int = 120, + workers: int = 64, + max_completion_tokens: int = 16384, + diagnostic_mode: bool = False, + diagnostic_instruction: str = "", + diagnostic_trace_context_by_id: dict[str, str] | None = None, + task_timeout: int = 600, +) -> list[dict]: + """Run QA agent on all items with ThreadPoolExecutor. Resume-aware.""" + task_timeout = max(int(task_timeout), int(exec_timeout) + 60) + results_path = os.path.join(out_root, "results.jsonl") + os.makedirs(out_root, exist_ok=True) + + # Resume: load already-done + done_ids: set[str] = set() + existing: list[dict] = [] + if os.path.exists(results_path): + with open(results_path) as f: + for line in f: + try: + r = json.loads(line) + done_ids.add(str(r["id"])) + existing.append(r) + except Exception: + pass + + pending = [it for it in items if str(it["id"]) not in done_ids] + if not pending: + return existing + + total = len(existing) + len(pending) + completed = len(existing) + correct_count = sum(1 for r in existing if r.get("hard", 0)) + if existing: + print(f" [rollout] resuming: {completed}/{total} already done", flush=True) + + results = list(existing) + + def _timeout_result(item: dict) -> dict: + return { + "id": str(item["id"]), + "question": item.get("question", ""), + "task_description": item.get("question", ""), + "task_type": item.get("task_type") or "searchqa", + "hard": 0, + "soft": 0.0, + "predicted_answer": "", + "response": "", + "fail_reason": f"task-timeout-{task_timeout}s", + "agent_ok": False, + "n_turns": 0, + "gold_answer": item.get("answers", []), + "phase": "timeout", + } + + def _error_result(item: dict, exc: Exception) -> dict: + row = _timeout_result(item) + row["phase"] = "error" + row["fail_reason"] = f"unexpected: {type(exc).__name__}: {exc}" + return row + + started_at: dict[str, float] = {} + + def _run_one(item: dict) -> dict: + started_at[str(item["id"])] = time.time() + return process_one( + item, + out_root, + skill_content, + max_turns, + diagnostic_mode, + diagnostic_instruction, + (diagnostic_trace_context_by_id or {}).get(str(item["id"]), ""), + exec_timeout, + max_completion_tokens, + ) + + with open(results_path, "a") as outf: + ex = ThreadPoolExecutor(max_workers=workers) + try: + futs = {ex.submit(_run_one, it): it for it in pending} + pending_futs = set(futs) + while pending_futs: + done, _ = wait(pending_futs, timeout=5, return_when=FIRST_COMPLETED) + now = time.time() + timed_out = [ + fut for fut in pending_futs - done + if str(futs[fut]["id"]) in started_at + and now - started_at[str(futs[fut]["id"])] >= task_timeout + ] + for fut in done: + pending_futs.remove(fut) + item = futs[fut] + try: + res = fut.result() + except Exception as exc: # noqa: BLE001 + res = _error_result(item, exc) + results.append(res) + completed += 1 + if res.get("hard", 0): + correct_count += 1 + acc = correct_count / completed if completed else 0 + print( + f" [rollout] {completed}/{total} " + f"(acc={acc:.3f}) id={res['id']} " + f"hard={res.get('hard', '?')}", + flush=True, + ) + outf.write(json.dumps(res, ensure_ascii=False) + "\n") + outf.flush() + for fut in timed_out: + pending_futs.remove(fut) + fut.cancel() + res = _timeout_result(futs[fut]) + results.append(res) + completed += 1 + acc = correct_count / completed if completed else 0 + print( + f" [rollout] {completed}/{total} " + f"(acc={acc:.3f}) id={res['id']} TIMEOUT", + flush=True, + ) + outf.write(json.dumps(res, ensure_ascii=False) + "\n") + outf.flush() + finally: + ex.shutdown(wait=False, cancel_futures=True) + + return results diff --git a/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/searchqa/skills/initial.md b/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/searchqa/skills/initial.md new file mode 100644 index 00000000..6bc64d80 --- /dev/null +++ b/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/searchqa/skills/initial.md @@ -0,0 +1,3 @@ +# Question Answering Skill + +(No learned rules yet. Rules will be added through the reflection process.) diff --git a/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/spreadsheetbench/__init__.py b/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/spreadsheetbench/__init__.py new file mode 100644 index 00000000..3db374b8 --- /dev/null +++ b/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/spreadsheetbench/__init__.py @@ -0,0 +1,5 @@ +"""SpreadsheetBench environment adapter for ReflACT.""" + +from skillopt.envs.spreadsheetbench.adapter import SpreadsheetBenchAdapter + +__all__ = ["SpreadsheetBenchAdapter"] diff --git a/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/spreadsheetbench/adapter.py b/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/spreadsheetbench/adapter.py new file mode 100644 index 00000000..5b2b6782 --- /dev/null +++ b/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/spreadsheetbench/adapter.py @@ -0,0 +1,192 @@ +"""SpreadsheetBench environment adapter for ReflACT. + +Connects the ReflACT training loop to SpreadsheetBench by implementing +:class:`~skillopt.envs.base.EnvAdapter`. +""" +from __future__ import annotations + +import json +import os + +from skillopt.datasets.base import BatchSpec +from skillopt.envs.base import EnvAdapter +from skillopt.envs.spreadsheetbench.dataloader import SpreadsheetBenchDataLoader +from skillopt.envs.spreadsheetbench.rollout import ( + process_one, + run_spreadsheet_batch, + run_spreadsheet_batch_codegen, +) +from skillopt.gradient.reflect import run_minibatch_reflect +from skillopt.model import get_target_backend, is_target_exec_backend + + +# Task types used for per-category breakdowns +TASK_TYPES = ["cell_level", "sheet_level"] + + +class SpreadsheetBenchAdapter(EnvAdapter): + """SpreadsheetBench environment adapter.""" + + def __init__( + self, + split_dir: str = "", + data_path: str = "", + split_mode: str = "ratio", + split_ratio: str = "2:1:7", + split_seed: int = 42, + split_output_dir: str = "", + data_root: str = "", + mode: str = "single", + max_turns: int = 30, + exec_timeout: int = 600, + workers: int = 64, + analyst_workers: int = 16, + failure_only: bool = False, + minibatch_size: int = 8, + edit_budget: int = 4, + seed: int = 42, + max_completion_tokens: int = 16384, + ) -> None: + self.data_root = data_root + self.mode = mode # "single", "multi", or "react" + self.max_turns = max_turns + self.exec_timeout = exec_timeout + self.workers = workers + self.max_completion_tokens = int(max_completion_tokens) + self.analyst_workers = analyst_workers + self.failure_only = failure_only + self.minibatch_size = minibatch_size + self.edit_budget = edit_budget + self.dataloader = SpreadsheetBenchDataLoader( + split_dir=split_dir, + data_path=data_path, + split_mode=split_mode, + split_ratio=split_ratio, + split_seed=split_seed, + split_output_dir=split_output_dir, + data_root=data_root, + seed=seed, + ) + + def setup(self, cfg: dict) -> None: + super().setup(cfg) + if is_target_exec_backend() and self.mode != "single": + raise NotImplementedError( + "Exec target backends are currently supported only for SpreadsheetBench mode=single." + ) + self.dataloader.setup(cfg) + + def get_dataloader(self): + return self.dataloader + + def build_env_from_batch(self, batch: BatchSpec, **kwargs): + return list(batch.payload or []) + + def build_train_env(self, batch_size: int, seed: int, **kwargs): + batch = self.dataloader.build_train_batch(batch_size=batch_size, seed=seed, **kwargs) + return self.build_env_from_batch(batch, **kwargs) + + def build_eval_env(self, env_num: int, split: str, seed: int, **kwargs): + batch = self.dataloader.build_eval_batch(env_num=env_num, split=split, seed=seed, **kwargs) + return self.build_env_from_batch(batch, **kwargs) + + def rollout( + self, + env_manager, + skill_content: str, + out_dir: str, + **kwargs, + ) -> list[dict]: + """Run agent on all items and return results. + + Dispatches based on ``self.mode``: + - ``"single"`` / ``"multi"``: codegen agent (no tool-call) + - ``"react"``: ReAct agent with tool-call (legacy) + """ + items = env_manager # For static datasets, env_manager is a list of items + results_path = os.path.join(out_dir, "results.jsonl") + os.makedirs(out_dir, exist_ok=True) + + # Resume support + if os.path.exists(results_path): + existing: list[dict] = [] + with open(results_path) as f: + for line in f: + try: + existing.append(json.loads(line)) + except Exception: + pass + if existing: + return existing + + if self.mode in ("single", "multi"): + results = run_spreadsheet_batch_codegen( + items=items, + data_root=self.data_root, + out_root=out_dir, + skill_content=skill_content, + mode=self.mode, + max_turns=self.max_turns, + max_completion_tokens=self.max_completion_tokens, + max_api_workers=self.workers, + task_timeout=self.exec_timeout, + use_eval_feedback=kwargs.get("use_eval_feedback", False), + diagnostic_mode=kwargs.get("diagnostic_mode", False), + diagnostic_instruction=kwargs.get("diagnostic_instruction", ""), + diagnostic_trace_context_by_id=kwargs.get("diagnostic_trace_context_by_id"), + ) + else: + results = run_spreadsheet_batch( + items=items, + data_root=self.data_root, + out_root=out_dir, + skill_content=skill_content, + max_turns=self.max_turns, + max_completion_tokens=self.max_completion_tokens, + max_api_workers=self.workers, + task_timeout=max(600, int(self.exec_timeout) + 60), + diagnostic_mode=kwargs.get("diagnostic_mode", False), + diagnostic_instruction=kwargs.get("diagnostic_instruction", ""), + diagnostic_trace_context_by_id=kwargs.get("diagnostic_trace_context_by_id"), + ) + + with open(results_path, "w") as f: + for r in results: + f.write(json.dumps(r, ensure_ascii=False) + "\n") + + return results + + def reflect( + self, + results: list[dict], + skill_content: str, + out_dir: str, + **kwargs, + ) -> list[dict | None]: + """Analyze rollout results and produce patches (minibatch mode).""" + prediction_dir = kwargs.get("prediction_dir", os.path.join(out_dir, "predictions")) + patches_dir = kwargs.get("patches_dir", os.path.join(out_dir, "patches")) + random_seed = kwargs.get("random_seed") + step_buffer_context = kwargs.get("step_buffer_context", "") + meta_skill_context = kwargs.get("meta_skill_context", "") + + return run_minibatch_reflect( + results=results, + skill_content=skill_content, + prediction_dir=prediction_dir, + patches_dir=patches_dir, + workers=self.analyst_workers, + failure_only=self.failure_only, + minibatch_size=self.minibatch_size, + edit_budget=self.edit_budget, + random_seed=random_seed, + error_system=self.get_error_minibatch_prompt(), + success_system=self.get_success_minibatch_prompt(), + step_buffer_context=step_buffer_context, + meta_skill_context=meta_skill_context, + update_mode=getattr(self, "_cfg", {}).get("skill_update_mode", "patch"), + ) + + + def get_task_types(self) -> list[str]: + return list(TASK_TYPES) diff --git a/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/spreadsheetbench/codegen_agent.py b/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/spreadsheetbench/codegen_agent.py new file mode 100644 index 00000000..865665da --- /dev/null +++ b/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/spreadsheetbench/codegen_agent.py @@ -0,0 +1,748 @@ +"""Codegen agent for SpreadsheetBench — no tool-call, pure code generation. + +Two modes: + - **single**: One LLM call → extract ```python``` block → done. + - **multi**: Up to max_turns LLM calls; after each, execute code and + feed errors back for correction. + +This matches the official SpreadsheetBench evaluation setting (LLM generates +a Python code block, no function-calling / tool-use). +""" +from __future__ import annotations + +import json +import os +import random +import signal +import time + +import openpyxl + + +# ── Timeout helper ────────────────────────────────────────────────────────── + +class TaskTimeout(Exception): + """Raised when a task exceeds its time budget.""" + + +def _timeout_handler(signum, frame): + raise TaskTimeout("Task timed out") + +from skillopt.model.azure_openai import ( + get_reasoning_effort, + get_target_client, + _needs_responses_api, + tracker, +) +from skillopt.model import get_codex_exec_config, get_target_backend, is_target_exec_backend +from skillopt.model.codex_harness import prepare_workspace, render_skill_md, run_target_exec +from skillopt.prompts import load_prompt +from skillopt.envs.spreadsheetbench.executor import run_generated_code +from skillopt.envs.spreadsheetbench.evaluator import evaluate + + +def _xskill_native_skill_mode() -> bool: + """xskill track: model is allowed/encouraged to invoke native Skills. + + Default (env unset) == reference/noskill track: the per-task prompt redirects + to the local .agents/skills markdown and suppresses the Skill tool (current + behavior, byte-unchanged). When XSKILL_SKILL_MODE=native, the per-task prompt + instead encourages native Skill-tool invocation and does NOT redirect to + .agents/skills (empty for the xskill track). The ONLY allowed turn-0 + difference between tracks is this skill-source wording (acceptance B3).""" + return str(os.environ.get("XSKILL_SKILL_MODE", "") or "").strip().lower() == "native" + + +# ── Eval feedback helper (no golden value leakage) ───────────────────────── + +def _build_eval_feedback(verify_report: str) -> str: + """Build Target feedback from a verify report, hiding expected values. + + The verify report contains lines like: + Sheet1!D2: got=None, expected=0 ✗ + Sheet1!D10: got=None, expected=None ✓ + + We strip the ``expected=...`` part so the Target sees only its own + output and whether each cell is correct or wrong. + """ + import re + lines = ["Your code executed successfully but produced incorrect results.", + "The following cells have wrong values:"] + for raw_line in verify_report.splitlines(): + raw_line = raw_line.strip() + if not raw_line: + continue + # Match enrichment lines like " Sheet1!D2: got=None, expected=0 ✗" + m = re.match( + r"(\S+!?\w+):\s*got=(.+?),\s*expected=.+?\s*(✓|✗)$", + raw_line, + ) + if m: + cell, got_val, mark = m.groups() + if mark == "✗": + lines.append(f" {cell}: your output = {got_val} (WRONG)") + else: + lines.append(f" {cell}: correct ✓") + lines.append( + "\nPlease analyze the spreadsheet data more carefully and fix the code. " + "Return a complete corrected Python script inside a ```python``` block." + ) + return "\n".join(lines) + + +# ── Workbook preview (same as official prompt.py) ──────────────────────────── + +def _preview_workbook(path: str, max_rows: int = 5, max_cols: int = 20) -> str: + """Generate a text preview of the first few rows of each sheet.""" + wb = openpyxl.load_workbook(path, data_only=False) + chunks: list[str] = [] + for sheet_name in wb.sheetnames: + ws = wb[sheet_name] + chunks.append( + f"## Sheet: {sheet_name} " + f"(dim={ws.dimensions}, max_row={ws.max_row}, max_col={ws.max_column})" + ) + for row in ws.iter_rows( + min_row=1, + max_row=min(ws.max_row, max_rows), + max_col=min(ws.max_column, max_cols), + values_only=False, + ): + cells = [] + for cell in row: + v = cell.value + if v is None: + cells.append(f"{cell.coordinate}=") + else: + s = str(v) + if len(s) > 40: + s = s[:37] + "..." + cells.append(f"{cell.coordinate}={s}") + chunks.append(" | ".join(cells)) + if ws.max_row > max_rows: + chunks.append(f"... ({ws.max_row - max_rows} more rows)") + chunks.append("") + wb.close() + return "\n".join(chunks) + + +# ── Code extraction (same as official prompt.py) ──────────────────────────── + +def extract_code(text: str) -> str: + """Extract the first ```python``` fenced code block from LLM output.""" + if "```" not in text: + return text.strip() + start = text.find("```") + nl = text.find("\n", start) + end = text.find("```", nl + 1) + if nl == -1 or end == -1: + return text.strip() + return text[nl + 1 : end].strip() + + +# ── Prompt construction (official SpreadsheetBench prompts) ───────────────── + + +def _build_system(skill_content: str) -> str: + base = load_prompt("codegen_system", env="spreadsheetbench") + if skill_content.strip(): + base += f"\n\n## Skill\n{skill_content.strip()}" + return base + + +def _build_user( + instruction: str, + input_xlsx: str, + instruction_type: str = "", + answer_position: str = "", + diagnostic_mode: bool = False, + diagnostic_instruction: str = "", + diagnostic_trace_context: str = "", +) -> str: + try: + preview = _preview_workbook(input_xlsx) + except Exception as e: # noqa: BLE001 + preview = f"(failed to preview workbook: {e})" + extra = "" + if instruction_type: + extra += f"\nInstruction type: {instruction_type}" + if answer_position: + extra += f"\nExpected answer position: {answer_position}" + task_suffix = "Return only a ```python``` code block." + diagnostic = "" + if diagnostic_mode and diagnostic_instruction.strip(): + task_suffix = ( + "First provide a short diagnostic readout that follows the training " + "instruction below, then return a single complete ```python``` code block." + ) + diagnostic = f"\n\n# Training readout\n{diagnostic_instruction.strip()}" + prefix = "" + if diagnostic_trace_context.strip(): + prefix = ( + "# Previous Codex Trace Snapshot\n" + "This is a partial transcript from an earlier attempt. Use it as your current reasoning context.\n\n" + f"{diagnostic_trace_context.strip()}\n\n" + ) + return ( + f"{prefix}" + f"# Instruction\n{instruction}\n{extra}\n\n" + f"# Input spreadsheet preview\n{preview}\n\n" + "# Task\n" + "Write a Python script that reads the workbook from the variable `INPUT_PATH`, " + "applies the instruction, and writes the modified workbook to `OUTPUT_PATH`. " + "Preserve all other cells unchanged. " + "The preview may be truncated — do not hardcode row counts or assume the data ends at the last previewed row; " + "iterate over all actual rows in the workbook instead. " + f"{task_suffix}" + f"{diagnostic}" + ) + + +# ── LLM call with retry ──────────────────────────────────────────────────── + +def _llm_call_with_retry(call_fn, *, retries: int = 5, timeout: int | None = 120): + """Wrap an LLM API call with retry and per-call timeout.""" + last_err = None + for attempt in range(retries): + try: + return call_fn(timeout=timeout) + except Exception as e: # noqa: BLE001 + last_err = e + sleep = min(2 ** attempt + random.random(), 60) + time.sleep(sleep) + raise RuntimeError(f"LLM call failed after {retries} retries: {last_err}") + + +def _get_deployment() -> str: + from skillopt.model import azure_openai as _llm + return _llm.TARGET_DEPLOYMENT + + +def _build_codex_skill(skill_content: str) -> str: + return render_skill_md( + skill_content, + description="Dynamic ReflACT skill for solving the current SpreadsheetBench task.", + preamble=( + "Use this skill when solving the current SpreadsheetBench task in this workspace.\n" + "Write a single self-contained Python solution to `solution.py`.\n" + "The solution must operate on the provided `INPUT_PATH` and `OUTPUT_PATH` variables.\n" + "You may inspect `input.xlsx` and run `python run_solution.py` to validate locally,\n" + "but do not hardcode values from the preview or from one specific workbook." + ), + ) + + +def _build_codex_task( + instruction: str, + input_xlsx: str, + instruction_type: str, + answer_position: str, + *, + diagnostic_mode: bool, + diagnostic_instruction: str, + diagnostic_trace_context: str, +) -> str: + prompt = _build_user( + instruction, + input_xlsx, + instruction_type, + answer_position, + diagnostic_mode=diagnostic_mode, + diagnostic_instruction=diagnostic_instruction, + diagnostic_trace_context=diagnostic_trace_context, + ) + skill_line = ( + "- Use any available Skill whose description matches this task (invoke it via the Skill tool) before writing code.\n" + if _xskill_native_skill_mode() + else "- Read `.agents/skills/skillopt-target/SKILL.md` before writing code; do not call a Skill tool.\n" + ) + return ( + f"{prompt}\n\n" + "## Codex Harness Task\n" + f"{skill_line}" + "- Read and optionally inspect `input.xlsx` in this workspace.\n" + "- Write the final Python solution to `solution.py`.\n" + "- The script should use the provided `INPUT_PATH` and `OUTPUT_PATH` variables.\n" + "- If you want to validate locally, run `python run_solution.py`.\n" + "- Do not return a code fence as the primary artifact; the source of truth is `solution.py`.\n" + ) + + +def _build_codex_driver() -> str: + return ( + "import pathlib\n" + "import re\n" + "import sys\n" + "import traceback\n\n" + 'INPUT_PATH = "input.xlsx"\n' + 'OUTPUT_PATH = "output.xlsx"\n' + "code = pathlib.Path('solution.py').read_text(encoding='utf-8')\n" + "code = re.sub(r'^\\s*(INPUT_PATH|OUTPUT_PATH)\\s*=\\s*.+$', '', code, flags=re.MULTILINE)\n" + "globals_dict = {'__name__': '__main__', 'INPUT_PATH': INPUT_PATH, 'OUTPUT_PATH': OUTPUT_PATH}\n" + "try:\n" + " exec(compile(code, 'solution.py', 'exec'), globals_dict, globals_dict)\n" + "except Exception:\n" + " traceback.print_exc()\n" + " sys.exit(2)\n" + ) + + +def _prepare_codex_workspace( + *, + instruction: str, + input_xlsx: str, + output_path: str, + instruction_type: str, + answer_position: str, + skill_content: str, + diagnostic_mode: bool, + diagnostic_instruction: str, + diagnostic_trace_context: str, + workspace_name: str = "codex_single", +) -> tuple[str, str, str, str]: + task_out_dir = os.path.dirname(output_path) + work_dir = os.path.join(task_out_dir, workspace_name) + skill_md = _build_codex_skill(skill_content) + task_md = _build_codex_task( + instruction, + input_xlsx, + instruction_type, + answer_position, + diagnostic_mode=diagnostic_mode, + diagnostic_instruction=diagnostic_instruction, + diagnostic_trace_context=diagnostic_trace_context, + ) + skill_directive = ( + "Use any available Skill whose description matches this task by invoking it via the Skill tool, then follow its guidance.\n" + if _xskill_native_skill_mode() + else "Read `.agents/skills/skillopt-target/SKILL.md` directly; do not call a Skill tool.\n" + ) + prompt = ( + f"{skill_directive}" + "Read `task.md`, inspect `input.xlsx` if useful, and write the final solution to `solution.py`.\n" + "You may run `python run_solution.py` to validate the script locally.\n" + "In your final response, briefly confirm whether `solution.py` was written and summarize the approach." + ) + prepare_workspace( + work_dir=work_dir, + skill_md=skill_md, + task_text=task_md, + extra_files={"run_solution.py": _build_codex_driver()}, + copy_files=[(input_xlsx, "input.xlsx")], + ) + + return work_dir, skill_md, task_md, prompt + + +def _run_exec_backend( + *, + work_dir: str, + prompt: str, + model: str, + timeout: int, +) -> tuple[str, str]: + return run_target_exec( + work_dir=work_dir, + prompt=prompt, + model=model, + timeout=timeout, + allow_file_edits=True, + ) + + +# ── Chat (no tools) ──────────────────────────────────────────────────────── + +def _chat_call( + client, + deployment: str, + messages: list[dict], + max_output_tokens: int, + llm_timeout: int | None = 120, +) -> str: + """Single LLM call, no tools. Returns raw text.""" + reasoning_effort = get_reasoning_effort() + if _needs_responses_api(deployment): + # Responses API + system = "" + api_input = [] + for m in messages: + if m["role"] == "system": + system = m["content"] + else: + api_input.append({"role": m["role"], "content": m["content"]}) + resp = _llm_call_with_retry(lambda timeout: client.responses.create( + model=deployment, + instructions=system, + input=api_input, + max_output_tokens=max_output_tokens, + **({"reasoning": {"effort": reasoning_effort}} if reasoning_effort else {}), + timeout=timeout, + ), timeout=llm_timeout) + if hasattr(resp, "usage") and resp.usage: + tracker.record( + "rollout", + getattr(resp.usage, "input_tokens", 0) or 0, + getattr(resp.usage, "output_tokens", 0) or 0, + ) + text = getattr(resp, "output_text", None) or "" + if text: + return text + for item in getattr(resp, "output", None) or []: + for part in getattr(item, "content", []): + if getattr(part, "type", "") == "output_text": + return part.text or "" + return "" + else: + # Chat Completions API — no tools + kwargs = { + "model": deployment, + "messages": messages, + "max_completion_tokens": max_output_tokens, + } + if reasoning_effort is not None: + kwargs["reasoning_effort"] = reasoning_effort + resp = _llm_call_with_retry(lambda timeout: client.chat.completions.create( + **kwargs, + timeout=timeout, + ), timeout=llm_timeout) + if resp.usage: + tracker.record( + "rollout", + resp.usage.prompt_tokens or 0, + resp.usage.completion_tokens or 0, + ) + return resp.choices[0].message.content or "" + + +# ── Public API ────────────────────────────────────────────────────────────── + +def run_single( + instruction: str, + input_xlsx: str, + output_path: str, + instruction_type: str = "", + answer_position: str = "", + skill_content: str = "", + max_output_tokens: int = 16384, + llm_timeout: int | None = 120, + task_timeout: int | None = 300, + diagnostic_mode: bool = False, + diagnostic_instruction: str = "", + diagnostic_trace_context: str = "", +) -> dict: + """Single-round code generation. One LLM call, no tools. + + Args: + llm_timeout: Per-LLM-call timeout in seconds (default 120). + task_timeout: Total task timeout in seconds (default 300). + + Returns ``{"code": str, "raw": str, "n_turns": 1}``. + """ + no_task_timeout = task_timeout is None or task_timeout <= 0 + if is_target_exec_backend(): + deadline = None if no_task_timeout else time.time() + task_timeout + deployment = _get_deployment() + work_dir, skill_md, task_md, prompt = _prepare_codex_workspace( + instruction=instruction, + input_xlsx=input_xlsx, + output_path=output_path, + instruction_type=instruction_type, + answer_position=answer_position, + skill_content=skill_content, + diagnostic_mode=diagnostic_mode, + diagnostic_instruction=diagnostic_instruction, + diagnostic_trace_context=diagnostic_trace_context, + ) + if deadline is None: + effective_timeout = 10**9 + else: + remaining = max(10, int(deadline - time.time())) + effective_timeout = min(task_timeout, remaining) + final_message, raw = _run_exec_backend( + work_dir=work_dir, + prompt=prompt, + model=deployment, + timeout=effective_timeout, + ) + solution_path = os.path.join(work_dir, "solution.py") + if os.path.exists(solution_path): + with open(solution_path, encoding="utf-8") as f: + code = f.read() + else: + code = extract_code(final_message or raw) + return { + "code": code, + "raw": raw or final_message, + "n_turns": 1, + "conversation": [{"role": "assistant", "content": final_message or raw}], + "target_system_prompt": skill_md, + "target_user_prompt": f"{prompt}\n\n## Task File\n\n{task_md}", + } + + deadline = None if no_task_timeout else time.time() + task_timeout + client = get_target_client() + deployment = _get_deployment() + system = _build_system(skill_content) + user = _build_user( + instruction, + input_xlsx, + instruction_type, + answer_position, + diagnostic_mode=diagnostic_mode, + diagnostic_instruction=diagnostic_instruction, + diagnostic_trace_context=diagnostic_trace_context, + ) + + messages = [ + {"role": "system", "content": system}, + {"role": "user", "content": user}, + ] + + if deadline is None: + effective_timeout = None + else: + remaining = max(10, int(deadline - time.time())) + effective_timeout = min(llm_timeout or remaining, remaining) + raw = _chat_call(client, deployment, messages, max_output_tokens, llm_timeout=effective_timeout) + time.sleep(3) # Rate-limit cooldown after successful LLM call + code = extract_code(raw) + + return { + "code": code, + "raw": raw, + "n_turns": 1, + "conversation": [{"role": "assistant", "content": raw}], + "target_system_prompt": system, + "target_user_prompt": user, + } + + +def run_multi( + instruction: str, + input_xlsx: str, + output_path: str, + instruction_type: str = "", + answer_position: str = "", + skill_content: str = "", + max_turns: int = 5, + max_output_tokens: int = 16384, + llm_timeout: int | None = 120, + task_timeout: int | None = 600, + gold_path: str = "", + diagnostic_mode: bool = False, + diagnostic_instruction: str = "", + diagnostic_trace_context: str = "", +) -> dict: + """Multi-round code generation with execution feedback. No tools. + + Each round: LLM generates code → execute → if error, feed back and retry. + + Args: + llm_timeout: Per-LLM-call timeout in seconds (default 120). + task_timeout: Total task timeout in seconds (default 600). + gold_path: Path to golden answer xlsx for eval feedback during + training. When non-empty, a successful execution is followed + by an eval check; if the output is wrong the agent receives + cell-level feedback (without revealing expected values) and + gets another turn. Leave empty for eval/test to avoid + data leakage. + + Returns ``{"code": str, "raw": str, "n_turns": int, "conversation": [...]}``. + """ + no_task_timeout = task_timeout is None or task_timeout <= 0 + if is_target_exec_backend(): + deadline = None if no_task_timeout else time.time() + task_timeout + deployment = _get_deployment() + work_dir, skill_md, task_md, initial_prompt = _prepare_codex_workspace( + instruction=instruction, + input_xlsx=input_xlsx, + output_path=output_path, + instruction_type=instruction_type, + answer_position=answer_position, + skill_content=skill_content, + diagnostic_mode=diagnostic_mode, + diagnostic_instruction=diagnostic_instruction, + diagnostic_trace_context=diagnostic_trace_context, + workspace_name="codex_multi", + ) + prompt = ( + f"{initial_prompt}\n\n" + "## Multi-Turn Repair Mode\n" + "- This is turn 1. Write or overwrite `solution.py`.\n" + "- After each turn, the harness will execute your `solution.py`; if it fails, you will receive feedback and may revise it.\n" + "- Keep the script general: use `INPUT_PATH` and `OUTPUT_PATH`, and do not hardcode one workbook's values." + ) + conversation: list[dict] = [] + code = "" + raw = "" + final_message = "" + solution_path = os.path.join(work_dir, "solution.py") + + for turn in range(max_turns): + if deadline is None: + effective_timeout = 10**9 + else: + remaining = deadline - time.time() + if remaining <= 10: + break + effective_timeout = max(10, int(remaining)) + final_message, raw = _run_exec_backend( + work_dir=work_dir, + prompt=prompt, + model=deployment, + timeout=effective_timeout, + ) + conversation.append({"role": "assistant", "content": final_message or raw}) + + if os.path.exists(solution_path): + with open(solution_path, encoding="utf-8") as f: + code = f.read() + else: + code = extract_code(final_message or raw) + if code.strip(): + with open(solution_path, "w", encoding="utf-8") as f: + f.write(code) + + if not code.strip(): + feedback = ( + "No usable `solution.py` or Python code block was produced. " + "Write a complete `solution.py` that reads `INPUT_PATH` and saves `OUTPUT_PATH`." + ) + else: + ok, err = run_generated_code( + code, + input_xlsx, + output_path, + timeout=None if no_task_timeout else 120, + ) + if ok: + if gold_path and answer_position: + from skillopt.envs.spreadsheetbench.rollout import _auto_verify_output + eval_result = evaluate( + output_path, gold_path, instruction_type, answer_position, + ) + if eval_result["ok"]: + break + verify = _auto_verify_output(output_path, gold_path, answer_position) + feedback = _build_eval_feedback(verify) + else: + break + else: + feedback = ( + "The current `solution.py` raised an error during harness execution:\n\n" + f"```\n{err[:3000]}\n```\n\n" + "Revise `solution.py` to fix the error. Keep using `INPUT_PATH` and `OUTPUT_PATH`." + ) + + feedback_path = os.path.join(work_dir, f"feedback_turn_{turn + 1:02d}.md") + with open(feedback_path, "w", encoding="utf-8") as f: + f.write(feedback) + conversation.append({"role": "user", "content": feedback}) + prompt = ( + f"The previous `solution.py` was evaluated and needs another revision.\n" + f"Read `{os.path.basename(feedback_path)}` and update `solution.py` accordingly.\n" + "You may run `python run_solution.py` for a local syntax/runtime check, but the harness will run the final code separately.\n" + "Do not hardcode workbook-specific answers; preserve unrelated cells." + ) + + return { + "code": code, + "raw": raw or final_message, + "n_turns": len([m for m in conversation if m["role"] == "assistant"]), + "conversation": conversation, + "target_system_prompt": skill_md, + "target_user_prompt": f"{initial_prompt}\n\n## Task File\n\n{task_md}", + } + + deadline = None if no_task_timeout else time.time() + task_timeout + client = get_target_client() + deployment = _get_deployment() + system = _build_system(skill_content) + user = _build_user( + instruction, + input_xlsx, + instruction_type, + answer_position, + diagnostic_mode=diagnostic_mode, + diagnostic_instruction=diagnostic_instruction, + diagnostic_trace_context=diagnostic_trace_context, + ) + + messages: list[dict] = [ + {"role": "system", "content": system}, + {"role": "user", "content": user}, + ] + conversation: list[dict] = [] + code = "" + raw = "" + + for turn in range(max_turns): + if deadline is None: + effective_timeout = None + else: + remaining = deadline - time.time() + if remaining <= 10: + # Not enough time for another round + break + effective_timeout = min(llm_timeout or int(remaining), int(remaining)) + raw = _chat_call(client, deployment, messages, max_output_tokens, llm_timeout=effective_timeout) + time.sleep(3) # Rate-limit cooldown after successful LLM call + code = extract_code(raw) + conversation.append({"role": "assistant", "content": raw}) + messages.append({"role": "assistant", "content": raw}) + + if not code.strip(): + # No code extracted — ask again + feedback = ( + "No Python code block was found in your response. " + "Please return a complete Python script inside a ```python``` block." + ) + messages.append({"role": "user", "content": feedback}) + conversation.append({"role": "user", "content": feedback}) + continue + + # Execute the code + ok, err = run_generated_code( + code, + input_xlsx, + output_path, + timeout=None if no_task_timeout else 120, + ) + if ok: + # Execution succeeded — check correctness if gold_path available + if gold_path and answer_position: + from skillopt.envs.spreadsheetbench.rollout import _auto_verify_output + eval_result = evaluate( + output_path, gold_path, instruction_type, answer_position, + ) + if eval_result["ok"]: + break # Genuinely correct — stop + + # Output is wrong — build feedback without leaking golden values + verify = _auto_verify_output(output_path, gold_path, answer_position) + feedback = _build_eval_feedback(verify) + messages.append({"role": "user", "content": feedback}) + conversation.append({"role": "user", "content": feedback}) + continue + else: + # No gold path (eval/test) — accept execution success + break + + # Execution failed — feed error back + feedback = ( + f"The code raised an error during execution:\n\n" + f"```\n{err[:3000]}\n```\n\n" + f"Please fix the code and return a complete corrected Python script " + f"inside a ```python``` block." + ) + messages.append({"role": "user", "content": feedback}) + conversation.append({"role": "user", "content": feedback}) + + return { + "code": code, + "raw": raw, + "n_turns": turn + 1, + "conversation": conversation, + "target_system_prompt": system, + "target_user_prompt": user, + } diff --git a/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/spreadsheetbench/dataloader.py b/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/spreadsheetbench/dataloader.py new file mode 100644 index 00000000..542ecc54 --- /dev/null +++ b/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/spreadsheetbench/dataloader.py @@ -0,0 +1,37 @@ +"""SpreadsheetBench task dataloader.""" +from __future__ import annotations + +from skillopt.datasets.base import SplitDataLoader + + +class SpreadsheetBenchDataLoader(SplitDataLoader): + """SpreadsheetBench dataloader. + + Each split directory contains a .json file (JSON array of task items). + Spreadsheet files referenced by items live under a separate ``data_root``. + """ + + def __init__( + self, + split_dir: str = "", + data_path: str = "", + split_mode: str = "ratio", + split_ratio: str = "2:1:7", + split_seed: int = 42, + split_output_dir: str = "", + data_root: str = "", + seed: int = 42, + limit: int = 0, + **kwargs, + ) -> None: + super().__init__( + split_dir=split_dir, + data_path=data_path, + split_mode=split_mode, + split_ratio=split_ratio, + split_seed=split_seed, + split_output_dir=split_output_dir, + seed=seed, + limit=limit, + ) + self.data_root = data_root diff --git a/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/spreadsheetbench/evaluator.py b/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/spreadsheetbench/evaluator.py new file mode 100644 index 00000000..3d8b84a6 --- /dev/null +++ b/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/spreadsheetbench/evaluator.py @@ -0,0 +1,158 @@ +"""Cell-value evaluator faithful to the official SpreadsheetBench +`evaluation/evaluation.py` (https://github.com/RUCKBReasoning/SpreadsheetBench). + +Key rules (copied from the official `transform_value` / `compare_cell_value`): + * numeric values (int/float and numeric strings) are compared after + ``round(float(v), 2)`` — a fixed 2-decimal quantization (NOT a tolerance); + * ``datetime.time`` is stringified and the trailing microseconds stripped; + * ``datetime.datetime`` is converted to an Excel serial day and rounded + to an integer day; + * an empty string ``""`` and ``None`` are considered equal, but otherwise + ``type(v1) != type(v2)`` fails the comparison. + +Format/style comparison is deliberately NOT performed — the official +reference evaluator also skips it (the relevant lines are commented out +in `cell_level_compare`). soft vs hard is defined at the run_bench level +across a task's multiple test cases, not here. +""" +from __future__ import annotations + +import datetime +import os +import re + +import openpyxl + + +# ---------- value transform / compare (official port) ---------- + +def _datetime_to_float(dt: datetime.datetime) -> float: + excel_start_date = datetime.datetime(1899, 12, 30) + delta = dt - excel_start_date + return delta.days + delta.seconds / 86400.0 + + +def _transform_value(v): + if isinstance(v, bool): + # openpyxl can return Python bool; official code doesn't special-case + # bools, but round(float(True), 2) == 1.0 which breaks 1 vs True. Keep + # parity with the official transform by promoting bool -> float. + return round(float(v), 2) + if isinstance(v, (int, float)): + return round(float(v), 2) + if isinstance(v, datetime.time): + return str(v)[:-3] + if isinstance(v, datetime.datetime): + return round(_datetime_to_float(v), 0) + if isinstance(v, str): + try: + return round(float(v), 2) + except ValueError: + return v + return v + + +def _compare_cell_value(v1, v2) -> bool: + v1 = _transform_value(v1) + v2 = _transform_value(v2) + if (v1 == "" and v2 is None) or (v1 is None and v2 == ""): + return True + if (v1 == "" and v2 == "") or (v1 is None and v2 is None): + return True + if type(v1) is not type(v2): + return False + return v1 == v2 + + +# ---------- range parsing (official port) ---------- + +def _col_num2name(n: int) -> str: + name = "" + while n > 0: + n, r = divmod(n - 1, 26) + name = chr(65 + r) + name + return name + + +def _col_name2num(name: str) -> int: + num = 0 + for c in name: + num = num * 26 + (ord(c) - ord("A") + 1) + return num + + +def _parse_range(range_str: str): + start_cell, end_cell = range_str.split(":") + sc = "".join(ch for ch in start_cell if ch.isalpha()) + sr = "".join(ch for ch in start_cell if ch.isdigit()) + ec = "".join(ch for ch in end_cell if ch.isalpha()) + er = "".join(ch for ch in end_cell if ch.isdigit()) + return (_col_name2num(sc), int(sr)), (_col_name2num(ec), int(er)) + + +def _generate_cell_names(range_str: str): + if ":" not in range_str: + return [range_str] + (sc, sr), (ec, er) = _parse_range(range_str) + cols = [_col_num2name(i) for i in range(sc, ec + 1)] + return [f"{c}{r}" for c in cols for r in range(sr, er + 1)] + + +def _cell_level_compare(wb_gt, wb_proc, sheet_name: str, cell_range: str): + if sheet_name not in wb_proc.sheetnames: + return False, f"worksheet not found: {sheet_name}" + ws_gt = wb_gt[sheet_name] + ws_proc = wb_proc[sheet_name] + for cn in _generate_cell_names(cell_range): + cg = ws_gt[cn] + cp = ws_proc[cn] + if not _compare_cell_value(cg.value, cp.value): + return False, f"value@{sheet_name}!{cn}: gt={cg.value!r} pred={cp.value!r}" + return True, "" + + +# ---------- public API ---------- + +def compare_workbooks(gt_file: str, proc_file: str, answer_position: str) -> tuple[bool, str]: + """Return (ok, msg). Single test-case comparison, official semantics.""" + if not os.path.exists(proc_file): + return False, "file not exist" + try: + wb_gt = openpyxl.load_workbook(filename=gt_file, data_only=True) + wb_proc = openpyxl.load_workbook(filename=proc_file, data_only=True) + except Exception as e: # noqa: BLE001 + return False, f"load error: {e}" + try: + ok_all = True + msg_first = "" + for scr in (answer_position or "").split(","): + scr = scr.strip() + if not scr: + continue + if "!" in scr: + sheet_name, cell_range = scr.split("!", 1) + sheet_name = sheet_name.strip().strip("'\"") + else: + sheet_name = wb_gt.sheetnames[0] + cell_range = scr + cell_range = cell_range.strip().strip("'\"") + ok, msg = _cell_level_compare(wb_gt, wb_proc, sheet_name, cell_range) + if not ok: + ok_all = False + if not msg_first: + msg_first = msg + return ok_all, msg_first + finally: + wb_gt.close() + wb_proc.close() + + +def evaluate(pred_path: str, gold_path: str, + instruction_type: str, answer_position: str) -> dict: + """Single test-case evaluate. soft/hard aggregation happens in run_bench.""" + ok, msg = compare_workbooks(gold_path, pred_path, answer_position) + return { + "ok": ok, + "reason": msg, + "instruction_type": instruction_type, + } diff --git a/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/spreadsheetbench/executor.py b/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/spreadsheetbench/executor.py new file mode 100644 index 00000000..24421f95 --- /dev/null +++ b/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/spreadsheetbench/executor.py @@ -0,0 +1,67 @@ +"""Execute LLM-generated Python code against an input xlsx to produce an output xlsx.""" +from __future__ import annotations + +import os +import re +import subprocess +import sys +import tempfile +import textwrap + + +RUNNER_TEMPLATE = textwrap.dedent( + """ + import os, sys, traceback + INPUT_PATH = {input_path!r} + OUTPUT_PATH = {output_path!r} + try: + {user_code_indented} + except Exception: + traceback.print_exc() + sys.exit(2) + """ +) + +# Regex to strip user-defined INPUT_PATH / OUTPUT_PATH assignments, +# since the runner template injects the correct values. +_PATH_ASSIGN_RE = re.compile( + r'^\s*(INPUT_PATH|OUTPUT_PATH)\s*=\s*.+$', re.MULTILINE +) + + +def _strip_path_assignments(code: str) -> str: + """Remove INPUT_PATH/OUTPUT_PATH assignments from user code.""" + return _PATH_ASSIGN_RE.sub("", code) + + +def run_generated_code(code: str, input_path: str, output_path: str, timeout: int | None = 120) -> tuple[bool, str]: + os.makedirs(os.path.dirname(output_path), exist_ok=True) + cleaned = _strip_path_assignments(code) + indented = textwrap.indent(cleaned, " ") + script = RUNNER_TEMPLATE.format( + input_path=input_path, + output_path=output_path, + user_code_indented=indented, + ) + with tempfile.NamedTemporaryFile("w", suffix=".py", delete=False) as f: + f.write(script) + tmp = f.name + try: + proc = subprocess.run( + [sys.executable, tmp], + capture_output=True, + text=True, + timeout=timeout if timeout and timeout > 0 else None, + ) + if proc.returncode != 0: + return False, (proc.stdout + "\n" + proc.stderr).strip() + if not os.path.exists(output_path): + return False, "output file was not created" + return True, "" + except subprocess.TimeoutExpired: + return False, f"timeout after {timeout}s" + finally: + try: + os.unlink(tmp) + except OSError: + pass diff --git a/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/spreadsheetbench/prompts/analyst_error.md b/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/spreadsheetbench/prompts/analyst_error.md new file mode 100644 index 00000000..dc7f352a --- /dev/null +++ b/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/spreadsheetbench/prompts/analyst_error.md @@ -0,0 +1,46 @@ +You are an expert failure-analysis agent for spreadsheet manipulation tasks. + +You will be given MULTIPLE failed agent trajectories from a single minibatch +and the current skill document. +Your job is to identify the most important COMMON failure patterns across +the batch and propose a concise set of skill edits. + +## Failure Type Categories +- **rule_missing**: the skill lacks a relevant rule for this type of task +- **rule_wrong**: an existing skill rule is misleading or incorrect +- **rule_ignored**: the skill has the right rule but the agent did not follow it +- **data_exploration**: the agent did not read enough data from the spreadsheet +- **code_error**: the agent's code has a bug unrelated to the skill +- **other**: none of the above + +## Analysis Process +1. Read ALL failed trajectories in the minibatch. +2. Identify the most prevalent, systematic failure patterns across them. +3. For each pattern, classify its failure type. +4. Propose skill edits that address the COMMON patterns — not individual edge cases. +5. Edits must be generalizable; do not hardcode task-specific values + (file paths, cell addresses, expected values). +6. Only patch gaps in the skill — do not duplicate existing content. +7. If the failure is because the agent did not read enough spreadsheet rows/columns + to understand the data, propose a patch encouraging broader data exploration. + +You will be told the maximum number of edits (the budget L). Produce AT MOST L edits, +focusing on the highest-impact patterns. You may produce fewer if warranted. + +Respond ONLY with a valid JSON object (no markdown fences, no extra text): +{ + "batch_size": , + "failure_summary": [ + {"failure_type": "", "count": , "description": ""} + ], + "patch": { + "reasoning": "", + "edits": [ + {"op": "append", "content": ""}, + {"op": "insert_after", "target": "", "content": ""}, + {"op": "replace", "target": "", "content": ""}, + {"op": "delete", "target": ""} + ] + } +} +Only include edits that are needed. "edits" can be an empty list if no patch is warranted. diff --git a/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/spreadsheetbench/prompts/analyst_success.md b/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/spreadsheetbench/prompts/analyst_success.md new file mode 100644 index 00000000..7e97330b --- /dev/null +++ b/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/spreadsheetbench/prompts/analyst_success.md @@ -0,0 +1,32 @@ +You are an expert success-pattern analyst for AI spreadsheet agents. + +You will be given MULTIPLE successful agent trajectories from a single minibatch +and the current skill document. Your job is to identify generalizable behavior +patterns that are COMMON across the batch and worth encoding in the skill. + +## Rules +- Only propose patches for patterns NOT already covered in the skill. +- Focus on patterns that appear across MULTIPLE trajectories in the batch. +- Be concise. Patterns must generalize beyond specific tasks. +- Prefer reinforcing existing sections over adding new top-level sections. +- If the agents' success involved reading enough data rows or using a smart + exploration strategy, consider reinforcing that in the patch. + +You will be told the maximum number of edits (the budget L). Produce AT MOST L edits, +focusing on the most broadly applicable patterns. You may produce fewer if warranted. + +Respond ONLY with a valid JSON object: +{ + "batch_size": , + "success_patterns": ["", ""], + "patch": { + "reasoning": "", + "edits": [ + {"op": "append", "content": ""}, + {"op": "insert_after", "target": "", "content": ""}, + {"op": "replace", "target": "", "content": ""}, + {"op": "delete", "target": ""} + ] + } +} +"edits" may be empty if the skill already covers all observed patterns. diff --git a/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/spreadsheetbench/prompts/codegen_system.md b/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/spreadsheetbench/prompts/codegen_system.md new file mode 100644 index 00000000..1ec71342 --- /dev/null +++ b/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/spreadsheetbench/prompts/codegen_system.md @@ -0,0 +1 @@ +You are an expert Python programmer specializing in spreadsheet manipulation. You will be given a user instruction together with a preview of an input .xlsx file. Your job is to write a single self-contained Python script that reads the input file at the path stored in the variable INPUT_PATH, performs the requested manipulation, and saves the result to OUTPUT_PATH. Use only the standard library, openpyxl, and pandas. Do not print anything. Do not use input(). Do not hardcode file paths. Return ONLY the Python code inside a single ```python ... ``` fenced block. \ No newline at end of file diff --git a/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/spreadsheetbench/prompts/critical_rules.md b/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/spreadsheetbench/prompts/critical_rules.md new file mode 100644 index 00000000..822da179 --- /dev/null +++ b/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/spreadsheetbench/prompts/critical_rules.md @@ -0,0 +1,9 @@ +## Critical Rules (MUST follow) +1. NEVER write Excel formulas to cells that will be graded on their displayed value. + openpyxl does NOT compute formulas -- the evaluator will see None. + Instead, compute results in Python and write literal values (numbers/strings). +2. After saving the workbook, ALWAYS reopen and verify the written values: + `wb2 = openpyxl.load_workbook(OUTPUT_PATH); print(wb2[sheet][cell].value)` +3. Use the `write_file` tool to create solution.py -- it avoids shell escaping issues. + Do NOT use `echo "..." > solution.py` for multi-line scripts. + diff --git a/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/spreadsheetbench/prompts/react_system.md b/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/spreadsheetbench/prompts/react_system.md new file mode 100644 index 00000000..afbbffb5 --- /dev/null +++ b/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/spreadsheetbench/prompts/react_system.md @@ -0,0 +1,21 @@ +You are an expert spreadsheet manipulation agent. + +{critical_rules}{skill_section}## Tools +You have two tools: +- `bash` -- execute any shell command and receive its output. +- `write_file` -- write content to a file (path, content). Use this for solution.py. + +## Protocol +1. Explore the input spreadsheet to understand its structure (sheets, headers, row count). +2. Use the `write_file` tool to create `solution.py` in the current directory. + solution.py MUST start with: + INPUT_PATH = "" + OUTPUT_PATH = "" + Then perform the manipulation and save the result to OUTPUT_PATH. + Use only: standard library, openpyxl, pandas. +3. Run `python solution.py` via `bash` and verify the output was created. +4. Fix any errors and re-run until the output is correct. +5. Once OUTPUT_PATH exists and is correct, stop calling tools. + +Do NOT use any libraries other than standard library, openpyxl, and pandas. +Do NOT hardcode cell values from the preview -- iterate over actual rows. diff --git a/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/spreadsheetbench/react_agent.py b/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/spreadsheetbench/react_agent.py new file mode 100644 index 00000000..2e729534 --- /dev/null +++ b/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/spreadsheetbench/react_agent.py @@ -0,0 +1,395 @@ +"""ReAct agent with bash tool for SpreadsheetBench evaluation. + +Adapted from the original SpreadsheetBench react agent implementation. + +Uses the unified ``skillopt.model`` router so SpreadsheetBench follows the same +backend selection as the rest of the framework. +""" +from __future__ import annotations + +import json +import os +import subprocess + +from skillopt.model import chat_target_messages +from skillopt.prompts import load_prompt + +# ── Tool schemas ───────────────────────────────────────────────────────────── + +BASH_TOOL_CHAT = { + "type": "function", + "function": { + "name": "bash", + "description": ( + "Execute a bash command and receive stdout+stderr (truncated to 4000 chars). " + "Use Python to read / write Excel files." + ), + "parameters": { + "type": "object", + "properties": { + "cmd": {"type": "string", "description": "Bash command to execute."} + }, + "required": ["cmd"], + }, + }, +} + +BASH_TOOL_RESPONSES = { + "type": "function", + "name": "bash", + "description": ( + "Execute a bash command and receive stdout+stderr (truncated to 4000 chars). " + "Use Python to read / write Excel files." + ), + "parameters": { + "type": "object", + "properties": { + "cmd": {"type": "string", "description": "Bash command to execute."} + }, + "required": ["cmd"], + }, +} + +WRITE_FILE_TOOL_CHAT = { + "type": "function", + "function": { + "name": "write_file", + "description": ( + "Write content to a file. Use this instead of echo/cat for multi-line " + "Python scripts to avoid shell escaping issues." + ), + "parameters": { + "type": "object", + "properties": { + "path": { + "type": "string", + "description": "File path to write (relative to working directory).", + }, + "content": { + "type": "string", + "description": "File content to write.", + }, + }, + "required": ["path", "content"], + }, + }, +} + +WRITE_FILE_TOOL_RESPONSES = { + "type": "function", + "name": "write_file", + "description": ( + "Write content to a file. Use this instead of echo/cat for multi-line " + "Python scripts to avoid shell escaping issues." + ), + "parameters": { + "type": "object", + "properties": { + "path": { + "type": "string", + "description": "File path to write (relative to working directory).", + }, + "content": { + "type": "string", + "description": "File content to write.", + }, + }, + "required": ["path", "content"], + }, +} + +# ── System prompt ───────────────────────────────────────────────────────────── + + +def _build_system(skill_content: str) -> str: + if skill_content.strip(): + skill_section = f"## Skill\n{skill_content.strip()}\n\n" + else: + skill_section = "" + return load_prompt("react_system", env="spreadsheetbench").format( + critical_rules=load_prompt("critical_rules", env="spreadsheetbench"), + skill_section=skill_section, + ) + + +def _build_user( + instruction: str, + input_path: str, + output_path: str, + instruction_type: str, + answer_position: str, + diagnostic_mode: bool = False, + diagnostic_instruction: str = "", + diagnostic_trace_context: str = "", +) -> str: + parts = [] + if diagnostic_trace_context.strip(): + parts.append( + "# Previous Codex Trace Snapshot\n" + "This is a partial transcript from an earlier attempt. Use it as your current reasoning context.\n\n" + f"{diagnostic_trace_context.strip()}" + ) + parts.extend([ + f"# Instruction\n{instruction}", + f"# Input file\n{input_path}", + f"# Output file\n{output_path}", + ]) + if instruction_type: + parts.append(f"# Instruction type\n{instruction_type}") + if answer_position: + parts.append(f"# Answer position\n{answer_position}") + if diagnostic_mode and diagnostic_instruction.strip(): + parts.append(f"# Training readout\n{diagnostic_instruction.strip()}") + parts.append( + "Manipulate the input spreadsheet according to the instruction " + "and save the result to the output file." + ) + return "\n\n".join(parts) + + +# ── File write (bypass shell escaping) ──────────────────────────────────────── + +def _write_file(path: str, content: str, work_dir: str) -> str: + """Write content to a file, bypassing shell escaping issues.""" + try: + full_path = os.path.join(work_dir, path) if not os.path.isabs(path) else path + parent = os.path.dirname(full_path) + if parent: + os.makedirs(parent, exist_ok=True) + with open(full_path, "w") as f: + f.write(content) + return f"File written: {full_path} ({len(content)} chars)" + except Exception as e: # noqa: BLE001 + return f"[write_file error: {e}]" + + +# ── Auto-verification ───────────────────────────────────────────────────────── + +def _auto_verify(work_dir: str) -> str: + """Auto-verify output xlsx after solution.py runs.""" + import glob as _glob + + sol_path = os.path.join(work_dir, "solution.py") + output_path = None + if os.path.exists(sol_path): + with open(sol_path) as f: + for line in f: + stripped = line.strip() + if stripped.startswith("OUTPUT_PATH"): + try: + val = stripped.split("=", 1)[1].strip() + output_path = val.strip("'\"").strip() + except Exception: # noqa: BLE001 + pass + break + + if not output_path or not os.path.exists(output_path): + xlsx_files = [ + f for f in _glob.glob(os.path.join(work_dir, "*.xlsx")) + if "_pred" in os.path.basename(f) + ] + if xlsx_files: + output_path = xlsx_files[0] + + if not output_path or not os.path.exists(output_path): + return ( + "\n\n[AUTO-VERIFY] WARNING: Output file not found! " + "Make sure OUTPUT_PATH is correct and wb.save(OUTPUT_PATH) is called." + ) + + try: + import openpyxl + + wb_formula = openpyxl.load_workbook(output_path, data_only=False) + wb_value = openpyxl.load_workbook(output_path, data_only=True) + lines = [f"\n\n[AUTO-VERIFY] Output file exists: {output_path}"] + + sn = wb_formula.sheetnames[0] + ws_f = wb_formula[sn] + ws_v = wb_value[sn] + lines.append(f" Sheet '{sn}': {ws_f.dimensions}") + + for row in ws_v.iter_rows( + min_row=1, max_row=min(5, ws_v.max_row), values_only=True, + ): + lines.append(f" {list(row)}") + + none_cells: list[str] = [] + for row_f, row_v in zip( + ws_f.iter_rows(min_row=1, max_row=min(30, ws_f.max_row)), + ws_v.iter_rows(min_row=1, max_row=min(30, ws_v.max_row)), + ): + for cf, cv in zip(row_f, row_v): + formula_val = cf.value + cached_val = cv.value + if ( + isinstance(formula_val, str) + and formula_val.startswith("=") + and cached_val is None + ): + none_cells.append(cf.coordinate) + + if none_cells: + lines.append( + f" WARNING: {len(none_cells)} cells have formulas but NO cached " + f"value -- evaluator will see None: {none_cells[:10]}" + ) + lines.append( + " FIX: Compute values in Python and write literal " + "numbers/strings instead of formulas." + ) + else: + lines.append(" All cells have concrete values. Looks good.") + + wb_formula.close() + wb_value.close() + return "\n".join(lines) + except Exception as e: # noqa: BLE001 + return f"\n\n[AUTO-VERIFY] Could not inspect output: {e}" + + +# ── Bash execution ──────────────────────────────────────────────────────────── + +def _run_bash(cmd: str, work_dir: str, timeout: int = 60) -> str: + try: + proc = subprocess.run( + cmd, + shell=True, + capture_output=True, + text=True, + timeout=timeout, + cwd=work_dir, + ) + out = (proc.stdout + proc.stderr).strip() + except subprocess.TimeoutExpired: + return f"[timeout after {timeout}s]" + except Exception as e: # noqa: BLE001 + return f"[error: {e}]" + if len(out) > 4000: + out = out[:3800] + f"\n...[truncated, {len(out)} total chars]" + result = out or "(no output)" + + if "solution.py" in cmd and "python" in cmd.lower(): + result += _auto_verify(work_dir) + + return result + + +def _assistant_tool_calls(message) -> list[dict]: + tool_calls = getattr(message, "tool_calls", None) or [] + return [ + tool_call.model_dump() if hasattr(tool_call, "model_dump") else dict(tool_call) + for tool_call in tool_calls + ] + + +def _react_loop( + system: str, + user: str, + work_dir: str, + max_turns: int, + max_output_tokens: int, +) -> dict: + messages: list[dict] = [ + {"role": "system", "content": system}, + {"role": "user", "content": user}, + ] + conversation: list[dict] = [] + n_turns = 0 + + for _ in range(max_turns): + message, _ = chat_target_messages( + messages=messages, + tools=[BASH_TOOL_CHAT, WRITE_FILE_TOOL_CHAT], + tool_choice="auto", + max_completion_tokens=max_output_tokens, + retries=5, + stage="rollout", + return_message=True, + ) + + assistant_text = str(getattr(message, "content", "") or "") + tool_calls = _assistant_tool_calls(message) + assistant_payload: dict = {"role": "assistant", "content": assistant_text} + if tool_calls: + assistant_payload["tool_calls"] = tool_calls + messages.append(assistant_payload) + + if not tool_calls: + conversation.append({"type": "message", "content": assistant_text}) + break + + for tool_call in tool_calls: + n_turns += 1 + function = tool_call.get("function", {}) or {} + try: + args = json.loads(str(function.get("arguments", "{}") or "{}")) + except json.JSONDecodeError: + args = {} + + if function.get("name") == "write_file": + obs = _write_file( + args.get("path", ""), + args.get("content", ""), + work_dir, + ) + conversation.append({ + "type": "tool_call", + "cmd": f"[write_file] {args.get('path', '')}", + "obs": obs, + }) + else: + cmd = args.get("cmd", "") + obs = _run_bash(cmd, work_dir) + conversation.append({"type": "tool_call", "cmd": cmd, "obs": obs}) + + messages.append( + { + "role": "tool", + "tool_call_id": tool_call.get("id", ""), + "content": obs, + } + ) + + return {"conversation": conversation, "n_turns": n_turns} + + +# ── Public API ──────────────────────────────────────────────────────────────── + +def run_react( + instruction: str, + input_path: str, + output_path: str, + work_dir: str, + instruction_type: str = "", + answer_position: str = "", + skill_content: str = "", + max_turns: int = 30, + max_output_tokens: int = 16384, + diagnostic_mode: bool = False, + diagnostic_instruction: str = "", + diagnostic_trace_context: str = "", +) -> dict: + """Run the ReAct agent for one task. + + Returns: + { + "conversation": [...], # list of {type, cmd/content, obs?} + "n_turns": int, # number of bash tool calls made + } + """ + system = _build_system(skill_content) + user = _build_user( + instruction, + input_path, + output_path, + instruction_type, + answer_position, + diagnostic_mode=diagnostic_mode, + diagnostic_instruction=diagnostic_instruction, + diagnostic_trace_context=diagnostic_trace_context, + ) + result = _react_loop(system, user, work_dir, max_turns, max_output_tokens) + result["target_system_prompt"] = system + result["target_user_prompt"] = user + return result diff --git a/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/spreadsheetbench/reflect.py b/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/spreadsheetbench/reflect.py new file mode 100644 index 00000000..cdfeaf6e --- /dev/null +++ b/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/spreadsheetbench/reflect.py @@ -0,0 +1,4 @@ +"""SpreadsheetBench Reflect stage. + +Prompts are now loaded from .md files by the base adapter. +""" diff --git a/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/spreadsheetbench/rollout.py b/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/spreadsheetbench/rollout.py new file mode 100644 index 00000000..0e918c7b --- /dev/null +++ b/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/spreadsheetbench/rollout.py @@ -0,0 +1,934 @@ +"""SpreadsheetBench rollout — codegen & ReAct batch execution. + +Provides: + - process_one_codegen(): single/multi-round code generation (no tool-call) + - run_spreadsheet_batch_codegen(): batch wrapper for codegen + - process_one(): ReAct agent with tool-call (legacy) + - run_spreadsheet_batch(): batch wrapper for ReAct (legacy) + - load_items(): load benchmark .json/.jsonl files +""" +from __future__ import annotations + +import glob as _glob +import json +import os +import shutil +import tempfile +import time +import traceback +from concurrent.futures import ( + FIRST_COMPLETED, + ThreadPoolExecutor, + wait, + TimeoutError as FuturesTimeoutError, +) + +import openpyxl + +from skillopt.envs.spreadsheetbench.react_agent import run_react +from skillopt.envs.spreadsheetbench.evaluator import evaluate, _generate_cell_names +from skillopt.envs.spreadsheetbench.executor import run_generated_code + + +# ── Data loading ───────────────────────────────────────────────────────────── + + +def load_items(path: str) -> list[dict]: + """Load a benchmark file. Supports both .jsonl and .json (list of dicts).""" + if path.endswith(".json"): + with open(path) as f: + data = json.load(f) + if isinstance(data, dict): + data = data.get("data") or list(data.values()) + return list(data) + items = [] + with open(path) as f: + for line in f: + line = line.strip() + if line: + items.append(json.loads(line)) + return items + + +# ── Test case discovery ────────────────────────────────────────────────────── + + +def _find_test_cases(task_dir: str) -> list[tuple[str, str, str]]: + """Return [(case_no, input_path, answer_path), ...] sorted by case_no. + + Supports naming conventions used by SpreadsheetBench releases: + * ``{no}_{id}_input.xlsx`` + ``{no}_{id}_answer.xlsx`` (original) + * ``{no}_{id}_init.xlsx`` + ``{no}_{id}_golden.xlsx`` (verified_400) + * ``initial.xlsx`` + ``golden.xlsx`` (verified_400, no prefix) + """ + cases: list[tuple[str, str, str]] = [] + inputs = sorted(_glob.glob(os.path.join(task_dir, "*_input.xlsx"))) + for ip in inputs: + no = os.path.basename(ip).split("_", 1)[0] + ap = ip.replace("_input.xlsx", "_answer.xlsx") + if os.path.exists(ap): + cases.append((no, ip, ap)) + inits = sorted(_glob.glob(os.path.join(task_dir, "*_init.xlsx"))) + for ip in inits: + no = os.path.basename(ip).split("_", 1)[0] + ap = ip.replace("_init.xlsx", "_golden.xlsx") + if os.path.exists(ap): + cases.append((no, ip, ap)) + + # Fallback: bare initial.xlsx + golden.xlsx (no numbered prefix) + if not cases: + bare_init = os.path.join(task_dir, "initial.xlsx") + bare_gold = os.path.join(task_dir, "golden.xlsx") + if os.path.exists(bare_init) and os.path.exists(bare_gold): + cases.append(("1", bare_init, bare_gold)) + + return cases + + +# ── Auto-verify helper ────────────────────────────────────────────────────── + + +def _auto_verify_output( + pred_path: str, + gold_path: str, + answer_position: str, +) -> str: + """Reopen the predicted xlsx and compare cells at answer_position with gold. + + Returns a human-readable verification report that can be appended to the + trajectory so the error analyst can see exactly what went wrong (e.g. + ``cell A1: got=None, expected=420``). + """ + if not os.path.exists(pred_path): + return "Verification: output file does not exist." + try: + wb_pred = openpyxl.load_workbook(pred_path, data_only=True) + wb_gold = openpyxl.load_workbook(gold_path, data_only=True) + except Exception as e: + return f"Verification: could not open workbooks: {e}" + + lines = ["## Output Verification"] + try: + for scr in (answer_position or "").split(","): + scr = scr.strip() + if not scr: + continue + if "!" in scr: + sheet_name, cell_range = scr.split("!", 1) + sheet_name = sheet_name.strip().strip("'\"") + else: + sheet_name = wb_gold.sheetnames[0] + cell_range = scr + cell_range = cell_range.strip().strip("'\"") + + cell_names = _generate_cell_names(cell_range) + ws_pred = wb_pred[sheet_name] if sheet_name in wb_pred.sheetnames else None + ws_gold = wb_gold[sheet_name] if sheet_name in wb_gold.sheetnames else None + + if ws_pred is None: + lines.append(f" Sheet '{sheet_name}' NOT FOUND in output.") + continue + + for cn in cell_names: + gv = ws_gold[cn].value if ws_gold else "N/A" + pv = ws_pred[cn].value + match = "✓" if repr(gv) == repr(pv) else "✗" + lines.append(f" {sheet_name}!{cn}: got={pv!r}, expected={gv!r} {match}") + + # Also check if any cells in the output contain formula strings + formula_cells = [] + for sn in wb_pred.sheetnames: + ws = wb_pred[sn] + for row in ws.iter_rows(max_row=min(ws.max_row, 200), values_only=False): + for cell in row: + if isinstance(cell.value, str) and cell.value.startswith("="): + formula_cells.append(f"{sn}!{cell.coordinate}={cell.value}") + if len(formula_cells) >= 10: + break + if len(formula_cells) >= 10: + break + if len(formula_cells) >= 10: + break + if formula_cells: + lines.append(f"\n WARNING: {len(formula_cells)} cells contain Excel formulas (openpyxl cannot evaluate them):") + for fc in formula_cells[:5]: + lines.append(f" {fc}") + if len(formula_cells) > 5: + lines.append(f" ... and {len(formula_cells) - 5} more") + finally: + wb_pred.close() + wb_gold.close() + + return "\n".join(lines) + + +# ── Per-task worker ────────────────────────────────────────────────────────── + + +def process_one( + item: dict, + data_root: str, + out_root: str, + skill_content: str, + max_turns: int, + diagnostic_mode: bool = False, + diagnostic_instruction: str = "", + diagnostic_trace_context: str = "", + max_completion_tokens: int = 16384, +) -> dict: + """Run the ReAct agent on a single SpreadsheetBench task. + + Returns a result dict compatible with ``compute_score()``. + """ + task_id = str(item["id"]) + instruction = item["instruction"] + instruction_type = item.get("instruction_type", "") + answer_position = item.get("answer_position", "") + answer_sheet = item.get("answer_sheet", "") + if answer_position and answer_sheet and "!" not in answer_position: + answer_position_eval = f"{answer_sheet}!{answer_position}" + else: + answer_position_eval = answer_position + + # Determine task_type from instruction_type + itype_lower = (instruction_type or "").lower() + if "cell" in itype_lower: + task_type = "cell_level" + elif "sheet" in itype_lower: + task_type = "sheet_level" + else: + task_type = "other" + + sp = item.get("spreadsheet_path", f"spreadsheet/{task_id}") + task_dir = sp if os.path.isabs(sp) else os.path.join(data_root, sp) + + result = { + "id": task_id, + "ok": False, + "instruction_type": instruction_type, + "task_type": task_type, + "task_description": instruction, + "phase": "setup", + "fail_reason": "", + "agent_ok": False, + "exec_ok": False, + "n_cases": 0, + "n_exec_pass": 0, + "n_pass": 0, + "soft": 0.0, + "hard": 0, + "n_turns": 0, + "cases": [], + "error": "", + } + + try: + cases = _find_test_cases(task_dir) + result["n_cases"] = len(cases) + if not cases: + result["fail_reason"] = "no-test-cases" + return result + + task_out_dir = os.path.join(out_root, "predictions", task_id) + os.makedirs(task_out_dir, exist_ok=True) + + no1, ip1, _ = cases[0] + pred_path_1 = os.path.join(task_out_dir, f"{no1}_pred.xlsx") + target_prompt_parts = [ + f"# Instruction\n{instruction}", + f"# Input file\n{ip1}", + f"# Output file\n{pred_path_1}", + ] + if instruction_type: + target_prompt_parts.append(f"# Instruction type\n{instruction_type}") + if answer_position_eval: + target_prompt_parts.append(f"# Answer position\n{answer_position_eval}") + if diagnostic_trace_context.strip(): + target_prompt_parts.insert( + 0, + "# Previous Codex Trace Snapshot\n" + "This is a partial transcript from an earlier attempt. Use it as your current reasoning context.\n\n" + f"{diagnostic_trace_context.strip()}", + ) + if diagnostic_mode and diagnostic_instruction.strip(): + target_prompt_parts.append(f"# Training readout\n{diagnostic_instruction.strip()}") + target_user_prompt = "\n\n".join(target_prompt_parts) + try: + from skillopt.envs.spreadsheetbench.react_agent import _build_system + target_system_prompt = _build_system(skill_content) + except Exception: + target_system_prompt = "" + if target_system_prompt: + with open(os.path.join(task_out_dir, "target_system_prompt.txt"), "w") as f: + f.write(target_system_prompt) + result["target_system_prompt"] = target_system_prompt + with open(os.path.join(task_out_dir, "target_user_prompt.txt"), "w") as f: + f.write(target_user_prompt) + result["target_user_prompt"] = target_user_prompt + + # ── Stage 1: run ReAct agent on test case 1 ───────────────────── + result["phase"] = "agent" + + work_dir = tempfile.mkdtemp(prefix=f"react_{task_id}_") + try: + # Copy input so agent works in an isolated directory + work_input = os.path.join(work_dir, os.path.basename(ip1)) + shutil.copy2(ip1, work_input) + + agent_result = run_react( + instruction=instruction, + input_path=work_input, + output_path=pred_path_1, + work_dir=work_dir, + instruction_type=instruction_type, + answer_position=answer_position_eval, + skill_content=skill_content, + max_turns=max_turns, + max_output_tokens=max_completion_tokens, + diagnostic_mode=diagnostic_mode, + diagnostic_instruction=diagnostic_instruction, + diagnostic_trace_context=diagnostic_trace_context, + ) + result["n_turns"] = agent_result.get("n_turns", 0) + if agent_result.get("target_system_prompt"): + with open(os.path.join(task_out_dir, "target_system_prompt.txt"), "w") as f: + f.write(agent_result["target_system_prompt"]) + result["target_system_prompt"] = agent_result["target_system_prompt"] + if agent_result.get("target_user_prompt"): + with open(os.path.join(task_out_dir, "target_user_prompt.txt"), "w") as f: + f.write(agent_result["target_user_prompt"]) + result["target_user_prompt"] = agent_result["target_user_prompt"] + + # Save conversation log + with open(os.path.join(task_out_dir, "conversation.json"), "w") as f: + json.dump( + agent_result.get("conversation", []), + f, ensure_ascii=False, indent=2, + ) + + # Copy solution.py if the agent wrote one + solution_src = os.path.join(work_dir, "solution.py") + solution_dst = os.path.join(task_out_dir, "solution.py") + if os.path.exists(solution_src): + shutil.copy2(solution_src, solution_dst) + + except Exception as e: + result["fail_reason"] = f"agent-error: {type(e).__name__}: {e}" + result["error"] = traceback.format_exc() + return result + finally: + shutil.rmtree(work_dir, ignore_errors=True) + + result["agent_ok"] = True + + # ── Stage 2: evaluate all test cases ───────────────────────────── + result["phase"] = "eval" + solution_path = os.path.join(task_out_dir, "solution.py") + all_exec = True + + for i, (no, ip, ap) in enumerate(cases): + pred_path = os.path.join(task_out_dir, f"{no}_pred.xlsx") + + if i > 0: + # Re-apply solution.py to subsequent test cases + if not os.path.exists(solution_path): + all_exec = False + result["cases"].append( + {"no": no, "stage": "exec", "ok": False, "error": "no-solution-py"} + ) + if not result["fail_reason"]: + result["fail_reason"] = "no-solution-py-for-other-cases" + continue + + with open(solution_path) as f: + code = f.read() + + # Prepend new INPUT_PATH / OUTPUT_PATH + preamble = ( + f"INPUT_PATH = {ip!r}\n" + f"OUTPUT_PATH = {pred_path!r}\n" + ) + full_code = preamble + code + + ok_exec, err = run_generated_code(full_code, ip, pred_path) + if not ok_exec: + all_exec = False + result["cases"].append( + {"no": no, "stage": "exec", "ok": False, "error": err[:500]} + ) + if not result["fail_reason"]: + tail = err.strip().splitlines()[-1][:200] if err.strip() else "unknown" + result["fail_reason"] = f"exec-error: {tail}" + continue + + # ── Evaluate ───────────────────────────────────────────────── + if not os.path.exists(pred_path): + all_exec = False + result["cases"].append( + {"no": no, "stage": "exec", "ok": False, "error": "output-not-found"} + ) + if not result["fail_reason"]: + result["fail_reason"] = "output-not-found" + continue + + result["n_exec_pass"] += 1 + try: + ev = evaluate(pred_path, ap, instruction_type, answer_position_eval) + except Exception as e: # noqa: BLE001 + ev = {"ok": False, "reason": f"eval-exception: {type(e).__name__}: {e}"} + + if ev["ok"]: + result["n_pass"] += 1 + else: + if not result["fail_reason"]: + result["fail_reason"] = f"eval-mismatch: {ev['reason'][:200]}" + result["cases"].append( + {"no": no, "stage": "eval", "ok": ev["ok"], "reason": ev.get("reason", "")} + ) + + result["exec_ok"] = all_exec + n_cases = result["n_cases"] + n_pass = result["n_pass"] + result["soft"] = (n_pass / n_cases) if n_cases else 0.0 + result["hard"] = 1 if (n_cases > 0 and n_pass == n_cases) else 0 + result["ok"] = bool(result["hard"]) + if result["ok"]: + result["fail_reason"] = "" + return result + + except Exception as e: # noqa: BLE001 + result["fail_reason"] = f"unexpected: {type(e).__name__}: {e}" + result["error"] = traceback.format_exc() + return result + + +# ── Batch runner ───────────────────────────────────────────────────────────── + + +def run_spreadsheet_batch( + items: list[dict], + data_root: str, + out_root: str, + skill_content: str, + max_turns: int = 30, + max_completion_tokens: int = 16384, + max_api_workers: int = 64, + task_timeout: int = 600, + diagnostic_mode: bool = False, + diagnostic_instruction: str = "", + diagnostic_trace_context_by_id: dict[str, str] | None = None, +) -> list[dict]: + """Run the ReAct agent on all items with ThreadPoolExecutor. + + Returns list of result dicts compatible with ``compute_score()``. + """ + os.makedirs(out_root, exist_ok=True) + + # Check for already-done items (resume support) + results_path = os.path.join(out_root, "results.jsonl") + done_ids: set[str] = set() + existing: list[dict] = [] + if os.path.exists(results_path): + with open(results_path) as f: + for line in f: + try: + r = json.loads(line) + done_ids.add(str(r["id"])) + existing.append(r) + except Exception: + pass + + pending = [it for it in items if str(it["id"]) not in done_ids] + print( + f" [spreadsheet rollout] total={len(items)} done={len(done_ids)} " + f"pending={len(pending)} workers={max_api_workers} task_timeout={task_timeout}s" + ) + + if not pending: + return existing + + t0 = time.time() + results = list(existing) + started_at: dict[str, float] = {} + + def _timeout_result(item: dict) -> dict: + return { + "id": str(item["id"]), + "ok": False, + "phase": "timeout", + "fail_reason": f"task-timeout-{task_timeout}s", + "n_cases": 0, "n_pass": 0, "soft": 0.0, "hard": 0, + "n_turns": 0, "cases": [], "error": "timeout", + } + + def _error_result(item: dict, exc: Exception) -> dict: + return { + "id": str(item["id"]), + "ok": False, + "phase": "error", + "fail_reason": f"unexpected: {type(exc).__name__}: {exc}", + "n_cases": 0, "n_pass": 0, "soft": 0.0, "hard": 0, + "n_turns": 0, "cases": [], "error": str(exc), + } + + def _run_one(it: dict) -> dict: + started_at[str(it["id"])] = time.time() + return process_one( + it, + data_root, + out_root, + skill_content, + max_turns, + diagnostic_mode, + diagnostic_instruction, + (diagnostic_trace_context_by_id or {}).get(str(it["id"]), ""), + max_completion_tokens, + ) + + ex = ThreadPoolExecutor(max_workers=max_api_workers) + try: + futs = {ex.submit(_run_one, it): it for it in pending} + pending_futs = set(futs) + finished = 0 + while pending_futs: + done, _ = wait(pending_futs, timeout=5, return_when=FIRST_COMPLETED) + now = time.time() + timed_out = [ + fut for fut in pending_futs - done + if str(futs[fut]["id"]) in started_at + and now - started_at[str(futs[fut]["id"])] >= task_timeout + ] + for fut in done: + pending_futs.remove(fut) + item = futs[fut] + try: + res = fut.result() + except FuturesTimeoutError: + res = _timeout_result(item) + except Exception as e: # noqa: BLE001 + res = _error_result(item, e) + results.append(res) + finished += 1 + status = "PASS" if res.get("hard") else ("TIMEOUT" if res.get("phase") == "timeout" else "FAIL") + dt = time.time() - t0 + print( + f" {finished}/{len(pending)} id={res['id']:<10} {status} " + f"turns={res.get('n_turns', 0):<3} " + f"cases={res.get('n_pass', 0)}/{res.get('n_cases', 0)} " + f"dt={dt:.0f}s" + ) + for fut in timed_out: + pending_futs.remove(fut) + res = _timeout_result(futs[fut]) + results.append(res) + finished += 1 + status = "TIMEOUT" + dt = time.time() - t0 + print( + f" {finished}/{len(pending)} id={res['id']:<10} {status} " + f"turns={res.get('n_turns', 0):<3} " + f"cases={res.get('n_pass', 0)}/{res.get('n_cases', 0)} " + f"dt={dt:.0f}s" + ) + finally: + ex.shutdown(wait=False, cancel_futures=True) + + return results + + +# ── Codegen per-task worker (no tool-call) ────────────────────────────────── + + +def process_one_codegen( + item: dict, + data_root: str, + out_root: str, + skill_content: str, + mode: str = "single", + max_turns: int = 5, + max_completion_tokens: int = 16384, + task_timeout: int = 600, + use_eval_feedback: bool = False, + diagnostic_mode: bool = False, + diagnostic_instruction: str = "", + diagnostic_trace_context: str = "", +) -> dict: + """Run codegen agent (single or multi-round) on one SpreadsheetBench task. + + This matches the official evaluation setting: LLM generates a Python code + block, no function-calling / tool-use. + """ + from skillopt.envs.spreadsheetbench.codegen_agent import run_single, run_multi + + task_id = str(item["id"]) + instruction = item["instruction"] + instruction_type = item.get("instruction_type", "") + answer_position = item.get("answer_position", "") + answer_sheet = item.get("answer_sheet", "") + if answer_position and answer_sheet and "!" not in answer_position: + answer_position_eval = f"{answer_sheet}!{answer_position}" + else: + answer_position_eval = answer_position + + itype_lower = (instruction_type or "").lower() + if "cell" in itype_lower: + task_type = "cell_level" + elif "sheet" in itype_lower: + task_type = "sheet_level" + else: + task_type = "other" + + sp = item.get("spreadsheet_path", f"spreadsheet/{task_id}") + task_dir = sp if os.path.isabs(sp) else os.path.join(data_root, sp) + + result = { + "id": task_id, + "ok": False, + "instruction_type": instruction_type, + "task_type": task_type, + "task_description": instruction, + "phase": "setup", + "fail_reason": "", + "llm_ok": False, + "code_ok": False, + "exec_ok": False, + "n_cases": 0, + "n_exec_pass": 0, + "n_pass": 0, + "soft": 0.0, + "hard": 0, + "n_turns": 0, + "cases": [], + "error": "", + } + + try: + cases = _find_test_cases(task_dir) + result["n_cases"] = len(cases) + if not cases: + result["fail_reason"] = "no-test-cases" + return result + + task_out_dir = os.path.join(out_root, "predictions", task_id) + os.makedirs(task_out_dir, exist_ok=True) + + # ── Save context for Optimizer (Reflect stage) ────────────────── + from skillopt.envs.spreadsheetbench.codegen_agent import ( + _preview_workbook, _build_system, _build_user, + ) + first_input_for_preview = cases[0][1] + try: + preview_text = _preview_workbook(first_input_for_preview) + except Exception: + preview_text = "(preview failed)" + target_system = _build_system(skill_content) + target_user = _build_user( + instruction, + first_input_for_preview, + instruction_type, + answer_position_eval, + diagnostic_mode=diagnostic_mode, + diagnostic_instruction=diagnostic_instruction, + diagnostic_trace_context=diagnostic_trace_context, + ) + + with open(os.path.join(task_out_dir, "spreadsheet_preview.txt"), "w") as f: + f.write(preview_text) + with open(os.path.join(task_out_dir, "target_system_prompt.txt"), "w") as f: + f.write(target_system) + with open(os.path.join(task_out_dir, "target_user_prompt.txt"), "w") as f: + f.write(target_user) + + result["spreadsheet_preview"] = preview_text + result["target_system_prompt"] = target_system + result["target_user_prompt"] = target_user + + # ── LLM phase ────────────────────────────────────────────────── + result["phase"] = "llm" + first_input = cases[0][1] + first_gold = cases[0][2] + first_pred = os.path.join(task_out_dir, f"{cases[0][0]}_pred.xlsx") + + try: + if mode == "multi": + agent_result = run_multi( + instruction=instruction, + input_xlsx=first_input, + output_path=first_pred, + instruction_type=instruction_type, + answer_position=answer_position_eval, + skill_content=skill_content, + max_turns=max_turns, + max_output_tokens=max_completion_tokens, + task_timeout=task_timeout, + gold_path=first_gold if use_eval_feedback else "", + diagnostic_mode=diagnostic_mode, + diagnostic_instruction=diagnostic_instruction, + diagnostic_trace_context=diagnostic_trace_context, + ) + else: + agent_result = run_single( + instruction=instruction, + input_xlsx=first_input, + output_path=first_pred, + instruction_type=instruction_type, + answer_position=answer_position_eval, + skill_content=skill_content, + max_output_tokens=max_completion_tokens, + task_timeout=task_timeout, + diagnostic_mode=diagnostic_mode, + diagnostic_instruction=diagnostic_instruction, + diagnostic_trace_context=diagnostic_trace_context, + ) + except Exception as e: # noqa: BLE001 + result["fail_reason"] = f"llm-call-failed: {type(e).__name__}: {e}" + result["error"] = traceback.format_exc() + return result + + result["llm_ok"] = True + result["n_turns"] = agent_result.get("n_turns", 1) + code = agent_result.get("code", "") + raw = agent_result.get("raw", "") + + # Save artifacts + with open(os.path.join(task_out_dir, "code.py"), "w") as f: + f.write(code) + with open(os.path.join(task_out_dir, "raw.txt"), "w") as f: + f.write(raw) + if agent_result.get("conversation"): + with open(os.path.join(task_out_dir, "conversation.json"), "w") as f: + json.dump(agent_result["conversation"], f, ensure_ascii=False, indent=2) + + if not code.strip(): + result["phase"] = "extract" + result["fail_reason"] = "empty-code-block" + return result + result["code_ok"] = True + + # ── Exec + eval per test case ────────────────────────────────── + result["phase"] = "exec" + all_exec = True + # Collect enrichment info for the conversation/trajectory + enrichment_parts: list[str] = [] + + for no, ip, ap in cases: + pred_path = os.path.join(task_out_dir, f"{no}_pred.xlsx") + + # For multi mode, the first case may already be produced + if not os.path.exists(pred_path): + ok_exec, err = run_generated_code(code, ip, pred_path) + if not ok_exec: + all_exec = False + result["cases"].append( + {"no": no, "stage": "exec", "ok": False, "error": err[:500]} + ) + if not result["fail_reason"]: + tail = err.strip().splitlines()[-1][:200] if err.strip() else "unknown" + result["fail_reason"] = f"exec-error: {tail}" + enrichment_parts.append( + f"## Execution (case {no})\nERROR: {err[:500]}" + ) + continue + + if not os.path.exists(pred_path): + all_exec = False + result["cases"].append( + {"no": no, "stage": "exec", "ok": False, "error": "output-not-found"} + ) + if not result["fail_reason"]: + result["fail_reason"] = "output-not-found" + continue + + result["n_exec_pass"] += 1 + try: + ev = evaluate(pred_path, ap, instruction_type, answer_position_eval) + except Exception as e: # noqa: BLE001 + ev = {"ok": False, "reason": f"eval-exception: {type(e).__name__}: {e}"} + + if ev["ok"]: + result["n_pass"] += 1 + else: + if not result["fail_reason"]: + result["fail_reason"] = f"eval-mismatch: {ev['reason'][:200]}" + result["cases"].append( + {"no": no, "stage": "eval", "ok": ev["ok"], "reason": ev.get("reason", "")} + ) + + # Auto-verify: reopen output and compare cells at answer_position + if answer_position_eval: + verify_report = _auto_verify_output(pred_path, ap, answer_position_eval) + enrichment_parts.append( + f"## Eval Result (case {no}): {'PASS' if ev['ok'] else 'FAIL'}\n" + f"{ev.get('reason', '')}\n\n{verify_report}" + ) + + result["exec_ok"] = all_exec + + # ── Enrich conversation with eval details ────────────────────── + if enrichment_parts: + enrichment_msg = "\n\n---\n\n".join(enrichment_parts) + conversation = agent_result.get("conversation", []) + conversation.append({ + "role": "system", + "content": f"[POST-EXECUTION VERIFICATION]\n\n{enrichment_msg}", + }) + # Re-save the enriched conversation + with open(os.path.join(task_out_dir, "conversation.json"), "w") as f: + json.dump(conversation, f, ensure_ascii=False, indent=2) + n_cases = result["n_cases"] + n_pass = result["n_pass"] + result["soft"] = (n_pass / n_cases) if n_cases else 0.0 + result["hard"] = 1 if (n_cases > 0 and n_pass == n_cases) else 0 + result["ok"] = bool(result["hard"]) + if result["ok"]: + result["fail_reason"] = "" + return result + + except Exception as e: # noqa: BLE001 + result["fail_reason"] = f"unexpected: {type(e).__name__}: {e}" + result["error"] = traceback.format_exc() + return result + + +# ── Codegen batch runner ──────────────────────────────────────────────────── + + +def run_spreadsheet_batch_codegen( + items: list[dict], + data_root: str, + out_root: str, + skill_content: str, + mode: str = "single", + max_turns: int = 5, + max_completion_tokens: int = 16384, + max_api_workers: int = 32, + task_timeout: int = 0, + use_eval_feedback: bool = False, + diagnostic_mode: bool = False, + diagnostic_instruction: str = "", + diagnostic_trace_context_by_id: dict[str, str] | None = None, +) -> list[dict]: + """Run codegen agent on all items (no tool-call). + + Args: + mode: "single" or "multi". + task_timeout: Hard per-task timeout in seconds at the future level. + 0 or negative disables the per-task timeout. + """ + no_task_timeout = task_timeout <= 0 + task_timeout_label = "none" if no_task_timeout else f"{task_timeout}s" + + os.makedirs(out_root, exist_ok=True) + + results_path = os.path.join(out_root, "results.jsonl") + done_ids: set[str] = set() + existing: list[dict] = [] + if os.path.exists(results_path): + with open(results_path) as f: + for line in f: + try: + r = json.loads(line) + done_ids.add(str(r["id"])) + existing.append(r) + except Exception: + pass + + pending = [it for it in items if str(it["id"]) not in done_ids] + print( + f" [spreadsheet codegen-{mode}] total={len(items)} done={len(done_ids)} " + f"pending={len(pending)} workers={max_api_workers} task_timeout={task_timeout_label}" + ) + + if not pending: + return existing + + t0 = time.time() + results = list(existing) + + started_at: dict[str, float] = {} + + def _run_one(it: dict) -> dict: + started_at[str(it["id"])] = time.time() + return process_one_codegen( + it, + data_root, + out_root, + skill_content, + mode, + max_turns, + max_completion_tokens, + task_timeout, + use_eval_feedback, + diagnostic_mode, + diagnostic_instruction, + (diagnostic_trace_context_by_id or {}).get(str(it["id"]), ""), + ) + + def _timeout_result(item: dict) -> dict: + return { + "id": str(item["id"]), + "ok": False, + "instruction_type": item.get("instruction_type", ""), + "task_type": "other", + "phase": "timeout", + "fail_reason": f"task-timeout-{task_timeout}s", + "n_cases": 0, "n_pass": 0, "soft": 0.0, "hard": 0, + "n_turns": 0, "cases": [], "error": "timeout", + } + + def _error_result(item: dict, e: Exception) -> dict: + return { + "id": str(item["id"]), + "ok": False, + "instruction_type": item.get("instruction_type", ""), + "task_type": "other", + "phase": "error", + "fail_reason": f"unexpected: {type(e).__name__}: {e}", + "n_cases": 0, "n_pass": 0, "soft": 0.0, "hard": 0, + "n_turns": 0, "cases": [], "error": str(e), + } + + def _record(res: dict, i: int) -> None: + results.append(res) + status = "PASS" if res.get("hard") else ("TIMEOUT" if res.get("phase") == "timeout" else "FAIL") + dt = time.time() - t0 + print( + f" {i}/{len(pending)} id={res['id']:<10} {status} " + f"turns={res.get('n_turns', 0):<3} " + f"cases={res.get('n_pass', 0)}/{res.get('n_cases', 0)} " + f"dt={dt:.0f}s" + ) + + ex = ThreadPoolExecutor(max_workers=max_api_workers) + try: + futs = {ex.submit(_run_one, it): it for it in pending} + pending_futs = set(futs) + finished = 0 + while pending_futs: + done, _ = wait(pending_futs, timeout=5, return_when=FIRST_COMPLETED) + now = time.time() + timed_out = [] if no_task_timeout else [ + fut for fut in pending_futs - done + if str(futs[fut]["id"]) in started_at + and now - started_at[str(futs[fut]["id"])] >= task_timeout + ] + for fut in done: + pending_futs.remove(fut) + item = futs[fut] + try: + res = fut.result() + except FuturesTimeoutError: + res = _timeout_result(item) + except Exception as e: # noqa: BLE001 + res = _error_result(item, e) + finished += 1 + _record(res, finished) + for fut in timed_out: + pending_futs.remove(fut) + fut.cancel() + finished += 1 + _record(_timeout_result(futs[fut]), finished) + finally: + ex.shutdown(wait=False, cancel_futures=True) + + return results diff --git a/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/spreadsheetbench/skills/initial.md b/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/spreadsheetbench/skills/initial.md new file mode 100644 index 00000000..17eb7cdc --- /dev/null +++ b/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/envs/spreadsheetbench/skills/initial.md @@ -0,0 +1,56 @@ +# Spreadsheet Manipulation Skill (xlsx) + +## Overview +This skill guides agents in manipulating Excel (.xlsx) spreadsheets using Python. + +**Primary libraries**: `openpyxl` (structure-preserving read/write), `pandas` (data transformation). +Never use any other third-party libraries. + +--- + +## Common Workflow + +1. **Explore** the input file: list sheets, inspect headers, check dimensions. +2. **Write `solution.py`** with `INPUT_PATH` and `OUTPUT_PATH` defined at the top. +3. **Execute** `python solution.py` and verify the output file was created. +4. **Confirm** the target cells/range contain the expected values. + +--- + +## Library Selection + +| Use case | Library | +|----------|---------| +| Preserve formulas, formatting, named ranges | `openpyxl` | +| Bulk data transformation, aggregation, sorting | `pandas` → write back with `openpyxl` | +| Simple cell read/write | `openpyxl` | + +**Warning**: `pandas.to_excel()` silently destroys existing formulas and named ranges. +When writing back to a spreadsheet that contains formulas, always use `openpyxl.save()`. + +--- + +## solution.py Template + +```python +import openpyxl +import pandas as pd + +INPUT_PATH = "..." # set to the actual input path +OUTPUT_PATH = "..." # set to the actual output path + +wb = openpyxl.load_workbook(INPUT_PATH) +ws = wb.active # or wb["SheetName"] + +# --- perform manipulation --- + +wb.save(OUTPUT_PATH) +``` + +--- + +## Output Requirements + +- Save the result to `OUTPUT_PATH`. +- Do not hardcode row counts or column letters — iterate over actual rows in the workbook. +- Preserve sheets and cells not mentioned in the instruction. diff --git a/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/evaluation/__init__.py b/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/evaluation/__init__.py new file mode 100644 index 00000000..bb89670e --- /dev/null +++ b/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/evaluation/__init__.py @@ -0,0 +1,13 @@ +"""ReflACT Evaluation -- candidate skill validation and model selection. + +Analogous to validation-based early stopping and model selection in neural +network training: evaluates candidate skills on held-out selection sets and +decides whether to accept or reject proposed updates. +""" +from skillopt.evaluation.gate import ( # noqa: F401 + GateAction, + GateMetric, + GateResult, + evaluate_gate, + select_gate_score, +) diff --git a/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/evaluation/gate.py b/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/evaluation/gate.py new file mode 100644 index 00000000..18564b0c --- /dev/null +++ b/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/evaluation/gate.py @@ -0,0 +1,148 @@ +"""Validation gate — accept / reject candidate skills. + +Analogous to validation-based early stopping and model selection in neural +network training: compares the candidate's score against the current and +best scores, then returns an accept/reject decision. + +The trainer owns side-effects (cache lookup, rollout, printing, state +mutation). This module is the pure decision function. + +Metric selection +---------------- +Three gate metrics are supported: + +* ``"hard"`` (default, backward-compatible): + Compare candidate vs current/best using *hard* exact-match accuracy. +* ``"soft"``: + Compare using *soft* per-item score (F1 / partial credit / etc.). + Use this when a small held-out selection set has too few items for + hard accuracy to be sensitive to incremental skill improvements. +* ``"mixed"``: + Compare using a weighted average ``(1 - w) * hard + w * soft``. + ``w`` is configurable via ``mixed_weight`` (default ``0.5``). +""" +from __future__ import annotations + +from dataclasses import dataclass +from typing import Literal + + +GateAction = Literal["accept_new_best", "accept", "reject"] +GateMetric = Literal["hard", "soft", "mixed"] + + +@dataclass(frozen=True) +class GateResult: + """Immutable outcome of the validation gate.""" + + action: GateAction + current_skill: str + current_score: float + best_skill: str + best_score: float + best_step: int + + +def select_gate_score( + hard: float, + soft: float, + metric: GateMetric = "hard", + mixed_weight: float = 0.5, +) -> float: + """Project (hard, soft) onto a single comparison metric. + + Parameters + ---------- + hard, soft + Aggregate hard / soft scores from a rollout batch (both 0..1). + metric + Which metric to compare on. + mixed_weight + For ``"mixed"``: weight given to ``soft``. Must be in ``[0, 1]``. + Ignored for ``"hard"`` / ``"soft"``. + """ + if metric == "hard": + return float(hard) + if metric == "soft": + return float(soft) + if metric == "mixed": + w = max(0.0, min(1.0, float(mixed_weight))) + return (1.0 - w) * float(hard) + w * float(soft) + raise ValueError( + f"unknown gate metric {metric!r}; expected 'hard', 'soft', or 'mixed'" + ) + + +def evaluate_gate( + candidate_skill: str, + cand_hard: float, + current_skill: str, + current_score: float, + best_skill: str, + best_score: float, + best_step: int, + global_step: int, + *, + cand_soft: float = 0.0, + metric: GateMetric = "hard", + mixed_weight: float = 0.5, +) -> GateResult: + """Pure gate decision: compare candidate score to current/best. + + Parameters + ---------- + candidate_skill + The candidate skill content being evaluated. + cand_hard, cand_soft + Aggregate hard / soft scores of the candidate on the selection set. + current_skill, current_score + The currently-active skill and its *metric-space* score. + best_skill, best_score, best_step + The best-so-far skill, its *metric-space* score, and the step + at which it was accepted. + global_step + Current global training step (recorded if a new best is accepted). + cand_soft + Soft score of the candidate; only consulted when ``metric != "hard"``. + Defaults to ``0.0`` for backward compatibility with callers that + previously passed only ``cand_hard``. + metric + Which metric to compare on. Defaults to ``"hard"`` to preserve + the original gate behavior. + mixed_weight + Weight on ``soft`` when ``metric == "mixed"``. + + Returns + ------- + GateResult + Updated state; the caller decides what to do with it (print, + mutate trainer state, log, etc.). + """ + cand_score = select_gate_score(cand_hard, cand_soft, metric, mixed_weight) + + if cand_score > current_score: + if cand_score > best_score: + return GateResult( + action="accept_new_best", + current_skill=candidate_skill, + current_score=cand_score, + best_skill=candidate_skill, + best_score=cand_score, + best_step=global_step, + ) + return GateResult( + action="accept", + current_skill=candidate_skill, + current_score=cand_score, + best_skill=best_skill, + best_score=best_score, + best_step=best_step, + ) + return GateResult( + action="reject", + current_skill=current_skill, + current_score=current_score, + best_skill=best_skill, + best_score=best_score, + best_step=best_step, + ) diff --git a/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/gradient/__init__.py b/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/gradient/__init__.py new file mode 100644 index 00000000..0b05ef84 --- /dev/null +++ b/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/gradient/__init__.py @@ -0,0 +1,15 @@ +"""SkillOpt Gradient -- trajectory analysis and patch generation. + +Analogous to gradient computation in neural network training: analyzes +minibatch rollout trajectories to produce skill-edit patches (the "gradient" +that drives skill updates). + +Modules +------- +- reflect: minibatch trajectory analysis (gradient computation) +- aggregate: hierarchical patch merging (gradient aggregation) +""" +from skillopt.gradient.reflect import ( # noqa: F401 + run_minibatch_reflect, +) +from skillopt.gradient.aggregate import merge_patches # noqa: F401 diff --git a/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/gradient/aggregate.py b/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/gradient/aggregate.py new file mode 100644 index 00000000..cdad87c0 --- /dev/null +++ b/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/gradient/aggregate.py @@ -0,0 +1,253 @@ +"""ReflACT Aggregate stage — hierarchical patch merging. + +The Aggregate stage takes independently-generated patches from the Reflect +stage and merges them into a single coherent patch via hierarchical LLM calls. +Failure-driven patches take priority over success-driven ones. +""" +from __future__ import annotations + +import json +from concurrent.futures import ThreadPoolExecutor, as_completed + +from skillopt.model import chat_optimizer +from skillopt.optimizer.meta_skill import format_meta_skill_context +from skillopt.optimizer.update_modes import ( + get_payload_items, + is_full_rewrite_minibatch_mode, + is_rewrite_mode, + normalize_update_mode, + payload_key, + payload_label, +) +from skillopt.prompts import load_prompt +from skillopt.utils import extract_json + + +# ── Internal helpers ────────────────────────────────────────────────────────── + +def _merge_batch( + skill_content: str, + patches: list[dict], + system_prompt: str, + update_mode: str, + meta_skill_context: str = "", + level: int = 1, +) -> dict: + """Call optimizer LLM to merge a batch of patches into one.""" + patches_text = json.dumps(patches, ensure_ascii=False, indent=2) + user = ( + f"## Current Skill\n{skill_content}\n\n" + f"## Patches to merge ({len(patches)} total, merge level {level})\n{patches_text}" + ) + optimizer_ctx = format_meta_skill_context(meta_skill_context) + if optimizer_ctx: + user = f"{optimizer_ctx}\n\n{user}" + try: + response, _ = chat_optimizer( + system=system_prompt, + user=user, + max_completion_tokens=64000 if is_full_rewrite_minibatch_mode(update_mode) else 4096, + retries=3, + stage="merge", + ) + merged = extract_json(response) + key = payload_key(update_mode) + if merged and key in merged: + for e in merged.get(key, []): + e["merge_level"] = level + return merged + except Exception: # noqa: BLE001 + pass + # Fallback: concatenate all edits + all_edits = [] + for p in patches: + for e in get_payload_items(p, update_mode): + e.setdefault("merge_level", level) + all_edits.append(e) + return {"reasoning": "fallback concatenation", payload_key(update_mode): all_edits} + + +def _hierarchical_merge( + skill_content: str, + patches: list[dict], + system_prompt: str, + update_mode: str, + batch_size: int, + verbose: bool, + label: str = "", + workers: int = 16, + meta_skill_context: str = "", +) -> dict: + """Hierarchically merge N patches using the given system prompt. + + Same-level batches are executed in PARALLEL via ThreadPoolExecutor. + """ + if not patches: + return {"reasoning": "no patches", payload_key(update_mode): []} + if len(patches) == 1: + return patches[0] + + current = list(patches) + level = 0 + while len(current) > 1: + level += 1 + batches: list[tuple[int, list[dict]]] = [] + for i in range(0, len(current), batch_size): + batch = current[i : i + batch_size] + batches.append((i, batch)) + + if verbose: + print( + f" [aggregate {label}] level={level} " + f"{len(current)} patches → {len(batches)} batches " + f"(parallel, batch_size={batch_size})" + ) + + next_level: list[dict | None] = [None] * len(batches) + + to_merge: list[tuple[int, list[dict]]] = [] + for idx, (i, batch) in enumerate(batches): + if len(batch) == 1: + next_level[idx] = batch[0] + else: + to_merge.append((idx, batch)) + + if to_merge: + with ThreadPoolExecutor(max_workers=workers) as ex: + futs = { + ex.submit( + _merge_batch, skill_content, batch, system_prompt, update_mode, + meta_skill_context, level, + ): idx + for idx, batch in to_merge + } + for fut in as_completed(futs): + idx = futs[fut] + next_level[idx] = fut.result() + if verbose: + batch_i, batch_data = batches[idx] + n_edits = len(get_payload_items(next_level[idx], update_mode)) + print( + f" [aggregate {label}] level={level} " + f"batch [{batch_i}:{batch_i+len(batch_data)}] " + f"→ 1 patch ({n_edits} {payload_label(update_mode)})" + ) + + current = [x for x in next_level if x is not None] + + return current[0] + + +# ── Public API ──────────────────────────────────────────────────────────────── + +def merge_patches( + skill_content: str, + failure_patches: list[dict], + success_patches: list[dict], + batch_size: int = 8, + verbose: bool = True, + workers: int = 16, + update_mode: str = "patch", + meta_skill_context: str = "", +) -> dict: + """Failure-first hierarchical merge with support count tracking. + + 1. Merge failure patches independently (parallel) + 2. Merge success patches independently (parallel) + 3. Final merge: combine both groups with failure priority + + Returns a merged :class:`~skillopt.types.Patch` dict (``edits`` + ``reasoning``). + """ + if verbose: + print( + f" [3/6 AGGREGATE] " + f"failure={len(failure_patches)} success={len(success_patches)} " + f"(parallel, workers={workers})" + ) + + update_mode = normalize_update_mode(update_mode) + if is_full_rewrite_minibatch_mode(update_mode): + merge_failure_prompt = load_prompt("merge_failure_full_rewrite") + merge_success_prompt = load_prompt("merge_success_full_rewrite") + merge_final_prompt = load_prompt("merge_final_full_rewrite") + elif is_rewrite_mode(update_mode): + merge_failure_prompt = load_prompt("merge_failure_rewrite") + merge_success_prompt = load_prompt("merge_success_rewrite") + merge_final_prompt = load_prompt("merge_final_rewrite") + else: + merge_failure_prompt = load_prompt("merge_failure") + merge_success_prompt = load_prompt("merge_success") + merge_final_prompt = load_prompt("merge_final") + + failure_merged = _hierarchical_merge( + skill_content, failure_patches, merge_failure_prompt, update_mode, + batch_size, verbose, label="failure", workers=workers, + meta_skill_context=meta_skill_context, + ) + + success_merged = _hierarchical_merge( + skill_content, success_patches, merge_success_prompt, update_mode, + batch_size, verbose, label="success", workers=workers, + meta_skill_context=meta_skill_context, + ) + + f_edits = get_payload_items(failure_merged, update_mode) + s_edits = get_payload_items(success_merged, update_mode) + + if not f_edits and not s_edits: + return {"reasoning": "no updates from either group", payload_key(update_mode): []} + if not s_edits: + return failure_merged + if not f_edits: + return success_merged + + combined_patches = [failure_merged, success_merged] + combined_text = json.dumps(combined_patches, ensure_ascii=False, indent=2) + if is_full_rewrite_minibatch_mode(update_mode): + item_label = payload_label(update_mode) + user = ( + f"## Current Skill\n{skill_content}\n\n" + f"## Two pre-merged candidate groups to combine\n" + f"Group 1 (from failed trajectories): " + f"{len(f_edits)} {item_label}\n" + f"Group 2 (from successful trajectories): " + f"{len(s_edits)} {item_label}\n\n" + f"{combined_text}" + ) + else: + user = ( + f"## Current Skill\n{skill_content}\n\n" + f"## Two pre-merged patch groups to combine\n" + f"Group 1 (failure-driven, HIGH priority): " + f"{len(f_edits)} edits\n" + f"Group 2 (success-driven, lower priority): " + f"{len(s_edits)} edits\n\n" + f"{combined_text}" + ) + optimizer_ctx = format_meta_skill_context(meta_skill_context) + if optimizer_ctx: + user = f"{optimizer_ctx}\n\n{user}" + try: + response, _ = chat_optimizer( + system=merge_final_prompt, + user=user, + max_completion_tokens=64000 if is_full_rewrite_minibatch_mode(update_mode) else 4096, + retries=3, + stage="merge", + ) + final = extract_json(response) + key = payload_key(update_mode) + if final and key in final: + if verbose: + print( + f" [aggregate final] " + f"{len(f_edits)}+{len(s_edits)} → {len(final[key])} {payload_label(update_mode)}" + ) + return final + except Exception: # noqa: BLE001 + pass + + return { + "reasoning": "fallback: failure first, then success", + payload_key(update_mode): f_edits + s_edits, + } diff --git a/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/gradient/reflect.py b/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/gradient/reflect.py new file mode 100644 index 00000000..b7649ce7 --- /dev/null +++ b/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/gradient/reflect.py @@ -0,0 +1,588 @@ +"""ReflACT core Reflect engine -- minibatch trajectory analysis. + +Provides environment-agnostic minibatch trajectory analysis: instead of +analyzing each trajectory independently, trajectories are grouped into +minibatches of size M and analyzed together -- analogous to minibatch SGD +vs per-sample SGD in neural network training. + +Two-level prompt priority system: + +1. **Custom prompt** (adapter returns non-None) -- used as-is. +2. **Generic default prompt** (adapter returns None) -- built-in defaults + that work for any environment without configuration. + +Public API +---------- +- :func:`fmt_trajectory` -- format one conversation into text +- :func:`fmt_minibatch_trajectories` -- format multiple trajectories for batch analysis +- :func:`run_error_analyst_minibatch` -- one optimizer call for a group of failures +- :func:`run_success_analyst_minibatch` -- one optimizer call for a group of successes +- :func:`run_minibatch_reflect` -- full reflect stage dispatcher +""" +from __future__ import annotations + +import json +import os +import random +import traceback +from concurrent.futures import ThreadPoolExecutor, as_completed + +from skillopt.model import chat_optimizer +from skillopt.optimizer.meta_skill import format_meta_skill_context +from skillopt.optimizer.update_modes import ( + get_payload_items, + is_full_rewrite_minibatch_mode, + normalize_update_mode, + payload_key, + payload_label, + truncate_payload, +) +from skillopt.prompts import load_prompt +from skillopt.utils import extract_json + + +# ── Trajectory formatting ──────────────────────────────────────────────────── + +_MAX_TRAJ_CHARS = 12_000 + + +def _clip_text(value, limit: int) -> str: + """Render optional trajectory fields safely before truncation.""" + if value is None: + return "" + return str(value)[:limit] + + +def fmt_trajectory( + conversation: list[dict], + max_chars: int = _MAX_TRAJ_CHARS, +) -> str: + """Format a conversation list into analyst-readable text. + + Accepts two common formats: + + 1. Tool-call records: ``{"type": "tool_call", "cmd": ..., "obs": ...}`` + 2. Step records: ``{"step": N, "action": ..., "env_feedback": ..., "reasoning": ...}`` + + Any other dict is rendered via its ``"content"`` key. + """ + lines: list[str] = [] + for item in conversation: + if not isinstance(item, dict): + lines.append(f"[agent] {_clip_text(item, 500)}") + continue + if item.get("type") == "tool_call": + cmd = _clip_text(item.get("cmd"), 500) + obs = _clip_text(item.get("obs"), 800) + lines.append(f"[action] {cmd}") + lines.append(f"[obs] {obs}") + elif "action" in item and "env_feedback" in item: + step = item.get("step", "?") + reasoning = _clip_text(item.get("reasoning"), 300) + action = _clip_text(item.get("action"), 200) + feedback = _clip_text(item.get("env_feedback"), 500) + if reasoning: + lines.append(f"[step {step} think] {reasoning}") + lines.append(f"[step {step} action] {action}") + lines.append(f"[step {step} obs] {feedback}") + elif item.get("role") == "system": + # Post-execution verification / enrichment info + msg = _clip_text(item.get("content"), 2000) + lines.append(f"[verification] {msg}") + else: + msg = _clip_text(item.get("content"), 500) + role = item.get("role", "agent") + lines.append(f"[{role}] {msg}") + + text = "\n".join(lines) + if len(text) > max_chars: + head = text[: max_chars // 2] + tail = text[-max_chars // 2 :] + text = head + "\n...[middle truncated]...\n" + tail + return text + + +# ── Minibatch trajectory formatting ────────────────────────────────────────── + + +def fmt_minibatch_trajectories( + items: list[dict], + prediction_dir: str, +) -> str: + """Format multiple trajectories for minibatch analyst consumption. + + Each item is a rollout result dict with ``"id"``, ``"task_description"``, + ``"task_type"``, ``"fail_reason"``, etc. Reads ``conversation.json`` + for each and formats them together with trajectory headers. + + If available, includes the spreadsheet preview and target system prompt + so the analyst can see what the agent saw. + + Parameters + ---------- + items : list[dict] + Rollout result dicts belonging to one minibatch. + prediction_dir : str + Path to ``predictions/`` directory containing per-task + ``/conversation.json`` files. + + Returns + ------- + str + Formatted text with all trajectories separated by ``---``. + """ + parts: list[str] = [] + for idx, item in enumerate(items, 1): + tid = str(item["id"]) + conv_path = os.path.join(prediction_dir, tid, "conversation.json") + if not os.path.exists(conv_path): + continue + with open(conv_path) as f: + conversation = json.load(f) + if not conversation: + continue + + traj_text = fmt_trajectory(conversation) + header = ( + f"### Trajectory {idx} (id={tid})\n" + f"Task: {item.get('task_description', item.get('instruction', ''))}\n" + f"Task type: {item.get('task_type', item.get('instruction_type', ''))}\n" + ) + fail_reason = item.get("fail_reason", "") + if fail_reason: + header += f"Failure reason: {fail_reason}\n" + header += f"Steps: {item.get('n_turns', '?')}\n" + + reference_text = str(item.get("reference_text") or "").strip() + if reference_text: + header += ( + f"\n#### Hidden Reference\n" + f"{reference_text[:4000]}\n" + ) + + # ── Append target context (what the agent saw) ────────────── + target_prompt = item.get("target_system_prompt", "") + if not target_prompt: + prompt_path = os.path.join(prediction_dir, tid, "target_system_prompt.txt") + if os.path.exists(prompt_path): + with open(prompt_path) as f: + target_prompt = f.read() + if target_prompt: + header += ( + f"\n#### Target System Prompt\n" + f"{target_prompt[:3000]}\n" + ) + + user_prompt = item.get("target_user_prompt", "") + if not user_prompt: + user_prompt_path = os.path.join(prediction_dir, tid, "target_user_prompt.txt") + if os.path.exists(user_prompt_path): + with open(user_prompt_path) as f: + user_prompt = f.read() + if user_prompt: + header += ( + f"\n#### Target User Prompt\n" + f"{user_prompt[:3000]}\n" + ) + + if os.environ.get("REFLACT_CODEX_TRACE_TO_OPTIMIZER", "0") == "1": + codex_trace_summary = item.get("codex_trace_summary", "") + if not codex_trace_summary: + codex_trace_summary_path = os.path.join(prediction_dir, tid, "codex_trace_summary.txt") + if os.path.exists(codex_trace_summary_path): + with open(codex_trace_summary_path) as f: + codex_trace_summary = f.read() + if codex_trace_summary: + header += ( + f"\n#### Codex Trace Summary\n" + f"{codex_trace_summary}\n" + ) + + codex_probe_trace_steps = str(item.get("codex_probe_trace_steps") or "").strip() + if codex_probe_trace_steps: + header += ( + f"\n#### Codex Trace Steps\n" + f"{codex_probe_trace_steps}\n" + ) + + preview = item.get("spreadsheet_preview", "") + if not preview: + preview_path = os.path.join(prediction_dir, tid, "spreadsheet_preview.txt") + if os.path.exists(preview_path): + with open(preview_path) as f: + preview = f.read() + if preview: + header += ( + f"\n#### Spreadsheet Preview\n" + f"{preview[:3000]}\n" + ) + + parts.append(header + "\n" + traj_text) + + return "\n\n---\n\n".join(parts) + + +# ── Prompt resolution ─────────────────────────────────────────────────────── + + +def _resolve_prompt(custom: str | None, default_name: str, update_mode: str = "patch") -> str: + """Return *custom* if provided (non-None), otherwise load from file.""" + if custom is not None: + return custom + mode = normalize_update_mode(update_mode) + actual_name = default_name + if is_full_rewrite_minibatch_mode(mode): + full_name = f"{default_name}_full_rewrite" + try: + return load_prompt(full_name) + except FileNotFoundError: + actual_name = default_name + elif mode == "rewrite_from_suggestions": + rewrite_name = f"{default_name}_rewrite" + try: + return load_prompt(rewrite_name) + except FileNotFoundError: + actual_name = default_name + return load_prompt(actual_name) + + +# ── Minibatch analysts ────────────────────────────────────────────────────── + + +def run_error_analyst_minibatch( + skill_content: str, + items: list[dict], + prediction_dir: str, + edit_budget: int = 4, + *, + system_prompt: str | None = None, + rejection_context: str = "", + trajectory_memory_context: str = "", + step_buffer_context: str = "", + meta_skill_context: str = "", + update_mode: str = "patch", +) -> dict | None: + """Analyze a minibatch of failed trajectories in one optimizer call. + + Parameters + ---------- + skill_content : str + Current skill document text. + items : list[dict] + Rollout result dicts (all should have ``hard=0``). + prediction_dir : str + Path to ``predictions/`` directory. + edit_budget : int + Maximum number of edits (L). + system_prompt : str | None + Custom system prompt. ``None`` = use generic default. + rejection_context : str + *Deprecated* — use ``step_buffer_context``. + trajectory_memory_context : str + *Deprecated* — use ``step_buffer_context``. + step_buffer_context : str + Unified summary of previous steps (failure patterns + rejected edits). + + Returns + ------- + dict | None + Patch dict with ``source_type="failure"``, or ``None`` on error. + """ + mode = normalize_update_mode(update_mode) + actual_system = _resolve_prompt(system_prompt, "analyst_error", mode) + + trajectories_text = fmt_minibatch_trajectories(items, prediction_dir) + if not trajectories_text.strip(): + return None + + user = ( + f"## Current Skill\n{skill_content}\n\n" + ) + if is_full_rewrite_minibatch_mode(mode): + user += ( + f"## Update Format\n" + f"Produce one complete replacement skill candidate for this minibatch. " + f"Do not output edits, patches, or revise suggestions.\n\n" + ) + else: + user += ( + f"## {payload_label(mode, title=True)} Budget\n" + f"Produce at most L={edit_budget} {payload_label(mode)}.\n\n" + ) + # Unified step buffer context (preferred) + ctx = step_buffer_context or rejection_context or "" + if trajectory_memory_context: + ctx = f"{ctx}\n{trajectory_memory_context}" if ctx else trajectory_memory_context + if ctx.strip(): + user += f"## Previous Steps in This Epoch\n{ctx}\n\n" + optimizer_ctx = format_meta_skill_context(meta_skill_context) + if optimizer_ctx: + user += optimizer_ctx + "\n\n" + user += f"## Failed Trajectories ({len(items)} total)\n{trajectories_text}" + + try: + response, _ = chat_optimizer( + system=actual_system, user=user, + max_completion_tokens=64000 if is_full_rewrite_minibatch_mode(mode) else 4096, + retries=3, + stage="analyst", + ) + result = extract_json(response) + if result and "patch" in result: + result["source_type"] = "failure" + if not is_full_rewrite_minibatch_mode(mode): + truncate_payload(result["patch"], edit_budget, mode) + return result + except Exception: # noqa: BLE001 + traceback.print_exc() + return None + + +def run_success_analyst_minibatch( + skill_content: str, + items: list[dict], + prediction_dir: str, + edit_budget: int = 4, + *, + system_prompt: str | None = None, + trajectory_memory_context: str = "", + step_buffer_context: str = "", + meta_skill_context: str = "", + update_mode: str = "patch", +) -> dict | None: + """Analyze a minibatch of successful trajectories in one optimizer call. + + Parameters + ---------- + system_prompt : str | None + Custom system prompt. ``None`` = use generic default. + trajectory_memory_context : str + *Deprecated* — use ``step_buffer_context``. + step_buffer_context : str + Unified summary of previous steps (failure patterns + rejected edits). + + Returns + ------- + dict | None + Patch dict with ``source_type="success"``, or ``None`` on error. + """ + mode = normalize_update_mode(update_mode) + actual_system = _resolve_prompt(system_prompt, "analyst_success", mode) + + trajectories_text = fmt_minibatch_trajectories(items, prediction_dir) + if not trajectories_text.strip(): + return None + + user = ( + f"## Current Skill\n{skill_content}\n\n" + ) + if is_full_rewrite_minibatch_mode(mode): + user += ( + f"## Update Format\n" + f"Produce one complete replacement skill candidate for this minibatch. " + f"Do not output edits, patches, or revise suggestions.\n\n" + ) + else: + user += ( + f"## {payload_label(mode, title=True)} Budget\n" + f"Produce at most L={edit_budget} {payload_label(mode)}.\n\n" + ) + ctx = step_buffer_context or trajectory_memory_context or "" + if ctx.strip(): + user += f"## Previous Steps in This Epoch\n{ctx}\n\n" + optimizer_ctx = format_meta_skill_context(meta_skill_context) + if optimizer_ctx: + user += optimizer_ctx + "\n\n" + user += f"## Successful Trajectories ({len(items)} total)\n{trajectories_text}" + + try: + response, _ = chat_optimizer( + system=actual_system, user=user, + max_completion_tokens=64000 if is_full_rewrite_minibatch_mode(mode) else 4096, + retries=3, + stage="analyst", + ) + result = extract_json(response) + if result and "patch" in result: + result["source_type"] = "success" + if not is_full_rewrite_minibatch_mode(mode): + truncate_payload(result["patch"], edit_budget, mode) + return result + except Exception: # noqa: BLE001 + traceback.print_exc() + return None + + +# ── Minibatch reflect dispatcher ──────────────────────────────────────────── + + +def _split_minibatches(items: list, batch_size: int) -> list[list]: + """Split items into minibatches of at most *batch_size*.""" + return [items[i : i + batch_size] for i in range(0, len(items), batch_size)] + + +def _shuffle_for_minibatch(items: list, seed: int | None) -> list: + """Return items in minibatch order. + + Uses a deterministic shuffle when a seed is provided so resume runs keep + the same minibatch composition. Falls back to input order when no seed is + available. + """ + ordered = list(items) + if seed is None: + return ordered + random.Random(seed).shuffle(ordered) + return ordered + + +def run_minibatch_reflect( + results: list[dict], + skill_content: str, + prediction_dir: str, + patches_dir: str, + workers: int, + failure_only: bool, + minibatch_size: int = 8, + edit_budget: int = 4, + random_seed: int | None = None, + *, + error_system: str | None = None, + success_system: str | None = None, + rejection_context: str = "", + trajectory_memory_context: str = "", + step_buffer_context: str = "", + meta_skill_context: str = "", + update_mode: str = "patch", +) -> list[dict | None]: + """Full minibatch reflect stage: group → parallel optimizer calls → patches. + + Separates failure and success trajectories, splits each into minibatches + of size M, runs all minibatches in parallel, and saves patch files. + + Parameters + ---------- + results : list[dict] + Rollout result dicts; see :class:`~skillopt.types.RolloutResult`. + skill_content : str + Current skill document. + prediction_dir : str + Path to ``predictions/`` with ``conversation.json`` files. + patches_dir : str + Path to save per-minibatch patch JSON files. + workers : int + Max parallel optimizer calls. + failure_only : bool + If True, skip success trajectories. + minibatch_size : int + Trajectories per group (M). + edit_budget : int + Max edits per minibatch (L). + random_seed : int | None + Optional seed used to shuffle trajectories before minibatch splitting. + error_system, success_system : str | None + Optional custom prompts. ``None`` = use generic defaults. + + Returns + ------- + list[dict | None] + Patch dicts (with ``source_type`` "failure" or "success"). + """ + os.makedirs(patches_dir, exist_ok=True) + + # Separate failure / success + failures = [r for r in results if not r.get("hard") or float(r.get("hard", 0)) < 1e-9] + successes = [r for r in results if r.get("hard")] if not failure_only else [] + + failures = _shuffle_for_minibatch(failures, random_seed) + successes = _shuffle_for_minibatch(successes, None if random_seed is None else random_seed + 1) + + # Split into minibatches + fail_batches = _split_minibatches(failures, minibatch_size) + succ_batches = _split_minibatches(successes, minibatch_size) + + n_fail_batches = len(fail_batches) + n_succ_batches = len(succ_batches) + print( + f" [2/6 REFLECT minibatch] " + f"failure={len(failures)}→{n_fail_batches} groups " + f"success={len(successes)}→{n_succ_batches} groups " + f"(M={minibatch_size}, L={edit_budget}, workers={workers})" + ) + + raw_patches: list[dict | None] = [] + + # Resume support: check for already-done minibatch patches + pending_fail: list[tuple[int, list[dict]]] = [] + for idx, batch in enumerate(fail_batches): + path = os.path.join(patches_dir, f"minibatch_fail_{idx:03d}.json") + if os.path.exists(path): + with open(path) as f: + raw_patches.append(json.load(f)) + else: + pending_fail.append((idx, batch)) + + pending_succ: list[tuple[int, list[dict]]] = [] + for idx, batch in enumerate(succ_batches): + path = os.path.join(patches_dir, f"minibatch_succ_{idx:03d}.json") + if os.path.exists(path): + with open(path) as f: + raw_patches.append(json.load(f)) + else: + pending_succ.append((idx, batch)) + + # ── Worker functions ────────────────────────────────────────────────── + def _do_fail(idx: int, batch: list[dict]) -> tuple[str, dict | None]: + patch = run_error_analyst_minibatch( + skill_content, batch, prediction_dir, + edit_budget=edit_budget, + system_prompt=error_system, + step_buffer_context=step_buffer_context, + # backward compat fallback + rejection_context=rejection_context, + trajectory_memory_context=trajectory_memory_context, + meta_skill_context=meta_skill_context, + update_mode=update_mode, + ) + return f"minibatch_fail_{idx:03d}", patch + + def _do_succ(idx: int, batch: list[dict]) -> tuple[str, dict | None]: + patch = run_success_analyst_minibatch( + skill_content, batch, prediction_dir, + edit_budget=edit_budget, + system_prompt=success_system, + step_buffer_context=step_buffer_context, + trajectory_memory_context=trajectory_memory_context, + meta_skill_context=meta_skill_context, + update_mode=update_mode, + ) + return f"minibatch_succ_{idx:03d}", patch + + # Run all pending minibatches in parallel + all_pending = ( + [("fail", idx, batch) for idx, batch in pending_fail] + + [("succ", idx, batch) for idx, batch in pending_succ] + ) + + with ThreadPoolExecutor(max_workers=workers) as ex: + futs = {} + for kind, idx, batch in all_pending: + if kind == "fail": + futs[ex.submit(_do_fail, idx, batch)] = (kind, idx, len(batch)) + else: + futs[ex.submit(_do_succ, idx, batch)] = (kind, idx, len(batch)) + + for i, fut in enumerate(as_completed(futs), 1): + kind, idx, batch_len = futs[fut] + tag, patch = fut.result() + if patch: + path = os.path.join(patches_dir, f"{tag}.json") + with open(path, "w") as f: + json.dump(patch, f, ensure_ascii=False, indent=2) + raw_patches.append(patch) + n_edits = len(get_payload_items(patch.get("patch", {}) if patch else {}, update_mode)) + print( + f" [analyst] {i}/{len(all_pending)} {tag} " + f"({batch_len} trajs) → {n_edits} {payload_label(update_mode)}" + ) + + return raw_patches diff --git a/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/model/__init__.py b/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/model/__init__.py new file mode 100644 index 00000000..6730ab39 --- /dev/null +++ b/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/model/__init__.py @@ -0,0 +1,512 @@ +"""ReflACT model API with runtime backend selection for the target path.""" + +from __future__ import annotations + +from typing import Any + +from skillopt.model import azure_openai as _openai +from skillopt.model import claude_backend as _claude +from skillopt.model import minimax_backend as _minimax +from skillopt.model import qwen_backend as _qwen +from skillopt.model.backend_config import ( # noqa: F401 + configure_claude_code_exec, + configure_codex_exec, + get_claude_code_exec_config, + get_codex_exec_config, + get_target_backend, + get_optimizer_backend, + is_target_chat_backend, + is_target_exec_backend, + is_optimizer_chat_backend, + set_target_backend, + set_optimizer_backend, +) + + +def set_backend(name: str | None) -> str: + """Backward-compatible global backend setter. + + Historically the codebase used one shared backend for both optimizer and + target. Keep that entry point so older scripts continue to work, while + mapping it onto the split optimizer/target backend model. + """ + normalized = str(name or "azure_openai").strip().lower() + if normalized in {"azure_openai", "openai_chat", "azure", "azure-openai"}: + set_optimizer_backend("openai_chat") + set_target_backend("openai_chat") + return "azure_openai" + if normalized in {"claude", "claude_chat", "anthropic"}: + set_optimizer_backend("claude_chat") + set_target_backend("claude_chat") + return "claude_chat" + if normalized == "codex": + set_optimizer_backend("openai_chat") + set_target_backend("codex_exec") + return "codex" + if normalized in {"codex_exec", "claude_code_exec"}: + set_optimizer_backend("openai_chat") + set_target_backend(normalized) + return normalized + if normalized in {"qwen", "qwen_chat"}: + set_optimizer_backend("openai_chat") + set_target_backend("qwen_chat") + return "qwen_chat" + if normalized in {"minimax", "minimax_chat"}: + set_optimizer_backend("openai_chat") + set_target_backend("minimax_chat") + return "minimax_chat" + raise ValueError(f"Unsupported legacy backend: {name!r}") + + +def get_backend_name() -> str: + """Best-effort backward-compatible backend summary.""" + optimizer = get_optimizer_backend() + target = get_target_backend() + if optimizer == "claude_chat" and target == "claude_chat": + return "claude_chat" + if optimizer == "qwen_chat" and target == "qwen_chat": + return "qwen_chat" + if optimizer == "openai_chat" and target == "openai_chat": + return "azure_openai" + if optimizer == "openai_chat" and target == "codex_exec": + return "codex" + if optimizer == "openai_chat" and target == "qwen_chat": + return "qwen_chat" + if optimizer == "openai_chat" and target == "minimax_chat": + return "minimax_chat" + return f"{optimizer}+{target}" + + +def chat_optimizer( + system: str, + user: str, + max_completion_tokens: int = 16384, + retries: int = 5, + stage: str = "optimizer", + reasoning_effort: str | None = None, + timeout: int | None = None, +) -> tuple[str, dict]: + if get_optimizer_backend() == "claude_chat": + return _claude.chat_optimizer( + system=system, + user=user, + max_completion_tokens=max_completion_tokens, + retries=retries, + stage=stage, + timeout=timeout, + ) + if get_optimizer_backend() == "qwen_chat": + return _qwen.chat_optimizer( + system=system, + user=user, + max_completion_tokens=max_completion_tokens, + retries=retries, + stage=stage, + reasoning_effort=reasoning_effort, + timeout=timeout, + ) + return _openai.chat_optimizer( + system=system, + user=user, + max_completion_tokens=max_completion_tokens, + retries=retries, + stage=stage, + reasoning_effort=reasoning_effort, + timeout=timeout, + ) + + +def chat_target( + system: str, + user: str, + max_completion_tokens: int = 16384, + retries: int = 5, + stage: str = "target", + reasoning_effort: str | None = None, + timeout: int | None = None, +) -> tuple[str, dict]: + if get_target_backend() == "claude_chat": + return _claude.chat_target( + system=system, + user=user, + max_completion_tokens=max_completion_tokens, + retries=retries, + stage=stage, + timeout=timeout, + ) + if get_target_backend() == "qwen_chat": + return _qwen.chat_target( + system=system, + user=user, + max_completion_tokens=max_completion_tokens, + retries=retries, + stage=stage, + reasoning_effort=reasoning_effort, + ) + if get_target_backend() == "minimax_chat": + return _minimax.chat_target( + system=system, + user=user, + max_completion_tokens=max_completion_tokens, + retries=retries, + stage=stage, + reasoning_effort=reasoning_effort, + ) + if not is_target_chat_backend(): + raise NotImplementedError( + "chat_target is only supported with target_backend=openai_chat, claude_chat, qwen_chat, or minimax_chat. " + "Exec backends are handled in environment-specific rollout code." + ) + return _openai.chat_target( + system=system, + user=user, + max_completion_tokens=max_completion_tokens, + retries=retries, + stage=stage, + reasoning_effort=reasoning_effort, + timeout=timeout, + ) + + +def chat_optimizer_messages( + messages: list[dict[str, Any]], + max_completion_tokens: int = 16384, + retries: int = 5, + stage: str = "optimizer", + reasoning_effort: str | None = None, + *, + tools: list[dict[str, Any]] | None = None, + tool_choice: str | dict[str, Any] | None = None, + return_message: bool = False, + timeout: int | None = None, +) -> tuple[Any, dict]: + if get_optimizer_backend() == "claude_chat": + return _claude.chat_optimizer_messages( + messages=messages, + max_completion_tokens=max_completion_tokens, + retries=retries, + stage=stage, + tools=tools, + tool_choice=tool_choice, + return_message=return_message, + timeout=timeout, + ) + if get_optimizer_backend() == "qwen_chat": + return _qwen.chat_optimizer_messages( + messages=messages, + max_completion_tokens=max_completion_tokens, + retries=retries, + stage=stage, + reasoning_effort=reasoning_effort, + tools=tools, + tool_choice=tool_choice, + return_message=return_message, + timeout=timeout, + ) + return _openai.chat_optimizer_messages( + messages=messages, + max_completion_tokens=max_completion_tokens, + retries=retries, + stage=stage, + reasoning_effort=reasoning_effort, + tools=tools, + tool_choice=tool_choice, + return_message=return_message, + timeout=timeout, + ) + + +def chat_target_messages( + messages: list[dict[str, Any]], + max_completion_tokens: int = 16384, + retries: int = 5, + stage: str = "target", + reasoning_effort: str | None = None, + *, + tools: list[dict[str, Any]] | None = None, + tool_choice: str | dict[str, Any] | None = None, + return_message: bool = False, + timeout: int | None = None, +) -> tuple[Any, dict]: + if get_target_backend() == "claude_chat": + return _claude.chat_target_messages( + messages=messages, + max_completion_tokens=max_completion_tokens, + retries=retries, + stage=stage, + tools=tools, + tool_choice=tool_choice, + return_message=return_message, + timeout=timeout, + ) + if get_target_backend() == "qwen_chat": + return _qwen.chat_target_messages( + messages=messages, + max_completion_tokens=max_completion_tokens, + retries=retries, + stage=stage, + reasoning_effort=reasoning_effort, + tools=tools, + tool_choice=tool_choice, + return_message=return_message, + ) + if get_target_backend() == "minimax_chat": + return _minimax.chat_target_messages( + messages=messages, + max_completion_tokens=max_completion_tokens, + retries=retries, + stage=stage, + reasoning_effort=reasoning_effort, + tools=tools, + tool_choice=tool_choice, + return_message=return_message, + ) + if not is_target_chat_backend(): + raise NotImplementedError( + "chat_target_messages is only supported with target_backend=openai_chat, claude_chat, qwen_chat, or minimax_chat. " + "Exec backends are handled in environment-specific rollout code." + ) + return _openai.chat_target_messages( + messages=messages, + max_completion_tokens=max_completion_tokens, + retries=retries, + stage=stage, + reasoning_effort=reasoning_effort, + tools=tools, + tool_choice=tool_choice, + return_message=return_message, + timeout=timeout, + ) + + +def chat_messages_with_deployment( + deployment: str, + messages: list[dict[str, Any]], + max_completion_tokens: int = 16384, + retries: int = 5, + stage: str = "custom", + reasoning_effort: str | None = None, + *, + tools: list[dict[str, Any]] | None = None, + tool_choice: str | dict[str, Any] | None = None, + return_message: bool = False, + timeout: int | None = None, +) -> tuple[Any, dict]: + return _openai.chat_messages_with_deployment( + deployment=deployment, + messages=messages, + max_completion_tokens=max_completion_tokens, + retries=retries, + stage=stage, + reasoning_effort=reasoning_effort, + tools=tools, + tool_choice=tool_choice, + return_message=return_message, + timeout=timeout, + ) + + +def chat_with_deployment( + deployment: str, + system: str, + user: str, + max_completion_tokens: int = 16384, + retries: int = 5, + stage: str = "custom", + reasoning_effort: str | None = None, + timeout: int | None = None, +) -> tuple[str, dict]: + return _openai.chat_with_deployment( + deployment=deployment, + system=system, + user=user, + max_completion_tokens=max_completion_tokens, + retries=retries, + stage=stage, + reasoning_effort=reasoning_effort, + timeout=timeout, + ) + + +def get_token_summary() -> dict: + summary = _openai.get_token_summary() + claude_summary = _claude.get_token_summary() + for stage, values in claude_summary.items(): + if stage == "_total": + continue + if stage not in summary: + summary[stage] = values + continue + summary[stage]["calls"] += values["calls"] + summary[stage]["prompt_tokens"] += values["prompt_tokens"] + summary[stage]["completion_tokens"] += values["completion_tokens"] + summary[stage]["total_tokens"] += values["total_tokens"] + qwen_summary = _qwen.get_token_summary() + for stage, values in qwen_summary.items(): + if stage == "_total": + continue + if stage not in summary: + summary[stage] = values + continue + summary[stage]["calls"] += values["calls"] + summary[stage]["prompt_tokens"] += values["prompt_tokens"] + summary[stage]["completion_tokens"] += values["completion_tokens"] + summary[stage]["total_tokens"] += values["total_tokens"] + minimax_summary = _minimax.get_token_summary() + for stage, values in minimax_summary.items(): + if stage == "_total": + continue + if stage not in summary: + summary[stage] = values + continue + summary[stage]["calls"] += values["calls"] + summary[stage]["prompt_tokens"] += values["prompt_tokens"] + summary[stage]["completion_tokens"] += values["completion_tokens"] + summary[stage]["total_tokens"] += values["total_tokens"] + total = { + "calls": 0, + "prompt_tokens": 0, + "completion_tokens": 0, + "total_tokens": 0, + } + for stage, values in summary.items(): + if stage == "_total": + continue + total["calls"] += values["calls"] + total["prompt_tokens"] += values["prompt_tokens"] + total["completion_tokens"] += values["completion_tokens"] + total["total_tokens"] += values["total_tokens"] + summary["_total"] = total + return summary + + +def reset_token_tracker() -> None: + _openai.reset_token_tracker() + _claude.reset_token_tracker() + _qwen.reset_token_tracker() + _minimax.reset_token_tracker() + + +def configure_azure_openai( + *, + endpoint: str | None = None, + api_version: str | None = None, + api_key: str | None = None, + auth_mode: str | None = None, + ad_scope: str | None = None, + managed_identity_client_id: str | None = None, + optimizer_endpoint: str | None = None, + optimizer_api_version: str | None = None, + optimizer_api_key: str | None = None, + optimizer_auth_mode: str | None = None, + optimizer_ad_scope: str | None = None, + optimizer_managed_identity_client_id: str | None = None, + target_endpoint: str | None = None, + target_api_version: str | None = None, + target_api_key: str | None = None, + target_auth_mode: str | None = None, + target_ad_scope: str | None = None, + target_managed_identity_client_id: str | None = None, +) -> None: + _openai.configure_azure_openai( + endpoint=endpoint, + api_version=api_version, + api_key=api_key, + auth_mode=auth_mode, + ad_scope=ad_scope, + managed_identity_client_id=managed_identity_client_id, + optimizer_endpoint=optimizer_endpoint, + optimizer_api_version=optimizer_api_version, + optimizer_api_key=optimizer_api_key, + optimizer_auth_mode=optimizer_auth_mode, + optimizer_ad_scope=optimizer_ad_scope, + optimizer_managed_identity_client_id=optimizer_managed_identity_client_id, + target_endpoint=target_endpoint, + target_api_version=target_api_version, + target_api_key=target_api_key, + target_auth_mode=target_auth_mode, + target_ad_scope=target_ad_scope, + target_managed_identity_client_id=target_managed_identity_client_id, + ) + + +def configure_qwen_chat( + *, + base_url: str | None = None, + api_key: str | None = None, + temperature: float | str | None = None, + timeout_seconds: float | str | None = None, + max_tokens: int | str | None = None, + enable_thinking: bool | str | None = None, + optimizer_base_url: str | None = None, + optimizer_api_key: str | None = None, + optimizer_temperature: float | str | None = None, + optimizer_timeout_seconds: float | str | None = None, + optimizer_max_tokens: int | str | None = None, + optimizer_enable_thinking: bool | str | None = None, + target_base_url: str | None = None, + target_api_key: str | None = None, + target_temperature: float | str | None = None, + target_timeout_seconds: float | str | None = None, + target_max_tokens: int | str | None = None, + target_enable_thinking: bool | str | None = None, +) -> None: + _qwen.configure_qwen_chat( + base_url=base_url, + api_key=api_key, + temperature=temperature, + timeout_seconds=timeout_seconds, + max_tokens=max_tokens, + enable_thinking=enable_thinking, + optimizer_base_url=optimizer_base_url, + optimizer_api_key=optimizer_api_key, + optimizer_temperature=optimizer_temperature, + optimizer_timeout_seconds=optimizer_timeout_seconds, + optimizer_max_tokens=optimizer_max_tokens, + optimizer_enable_thinking=optimizer_enable_thinking, + target_base_url=target_base_url, + target_api_key=target_api_key, + target_temperature=target_temperature, + target_timeout_seconds=target_timeout_seconds, + target_max_tokens=target_max_tokens, + target_enable_thinking=target_enable_thinking, + ) + + +def configure_minimax_chat( + *, + base_url: str | None = None, + api_key: str | None = None, + temperature: float | str | None = None, + timeout_seconds: float | str | None = None, + max_tokens: int | str | None = None, + enable_thinking: bool | str | None = None, +) -> None: + _minimax.configure_minimax_chat( + base_url=base_url, + api_key=api_key, + temperature=temperature, + timeout_seconds=timeout_seconds, + max_tokens=max_tokens, + enable_thinking=enable_thinking, + ) + + +def set_reasoning_effort(effort: str | None) -> None: + _openai.set_reasoning_effort(effort) + _claude.set_reasoning_effort(effort) + _qwen.set_reasoning_effort(effort) + _minimax.set_reasoning_effort(effort) + + +def set_target_deployment(deployment: str) -> None: + _openai.set_target_deployment(deployment) + _claude.set_target_deployment(deployment) + _qwen.set_target_deployment(deployment) + _minimax.set_target_deployment(deployment) + + +def set_optimizer_deployment(deployment: str) -> None: + _openai.set_optimizer_deployment(deployment) + _claude.set_optimizer_deployment(deployment) + _qwen.set_optimizer_deployment(deployment) diff --git a/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/model/azure_openai.py b/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/model/azure_openai.py new file mode 100644 index 00000000..e7c139cb --- /dev/null +++ b/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/model/azure_openai.py @@ -0,0 +1,915 @@ +"""ReflACT Model backend — Azure OpenAI wrapper with token tracking. + +Provides optimizer/target dual-deployment chat functions and a global +TokenTracker for per-stage cost accounting. Previously llm/azure_openai.py. +""" +from __future__ import annotations + +import json +import os +import subprocess +import threading +import time +from types import SimpleNamespace +from typing import Any +from openai import AzureOpenAI, OpenAI + +# Sentinel value used as the api_version when the "openai_compatible" +# auth_mode is selected. Real Azure deployments never use this string, +# so it doubles as a marker for downstream type narrowing. +_OPENAI_COMPATIBLE_API_VERSION = "openai-compat" + +# ── Configuration ───────────────────────────────────────────────────────────── + +ENDPOINT = os.environ.get( + "AZURE_OPENAI_ENDPOINT", + "", # Set via env var or config: e.g. "https://your-resource.openai.azure.com/" +) +API_VERSION = os.environ.get("AZURE_OPENAI_API_VERSION", "2024-12-01-preview") +API_KEY = os.environ.get( + "AZURE_OPENAI_API_KEY", + "", +) +AUTH_MODE = os.environ.get("AZURE_OPENAI_AUTH_MODE", "azure_cli").strip().lower() +AD_SCOPE = os.environ.get( + "AZURE_OPENAI_AD_SCOPE", + "https://cognitiveservices.azure.com/.default", +) +MANAGED_IDENTITY_CLIENT_ID = os.environ.get( + "AZURE_OPENAI_MANAGED_IDENTITY_CLIENT_ID", + "", +).strip() + +OPTIMIZER_ENDPOINT = ( + os.environ.get("OPTIMIZER_AZURE_OPENAI_ENDPOINT") + or os.environ.get("AZURE_OPENAI_OPTIMIZER_ENDPOINT") + or ENDPOINT +) +TARGET_ENDPOINT = ( + os.environ.get("TARGET_AZURE_OPENAI_ENDPOINT") + or os.environ.get("AZURE_OPENAI_TARGET_ENDPOINT") + or ENDPOINT +) +OPTIMIZER_API_VERSION = ( + os.environ.get("OPTIMIZER_AZURE_OPENAI_API_VERSION") + or os.environ.get("AZURE_OPENAI_OPTIMIZER_API_VERSION") + or API_VERSION +) +TARGET_API_VERSION = ( + os.environ.get("TARGET_AZURE_OPENAI_API_VERSION") + or os.environ.get("AZURE_OPENAI_TARGET_API_VERSION") + or API_VERSION +) +OPTIMIZER_API_KEY = ( + os.environ.get("OPTIMIZER_AZURE_OPENAI_API_KEY") + or os.environ.get("AZURE_OPENAI_OPTIMIZER_API_KEY") + or API_KEY +) +TARGET_API_KEY = ( + os.environ.get("TARGET_AZURE_OPENAI_API_KEY") + or os.environ.get("AZURE_OPENAI_TARGET_API_KEY") + or API_KEY +) +OPTIMIZER_AUTH_MODE = ( + os.environ.get("OPTIMIZER_AZURE_OPENAI_AUTH_MODE") + or os.environ.get("AZURE_OPENAI_OPTIMIZER_AUTH_MODE") + or AUTH_MODE +).strip().lower() +TARGET_AUTH_MODE = ( + os.environ.get("TARGET_AZURE_OPENAI_AUTH_MODE") + or os.environ.get("AZURE_OPENAI_TARGET_AUTH_MODE") + or AUTH_MODE +).strip().lower() +OPTIMIZER_AD_SCOPE = ( + os.environ.get("OPTIMIZER_AZURE_OPENAI_AD_SCOPE") + or os.environ.get("AZURE_OPENAI_OPTIMIZER_AD_SCOPE") + or AD_SCOPE +) +TARGET_AD_SCOPE = ( + os.environ.get("TARGET_AZURE_OPENAI_AD_SCOPE") + or os.environ.get("AZURE_OPENAI_TARGET_AD_SCOPE") + or AD_SCOPE +) +OPTIMIZER_MANAGED_IDENTITY_CLIENT_ID = ( + os.environ.get("OPTIMIZER_AZURE_OPENAI_MANAGED_IDENTITY_CLIENT_ID") + or os.environ.get("AZURE_OPENAI_OPTIMIZER_MANAGED_IDENTITY_CLIENT_ID") + or MANAGED_IDENTITY_CLIENT_ID +).strip() +TARGET_MANAGED_IDENTITY_CLIENT_ID = ( + os.environ.get("TARGET_AZURE_OPENAI_MANAGED_IDENTITY_CLIENT_ID") + or os.environ.get("AZURE_OPENAI_TARGET_MANAGED_IDENTITY_CLIENT_ID") + or MANAGED_IDENTITY_CLIENT_ID +).strip() + +OPTIMIZER_DEPLOYMENT = os.environ.get("OPTIMIZER_DEPLOYMENT", "gpt-4o") +TARGET_DEPLOYMENT = os.environ.get("TARGET_DEPLOYMENT", "gpt-4o") + +REASONING_EFFORT: str | None = None + +_AZ_CLI_TOKEN_CACHE: dict[str, dict[str, Any]] = {} + +# Deployments that require Responses API +_RESPONSES_API_MODELS = { + "gpt-5.3-codex", "gpt-5.1-codex", "gpt-5.2-codex", + "gpt-5-codex", "codex-mini", "gpt-5.4-pro", +} + + +# ── Token Tracker ───────────────────────────────────────────────────────────── + +class TokenTracker: + """Thread-safe per-stage token counter.""" + + def __init__(self) -> None: + self._lock = threading.Lock() + self._data: dict[str, dict] = {} + + def record( + self, stage: str, prompt_tokens: int, completion_tokens: int, + ) -> None: + with self._lock: + if stage not in self._data: + self._data[stage] = { + "calls": 0, + "prompt_tokens": 0, + "completion_tokens": 0, + } + d = self._data[stage] + d["calls"] += 1 + d["prompt_tokens"] += prompt_tokens + d["completion_tokens"] += completion_tokens + + def summary(self) -> dict: + with self._lock: + out: dict = {} + total_p = total_c = total_calls = 0 + for stage, d in sorted(self._data.items()): + out[stage] = { + "calls": d["calls"], + "prompt_tokens": d["prompt_tokens"], + "completion_tokens": d["completion_tokens"], + "total_tokens": d["prompt_tokens"] + d["completion_tokens"], + } + total_p += d["prompt_tokens"] + total_c += d["completion_tokens"] + total_calls += d["calls"] + out["_total"] = { + "calls": total_calls, + "prompt_tokens": total_p, + "completion_tokens": total_c, + "total_tokens": total_p + total_c, + } + return out + + def reset(self) -> None: + with self._lock: + self._data.clear() + + def stage_snapshot(self, stage: str) -> dict: + """Return a copy of one stage's counters (or zeros if not tracked yet).""" + with self._lock: + d = self._data.get(stage, {}) + return { + "calls": d.get("calls", 0), + "prompt_tokens": d.get("prompt_tokens", 0), + "completion_tokens": d.get("completion_tokens", 0), + "total_tokens": d.get("prompt_tokens", 0) + d.get("completion_tokens", 0), + } + + +tracker = TokenTracker() + + +# ── Client management ───────────────────────────────────────────────────────── + +_optimizer_client: AzureOpenAI | OpenAI | None = None +_target_client: AzureOpenAI | OpenAI | None = None +_optimizer_lock = threading.Lock() +_target_lock = threading.Lock() + + +def _role_config(role: str) -> dict[str, str]: + if role == "optimizer": + return { + "endpoint": OPTIMIZER_ENDPOINT, + "api_version": OPTIMIZER_API_VERSION, + "api_key": OPTIMIZER_API_KEY, + "auth_mode": OPTIMIZER_AUTH_MODE, + "ad_scope": OPTIMIZER_AD_SCOPE, + "managed_identity_client_id": OPTIMIZER_MANAGED_IDENTITY_CLIENT_ID, + } + if role == "target": + return { + "endpoint": TARGET_ENDPOINT, + "api_version": TARGET_API_VERSION, + "api_key": TARGET_API_KEY, + "auth_mode": TARGET_AUTH_MODE, + "ad_scope": TARGET_AD_SCOPE, + "managed_identity_client_id": TARGET_MANAGED_IDENTITY_CLIENT_ID, + } + raise ValueError(f"Unknown Azure OpenAI client role: {role!r}") + + +def _make_token_provider( + auth_mode: str, + ad_scope: str, + managed_identity_client_id: str, +): + try: + from azure.identity import ( # type: ignore[import-not-found] + AzureCliCredential, + ManagedIdentityCredential, + get_bearer_token_provider, + ) + except ImportError as e: + if auth_mode == "azure_cli": + return _make_azure_cli_token_provider(ad_scope) + raise ImportError( + "Azure AD auth requires azure-identity. Install it with `pip install azure-identity`." + ) from e + + if auth_mode in {"managed_identity", "aad", "azure_ad"}: + if managed_identity_client_id: + credential = ManagedIdentityCredential(client_id=managed_identity_client_id) + else: + credential = ManagedIdentityCredential() + elif auth_mode == "azure_cli": + credential = AzureCliCredential() + else: + raise ValueError( + "Unsupported Azure OpenAI auth mode " + f"{auth_mode!r}; expected api_key, managed_identity, azure_ad, aad, or azure_cli." + ) + return get_bearer_token_provider(credential, ad_scope) + + +def _make_azure_cli_token_provider(ad_scope: str): + """Return an Azure CLI token provider compatible with AzureOpenAI. + + This fallback avoids requiring azure-identity in environments where `az` + is already logged in. The SDK calls this provider whenever it needs a + bearer token. + """ + + resource = ad_scope.removesuffix("/.default") + + def _provider() -> str: + now = int(time.time()) + cache = _AZ_CLI_TOKEN_CACHE.setdefault(resource, {"token": "", "expires_on": 0}) + cached = str(cache.get("token") or "") + expires_on = int(cache.get("expires_on") or 0) + if cached and expires_on - now > 300: + return cached + + raw = subprocess.check_output( + [ + "az", + "account", + "get-access-token", + "--resource", + resource, + "-o", + "json", + ], + text=True, + stderr=subprocess.STDOUT, + ) + payload = json.loads(raw) + token = str(payload["accessToken"]) + cache["token"] = token + cache["expires_on"] = int(payload.get("expires_on") or now + 3000) + return token + + return _provider + + +def _make_client(role: str) -> AzureOpenAI | OpenAI: + cfg = _role_config(role) + if not cfg["endpoint"]: + raise ValueError( + f"Azure OpenAI endpoint is not configured for {role}. " + "Pass --azure_openai_endpoint https://your-resource.openai.azure.com/ " + "or set AZURE_OPENAI_ENDPOINT in your environment." + ) + auth_mode = cfg["auth_mode"] + if auth_mode in {"openai_compatible", "compat", "openai"}: + return OpenAI( + base_url=cfg["endpoint"].rstrip("/"), + api_key=cfg["api_key"] or "dummy", + default_headers={"User-Agent": "SkillOpt"}, + ) + if auth_mode in {"api_key", "key"}: + if not cfg["api_key"]: + raise ValueError( + f"Azure OpenAI API key is not configured for {role}. " + "Set model.azure_openai_api_key in the config or export AZURE_OPENAI_API_KEY." + ) + return AzureOpenAI( + api_version=cfg["api_version"], + azure_endpoint=cfg["endpoint"], + api_key=cfg["api_key"], + ) + return AzureOpenAI( + api_version=cfg["api_version"], + azure_endpoint=cfg["endpoint"], + azure_ad_token_provider=_make_token_provider( + auth_mode, + cfg["ad_scope"], + cfg["managed_identity_client_id"], + ), + ) + + +def get_optimizer_client() -> AzureOpenAI | OpenAI: + global _optimizer_client + with _optimizer_lock: + if _optimizer_client is None: + _optimizer_client = _make_client("optimizer") + return _optimizer_client + + +def get_target_client() -> AzureOpenAI | OpenAI: + global _target_client + with _target_lock: + if _target_client is None: + # When using qwen_chat backend, return an OpenAI client pointing to vLLM + from skillopt.model.backend_config import get_target_backend + if get_target_backend() == "qwen_chat": + from skillopt.model import qwen_backend as _qwen + target_config = _qwen.TARGET_CONFIG + _target_client = OpenAI( + base_url=target_config.base_url, + api_key=target_config.api_key or "dummy", + ) + else: + _target_client = _make_client("target") + return _target_client + + +def _needs_responses_api(deployment: str) -> bool: + dep = deployment.lower() + return any(dep == m or dep.startswith(m + "-") for m in _RESPONSES_API_MODELS) + + +# ── Core chat function ──────────────────────────────────────────────────────── + +def _chat_impl( + client: AzureOpenAI | OpenAI, + deployment: str, + system: str, + user: str, + max_completion_tokens: int, + retries: int, + stage: str, + reasoning_effort: str | None = None, + timeout: int | None = None, +) -> tuple[str, dict]: + """Call LLM, track tokens, return (text, usage_dict).""" + last_err = None + usage_info = {"prompt_tokens": 0, "completion_tokens": 0, "total_tokens": 0} + + for attempt in range(retries): + try: + if _needs_responses_api(deployment): + kwargs: dict[str, Any] = { + "model": deployment, + "instructions": system, + "input": [{"role": "user", "content": user}], + "max_output_tokens": max_completion_tokens, + } + actual_effort = reasoning_effort or REASONING_EFFORT + if actual_effort: + kwargs["reasoning"] = {"effort": actual_effort} + if timeout is not None: + kwargs["timeout"] = timeout + resp = client.responses.create(**kwargs) + text = getattr(resp, "output_text", None) or "" + if not text: + for item in getattr(resp, "output", None) or []: + for part in getattr(item, "content", []): + if getattr(part, "type", "") == "output_text": + text = part.text or "" + if hasattr(resp, "usage") and resp.usage: + usage_info = { + "prompt_tokens": getattr(resp.usage, "input_tokens", 0) or 0, + "completion_tokens": getattr(resp.usage, "output_tokens", 0) or 0, + "total_tokens": ( + (getattr(resp.usage, "input_tokens", 0) or 0) + + (getattr(resp.usage, "output_tokens", 0) or 0) + ), + } + else: + kwargs: dict[str, Any] = dict( + model=deployment, + messages=[ + {"role": "system", "content": system}, + {"role": "user", "content": user}, + ], + max_completion_tokens=max_completion_tokens, + ) + actual_effort = reasoning_effort or REASONING_EFFORT + if actual_effort is not None: + kwargs["reasoning_effort"] = actual_effort + if timeout is not None: + kwargs["timeout"] = timeout + resp = client.chat.completions.create(**kwargs) + text = resp.choices[0].message.content or "" + if resp.usage: + usage_info = { + "prompt_tokens": resp.usage.prompt_tokens or 0, + "completion_tokens": resp.usage.completion_tokens or 0, + "total_tokens": resp.usage.total_tokens or 0, + } + + tracker.record( + stage, + usage_info["prompt_tokens"], + usage_info["completion_tokens"], + ) + return text, usage_info + + except Exception as e: # noqa: BLE001 + last_err = e + sleep = min(2 ** attempt, 30) + time.sleep(sleep) + + raise RuntimeError(f"LLM call failed after {retries} retries: {last_err}") + + +def _chat_messages_impl( + client: AzureOpenAI | OpenAI, + deployment: str, + messages: list[dict[str, Any]], + max_completion_tokens: int, + retries: int, + stage: str, + reasoning_effort: str | None = None, + *, + tools: list[dict[str, Any]] | None = None, + tool_choice: str | dict[str, Any] | None = None, + return_message: bool = False, + timeout: int | None = None, +) -> tuple[Any, dict]: + """Call the model with a pre-built message list.""" + last_err = None + usage_info = {"prompt_tokens": 0, "completion_tokens": 0, "total_tokens": 0} + + for attempt in range(retries): + try: + if _needs_responses_api(deployment): + input_items, instructions = _messages_to_responses_input(messages) + kwargs: dict[str, Any] = { + "model": deployment, + "input": input_items, + "max_output_tokens": max_completion_tokens, + } + if instructions: + kwargs["instructions"] = instructions + actual_effort = reasoning_effort or REASONING_EFFORT + if actual_effort: + kwargs["reasoning"] = {"effort": actual_effort} + if tools: + kwargs["tools"] = [_chat_tool_to_responses_tool(tool) for tool in tools] + if tool_choice is not None: + kwargs["tool_choice"] = tool_choice + if timeout is not None: + kwargs["timeout"] = timeout + resp = client.responses.create(**kwargs) + message, text = _responses_to_chat_message(resp) + if hasattr(resp, "usage") and resp.usage: + usage_info = { + "prompt_tokens": getattr(resp.usage, "input_tokens", 0) or 0, + "completion_tokens": getattr(resp.usage, "output_tokens", 0) or 0, + "total_tokens": ( + (getattr(resp.usage, "input_tokens", 0) or 0) + + (getattr(resp.usage, "output_tokens", 0) or 0) + ), + } + else: + kwargs = dict( + model=deployment, + messages=messages, + max_completion_tokens=max_completion_tokens, + ) + actual_effort = reasoning_effort or REASONING_EFFORT + if tools: + kwargs["tools"] = tools + if tool_choice is not None: + kwargs["tool_choice"] = tool_choice + # Some models (e.g. gpt-5.5) don't support reasoning_effort with function tools + elif actual_effort is not None: + kwargs["reasoning_effort"] = actual_effort + if timeout is not None: + kwargs["timeout"] = timeout + resp = client.chat.completions.create(**kwargs) + message = resp.choices[0].message + text = message.content or "" + if resp.usage: + usage_info = { + "prompt_tokens": resp.usage.prompt_tokens or 0, + "completion_tokens": resp.usage.completion_tokens or 0, + "total_tokens": resp.usage.total_tokens or 0, + } + tracker.record( + stage, + usage_info["prompt_tokens"], + usage_info["completion_tokens"], + ) + return (message if return_message else text), usage_info + except Exception as e: # noqa: BLE001 + last_err = e + sleep = min(2 ** attempt, 30) + time.sleep(sleep) + + raise RuntimeError(f"LLM message call failed after {retries} retries: {last_err}") + + +def _chat_tool_to_responses_tool(tool: dict[str, Any]) -> dict[str, Any]: + """Convert a Chat Completions function tool to Responses API format.""" + if tool.get("type") == "function" and isinstance(tool.get("function"), dict): + fn = tool["function"] + return { + "type": "function", + "name": fn.get("name", ""), + "description": fn.get("description", ""), + "parameters": fn.get("parameters", {"type": "object", "properties": {}}), + } + return tool + + +def _messages_to_responses_input(messages: list[dict[str, Any]]) -> tuple[list[dict[str, Any]], str]: + """Convert chat-style messages, including tool results, to Responses input.""" + instructions: list[str] = [] + input_items: list[dict[str, Any]] = [] + for message in messages: + role = message.get("role") + content = message.get("content") or "" + if role == "system": + if content: + instructions.append(str(content)) + continue + if role == "tool": + input_items.append({ + "type": "function_call_output", + "call_id": str(message.get("tool_call_id", "")), + "output": str(content), + }) + continue + if role == "assistant": + if content: + input_items.append({"role": "assistant", "content": str(content)}) + for tool_call in message.get("tool_calls") or []: + function = tool_call.get("function", {}) or {} + input_items.append({ + "type": "function_call", + "call_id": str(tool_call.get("id", "")), + "name": str(function.get("name", "")), + "arguments": str(function.get("arguments", "{}") or "{}"), + }) + continue + if role in {"user", "developer"}: + input_items.append({"role": "user", "content": str(content)}) + return input_items, "\n\n".join(instructions) + + +def _responses_to_chat_message(resp: Any) -> tuple[Any, str]: + """Convert Responses output into the subset of Chat message API we use.""" + text = getattr(resp, "output_text", None) or "" + tool_calls: list[dict[str, Any]] = [] + for item in getattr(resp, "output", None) or []: + item_type = getattr(item, "type", "") + if item_type == "function_call": + tool_calls.append({ + "id": getattr(item, "call_id", "") or getattr(item, "id", ""), + "type": "function", + "function": { + "name": getattr(item, "name", ""), + "arguments": getattr(item, "arguments", "") or "{}", + }, + }) + elif item_type == "message" and not text: + content_parts = getattr(item, "content", []) or [] + for part in content_parts: + if getattr(part, "type", "") == "output_text": + text += getattr(part, "text", "") or "" + return SimpleNamespace(content=text, tool_calls=tool_calls), text + + +# ── Public API ──────────────────────────────────────────────────────────────── + +def configure_azure_openai( + *, + endpoint: str | None = None, + api_version: str | None = None, + api_key: str | None = None, + auth_mode: str | None = None, + ad_scope: str | None = None, + managed_identity_client_id: str | None = None, + optimizer_endpoint: str | None = None, + optimizer_api_version: str | None = None, + optimizer_api_key: str | None = None, + optimizer_auth_mode: str | None = None, + optimizer_ad_scope: str | None = None, + optimizer_managed_identity_client_id: str | None = None, + target_endpoint: str | None = None, + target_api_version: str | None = None, + target_api_key: str | None = None, + target_auth_mode: str | None = None, + target_ad_scope: str | None = None, + target_managed_identity_client_id: str | None = None, +) -> None: + global ENDPOINT, API_VERSION, API_KEY, AUTH_MODE, AD_SCOPE, MANAGED_IDENTITY_CLIENT_ID + global OPTIMIZER_ENDPOINT, OPTIMIZER_API_VERSION, OPTIMIZER_API_KEY, OPTIMIZER_AUTH_MODE + global OPTIMIZER_AD_SCOPE, OPTIMIZER_MANAGED_IDENTITY_CLIENT_ID + global TARGET_ENDPOINT, TARGET_API_VERSION, TARGET_API_KEY, TARGET_AUTH_MODE + global TARGET_AD_SCOPE, TARGET_MANAGED_IDENTITY_CLIENT_ID + global _optimizer_client, _target_client + + def _clean(value: str | None, *, lower: bool = False) -> str | None: + if value is None: + return None + str_value = str(value).strip() + if not str_value: + return None + if lower: + str_value = str_value.lower() + return str_value + + def _set(global_name: str, value: str | None, env_key: str) -> None: + if value is None: + return + globals()[global_name] = value + os.environ[env_key] = value + + shared_endpoint = _clean(endpoint) + shared_api_version = _clean(api_version) + shared_api_key = _clean(api_key) + shared_auth_mode = _clean(auth_mode, lower=True) + shared_ad_scope = _clean(ad_scope) + shared_managed_identity_client_id = _clean(managed_identity_client_id) + + # Auto-configure for openai_compatible mode + if shared_auth_mode in {"openai_compatible", "compat", "openai"}: + if shared_api_version is None: + shared_api_version = _OPENAI_COMPATIBLE_API_VERSION + + _set("ENDPOINT", shared_endpoint, "AZURE_OPENAI_ENDPOINT") + _set("API_VERSION", shared_api_version, "AZURE_OPENAI_API_VERSION") + _set("API_KEY", shared_api_key, "AZURE_OPENAI_API_KEY") + _set("AUTH_MODE", shared_auth_mode, "AZURE_OPENAI_AUTH_MODE") + _set("AD_SCOPE", shared_ad_scope, "AZURE_OPENAI_AD_SCOPE") + _set( + "MANAGED_IDENTITY_CLIENT_ID", + shared_managed_identity_client_id, + "AZURE_OPENAI_MANAGED_IDENTITY_CLIENT_ID", + ) + + resolved_optimizer_endpoint = _clean(optimizer_endpoint) or shared_endpoint + resolved_optimizer_api_version = _clean(optimizer_api_version) or shared_api_version + resolved_optimizer_api_key = _clean(optimizer_api_key) or shared_api_key + resolved_optimizer_auth_mode = _clean(optimizer_auth_mode, lower=True) or shared_auth_mode + resolved_optimizer_ad_scope = _clean(optimizer_ad_scope) or shared_ad_scope + resolved_optimizer_mi = ( + _clean(optimizer_managed_identity_client_id) + or shared_managed_identity_client_id + ) + + # Auto-configure for openai_compatible mode + if resolved_optimizer_auth_mode in {"openai_compatible", "compat", "openai"}: + if resolved_optimizer_api_version is None: + resolved_optimizer_api_version = _OPENAI_COMPATIBLE_API_VERSION + + resolved_target_endpoint = _clean(target_endpoint) or shared_endpoint + resolved_target_api_version = _clean(target_api_version) or shared_api_version + resolved_target_api_key = _clean(target_api_key) or shared_api_key + resolved_target_auth_mode = _clean(target_auth_mode, lower=True) or shared_auth_mode + resolved_target_ad_scope = _clean(target_ad_scope) or shared_ad_scope + resolved_target_mi = ( + _clean(target_managed_identity_client_id) + or shared_managed_identity_client_id + ) + + # Auto-configure for openai_compatible mode + if resolved_target_auth_mode in {"openai_compatible", "compat", "openai"}: + if resolved_target_api_version is None: + resolved_target_api_version = _OPENAI_COMPATIBLE_API_VERSION + + _set("OPTIMIZER_ENDPOINT", resolved_optimizer_endpoint, "OPTIMIZER_AZURE_OPENAI_ENDPOINT") + _set( + "OPTIMIZER_API_VERSION", + resolved_optimizer_api_version, + "OPTIMIZER_AZURE_OPENAI_API_VERSION", + ) + _set("OPTIMIZER_API_KEY", resolved_optimizer_api_key, "OPTIMIZER_AZURE_OPENAI_API_KEY") + _set("OPTIMIZER_AUTH_MODE", resolved_optimizer_auth_mode, "OPTIMIZER_AZURE_OPENAI_AUTH_MODE") + _set("OPTIMIZER_AD_SCOPE", resolved_optimizer_ad_scope, "OPTIMIZER_AZURE_OPENAI_AD_SCOPE") + _set( + "OPTIMIZER_MANAGED_IDENTITY_CLIENT_ID", + resolved_optimizer_mi, + "OPTIMIZER_AZURE_OPENAI_MANAGED_IDENTITY_CLIENT_ID", + ) + _set("TARGET_ENDPOINT", resolved_target_endpoint, "TARGET_AZURE_OPENAI_ENDPOINT") + _set( + "TARGET_API_VERSION", + resolved_target_api_version, + "TARGET_AZURE_OPENAI_API_VERSION", + ) + _set("TARGET_API_KEY", resolved_target_api_key, "TARGET_AZURE_OPENAI_API_KEY") + _set("TARGET_AUTH_MODE", resolved_target_auth_mode, "TARGET_AZURE_OPENAI_AUTH_MODE") + _set("TARGET_AD_SCOPE", resolved_target_ad_scope, "TARGET_AZURE_OPENAI_AD_SCOPE") + _set( + "TARGET_MANAGED_IDENTITY_CLIENT_ID", + resolved_target_mi, + "TARGET_AZURE_OPENAI_MANAGED_IDENTITY_CLIENT_ID", + ) + + with _optimizer_lock: + _optimizer_client = None + with _target_lock: + _target_client = None + + +def chat_optimizer( + system: str, + user: str, + max_completion_tokens: int = 16384, + retries: int = 5, + stage: str = "optimizer", + reasoning_effort: str | None = None, + timeout: int | None = None, +) -> tuple[str, dict]: + """Call the optimizer model. Returns (response_text, usage_dict).""" + return _chat_impl( + get_optimizer_client(), OPTIMIZER_DEPLOYMENT, + system, user, max_completion_tokens, retries, stage, reasoning_effort, timeout, + ) + + +def chat_with_deployment( + deployment: str, + system: str, + user: str, + max_completion_tokens: int = 16384, + retries: int = 5, + stage: str = "custom", + reasoning_effort: str | None = None, + timeout: int | None = None, +) -> tuple[str, dict]: + """Call an arbitrary deployment using the shared Azure client.""" + return _chat_impl( + get_optimizer_client(), + deployment, + system, + user, + max_completion_tokens, + retries, + stage, + reasoning_effort, + timeout, + ) + + +def chat_target( + system: str, + user: str, + max_completion_tokens: int = 16384, + retries: int = 5, + stage: str = "target", + reasoning_effort: str | None = None, + timeout: int | None = None, +) -> tuple[str, dict]: + """Call the target model. Returns (response_text, usage_dict).""" + return _chat_impl( + get_target_client(), TARGET_DEPLOYMENT, + system, user, max_completion_tokens, retries, stage, reasoning_effort, timeout, + ) + + +def chat_optimizer_messages( + messages: list[dict[str, Any]], + max_completion_tokens: int = 16384, + retries: int = 5, + stage: str = "optimizer", + reasoning_effort: str | None = None, + *, + tools: list[dict[str, Any]] | None = None, + tool_choice: str | dict[str, Any] | None = None, + return_message: bool = False, + timeout: int | None = None, +) -> tuple[Any, dict]: + """Call the optimizer model with a pre-built chat message list.""" + return _chat_messages_impl( + get_optimizer_client(), + OPTIMIZER_DEPLOYMENT, + messages, + max_completion_tokens, + retries, + stage, + reasoning_effort, + tools=tools, + tool_choice=tool_choice, + return_message=return_message, + timeout=timeout, + ) + + +def chat_messages_with_deployment( + deployment: str, + messages: list[dict[str, Any]], + max_completion_tokens: int = 16384, + retries: int = 5, + stage: str = "custom", + reasoning_effort: str | None = None, + *, + tools: list[dict[str, Any]] | None = None, + tool_choice: str | dict[str, Any] | None = None, + return_message: bool = False, + timeout: int | None = None, +) -> tuple[Any, dict]: + """Call an arbitrary deployment with a pre-built chat message list.""" + return _chat_messages_impl( + get_optimizer_client(), + deployment, + messages, + max_completion_tokens, + retries, + stage, + reasoning_effort, + tools=tools, + tool_choice=tool_choice, + return_message=return_message, + timeout=timeout, + ) + + +def chat_target_messages( + messages: list[dict[str, Any]], + max_completion_tokens: int = 16384, + retries: int = 5, + stage: str = "target", + reasoning_effort: str | None = None, + *, + tools: list[dict[str, Any]] | None = None, + tool_choice: str | dict[str, Any] | None = None, + return_message: bool = False, + timeout: int | None = None, +) -> tuple[Any, dict]: + """Call the target model with a pre-built chat message list.""" + return _chat_messages_impl( + get_target_client(), + TARGET_DEPLOYMENT, + messages, + max_completion_tokens, + retries, + stage, + reasoning_effort, + tools=tools, + tool_choice=tool_choice, + return_message=return_message, + timeout=timeout, + ) + + +def get_token_summary() -> dict: + """Return per-stage and total token usage.""" + return tracker.summary() + + +def reset_token_tracker() -> None: + tracker.reset() + + +def set_target_deployment(deployment: str) -> None: + """Change target deployment at runtime.""" + global _target_client, TARGET_DEPLOYMENT + TARGET_DEPLOYMENT = deployment + os.environ["TARGET_DEPLOYMENT"] = deployment + os.environ["AZURE_OPENAI_DEPLOYMENT"] = deployment + with _target_lock: + _target_client = None + try: + import llm_client as _legacy + _legacy.DEPLOYMENT = deployment + _legacy._client = None + except Exception: + pass + + +def set_reasoning_effort(effort: str | None) -> None: + """Set reasoning effort for all LLM calls. None = off.""" + global REASONING_EFFORT + REASONING_EFFORT = effort if effort else None + + +def get_reasoning_effort() -> str | None: + """Return the process-wide reasoning effort for direct Azure client users.""" + return REASONING_EFFORT + + +def set_optimizer_deployment(deployment: str) -> None: + """Change optimizer deployment at runtime.""" + global _optimizer_client, OPTIMIZER_DEPLOYMENT + OPTIMIZER_DEPLOYMENT = deployment + os.environ["OPTIMIZER_DEPLOYMENT"] = deployment + with _optimizer_lock: + _optimizer_client = None diff --git a/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/model/backend_config.py b/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/model/backend_config.py new file mode 100644 index 00000000..f23725c5 --- /dev/null +++ b/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/model/backend_config.py @@ -0,0 +1,185 @@ +"""Runtime backend configuration for optimizer/target model calls.""" +from __future__ import annotations + +import os + +from skillopt.model.common import default_model_for_backend, normalize_backend_name + + +def _parse_bool(value: str | None, default: bool) -> bool: + if value is None: + return default + return str(value).strip().lower() in {"1", "true", "yes", "on"} + + +OPTIMIZER_BACKEND = normalize_backend_name(os.environ.get("OPTIMIZER_BACKEND", "openai_chat")) +TARGET_BACKEND = normalize_backend_name(os.environ.get("TARGET_BACKEND", "openai_chat")) + +CODEX_EXEC_PATH = os.environ.get("CODEX_EXEC_PATH", "codex") +CODEX_EXEC_SANDBOX = os.environ.get("CODEX_EXEC_SANDBOX", "workspace-write") +CODEX_EXEC_PROFILE = os.environ.get("CODEX_EXEC_PROFILE", "") +CODEX_EXEC_FULL_AUTO = _parse_bool(os.environ.get("CODEX_EXEC_FULL_AUTO"), True) +CODEX_EXEC_REASONING_EFFORT = os.environ.get("CODEX_EXEC_REASONING_EFFORT", "none") +CODEX_EXEC_USE_SDK = os.environ.get("CODEX_EXEC_USE_SDK", "auto") +CODEX_EXEC_NETWORK_ACCESS = _parse_bool(os.environ.get("CODEX_EXEC_NETWORK_ACCESS"), False) +CODEX_EXEC_WEB_SEARCH = _parse_bool(os.environ.get("CODEX_EXEC_WEB_SEARCH"), False) +CODEX_EXEC_APPROVAL_POLICY = os.environ.get("CODEX_EXEC_APPROVAL_POLICY", "never") +CLAUDE_CODE_EXEC_PATH = os.environ.get("CLAUDE_CODE_EXEC_PATH", "claude") +CLAUDE_CODE_EXEC_PROFILE = os.environ.get("CLAUDE_CODE_EXEC_PROFILE", "") +CLAUDE_CODE_EXEC_USE_SDK = os.environ.get("CLAUDE_CODE_EXEC_USE_SDK", "auto") +CLAUDE_CODE_EXEC_EFFORT = os.environ.get("CLAUDE_CODE_EXEC_EFFORT", "medium") + + +def _parse_int(value: str | None, default: int) -> int: + if value is None: + return default + try: + return int(str(value).strip()) + except ValueError: + return default + + +EXEC_EMPTY_RESPONSE_RETRIES = max(0, _parse_int(os.environ.get("EXEC_EMPTY_RESPONSE_RETRIES"), 1)) +CLAUDE_CODE_EXEC_MAX_THINKING_TOKENS = max( + 0, + _parse_int(os.environ.get("CLAUDE_CODE_EXEC_MAX_THINKING_TOKENS"), 16384), +) + + +def set_optimizer_backend(backend: str) -> None: + global OPTIMIZER_BACKEND + OPTIMIZER_BACKEND = normalize_backend_name(backend or "openai_chat") + if OPTIMIZER_BACKEND not in {"openai_chat", "claude_chat", "qwen_chat", "minimax_chat"}: + raise ValueError( + f"Unsupported optimizer backend: {OPTIMIZER_BACKEND!r}. " + "Supported values are 'openai_chat', 'claude_chat', 'qwen_chat', and 'minimax_chat'." + ) + os.environ["OPTIMIZER_BACKEND"] = OPTIMIZER_BACKEND + + +def get_optimizer_backend() -> str: + return OPTIMIZER_BACKEND + + +def set_target_backend(backend: str) -> None: + global TARGET_BACKEND + TARGET_BACKEND = normalize_backend_name(backend or "openai_chat") + if TARGET_BACKEND not in {"openai_chat", "claude_chat", "qwen_chat", "minimax_chat", "codex_exec", "claude_code_exec"}: + raise ValueError( + f"Unsupported target backend: {TARGET_BACKEND!r}. " + "Supported values are 'openai_chat', 'claude_chat', 'qwen_chat', 'minimax_chat', 'codex_exec', and 'claude_code_exec'." + ) + os.environ["TARGET_BACKEND"] = TARGET_BACKEND + + +def get_target_backend() -> str: + return TARGET_BACKEND + + +def is_target_exec_backend() -> bool: + return TARGET_BACKEND in {"codex_exec", "claude_code_exec"} + + +def is_optimizer_chat_backend() -> bool: + return OPTIMIZER_BACKEND in {"openai_chat", "claude_chat", "qwen_chat", "minimax_chat"} + + +def is_target_chat_backend() -> bool: + return TARGET_BACKEND in {"openai_chat", "claude_chat", "qwen_chat", "minimax_chat"} + + +def configure_codex_exec( + *, + path: str | None = None, + sandbox: str | None = None, + profile: str | None = None, + full_auto: bool | None = None, + reasoning_effort: str | None = None, + use_sdk: str | None = None, + network_access: bool | None = None, + web_search: bool | None = None, + approval_policy: str | None = None, +) -> None: + global CODEX_EXEC_PATH, CODEX_EXEC_SANDBOX, CODEX_EXEC_PROFILE, CODEX_EXEC_FULL_AUTO, CODEX_EXEC_REASONING_EFFORT, CODEX_EXEC_USE_SDK, CODEX_EXEC_NETWORK_ACCESS, CODEX_EXEC_WEB_SEARCH, CODEX_EXEC_APPROVAL_POLICY + if path is not None: + CODEX_EXEC_PATH = str(path).strip() or "codex" + os.environ["CODEX_EXEC_PATH"] = CODEX_EXEC_PATH + if sandbox is not None: + CODEX_EXEC_SANDBOX = str(sandbox).strip() or "workspace-write" + os.environ["CODEX_EXEC_SANDBOX"] = CODEX_EXEC_SANDBOX + if profile is not None: + CODEX_EXEC_PROFILE = str(profile).strip() + os.environ["CODEX_EXEC_PROFILE"] = CODEX_EXEC_PROFILE + if full_auto is not None: + CODEX_EXEC_FULL_AUTO = bool(full_auto) + os.environ["CODEX_EXEC_FULL_AUTO"] = "true" if CODEX_EXEC_FULL_AUTO else "false" + if reasoning_effort is not None: + CODEX_EXEC_REASONING_EFFORT = str(reasoning_effort).strip() or "none" + os.environ["CODEX_EXEC_REASONING_EFFORT"] = CODEX_EXEC_REASONING_EFFORT + if use_sdk is not None: + CODEX_EXEC_USE_SDK = str(use_sdk).strip().lower() or "auto" + os.environ["CODEX_EXEC_USE_SDK"] = CODEX_EXEC_USE_SDK + if network_access is not None: + CODEX_EXEC_NETWORK_ACCESS = bool(network_access) + os.environ["CODEX_EXEC_NETWORK_ACCESS"] = "true" if CODEX_EXEC_NETWORK_ACCESS else "false" + if web_search is not None: + CODEX_EXEC_WEB_SEARCH = bool(web_search) + os.environ["CODEX_EXEC_WEB_SEARCH"] = "true" if CODEX_EXEC_WEB_SEARCH else "false" + if approval_policy is not None: + CODEX_EXEC_APPROVAL_POLICY = str(approval_policy).strip() or "never" + os.environ["CODEX_EXEC_APPROVAL_POLICY"] = CODEX_EXEC_APPROVAL_POLICY + + +def get_codex_exec_config() -> dict[str, str | bool | int]: + return { + "path": CODEX_EXEC_PATH, + "sandbox": CODEX_EXEC_SANDBOX, + "profile": CODEX_EXEC_PROFILE, + "full_auto": CODEX_EXEC_FULL_AUTO, + "reasoning_effort": CODEX_EXEC_REASONING_EFFORT, + "use_sdk": CODEX_EXEC_USE_SDK, + "network_access": CODEX_EXEC_NETWORK_ACCESS, + "web_search": CODEX_EXEC_WEB_SEARCH, + "approval_policy": CODEX_EXEC_APPROVAL_POLICY, + "empty_response_retries": EXEC_EMPTY_RESPONSE_RETRIES, + } + + +def configure_claude_code_exec( + *, + path: str | None = None, + profile: str | None = None, + use_sdk: str | None = None, + effort: str | None = None, + max_thinking_tokens: int | str | None = None, +) -> None: + global CLAUDE_CODE_EXEC_PATH, CLAUDE_CODE_EXEC_PROFILE, CLAUDE_CODE_EXEC_USE_SDK, CLAUDE_CODE_EXEC_EFFORT, CLAUDE_CODE_EXEC_MAX_THINKING_TOKENS + if path is not None: + CLAUDE_CODE_EXEC_PATH = str(path).strip() or "claude" + os.environ["CLAUDE_CODE_EXEC_PATH"] = CLAUDE_CODE_EXEC_PATH + if profile is not None: + CLAUDE_CODE_EXEC_PROFILE = str(profile).strip() + os.environ["CLAUDE_CODE_EXEC_PROFILE"] = CLAUDE_CODE_EXEC_PROFILE + if use_sdk is not None: + CLAUDE_CODE_EXEC_USE_SDK = str(use_sdk).strip().lower() or "auto" + os.environ["CLAUDE_CODE_EXEC_USE_SDK"] = CLAUDE_CODE_EXEC_USE_SDK + if effort is not None: + CLAUDE_CODE_EXEC_EFFORT = str(effort).strip().lower() or "medium" + os.environ["CLAUDE_CODE_EXEC_EFFORT"] = CLAUDE_CODE_EXEC_EFFORT + if max_thinking_tokens is not None: + CLAUDE_CODE_EXEC_MAX_THINKING_TOKENS = max( + 0, + _parse_int(str(max_thinking_tokens), 16384), + ) + os.environ["CLAUDE_CODE_EXEC_MAX_THINKING_TOKENS"] = str(CLAUDE_CODE_EXEC_MAX_THINKING_TOKENS) + + +def get_claude_code_exec_config() -> dict[str, str | int]: + return { + "path": CLAUDE_CODE_EXEC_PATH, + "profile": CLAUDE_CODE_EXEC_PROFILE, + "use_sdk": CLAUDE_CODE_EXEC_USE_SDK, + "effort": CLAUDE_CODE_EXEC_EFFORT, + "max_thinking_tokens": CLAUDE_CODE_EXEC_MAX_THINKING_TOKENS, + "empty_response_retries": EXEC_EMPTY_RESPONSE_RETRIES, + } diff --git a/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/model/claude_backend.py b/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/model/claude_backend.py new file mode 100644 index 00000000..4b2900b8 --- /dev/null +++ b/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/model/claude_backend.py @@ -0,0 +1,365 @@ +"""Claude CLI chat backend for ReflACT.""" +from __future__ import annotations + +import base64 +import json +import mimetypes +import os +import shutil +import subprocess +import tempfile +import time +from typing import Any +from urllib.parse import unquote, urlparse + +from skillopt.model.common import CompatAssistantMessage, CompatToolCall, CompatToolFunction, default_model_for_backend, tracker + +CLAUDE_BIN = os.environ.get("CLAUDE_CLI_BIN", "claude") +CLAUDE_PERMISSION_MODE = os.environ.get("CLAUDE_PERMISSION_MODE", "dontAsk") +CLAUDE_SETTING_SOURCES = os.environ.get("CLAUDE_SETTING_SOURCES", "user,project") +CLAUDE_ALLOW_ATTACHMENT_READ = os.environ.get("CLAUDE_ALLOW_ATTACHMENT_READ", "1").strip().lower() not in {"0", "false", "no"} + +OPTIMIZER_DEPLOYMENT = os.environ.get("OPTIMIZER_DEPLOYMENT", "claude-sonnet-4-6") +TARGET_DEPLOYMENT = os.environ.get("TARGET_DEPLOYMENT", "claude-sonnet-4-6") +REASONING_EFFORT: str | None = None +_VALID_EFFORTS = {"low", "medium", "high", "xhigh", "max"} + + +def _parse_data_uri(url: str) -> tuple[bytes, str]: + header, data = url.split(",", 1) + mime = header[5:].split(";", 1)[0] or "image/png" + return base64.b64decode(data), mime + + +def _content_to_text(content: Any, attachments: list[dict[str, Any]], *, image_counter: int) -> tuple[str, int]: + if isinstance(content, str): + return content, image_counter + if not isinstance(content, list): + return str(content), image_counter + parts: list[str] = [] + for item in content: + if not isinstance(item, dict): + continue + item_type = item.get("type") + if item_type == "text": + parts.append(str(item.get("text", ""))) + continue + if item_type != "image_url": + continue + image_counter += 1 + label = f"[Attached image {image_counter}]" + parts.append(label) + image_url = item.get("image_url", {}) or {} + url = str(image_url.get("url", "") or "") + if not url: + continue + if url.startswith("data:") and ";base64," in url: + data, mime = _parse_data_uri(url) + attachments.append({"bytes": data, "mime": mime, "label": label}) + continue + if url.startswith("file://"): + parsed = urlparse(url) + path = unquote(parsed.path) + if path: + attachments.append({"path": path, "label": label}) + continue + if os.path.exists(url): + attachments.append({"path": url, "label": label}) + return "".join(parts), image_counter + + +def _simplify_tool_schemas(tools: list[dict[str, Any]] | None) -> list[dict[str, Any]]: + simplified: list[dict[str, Any]] = [] + for tool in tools or []: + function = tool.get("function", tool) + simplified.append({ + "name": function.get("name", ""), + "description": function.get("description", ""), + "parameters": function.get("parameters", {}), + }) + return simplified + + +def _build_prompt_from_messages(messages: list[dict[str, Any]], *, tools: list[dict[str, Any]] | None = None, tool_choice: str | dict[str, Any] | None = None, structured_output: bool = False) -> tuple[str, str, list[dict[str, Any]]]: + system_parts: list[str] = [] + history_parts: list[str] = [] + attachments: list[dict[str, Any]] = [] + image_counter = 0 + + def _history_line(label: str, body: str) -> str: + stripped = body.strip() + if not stripped: + return f"- {label}:" + indented = stripped.replace("\n", "\n ") + return f"- {label}: {indented}" + + for message in messages: + role = str(message.get("role", "user")) + text, image_counter = _content_to_text(message.get("content", ""), attachments, image_counter=image_counter) + if role == "system": + if text.strip(): + system_parts.append(text.strip()) + continue + if role == "assistant": + block = _history_line("Assistant", text) + tool_calls = message.get("tool_calls") or [] + if tool_calls: + simplified_calls = [] + for tool_call in tool_calls: + function = tool_call.get("function", {}) or {} + simplified_calls.append({ + "name": function.get("name", ""), + "arguments": function.get("arguments", "{}"), + }) + block += "\n Compatibility tool requests:\n" + json.dumps(simplified_calls, ensure_ascii=False, indent=2) + history_parts.append(block) + continue + if role == "tool": + tool_call_id = str(message.get("tool_call_id", "") or "") + history_parts.append(_history_line(f"Tool result (tool_call_id={tool_call_id})", text)) + continue + history_parts.append(_history_line(role.capitalize(), text)) + + prompt_parts: list[str] = [] + if tools: + simplified_tools = _simplify_tool_schemas(tools) + prompt_parts.append("Available compatibility tools:\n" + json.dumps(simplified_tools, ensure_ascii=False, indent=2)) + prompt_parts.append("Do not execute these compatibility tools yourself. If you need one, request it in `tool_calls`. Each `arguments` field must be a JSON string.") + if tool_choice == "required": + prompt_parts.append("Tool choice policy: you must request at least one compatibility tool.") + elif isinstance(tool_choice, dict) and tool_choice.get("type") == "function": + function = tool_choice.get("function", {}) or {} + prompt_parts.append(f"Tool choice policy: you must request the compatibility tool `{function.get('name', '')}`.") + history_text = "\n".join(part for part in history_parts if part).strip() + if history_text: + prompt_parts.append("History:\n" + history_text) + if structured_output: + prompt_parts.append("Return only JSON matching the provided schema.") + if tools: + prompt_parts.append("Set `content` to the assistant-visible reply. Set `tool_calls` to an empty array when no compatibility tool is needed.") + else: + prompt_parts.append("Answer the latest user request.") + return "\n\n".join(part for part in system_parts if part).strip(), "\n\n".join(prompt_parts), attachments + + +def _copy_attachments_to_temp(attachments: list[dict[str, Any]], temp_dir: str) -> list[dict[str, str]]: + copied: list[dict[str, str]] = [] + for index, attachment in enumerate(attachments, 1): + source_path = attachment.get("path") + if source_path: + source_path = str(source_path) + source_suffix = os.path.splitext(source_path)[1] + target_path = os.path.join(temp_dir, f"image_{index}{source_suffix or '.bin'}") + shutil.copyfile(source_path, target_path) + copied.append({"path": target_path, "label": str(attachment.get("label", ""))}) + continue + mime = str(attachment.get("mime", "image/png")) + suffix = mimetypes.guess_extension(mime) or ".png" + target_path = os.path.join(temp_dir, f"image_{index}{suffix}") + with open(target_path, "wb") as f: + f.write(attachment.get("bytes", b"") or b"") + copied.append({"path": target_path, "label": str(attachment.get("label", ""))}) + return copied + + +def _append_attachment_instructions(prompt: str, copied_attachments: list[dict[str, str]]) -> str: + if not copied_attachments or not CLAUDE_ALLOW_ATTACHMENT_READ: + return prompt + lines = [ + "Attached image files:", + *[f"- {item['label'] or f'Attached image {index}'}: {item['path']}" for index, item in enumerate(copied_attachments, 1)], + "If you need to inspect an attached image, you may use the built-in `Read` tool on those listed paths only. Do not use built-in tools for any other purpose.", + ] + return prompt.rstrip() + "\n\n" + "\n".join(lines) + + +def _usage_from_result(result_event: dict[str, Any] | None) -> dict[str, int]: + usage = (result_event or {}).get("usage", {}) or {} + input_tokens = int(usage.get("input_tokens", 0) or 0) + output_tokens = int(usage.get("output_tokens", 0) or 0) + return { + "prompt_tokens": input_tokens, + "completion_tokens": output_tokens, + "total_tokens": input_tokens + output_tokens, + } + + +def _extract_result(event_stream: list[dict[str, Any]]) -> tuple[str, dict[str, Any] | None]: + result_event = None + for event in reversed(event_stream): + if event.get("type") == "result": + result_event = event + break + if result_event is None: + raise RuntimeError("Claude backend did not return a result event.") + content = result_event.get("result") or result_event.get("content") or "" + return str(content), result_event + + +def _check_claude_error(stderr_text: str, model: str) -> None: + lowered = stderr_text.lower() + if "invalid api key" in lowered or "authentication" in lowered or "login" in lowered: + raise RuntimeError("Claude CLI is not logged in. Run `claude auth login` (or start `claude` and use `/login`) first.") + if "unknown model" in lowered or "not available" in lowered or "invalid model" in lowered: + default_model = default_model_for_backend("claude") + raise RuntimeError(f"Claude backend tried to use model {model!r}, but your current Claude CLI/account rejected it. Try an available Claude model such as {default_model!r}.") + + +def _normalize_reasoning_effort(effort: str | None) -> str | None: + normalized = str(effort or "").strip().lower() + if not normalized or normalized == "off": + return None + if normalized in _VALID_EFFORTS: + return normalized + return None + + +def _assistant_message_schema() -> dict[str, Any]: + return { + "type": "object", + "properties": { + "content": {"type": "string"}, + "tool_calls": { + "type": "array", + "items": { + "type": "object", + "properties": { + "name": {"type": "string"}, + "arguments": {"type": "string"}, + }, + "required": ["name", "arguments"], + "additionalProperties": False, + }, + }, + }, + "required": ["content", "tool_calls"], + "additionalProperties": False, + } + + +def _assistant_message_schema_wrapper() -> str: + return json.dumps(_assistant_message_schema(), ensure_ascii=False) + + +def _run_claude_print(*, system: str, prompt: str, model: str, tools: list[dict[str, Any]] | None, tool_choice: str | dict[str, Any] | None, return_message: bool, timeout: int | None, attachments: list[dict[str, Any]] | None = None) -> tuple[str, dict[str, Any], dict[str, int]]: + effort = _normalize_reasoning_effort(REASONING_EFFORT) + with tempfile.TemporaryDirectory(prefix="skillopt_claude_") as temp_dir: + copied_attachments = _copy_attachments_to_temp(attachments or [], temp_dir) + prompt_for_cli = _append_attachment_instructions(prompt, copied_attachments) + # NOTE: this is the `claude_chat` backend path; it is NOT used by the + # xskill experiment (single + claude_code_exec). The real isolation fix + # lives in codex_harness.py::_run_claude_code_cli_exec (gated on + # XSKILL_CLAUDE_HOME). The earlier dead-code XSKILL_EXP_DIR cwd hook here + # was reverted to avoid a misleading half-isolation path. + claude_cwd = temp_dir + cmd = [CLAUDE_BIN, "-p", "--output-format", "json", "--permission-mode", CLAUDE_PERMISSION_MODE, "--add-dir", temp_dir] + if model: + cmd.extend(["--model", model]) + if CLAUDE_SETTING_SOURCES: + cmd.extend(["--setting-sources", CLAUDE_SETTING_SOURCES]) + if system: + cmd.extend(["--append-system-prompt", system]) + if effort: + cmd.extend(["--effort", effort]) + structured_output = bool(return_message) + if structured_output: + cmd.extend(["--schema", _assistant_message_schema_wrapper()]) + proc = subprocess.run(cmd + [prompt_for_cli], capture_output=True, text=True, timeout=timeout or 300, cwd=claude_cwd) + stderr_text = (proc.stderr or "").strip() + if proc.returncode != 0: + _check_claude_error(stderr_text, model) + raise RuntimeError(stderr_text or f"Claude CLI exited with code {proc.returncode}") + stream = [] + for raw_line in (proc.stdout or "").splitlines(): + raw_line = raw_line.strip() + if not raw_line: + continue + try: + stream.append(json.loads(raw_line)) + except json.JSONDecodeError: + continue + raw_text, result_event = _extract_result(stream) + usage_info = _usage_from_result(result_event) + return raw_text, result_event or {}, usage_info + + +def _compat_message_from_payload(payload: Any) -> CompatAssistantMessage: + if not isinstance(payload, dict): + return CompatAssistantMessage(content=str(payload or ""), tool_calls=[]) + content = str(payload.get("content", "") or "") + tool_calls: list[CompatToolCall] = [] + for index, tool_call in enumerate(payload.get("tool_calls", []) or [], start=1): + name = str(tool_call.get("name", "") or "") + arguments = str(tool_call.get("arguments", "{}") or "{}") + tool_calls.append(CompatToolCall(id=f"claude_tool_{index}", function=CompatToolFunction(name=name, arguments=arguments))) + return CompatAssistantMessage(content=content, tool_calls=tool_calls) + + +def _call_messages(messages: list[dict[str, Any]], max_completion_tokens: int, retries: int, stage: str, *, tools: list[dict[str, Any]] | None = None, tool_choice: str | dict[str, Any] | None = None, return_message: bool = False, deployment: str | None = None, timeout: int | None = None) -> tuple[Any, dict[str, int]]: + del max_completion_tokens + system, prompt, attachments = _build_prompt_from_messages(messages, tools=tools, tool_choice=tool_choice, structured_output=return_message) + model = deployment or TARGET_DEPLOYMENT + last_err = None + for attempt in range(retries): + try: + raw_text, payload, usage_info = _run_claude_print(system=system, prompt=prompt, model=model, tools=tools, tool_choice=tool_choice, return_message=return_message, timeout=timeout, attachments=attachments) + tracker.record(stage, usage_info["prompt_tokens"], usage_info["completion_tokens"]) + if return_message: + return _compat_message_from_payload(payload.get("result", payload)), usage_info + return raw_text, usage_info + except Exception as e: # noqa: BLE001 + last_err = e + time.sleep(min(2 ** attempt, 15)) + raise RuntimeError(f"Claude backend failed after {retries} retries: {last_err}") + + +def chat_optimizer(system: str, user: str, max_completion_tokens: int = 16384, retries: int = 5, stage: str = "optimizer", timeout: int | None = None) -> tuple[str, dict[str, int]]: + messages = [{"role": "system", "content": system}, {"role": "user", "content": user}] + return _call_messages(messages, max_completion_tokens, retries, stage, deployment=OPTIMIZER_DEPLOYMENT, timeout=timeout) + + +def chat_target(system: str, user: str, max_completion_tokens: int = 16384, retries: int = 5, stage: str = "target", timeout: int | None = None) -> tuple[str, dict[str, int]]: + messages = [{"role": "system", "content": system}, {"role": "user", "content": user}] + return _call_messages(messages, max_completion_tokens, retries, stage, deployment=TARGET_DEPLOYMENT, timeout=timeout) + + +def chat_with_deployment(deployment: str, system: str, user: str, max_completion_tokens: int = 16384, retries: int = 5, stage: str = "custom", timeout: int | None = None) -> tuple[str, dict[str, int]]: + messages = [{"role": "system", "content": system}, {"role": "user", "content": user}] + return _call_messages(messages, max_completion_tokens, retries, stage, deployment=deployment, timeout=timeout) + + +def chat_optimizer_messages(messages: list[dict[str, Any]], max_completion_tokens: int = 16384, retries: int = 5, stage: str = "optimizer", *, tools: list[dict[str, Any]] | None = None, tool_choice: str | dict[str, Any] | None = None, return_message: bool = False, timeout: int | None = None) -> tuple[Any, dict[str, int]]: + return _call_messages(messages, max_completion_tokens, retries, stage, tools=tools, tool_choice=tool_choice, return_message=return_message, deployment=OPTIMIZER_DEPLOYMENT, timeout=timeout) + + +def chat_target_messages(messages: list[dict[str, Any]], max_completion_tokens: int = 16384, retries: int = 5, stage: str = "target", *, tools: list[dict[str, Any]] | None = None, tool_choice: str | dict[str, Any] | None = None, return_message: bool = False, timeout: int | None = None) -> tuple[Any, dict[str, int]]: + return _call_messages(messages, max_completion_tokens, retries, stage, tools=tools, tool_choice=tool_choice, return_message=return_message, deployment=TARGET_DEPLOYMENT, timeout=timeout) + + +def chat_messages_with_deployment(deployment: str, messages: list[dict[str, Any]], max_completion_tokens: int = 16384, retries: int = 5, stage: str = "custom", *, tools: list[dict[str, Any]] | None = None, tool_choice: str | dict[str, Any] | None = None, return_message: bool = False, timeout: int | None = None) -> tuple[Any, dict[str, int]]: + return _call_messages(messages, max_completion_tokens, retries, stage, tools=tools, tool_choice=tool_choice, return_message=return_message, deployment=deployment, timeout=timeout) + + +def get_token_summary() -> dict[str, dict[str, int]]: + return tracker.summary() + + +def reset_token_tracker() -> None: + tracker.reset() + + +def set_reasoning_effort(effort: str | None) -> None: + global REASONING_EFFORT + REASONING_EFFORT = effort if effort else None + + +def set_target_deployment(deployment: str) -> None: + global TARGET_DEPLOYMENT + TARGET_DEPLOYMENT = deployment or default_model_for_backend("claude") + os.environ["TARGET_DEPLOYMENT"] = TARGET_DEPLOYMENT + + +def set_optimizer_deployment(deployment: str) -> None: + global OPTIMIZER_DEPLOYMENT + OPTIMIZER_DEPLOYMENT = deployment or default_model_for_backend("claude") + os.environ["OPTIMIZER_DEPLOYMENT"] = OPTIMIZER_DEPLOYMENT diff --git a/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/model/claude_backend.py.orig b/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/model/claude_backend.py.orig new file mode 100644 index 00000000..04a17a30 --- /dev/null +++ b/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/model/claude_backend.py.orig @@ -0,0 +1,359 @@ +"""Claude CLI chat backend for ReflACT.""" +from __future__ import annotations + +import base64 +import json +import mimetypes +import os +import shutil +import subprocess +import tempfile +import time +from typing import Any +from urllib.parse import unquote, urlparse + +from skillopt.model.common import CompatAssistantMessage, CompatToolCall, CompatToolFunction, default_model_for_backend, tracker + +CLAUDE_BIN = os.environ.get("CLAUDE_CLI_BIN", "claude") +CLAUDE_PERMISSION_MODE = os.environ.get("CLAUDE_PERMISSION_MODE", "dontAsk") +CLAUDE_SETTING_SOURCES = os.environ.get("CLAUDE_SETTING_SOURCES", "user,project") +CLAUDE_ALLOW_ATTACHMENT_READ = os.environ.get("CLAUDE_ALLOW_ATTACHMENT_READ", "1").strip().lower() not in {"0", "false", "no"} + +OPTIMIZER_DEPLOYMENT = os.environ.get("OPTIMIZER_DEPLOYMENT", "claude-sonnet-4-6") +TARGET_DEPLOYMENT = os.environ.get("TARGET_DEPLOYMENT", "claude-sonnet-4-6") +REASONING_EFFORT: str | None = None +_VALID_EFFORTS = {"low", "medium", "high", "xhigh", "max"} + + +def _parse_data_uri(url: str) -> tuple[bytes, str]: + header, data = url.split(",", 1) + mime = header[5:].split(";", 1)[0] or "image/png" + return base64.b64decode(data), mime + + +def _content_to_text(content: Any, attachments: list[dict[str, Any]], *, image_counter: int) -> tuple[str, int]: + if isinstance(content, str): + return content, image_counter + if not isinstance(content, list): + return str(content), image_counter + parts: list[str] = [] + for item in content: + if not isinstance(item, dict): + continue + item_type = item.get("type") + if item_type == "text": + parts.append(str(item.get("text", ""))) + continue + if item_type != "image_url": + continue + image_counter += 1 + label = f"[Attached image {image_counter}]" + parts.append(label) + image_url = item.get("image_url", {}) or {} + url = str(image_url.get("url", "") or "") + if not url: + continue + if url.startswith("data:") and ";base64," in url: + data, mime = _parse_data_uri(url) + attachments.append({"bytes": data, "mime": mime, "label": label}) + continue + if url.startswith("file://"): + parsed = urlparse(url) + path = unquote(parsed.path) + if path: + attachments.append({"path": path, "label": label}) + continue + if os.path.exists(url): + attachments.append({"path": url, "label": label}) + return "".join(parts), image_counter + + +def _simplify_tool_schemas(tools: list[dict[str, Any]] | None) -> list[dict[str, Any]]: + simplified: list[dict[str, Any]] = [] + for tool in tools or []: + function = tool.get("function", tool) + simplified.append({ + "name": function.get("name", ""), + "description": function.get("description", ""), + "parameters": function.get("parameters", {}), + }) + return simplified + + +def _build_prompt_from_messages(messages: list[dict[str, Any]], *, tools: list[dict[str, Any]] | None = None, tool_choice: str | dict[str, Any] | None = None, structured_output: bool = False) -> tuple[str, str, list[dict[str, Any]]]: + system_parts: list[str] = [] + history_parts: list[str] = [] + attachments: list[dict[str, Any]] = [] + image_counter = 0 + + def _history_line(label: str, body: str) -> str: + stripped = body.strip() + if not stripped: + return f"- {label}:" + indented = stripped.replace("\n", "\n ") + return f"- {label}: {indented}" + + for message in messages: + role = str(message.get("role", "user")) + text, image_counter = _content_to_text(message.get("content", ""), attachments, image_counter=image_counter) + if role == "system": + if text.strip(): + system_parts.append(text.strip()) + continue + if role == "assistant": + block = _history_line("Assistant", text) + tool_calls = message.get("tool_calls") or [] + if tool_calls: + simplified_calls = [] + for tool_call in tool_calls: + function = tool_call.get("function", {}) or {} + simplified_calls.append({ + "name": function.get("name", ""), + "arguments": function.get("arguments", "{}"), + }) + block += "\n Compatibility tool requests:\n" + json.dumps(simplified_calls, ensure_ascii=False, indent=2) + history_parts.append(block) + continue + if role == "tool": + tool_call_id = str(message.get("tool_call_id", "") or "") + history_parts.append(_history_line(f"Tool result (tool_call_id={tool_call_id})", text)) + continue + history_parts.append(_history_line(role.capitalize(), text)) + + prompt_parts: list[str] = [] + if tools: + simplified_tools = _simplify_tool_schemas(tools) + prompt_parts.append("Available compatibility tools:\n" + json.dumps(simplified_tools, ensure_ascii=False, indent=2)) + prompt_parts.append("Do not execute these compatibility tools yourself. If you need one, request it in `tool_calls`. Each `arguments` field must be a JSON string.") + if tool_choice == "required": + prompt_parts.append("Tool choice policy: you must request at least one compatibility tool.") + elif isinstance(tool_choice, dict) and tool_choice.get("type") == "function": + function = tool_choice.get("function", {}) or {} + prompt_parts.append(f"Tool choice policy: you must request the compatibility tool `{function.get('name', '')}`.") + history_text = "\n".join(part for part in history_parts if part).strip() + if history_text: + prompt_parts.append("History:\n" + history_text) + if structured_output: + prompt_parts.append("Return only JSON matching the provided schema.") + if tools: + prompt_parts.append("Set `content` to the assistant-visible reply. Set `tool_calls` to an empty array when no compatibility tool is needed.") + else: + prompt_parts.append("Answer the latest user request.") + return "\n\n".join(part for part in system_parts if part).strip(), "\n\n".join(prompt_parts), attachments + + +def _copy_attachments_to_temp(attachments: list[dict[str, Any]], temp_dir: str) -> list[dict[str, str]]: + copied: list[dict[str, str]] = [] + for index, attachment in enumerate(attachments, 1): + source_path = attachment.get("path") + if source_path: + source_path = str(source_path) + source_suffix = os.path.splitext(source_path)[1] + target_path = os.path.join(temp_dir, f"image_{index}{source_suffix or '.bin'}") + shutil.copyfile(source_path, target_path) + copied.append({"path": target_path, "label": str(attachment.get("label", ""))}) + continue + mime = str(attachment.get("mime", "image/png")) + suffix = mimetypes.guess_extension(mime) or ".png" + target_path = os.path.join(temp_dir, f"image_{index}{suffix}") + with open(target_path, "wb") as f: + f.write(attachment.get("bytes", b"") or b"") + copied.append({"path": target_path, "label": str(attachment.get("label", ""))}) + return copied + + +def _append_attachment_instructions(prompt: str, copied_attachments: list[dict[str, str]]) -> str: + if not copied_attachments or not CLAUDE_ALLOW_ATTACHMENT_READ: + return prompt + lines = [ + "Attached image files:", + *[f"- {item['label'] or f'Attached image {index}'}: {item['path']}" for index, item in enumerate(copied_attachments, 1)], + "If you need to inspect an attached image, you may use the built-in `Read` tool on those listed paths only. Do not use built-in tools for any other purpose.", + ] + return prompt.rstrip() + "\n\n" + "\n".join(lines) + + +def _usage_from_result(result_event: dict[str, Any] | None) -> dict[str, int]: + usage = (result_event or {}).get("usage", {}) or {} + input_tokens = int(usage.get("input_tokens", 0) or 0) + output_tokens = int(usage.get("output_tokens", 0) or 0) + return { + "prompt_tokens": input_tokens, + "completion_tokens": output_tokens, + "total_tokens": input_tokens + output_tokens, + } + + +def _extract_result(event_stream: list[dict[str, Any]]) -> tuple[str, dict[str, Any] | None]: + result_event = None + for event in reversed(event_stream): + if event.get("type") == "result": + result_event = event + break + if result_event is None: + raise RuntimeError("Claude backend did not return a result event.") + content = result_event.get("result") or result_event.get("content") or "" + return str(content), result_event + + +def _check_claude_error(stderr_text: str, model: str) -> None: + lowered = stderr_text.lower() + if "invalid api key" in lowered or "authentication" in lowered or "login" in lowered: + raise RuntimeError("Claude CLI is not logged in. Run `claude auth login` (or start `claude` and use `/login`) first.") + if "unknown model" in lowered or "not available" in lowered or "invalid model" in lowered: + default_model = default_model_for_backend("claude") + raise RuntimeError(f"Claude backend tried to use model {model!r}, but your current Claude CLI/account rejected it. Try an available Claude model such as {default_model!r}.") + + +def _normalize_reasoning_effort(effort: str | None) -> str | None: + normalized = str(effort or "").strip().lower() + if not normalized or normalized == "off": + return None + if normalized in _VALID_EFFORTS: + return normalized + return None + + +def _assistant_message_schema() -> dict[str, Any]: + return { + "type": "object", + "properties": { + "content": {"type": "string"}, + "tool_calls": { + "type": "array", + "items": { + "type": "object", + "properties": { + "name": {"type": "string"}, + "arguments": {"type": "string"}, + }, + "required": ["name", "arguments"], + "additionalProperties": False, + }, + }, + }, + "required": ["content", "tool_calls"], + "additionalProperties": False, + } + + +def _assistant_message_schema_wrapper() -> str: + return json.dumps(_assistant_message_schema(), ensure_ascii=False) + + +def _run_claude_print(*, system: str, prompt: str, model: str, tools: list[dict[str, Any]] | None, tool_choice: str | dict[str, Any] | None, return_message: bool, timeout: int | None, attachments: list[dict[str, Any]] | None = None) -> tuple[str, dict[str, Any], dict[str, int]]: + effort = _normalize_reasoning_effort(REASONING_EFFORT) + with tempfile.TemporaryDirectory(prefix="skillopt_claude_") as temp_dir: + copied_attachments = _copy_attachments_to_temp(attachments or [], temp_dir) + prompt_for_cli = _append_attachment_instructions(prompt, copied_attachments) + cmd = [CLAUDE_BIN, "-p", "--output-format", "json", "--permission-mode", CLAUDE_PERMISSION_MODE, "--add-dir", temp_dir] + if model: + cmd.extend(["--model", model]) + if CLAUDE_SETTING_SOURCES: + cmd.extend(["--setting-sources", CLAUDE_SETTING_SOURCES]) + if system: + cmd.extend(["--append-system-prompt", system]) + if effort: + cmd.extend(["--effort", effort]) + structured_output = bool(return_message) + if structured_output: + cmd.extend(["--schema", _assistant_message_schema_wrapper()]) + proc = subprocess.run(cmd + [prompt_for_cli], capture_output=True, text=True, timeout=timeout or 300, cwd=temp_dir) + stderr_text = (proc.stderr or "").strip() + if proc.returncode != 0: + _check_claude_error(stderr_text, model) + raise RuntimeError(stderr_text or f"Claude CLI exited with code {proc.returncode}") + stream = [] + for raw_line in (proc.stdout or "").splitlines(): + raw_line = raw_line.strip() + if not raw_line: + continue + try: + stream.append(json.loads(raw_line)) + except json.JSONDecodeError: + continue + raw_text, result_event = _extract_result(stream) + usage_info = _usage_from_result(result_event) + return raw_text, result_event or {}, usage_info + + +def _compat_message_from_payload(payload: Any) -> CompatAssistantMessage: + if not isinstance(payload, dict): + return CompatAssistantMessage(content=str(payload or ""), tool_calls=[]) + content = str(payload.get("content", "") or "") + tool_calls: list[CompatToolCall] = [] + for index, tool_call in enumerate(payload.get("tool_calls", []) or [], start=1): + name = str(tool_call.get("name", "") or "") + arguments = str(tool_call.get("arguments", "{}") or "{}") + tool_calls.append(CompatToolCall(id=f"claude_tool_{index}", function=CompatToolFunction(name=name, arguments=arguments))) + return CompatAssistantMessage(content=content, tool_calls=tool_calls) + + +def _call_messages(messages: list[dict[str, Any]], max_completion_tokens: int, retries: int, stage: str, *, tools: list[dict[str, Any]] | None = None, tool_choice: str | dict[str, Any] | None = None, return_message: bool = False, deployment: str | None = None, timeout: int | None = None) -> tuple[Any, dict[str, int]]: + del max_completion_tokens + system, prompt, attachments = _build_prompt_from_messages(messages, tools=tools, tool_choice=tool_choice, structured_output=return_message) + model = deployment or TARGET_DEPLOYMENT + last_err = None + for attempt in range(retries): + try: + raw_text, payload, usage_info = _run_claude_print(system=system, prompt=prompt, model=model, tools=tools, tool_choice=tool_choice, return_message=return_message, timeout=timeout, attachments=attachments) + tracker.record(stage, usage_info["prompt_tokens"], usage_info["completion_tokens"]) + if return_message: + return _compat_message_from_payload(payload.get("result", payload)), usage_info + return raw_text, usage_info + except Exception as e: # noqa: BLE001 + last_err = e + time.sleep(min(2 ** attempt, 15)) + raise RuntimeError(f"Claude backend failed after {retries} retries: {last_err}") + + +def chat_optimizer(system: str, user: str, max_completion_tokens: int = 16384, retries: int = 5, stage: str = "optimizer", timeout: int | None = None) -> tuple[str, dict[str, int]]: + messages = [{"role": "system", "content": system}, {"role": "user", "content": user}] + return _call_messages(messages, max_completion_tokens, retries, stage, deployment=OPTIMIZER_DEPLOYMENT, timeout=timeout) + + +def chat_target(system: str, user: str, max_completion_tokens: int = 16384, retries: int = 5, stage: str = "target", timeout: int | None = None) -> tuple[str, dict[str, int]]: + messages = [{"role": "system", "content": system}, {"role": "user", "content": user}] + return _call_messages(messages, max_completion_tokens, retries, stage, deployment=TARGET_DEPLOYMENT, timeout=timeout) + + +def chat_with_deployment(deployment: str, system: str, user: str, max_completion_tokens: int = 16384, retries: int = 5, stage: str = "custom", timeout: int | None = None) -> tuple[str, dict[str, int]]: + messages = [{"role": "system", "content": system}, {"role": "user", "content": user}] + return _call_messages(messages, max_completion_tokens, retries, stage, deployment=deployment, timeout=timeout) + + +def chat_optimizer_messages(messages: list[dict[str, Any]], max_completion_tokens: int = 16384, retries: int = 5, stage: str = "optimizer", *, tools: list[dict[str, Any]] | None = None, tool_choice: str | dict[str, Any] | None = None, return_message: bool = False, timeout: int | None = None) -> tuple[Any, dict[str, int]]: + return _call_messages(messages, max_completion_tokens, retries, stage, tools=tools, tool_choice=tool_choice, return_message=return_message, deployment=OPTIMIZER_DEPLOYMENT, timeout=timeout) + + +def chat_target_messages(messages: list[dict[str, Any]], max_completion_tokens: int = 16384, retries: int = 5, stage: str = "target", *, tools: list[dict[str, Any]] | None = None, tool_choice: str | dict[str, Any] | None = None, return_message: bool = False, timeout: int | None = None) -> tuple[Any, dict[str, int]]: + return _call_messages(messages, max_completion_tokens, retries, stage, tools=tools, tool_choice=tool_choice, return_message=return_message, deployment=TARGET_DEPLOYMENT, timeout=timeout) + + +def chat_messages_with_deployment(deployment: str, messages: list[dict[str, Any]], max_completion_tokens: int = 16384, retries: int = 5, stage: str = "custom", *, tools: list[dict[str, Any]] | None = None, tool_choice: str | dict[str, Any] | None = None, return_message: bool = False, timeout: int | None = None) -> tuple[Any, dict[str, int]]: + return _call_messages(messages, max_completion_tokens, retries, stage, tools=tools, tool_choice=tool_choice, return_message=return_message, deployment=deployment, timeout=timeout) + + +def get_token_summary() -> dict[str, dict[str, int]]: + return tracker.summary() + + +def reset_token_tracker() -> None: + tracker.reset() + + +def set_reasoning_effort(effort: str | None) -> None: + global REASONING_EFFORT + REASONING_EFFORT = effort if effort else None + + +def set_target_deployment(deployment: str) -> None: + global TARGET_DEPLOYMENT + TARGET_DEPLOYMENT = deployment or default_model_for_backend("claude") + os.environ["TARGET_DEPLOYMENT"] = TARGET_DEPLOYMENT + + +def set_optimizer_deployment(deployment: str) -> None: + global OPTIMIZER_DEPLOYMENT + OPTIMIZER_DEPLOYMENT = deployment or default_model_for_backend("claude") + os.environ["OPTIMIZER_DEPLOYMENT"] = OPTIMIZER_DEPLOYMENT diff --git a/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/model/codex_backend.py b/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/model/codex_backend.py new file mode 100644 index 00000000..d9ab6159 --- /dev/null +++ b/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/model/codex_backend.py @@ -0,0 +1,664 @@ +"""Codex CLI backend for ReflACT.""" +from __future__ import annotations + +import base64 +import json +import mimetypes +import os +import subprocess +import tempfile +import time +import uuid +from typing import Any +from urllib.parse import unquote, urlparse + +from skillopt.model.common import ( + CompatAssistantMessage, + CompatToolCall, + CompatToolFunction, + tracker, +) + + +CODEX_BIN = os.environ.get("CODEX_CLI_BIN", "codex") +CODEX_PROFILE = os.environ.get("CODEX_PROFILE", "review") +CODEX_SANDBOX_MODE = os.environ.get("CODEX_SANDBOX_MODE", "read-only") + +OPTIMIZER_DEPLOYMENT = os.environ.get("OPTIMIZER_DEPLOYMENT", "gpt-4o") +TARGET_DEPLOYMENT = os.environ.get("TARGET_DEPLOYMENT", "gpt-4o") + +REASONING_EFFORT: str | None = None + + +def _default_working_directory() -> str: + return os.environ.get("CODEX_WORKING_DIRECTORY", os.getcwd()) + + +def _parse_data_uri(url: str) -> tuple[bytes, str]: + header, data = url.split(",", 1) + mime = header[5:].split(";", 1)[0] or "image/png" + return base64.b64decode(data), mime + + +def _content_to_text( + content: Any, + attachments: list[dict[str, Any]], + *, + image_counter: int, +) -> tuple[str, int]: + if isinstance(content, str): + return content, image_counter + + if not isinstance(content, list): + return str(content), image_counter + + parts: list[str] = [] + for item in content: + if not isinstance(item, dict): + continue + item_type = item.get("type") + if item_type == "text": + parts.append(str(item.get("text", ""))) + continue + if item_type != "image_url": + continue + + image_counter += 1 + label = f"[Attached image {image_counter}]" + parts.append(label) + + image_url = item.get("image_url", {}) or {} + url = str(image_url.get("url", "") or "") + if not url: + continue + if url.startswith("data:") and ";base64," in url: + data, mime = _parse_data_uri(url) + attachments.append({"bytes": data, "mime": mime}) + continue + if url.startswith("file://"): + parsed = urlparse(url) + path = unquote(parsed.path) + if path: + attachments.append({"path": path}) + continue + if os.path.exists(url): + attachments.append({"path": url}) + + return "".join(parts), image_counter + + +def _simplify_tool_schemas(tools: list[dict[str, Any]] | None) -> list[dict[str, Any]]: + simplified: list[dict[str, Any]] = [] + for tool in tools or []: + function = tool.get("function", tool) + simplified.append( + { + "name": function.get("name", ""), + "description": function.get("description", ""), + "parameters": function.get("parameters", {}), + } + ) + return simplified + + +def _build_prompt_from_messages( + messages: list[dict[str, Any]], + *, + tools: list[dict[str, Any]] | None = None, + tool_choice: str | dict[str, Any] | None = None, + structured_output: bool = False, +) -> tuple[str, list[dict[str, Any]]]: + system_parts: list[str] = [] + history_parts: list[str] = [] + attachments: list[dict[str, Any]] = [] + image_counter = 0 + + def _history_line(label: str, body: str) -> str: + stripped = body.strip() + if not stripped: + return f"- {label}:" + indented = stripped.replace("\n", "\n ") + return f"- {label}: {indented}" + + for message in messages: + role = str(message.get("role", "user")) + text, image_counter = _content_to_text( + message.get("content", ""), + attachments, + image_counter=image_counter, + ) + + if role == "system": + if text.strip(): + system_parts.append(text.strip()) + continue + + if role == "assistant": + block = _history_line("Assistant", text) + tool_calls = message.get("tool_calls") or [] + if tool_calls: + simplified_calls = [] + for tool_call in tool_calls: + function = tool_call.get("function", {}) or {} + simplified_calls.append( + { + "name": function.get("name", ""), + "arguments": function.get("arguments", "{}"), + } + ) + block += ( + "\n Compatibility tool requests:\n" + + json.dumps(simplified_calls, ensure_ascii=False, indent=2) + ) + history_parts.append(block) + continue + + if role == "tool": + tool_call_id = str(message.get("tool_call_id", "") or "") + label = f"Tool result (tool_call_id={tool_call_id})" + history_parts.append(_history_line(label, text)) + continue + + history_parts.append(_history_line(role.capitalize(), text)) + + prompt_parts: list[str] = [] + + system_text = "\n\n".join(part for part in system_parts if part).strip() + if system_text: + prompt_parts.append(system_text) + + if tools: + simplified_tools = _simplify_tool_schemas(tools) + prompt_parts.append( + "Available compatibility tools:\n" + + json.dumps(simplified_tools, ensure_ascii=False, indent=2) + ) + prompt_parts.append( + "Do not execute these tools yourself. If you need one, request it in " + "`tool_calls`. Each `arguments` field must be a JSON string." + ) + + if tool_choice == "required": + prompt_parts.append( + "Tool choice policy: you must request at least one compatibility tool." + ) + elif isinstance(tool_choice, dict) and tool_choice.get("type") == "function": + function = tool_choice.get("function", {}) or {} + prompt_parts.append( + "Tool choice policy: you must request the compatibility tool " + f"`{function.get('name', '')}`." + ) + + history_text = "\n".join(part for part in history_parts if part).strip() + if history_text: + prompt_parts.append("History:\n" + history_text) + + if structured_output: + prompt_parts.append("Return only JSON matching the provided schema.") + if tools: + prompt_parts.append( + "Set `content` to the assistant-visible reply. Set `tool_calls` to " + "an empty array when no tool is needed." + ) + else: + prompt_parts.append("Answer the latest user request.") + + return "\n\n".join(prompt_parts), attachments + + +def _assistant_message_schema() -> dict[str, Any]: + return { + "type": "object", + "properties": { + "content": {"type": "string"}, + "tool_calls": { + "type": "array", + "items": { + "type": "object", + "properties": { + "name": {"type": "string"}, + "arguments": {"type": "string"}, + }, + "required": ["name", "arguments"], + "additionalProperties": False, + }, + }, + }, + "required": ["content", "tool_calls"], + "additionalProperties": False, + } + + +def _materialize_attachments( + attachments: list[dict[str, Any]], + temp_dir: str, +) -> list[str]: + image_paths: list[str] = [] + for index, attachment in enumerate(attachments, 1): + path = attachment.get("path") + if path: + image_paths.append(str(path)) + continue + + mime = str(attachment.get("mime", "image/png")) + suffix = mimetypes.guess_extension(mime) or ".png" + image_path = os.path.join(temp_dir, f"image_{index}{suffix}") + with open(image_path, "wb") as f: + f.write(attachment.get("bytes", b"")) + image_paths.append(image_path) + return image_paths + + +def _usage_from_event(usage: dict[str, Any] | None) -> dict[str, int]: + usage = usage or {} + prompt_tokens = int(usage.get("input_tokens", 0) or 0) + completion_tokens = int(usage.get("output_tokens", 0) or 0) + return { + "prompt_tokens": prompt_tokens, + "completion_tokens": completion_tokens, + "total_tokens": prompt_tokens + completion_tokens, + } + + +def _extract_error(stdout: str, stderr: str) -> str: + for raw_line in reversed(stdout.splitlines()): + line = raw_line.strip() + if not line: + continue + try: + payload = json.loads(line) + except json.JSONDecodeError: + continue + if payload.get("type") == "turn.failed": + error = payload.get("error", {}) or {} + return str(error.get("message", "") or "Codex turn failed") + if payload.get("type") == "error": + return str(payload.get("message", "") or "Codex execution failed") + return stderr.strip() or stdout.strip() or "Codex execution failed" + + +def _run_codex_exec( + *, + model: str, + prompt: str, + attachments: list[dict[str, Any]], + output_schema: dict[str, Any] | None, + timeout: int | None, +) -> tuple[str, dict[str, int]]: + with tempfile.TemporaryDirectory(prefix="skillopt_codex_") as temp_dir: + output_path = os.path.join(temp_dir, "last_message.txt") + image_paths = _materialize_attachments(attachments, temp_dir) + + command = [ + CODEX_BIN, + "exec", + "--json", + "--ephemeral", + "--profile", + CODEX_PROFILE, + "-c", + "approval_policy=\"never\"", + "--sandbox", + CODEX_SANDBOX_MODE, + "--skip-git-repo-check", + "--cd", + _default_working_directory(), + "--model", + model, + "--output-last-message", + output_path, + ] + + if REASONING_EFFORT: + command.extend(["-c", f"model_reasoning_effort={json.dumps(REASONING_EFFORT)}"]) + + schema_path = None + if output_schema is not None: + schema_path = os.path.join(temp_dir, "schema.json") + with open(schema_path, "w", encoding="utf-8") as f: + json.dump(output_schema, f, ensure_ascii=False) + command.extend(["--output-schema", schema_path]) + + for image_path in image_paths: + command.extend(["--image", image_path]) + + command.append("-") + + proc = subprocess.run( + command, + input=prompt, + text=True, + capture_output=True, + timeout=timeout, + check=False, + ) + + usage_info = {"prompt_tokens": 0, "completion_tokens": 0, "total_tokens": 0} + fallback_text = "" + for raw_line in proc.stdout.splitlines(): + line = raw_line.strip() + if not line: + continue + try: + payload = json.loads(line) + except json.JSONDecodeError: + continue + if payload.get("type") == "item.completed": + item = payload.get("item", {}) or {} + if item.get("type") == "agent_message": + fallback_text = str(item.get("text", "") or fallback_text) + if payload.get("type") == "turn.completed": + usage_info = _usage_from_event(payload.get("usage")) + + last_message = "" + if os.path.exists(output_path): + with open(output_path, encoding="utf-8") as f: + last_message = f.read().strip() + if not last_message: + last_message = fallback_text.strip() + + if proc.returncode != 0: + raise RuntimeError(_extract_error(proc.stdout, proc.stderr)) + if not last_message: + raise RuntimeError("Codex returned an empty final message") + return last_message, usage_info + + +def _tool_name_from_choice(tool_choice: str | dict[str, Any] | None) -> str | None: + if not isinstance(tool_choice, dict): + return None + if tool_choice.get("type") != "function": + return None + function = tool_choice.get("function", {}) or {} + return str(function.get("name", "") or "") or None + + +def _compat_message_from_payload( + payload: dict[str, Any], + *, + tool_choice: str | dict[str, Any] | None = None, +) -> CompatAssistantMessage: + content = str(payload.get("content", "") or "") + tool_calls: list[CompatToolCall] = [] + for index, raw_tool_call in enumerate(payload.get("tool_calls", []) or [], 1): + if not isinstance(raw_tool_call, dict): + continue + name = str(raw_tool_call.get("name", "") or "") + arguments = raw_tool_call.get("arguments", "{}") + if not isinstance(arguments, str): + arguments = json.dumps(arguments, ensure_ascii=False) + tool_calls.append( + CompatToolCall( + id=f"tool_{index}_{uuid.uuid4().hex[:12]}", + function=CompatToolFunction(name=name, arguments=arguments), + ) + ) + + if tool_choice == "required" and not tool_calls: + raise RuntimeError("Codex response did not request a tool under tool_choice='required'") + + required_name = _tool_name_from_choice(tool_choice) + if required_name and all( + tool_call.function.name != required_name for tool_call in tool_calls + ): + raise RuntimeError( + f"Codex response did not request the required tool {required_name!r}" + ) + + return CompatAssistantMessage(content=content, tool_calls=tool_calls) + + +def _chat_messages_impl( + model: str, + messages: list[dict[str, Any]], + max_completion_tokens: int, + retries: int, + stage: str, + *, + tools: list[dict[str, Any]] | None = None, + tool_choice: str | dict[str, Any] | None = None, + return_message: bool = False, + timeout: int | None = None, +) -> tuple[Any, dict[str, int]]: + del max_completion_tokens + last_err = None + structured_output = bool(tools) or return_message + + for attempt in range(retries): + try: + prompt, attachments = _build_prompt_from_messages( + messages, + tools=tools, + tool_choice=tool_choice, + structured_output=structured_output, + ) + raw_text, usage_info = _run_codex_exec( + model=model, + prompt=prompt, + attachments=attachments, + output_schema=_assistant_message_schema() if structured_output else None, + timeout=timeout, + ) + tracker.record( + stage, + usage_info["prompt_tokens"], + usage_info["completion_tokens"], + ) + + if not structured_output: + return raw_text, usage_info + + payload = json.loads(raw_text) + compat = _compat_message_from_payload(payload, tool_choice=tool_choice) + return (compat if return_message else compat.content), usage_info + except subprocess.TimeoutExpired as exc: + last_err = RuntimeError(f"Codex CLI timed out after {timeout}s") if timeout else exc + except Exception as exc: # noqa: BLE001 + last_err = exc + time.sleep(min(2 ** attempt, 30)) + + raise RuntimeError(f"Codex call failed after {retries} retries: {last_err}") + + +def chat_with_model( + model: str, + system: str, + user: str, + max_completion_tokens: int = 16384, + retries: int = 5, + stage: str = "custom", + timeout: int | None = None, +) -> tuple[str, dict[str, int]]: + messages = [ + {"role": "system", "content": system}, + {"role": "user", "content": user}, + ] + return _chat_messages_impl( + model, + messages, + max_completion_tokens, + retries, + stage, + timeout=timeout, + ) + + +def chat_messages_with_model( + model: str, + messages: list[dict[str, Any]], + max_completion_tokens: int = 16384, + retries: int = 5, + stage: str = "custom", + *, + tools: list[dict[str, Any]] | None = None, + tool_choice: str | dict[str, Any] | None = None, + return_message: bool = False, + timeout: int | None = None, +) -> tuple[Any, dict[str, int]]: + return _chat_messages_impl( + model, + messages, + max_completion_tokens, + retries, + stage, + tools=tools, + tool_choice=tool_choice, + return_message=return_message, + timeout=timeout, + ) + + +def chat_optimizer( + system: str, + user: str, + max_completion_tokens: int = 16384, + retries: int = 5, + stage: str = "optimizer", + timeout: int | None = None, +) -> tuple[str, dict[str, int]]: + return chat_with_model( + model=OPTIMIZER_DEPLOYMENT, + system=system, + user=user, + max_completion_tokens=max_completion_tokens, + retries=retries, + stage=stage, + timeout=timeout, + ) + + +def chat_with_deployment( + deployment: str, + system: str, + user: str, + max_completion_tokens: int = 16384, + retries: int = 5, + stage: str = "custom", + timeout: int | None = None, +) -> tuple[str, dict[str, int]]: + return chat_with_model( + model=deployment, + system=system, + user=user, + max_completion_tokens=max_completion_tokens, + retries=retries, + stage=stage, + timeout=timeout, + ) + + +def chat_target( + system: str, + user: str, + max_completion_tokens: int = 16384, + retries: int = 5, + stage: str = "target", + timeout: int | None = None, +) -> tuple[str, dict[str, int]]: + return chat_with_model( + model=TARGET_DEPLOYMENT, + system=system, + user=user, + max_completion_tokens=max_completion_tokens, + retries=retries, + stage=stage, + timeout=timeout, + ) + + +def chat_optimizer_messages( + messages: list[dict[str, Any]], + max_completion_tokens: int = 16384, + retries: int = 5, + stage: str = "optimizer", + *, + tools: list[dict[str, Any]] | None = None, + tool_choice: str | dict[str, Any] | None = None, + return_message: bool = False, + timeout: int | None = None, +) -> tuple[Any, dict[str, int]]: + return _chat_messages_impl( + OPTIMIZER_DEPLOYMENT, + messages, + max_completion_tokens, + retries, + stage, + tools=tools, + tool_choice=tool_choice, + return_message=return_message, + timeout=timeout, + ) + + +def chat_messages_with_deployment( + deployment: str, + messages: list[dict[str, Any]], + max_completion_tokens: int = 16384, + retries: int = 5, + stage: str = "custom", + *, + tools: list[dict[str, Any]] | None = None, + tool_choice: str | dict[str, Any] | None = None, + return_message: bool = False, + timeout: int | None = None, +) -> tuple[Any, dict[str, int]]: + return _chat_messages_impl( + deployment, + messages, + max_completion_tokens, + retries, + stage, + tools=tools, + tool_choice=tool_choice, + return_message=return_message, + timeout=timeout, + ) + + +def chat_target_messages( + messages: list[dict[str, Any]], + max_completion_tokens: int = 16384, + retries: int = 5, + stage: str = "target", + *, + tools: list[dict[str, Any]] | None = None, + tool_choice: str | dict[str, Any] | None = None, + return_message: bool = False, + timeout: int | None = None, +) -> tuple[Any, dict[str, int]]: + return _chat_messages_impl( + TARGET_DEPLOYMENT, + messages, + max_completion_tokens, + retries, + stage, + tools=tools, + tool_choice=tool_choice, + return_message=return_message, + timeout=timeout, + ) + + +def get_token_summary() -> dict[str, dict[str, int]]: + return tracker.summary() + + +def reset_token_tracker() -> None: + tracker.reset() + + +def set_target_deployment(deployment: str) -> None: + global TARGET_DEPLOYMENT + TARGET_DEPLOYMENT = deployment + os.environ["TARGET_DEPLOYMENT"] = deployment + + +def set_reasoning_effort(effort: str | None) -> None: + global REASONING_EFFORT + REASONING_EFFORT = effort if effort else None + + +def set_optimizer_deployment(deployment: str) -> None: + global OPTIMIZER_DEPLOYMENT + OPTIMIZER_DEPLOYMENT = deployment + os.environ["OPTIMIZER_DEPLOYMENT"] = deployment diff --git a/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/model/codex_harness.py b/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/model/codex_harness.py new file mode 100644 index 00000000..f0217795 --- /dev/null +++ b/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/model/codex_harness.py @@ -0,0 +1,1160 @@ +"""Helpers for running exec backends as the target harness.""" +from __future__ import annotations + +import asyncio +import json +import os +import re +import shutil +import subprocess +import threading +import traceback +from typing import Any + +from skillopt.model.backend_config import ( + get_claude_code_exec_config, + get_codex_exec_config, + get_target_backend, +) + + +ANSWER_SCHEMA: dict[str, Any] = { + "type": "object", + "properties": { + "final_response": { + "type": "string", + "description": "The exact final answer text to return, preserving required ... tags.", + }, + "final_answer": { + "type": "string", + "description": "The concise answer value without explanation, if separable.", + }, + }, + "required": ["final_response", "final_answer"], + "additionalProperties": False, +} + + +def render_skill_md( + skill_content: str, + *, + name: str = "skillopt-target", + description: str = "Dynamic ReflACT skill for the current benchmark task.", + preamble: str = "", +) -> str: + body = skill_content.strip() or "No additional dynamic guidance was provided for this task." + chunks = [ + "---", + f'name: "{name}"', + f'description: "{description}"', + "---", + "", + "# ReflACT Target Skill", + "", + ] + if preamble.strip(): + chunks.append(preamble.strip()) + chunks.append("") + chunks.extend([ + "## Dynamic Guidance", + "", + body, + "", + ]) + return "\n".join(chunks) + + +def prepare_workspace( + *, + work_dir: str, + skill_md: str, + task_text: str = "", + task_filename: str = "task.md", + images: list[str] | None = None, + extra_files: dict[str, str] | None = None, + copy_files: list[tuple[str, str]] | None = None, + link_dirs: list[tuple[str, str]] | None = None, +) -> tuple[str, str]: + if os.path.exists(work_dir): + shutil.rmtree(work_dir) + os.makedirs(os.path.join(work_dir, ".agents", "skills", "skillopt-target"), exist_ok=True) + + skill_path = os.path.join(work_dir, ".agents", "skills", "skillopt-target", "SKILL.md") + with open(skill_path, "w", encoding="utf-8") as f: + f.write(skill_md) + + task_path = os.path.join(work_dir, task_filename) + if task_text: + with open(task_path, "w", encoding="utf-8") as f: + f.write(task_text) + + if extra_files: + for rel_path, content in extra_files.items(): + full_path = os.path.join(work_dir, rel_path) + parent = os.path.dirname(full_path) + if parent: + os.makedirs(parent, exist_ok=True) + with open(full_path, "w", encoding="utf-8") as f: + f.write(content) + + if copy_files: + for src, rel_dst in copy_files: + dst = os.path.join(work_dir, rel_dst) + parent = os.path.dirname(dst) + if parent: + os.makedirs(parent, exist_ok=True) + shutil.copy2(src, dst) + + if link_dirs: + for src, rel_dst in link_dirs: + dst = os.path.join(work_dir, rel_dst) + parent = os.path.dirname(dst) + if parent: + os.makedirs(parent, exist_ok=True) + os.symlink(os.path.abspath(src), dst) + + attachment_lines: list[str] = [] + if images: + attachments_dir = os.path.join(work_dir, "attachments") + os.makedirs(attachments_dir, exist_ok=True) + for index, image in enumerate(images, 1): + if not os.path.exists(image): + raise FileNotFoundError(image) + src = os.path.abspath(image) + base = os.path.basename(src) or f"image_{index}" + dst_name = f"{index:02d}_{base}" + dst = os.path.join(attachments_dir, dst_name) + if os.path.abspath(src) != os.path.abspath(dst): + shutil.copy2(src, dst) + rel_dst = os.path.relpath(dst, work_dir) + attachment_lines.append(f"- `{rel_dst}` (source: `{src}`)") + + if attachment_lines: + with open(os.path.join(work_dir, "ATTACHMENTS.md"), "w", encoding="utf-8") as f: + f.write( + "# Attachments\n\n" + "Use these local files when the task refers to attached images or documents.\n\n" + + "\n".join(attachment_lines) + + "\n" + ) + + return skill_path, task_path + + +def _build_codex_trace_summary(raw: str, response: str) -> str: + lines = [ln.rstrip() for ln in (raw or "").splitlines()] + + def _find(prefix: str) -> str: + for ln in lines: + if ln.startswith(prefix): + return ln[len(prefix):].strip() + return "" + + sandbox = _find("sandbox: ") + reasoning = _find("reasoning effort: ") + task_read = "unknown" + skill_read = "unknown" + exec_errors: list[str] = [] + tokens_used = "" + + for idx, ln in enumerate(lines): + if ln.startswith("exec"): + cmd = lines[idx + 1] if idx + 1 < len(lines) else "" + outcome = lines[idx + 2] if idx + 2 < len(lines) else "" + joined = f"{cmd}\n{outcome}" + if "task.md" in joined: + if "succeeded" in outcome: + task_read = "success" + elif "failed" in outcome or "ERROR" in outcome: + task_read = "failed" + if "SKILL.md" in joined: + if "succeeded" in outcome: + skill_read = "success" + elif "failed" in outcome or "ERROR" in outcome: + skill_read = "failed" + if ln.startswith("ERROR:"): + exec_errors.append(ln[len("ERROR:"):].strip()) + if ln == "tokens used" and idx + 1 < len(lines): + tokens_used = lines[idx + 1].strip() + + match = re.search(r"\s*([A-E])\s*", response or "", re.IGNORECASE) + if match: + answer_format = "well_formed" + answer_label = match.group(1).upper() + elif "" in (response or "").lower(): + answer_format = "tagged_nonlabel" + answer_label = "" + elif (response or "").strip(): + answer_format = "plain_text" + answer_label = "" + else: + answer_format = "missing" + answer_label = "" + + parts = ["Codex Trace Summary"] + if sandbox: + parts.append(f"- sandbox: {sandbox}") + if reasoning: + parts.append(f"- reasoning: {reasoning}") + parts.append(f"- read task.md: {task_read}") + parts.append(f"- read SKILL.md: {skill_read}") + if exec_errors: + parts.append(f"- shell/tool errors: {' | '.join(exec_errors[:3])}") + else: + parts.append("- shell/tool errors: none") + parts.append(f"- final answer format: {answer_format}") + parts.append(f"- final answer label: {answer_label or '(none)'}") + if tokens_used: + parts.append(f"- tokens used: {tokens_used}") + return "\n".join(parts) + + +def _build_claude_trace_summary(raw: str, response: str) -> str: + answer_format = "missing" + if "" in (response or "").lower(): + answer_format = "tagged" + elif (response or "").strip(): + answer_format = "plain_text" + errors: list[str] = [] + for ln in (raw or "").splitlines(): + if "error" in ln.lower() or "traceback" in ln.lower(): + errors.append(ln.strip()) + if len(errors) >= 3: + break + parts = ["Claude Code Trace Summary", f"- final answer format: {answer_format}"] + parts.append(f"- final response chars: {len(response or '')}") + parts.append(f"- errors: {' | '.join(errors) if errors else 'none'}") + return "\n".join(parts) + + +def _persist_artifacts( + *, + work_dir: str, + raw: str, + response: str, + prefix: str, + summary_builder, +) -> None: + pred_dir = os.path.dirname(work_dir.rstrip(os.sep)) + raw_path = os.path.join(pred_dir, f"{prefix}_raw.txt") + summary_path = os.path.join(pred_dir, f"{prefix}_trace_summary.txt") + + combined_raw = raw + if os.path.exists(raw_path): + with open(raw_path, encoding="utf-8") as f: + prev = f.read() + combined_raw = f"{prev}\n\n===== TURN BREAK =====\n\n{raw}" if prev.strip() else raw + + with open(raw_path, "w", encoding="utf-8") as f: + f.write(combined_raw) + with open(summary_path, "w", encoding="utf-8") as f: + f.write(summary_builder(combined_raw, response)) + + +def _persist_codex_artifacts(work_dir: str, raw: str, response: str) -> None: + _persist_artifacts( + work_dir=work_dir, + raw=raw, + response=response, + prefix="codex", + summary_builder=_build_codex_trace_summary, + ) + + +def _persist_claude_artifacts(work_dir: str, raw: str, response: str) -> None: + _persist_artifacts( + work_dir=work_dir, + raw=raw, + response=response, + prefix="claude", + summary_builder=_build_claude_trace_summary, + ) + + +def parse_codex_raw(raw: str) -> dict: + """Parse raw Codex CLI output into step sections. + + Returns a dict with: + - ``steps``: ordered sections beginning at the first ``user/codex/exec`` marker + - ``trace_body``: raw trace starting at the first marker + """ + lines = (raw or "").splitlines() + markers = {"user", "codex", "exec"} + first_step_line: int | None = None + for idx, line in enumerate(lines): + if line in markers: + first_step_line = idx + break + if first_step_line is None: + return {"steps": [], "trace_body": ""} + + steps: list[dict] = [] + current: dict | None = None + for idx in range(first_step_line, len(lines)): + line = lines[idx] + if line in markers: + if current is not None: + current["end_line"] = idx + current["content"] = "\n".join(current["content_lines"]).strip() + current.pop("content_lines", None) + steps.append(current) + current = { + "index": len(steps) + 1, + "type": line, + "start_line": idx, + "content_lines": [], + } + continue + if current is not None: + current["content_lines"].append(line) + if current is not None: + current["end_line"] = len(lines) + current["content"] = "\n".join(current["content_lines"]).strip() + current.pop("content_lines", None) + steps.append(current) + + trace_body = "\n".join(lines[first_step_line:]).strip() + return {"steps": steps, "trace_body": trace_body} + + +def format_codex_trace_steps(raw: str, *, max_chars: int = 4000) -> str: + """Render parsed Codex trace into numbered compact steps for optimizer prompts.""" + parsed = parse_codex_raw(raw) + steps = parsed["steps"] + if not steps: + return "" + + rendered: list[str] = [] + for step in steps: + summary = "" + content = str(step.get("content") or "").strip() + if step["type"] == "exec": + body_lines = [ln.strip() for ln in content.splitlines() if ln.strip()] + cmd = body_lines[0] if body_lines else "" + status = "" + for ln in body_lines[1:]: + low = ln.lower() + if "succeeded in" in low or "failed in" in low or "timed out" in low or low.startswith("error"): + status = ln + break + summary = cmd + if status: + summary = f"{summary} | {status}" if summary else status + else: + summary = " ".join(content.splitlines()) + summary = summary[:500] if summary else "(empty)" + rendered.append(f"[{step['index']}] {step['type']}: {summary}") + + text = "\n".join(rendered) + if len(text) > max_chars: + text = text[:max_chars] + "\n...[trace steps truncated]..." + return text + + +def extract_codex_trace_prefix(raw: str, *, after_step: int) -> str: + """Return raw trace body up to and including ``after_step``. + + ``after_step <= 0`` yields an empty string. + """ + if after_step <= 0: + return "" + parsed = parse_codex_raw(raw) + steps = parsed["steps"] + if not steps: + return "" + clamped = min(after_step, len(steps)) + lines = parsed["trace_body"].splitlines() + end_line = int(steps[clamped - 1]["end_line"]) - int(steps[0]["start_line"]) + return "\n".join(lines[:end_line]).strip() + + +_DENIED_DATA_DIR_NAMES = {"officeqa_split", "sealqa_split"} + + +def _normalize_tools(allowed_tools: list[str] | str | None) -> str: + if allowed_tools is None: + return "" + if isinstance(allowed_tools, str): + return ",".join(part.strip() for part in allowed_tools.split(",") if part.strip()) + return ",".join(str(tool).strip() for tool in allowed_tools if str(tool).strip()) + + +def _tools_list(allowed_tools: list[str] | str | None) -> list[str]: + tools = _normalize_tools(allowed_tools) + return [part.strip() for part in tools.split(",") if part.strip()] + + +def _validate_exec_path(path: str) -> str: + resolved = os.path.realpath(os.path.abspath(path)) + parts = set(resolved.split(os.sep)) + denied = parts & _DENIED_DATA_DIR_NAMES + if denied: + raise ValueError(f"Refusing to expose denied data directory to exec backend: {', '.join(sorted(denied))}") + return resolved + + +def _validated_add_dirs(work_dir: str, data_dirs: list[str] | None, images: list[str] | None) -> list[str]: + add_dirs = [_validate_exec_path(work_dir)] + for data_dir in data_dirs or []: + add_dirs.append(_validate_exec_path(data_dir)) + for image in images or []: + add_dirs.append(_validate_exec_path(os.path.dirname(image) or work_dir)) + deduped: list[str] = [] + for path in add_dirs: + if path not in deduped: + deduped.append(path) + return deduped + + +def _sdk_mode(value: Any) -> str: + mode = str(value or "auto").strip().lower() + if mode in {"1", "true", "yes", "on", "sdk"}: + return "sdk" + if mode in {"0", "false", "no", "off", "cli"}: + return "cli" + return "auto" + + +def _claude_effort(value: Any) -> str: + effort = str(value or "medium").strip().lower() + if effort in {"", "none", "off"}: + return "" + if effort == "xhigh": + return "max" + if effort not in {"low", "medium", "high", "max"}: + return "medium" + return effort + + +def _json_default(obj: Any) -> Any: + if isinstance(obj, (str, int, float, bool)) or obj is None: + return obj + if isinstance(obj, (list, tuple)): + return list(obj) + if isinstance(obj, dict): + return obj + if hasattr(obj, "model_dump"): + return obj.model_dump(mode="json") + if hasattr(obj, "__dict__"): + return {k: v for k, v in vars(obj).items() if not k.startswith("_")} + return str(obj) + + +def _json_dumps(data: Any) -> str: + return json.dumps(data, ensure_ascii=False, indent=2, default=_json_default) + + +def _run_async(coro): + try: + asyncio.get_running_loop() + except RuntimeError: + return asyncio.run(coro) + + box: dict[str, Any] = {} + + def _target() -> None: + try: + box["result"] = asyncio.run(coro) + except BaseException as exc: # noqa: BLE001 + box["exception"] = exc + + thread = threading.Thread(target=_target, daemon=True) + thread.start() + thread.join() + if "exception" in box: + raise box["exception"] + return box.get("result") + + +def _xskill_skill_mode() -> str: + """xskill-track skill-delivery mode. + + Default (env unset) == 'reference': current behavior, byte-unchanged for the + reference/noskill tracks (skills come from the local + .agents/skills/skillopt-target/SKILL.md markdown, the native Skill tool is + suppressed). 'native' == xskill track: the model is allowed and encouraged to + invoke available Skills via the native Skill tool, and we do NOT redirect to + .agents/skills (empty for the xskill track). The ONLY allowed turn-0 + difference between the two tracks is this skill-source wording (checklist B3). + """ + mode = str(os.environ.get("XSKILL_SKILL_MODE", "") or "").strip().lower() + return "native" if mode == "native" else "reference" + + +def _exec_prompt(prompt: str, *, allow_file_edits: bool = False) -> str: + edit_instruction = ( + "You may modify files in the workspace when the task asks you to create an artifact. " + if allow_file_edits + else "Do not modify files. " + ) + if _xskill_skill_mode() == "native": + # xskill track: encourage native Skill-tool invocation; no .agents/skills redirect. + skill_source = ( + "Use the workspace files to solve the task. Read task.md before answering. " + "Relevant Skills are available to you: when a Skill's description matches " + "this task, invoke it via the Skill tool and follow its guidance. " + "If ATTACHMENTS.md exists, read it and inspect the listed local files. " + ) + else: + skill_source = ( + "Use the workspace files to solve the task. Read task.md and the skill at " + ".agents/skills/skillopt-target/SKILL.md before answering. " + "If ATTACHMENTS.md exists, read it and inspect the listed local files. " + "Do not call a Skill tool; the ReflACT guidance is a local markdown file. " + ) + return ( + f"{skill_source}" + f"Do not ask for permission. {edit_instruction}" + "Return only the final answer text, keeping any required ... tags exactly.\n\n" + f"{_normalize_target_exec_prompt(prompt)}" + ) + + +def _retry_prompt(prompt: str, attempt: int) -> str: + if attempt <= 0: + return prompt + if _xskill_skill_mode() == "native": + skill_reminder = ( + "Re-read task.md and invoke any available Skill whose description matches this task. " + ) + else: + skill_reminder = ( + "Re-read task.md and .agents/skills/skillopt-target/SKILL.md. " + ) + return ( + f"{prompt}\n\n" + "Previous execution returned an empty final response. " + f"{skill_reminder}" + "If ATTACHMENTS.md exists, use the listed files. " + "Then produce the final answer inside ...." + ) + + +def _normalize_target_exec_prompt(prompt: str) -> str: + """Rewrite the embedded skill-source wording per track. + + Reference track (default): avoid wording that makes Claude Code call an + unregistered Skill tool -> redirect to the local markdown. + xskill track (XSKILL_SKILL_MODE=native): the Skill tool IS registered and the + skill body lives in the native Skill listing, so encourage native invocation + and do NOT redirect to .agents/skills (empty for the xskill track).""" + text = prompt or "" + if _xskill_skill_mode() == "native": + replacements = { + "Use the `skillopt-target` skill available in this workspace.": ( + "Use any available Skill whose description matches this task by invoking it via the Skill tool." + ), + "- Use the local `skillopt-target` skill before writing code.": ( + "- Use any available Skill whose description matches this task (invoke it via the Skill tool) before writing code." + ), + } + else: + replacements = { + "Use the `skillopt-target` skill available in this workspace.": ( + "Read `.agents/skills/skillopt-target/SKILL.md` directly; do not call a Skill tool." + ), + "- Use the local `skillopt-target` skill before writing code.": ( + "- Read `.agents/skills/skillopt-target/SKILL.md` before writing code; do not call a Skill tool." + ), + } + for old, new in replacements.items(): + text = text.replace(old, new) + return text + + +def _strict_schema(schema: dict[str, Any]) -> dict[str, Any]: + strict = json.loads(json.dumps(schema)) + strict["additionalProperties"] = False + properties = strict.get("properties") or {} + strict["required"] = list(properties.keys()) + return strict + + +def _structured_response(data: Any) -> tuple[str, str]: + if not isinstance(data, dict): + return "", f"Structured output was not an object: {type(data).__name__}" + final_response = str(data.get("final_response") or "").strip() + final_answer = str(data.get("final_answer") or "").strip() + if final_response: + return final_response, "" + if final_answer: + if "" in final_answer.lower(): + return final_answer, "" + return f"{final_answer}", "" + return "", "Structured output did not contain a final response." + + +def _extract_claude_structured_output(messages: list[Any]) -> Any: + """Claude Code SDK can finish with error_during_execution after StructuredOutput.""" + for msg in reversed(messages): + structured = getattr(msg, "structured_output", None) + if isinstance(structured, dict): + return structured + + content = getattr(msg, "content", None) + if content is None and isinstance(msg, dict): + content = msg.get("content") + if not isinstance(content, list): + continue + + for item in reversed(content): + name = getattr(item, "name", None) + payload = getattr(item, "input", None) + if isinstance(item, dict): + name = item.get("name", name) + payload = item.get("input", payload) + if name == "StructuredOutput" and isinstance(payload, dict): + return payload + return None + + +def _raw_exception(label: str, exc: BaseException) -> str: + return _json_dumps({ + "backend": label, + "is_error": True, + "error_type": type(exc).__name__, + "error": str(exc), + "traceback": traceback.format_exc(), + }) + + +def _run_claude_code_sdk_exec( + *, + work_dir: str, + prompt: str, + model: str, + timeout: int, + images: list[str] | None = None, + data_dirs: list[str] | None = None, + allowed_tools: list[str] | str | None = None, + permission_mode: str | None = None, + allow_file_edits: bool = False, +) -> tuple[str, str]: + # XSKILL FIX 1 (defense-in-depth): this SDK path has NO isolation support; + # it would load the user's ambient ~/.claude skills. Never run it while the + # xskill isolation env is active. Fail closed. + if os.environ.get("XSKILL_CLAUDE_HOME"): + raise RuntimeError( + "XSKILL isolation guard: _run_claude_code_sdk_exec invoked while " + "XSKILL_CLAUDE_HOME is set. The SDK path is not isolated; use the CLI " + "path (claude_code_exec_use_sdk=cli)." + ) + from claude_agent_sdk import ClaudeAgentOptions, ClaudeSDKClient + + async def _query() -> tuple[str, str]: + system_prompt: dict[str, Any] = { + "type": "preset", + "preset": "claude_code", + "append": ( + "Use the workspace files to solve the task. Read task.md and the skill at " + ".agents/skills/skillopt-target/SKILL.md before answering. " + "If ATTACHMENTS.md exists, read it and inspect the listed local files. " + "Do not call a Skill tool; the ReflACT guidance is a local markdown file. " + + ( + "You may modify files in the workspace when the task asks you to create an artifact. " + if allow_file_edits + else "Do not modify files. " + ) + + "Return structured output whose final_response preserves required ... tags." + ), + } + kwargs: dict[str, Any] = { + "system_prompt": system_prompt, + "output_format": {"type": "json_schema", "schema": ANSWER_SCHEMA}, + "allowed_tools": _tools_list(allowed_tools) or ["Read", "Bash"], + "cwd": str(work_dir), + "permission_mode": permission_mode or "bypassPermissions", + "add_dirs": _validated_add_dirs(work_dir, data_dirs, images), + "max_buffer_size": 8 * 1024 * 1024, + } + config = get_claude_code_exec_config() + effort = _claude_effort(config.get("effort")) + if effort: + kwargs["effort"] = effort + max_thinking_tokens = int(config.get("max_thinking_tokens", 0) or 0) + if max_thinking_tokens > 0: + kwargs["max_thinking_tokens"] = max_thinking_tokens + options = ClaudeAgentOptions(**kwargs) + if model: + options.model = model.split("/", 1)[1] if model.startswith("anthropic/") else model + + messages = [] + async with ClaudeSDKClient(options) as client: + await client.query(_normalize_target_exec_prompt(prompt)) + messages = [msg async for msg in client.receive_response()] + last = messages[-1] if messages else None + raw_structured_output = _extract_claude_structured_output(messages) + response, parse_error = _structured_response(raw_structured_output) + first = messages[0] if messages else None + first_data = getattr(first, "data", {}) if first is not None else {} + terminal_is_error = bool(getattr(last, "is_error", False)) if last is not None else False + raw = _json_dumps({ + "backend": "claude_code_sdk", + "uuid": first_data.get("uuid", "") if isinstance(first_data, dict) else "", + "session_id": getattr(last, "session_id", "") if last is not None else "", + "model": first_data.get("model", model) if isinstance(first_data, dict) else model, + "tools": first_data.get("tools", _tools_list(allowed_tools)) if isinstance(first_data, dict) else _tools_list(allowed_tools), + "duration_ms": getattr(last, "duration_ms", 0) if last is not None else 0, + "total_cost_usd": getattr(last, "total_cost_usd", 0.0) if last is not None else 0.0, + "num_turns": getattr(last, "num_turns", 0) if last is not None else 0, + "usage": getattr(last, "usage", {}) if last is not None else {}, + "result": getattr(last, "result", "") if last is not None else "", + "is_error": bool(parse_error) or (terminal_is_error and not response.strip()), + "terminal_is_error": terminal_is_error, + "parse_error": parse_error, + "raw_structured_output": raw_structured_output, + "messages": messages, + }) + return response, raw + + return _run_async(asyncio.wait_for(_query(), timeout=timeout)) + + +def _run_claude_code_cli_exec( + *, + work_dir: str, + prompt: str, + model: str, + timeout: int, + images: list[str] | None = None, + data_dirs: list[str] | None = None, + allowed_tools: list[str] | str | None = None, + permission_mode: str | None = None, + allow_file_edits: bool = False, +) -> tuple[str, str]: + config = get_claude_code_exec_config() + tools = "Read,Bash" if allowed_tools is None else _normalize_tools(allowed_tools) + cmd = [ + str(config["path"]), + "-p", + "--output-format", + "text", + "--permission-mode", + permission_mode or "bypassPermissions", + "--add-dir", + work_dir, + "--tools", + tools, + "--allowedTools", + tools, + ] + if config.get("profile"): + cmd.extend(["--settings", '{"env":{"CLAUDE_CODE_USE_BEDROCK":"0"}}']) + cmd.extend(["--append-system-prompt", f"Profile: {config['profile']}"]) + if model: + cmd.extend(["--model", model]) + effort = _claude_effort(config.get("effort")) + if effort: + cmd.extend(["--effort", effort]) + max_thinking_tokens = int(config.get("max_thinking_tokens", 0) or 0) + if max_thinking_tokens > 0: + cmd.extend(["--max-thinking-tokens", str(max_thinking_tokens)]) + for data_dir in data_dirs or []: + cmd.extend(["--add-dir", _validate_exec_path(data_dir)]) + if images: + for image in images: + cmd.extend(["--add-dir", _validate_exec_path(os.path.dirname(image) or work_dir)]) + # XSKILL isolation gate: when XSKILL_CLAUDE_HOME is set, point the claude CLI + # at an isolated config dir so the user's real ~/.claude (108 skills) and + # plugin skills (73) do NOT load. Only experiment skills under + # /skills (user scope) + the CLI's built-in skills remain. + # Gated on the env var: default behavior is unchanged when it is unset. + run_env: dict[str, str] | None = None + xskill_home = os.environ.get("XSKILL_CLAUDE_HOME") + if xskill_home: + run_env = dict(os.environ) + run_env["CLAUDE_CONFIG_DIR"] = xskill_home + # user,project so the experiment skills under /skills load + # (project alone drops the user-scope skills dir under the isolated config). + cmd.extend(["--setting-sources", "user,project"]) + # The Skill tool MUST be in the allowed set, otherwise Claude Code strips + # the skill listing entirely (restricting --tools to Read,Bash removes the + # Skill tool => no skill_listing => experiment skills never enter the budget). + if "Skill" not in tools.split(","): + tools_with_skill = f"{tools},Skill" if tools else "Skill" + for i, arg in enumerate(cmd): + if arg in ("--tools", "--allowedTools") and i + 1 < len(cmd): + cmd[i + 1] = tools_with_skill + cmd.extend(["--", _exec_prompt(prompt, allow_file_edits=allow_file_edits)]) + + try: + proc = subprocess.run( + cmd, + cwd=work_dir, + capture_output=True, + text=True, + timeout=timeout, + env=run_env, + ) + except subprocess.TimeoutExpired as exc: + stdout = exc.stdout or "" + stderr = exc.stderr or "" + raw = stdout + if stderr: + raw = f"{raw}\n[stderr]\n{stderr}" if raw else stderr + return "", raw + + stdout = proc.stdout or "" + stderr = proc.stderr or "" + raw = stdout + if stderr: + raw = f"{raw}\n[stderr]\n{stderr}" if raw else stderr + response = stdout.strip() + if proc.returncode != 0 and not response: + return "", raw + return response, raw + + +def run_claude_code_exec( + *, + work_dir: str, + prompt: str, + model: str, + timeout: int, + images: list[str] | None = None, + data_dirs: list[str] | None = None, + allowed_tools: list[str] | str | None = None, + permission_mode: str | None = None, + allow_file_edits: bool = False, +) -> tuple[str, str]: + config = get_claude_code_exec_config() + mode = _sdk_mode(config.get("use_sdk")) + # XSKILL FIX 1 (fail-closed isolation): the SDK exec path + # (_run_claude_code_sdk_exec) does NOT honor XSKILL_CLAUDE_HOME, so if a run + # were to resolve to the SDK path the user's ambient ~/.claude skills (108 + # user + 73 plugin) could silently leak into the isolated experiment. When + # XSKILL_CLAUDE_HOME is set we therefore force the CLI path. If the SDK path + # was *explicitly* requested (use_sdk=sdk) while the isolation env is on, we + # refuse loudly rather than run unisolated. Gated on the env var: default + # behavior (env unset) is byte-unchanged. + if os.environ.get("XSKILL_CLAUDE_HOME"): + if mode == "sdk": + raise RuntimeError( + "XSKILL isolation guard: XSKILL_CLAUDE_HOME is set but " + "claude_code_exec_use_sdk resolved to 'sdk'. The SDK exec path " + "(_run_claude_code_sdk_exec) ignores XSKILL_CLAUDE_HOME and would " + "load the user's ambient ~/.claude skills, breaking isolation. " + "Set claude_code_exec_use_sdk=cli (or CLAUDE_CODE_EXEC_USE_SDK=cli) " + "for xskill-track runs." + ) + # auto -> force CLI so isolation is guaranteed (SDK path is never taken). + mode = "cli" + retries = int(config.get("empty_response_retries", 0) or 0) + last_response = "" + all_raw: list[str] = [] + + for attempt in range(retries + 1): + attempt_prompt = _retry_prompt(prompt, attempt) + if mode != "cli": + try: + response, raw = _run_claude_code_sdk_exec( + work_dir=work_dir, + prompt=attempt_prompt, + model=model, + timeout=timeout, + images=images, + data_dirs=data_dirs, + allowed_tools=allowed_tools, + permission_mode=permission_mode, + allow_file_edits=allow_file_edits, + ) + all_raw.append(f"===== CLAUDE SDK ATTEMPT {attempt + 1} =====\n{raw}") + if response.strip(): + combined = "\n\n".join(all_raw) + _persist_claude_artifacts(work_dir, combined, response) + return response, combined + except (ImportError, ModuleNotFoundError) as exc: + raw = _raw_exception("claude_code_sdk", exc) + all_raw.append(f"===== CLAUDE SDK ATTEMPT {attempt + 1} =====\n{raw}") + if mode == "sdk": + _persist_claude_artifacts(work_dir, "\n\n".join(all_raw), "") + raise + except Exception as exc: # noqa: BLE001 + raw = _raw_exception("claude_code_sdk", exc) + all_raw.append(f"===== CLAUDE SDK ATTEMPT {attempt + 1} =====\n{raw}") + if mode == "sdk" and attempt >= retries: + _persist_claude_artifacts(work_dir, "\n\n".join(all_raw), "") + raise + if mode != "sdk": + response, raw = _run_claude_code_cli_exec( + work_dir=work_dir, + prompt=attempt_prompt, + model=model, + timeout=timeout, + images=images, + data_dirs=data_dirs, + allowed_tools=allowed_tools, + permission_mode=permission_mode, + allow_file_edits=allow_file_edits, + ) + all_raw.append(f"===== CLAUDE CLI ATTEMPT {attempt + 1} =====\n{raw}") + last_response = response + if response.strip(): + combined = "\n\n".join(all_raw) + _persist_claude_artifacts(work_dir, combined, response) + return response, combined + + combined = "\n\n".join(all_raw) + _persist_claude_artifacts(work_dir, combined, last_response) + return last_response, combined + + +def _run_codex_sdk_exec( + *, + work_dir: str, + prompt: str, + model: str, + timeout: int, + images: list[str] | None = None, + data_dirs: list[str] | None = None, +) -> tuple[str, str]: + from openai_codex_sdk import Codex + + for data_dir in data_dirs or []: + _validate_exec_path(data_dir) + for image in images or []: + _validate_exec_path(os.path.dirname(image) or work_dir) + + async def _query() -> tuple[str, str]: + config = get_codex_exec_config() + reasoning_effort = str(config.get("reasoning_effort", "") or "").strip() + thread_options: dict[str, Any] = { + "working_directory": work_dir, + "skip_git_repo_check": True, + "sandbox_mode": str(config.get("sandbox") or "workspace-write"), + "network_access_enabled": bool(config.get("network_access", False)), + "web_search_enabled": bool(config.get("web_search", False)), + "approval_policy": str(config.get("approval_policy") or "never"), + } + if model: + thread_options["model"] = model + if data_dirs: + thread_options["additional_directories"] = data_dirs + if reasoning_effort and reasoning_effort != "none": + thread_options["model_reasoning_effort"] = reasoning_effort + + codex_options: dict[str, Any] = {"env": os.environ.copy()} + codex_path = str(config.get("path") or "").strip() + if codex_path: + codex_options["codexPathOverride"] = codex_path + codex = Codex(codex_options) + thread = codex.start_thread(thread_options) + turn = await thread.run(prompt, {"output_schema": _strict_schema(ANSWER_SCHEMA)}) + result_text = str(getattr(turn, "final_response", "") or "") + parsed: Any = None + parse_error = "" + response = "" + if result_text.strip(): + try: + parsed = json.loads(result_text) + response, parse_error = _structured_response(parsed) + except Exception as exc: # noqa: BLE001 + parse_error = f"{type(exc).__name__}: {exc}" + else: + parse_error = "No response from Codex SDK (final_response is empty)." + raw = _json_dumps({ + "backend": "codex_sdk", + "id": getattr(turn, "id", ""), + "thread_id": getattr(turn, "thread_id", ""), + "model": model, + "thread_options": thread_options, + "final_response": result_text, + "raw_structured_output": parsed, + "parse_error": parse_error, + "is_error": bool(parse_error), + "items": getattr(turn, "items", []), + }) + return response, raw + + return _run_async(asyncio.wait_for(_query(), timeout=timeout)) + + +def _run_codex_cli_exec( + *, + work_dir: str, + prompt: str, + model: str, + timeout: int, + images: list[str] | None = None, + data_dirs: list[str] | None = None, + sandbox: str | None = None, + full_auto: bool | None = None, +) -> tuple[str, str]: + config = get_codex_exec_config() + last_message_path = os.path.join(work_dir, "codex_last_message.txt") + cmd = [ + str(config["path"]), + "exec", + "--skip-git-repo-check", + "--color", + "never", + "-C", + work_dir, + ] + if config.get("profile"): + cmd.extend(["-p", str(config["profile"])]) + reasoning_effort = str(config.get("reasoning_effort", "")).strip() + if reasoning_effort: + cmd.extend(["-c", f'model_reasoning_effort="{reasoning_effort}"']) + actual_full_auto = bool(config.get("full_auto", True)) if full_auto is None else bool(full_auto) + actual_sandbox = str(sandbox or config["sandbox"]) + if actual_full_auto: + cmd.append("--full-auto") + else: + cmd.extend(["--sandbox", actual_sandbox]) + if model: + cmd.extend(["-m", model]) + for data_dir in data_dirs or []: + _validate_exec_path(data_dir) + for image in images or []: + _validate_exec_path(os.path.dirname(image) or work_dir) + cmd.extend(["-i", image]) + cmd.extend(["--output-last-message", last_message_path, prompt]) + + try: + proc = subprocess.run( + cmd, + cwd=work_dir, + capture_output=True, + text=True, + timeout=timeout, + ) + except subprocess.TimeoutExpired as exc: + stdout = exc.stdout or "" + stderr = exc.stderr or "" + raw = stdout + if stderr: + raw = f"{raw}\n[stderr]\n{stderr}" if raw else stderr + _persist_codex_artifacts(work_dir, raw, "") + raise + try: + from skillopt.model import azure_openai as _openai + _openai.tracker.record("rollout", 0, 0) + except Exception: + pass + stdout = proc.stdout or "" + stderr = proc.stderr or "" + last_message = "" + if os.path.exists(last_message_path): + with open(last_message_path, encoding="utf-8") as f: + last_message = f.read() + raw = stdout + if stderr: + raw = f"{raw}\n[stderr]\n{stderr}" if raw else stderr + if proc.returncode != 0: + _persist_codex_artifacts(work_dir, raw, last_message) + detail = (stderr or stdout).strip() + raise RuntimeError( + f"codex exec failed with exit code {proc.returncode}: {detail[:4000]}" + ) + return last_message, raw + + +def run_codex_exec( + *, + work_dir: str, + prompt: str, + model: str, + timeout: int, + images: list[str] | None = None, + data_dirs: list[str] | None = None, + sandbox: str | None = None, + full_auto: bool | None = None, +) -> tuple[str, str]: + config = get_codex_exec_config() + mode = _sdk_mode(config.get("use_sdk")) + retries = int(config.get("empty_response_retries", 0) or 0) + last_response = "" + all_raw: list[str] = [] + + for attempt in range(retries + 1): + attempt_prompt = _retry_prompt(prompt, attempt) + if mode != "cli": + try: + response, raw = _run_codex_sdk_exec( + work_dir=work_dir, + prompt=attempt_prompt, + model=model, + timeout=timeout, + images=images, + data_dirs=data_dirs, + ) + all_raw.append(f"===== CODEX SDK ATTEMPT {attempt + 1} =====\n{raw}") + if response.strip(): + combined = "\n\n".join(all_raw) + _persist_codex_artifacts(work_dir, combined, response) + return response, combined + except (ImportError, ModuleNotFoundError) as exc: + raw = _raw_exception("codex_sdk", exc) + all_raw.append(f"===== CODEX SDK ATTEMPT {attempt + 1} =====\n{raw}") + if mode == "sdk": + _persist_codex_artifacts(work_dir, "\n\n".join(all_raw), "") + raise + except Exception as exc: # noqa: BLE001 + raw = _raw_exception("codex_sdk", exc) + all_raw.append(f"===== CODEX SDK ATTEMPT {attempt + 1} =====\n{raw}") + if mode == "sdk" and attempt >= retries: + _persist_codex_artifacts(work_dir, "\n\n".join(all_raw), "") + raise + if mode != "sdk": + response, raw = _run_codex_cli_exec( + work_dir=work_dir, + prompt=attempt_prompt, + model=model, + timeout=timeout, + images=images, + data_dirs=data_dirs, + sandbox=sandbox, + full_auto=full_auto, + ) + all_raw.append(f"===== CODEX CLI ATTEMPT {attempt + 1} =====\n{raw}") + last_response = response + if response.strip(): + combined = "\n\n".join(all_raw) + _persist_codex_artifacts(work_dir, combined, response) + return response, combined + + combined = "\n\n".join(all_raw) + _persist_codex_artifacts(work_dir, combined, last_response) + return last_response, combined + + +def run_target_exec( + *, + work_dir: str, + prompt: str, + model: str, + timeout: int, + images: list[str] | None = None, + data_dirs: list[str] | None = None, + allowed_tools: list[str] | str | None = None, + permission_mode: str | None = None, + sandbox: str | None = None, + full_auto: bool | None = None, + allow_file_edits: bool = False, +) -> tuple[str, str]: + backend = get_target_backend() + if backend == "codex_exec": + return run_codex_exec( + work_dir=work_dir, + prompt=prompt, + model=model, + timeout=timeout, + images=images, + data_dirs=data_dirs, + sandbox=sandbox, + full_auto=full_auto, + ) + if backend == "claude_code_exec": + return run_claude_code_exec( + work_dir=work_dir, + prompt=prompt, + model=model, + timeout=timeout, + images=images, + data_dirs=data_dirs, + allowed_tools=allowed_tools, + permission_mode=permission_mode, + allow_file_edits=allow_file_edits, + ) + raise ValueError(f"Unsupported exec backend: {backend}") diff --git a/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/model/common.py b/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/model/common.py new file mode 100644 index 00000000..80983b52 --- /dev/null +++ b/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/model/common.py @@ -0,0 +1,229 @@ +"""Shared model utilities for ReflACT backends.""" +from __future__ import annotations + +import json +import threading +from dataclasses import dataclass, field +from typing import Any + + +_RESPONSES_API_MODELS = { + "gpt-5.3-codex", + "gpt-5.1-codex", + "gpt-5.2-codex", + "gpt-5-codex", + "codex-mini", + "gpt-5.4-pro", +} + +_BACKEND_DEFAULT_MODELS = { + "azure_openai": "gpt-4o", + "openai_chat": "gpt-4o", + "codex": "gpt-4o", + "codex_exec": "gpt-4o", + "claude": "claude-sonnet-4-6", + "claude_chat": "claude-sonnet-4-6", + "claude_code_exec": "claude-sonnet-4-6", + "qwen_chat": "Qwen/Qwen3.5-4B", + "minimax_chat": "MiniMax-M2.7", +} + +_BACKEND_ALIASES = { + "azure": "azure_openai", + "azure_openai": "azure_openai", + "azure-openai": "azure_openai", + "openai_chat": "openai_chat", + "openai": "codex", + "codex": "codex", + "codex_exec": "codex_exec", + "claude": "claude_chat", + "claude_chat": "claude_chat", + "claude_code_exec": "claude_code_exec", + "anthropic": "claude_chat", + "qwen": "qwen_chat", + "qwen_chat": "qwen_chat", + "minimax": "minimax_chat", + "minimax_chat": "minimax_chat", +} + + +def normalize_backend_name(name: str | None) -> str: + normalized = str(name or "").strip().lower() + return _BACKEND_ALIASES.get(normalized, normalized or "azure_openai") + + +def default_model_for_backend(backend: str | None) -> str: + return _BACKEND_DEFAULT_MODELS.get( + normalize_backend_name(backend), + _BACKEND_DEFAULT_MODELS["azure_openai"], + ) + + +def needs_responses_api(model: str) -> bool: + normalized = str(model or "").strip().lower() + return any( + normalized == prefix or normalized.startswith(prefix + "-") + for prefix in _RESPONSES_API_MODELS + ) + + +class TokenTracker: + def __init__(self) -> None: + self._lock = threading.Lock() + self._data: dict[str, dict[str, int]] = {} + + def record(self, stage: str, prompt_tokens: int, completion_tokens: int) -> None: + with self._lock: + if stage not in self._data: + self._data[stage] = { + "calls": 0, + "prompt_tokens": 0, + "completion_tokens": 0, + } + entry = self._data[stage] + entry["calls"] += 1 + entry["prompt_tokens"] += prompt_tokens + entry["completion_tokens"] += completion_tokens + + def summary(self) -> dict[str, dict[str, int]]: + with self._lock: + out: dict[str, dict[str, int]] = {} + total_prompt = total_completion = total_calls = 0 + for stage, entry in sorted(self._data.items()): + prompt_tokens = entry["prompt_tokens"] + completion_tokens = entry["completion_tokens"] + out[stage] = { + "calls": entry["calls"], + "prompt_tokens": prompt_tokens, + "completion_tokens": completion_tokens, + "total_tokens": prompt_tokens + completion_tokens, + } + total_prompt += prompt_tokens + total_completion += completion_tokens + total_calls += entry["calls"] + out["_total"] = { + "calls": total_calls, + "prompt_tokens": total_prompt, + "completion_tokens": total_completion, + "total_tokens": total_prompt + total_completion, + } + return out + + def reset(self) -> None: + with self._lock: + self._data.clear() + + +tracker = TokenTracker() + + +@dataclass +class CompatToolFunction: + name: str + arguments: str + + def model_dump(self, mode: str = "json") -> dict[str, str]: + del mode + return { + "name": self.name, + "arguments": self.arguments, + } + + +@dataclass +class CompatToolCall: + id: str + function: CompatToolFunction + type: str = "function" + + def model_dump(self, mode: str = "json") -> dict[str, Any]: + del mode + return { + "id": self.id, + "type": self.type, + "function": self.function.model_dump(), + } + + +@dataclass +class CompatAssistantMessage: + content: str + tool_calls: list[CompatToolCall] = field(default_factory=list) + metadata: dict[str, Any] = field(default_factory=dict) + + def model_dump(self, mode: str = "json") -> dict[str, Any]: + del mode + data: dict[str, Any] = {"role": "assistant", "content": self.content} + if self.tool_calls: + data["tool_calls"] = [tool_call.model_dump() for tool_call in self.tool_calls] + return data + + +def usage_from_openai_usage(usage: Any) -> dict[str, int]: + if not usage: + return {"prompt_tokens": 0, "completion_tokens": 0, "total_tokens": 0} + prompt_tokens = getattr(usage, "prompt_tokens", 0) or 0 + completion_tokens = getattr(usage, "completion_tokens", 0) or 0 + total_tokens = getattr(usage, "total_tokens", 0) or (prompt_tokens + completion_tokens) + return { + "prompt_tokens": prompt_tokens, + "completion_tokens": completion_tokens, + "total_tokens": total_tokens, + } + + +def usage_from_responses_usage(usage: Any) -> dict[str, int]: + if not usage: + return {"prompt_tokens": 0, "completion_tokens": 0, "total_tokens": 0} + prompt_tokens = getattr(usage, "input_tokens", 0) or 0 + completion_tokens = getattr(usage, "output_tokens", 0) or 0 + return { + "prompt_tokens": prompt_tokens, + "completion_tokens": completion_tokens, + "total_tokens": prompt_tokens + completion_tokens, + } + + +def compat_message_from_chat_message(message: Any) -> CompatAssistantMessage: + content = getattr(message, "content", "") or "" + tool_calls = [] + for tool_call in getattr(message, "tool_calls", None) or []: + function = getattr(tool_call, "function", None) + tool_calls.append( + CompatToolCall( + id=getattr(tool_call, "id", "") or "", + function=CompatToolFunction( + name=getattr(function, "name", "") or "", + arguments=getattr(function, "arguments", "") or "{}", + ), + ) + ) + return CompatAssistantMessage(content=content, tool_calls=tool_calls) + + +def compat_message_from_responses_output(output: list[Any]) -> CompatAssistantMessage: + text_parts: list[str] = [] + tool_calls: list[CompatToolCall] = [] + for item in output: + item_type = getattr(item, "type", "") or "" + if item_type == "function_call": + raw_arguments = getattr(item, "arguments", None) + if raw_arguments is None: + raw_arguments = json.dumps(getattr(item, "input", {}) or {}) + tool_calls.append( + CompatToolCall( + id=getattr(item, "call_id", "") or getattr(item, "id", "") or "", + function=CompatToolFunction( + name=getattr(item, "name", "") or "", + arguments=str(raw_arguments or "{}"), + ), + ) + ) + continue + if item_type != "message": + continue + for part in getattr(item, "content", []) or []: + part_type = getattr(part, "type", "") or "" + if part_type in {"output_text", "text"}: + text_parts.append(getattr(part, "text", "") or "") + return CompatAssistantMessage(content="".join(text_parts), tool_calls=tool_calls) diff --git a/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/model/minimax_backend.py b/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/model/minimax_backend.py new file mode 100644 index 00000000..8c6add9c --- /dev/null +++ b/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/model/minimax_backend.py @@ -0,0 +1,277 @@ +"""OpenAI-compatible MiniMax chat backend for the target path.""" +from __future__ import annotations + +import json +import os +import threading +import time +import urllib.error +import urllib.request +from typing import Any + +from skillopt.model.common import ( + CompatAssistantMessage, + CompatToolCall, + CompatToolFunction, + TokenTracker, + default_model_for_backend, +) + +BASE_URL = os.environ.get("MINIMAX_BASE_URL", "https://api.minimax.io/v1") +API_KEY = os.environ.get("MINIMAX_API_KEY", "") +TIMEOUT_SECONDS = float(os.environ.get("MINIMAX_TIMEOUT_SECONDS", "300") or 300) +MAX_TOKENS = int(os.environ.get("MINIMAX_MAX_TOKENS", "8000") or 8000) +TEMPERATURE: float | None = None +_raw_temperature = os.environ.get("MINIMAX_TEMPERATURE", "0.7").strip() +if _raw_temperature: + TEMPERATURE = float(_raw_temperature) +ENABLE_THINKING = os.environ.get("MINIMAX_ENABLE_THINKING", "false").strip().lower() in { + "1", + "true", + "yes", + "on", +} + +TARGET_DEPLOYMENT = os.environ.get( + "TARGET_DEPLOYMENT", + default_model_for_backend("minimax_chat"), +) + +_config_lock = threading.Lock() +tracker = TokenTracker() + + +def _chat_url() -> str: + base = BASE_URL.rstrip("/") + if base.endswith("/chat/completions"): + return base + return f"{base}/chat/completions" + + +def _json_safe(value: Any) -> Any: + if value is None or isinstance(value, (str, int, float, bool)): + return value + if isinstance(value, list): + return [_json_safe(item) for item in value] + if isinstance(value, dict): + return {str(key): _json_safe(val) for key, val in value.items()} + model_dump = getattr(value, "model_dump", None) + if callable(model_dump): + try: + return model_dump(mode="json") + except TypeError: + return model_dump() + return str(value) + + +def _usage_from_payload(payload: dict[str, Any]) -> dict[str, int]: + usage = payload.get("usage") or {} + prompt_tokens = int(usage.get("prompt_tokens") or usage.get("input_tokens") or 0) + completion_tokens = int(usage.get("completion_tokens") or usage.get("output_tokens") or 0) + total_tokens = int(usage.get("total_tokens") or (prompt_tokens + completion_tokens)) + return { + "prompt_tokens": prompt_tokens, + "completion_tokens": completion_tokens, + "total_tokens": total_tokens, + } + + +def _compat_message_from_payload(message: dict[str, Any], choice: dict[str, Any]) -> CompatAssistantMessage: + content = message.get("content") or "" + if not isinstance(content, str): + content = json.dumps(content, ensure_ascii=False) + tool_calls: list[CompatToolCall] = [] + for index, tool_call in enumerate(message.get("tool_calls") or [], start=1): + function = tool_call.get("function") or {} + tool_calls.append( + CompatToolCall( + id=str(tool_call.get("id") or f"minimax_tool_{index}"), + type=str(tool_call.get("type") or "function"), + function=CompatToolFunction( + name=str(function.get("name") or ""), + arguments=str(function.get("arguments") or "{}"), + ), + ) + ) + return CompatAssistantMessage( + content=content, + tool_calls=tool_calls, + metadata={ + "finish_reason": choice.get("finish_reason"), + "choice0": _json_safe(choice), + }, + ) + + +def _post_chat_completion(payload: dict[str, Any], timeout: float | None) -> dict[str, Any]: + headers = {"Content-Type": "application/json"} + if API_KEY: + headers["Authorization"] = f"Bearer {API_KEY}" + req = urllib.request.Request( + _chat_url(), + data=json.dumps(payload, ensure_ascii=False).encode("utf-8"), + headers=headers, + method="POST", + ) + try: + with urllib.request.urlopen(req, timeout=timeout or TIMEOUT_SECONDS) as resp: + raw = resp.read().decode("utf-8") + except urllib.error.HTTPError as e: + body = e.read().decode("utf-8", errors="replace") + raise RuntimeError(f"MiniMax chat API returned HTTP {e.code}: {body}") from e + except urllib.error.URLError as e: + raise RuntimeError(f"MiniMax chat API request failed: {e}") from e + try: + return json.loads(raw) + except json.JSONDecodeError as e: + raise RuntimeError(f"MiniMax chat API returned non-JSON response: {raw[:1000]}") from e + + +def _chat_messages_impl( + messages: list[dict[str, Any]], + max_completion_tokens: int, + retries: int, + stage: str, + *, + tools: list[dict[str, Any]] | None = None, + tool_choice: str | dict[str, Any] | None = None, + return_message: bool = False, + deployment: str | None = None, + timeout: float | None = None, +) -> tuple[Any, dict[str, int]]: + payload: dict[str, Any] = { + "model": deployment or TARGET_DEPLOYMENT, + "messages": _json_safe(messages), + "max_tokens": min(max_completion_tokens, MAX_TOKENS), + } + payload["chat_template_kwargs"] = {"enable_thinking": ENABLE_THINKING} + if TEMPERATURE is not None: + payload["temperature"] = TEMPERATURE + if tools: + payload["tools"] = _json_safe(tools) + if tool_choice is not None: + payload["tool_choice"] = _json_safe(tool_choice) + + last_err: Exception | None = None + for attempt in range(retries): + try: + data = _post_chat_completion(payload, timeout) + choices = data.get("choices") or [] + if not choices: + raise RuntimeError(f"MiniMax chat API returned no choices: {data}") + choice0 = choices[0] + message = choice0.get("message") or {} + text = message.get("content") or "" + if not isinstance(text, str): + text = json.dumps(text, ensure_ascii=False) + usage_info = _usage_from_payload(data) + tracker.record(stage, usage_info["prompt_tokens"], usage_info["completion_tokens"]) + if return_message: + return _compat_message_from_payload(message, choice0), usage_info + return text, usage_info + except Exception as e: # noqa: BLE001 + last_err = e + time.sleep(min(2 ** attempt, 30)) + raise RuntimeError(f"MiniMax chat call failed after {retries} retries: {last_err}") + + +def configure_minimax_chat( + *, + base_url: str | None = None, + api_key: str | None = None, + temperature: float | str | None = None, + timeout_seconds: float | str | None = None, + max_tokens: int | str | None = None, + enable_thinking: bool | str | None = None, +) -> None: + global BASE_URL, API_KEY, TEMPERATURE, TIMEOUT_SECONDS, MAX_TOKENS, ENABLE_THINKING + with _config_lock: + if base_url is not None: + BASE_URL = str(base_url).strip() or BASE_URL + os.environ["MINIMAX_BASE_URL"] = BASE_URL + if api_key is not None: + API_KEY = str(api_key).strip() + os.environ["MINIMAX_API_KEY"] = API_KEY + if temperature is not None: + raw = str(temperature).strip() + TEMPERATURE = float(raw) if raw else None + os.environ["MINIMAX_TEMPERATURE"] = raw + if timeout_seconds is not None: + TIMEOUT_SECONDS = float(timeout_seconds) + os.environ["MINIMAX_TIMEOUT_SECONDS"] = str(timeout_seconds) + if max_tokens is not None: + MAX_TOKENS = int(max_tokens) + os.environ["MINIMAX_MAX_TOKENS"] = str(max_tokens) + if enable_thinking is not None: + if isinstance(enable_thinking, str): + ENABLE_THINKING = enable_thinking.strip().lower() in {"1", "true", "yes", "on"} + else: + ENABLE_THINKING = bool(enable_thinking) + os.environ["MINIMAX_ENABLE_THINKING"] = "true" if ENABLE_THINKING else "false" + + +def get_max_tokens() -> int: + return MAX_TOKENS + + +def chat_target( + system: str, + user: str, + max_completion_tokens: int = 16384, + retries: int = 5, + stage: str = "target", + reasoning_effort: str | None = None, + timeout: float | None = None, +) -> tuple[str, dict[str, int]]: + del reasoning_effort + messages = [{"role": "system", "content": system}, {"role": "user", "content": user}] + return _chat_messages_impl( + messages, + max_completion_tokens, + retries, + stage, + timeout=timeout, + ) + + +def chat_target_messages( + messages: list[dict[str, Any]], + max_completion_tokens: int = 16384, + retries: int = 5, + stage: str = "target", + reasoning_effort: str | None = None, + *, + tools: list[dict[str, Any]] | None = None, + tool_choice: str | dict[str, Any] | None = None, + return_message: bool = False, + timeout: float | None = None, +) -> tuple[Any, dict[str, int]]: + del reasoning_effort + return _chat_messages_impl( + messages, + max_completion_tokens, + retries, + stage, + tools=tools, + tool_choice=tool_choice, + return_message=return_message, + timeout=timeout, + ) + + +def get_token_summary() -> dict[str, dict[str, int]]: + return tracker.summary() + + +def reset_token_tracker() -> None: + tracker.reset() + + +def set_reasoning_effort(effort: str | None) -> None: + del effort + + +def set_target_deployment(deployment: str) -> None: + global TARGET_DEPLOYMENT + TARGET_DEPLOYMENT = deployment or default_model_for_backend("minimax_chat") + os.environ["TARGET_DEPLOYMENT"] = TARGET_DEPLOYMENT \ No newline at end of file diff --git a/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/model/qwen_backend.py b/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/model/qwen_backend.py new file mode 100644 index 00000000..be193d46 --- /dev/null +++ b/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/model/qwen_backend.py @@ -0,0 +1,455 @@ +"""OpenAI-compatible Qwen chat backend for optimizer and target paths.""" +from __future__ import annotations + +from dataclasses import dataclass +import json +import os +import threading +import time +import urllib.error +import urllib.request +from typing import Any + +from skillopt.model.common import ( + CompatAssistantMessage, + CompatToolCall, + CompatToolFunction, + TokenTracker, + default_model_for_backend, +) + + +@dataclass +class QwenChatConfig: + base_url: str + api_key: str + timeout_seconds: float + max_tokens: int + temperature: float | None + enable_thinking: bool + deployment: str + + +def _parse_bool(value: Any, default: bool = False) -> bool: + if value is None: + return default + return str(value).strip().lower() in {"1", "true", "yes", "on"} + + +def _parse_optional_float(value: Any) -> float | None: + if value is None: + return None + raw = str(value).strip() + return float(raw) if raw else None + + +def _parse_int(value: Any, default: int) -> int: + if value is None: + return default + raw = str(value).strip() + return int(raw) if raw else default + + +def _role_env(role: str, key: str, default: str) -> str: + role_key = f"{role.upper()}_QWEN_CHAT_{key}" + generic_key = f"QWEN_CHAT_{key}" + return os.environ.get(role_key) or os.environ.get(generic_key) or default + + +def _initial_config(role: str) -> QwenChatConfig: + role_upper = role.upper() + deployment_env = "OPTIMIZER_DEPLOYMENT" if role == "optimizer" else "TARGET_DEPLOYMENT" + return QwenChatConfig( + base_url=_role_env(role, "BASE_URL", "http://localhost:8000/v1"), + api_key=_role_env(role, "API_KEY", ""), + timeout_seconds=float(_role_env(role, "TIMEOUT_SECONDS", "300") or 300), + max_tokens=_parse_int(_role_env(role, "MAX_TOKENS", "8000"), 8000), + temperature=_parse_optional_float(_role_env(role, "TEMPERATURE", "0.7")), + enable_thinking=_parse_bool(_role_env(role, "ENABLE_THINKING", "false")), + deployment=( + os.environ.get(f"{role_upper}_QWEN_CHAT_MODEL") + or os.environ.get("QWEN_CHAT_MODEL") + or os.environ.get(deployment_env) + or default_model_for_backend("qwen_chat") + ), + ) + + +OPTIMIZER_CONFIG = _initial_config("optimizer") +TARGET_CONFIG = _initial_config("target") + +_config_lock = threading.Lock() +tracker = TokenTracker() + + +def _chat_url(config: QwenChatConfig) -> str: + base = config.base_url.rstrip("/") + if base.endswith("/chat/completions"): + return base + return f"{base}/chat/completions" + + +def _json_safe(value: Any) -> Any: + if value is None or isinstance(value, (str, int, float, bool)): + return value + if isinstance(value, list): + return [_json_safe(item) for item in value] + if isinstance(value, dict): + return {str(key): _json_safe(val) for key, val in value.items()} + model_dump = getattr(value, "model_dump", None) + if callable(model_dump): + try: + return model_dump(mode="json") + except TypeError: + return model_dump() + return str(value) + + +def _usage_from_payload(payload: dict[str, Any]) -> dict[str, int]: + usage = payload.get("usage") or {} + prompt_tokens = int(usage.get("prompt_tokens") or usage.get("input_tokens") or 0) + completion_tokens = int(usage.get("completion_tokens") or usage.get("output_tokens") or 0) + total_tokens = int(usage.get("total_tokens") or (prompt_tokens + completion_tokens)) + return { + "prompt_tokens": prompt_tokens, + "completion_tokens": completion_tokens, + "total_tokens": total_tokens, + } + + +def _compat_message_from_payload(message: dict[str, Any], choice: dict[str, Any]) -> CompatAssistantMessage: + content = message.get("content") or "" + if not isinstance(content, str): + content = json.dumps(content, ensure_ascii=False) + tool_calls: list[CompatToolCall] = [] + for index, tool_call in enumerate(message.get("tool_calls") or [], start=1): + function = tool_call.get("function") or {} + tool_calls.append( + CompatToolCall( + id=str(tool_call.get("id") or f"qwen_tool_{index}"), + type=str(tool_call.get("type") or "function"), + function=CompatToolFunction( + name=str(function.get("name") or ""), + arguments=str(function.get("arguments") or "{}"), + ), + ) + ) + return CompatAssistantMessage( + content=content, + tool_calls=tool_calls, + metadata={ + "finish_reason": choice.get("finish_reason"), + "choice0": _json_safe(choice), + }, + ) + + +def _post_chat_completion( + payload: dict[str, Any], + timeout: float | None, + config: QwenChatConfig, +) -> dict[str, Any]: + headers = {"Content-Type": "application/json"} + if config.api_key: + headers["Authorization"] = f"Bearer {config.api_key}" + req = urllib.request.Request( + _chat_url(config), + data=json.dumps(payload, ensure_ascii=False).encode("utf-8"), + headers=headers, + method="POST", + ) + try: + with urllib.request.urlopen(req, timeout=timeout or config.timeout_seconds) as resp: + raw = resp.read().decode("utf-8") + except urllib.error.HTTPError as e: + body = e.read().decode("utf-8", errors="replace") + raise RuntimeError(f"Qwen chat API returned HTTP {e.code}: {body}") from e + except urllib.error.URLError as e: + raise RuntimeError(f"Qwen chat API request failed: {e}") from e + try: + return json.loads(raw) + except json.JSONDecodeError as e: + raise RuntimeError(f"Qwen chat API returned non-JSON response: {raw[:1000]}") from e + + +def _chat_messages_impl( + messages: list[dict[str, Any]], + max_completion_tokens: int, + retries: int, + stage: str, + *, + role: str, + tools: list[dict[str, Any]] | None = None, + tool_choice: str | dict[str, Any] | None = None, + return_message: bool = False, + deployment: str | None = None, + timeout: float | None = None, +) -> tuple[Any, dict[str, int]]: + config = OPTIMIZER_CONFIG if role == "optimizer" else TARGET_CONFIG + payload: dict[str, Any] = { + "model": deployment or config.deployment, + "messages": _json_safe(messages), + "max_tokens": min(max_completion_tokens, config.max_tokens), + } + payload["chat_template_kwargs"] = {"enable_thinking": config.enable_thinking} + if config.temperature is not None: + payload["temperature"] = config.temperature + if tools: + payload["tools"] = _json_safe(tools) + if tool_choice is not None: + payload["tool_choice"] = _json_safe(tool_choice) + + last_err: Exception | None = None + for attempt in range(retries): + try: + data = _post_chat_completion(payload, timeout, config) + choices = data.get("choices") or [] + if not choices: + raise RuntimeError(f"Qwen chat API returned no choices: {data}") + choice0 = choices[0] + message = choice0.get("message") or {} + text = message.get("content") or "" + if not isinstance(text, str): + text = json.dumps(text, ensure_ascii=False) + usage_info = _usage_from_payload(data) + tracker.record(stage, usage_info["prompt_tokens"], usage_info["completion_tokens"]) + if return_message: + return _compat_message_from_payload(message, choice0), usage_info + return text, usage_info + except Exception as e: # noqa: BLE001 + last_err = e + time.sleep(min(2 ** attempt, 30)) + raise RuntimeError(f"Qwen chat call failed after {retries} retries: {last_err}") + + +def configure_qwen_chat( + *, + base_url: str | None = None, + api_key: str | None = None, + temperature: float | str | None = None, + timeout_seconds: float | str | None = None, + max_tokens: int | str | None = None, + enable_thinking: bool | str | None = None, + optimizer_base_url: str | None = None, + optimizer_api_key: str | None = None, + optimizer_temperature: float | str | None = None, + optimizer_timeout_seconds: float | str | None = None, + optimizer_max_tokens: int | str | None = None, + optimizer_enable_thinking: bool | str | None = None, + target_base_url: str | None = None, + target_api_key: str | None = None, + target_temperature: float | str | None = None, + target_timeout_seconds: float | str | None = None, + target_max_tokens: int | str | None = None, + target_enable_thinking: bool | str | None = None, +) -> None: + with _config_lock: + if base_url is not None: + os.environ["QWEN_CHAT_BASE_URL"] = str(base_url).strip() + if api_key is not None: + os.environ["QWEN_CHAT_API_KEY"] = str(api_key).strip() + if temperature is not None: + os.environ["QWEN_CHAT_TEMPERATURE"] = str(temperature).strip() + if timeout_seconds is not None: + os.environ["QWEN_CHAT_TIMEOUT_SECONDS"] = str(timeout_seconds) + if max_tokens is not None: + os.environ["QWEN_CHAT_MAX_TOKENS"] = str(max_tokens) + if enable_thinking is not None: + os.environ["QWEN_CHAT_ENABLE_THINKING"] = ( + "true" if _parse_bool(enable_thinking) else "false" + ) + _update_config( + OPTIMIZER_CONFIG, + "optimizer", + base_url=optimizer_base_url if optimizer_base_url is not None else base_url, + api_key=optimizer_api_key if optimizer_api_key is not None else api_key, + temperature=( + optimizer_temperature + if optimizer_temperature is not None + else temperature + ), + timeout_seconds=( + optimizer_timeout_seconds + if optimizer_timeout_seconds is not None + else timeout_seconds + ), + max_tokens=optimizer_max_tokens if optimizer_max_tokens is not None else max_tokens, + enable_thinking=( + optimizer_enable_thinking + if optimizer_enable_thinking is not None + else enable_thinking + ), + ) + _update_config( + TARGET_CONFIG, + "target", + base_url=target_base_url if target_base_url is not None else base_url, + api_key=target_api_key if target_api_key is not None else api_key, + temperature=target_temperature if target_temperature is not None else temperature, + timeout_seconds=( + target_timeout_seconds + if target_timeout_seconds is not None + else timeout_seconds + ), + max_tokens=target_max_tokens if target_max_tokens is not None else max_tokens, + enable_thinking=( + target_enable_thinking + if target_enable_thinking is not None + else enable_thinking + ), + ) + + +def _update_config( + config: QwenChatConfig, + role: str, + *, + base_url: str | None = None, + api_key: str | None = None, + temperature: float | str | None = None, + timeout_seconds: float | str | None = None, + max_tokens: int | str | None = None, + enable_thinking: bool | str | None = None, +) -> None: + env_prefix = role.upper() + if base_url is not None: + config.base_url = str(base_url).strip() or config.base_url + os.environ[f"{env_prefix}_QWEN_CHAT_BASE_URL"] = config.base_url + if api_key is not None: + config.api_key = str(api_key).strip() + os.environ[f"{env_prefix}_QWEN_CHAT_API_KEY"] = config.api_key + if temperature is not None: + raw = str(temperature).strip() + config.temperature = float(raw) if raw else None + os.environ[f"{env_prefix}_QWEN_CHAT_TEMPERATURE"] = raw + if timeout_seconds is not None: + config.timeout_seconds = float(timeout_seconds) + os.environ[f"{env_prefix}_QWEN_CHAT_TIMEOUT_SECONDS"] = str(timeout_seconds) + if max_tokens is not None: + config.max_tokens = int(max_tokens) + os.environ[f"{env_prefix}_QWEN_CHAT_MAX_TOKENS"] = str(max_tokens) + if enable_thinking is not None: + config.enable_thinking = _parse_bool(enable_thinking) + os.environ[f"{env_prefix}_QWEN_CHAT_ENABLE_THINKING"] = ( + "true" if config.enable_thinking else "false" + ) + + +def get_max_tokens() -> int: + return TARGET_CONFIG.max_tokens + + +def chat_optimizer( + system: str, + user: str, + max_completion_tokens: int = 16384, + retries: int = 5, + stage: str = "optimizer", + reasoning_effort: str | None = None, + timeout: float | None = None, +) -> tuple[str, dict[str, int]]: + del reasoning_effort + messages = [{"role": "system", "content": system}, {"role": "user", "content": user}] + return _chat_messages_impl( + messages, + max_completion_tokens, + retries, + stage, + role="optimizer", + timeout=timeout, + ) + + +def chat_target( + system: str, + user: str, + max_completion_tokens: int = 16384, + retries: int = 5, + stage: str = "target", + reasoning_effort: str | None = None, + timeout: float | None = None, +) -> tuple[str, dict[str, int]]: + del reasoning_effort + messages = [{"role": "system", "content": system}, {"role": "user", "content": user}] + return _chat_messages_impl( + messages, + max_completion_tokens, + retries, + stage, + role="target", + timeout=timeout, + ) + + +def chat_optimizer_messages( + messages: list[dict[str, Any]], + max_completion_tokens: int = 16384, + retries: int = 5, + stage: str = "optimizer", + reasoning_effort: str | None = None, + *, + tools: list[dict[str, Any]] | None = None, + tool_choice: str | dict[str, Any] | None = None, + return_message: bool = False, + timeout: float | None = None, +) -> tuple[Any, dict[str, int]]: + del reasoning_effort + return _chat_messages_impl( + messages, + max_completion_tokens, + retries, + stage, + role="optimizer", + tools=tools, + tool_choice=tool_choice, + return_message=return_message, + timeout=timeout, + ) + + +def chat_target_messages( + messages: list[dict[str, Any]], + max_completion_tokens: int = 16384, + retries: int = 5, + stage: str = "target", + reasoning_effort: str | None = None, + *, + tools: list[dict[str, Any]] | None = None, + tool_choice: str | dict[str, Any] | None = None, + return_message: bool = False, + timeout: float | None = None, +) -> tuple[Any, dict[str, int]]: + del reasoning_effort + return _chat_messages_impl( + messages, + max_completion_tokens, + retries, + stage, + role="target", + tools=tools, + tool_choice=tool_choice, + return_message=return_message, + timeout=timeout, + ) + + +def get_token_summary() -> dict[str, dict[str, int]]: + return tracker.summary() + + +def reset_token_tracker() -> None: + tracker.reset() + + +def set_reasoning_effort(effort: str | None) -> None: + del effort + + +def set_target_deployment(deployment: str) -> None: + TARGET_CONFIG.deployment = deployment or default_model_for_backend("qwen_chat") + os.environ["TARGET_DEPLOYMENT"] = TARGET_CONFIG.deployment + + +def set_optimizer_deployment(deployment: str) -> None: + OPTIMIZER_CONFIG.deployment = deployment or default_model_for_backend("qwen_chat") + os.environ["OPTIMIZER_DEPLOYMENT"] = OPTIMIZER_CONFIG.deployment diff --git a/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/model/router.py b/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/model/router.py new file mode 100644 index 00000000..08637614 --- /dev/null +++ b/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/model/router.py @@ -0,0 +1,236 @@ +"""Runtime backend router for ReflACT model calls.""" +from __future__ import annotations + +import os +from typing import Any + +from . import azure_openai, claude_backend, codex_backend +from .common import normalize_backend_name + + +_ACTIVE_BACKEND = normalize_backend_name( + os.environ.get("REFLACT_MODEL_BACKEND", "azure_openai") +) + + +def _backend_module(name: str): + if name == "azure_openai": + return azure_openai + if name == "codex": + return codex_backend + if name == "claude": + return claude_backend + raise ValueError(f"Unknown backend: {name!r}") + + +def _all_backend_modules() -> list[Any]: + return [azure_openai, codex_backend, claude_backend] + + +def set_backend(name: str | None) -> str: + """Select the active model backend for subsequent calls.""" + global _ACTIVE_BACKEND + normalized = normalize_backend_name(name) + if normalized not in {"azure_openai", "codex", "claude"}: + valid = ", ".join(sorted({"azure_openai", "codex", "claude"})) + raise ValueError(f"Unknown backend {name!r}. Expected one of: {valid}") + _ACTIVE_BACKEND = normalized + os.environ["REFLACT_MODEL_BACKEND"] = normalized + return _ACTIVE_BACKEND + + +def get_backend_name() -> str: + return _ACTIVE_BACKEND + + +def chat_optimizer( + system: str, + user: str, + max_completion_tokens: int = 16384, + retries: int = 5, + stage: str = "optimizer", + timeout: int | None = None, +) -> tuple[str, dict[str, int]]: + return _backend_module(_ACTIVE_BACKEND).chat_optimizer( + system=system, + user=user, + max_completion_tokens=max_completion_tokens, + retries=retries, + stage=stage, + timeout=timeout, + ) + + +def chat_target( + system: str, + user: str, + max_completion_tokens: int = 16384, + retries: int = 5, + stage: str = "target", + timeout: int | None = None, +) -> tuple[str, dict[str, int]]: + return _backend_module(_ACTIVE_BACKEND).chat_target( + system=system, + user=user, + max_completion_tokens=max_completion_tokens, + retries=retries, + stage=stage, + timeout=timeout, + ) + + +def chat_with_deployment( + deployment: str, + system: str, + user: str, + max_completion_tokens: int = 16384, + retries: int = 5, + stage: str = "custom", + timeout: int | None = None, +) -> tuple[str, dict[str, int]]: + return _backend_module(_ACTIVE_BACKEND).chat_with_deployment( + deployment=deployment, + system=system, + user=user, + max_completion_tokens=max_completion_tokens, + retries=retries, + stage=stage, + timeout=timeout, + ) + + +def chat_optimizer_messages( + messages: list[dict[str, Any]], + max_completion_tokens: int = 16384, + retries: int = 5, + stage: str = "optimizer", + *, + tools: list[dict[str, Any]] | None = None, + tool_choice: str | dict[str, Any] | None = None, + return_message: bool = False, + timeout: int | None = None, +) -> tuple[Any, dict[str, int]]: + return _backend_module(_ACTIVE_BACKEND).chat_optimizer_messages( + messages=messages, + max_completion_tokens=max_completion_tokens, + retries=retries, + stage=stage, + tools=tools, + tool_choice=tool_choice, + return_message=return_message, + timeout=timeout, + ) + + +def chat_target_messages( + messages: list[dict[str, Any]], + max_completion_tokens: int = 16384, + retries: int = 5, + stage: str = "target", + *, + tools: list[dict[str, Any]] | None = None, + tool_choice: str | dict[str, Any] | None = None, + return_message: bool = False, + timeout: int | None = None, +) -> tuple[Any, dict[str, int]]: + return _backend_module(_ACTIVE_BACKEND).chat_target_messages( + messages=messages, + max_completion_tokens=max_completion_tokens, + retries=retries, + stage=stage, + tools=tools, + tool_choice=tool_choice, + return_message=return_message, + timeout=timeout, + ) + + +def chat_messages_with_deployment( + deployment: str, + messages: list[dict[str, Any]], + max_completion_tokens: int = 16384, + retries: int = 5, + stage: str = "custom", + *, + tools: list[dict[str, Any]] | None = None, + tool_choice: str | dict[str, Any] | None = None, + return_message: bool = False, + timeout: int | None = None, +) -> tuple[Any, dict[str, int]]: + return _backend_module(_ACTIVE_BACKEND).chat_messages_with_deployment( + deployment=deployment, + messages=messages, + max_completion_tokens=max_completion_tokens, + retries=retries, + stage=stage, + tools=tools, + tool_choice=tool_choice, + return_message=return_message, + timeout=timeout, + ) + + +def get_token_summary() -> dict[str, dict[str, int]]: + return _backend_module(_ACTIVE_BACKEND).get_token_summary() + + +def reset_token_tracker() -> None: + _backend_module(_ACTIVE_BACKEND).reset_token_tracker() + + +def set_reasoning_effort(effort: str | None) -> None: + for module in _all_backend_modules(): + module.set_reasoning_effort(effort) + + +def set_target_deployment(deployment: str) -> None: + for module in _all_backend_modules(): + module.set_target_deployment(deployment) + + +def set_optimizer_deployment(deployment: str) -> None: + for module in _all_backend_modules(): + module.set_optimizer_deployment(deployment) + + +def configure_azure_openai( + *, + endpoint: str | None = None, + api_version: str | None = None, + api_key: str | None = None, + auth_mode: str | None = None, + ad_scope: str | None = None, + managed_identity_client_id: str | None = None, + optimizer_endpoint: str | None = None, + optimizer_api_version: str | None = None, + optimizer_api_key: str | None = None, + optimizer_auth_mode: str | None = None, + optimizer_ad_scope: str | None = None, + optimizer_managed_identity_client_id: str | None = None, + target_endpoint: str | None = None, + target_api_version: str | None = None, + target_api_key: str | None = None, + target_auth_mode: str | None = None, + target_ad_scope: str | None = None, + target_managed_identity_client_id: str | None = None, +) -> None: + azure_openai.configure_azure_openai( + endpoint=endpoint, + api_version=api_version, + api_key=api_key, + auth_mode=auth_mode, + ad_scope=ad_scope, + managed_identity_client_id=managed_identity_client_id, + optimizer_endpoint=optimizer_endpoint, + optimizer_api_version=optimizer_api_version, + optimizer_api_key=optimizer_api_key, + optimizer_auth_mode=optimizer_auth_mode, + optimizer_ad_scope=optimizer_ad_scope, + optimizer_managed_identity_client_id=optimizer_managed_identity_client_id, + target_endpoint=target_endpoint, + target_api_version=target_api_version, + target_api_key=target_api_key, + target_auth_mode=target_auth_mode, + target_ad_scope=target_ad_scope, + target_managed_identity_client_id=target_managed_identity_client_id, + ) diff --git a/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/optimizer/__init__.py b/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/optimizer/__init__.py new file mode 100644 index 00000000..c9e690bb --- /dev/null +++ b/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/optimizer/__init__.py @@ -0,0 +1,15 @@ +"""SkillOpt Optimizer -- skill update operations. + +Analogous to the optimizer in neural network training: applies the computed +"gradient" (patches) to the current skill document to produce an updated +candidate skill. + +Modules +------- +- skill: edit application (optimizer.step() / parameter update) +- clip: edit ranking and selection (gradient clipping) +- slow_update: longitudinal comparison and guidance (EMA / regularization) +- meta_skill: cross-epoch memory for optimizer context +""" +from skillopt.optimizer.skill import apply_edit, apply_patch # noqa: F401 +from skillopt.optimizer.clip import rank_and_select # noqa: F401 diff --git a/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/optimizer/clip.py b/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/optimizer/clip.py new file mode 100644 index 00000000..7add26d7 --- /dev/null +++ b/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/optimizer/clip.py @@ -0,0 +1,109 @@ +"""ReflACT gradient clipping — LLM-driven edit ranking and selection. + +Analogous to gradient clipping in neural network training: ranks candidate +edits by importance and selects the top-L to apply, controlling the +effective step size. Previously core/select.py. +""" +from __future__ import annotations + +from skillopt.model import chat_optimizer +from skillopt.optimizer.meta_skill import format_meta_skill_context +from skillopt.optimizer.update_modes import ( + describe_item, + get_payload_items, + is_rewrite_mode, + normalize_update_mode, + payload_key, + payload_label, +) +from skillopt.prompts import load_prompt +from skillopt.utils import extract_json + + +# ── Public API ──────────────────────────────────────────────────────────────── + +def rank_and_select( + skill_content: str, + patch: dict, + max_edits: int, + meta_skill_context: str = "", + update_mode: str = "patch", +) -> dict: + """Use a optimizer LLM to rank edits by importance, then keep top-L. + + If the edit pool is within budget, returns the patch unchanged. + Otherwise, calls the optimizer to rank and select the most impactful edits. + + Parameters + ---------- + skill_content : str + Current skill document. + patch : dict + Merged :class:`~skillopt.types.Patch` dict with ``edits`` list. + max_edits : int + Maximum number of edits to keep (the "edit budget"). + + Returns + ------- + dict + :class:`~skillopt.types.Patch` dict with selected edits and + optional ``ranking_details``. + """ + update_mode = normalize_update_mode(update_mode) + edits = get_payload_items(patch, update_mode) + if len(edits) <= max_edits: + return patch + + # Build the edit pool description for the optimizer + edits_desc = [] + for i, edit in enumerate(edits): + edits_desc.append(f"[{i}] {describe_item(edit, update_mode, max_chars=500)}") + + user = ( + f"## Current Skill\n{skill_content}\n\n" + f"## {payload_label(update_mode, title=True)} Pool ({len(edits)} {payload_label(update_mode)}, budget={max_edits})\n" + + "\n".join(edits_desc) + + f"\n\nSelect the {max_edits} most important {payload_label(update_mode)}. " + f"Return their 0-based indices in priority order." + ) + optimizer_ctx = format_meta_skill_context(meta_skill_context) + if optimizer_ctx: + user = f"{optimizer_ctx}\n\n{user}" + prompt_name = "ranking_rewrite" if is_rewrite_mode(update_mode) else "ranking" + + try: + response, _ = chat_optimizer( + system=load_prompt(prompt_name), user=user, + max_completion_tokens=2048, retries=3, stage="ranking", + ) + result = extract_json(response) + if result and "selected_indices" in result: + indices = result["selected_indices"] + selected = [] + seen: set[int] = set() + for idx in indices: + if ( + isinstance(idx, int) + and 0 <= idx < len(edits) + and idx not in seen + ): + selected.append(edits[idx]) + seen.add(idx) + if len(selected) >= max_edits: + break + if selected: + return { + "reasoning": patch.get("reasoning", "") + + f" [optimizer-ranked: selected {len(selected)}/{len(edits)} {payload_label(update_mode)}]", + payload_key(update_mode): selected, + "ranking_details": result, + } + except Exception: # noqa: BLE001 + pass + + # Fallback: simple truncation + return { + "reasoning": patch.get("reasoning", "") + + f" [fallback truncated {len(edits)}->{max_edits} {payload_label(update_mode)}]", + payload_key(update_mode): edits[:max_edits], + } diff --git a/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/optimizer/lr_autonomous.py b/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/optimizer/lr_autonomous.py new file mode 100644 index 00000000..95a4bba9 --- /dev/null +++ b/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/optimizer/lr_autonomous.py @@ -0,0 +1,108 @@ +"""Optimizer-driven autonomous update-size decisions.""" +from __future__ import annotations + +import json +import re +from typing import Any + +from skillopt.model import chat_optimizer +from skillopt.optimizer.meta_skill import format_meta_skill_context +from skillopt.optimizer.update_modes import describe_item, get_payload_items, payload_label +from skillopt.prompts import load_prompt +from skillopt.utils import extract_json + + +def _coerce_nonnegative_int(value: Any) -> int | None: + if isinstance(value, bool): + return None + if isinstance(value, int): + return max(0, value) + if isinstance(value, float) and value.is_integer(): + return max(0, int(value)) + text = str(value or "").strip() + if not text: + return None + match = re.search(r"-?\d+", text) + if not match: + return None + return max(0, int(match.group(0))) + + +def decide_autonomous_learning_rate( + *, + skill_content: str, + merged_patch: dict, + update_mode: str, + rollout_hard: float, + rollout_soft: float, + rollout_n: int, + step_buffer_context: str = "", + meta_skill_context: str = "", +) -> dict: + """Ask the optimizer to choose the number of update items for this step. + + The prompt intentionally avoids default budgets, candidate budget lists, or + scheduler history. The only hard post-processing is validity: the returned + integer is clamped to the available item count. + """ + items = get_payload_items(merged_patch, update_mode) + available = len(items) + item_lines = [ + f"[{idx}] {describe_item(item, update_mode, max_chars=700)}" + for idx, item in enumerate(items) + ] + user = ( + f"## Current Skill\n{skill_content}\n\n" + f"## Current Step Evidence\n" + f"rollout_n={rollout_n}\n" + f"rollout_hard={rollout_hard:.6f}\n" + f"rollout_soft={rollout_soft:.6f}\n" + f"proposed_update_items={available}\n" + f"update_item_type={payload_label(update_mode)}\n\n" + f"## Proposed Update Items\n" + + "\n".join(item_lines) + + "\n\nDecide how many proposed update items should be applied now." + ) + if step_buffer_context.strip(): + user += f"\n\n## Previous Steps in This Epoch\n{step_buffer_context}" + optimizer_ctx = format_meta_skill_context(meta_skill_context) + if optimizer_ctx: + user = f"{optimizer_ctx}\n\n{user}" + + response = "" + parsed: dict | None = None + decision: int | None = None + try: + response, _ = chat_optimizer( + system=load_prompt("lr_autonomous"), + user=user, + max_completion_tokens=2048, + retries=3, + stage="lr_autonomous", + ) + parsed = extract_json(response) + if parsed: + decision = _coerce_nonnegative_int(parsed.get("learning_rate")) + except Exception as exc: # noqa: BLE001 + parsed = {"error": str(exc)} + + fallback = False + if decision is None: + decision = 0 + fallback = True + + chosen = min(decision, available) + record = { + "learning_rate": chosen, + "raw_learning_rate": decision, + "available_update_items": available, + "clamped": chosen != decision, + "fallback": fallback, + "reasoning": (parsed or {}).get("reasoning", ""), + "confidence": (parsed or {}).get("confidence", ""), + "risk_notes": (parsed or {}).get("risk_notes", []), + "raw_response": response, + } + if parsed and "error" in parsed: + record["error"] = parsed["error"] + return record diff --git a/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/optimizer/meta_skill.py b/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/optimizer/meta_skill.py new file mode 100644 index 00000000..6e34ff10 --- /dev/null +++ b/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/optimizer/meta_skill.py @@ -0,0 +1,79 @@ +"""Optimizer-side meta skill memory for cross-epoch optimization guidance. + +This module maintains a compact optimizer-facing memory distilled from +adjacent-epoch skill comparisons. Unlike ``slow_update``, it does not +modify the target skill document. Instead, it produces guidance meant to +improve future optimizer behavior when proposing, merging, and ranking edits. +""" +from __future__ import annotations + +import traceback + +from skillopt.model import chat_optimizer +from skillopt.optimizer.slow_update import format_comparison_text +from skillopt.prompts import load_prompt +from skillopt.utils import extract_json + + +def format_meta_skill_context(meta_skill_content: str) -> str: + """Render optimizer memory into a prompt-ready context block.""" + content = (meta_skill_content or "").strip() + if not content: + return "" + return ( + "## Optimizer Meta Skill\n" + "This is optimizer-side memory distilled from prior epoch transitions in " + "this environment. Use it to improve how you propose, merge, and rank " + "skill edits. Prefer it when the current evidence is ambiguous, but do " + "not force it if the current trajectories clearly contradict it.\n\n" + f"{content}" + ) + + +def run_meta_skill( + prev_skill: str, + curr_skill: str, + comparison_pairs: list[dict], + *, + prev_meta_skill_content: str = "", + system_prompt: str | None = None, +) -> dict | None: + """Produce updated optimizer-side meta skill from adjacent epochs.""" + actual_system = system_prompt if system_prompt is not None else load_prompt("meta_skill") + + prev_meta_section = ( + prev_meta_skill_content.strip() + if prev_meta_skill_content and prev_meta_skill_content.strip() + else "(No previous optimizer meta skill — this is the first update.)" + ) + + comparison_text = format_comparison_text(comparison_pairs) + user = ( + f"## Previous Epoch Last-Step Skill\n{prev_skill}\n\n" + f"## Current Epoch Last-Step Skill\n{curr_skill}\n\n" + f"## Previous Optimizer Meta Skill\n" + f"The following optimizer memory was available during the current epoch. " + f"Reflect on whether it improved or harmed the quality of edits.\n\n" + f"{prev_meta_section}\n\n" + f"## Longitudinal Comparison (same tasks, two last-step skills)\n" + f"{comparison_text}" + ) + + try: + response, _ = chat_optimizer( + system=actual_system, + user=user, + max_completion_tokens=16384, + retries=3, + stage="meta_skill", + ) + result = extract_json(response) + if result and result.get("meta_skill_content"): + return { + "reasoning": str(result.get("reasoning", "")).strip(), + "meta_skill_content": str(result["meta_skill_content"]).strip(), + } + except Exception: # noqa: BLE001 + traceback.print_exc() + + return None diff --git a/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/optimizer/rewrite.py b/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/optimizer/rewrite.py new file mode 100644 index 00000000..f8b062ba --- /dev/null +++ b/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/optimizer/rewrite.py @@ -0,0 +1,59 @@ +"""Optimizer-driven full skill rewrite from selected revise_suggestions.""" +from __future__ import annotations + +import json + +from skillopt.model import chat_optimizer +from skillopt.prompts import load_prompt +from skillopt.optimizer.update_modes import get_payload_items +from skillopt.utils import extract_json + + +def rewrite_skill_from_suggestions( + skill_content: str, + patch: dict, + *, + system_prompt: str | None = None, + step_buffer_context: str = "", + env: str | None = None, + reasoning_effort: str | None = "high", + max_completion_tokens: int = 64000, +) -> dict | None: + suggestions = get_payload_items(patch, "rewrite_from_suggestions") + if not suggestions: + return None + + user = ( + f"## Current Skill\n{skill_content}\n\n" + f"## Selected Revise Suggestions ({len(suggestions)} total)\n" + f"{json.dumps(suggestions, ensure_ascii=False, indent=2)}\n\n" + ) + if step_buffer_context.strip(): + user += f"## Previous Steps in This Epoch\n{step_buffer_context}\n\n" + user += ( + "Rewrite the full skill document so it integrates the selected suggestions. " + "Return the complete new skill in `new_skill`." + ) + + actual_system = system_prompt if system_prompt is not None else load_prompt( + "rewrite_skill", env=env, + ) + + try: + response, _ = chat_optimizer( + system=actual_system, + user=user, + max_completion_tokens=max_completion_tokens, + retries=3, + stage="rewrite", + reasoning_effort=reasoning_effort, + ) + result = extract_json(response) + if result and str(result.get("new_skill", "")).strip(): + result["new_skill"] = str(result["new_skill"]).rstrip() + "\n" + if "change_summary" not in result or not isinstance(result["change_summary"], list): + result["change_summary"] = [] + return result + except Exception: # noqa: BLE001 + return None + return None diff --git a/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/optimizer/scheduler.py b/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/optimizer/scheduler.py new file mode 100644 index 00000000..63b944e0 --- /dev/null +++ b/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/optimizer/scheduler.py @@ -0,0 +1,127 @@ +"""Learning-rate (edit budget) schedulers for ReflACT. + +The "learning rate" in ReflACT is the maximum number of skill edits allowed +per optimization step. A scheduler controls how this budget changes over +the course of training. + +Supported modes +--------------- +- ``constant`` : Fixed budget throughout training. +- ``linear`` : Linear decay from ``max_lr`` to ``min_lr``. +- ``cosine`` : Cosine annealing from ``max_lr`` to ``min_lr``. +- ``autonomous`` : No limit — the model decides how many edits to make. + +Usage:: + + scheduler = build_scheduler(cfg) + for step in range(1, total_steps + 1): + lr = scheduler.step() # returns edit budget for this step + # ... use lr as max_edits ... +""" +from __future__ import annotations + +import math +from abc import ABC, abstractmethod + + +class LRScheduler(ABC): + """Base class for edit-budget schedulers.""" + + def __init__(self, max_lr: int, min_lr: int, total_steps: int) -> None: + self.max_lr = max_lr + self.min_lr = min_lr + self.total_steps = total_steps + self._current_step = 0 + + @abstractmethod + def _compute_lr(self, step: int) -> int: + """Return the edit budget for the given 1-indexed step.""" + + def step(self) -> int: + """Advance one step and return the edit budget.""" + self._current_step += 1 + return self._compute_lr(self._current_step) + + def get_lr(self, step: int) -> int: + """Return the edit budget for an arbitrary step (1-indexed).""" + return self._compute_lr(step) + + def state_dict(self) -> dict: + return {"current_step": self._current_step} + + def load_state_dict(self, state: dict) -> None: + self._current_step = state.get("current_step", 0) + + +class ConstantScheduler(LRScheduler): + """Fixed edit budget throughout training.""" + + def _compute_lr(self, step: int) -> int: + return self.max_lr + + +class LinearScheduler(LRScheduler): + """Linear decay from ``max_lr`` to ``min_lr`` over ``total_steps``.""" + + def _compute_lr(self, step: int) -> int: + if self.total_steps <= 1: + return self.max_lr + t = min(step, self.total_steps) / self.total_steps + lr = self.max_lr + (self.min_lr - self.max_lr) * t + return max(self.min_lr, round(lr)) + + +class CosineScheduler(LRScheduler): + """Cosine annealing from ``max_lr`` to ``min_lr`` over ``total_steps``.""" + + def _compute_lr(self, step: int) -> int: + if self.total_steps <= 1: + return self.max_lr + t = min(step, self.total_steps) / self.total_steps + lr = self.min_lr + 0.5 * (self.max_lr - self.min_lr) * (1 + math.cos(math.pi * t)) + return max(self.min_lr, round(lr)) + + +class AutonomousScheduler(LRScheduler): + """No edit limit — the model decides freely.""" + + NO_LIMIT = 999 + + def _compute_lr(self, step: int) -> int: + return self.NO_LIMIT + + +# ── Factory ────────────────────────────────────────────────────────────── + +_REGISTRY: dict[str, type[LRScheduler]] = { + "constant": ConstantScheduler, + "linear": LinearScheduler, + "cosine": CosineScheduler, + "autonomous": AutonomousScheduler, +} + + +def build_scheduler( + mode: str = "constant", + max_lr: int = 8, + min_lr: int = 2, + total_steps: int = 8, +) -> LRScheduler: + """Build a scheduler from config parameters. + + Parameters + ---------- + mode : str + One of ``constant``, ``linear``, ``cosine``, ``autonomous``. + max_lr : int + Initial / maximum edit budget. + min_lr : int + Minimum edit budget (for decay modes). + total_steps : int + Total number of optimization steps in training. + """ + if mode not in _REGISTRY: + raise ValueError( + f"Unknown scheduler mode '{mode}'. Available: {list(_REGISTRY.keys())}" + ) + return _REGISTRY[mode](max_lr=max_lr, min_lr=min_lr, total_steps=total_steps) diff --git a/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/optimizer/select.py b/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/optimizer/select.py new file mode 100644 index 00000000..fc49eeb0 --- /dev/null +++ b/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/optimizer/select.py @@ -0,0 +1,4 @@ +"""Backward-compat stub — moved to skillopt.optimizer.clip.""" +from skillopt.optimizer.clip import rank_and_select # noqa: F401 + +__all__ = ["rank_and_select"] diff --git a/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/optimizer/skill.py b/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/optimizer/skill.py new file mode 100644 index 00000000..0a8855f9 --- /dev/null +++ b/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/optimizer/skill.py @@ -0,0 +1,164 @@ +"""ReflACT skill operations — edit application and patch processing. + +The Update stage (⑤) of the ReflACT pipeline: apply a ranked set of +edits to the current skill document, producing an updated candidate. +Analogous to optimizer.step() in neural network training. +""" +from __future__ import annotations + +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from skillopt.types import Edit as EditType, Patch as PatchType + +SLOW_UPDATE_START = "" +SLOW_UPDATE_END = "" + + +def _is_in_slow_update_region(skill: str, target: str) -> bool: + """Check if *target* text falls within the protected slow update region.""" + start_idx = skill.find(SLOW_UPDATE_START) + end_idx = skill.find(SLOW_UPDATE_END) + if start_idx == -1 or end_idx == -1: + return False + target_idx = skill.find(target) + if target_idx == -1: + return False + region_end = end_idx + len(SLOW_UPDATE_END) + return start_idx <= target_idx < region_end + + +def _strip_slow_update_markers(text: str) -> str: + """Remove any SLOW_UPDATE markers from edit content to prevent duplication.""" + return ( + text.replace(SLOW_UPDATE_START, "") + .replace(SLOW_UPDATE_END, "") + ) + + +def _edit_fields(edit: EditType | dict) -> tuple[str, str, str]: + op = edit.op if hasattr(edit, "op") else edit.get("op", "") + content = _strip_slow_update_markers( + (edit.content if hasattr(edit, "content") else edit.get("content", "")).strip() + ) + target = edit.target if hasattr(edit, "target") else edit.get("target", "") + return op, content, target + + +def _apply_edit_with_report(skill: str, edit: EditType | dict) -> tuple[str, dict]: + op, content, target = _edit_fields(edit) + report = { + "op": op, + "target": target[:200], + "content_preview": content[:200], + "status": "unknown", + } + + if target and _is_in_slow_update_region(skill, target): + report["status"] = "skipped_protected_slow_update_region" + return skill, report + + if op == "append": + su_start = skill.find(SLOW_UPDATE_START) + if su_start != -1: + before = skill[:su_start].rstrip() + after = skill[su_start:] + report["status"] = "applied_append_before_slow_update" + return before + "\n\n" + content + "\n\n" + after, report + report["status"] = "applied_append" + return skill.rstrip() + "\n\n" + content + "\n", report + + if op == "insert_after": + if not target or target not in skill: + su_start = skill.find(SLOW_UPDATE_START) + if su_start != -1: + before = skill[:su_start].rstrip() + after = skill[su_start:] + report["status"] = "applied_insert_after_fallback_before_slow_update" + return before + "\n\n" + content + "\n\n" + after, report + report["status"] = "applied_insert_after_fallback_append" + return skill.rstrip() + "\n\n" + content + "\n", report + idx = skill.index(target) + len(target) + newline = skill.find("\n", idx) + insert_at = newline + 1 if newline != -1 else len(skill) + report["status"] = "applied_insert_after" + return skill[:insert_at] + "\n" + content + "\n" + skill[insert_at:], report + + if op == "replace": + if not target: + report["status"] = "skipped_replace_missing_target" + return skill, report + if target not in skill: + report["status"] = "skipped_replace_target_not_found" + return skill, report + report["status"] = "applied_replace" + return skill.replace(target, content, 1), report + + if op == "delete": + if not target: + report["status"] = "skipped_delete_missing_target" + return skill, report + if target not in skill: + report["status"] = "skipped_delete_target_not_found" + return skill, report + report["status"] = "applied_delete" + return skill.replace(target, "", 1), report + + report["status"] = "skipped_unknown_op" + return skill, report + + +def apply_edit(skill: str, edit: EditType | dict) -> str: + """Apply a single edit operation to the skill document. + + Parameters + ---------- + skill : str + Current skill document content. + edit : Edit | dict + An :class:`~skillopt.types.Edit` instance or a plain dict with + keys ``op``, ``content``, ``target``. + + Edits targeting the protected slow-update region are silently skipped. + """ + updated_skill, _ = _apply_edit_with_report(skill, edit) + return updated_skill + + +def apply_patch_with_report( + skill: str, + patch: PatchType | dict, +) -> tuple[str, list[dict]]: + """Apply a patch and return a per-edit report for observability.""" + edits = patch.edits if hasattr(patch, "edits") else patch.get("edits", []) + reports: list[dict] = [] + for idx, edit in enumerate(edits, 1): + try: + skill, report = _apply_edit_with_report(skill, edit) + report["index"] = idx + except Exception as exc: # noqa: BLE001 + report = { + "index": idx, + "op": "", + "target": "", + "content_preview": "", + "status": "error", + "error": str(exc), + } + reports.append(report) + return skill, reports + + +def apply_patch(skill: str, patch: PatchType | dict) -> str: + """Apply a patch (list of edits) to the skill document sequentially. + + Parameters + ---------- + skill : str + Current skill document content. + patch : Patch | dict + A :class:`~skillopt.types.Patch` instance or a plain dict with + key ``edits`` containing a list of edit operations. + """ + updated_skill, _ = apply_patch_with_report(skill, patch) + return updated_skill diff --git a/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/optimizer/slow_update.py b/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/optimizer/slow_update.py new file mode 100644 index 00000000..3d349544 --- /dev/null +++ b/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/optimizer/slow_update.py @@ -0,0 +1,396 @@ +"""ReflACT Slow Update — epoch-level longitudinal skill refinement. + +At the end of each epoch, the slow update compares rollout performance of the +same sample set under the previous epoch's skill vs. the current epoch's skill +(Markov: only adjacent epochs). A optimizer analyzes regressions, improvements, +and persistent failures, then writes a free-form guidance block into a +**protected** section of the skill document. This section cannot be modified by +step-level analyst edits — only the slow update process overwrites it. + +Public API +---------- +- :func:`inject_empty_slow_update_field` — add empty placeholder (epoch 1) +- :func:`extract_slow_update_field` — read current content +- :func:`replace_slow_update_field` — overwrite content +- :func:`has_slow_update_field` — check if markers are present +- :func:`build_comparison_text` — format side-by-side rollout results +- :func:`run_slow_update` — optimizer call to produce guidance +""" +from __future__ import annotations + +import json +import os +import traceback + +from skillopt.model import chat_optimizer +from skillopt.prompts import load_prompt +from skillopt.utils import extract_json + +# ── Protected field markers ───────────────────────────────────────────────── + +SLOW_UPDATE_START = "" +SLOW_UPDATE_END = "" + +# ── Field manipulation helpers ────────────────────────────────────────────── + + +def has_slow_update_field(skill: str) -> bool: + return SLOW_UPDATE_START in skill and SLOW_UPDATE_END in skill + + +def inject_empty_slow_update_field(skill: str) -> str: + if has_slow_update_field(skill): + return skill + block = ( + f"\n\n{SLOW_UPDATE_START}\n" + f"{SLOW_UPDATE_END}\n" + ) + return skill.rstrip() + block + + +def extract_slow_update_field(skill: str) -> str: + start = skill.find(SLOW_UPDATE_START) + end = skill.find(SLOW_UPDATE_END) + if start == -1 or end == -1: + return "" + inner_start = start + len(SLOW_UPDATE_START) + return skill[inner_start:end].strip() + + +def _strip_all_slow_update_fields(skill: str) -> str: + """Remove every SLOW_UPDATE_START/END pair (and content between) from *skill*.""" + while True: + start = skill.find(SLOW_UPDATE_START) + if start == -1: + break + end = skill.find(SLOW_UPDATE_END, start) + if end == -1: + # Orphan start marker — remove it + skill = skill[:start] + skill[start + len(SLOW_UPDATE_START):] + break + skill = skill[:start] + skill[end + len(SLOW_UPDATE_END):] + # Clean up stray end markers + skill = skill.replace(SLOW_UPDATE_END, "") + # Collapse excess blank lines left behind + while "\n\n\n" in skill: + skill = skill.replace("\n\n\n", "\n\n") + return skill.rstrip() + + +def replace_slow_update_field(skill: str, new_content: str) -> str: + # Remove all existing slow update regions first to guarantee exactly one. + skill = _strip_all_slow_update_fields(skill) + block = ( + f"\n\n{SLOW_UPDATE_START}\n" + f"{new_content.strip()}\n" + f"{SLOW_UPDATE_END}\n" + ) + return skill + block + + +# ── Comparison text builder ───────────────────────────────────────────────── + + +# NOTE: The character limits below (whole-trajectory cap + the per-field caps in +# _read_trajectory and the comparison metadata) only trim the comparison samples +# fed to the slow-update optimizer. They exist to cut token usage and speed up the +# call; they do NOT affect what gets written into the skill. If you need richer +# context for the longitudinal comparison, feel free to raise them. +_MAX_TRAJ_CHARS = 3000 + + +def _clip_text(value, limit: int) -> str: + if value is None: + return "" + return str(value)[:limit] + + +def _read_trajectory(rollout_dir: str, task_id: str) -> str: + """Read and format a single trajectory from a rollout directory.""" + conv_path = os.path.join(rollout_dir, "predictions", task_id, "conversation.json") + if not os.path.exists(conv_path): + return "(trajectory not available)" + try: + with open(conv_path) as f: + conversation = json.load(f) + except Exception: + return "(trajectory read error)" + if not conversation: + return "(empty trajectory)" + + lines: list[str] = [] + for entry in conversation: + if not isinstance(entry, dict): + continue + # Per-field caps (cmd/obs/reasoning/etc.) keep each trajectory compact to + # save tokens / time; raise them if you want fuller step detail. + if entry.get("type") == "tool_call": + cmd = _clip_text(entry.get("cmd"), 500) + obs = _clip_text(entry.get("obs"), 800) + lines.append(f"[action] {cmd}") + lines.append(f"[obs] {obs}") + elif "action" in entry and "env_feedback" in entry: + step = entry.get("step", "?") + reasoning = _clip_text(entry.get("reasoning"), 300) + action = _clip_text(entry.get("action"), 200) + feedback = _clip_text(entry.get("env_feedback"), 500) + if reasoning: + lines.append(f"[step {step} think] {reasoning}") + lines.append(f"[step {step} action] {action}") + lines.append(f"[step {step} obs] {feedback}") + elif entry.get("role") == "system": + msg = _clip_text(entry.get("content"), 1000) + lines.append(f"[verification] {msg}") + else: + msg = _clip_text(entry.get("content"), 500) + role = entry.get("role", "agent") + lines.append(f"[{role}] {msg}") + + text = "\n".join(lines) + if len(text) > _MAX_TRAJ_CHARS: + half = _MAX_TRAJ_CHARS // 2 + text = text[:half] + "\n...[truncated]...\n" + text[-half:] + return text + + +# ── Structured comparison pairs ───────────────────────────────────────────── + + +def build_comparison_pairs( + results_prev: list[dict], + results_curr: list[dict], + items: list[dict], + prev_rollout_dir: str = "", + curr_rollout_dir: str = "", +) -> list[dict]: + """Build a structured list of per-sample comparison entries. + + Each entry bundles the original item, both rollout results, the change + category, and both trajectories into one dict — the single source of + truth for this sample's longitudinal comparison. + + Returns + ------- + list[dict] + One dict per sample with keys: + ``id, task, category, prev, curr, prev_trajectory, curr_trajectory`` + """ + prev_by_id = {str(r["id"]): r for r in results_prev} + curr_by_id = {str(r["id"]): r for r in results_curr} + + pairs: list[dict] = [] + for item in items: + tid = str(item.get("id", "")) + prev = prev_by_id.get(tid, {}) + curr = curr_by_id.get(tid, {}) + prev_ok = bool(prev.get("hard", 0)) + curr_ok = bool(curr.get("hard", 0)) + + if not prev_ok and curr_ok: + category = "improved" + elif prev_ok and not curr_ok: + category = "regressed" + elif not prev_ok and not curr_ok: + category = "persistent_fail" + else: + category = "stable_success" + + pairs.append({ + "id": tid, + "task": item.get("question", item.get("task_description", item.get("instruction", tid))), + "category": category, + "prev": { + "hard": int(prev_ok), + "soft": float(prev.get("soft", 0.0)), + "predicted_answer": prev.get("predicted_answer", prev.get("answer", "N/A")), + "fail_reason": prev.get("fail_reason", ""), + }, + "curr": { + "hard": int(curr_ok), + "soft": float(curr.get("soft", 0.0)), + "predicted_answer": curr.get("predicted_answer", curr.get("answer", "N/A")), + "fail_reason": curr.get("fail_reason", ""), + }, + "prev_trajectory": ( + _read_trajectory(prev_rollout_dir, tid) if prev_rollout_dir else "" + ), + "curr_trajectory": ( + _read_trajectory(curr_rollout_dir, tid) if curr_rollout_dir else "" + ), + }) + + return pairs + + +def save_comparison_pairs(pairs: list[dict], out_path: str) -> None: + """Persist comparison pairs to JSON (without trajectory text to save space).""" + slim = [] + for p in pairs: + slim.append({ + "id": p["id"], + "task": p["task"][:300], + "category": p["category"], + "prev": p["prev"], + "curr": p["curr"], + }) + with open(out_path, "w") as f: + json.dump(slim, f, ensure_ascii=False, indent=2) + + +def format_comparison_text(pairs: list[dict]) -> str: + """Format structured comparison pairs into optimizer-readable text.""" + by_cat: dict[str, list[dict]] = { + "regressed": [], + "persistent_fail": [], + "improved": [], + "stable_success": [], + } + for p in pairs: + by_cat.setdefault(p["category"], []).append(p) + + total = len(pairs) + parts = [ + f"## Longitudinal Comparison Summary\n" + f"Total samples: {total}\n" + f"- Improved (wrong→right): {len(by_cat['improved'])}\n" + f"- Regressed (right→wrong): {len(by_cat['regressed'])}\n" + f"- Persistent failures (wrong→wrong): {len(by_cat['persistent_fail'])}\n" + f"- Stable successes (right→right): {len(by_cat['stable_success'])}\n" + ] + + categories = [ + ("regressed", "Regressions (right→wrong) — HIGHEST PRIORITY", True), + ("persistent_fail", "Persistent Failures (wrong→wrong)", True), + ("improved", "Improvements (wrong→right)", True), + ("stable_success", "Stable Successes (right→right)", False), + ] + + for cat_key, label, show_traj in categories: + entries = by_cat[cat_key] + if not entries: + parts.append(f"### {label}\n(none)\n") + continue + + lines = [f"### {label}"] + for e in entries: + prev = e["prev"] + curr = e["curr"] + lines.append( + f"\n#### Task {e['id']}: {e['task'][:300]}\n" + f"- Prev epoch: {'PASS' if prev['hard'] else 'FAIL'} " + f"(soft={prev['soft']:.2f}) — answer: {str(prev['predicted_answer'])[:200]}\n" + f"- Curr epoch: {'PASS' if curr['hard'] else 'FAIL'} " + f"(soft={curr['soft']:.2f}) — answer: {str(curr['predicted_answer'])[:200]}" + ) + if curr.get("fail_reason"): + lines.append(f"- Curr fail reason: {curr['fail_reason'][:300]}") + if prev.get("fail_reason") and not prev["hard"]: + lines.append(f"- Prev fail reason: {prev['fail_reason'][:300]}") + + if show_traj: + if e.get("prev_trajectory"): + lines.append( + f"\n**Previous epoch trajectory:**\n```\n{e['prev_trajectory']}\n```" + ) + if e.get("curr_trajectory"): + lines.append( + f"\n**Current epoch trajectory:**\n```\n{e['curr_trajectory']}\n```" + ) + + parts.append("\n".join(lines)) + + return "\n\n".join(parts) + + + +# ── Optimizer call ──────────────────────────────────────────────────────────── + + +def run_slow_update( + skill_content: str, + results_prev: list[dict], + results_curr: list[dict], + items: list[dict], + *, + prev_skill: str = "", + prev_slow_update_content: str = "", + prev_rollout_dir: str = "", + curr_rollout_dir: str = "", + comparison_pairs: list[dict] | None = None, + system_prompt: str | None = None, +) -> dict | None: + """Run the slow update optimizer call for one epoch boundary. + + Parameters + ---------- + skill_content : str + Current epoch's skill (after fast updates). + results_prev : list[dict] + Rollout results of the 20 samples under previous epoch's skill. + results_curr : list[dict] + Rollout results of the 20 samples under current epoch's skill. + items : list[dict] + The 20 sample items used for comparison. + prev_skill : str + Previous epoch's skill content. + prev_slow_update_content : str + The slow update guidance from the previous epoch (to reflect on). + prev_rollout_dir : str + Path to previous epoch rollout output (contains predictions/). + curr_rollout_dir : str + Path to current epoch rollout output (contains predictions/). + system_prompt : str | None + Custom system prompt override. + + Returns + ------- + dict | None + Conforms to :class:`~skillopt.types.SlowUpdateResult`: + ``{"reasoning": str, "slow_update_content": str}`` or ``None``. + """ + actual_system = system_prompt if system_prompt is not None else load_prompt("slow_update") + + pairs = comparison_pairs + if pairs is None: + pairs = build_comparison_pairs( + results_prev, results_curr, items, + prev_rollout_dir=prev_rollout_dir, + curr_rollout_dir=curr_rollout_dir, + ) + comparison_text = format_comparison_text(pairs) + + prev_guidance_section = ( + prev_slow_update_content.strip() + if prev_slow_update_content and prev_slow_update_content.strip() + else "(No previous guidance — this is the first slow update.)" + ) + + user = ( + f"## Previous Epoch's Skill\n{prev_skill}\n\n" + f"## Current Epoch's Skill\n{skill_content}\n\n" + f"## Previous Slow Update Guidance\n" + f"The following guidance was active during the current epoch. " + f"Reflect on its effectiveness before writing the new version.\n\n" + f"{prev_guidance_section}\n\n" + f"## Longitudinal Comparison (same 20 tasks, two skill versions)\n" + f"{comparison_text}" + ) + + try: + response, _ = chat_optimizer( + system=actual_system, + user=user, + max_completion_tokens=16384, + retries=3, + stage="slow_update", + ) + result = extract_json(response) + if result and result.get("slow_update_content"): + return { + "reasoning": str(result.get("reasoning", "")).strip(), + "slow_update_content": str(result["slow_update_content"]).strip(), + } + except Exception: # noqa: BLE001 + traceback.print_exc() + + return None diff --git a/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/optimizer/update_modes.py b/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/optimizer/update_modes.py new file mode 100644 index 00000000..59dddda6 --- /dev/null +++ b/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/optimizer/update_modes.py @@ -0,0 +1,136 @@ +"""Helpers for switching between patch edits and rewrite-from-suggestions.""" +from __future__ import annotations + +from typing import Any + +PATCH_MODE = "patch" +REWRITE_MODE = "rewrite_from_suggestions" +FULL_REWRITE_MINIBATCH_MODE = "full_rewrite_minibatch" + + +def normalize_update_mode(mode: str | None) -> str: + raw = str(mode or PATCH_MODE).strip().lower() + aliases = { + "patch": PATCH_MODE, + "edits": PATCH_MODE, + "rewrite": REWRITE_MODE, + "rewrite_from_suggestions": REWRITE_MODE, + "suggestions": REWRITE_MODE, + "rewrite_suggestions": REWRITE_MODE, + "full_rewrite": FULL_REWRITE_MINIBATCH_MODE, + "full_rewrite_minibatch": FULL_REWRITE_MINIBATCH_MODE, + "minibatch_full_rewrite": FULL_REWRITE_MINIBATCH_MODE, + "skill_rewrite_minibatch": FULL_REWRITE_MINIBATCH_MODE, + } + return aliases.get(raw, PATCH_MODE) + + +def is_rewrite_mode(mode: str | None) -> bool: + return normalize_update_mode(mode) == REWRITE_MODE + + +def is_full_rewrite_minibatch_mode(mode: str | None) -> bool: + return normalize_update_mode(mode) == FULL_REWRITE_MINIBATCH_MODE + + +def payload_key(mode: str | None) -> str: + if is_full_rewrite_minibatch_mode(mode): + return "skill_candidates" + return "revise_suggestions" if is_rewrite_mode(mode) else "edits" + + +def payload_label(mode: str | None, *, singular: bool = False, title: bool = False) -> str: + if is_full_rewrite_minibatch_mode(mode): + word = "skill candidate" if singular else "skill candidates" + elif is_rewrite_mode(mode): + word = "suggestion" if singular else "suggestions" + else: + word = "edit" if singular else "edits" + return word.title() if title else word + + +def get_payload_items(container: dict | None, mode: str | None) -> list[dict]: + if not isinstance(container, dict): + return [] + items = container.get(payload_key(mode), []) + return items if isinstance(items, list) else [] + + +def set_payload_items(container: dict, items: list[dict], mode: str | None) -> dict: + container[payload_key(mode)] = items + return container + + +def truncate_payload(container: dict, max_items: int, mode: str | None) -> dict: + if max_items < 0: + return container + items = get_payload_items(container, mode) + if len(items) > max_items: + set_payload_items(container, items[:max_items], mode) + return container + + +def describe_item(item: dict, mode: str | None, *, max_chars: int = 240) -> str: + if not isinstance(item, dict): + return "" + if is_full_rewrite_minibatch_mode(mode): + parts = [ + f"title={item.get('title', '')!r}", + f"change_summary={item.get('change_summary', [])!r}", + ] + if item.get("source_type"): + parts.append(f"source={item.get('source_type')}") + if item.get("support_count") is not None: + parts.append(f"support={item.get('support_count')}") + new_skill = str(item.get("new_skill", "")).strip() + if new_skill: + parts.append(f"new_skill_preview={new_skill[:120]!r}") + text = " ".join(parts) + elif is_rewrite_mode(mode): + parts = [ + f"type={item.get('type', '?')}", + f"title={item.get('title', '')!r}", + f"instruction={item.get('instruction', '')!r}", + ] + if item.get("priority_hint"): + parts.append(f"priority={item.get('priority_hint')}") + if item.get("support_count") is not None: + parts.append(f"support={item.get('support_count')}") + text = " ".join(parts) + else: + op = item.get("op", "?") + target = item.get("target", "") + content = item.get("content", "") + parts = [f"op={op}"] + if target: + parts.append(f"target={target!r}") + if content: + parts.append(f"content={content!r}") + if item.get("support_count") is not None: + parts.append(f"support={item.get('support_count')}") + text = " ".join(parts) + if len(text) <= max_chars: + return text + return text[: max_chars - 3].rstrip() + "..." + + +def short_item_summary(item: dict, mode: str | None, *, max_chars: int = 200) -> dict[str, Any]: + if is_full_rewrite_minibatch_mode(mode): + return { + "title": str(item.get("title", ""))[:max_chars], + "change_summary": [ + str(x)[:max_chars] for x in item.get("change_summary", [])[:3] + ] if isinstance(item.get("change_summary"), list) else [], + "source_type": item.get("source_type", ""), + } + if is_rewrite_mode(mode): + return { + "type": item.get("type", "?"), + "title": str(item.get("title", ""))[:max_chars], + "instruction": str(item.get("instruction", ""))[:max_chars], + } + return { + "op": item.get("op", "?"), + "content": str(item.get("content", ""))[:max_chars], + "target": item.get("target", ""), + } diff --git a/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/prompts/__init__.py b/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/prompts/__init__.py new file mode 100644 index 00000000..af1bc58a --- /dev/null +++ b/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/prompts/__init__.py @@ -0,0 +1,63 @@ +"""Prompt loading utilities for ReflACT. + +Prompts are stored as ``.md`` files and loaded at runtime: + +- **Generic** prompts live in ``skillopt/prompts/*.md`` +- **Env-specific** prompts live in ``skillopt/envs//prompts/*.md`` + +``load_prompt(name, env)`` tries the env-specific path first, then falls +back to the generic default. +""" +from __future__ import annotations + +import os + +_PROMPTS_DIR = os.path.dirname(os.path.abspath(__file__)) +_REFLACT_DIR = os.path.dirname(_PROMPTS_DIR) + +_cache: dict[str, str] = {} + + +def _read_file(path: str) -> str | None: + if path in _cache: + return _cache[path] + if not os.path.isfile(path): + return None + with open(path, encoding="utf-8") as f: + content = f.read() + _cache[path] = content + return content + + +def load_prompt(name: str, env: str | None = None) -> str: + """Load a prompt by name with env-specific override and generic fallback. + + Lookup order: + 1. ``skillopt/envs/{env}/prompts/{name}.md`` (if *env* given) + 2. ``skillopt/prompts/{name}.md`` (generic default) + + Raises ``FileNotFoundError`` if neither path exists. + """ + if env is not None: + env_path = os.path.join(_REFLACT_DIR, "envs", env, "prompts", f"{name}.md") + content = _read_file(env_path) + if content is not None: + return content + + generic_path = os.path.join(_PROMPTS_DIR, f"{name}.md") + content = _read_file(generic_path) + if content is not None: + return content + + searched = [] + if env is not None: + searched.append(os.path.join("skillopt/envs", env, "prompts", f"{name}.md")) + searched.append(f"skillopt/prompts/{name}.md") + raise FileNotFoundError( + f"Prompt '{name}' not found. Searched: {', '.join(searched)}" + ) + + +def clear_cache() -> None: + """Clear the prompt file cache (useful for testing).""" + _cache.clear() diff --git a/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/prompts/analyst_error.md b/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/prompts/analyst_error.md new file mode 100644 index 00000000..af1c0c5e --- /dev/null +++ b/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/prompts/analyst_error.md @@ -0,0 +1,41 @@ +You are an expert failure-analysis agent for AI agent tasks. + +You will be given MULTIPLE failed agent trajectories from a single minibatch +and the current skill document. +Your job is to identify the most important COMMON failure patterns across +the batch and propose a concise set of skill edits. + +## Analysis Process +1. Read ALL trajectories in the minibatch. +2. Identify the most prevalent, systematic failure patterns across them. +3. For each pattern, classify its failure type. +4. Propose skill edits that address the COMMON patterns — not individual edge cases. +5. Edits must be generalizable; do not hardcode task-specific values. +6. Only patch gaps in the skill — do not duplicate existing content. + +You will be told the maximum number of edits (the budget L). Produce AT MOST L edits, +focusing on the highest-impact patterns. You may produce fewer if warranted. + +Respond ONLY with a valid JSON object (no markdown fences, no extra text): +{ + "batch_size": , + "failure_summary": [ + {"failure_type": "", "count": , "description": ""} + ], + "patch": { + "reasoning": "", + "edits": [ + {"op": "append", "content": ""}, + {"op": "insert_after", "target": "", "content": ""}, + {"op": "replace", "target": "", "content": ""}, + {"op": "delete", "target": ""} + ] + } +} +Only include edits that are needed. "edits" can be an empty list if no patch is warranted. + +IMPORTANT: The skill document may contain a section between + and markers. +This is a PROTECTED section managed by a separate slow-update process. +Do NOT propose any edits that target, modify, or delete content within +these markers. diff --git a/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/prompts/analyst_error_full_rewrite.md b/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/prompts/analyst_error_full_rewrite.md new file mode 100644 index 00000000..5d7e2c53 --- /dev/null +++ b/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/prompts/analyst_error_full_rewrite.md @@ -0,0 +1,32 @@ +You will be given several failed agent trajectories from one minibatch and the current skill document. + +Summarize the lessons from these trajectories into one complete replacement skill document. + +When rewriting from a minibatch, use the current trajectories as the primary +evidence for updates. Preserve essential task-format instructions, but avoid mechanically carrying over +stale, redundant, or conflicting rules. Prefer a concise, coherent replacement +skill over a long document with weakly supported guidance. + +Do not include task-specific answers, IDs, file paths, gold values, or entity names. +If the skill contains a protected block between and +, keep that block unchanged. + +Respond ONLY with a valid JSON object: +{ + "batch_size": , + "failure_summary": [ + {"failure_type": "", "count": , "description": ""} + ], + "patch": { + "reasoning": "", + "skill_candidates": [ + { + "title": "", + "change_summary": ["", ""], + "new_skill": "" + } + ] + } +} + +Return exactly one item in "skill_candidates". diff --git a/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/prompts/analyst_error_rewrite.md b/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/prompts/analyst_error_rewrite.md new file mode 100644 index 00000000..1d34d0e0 --- /dev/null +++ b/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/prompts/analyst_error_rewrite.md @@ -0,0 +1,44 @@ +You are an expert failure-analysis agent for AI agent tasks. + +You will be given MULTIPLE failed agent trajectories from a single minibatch +and the current skill document. +Your job is to identify the most important COMMON failure patterns across +the batch and propose a concise set of skill-revision suggestions. + +## Analysis Process +1. Read ALL trajectories in the minibatch. +2. Identify the most prevalent, systematic failure patterns across them. +3. For each pattern, classify its failure type. +4. Propose revision suggestions that address the COMMON patterns, not individual edge cases. +5. Suggestions must be generalizable and should help a later optimizer rewrite the full skill document. +6. Do not hardcode task-specific values. + +You will be told the maximum number of suggestions (the budget L). Produce AT MOST L suggestions, +focusing on the highest-impact patterns. You may produce fewer if warranted. + +Respond ONLY with a valid JSON object (no markdown fences, no extra text): +{ + "batch_size": , + "failure_summary": [ + {"failure_type": "", "count": , "description": ""} + ], + "patch": { + "reasoning": "", + "revise_suggestions": [ + { + "type": "add_rule|remove_rule|merge_rules|reorganize|compress|clarify", + "title": "", + "motivation": "", + "instruction": "", + "priority_hint": "high|medium|low" + } + ] + } +} +"revise_suggestions" may be an empty list if no revision is warranted. + +IMPORTANT: The skill document may contain a section between + and markers. +This is a PROTECTED section managed by a separate slow-update process. +Do NOT propose suggestions that target, modify, or delete content within +these markers. diff --git a/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/prompts/analyst_success.md b/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/prompts/analyst_success.md new file mode 100644 index 00000000..f79336d1 --- /dev/null +++ b/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/prompts/analyst_success.md @@ -0,0 +1,36 @@ +You are an expert success-pattern analyst for AI agents. + +You will be given MULTIPLE successful agent trajectories from a single minibatch +and the current skill document. Your job is to identify generalizable behavior +patterns that are COMMON across the batch and worth encoding in the skill. + +## Rules +- Only propose patches for patterns NOT already covered in the skill. +- Focus on patterns that appear across MULTIPLE trajectories in the batch. +- Be concise. Patterns must generalize beyond specific tasks. +- Prefer reinforcing existing sections over adding new top-level sections. + +You will be told the maximum number of edits (the budget L). Produce AT MOST L edits, +focusing on the most broadly applicable patterns. You may produce fewer if warranted. + +Respond ONLY with a valid JSON object: +{ + "batch_size": , + "success_patterns": ["", ""], + "patch": { + "reasoning": "", + "edits": [ + {"op": "append", "content": ""}, + {"op": "insert_after", "target": "", "content": ""}, + {"op": "replace", "target": "", "content": ""}, + {"op": "delete", "target": ""} + ] + } +} +"edits" may be empty if the skill already covers all observed patterns. + +IMPORTANT: The skill document may contain a section between + and markers. +This is a PROTECTED section managed by a separate slow-update process. +Do NOT propose any edits that target, modify, or delete content within +these markers. diff --git a/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/prompts/analyst_success_full_rewrite.md b/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/prompts/analyst_success_full_rewrite.md new file mode 100644 index 00000000..eabfcf58 --- /dev/null +++ b/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/prompts/analyst_success_full_rewrite.md @@ -0,0 +1,30 @@ +You will be given several successful agent trajectories from one minibatch and the current skill document. + +Summarize any useful lessons from these trajectories into one complete replacement skill document. + +When rewriting from a minibatch, use the current trajectories as the primary +evidence for updates. Preserve essential task-format instructions, but avoid mechanically carrying over +stale, redundant, or conflicting rules. Prefer a concise, coherent replacement +skill over a long document with weakly supported guidance. + +Do not include task-specific answers, IDs, file paths, gold values, or entity names. +If the skill contains a protected block between and +, keep that block unchanged. + +Respond ONLY with a valid JSON object: +{ + "batch_size": , + "success_patterns": ["", ""], + "patch": { + "reasoning": "", + "skill_candidates": [ + { + "title": "", + "change_summary": ["", ""], + "new_skill": "" + } + ] + } +} + +Return exactly one item in "skill_candidates". diff --git a/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/prompts/analyst_success_rewrite.md b/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/prompts/analyst_success_rewrite.md new file mode 100644 index 00000000..1291b6dc --- /dev/null +++ b/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/prompts/analyst_success_rewrite.md @@ -0,0 +1,33 @@ +You are an expert success-pattern analyst for AI agent tasks. + +You will be given MULTIPLE successful agent trajectories from a single minibatch +and the current skill document. Your job is to identify broadly useful patterns +worth preserving in a later full-skill rewrite. + +## Rules +- Only propose revise_suggestions for patterns NOT already covered in the skill. +- Focus on patterns that appear across MULTIPLE trajectories in the batch. +- Keep suggestions general, concise, and rewrite-friendly. +- Prefer guidance that improves organization, clarity, or reusable behavior. + +You will be told the maximum number of suggestions (the budget L). Produce AT MOST L suggestions, +focusing on the most broadly applicable patterns. You may produce fewer if warranted. + +Respond ONLY with a valid JSON object: +{ + "batch_size": , + "success_patterns": ["", ""], + "patch": { + "reasoning": "", + "revise_suggestions": [ + { + "type": "add_rule|remove_rule|merge_rules|reorganize|compress|clarify", + "title": "", + "motivation": "", + "instruction": "", + "priority_hint": "high|medium|low" + } + ] + } +} +"revise_suggestions" may be empty if the skill already captures all useful patterns. diff --git a/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/prompts/lr_autonomous.md b/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/prompts/lr_autonomous.md new file mode 100644 index 00000000..81d1bc07 --- /dev/null +++ b/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/prompts/lr_autonomous.md @@ -0,0 +1,20 @@ +You are an update-size controller for a skill-learning system. + +You will receive: +1. The current skill document. +2. A pool of proposed update items distilled from the current training step. +3. Brief evidence about the current rollout and training step. + +Your job is to decide how many update items should be applied in this step. +Use only the evidence shown in the prompt. Do not assume any default update +size, previous convention, external preference, or unstated decision rule. + +Do not rank the update items. Only decide the count. + +Respond ONLY with a valid JSON object: +{ + "learning_rate": , + "reasoning": "", + "confidence": "low|medium|high", + "risk_notes": ["", "..."] +} diff --git a/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/prompts/merge_failure.md b/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/prompts/merge_failure.md new file mode 100644 index 00000000..e448999b --- /dev/null +++ b/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/prompts/merge_failure.md @@ -0,0 +1,30 @@ +You are a skill-edit coordinator. You receive multiple independently-proposed patches +from FAILURE analysis of agent trajectories. Merge them into ONE coherent, non-redundant patch. + +Merge guidelines: +1. **Deduplicate**: keep the best-worded version of similar edits. +2. **Resolve conflicts**: if patches contradict on the same point, + choose the one with stronger justification or synthesize both. +3. **Preserve unique insights**: include all non-redundant corrective edits. +4. **Prevalent-pattern bias**: edits appearing consistently across multiple patches + address systematic failures — preserve them with HIGH priority. + Edits from only one patch may be discarded if task-specific. +5. **Independence**: no two edits in the merged patch may target the same text region. +6. **Support count**: for each merged edit, estimate how many source patches support it. +7. **PROTECTED SECTION**: The skill may contain a section between + and markers. + Do NOT merge or produce any edits that target content within these markers. + +Respond ONLY with a valid JSON object: +{ + "reasoning": "", + "edits": [ + { + "op": "append|insert_after|replace|delete", + "target": "", + "content": "", + "support_count": , + "source_type": "failure" + } + ] +} diff --git a/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/prompts/merge_failure_full_rewrite.md b/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/prompts/merge_failure_full_rewrite.md new file mode 100644 index 00000000..0b5c20b4 --- /dev/null +++ b/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/prompts/merge_failure_full_rewrite.md @@ -0,0 +1,28 @@ +You will be given complete skill candidates written from failed trajectories and the current skill document. + +Combine them into one complete replacement skill document. + +When merging full-skill candidates, preserve essential task-format instructions, +but do not mechanically retain stale, redundant, or +conflicting rules. If candidates disagree, prefer the concise rule with clearer +trajectory support and better consistency with the replacement skill. + +Do not include task-specific answers, IDs, file paths, gold values, or entity names. +If the current skill contains a protected block between and +, keep that block unchanged. + +Respond ONLY with a valid JSON object: +{ + "reasoning": "", + "skill_candidates": [ + { + "title": "", + "change_summary": ["", ""], + "new_skill": "", + "support_count": , + "source_type": "failure" + } + ] +} + +Return exactly one item in "skill_candidates". diff --git a/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/prompts/merge_failure_rewrite.md b/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/prompts/merge_failure_rewrite.md new file mode 100644 index 00000000..f86c079a --- /dev/null +++ b/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/prompts/merge_failure_rewrite.md @@ -0,0 +1,26 @@ +You are a skill-revision coordinator. You receive multiple independently-proposed +revision suggestion sets from FAILURE analysis of agent trajectories. Merge them +into ONE coherent, non-redundant set of revise_suggestions. + +Merge guidelines: +1. Deduplicate overlapping suggestions. +2. Resolve conflicts by keeping the more general, better-justified direction. +3. Preserve unique high-impact corrective insights. +4. Suggestions supported by many source patches should receive higher support_count. +5. The output suggestions should help a later optimizer rewrite the full skill. + +Respond ONLY with a valid JSON object: +{ + "reasoning": "", + "revise_suggestions": [ + { + "type": "add_rule|remove_rule|merge_rules|reorganize|compress|clarify", + "title": "", + "motivation": "", + "instruction": "", + "priority_hint": "high|medium|low", + "support_count": , + "source_type": "failure" + } + ] +} diff --git a/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/prompts/merge_final.md b/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/prompts/merge_final.md new file mode 100644 index 00000000..5dd8be1c --- /dev/null +++ b/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/prompts/merge_final.md @@ -0,0 +1,33 @@ +You are a skill-edit coordinator performing the FINAL merge. You receive two +pre-merged patch groups: +1. **Failure-driven patches** (corrective, high priority) +2. **Success-driven patches** (reinforcement, lower priority) + +Merge guidelines: +1. **FAILURE PATCHES TAKE PRIORITY**: the primary goal of skill reflection is to + fix failures. Failure-driven edits should be preserved unless they directly + conflict with a well-supported success pattern. +2. **Deduplicate**: if a failure edit and success edit cover the same point, + keep the failure version. +3. **Preserve success insights**: include success edits that cover patterns + NOT addressed by failure edits. +4. **Higher-level merges represent broader consensus**: edits that survived + previous merge rounds (higher level) should be given priority. +5. **Carry forward support_count and source_type for each edit.** +6. **PROTECTED SECTION**: The skill may contain a section between + and markers. + Do NOT merge or produce any edits that target content within these markers. + +Respond ONLY with a valid JSON object: +{ + "reasoning": "", + "edits": [ + { + "op": "append|insert_after|replace|delete", + "target": "", + "content": "", + "support_count": , + "source_type": "failure|success" + } + ] +} diff --git a/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/prompts/merge_final_full_rewrite.md b/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/prompts/merge_final_full_rewrite.md new file mode 100644 index 00000000..9976a460 --- /dev/null +++ b/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/prompts/merge_final_full_rewrite.md @@ -0,0 +1,28 @@ +You will be given complete skill candidates and the current skill document. + +Combine them into one complete replacement skill document. + +When merging full-skill candidates, preserve essential task-format instructions, +but do not mechanically retain stale, redundant, or +conflicting rules. Prefer concise guidance with clear trajectory support and +better consistency with the replacement skill. + +Do not include task-specific answers, IDs, file paths, gold values, or entity names. +If the current skill contains a protected block between and +, keep that block unchanged. + +Respond ONLY with a valid JSON object: +{ + "reasoning": "", + "skill_candidates": [ + { + "title": "", + "change_summary": ["", ""], + "new_skill": "", + "support_count": , + "source_type": "failure|success|mixed" + } + ] +} + +Return exactly one item in "skill_candidates". diff --git a/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/prompts/merge_final_rewrite.md b/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/prompts/merge_final_rewrite.md new file mode 100644 index 00000000..7fe3e294 --- /dev/null +++ b/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/prompts/merge_final_rewrite.md @@ -0,0 +1,25 @@ +You are a skill-revision coordinator performing the FINAL merge. You receive: +1. Failure-driven revise_suggestions (higher priority) +2. Success-driven revise_suggestions (lower priority) + +Merge guidelines: +1. Failure-driven suggestions take priority when they overlap. +2. Keep success-driven suggestions that add distinct value. +3. Prefer general, rewrite-friendly, non-redundant suggestions. +4. Carry forward support_count and source_type. + +Respond ONLY with a valid JSON object: +{ + "reasoning": "", + "revise_suggestions": [ + { + "type": "add_rule|remove_rule|merge_rules|reorganize|compress|clarify", + "title": "", + "motivation": "", + "instruction": "", + "priority_hint": "high|medium|low", + "support_count": , + "source_type": "failure|success" + } + ] +} diff --git a/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/prompts/merge_success.md b/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/prompts/merge_success.md new file mode 100644 index 00000000..a467bb16 --- /dev/null +++ b/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/prompts/merge_success.md @@ -0,0 +1,28 @@ +You are a skill-edit coordinator. You receive multiple independently-proposed patches +from SUCCESS analysis of agent trajectories. Merge them into ONE coherent patch +that reinforces effective patterns. + +Merge guidelines: +1. **Deduplicate**: keep only the most generalizable version of similar patterns. +2. **Be conservative**: success-driven patches reinforce existing behavior. + Only include edits for patterns NOT already in the skill. +3. **Prevalent-pattern bias**: patterns seen across many successful trajectories + are most worth encoding. +4. **Support count**: estimate how many source patches support each merged edit. +5. **PROTECTED SECTION**: The skill may contain a section between + and markers. + Do NOT merge or produce any edits that target content within these markers. + +Respond ONLY with a valid JSON object: +{ + "reasoning": "", + "edits": [ + { + "op": "append|insert_after|replace|delete", + "target": "", + "content": "", + "support_count": , + "source_type": "success" + } + ] +} diff --git a/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/prompts/merge_success_full_rewrite.md b/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/prompts/merge_success_full_rewrite.md new file mode 100644 index 00000000..a508c0da --- /dev/null +++ b/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/prompts/merge_success_full_rewrite.md @@ -0,0 +1,28 @@ +You will be given complete skill candidates written from successful trajectories and the current skill document. + +Combine them into one complete replacement skill document. + +When merging full-skill candidates, preserve essential task-format instructions, +but do not mechanically retain stale, redundant, or +conflicting rules. If candidates disagree, prefer the concise rule with clearer +trajectory support and better consistency with the replacement skill. + +Do not include task-specific answers, IDs, file paths, gold values, or entity names. +If the current skill contains a protected block between and +, keep that block unchanged. + +Respond ONLY with a valid JSON object: +{ + "reasoning": "", + "skill_candidates": [ + { + "title": "", + "change_summary": ["", ""], + "new_skill": "", + "support_count": , + "source_type": "success" + } + ] +} + +Return exactly one item in "skill_candidates". diff --git a/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/prompts/merge_success_rewrite.md b/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/prompts/merge_success_rewrite.md new file mode 100644 index 00000000..e8238939 --- /dev/null +++ b/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/prompts/merge_success_rewrite.md @@ -0,0 +1,25 @@ +You are a skill-revision coordinator. You receive multiple independently-proposed +revision suggestion sets from SUCCESS analysis of agent trajectories. Merge them +into ONE coherent, non-redundant set of revise_suggestions. + +Merge guidelines: +1. Deduplicate overlapping success patterns. +2. Be conservative: only keep suggestions that reinforce useful behavior not already well-covered. +3. Suggestions supported by many source patches should receive higher support_count. +4. The output suggestions should help a later optimizer rewrite the full skill. + +Respond ONLY with a valid JSON object: +{ + "reasoning": "", + "revise_suggestions": [ + { + "type": "add_rule|remove_rule|merge_rules|reorganize|compress|clarify", + "title": "", + "motivation": "", + "instruction": "", + "priority_hint": "high|medium|low", + "support_count": , + "source_type": "success" + } + ] +} diff --git a/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/prompts/meta_skill.md b/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/prompts/meta_skill.md new file mode 100644 index 00000000..f094973c --- /dev/null +++ b/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/prompts/meta_skill.md @@ -0,0 +1,40 @@ +You are a optimizer-coach for an AI agent skill optimization system. + +Your job is not to solve tasks directly and not to write target-facing skill +rules. Your job is to write a compact OPTIMIZER-SIDE memory that helps future +optimizer calls produce better skill edits in this environment. + +## What You Receive + +1. The previous epoch's last-step skill. +2. The current epoch's last-step skill. +3. A longitudinal comparison on the SAME sampled tasks under those two skills. +4. The previous optimizer meta skill, if one existed. + +## Your Goal + +Write a concise meta skill that improves future optimizer behavior in stages such +as failure analysis, success analysis, patch merging, and edit ranking. + +This meta skill should capture things like: +- Which kinds of edits tend to help in this environment. +- Which kinds of edits tend to be too vague, redundant, brittle, or harmful. +- What level of abstraction works best for rules here. +- What failure-repair patterns should be prioritized. +- What regression risks future optimizer calls should guard against. + +## Important Constraints + +- Address the FUTURE OPTIMIZER directly, not the target. +- Focus on how to write better edits and organize better skill updates. +- Use evidence from the adjacent-epoch comparison, not generic advice. +- Keep it compact and high-signal. Prefer a few durable principles. +- Revise or remove parts of the previous meta skill if they did not help. +- Do not output target-facing task instructions. +- Do not restate the whole skill; summarize editing strategy. + +Respond ONLY with a valid JSON object: +{ + "reasoning": "", + "meta_skill_content": "" +} diff --git a/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/prompts/ranking.md b/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/prompts/ranking.md new file mode 100644 index 00000000..05575a74 --- /dev/null +++ b/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/prompts/ranking.md @@ -0,0 +1,20 @@ +You are an expert skill-optimization optimizer. You receive a skill document and a pool +of proposed edits. Your job is to RANK the edits by importance and select the top ones. + +Ranking criteria (in order of priority): +1. **Systematic impact**: edits that address widespread, recurring failure patterns + across many tasks should rank highest. A rule that fixes 50%% of failures beats + one that fixes a single edge case. +2. **Complementarity**: edits that fill gaps in the current skill (not duplicate + existing content) rank higher. +3. **Generality**: edits phrased as general principles rank higher than those + tied to specific question types or entities. +4. **Actionability**: edits with clear, concrete guidance rank higher than vague advice. + +You will be told how many edits to select (the budget). + +Respond ONLY with a valid JSON object: +{ + "reasoning": "", + "selected_indices": [<0-based indices of the top edits, in priority order>] +} diff --git a/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/prompts/ranking_rewrite.md b/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/prompts/ranking_rewrite.md new file mode 100644 index 00000000..065787a0 --- /dev/null +++ b/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/prompts/ranking_rewrite.md @@ -0,0 +1,15 @@ +You are an expert skill-optimization optimizer. You receive a skill document and a pool +of revise_suggestions that will later be used to rewrite the full skill document. +Rank the suggestions by importance and select the top ones. + +Ranking criteria: +1. Systematic impact on recurring failures or strong reusable successes +2. Complementarity with the current skill +3. Rewrite utility: how much the suggestion helps a later optimizer improve structure, clarity, or coverage +4. Generality and actionability + +Respond ONLY with a valid JSON object: +{ + "reasoning": "", + "selected_indices": [<0-based indices in priority order>] +} diff --git a/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/prompts/rewrite_skill.md b/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/prompts/rewrite_skill.md new file mode 100644 index 00000000..78f26880 --- /dev/null +++ b/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/prompts/rewrite_skill.md @@ -0,0 +1,25 @@ +You are an expert skill-document rewriter for an AI agent training system. + +You will receive: +1. The current skill document +2. A selected set of revise_suggestions distilled from trajectory analysis + +Your job is to rewrite the FULL target skill document so it incorporates the +selected suggestions coherently. + +Hard requirements: +1. Produce a complete standalone skill document, not a patch. +2. Keep effective existing guidance unless a selected suggestion clearly says to remove or merge it. +3. Prefer consolidation and clarity over making the document longer. +4. Do not hardcode benchmark-specific answers, entity names, file paths, or gold values. +5. Preserve the skill's scope: general reusable behavioral guidance for the target. +6. Do not modify content inside the protected slow-update block between + and except to keep it intact. +7. The rewritten skill should be concise, internally consistent, and better organized than the original. + +Respond ONLY with a valid JSON object: +{ + "reasoning": "", + "change_summary": ["", ""], + "new_skill": "" +} diff --git a/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/prompts/slow_update.md b/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/prompts/slow_update.md new file mode 100644 index 00000000..38b1c66a --- /dev/null +++ b/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/prompts/slow_update.md @@ -0,0 +1,60 @@ +You are a strategic skill advisor for an AI agent optimization system. + +Your role is different from the per-step analyst. The per-step analyst sees +individual trajectories and proposes local patches. YOU see how the skill has +evolved across an entire epoch by comparing the SAME tasks under two consecutive +skill versions. This longitudinal view lets you identify systemic drift, +regressions, and persistent blind spots that step-level edits cannot catch. + +## What You Receive + +1. **Previous epoch's skill** and **current epoch's skill** — to see what changed. +2. **Longitudinal comparison** — the same 20 training tasks rolled out under + both skills, categorized into: regressions, persistent failures, + improvements, and stable successes. +3. **Previous slow update guidance** (if any) — the guidance you (or a prior + invocation of you) wrote at the end of the last epoch. This guidance was + active during the current epoch's step-level optimization. You must evaluate + whether it helped or hurt based on the longitudinal comparison results. + +## Your Process + +1. **Reflect on the previous guidance** (if provided): + - Which parts of the previous guidance were effective? (Evidence: tasks that + improved or stayed correct.) + - Which parts failed or backfired? (Evidence: regressions or persistent + failures that the guidance was supposed to address.) + - Were there blind spots the previous guidance missed entirely? + Include this reflection in your "reasoning" field. + +2. **Write updated guidance** that: + - Retains and strengthens parts of the previous guidance that proved effective. + - Revises or removes parts that were ineffective or counterproductive. + - Adds new instructions to address newly observed regressions and persistent + failures. + +## Output Requirements + +Write a **strategic guidance block** that will OVERWRITE the previous guidance +in the protected section of the skill document. This section is READ-ONLY to +all subsequent step-level optimization — only you can overwrite it at the next +epoch boundary. + +Your guidance must: +- Be written as **direct, actionable instructions** to the target model + (the AI agent that will read and follow the skill). +- Focus on helping the target get problems RIGHT — not on analysis or + explanation of what went wrong. +- Prioritize: (1) preventing regressions, (2) fixing persistent failures, + (3) reinforcing successful patterns. +- Be concise but comprehensive — you have no length limit, but every sentence + should earn its place. +- NOT duplicate content already in the main skill body — complement it. +- Address the target directly (e.g., "When you encounter X, always do Y" + rather than "The agent should..."). + +Respond ONLY with a valid JSON object (no markdown fences, no extra text): +{ + "reasoning": "", + "slow_update_content": "" +} diff --git a/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/scheduler/__init__.py b/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/scheduler/__init__.py new file mode 100644 index 00000000..9378ee68 --- /dev/null +++ b/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/scheduler/__init__.py @@ -0,0 +1,8 @@ +"""ReflACT Scheduler -- edit budget and learning rate scheduling. + +Analogous to learning rate schedulers (cosine annealing, step decay, warmup) +in neural network training. Controls how the edit_budget evolves over the +course of training. + +Placeholder for future implementations. +""" diff --git a/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/types.py b/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/types.py new file mode 100644 index 00000000..9c23edb4 --- /dev/null +++ b/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/types.py @@ -0,0 +1,306 @@ +"""Standardized I/O types for the ReflACT pipeline. + +Shared dataclass definitions for the 6-stage per-step pipeline +and the 2 epoch-level stages. All types support round-trip +conversion to/from plain dicts for incremental adoption. + +Re-exports +---------- +GateResult, GateAction — from skillopt.evaluation.gate +BatchSpec — from skillopt.datasets.base +""" +from __future__ import annotations + +from dataclasses import dataclass, field, fields as dc_fields +from typing import Any, Literal + +from skillopt.evaluation.gate import GateAction, GateResult # noqa: F401 +from skillopt.datasets.base import BatchSpec # noqa: F401 + + +# ── Atomic types ───────────────────────────────────────────────────────── + +EditOp = Literal["append", "insert_after", "replace", "delete"] + + +@dataclass +class Edit: + """A single edit operation on a skill document. + + Used across Reflect → Aggregate → Select → Update → MetaReflect. + """ + + op: EditOp + content: str = "" + target: str = "" + support_count: int | None = None + source_type: Literal["failure", "success"] | None = None + merge_level: int | None = None + update_origin: str = "" + update_target: str = "" + + @classmethod + def from_dict(cls, d: dict) -> Edit: + return cls( + op=d.get("op", "append"), + content=d.get("content", ""), + target=d.get("target", ""), + support_count=d.get("support_count"), + source_type=d.get("source_type"), + merge_level=d.get("merge_level"), + update_origin=d.get("update_origin", ""), + update_target=d.get("update_target", ""), + ) + + def to_dict(self) -> dict: + d: dict[str, Any] = {"op": self.op, "content": self.content} + if self.target: + d["target"] = self.target + if self.support_count is not None: + d["support_count"] = self.support_count + if self.source_type is not None: + d["source_type"] = self.source_type + if self.merge_level is not None: + d["merge_level"] = self.merge_level + if self.update_origin: + d["update_origin"] = self.update_origin + if self.update_target: + d["update_target"] = self.update_target + return d + + +@dataclass +class Patch: + """A set of edits with reasoning. + + Output of Aggregate (③), Select (④); input to Update (⑤). + """ + + edits: list[Edit] = field(default_factory=list) + reasoning: str = "" + ranking_details: dict[str, Any] | None = None + + @classmethod + def from_dict(cls, d: dict) -> Patch: + edits_raw = d.get("edits", []) + return cls( + edits=[Edit.from_dict(e) if isinstance(e, dict) else e for e in edits_raw], + reasoning=d.get("reasoning", ""), + ranking_details=d.get("ranking_details"), + ) + + def to_dict(self) -> dict: + d: dict[str, Any] = { + "reasoning": self.reasoning, + "edits": [e.to_dict() if isinstance(e, Edit) else e for e in self.edits], + } + if self.ranking_details is not None: + d["ranking_details"] = self.ranking_details + return d + + +# ── Stage ① ROLLOUT ────────────────────────────────────────────────────── + +@dataclass +class RolloutResult: + """Result of a single episode/task rollout. + + Universal fields are required; env-specific fields live in ``extras``. + """ + + id: str + hard: int + soft: float + n_turns: int = 0 + fail_reason: str = "" + task_type: str = "" + task_description: str = "" + predicted_answer: str = "" + question: str = "" + reference_text: str = "" + target_system_prompt: str = "" + target_user_prompt: str = "" + spreadsheet_preview: str = "" + extras: dict[str, Any] = field(default_factory=dict) + + _KNOWN_FIELDS: frozenset[str] | None = field( + default=None, init=False, repr=False, compare=False, # type: ignore[assignment] + ) + + @classmethod + def _get_known_fields(cls) -> frozenset[str]: + if cls._KNOWN_FIELDS is None: + cls._KNOWN_FIELDS = frozenset( + f.name for f in dc_fields(cls) + if f.name != "_KNOWN_FIELDS" + ) + return cls._KNOWN_FIELDS + + @classmethod + def from_dict(cls, d: dict) -> RolloutResult: + known = cls._get_known_fields() + extras = {k: v for k, v in d.items() if k not in known} + return cls( + id=str(d.get("id", "")), + hard=int(d.get("hard", 0)), + soft=float(d.get("soft", 0.0)), + n_turns=int(d.get("n_turns", 0)), + fail_reason=str(d.get("fail_reason", "")), + task_type=str(d.get("task_type", "")), + task_description=str(d.get("task_description", "")), + predicted_answer=str(d.get("predicted_answer", "")), + question=str(d.get("question", "")), + reference_text=str(d.get("reference_text", "")), + target_system_prompt=str(d.get("target_system_prompt", "")), + target_user_prompt=str(d.get("target_user_prompt", "")), + spreadsheet_preview=str(d.get("spreadsheet_preview", "")), + extras=extras, + ) + + def to_dict(self) -> dict: + d: dict[str, Any] = { + "id": self.id, + "hard": self.hard, + "soft": self.soft, + } + for attr in ( + "n_turns", "fail_reason", "task_type", "task_description", + "predicted_answer", "question", "reference_text", + "target_system_prompt", "target_user_prompt", + "spreadsheet_preview", + ): + val = getattr(self, attr) + if val: + d[attr] = val + d.update(self.extras) + return d + + +# ── Stage ② REFLECT ────────────────────────────────────────────────────── + +@dataclass +class FailureSummaryEntry: + """One entry in the failure summary produced by error analysts.""" + + failure_type: str + count: int = 0 + description: str = "" + + @classmethod + def from_dict(cls, d: dict) -> FailureSummaryEntry: + return cls( + failure_type=d.get("failure_type", ""), + count=int(d.get("count", 0)), + description=d.get("description", ""), + ) + + def to_dict(self) -> dict: + return { + "failure_type": self.failure_type, + "count": self.count, + "description": self.description, + } + + +@dataclass +class RawPatch: + """Analyst output from the Reflect stage — a patch with provenance. + + Wraps the dict produced by ``run_error_analyst_minibatch`` + and ``run_success_analyst_minibatch``. + """ + + patch: Patch + source_type: Literal["failure", "success"] = "failure" + batch_size: int = 0 + failure_summary: list[FailureSummaryEntry] = field(default_factory=list) + + @classmethod + def from_dict(cls, d: dict | None) -> RawPatch | None: + if d is None: + return None + inner = d.get("patch", d) + if not isinstance(inner, dict): + return None + patch = Patch.from_dict(inner) + return cls( + patch=patch, + source_type=d.get("source_type", "failure"), + batch_size=int(d.get("batch_size", 0)), + failure_summary=[ + FailureSummaryEntry.from_dict(fs) + for fs in d.get("failure_summary", []) + ], + ) + + def to_dict(self) -> dict: + d: dict[str, Any] = { + "patch": self.patch.to_dict(), + "source_type": self.source_type, + "batch_size": self.batch_size, + } + if self.failure_summary: + d["failure_summary"] = [fs.to_dict() for fs in self.failure_summary] + return d + + +# ── Epoch-level: SLOW_UPDATE ───────────────────────────────────────────── + +@dataclass +class SlowUpdateResult: + """Output of the epoch-level slow update stage (EMA / regularization).""" + + reasoning: str = "" + slow_update_content: str = "" + action: str = "" + time_s: float | None = None + prev_hard: float | None = None + curr_hard: float | None = None + selection_hard: float | None = None + selection_soft: float | None = None + candidate_hash: str = "" + update_origin: str = "" + update_target: str = "" + + @classmethod + def from_dict(cls, d: dict | None) -> SlowUpdateResult | None: + if d is None: + return None + return cls( + reasoning=d.get("reasoning", ""), + slow_update_content=d.get("slow_update_content", ""), + action=d.get("action", ""), + time_s=d.get("time_s"), + prev_hard=d.get("prev_hard"), + curr_hard=d.get("curr_hard"), + selection_hard=d.get("selection_hard"), + selection_soft=d.get("selection_soft"), + candidate_hash=d.get("candidate_hash", ""), + update_origin=d.get("update_origin", ""), + update_target=d.get("update_target", ""), + ) + + def to_dict(self) -> dict: + d: dict[str, Any] = { + "reasoning": self.reasoning, + "slow_update_content": self.slow_update_content, + } + if self.action: + d["action"] = self.action + if self.time_s is not None: + d["time_s"] = self.time_s + if self.prev_hard is not None: + d["prev_hard"] = self.prev_hard + if self.curr_hard is not None: + d["curr_hard"] = self.curr_hard + if self.selection_hard is not None: + d["selection_hard"] = self.selection_hard + if self.selection_soft is not None: + d["selection_soft"] = self.selection_soft + if self.candidate_hash: + d["candidate_hash"] = self.candidate_hash + if self.update_origin: + d["update_origin"] = self.update_origin + if self.update_target: + d["update_target"] = self.update_target + return d diff --git a/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/utils/__init__.py b/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/utils/__init__.py new file mode 100644 index 00000000..d59520e0 --- /dev/null +++ b/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/utils/__init__.py @@ -0,0 +1,4 @@ +"""ReflACT utilities — JSON extraction, scoring, hashing.""" + +from skillopt.utils.json_utils import extract_json, extract_json_array # noqa: F401 +from skillopt.utils.scoring import compute_score, skill_hash # noqa: F401 diff --git a/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/utils/json_utils.py b/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/utils/json_utils.py new file mode 100644 index 00000000..011241b8 --- /dev/null +++ b/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/utils/json_utils.py @@ -0,0 +1,42 @@ +"""JSON extraction helpers for LLM responses.""" +from __future__ import annotations + +import json +import re + + +def extract_json(text: str) -> dict | None: + """Extract a JSON object from LLM response text. + + Tries ```json fences first, then bare {...} patterns. + """ + m = re.search(r"```json\s*(.*?)```", text, re.DOTALL) + if m: + try: + return json.loads(m.group(1)) + except json.JSONDecodeError: + pass + m = re.search(r"\{.*\}", text, re.DOTALL) + if m: + try: + return json.loads(m.group(0)) + except json.JSONDecodeError: + pass + return None + + +def extract_json_array(text: str) -> list | None: + """Extract a JSON array from LLM response text.""" + m = re.search(r"```json\s*(.*?)```", text, re.DOTALL) + if m: + try: + return json.loads(m.group(1)) + except json.JSONDecodeError: + pass + m = re.search(r"\[.*\]", text, re.DOTALL) + if m: + try: + return json.loads(m.group(0)) + except json.JSONDecodeError: + pass + return None diff --git a/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/utils/scoring.py b/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/utils/scoring.py new file mode 100644 index 00000000..df5d1fe0 --- /dev/null +++ b/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt/utils/scoring.py @@ -0,0 +1,28 @@ +"""Scoring and hashing utilities.""" +from __future__ import annotations + +import hashlib + + +def compute_score(results: list) -> tuple[float, float]: + """Compute hard and soft accuracy from a list of episode results. + + Accepts both plain dicts and :class: instances. hard may be continuous (0.0-1.0) when using smoothed reward. + """ + if not results: + return 0.0, 0.0 + + def _hard(r: object) -> float: + return float(r.hard if hasattr(r, "hard") else r.get("hard", 0)) + + def _soft(r: object) -> float: + return float(r.soft if hasattr(r, "soft") else r.get("soft", 0.0)) + + hard = sum(_hard(r) for r in results) / len(results) + soft = sum(_soft(r) for r in results) / len(results) + return hard, soft + + +def skill_hash(content: str) -> str: + """Return a short deterministic hash of skill content (for caching).""" + return hashlib.sha256(content.encode()).hexdigest()[:16] diff --git a/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt_webui/__init__.py b/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt_webui/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt_webui/__main__.py b/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt_webui/__main__.py new file mode 100644 index 00000000..6014cd12 --- /dev/null +++ b/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt_webui/__main__.py @@ -0,0 +1,3 @@ +# SkillOpt WebUI — `__main__` entry point +from skillopt_webui.app import main +main() diff --git a/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt_webui/app.py b/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt_webui/app.py new file mode 100644 index 00000000..ef0c68f0 --- /dev/null +++ b/benchmark/spreadsheet_xarena/third_party/SkillOpt/skillopt_webui/app.py @@ -0,0 +1,550 @@ +""" +SkillOpt WebUI — Configure, launch, and monitor training from your browser. + +Usage: + python -m skillopt_webui.app [--port PORT] [--share] +""" +import argparse +import glob +import json +import os +import signal +import subprocess +import sys +import threading +import time +from pathlib import Path + +import gradio as gr +import yaml + +PROJECT_ROOT = Path(__file__).resolve().parent.parent + + +# ─── Config helpers ────────────────────────────────────────────────────────── + +def discover_configs() -> list[str]: + """Find all YAML configs under configs/.""" + pattern = str(PROJECT_ROOT / "configs" / "**" / "*.yaml") + paths = sorted(glob.glob(pattern, recursive=True)) + return [os.path.relpath(p, PROJECT_ROOT) for p in paths + if "_base_" not in p] + + +def load_config(path: str) -> dict: + """Load a YAML config file.""" + with open(PROJECT_ROOT / path) as f: + return yaml.safe_load(f) + + +def config_to_display(cfg: dict) -> str: + """Pretty-print config for display.""" + return yaml.dump(cfg, default_flow_style=False, sort_keys=False) + + +# ─── Training process management ──────────────────────────────────────────── + +class TrainingManager: + """Manages a single training subprocess.""" + + def __init__(self): + self._lock = threading.Lock() + self.process = None + self.log_lines: list[str] = [] + self.stage = "Idle" + self.step = 0 + self.total_steps = 0 + self.epoch = 0 + self.total_epochs = 0 + self.running = False + + def start(self, config_path: str, overrides: dict) -> str: + with self._lock: + if self.running: + return "⚠️ Training already running. Stop it first." + + cmd = [ + sys.executable, "scripts/train.py", + "--config", config_path, + ] + cfg_options = [] + for k, v in overrides.items(): + if v is not None and v != "": + cfg_options.append(f"{k}={v}") + if cfg_options: + cmd.append("--cfg-options") + cmd.extend(cfg_options) + + env = os.environ.copy() + env["PYTHONUNBUFFERED"] = "1" + # Auto-load API credentials from .secrets/*.env + secrets_dir = PROJECT_ROOT / ".secrets" + if secrets_dir.is_dir(): + for env_file in sorted(secrets_dir.glob("*.env")): + for line in env_file.read_text().splitlines(): + line = line.strip() + if line and not line.startswith("#") and "=" in line: + k, v = line.split("=", 1) + env[k] = v + # Propagate OPTIMIZER_* to base AZURE_OPENAI_* when base is missing, + # so target/default endpoints inherit from optimizer config. + _propagate = [ + ("ENDPOINT", ""), ("API_VERSION", ""), ("AUTH_MODE", ""), + ("MANAGED_IDENTITY_CLIENT_ID", ""), ("AD_SCOPE", ""), + ("API_KEY", ""), + ] + for suffix, _ in _propagate: + base_key = f"AZURE_OPENAI_{suffix}" + optimizer_key = f"OPTIMIZER_AZURE_OPENAI_{suffix}" + if not env.get(base_key) and env.get(optimizer_key): + env[base_key] = env[optimizer_key] + + try: + proc = subprocess.Popen( + cmd, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, + cwd=str(PROJECT_ROOT), + bufsize=1, + env=env, + start_new_session=True, # create process group for clean kill + ) + except Exception as e: + return f"❌ Failed to start training: {e}" + + with self._lock: + self.process = proc + self.log_lines = [f"$ {' '.join(cmd)}\n"] + self.stage = "Starting" + self.step = 0 + self.total_steps = 0 + self.epoch = 0 + self.total_epochs = 0 + self.running = True + + thread = threading.Thread(target=self._read_output, daemon=True) + thread.start() + + return "✅ Training started!" + + def _read_output(self): + for line in self.process.stdout: + with self._lock: + self.log_lines.append(line) + self._parse_stage(line) + if len(self.log_lines) > 5000: + self.log_lines = self.log_lines[-4000:] + self.process.wait() + with self._lock: + self.running = False + self.stage = f"Finished (exit={self.process.returncode})" + + def _parse_stage(self, line: str): + line_lower = line.lower() + if "1/6 rollout" in line_lower or ("rollout" in line_lower and "worker" in line_lower): + self.stage = "🎯 Rollout" + elif "2/6 reflect" in line_lower or ("reflect" in line_lower and "patch" in line_lower): + self.stage = "🔍 Reflect" + elif "3/6 aggregate" in line_lower or "merge" in line_lower: + self.stage = "🔗 Aggregate" + elif "4/6 select" in line_lower: + self.stage = "✂️ Select" + elif "5/6 update" in line_lower: + self.stage = "📝 Update" + elif "6/6" in line_lower or ("gate" in line_lower and "score" in line_lower): + self.stage = "🚦 Gate" + elif "slow update" in line_lower: + self.stage = "🔄 Slow Update" + elif "meta skill" in line_lower: + self.stage = "🧠 Meta Skill" + elif "baseline" in line_lower and "evaluate" in line_lower: + self.stage = "📊 Baseline" + if "[step" in line_lower: + try: + parts = line.split("[STEP")[1].split("]")[0].split("/") + self.step = int(parts[0].strip()) + self.total_steps = int(parts[1].strip()) + except (IndexError, ValueError): + pass + if "[epoch" in line_lower: + try: + parts = line.split("[EPOCH")[1].split("]")[0].split("/") + self.epoch = int(parts[0].strip()) + self.total_epochs = int(parts[1].strip()) + except (IndexError, ValueError): + pass + + def stop(self) -> str: + with self._lock: + if self.process and self.running: + try: + # Kill entire process group (children included) + os.killpg(os.getpgid(self.process.pid), signal.SIGTERM) + except (ProcessLookupError, OSError): + self.process.terminate() + self.process.wait(timeout=5) + self.running = False + self.stage = "Stopped" + return "🛑 Training stopped." + return "No training running." + + def get_logs(self) -> str: + with self._lock: + return "".join(self.log_lines[-500:]) + + def get_colored_logs_html(self) -> str: + """Render last 300 log lines with color-coded stages.""" + import html as html_mod + with self._lock: + lines = list(self.log_lines[-300:]) + parts = [] + for line in lines: + # Rebrand: display "skillopt" instead of "reflact" in logs + line_display = line.replace("reflact", "skillopt").replace("ReflACT", "SkillOpt").replace("Reflact", "Skillopt").replace("REFLACT", "SKILLOPT") + escaped = html_mod.escape(line_display.rstrip("\n")) + low = line.lower() + if "[epoch" in low: + color = "#f59e0b" # amber + weight = "700" + elif "[step" in low: + color = "#8b5cf6" # purple + weight = "700" + elif "rollout]" in low or "1/6" in low: + color = "#3b82f6" # blue + elif "reflect" in low or "2/6" in low: + color = "#f97316" # orange + elif "aggregate" in low or "3/6" in low or "merge" in low: + color = "#06b6d4" # cyan + elif "select" in low or "4/6" in low: + color = "#ec4899" # pink + elif "update" in low or "5/6" in low: + color = "#10b981" # green + elif "gate" in low or "6/6" in low: + color = "#ef4444" # red + elif "slow update" in low: + color = "#f59e0b" # amber + weight = "700" + elif "meta skill" in low: + color = "#a855f7" # violet + weight = "700" + elif "baseline" in low: + color = "#6366f1" # indigo + weight = "700" + elif "[rollout]" in low: + # per-item rollout progress + if "hard=1" in line: + color = "#22c55e" # green for correct + elif "hard=0" in line: + color = "#f87171" # red for wrong + elif "timeout" in low: + color = "#fbbf24" # yellow for timeout + else: + color = "#94a3b8" # gray + weight = "400" + elif "error" in low or "fail" in low: + color = "#ef4444" + weight = "700" + elif "========" in line: + color = "#64748b" # separator + weight = "400" + else: + color = "#e2e8f0" # default light gray + weight = "400" + if "weight" not in dir(): + weight = "400" + parts.append(f'{escaped}') + weight = "400" # reset + + log_html = "
".join(parts) if parts else 'No logs yet. Click Refresh after launching training.' + return f'''
{log_html}
''' + + def get_progress_html(self) -> str: + """Render a visual progress bar.""" + s = self.get_status() + step = s["step"] + total = s["total_steps"] + epoch = self.epoch + total_epochs = self.total_epochs + pct = s["progress"] * 100 + + if not self.running and step == 0: + return '
Waiting for training to start...
' + + # Color based on progress + if pct < 25: + bar_color = "linear-gradient(90deg, #3b82f6, #6366f1)" + elif pct < 50: + bar_color = "linear-gradient(90deg, #6366f1, #8b5cf6)" + elif pct < 75: + bar_color = "linear-gradient(90deg, #8b5cf6, #a855f7)" + else: + bar_color = "linear-gradient(90deg, #a855f7, #22c55e)" + + stage_icon = self.stage if self.stage != "Idle" else "⏳" + status_dot = "🟢" if self.running else ("✅" if "Finished" in self.stage else "⚪") + + epoch_str = f"Epoch {epoch}/{total_epochs}" if total_epochs > 0 else "" + step_str = f"Step {step}/{total}" if total > 0 else "" + + return f''' +
+
+ {status_dot} {stage_icon} + {epoch_str}   {step_str} + {pct:.1f}% +
+
+
+
+
''' + + def get_status(self) -> dict: + with self._lock: + progress = 0 + if self.total_steps > 0: + progress = self.step / self.total_steps + return { + "running": self.running, + "stage": self.stage, + "step": self.step, + "total_steps": self.total_steps, + "progress": progress, + } + + +manager = TrainingManager() + + +# ─── Pipeline Stage HTML ──────────────────────────────────────────────────── + +STAGES = ["Rollout", "Reflect", "Aggregate", "Select", "Update", "Gate"] +STAGE_ICONS = ["🎯", "🔍", "🔗", "✂️", "📝", "🚦"] + + +def render_pipeline_html(active_stage: str = "") -> str: + """Render animated pipeline HTML.""" + html = '
' + for i, (name, icon) in enumerate(zip(STAGES, STAGE_ICONS)): + is_active = name.lower() in active_stage.lower() if active_stage else False + bg = "#6366f1" if is_active else "#f3f4f6" + color = "white" if is_active else "#374151" + border = "3px solid #4f46e5" if is_active else "2px solid #d1d5db" + shadow = "0 0 20px rgba(99,102,241,0.4)" if is_active else "none" + pulse = "animation: pulse 1.5s ease-in-out infinite;" if is_active else "" + html += f''' +
+ {icon} + {name} +
''' + if i < len(STAGES) - 1: + arrow_color = "#6366f1" if is_active else "#d1d5db" + html += f'
→
' + html += '
' + html += '' + return html + + +# ─── Gradio UI ────────────────────────────────────────────────────────────── + +def build_ui(): + configs = discover_configs() + + with gr.Blocks( + title="SkillOpt WebUI", + ) as app: + gr.Markdown("# 🧠 SkillOpt Training Dashboard") + gr.Markdown("*SKILLOPT: Executive Strategy for Self-Evolving Agent Skills — Configure, launch, and monitor training.*") + + with gr.Tabs(): + # ── Tab 1: Configure & Launch ──────────────────────────── + with gr.Tab("⚙️ Configure & Launch"): + with gr.Row(): + with gr.Column(scale=1): + config_dropdown = gr.Dropdown( + choices=configs, + label="Config File", + value=configs[0] if configs else None, + ) + config_preview = gr.Code( + label="Config Preview", + language="yaml", + interactive=False, + ) + + with gr.Column(scale=1): + gr.Markdown("### Hyperparameters (DL Analogy)") + lr = gr.Slider(1, 32, value=4, step=1, + label="Learning Rate (max edits/step)") + scheduler = gr.Dropdown( + ["cosine", "linear", "constant", "autonomous"], + value="cosine", + label="LR Scheduler", + ) + num_epochs = gr.Slider(1, 8, value=4, step=1, + label="Epochs") + batch_size = gr.Slider(10, 100, value=40, step=5, + label="Batch Size (tasks per step)") + analyst_workers = gr.Slider(1, 32, value=16, step=1, + label="Analyst Workers (parallel reflection)") + use_slow_update = gr.Checkbox(value=True, + label="Slow Update (epoch-boundary momentum)") + use_meta_skill = gr.Checkbox(value=True, + label="Meta Skill (cross-epoch optimizer memory)") + use_gate = gr.Checkbox(value=True, + label="Gate (validation-based accept/reject)") + + with gr.Row(): + launch_btn = gr.Button("🚀 Launch Training", + variant="primary", size="lg") + stop_btn = gr.Button("🛑 Stop", variant="stop") + + status_text = gr.Textbox(label="Status", interactive=False) + + def on_config_change(path): + if path: + try: + return config_to_display(load_config(path)) + except Exception as e: + return f"Error: {e}" + return "" + + config_dropdown.change(on_config_change, config_dropdown, config_preview) + + def on_launch(cfg_path, lr_val, sched, epochs, batch, workers, + slow_update, meta_skill, gate): + overrides = { + "optimizer.learning_rate": lr_val, + "optimizer.lr_scheduler": sched, + "train.num_epochs": epochs, + "train.batch_size": batch, + "gradient.analyst_workers": workers, + "optimizer.use_slow_update": slow_update, + "optimizer.use_meta_skill": meta_skill, + "evaluation.use_gate": gate, + } + return manager.start(cfg_path, overrides) + + launch_btn.click( + on_launch, + [config_dropdown, lr, scheduler, num_epochs, batch_size, + analyst_workers, use_slow_update, use_meta_skill, use_gate], + status_text, + ) + stop_btn.click(lambda: manager.stop(), outputs=status_text) + + # ── Tab 2: Monitor ─────────────────────────────────────── + with gr.Tab("📊 Monitor"): + pipeline_html = gr.HTML( + value=render_pipeline_html(), + label="Pipeline Stage", + ) + + progress_html = gr.HTML( + value=manager.get_progress_html(), + label="Progress", + ) + + log_html = gr.HTML( + value=manager.get_colored_logs_html(), + label="Training Logs", + ) + + refresh_btn = gr.Button("🔄 Refresh Logs", variant="primary", size="lg") + + def on_refresh(): + s = manager.get_status() + pipeline = render_pipeline_html(s["stage"]) + progress = manager.get_progress_html() + logs = manager.get_colored_logs_html() + return pipeline, progress, logs + + refresh_btn.click( + on_refresh, + outputs=[pipeline_html, progress_html, log_html], + ) + + # ── Tab 3: Results ─────────────────────────────────────── + with gr.Tab("📈 Results"): + gr.Markdown("### Output Explorer") + output_dir = gr.Textbox( + label="Output Directory", + value="outputs/", + interactive=True, + ) + scan_btn = gr.Button("🔍 Scan Results") + results_table = gr.Dataframe( + headers=["Experiment", "Benchmark", "Best Score", "Steps"], + label="Experiments", + ) + + def scan_outputs(out_dir): + rows = [] + base = PROJECT_ROOT / out_dir + if not base.exists(): + return rows + for bench_dir in sorted(base.iterdir()): + if not bench_dir.is_dir(): + continue + for run_dir in sorted(bench_dir.iterdir()): + if not run_dir.is_dir(): + continue + cfg_file = run_dir / "config.yaml" + score = "—" + steps = "—" + if cfg_file.exists(): + try: + c = yaml.safe_load(cfg_file.read_text()) + steps = str(c.get("train", {}).get("num_steps", "—")) + except Exception: + pass + # Try to find best score from logs + for log_f in run_dir.glob("**/*.jsonl"): + try: + with open(log_f) as f: + for line in f: + d = json.loads(line) + if "score" in d: + score = f"{d['score']:.4f}" + except Exception: + pass + rows.append([ + run_dir.name, + bench_dir.name, + score, + steps, + ]) + return rows + + scan_btn.click(scan_outputs, output_dir, results_table) + + return app + + +def main(): + parser = argparse.ArgumentParser(description="SkillOpt WebUI") + parser.add_argument("--port", type=int, default=7860) + parser.add_argument("--share", action="store_true") + parser.add_argument("--host", type=str, default="0.0.0.0", + help="Server host. Use 0.0.0.0 for public access.") + args = parser.parse_args() + + app = build_ui() + app.launch( + server_name=args.host, + server_port=args.port, + share=args.share, + theme=gr.themes.Soft(primary_hue="indigo"), + ) + + +if __name__ == "__main__": + main() diff --git a/benchmark/spreadsheet_xarena/third_party/SkillOpt/tests/__init__.py b/benchmark/spreadsheet_xarena/third_party/SkillOpt/tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/benchmark/spreadsheet_xarena/third_party/SkillOpt/tests/test_json_utils.py b/benchmark/spreadsheet_xarena/third_party/SkillOpt/tests/test_json_utils.py new file mode 100644 index 00000000..d9a4b06e --- /dev/null +++ b/benchmark/spreadsheet_xarena/third_party/SkillOpt/tests/test_json_utils.py @@ -0,0 +1,112 @@ +"""Tests for skillopt.utils.json_utils.""" +from __future__ import annotations + +import pytest + +from skillopt.utils.json_utils import extract_json, extract_json_array + + +class TestExtractJson: + """extract_json — extract a JSON object from LLM response text.""" + + def test_code_fence_json(self) -> None: + text = 'Some text\n```json\n{"key": "value", "num": 42}\n```\nmore text' + assert extract_json(text) == {"key": "value", "num": 42} + + def test_bare_json_object(self) -> None: + text = 'The result is {"answer": "yes", "score": 0.95}.' + assert extract_json(text) == {"answer": "yes", "score": 0.95} + + def test_code_fence_takes_precedence(self) -> None: + """If fence content parses successfully it should be preferred over bare.""" + text = ( + '```json\n{"source": "fence"}\n```\n' + 'Then also {"source": "bare"}' + ) + assert extract_json(text) == {"source": "fence"} + + def test_broken_fence_falls_back_to_bare(self) -> None: + """When fence content is invalid JSON, fall back to bare {...} match.""" + # Use invalid fence content that has no braces so the greedy bare + # regex doesn't swallow the valid object. + text = ( + '```json\nnot json at all\n```\n' + 'Answer: {"fallback": "yes"}' + ) + assert extract_json(text) == {"fallback": "yes"} + + def test_nested_json(self) -> None: + text = '```json\n{"outer": {"inner": [1, 2, 3]}}\n```' + assert extract_json(text) == {"outer": {"inner": [1, 2, 3]}} + + def test_no_json_returns_none(self) -> None: + assert extract_json("Just plain text without JSON.") is None + + def test_empty_string_returns_none(self) -> None: + assert extract_json("") is None + + def test_malformed_json_returns_none(self) -> None: + assert extract_json("{broken") is None + + def test_empty_json_object(self) -> None: + assert extract_json('{"empty": {}}') == {"empty": {}} + + def test_json_with_escaped_chars(self) -> None: + text = '{"message": "hello\\nworld"}' + assert extract_json(text) == {"message": "hello\nworld"} + + def test_only_fence_with_no_json_syntax(self) -> None: + """Code fences without valid JSON content should not match.""" + text = "```\nplain code block\n```" + assert extract_json(text) is None + + +class TestExtractJsonArray: + """extract_json_array — extract a JSON array from LLM response text.""" + + def test_code_fence_array(self) -> None: + text = '```json\n["a", "b", "c"]\n```' + assert extract_json_array(text) == ["a", "b", "c"] + + def test_bare_array(self) -> None: + text = "The items are [1, 2, 3]." + assert extract_json_array(text) == [1, 2, 3] + + def test_code_fence_takes_precedence(self) -> None: + text = ( + '```json\n["from_fence"]\n```\n' + 'also ["from_bare"]' + ) + assert extract_json_array(text) == ["from_fence"] + + def test_broken_fence_falls_back_to_bare(self) -> None: + text = ( + '```json\nnot json at all\n```\n' + 'values: [42]' + ) + assert extract_json_array(text) == [42] + + def test_nested_array(self) -> None: + text = '```json\n[[1, 2], [3, 4]]\n```' + assert extract_json_array(text) == [[1, 2], [3, 4]] + + def test_no_array_returns_none(self) -> None: + assert extract_json_array("no brackets here") is None + + def test_empty_string_returns_none(self) -> None: + assert extract_json_array("") is None + + def test_malformed_array_returns_none(self) -> None: + assert extract_json_array("[1, 2, ") is None + + def test_empty_json_array(self) -> None: + assert extract_json_array("[]") == [] + + def test_array_of_objects(self) -> None: + text = '[{"x": 1}, {"x": 2}]' + assert extract_json_array(text) == [{"x": 1}, {"x": 2}] + + def test_object_not_confused_with_array(self) -> None: + """extract_json_array should not match a bare JSON object.""" + text = '{"this is an object": true}' + assert extract_json_array(text) is None diff --git a/benchmark/spreadsheet_xarena/third_party/SkillOpt/tests/test_scoring.py b/benchmark/spreadsheet_xarena/third_party/SkillOpt/tests/test_scoring.py new file mode 100644 index 00000000..281c6b82 --- /dev/null +++ b/benchmark/spreadsheet_xarena/third_party/SkillOpt/tests/test_scoring.py @@ -0,0 +1,106 @@ +"""Tests for skillopt.utils.scoring.""" +from __future__ import annotations + +import pytest + +from skillopt.utils.scoring import compute_score, skill_hash + + +class _ResultObject: + """Minimal object with hard/soft attrs (duck-typing path).""" + + def __init__(self, hard: float, soft: float) -> None: + self.hard = hard + self.soft = soft + + +class TestComputeScore: + """compute_score — hard/soft accuracy from a list of episode results.""" + + def test_empty_list_returns_zeros(self) -> None: + assert compute_score([]) == (0.0, 0.0) + + def test_dict_results_happy_path(self) -> None: + results = [ + {"hard": 1, "soft": 0.8}, + {"hard": 0, "soft": 0.5}, + {"hard": 1, "soft": 0.9}, + ] + hard, soft = compute_score(results) + assert hard == pytest.approx(2 / 3) + assert soft == pytest.approx((0.8 + 0.5 + 0.9) / 3) + + def test_object_results(self) -> None: + results = [ + _ResultObject(1.0, 0.75), + _ResultObject(0.0, 0.25), + ] + hard, soft = compute_score(results) + assert hard == 0.5 + assert soft == 0.5 + + def test_mixed_dict_and_object_results(self) -> None: + results = [ + {"hard": 1, "soft": 1.0}, + _ResultObject(0, 0.0), + ] + hard, soft = compute_score(results) + assert hard == 0.5 + assert soft == 0.5 + + def test_missing_keys_default_to_zero(self) -> None: + results = [ + {"hard": 1}, + {}, + ] + hard, soft = compute_score(results) + assert hard == 0.5 + assert soft == 0.0 + + def test_single_result(self) -> None: + results = [{"hard": 1, "soft": 0.95}] + assert compute_score(results) == (1.0, 0.95) + + def test_continuous_hard_values(self) -> None: + """Hard may be continuous 0.0-1.0 when using smoothed reward.""" + results = [ + {"hard": 0.75, "soft": 0.6}, + {"hard": 0.25, "soft": 0.4}, + ] + hard, soft = compute_score(results) + assert hard == 0.5 + assert soft == 0.5 + + +class TestSkillHash: + """skill_hash — a short, deterministic hash of skill content.""" + + def test_deterministic(self) -> None: + assert skill_hash("hello") == skill_hash("hello") + + def test_different_input_produces_different_hash(self) -> None: + assert skill_hash("hello") != skill_hash("world") + + def test_empty_string(self) -> None: + h = skill_hash("") + assert isinstance(h, str) + assert len(h) == 16 + + def test_output_length(self) -> None: + h = skill_hash("some skill content here") + assert len(h) == 16 + + def test_hex_characters(self) -> None: + h = skill_hash("any content") + assert all(c in "0123456789abcdef" for c in h) + + def test_unicode_content(self) -> None: + h1 = skill_hash("cafe") + h2 = skill_hash("cafe") + assert h1 == h2 + + def test_multiline_content(self) -> None: + content = "line1\nline2\nline3" + h = skill_hash(content) + assert len(h) == 16 + assert isinstance(h, str) diff --git a/benchmark/spreadsheet_xarena/third_party/SkillOpt/tests/test_types.py b/benchmark/spreadsheet_xarena/third_party/SkillOpt/tests/test_types.py new file mode 100644 index 00000000..f39c8f62 --- /dev/null +++ b/benchmark/spreadsheet_xarena/third_party/SkillOpt/tests/test_types.py @@ -0,0 +1,249 @@ +"""Tests for skillopt.types — Edit and Patch dataclass serialization.""" +from __future__ import annotations + +import pytest + +from skillopt.types import Edit, Patch + + +# ── Edit ──────────────────────────────────────────────────────────────────── + + +class TestEditCreation: + """Edit dataclass construction.""" + + def test_minimal_edit(self) -> None: + e = Edit(op="append") + assert e.op == "append" + assert e.content == "" + assert e.target == "" + assert e.support_count is None + assert e.source_type is None + assert e.merge_level is None + assert e.update_origin == "" + assert e.update_target == "" + + def test_full_edit(self) -> None: + e = Edit( + op="replace", + content="new content", + target="old content", + support_count=5, + source_type="failure", + merge_level=2, + update_origin="reflect", + update_target="skill", + ) + assert e.op == "replace" + assert e.content == "new content" + assert e.target == "old content" + assert e.support_count == 5 + assert e.source_type == "failure" + assert e.merge_level == 2 + assert e.update_origin == "reflect" + assert e.update_target == "skill" + + def test_insert_after_op(self) -> None: + e = Edit(op="insert_after", content="insertion", target="anchor") + assert e.op == "insert_after" + assert e.content == "insertion" + assert e.target == "anchor" + + def test_delete_op(self) -> None: + e = Edit(op="delete", target="thing_to_remove") + assert e.op == "delete" + assert e.target == "thing_to_remove" + + +class TestEditRoundTrip: + """Edit.to_dict() / Edit.from_dict() round-trip.""" + + def test_round_trip_minimal(self) -> None: + e = Edit(op="append") + d = e.to_dict() + restored = Edit.from_dict(d) + assert restored == e + + def test_round_trip_full(self) -> None: + e = Edit( + op="replace", + content="new content", + target="old content", + support_count=3, + source_type="success", + merge_level=1, + update_origin="meta_reflect", + update_target="system_prompt", + ) + d = e.to_dict() + restored = Edit.from_dict(d) + assert restored == e + + def test_round_trip_delete_without_content(self) -> None: + e = Edit(op="delete", target="obsolete_line") + d = e.to_dict() + restored = Edit.from_dict(d) + assert restored == e + + def test_optional_fields_omitted_when_default(self) -> None: + e = Edit(op="append") + d = e.to_dict() + assert d == {"op": "append", "content": ""} + # support_count, source_type, etc. should be absent + assert "support_count" not in d + assert "source_type" not in d + assert "merge_level" not in d + assert "target" not in d + assert "update_origin" not in d + assert "update_target" not in d + + def test_from_dict_with_defaults(self) -> None: + d = {"op": "replace", "content": "abc"} + e = Edit.from_dict(d) + assert e.op == "replace" + assert e.content == "abc" + assert e.target == "" + assert e.support_count is None + assert e.source_type is None + + def test_from_dict_with_extra_keys(self) -> None: + """Extra keys in dict should be ignored.""" + d = {"op": "append", "content": "", "unknown_field": 42} + e = Edit.from_dict(d) + assert e.op == "append" + assert not hasattr(e, "unknown_field") + + +class TestEditEdgeCases: + """Edge cases around Edit.""" + + def test_support_count_zero(self) -> None: + """0 is a valid support_count and should be serialized.""" + e = Edit(op="append", support_count=0) + d = e.to_dict() + assert d["support_count"] == 0 + restored = Edit.from_dict(d) + assert restored.support_count == 0 + + def test_merge_level_zero(self) -> None: + e = Edit(op="replace", merge_level=0) + d = e.to_dict() + assert d["merge_level"] == 0 + restored = Edit.from_dict(d) + assert restored.merge_level == 0 + + def test_empty_target_stays_empty(self) -> None: + e = Edit(op="append", target="") + d = e.to_dict() + assert "target" not in d + + +# ── Patch ─────────────────────────────────────────────────────────────────── + + +class TestPatchCreation: + """Patch dataclass construction.""" + + def test_empty_patch(self) -> None: + p = Patch() + assert p.edits == [] + assert p.reasoning == "" + assert p.ranking_details is None + + def test_patch_with_edits(self) -> None: + edits = [ + Edit(op="append", content="step 1"), + Edit(op="append", content="step 2"), + ] + p = Patch(edits=edits, reasoning="Added two steps") + assert len(p.edits) == 2 + assert p.reasoning == "Added two steps" + + def test_patch_with_ranking_details(self) -> None: + p = Patch(ranking_details={"score": 0.95, "rank": 1}) + assert p.ranking_details == {"score": 0.95, "rank": 1} + + +class TestPatchRoundTrip: + """Patch.to_dict() / Patch.from_dict() round-trip.""" + + def test_round_trip_empty(self) -> None: + p = Patch() + d = p.to_dict() + restored = Patch.from_dict(d) + assert restored.edits == [] + assert restored.reasoning == "" + assert restored.ranking_details is None + + def test_round_trip_with_edits(self) -> None: + edits = [ + Edit(op="insert_after", content="new step", target="existing step"), + Edit(op="replace", content="updated", target="old"), + ] + p = Patch(edits=edits, reasoning="Batch update") + d = p.to_dict() + restored = Patch.from_dict(d) + assert len(restored.edits) == 2 + for original, restored_edit in zip(p.edits, restored.edits): + assert isinstance(restored_edit, Edit) + assert original == restored_edit + assert restored.reasoning == "Batch update" + assert restored.ranking_details is None + + def test_round_trip_with_ranking_details(self) -> None: + details = {"strategy": "rouge", "scores": [0.9, 0.8, 0.7]} + p = Patch( + edits=[Edit(op="append", content="a")], + reasoning="selected best", + ranking_details=details, + ) + d = p.to_dict() + restored = Patch.from_dict(d) + assert restored.ranking_details == details + + def test_to_dict_contains_reasoning_and_edits(self) -> None: + p = Patch(edits=[Edit(op="append", content="test")], reasoning="reason") + d = p.to_dict() + assert "reasoning" in d + assert "edits" in d + assert isinstance(d["edits"], list) + + def test_from_dict_preserves_edit_order(self) -> None: + edits = [ + Edit(op="append", content="first"), + Edit(op="insert_after", content="second", target="first"), + Edit(op="append", content="third"), + ] + p = Patch(edits=edits, reasoning="ordered") + d = p.to_dict() + restored = Patch.from_dict(d) + assert restored.edits[0].content == "first" + assert restored.edits[1].content == "second" + assert restored.edits[2].content == "third" + + +class TestPatchEdgeCases: + """Edge cases around Patch.""" + + def test_reasoning_empty_string(self) -> None: + p = Patch(reasoning="") + d = p.to_dict() + assert d["reasoning"] == "" + + def test_zero_edits(self) -> None: + """Patch with explicitly empty edit list.""" + p = Patch(edits=[]) + d = p.to_dict() + assert d["edits"] == [] + + def test_nested_edit_from_dict_handles_dicts(self) -> None: + """from_dict should accept dicts in the 'edits' list.""" + d = { + "reasoning": "test", + "edits": [{"op": "append", "content": "hello"}], + } + p = Patch.from_dict(d) + assert len(p.edits) == 1 + assert isinstance(p.edits[0], Edit) + assert p.edits[0].op == "append" + assert p.edits[0].content == "hello"