diff --git a/.dockerignore b/.dockerignore index c5d724a4..3ea5b1ba 100644 --- a/.dockerignore +++ b/.dockerignore @@ -2,4 +2,7 @@ .git __pycache__ *.pyc +.pytest_cache results +.jetskicli + diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index ff2cd69e..407677e0 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -5,4 +5,4 @@ # the repo. Unless a later match takes precedence, # @global-owner1 and @global-owner2 will be requested for # review when someone opens a pull request. -* @ismailmehdi +* @ismailmehdi @helloeve @prernakakkar-google diff --git a/.github/labels.yml b/.github/labels.yml new file mode 100644 index 00000000..ef534bc7 --- /dev/null +++ b/.github/labels.yml @@ -0,0 +1,105 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: duplicate + color: ededed + description: "" + +- name: 'type: bug' + color: db4437 + description: Error or flaw in code with unintended results or allowing sub-optimal usage patterns. + +- name: 'type: cleanup' + color: c5def5 + description: An internal cleanup or hygiene concern. + +- name: 'type: docs' + color: 0000A0 + description: Improvement to the documentation for an API. + +- name: 'type: feature request' + color: c5def5 + description: "'Nice-to-have' improvement, new feature or different behavior or design." + +- name: 'type: process' + color: c5def5 + description: A process-related concern. May include testing, release, or the like. + +- name: 'type: question' + color: c5def5 + description: Request for information or clarification. + +- name: 'priority: p0' + color: b60205 + description: Highest priority. Critical issue. P0 implies highest priority. + +- name: 'priority: p1' + color: ffa03e + description: Important issue which blocks shipping the next release. Will be fixed prior to next release. + +- name: 'priority: p2' + color: fef2c0 + description: Moderately-important priority. Fix may not be included in next release. + +- name: 'priority: p3' + color: ffffc7 + description: Desirable enhancement or fix. May not be included in next release. + +- name: 'do not merge' + color: d93f0b + description: Indicates a pull request not ready for merge, due to either quality or timing. + +- name: 'autorelease: pending' + color: ededed + description: Release please needs to do its work on this. + +- name: 'autorelease: triggered' + color: ededed + description: Release please has triggered a release for this. + +- name: 'autorelease: tagged' + color: ededed + description: Release please has completed a release for this. + +- name: 'status: help wanted' + color: 8befd7 + description: 'Status: Unplanned work open to contributions from the community.' + +- name: 'status: feedback wanted' + color: 8befd7 + description: 'Status: waiting for feedback from community or issue author.' + +- name: 'status: waiting for response' + color: 8befd7 + description: 'Status: reviewer is awaiting feedback or responses from the author before proceeding.' + +- name: good first issue + color: 7057ff + description: Good for newcomers + +- name: generator + color: bfd4f2 + description: Related to model/CLI generators (Jetski, Gemini CLI, Claude Code) + +- name: scorer + color: 7057ff + description: Related to evaluation scoring metrics + +- name: orchestrator + color: 008672 + description: Related to orchestrators and dataset execution + +- name: database + color: e4e669 + description: Database driver or connection pooling logic diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml new file mode 100644 index 00000000..07bdb74d --- /dev/null +++ b/.github/workflows/pytest.yml @@ -0,0 +1,31 @@ +name: run-pytest + +on: + pull_request: + branches: + - main + push: + branches: + - main + +jobs: + test: + runs-on: [self-hosted, bastion] + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: "3.11" + - name: Install uv + run: | + curl -LsSf https://astral.sh/uv/install.sh | sh + echo "$HOME/.local/bin" >> $GITHUB_PATH + - name: Generate Proto Files + run: uv run make proto + - name: Run tests + run: | + uv run --with pytest pytest evalbench/test -vv + env: + PYTHONPATH: ${{ github.workspace }}/evalbench:${{ github.workspace }} diff --git a/.gitignore b/.gitignore index 17318955..961a2fa8 100644 --- a/.gitignore +++ b/.gitignore @@ -176,6 +176,7 @@ results/* evalbench/results/* db_connections/bat/db_blog.db datasets/bird-interact/livesqlbench-base-lite-dumps +datasets/bird-interact/bird-interact-lite/ datasets/bird/datasets/bird/prompts.json datasets/bird/db_connections/bird/california_schools.sqlite datasets/bird/db_connections/bird/card_games.sqlite @@ -192,3 +193,14 @@ evalbench/db_connections/bat/db_blog.db # Autogenerated version file viewer/version.txt +.jetskicli/project.json + +# Local testing datasets/configs (do not commit) +datasets/bqml/ +datasets/model_configs/gemini_cli_test_model.yaml +datasets/model_configs/gemini_2.5_pro_test_model.yaml +datasets/model_configs/gemini_3_flash_test_model.yaml +datasets/model_configs/codex_cli_dak_model.yaml + + + diff --git a/.release-please-manifest.json b/.release-please-manifest.json index dd8fde77..23860f90 100644 --- a/.release-please-manifest.json +++ b/.release-please-manifest.json @@ -1,3 +1,3 @@ { - ".": "1.5.0" + ".": "1.13.0" } diff --git a/AGENTS.md b/AGENTS.md index 812c2032..53b801a2 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -52,7 +52,7 @@ graph TD ### Core Components - **Orchestrators**: Manages dataset breakdown and parallel execution stages (`OneShotOrchestrator`, `InteractOrchestrator`, `AgentOrchestrator`, `DataAgentOrchestrator`). - **Evaluators**: Executes the test scenario lifecycles for a given dialect or database (`Evaluator`, `InteractEvaluator`, `AgentEvaluator`, `DataAgentEvaluator`). -- **Generators**: Acts as the driver for the tested model or CLI (`GeminiCliGenerator`, `ClaudeCodeGenerator`, `QueryData`). +- **Generators**: Acts as the driver for the tested model or CLI (`GeminiCliGenerator`, `ClaudeCodeGenerator`, `CodexCliGenerator`, `AgyCliGenerator`, `QueryData`). - **Simulated Users**: Drives conversations autonomously by translating conversation plans into user messages. - **Scorers**: Computes correctness metrics (Exact Match, LLM-Rater, Trajectory Matcher, Behavioral Metrics). @@ -81,7 +81,9 @@ evalbench/ ├── generators/ # Tested system adapters │ ├── models/ │ │ ├── claude_code.py # Claude Code driver -│ │ └── gemini_cli.py # Gemini CLI driver +│ │ ├── codex_cli.py # Codex CLI driver +│ │ ├── gemini_cli.py # Gemini CLI driver +│ │ └── agy_cli.py # Antigravity (agy) CLI driver │ └── prompts/ ├── mp/ # Multi-processing / multi-threading runners │ └── mprunner.py @@ -105,9 +107,9 @@ When evaluating agentic frameworks that leverage external tools (e.g., Gemini CL | Paradigm | Supported Generators | How it Works | |---|---|---| -| **MCP Servers** | `gemini_cli`, `claude_code` | Remote HTTP/SSE or local stdio-based Model Context Protocol servers | -| **Extensions** | `gemini_cli` | GitHub-hosted plugin packages installed idempotently via CLI | -| **Skills** | `gemini_cli` | Local or registry skill packages enabled/linked in the sandboxed environment | +| **MCP Servers** | `gemini_cli`, `claude_code`, `codex_cli`, `agy_cli` | Remote HTTP/SSE or local stdio-based Model Context Protocol servers | +| **Extensions** | `gemini_cli`, `claude_code`, `codex_cli` | GitHub-hosted plugin packages installed idempotently via CLI | +| **Skills** | `gemini_cli`, `claude_code`, `codex_cli`, `agy_cli` | Skill packages installed into the sandboxed environment via each CLI's native mechanism (e.g. link/enable for gemini, `agy plugin install` for agy) | --- @@ -207,7 +209,7 @@ Contains the test cases. "id": "list-instances-01", "starting_prompt": "List all Cloud SQL instances in project my-evaluation-project", "conversation_plan": "Ensure the agent accurately calls list_instances. Verify the output is returned correctly.", - "expected_trajectory": ["list_instances"], + "expected_trajectory": ["cloud-sql__list_instances"], "env": { "GOOGLE_CLOUD_PROJECT": "my-evaluation-project" }, "max_turns": 4 } @@ -222,6 +224,7 @@ To upload metrics directly to Google BigQuery and create Looker Studio dashboard reporting: bigquery: gcp_project_id: 'your-gcp-project-id' + dataset_name: 'evalbench' # Optional, defaults to 'evalbench' dataset_location: 'US' # Optional, defaults to 'US' ``` @@ -231,10 +234,12 @@ reporting: ### Setup Python Virtual Environment +Using `uv` to manage the environment: + ```bash -python3 -m venv venv -source venv/bin/activate -pip install -r requirements.txt +uv venv +source .venv/bin/activate +uv sync ``` ### GCP Authentication @@ -271,6 +276,7 @@ If settings from a previous run conflict or cause issues, clear out the generato ```bash rm -rf .venv/fake_home rm -rf .venv/fake_home_claude +rm -rf .venv/fake_home_agy ``` ### Forcing Sequential Runs diff --git a/CHANGELOG.md b/CHANGELOG.md index a6a4e107..dfa2b7fb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,186 @@ # Changelog +## [1.13.0](https://github.com/GoogleCloudPlatform/evalbench/compare/v1.12.1...v1.13.0) (2026-07-24) + + +### Features + +* **mcp_readability:** move style judge to gemini-3.1-pro with truncation handling ([b4665f3](https://github.com/GoogleCloudPlatform/evalbench/commit/b4665f3d702e31198e8f511358415b92c04260cb)) + +## [1.12.1](https://github.com/GoogleCloudPlatform/evalbench/compare/v1.12.0...v1.12.1) (2026-07-23) + + +### Bug Fixes + +* add type checking to stream event parsing to safely handle non-dict payloads ([1003e08](https://github.com/GoogleCloudPlatform/evalbench/commit/1003e085fddc8a577108e96b5a1012c68048b283)) +* skip non-object lines in agy stream and report model errors based on result status ([e3ac999](https://github.com/GoogleCloudPlatform/evalbench/commit/e3ac99998a769c65b8248133dce0812df60d1ca3)) + +## [1.12.0](https://github.com/GoogleCloudPlatform/evalbench/compare/v1.11.0...v1.12.0) (2026-07-22) + + +### Features + +* **query_data_api:** add option to use REST API fallback for unreleased proto fields ([#503](https://github.com/GoogleCloudPlatform/evalbench/issues/503)) ([b29d37f](https://github.com/GoogleCloudPlatform/evalbench/commit/b29d37f81ea34704923b0358ea4a6c0fa072ce20)) + +## [1.11.0](https://github.com/GoogleCloudPlatform/evalbench/compare/v1.10.0...v1.11.0) (2026-07-21) + + +### Features + +* **dea:** parameterize url_agent_name and agent_type_uri options ([#497](https://github.com/GoogleCloudPlatform/evalbench/issues/497)) ([30a065f](https://github.com/GoogleCloudPlatform/evalbench/commit/30a065f3ca2005c6a0d7dbecfe639cbe811534cb)) +* **mcp_readability:** human-readable LLM feedback without score, with waivers ([c49f45f](https://github.com/GoogleCloudPlatform/evalbench/commit/c49f45f5dcc83f71818e5e8e3780083538e788b9)) +* **mcp_readability:** human-readable LLM feedback without score, with waivers ([ffa4d1f](https://github.com/GoogleCloudPlatform/evalbench/commit/ffa4d1f720142f50884a77e68505bafd8c8277dd)) + +## [1.10.0](https://github.com/GoogleCloudPlatform/evalbench/compare/v1.9.0...v1.10.0) (2026-07-13) + + +### Features + +* add 300-second timeout to QueryData API calls to prevent indefinite blocking ([76c7ca4](https://github.com/GoogleCloudPlatform/evalbench/commit/76c7ca4f4a7e0c56b280e855d791f7a823c06993)) +* Add ability to filter to specific json scenario/examples ([#453](https://github.com/GoogleCloudPlatform/evalbench/issues/453)) ([7c99d3d](https://github.com/GoogleCloudPlatform/evalbench/commit/7c99d3dc4f5b2f647de6aeab37ec62918103708f)) +* Add hybrid evaluation Python notebook ([#487](https://github.com/GoogleCloudPlatform/evalbench/issues/487)) ([ace0149](https://github.com/GoogleCloudPlatform/evalbench/commit/ace01490753ea35710a32dee0a8d557c6f7fe409)) +* **dea:** adjust conversation turns and plan for programmatic scenarios ([#479](https://github.com/GoogleCloudPlatform/evalbench/issues/479)) ([7434a91](https://github.com/GoogleCloudPlatform/evalbench/commit/7434a91bd7a5a2b2403b123b8697c45277fc9563)) +* **dea:** adjust conversation turns and plan for programmatic scenarios ([#479](https://github.com/GoogleCloudPlatform/evalbench/issues/479)) ([85b90ec](https://github.com/GoogleCloudPlatform/evalbench/commit/85b90ec384e437b1826b0f851cad2036da5d4caf)) +* **dea:** implement GCS archival for DataEngineeringAgent ([#476](https://github.com/GoogleCloudPlatform/evalbench/issues/476)) ([5e36350](https://github.com/GoogleCloudPlatform/evalbench/commit/5e36350fadacef0c207a792d031c8df0479fe4b5)) +* **dea:** implement GCS archival for DataEngineeringAgent ([#476](https://github.com/GoogleCloudPlatform/evalbench/issues/476)) ([89c24e4](https://github.com/GoogleCloudPlatform/evalbench/commit/89c24e4affe8a74c73c203abef3c115b7dcba38c)) +* **dea:** support dynamic workspace environment files mapping and injection ([#491](https://github.com/GoogleCloudPlatform/evalbench/issues/491)) ([a409608](https://github.com/GoogleCloudPlatform/evalbench/commit/a409608a7d7b7726f9762c3a7d78708692bd2bbe)) +* **dea:** support dynamic workspace lifecycle and parameter passing in evaluator, generator, and scorers ([#474](https://github.com/GoogleCloudPlatform/evalbench/issues/474)) ([b5e97f0](https://github.com/GoogleCloudPlatform/evalbench/commit/b5e97f0e40eae55ea4b33e053f03c9c3cc041be8)) +* **dea:** support dynamic workspace lifecycle and parameter passing in evaluator, generator, and scorers ([#474](https://github.com/GoogleCloudPlatform/evalbench/issues/474)) ([94b63ae](https://github.com/GoogleCloudPlatform/evalbench/commit/94b63aed03d70a07de0eed794c6c043bfb761140)) +* **dea:** support static workspaces (Mode 2) and update README ([#486](https://github.com/GoogleCloudPlatform/evalbench/issues/486)) ([39cb2b8](https://github.com/GoogleCloudPlatform/evalbench/commit/39cb2b8bdf87abe64c37ff634d01de6c932feb45)) +* **mcp-readability:** add MCP tool man-page formatter ([620cc0b](https://github.com/GoogleCloudPlatform/evalbench/commit/620cc0b62e81204d4b9f72d9b801b104c369b158)) +* **mcp-readability:** add MCP tool man-page formatter ([3e4c16e](https://github.com/GoogleCloudPlatform/evalbench/commit/3e4c16eafecd0752ed21c31e404384f7fd37b1c0)) +* **mcp-readability:** add McpToolsGenerator with pluggable tools_source ([66b67d5](https://github.com/GoogleCloudPlatform/evalbench/commit/66b67d566bd1131907368f180836f0bd309a3ad3)) +* **mcp-readability:** add sample run config, endpoints, style guide, exceptions ([a21ec5e](https://github.com/GoogleCloudPlatform/evalbench/commit/a21ec5ed49055617fdd624c7b00da698f2a6de41)) +* **mcp-readability:** compliance orchestrator, LLM judge, and metrics scorer ([52ffade](https://github.com/GoogleCloudPlatform/evalbench/commit/52ffade7567373a0781cf2c841a4fd96e1ee5bb5)) +* **mcp-readability:** compliance orchestrator, LLM judge, and metrics scorer ([fefb034](https://github.com/GoogleCloudPlatform/evalbench/commit/fefb0341314e6a28237d67133d4d6de60cc3cb12)) +* **mcp-readability:** compliance orchestrator, LLM judge, and metrics scorer ([222fd49](https://github.com/GoogleCloudPlatform/evalbench/commit/222fd4933abde6cbc81bbc7e5f1bf7737ef0e505)) +* natively copy declared env_files to agent sandbox ([#434](https://github.com/GoogleCloudPlatform/evalbench/issues/434)) ([db4e5b8](https://github.com/GoogleCloudPlatform/evalbench/commit/db4e5b8e6883b38831b3359aba0227a316dae94c)) +* **query_data_api:** support api_endpoint override in model config ([#480](https://github.com/GoogleCloudPlatform/evalbench/issues/480)) ([3f1868c](https://github.com/GoogleCloudPlatform/evalbench/commit/3f1868cc1983b41d4a61ae9c49c29ebf267f30b2)) +* **query_data_api:** support api_endpoint override in model config ([#480](https://github.com/GoogleCloudPlatform/evalbench/issues/480)) ([0d59e90](https://github.com/GoogleCloudPlatform/evalbench/commit/0d59e90632c83a2976dd22ce5edcd31b41d92da9)) +* Support cross-database evaluation with SQLite ground truth ([#465](https://github.com/GoogleCloudPlatform/evalbench/issues/465)) ([e71692c](https://github.com/GoogleCloudPlatform/evalbench/commit/e71692c3ec32e4fca62f937c331d82b05b491f72)) +* Support cross-database evaluation with SQLite ground truth ([#465](https://github.com/GoogleCloudPlatform/evalbench/issues/465)) ([3cb207b](https://github.com/GoogleCloudPlatform/evalbench/commit/3cb207b83084547887d7a96dc553d26172e425c5)) +* **util:** add DataformHelper utility for managing cloud sandboxes and mock-based unit tests ([#458](https://github.com/GoogleCloudPlatform/evalbench/issues/458)) ([aa1642a](https://github.com/GoogleCloudPlatform/evalbench/commit/aa1642ab51ef960f7035fb4f39060961950f66af)) +* **util:** update DataformHelper utility with high-level workspace lifecycle methods ([#466](https://github.com/GoogleCloudPlatform/evalbench/issues/466)) ([9a75a7a](https://github.com/GoogleCloudPlatform/evalbench/commit/9a75a7ab9a7d1651bc094ed933925c7fcb6c8ffb)) + + +### Bug Fixes + +* add timeout to data agent request in querydata generator ([a8b148e](https://github.com/GoogleCloudPlatform/evalbench/commit/a8b148e2397383bac94285f3c5ca6648ec18df1c)) +* add timeout to data agent request in querydata generator ([9573c14](https://github.com/GoogleCloudPlatform/evalbench/commit/9573c146a4eb3be73b0a49cd087d0b26163a6259)) +* **agy_cli:** drop unsupported -- delimiter from plugin install ([4f55589](https://github.com/GoogleCloudPlatform/evalbench/commit/4f55589d16467f9dfb1b64b57c3f06a6611bdde2)) +* **agy_cli:** drop unsupported -- delimiter from plugin install ([87f0945](https://github.com/GoogleCloudPlatform/evalbench/commit/87f094510830b063e2e9fea05360f9f4f8431042)) +* **dataform:** synchronize GCP project env vars in dataform scorer ([a15a54f](https://github.com/GoogleCloudPlatform/evalbench/commit/a15a54f0a593a6e3cb9c25493a0b86acc62eb554)) +* **dataform:** synchronize GCP project env vars in dataform scorer ([de7dc22](https://github.com/GoogleCloudPlatform/evalbench/commit/de7dc225508360dc01f0b6e3171fe12addeb4704)) +* **dataform:** synchronize GCP project env vars in dataform scorer ([be31ac6](https://github.com/GoogleCloudPlatform/evalbench/commit/be31ac6dffea917ce0abe5f6ad0d44a74be454b6)) +* **dataset:** load all knowledge-base entries, not just the last line ([b8b96c8](https://github.com/GoogleCloudPlatform/evalbench/commit/b8b96c8db96c820f81ea81e8c21fae151f00b046)) +* **dataset:** load all knowledge-base entries, not just the last line ([63441ce](https://github.com/GoogleCloudPlatform/evalbench/commit/63441ce5fde578055fd19afc6a6972908e70d445)) +* **dea:** accumulate all agent text parts and increase timeout ([#456](https://github.com/GoogleCloudPlatform/evalbench/issues/456)) ([cf3f758](https://github.com/GoogleCloudPlatform/evalbench/commit/cf3f7584ec3ec16698167ad312deac54d65a89df)) +* **deps:** add mcp to pyproject.toml dependencies ([62c7436](https://github.com/GoogleCloudPlatform/evalbench/commit/62c7436274ae3aac31e8ebdacbf8798f7f256ca1)) +* **deps:** add mcp to pyproject.toml dependencies ([5928137](https://github.com/GoogleCloudPlatform/evalbench/commit/5928137ae972dda50f3fe69c748547fe28d320f9)) +* **interact:** count total_db_len so DB-setup progress isn't always zero ([#446](https://github.com/GoogleCloudPlatform/evalbench/issues/446)) ([0d66535](https://github.com/GoogleCloudPlatform/evalbench/commit/0d66535eaa20e9b88fded96e0d498b4f9d68cb7e)) +* log metadata reflection failures instead of swallowing them ([cce4eb2](https://github.com/GoogleCloudPlatform/evalbench/commit/cce4eb20dc4b61497cac720b7b7bfed3ff16eb05)) +* log metadata reflection failures instead of swallowing them ([fa3c508](https://github.com/GoogleCloudPlatform/evalbench/commit/fa3c5086fb976310dbfea9e5ece1fc4d6f5b6bad)) +* **make:** eliminate container name race in container/shell targets ([#449](https://github.com/GoogleCloudPlatform/evalbench/issues/449)) ([8f1803d](https://github.com/GoogleCloudPlatform/evalbench/commit/8f1803d53ed214a9976ce7697d3fd9098a07ecda)) +* re-raise 404 as bare HTTPError instead of ResourceExhaustedError ([3307570](https://github.com/GoogleCloudPlatform/evalbench/commit/330757065d1c23d670b6668ae5224a25cf45c6df)) +* re-raise 404 errors in querydata generate_internal ([7a2bf1f](https://github.com/GoogleCloudPlatform/evalbench/commit/7a2bf1f246de36bc6ae5d88734b6108ca18a90b5)) +* re-raise 404 errors in querydata generate_internal ([b7f7b30](https://github.com/GoogleCloudPlatform/evalbench/commit/b7f7b304521d2363f0bbc5622327efca2984a713)) +* **reporting:** Add retry logic and configurable chunk size for BigQuery writes ([6b5f0a6](https://github.com/GoogleCloudPlatform/evalbench/commit/6b5f0a63ceafe713a4034cd80ad23244a517d5e4)) +* **reporting:** Add retry logic and configurable chunk size for BigQuery writes ([8774d02](https://github.com/GoogleCloudPlatform/evalbench/commit/8774d02db9391306144f9845e8bdb741fe561b25)) +* restrict unpickling of cached results to safe types ([#439](https://github.com/GoogleCloudPlatform/evalbench/issues/439)) ([067dfe3](https://github.com/GoogleCloudPlatform/evalbench/commit/067dfe39fdb2f8d47d6de2ff787d433d28c6edaa)) +* **scorers:** correct digit regex in behavioral metrics scorer ([78ecdfc](https://github.com/GoogleCloudPlatform/evalbench/commit/78ecdfc67cd0c88ac394d3eb41be57bce8b08b27)) +* **scorers:** correct digit regex in behavioral metrics scorer ([c8d82ea](https://github.com/GoogleCloudPlatform/evalbench/commit/c8d82ea2e784f87273135532855cf2b2f2352463)) +* tight copy of claude settings from local environment ([#467](https://github.com/GoogleCloudPlatform/evalbench/issues/467)) ([474a9be](https://github.com/GoogleCloudPlatform/evalbench/commit/474a9beebb42c26223951d25a1435202fb47c5ca)) + +## [1.9.0](https://github.com/GoogleCloudPlatform/evalbench/compare/v1.8.0...v1.9.0) (2026-06-11) + + +### Features + +* add AgyCliGenerator support to evaluator and models, including test suite and configuration datasets ([9e6d71f](https://github.com/GoogleCloudPlatform/evalbench/commit/9e6d71fb88fd652acb3d1e1ff2109b8a7e92f3bc)) +* add AgyCliGenerator support to evaluator and models, including test suite and configuration datasets ([9e6d71f](https://github.com/GoogleCloudPlatform/evalbench/commit/9e6d71fb88fd652acb3d1e1ff2109b8a7e92f3bc)) +* add Antigravity agent tab and update CLI version retrieval logic ([fe1a0ae](https://github.com/GoogleCloudPlatform/evalbench/commit/fe1a0ae2415a87bd090d18c0cde5548030c9e439)) +* add Antigravity agent tab and update CLI version retrieval logic ([147680b](https://github.com/GoogleCloudPlatform/evalbench/commit/147680b34e375aa9c2d33e7cf6a8d1e197ce0553)) +* **dea:** support YAML-only configuration and penguins dataset for DEA conversational evaluation ([#410](https://github.com/GoogleCloudPlatform/evalbench/issues/410)) ([2e37c1c](https://github.com/GoogleCloudPlatform/evalbench/commit/2e37c1ca804fe51a21d824892c9c6dd67821c406)) +* recover resolved model label from agy cli logs for statistics bucket tagging ([ee7635f](https://github.com/GoogleCloudPlatform/evalbench/commit/ee7635f8d6f89647cef9bc66b79f637c83c30b2d)) +* update MCP config translation to support both serverUrl and url fields in agy cli ([685b1b2](https://github.com/GoogleCloudPlatform/evalbench/commit/685b1b2c6b2699a6b87749e03a55b60e0ff0d7ed)) + + +### Bug Fixes + +* **dea:** resolve concurrency deadlock in GcpAdcCredentialService ([#422](https://github.com/GoogleCloudPlatform/evalbench/issues/422)) ([6619ae5](https://github.com/GoogleCloudPlatform/evalbench/commit/6619ae50d18c5a2d096ad4c2293dd9502683dcda)) +* pin pyopenssl<26.2 to avoid google-auth mTLS regression ([#424](https://github.com/GoogleCloudPlatform/evalbench/issues/424)) ([c8916ae](https://github.com/GoogleCloudPlatform/evalbench/commit/c8916aeac729d2cf5da81838001195ec04a05d2a)) +* update error message to list supported AgentCliGenerator subclasses ([dfe1e8e](https://github.com/GoogleCloudPlatform/evalbench/commit/dfe1e8e27063208f710666f3634624050a527813)) +* update generator key path in trends data mapping to model_config.generator ([3739f9d](https://github.com/GoogleCloudPlatform/evalbench/commit/3739f9dd3dd7d5f8b5038f8b51b2bceb396e7a06)) + +## [1.8.0](https://github.com/GoogleCloudPlatform/evalbench/compare/v1.7.1...v1.8.0) (2026-06-03) + + +### Features + +* add Codex to agent filters in viewer ([a2be23c](https://github.com/GoogleCloudPlatform/evalbench/commit/a2be23caccb27a66d7a638063fc086445771a80b)) +* add Codex to agent filters in viewer ([#387](https://github.com/GoogleCloudPlatform/evalbench/issues/387)) ([2ac6c1f](https://github.com/GoogleCloudPlatform/evalbench/commit/2ac6c1f7cb493e919c97520736e34ad59ba61d3f)) +* add filter_native_tools option to trajectory_matcher to optionally ignore native harness tools during scoring ([2c34d94](https://github.com/GoogleCloudPlatform/evalbench/commit/2c34d94e13d71b2e491009f4858108388ea040de)) +* add packaged console script entrypoint to support `uvx` execution ([#385](https://github.com/GoogleCloudPlatform/evalbench/issues/385)) ([8ea07f8](https://github.com/GoogleCloudPlatform/evalbench/commit/8ea07f80fa8b58184b8677480dc886470c1e0662)) +* **dea:** define EvalDeaRequest input model for conversational evaluations ([#407](https://github.com/GoogleCloudPlatform/evalbench/issues/407)) ([04b91cf](https://github.com/GoogleCloudPlatform/evalbench/commit/04b91cfafa9c049122b5530e89f790572f479058)) +* opt-in function-calling for the Gemini SDK judge ([#409](https://github.com/GoogleCloudPlatform/evalbench/issues/409)) ([d97f511](https://github.com/GoogleCloudPlatform/evalbench/commit/d97f511770941c7cf1acf975ec38b52b5030f51b)) +* Rename package to google-evalbench and decouple viewer dependencies ([#390](https://github.com/GoogleCloudPlatform/evalbench/issues/390)) ([0d75811](https://github.com/GoogleCloudPlatform/evalbench/commit/0d758112ae9f6618a7c8058f032f44a440a381f8)) +* **scorers:** filter native tools in trajectory_matcher with opt-out flag ([2c1ab58](https://github.com/GoogleCloudPlatform/evalbench/commit/2c1ab58dee8e52ebdc8b39751032ca6ade0000d2)) +* stabilize Cloud Run deployment and polish standalone CLI UX ([#389](https://github.com/GoogleCloudPlatform/evalbench/issues/389)) ([4720eef](https://github.com/GoogleCloudPlatform/evalbench/commit/4720eefbebed97d9f21a01a6d4541d77267b443c)) +* support work_dir for claude code eval ([#403](https://github.com/GoogleCloudPlatform/evalbench/issues/403)) ([179e0d3](https://github.com/GoogleCloudPlatform/evalbench/commit/179e0d3f41584112e686675b68ee37a52bea3492)) + + +### Bug Fixes + +* add --no-sync flag to runtime uv run commands to prevent PyPI timeouts ([#392](https://github.com/GoogleCloudPlatform/evalbench/issues/392)) ([0c4783c](https://github.com/GoogleCloudPlatform/evalbench/commit/0c4783c2ca3d1c6a62d844faf9f359119c9dd6f1)) +* allow-list files in fake home directory for Gemini CLI ([#395](https://github.com/GoogleCloudPlatform/evalbench/issues/395)) ([734bc2a](https://github.com/GoogleCloudPlatform/evalbench/commit/734bc2a154a264e4066d4e826d51c44bcbc1c603)) +* fix Mesop event routing bug in trends dropdown ([023d150](https://github.com/GoogleCloudPlatform/evalbench/commit/023d1505a73e77835f75746dfc5624a921ce2832)) +* **gemini-cli:** support 'name' parameter key in skill extraction ([#378](https://github.com/GoogleCloudPlatform/evalbench/issues/378)) ([62400da](https://github.com/GoogleCloudPlatform/evalbench/commit/62400daf314926a0cd7bfd916997e9203e91f8b5)) +* patch absl help output when running via uvx/launcher ([85048e2](https://github.com/GoogleCloudPlatform/evalbench/commit/85048e2760d7951351423facbf0c6fa2106b439e)) +* prevent silent errors on DB query timeouts and extend deadline ([#406](https://github.com/GoogleCloudPlatform/evalbench/issues/406)) ([fbbd31d](https://github.com/GoogleCloudPlatform/evalbench/commit/fbbd31d42a3af782411ae1e0c131f4ef2beef143)) +* surface eval failures instead of silently terminating or crashing ([#398](https://github.com/GoogleCloudPlatform/evalbench/issues/398)) ([9c36108](https://github.com/GoogleCloudPlatform/evalbench/commit/9c361083f53ae3b8ae622aafedd7d483701ca413)) + +## [1.7.1](https://github.com/GoogleCloudPlatform/evalbench/compare/v1.7.0...v1.7.1) (2026-05-07) + + +### Bug Fixes + +* trigger release-please for username/password issue ([#371](https://github.com/GoogleCloudPlatform/evalbench/issues/371)) ([506a717](https://github.com/GoogleCloudPlatform/evalbench/commit/506a71712062ded66108900460cbb73afc2a9a0c)) + +## [1.7.0](https://github.com/GoogleCloudPlatform/evalbench/compare/v1.6.0...v1.7.0) (2026-05-05) + + +### Features + +* add Dataform scorers and plumb isolated fake_home workspace directory tracking ([#349](https://github.com/GoogleCloudPlatform/evalbench/issues/349)) ([b4ddfda](https://github.com/GoogleCloudPlatform/evalbench/commit/b4ddfda9241eae431ea143ea87c5cb8a99cea989)) +* Add dbt Scorers for Agent Evaluations ([#367](https://github.com/GoogleCloudPlatform/evalbench/issues/367)) ([5496d59](https://github.com/GoogleCloudPlatform/evalbench/commit/5496d59948b6e6f86322c93e25aecc664ed5e73c)) +* Add GCS Artifacts Reporter for Agent Evaluations ([#366](https://github.com/GoogleCloudPlatform/evalbench/issues/366)) ([11def06](https://github.com/GoogleCloudPlatform/evalbench/commit/11def06bc3f7a68acf17c890127dda13c0ebe50c)) +* implement lifecycle execution for setup and teardown scripts ([#360](https://github.com/GoogleCloudPlatform/evalbench/issues/360)) ([9de38da](https://github.com/GoogleCloudPlatform/evalbench/commit/9de38da0fca549b8e160044efe6891e630607450)) + + +### Bug Fixes + +* correct indentation in eval_service.py to resolve SyntaxError ([8512441](https://github.com/GoogleCloudPlatform/evalbench/commit/851244137ddc01271216cfe44c5bc090e0ab5bac)) +* correct return signature in base Orchestrator.process() ([ad8228e](https://github.com/GoogleCloudPlatform/evalbench/commit/ad8228e0db3bea785b030da354020ebc8e6b83df)) +* correct return signature in remaining Orchestrators ([b78a9e9](https://github.com/GoogleCloudPlatform/evalbench/commit/b78a9e9759b364755b26898e459a1821cdae1b6f)) +* pass None for missing metrics parameter in AgentOrchestrator results initialization ([c72c758](https://github.com/GoogleCloudPlatform/evalbench/commit/c72c7580dedc3fdfe82121d229c2582680d13a4e)) + +## [1.6.0](https://github.com/GoogleCloudPlatform/evalbench/compare/v1.5.0...v1.6.0) (2026-04-30) + + +### Features + +* add exponential backoff for Gemini 429 resource exhausted errors ([#352](https://github.com/GoogleCloudPlatform/evalbench/issues/352)) ([11d4236](https://github.com/GoogleCloudPlatform/evalbench/commit/11d423658a0b79385e5e05644cb614b32674b0dd)) +* allow execution of mixed DDL and DML statements in Spanner driver ([#351](https://github.com/GoogleCloudPlatform/evalbench/issues/351)) ([64c0b63](https://github.com/GoogleCloudPlatform/evalbench/commit/64c0b630ac95e9ad372afa516d6dc7b34be87ec9)) +* chunk Spanner DDL statements into groups of 10 to avoid limits ([#353](https://github.com/GoogleCloudPlatform/evalbench/issues/353)) ([306caad](https://github.com/GoogleCloudPlatform/evalbench/commit/306caad98bb5ee6825d834441edbf3a95067b030)) +* handle quoted fields in CSV setup data ([#350](https://github.com/GoogleCloudPlatform/evalbench/issues/350)) ([3ba2334](https://github.com/GoogleCloudPlatform/evalbench/commit/3ba2334d5a816fb4d4614a814e9c8a0a53ab1059)) +* handle recursive dependency cleanup in Spanner table drop ([#356](https://github.com/GoogleCloudPlatform/evalbench/issues/356)) ([3eb4d45](https://github.com/GoogleCloudPlatform/evalbench/commit/3eb4d45c2ecbc8bda538f5df7748f32e77ca0b17)) +* use parameterized queries for data insertion in MySQL, Postgres, and SQLite ([#358](https://github.com/GoogleCloudPlatform/evalbench/issues/358)) ([733724d](https://github.com/GoogleCloudPlatform/evalbench/commit/733724d9978b73c274dd0a489b1c35cd78c2cb75)) + + +### Bug Fixes + +* ensure setup and cleanup sql run for ddl and dml queries ([#354](https://github.com/GoogleCloudPlatform/evalbench/issues/354)) ([89b7c4d](https://github.com/GoogleCloudPlatform/evalbench/commit/89b7c4d438680cf3c96d6399d72fc46ff1ad6bdd)) + ## [1.5.0](https://github.com/GoogleCloudPlatform/evalbench/compare/v1.4.0...v1.5.0) (2026-04-26) diff --git a/Makefile b/Makefile index ab31d2f9..6ec2ce13 100644 --- a/Makefile +++ b/Makefile @@ -38,12 +38,18 @@ build-test: $(CONTAINER_ENGINE) build -t evalbench-test -f evalbench_service/Dockerfile . container: - $(CONTAINER_ENGINE) stop evalbench_server || true - $(CONTAINER_ENGINE) rm evalbench_server || true + $(CONTAINER_ENGINE) rm -f evalbench_server 2>/dev/null || true; \ + retries=60; \ + while $(CONTAINER_ENGINE) ps -a --format '{{.Names}}' | grep -qx evalbench_server; do \ + sleep 0.5; \ + retries=$$((retries - 1)); \ + if [ $$retries -le 0 ]; then echo "Timeout waiting for evalbench_server removal" >&2; exit 1; fi; \ + done $(CONTAINER_ENGINE) run --rm --name=evalbench_server \ $(if $(filter podman,$(CONTAINER_ENGINE)),--sysctl net.ipv6.conf.all.disable_ipv6=1) \ $(if $(filter docker,$(CONTAINER_ENGINE)),--net=host) \ -v ~/.config/gcloud:/root/.config/gcloud \ + -v ~/.gemini/antigravity-cli:/root/.gemini/antigravity-cli:ro \ -e GOOGLE_CLOUD_PROJECT=cloud-db-nl2sql \ -e MESOP_XSRF_CHECK=false \ --cap-add=SYS_PTRACE \ @@ -52,14 +58,19 @@ container: -e TYPE=$(TYPE) evalbench:latest shell: - $(CONTAINER_ENGINE) stop evalbench_server || true - $(CONTAINER_ENGINE) rm evalbench_server || true + $(CONTAINER_ENGINE) rm -f evalbench_server 2>/dev/null || true; \ + retries=60; \ + while $(CONTAINER_ENGINE) ps -a --format '{{.Names}}' | grep -qx evalbench_server; do \ + sleep 0.5; \ + retries=$$((retries - 1)); \ + if [ $$retries -le 0 ]; then echo "Timeout waiting for evalbench_server removal" >&2; exit 1; fi; \ + done $(CONTAINER_ENGINE) run -ti --rm --name=evalbench_server \ $(if $(filter podman,$(CONTAINER_ENGINE)),--sysctl net.ipv6.conf.all.disable_ipv6=1) \ $(if $(filter docker,$(CONTAINER_ENGINE)),--net=host) \ --cap-add=SYS_PTRACE \ -v ~/.config/gcloud:/root/.config/gcloud \ - -v $(PWD)/requirements.txt:/evalbench/requirements.txt \ + -v ~/.gemini/antigravity-cli:/root/.gemini/antigravity-cli:ro \ -v $(PWD)/evalbench:/evalbench/evalbench \ -v $(PWD)/viewer:/evalbench/viewer \ -p 3000:3000 \ diff --git a/README.md b/README.md index bd14b35c..280c002e 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,7 @@ EvalBench is a flexible framework designed to measure the quality of generative ## Getting Started      [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/GoogleCloudPlatform/evalbench/blob/main/docs/examples/sqlite_example.ipynb) Follow the steps below to run EvalBench on your local VM: -> *Note*: Evalbench requires python 3.10 or higher. +> *Note*: Evalbench requires Python 3.10 or higher and **uv** for dependency management. ### 1. Clone the Repository @@ -19,27 +19,22 @@ git clone git@github.com:GoogleCloudPlatform/evalbench.git ### 2. Set Up a Virtual Environment -Navigate to the repository directory and create a virtual environment: +Navigate to the repository directory and create a virtual environment using `uv`: ```bash cd evalbench -python3 -m venv venv -source venv/bin/activate +uv venv +source .venv/bin/activate ``` ### 3. Install Dependencies -Install the required Python dependencies: +Install the required Python dependencies using `uv`: ```bash -pip install -r requirements.txt +uv sync ``` -Due to proto conflict between google-cloud packages you may need to force install common-protos: - ``` - pip install --force-reinstall googleapis-common-protos==1.64.0 - ``` - ### 4. Configure GCP Authentication (For Vertex AI | Gemini Examples) If gcloud is not installed already, follow the steps in [gcloud installation guide](https://cloud.google.com/sdk/docs/install#installation_instructions). @@ -63,7 +58,7 @@ export EVAL_GCP_PROJECT_REGION=your_region_here For a quick start, let's run NL2SQL on some sqlite DQL queries. -1. First, read through [sqlite/run_dql.yaml](/datasets/bat/example_run_config.yaml) and see the configuration settings we will be running. +1. First, read through [datasets/bat/example_run_config.yaml](/datasets/bat/example_run_config.yaml) and see the configuration settings we will be running. Now, configure your evaluation by setting the `EVAL_CONFIG` environment variable. For example, to run a configuration using the `db_blog` dataset on SQLite: diff --git a/cloudbuild.yaml b/cloudbuild.yaml index 10d71752..3f8c53a5 100644 --- a/cloudbuild.yaml +++ b/cloudbuild.yaml @@ -1,6 +1,19 @@ # Building deployer image steps: +- name: 'python:3.11' + entrypoint: 'bash' + args: + - '-c' + - | + curl -LsSf https://astral.sh/uv/install.sh | sh + $$HOME/.local/bin/uv sync + $$HOME/.local/bin/uv run python -m grpc_tools.protoc --proto_path=evalbench/evalproto --python_out=evalbench/evalproto --pyi_out=evalbench/evalproto --grpc_python_out=evalbench/evalproto --experimental_editions evalbench/evalproto/*.proto + $$HOME/.local/bin/uv run pytest evalbench/test -vv + env: + - 'PYTHONPATH=evalbench:.' + - 'SKIP_CLOUD_TESTS=true' - name: 'gcr.io/cloud-builders/docker' + args: ['build', '-t', 'us-central1-docker.pkg.dev/cloud-db-nl2sql/evalbench/eval_server:$COMMIT_SHA', '-f', 'evalbench_service/Dockerfile', '.'] - name: 'us-central1-docker.pkg.dev/cloud-db-nl2sql/evalbench/eval_server:$COMMIT_SHA' dir: '/evalbench' @@ -10,6 +23,15 @@ steps: - 'EVAL_GCP_PROJECT_REGION=${_VAR_REGION}' - 'EVAL_CONFIG=${_VAR_EVAL_CONFIG}' - 'UV_CACHE_DIR=/tmp/uv-cache' + volumes: + - name: 'eval_results' + path: '/evalbench/results' +- name: 'us-central1-docker.pkg.dev/cloud-db-nl2sql/evalbench/eval_server:$COMMIT_SHA' + dir: '/evalbench' + args: ['uv', 'run', 'python', 'verifier/verify.py'] + volumes: + - name: 'eval_results' + path: '/evalbench/results' images: - 'us-central1-docker.pkg.dev/cloud-db-nl2sql/evalbench/eval_server:$COMMIT_SHA' substitutions: diff --git a/datasets/agy-cli-tools/agy-cli-fake.evalset.json b/datasets/agy-cli-tools/agy-cli-fake.evalset.json new file mode 100644 index 00000000..0db8e2a4 --- /dev/null +++ b/datasets/agy-cli-tools/agy-cli-fake.evalset.json @@ -0,0 +1,30 @@ +{ + "scenarios": [ + { + "id": "fake-csql-create-instance-success", + "starting_prompt": "Create a new Cloud SQL instance named 'my-fake-db' in project 'example-project'. Use PostgreSQL 17, and set the password to 'password123'. Also use the 'Development' edition preset.", + "conversation_plan": "The user wants to create a database. All required parameters are in the starting prompt. The agent should call create_instance and report the success message back.", + "expected_trajectory": [ + "cloud-sql__create_instance" + ], + "env": { + "GOOGLE_CLOUD_PROJECT": "example-project" + }, + "kind": "tools", + "max_turns": 3 + }, + { + "id": "fake-csql-get-instance-failure", + "starting_prompt": "Get the details for the Cloud SQL instance named 'missing-db' in project 'example-project'.", + "conversation_plan": "The user wants to get instance details. The agent should call get_instance, which is hardcoded to fail with an error 'Instance not found or permission denied'. The agent should explain that the instance could not be found based on the error.", + "expected_trajectory": [ + "cloud-sql__get_instance" + ], + "env": { + "GOOGLE_CLOUD_PROJECT": "example-project" + }, + "kind": "tools", + "max_turns": 3 + } + ] +} diff --git a/datasets/agy-cli-tools/agy-cli.evalset.json b/datasets/agy-cli-tools/agy-cli.evalset.json new file mode 100644 index 00000000..eb5ccb71 --- /dev/null +++ b/datasets/agy-cli-tools/agy-cli.evalset.json @@ -0,0 +1,33 @@ +{ + "scenarios": [ + { + "id": "cloud-sql-debug-01", + "starting_prompt": "list all instances in project ${EVAL_GCP_PROJECT_ID}", + "conversation_plan": "Ask the agent to list all instances in project ${EVAL_GCP_PROJECT_ID}. Once the instances are listed, if an instance named 'nl2code' exists, get its details and verify that its state is RUNNABLE.", + "expected_trajectory": [ + "cloud-sql__list_instances", + "cloud-sql__get_instance" + ], + "env": { + "GOOGLE_CLOUD_PROJECT": "${EVAL_GCP_PROJECT_ID}" + }, + "kind": "tools", + "max_turns": 3 + }, + { + "id": "csql-update-instance-scaling", + "starting_prompt": "Scale up my database instance.", + "conversation_plan": "The user wants to increase the resources for 'nl2code' in ${EVAL_GCP_PROJECT_ID} project. When asked for specifics, state you want to change the tier/machine type to 'db-custom-4-15360' (4 vCPUs, 15GB RAM). Confirm the update when asked. Verify the operation status.", + "expected_trajectory": [ + "cloud-sql__get_instance", + "cloud-sql__update_instance", + "cloud-sql__get_operation" + ], + "env": { + "GOOGLE_CLOUD_PROJECT": "${EVAL_GCP_PROJECT_ID}" + }, + "kind": "tools", + "max_turns": 6 + } + ] +} diff --git a/datasets/agy-cli-tools/example_run_config.yaml b/datasets/agy-cli-tools/example_run_config.yaml new file mode 100644 index 00000000..ea40b9d0 --- /dev/null +++ b/datasets/agy-cli-tools/example_run_config.yaml @@ -0,0 +1,39 @@ +############################################################ +### Dataset / Eval Items +############################################################ +dataset_config: datasets/agy-cli-tools/agy-cli.evalset.json +dataset_format: agent-format + +# Orchestrator Configuration +orchestrator: agent +model_config: datasets/model_configs/agy_cli_model.yaml +simulated_user_model_config: datasets/model_configs/gemini_2.5_pro_model.yaml + +############################################################ +### Scorer Related Configs +############################################################ +scorers: + # By default trajectory_matcher drops native/harness-internal tools + # (run_shell_command, write_file, update_topic, ...) from both expected + # and actual trajectories before scoring. Uncomment the block below to + # keep them and score native-tool usage too. + trajectory_matcher: {} + # trajectory_matcher: + # filter_native_tools: false + goal_completion: + model_config: datasets/model_configs/gemini_2.5_pro_model.yaml + behavioral_metrics: + model_config: datasets/model_configs/gemini_2.5_pro_model.yaml + parameter_analysis: + model_config: datasets/model_configs/gemini_2.5_pro_model.yaml + turn_count: {} + end_to_end_latency: {} + tool_call_latency: {} + token_consumption: {} + +############################################################ +### Reporting Related Configs +############################################################ +reporting: + csv: + output_directory: 'results' diff --git a/datasets/agy-cli-tools/example_run_fake_config.yaml b/datasets/agy-cli-tools/example_run_fake_config.yaml new file mode 100644 index 00000000..8719bdee --- /dev/null +++ b/datasets/agy-cli-tools/example_run_fake_config.yaml @@ -0,0 +1,40 @@ +############################################################ +### Dataset / Eval Items +############################################################ +dataset_config: datasets/agy-cli-tools/agy-cli-fake.evalset.json +dataset_format: agent-format + +# Orchestrator Configuration +orchestrator: agent +model_config: datasets/model_configs/agy_cli_fake_model.yaml +simulated_user_model_config: datasets/model_configs/gemini_2.5_pro_model.yaml + +############################################################ +### Scorer Related Configs +############################################################ +scorers: + # By default trajectory_matcher drops native/harness-internal tools + # (run_shell_command, write_file, update_topic, ...) from both expected + # and actual trajectories before scoring. Uncomment the block below to + # keep them and score native-tool usage too. + trajectory_matcher: {} + # trajectory_matcher: + # filter_native_tools: false + goal_completion: + model_config: datasets/model_configs/gemini_2.5_pro_model.yaml + behavioral_metrics: + model_config: datasets/model_configs/gemini_2.5_pro_model.yaml + parameter_analysis: + model_config: datasets/model_configs/gemini_2.5_pro_model.yaml + turn_count: {} + end_to_end_latency: {} + tool_call_latency: {} + # token_consumption omitted: agy's transcript exposes no token usage, so the + # scorer would only ever report 0. Re-add if agy starts emitting usage data. + +############################################################ +### Reporting Related Configs +############################################################ +reporting: + csv: + output_directory: 'results' diff --git a/datasets/agy-cli-tools/example_run_skills_config.yaml b/datasets/agy-cli-tools/example_run_skills_config.yaml new file mode 100644 index 00000000..62fe7d87 --- /dev/null +++ b/datasets/agy-cli-tools/example_run_skills_config.yaml @@ -0,0 +1,36 @@ +############################################################ +### Dataset / Eval Items +############################################################ +dataset_config: datasets/agy-cli-tools/agy-cli.evalset.json +dataset_format: agent-format + +# Orchestrator Configuration +orchestrator: agent +model_config: datasets/model_configs/agy_cli_skills_model.yaml +simulated_user_model_config: datasets/model_configs/gemini_2.5_pro_model.yaml + +############################################################ +### Scorer Related Configs +############################################################ +scorers: + # Note: trajectory_matcher and parameter_analysis are omitted here because + # skill-driven workflows use a different execution surface. The agent activates + # high-level skills rather than directly invoking granular tools. + # To evaluate skill execution paths, use the `skills_trajectory` scorer instead. + + goal_completion: + model_config: datasets/model_configs/gemini_2.5_pro_model.yaml + behavioral_metrics: + model_config: datasets/model_configs/gemini_2.5_pro_model.yaml + turn_count: {} + end_to_end_latency: {} + tool_call_latency: {} + # token_consumption omitted: agy's transcript exposes no token usage, so the + # scorer would only ever report 0. Re-add if agy starts emitting usage data. + +############################################################ +### Reporting Related Configs +############################################################ +reporting: + csv: + output_directory: 'results' diff --git a/datasets/bat/example_run_config.yaml b/datasets/bat/example_run_config.yaml index 88e4f9c3..95546025 100644 --- a/datasets/bat/example_run_config.yaml +++ b/datasets/bat/example_run_config.yaml @@ -3,6 +3,8 @@ ############################################################ # The JSON list of prompts / golden SQLs and eval attributes for the run dataset_config: datasets/bat/prompts.json +# Number of trials for multi-trial evaluation +num_trials: 2 ### Database Info # List of paths to the YAML files that provide the database connection details database_configs: @@ -42,6 +44,9 @@ scorers: returned_sql: null set_match: null executable_sql: null + exact_match_consistency: null + llm_consistency: + model_config: datasets/model_configs/gemini_2.5_pro_model.yaml # # ############################################################ @@ -50,3 +55,7 @@ scorers: reporting: csv: output_directory: 'results' +# bigquery: +# gcp_project_id: cloud-db-nl2sql + gcs_artifacts: + bucket: 'evalbench-sessions-cloud-db-nl2sql' diff --git a/datasets/bird-interact/bird-interact-lite b/datasets/bird-interact/bird-interact-lite deleted file mode 160000 index 10bb6194..00000000 --- a/datasets/bird-interact/bird-interact-lite +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 10bb61942c8d67d1df7dfc732ab30a6e0d33035c diff --git a/datasets/bird/example_hybrid_run_config.yaml b/datasets/bird/example_hybrid_run_config.yaml new file mode 100644 index 00000000..35217eb4 --- /dev/null +++ b/datasets/bird/example_hybrid_run_config.yaml @@ -0,0 +1,44 @@ +############################################################ +### Dataset / Eval Items (Hybrid BigQuery -> SQLite Mode) +############################################################ +# The JSON list of prompts / golden SQLs and eval attributes for the run +dataset_config: datasets/bird/prompts.json + +# Database configs mapping to BigQuery (for executing generated SQL) +database_configs: + - datasets/bat/db_configs/bigquery.yaml + +# Filter dialet to bigquery for query execution +dialects: + - bigquery +query_types: + - dql +dataset_format: bird-standard-format + +############################################################ +### Prompt and Generation Modules +############################################################ +# The YAML config for the model to be used for SQL generation +model_config: datasets/model_configs/gemini_2.5_pro_model.yaml +# The prompt generator module id +prompt_generator: 'SQLGenBasePromptGenerator' + +############################################################ +### Scorer Related Configs +############################################################ +scorers: + llmrater: + model_config: datasets/model_configs/gemini_2.5_pro_model.yaml + # Fallback to local SQLite database when golden query execution fails on BQ + hybrid_ground_truth: true + python_scorer: + # Isolated cross-database Execution Accuracy (XA) judge script + script_path: 'evalbench/scorers/judges/hybrid_xa_judge.py' + scorer_name: 'hybrid_cross_db' + +############################################################ +### Reporting Related Configs +############################################################ +reporting: + csv: + output_directory: 'results' diff --git a/datasets/claude-code-tools/claude-code-fake.evalset.json b/datasets/claude-code-tools/claude-code-fake.evalset.json index 1d9f584b..f02e8424 100644 --- a/datasets/claude-code-tools/claude-code-fake.evalset.json +++ b/datasets/claude-code-tools/claude-code-fake.evalset.json @@ -5,7 +5,7 @@ "starting_prompt": "Create a new Cloud SQL instance named 'my-fake-db' in project 'astana-evaluation'. Use PostgreSQL 17, and set the password to 'password123'. Also use the 'Development' edition preset.", "conversation_plan": "The user wants to create a database. All required parameters are in the starting prompt. The agent should call create_instance and report the success message back.", "expected_trajectory": [ - "create_instance" + "cloud-sql__create_instance" ], "env": { "GOOGLE_CLOUD_PROJECT": "astana-evaluation" @@ -18,7 +18,7 @@ "starting_prompt": "Get the details for the Cloud SQL instance named 'missing-db' in project 'astana-evaluation'.", "conversation_plan": "The user wants to get instance details. The agent should call get_instance, which is hardcoded to fail with an error 'Instance not found or permission denied'. The agent should explain that the instance could not be found based on the error.", "expected_trajectory": [ - "get_instance" + "cloud-sql__get_instance" ], "env": { "GOOGLE_CLOUD_PROJECT": "astana-evaluation" diff --git a/datasets/claude-code-tools/claude-code-skills.evalset.json b/datasets/claude-code-tools/claude-code-skills.evalset.json new file mode 100644 index 00000000..71be751f --- /dev/null +++ b/datasets/claude-code-tools/claude-code-skills.evalset.json @@ -0,0 +1,55 @@ +{ + "scenarios": [ + { + "id": "cloud-sql-list-instances-01", + "starting_prompt": "list all Cloud SQL instances in project astana-evaluation", + "conversation_plan": "Ask the agent to list instances in project astana-evaluation. Once all instances are listed if nl2code exist get its state and validate its RUNNABLE", + "expected_trajectory": [ + "list_instances.js", + "get_instance.js" + ], + "expected_skills": [ + "cloud-sql-postgres-admin" + ], + "env": { + "GOOGLE_CLOUD_PROJECT": "astana-evaluation" + }, + "kind": "tools", + "max_turns": 3 + }, + { + "id": "csql-create-ambiguous-multiturn-01", + "starting_prompt": "I need a database.", + "conversation_plan": "The user starts with a vague request. You want to CREATE a NEW Cloud SQL instance named 'my-pg-app'. If the agent offers to create one, say YES. When asked for details, provide 'my-pg-app' as the instance name and 'user_data' as the database name. Never claim to have an existing instance. The goal is for the agent to eventually create the database 'user_data' inside 'my-pg-app' in astana-evaluation project.", + "expected_trajectory": [ + "list_instances.js", + "create_instance.js", + "create_database.js" + ], + "expected_skills": [ + "cloud-sql-postgres-admin" + ], + "env": { + "GOOGLE_CLOUD_PROJECT": "astana-evaluation" + }, + "kind": "tools", + "max_turns": 6 + }, + { + "id": "csql-instance-not-found-failure", + "starting_prompt": "Update the instance 'non-existent-db-123' to have 8 cores.", + "conversation_plan": "The user asks to interact with an instance named 'non-existent-db-123' in astana-evaluation project that doesn't exist. The agent should try to get the instance details or update it directly, fail to find it, and inform the user. The user will then ask to list instances to find the correct name.", + "expected_trajectory": [ + "list_instances.js" + ], + "expected_skills": [ + "cloud-sql-postgres-admin" + ], + "env": { + "GOOGLE_CLOUD_PROJECT": "astana-evaluation" + }, + "kind": "tools", + "max_turns": 4 + } + ] +} diff --git a/datasets/claude-code-tools/claude-code.evalset.json b/datasets/claude-code-tools/claude-code.evalset.json index 29b79df0..9ce302ce 100644 --- a/datasets/claude-code-tools/claude-code.evalset.json +++ b/datasets/claude-code-tools/claude-code.evalset.json @@ -5,8 +5,8 @@ "starting_prompt": "list all Cloud SQL instances in project astana-evaluation", "conversation_plan": "Ask the agent to list instances in project astana-evaluation. Once all instances are listed if nl2code exist get its state and validate its RUNNABLE", "expected_trajectory": [ - "list_instances", - "get_instance" + "cloud-sql__list_instances", + "cloud-sql__get_instance" ], "env": { "GOOGLE_CLOUD_PROJECT": "astana-evaluation" @@ -19,9 +19,9 @@ "starting_prompt": "I need a database.", "conversation_plan": "The user starts with a vague request. You want to CREATE a NEW Cloud SQL instance named 'my-pg-app'. If the agent offers to create one, say YES. When asked for details, provide 'my-pg-app' as the instance name and 'user_data' as the database name. Never claim to have an existing instance. The goal is for the agent to eventually create the database 'user_data' inside 'my-pg-app' in astana-evaluation project.", "expected_trajectory": [ - "list_instances", - "create_instance", - "create_database" + "cloud-sql__list_instances", + "cloud-sql__create_instance", + "cloud-sql__create_database" ], "env": { "GOOGLE_CLOUD_PROJECT": "astana-evaluation" @@ -34,8 +34,8 @@ "starting_prompt": "Update the instance 'non-existent-db-123' to have 8 cores.", "conversation_plan": "The user asks to interact with an instance named 'non-existent-db-123' in astana-evaluation project that doesn't exist. The agent should try to get the instance details or update it directly, fail to find it, and inform the user. The user will then ask to list instances to find the correct name.", "expected_trajectory": [ - "update_instance", - "list_instances" + "cloud-sql__update_instance", + "cloud-sql__list_instances" ], "env": { "GOOGLE_CLOUD_PROJECT": "astana-evaluation" diff --git a/datasets/claude-code-tools/example_run_config.yaml b/datasets/claude-code-tools/example_run_config.yaml index 46333403..a15de685 100644 --- a/datasets/claude-code-tools/example_run_config.yaml +++ b/datasets/claude-code-tools/example_run_config.yaml @@ -20,7 +20,13 @@ runners: ### Scorer Related Configs ############################################################ scorers: + # By default trajectory_matcher drops native/harness-internal tools + # (Read, Bash, Edit, ToolSearch, ...) from both expected and actual + # trajectories before scoring. Uncomment the block below to keep them + # and score native-tool usage too. trajectory_matcher: {} + # trajectory_matcher: + # filter_native_tools: false goal_completion: model_config: datasets/model_configs/gemini_2.5_pro_model.yaml behavioral_metrics: @@ -28,9 +34,12 @@ scorers: parameter_analysis: model_config: datasets/model_configs/gemini_2.5_pro_model.yaml turn_count: {} + agent_steps: {} end_to_end_latency: {} tool_call_latency: {} token_consumption: {} + tokens_processed: {} + effective_billed_tokens: {} ############################################################ ### Reporting Related Configs diff --git a/datasets/claude-code-tools/example_run_fake_config.yaml b/datasets/claude-code-tools/example_run_fake_config.yaml index f4f15ef9..7bcdf73a 100644 --- a/datasets/claude-code-tools/example_run_fake_config.yaml +++ b/datasets/claude-code-tools/example_run_fake_config.yaml @@ -24,6 +24,8 @@ scorers: end_to_end_latency: {} tool_call_latency: {} token_consumption: {} + tokens_processed: {} + effective_billed_tokens: {} ############################################################ ### Reporting Related Configs diff --git a/datasets/claude-code-tools/example_run_mcp_repo_config.yaml b/datasets/claude-code-tools/example_run_mcp_repo_config.yaml new file mode 100644 index 00000000..12cd387e --- /dev/null +++ b/datasets/claude-code-tools/example_run_mcp_repo_config.yaml @@ -0,0 +1,39 @@ +############################################################ +### Dataset / Eval Items +############################################################ +dataset_config: datasets/claude-code-tools/claude-code-skills.evalset.json +dataset_format: agent-format + +# Orchestrator Configuration +orchestrator: agent +model_config: datasets/model_configs/claude_code_mcp_repo_model.yaml +simulated_user_model_config: datasets/model_configs/gemini_2.5_pro_model.yaml + +runners: + agent_runners: 1 + +############################################################ +### Scorer Related Configs +############################################################ +scorers: + skills_trajectory: + enforce_order: false + skills_best_practices: + model_config: datasets/model_configs/gemini_2.5_pro_model.yaml + goal_completion: + model_config: datasets/model_configs/gemini_2.5_pro_model.yaml + behavioral_metrics: + model_config: datasets/model_configs/gemini_2.5_pro_model.yaml + turn_count: {} + end_to_end_latency: {} + tool_call_latency: {} + token_consumption: {} + tokens_processed: {} + effective_billed_tokens: {} + +############################################################ +### Reporting Related Configs +############################################################ +reporting: + csv: + output_directory: 'results' diff --git a/datasets/claude-code-tools/example_run_skills_config.yaml b/datasets/claude-code-tools/example_run_skills_config.yaml new file mode 100644 index 00000000..18dea355 --- /dev/null +++ b/datasets/claude-code-tools/example_run_skills_config.yaml @@ -0,0 +1,39 @@ +############################################################ +### Dataset / Eval Items +############################################################ +dataset_config: datasets/claude-code-tools/claude-code-skills.evalset.json +dataset_format: agent-format + +# Orchestrator Configuration +orchestrator: agent +model_config: datasets/model_configs/claude_code_skills_model.yaml +simulated_user_model_config: datasets/model_configs/gemini_2.5_pro_model.yaml + +runners: + agent_runners: 1 + +############################################################ +### Scorer Related Configs +############################################################ +scorers: + skills_trajectory: + enforce_order: false + skills_best_practices: + model_config: datasets/model_configs/gemini_2.5_pro_model.yaml + goal_completion: + model_config: datasets/model_configs/gemini_2.5_pro_model.yaml + behavioral_metrics: + model_config: datasets/model_configs/gemini_2.5_pro_model.yaml + turn_count: {} + end_to_end_latency: {} + tool_call_latency: {} + token_consumption: {} + tokens_processed: {} + effective_billed_tokens: {} + +############################################################ +### Reporting Related Configs +############################################################ +reporting: + csv: + output_directory: 'results' diff --git a/datasets/codex-cli-tools/codex-cli-fake.evalset.json b/datasets/codex-cli-tools/codex-cli-fake.evalset.json new file mode 100644 index 00000000..f02e8424 --- /dev/null +++ b/datasets/codex-cli-tools/codex-cli-fake.evalset.json @@ -0,0 +1,30 @@ +{ + "scenarios": [ + { + "id": "fake-csql-create-instance-success", + "starting_prompt": "Create a new Cloud SQL instance named 'my-fake-db' in project 'astana-evaluation'. Use PostgreSQL 17, and set the password to 'password123'. Also use the 'Development' edition preset.", + "conversation_plan": "The user wants to create a database. All required parameters are in the starting prompt. The agent should call create_instance and report the success message back.", + "expected_trajectory": [ + "cloud-sql__create_instance" + ], + "env": { + "GOOGLE_CLOUD_PROJECT": "astana-evaluation" + }, + "kind": "tools", + "max_turns": 3 + }, + { + "id": "fake-csql-get-instance-failure", + "starting_prompt": "Get the details for the Cloud SQL instance named 'missing-db' in project 'astana-evaluation'.", + "conversation_plan": "The user wants to get instance details. The agent should call get_instance, which is hardcoded to fail with an error 'Instance not found or permission denied'. The agent should explain that the instance could not be found based on the error.", + "expected_trajectory": [ + "cloud-sql__get_instance" + ], + "env": { + "GOOGLE_CLOUD_PROJECT": "astana-evaluation" + }, + "kind": "tools", + "max_turns": 3 + } + ] +} diff --git a/datasets/codex-cli-tools/codex-cli.evalset.json b/datasets/codex-cli-tools/codex-cli.evalset.json new file mode 100644 index 00000000..9ce302ce --- /dev/null +++ b/datasets/codex-cli-tools/codex-cli.evalset.json @@ -0,0 +1,47 @@ +{ + "scenarios": [ + { + "id": "cloud-sql-list-instances-01", + "starting_prompt": "list all Cloud SQL instances in project astana-evaluation", + "conversation_plan": "Ask the agent to list instances in project astana-evaluation. Once all instances are listed if nl2code exist get its state and validate its RUNNABLE", + "expected_trajectory": [ + "cloud-sql__list_instances", + "cloud-sql__get_instance" + ], + "env": { + "GOOGLE_CLOUD_PROJECT": "astana-evaluation" + }, + "kind": "tools", + "max_turns": 3 + }, + { + "id": "csql-create-ambiguous-multiturn-01", + "starting_prompt": "I need a database.", + "conversation_plan": "The user starts with a vague request. You want to CREATE a NEW Cloud SQL instance named 'my-pg-app'. If the agent offers to create one, say YES. When asked for details, provide 'my-pg-app' as the instance name and 'user_data' as the database name. Never claim to have an existing instance. The goal is for the agent to eventually create the database 'user_data' inside 'my-pg-app' in astana-evaluation project.", + "expected_trajectory": [ + "cloud-sql__list_instances", + "cloud-sql__create_instance", + "cloud-sql__create_database" + ], + "env": { + "GOOGLE_CLOUD_PROJECT": "astana-evaluation" + }, + "kind": "tools", + "max_turns": 6 + }, + { + "id": "csql-instance-not-found-failure", + "starting_prompt": "Update the instance 'non-existent-db-123' to have 8 cores.", + "conversation_plan": "The user asks to interact with an instance named 'non-existent-db-123' in astana-evaluation project that doesn't exist. The agent should try to get the instance details or update it directly, fail to find it, and inform the user. The user will then ask to list instances to find the correct name.", + "expected_trajectory": [ + "cloud-sql__update_instance", + "cloud-sql__list_instances" + ], + "env": { + "GOOGLE_CLOUD_PROJECT": "astana-evaluation" + }, + "kind": "tools", + "max_turns": 4 + } + ] +} diff --git a/datasets/codex-cli-tools/example_run_config.yaml b/datasets/codex-cli-tools/example_run_config.yaml new file mode 100644 index 00000000..ac9c1bb1 --- /dev/null +++ b/datasets/codex-cli-tools/example_run_config.yaml @@ -0,0 +1,44 @@ +############################################################ +### Dataset / Eval Items +############################################################ +dataset_config: datasets/codex-cli-tools/codex-cli.evalset.json +dataset_format: agent-format + +orchestrator: agent +model_config: datasets/model_configs/codex_cli_model.yaml +simulated_user_model_config: datasets/model_configs/gemini_2.5_pro_model.yaml + +# Concurrency: number of scenarios to run in parallel. +# Set to 1 for sequential runs (easier to follow logs, avoids session conflicts +# on the shared sandboxed ~/.codex store). +runners: + agent_runners: 1 + +############################################################ +### Scorer Related Configs +############################################################ +scorers: + # By default trajectory_matcher drops native/harness-internal tools + # (shell, file ops, ...) from both expected and actual trajectories + # before scoring. Uncomment the block below to keep them and score + # native-tool usage too. + trajectory_matcher: {} + # trajectory_matcher: + # filter_native_tools: false + goal_completion: + model_config: datasets/model_configs/gemini_2.5_pro_model.yaml + behavioral_metrics: + model_config: datasets/model_configs/gemini_2.5_pro_model.yaml + parameter_analysis: + model_config: datasets/model_configs/gemini_2.5_pro_model.yaml + turn_count: {} + end_to_end_latency: {} + tool_call_latency: {} + token_consumption: {} + +############################################################ +### Reporting Related Configs +############################################################ +reporting: + csv: + output_directory: 'results' diff --git a/datasets/codex-cli-tools/example_run_fake_config.yaml b/datasets/codex-cli-tools/example_run_fake_config.yaml new file mode 100644 index 00000000..2d96cc2e --- /dev/null +++ b/datasets/codex-cli-tools/example_run_fake_config.yaml @@ -0,0 +1,33 @@ +############################################################ +### Dataset / Eval Items +############################################################ +dataset_config: datasets/codex-cli-tools/codex-cli-fake.evalset.json +dataset_format: agent-format + +# Orchestrator Configuration +orchestrator: agent +model_config: datasets/model_configs/codex_cli_fake_model.yaml +simulated_user_model_config: datasets/model_configs/gemini_2.5_pro_model.yaml + +############################################################ +### Scorer Related Configs +############################################################ +scorers: + trajectory_matcher: {} + goal_completion: + model_config: datasets/model_configs/gemini_2.5_pro_model.yaml + behavioral_metrics: + model_config: datasets/model_configs/gemini_2.5_pro_model.yaml + parameter_analysis: + model_config: datasets/model_configs/gemini_2.5_pro_model.yaml + turn_count: {} + end_to_end_latency: {} + tool_call_latency: {} + token_consumption: {} + +############################################################ +### Reporting Related Configs +############################################################ +reporting: + csv: + output_directory: 'results' diff --git a/datasets/codex-skills/codex-skills.evalset.json b/datasets/codex-skills/codex-skills.evalset.json new file mode 100644 index 00000000..91501e29 --- /dev/null +++ b/datasets/codex-skills/codex-skills.evalset.json @@ -0,0 +1,55 @@ +{ + "scenarios": [ + { + "id": "codex-cloud-sql-list-instances-01", + "starting_prompt": "list all Cloud SQL instances in project astana-evaluation", + "conversation_plan": "Ask the agent to list instances in project astana-evaluation. Once all instances are listed if nl2code exists get its state and validate it is RUNNABLE.", + "expected_trajectory": [ + "list_instances.js", + "get_instance.js" + ], + "expected_skills": [ + "cloud-sql-postgres-admin" + ], + "env": { + "GOOGLE_CLOUD_PROJECT": "astana-evaluation" + }, + "kind": "tools", + "max_turns": 3 + }, + { + "id": "codex-csql-create-ambiguous-multiturn-01", + "starting_prompt": "I need a database.", + "conversation_plan": "The user starts with a vague request. You want to CREATE a NEW Cloud SQL instance named 'my-pg-app'. If the agent offers to create one, say YES. When asked for details, provide 'my-pg-app' as the instance name and 'user_data' as the database name. Never claim to have an existing instance. The goal is for the agent to eventually create the database 'user_data' inside 'my-pg-app' in astana-evaluation project.", + "expected_trajectory": [ + "list_instances.js", + "create_instance.js", + "create_database.js" + ], + "expected_skills": [ + "cloud-sql-postgres-admin" + ], + "env": { + "GOOGLE_CLOUD_PROJECT": "astana-evaluation" + }, + "kind": "tools", + "max_turns": 6 + }, + { + "id": "codex-csql-instance-not-found-failure", + "starting_prompt": "Update the instance 'non-existent-db-123' to have 8 cores.", + "conversation_plan": "The user asks to interact with an instance named 'non-existent-db-123' in astana-evaluation project that doesn't exist. The agent should try to get the instance details or update it directly, fail to find it, and inform the user. The user will then ask to list instances to find the correct name.", + "expected_trajectory": [ + "list_instances.js" + ], + "expected_skills": [ + "cloud-sql-postgres-admin" + ], + "env": { + "GOOGLE_CLOUD_PROJECT": "astana-evaluation" + }, + "kind": "tools", + "max_turns": 4 + } + ] +} \ No newline at end of file diff --git a/datasets/codex-skills/codex_skills_model.yaml b/datasets/codex-skills/codex_skills_model.yaml new file mode 100644 index 00000000..b003b88a --- /dev/null +++ b/datasets/codex-skills/codex_skills_model.yaml @@ -0,0 +1,22 @@ +codex_cli_version: "@openai/codex@latest" +generator: codex_cli +model: "gpt-5.5" +openai_api_key_secret: "projects/393137573/secrets/OPENAI_API_KEY/versions/1" + +env: + GOOGLE_CLOUD_PROJECT: "astana-evaluation" + GOOGLE_CLOUD_LOCATION: "us-central1" + # Cloud SQL PostgreSQL skills configuration. + # Substitute the placeholders below with real values per scenario before running. + CLOUD_SQL_POSTGRES_PROJECT: "astana-evaluation" + CLOUD_SQL_POSTGRES_REGION: "us-central1" + CLOUD_SQL_POSTGRES_INSTANCE: + CLOUD_SQL_POSTGRES_DATABASE: + CLOUD_SQL_POSTGRES_USER: + CLOUD_SQL_POSTGRES_PASSWORD: + CLOUD_SQL_POSTGRES_IP_TYPE: "PUBLIC" + +setup: + skills: + - action: install_from_repo + url: "https://github.com/gemini-cli-extensions/cloud-sql-postgresql.git" diff --git a/datasets/codex-skills/codex_skills_run_config.yaml b/datasets/codex-skills/codex_skills_run_config.yaml new file mode 100644 index 00000000..89c3e589 --- /dev/null +++ b/datasets/codex-skills/codex_skills_run_config.yaml @@ -0,0 +1,23 @@ +dataset_config: datasets/codex-skills/codex-skills.evalset.json +dataset_format: agent-format + +orchestrator: agent +model_config: datasets/codex-skills/codex_skills_model.yaml +simulated_user_model_config: datasets/model_configs/gemini_3.1_pro_model.yaml + +runners: + agent_runners: 1 + +scorers: + skills_trajectory: + enforce_order: false + skills_best_practices: + model_config: datasets/model_configs/gemini_3.1_pro_model.yaml + goal_completion: + model_config: datasets/model_configs/gemini_3.1_pro_model.yaml + behavioral_metrics: + model_config: datasets/model_configs/gemini_3.1_pro_model.yaml + turn_count: {} + end_to_end_latency: {} + tool_call_latency: {} + token_consumption: {} diff --git a/datasets/db-engine-convertor/README.md b/datasets/db-engine-convertor/README.md index 28d23a3e..b82a34e4 100644 --- a/datasets/db-engine-convertor/README.md +++ b/datasets/db-engine-convertor/README.md @@ -35,11 +35,12 @@ This toolkit uses Google Gemini AI to automatically generate database-specific s ### Installation -```bash -# Install dependencies -cd db-engine-convertor -pip install -r requirements.txt +EvalBench uses `uv` for dependency management. It is recommended to set up the environment from the project root. +```bash +# From the project root +uv sync +``` # Set up environment variables cp .env.example .env # Edit .env with your database credentials and API keys diff --git a/datasets/dea-tools/README.md b/datasets/dea-tools/README.md new file mode 100644 index 00000000..7c54933f --- /dev/null +++ b/datasets/dea-tools/README.md @@ -0,0 +1,120 @@ +# How to run Evalbench for Data Engineering Agent (DEA) + +This directory contains a sample configuration for evaluating the Data Engineering Agent (DEA) in a 100% programmatic, CLI-free, and SQL-free stateful multi-turn conversation. + +## 1. Bootstrapping a Fresh Repository Clone + +Run these exact commands from the repository root: +1. **Initialize the Virtual Environment**: Run `uv sync` to build the `.venv` directory and locally install all required pip packages. +2. **Compile Protobufs**: Run `make proto` (or explicitly run `.venv/bin/python3 -m grpc_tools.protoc --proto_path=evalbench/evalproto --python_out=evalbench/evalproto --pyi_out=evalbench/evalproto --grpc_python_out=evalbench/evalproto evalbench/evalproto/*.proto`) to generate the required Python `.pb2` module bindings. +3. **Execute using venv**: Always prepend your shell run commands with `.venv/bin/python3 ` instead of calling python files directly to avoid invoking the system interpreter. + +## 2. Supply Your Evaluation Dataset +The dataset file is defined in `datasets/dea-tools/dea-live-conversational.evalset.json`. It defines conversational turns (such as reading table schemas and modifying schemas) along with evaluation metrics and rubrics. + +### Dataset JSON Structure: +Key fields defined in the dataset include: +* `scenarios`: An array of conversational steps. In multi-turn/follow-up flows, scenarios run sequentially, sharing the same agent session. +* `starting_prompt`: The initial message sent to the agent at the beginning of the scenario. +* `max_turns`: The maximum number of conversational turns allowed between the Simulated User and the Agent (e.g., set to 2 for multi-turn validation). +* `conversation_plan`: A plan (like *"Approve any plans generated by the agent"*) that guides the simulated user's responses, enabling natural, goal-directed interactions instead of rigid scripted responses. +* `binary_rubric`: The grading rules used by the LLM-Judge to grade the scenario (gives 0 or 100). + +## 3. Supply Your Run Configuration +The run configuration file is defined in `datasets/dea-tools/example_run_config.yaml`. It configures the evaluation orchestrator, dataset format, model configurations, setup/teardown scripts, and reporting/database settings. + +### Run Configuration Key Fields: +This file is the main driver that dictates how evaluations are executed. Key parameters include: +* `orchestrator` & `dataset_format`: Configured to `dea` and `dea-format` for Data Engineering Agent evaluations. +* `model_config`: Points to the GCP DEA agent under test. +* `simulated_user_model_config`: Points to the model acting as the user (Gemini 2.5 Pro). +* `dataset_config`: Points to the Dataset JSON file (described in Step 2). +* `env` block: Sets local variables like `EVAL_DATAFORM_SETUP_ENV_FILES_DIR` to inject existing files (e.g., workspace settings, schema definitions) into the workspace before the agent runs. This allows the agent to build on top of an established environment instead of starting from scratch. +* `scorers`: Defines the evaluation scorers: + * `binary_rubric_scorer`: Grader that evaluates the conversation against the `binary_rubric` (gives 0 or 100). + * `dataform_cloud_compile` & `dataform_cloud_run`: Cloud verifiers that compile and run the generated SQLX files in GCP to ensure they are physically valid. +* `reporting`: Outputs results locally as CSVs, and pushes them to your BigQuery table. +* `set_up_script` & `tear_down_script`: Scripts that prep and clean up the Dataform environment. You can comment out `tear_down_script` if you want to keep the workspace alive after the run for debugging. +* `dataform_workspace_gcs_archive`: Saves a ZIP archive of the workspace artifacts to a GCS bucket. + +### Parameterized Agent & Environment Configuration: +The model configuration file is defined in `datasets/model_configs/gcp_data_engineering_agent_model.yaml`. In your model configuration YAML file, you can configure target environments and agent personas: + +```yaml +generator: "data_engineering_agent" +gcp_project_id: !ENV ${EVAL_GCP_PROJECT_ID} +gcp_region: !ENV ${EVAL_GCP_PROJECT_REGION} + +# Parameterized Agent Options: +model_env: local # Target environment: "local", "staging", "autopush", or "prod" (defaults to "prod") +port: 9876 # Required port for local Boq servers when model_env="local" +agent_type: sparkagent # Desired agent persona (e.g., "sparkagent", "dataengineeringagent") + +``` + +* **`model_env`**: Configures the host environment (`local` uses `http://localhost:{port}`, `staging` uses `https://staging-geminidataanalytics.sandbox.googleapis.com`, `autopush` uses `https://autopush-geminidataanalytics.sandbox.googleapis.com`, `prod` uses `https://geminidataanalytics.googleapis.com`). + +* **`port`**: Specifies the HTTP port for local Boq servers (required when `model_env="local"`). +* **`agent_type`**: Sets the desired agent persona (e.g., "sparkagent", "dataengineeringagent"). + +## 4. Run EvalBench +You can run EvalBench for DEA in two modes: + +### Mode 1: Dynamic Sandbox +EvalBench automatically creates a temporary Dataform repository and workspace for the evaluation and tears them down afterwards. This ensures a clean slate for every run. + +To use this mode, ensure `set_up_script` and `tear_down_script` are enabled in your run config (default behavior in `example_run_config.yaml`). + +```bash +EVAL_GCP_PROJECT_ID= \ +EVAL_GCP_PROJECT_REGION= \ +.venv/bin/python3 evalbench/evalbench.py --experiment_config=datasets/dea-tools/example_run_config.yaml +``` + +### Mode 2: Static Workspace +EvalBench uses a specific, pre-existing Dataform repository and workspace that you manage. + +To use this mode: +1. In `example_run_config.yaml`, comment out `set_up_script` and `tear_down_script`. +2. Uncomment `dataform_repository` and `dataform_workspace`. +3. Provide the repository and workspace IDs in the environment: + +```bash +EVAL_GCP_PROJECT_ID= \ +EVAL_GCP_PROJECT_REGION= \ +EVAL_DEA_REPOSITORY_ID= \ +EVAL_DEA_WORKSPACE_ID= \ +.venv/bin/python3 evalbench/evalbench.py --experiment_config=datasets/dea-tools/example_run_config.yaml +``` + +> [!NOTE] +> If both the setup/teardown scripts and the static repository/workspace coordinates are configured in the YAML run config, the dynamic sandbox logic will override the static configuration. + +### Automated Suite Run +To run all 10 core evaluation cases sequentially with strict environment isolation, you can execute them as a **test suite**. EvalBench will run the setup and teardown scripts for each individual test case, creating a fresh, independent Dataform repository for every run. + +To run the suite, use the `--suite_config` flag: + +```bash +EVAL_GCP_PROJECT_ID= \ +EVAL_GCP_PROJECT_REGION= \ +.venv/bin/python3 evalbench/evalbench.py --suite_config=datasets/dea-tools/core_10_cases_suite.yaml +``` + +### Key Environment Variables: +* `EVAL_GCP_PROJECT_ID`: The GCP Project ID where your DEA agent is deployed. +* `EVAL_GCP_PROJECT_REGION`: The GCP Region (e.g., `us-west4`) of the agent. +* `EVAL_DEA_REPOSITORY_ID`: The target Dataform repository ID (Required for Mode 2, not needed for Mode 1). +* `EVAL_DEA_WORKSPACE_ID`: The target Dataform workspace ID (Required for Mode 2, not needed for Mode 1). + +## 5. Inspect Results +Upon completion, results will be generated in two locations: + +### Local Files (under the `results/` folder): +* `evals.csv`: Contains the full conversation history. +* `scores.csv`: Contains LLM-Judge scores and detailed reasoning for the rubric checks. + +### Google BigQuery (Cloud Database): +If enabled in `example_run_config.yaml`, results are automatically uploaded to your GCP project under the table `.evalbench.results`. + +A clickable **Looker Studio Dashboard** link will be printed in the terminal console upon completion to visually inspect the conversation flows and scores. diff --git a/datasets/dea-tools/dea-live-conversational.evalset.json b/datasets/dea-tools/dea-live-conversational.evalset.json new file mode 100644 index 00000000..21192e8a --- /dev/null +++ b/datasets/dea-tools/dea-live-conversational.evalset.json @@ -0,0 +1,34 @@ +{ + "scenarios": [ + { + "id": "dea-programmatic-scenario-0", + "starting_prompt": "Please read the schema of the BigQuery table bigquery-public-data.ml_datasets.penguins. Don't be chatty, just go straight to the answer.", + "max_turns": 1, + "conversation_plan": [], + "binary_rubric": [ + "Verify that the agent listed the table columns species, island, culmen_length_mm, culmen_depth_mm, flipper_length_mm, body_mass_g, sex." + ] + }, + { + "id": "dea-programmatic-scenario-1", + "starting_prompt": "Show the table schema columns again, but reversed in a pure conversational text/markdown table format. Do NOT generate or offer any SQL views or queries, but strictly show the schema table in text. Don't be chatty, go straight to the answer.", + "max_turns": 1, + "conversation_plan": [], + "binary_rubric": [ + "Verify that the agent successfully reversed the columns order in a conversational text/table format, showing sex first and species last." + ] + }, + { + "id": "dea-programmatic-scenario-2", + "starting_prompt": "In my Dataform workspace, create a new SQLX view called `simple_constant_view` that selects a constant value (1 as value). Put it in `definitions/simple_constant_view.sqlx` and write it to the workspace. Overwrite `workflow_settings.yaml` to set `defaultLocation` to 'US' (not us-central1). Go straight to the answer.", + "max_turns": 2, + "conversation_plan": [ + "Approve the plan the agent generates.", + "Verify that the agent confirms it has written the file simple_constant_view.sqlx." + ], + "binary_rubric": [ + "Verify that the agent's response indicates it has created the file successfully in the workspace." + ] + } + ] +} diff --git a/datasets/dea-tools/example_run_config.yaml b/datasets/dea-tools/example_run_config.yaml new file mode 100644 index 00000000..9118e86e --- /dev/null +++ b/datasets/dea-tools/example_run_config.yaml @@ -0,0 +1,33 @@ +orchestrator: dea +dataset_format: dea-format +model_config: datasets/model_configs/gcp_data_engineering_agent_model.yaml +simulated_user_model_config: datasets/model_configs/gemini_2.5_pro_model.yaml +dataset_config: datasets/dea-tools/dea-live-conversational.evalset.json +env: + SCENARIO_ID: "dea-live-conversational" + EVAL_DATAFORM_SETUP_ENV_FILES_DIR: "datasets/dea-tools/templates/default_workspace" +scorers: + binary_rubric_scorer: + model_config: datasets/model_configs/gemini_2.5_pro_model.yaml + dataform_cloud_compile: + gcp_project_id: !ENV ${EVAL_GCP_PROJECT_ID} + gcp_region: !ENV ${EVAL_GCP_PROJECT_REGION} + timeout_seconds: 300 + dataform_cloud_run: + gcp_project_id: !ENV ${EVAL_GCP_PROJECT_ID} + gcp_region: !ENV ${EVAL_GCP_PROJECT_REGION} + timeout_seconds: 300 +reporting: + csv: + output_directory: results + bigquery: + gcp_project_id: !ENV ${EVAL_GCP_PROJECT_ID} +runners: + agent_runners: 1 +set_up_script: datasets/dea-tools/scripts/setup_dataform.sh +tear_down_script: datasets/dea-tools/scripts/teardown_dataform.sh +dataform_workspace_gcs_archive: + bucket: !ENV ${EVAL_GCP_PROJECT_ID}-evalbench-archives + path_prefix: workspaces + gcp_project_id: !ENV ${EVAL_GCP_PROJECT_ID} + gcp_region: !ENV ${EVAL_GCP_PROJECT_REGION} diff --git a/datasets/dea-tools/scripts/setup_dataform.sh b/datasets/dea-tools/scripts/setup_dataform.sh new file mode 100755 index 00000000..7d3fc49b --- /dev/null +++ b/datasets/dea-tools/scripts/setup_dataform.sh @@ -0,0 +1,27 @@ +#!/bin/bash +set -e + +SESSION_DIR="$1" +if [ -z "$SESSION_DIR" ]; then + echo "Error: SESSION_DIR not provided to setup script." + exit 1 +fi + +PROJECT_ID="${EVAL_GCP_PROJECT_ID}" +REGION="${EVAL_GCP_PROJECT_REGION}" + +JOB_ID=$(basename "$SESSION_DIR") + +SCENARIO_ID="${SCENARIO_ID:-default}" +ENV_FILES_DIR="${EVAL_DATAFORM_SETUP_ENV_FILES_DIR}" + +WORKSPACE_URI=$(PYTHONPATH=evalbench .venv/bin/python3 -c " +from util.dataform_workspace import DataformWorkspaceManager +manager = DataformWorkspaceManager('$PROJECT_ID', '$REGION') +uri = manager.setup_workspace('$JOB_ID', '$SCENARIO_ID', env_files_dir='$ENV_FILES_DIR') +print(uri) +") + +mkdir -p "$SESSION_DIR" +echo "$WORKSPACE_URI" > "$SESSION_DIR/target_workspace.txt" +echo "Setup complete. Workspace URI saved to $SESSION_DIR/target_workspace.txt" diff --git a/datasets/dea-tools/scripts/teardown_dataform.sh b/datasets/dea-tools/scripts/teardown_dataform.sh new file mode 100755 index 00000000..4f9d00bb --- /dev/null +++ b/datasets/dea-tools/scripts/teardown_dataform.sh @@ -0,0 +1,27 @@ +#!/bin/bash +set -e + +SESSION_DIR="$1" +if [ -z "$SESSION_DIR" ]; then + echo "Error: SESSION_DIR not provided to teardown script." + exit 1 +fi + +STATE_FILE="$SESSION_DIR/target_workspace.txt" +if [ ! -f "$STATE_FILE" ]; then + echo "State file $STATE_FILE not found. Skipping teardown." + exit 0 +fi + +WORKSPACE_URI=$(cat "$STATE_FILE") +PROJECT_ID="${EVAL_GCP_PROJECT_ID}" +REGION="${EVAL_GCP_PROJECT_REGION}" + +PYTHONPATH=evalbench .venv/bin/python3 -c " +from util.dataform_workspace import DataformWorkspaceManager +manager = DataformWorkspaceManager('$PROJECT_ID', '$REGION') +manager.teardown_workspace('$WORKSPACE_URI') +" + +rm -f "$STATE_FILE" +echo "Teardown complete. Deleted cloud resources for $WORKSPACE_URI" diff --git a/datasets/dea-tools/templates/default_workspace/example.txt b/datasets/dea-tools/templates/default_workspace/example.txt new file mode 100644 index 00000000..49448cb9 --- /dev/null +++ b/datasets/dea-tools/templates/default_workspace/example.txt @@ -0,0 +1 @@ +This is an example file injected by EvalBench. \ No newline at end of file diff --git a/datasets/gemini-cli-tools/bigtable/gemini-cli-bigtable.evalset.json b/datasets/gemini-cli-tools/bigtable/gemini-cli-bigtable.evalset.json index 9fa8b67d..9d3ccf81 100644 --- a/datasets/gemini-cli-tools/bigtable/gemini-cli-bigtable.evalset.json +++ b/datasets/gemini-cli-tools/bigtable/gemini-cli-bigtable.evalset.json @@ -5,10 +5,10 @@ "starting_prompt": "I need to initialize my Bigtable environment for evaluation if they don't exist already. Please create the following instances in project cloud-bigtable-spacewalk: 'evalbench-prod-data', 'evalbench-legacy-archive', 'evalbench-staging-db', 'evalbench-customers-v2', 'evalbench-dev-db', and 'evalbench-test-instance-123'. All clusters should be in us-central1-a. Also, in 'evalbench-legacy-archive' create a table 'evalbench-v1-data' and in 'evalbench-staging-db' create a table 'evalbench-temporary-data' if they don't exist already.", "conversation_plan": "The user is seeding the environment. The agent should sequentially or in parallel create all requested instances and tables. Confirm when everything is ready.", "expected_trajectory": [ - "mcp_Bigtable_list_instances", - "mcp_Bigtable_create_instance", - "mcp_Bigtable_list_tables", - "mcp_Bigtable_create_table" + "bigtable__list_instances", + "bigtable__create_instance", + "bigtable__list_tables", + "bigtable__create_table" ], "env": { "GOOGLE_CLOUD_PROJECT": "cloud-bigtable-spacewalk" @@ -21,8 +21,8 @@ "starting_prompt": "What Bigtable instances are running in project cloud-bigtable-spacewalk?", "conversation_plan": "The user starts with a discovery question. The agent should list instances. The user then asks for the configuration details of the 'evalbench-prod-data' instance to check its clusters.", "expected_trajectory": [ - "mcp_Bigtable_list_instances", - "mcp_Bigtable_get_instance" + "bigtable__list_instances", + "bigtable__get_instance" ], "env": { "GOOGLE_CLOUD_PROJECT": "cloud-bigtable-spacewalk" @@ -35,9 +35,9 @@ "starting_prompt": "Set up a new Bigtable instance 'evalbench-metrics-db' in us-east1-c and add a 'evalbench-raw-metrics' table with a 'data' column family. If the instance creation is pending, please wait or check until it's ready before creating the table.", "conversation_plan": "The agent must create the instance. If the table creation fails because the instance isn't ready, the agent should poll 'get_instance' or wait and try again. The user expects both to be completed.", "expected_trajectory": [ - "mcp_Bigtable_create_instance", - "mcp_Bigtable_get_instance", - "mcp_Bigtable_create_table" + "bigtable__create_instance", + "bigtable__get_instance", + "bigtable__create_table" ], "env": { "GOOGLE_CLOUD_PROJECT": "cloud-bigtable-spacewalk" @@ -50,7 +50,7 @@ "starting_prompt": "I need to ensure the Bigtable instance 'evalbench-prod-data' exists. If it's already there, just let me know its current state. If not, create it in us-central1-a.", "conversation_plan": "The agent should first check for the instance's existence using 'get_instance'. If it exists, it should report the state. If it returns a 404/not found, it should proceed to 'create_instance'.", "expected_trajectory": [ - "mcp_Bigtable_get_instance" + "bigtable__get_instance" ], "env": { "GOOGLE_CLOUD_PROJECT": "cloud-bigtable-spacewalk" @@ -63,8 +63,8 @@ "starting_prompt": "I need to find the 'evalbench-audit' table in my 'evalbench-prod-data'. Can you get its metadata?", "conversation_plan": "The user wants to find a specific table. The agent should list tables in the instance to verify existence and then get the table details.", "expected_trajectory": [ - "mcp_Bigtable_list_tables", - "mcp_Bigtable_get_table" + "bigtable__list_tables", + "bigtable__get_table" ], "env": { "GOOGLE_CLOUD_PROJECT": "cloud-bigtable-spacewalk" @@ -77,7 +77,7 @@ "starting_prompt": "Delete the 'evalbench-temporary-data' table in the 'evalbench-staging-db' instance.", "conversation_plan": "A direct administrative action. The agent must call delete_table. Since the tool requires confirmation, the agent should ask and the user will say 'yes'.", "expected_trajectory": [ - "mcp_Bigtable_delete_table" + "bigtable__delete_table" ], "env": { "GOOGLE_CLOUD_PROJECT": "cloud-bigtable-spacewalk" @@ -90,9 +90,9 @@ "starting_prompt": "Get details for the 'evalbench-customer-db' instance.", "conversation_plan": "The user refers to an instance by a name that doesn't exist. The agent's 'get_instance' call will fail. The agent should then list instances, and the user will say 'Oh, I meant evalbench-customers-v2, get that one'.", "expected_trajectory": [ - "mcp_Bigtable_get_instance", - "mcp_Bigtable_list_instances", - "mcp_Bigtable_get_instance" + "bigtable__get_instance", + "bigtable__list_instances", + "bigtable__get_instance" ], "env": { "GOOGLE_CLOUD_PROJECT": "cloud-bigtable-spacewalk" @@ -105,8 +105,8 @@ "starting_prompt": "Add a new table to my instance.", "conversation_plan": "A vague request. The agent should ask which instance. The user provides 'evalbench-dev-db'. The agent then asks for table name and families. The user provides name 'evalbench-app-logs' and family 'metadata'.", "expected_trajectory": [ - "mcp_Bigtable_list_instances", - "mcp_Bigtable_create_table" + "bigtable__list_instances", + "bigtable__create_table" ], "env": { "GOOGLE_CLOUD_PROJECT": "cloud-bigtable-spacewalk" @@ -119,7 +119,7 @@ "starting_prompt": "I no longer need the 'evalbench-test-instance-123'. Please delete it.", "conversation_plan": "The user wants to delete a specific instance. The agent must call delete_instance. The tool requires explicit user confirmation, so the agent should ask and the user will confirm with 'yes'.", "expected_trajectory": [ - "mcp_Bigtable_delete_instance" + "bigtable__delete_instance" ], "env": { "GOOGLE_CLOUD_PROJECT": "cloud-bigtable-spacewalk" @@ -132,7 +132,7 @@ "starting_prompt": "Evaluation is complete. Please delete all the Bigtable instances we used in project cloud-bigtable-spacewalk: 'evalbench-prod-data', 'evalbench-legacy-archive', 'evalbench-staging-db', 'evalbench-customers-v2', 'evalbench-dev-db', and 'evalbench-metrics-db'. Confirm each deletion when asked.", "conversation_plan": "The user is cleaning up. The agent should call delete_instance for each instance. The user will provide 'yes' for each confirmation request.", "expected_trajectory": [ - "mcp_Bigtable_delete_instance" + "bigtable__delete_instance" ], "env": { "GOOGLE_CLOUD_PROJECT": "cloud-bigtable-spacewalk" diff --git a/datasets/gemini-cli-tools/example_run_config.yaml b/datasets/gemini-cli-tools/example_run_config.yaml index 7fea123e..8ee43816 100644 --- a/datasets/gemini-cli-tools/example_run_config.yaml +++ b/datasets/gemini-cli-tools/example_run_config.yaml @@ -13,7 +13,13 @@ simulated_user_model_config: datasets/model_configs/gemini_2.5_pro_model.yaml ### Scorer Related Configs ############################################################ scorers: + # By default trajectory_matcher drops native/harness-internal tools + # (run_shell_command, write_file, update_topic, ...) from both expected + # and actual trajectories before scoring. Uncomment the block below to + # keep them and score native-tool usage too. trajectory_matcher: {} + # trajectory_matcher: + # filter_native_tools: false goal_completion: model_config: datasets/model_configs/gemini_2.5_pro_model.yaml behavioral_metrics: diff --git a/datasets/gemini-cli-tools/gemini-cli-fake.evalset.json b/datasets/gemini-cli-tools/gemini-cli-fake.evalset.json index 1d9f584b..f02e8424 100644 --- a/datasets/gemini-cli-tools/gemini-cli-fake.evalset.json +++ b/datasets/gemini-cli-tools/gemini-cli-fake.evalset.json @@ -5,7 +5,7 @@ "starting_prompt": "Create a new Cloud SQL instance named 'my-fake-db' in project 'astana-evaluation'. Use PostgreSQL 17, and set the password to 'password123'. Also use the 'Development' edition preset.", "conversation_plan": "The user wants to create a database. All required parameters are in the starting prompt. The agent should call create_instance and report the success message back.", "expected_trajectory": [ - "create_instance" + "cloud-sql__create_instance" ], "env": { "GOOGLE_CLOUD_PROJECT": "astana-evaluation" @@ -18,7 +18,7 @@ "starting_prompt": "Get the details for the Cloud SQL instance named 'missing-db' in project 'astana-evaluation'.", "conversation_plan": "The user wants to get instance details. The agent should call get_instance, which is hardcoded to fail with an error 'Instance not found or permission denied'. The agent should explain that the instance could not be found based on the error.", "expected_trajectory": [ - "get_instance" + "cloud-sql__get_instance" ], "env": { "GOOGLE_CLOUD_PROJECT": "astana-evaluation" diff --git a/datasets/gemini-cli-tools/gemini-cli.evalset.json b/datasets/gemini-cli-tools/gemini-cli.evalset.json index 050e4640..25ec2b14 100644 --- a/datasets/gemini-cli-tools/gemini-cli.evalset.json +++ b/datasets/gemini-cli-tools/gemini-cli.evalset.json @@ -5,8 +5,8 @@ "starting_prompt": "list all instances in project astana-evaluation", "conversation_plan": "Ask the agent to list instances in project astana-evaluation. Once all instances are listed if nl2code exist get its state and validate its RUNNABLE", "expected_trajectory": [ - "list_instances", - "get_instance" + "cloud-sql__list_instances", + "cloud-sql__get_instance" ], "env": { "GOOGLE_CLOUD_PROJECT": "astana-evaluation" @@ -19,9 +19,9 @@ "starting_prompt": "I need a database.", "conversation_plan": "The user starts with a vague request. You want to CREATE a NEW Cloud SQL instance named 'my-pg-app'. If the agent offers to create one, say YES. When asked for details, provide 'my-pg-app' as the instance name and 'user_data' as the database name. Never claim to have an existing instance. The goal is for the agent to eventually create the database 'user_data' inside 'my-pg-app' in astana-evaluation project.", "expected_trajectory": [ - "list_instances", - "create_instance", - "create_database" + "cloud-sql__list_instances", + "cloud-sql__create_instance", + "cloud-sql__create_database" ], "env": { "GOOGLE_CLOUD_PROJECT": "astana-evaluation", @@ -35,9 +35,9 @@ "starting_prompt": "I want to clone the 'production-db' instance.", "conversation_plan": "The user wants to create a clone of the instance 'nl2code'. If the agent asks for a new instance name, provide 'staging-nl2code'. If asked to confirm the project, say 'astana-evaluation'.", "expected_trajectory": [ - "get_instance", - "clone_instance", - "get_operation" + "cloud-sql__get_instance", + "cloud-sql__clone_instance", + "cloud-sql__get_operation" ], "env": { "GOOGLE_CLOUD_PROJECT": "astana-evaluation" @@ -50,9 +50,9 @@ "starting_prompt": "Scale up my database instance.", "conversation_plan": "The user wants to increase the resources for 'nl2code' in astana-evaluation project. When asked for specifics, state you want to change the tier/machine type to 'db-custom-4-15360' (4 vCPUs, 15GB RAM). Confirm the update when asked. Verify the operation status.", "expected_trajectory": [ - "get_instance", - "update_instance", - "get_operation" + "cloud-sql__get_instance", + "cloud-sql__update_instance", + "cloud-sql__get_operation" ], "env": { "GOOGLE_CLOUD_PROJECT": "astana-evaluation" @@ -65,8 +65,8 @@ "starting_prompt": "I need to rotate the password for a user on the 'magic' instance, but I forgot their exact username.", "conversation_plan": "First, the user asks to list all users on the 'magic' instance. The agent will show the users. The user then identifies 'webapp_service_account' and asks to update its password to 'NewStrictPassword99!'. Confirm the password update. The project name is astana-evaluation", "expected_trajectory": [ - "list_users", - "update_user" + "cloud-sql__list_users", + "cloud-sql__update_user" ], "env": { "GOOGLE_CLOUD_PROJECT": "astana-evaluation" @@ -79,8 +79,8 @@ "starting_prompt": "Update the instance 'non-existent-db-123' to have 8 cores.", "conversation_plan": "The user asks to interact with an instance named 'non-existent-db-123' in astana-evaluation project that doesn't exist. The agent should try to get the instance details or update it directly, fail to find it, and inform the user. The user will then ask to list instances to find the correct name.", "expected_trajectory": [ - "update_instance", - "list_instances" + "cloud-sql__update_instance", + "cloud-sql__list_instances" ], "env": { "GOOGLE_CLOUD_PROJECT": "astana-evaluation" @@ -89,4 +89,4 @@ "max_turns": 4 } ] -} \ No newline at end of file +} diff --git a/datasets/mcp_readability/endpoints.yaml b/datasets/mcp_readability/endpoints.yaml new file mode 100644 index 00000000..a0ff07b5 --- /dev/null +++ b/datasets/mcp_readability/endpoints.yaml @@ -0,0 +1,34 @@ +# Sample MCP endpoints. Each entry has: product_name, endpoint_type +# (PROD | AUTOPUSH | STAGING | DEV), and a pluggable tools_source. +# +# An endpoint's identity for reporting is (product_name, endpoint_type). The +# source is defined entirely by tools_source — there is no top-level endpoint_url. +# tools_source.type selects how tools are obtained (independent of endpoint_type, +# which is just the deployment channel / reporting metadata): +# http - live tools/list over Streamable HTTP; url is the endpoint address +# stdio - launch a local MCP server and speak MCP over its stdio pipes +# file - read a raw tools/list JSON dump (offline / deterministic) + +endpoints: + - product_name: "Cloud SQL" + endpoint_type: PROD + tools_source: + type: http + url: "https://sqladmin.googleapis.com/mcp" + + - product_name: "MCP Toolbox (BigQuery)" + endpoint_type: DEV + tools_source: + type: stdio + # MCP Toolbox for Databases, run over stdio via its npm wrapper. + command: "npx" + args: ["-y", "@toolbox-sdk/server", "--prebuilt=bigquery", "--stdio"] + # Needs BigQuery credentials in the environment (ADC + project), e.g.: + # env: + # BIGQUERY_PROJECT: "your-gcp-project" + + - product_name: "Sample (offline)" + endpoint_type: DEV + tools_source: + type: file + path: datasets/mcp_readability/sample_tools.json diff --git a/datasets/mcp_readability/exceptions.yaml b/datasets/mcp_readability/exceptions.yaml new file mode 100644 index 00000000..e40a6036 --- /dev/null +++ b/datasets/mcp_readability/exceptions.yaml @@ -0,0 +1,14 @@ +# Per-endpoint rule waivers (sample). A waived rule is excluded from the +# P0/P1/P2 counts and reported separately under "waived". Matchers (all present +# must match; absent field or "*" = match-all): +# product_name, endpoint_type. +# rule_id references a style-guide rule's {#tag} anchor. + +exceptions: + - product_name: "Cloud SQL" # waive one rule for one product + rule_id: "tool-names" + reason: "Legacy tool names retained for backward compatibility." + + - endpoint_type: AUTOPUSH # waive a rule across a whole channel + rule_id: "use-enums" + reason: "Autopush builds defer enum constraints until promotion to prod." diff --git a/datasets/mcp_readability/run_config.yaml b/datasets/mcp_readability/run_config.yaml new file mode 100644 index 00000000..b8200197 --- /dev/null +++ b/datasets/mcp_readability/run_config.yaml @@ -0,0 +1,52 @@ +############################################################ +### MCP Readability — sample run config +############################################################ +# Drives the `mcp_readability` orchestrator: for each endpoint, fetch its tools +# (rendered as a man-page), score them against the style guide with an LLM, and +# emit one result row per endpoint to the shared EvalBench reporters. +# +# NOTE: this is a sample/illustrative config. The `mcp_tools` generator (this PR) +# already consumes `endpoints.yaml` and each entry's `tools_source`. The +# `mcp_readability` orchestrator and the LLM judge referenced below land in a +# follow-up PR. + +orchestrator: mcp_readability + +# Plug-and-play scorers. Each key names a registered mcp_readability scorer +# (SCORER_REGISTRY in the orchestrator) and carries that scorer's own config. +# The orchestrator runs every scorer declared here against each endpoint and +# merges their result columns; the shared analyzer aggregates a per-scorer pass +# rate (comparator = scorer name) for the scores / summary reports. Add a scorer +# (e.g. conformance) by registering the class and adding a block here. +scorers: + # Deterministic size metrics. Pass = within token budget. `token_budget` feeds + # mcp_readability_token_budget_used_percent (endpoints may override per-entry). + mcp_tool_metrics: + token_budget: 200000 + # LLM judge vs the style guide. Pass = no P0 findings. Owns its model config + # and style-guide path. Run with EVAL_GCP_PROJECT_REGION=global. + mcp_style_readability: + model_config: datasets/model_configs/gemini_3.1_pro_model.yaml + style_guide: datasets/mcp_readability/style_guide.md + +# Inputs (paths relative to the repo root / run cwd). +endpoints_config: datasets/mcp_readability/endpoints.yaml +exceptions_config: datasets/mcp_readability/exceptions.yaml # optional +tools_generator_config: datasets/mcp_readability/tools_generator.yaml + +# Optional filter: only check endpoints whose endpoint_type is in this list. +# Empty = check all. e.g. [PROD] +endpoint_types: [] + +# Per-endpoint check concurrency. +runners: + endpoint_runners: 4 + +# Shared EvalBench reporters. CSV writes results//evals.csv; uncomment +# bigquery to also append rows to .evalbench.results +# (schema auto-evolves). +reporting: + csv: + output_directory: 'results' + # bigquery: + # gcp_project_id: your-gcp-project diff --git a/datasets/mcp_readability/sample_tools.json b/datasets/mcp_readability/sample_tools.json new file mode 100644 index 00000000..e77c92a9 --- /dev/null +++ b/datasets/mcp_readability/sample_tools.json @@ -0,0 +1,51 @@ +{ + "tools": [ + { + "name": "list_datasets", + "description": "List all BigQuery datasets in the given project.", + "inputSchema": { + "type": "object", + "properties": { + "project_id": { + "type": "string", + "description": "The Google Cloud project ID." + }, + "page_size": { + "type": "integer", + "description": "Maximum number of datasets to return.", + "default": 50 + }, + "page_token": { + "type": "string", + "description": "Opaque token from a previous response for the next page." + } + }, + "required": ["project_id"] + } + }, + { + "name": "get_job_state", + "description": "Return the current state of a BigQuery job.", + "inputSchema": { + "type": "object", + "properties": { + "project_id": { + "type": "string", + "description": "The Google Cloud project ID." + }, + "job_id": { + "type": "string", + "description": "The BigQuery job identifier." + }, + "view": { + "type": "string", + "description": "How much job detail to return.", + "enum": ["BASIC", "FULL"], + "default": "BASIC" + } + }, + "required": ["project_id", "job_id"] + } + } + ] +} diff --git a/datasets/mcp_readability/style_guide.md b/datasets/mcp_readability/style_guide.md new file mode 100644 index 00000000..369e959c --- /dev/null +++ b/datasets/mcp_readability/style_guide.md @@ -0,0 +1,31 @@ +# Data Cloud MCP Tool Style Guide (sample) + +This is a small, illustrative style guide for the MCP readability check. Each rule +heading is annotated with a priority (``) and a stable +`{#tag}` anchor. The readability judge uses the priority to assign severity +(`p0`→P0, `p1`→P1, `p2`→P2) and uses the `{#tag}` anchor as the `rule_id`, so +waivers in `exceptions.yaml` stay valid even when a heading is reworded. + +### Tool Names {#tool-names} + +Tool names should be `snake_case` and read as `_` (e.g. +`list_datasets`, `get_instance`). Avoid abbreviations or internal jargon an agent +is unlikely to recognize. + +### Concise, Complete Descriptions {#descriptions} + +Every tool and every parameter must have a description that says what it does and +when to use it. Descriptions should be concise but self-contained — the agent +should not need external documentation to call the tool correctly. + +### Use Enums for Closed Sets {#use-enums} + +When a parameter accepts a fixed set of values, declare them as an `enum` in the +input schema rather than describing the options in prose, so the agent always +picks a valid value. + +### Safe Pagination {#safe-pagination} + +List operations that can return large result sets must expose pagination (for +example `page_size` and `page_token`) so an agent can page through results safely +instead of requesting an unbounded response. diff --git a/datasets/mcp_readability/tools_generator.yaml b/datasets/mcp_readability/tools_generator.yaml new file mode 100644 index 00000000..c2c3dbf0 --- /dev/null +++ b/datasets/mcp_readability/tools_generator.yaml @@ -0,0 +1,4 @@ +# Config for the mcp_tools generator (the system-under-test fetcher). +# Selected by `tools_generator_config` in the run config; built via get_generator. +generator: mcp_tools +timeout: 30 diff --git a/datasets/model_configs/agy_cli_fake_model.yaml b/datasets/model_configs/agy_cli_fake_model.yaml new file mode 100644 index 00000000..89d5bf63 --- /dev/null +++ b/datasets/model_configs/agy_cli_fake_model.yaml @@ -0,0 +1,62 @@ +# NOTE: the --config arg below points at THIS file's own path. +# If you rename/move this file, update that path too, or the +# fake MCP server will fail to load its tool definitions. +generator: agy_cli +env: + GOOGLE_CLOUD_PROJECT: "example-project" + GOOGLE_CLOUD_LOCATION: "us-central1" + GOOGLE_GENAI_USE_VERTEXAI: "true" +setup: + fake_mcp_servers: + "cloud-sql": + command: "python" + args: + - "evalbench/util/fake_mcp_server.py" + - "--server-name" + - "cloud-sql" + - "--config" + - "datasets/model_configs/agy_cli_fake_model.yaml" + +fake_mcp_tools: + "cloud-sql": + - name: create_instance + description: "Creates a Postgres instance using `Production` and `Development` presets. For the `Development` template, it chooses a 2 vCPU, 16 GiB RAM, 100 GiB SSD configuration with Non-HA/zonal availability. For the `Production` template, it chooses an 8 vCPU, 64 GiB RAM, 250 GiB SSD configuration with HA/regional availability. The Enterprise Plus edition is used in both cases. The default database version is `POSTGRES_17`. The agent should ask the user if they want to use a different version." + parameters: + type: object + properties: + project_id: + type: string + description: "The ID of the project." + instance_name: + type: string + description: "The name of the Cloud SQL instance." + database_version: + type: string + description: "The database engine type and version." + password: + type: string + description: "The password for the default user." + edition_preset: + type: string + description: "The edition preset for the instance." + required: ["project_id", "instance_name", "database_version", "password", "edition_preset"] + response: + status: "success" + message: "Instance created successfully" + - name: get_instance + description: "gets the details of a Cloud SQL instance." + parameters: + type: object + properties: + project_id: + type: string + description: "The ID of the project." + instance_name: + type: string + description: "The name of the Cloud SQL instance." + required: ["project_id", "instance_name"] + response: + status: "failure" + error: + code: 404 + message: "Instance not found or permission denied" diff --git a/datasets/model_configs/agy_cli_model.yaml b/datasets/model_configs/agy_cli_model.yaml new file mode 100644 index 00000000..d30e6347 --- /dev/null +++ b/datasets/model_configs/agy_cli_model.yaml @@ -0,0 +1,39 @@ +# Antigravity (agy) CLI -- self-updating native binary installed via +# https://antigravity.google/cli/install.sh. No npm package and no pinning +# mechanism is exposed by the installer; the `agy` binary on PATH is what runs. + +generator: agy_cli + +# Model to use. The harness passes this via agy's `--model` flag (agy +# >=1.0.5). The value must be the exact agy UI label (NOT an API id like +# "gemini-2.5-pro") -- list the valid labels with `agy models`. An +# unrecognized label is silently ignored and agy falls back to its default +# model. Omit this key to leave the flag off, so agy uses its own default. +model: "Gemini 3.1 Pro (High)" + +# Timeout for print mode wait (default 5m). Passed via --print-timeout. +timeout: "20m" + +env: + # Set to your project, or export EVAL_GCP_PROJECT_ID and keep the !ENV form. + # Key stays GOOGLE_CLOUD_PROJECT -- the var the agy subprocess reads. + GOOGLE_CLOUD_PROJECT: !ENV ${EVAL_GCP_PROJECT_ID} + GOOGLE_CLOUD_LOCATION: "global" + GOOGLE_GENAI_USE_VERTEXAI: "true" + +setup: + mcp_servers: + "cloud-sql": + # agy's native HTTP endpoint field is "serverUrl"; "url" also works + # as of v1.0.5. A gemini-style "httpUrl" is auto-translated to + # "serverUrl" by the harness (_translate_mcp_config), so it works + # too -- but prefer "serverUrl" here. authProviderType/oauth/headers + # are native agy fields, so Google auth works without Bearer-header + # injection. + serverUrl: "https://sqladmin.googleapis.com/mcp" + authProviderType: google_credentials + oauth: + scopes: + - https://www.googleapis.com/auth/cloud-platform + headers: + X-Goog-User-Project: !ENV ${EVAL_GCP_PROJECT_ID} diff --git a/datasets/model_configs/agy_cli_skills_model.yaml b/datasets/model_configs/agy_cli_skills_model.yaml new file mode 100644 index 00000000..952a6e4b --- /dev/null +++ b/datasets/model_configs/agy_cli_skills_model.yaml @@ -0,0 +1,32 @@ +# Antigravity (agy) CLI -- self-updating native binary. The `agy` command on +# PATH is what runs. +# +# Model selection: the harness passes the model via agy's `--model` flag +# (agy >=1.0.5). The value must be the exact agy UI label (e.g. +# "Gemini 3.1 Pro (High)"), as listed by `agy models`. Left unset here so the +# flag is omitted and agy uses its own default model. +generator: agy_cli +env: + # Set to your project, or export EVAL_GCP_PROJECT_ID and keep the !ENV form. + # Key stays GOOGLE_CLOUD_PROJECT -- the var the agy subprocess reads. + GOOGLE_CLOUD_PROJECT: !ENV ${EVAL_GCP_PROJECT_ID} + GOOGLE_CLOUD_LOCATION: "global" + GOOGLE_GENAI_USE_VERTEXAI: "true" + + # Set the following environment variables for sql skills + CLOUD_SQL_POSTGRES_PROJECT: !ENV ${EVAL_GCP_PROJECT_ID} + CLOUD_SQL_POSTGRES_REGION: "us-central1" + CLOUD_SQL_POSTGRES_INSTANCE: + CLOUD_SQL_POSTGRES_DATABASE: + CLOUD_SQL_POSTGRES_USER: + CLOUD_SQL_POSTGRES_PASSWORD: + CLOUD_SQL_POSTGRES_IP_TYPE: "PUBLIC" +setup: + # `skills` is named for parity with the claude_code/codex_cli harnesses. + # For agy each entry installs a *plugin* (`agy plugin install`), which may + # bundle both skills and its own MCP servers. To attach a standalone MCP + # server not packaged in a plugin, use a separate top-level `mcp_servers` + # block instead. + skills: + - action: install_from_repo + path: "https://github.com/gemini-cli-extensions/cloud-sql-postgresql.git" diff --git a/datasets/model_configs/claude_code_mcp_repo_model.yaml b/datasets/model_configs/claude_code_mcp_repo_model.yaml new file mode 100644 index 00000000..e928b480 --- /dev/null +++ b/datasets/model_configs/claude_code_mcp_repo_model.yaml @@ -0,0 +1,52 @@ +# Claude Code CLI version. +# - Globally installed binary: "claude" +# - Pin to a specific npm version (uses `npm exec --yes` like Gemini CLI): +# "@anthropic-ai/claude-code@2.1.156" +claude_code_version: "@anthropic-ai/claude-code@2.1.156" + +generator: claude_code + +model: "claude-opus-4-8" + +use_vertex: true +env: + ANTHROPIC_VERTEX_PROJECT_ID: "" + CLOUD_ML_REGION: "global" + +setup: + # MCP servers can be set up in two ways: + # + # 1. Inline (dict): declare server configs directly. Written to + # mcp_servers.json and passed via --mcp-config. + # + # mcp_servers: + # "cloud-sql": + # httpUrl: "https://sqladmin.googleapis.com/mcp" + # authProviderType: google_credentials + # headers: + # X-Goog-User-Project: astana-evaluation + # + # 2. Install from repo (list): clone a Claude Code plugin marketplace repo + # whose plugin bundles MCP servers (.mcp.json / mcpServers in plugin.json). + # The marketplace is registered as a `directory` source and the plugin + # enabled, so Claude Code auto-starts the bundled MCP servers -- mirroring + # the `skills` install_from_repo flow. + # - url: git URL; append `#tag` to pin a version + # - plugin: optional; enable a specific plugin (defaults to the + # marketplace's first) + # - config: optional; values for the plugin's `userConfig`, written to + # pluginConfigs[].options so `${user_config.*}` + # placeholders in the bundled MCP config resolve non-interactively + # + # The data-agent-kit-starter-pack plugin ("dak") bundles 19 skills and 11 MCP + # servers (bigquery, spanner, cloud-sql-postgresql, dataproc, ...). Its MCP + # servers reference ${user_config.PROJECT_ID}, ${user_config.GCP_REGION}, and + # ${user_config.BIGQUERY_LOCATION}, supplied below via `config`. + mcp_servers: + - action: install_from_repo + url: "https://github.com/gemini-cli-extensions/data-agent-kit-starter-pack.git" + plugin: "dak" + config: + PROJECT_ID: "" + GCP_REGION: "us-central1" + BIGQUERY_LOCATION: "US" diff --git a/datasets/model_configs/claude_code_model.yaml b/datasets/model_configs/claude_code_model.yaml index dfa89618..86fba8e4 100644 --- a/datasets/model_configs/claude_code_model.yaml +++ b/datasets/model_configs/claude_code_model.yaml @@ -13,13 +13,11 @@ model: "claude-opus-4-6" # --- Auth Option 1: Vertex AI (GCP ADC, no API key) --- # Requires Claude models enabled in your GCP project via Model Garden. -# use_vertex: true -# vertex_project_id: "astana-evaluation" -# vertex_region: "us-east5" +use_vertex: true +vertex_project_id: "astana-evaluation" +vertex_region: "us-east5" -use_vertex: false env: - ANTHROPIC_API_KEY: GOOGLE_CLOUD_PROJECT: "astana-evaluation" GOOGLE_CLOUD_LOCATION: "us-central1" @@ -32,4 +30,4 @@ setup: scopes: - https://www.googleapis.com/auth/cloud-platform headers: - X-Goog-User-Project: astana-evaluation + X-Goog-User-Project: astana-evaluation \ No newline at end of file diff --git a/datasets/model_configs/claude_code_skills_model.yaml b/datasets/model_configs/claude_code_skills_model.yaml new file mode 100644 index 00000000..a4862fd8 --- /dev/null +++ b/datasets/model_configs/claude_code_skills_model.yaml @@ -0,0 +1,31 @@ +# Claude Code CLI version. +# - Globally installed binary: "claude" +# - Pin to a specific npm version (uses `npm exec --yes` like Gemini CLI): +# "@anthropic-ai/claude-code@2.1.85" +claude_code_version: "@anthropic-ai/claude-code@2.1.119" + +generator: claude_code + +model: "claude-opus-4-7" + +use_vertex: true +env: + ANTHROPIC_VERTEX_PROJECT_ID: "astana-evaluation" + CLOUD_ML_REGION: "global" + # Cloud SQL PostgreSQL extension configuration + CLOUD_SQL_POSTGRES_PROJECT: "astana-evaluation" + CLOUD_SQL_POSTGRES_REGION: "us-central1" + CLOUD_SQL_POSTGRES_INSTANCE: + CLOUD_SQL_POSTGRES_DATABASE: + CLOUD_SQL_POSTGRES_USER: + CLOUD_SQL_POSTGRES_PASSWORD: + CLOUD_SQL_POSTGRES_IP_TYPE: "PUBLIC" + +setup: + # Install skills from main branch (uses npx toolbox-sdk, no local binary needed) + # For full plugin marketplace setup (interactive), manually run in Claude Code: + # /plugin marketplace add https://github.com/gemini-cli-extensions/cloud-sql-postgresql.git + # /plugin install cloud-sql-postgresql@cloud-sql-postgresql-marketplace + skills: + - action: install_from_repo + url: "https://github.com/gemini-cli-extensions/cloud-sql-postgresql.git" diff --git a/datasets/model_configs/codex_cli_fake_model.yaml b/datasets/model_configs/codex_cli_fake_model.yaml new file mode 100644 index 00000000..16986c38 --- /dev/null +++ b/datasets/model_configs/codex_cli_fake_model.yaml @@ -0,0 +1,85 @@ +codex_cli_version: "@openai/codex@latest" +generator: codex_cli +model: "gpt-5.5" + +# Fetched from Secret Manager (override per-environment as needed). +openai_api_key_secret: "projects/393137573/secrets/OPENAI_API_KEY/versions/1" + +env: + GOOGLE_CLOUD_PROJECT: "astana-evaluation" + +setup: + mcp_servers: + "cloud-sql": + command: "python" + args: + - "evalbench/util/fake_mcp_server.py" + - "--server-name" + - "cloud-sql" + - "--config" + - "datasets/model_configs/codex_cli_fake_model.yaml" + +fake_mcp_tools: + "cloud-sql": + - name: create_instance + description: "Creates a Cloud SQL instance" + parameters: + type: object + properties: + project_id: + type: string + description: "GCP project ID" + instance_name: + type: string + description: "Name for the new instance" + required: ["project_id", "instance_name"] + response: + status: "success" + message: "Instance created successfully" + - name: get_instance + description: "Gets details of a Cloud SQL instance" + parameters: + type: object + properties: + project_id: + type: string + description: "GCP project ID" + instance_name: + type: string + description: "Instance name" + required: ["project_id", "instance_name"] + response: + status: "failure" + error: + code: 404 + message: "Instance not found or permission denied" + - name: list_instances + description: "Lists all Cloud SQL instances in a project" + parameters: + type: object + properties: + project_id: + type: string + description: "GCP project ID" + required: ["project_id"] + response: + status: "success" + instances: + - name: "nl2code" + state: "RUNNABLE" + databaseVersion: "POSTGRES_15" + - name: update_instance + description: "Updates a Cloud SQL instance" + parameters: + type: object + properties: + project_id: + type: string + description: "GCP project ID" + instance_name: + type: string + description: "Instance name" + required: ["project_id", "instance_name"] + response: + status: "success" + message: "Instance updated successfully" diff --git a/datasets/model_configs/codex_cli_model.yaml b/datasets/model_configs/codex_cli_model.yaml new file mode 100644 index 00000000..d762d4b7 --- /dev/null +++ b/datasets/model_configs/codex_cli_model.yaml @@ -0,0 +1,36 @@ +# OpenAI Codex CLI version. +# - Globally installed binary: "codex" +# - Pin to a specific npm version (uses `npm exec --yes` like Gemini CLI): +# "@openai/codex@latest" +codex_cli_version: "@openai/codex@latest" + +generator: codex_cli + +# Model to use. Pass any model id supported by your OpenAI account, e.g. +# "gpt-5.5", "o4-mini", "gpt-4.1", etc. +model: "gpt-5.5" + +# OPENAI_API_KEY is fetched from Google Secret Manager. Provide the resource +# path; either the bare form or the `secret_manager://` URL form works. +openai_api_key_secret: "projects/393137573/secrets/OPENAI_API_KEY/versions/1" + +# Codex's NDJSON has token counts but no cost. Provide rates here and +# evalbench will compute `cost_usd` per turn. Update these to match the +# model you set above; rates below are placeholder OpenAI list prices. +# `cached_input_per_million_usd` is optional — defaults to 10% of input. +pricing: + input_per_million_usd: 1.25 + cached_input_per_million_usd: 0.125 + output_per_million_usd: 10.0 + +env: + GOOGLE_CLOUD_PROJECT: "astana-evaluation" + GOOGLE_CLOUD_LOCATION: "us-central1" + +setup: + mcp_servers: + "cloud-sql": + httpUrl: "https://sqladmin.googleapis.com/mcp" + authProviderType: google_credentials + headers: + X-Goog-User-Project: astana-evaluation diff --git a/datasets/model_configs/gcp_data_engineering_agent_model.yaml b/datasets/model_configs/gcp_data_engineering_agent_model.yaml new file mode 100644 index 00000000..5862278c --- /dev/null +++ b/datasets/model_configs/gcp_data_engineering_agent_model.yaml @@ -0,0 +1,9 @@ +generator: "data_engineering_agent" +gcp_project_id: !ENV ${EVAL_GCP_PROJECT_ID} +gcp_region: !ENV ${EVAL_GCP_PROJECT_REGION:us-west4} +dataform_repository: !ENV ${EVAL_DEA_REPOSITORY_ID} +dataform_workspace: !ENV ${EVAL_DEA_WORKSPACE_ID} + +model_env: prod +agent_type: dataengineeringagent +execs_per_minute: 10 diff --git a/datasets/query_data_api/model_configs/querydata_config.yaml b/datasets/query_data_api/model_configs/querydata_config.yaml index 13520013..6e09bafb 100644 --- a/datasets/query_data_api/model_configs/querydata_config.yaml +++ b/datasets/query_data_api/model_configs/querydata_config.yaml @@ -3,6 +3,10 @@ project_id: "" location: "" generator: "query_data_api" +# Optional. Override the default production endpoint (leave unset for prod). +# Example — sandbox: "autopush-geminidataanalytics.sandbox.googleapis.com" +# api_endpoint: "" + # This context block directly maps to the gda.QueryDataContext protobuf definition context: datasource_references: diff --git a/docs/agy_cli_agent_testing.md b/docs/agy_cli_agent_testing.md new file mode 100644 index 00000000..5731aca0 --- /dev/null +++ b/docs/agy_cli_agent_testing.md @@ -0,0 +1,393 @@ +# Antigravity (agy) CLI Evaluation Guide + +This guide covers how to use EvalBench for evaluating [Antigravity CLI](https://antigravity.google/product/antigravity-cli) (`agy`) +agent workflows using **MCP Servers** and **Skills**. It mirrors the structure +of [`gemini_cli_agent_testing.md`](gemini_cli_agent_testing.md) and only calls +out where the two harnesses differ. + +> [!IMPORTANT] +> **First-run auth:** agy uses an OAuth consumer flow backed by the system +> keyring. Before evals can run, complete `agy` +> login interactively at least once on the host so a refreshable token +> exists. After that, the harness can run non-interactively. + +--- + +## Table of Contents + +- [Overview](#overview) +- [Architecture](#architecture) +- [Prerequisites](#prerequisites) +- [Quick Start](#quick-start) +- [Configuration Reference](#configuration-reference) + - [Run Configuration](#1-run-configuration) + - [Model Configuration](#2-model-configuration) + - [Evaluation Dataset (Evalset)](#3-evaluation-dataset-evalset) +- [Tool Paradigms](#tool-paradigms) + - [MCP Servers](#mcp-servers) + - [Skills](#skills) + - [Fake MCP Servers (Testing)](#fake-mcp-servers-testing) +- [Differences vs. Gemini CLI](#differences-vs-gemini-cli) +- [Scorers](#scorers) +- [Reporting](#reporting) +- [Troubleshooting](#troubleshooting) + +--- + +## Overview + +EvalBench's agy CLI integration enables automated, multi-turn evaluation of +agentic workflows that run on the Antigravity CLI binary. Same evaluator, +same scorers, same evalset format as the Gemini CLI guide -- the generator +just shells out to the `agy` binary instead of `npm exec @google/gemini-cli`. + +### Key Capabilities + +- **Multi-turn evaluation** with LLM-powered simulated users +- **Two tool paradigms** today: MCP servers and skills (agy does not expose a + Gemini-CLI-style extension manager) +- **Fake MCP server support** for deterministic, offline testing +- **Same 8 built-in scorers** as Gemini CLI +- **CSV and BigQuery reporting** + +--- + +## Architecture + +Identical to the Gemini CLI flow; only the generator class changes: + +``` +Run Config -> AgentOrchestrator -> AgentEvaluator -> AgyCliGenerator -> agy + | + v + MCP servers / skills +``` + +The `orchestrator: agent` keyword in your run config selects the `AgentOrchestrator`, while the concrete CLI generator (`agy_cli`) is chosen via the `generator` field in your `model_config`. + +--- + +## Prerequisites + +1. **Python 3.10+** and project dependencies installed using `uv`: + ```bash + cd evalbench + uv sync + ``` + +2. **Antigravity CLI installed (for the one-time interactive auth)**: + ```bash + curl -fsSL https://antigravity.google/cli/install.sh | sh -s -- --dir ~/.local/bin + export PATH="$HOME/.local/bin:$PATH" + agy --version # sanity check + ``` + + The installer writes a native binary; it self-updates in the background + and does not expose a pinning flag. + +> [!NOTE] +> **The harness does not run this host binary.** The testing harness +> installs its own isolated copy of `agy` per session into +> `/.local/bin` and always +> launches that one, never the host `PATH` binary. Installing on the host +> is only needed for the one-time interactive login (next step) that +> seeds the OAuth token the harness mirrors into the sandbox. +> Per-session install keeps concurrent evals isolated and stops agy's +> background self-update from swapping the binary mid-run. + +3. **GCP Authentication** (ADC -- for Google-auth MCP servers' outbound + credentials; agy's own model backend uses the first-run OAuth token, not + ADC): + ```bash + gcloud auth application-default login + ``` + +4. **Environment Variables**: + ```bash + export EVAL_GCP_PROJECT_ID=your_project_id + export EVAL_GCP_PROJECT_REGION=us-central1 + ``` + +--- + +## Quick Start + +### 1. Set the run configuration + +```bash +# For MCP Server evaluation: +export EVAL_CONFIG=datasets/agy-cli-tools/example_run_config.yaml + +# For Skills evaluation: +export EVAL_CONFIG=datasets/agy-cli-tools/example_run_skills_config.yaml + +# For Fake MCP (offline testing): +export EVAL_CONFIG=datasets/agy-cli-tools/example_run_fake_config.yaml +``` + +### 2. Run the evaluation + +```bash +./evalbench/run.sh +``` + +--- + +## Configuration Reference + +### 1. Run Configuration + +For agy CLI, set `orchestrator: agent` (the modern agent-CLI keyword, +shared with `claude_code` and `codex_cli`) and +`dataset_format: agent-format`. The legacy `geminicli` / +`gemini-cli-format` values still work -- both route to +`AgentOrchestrator` -- but the `agent*` names are the right ones for +non-gemini CLIs. + +| Key | Required | Description | +|-----|----------|-------------| +| `dataset_config` | Yes | Path to the evalset JSON file | +| `dataset_format` | Yes | `agent-format` (recommended) or the legacy `gemini-cli-format` | +| `orchestrator` | Yes | `agent` (recommended) or the legacy `geminicli` | +| `model_config` | Yes | Path to the agy CLI model config YAML | +| `simulated_user_model_config` | Yes | Path to the model config for the simulated user LLM | +| `scorers` | Yes | Dictionary of scorer configurations | +| `reporting` | Yes | CSV and/or BigQuery output options | + +**Example** ([example_run_config.yaml](/datasets/agy-cli-tools/example_run_config.yaml)): + +```yaml +dataset_config: datasets/agy-cli-tools/agy-cli.evalset.json +dataset_format: agent-format +orchestrator: agent +model_config: datasets/model_configs/agy_cli_model.yaml +simulated_user_model_config: datasets/model_configs/gemini_2.5_pro_model.yaml + +scorers: + trajectory_matcher: {} + goal_completion: + model_config: datasets/model_configs/gemini_2.5_pro_model.yaml + turn_count: {} + +reporting: + csv: + output_directory: 'results' +``` + +--- + +### 2. Model Configuration + +| Key | Required | Description | +|-----|----------|-------------| +| `generator` | Yes | Must be `agy_cli` | +| `model` | Optional | agy UI model label, e.g. `"Gemini 3.1 Pro (High)"`. Passed via the `--model` flag. Must be a valid label, not an API id. | +| `timeout` | Optional | Timeout duration string, e.g. `"20m"`. Passed via the `--print-timeout` flag. If omitted, defaults to agy's internal default (5 minutes). | +| `env` | Optional | Environment variables passed to the CLI process | +| `setup` | Optional | Tool setup block containing `mcp_servers`, `skills`, or `fake_mcp_servers` | + +> [!IMPORTANT] +> **Explicit Project Configuration Required:** agy does not read GCP project details from the host environment. You **must** explicitly set `GOOGLE_CLOUD_PROJECT` and `GOOGLE_CLOUD_LOCATION` in the `env` block of your model config. If omitted, agy will return empty responses and fail to make tool calls, even if those variables are exported in your shell. + +> [!NOTE] +> **Model selection uses the `--model` flag; use a UI label.** The harness +> passes the configured `model` via agy's `--model` flag (agy >=1.0.5). The +> value must be the **exact agy UI label** (e.g. `"Gemini 3.1 Pro (High)"`, +> `"Gemini 3.5 Flash (Medium)"`), not an API id like `gemini-2.5-pro` -- an +> unrecognized value is silently ignored and agy falls back to its default +> model. List the valid labels with `agy models`. Omit the key to leave the +> flag off, so agy uses its own default model. + +--- + +### 3. Evaluation Dataset (Evalset) + +Same format as Gemini CLI. See the [Gemini guide's evalset section](gemini_cli_agent_testing.md#3-evaluation-dataset-evalset) +for full field reference; the same `expected_trajectory` canonical form +(`__`) applies. The `agy-cli-tools/` directory ships copies of +the Gemini Cloud SQL evalsets so the two harnesses score against an +identical baseline. + +--- + +## Tool Paradigms + +### MCP Servers + +Configured under `setup.mcp_servers` in the model config. EvalBench writes +the block under the `mcpServers` key of a sandboxed +`/.gemini/config/mcp_config.json` (a separate file from +`settings.json`) and lets agy pick it up at startup. + +> [!IMPORTANT] +> **Use `serverUrl` for the HTTP endpoint** (`url` also works as of v1.0.5); +> a Gemini-style `httpUrl` is auto-translated by the harness. +> `authProviderType`, `oauth.scopes`, and `headers` are native agy fields, so +> Google auth works without Bearer-header injection (unlike `claude_code`). + +The harness pre-verifies attach: at setup it runs +a short probe, then confirms each configured server discovered +tools by checking that agy wrote the expected per-tool schema files to +the local app data cache. A server that attaches no tools fails the evaluation +with the offending server name rather than silently degrading. + +### Skills + +> [!NOTE] +> The field is named `setup.skills` for parity with the `claude_code` and +> `codex_cli` harnesses, which use the same key. For agy each entry is +> installed as a **plugin** (`agy plugin install`), and a plugin bundle may +> carry skills *and* its own MCP servers. The separate top-level +> `setup.mcp_servers` block is for attaching a **standalone** MCP server (by +> URL/stdio) that is not packaged in a plugin -- the two are distinct attach +> paths and are configured independently. + +Configured under `setup.skills`. Skills are delivered via **plugins**: +verified against agy v1.0.5, `agy plugin install ` reads a plugin +manifest (Claude/Gemini/Codex formats), processes any bundled skills, +materializes them under `/.gemini/config/plugins//`, and +records the install in `/.gemini/config/import_manifest.json`. There +is no `agy skills` subcommand, and dropping `SKILL.md` folders on disk +registers nothing (`agy plugin list` stays empty). The harness therefore +shells out to `agy plugin install` for every entry. Two input shapes are +supported: + +```yaml +setup: + skills: + # String form: an install target passed straight to + # `agy plugin install`. May be a local plugin directory or a git URL + # (cloned first). `agy plugin install` requires the target to resolve + # to a directory, so a bare git URL is cloned before install. + - "/path/to/a/local/plugin" + + # Dict form: same, via an explicit target. Git URLs (scheme:// or + # trailing .git) are cloned first, then the clone dir is installed; + # local paths are installed in place. `url:` is conventional; `path:` + # is accepted as a synonym. Append `#` to a git URL to + # pin a version -- the clone uses `git clone --branch`, which resolves + # branch and tag names only, not raw commit SHAs. + - action: install_from_repo + url: "https://github.com/gemini-cli-extensions/cloud-sql-postgresql.git#v1.2.3" +``` + +> [!NOTE] +> A `plugin@marketplace` spec is not a reliable target (unlike `claude_code`/ +> `gemini_cli`); use a git URL or local directory. Legacy dict actions +> (`link`, `install`, `enable`, `disable`, `uninstall`) that the gemini-cli +> generator supports are **not** supported here either -- use a string target +> or `install_from_repo`. Unsupported entries are logged and skipped. + +### Fake MCP Servers (Testing) + +Identical setup to Gemini CLI -- a stdio MCP server defined in +`setup.fake_mcp_servers` with tool definitions in the top-level +`fake_mcp_tools` block. See +[`datasets/model_configs/agy_cli_fake_model.yaml`](../datasets/model_configs/agy_cli_fake_model.yaml) +for a working example. + +--- + +## Differences vs. Gemini CLI + +| Area | Gemini CLI | Antigravity (agy) | +|------|-----------|--------------------| +| Install | `npm install -g @google/gemini-cli@` | `curl install.sh \| sh -- --dir ` | +| Version pinning | NPM specifier in `gemini_cli_version` | No pinning mechanism; the latest binary is installed dynamically at runtime. | +| Invocation | `npm exec --yes @google/gemini-cli@ -- ...` | `agy ...` (bare binary) | +| Non-interactive flag | `--yolo` / `--prompt` | `--dangerously-skip-permissions` and `-p` (alias `--print`) | +| Output format | `--output-format stream-json` (NDJSON on stdout) | `--output-format stream-json` (NDJSON on stdout); the harness parses this event stream (see below) | +| Session resume | `--resume ` | `--continue` (most recent in cwd) or `--conversation ` | +| Settings dir (`appDataDir`) | `~/.gemini/` | `~/.gemini/antigravity-cli/` | +| Settings file | `~/.gemini/settings.json` | `~/.gemini/antigravity-cli/settings.json` | +| Skills dir | `~/.gemini/skills//SKILL.md` | `~/.gemini/config/plugins//` (materialized by `agy plugin install`) | +| Skill management | `gemini skills ` subcommands | `agy plugin install ` (plugin manifests carry skills); no `agy skills` subcommand | +| Extensions | Supported via `setup.extensions` | Not modeled; drop the block | +| MCP config location | `mcpServers` in `settings.json` | `mcpServers` in a separate `~/.gemini/config/mcp_config.json` | +| MCP HTTP transport field | `httpUrl` | `serverUrl` (native); `url` also accepted as of v1.0.5; a Gemini-style `httpUrl` is auto-translated to `serverUrl` by the harness | +| MCP tool name format | `mcp__` (single underscore) | No per-tool functions -- every MCP call goes through a single native `call_mcp_tool` wrapper whose args carry `ServerName`/`ToolName`/`Arguments`; the harness unwraps it to the canonical `__` | +| Model selection | `GEMINI_API_MODEL` / `GEMINI_MODEL` env var | `--model` flag (agy >=1.0.5); value is a UI label (e.g. `"Gemini 3.1 Pro (High)"`), not an API id | +| Auth | NPM auth token via `gcloud auth print-access-token` plus ADC | OAuth (keyring-backed); ADC not required by agy itself | +| Token-usage stats | Reported per request | Exposed via the stream-json `result` event's `usage` block (input/output/thinking/total tokens) | + +### Tool-call extraction + +The harness runs agy with `--output-format stream-json` and parses the event +stream (`init`, `step_update` x N, `result`) from stdout. Each tool call is a +`step_update` whose `ACTIVE` -> `DONE`/`ERROR` states give success/failure +directly; `call_mcp_tool` invocations are unwrapped into the canonical +`__` format. The stream is per-invocation, so no turn-slicing is +needed under `--continue`. A run that times out or errors exits non-zero but +still emits a full stream ending in an `ERROR` `result`, so its tool calls are +captured too. + +--- + +## Scorers + +Identical to Gemini CLI -- see the +[scorers section of the Gemini guide](gemini_cli_agent_testing.md#scorers). +The `trajectory_matcher` default of dropping native/harness-internal tools +also applies. + +--- + +## Reporting + +Identical to Gemini CLI. CSV under `reporting.csv.output_directory`, +BigQuery under `reporting.bigquery.gcp_project_id`. + +--- + +## Troubleshooting + +### `agy: command not found` + +This affects the **host** `agy` command -- used for the first-run auth and +for `agy models` / `agy --version` -- not the eval run, which launches the +harness's own per-session binary. Re-run the installer with `--dir` pointing +at a directory on your `PATH`, or symlink the binary into one. + +### MCP Server Doesn't Attach + +The harness pre-verifies attach at setup: it runs a +short probe and fails fast if a configured +server discovered no tools. If you hit that error: + +- **Check the URL field** (see the MCP Servers callout above): a typo'd or + unrecognized URL key is accepted silently and exposes zero tools, so it + looks like a load failure. +- Confirm the block lives under `mcpServers` in + `/.gemini/config/mcp_config.json` (not `settings.json`) after + setup runs. +- Confirm agy wrote per-tool schemas to + `/.gemini/antigravity-cli/mcp//*.json` -- an empty + directory means the server failed to attach. +- For Google-auth servers, run `gcloud auth application-default login` + (used for outbound credentials to the MCP server -- agy's own auth is + separate OAuth). +- Verify OAuth scopes and project ID in the headers. + +### Skill Not Picked Up + +- Confirm the plugin registered: check that the name appears in + `/.gemini/config/import_manifest.json` and that + `/.gemini/config/plugins//` was materialized after + setup runs. The setup log line `agy registered plugins: [...]` reports + what `agy plugin install` recorded. +- If `agy plugin install` failed, the harness logs an `agy plugin install + '' failed` line with the install command's exit code and stderr -- + read that line for the reason. A bad target is the usual cause: a wrong + path, an unreachable git URL, or a directory with no valid plugin manifest + (agy accepts Claude/Gemini/Codex manifest formats, e.g. `plugin.json` or + `gemini-extension.json`). +- If you have an `action: link` / `enable` / `install` entry in your + config, drop it -- those gemini-cli-style actions are not supported + here and are logged-and-skipped. Use a string target or + `install_from_repo`. + +### Empty or Missing Results + +- Confirm `dataset_format` is `agent-format` (or the legacy `gemini-cli-format`). +- Verify the `model_config` path is correct relative to the repo root. +- Check that `agy --version` works from the same shell. diff --git a/docs/architecture.md b/docs/architecture.md new file mode 100644 index 00000000..ffe601f6 --- /dev/null +++ b/docs/architecture.md @@ -0,0 +1,212 @@ +# External Dependency Graph + +Generated from `pyproject.toml` cross-referenced against actual imports under +`evalbench/`. Each edge is a real import path in the code — deps with no +incoming edge are unused-by-evalbench (none currently). + +## Graph + +```mermaid +graph LR + %% ============= Evalbench internal packages ============= + subgraph EVB ["evalbench/"] + GEN_SDK[generators/models
SDK judges] + GEN_AGENT[generators/models
agent CLIs] + GEN_DEA[generators/models
data_engineering_agent.py
query_data_api.py] + DB[databases/] + DBUTIL[databases/util.py] + SCORE[scorers/
dbtscorer.py] + REPORT[reporting/
gcs_artifact.py] + SESSION[util/session.py] + EVALBENCH[evalbench.py
util/sanitizer.py
etc.] + end + + %% ============= External dep groups ============= + subgraph LLM ["LLM SDKs"] + GENAI[google-genai] + ANTHROPIC["anthropic[vertex]"] + A2A["a2a-sdk ≥1.0.3"] + ADK[google-adk] + GDA["google-cloud-geminidataanalytics
≥0.11.0"] + end + + subgraph GCP ["GCP services"] + SPANNER[google-cloud-spanner] + SQLA_SP[sqlalchemy-spanner] + BIGTABLE[google-cloud-bigtable] + SECRETS[google-cloud-secret-manager] + FIRESTORE[google-cloud-firestore] + STORAGE[google-cloud-storage] + IAM[google-cloud-iam] + ALLOY["google-cloud-alloydb-connector
[pg8000]"] + CSQL_PG["cloud-sql-python-connector
[pg8000]"] + CSQL_TDS["cloud-sql-python-connector
[pytds]"] + PGBQ[pandas-gbq] + end + + subgraph SQL ["DB drivers / ORM"] + SQLA[sqlalchemy] + SQLA_TDS[sqlalchemy-pytds] + PYMYSQL[pymysql] + PYMONGO[pymongo] + MONGOMOCK[mongomock] + REDIS[redis] + end + + subgraph PARSE ["SQL parsing"] + SQLGLOT[sqlglot] + SQLPARSE[sqlparse] + end + + subgraph DBT ["dbt"] + DBT_CORE[dbt-core] + DBT_BQ[dbt-bigquery] + DBT_PG[dbt-postgres] + end + + subgraph DATA ["Data processing"] + PANDAS[pandas] + ARROW[pyarrow] + PROTO[protobuf] + end + + subgraph INFRA ["Infra / runtime"] + GRPC[grpcio ≥1.80] + GRPCT[grpcio-tools] + JSCHEMA[jsonschema] + RL[ratelimit] + BACKOFF[backoff] + AIOLOG[aiologger] + YAMLENV[pyaml_env] + ABSL[absl-py] + GITPY[GitPython] + RICH[rich] + TAB[tabulate] + end + + subgraph DEV ["Dev only"] + PYCS["pycodestyle ≥2.14"] + PYTEST["pytest ≥8.0"] + end + + %% ============= Edges ============= + GEN_SDK --> GENAI + GEN_SDK --> ANTHROPIC + + GEN_AGENT --> STORAGE + GEN_AGENT -. shell-out .-> GENAI + + GEN_DEA --> A2A + GEN_DEA --> GDA + + DB --> SQLA + DB --> SQLA_SP + DB --> SQLA_TDS + DB --> SPANNER + DB --> BIGTABLE + DB --> PYMYSQL + DB --> PYMONGO + DB --> MONGOMOCK + DB --> ALLOY + DB --> CSQL_PG + DB --> CSQL_TDS + DB --> PGBQ + + DBUTIL --> SECRETS + DBUTIL --> REDIS + + SCORE --> DBT_CORE + SCORE --> DBT_BQ + SCORE --> DBT_PG + + REPORT --> STORAGE + + SESSION --> FIRESTORE + SESSION --> STORAGE + SESSION --> ADK + + EVALBENCH --> SQLGLOT + EVALBENCH --> SQLPARSE + EVALBENCH --> PANDAS + EVALBENCH --> ARROW + EVALBENCH --> PROTO + EVALBENCH --> GRPC + EVALBENCH --> GRPCT + EVALBENCH --> JSCHEMA + EVALBENCH --> RL + EVALBENCH --> BACKOFF + EVALBENCH --> AIOLOG + EVALBENCH --> YAMLENV + EVALBENCH --> ABSL + EVALBENCH --> GITPY + EVALBENCH --> RICH + EVALBENCH --> TAB + EVALBENCH --> IAM + + classDef internal fill:#dbeafe,stroke:#1e40af,color:#1e3a8a + classDef llm fill:#fef3c7,stroke:#b45309,color:#78350f + classDef gcp fill:#d1fae5,stroke:#047857,color:#064e3b + classDef sql fill:#ede9fe,stroke:#6d28d9,color:#4c1d95 + classDef parse fill:#fce7f3,stroke:#be185d,color:#831843 + classDef dbt fill:#ffedd5,stroke:#c2410c,color:#7c2d12 + classDef data fill:#f3f4f6,stroke:#4b5563,color:#1f2937 + classDef infra fill:#e0f2fe,stroke:#0369a1,color:#0c4a6e + classDef dev fill:#f5f5f4,stroke:#78716c,color:#44403c + + class GEN_SDK,GEN_AGENT,GEN_DEA,DB,DBUTIL,SCORE,REPORT,SESSION,EVALBENCH internal + class GENAI,ANTHROPIC,A2A,ADK,GDA llm + class SPANNER,SQLA_SP,BIGTABLE,SECRETS,FIRESTORE,STORAGE,IAM,ALLOY,CSQL_PG,CSQL_TDS,PGBQ gcp + class SQLA,SQLA_TDS,PYMYSQL,PYMONGO,MONGOMOCK,REDIS sql + class SQLGLOT,SQLPARSE parse + class DBT_CORE,DBT_BQ,DBT_PG dbt + class PANDAS,ARROW,PROTO data + class GRPC,GRPCT,JSCHEMA,RL,BACKOFF,AIOLOG,YAMLENV,ABSL,GITPY,RICH,TAB infra + class PYCS,PYTEST dev +``` + +## Notes + +- **Agent CLIs** (`gemini_cli`, `claude_code`, `codex_cli`, `agy_cli`) shell out + to external binaries; they don't import the SDKs directly. The dashed edge + `agent CLIs ⇢ google-genai` shows that the *agent under test* uses Gemini + internally, not that evalbench imports it for them. +- **`google-genai`** is consumed by both the Gemini SDK judge + (`generators/models/gemini.py`) and indirectly by tests + (`test/gemini_tools_test.py`). +- **`google-cloud-iam`** has no obvious consumer in `evalbench/` import + scanning — likely pulled in transitively by other GCP SDKs or used by deploy + tooling. Worth confirming before assuming it's load-bearing. +- **`dbt-core` + adapters** are consumed only by `scorers/dbtscorer.py` — + pruning them would only break dbt-based scoring. +- **`mongomock`** is a test-time dependency for `databases/mongodb.py`. Listed + in main `dependencies` rather than `dev` — worth flagging if you want to slim + the runtime install. + +## Dep groups by purpose + +| Group | Purpose | If removed, what breaks | +|---|---|---| +| LLM SDKs | Direct Gemini/Claude judges + DEA agent | SDK-judged scorers, DEA generator | +| GCP services | Backend connectivity for managed DBs + storage | Spanner/BigQuery/Bigtable/AlloyDB/CloudSQL/Firestore/GCS paths | +| DB drivers | Raw drivers for self-managed DBs | Postgres/MySQL/MSSQL/Mongo backends + cache | +| SQL parsing | Pre-execution analysis (e.g. trajectory matching) | scorers that compare structural SQL | +| dbt | dbt-based scoring | `dbtscorer` only | +| Data processing | Result-set handling | Most CSV/Parquet reporting paths | +| Infra | Cross-cutting (RPC, rate limiting, logging) | Most things — these are foundational | +| Dev | Linting + tests | CI; not runtime | + +## Surfacing supply-chain risk + +For a quick sweep of high-blast-radius deps: + +```bash +# Direct deps only (not transitive) +grep -A 100 '^dependencies = \[' pyproject.toml | sed -n 's/^ "\(.*\)",/\1/p' + +# Per-package recent versions (lockfile) +grep -A1 '^name = ' uv.lock | grep -E '^(name|version)' | paste - - | head -40 +``` + +GitHub Dependabot covers most of the CVE noise (already enabled — the recent +push showed *"3 vulnerabilities on default branch"*). The graph above tells +you which deps an unpatched CVE actually reaches in this codebase. diff --git a/docs/claude_code_agent_testing.md b/docs/claude_code_agent_testing.md index 246bb0f1..0e76f09c 100644 --- a/docs/claude_code_agent_testing.md +++ b/docs/claude_code_agent_testing.md @@ -213,7 +213,7 @@ The model config defines the Claude Code CLI version, model, auth, environment, ### 3. Evaluation Dataset (Evalset) -**Identical schema** to the Gemini CLI evalset. See [Gemini CLI doc — Evalset](./gemini_cli_agent_testing.md#3-evaluation-dataset-evalset) for details. +**Identical schema** to the Gemini CLI evalset. See [Gemini CLI doc — Evalset](./gemini_cli_agent_testing.md#3-evaluation-dataset-evalset) for details, including the canonical [tool name format](./gemini_cli_agent_testing.md#tool-name-format) used in `expected_trajectory`. Minimal example: @@ -224,7 +224,7 @@ Minimal example: "id": "cloud-sql-list-instances-01", "starting_prompt": "list all Cloud SQL instances in project astana-evaluation", "conversation_plan": "Ask the agent to list instances. If nl2code exists, get its state and verify it is RUNNABLE.", - "expected_trajectory": ["list_instances", "get_instance"], + "expected_trajectory": ["cloud-sql__list_instances", "cloud-sql__get_instance"], "env": { "GOOGLE_CLOUD_PROJECT": "astana-evaluation" }, "kind": "tools", "max_turns": 3 @@ -283,7 +283,7 @@ EvalBench accepts the **same MCP server config schema as Gemini CLI** for HTTP s | Gemini-style field | Claude Code translation | |---|---| | `httpUrl` | → `url` + auto-adds `type: "http"` | -| `authProviderType: google_credentials` | → fetches a token via `gcloud auth print-access-token` and injects `Authorization: Bearer ` into headers | +| `authProviderType: google_credentials` | → injects `Authorization: Bearer ` (from `gcloud auth application-default print-access-token`, falling back to `gcloud auth print-access-token`) **and** sets a `headersHelper` so Claude Code re-mints a fresh ADC token on every connection (avoids ~1h expiry). Google API MCP endpoints reject the plain user token on tool calls — see [Troubleshooting](#mcp-tool-call-fails-with-incompatible-auth-server-does-not-support-dynamic-client-registration). | | `oauth.scopes` | (dropped — Claude Code doesn't use Gemini's OAuth delegation) | | `headers` | → passed through as-is | | `command` / `args` (stdio) | → passed through as-is | @@ -347,7 +347,7 @@ Quick reference: | Scorer | Type | Description | |---|---|---| -| `trajectory_matcher` | Deterministic | Jaccard or Levenshtein match between expected and actual tool trajectory | +| `trajectory_matcher` | Deterministic | Jaccard or Levenshtein match between expected and actual tool trajectory. Native Claude Code tools (`Read`, `Bash`, `Edit`, `ToolSearch`, ...) are dropped from both sides by default — set `filter_native_tools: false` to score them too. | | `goal_completion` | LLM | Did the agent accomplish the conversation plan? | | `behavioral_metrics` | LLM | Hallucination rate + clarification rate | | `parameter_analysis` | LLM | Qualitative feedback on tool parameters | @@ -470,9 +470,25 @@ runners: agent_runners: 1 ``` +### MCP tool call fails with `Incompatible auth server: does not support dynamic client registration` + +**Symptom:** the agent connects to the MCP server and can list its tools, but the first real tool call fails with `Incompatible auth server: does not support dynamic client registration`. + +**Root cause:** this is a *misleading* error. Google API MCP endpoints (e.g. `sqladmin.googleapis.com/mcp`) leave `initialize` and `tools/list` unauthenticated but require a valid bearer token on the actual tool call. When the injected token is missing, expired, or the **wrong kind**, the tool call returns `401` with a `WWW-Authenticate: Bearer resource_metadata=...` challenge. Claude Code always attaches an OAuth authProvider to HTTP MCP servers, so on that 401 it tries to complete an OAuth 2.0 flow — which begins with [dynamic client registration (RFC 7591)](https://datatracker.ietf.org/doc/html/rfc7591). These Google endpoints **don't support dynamic client registration**, so the OAuth fallback dies with that error instead of surfacing the underlying 401. + +**Fixes:** +- **Use ADC, not the user token.** The generator now injects `gcloud auth application-default print-access-token` (ADC) rather than `gcloud auth print-access-token`. The plain gcloud *user* token is rejected by these endpoints ("Request had invalid authentication credentials"), which is what triggered the fallback. Make sure you ran **`gcloud auth application-default login`** (not just `gcloud auth login`). +- **Token expiry on long runs (handled automatically).** Google access tokens expire after ~1 hour. To avoid later scenarios hitting the same 401 → DCR error, the generator also emits a [`headersHelper`](https://code.claude.com/docs/en/mcp) on the MCP server config — a command Claude Code runs on **every connection** to mint a fresh ADC token (its JSON stdout overrides the baked static header). The baked token remains as a fallback for when `gcloud`/ADC isn't available in Claude Code's environment. No action needed; if you see stale-token 401s, confirm `gcloud` is on `PATH` and ADC is valid in the run environment. + +> **Limitation:** OneMCP / Google API MCP endpoints do not support OAuth dynamic client registration. Static bearer-token auth (via `authProviderType: google_credentials`) is the only supported path — Claude Code's interactive OAuth login flow will not work against them. + ### MCP server fails with `401 Unauthorized` (real Cloud SQL endpoint) -The injected gcloud access token may have insufficient scopes. Make sure `gcloud auth application-default login` was run with `--scopes=https://www.googleapis.com/auth/cloud-platform`, and your account has the required IAM roles (e.g., `roles/cloudsql.admin`). +Usually a token problem (see the DCR entry above). Checklist: +- Run `gcloud auth application-default login` with `--scopes=https://www.googleapis.com/auth/cloud-platform`. +- Confirm your account has the required IAM roles (e.g., `roles/cloudsql.admin`). +- Set the quota project header: `headers: { X-Goog-User-Project: }`. +- Verify directly: `curl -H "Authorization: Bearer $(gcloud auth application-default print-access-token)" -H "X-Goog-User-Project: " -H "Content-Type: application/json" -H "Accept: application/json, text/event-stream" -d '{"jsonrpc":"2.0","id":1,"method":"tools/call","params":{"name":"list_instances","arguments":{"project":""}}}' https://sqladmin.googleapis.com/mcp` ### `npm exec` is slow on first run diff --git a/docs/codex_cli_agent_testing.md b/docs/codex_cli_agent_testing.md new file mode 100644 index 00000000..5489ee1f --- /dev/null +++ b/docs/codex_cli_agent_testing.md @@ -0,0 +1,637 @@ +# OpenAI Codex CLI Evaluation Guide + +This guide covers how to use EvalBench for evaluating **OpenAI Codex CLI** (`codex exec`) agent workflows using **MCP Servers** (HTTP/streamable and stdio). It includes configuration reference, evaluation dataset format, scoring metrics, and step-by-step instructions for running evaluations locally. + +The Codex CLI generator mirrors the Gemini CLI and Claude Code generators in this repo — same evalset format, same orchestrator, same scorers — so most of what you know about [Gemini CLI evaluation](./gemini_cli_agent_testing.md) and [Claude Code evaluation](./claude_code_agent_testing.md) carries over. + +--- + +## Table of Contents + +- [Overview](#overview) +- [Architecture](#architecture) +- [Prerequisites](#prerequisites) +- [Quick Start](#quick-start) +- [Configuration Reference](#configuration-reference) + - [Run Configuration](#1-run-configuration) + - [Model Configuration](#2-model-configuration) + - [Evaluation Dataset (Evalset)](#3-evaluation-dataset-evalset) +- [Authentication](#authentication) +- [MCP Servers](#mcp-servers) +- [Sandbox & Approval Modes](#sandbox--approval-modes) +- [Pricing & Cost Tracking](#pricing--cost-tracking) +- [Scorers](#scorers) +- [End-to-End Examples](#end-to-end-examples) +- [Troubleshooting](#troubleshooting) + +--- + +## Overview + +EvalBench's Codex CLI integration enables automated, multi-turn evaluation of agentic AI workflows powered by OpenAI's [Codex CLI](https://github.com/openai/codex). The CLI acts as the orchestrator that connects to MCP server backends and executes scenarios defined in an evaluation dataset. A **simulated user** powered by an LLM drives multi-turn conversations following a conversation plan. + +### Key Capabilities + +- **Multi-turn evaluation** with LLM-powered simulated users (uses `codex exec resume ` to continue prior threads) +- **API-key auth** sourced from `OPENAI_API_KEY` env or Google Secret Manager +- **Two MCP transport modes**: streamable HTTP (with Google Cloud OAuth auto-injection) and stdio +- **Pinned CLI versions** via `npm exec` (matches Gemini CLI / Claude Code) +- **Cost tracking** via configurable per-model pricing (Codex NDJSON ships tokens but not USD) +- **8 built-in scorers** covering correctness, efficiency, and behavior quality +- **CSV and BigQuery reporting** + +### What carries over from the other agent integrations + +| Aspect | Same / Different | +|---|---| +| Evalset JSON format | **Same** — `scenarios[]` with `id`, `starting_prompt`, `conversation_plan`, `expected_trajectory`, `max_turns`, `env` | +| `dataset_format` | `agent-format` | +| `orchestrator` | `agent` | +| Scorers | **Same** (`trajectory_matcher`, `goal_completion`, `behavioral_metrics`, etc.) | +| Simulated user | **Same** (`simulated_user_model_config`) | +| Reporting | **Same** (CSV / BigQuery) | +| MCP server config | **Same** Gemini-style schema (`httpUrl`, `authProviderType: google_credentials`, `headers`) — auto-translated to Codex's TOML format | +| Skills & Extensions | **Same** — supports installing skills from git repos or local directories in the sandboxed environment | + +--- + +## Architecture + +``` +┌─────────────────────────────────────────────────────────────────────┐ +│ EvalBench Pipeline │ +│ │ +│ ┌──────────────┐ ┌──────────────────┐ ┌───────────────────┐ │ +│ │ Run Config │───▶│ AgentOrchestrator│───▶│ AgentEvaluator │ │ +│ │ (YAML) │ │ │ │ │ │ +│ └──────────────┘ └──────────────────┘ └────────┬──────────┘ │ +│ │ │ +│ ┌──────────────┐ ┌──────────────────────┼──────────┐ │ +│ │ Eval Dataset│ │ Per Scenario │ │ │ +│ │ (JSON) │─────────────▶│ ▼ │ │ +│ └──────────────┘ │ ┌──────────────────────────┐ │ │ +│ │ │ CodexCliGenerator │ │ │ +│ ┌──────────────┐ │ │ ┌──────────┐ ┌────────┐ │ │ │ +│ │ Model Config │──────────────│─▶│ │ MCP / │ │Sim. │ │ │ │ +│ │ (YAML) │ │ │ │ codex CLI│ │User │ │ │ │ +│ └──────────────┘ │ │ └──────────┘ └────────┘ │ │ │ +│ │ └───────────┬──────────────┘ │ │ +│ │ │ │ │ +│ │ ▼ │ │ +│ │ ┌──────────────────────────┐ │ │ +│ │ │ Scorers (8 metrics) │ │ │ +│ │ └──────────────────────────┘ │ │ +│ └─────────────────────────────────┘ │ +│ │ +│ ┌──────────────────────────────────────────────────────────────┐ │ +│ │ Reporting (CSV / BigQuery) │ │ +│ └──────────────────────────────────────────────────────────────┘ │ +└─────────────────────────────────────────────────────────────────────┘ +``` + +**Flow:** +1. The **Run Config** ties together the dataset, model config, scorers, and reporting. +2. The **AgentOrchestrator** (`orchestrator: agent`) loads the `AgentEvaluator`. +3. The evaluator instantiates [CodexCliGenerator](../evalbench/generators/models/codex_cli.py) based on `generator: codex_cli` in the model config. +4. On startup, the generator writes a sandboxed `~/.codex/config.toml` (with translated MCP servers) and `~/.codex/auth.json` (with the API key) into `.venv/fake_home_codex/` so the host machine's `~/.codex` is not touched. +5. For each scenario, the evaluator runs a multi-turn loop: + - Sends the starting prompt to Codex via `codex exec --json --skip-git-repo-check ... ` + - A **SimulatedUser** (LLM) generates realistic follow-up responses + - Subsequent turns use `codex exec resume ` to continue the same Codex thread + - Tools and stats are accumulated across turns from the NDJSON ThreadEvent stream + - Conversation continues until `max_turns` is reached or the simulated user sends `TERMINATE` +6. Results are scored and written to CSV and/or BigQuery. + +--- + +## Prerequisites + +1. **Python 3.10+** and project dependencies installed +2. **Node.js and npm** (for running Codex CLI via `npm exec`) +3. **Codex CLI** — either: + - Globally installed: `npm install -g @openai/codex` (then use `codex_cli_version: "codex"`), or + - Pinned version (recommended for reproducibility): `codex_cli_version: "@openai/codex@latest"` (or a specific version like `"@openai/codex@0.30.0"`) — `npm exec --yes` will install it on first use +4. **OpenAI API key** — see [Authentication](#authentication). The key must be supplied in the model config (or env). Codex's ChatGPT-OAuth fallback is **not** usable from CI — accounts without ChatGPT-Plus get `402 deactivated_workspace`. +5. **Environment variables** for the simulated user / scorer model: + ```bash + export EVAL_GCP_PROJECT_ID=your_project_id + export EVAL_GCP_PROJECT_REGION=us-central1 + ``` +6. **gcloud** (only if any MCP server uses `authProviderType: google_credentials` — the generator shells out to `gcloud auth application-default print-access-token`; run `gcloud auth application-default login` first) + +--- + +## Quick Start + +### 1. Choose a run config + +```bash +# Real MCP server (Cloud SQL Admin API): +export EVAL_CONFIG=datasets/codex-cli-tools/example_run_config.yaml + +# Fake MCP (offline testing, deterministic): +export EVAL_CONFIG=datasets/codex-cli-tools/example_run_fake_config.yaml +``` + +### 2. Run the evaluation + +```bash +./evalbench/run.sh +``` + +Results land in `results//` as CSV files. + +--- + +## Configuration Reference + +### 1. Run Configuration + +The top-level config that ties everything together. **Identical** to the other agent run configs. + +| Key | Required | Description | +|-----|----------|-------------| +| `dataset_config` | Yes | Path to the evalset JSON file | +| `dataset_format` | Yes | `agent-format` | +| `orchestrator` | Yes | `agent` | +| `model_config` | Yes | Path to the Codex CLI model config YAML | +| `simulated_user_model_config` | Yes | Path to the model config for the simulated user LLM | +| `scorers` | Yes | Dictionary of scorer configurations | +| `runners.agent_runners` | Optional | Concurrency (default `10`). Set to `1` for sequential runs — recommended for Codex because all scenarios share the same sandboxed `~/.codex` store. | +| `reporting` | Optional | CSV and/or BigQuery output options | + +**Example** ([example_run_config.yaml](../datasets/codex-cli-tools/example_run_config.yaml)): + +```yaml +dataset_config: datasets/codex-cli-tools/codex-cli.evalset.json +dataset_format: agent-format + +orchestrator: agent +model_config: datasets/model_configs/codex_cli_model.yaml +simulated_user_model_config: datasets/model_configs/gemini_2.5_pro_model.yaml + +# Run scenarios sequentially. Codex shares one ~/.codex/config.toml across +# the runner pool; serial runs avoid session-id collisions on `codex exec resume`. +runners: + agent_runners: 1 + +scorers: + trajectory_matcher: {} + goal_completion: + model_config: datasets/model_configs/gemini_2.5_pro_model.yaml + behavioral_metrics: + model_config: datasets/model_configs/gemini_2.5_pro_model.yaml + parameter_analysis: + model_config: datasets/model_configs/gemini_2.5_pro_model.yaml + turn_count: {} + end_to_end_latency: {} + tool_call_latency: {} + token_consumption: {} + +reporting: + csv: + output_directory: 'results' +``` + +--- + +### 2. Model Configuration + +The model config defines the Codex CLI version, model, auth, sandbox/approval policy, environment, MCP server setup, and pricing. + +#### Common Fields + +| Key | Required | Description | +|-----|----------|-------------| +| `codex_cli_version` | Yes | Either `"codex"` (uses the globally installed binary) or an npm spec like `"@openai/codex@latest"` / `"@openai/codex@0.30.0"` (uses `npm exec --yes`) | +| `generator` | Yes | Must be `codex_cli` | +| `model` | Yes | Model id passed to `codex exec -m ` (e.g., `"gpt-5.5"`, `"o4-mini"`, `"gpt-4.1"`) | +| `openai_api_key_secret` | Optional | Google Secret Manager resource path for the API key. Bare form `projects/.../secrets/.../versions/` or `secret_manager://projects/.../secrets/.../versions/` URL form. **Numeric version required — `latest` not supported.** | +| `env.OPENAI_API_KEY` | Optional | Direct API key. Also accepts a Secret Manager path here (auto-detected and resolved). Prefer `openai_api_key_secret` or shell env over hardcoding. | +| `sandbox_mode` | Optional | `"danger-full-access"` (default — passes `--dangerously-bypass-approvals-and-sandbox`), or any value Codex's `--sandbox` flag accepts (e.g., `"workspace-write"`, `"read-only"`) | +| `approval_mode` | Optional | Forwarded to `--ask-for-approval` when `sandbox_mode != "danger-full-access"`. Default `"never"`. | +| `profile` | Optional | Codex profile name (forwarded as `--profile `) | +| `json_flag` | Optional | `"--json"` (default, newer Codex versions) or `"--experimental-json"` (older versions). Codex requires NDJSON for the eval pipeline to extract tool calls and tokens. | +| `pricing` | Optional | Per-model rates used to compute `cost_usd` per turn. See [Pricing & Cost Tracking](#pricing--cost-tracking). | +| `env` | Optional | Environment variables passed to the CLI process (e.g., `GOOGLE_CLOUD_PROJECT` for Cloud SQL MCP) | +| `setup.mcp_servers` | Optional | MCP server configurations (see [MCP Servers](#mcp-servers)) | +| `setup.config` | Optional | Free-form key/value pairs written to the top of `~/.codex/config.toml`. Merged on top of the default `forced_login_method = "api"`. | + +**Example** ([codex_cli_model.yaml](../datasets/model_configs/codex_cli_model.yaml)): + +```yaml +codex_cli_version: "@openai/codex@latest" +generator: codex_cli +model: "gpt-5.5" + +openai_api_key_secret: + +pricing: + input_per_million_usd: 1.25 + cached_input_per_million_usd: 0.125 + output_per_million_usd: 10.0 + +env: + GOOGLE_CLOUD_PROJECT: "astana-evaluation" + GOOGLE_CLOUD_LOCATION: "us-central1" + +setup: + mcp_servers: + "cloud-sql": + httpUrl: "https://sqladmin.googleapis.com/mcp" + authProviderType: google_credentials + headers: + X-Goog-User-Project: astana-evaluation +``` + +--- + +### 3. Evaluation Dataset (Evalset) + +**Identical schema** to the Gemini CLI / Claude Code evalset. See [Gemini CLI doc — Evalset](./gemini_cli_agent_testing.md#3-evaluation-dataset-evalset) for full details, including the canonical [tool name format](./gemini_cli_agent_testing.md#tool-name-format) used in `expected_trajectory`. + +Minimal example ([codex-cli.evalset.json](../datasets/codex-cli-tools/codex-cli.evalset.json)): + +```json +{ + "scenarios": [ + { + "id": "cloud-sql-list-instances-01", + "starting_prompt": "list all Cloud SQL instances in project astana-evaluation", + "conversation_plan": "Ask the agent to list instances in project astana-evaluation. Once all instances are listed if nl2code exists get its state and validate it is RUNNABLE.", + "expected_trajectory": ["cloud-sql__list_instances", "cloud-sql__get_instance"], + "env": { "GOOGLE_CLOUD_PROJECT": "astana-evaluation" }, + "kind": "tools", + "max_turns": 3 + } + ] +} +``` + +--- + +## Authentication + +Codex CLI uses an OpenAI API key. The generator resolves it in this order, then writes it to `~/.codex/auth.json` (the same file `codex login --api-key` produces) so `codex exec` picks it up: + +1. **`openai_api_key_secret`** in the model config — a Google Secret Manager resource path. Bare form `projects//secrets//versions/` or `secret_manager://projects/...` URL form. Requires a numeric version (no `latest`). +2. **`env.OPENAI_API_KEY`** in the model config, or **`OPENAI_API_KEY`** in the shell env. If the value itself looks like a Secret Manager path, it's resolved transparently. + +```yaml +# Option 1: Secret Manager (recommended for shared / CI environments) +openai_api_key_secret: + +# Option 2: shell env (recommended for local dev) +# export OPENAI_API_KEY=sk-... +env: {} + +# Option 3: inline (avoid — gets committed) +env: + OPENAI_API_KEY: "sk-..." +``` + +The generator writes `/.codex/auth.json` with `{"auth_mode": "apikey", "OPENAI_API_KEY": ""}` and chmods it to `0600`. It also sets `forced_login_method = "api"` in `config.toml` so Codex never tries the ChatGPT-OAuth flow. + +> **Why not just rely on the env var?** Codex's auth manager only honors `OPENAI_API_KEY` when an internal `enable_codex_api_key_env` flag is set. For `codex exec` the canonical path is `auth.json`, so the generator writes both. + +--- + +## MCP Servers + +EvalBench accepts the **same MCP server config schema as Gemini CLI and Claude Code**. The Codex generator auto-translates it into Codex's TOML schema at runtime (see [_translate_mcp_config](../evalbench/generators/models/codex_cli.py)): + +| Gemini-style field | Codex translation | +|---|---| +| `httpUrl` | → `url` (TOML, streamable HTTP server) | +| `headers` | → `http_headers` (TOML inline table) | +| `authProviderType: google_credentials` | → mints an **ADC** token (`gcloud auth application-default print-access-token`, falling back to `gcloud auth print-access-token`) and passes it to Codex via `bearer_token_env_var` (env var `EVALBENCH_GCLOUD_MCP_TOKEN`). The generator re-mints a fresh token before **every turn** (each `codex exec` re-reads the env var), so it doesn't expire mid-suite. `X-Goog-User-Project` stays a static `http_headers` entry. Google API MCP endpoints reject the plain user token on tool calls — see [Troubleshooting](#mcp-server-fails-with-401-unauthorized-real-cloud-sql-endpoint). | +| `oauth.scopes` | (dropped — Codex doesn't use Gemini's OAuth delegation) | +| `command` / `args` / `env` / `cwd` (stdio) | → passed through as-is into a `[mcp_servers.NAME]` stdio block | + +### HTTP MCP server (Cloud SQL Managed) + +```yaml +setup: + mcp_servers: + "cloud-sql": + httpUrl: "https://sqladmin.googleapis.com/mcp" + authProviderType: google_credentials + headers: + X-Goog-User-Project: astana-evaluation +``` + +This generates the following block in `~/.codex/config.toml`: + +```toml +forced_login_method = "api" + +[mcp_servers.cloud-sql] +url = "https://sqladmin.googleapis.com/mcp" +http_headers = { "X-Goog-User-Project" = "astana-evaluation", "Authorization" = "Bearer ya29...." } +``` + +### Stdio MCP server + +```yaml +setup: + mcp_servers: + "cloud-sql": + command: "python" + args: + - "evalbench/util/fake_mcp_server.py" + - "--server-name" + - "cloud-sql" + - "--config" + - "datasets/model_configs/codex_cli_fake_model.yaml" +``` + +→ TOML: + +```toml +[mcp_servers.cloud-sql] +command = "python" +args = ["evalbench/util/fake_mcp_server.py", "--server-name", "cloud-sql", "--config", "datasets/model_configs/codex_cli_fake_model.yaml"] +``` + +### How it works under the hood + +1. `CodexCliGenerator._setup` writes the translated config to `/.codex/config.toml` +2. `_write_codex_auth_json` writes the API key to `/.codex/auth.json` +3. The CLI is invoked with `HOME=` so it loads only the configured servers (no host-machine pollution) +4. Each scenario runs in a sandboxed `HOME` (`.venv/fake_home_codex/` locally, `/tmp_sessions//fake_home` in gRPC mode) + +--- + +## Skills & Extensions + +Codex CLI evaluations support **Skills** and **Extensions** (plugins). These are installed during the setup phase into the sandboxed `~/.codex` directory. + +### Configuration + +You can specify skills to install in the `setup` section of your model configuration: + +```yaml +setup: + skills: + - action: install_from_repo + url: "https://github.com/gemini-cli-extensions/cloud-sql-postgresql.git" +``` + +The generator will: +1. Clone the repository into `/.codex/plugins/` +2. Register the plugin in `/.codex/plugins/marketplace.json` +3. Install the individual skills into `/.codex/skills/` + +These skills are then available for the Codex agent to use during the evaluation turns. + +--- + +## Sandbox & Approval Modes + +Codex CLI sandboxes file/network access by default and prompts for approval before tool calls. For automated evaluation, the generator disables both by default — equivalent to Gemini CLI's `--yolo` and Claude Code's `--dangerously-skip-permissions`. + +| `sandbox_mode` value | Behavior | Resulting flags | +|---|---|---| +| `"danger-full-access"` (default) | Fully bypass sandbox + approvals | `--dangerously-bypass-approvals-and-sandbox` | +| `"workspace-write"` | Allow writes only inside the working tree | `--sandbox workspace-write --ask-for-approval ` | +| `"read-only"` | No writes anywhere | `--sandbox read-only --ask-for-approval ` | + +`approval_mode` (default `"never"`) is forwarded as `--ask-for-approval ` whenever `sandbox_mode` is not `danger-full-access`. Use `"never"` for unattended runs; any prompt-driven mode will hang the evaluator. + +```yaml +# Default (most permissive — recommended for CI) +sandbox_mode: "danger-full-access" + +# Stricter — block writes outside repo, never prompt +sandbox_mode: "workspace-write" +approval_mode: "never" +``` + +--- + +## Pricing & Cost Tracking + +Codex's NDJSON includes `usage.input_tokens`, `usage.output_tokens`, and `usage.cached_input_tokens` per turn but **does not** include cost. Provide a `pricing` block in the model config and the generator will compute `cost_usd` per turn for the `token_consumption` scorer. + +Two equivalent forms are accepted; pick whichever is easier to copy from OpenAI's pricing page. + +**Per-million form (matches the OpenAI pricing page):** + +```yaml +pricing: + input_per_million_usd: 1.25 + cached_input_per_million_usd: 0.125 # optional; defaults to 10% of input + output_per_million_usd: 10.0 +``` + +**Per-token form:** + +```yaml +pricing: + input_per_token_usd: 0.00000125 + cached_input_per_token_usd: 0.000000125 + output_per_token_usd: 0.00001 +``` + +**Cost formula:** + +``` +billable_input = max(0, input_tokens - cached_input_tokens) +cost_usd = (billable_input * input_rate) + + (cached_input_tokens * cached_input_rate) + + (output_tokens * output_rate) +``` + +If `pricing` is missing or malformed, `cost_usd` is reported as `0.0` (rather than guessing). Update the rates whenever you change `model:` — pricing differs by model. + +--- + +## Scorers + +**Identical** to the Gemini CLI / Claude Code scorers. See [Gemini CLI doc — Scorers](./gemini_cli_agent_testing.md#scorers) for the full list. + +Quick reference: + +| Scorer | Type | Description | +|---|---|---| +| `trajectory_matcher` | Deterministic | Jaccard or Levenshtein match between expected and actual tool trajectory. Native Codex tools (`shell`, file ops, ...) are dropped from both sides by default — set `filter_native_tools: false` to score them too. | +| `goal_completion` | LLM | Did the agent accomplish the conversation plan? | +| `behavioral_metrics` | LLM | Hallucination rate + clarification rate | +| `parameter_analysis` | LLM | Qualitative feedback on tool parameters | +| `turn_count` | Deterministic | Number of conversation turns | +| `end_to_end_latency` | Deterministic | Total wall-clock latency of the `codex exec` subprocess | +| `tool_call_latency` | Deterministic | Sum of per-tool durations measured between `item.started` and `item.completed` arrival times (Codex events carry no timestamps, so the generator stamps arrival in-process) | +| `token_consumption` | Deterministic | Total input + output + cached tokens, plus `cost_usd` from the `pricing` block | + +The Codex generator extracts tool calls from the following NDJSON ThreadItem kinds: `mcp_tool_call`, `command_execution` (reported as `shell`), `web_search`, and `file_change`. + +--- + +## End-to-End Examples + +### Example 1: Real Cloud SQL Managed MCP + +**Goal**: Use Codex to manage Cloud SQL instances via the public Cloud SQL Admin MCP. + +```yaml +# datasets/model_configs/codex_cli_model.yaml +codex_cli_version: "@openai/codex@latest" +generator: codex_cli +model: "gpt-5.5" + +openai_api_key_secret: "projects/393137573/secrets/OPENAI_API_KEY/versions/1" + +pricing: + input_per_million_usd: 1.25 + cached_input_per_million_usd: 0.125 + output_per_million_usd: 10.0 + +env: + GOOGLE_CLOUD_PROJECT: "astana-evaluation" + +setup: + mcp_servers: + "cloud-sql": + httpUrl: "https://sqladmin.googleapis.com/mcp" + authProviderType: google_credentials + headers: + X-Goog-User-Project: astana-evaluation +``` + +Run: + +```bash +gcloud auth login # for the OpenAI Secret Manager fetch +gcloud auth application-default login # for the Cloud SQL MCP token +export EVAL_GCP_PROJECT_ID=astana-evaluation +export EVAL_CONFIG=datasets/codex-cli-tools/example_run_config.yaml +./evalbench/run.sh +``` + +### Example 2: Offline / fake MCP (deterministic) + +Use the bundled fake MCP server when you want to exercise the agent without any network calls — useful in CI or when iterating on a scenario file. + +```bash +export EVAL_CONFIG=datasets/codex-cli-tools/example_run_fake_config.yaml +./evalbench/run.sh +``` + +This config ([example_run_fake_config.yaml](../datasets/codex-cli-tools/example_run_fake_config.yaml)) launches `evalbench/util/fake_mcp_server.py` over stdio with the canned tool responses defined under `fake_mcp_tools` in [codex_cli_fake_model.yaml](../datasets/model_configs/codex_cli_fake_model.yaml). + +### Example 3: Local dev with `OPENAI_API_KEY` from your shell + +Skip Secret Manager entirely: + +```yaml +codex_cli_version: "@openai/codex@latest" +generator: codex_cli +model: "gpt-5.5" +# no openai_api_key_secret — OPENAI_API_KEY picked up from shell env +setup: + mcp_servers: + "cloud-sql": + httpUrl: "https://sqladmin.googleapis.com/mcp" + authProviderType: google_credentials + headers: + X-Goog-User-Project: astana-evaluation +``` + +```bash +export OPENAI_API_KEY=sk-... +export EVAL_CONFIG=datasets/codex-cli-tools/example_run_config.yaml +./evalbench/run.sh +``` + +--- + +## Troubleshooting + +### `402 deactivated_workspace` / "ChatGPT-Plus required" + +Codex fell back to ChatGPT-OAuth because no API key was found. Check: +- `openai_api_key_secret` resolves successfully (look for `Failed to fetch OPENAI_API_KEY from Secret Manager` in logs) +- Or `OPENAI_API_KEY` is set in your shell / model config `env` +- The generator logs `Codex API key resolved (length=N) and written to .../auth.json` on a successful resolve. If you see `Codex API key could not be resolved`, fix that first. + +### `error: unexpected argument '--json' found` (or similar) + +Your installed Codex predates the `--json` flag. Either upgrade (`codex_cli_version: "@openai/codex@latest"`) or fall back to the experimental flag: + +```yaml +json_flag: "--experimental-json" +``` + +### `Error: command not found: codex` / `npm: command not found` + +- For pinned versions (`@openai/codex@...`): make sure `node` and `npm` are on `PATH`. `npm exec --yes` will download Codex on first use. +- For `codex_cli_version: "codex"`: install globally with `npm install -g @openai/codex`. + +### MCP server fails with `401 Unauthorized` (real Cloud SQL endpoint) + +The injected token is missing, expired, the **wrong kind**, or your principal lacks IAM access. Note Google API MCP endpoints leave `initialize`/`tools/list` unauthenticated but require a valid bearer token on the actual **tool call**, and they only accept an **ADC** token — the plain gcloud *user* token (`gcloud auth print-access-token`) is rejected. The generator now injects the ADC token (`gcloud auth application-default print-access-token`); make sure: +- You ran **`gcloud auth application-default login`** (with `--scopes=https://www.googleapis.com/auth/cloud-platform`) — not just `gcloud auth login`. +- Your account / service account has the required IAM roles (e.g., `roles/cloudsql.admin`). +- `X-Goog-User-Project` header points at a project that has the Cloud SQL Admin API enabled. +- Token expiry is handled automatically: the generator mints a fresh ADC token into `EVALBENCH_GCLOUD_MCP_TOKEN` before every turn and Codex reads it via `bearer_token_env_var`, so long suites don't hit stale-token 401s. (Requires a Codex build that supports `bearer_token_env_var`; if yours doesn't, the token won't be applied — fall back to a static `http_headers` `Authorization`.) + +### `Invalid TOML` when Codex starts + +Either a manual edit to `~/.codex/config.toml` clobbered the generated file, or a hand-written `setup.config` value contains an unsupported type. Fix: + +```bash +rm -rf .venv/fake_home_codex +``` + +…and re-run. The generator regenerates `config.toml` on every invocation. + +### `[error] thread not found` events in stdout + +These show up as `[error] ...` in the response field when `codex exec resume` is called against a thread that Codex couldn't write to. The matching `failed to record rollout items: thread not found` lines on stderr are scrubbed by the generator (see `_STDERR_NOISE_PATTERNS`). Causes: +- Two scenarios racing on the same fake `HOME` — set `runners.agent_runners: 1`. +- A previous run left a corrupt `~/.codex/sessions/` — `rm -rf .venv/fake_home_codex/.codex/sessions` and re-run. + +### Empty/zero results in the summary + +The simulated user failed to initialize. Check `EVAL_GCP_PROJECT_ID` is set if your simulated user model uses Vertex AI: + +```bash +export EVAL_GCP_PROJECT_ID=astana-evaluation +``` + +### `cost_usd` is always `0.0` + +Either no `pricing` block was provided, or it's malformed. Check the model-config YAML and look for `Codex pricing config missing input/output rates; cost_usd will be 0.` in the logs. Pricing keys must be strict `_per_million_usd` or `_per_token_usd` — typos silently fall through. + +### Scenarios appear to run in interleaved order + +This is **expected** when `agent_runners > 1`. Codex shares one `~/.codex/config.toml` and one `auth.json` across the runner pool, so concurrent scenarios are safe **but** their NDJSON streams will interleave in the eval logs. To force sequential execution: + +```yaml +runners: + agent_runners: 1 +``` + +### `npm exec` is slow on first run + +`npm exec --yes @openai/codex@` downloads the package on first use (~30–60 sec). Subsequent runs hit the npm cache. + +### Cleaning the sandbox + +If a previous run left bad state: + +```bash +rm -rf .venv/fake_home_codex +``` + +The generator recreates `~/.codex/config.toml` and `~/.codex/auth.json` on the next invocation. + +--- + +## See Also + +- [Gemini CLI Evaluation Guide](./gemini_cli_agent_testing.md) — sister doc, shares most concepts +- [Claude Code Evaluation Guide](./claude_code_agent_testing.md) — sister doc, shares most concepts +- [Codex CLI source / docs](https://github.com/openai/codex) — official CLI reference +- [Model Context Protocol (MCP)](https://modelcontextprotocol.io/) — protocol used by tool servers +- [CodexCliGenerator implementation](../evalbench/generators/models/codex_cli.py) diff --git a/docs/configs/dataset-config.md b/docs/configs/dataset-config.md index 44157576..ef47da3a 100644 --- a/docs/configs/dataset-config.md +++ b/docs/configs/dataset-config.md @@ -24,6 +24,7 @@ Each evaluation item in the JSON file includes the following keys: ## Important Notes - **Multiple Dialects:** Each SQL-related key (`golden_sql`, `eval_query`, `setup_sql`, `cleanup_sql`) maps dialects to their corresponding queries. This ensures that the evaluation items can be used across different database systems. +- **Hybrid Engine Evaluations (Cross-Database Mode):** When running hybrid cross-database benchmarks (e.g. generating and executing queries on BigQuery but validating them against SQLite references), the `golden_sql` queries must map to the execution engine's dialect (e.g., `"bigquery"`), even if those reference queries themselves are written in SQLite syntax. - **Custom Metadata:** The `other` field is optional and can contain any additional information you deem necessary for reporting or contextual purposes. - **Structured Testing:** By defining separate SQL statements for setup, evaluation, and cleanup, this configuration supports robust testing of DDL operations. diff --git a/docs/configs/db-config.md b/docs/configs/db-config.md index 83bbc244..d29fe5cd 100644 --- a/docs/configs/db-config.md +++ b/docs/configs/db-config.md @@ -15,8 +15,8 @@ The YAML configuration file is structured as follows: | `database_name` | Yes | The name of your database that is used for the default connection. This can be the default admin database (i.e. `postgres`) on the instance. This DB is only used to create databases needed for running evaluations. | | `database_path` | Yes | The path or instance reference to your database (e.g., cloud instance path, local path). Please see note above on database_path for more information on SQLAlchemy with more instructions on how to connect to local or GCP databases. *NOTE: For Sqlite, database_path is the directory that the .db files are found or stored in.* | | `max_executions_per_minute` | No | Optional throttle limit for the number of executions per minute. | -| `user_name` | Conditionally (if needed) | Required only for databases that need authentication (e.g., MySQL, PostgreSQL). Not needed for databases like SQLite. | -| `password` | Conditionally (if needed) | The password for the database. Can be interchanged with `secret_manager_path` if you prefer using GCP Secret Manager for secure storage. | +| `user_name` | Conditionally (if needed) | Required only for databases that need authentication (e.g., MySQL, PostgreSQL). Not needed for databases like SQLite or when using IAM authentication (username derived from ADC). | +| `password` | Conditionally (if needed) | The password for the database. Can be interchanged with `secret_manager_path` if you prefer using GCP Secret Manager for secure storage. Omit if using IAM authentication. | | `secret_manager_path` | No (alternative to password) | An alternative to `password` that specifies the path to your secret in GCP Secret Manager. Use this if you prefer not to store the password directly. | | `extension` | Conditionally (if needed) | Required only for SQLite when datasets do not use the default `.db` extension. | | `location` | Conditionally (if needed) | Specifies the location of your dataset. Required for BigQuery. Default is "US".| @@ -29,6 +29,7 @@ The YAML configuration file is structured as follows: - **Authentication:** - For databases that require authentication (e.g., MySQL), provide both `user_name` and either `password` or `secret_manager_path`. - For databases like SQLite, omit `user_name` and `password`. + - For IAM authentication (e.g., AlloyDB, CloudSQL), omit `user_name` and `password`. The user name will be derived from Application Default Credentials (ADC). - **Optional Parameters:** - `max_executions_per_minute` is optional and can be adjusted according to your application's needs. - **Secret Management:** diff --git a/docs/configs/run-config.md b/docs/configs/run-config.md index 555adf19..87dcd95b 100644 --- a/docs/configs/run-config.md +++ b/docs/configs/run-config.md @@ -14,6 +14,9 @@ This section defines the primary resources used during evaluation, including the | `databases` | Optional | Specifies the databases (e.g., `db_blog`, `california_schools`, etc.). This filters the dataset to the provided list of databases and ignores all other evals. If not provided, all databases found in the dataset_config json file will be tried. | | `query_types` | Optional | Specifies the query_types (`dql`, `dml`, `dd`). This filters the dataset to the list of evals that are of the query_types provided. If not provided, all eval types (dql, dml and ddl) found in the dataset_config json file will be tried. | | `dataset_format` | Conditional (if needed) | Defines the dataset format, with `evalbench-standard-format` as the default. For BIRD datasets, it must be set to `bird-standard-format`.| +| `num_trials` | Optional | Number of trials to run for each prompt. | +| `scenarios` | Optional | A list of specific scenario IDs to run (only applies to scenario-based agentic datasets like `gemini-cli-format` or `cortado-format`). Defaults to empty (runs all scenarios). | +| `scenario_pattern` | Optional | A glob pattern of scenario IDs to run (only applies to scenario-based agentic datasets). Defaults to None (runs all scenarios). | --- ## 2. Prompt and Generation Modules @@ -69,14 +72,17 @@ setup_directory/ The `scorers` section defines various scoring strategies to evaluate the quality of the generated SQL queries. Each scorer applies a different metric or comparison strategy. -| **Scorer Key** | **Required** | **Description** | -| ----------------- | ------------ | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `exact_match` | Optional | Evaluates whether the generated SQL query result exactly matches the expected (golden) query result. | -| `returned_sql` | Optional | Checks that the generated output contains valid SQL code rather than just comments. | -| `regexp_matcher` | Optional | Uses regular expressions to determine if the generated query satisfies specific patterns.

**Run Configuration Options:**
- `regexp_string_list` (required): A list of regex patterns to match against the generated query.
- `invert_results` (Optional, default: `False`): When set to true, non-matching queries score 100 and matching queries score 0.
- `match_all_patterns` (Optional, default: `False`): If true, a score of 100 is given only if all regex patterns are matched; otherwise, a match with at least one pattern suffices.
- `match_whole_query` (Optional, default: `False`): When true, forces the pattern to match the entire query rather than a substring. | -| `llmrater` | Optional | Compares the execution results of the golden SQL query with those produced by the model. It scores 100 for concrete positive cases, such as mismatches in column names or extra columns in the generated SQL. This scorer requires its own `model_config` for proper operation. | -| `recall_match` | Optional | Computes the precision and recall by comparing the generated and expected results, ignoring `None` and duplicate values. The default scoring mode is based on recall, where matching results are compared against the expected outputs regardless of their order. | -| `set_match` | Optional | Measures the execution accuracy by comparing the results of the golden query execution with those of the generated query, as defined by the BIRD methodology. | +| **Scorer Key** | **Required** | **Description** | +| :--- | :--- | :--- | +| `exact_match` | Optional | Evaluates whether the generated SQL query result exactly matches the expected (golden) query result. | +| `returned_sql` | Optional | Checks that the generated output contains valid SQL code rather than just comments. | +| `regexp_matcher` | Optional | Uses regular expressions to determine if the generated query satisfies specific patterns.

**Run Configuration Options:**
- `regexp_string_list` (required): A list of regex patterns to match against the generated query.
- `invert_results` (Optional, default: `False`): When set to true, non-matching queries score 100 and matching queries score 0.
- `match_all_patterns` (Optional, default: `False`): If true, a score of 100 is given only if all regex patterns are matched; otherwise, a match with at least one pattern suffices.
- `match_whole_query` (Optional, default: `False`): When true, forces the pattern to match the entire query rather than a substring. | +| `llmrater` | Optional | Compares the execution results of the golden SQL query with those produced by the model. It scores 100 for concrete positive cases, such as mismatches in column names or extra columns in the generated SQL. This scorer requires its own `model_config` for proper operation.

**Run Configuration Options:**
- `hybrid_ground_truth` (Optional, default: `False`): When set to true, if the reference (golden) query execution fails on the target BigQuery engine, it dynamically falls back to resolve the correct reference rows from the local SQLite database file. | +| `python_scorer` | Optional | A generic scorer that executes an external Python script in an isolated sandbox (`uv run --isolated`) to perform custom evaluation logic.

**Run Configuration Options:**
- `script_path` (Required): Path to the Python evaluation script (e.g. `evalbench/scorers/judges/hybrid_xa_judge.py`).
- `scorer_name` (Optional): A custom name for the scorer instance (e.g. `hybrid_cross_db`).

**Included Hybrid Evaluator (`hybrid_xa_judge.py`):**
When set to `evalbench/scorers/judges/hybrid_xa_judge.py`, this operates as a cross-database Execution Accuracy (XA) judge. It compares BigQuery execution results against SQLite references by applying strict cell normalization rules: (1) rounding float values to 4 decimal places, (2) sorting rows lexicographically, (3) stripping trailing `.0` string suffixes, and (4) ignoring column headers. | +| `recall_match` | Optional | Computes the precision and recall by comparing the generated and expected results, ignoring `None` and duplicate values. The default scoring mode is based on recall, where matching results are compared against the expected outputs regardless of their order. | +| `set_match` | Optional | Measures the execution accuracy by comparing the results of the golden query execution with those of the generated query, as defined by the BIRD methodology. | +| `exact_match_consistency` | Optional | Evaluates consistency across multiple trials using exact match on execution results. Multiple trials are aggregated at the prompt level using a strict "All-or-Nothing" rule—the prompt is deemed consistent only if ALL trial pairs are consistent. | +| `llm_consistency` | Optional | Evaluates consistency across multiple trials using an LLM to compare results and errors. Requires `model_config`. Multiple trials are aggregated at the prompt level using a strict "All-or-Nothing" rule—the prompt is deemed consistent only if ALL trial pairs are consistent. | ## 5. Reporting Configurations @@ -86,7 +92,7 @@ The `reporting` section specifies how and where the evaluation results will be r | ---------- | ------------ | ---------------------------------------------------------------------------------------------------------------------------------------------------------------- | | `truncate_execution_outputs`| Optional (defaults to 250 rows) | This allows overriding the truncation of outputs in reporting (CSVs, BQ) to the number of rows specified. This affects the following reporting fields: `generated_result`, `golden_result`, `golden_eval_results` `eval_results`. This prevents logging incredibly large results with potentially thousands or millions of rows. *NOTE: This does not affect any logic other than reporting.* | | `csv` | Optional | Configuration for CSV reporting.
**Subkey:** `output_directory` specifies the directory where CSV results will be saved (e.g., `'results'`). | -| `bigquery` | Optional | Configuration for reporting to Google BigQuery.
**Subkey:** `gcp_project_id` specifies the Google Cloud Project ID for BigQuery integration (e.g., `my_cool_gcp_project`). | +| `bigquery` | Optional | Configuration for reporting to Google BigQuery.
**Subkeys:** `gcp_project_id` specifies the Google Cloud Project ID for BigQuery integration (e.g., `my_cool_gcp_project`). `dataset_name` overrides the BigQuery dataset that results are written to (`.`), defaulting to `evalbench`. | --- > bigquery project_id: You can globally set your GCP project_id using the environment variables `EVAL_GCP_PROJECT_ID` or identify it separately. @@ -135,4 +141,5 @@ reporting: output_directory: 'results' bigquery: gcp_project_id: my_cool_gcp_project + dataset_name: evalbench # Optional, defaults to 'evalbench' ``` diff --git a/docs/examples/GCP_CloudSQL_Example.ipynb b/docs/examples/GCP_CloudSQL_Example.ipynb index a6e11d1a..f43b0825 100644 --- a/docs/examples/GCP_CloudSQL_Example.ipynb +++ b/docs/examples/GCP_CloudSQL_Example.ipynb @@ -1,443 +1,470 @@ { - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "id": "V5Bx7tBafUEP" - }, - "source": [ - "[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/GoogleCloudPlatform/evalbench/blob/main/docs/examples/GCP_CloudSQL_Example.ipynb)\n", - "\n", - "# Getting Started With Evalbench\n", - "\n", - "EvalBench is a flexible framework designed to measure the quality of generative AI (GenAI) workflows around database specific tasks. As of now, it provides a comprehensive set of tools, and modules to evaluate models on NL2SQL tasks, including capability of running and scoring DQL, DML, and DDL queries across multiple supported databases. Its modular, plug-and-play architecture allows you to seamlessly integrate custom components while leveraging a robust evaluation pipeline, result storage, scoring strategies, and dashboarding capabilities." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "6GKI9_72faY9" - }, - "source": [ - "## Quick Start Example w/ GCP CloudSQL offerings (MySQL, Postgres, SQL Server)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "ZWK6-XVs116G" - }, - "source": [ - "### Before You Begin\n", - "\n", - "To run this notebook, you will need to do the following:\n", - "\n", - "* [Create a Google Cloud Project](https://developers.google.com/workspace/guides/create-project)\n", - "* MySQL\n", - " * [Create a Cloud SQL for MySQL instance](https://cloud.google.com/sql/docs/mysql/create-instance)\n", - " * [Create a MySQL Database](https://cloud.google.com/sql/docs/mysql/create-manage-databases)\n", - " * [Create an evalbench user](https://cloud.google.com/sql/docs/mysql/**create**-manage-users)\n", - "* PostgreSQL\n", - " * [Create a Cloud SQL for PostgreSQL instance](https://cloud.google.com/sql/docs/postgres/create-instance)\n", - " * [Create a PostgreSQL Database](https://cloud.google.com/sql/docs/postgres/create-manage-databases)\n", - " * [Create an evalbench user](https://cloud.google.com/sql/docs/postgres/create-manage-users)\n", - "* SQL Server\n", - " * [Create a Cloud SQL for SQL server instance](https://cloud.google.com/sql/docs/sqlserver/create-instance)\n", - " * [Create a SQL Server Database](https://cloud.google.com/sql/docs/sqlserver/create-manage-databases)\n", - " * [Create an evalbench user](https://cloud.google.com/sql/docs/sqlserver/create-manage-users)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "zXXvsHVi7R5M" - }, - "source": [ - "\n", - "**After confirmed access to database in the runtime environment of this notebook, fill in the database details below and proceed to run the script**" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "lVL_YPR46njA" - }, - "outputs": [], - "source": [ - "GCP_PROJECT_ID = \"\" # @param {\"type\":\"string\",\"placeholder\":\"your_gcp_project_id\"}\n", - "\n", - "DATABASE_TYPE = \"\" # @param [\"mysql\",\"postgres\",\"sqlserver\"]\n", - "DATABASE_INSTANCE_REGION = \"\" # @param {\"type\":\"string\",\"placeholder\":\"your_gcp_project_region\"}\n", - "DATABASE_NAME = \"\" # @param {\"type\":\"string\",\"placeholder\":\"your_database_name\"}\n", - "DATABASE_USERNAME = \"\" # @param {\"type\":\"string\",\"placeholder\":\"your_database_username\"}\n", - "DATABASE_PASSWORD = \"\" # @param {\"type\":\"string\",\"placeholder\":\"your_database_password\"}" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "9U5aGmLW5gip" - }, - "source": [ - "### 1. Set up the environment" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "6-4In2FAz1T5" - }, - "source": [ - "#### 1. Clone the EvalBench repository from GitHub:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "EFazUzuzTDvh" - }, - "outputs": [], - "source": [ - "!git clone https://github.com/GoogleCloudPlatform/evalbench.git" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "J0Bprx7I1E-4" - }, - "outputs": [], - "source": [ - "cd evalbench" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "n7IDm5ZffnqH" - }, - "source": [ - "#### 2. Install Dependencies" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "Nz-oC6c-0HbC" - }, - "outputs": [], - "source": [ - "!pip install -r requirements.txt" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "K9rvyXx_5_Pi" - }, - "source": [ - "#### 3. Connect with GCP" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "V3_MT1r_z-mR" - }, - "outputs": [], - "source": [ - "import os\n", - "os.environ['EVAL_GCP_PROJECT_ID'] = GCP_PROJECT_ID\n", - "os.environ['EVAL_GCP_PROJECT_REGION'] = DATABASE_INSTANCE_REGION" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "AwQBxt_a0G04" - }, - "outputs": [], - "source": [ - "from google.colab import auth\n", - "auth.authenticate_user(project_id=os.environ['EVAL_GCP_PROJECT_ID'])" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "Z3v0MsBsT5n0" - }, - "source": [ - "#### 4. Update the run config file" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "Kzw7aqTLeXxJ" - }, - "outputs": [], - "source": [ - "run_config = f\"\"\"############################################################\n", - "### Dataset / Eval Items\n", - "############################################################\n", - "# The JSON list of prompts / golden SQLs and eval attributes for the run\n", - "dataset_config: datasets/bat/prompts.json\n", - "### Database Info\n", - "# The YAML config for the database connection information\n", - "database_configs:\n", - " - datasets/bat/db_configs/{DATABASE_TYPE}.yaml\n", - "# The dialect that the dataset_config will be filtered by. Only one can be\n", - "# specified at a time (per run). See above for list of supported Dialects\n", - "dialects:\n", - " - {DATABASE_TYPE}\n", - "query_types:\n", - " - dql\n", - "#\n", - "#\n", - "############################################################\n", - "### Prompt and Generation Modules\n", - "############################################################\n", - "# The YAML config for the model to be used for generation.\n", - "model_config: datasets/bat/model_configs/gemini_2.5_pro_model.yaml\n", - "# The prompt generator module id for prompt generation.\n", - "prompt_generator: 'SQLGenBasePromptGenerator'\n", - "#\n", - "#\n", - "############################################################\n", - "### Optional - Setup / Teardown related configs (Required for testing DDL)\n", - "############################################################\n", - "# Used for setup / teardown, the directory path to the sql files used for setting up\n", - "# / tearing down a database instance. This is required for running DDL\n", - "setup_directory: datasets/bat/setup\n", - "#\n", - "#\n", - "############################################################\n", - "### Scorer Related Configs - See /scorers directory for each scorer.\n", - "############################################################\n", - "scorers:\n", - " exact_match: null\n", - " llmrater:\n", - " model_config: datasets/model_configs/gemini_1.5-pro-002_model.yaml\n", - " returned_sql: null\n", - " set_match: null\n", - " executable_sql: null\n", - "#\n", - "#\n", - "############################################################\n", - "### Reporting Related Configs\n", - "############################################################\n", - "reporting:\n", - " csv:\n", - " output_directory: 'results'\"\"\"" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "cZ34McRSQT8y" - }, - "outputs": [], - "source": [ - "!echo \"{run_config}\" > datasets/bat/example_run_config.yaml" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "gxhh9KeEe6BF" - }, - "source": [ - "#### 5. Update the database config file" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "oWNuMRkle_RC" - }, - "outputs": [], - "source": [ - "db_config = f\"\"\"db_type: {DATABASE_TYPE}\n", - "database_name: {DATABASE_NAME}\n", - "database_path: {GCP_PROJECT_ID}:{DATABASE_INSTANCE_REGION}:{DATABASE_NAME}\n", - "max_executions_per_minute: 180\n", - "user_name: {DATABASE_USERNAME}\n", - "password: {DATABASE_PASSWORD}\"\"\"" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "c4H0JZEYfHQ4" - }, - "outputs": [], - "source": [ - "!echo \"{db_config}\" > datasets/bat/db_configs/{DATABASE_TYPE}.yaml" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "QjT5t3vm10kQ" - }, - "source": [ - "### 2. Run Evalbench" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "wFvcdn0AT1pt" - }, - "outputs": [], - "source": [ - "%run evalbench/evalbench.py --experiment_config=\"datasets/bat/example_run_config.yaml\"" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "PrPvYYX817yt" - }, - "source": [ - "### 3. Build Reports" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "VMEmWAdI2IG-" - }, - "outputs": [], - "source": [ - "import os\n", - "import pandas as pd\n", - "import matplotlib.pyplot as plt\n", - "\n", - "results_dir = \"results/\"\n", - "\n", - "# Find the first folder in the results directory\n", - "# NOTE: Change this logic if you have a specific job_id you want to find\n", - "first_folder = None\n", - "for folder in os.listdir(results_dir):\n", - " folder_path = os.path.join(results_dir, folder)\n", - " if not folder.startswith(\".\") and os.path.isdir(folder_path):\n", - " first_folder = folder_path\n", - " break\n", - "\n", - "if first_folder:\n", - " summary_file = os.path.join(first_folder, \"summary.csv\")\n", - " if os.path.exists(summary_file):\n", - " df = pd.read_csv(summary_file)\n", - " df['percentage'] = (df['correct_results_count'] / df['total_results_count']) * 100\n", - " df_sorted = df.sort_values(by='percentage', ascending=False)\n", - " plt.figure(figsize=(8, 6))\n", - " bars = plt.bar(df_sorted['metric_name'], df_sorted['percentage'], color='skyblue')\n", - " plt.xlabel('Metric Name')\n", - " plt.ylabel('Correct Results (%)')\n", - " plt.title('Percentage of Correct Results per Metric')\n", - " plt.ylim(0, 110)\n", - " plt.xticks(rotation=45)\n", - " for bar in bars:\n", - " height = bar.get_height()\n", - " plt.text(bar.get_x() + bar.get_width() / 2, height + 1, f'{height:.1f}%',\n", - " ha='center', va='bottom', fontsize=9)\n", - " plt.tight_layout()\n", - " plt.show()\n", - " else:\n", - " print(f\"summary.csv not found in {os.path.join(first_folder, 'summary.csv')}.\")\n", - "else:\n", - " print(\"No results found.\")\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "u41cgkZQc8QS" - }, - "source": [ - "Now report on the overall evaluations and their scoring" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "bWUh0Mzfc7IH" - }, - "outputs": [], - "source": [ - "from google.colab import data_table\n", - "\n", - "evals_file = os.path.join(first_folder, \"evals.csv\")\n", - "scores_file = os.path.join(first_folder, \"scores.csv\")\n", - "if not os.path.exists(scores_file) or not os.path.exists(evals_file):\n", - " print(\"No results found.\")\n", - " exit()\n", - "\n", - "evals_df = pd.read_csv(evals_file)\n", - "scores_df = pd.read_csv(scores_file)\n", - "scores_pivot = scores_df.pivot_table(\n", - " index=[\"id\", \"job_id\"],\n", - " columns=\"comparator\",\n", - " values=\"score\",\n", - " aggfunc=\"first\"\n", - ").reset_index()\n", - "scores_pivot.columns.name = None\n", - "scores_pivot = scores_pivot.rename(columns={\n", - " \"returned_sql\": \"score_returned_sql\",\n", - " \"llmrater\": \"score_llmrater\",\n", - " \"set_match\": \"score_set_match\",\n", - " \"exact_match\": \"score_exact_match\"\n", - "})\n", - "merged_df = pd.merge(evals_df, scores_pivot, on=[\"id\", \"job_id\"], how=\"left\")\n", - "merged_df[\"score_executable\"] = merged_df[\"generated_error\"].isna().astype(int) * 100\n", - "final_df = merged_df[[\n", - " \"id\",\n", - " \"nl_prompt\",\n", - " \"generated_sql\",\n", - " \"golden_sql\",\n", - " \"generated_result\",\n", - " \"golden_result\",\n", - " \"score_returned_sql\",\n", - " \"score_executable\",\n", - " \"score_llmrater\",\n", - " \"score_set_match\",\n", - " \"score_exact_match\"\n", - "]].rename(columns={\n", - " \"generated_sql\": \"generated_query\",\n", - " \"golden_sql\": \"golden_query\"\n", - "})\n", - "data_table.enable_dataframe_formatter()\n", - "final_df" - ] - } - ], - "metadata": { - "colab": { - "provenance": [] - }, - "kernelspec": { - "display_name": "Python 3", - "name": "python3" - }, - "language_info": { - "name": "python" - } + "cells": [ + { + "id": "2a621b74", + "cell_type": "markdown", + "source": [ + "[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/GoogleCloudPlatform/evalbench/blob/main/docs/examples/GCP_CloudSQL_Example.ipynb)\n", + "\n", + "# Getting Started With Evalbench\n", + "\n", + "EvalBench is a flexible framework designed to measure the quality of generative AI (GenAI) workflows around database specific tasks. As of now, it provides a comprehensive set of tools, and modules to evaluate models on NL2SQL tasks, including capability of running and scoring DQL, DML, and DDL queries across multiple supported databases. Its modular, plug-and-play architecture allows you to seamlessly integrate custom components while leveraging a robust evaluation pipeline, result storage, scoring strategies, and dashboarding capabilities." + ], + "metadata": { + "id": "V5Bx7tBafUEP" + }, + "execution_count": null }, - "nbformat": 4, - "nbformat_minor": 0 + { + "id": "08844b0f", + "cell_type": "markdown", + "source": [ + "## Quick Start Example w/ GCP CloudSQL offerings (MySQL, Postgres, SQL Server)" + ], + "metadata": { + "id": "6GKI9_72faY9" + }, + "execution_count": null + }, + { + "id": "25d95bbe", + "cell_type": "markdown", + "source": [ + "### Before You Begin\n", + "\n", + "To run this notebook, you will need to do the following:\n", + "\n", + "* [Create a Google Cloud Project](https://developers.google.com/workspace/guides/create-project)\n", + "* MySQL\n", + " * [Create a Cloud SQL for MySQL instance](https://cloud.google.com/sql/docs/mysql/create-instance)\n", + " * [Create a MySQL Database](https://cloud.google.com/sql/docs/mysql/create-manage-databases)\n", + " * [Create an evalbench user](https://cloud.google.com/sql/docs/mysql/**create**-manage-users)\n", + "* PostgreSQL\n", + " * [Create a Cloud SQL for PostgreSQL instance](https://cloud.google.com/sql/docs/postgres/create-instance)\n", + " * [Create a PostgreSQL Database](https://cloud.google.com/sql/docs/postgres/create-manage-databases)\n", + " * [Create an evalbench user](https://cloud.google.com/sql/docs/postgres/create-manage-users)\n", + "* SQL Server\n", + " * [Create a Cloud SQL for SQL server instance](https://cloud.google.com/sql/docs/sqlserver/create-instance)\n", + " * [Create a SQL Server Database](https://cloud.google.com/sql/docs/sqlserver/create-manage-databases)\n", + " * [Create an evalbench user](https://cloud.google.com/sql/docs/sqlserver/create-manage-users)" + ], + "metadata": { + "id": "ZWK6-XVs116G" + }, + "execution_count": null + }, + { + "id": "23a4dcbf", + "cell_type": "markdown", + "source": [ + "\n", + "**After confirmed access to database in the runtime environment of this notebook, fill in the database details below and proceed to run the script**" + ], + "metadata": { + "id": "zXXvsHVi7R5M" + }, + "execution_count": null + }, + { + "id": "bf6caa44", + "cell_type": "code", + "source": [ + "GCP_PROJECT_ID = \"\" # @param {\"type\":\"string\",\"placeholder\":\"your_gcp_project_id\"}\n", + "\n", + "DATABASE_TYPE = \"\" # @param [\"mysql\",\"postgres\",\"sqlserver\"]\n", + "DATABASE_INSTANCE_REGION = \"\" # @param {\"type\":\"string\",\"placeholder\":\"your_gcp_project_region\"}\n", + "DATABASE_NAME = \"\" # @param {\"type\":\"string\",\"placeholder\":\"your_database_name\"}\n", + "DATABASE_USERNAME = \"\" # @param {\"type\":\"string\",\"placeholder\":\"your_database_username\"}\n", + "DATABASE_PASSWORD = \"\" # @param {\"type\":\"string\",\"placeholder\":\"your_database_password\"}" + ], + "metadata": { + "id": "lVL_YPR46njA" + }, + "execution_count": null + }, + { + "id": "97fead51", + "cell_type": "markdown", + "source": [ + "### 1. Set up the environment" + ], + "metadata": { + "id": "9U5aGmLW5gip" + }, + "execution_count": null + }, + { + "id": "88b6ed98", + "cell_type": "markdown", + "source": [ + "#### 1. Clone the EvalBench repository from GitHub:" + ], + "metadata": { + "id": "6-4In2FAz1T5" + }, + "execution_count": null + }, + { + "id": "d1f053ce", + "cell_type": "code", + "source": [ + "!git clone https://github.com/GoogleCloudPlatform/evalbench.git" + ], + "metadata": { + "id": "EFazUzuzTDvh" + }, + "execution_count": null + }, + { + "id": "dcb118c5", + "cell_type": "code", + "source": [ + "cd evalbench" + ], + "metadata": { + "id": "J0Bprx7I1E-4" + }, + "execution_count": null + }, + { + "id": "b70ce16d", + "cell_type": "markdown", + "source": [ + "#### 2. Install Dependencies" + ], + "metadata": { + "id": "n7IDm5ZffnqH" + }, + "execution_count": null + }, + { + "id": "4cb30cc7", + "cell_type": "code", + "source": [ + "!pip install uv\n", + "!uv sync" + ], + "metadata": { + "id": "Nz-oC6c-0HbC" + }, + "execution_count": null + }, + { + "id": "c4ee057c", + "cell_type": "markdown", + "source": [ + "#### 3. Connect with GCP" + ], + "metadata": { + "id": "K9rvyXx_5_Pi" + }, + "execution_count": null + }, + { + "id": "8ac4dc9a", + "cell_type": "code", + "source": [ + "import os\n", + "os.environ['EVAL_GCP_PROJECT_ID'] = GCP_PROJECT_ID\n", + "os.environ['EVAL_GCP_PROJECT_REGION'] = DATABASE_INSTANCE_REGION" + ], + "metadata": { + "id": "V3_MT1r_z-mR" + }, + "execution_count": null + }, + { + "id": "bb25f855", + "cell_type": "code", + "source": [ + "from google.colab import auth\n", + "auth.authenticate_user(project_id=os.environ['EVAL_GCP_PROJECT_ID'])" + ], + "metadata": { + "id": "AwQBxt_a0G04" + }, + "execution_count": null + }, + { + "id": "5c721778", + "cell_type": "markdown", + "source": [ + "#### 4. Update the run config file" + ], + "metadata": { + "id": "Z3v0MsBsT5n0" + }, + "execution_count": null + }, + { + "id": "af1cbc92", + "cell_type": "code", + "source": [ + "run_config = f\"\"\"############################################################\n", + "### Dataset / Eval Items\n", + "############################################################\n", + "# The JSON list of prompts / golden SQLs and eval attributes for the run\n", + "dataset_config: datasets/bat/prompts.json\n", + "### Database Info\n", + "# The YAML config for the database connection information\n", + "database_configs:\n", + " - datasets/bat/db_configs/{DATABASE_TYPE}.yaml\n", + "# The dialect that the dataset_config will be filtered by. Only one can be\n", + "# specified at a time (per run). See above for list of supported Dialects\n", + "dialects:\n", + " - {DATABASE_TYPE}\n", + "query_types:\n", + " - dql\n", + "#\n", + "#\n", + "############################################################\n", + "### Prompt and Generation Modules\n", + "############################################################\n", + "# The YAML config for the model to be used for generation.\n", + "model_config: datasets/bat/model_configs/gemini_2.5_pro_model.yaml\n", + "# The prompt generator module id for prompt generation.\n", + "prompt_generator: 'SQLGenBasePromptGenerator'\n", + "#\n", + "#\n", + "############################################################\n", + "### Optional - Setup / Teardown related configs (Required for testing DDL)\n", + "############################################################\n", + "# Used for setup / teardown, the directory path to the sql files used for setting up\n", + "# / tearing down a database instance. This is required for running DDL\n", + "setup_directory: datasets/bat/setup\n", + "#\n", + "#\n", + "############################################################\n", + "### Scorer Related Configs - See /scorers directory for each scorer.\n", + "############################################################\n", + "scorers:\n", + " exact_match: null\n", + " llmrater:\n", + " model_config: datasets/model_configs/gemini_1.5-pro-002_model.yaml\n", + " returned_sql: null\n", + " set_match: null\n", + " executable_sql: null\n", + "#\n", + "#\n", + "############################################################\n", + "### Reporting Related Configs\n", + "############################################################\n", + "reporting:\n", + " csv:\n", + " output_directory: 'results'\"\"\"" + ], + "metadata": { + "id": "Kzw7aqTLeXxJ" + }, + "execution_count": null + }, + { + "id": "1271560c", + "cell_type": "code", + "source": [ + "!echo \"{run_config}\" \u003e datasets/bat/example_run_config.yaml" + ], + "metadata": { + "id": "cZ34McRSQT8y" + }, + "execution_count": null + }, + { + "id": "26f75ceb", + "cell_type": "markdown", + "source": [ + "#### 5. Update the database config file" + ], + "metadata": { + "id": "gxhh9KeEe6BF" + }, + "execution_count": null + }, + { + "id": "0356d1b9", + "cell_type": "code", + "source": [ + "db_config = f\"\"\"db_type: {DATABASE_TYPE}\n", + "database_name: {DATABASE_NAME}\n", + "database_path: {GCP_PROJECT_ID}:{DATABASE_INSTANCE_REGION}:{DATABASE_NAME}\n", + "max_executions_per_minute: 180\n", + "user_name: {DATABASE_USERNAME}\n", + "password: {DATABASE_PASSWORD}\"\"\"" + ], + "metadata": { + "id": "oWNuMRkle_RC" + }, + "execution_count": null + }, + { + "id": "de21be38", + "cell_type": "code", + "source": [ + "!echo \"{db_config}\" \u003e datasets/bat/db_configs/{DATABASE_TYPE}.yaml" + ], + "metadata": { + "id": "c4H0JZEYfHQ4" + }, + "execution_count": null + }, + { + "id": "3881cc8a", + "cell_type": "markdown", + "source": [ + "### 2. Run Evalbench" + ], + "metadata": { + "id": "QjT5t3vm10kQ" + }, + "execution_count": null + }, + { + "id": "8cc36f34", + "cell_type": "code", + "source": [ + "%run evalbench/evalbench.py --experiment_config=\"datasets/bat/example_run_config.yaml\"" + ], + "metadata": { + "id": "wFvcdn0AT1pt" + }, + "execution_count": null + }, + { + "id": "27e316c5", + "cell_type": "markdown", + "source": [ + "### 3. Build Reports" + ], + "metadata": { + "id": "PrPvYYX817yt" + }, + "execution_count": null + }, + { + "id": "5938c115", + "cell_type": "code", + "source": [ + "import os\n", + "import pandas as pd\n", + "import matplotlib.pyplot as plt\n", + "\n", + "results_dir = \"results/\"\n", + "\n", + "# Find the first folder in the results directory\n", + "# NOTE: Change this logic if you have a specific job_id you want to find\n", + "first_folder = None\n", + "for folder in os.listdir(results_dir):\n", + " folder_path = os.path.join(results_dir, folder)\n", + " if not folder.startswith(\".\") and os.path.isdir(folder_path):\n", + " first_folder = folder_path\n", + " break\n", + "\n", + "if first_folder:\n", + " summary_file = os.path.join(first_folder, \"summary.csv\")\n", + " if os.path.exists(summary_file):\n", + " df = pd.read_csv(summary_file)\n", + " df['percentage'] = (df['correct_results_count'] / df['total_results_count']) * 100\n", + " df_sorted = df.sort_values(by='percentage', ascending=False)\n", + " plt.figure(figsize=(8, 6))\n", + " bars = plt.bar(df_sorted['metric_name'], df_sorted['percentage'], color='skyblue')\n", + " plt.xlabel('Metric Name')\n", + " plt.ylabel('Correct Results (%)')\n", + " plt.title('Percentage of Correct Results per Metric')\n", + " plt.ylim(0, 110)\n", + " plt.xticks(rotation=45)\n", + " for bar in bars:\n", + " height = bar.get_height()\n", + " plt.text(bar.get_x() + bar.get_width() / 2, height + 1, f'{height:.1f}%',\n", + " ha='center', va='bottom', fontsize=9)\n", + " plt.tight_layout()\n", + " plt.show()\n", + " else:\n", + " print(f\"summary.csv not found in {os.path.join(first_folder, 'summary.csv')}.\")\n", + "else:\n", + " print(\"No results found.\")\n" + ], + "metadata": { + "id": "VMEmWAdI2IG-" + }, + "execution_count": null + }, + { + "id": "8221e538", + "cell_type": "markdown", + "source": [ + "Now report on the overall evaluations and their scoring" + ], + "metadata": { + "id": "u41cgkZQc8QS" + }, + "execution_count": null + }, + { + "id": "a0125114", + "cell_type": "code", + "source": [ + "from google.colab import data_table\n", + "\n", + "evals_file = os.path.join(first_folder, \"evals.csv\")\n", + "scores_file = os.path.join(first_folder, \"scores.csv\")\n", + "if not os.path.exists(scores_file) or not os.path.exists(evals_file):\n", + " print(\"No results found.\")\n", + " exit()\n", + "\n", + "evals_df = pd.read_csv(evals_file)\n", + "scores_df = pd.read_csv(scores_file)\n", + "scores_pivot = scores_df.pivot_table(\n", + " index=[\"id\", \"job_id\"],\n", + " columns=\"comparator\",\n", + " values=\"score\",\n", + " aggfunc=\"first\"\n", + ").reset_index()\n", + "scores_pivot.columns.name = None\n", + "scores_pivot = scores_pivot.rename(columns={\n", + " \"returned_sql\": \"score_returned_sql\",\n", + " \"llmrater\": \"score_llmrater\",\n", + " \"set_match\": \"score_set_match\",\n", + " \"exact_match\": \"score_exact_match\"\n", + "})\n", + "merged_df = pd.merge(evals_df, scores_pivot, on=[\"id\", \"job_id\"], how=\"left\")\n", + "merged_df[\"score_executable\"] = merged_df[\"generated_error\"].isna().astype(int) * 100\n", + "final_df = merged_df[[\n", + " \"id\",\n", + " \"nl_prompt\",\n", + " \"generated_sql\",\n", + " \"golden_sql\",\n", + " \"generated_result\",\n", + " \"golden_result\",\n", + " \"score_returned_sql\",\n", + " \"score_executable\",\n", + " \"score_llmrater\",\n", + " \"score_set_match\",\n", + " \"score_exact_match\"\n", + "]].rename(columns={\n", + " \"generated_sql\": \"generated_query\",\n", + " \"golden_sql\": \"golden_query\"\n", + "})\n", + "data_table.enable_dataframe_formatter()\n", + "final_df" + ], + "metadata": { + "id": "bWUh0Mzfc7IH" + }, + "execution_count": null + } + ], + "metadata": { + "colab": { + "provenance": [] + }, + "kernelspec": { + "display_name": "Python 3", + "name": "python3" + }, + "language_info": { + "name": "python" + } + }, + "nbformat_minor": 0, + "nbformat": 4 } diff --git a/docs/examples/bigquery_hybrid_example.ipynb b/docs/examples/bigquery_hybrid_example.ipynb new file mode 100644 index 00000000..d8edaf53 --- /dev/null +++ b/docs/examples/bigquery_hybrid_example.ipynb @@ -0,0 +1,274 @@ +{ + "cells": [ + { + "id": "96dbebdd", + "cell_type": "markdown", + "source": [ + "# BIRD Cross-Database Evaluation (BigQuery Generation + SQLite Ground Truth)\n", + "\n", + "This notebook demonstrates how to set up and run a hybrid cross-database evaluation on the BIRD benchmark.\n", + "\n", + "When deploying Text-to-SQL models in enterprise data warehouses, you want the model to generate SQL optimized specifically for Google BigQuery (utilizing its dialect, functions, and schemas). However, standard benchmark datasets like BIRD are packaged with reference (golden) SQL queries written in SQLite dialect, which often fail when executed directly on BigQuery.\n", + "\n", + "Instead of manually translating thousands of complex reference queries to BigQuery dialect, Hybrid Mode executes the model's generated queries on BigQuery, but executes the reference queries on local SQLite databases to resolve the ground truth. This enables developers to reliably measure Execution Accuracy (XA) for BigQuery SQL generation out-of-the-box using standard benchmark datasets, without having to rewrite or modify any reference SQL queries.\n", + "\n", + "### Process Flow:\n", + "1. **Model Generation**: The model generates SQL targeting Google BigQuery.\n", + "2. **Execution**: The generated query executes on BigQuery.\n", + "3. **Hybrid Verification**: If the reference/golden query fails on BigQuery, EvalBench executes it on a local SQLite fallback database to fetch the ground truth rows, then performs a structure-agnostic comparison.\n", + "4. **Reporting**: Results are stored directly to Google BigQuery, generating a Looker Studio visualization dashboard link." + ], + "metadata": {} + }, + { + "id": "065fc18a", + "cell_type": "markdown", + "source": [ + "#### 1. Clone the EvalBench repository from GitHub" + ], + "metadata": {} + }, + { + "id": "620dc588", + "cell_type": "code", + "source": [ + "!git clone https://github.com/GoogleCloudPlatform/evalbench.git" + ], + "metadata": {}, + "execution_count": null, + "outputs": [] + }, + { + "id": "d4487279", + "cell_type": "markdown", + "source": [ + "#### 2. Install Dependencies\n", + "We install `uv` to manage python packages, sync the virtual environment, install the `mcp` library package, and compile the protobuf message files." + ], + "metadata": {} + }, + { + "id": "6dd67d2e", + "cell_type": "code", + "source": [ + "!pip install uv\n", + "!cd evalbench && uv sync\n", + "!cd evalbench && .venv/bin/pip3 install mcp\n", + "!cd evalbench && .venv/bin/python3 -m grpc_tools.protoc \\\n", + " --proto_path=evalbench/evalproto \\\n", + " --python_out=evalbench/evalproto \\\n", + " --pyi_out=evalbench/evalproto \\\n", + " --grpc_python_out=evalbench/evalproto \\\n", + " --experimental_editions \\\n", + " evalbench/evalproto/*.proto" + ], + "metadata": {}, + "execution_count": null, + "outputs": [] + }, + { + "id": "beca689a", + "cell_type": "markdown", + "source": [ + "#### 3. Download the BIRD Dataset and Database Connections\n", + "This script downloads the natural language prompts and all the SQLite databases required for resolving the ground truth evaluation rows." + ], + "metadata": {} + }, + { + "id": "2e441178", + "cell_type": "code", + "source": [ + "!cd evalbench && bash datasets/bird/download_dataset.sh" + ], + "metadata": {}, + "execution_count": null, + "outputs": [] + }, + { + "id": "4cf343ad", + "cell_type": "markdown", + "source": [ + "#### 3.5. Create Sliced Dataset and Custom Config dynamically\n", + "Since running evaluation on the entire BIRD developer set (1,500+ queries) can take several hours, we dynamically slice the prompts to keep only 20 examples from two databases (`california_schools` and `card_games`), and create a custom configuration pointing to this slice." + ], + "metadata": {} + }, + { + "id": "5eb35639", + "cell_type": "code", + "source": [ + "import json\n", + "\n", + "repo_prefix = \"evalbench/\"\n", + "prompts_path = repo_prefix + \"datasets/bird/prompts.json\"\n", + "temp_prompts_path_rel = \"datasets/bird/prompts_colab_temp.json\"\n", + "temp_prompts_path = repo_prefix + temp_prompts_path_rel\n", + "colab_config_path = repo_prefix + \"datasets/bird/colab_run_config.yaml\"\n", + "\n", + "# 1. Read prompts.json and write a 20-prompt slice (10 from each\n", + "# database) to prompts_colab_temp.json.\n", + "with open(prompts_path, \"r\") as f:\n", + " prompts = json.load(f)\n", + "\n", + "filtered_prompts = []\n", + "for db_id in (\"california_schools\", \"card_games\"):\n", + " filtered_prompts.extend([\n", + " p for p in prompts \n", + " if p.get(\"db_id\") == db_id\n", + " ][:10])\n", + "\n", + "with open(temp_prompts_path, \"w\") as f:\n", + " json.dump(filtered_prompts, f, indent=2)\n", + "\n", + "# 2. Write a new configuration file pointing to the temporary prompts.\n", + "config_content = f\"\"\"\n", + "dataset_config: {temp_prompts_path_rel}\n", + "\n", + "database_configs:\n", + " - datasets/bat/db_configs/bigquery.yaml\n", + " - datasets/bird/db_configs/sqlite.yaml\n", + "\n", + "dialects:\n", + " - bigquery\n", + "query_types:\n", + " - dql\n", + "dataset_format: bird-standard-format\n", + "\n", + "model_config: datasets/model_configs/gemini_2.5_pro_model.yaml\n", + "prompt_generator: 'SQLGenBasePromptGenerator'\n", + "\n", + "scorers:\n", + " llmrater:\n", + " model_config: datasets/model_configs/gemini_2.5_pro_model.yaml\n", + " hybrid_ground_truth: true\n", + " python_scorer:\n", + " script_path: 'evalbench/scorers/judges/hybrid_xa_judge.py'\n", + " scorer_name: 'hybrid_cross_db'\n", + "\n", + "reporting:\n", + " bigquery:\n", + " dataset_location: \"US\"\n", + "\"\"\"\n", + "\n", + "with open(colab_config_path, \"w\") as f:\n", + " f.write(config_content.strip())\n", + "\n", + "print(\"Created colab_run_config.yaml successfully!\")" + ], + "metadata": {}, + "execution_count": null, + "outputs": [] + }, + { + "id": "8adb4b6a", + "cell_type": "markdown", + "source": [ + "#### 4. Setup Evalbench Environment and GCP Credentials\n", + "Enter your GCP Project ID and region where the BigQuery datasets will be loaded." + ], + "metadata": {} + }, + { + "id": "7bf5cb51", + "cell_type": "code", + "source": [ + "import os\n", + "os.environ['EVAL_GCP_PROJECT_ID'] = ''\n", + "os.environ['EVAL_GCP_PROJECT_REGION'] = ''" + ], + "metadata": {}, + "execution_count": null, + "outputs": [] + }, + { + "id": "844a132b", + "cell_type": "code", + "source": [ + "from google.colab import auth\n", + "auth.authenticate_user(project_id=os.environ['EVAL_GCP_PROJECT_ID'])" + ], + "metadata": {}, + "execution_count": null, + "outputs": [] + }, + { + "id": "acc562bb", + "cell_type": "markdown", + "source": [ + "#### 5. Run the Hybrid Cross-Database Evaluation\n", + "This runs the evaluation using our dynamically generated `colab_run_config.yaml` configuration." + ], + "metadata": {} + }, + { + "id": "8711b57c", + "cell_type": "code", + "source": [ + "!cd evalbench && .venv/bin/python3 evalbench/evalbench.py \\\n", + " --experiment_config=\"datasets/bird/colab_run_config.yaml\"" + ], + "metadata": {}, + "execution_count": null, + "outputs": [] + }, + { + "id": "bad9bd64", + "cell_type": "markdown", + "source": [ + "#### 6. Inspecting Results in BigQuery\n", + "Once the run is complete, you can click the Looker Studio URL printed at the end of the logs to view the visual report. \n", + "\n", + "Alternatively, you can query the results directly in Python from BigQuery:" + ], + "metadata": {} + }, + { + "id": "d45dbc13", + "cell_type": "code", + "source": [ + "from google.cloud import bigquery\n", + "import pandas as pd\n", + "\n", + "project_id = os.environ['EVAL_GCP_PROJECT_ID']\n", + "client = bigquery.Client(project=project_id)\n", + "\n", + "# Dynamically filters for the most recent evaluation run\n", + "query = f\"\"\"\n", + "SELECT \n", + " id, \n", + " database, \n", + " nl_prompt, \n", + " generated_sql, \n", + " golden_sql, \n", + " generated_result, \n", + " golden_result \n", + "FROM `{project_id}.evalbench.results` \n", + "WHERE job_id = (\n", + " SELECT job_id \n", + " FROM `{project_id}.evalbench.results` \n", + " ORDER BY run_time DESC \n", + " LIMIT 1\n", + ")\n", + "\"\"\"\n", + "df = client.query(query).to_dataframe()\n", + "df" + ], + "metadata": {}, + "execution_count": null, + "outputs": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python" + } + }, + "nbformat_minor": 5, + "nbformat": 4 +} \ No newline at end of file diff --git a/docs/examples/sqlite_example.ipynb b/docs/examples/sqlite_example.ipynb index f60422cd..864dffd7 100644 --- a/docs/examples/sqlite_example.ipynb +++ b/docs/examples/sqlite_example.ipynb @@ -1,279 +1,300 @@ { - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "id": "V5Bx7tBafUEP" - }, - "source": [ - "[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/GoogleCloudPlatform/evalbench/blob/main/docs/examples/sqlite_example.ipynb)\n", - "\n", - "# Getting Started With Evalbench\n", - "\n", - "EvalBench is a flexible framework designed to measure the quality of generative AI (GenAI) workflows around database specific tasks. As of now, it provides a comprehensive set of tools, and modules to evaluate models on NL2SQL tasks, including capability of running and scoring DQL, DML, and DDL queries across multiple supported databases. Its modular, plug-and-play architecture allows you to seamlessly integrate custom components while leveraging a robust evaluation pipeline, result storage, scoring strategies, and dashboarding capabilities." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "6GKI9_72faY9" - }, - "source": [ - "### Quick Start Example w/ sqllite\n", - "#### 1. Clone the EvalBench repository from GitHub:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "HVZAQXctfFnx" - }, - "outputs": [], - "source": [ - "!git clone https://github.com/GoogleCloudPlatform/evalbench.git" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "n7IDm5ZffnqH" - }, - "source": [ - "#### 2. Install Dependencies" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "L37hxmGLf-wM" - }, - "outputs": [], - "source": [ - "!pip install -r evalbench/requirements.txt" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "uDoc62MDz5UO" - }, - "source": [ - "#### 3. Setup Evalbench environment" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "S8VSHGEG0Bmn" - }, - "outputs": [], - "source": [ - "cd evalbench" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "ym5qP0zfhDnf" - }, - "source": [ - "#### 4. Connect with GCP" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "**NOTE: Update Your GCP Project ID and Region Below.** " - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "V3_MT1r_z-mR" - }, - "outputs": [], - "source": [ - "import os\n", - "os.environ['EVAL_GCP_PROJECT_ID'] = ''\n", - "os.environ['EVAL_GCP_PROJECT_REGION'] = ''" - ] - }, - { - "cell_type": "code", - "execution_count": 52, - "metadata": { - "id": "AwQBxt_a0G04" - }, - "outputs": [], - "source": [ - "from google.colab import auth\n", - "auth.authenticate_user(project_id=os.environ['EVAL_GCP_PROJECT_ID'])" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "QjT5t3vm10kQ" - }, - "source": [ - "#### 5. Run Evalbench" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "ZCPUOdUkhGiE" - }, - "outputs": [], - "source": [ - "%run evalbench/evalbench.py --experiment_config=\"datasets/bat/example_run_config.yaml\"" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "PrPvYYX817yt" - }, - "source": [ - "#### 6. Build a Report" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "VMEmWAdI2IG-" - }, - "outputs": [], - "source": [ - "import os\n", - "import pandas as pd\n", - "import matplotlib.pyplot as plt\n", - "\n", - "results_dir = \"results/\"\n", - "\n", - "# Find the first folder in the results directory\n", - "# NOTE: Change this logic if you have a specific job_id you want to find\n", - "first_folder = None\n", - "for folder in os.listdir(results_dir):\n", - " folder_path = os.path.join(results_dir, folder)\n", - " if not folder.startswith(\".\") and os.path.isdir(folder_path):\n", - " first_folder = folder_path\n", - " break\n", - "\n", - "if first_folder:\n", - " summary_file = os.path.join(first_folder, \"summary.csv\")\n", - " if os.path.exists(summary_file):\n", - " df = pd.read_csv(summary_file)\n", - " df['percentage'] = (df['correct_results_count'] / df['total_results_count']) * 100\n", - " df_sorted = df.sort_values(by='percentage', ascending=False)\n", - " plt.figure(figsize=(8, 6))\n", - " bars = plt.bar(df_sorted['metric_name'], df_sorted['percentage'], color='skyblue')\n", - " plt.xlabel('Metric Name')\n", - " plt.ylabel('Correct Results (%)')\n", - " plt.title('Percentage of Correct Results per Metric')\n", - " plt.ylim(0, 110)\n", - " plt.xticks(rotation=45)\n", - " for bar in bars:\n", - " height = bar.get_height()\n", - " plt.text(bar.get_x() + bar.get_width() / 2, height + 1, f'{height:.1f}%',\n", - " ha='center', va='bottom', fontsize=9)\n", - " plt.tight_layout()\n", - " plt.show()\n", - " else:\n", - " print(f\"summary.csv not found in {os.path.join(first_folder, 'summary.csv')}.\")\n", - "else:\n", - " print(\"No results found.\")\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "u41cgkZQc8QS" - }, - "source": [ - "Now report on the overall evaluations and their scoring" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "bWUh0Mzfc7IH" - }, - "outputs": [], - "source": [ - "from google.colab import data_table\n", - "\n", - "evals_file = os.path.join(first_folder, \"evals.csv\")\n", - "scores_file = os.path.join(first_folder, \"scores.csv\")\n", - "if not os.path.exists(scores_file) or not os.path.exists(evals_file):\n", - " print(\"No results found.\")\n", - " exit()\n", - "\n", - "evals_df = pd.read_csv(evals_file)\n", - "scores_df = pd.read_csv(scores_file)\n", - "scores_pivot = scores_df.pivot_table(\n", - " index=[\"id\", \"job_id\"],\n", - " columns=\"comparator\",\n", - " values=\"score\",\n", - " aggfunc=\"first\"\n", - ").reset_index()\n", - "scores_pivot.columns.name = None\n", - "scores_pivot = scores_pivot.rename(columns={\n", - " \"returned_sql\": \"score_returned_sql\",\n", - " \"llmrater\": \"score_llmrater\",\n", - " \"set_match\": \"score_set_match\",\n", - " \"exact_match\": \"score_exact_match\"\n", - "})\n", - "merged_df = pd.merge(evals_df, scores_pivot, on=[\"id\", \"job_id\"], how=\"left\")\n", - "merged_df[\"score_executable\"] = merged_df[\"generated_error\"].isna().astype(int) * 100\n", - "final_df = merged_df[[\n", - " \"id\",\n", - " \"nl_prompt\",\n", - " \"generated_sql\",\n", - " \"golden_sql\",\n", - " \"generated_result\",\n", - " \"golden_result\",\n", - " \"score_returned_sql\",\n", - " \"score_executable\",\n", - " \"score_llmrater\",\n", - " \"score_set_match\",\n", - " \"score_exact_match\"\n", - "]].rename(columns={\n", - " \"generated_sql\": \"generated_query\",\n", - " \"golden_sql\": \"golden_query\"\n", - "})\n", - "data_table.enable_dataframe_formatter()\n", - "final_df" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Congrats! You have done it!! Please refer to the [EvalBench documentation](https://github.com/GoogleCloudPlatform/evalbench) for additional information including how to configure more complicated [run-configs](https://github.com/GoogleCloudPlatform/evalbench/blob/main/docs/configs/run-config.md). Enjoy evaluating your GenAI models!" - ] - } - ], - "metadata": { - "colab": { - "provenance": [] - }, - "kernelspec": { - "display_name": "Python 3", - "name": "python3" - }, - "language_info": { - "name": "python" - } + "cells": [ + { + "id": "59362d01", + "cell_type": "markdown", + "source": [ + "[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/GoogleCloudPlatform/evalbench/blob/main/docs/examples/sqlite_example.ipynb)\n", + "\n", + "# Getting Started With Evalbench\n", + "\n", + "EvalBench is a flexible framework designed to measure the quality of generative AI (GenAI) workflows around database specific tasks. As of now, it provides a comprehensive set of tools, and modules to evaluate models on NL2SQL tasks, including capability of running and scoring DQL, DML, and DDL queries across multiple supported databases. Its modular, plug-and-play architecture allows you to seamlessly integrate custom components while leveraging a robust evaluation pipeline, result storage, scoring strategies, and dashboarding capabilities." + ], + "metadata": { + "id": "V5Bx7tBafUEP" + }, + "execution_count": null }, - "nbformat": 4, - "nbformat_minor": 0 + { + "id": "956b7915", + "cell_type": "markdown", + "source": [ + "### Quick Start Example w/ sqllite\n", + "#### 1. Clone the EvalBench repository from GitHub:" + ], + "metadata": { + "id": "6GKI9_72faY9" + }, + "execution_count": null + }, + { + "id": "ae90f7e2", + "cell_type": "code", + "source": [ + "!git clone https://github.com/GoogleCloudPlatform/evalbench.git" + ], + "metadata": { + "id": "HVZAQXctfFnx" + }, + "execution_count": null + }, + { + "id": "f43df1f9", + "cell_type": "markdown", + "source": [ + "#### 2. Install Dependencies" + ], + "metadata": { + "id": "n7IDm5ZffnqH" + }, + "execution_count": null + }, + { + "id": "917caafe", + "cell_type": "code", + "source": [ + "!pip install uv\n", + "!cd evalbench \u0026\u0026 uv sync" + ], + "metadata": { + "id": "L37hxmGLf-wM" + }, + "execution_count": null + }, + { + "id": "67fbc7c1", + "cell_type": "markdown", + "source": [ + "#### 3. Setup Evalbench environment" + ], + "metadata": { + "id": "uDoc62MDz5UO" + }, + "execution_count": null + }, + { + "id": "518c3ca7", + "cell_type": "code", + "source": [ + "cd evalbench" + ], + "metadata": { + "id": "S8VSHGEG0Bmn" + }, + "execution_count": null + }, + { + "id": "cf50fa95", + "cell_type": "markdown", + "source": [ + "#### 4. Connect with GCP" + ], + "metadata": { + "id": "ym5qP0zfhDnf" + }, + "execution_count": null + }, + { + "id": "499b442b", + "cell_type": "markdown", + "source": [ + "**NOTE: Update Your GCP Project ID and Region Below.** " + ], + "metadata": {}, + "execution_count": null + }, + { + "id": "b947d526", + "cell_type": "code", + "source": [ + "import os\n", + "os.environ['EVAL_GCP_PROJECT_ID'] = '\u003cput-your-project-id-here\u003e'\n", + "os.environ['EVAL_GCP_PROJECT_REGION'] = '\u003cgcp-region-here\u003e'" + ], + "metadata": { + "id": "V3_MT1r_z-mR" + }, + "execution_count": null + }, + { + "id": "2675c28a", + "cell_type": "code", + "source": [ + "from google.colab import auth\n", + "auth.authenticate_user(project_id=os.environ['EVAL_GCP_PROJECT_ID'])" + ], + "metadata": { + "id": "AwQBxt_a0G04" + }, + "execution_count": 52 + }, + { + "id": "a6249f3b", + "cell_type": "markdown", + "source": [ + "#### 5. Run Evalbench" + ], + "metadata": { + "id": "QjT5t3vm10kQ" + }, + "execution_count": null + }, + { + "id": "ccd6e4eb", + "cell_type": "code", + "source": [ + "%run evalbench/evalbench.py --experiment_config=\"datasets/bat/example_run_config.yaml\"" + ], + "metadata": { + "id": "ZCPUOdUkhGiE" + }, + "execution_count": null + }, + { + "id": "0c088570", + "cell_type": "markdown", + "source": [ + "#### 6. Build a Report" + ], + "metadata": { + "id": "PrPvYYX817yt" + }, + "execution_count": null + }, + { + "id": "1d604d7e", + "cell_type": "code", + "source": [ + "import os\n", + "import pandas as pd\n", + "import matplotlib.pyplot as plt\n", + "\n", + "results_dir = \"results/\"\n", + "\n", + "# Find the first folder in the results directory\n", + "# NOTE: Change this logic if you have a specific job_id you want to find\n", + "first_folder = None\n", + "for folder in os.listdir(results_dir):\n", + " folder_path = os.path.join(results_dir, folder)\n", + " if not folder.startswith(\".\") and os.path.isdir(folder_path):\n", + " first_folder = folder_path\n", + " break\n", + "\n", + "if first_folder:\n", + " summary_file = os.path.join(first_folder, \"summary.csv\")\n", + " if os.path.exists(summary_file):\n", + " df = pd.read_csv(summary_file)\n", + " df['percentage'] = (df['correct_results_count'] / df['total_results_count']) * 100\n", + " df_sorted = df.sort_values(by='percentage', ascending=False)\n", + " plt.figure(figsize=(8, 6))\n", + " bars = plt.bar(df_sorted['metric_name'], df_sorted['percentage'], color='skyblue')\n", + " plt.xlabel('Metric Name')\n", + " plt.ylabel('Correct Results (%)')\n", + " plt.title('Percentage of Correct Results per Metric')\n", + " plt.ylim(0, 110)\n", + " plt.xticks(rotation=45)\n", + " for bar in bars:\n", + " height = bar.get_height()\n", + " plt.text(bar.get_x() + bar.get_width() / 2, height + 1, f'{height:.1f}%',\n", + " ha='center', va='bottom', fontsize=9)\n", + " plt.tight_layout()\n", + " plt.show()\n", + " else:\n", + " print(f\"summary.csv not found in {os.path.join(first_folder, 'summary.csv')}.\")\n", + "else:\n", + " print(\"No results found.\")\n" + ], + "metadata": { + "id": "VMEmWAdI2IG-" + }, + "execution_count": null + }, + { + "id": "7b5b534d", + "cell_type": "markdown", + "source": [ + "Now report on the overall evaluations and their scoring" + ], + "metadata": { + "id": "u41cgkZQc8QS" + }, + "execution_count": null + }, + { + "id": "4261e6e6", + "cell_type": "code", + "source": [ + "from google.colab import data_table\n", + "\n", + "evals_file = os.path.join(first_folder, \"evals.csv\")\n", + "scores_file = os.path.join(first_folder, \"scores.csv\")\n", + "if not os.path.exists(scores_file) or not os.path.exists(evals_file):\n", + " print(\"No results found.\")\n", + " exit()\n", + "\n", + "evals_df = pd.read_csv(evals_file)\n", + "scores_df = pd.read_csv(scores_file)\n", + "scores_pivot = scores_df.pivot_table(\n", + " index=[\"id\", \"job_id\"],\n", + " columns=\"comparator\",\n", + " values=\"score\",\n", + " aggfunc=\"first\"\n", + ").reset_index()\n", + "scores_pivot.columns.name = None\n", + "scores_pivot = scores_pivot.rename(columns={\n", + " \"returned_sql\": \"score_returned_sql\",\n", + " \"llmrater\": \"score_llmrater\",\n", + " \"set_match\": \"score_set_match\",\n", + " \"exact_match\": \"score_exact_match\"\n", + "})\n", + "merged_df = pd.merge(evals_df, scores_pivot, on=[\"id\", \"job_id\"], how=\"left\")\n", + "merged_df[\"score_executable\"] = merged_df[\"generated_error\"].isna().astype(int) * 100\n", + "final_df = merged_df[[\n", + " \"id\",\n", + " \"nl_prompt\",\n", + " \"generated_sql\",\n", + " \"golden_sql\",\n", + " \"generated_result\",\n", + " \"golden_result\",\n", + " \"score_returned_sql\",\n", + " \"score_executable\",\n", + " \"score_llmrater\",\n", + " \"score_set_match\",\n", + " \"score_exact_match\"\n", + "]].rename(columns={\n", + " \"generated_sql\": \"generated_query\",\n", + " \"golden_sql\": \"golden_query\"\n", + "})\n", + "data_table.enable_dataframe_formatter()\n", + "final_df" + ], + "metadata": { + "id": "bWUh0Mzfc7IH" + }, + "execution_count": null + }, + { + "id": "568767b0", + "cell_type": "markdown", + "source": [ + "Congrats! You have done it!! Please refer to the [EvalBench documentation](https://github.com/GoogleCloudPlatform/evalbench) for additional information including how to configure more complicated [run-configs](https://github.com/GoogleCloudPlatform/evalbench/blob/main/docs/configs/run-config.md). Enjoy evaluating your GenAI models!" + ], + "metadata": {}, + "execution_count": null + } + ], + "metadata": { + "colab": { + "provenance": [] + }, + "kernelspec": { + "display_name": "Python 3", + "name": "python3" + }, + "language_info": { + "name": "python" + } + }, + "nbformat_minor": 0, + "nbformat": 4 } diff --git a/docs/gemini_cli_agent_testing.md b/docs/gemini_cli_agent_testing.md index ae8ad8d2..79ee50b3 100644 --- a/docs/gemini_cli_agent_testing.md +++ b/docs/gemini_cli_agent_testing.md @@ -89,10 +89,10 @@ EvalBench's Gemini CLI integration enables automated, multi-turn evaluation of a ## Prerequisites -1. **Python 3.10+** and project dependencies installed: +1. **Python 3.10+** and project dependencies installed using `uv`: ```bash cd evalbench - pip install -r requirements.txt + uv sync ``` 2. **Node.js and npm** (for Gemini CLI execution) @@ -249,11 +249,17 @@ The evalset JSON file defines the test scenarios. Each scenario represents an ag | `id` | Yes | Unique identifier for the scenario | | `starting_prompt` | Yes | The first user message sent to Gemini CLI | | `conversation_plan` | Yes | Natural language instructions that guide the simulated user's behavior across turns. This defines the goals, expected information to provide, and how to react to agent responses. | -| `expected_trajectory` | Yes | Ordered list of tool names the agent is expected to call. Used by `trajectory_matcher` scorer. | +| `expected_trajectory` | Yes | Ordered list of tool names the agent is expected to call. Used by `trajectory_matcher` scorer. See [Tool name format](#tool-name-format) below. | | `env` | Optional | Per-scenario environment variables (merged with model config env) | | `kind` | Optional | Category label (e.g., `"tools"`) | | `max_turns` | Yes | Maximum number of conversation turns before the evaluation stops | +#### Tool name format + +Entries in `expected_trajectory` use the canonical form `__` (double-underscore separator) for MCP tools, and the bare name for native harness tools (e.g. `Read`, `Bash`, `run_shell_command`). Each harness adapter normalizes its raw tool-call event into this form at the boundary, so the same evalset can score runs from Codex, Claude Code, and Gemini CLI without modification. The `` segment comes from the MCP server key in your model config and is case-sensitive — e.g. `cloud-sql` or `bigtable`. See `evalbench/generators/models/tool_naming.py` for the canonicalization helper. + +By default the `trajectory_matcher` scorer drops native/harness-internal tools (anything that is **not** in canonical `__` form) from both the expected and actual lists before scoring, so authors can keep `expected_trajectory` focused on user-visible MCP intent without the score being dragged down by harness-internal calls like `update_topic` or `Bash`. Set `filter_native_tools: false` on the scorer if you intentionally want to score native-tool usage — see the [scorer configuration example](#scorer-configuration-example). + #### Writing Good Conversation Plans The `conversation_plan` is a critical part of each scenario. It instructs the simulated user LLM how to behave. Best practices: @@ -270,7 +276,7 @@ The `conversation_plan` is a critical part of each scenario. It instructs the si "id": "csql-create-ambiguous-multiturn-01", "starting_prompt": "I need a database.", "conversation_plan": "The user starts with a vague request. You want to CREATE a NEW Cloud SQL instance named 'my-pg-app'. If the agent offers to create one, say YES. When asked for details, provide 'my-pg-app' as the instance name and 'user_data' as the database name. Never claim to have an existing instance. The goal is for the agent to eventually create the database 'user_data' inside 'my-pg-app' in astana-evaluation project.", - "expected_trajectory": ["list_instances", "create_instance", "create_database"], + "expected_trajectory": ["cloud-sql__list_instances", "cloud-sql__create_instance", "cloud-sql__create_database"], "env": { "GOOGLE_CLOUD_PROJECT": "astana-evaluation" }, @@ -612,7 +618,7 @@ These require no additional model: | Scorer | Score Range | Description | |--------|------------|-------------| -| `trajectory_matcher` | 0–100 | Compares expected vs. actual tool usage. Uses **Jaccard Similarity** by default (set-based, order-insensitive). Set `enforce_order: true` for **Levenshtein distance** (order-sensitive). | +| `trajectory_matcher` | 0–100 | Compares expected vs. actual tool usage. Uses **Jaccard Similarity** by default (set-based, order-insensitive). Set `enforce_order: true` for **Levenshtein distance** (order-sensitive). Native/harness-internal tools are dropped from both sides before scoring by default; set `filter_native_tools: false` to keep them. See [Tool name format](#tool-name-format) for the canonical-name rule the filter uses. | | `turn_count` | Count | Reports the number of conversation turns the agent took. Lower is generally better. | | `end_to_end_latency` | Milliseconds | Total latency = model API latency + tool execution latency. | | `tool_call_latency` | Milliseconds | Sum of all tool execution durations across all turns. | @@ -624,9 +630,13 @@ These require no additional model: ```yaml scorers: # Deterministic scorers (no model needed) + # trajectory_matcher drops native/harness-internal tools (Read, Bash, + # run_shell_command, ...) by default — uncomment the expanded block + # below to opt out, or to enforce ordered matching. trajectory_matcher: {} # trajectory_matcher: - # enforce_order: true # Use Levenshtein for ordered matching + # filter_native_tools: false # keep native tools in the score + # enforce_order: true # use Levenshtein for ordered matching turn_count: {} end_to_end_latency: {} tool_call_latency: {} @@ -790,7 +800,7 @@ reporting: "id": "list-and-inspect-01", "starting_prompt": "list all instances in project my-project", "conversation_plan": "Ask the agent to list instances. Once listed, get details of the 'prod-db' instance and verify it is RUNNABLE.", - "expected_trajectory": ["list_instances", "get_instance"], + "expected_trajectory": ["cloud-sql__list_instances", "cloud-sql__get_instance"], "env": { "GOOGLE_CLOUD_PROJECT": "my-project" }, "kind": "tools", "max_turns": 3 @@ -866,7 +876,7 @@ reporting: "id": "fake-create-success", "starting_prompt": "Create a new Cloud SQL instance named 'test-db' in project 'my-project'.", "conversation_plan": "All details are in the prompt. The agent should call create_instance and report success.", - "expected_trajectory": ["create_instance"], + "expected_trajectory": ["cloud-sql__create_instance"], "env": { "GOOGLE_CLOUD_PROJECT": "my-project" }, "kind": "tools", "max_turns": 3 diff --git a/docs/judge_tools.md b/docs/judge_tools.md new file mode 100644 index 00000000..ebffc258 --- /dev/null +++ b/docs/judge_tools.md @@ -0,0 +1,179 @@ +# Judge Tools (Function Calling) + +LLM-judged scorers (`BinaryRubricScorer`, `LlmRater`, `GoalCompletionRate`, +`BehavioralMetrics`, `ParameterAnalysis`, `SkillsBestPractices`) call +`judge.generate(prompt)` and parse the text back. By default the judge is +single-shot: it sees the prompt and produces an answer with no access to +external state. + +Tool support lets a judge **invoke registered tools mid-generation** so it can +ground its answer in something the prompt alone can't tell it — for example, +fetching `https://beam.apache.org/get-started/downloads/` to check that an +agent named the most-recent Apache Beam version. + +> **Status:** Gemini (`gcp_vertex_gemini`) only. Claude SDK judge support is +> tracked as a follow-up. + +## Activation + +Tools are **opt-in per judge** via a `tools:` list in the model config YAML. +Configs without the key keep the original single-shot codepath. + +```yaml +generator: gcp_vertex_gemini +vertex_model: gemini-2.5-pro +base_prompt: "" +execs_per_minute: 5 +tools: + - fetch_url +``` + +Unknown tool names fail fast at `GeminiGenerator.__init__` with a `ValueError` +that lists the available tools. + +## Available tools + +| Name | Purpose | Constraints | +|---|---|---| +| `fetch_url` | Fetch an HTTPS URL and return its body as text. HTML is stripped to plain text. | HTTPS only; SSRF guard against private/loopback/link-local/multicast/reserved IPs; 10s timeout; 50 KB body cap. | + +`fetch_url` returns `"Error: ..."` strings on any failure (bad scheme, blocked +host, HTTP error, timeout) so the model can observe the failure and react +rather than crash the judge. + +## End-to-end example + +**`datasets/model_configs/gemini_2.5_pro_with_tools_model.yaml`** (new) + +```yaml +generator: gcp_vertex_gemini +vertex_model: gemini-2.5-pro +base_prompt: "" +execs_per_minute: 5 +tools: + - fetch_url +``` + +**`datasets/your-eval/example_run_config.yaml`** + +```yaml +dataset_config: datasets/your-eval/scenarios.evalset.json +dataset_format: agent-format +orchestrator: agent +model_config: datasets/model_configs/gemini_cli_model.yaml +simulated_user_model_config: datasets/model_configs/gemini_2.5_pro_model.yaml + +scorers: + binary_rubric_scorer: + model_config: datasets/model_configs/gemini_2.5_pro_with_tools_model.yaml + goal_completion: + model_config: datasets/model_configs/gemini_2.5_pro_model.yaml # no tools + trajectory_matcher: {} + turn_count: {} + +reporting: + csv: + output_directory: 'results' +``` + +**`datasets/your-eval/scenarios.evalset.json`** (rubric lives on the scenario) + +```json +{ + "scenarios": [ + { + "id": "beam-version-check", + "starting_prompt": "What is the most recent version of Apache Beam?", + "conversation_plan": "User wants the agent to look up the current Beam release.", + "binary_rubric": [ + "Did the agent's final answer state the most-recent Apache Beam version listed on https://beam.apache.org/get-started/downloads/?" + ], + "max_turns": 2 + } + ] +} +``` + +## How it threads through + +1. The run config's `scorers.binary_rubric_scorer.model_config` is loaded via + `get_generator(...)` inside `BinaryRubricScorer.__init__` + (`evalbench/scorers/binaryrubricscorer.py`). +2. That YAML has `generator: gcp_vertex_gemini` plus `tools: [fetch_url]`, so + `GeminiGenerator` resolves the names through + `evalbench/generators/models/tools/__init__.py::get_tools(...)` and builds + the native `types.Tool` config once at construction. +3. When the scorer calls `self.model.generate(prompt)`, the judge enters the + tool-calling loop. If the model emits a `function_call`, the registered + tool runs, its result is appended as a `FunctionResponse`, and the loop + continues until the model returns a final text answer. +4. The loop is bounded at `MAX_TOOL_ITERATIONS = 5` + (`evalbench/generators/models/gemini.py`). On exhaust the judge returns + the last text and logs a warning. + +Per-scorer judges are independent, so you can give `fetch_url` only to the +rubric judge and leave latency-sensitive judges (`behavioral_metrics`, +`parameter_analysis`) on plain single-shot configs. + +## Security notes + +- **HTTPS-only.** Other schemes (`http`, `file`, ...) are rejected before any + network call. +- **SSRF guard.** `fetch_url` resolves the hostname and rejects responses + pointing at private / loopback / link-local / multicast / reserved address + ranges (Python's `ipaddress` classification). +- **Known follow-ups:** + - **Redirect SSRF.** `urlopen` follows 3xx without re-validating each hop. + A malicious site can return a 302 to a private IP and bypass the upfront + check. Fix is a custom `HTTPRedirectHandler` that re-runs the host check + on every hop, or disabling redirects entirely. + - **DNS TOCTOU.** `_is_blocked_host` resolves the name once; `urlopen` + resolves it again. DNS rebinding can flip between the two. Hardening + requires socket-level interception. + +## Adding a new tool + +1. Create `evalbench/generators/models/tools/.py` exporting a + `Tool` instance: + + ```python + from .base import Tool + + def _impl(args: dict) -> str: + return f"got args: {args}" + + MY_TOOL = Tool( + name="my_tool", + description="One-sentence summary the model will read.", + input_schema={ + "type": "object", + "properties": {"x": {"type": "string"}}, + "required": ["x"], + }, + fn=_impl, + ) + ``` + +2. Register it in `evalbench/generators/models/tools/__init__.py`: + + ```python + from .my_tool import MY_TOOL + + TOOL_REGISTRY = { + FETCH_URL_TOOL.name: FETCH_URL_TOOL, + MY_TOOL.name: MY_TOOL, + } + ``` + +3. Reference it in any judge model config: + + ```yaml + tools: + - fetch_url + - my_tool + ``` + +Tool implementations should return strings (errors as `"Error: ..."`), keep +side effects bounded, and avoid raising — exceptions are caught and converted +into `"Error: : "` strings so the loop can continue, but a +well-behaved tool produces actionable error text itself. diff --git a/evalbench/__init__.py b/evalbench/__init__.py index 58a5b2b9..c846848d 100644 --- a/evalbench/__init__.py +++ b/evalbench/__init__.py @@ -1,3 +1,11 @@ +import os +import sys + +# Append package root to sys.path for legacy absolute imports. +# Using append (rather than insert) prevents namespace collisions in spawned child processes. +sys.path.append(os.path.dirname(__file__)) + + from . import reporting from . import util from . import dataset diff --git a/evalbench/databases/bigquery.py b/evalbench/databases/bigquery.py index b8cad5f5..09d4cfca 100644 --- a/evalbench/databases/bigquery.py +++ b/evalbench/databases/bigquery.py @@ -22,7 +22,7 @@ class BQDB(DB): def __init__(self, db_config): super().__init__(db_config) - self.project_id = get_gcp_project("") + self.project_id = get_gcp_project(db_config.get("gcp_project_id", "")) self.location = db_config.get("location", "US") self.client = bigquery.Client(project=self.project_id) self.tmp_users = [] @@ -176,6 +176,11 @@ def generate_ddl(self, schema: DatabaseSchema) -> List[str]: print(f"Error generating DDL statements: {e}") return ddl_statements + def ensure_database_exists(self, database_name: str) -> None: + dataset_ref = bigquery.Dataset(f"{self.project_id}.{database_name}") + dataset_ref.location = self.location + self.client.create_dataset(dataset_ref, exists_ok=True) + def create_tmp_database(self, database_name: str): dataset_ref = bigquery.Dataset(f"{self.project_id}.{database_name}") dataset_ref.location = self.location diff --git a/evalbench/databases/db.py b/evalbench/databases/db.py index e287b3ee..7d6a5fb2 100644 --- a/evalbench/databases/db.py +++ b/evalbench/databases/db.py @@ -231,6 +231,21 @@ def delete_tmp_user(self, username: str) -> None: """ raise NotImplementedError("Subclasses must implement this method") + def _format_boolean_value(self, val: str) -> Any: + return val + + def _clean_insert_value(self, val: Any) -> Any: + if val is None or str(val).strip().lower() in ("", "null", "none"): + return None + v = str(val).strip() + if v.startswith("'") and v.endswith("'"): + v = v[1:-1] + + lower_v = v.lower() + if lower_v in ("true", "false"): + return self._format_boolean_value(lower_v) + return v + @abstractmethod def close_connections(self) -> None: """ diff --git a/evalbench/databases/mysql.py b/evalbench/databases/mysql.py index 30e78545..52558bbc 100644 --- a/evalbench/databases/mysql.py +++ b/evalbench/databases/mysql.py @@ -1,5 +1,5 @@ -import sqlalchemy import sqlparse +import sqlalchemy import pymysql from sqlalchemy import text, MetaData from sqlalchemy.engine.base import Connection @@ -61,6 +61,8 @@ def __init__(self, db_config): self.use_adc = not self.username and not self.password if self.use_adc: self.username = get_adc_user_email() + if self.username and self.username.endswith(".gserviceaccount.com"): + self.username = self.username.replace(".gserviceaccount.com", "") def get_conn(): """Callable for sqlalchemy 'creator' parameter.""" @@ -239,8 +241,8 @@ def get_metadata(self) -> dict: columns.append( {"name": column.name, "type": str(column.type)}) db_metadata[table.name] = columns - except Exception: - pass + except Exception as e: + logging.warning(f"Failed to reflect database metadata: {e}") return db_metadata @@ -313,23 +315,35 @@ def drop_all_tables(self): DROP_ALL_TABLES_QUERY.format(DATABASE=self.db_name).split(";") ) - def insert_data(self, data: dict[str, List[str]], - setup: Optional[List[str]] = None): + def insert_data(self, data: dict[str, List[str]], setup: Optional[List[str]] = None) -> None: if not data: return - insertion_statements = [] - for table_name in data: - for row in data[table_name]: - inline_columns = ", ".join([f"{value}" for value in row]) - insertion_statements.append( - f"INSERT INTO `{table_name}` VALUES ({inline_columns});" - ) + try: - self.batch_execute(insertion_statements) - except RuntimeError as error: + with self.engine.begin() as connection: + for table_name in data: + rows = data[table_name] + if not rows: + continue + + num_cols = len(rows[0]) + param_placeholders = ", ".join([f":v{i}" for i in range(num_cols)]) + stmt = text(f"INSERT INTO `{table_name}` VALUES ({param_placeholders})") + + params = [] + for row in rows: + p = {} + for i, val in enumerate(row): + p[f"v{i}"] = self._clean_insert_value(val) + params.append(p) + + connection.execute(stmt, params) + except Exception as error: raise RuntimeError(f"Could not insert data into database: {error}") - ##################################################### + def _format_boolean_value(self, val: str) -> Any: + return 1 if val == "true" else 0 + ###################################################### ##################################################### # Database User Management ##################################################### diff --git a/evalbench/databases/postgres.py b/evalbench/databases/postgres.py index b4fa6207..de2b77bc 100644 --- a/evalbench/databases/postgres.py +++ b/evalbench/databases/postgres.py @@ -3,6 +3,7 @@ from sqlalchemy import text, MetaData from sqlalchemy.engine.base import Connection import logging +import os from .db import DB from google.cloud.sql.connector import Connector from util.auth import get_adc_user_email @@ -54,11 +55,14 @@ def __init__(self, db_config): self.use_cloud_sql = (self.db_path.count(":") == 2) # Normalize password for drivers that dislike None - effective_password = self.password if self.password is not None else "" + if self.password is None: + self.password = "" self.use_adc = not self.username and not self.password if self.use_adc: self.username = get_adc_user_email() + if self.username and self.username.endswith(".gserviceaccount.com"): + self.username = self.username.replace(".gserviceaccount.com", "") def get_conn(): # Only used for Cloud SQL Connector path @@ -66,7 +70,7 @@ def get_conn(): self.db_path, "pg8000", user=self.username, - password=effective_password, + password=self.password, db=self.db_name, enable_iam_auth=self.use_adc, ) @@ -86,17 +90,14 @@ def get_engine_config(): else: # Standard local connection via URL args["connect_args"]["timeout"] = 60 - pass_str = effective_password if effective_password is not None else "" - # Check for local UNIX socket - import os socket_path = "/var/run/postgresql/.s.PGSQL.5432" if self.db_path == "localhost" and os.path.exists(socket_path): args["connect_args"]["unix_sock"] = socket_path # Use a slash-only URL so SQLAlchemy doesn't force a TCP host - url = f"postgresql+pg8000://{self.username}:{pass_str}@/{self.db_name}" + url = f"postgresql+pg8000://{self.username}:{self.password}@/{self.db_name}" else: - url = f"postgresql+pg8000://{self.username}:{pass_str}@{self.db_path}/{self.db_name}" + url = f"postgresql+pg8000://{self.username}:{self.password}@{self.db_path}/{self.db_name}" if "is_tmp_db" in db_config: args["poolclass"] = NullPool @@ -204,8 +205,8 @@ def get_metadata(self) -> dict: columns.append( {"name": column.name, "type": str(column.type)}) db_metadata[table.name] = columns - except Exception: - pass + except Exception as e: + logging.warning(f"Failed to reflect database metadata: {e}") return db_metadata @@ -243,9 +244,6 @@ def drop_tmp_database(self, database_name: str): logging.error(f"Could not delete database: {error}") def ensure_database_exists(self, database_name: str) -> None: - from google.cloud.sql.connector import Connector - import sqlalchemy - from sqlalchemy import text connector = Connector() try: @@ -268,24 +266,35 @@ def drop_all_tables(self): if error: raise RuntimeError(error) - def insert_data( - self, data: dict[str, List[str]], setup: Optional[List[str]] = None - ): + def insert_data(self, data: dict[str, List[str]], setup: Optional[List[str]] = None) -> None: if not data: return - insertion_statements = [] - for table_name in data: - for row in data[table_name]: - inline_columns = ", ".join([f"{value}" for value in row]) - insertion_statements.append( - f"INSERT INTO public.{table_name} VALUES ({inline_columns});" - ) + try: - self.batch_execute(insertion_statements) - except RuntimeError as error: + with self.engine.begin() as connection: + for table_name in data: + rows = data[table_name] + if not rows: + continue + + num_cols = len(rows[0]) + param_placeholders = ", ".join([f":v{i}" for i in range(num_cols)]) + stmt = text(f"INSERT INTO public.\"{table_name}\" VALUES ({param_placeholders})") + + params = [] + for row in rows: + p = {} + for i, val in enumerate(row): + p[f"v{i}"] = self._clean_insert_value(val) + params.append(p) + + connection.execute(stmt, params) + except Exception as error: raise RuntimeError(f"Could not insert data into database: {error}") - ##################################################### + def _format_boolean_value(self, val: str) -> Any: + return val + ###################################################### ##################################################### # Database User Management ##################################################### diff --git a/evalbench/databases/spanner.py b/evalbench/databases/spanner.py index aa5555d8..24c4628d 100644 --- a/evalbench/databases/spanner.py +++ b/evalbench/databases/spanner.py @@ -30,6 +30,7 @@ def __init__(self, db_config): self.config = db_config self.dialect = db_config.get("dialect", "spanner_gsql") self.db_type = "spanner" + self.query_timeout = db_config.get("query_timeout", 120) self.engine = None self.emulator_manager = None @@ -37,7 +38,8 @@ def __init__(self, db_config): "use_managed_emulator", False) raw_dialect = self.dialect.lower() - logging.debug(f"SpannerDB init for {db_config.get('database_name')} with self.dialect={self.dialect}") + logging.debug( + f"SpannerDB init for {db_config.get('database_name')} with self.dialect={self.dialect}") if "pg" in raw_dialect or "postgres" in raw_dialect: self.dialect_enum = DatabaseDialect.POSTGRESQL self.expected_dialect_str = "POSTGRESQL" @@ -67,7 +69,8 @@ def __init__(self, db_config): client_kwargs["disable_builtin_metrics"] = True client = spanner.Client(**client_kwargs) self.spanner_instance = client.instance(self.instance_id) - self.database = self.spanner_instance.database(db_name, database_dialect=self.dialect_enum) + self.database = self.spanner_instance.database( + db_name, database_dialect=self.dialect_enum) def close_connections(self): if self.emulator_manager: @@ -76,25 +79,34 @@ def close_connections(self): def batch_execute(self, commands: list[str]): if not commands: return - logging.debug(f"Executing batch in {self.database.database_id}. Object dialect: {self.database.database_dialect}") - logging.debug(f"Executing batch of {len(commands)} statements in Spanner {self.expected_dialect_str} for {self.database.database_id}") + logging.debug( + f"Executing batch in {self.database.database_id}. Object dialect: {self.database.database_dialect}") + logging.debug( + f"Executing batch of {len(commands)} statements in Spanner {self.expected_dialect_str} for {self.database.database_id}") if commands: logging.debug(f"First statement: {commands[0][:100]}...") - try: - op = self.database.update_ddl(commands) - op.result(timeout=600) - except Exception as e: - logging.warning( - f"update_ddl failed, trying individual execution: {e}") - for stmt in commands: - _, _, error = self.execute(stmt) - if error: - # Ignore 'already exists' / 'Duplicate name' errors during fallback - if "Duplicate name" in error or "already exists" in error: - logging.info(f"Ignoring duplicate error during fallback: {error}") - else: - raise RuntimeError( - f"Error in batch statement: {stmt}\nError: {error}") + # Chunk DDL statements into groups of 10 to avoid limit + chunk_size = 10 + for i in range(0, len(commands), chunk_size): + chunk = commands[i:i + chunk_size] + try: + print( + f"Creating tables in {self.database.name} with {len(chunk)} commands (chunk {i // chunk_size + 1})") + op = self.database.update_ddl(chunk) + op.result(timeout=600) + except Exception as e: + logging.warning( + f"update_ddl failed for chunk {i // chunk_size + 1}, trying individual execution: {e}") + for stmt in chunk: + _, _, error = self.execute(stmt) + if error: + # Ignore 'already exists' / 'Duplicate name' errors during fallback + if "Duplicate name" in error or "already exists" in error: + logging.info( + f"Ignoring duplicate error during fallback: {error}") + else: + raise RuntimeError( + f"Error in batch statement: {stmt}\nError: {error}") def execute(self, query, eval_query=None, use_cache=False, rollback=False): if query.strip() == "": @@ -102,13 +114,28 @@ def execute(self, query, eval_query=None, use_cache=False, rollback=False): # Detect DDL upper_query = query.strip().upper() - is_ddl = any(upper_query.startswith(prefix) for prefix in ["CREATE", "ALTER", "DROP", "RENAME"]) + is_ddl = any(upper_query.startswith(prefix) + for prefix in ["CREATE", "ALTER", "DROP", "RENAME"]) if is_ddl: - logging.info(f"Executing DDL in Spanner: {query[:100]}...") + logging.info(f"Executing mixed DDL/DML sequence in Spanner...") try: - op = self.database.update_ddl([query]) - op.result(timeout=600) + statements = [s.strip() for s in query.split(";") if s.strip()] + for stmt in statements: + stmt = stmt.strip() + if stmt.endswith(";"): + stmt = stmt[:-1].strip() + if not stmt: + continue + upper_stmt = stmt.upper() + is_sub_ddl = any(upper_stmt.startswith(prefix) for prefix in [ + "CREATE", "ALTER", "DROP", "RENAME"]) + if is_sub_ddl: + op = self.database.update_ddl([stmt]) + op.result(timeout=600) + else: + # Route DML statement to normal execution + self._execute(stmt, eval_query=None, rollback=False) return [{"status": "success"}], None, None except Exception as e: return None, None, str(e) @@ -136,10 +163,11 @@ def _tx_logic(transaction): try: if is_dml: rows_affected = transaction.execute_update( - query, timeout=15) + query, timeout=self.query_timeout) result = [{"rows_affected": rows_affected}] else: - res = transaction.execute_sql(query, timeout=15) + res = transaction.execute_sql( + query, timeout=self.query_timeout) rows = list(res) fields = [ f.name for f in res.fields] if res.fields else [] @@ -147,7 +175,7 @@ def _tx_logic(transaction): if eval_query: res_eval = transaction.execute_sql( - eval_query, timeout=15) + eval_query, timeout=self.query_timeout) rows_eval = list(res_eval) fields_eval = [ f.name for f in res_eval.fields] if res_eval.fields else [] @@ -167,7 +195,8 @@ def _tx_logic(transaction): else: try: with self.database.snapshot() as snapshot: - res = snapshot.execute_sql(query, timeout=15) + res = snapshot.execute_sql( + query, timeout=self.query_timeout) rows = list(res) fields = [ f.name for f in res.fields] if res.fields else [] @@ -175,7 +204,7 @@ def _tx_logic(transaction): if eval_query: res_eval = snapshot.execute_sql( - eval_query, timeout=15) + eval_query, timeout=self.query_timeout) rows_eval = list(res_eval) fields_eval = [ f.name for f in res_eval.fields] if res_eval.fields else [] @@ -204,7 +233,7 @@ def get_metadata(self): type_col = "spanner_type" if self.expected_dialect_str == "GOOGLESQL" else "data_type" query = f"SELECT table_name, column_name, {type_col} FROM information_schema.columns WHERE table_schema = '{schema_name}' ORDER BY table_name, ordinal_position" with self.database.snapshot() as snapshot: - res = snapshot.execute_sql(query, timeout=15) + res = snapshot.execute_sql(query, timeout=self.query_timeout) for row in res: t_name, c_name, d_type = row[0], row[1], row[2] if t_name not in db_metadata: @@ -213,10 +242,12 @@ def get_metadata(self): {"name": c_name, "type": str(d_type)}) if db_metadata: - logging.info(f"Metadata extracted for {len(db_metadata)} tables in Spanner {self.expected_dialect_str}") + logging.info( + f"Metadata extracted for {len(db_metadata)} tables in Spanner {self.expected_dialect_str}") return db_metadata else: - logging.warning(f"No metadata found in Spanner {self.expected_dialect_str} information_schema for schema '{schema_name}'") + logging.warning( + f"No metadata found in Spanner {self.expected_dialect_str} information_schema for schema '{schema_name}'") except Exception as e: logging.error(f"Native metadata inspection failed: {e}") return db_metadata @@ -231,20 +262,24 @@ def generate_ddl(self, schema: DatabaseSchema) -> str: def create_tmp_database(self, database_name): # Spanner database IDs cannot end with an underscore database_name = database_name.rstrip("_") - logging.info(f"Creating temporary Spanner database: {database_name}...") + logging.info( + f"Creating temporary Spanner database: {database_name}...") self.ensure_database_exists(database_name) def drop_tmp_database(self, database_name): database_name = database_name.rstrip("_") - logging.info(f"Dropping temporary Spanner database: {database_name}...") + logging.info( + f"Dropping temporary Spanner database: {database_name}...") try: spanner_client = spanner.Client(disable_builtin_metrics=True) instance = spanner_client.instance(self.instance_id) database = instance.database(database_name) database.drop() - logging.info(f"Successfully dropped Spanner database {database_name}.") + logging.info( + f"Successfully dropped Spanner database {database_name}.") except Exception as e: - logging.warning(f"Failed to drop temporary Spanner database {database_name}: {e}") + logging.warning( + f"Failed to drop temporary Spanner database {database_name}: {e}") def resetup_database(self, force=False, setup_users=False) -> None: # For Spanner, we need to ensure the database exists before we can resetup it @@ -254,13 +289,15 @@ def resetup_database(self, force=False, setup_users=False) -> None: # Verify the backend dialect matches what we expect self.database.reload() if self.database.database_dialect != self.dialect_enum: - logging.warning(f"Database {db_id} exists but has wrong dialect ({self.database.database_dialect} != {self.dialect_enum}). Dropping it.") + logging.warning( + f"Database {db_id} exists but has wrong dialect ({self.database.database_dialect} != {self.dialect_enum}). Dropping it.") self.drop_tmp_database(db_id) # Wait for drop to complete (drop is usually fast, but just in case) time.sleep(2) if not self.database.exists(): - logging.info(f"Database {db_id} does not exist. Creating it before setup...") + logging.info( + f"Database {db_id} does not exist. Creating it before setup...") self.create_tmp_database(db_id) super().resetup_database(force=force, setup_users=setup_users) @@ -270,11 +307,13 @@ def ensure_database_exists(self, database_name: str) -> None: instance_id = self.instance_id instance = spanner_client.instance(instance_id) # Create database with the configured dialect - database = instance.database(database_name, database_dialect=self.dialect_enum) + database = instance.database( + database_name, database_dialect=self.dialect_enum) try: op = database.create() op.result() # Wait for completion - logging.info(f"Successfully created Spanner database {database_name}.") + logging.info( + f"Successfully created Spanner database {database_name}.") except exceptions.AlreadyExists: pass except Exception as e: @@ -297,32 +336,124 @@ def ensure_database_exists(self, database_name: str) -> None: raise RuntimeError( f"Failed to create Spanner DB {database_name}: {e}") from e + def _get_quote_char(self): + return '"' if self.expected_dialect_str == "POSTGRESQL" else '`' + + def _execute_ddl_batch(self, commands, batch_size=50): + for i in range(0, len(commands), batch_size): + op = self.database.update_ddl(commands[i:i + batch_size]) + op.result(timeout=300) + + def _drop_views(self): + with self.database.snapshot() as snapshot: + if self.expected_dialect_str == "POSTGRESQL": + res = snapshot.execute_sql( + "SELECT table_name FROM information_schema.tables WHERE table_schema = 'public' AND table_type = 'VIEW'", timeout=self.query_timeout) + else: + res = snapshot.execute_sql( + "SELECT table_name FROM information_schema.tables WHERE (table_schema = '' OR table_schema IS NULL) AND table_type = 'VIEW'", timeout=self.query_timeout) + view_names = [row[0] for row in res] + + if view_names: + quote = self._get_quote_char() + commands = [f"DROP VIEW {quote}{v}{quote}" for v in view_names] + try: + self._execute_ddl_batch(commands) + except Exception: + for v in view_names: + try: + self.database.update_ddl( + [f"DROP VIEW {quote}{v}{quote}"]).result(timeout=60) + except Exception: + pass + + def _drop_indices(self): + with self.database.snapshot() as snapshot: + query = "SELECT index_name FROM information_schema.indexes WHERE (table_schema = '' OR table_schema IS NULL OR table_schema = 'public') AND index_name != 'PRIMARY_KEY'" + res = snapshot.execute_sql(query, timeout=self.query_timeout) + index_names = [row[0] for row in res] + + if index_names: + quote = self._get_quote_char() + commands = [ + f"DROP INDEX {quote}{idx}{quote}" for idx in index_names] + try: + self._execute_ddl_batch(commands) + except Exception: + for idx in index_names: + try: + self.database.update_ddl( + [f"DROP INDEX {quote}{idx}{quote}"]).result(timeout=60) + except Exception: + pass + + def _drop_foreign_keys(self): + with self.database.snapshot() as snapshot: + if self.expected_dialect_str == "POSTGRESQL": + query = "SELECT table_name, constraint_name FROM information_schema.table_constraints WHERE table_schema = 'public' AND constraint_type = 'FOREIGN KEY'" + else: + query = "SELECT table_name, constraint_name FROM information_schema.table_constraints WHERE (table_schema = '' OR table_schema IS NULL) AND constraint_type = 'FOREIGN KEY'" + res = snapshot.execute_sql(query, timeout=self.query_timeout) + fk_info = [(row[0], row[1]) for row in res] + + if fk_info: + quote = self._get_quote_char() + commands = [ + f"ALTER TABLE {quote}{table}{quote} DROP CONSTRAINT {quote}{fk}{quote}" for table, fk in fk_info] + try: + self._execute_ddl_batch(commands) + except Exception: + for table, fk in fk_info: + try: + self.database.update_ddl( + [f"ALTER TABLE {quote}{table}{quote} DROP CONSTRAINT {quote}{fk}{quote}"]).result(timeout=60) + except Exception: + pass + + def _drop_tables(self): + with self.database.snapshot() as snapshot: + if self.expected_dialect_str == "POSTGRESQL": + res = snapshot.execute_sql( + "SELECT table_name FROM information_schema.tables WHERE table_schema = 'public' AND table_type = 'BASE TABLE'", timeout=self.query_timeout) + else: + res = snapshot.execute_sql( + "SELECT table_name FROM information_schema.tables WHERE (table_schema = '' OR table_schema IS NULL) AND table_type = 'BASE TABLE'", timeout=self.query_timeout) + table_names = [row[0] for row in res] + + if not table_names: + return + + quote = self._get_quote_char() + commands = [f"DROP TABLE {quote}{t}{quote}" for t in table_names] + + try: + self._execute_ddl_batch(commands) + except Exception: + pending_tables = table_names + for _ in range(5): + if not pending_tables: + break + next_pending = [] + for t in pending_tables: + try: + self.database.update_ddl( + [f"DROP TABLE {quote}{t}{quote}"]).result(timeout=30) + except Exception as e: + if "Table not found" not in str(e) and "does not exist" not in str(e): + next_pending.append(t) + pending_tables = next_pending + def drop_all_tables(self): try: if not self.database.exists(): - logging.info(f"Database {self.database.database_id} does not exist. Skipping drop_all_tables.") + logging.info( + f"Database {self.database.database_id} does not exist. Skipping drop_all_tables.") return - with self.database.snapshot() as snapshot: - schema_name = 'public' if self.expected_dialect_str == "POSTGRESQL" else '' - res = snapshot.execute_sql(f"SELECT table_name FROM information_schema.tables WHERE table_schema = '{schema_name}' AND table_type = 'BASE TABLE'", timeout=15) - table_names = [row[0] for row in res] - if not table_names: - return - pending_tables = table_names - for _ in range(5): - if not pending_tables: - break - next_pending = [] - quote = '"' if self.expected_dialect_str == "POSTGRESQL" else '`' - for t in pending_tables: - try: - op = self.database.update_ddl( - [f"DROP TABLE {quote}{t}{quote}"]) - op.result(timeout=60) - except Exception: - next_pending.append(t) - pending_tables = next_pending + self._drop_views() + self._drop_indices() + self._drop_foreign_keys() + self._drop_tables() except Exception: pass @@ -335,7 +466,7 @@ def insert_data(self, data, setup=None): with self.database.snapshot() as snapshot: type_col = "spanner_type" if self.expected_dialect_str == "GOOGLESQL" else "data_type" query = f"SELECT table_name, column_name, {type_col} FROM information_schema.columns WHERE table_schema = '{schema_name}' ORDER BY table_name, ordinal_position" - res = snapshot.execute_sql(query, timeout=15) + res = snapshot.execute_sql(query, timeout=self.query_timeout) for row in res: t_name, c_name, d_type = row[0], row[1], row[2] if t_name not in table_info: diff --git a/evalbench/databases/sqlite.py b/evalbench/databases/sqlite.py index f3f6d3ae..33f6f470 100644 --- a/evalbench/databases/sqlite.py +++ b/evalbench/databases/sqlite.py @@ -3,9 +3,10 @@ from sqlalchemy import text, MetaData from sqlalchemy.engine.base import Connection import logging -import sqlite3 -import os import sqlparse +import os +import sqlite3 +import shutil from .db import DB from .util import ( with_cache_execute, @@ -162,8 +163,8 @@ def get_metadata(self) -> dict: columns.append( {"name": column.name, "type": str(column.type)}) db_metadata[table.name] = columns - except Exception: - pass + except Exception as e: + logging.warning(f"Failed to reflect database metadata: {e}") return db_metadata @@ -205,7 +206,6 @@ def create_tmp_database(self, database_name: str): for src_name in source_candidates: source_path = self._get_connection_path(self.db_path, src_name) if os.path.exists(source_path) and os.path.getsize(source_path) > 0: - import shutil shutil.copy2(source_path, target_path) copied = True break @@ -228,8 +228,6 @@ def drop_tmp_database(self, database_name: str): logging.error(f"Could not delete database: {error}") def ensure_database_exists(self, database_name: str) -> None: - import sqlite3 - import os filename = f"{self.db_path}/{database_name}{self.extension}" os.makedirs(self.db_path, exist_ok=True) conn = sqlite3.connect(filename) @@ -249,22 +247,36 @@ def drop_all_tables(self): except Exception as error: logging.error(f"Failed to drop all tables: {error}") - def insert_data(self, data: dict[str, List[str]], setup: Optional[List[str]] = None): + def insert_data(self, data: dict[str, List[str]], setup: Optional[List[str]] = None) -> None: if not data: return - insertion_statements = [] - for table_name in data: - for row in data[table_name]: - inline_columns = ", ".join([f"{value}" for value in row]) - insertion_statements.append( - f"INSERT INTO `{table_name}` VALUES ({inline_columns});" - ) + try: - self.batch_execute(insertion_statements) - except RuntimeError as error: + with self.engine.begin() as connection: + for table_name in data: + rows = data[table_name] + if not rows: + continue + + num_cols = len(rows[0]) + param_placeholders = ", ".join([f":v{i}" for i in range(num_cols)]) + stmt = text(f"INSERT INTO `{table_name}` VALUES ({param_placeholders})") + + params = [] + for row in rows: + p = {} + for i, val in enumerate(row): + p[f"v{i}"] = self._clean_insert_value(val) + params.append(p) + + connection.execute(stmt, params) + except Exception as error: raise RuntimeError(f"Could not insert data into database: {error}") - ##################################################### + def _format_boolean_value(self, val: str) -> Any: + return 1 if val == "true" else 0 + + ###################################################### ##################################################### # Database User Management ##################################################### diff --git a/evalbench/databases/sqlserver.py b/evalbench/databases/sqlserver.py index 3aa70a13..dd07cede 100644 --- a/evalbench/databases/sqlserver.py +++ b/evalbench/databases/sqlserver.py @@ -189,8 +189,8 @@ def get_metadata(self) -> dict: columns.append( {"name": column.name, "type": str(column.type)}) db_metadata[table.name] = columns - except Exception: - pass + except Exception as e: + logging.warning(f"Failed to reflect database metadata: {e}") return db_metadata diff --git a/evalbench/databases/util.py b/evalbench/databases/util.py index 7d6ae908..5ea95c20 100644 --- a/evalbench/databases/util.py +++ b/evalbench/databases/util.py @@ -7,6 +7,7 @@ import redis import re from dataclasses import dataclass, field +from util.safe_pickle import safe_pickle_loads # CLIENT is initialized lazily to avoid blocking network calls during module import, # which can cause hangs in restricted environments like Cloud Build. @@ -52,21 +53,22 @@ def _is_db_secret_path(secret: str) -> bool: return bool(re.match(pattern, secret)) +def access_secret(secret_path: str) -> str: + """Returns the decoded UTF-8 payload of a Secret Manager version. + + Caller is responsible for validating the resource path shape. + """ + request = secretmanager_v1.AccessSecretVersionRequest(name=secret_path) + response = get_client().access_secret_version(request=request) + return response.payload.data.decode("utf-8") + + def get_db_secret(secret): if not _is_db_secret_path(secret): raise ValueError( "secret manager path not parsable. Could not recover password for DB." ) - secret_path = secret - # Initialize request argument(s) - request = secretmanager_v1.AccessSecretVersionRequest( - name=secret_path, - ) - # Make the request - response = get_client().access_secret_version(request=request) - - # Return the secret - return response.payload.data.decode("utf-8") + return access_secret(secret) def generate_ddl(data, db_name, comments_data=None): @@ -124,7 +126,7 @@ def with_cache_execute( cached_result = cache_client.get(query_hash) if cached_result: logging.debug(f"Using cached result for query: {query}") - return pickle.loads(cached_result), None, None + return safe_pickle_loads(cached_result), None, None except Exception as e: logging.warning(f"Failed to retrieve query from cache: {e}") diff --git a/evalbench/dataset/cortadoinput.py b/evalbench/dataset/cortadoinput.py new file mode 100644 index 00000000..0eec1e3d --- /dev/null +++ b/evalbench/dataset/cortadoinput.py @@ -0,0 +1,61 @@ +import json +import copy + + +class EvalCortadoRequest: + def __init__(self, raw_dict: dict, job_id: str = "", trace_id: str = ""): + """ + Initializes an EvalCortadoRequest from a parsed JSON dictionary. + """ + # Store the raw dictionary so process_scenario can read max_turns and the plan + self.scenario = raw_dict + + # Extract top-level identification + self.id = str(raw_dict.get("id", "-1")) + self.job_id = job_id + self.trace_id = trace_id + + # Evalbench core routing needs these to match the YAML + self.dialect = raw_dict.get("dialect", "") + self.dialects = [self.dialect] + self.database = raw_dict.get("database", "") + self.nl_prompt = raw_dict.get("starting_prompt", "") + + # Ensure the stringified payload is ready for the gRPC Proxy + self.payload_str = json.dumps(raw_dict) + self.payload = self.payload_str + + self.agent_results = [] + self.scoring_results = [] + + @classmethod + def init_from_proto(cls, proto): + """Unpacks the Protobuf from Google3 back into the object.""" + payload_str = getattr(proto, "payload", "{}") + try: + raw_dict = json.loads(payload_str) + except json.JSONDecodeError: + raw_dict = {} + + raw_dict["id"] = str(getattr(proto, "id", "-1")) + + return cls( + raw_dict=raw_dict, + job_id=getattr(proto, "job_id", ""), + trace_id=getattr(proto, "trace_id", ""), + ) + + def to_proto(self): + """Packs the object into the Protobuf to send to Google3.""" + # Note: You must import eval_request_pb2 here to prevent circular dependencies + from evalproto import eval_request_pb2 + + return eval_request_pb2.EvalInputRequest( + id=int(self.id) if self.id.isdigit() else 0, + payload=self.payload_str, + # We map starting_prompt to nl_prompt for backwards compatibility + nl_prompt=self.nl_prompt + ) + + def copy(self): + return copy.deepcopy(self) diff --git a/evalbench/dataset/dataengineeringagentinput.py b/evalbench/dataset/dataengineeringagentinput.py new file mode 100644 index 00000000..ea1d084e --- /dev/null +++ b/evalbench/dataset/dataengineeringagentinput.py @@ -0,0 +1,62 @@ +import json +import copy +from typing import Any + + +class EvalDeaRequest: + def __init__(self, raw_dict: dict, job_id: str = "", trace_id: str = ""): + """ + Initializes an EvalDeaRequest from a parsed JSON dictionary. + """ + # Store the raw dictionary so process_scenario can read max_turns and the plan + self.scenario = raw_dict + + # Extract top-level identification + self.id = str(raw_dict.get("id", "-1")) + self.job_id = job_id + self.trace_id = trace_id + + self.nl_prompt = raw_dict.get("starting_prompt", "") + + # Ensure the stringified payload is ready for the A2A connection + self.payload_str = json.dumps(raw_dict) + self.payload = self.payload_str + + self.agent_results: list[dict[str, Any]] = [] + self.scoring_results: list[dict[str, Any]] = [] + + self.accumulated_tools: list[str] = [] + self.this_turn_tool_details: list[dict[str, Any]] = [] + self.seen_tool_ids: set[str] = set() + + @classmethod + def init_from_proto(cls, proto): + """Unpacks the Protobuf from Google3 back into the object.""" + payload_str = getattr(proto, "payload", "{}") + try: + raw_dict = json.loads(payload_str) + except json.JSONDecodeError: + raw_dict = {} + + raw_dict["id"] = str(getattr(proto, "id", "-1")) + + return cls( + raw_dict=raw_dict, + job_id=getattr(proto, "job_id", ""), + trace_id=getattr(proto, "trace_id", ""), + ) + + def to_proto(self): + """Packs the object into the Protobuf to send to Google3.""" + # Note: You must import eval_request_pb2 here to prevent circular dependencies + from evalproto import eval_request_pb2 + + return eval_request_pb2.EvalInputRequest( + id=int(self.id) if self.id.isdigit() else 0, + payload=self.payload_str, + # We map starting_prompt to nl_prompt for backwards compatibility + nl_prompt=self.nl_prompt + ) + + def copy(self): + return copy.deepcopy(self) diff --git a/evalbench/dataset/dataset.py b/evalbench/dataset/dataset.py index fa1ed066..ab0feb5b 100644 --- a/evalbench/dataset/dataset.py +++ b/evalbench/dataset/dataset.py @@ -3,12 +3,38 @@ from typing import Any, Optional import json import logging +import fnmatch from collections.abc import Sequence from dataset.evalinput import EvalInputRequest from dataset.evalinteractinput import EvalInteractInputRequest from dataset.evalgeminicliinput import EvalGeminiCliRequest +from dataset.cortadoinput import EvalCortadoRequest +from dataset.dataengineeringagentinput import EvalDeaRequest from itertools import chain import os +import re + + +_ENV_PLACEHOLDER_RE = re.compile(r"\$\{([^}]+)\}") + + +def _expand_env_placeholders(text: str, source_path: str) -> str: + """Substitutes ``${VAR}`` placeholders in a raw dataset from the + environment, so values like a GCP project are supplied at runtime instead + of hard-coded. Fails fast on any unresolved placeholder rather than + passing a literal ``${VAR}`` to the agent. + """ + expanded = os.path.expandvars(text) + unresolved = sorted( + {m.group(1) for m in _ENV_PLACEHOLDER_RE.finditer(expanded)} + ) + if unresolved: + raise ValueError( + f"Unresolved ${{...}} placeholder(s) in dataset {source_path}: " + f"{unresolved}. Export the corresponding environment variable(s) " + f"before running (e.g. `export {unresolved[0]}=...`)." + ) + return expanded def load_schema(dataset_dir: str, selected_database: str): @@ -35,8 +61,8 @@ def load_knowledge( if not line.strip(): continue obj = json.loads(line) - if obj.get("id") not in exclude_ids: - external_kg_list.append(json.dumps(obj)) + if obj.get("id") not in exclude_ids: + external_kg_list.append(json.dumps(obj)) external_kg = "\n".join(external_kg_list) return external_kg @@ -95,16 +121,114 @@ def load_bird_interact_dataset(json_file_path, config): return input_items -def load_gemini_cli_json(json_file_path): +def load_dea_json(json_file_path, config): + all_items: dict[str, list[EvalDeaRequest]] = { + "dea-format": [], + } + with open(json_file_path, "r") as json_file: + content = json_file.read() + data = json.loads(content) + + # Filter scenarios + scenarios = data.get("scenarios", []) + filtered_scenarios = _filter_scenarios(scenarios, config) + if scenarios and not filtered_scenarios: + return all_items + + if "scenarios" in data: + data["scenarios"] = filtered_scenarios + + eval_input = EvalDeaRequest( + raw_dict=data + ) + all_items["dea-format"].append(eval_input) + + return all_items + + +def _filter_scenarios(scenarios: list[dict], config: dict) -> list[dict]: + """Filters a list of scenarios based on explicit IDs or glob pattern in config.""" + scenarios_to_run = config.get("scenarios", []) + if isinstance(scenarios_to_run, str): + scenarios_to_run = [s.strip() for s in scenarios_to_run.split(",") if s.strip()] + + scenario_pattern = config.get("scenario_pattern", None) + + # If no filters are specified, return the original list (run all) + if not scenarios_to_run and not scenario_pattern: + return scenarios + + filtered_scenarios = [] + for scenario in scenarios: + scenario_id = scenario.get("id") + if not scenario_id: + continue + + # Match explicit list of IDs + if scenarios_to_run and scenario_id not in scenarios_to_run: + continue + + # Match glob pattern + if scenario_pattern: + if not fnmatch.fnmatch(scenario_id, scenario_pattern): + continue + + filtered_scenarios.append(scenario) + + return filtered_scenarios + + +def load_cortado_json(json_file_path, config): + all_items: dict[str, list[EvalCortadoRequest]] = { + "cortado-format": [], + } + with open(json_file_path, "r") as json_file: + data = json.load(json_file) + + scenarios = data.get("scenarios", []) + filtered_scenarios = _filter_scenarios(scenarios, config) + for scenario in filtered_scenarios: + eval_input = EvalCortadoRequest( + raw_dict=scenario + ) + all_items["cortado-format"].extend([eval_input]) + + return all_items + + +def load_gemini_cli_json(json_file_path, config): all_items: dict[str, list[EvalGeminiCliRequest]] = { "gemini-cli-format": [], } with open(json_file_path, "r") as json_file: json_item = json_file.read() + json_item = _expand_env_placeholders(json_item, json_file_path) item = json.loads(json_item) + + # Filter scenarios + scenarios = item.get("scenarios", []) + filtered_scenarios = _filter_scenarios(scenarios, config) + if scenarios and not filtered_scenarios: + return all_items + + item["scenarios"] = filtered_scenarios + + # Resolve work_dir for scenarios + dataset_dir = os.path.dirname(json_file_path) + for scenario in item["scenarios"]: + if "work_dir" in scenario: + work_dir = scenario["work_dir"] + # Resolve relative to dataset file + if not os.path.isabs(work_dir): + work_dir = os.path.abspath(os.path.join(dataset_dir, work_dir)) + scenario["resolved_work_dir"] = work_dir + + # Update payload with modified JSON + updated_json_item = json.dumps(item) + eval_input = EvalGeminiCliRequest( id=item.get("id", "0"), - payload=json_item, + payload=updated_json_item, ) all_items["gemini-cli-format"].extend([eval_input]) return all_items @@ -118,12 +242,27 @@ def load_json(json_file_path): def load_dataset_from_json(json_file_path, config): + # No dataset path (e.g. orchestrators driven by their run config rather than a + # prompt dataset): nothing to load. flatten_dataset({}) yields []. Log it so + # a run that *did* expect a dataset (missing/misconfigured dataset_config) + # doesn't fail silently. + if not json_file_path: + logging.info( + "load_dataset_from_json: no dataset path provided; returning an " + "empty dataset. Expected only for orchestrators that drive their " + "own inputs from the run config." + ) + return {} input_items = {} dataset_format = config.get("dataset_format", "evalbench-standard-format") if dataset_format == "bird-interact-format": all_items = load_bird_interact_dataset(json_file_path, config) elif dataset_format in ("gemini-cli-format", "agent-format"): - all_items = load_gemini_cli_json(json_file_path) + all_items = load_gemini_cli_json(json_file_path, config) + elif dataset_format == "cortado-format": + all_items = load_cortado_json(json_file_path, config) + elif dataset_format == "dea-format": + all_items = load_dea_json(json_file_path, config) else: all_items = load_json(json_file_path) @@ -137,6 +276,14 @@ def load_dataset_from_json(json_file_path, config): if "orchestrator" not in config: config["orchestrator"] = "interact" input_items = all_items + elif dataset_format == "cortado-format": + if "orchestrator" not in config: + config["orchestrator"] = "cortado" + input_items = all_items + elif dataset_format == "dea-format": + if "orchestrator" not in config: + config["orchestrator"] = "dea" + input_items = all_items elif dataset_format in ("gemini-cli-format", "agent-format"): if "orchestrator" not in config: config["orchestrator"] = "agent" if dataset_format == "agent-format" else "geminicli" @@ -144,7 +291,7 @@ def load_dataset_from_json(json_file_path, config): else: raise ValueError("Dataset not in any of the recognised formats") - if dataset_format not in ["gemini-cli-format", "agent-format", "bird-interact-format"]: + if dataset_format not in ["gemini-cli-format", "bird-interact-format", "agent-format", "cortado-format"]: totalEntries = sum(len(input_items.get(q, [])) for q in ["dql", "dml", "ddl"]) logging.info(f"Converted {totalEntries} entries to EvalInput.") diff --git a/evalbench/dataset/evalinteractinput.py b/evalbench/dataset/evalinteractinput.py index 6ed228d6..9d498d25 100644 --- a/evalbench/dataset/evalinteractinput.py +++ b/evalbench/dataset/evalinteractinput.py @@ -117,6 +117,7 @@ def breakdown_datasets(total_dataset: list[EvalInteractInputRequest]): datasets[dialect][input.database] = {} if input.query_type not in datasets[dialect][input.database]: datasets[dialect][input.database][input.query_type] = [] + total_db_len += 1 datasets[dialect][input.database][input.query_type].append( input.copy()) total_dataset_len += 1 diff --git a/evalbench/eval_service.py b/evalbench/eval_service.py index 3529f60b..bf315503 100644 --- a/evalbench/eval_service.py +++ b/evalbench/eval_service.py @@ -2,6 +2,7 @@ import asyncio import json +import os from collections.abc import AsyncIterator from typing import AsyncGenerator @@ -11,6 +12,7 @@ import yaml import grpc import pathlib +import queue from dataset.dataset import load_json from dataset import evalinput from evaluator import get_orchestrator, get_streaming_orchestrator @@ -20,6 +22,8 @@ import reporting.analyzer as analyzer from util.config import update_google3_relative_paths, set_session_configs, config_to_df from util import get_SessionManager +from util.scriptrunner import run_script +from util.sessionmgr import SESSION_RESOURCES_PATH from dataset.dataset import load_dataset_from_json from evalproto import ( eval_request_pb2, @@ -30,14 +34,21 @@ load_session_configs, get_dataset_from_request, ) +from generators.models.grpc_proxy import PROXY_QUEUES import threading from util.context import rpc_id_var from util import get_SessionManager + SESSIONMANAGER = get_SessionManager() +class EmptyEvalResultError(Exception): + """Raised when an Eval run produces no result rows — a client/config issue, + not a server fault. Translated to gRPC FAILED_PRECONDITION.""" + + class SessionManagerInterceptor(grpc.aio.ServerInterceptor): def __init__(self, tag: str, rpc_id: Optional[str] = None) -> None: self.tag = tag @@ -82,6 +93,7 @@ async def Connect( session = SESSIONMANAGER.get_session(session_id) if session is not None: session["streaming_eval"] = request.streaming_eval + session["bidirectional_stream"] = request.bidirectional_stream return eval_response_pb2.EvalResponse(response="ack", session_id=session_id) async def EvalConfig( @@ -91,14 +103,17 @@ async def EvalConfig( ) -> eval_response_pb2.EvalResponse: resource_map = {r.address: r.address for r in request.resources} experiment_config = yaml.safe_load(request.yaml_config.decode("utf-8")) - update_google3_relative_paths(experiment_config, rpc_id_var.get(), resource_map) + update_google3_relative_paths( + experiment_config, rpc_id_var.get(), resource_map) for resource in request.resources: if resource.address.endswith(".yaml"): yaml_config = yaml.safe_load(resource.content.decode("utf-8")) - update_google3_relative_paths(yaml_config, rpc_id_var.get(), resource_map) + update_google3_relative_paths( + yaml_config, rpc_id_var.get(), resource_map) resource.content = yaml.dump(yaml_config).encode("utf-8") session = SESSIONMANAGER.get_session(rpc_id_var.get()) - SESSIONMANAGER.write_resource_files(rpc_id_var.get(), request.resources) + SESSIONMANAGER.write_resource_files( + rpc_id_var.get(), request.resources) set_session_configs(session, experiment_config) session_id = rpc_id_var.get() return eval_response_pb2.EvalResponse(response="ack", session_id=session_id) @@ -111,8 +126,11 @@ async def ListEvalInputs( session = SESSIONMANAGER.get_session(rpc_id_var.get()) logging.info("Retrieving Evals for: %s.", rpc_id_var.get()) experiment_config = session["config"] - dataset_config_json = experiment_config["dataset_config"] - dataset = load_dataset_from_json(dataset_config_json, experiment_config) + # dataset_config is optional: some orchestrators drive their work from + # the run config itself, in which case load_dataset_from_json returns {}. + dataset_config_json = experiment_config.get("dataset_config") + dataset = load_dataset_from_json( + dataset_config_json, experiment_config) for _, eval_inputs in dataset.items(): for eval_input in eval_inputs: eval_input_request = eval_input.to_proto() @@ -123,85 +141,321 @@ async def Eval( request_iterator: AsyncIterator[eval_request_pb2.EvalInputRequest], context: grpc.ServicerContext, ) -> eval_response_pb2.EvalResponse: + try: + session_id = rpc_id_var.get() + session = SESSIONMANAGER.get_session(session_id) + config, db_configs, model_config, setup_config = load_session_configs( + session) + if config is None: + context.set_code(grpc.StatusCode.FAILED_PRECONDITION) + context.set_details("Session not configured") + return eval_response_pb2.EvalResponse() + + config["session_id"] = session_id + session_dir = os.path.join(SESSION_RESOURCES_PATH, session_id) + + set_up_script = config.get("set_up_script") + if set_up_script: + if os.path.exists(set_up_script): + logging.info( + f"Eval: Executing set_up_script '{set_up_script}'") + run_script(set_up_script, session_dir, "setup") + else: + logging.error( + f"Eval: Cannot run set_up_script, file not found at '{set_up_script}'") + + streaming_eval = session.get( + "streaming_eval", False) if session else False + loop = asyncio.get_event_loop() + + if streaming_eval: + evaluator = get_streaming_orchestrator( + config, db_configs, setup_config, report_progress=True + ) + logging.info( + "Streaming eval mode: evaluating items as they arrive..." + ) + tasks = [] + async for request in request_iterator: + eval_input = evalinput.EvalInputRequest.init_from_proto( + request + ) + ctx = contextvars.copy_context() + + task = loop.run_in_executor( + None, ctx.run, evaluator.evaluate_item, eval_input + ) + tasks.append(task) + await asyncio.gather(*tasks) + else: + dataset = await get_dataset_from_request(request_iterator) + evaluator = get_orchestrator( + config, db_configs, setup_config, report_progress=True + ) + logging.info( + "Batch eval mode: evaluating all items together...") + ctx = contextvars.copy_context() + await loop.run_in_executor( + None, ctx.run, evaluator.evaluate, dataset + ) + + job_id, run_time, results_tf, scores_tf, multi_trial_scores_tf = evaluator.process() + # Fallback to empty dict if reporting is present but null in YAML + reporters = get_reporters( + config.get("reporting") or {}, job_id, run_time + ) + + # Offload blocking results processing to a thread pool + logging.info("Offloading results processing to thread pool...") + ctx = contextvars.copy_context() + summary = await loop.run_in_executor( + None, + ctx.run, + _process_results, + reporters, + job_id, + run_time, + results_tf, + scores_tf, + multi_trial_scores_tf, + config, + model_config, + db_configs, + ) + + logging.info( + f"Finished Job ID {job_id} Thread count:{threading.active_count()}" + ) + + if config.get("summary_in_response"): + response = json.dumps({"job_id": job_id, "summary": summary}) + else: + response = f"{job_id}" + + tear_down_script = config.get("tear_down_script") + if tear_down_script: + if os.path.exists(tear_down_script): + logging.info( + f"Eval: Executing tear_down_script '{tear_down_script}'") + run_script(tear_down_script, session_dir, "teardown") + else: + logging.error( + f"Eval: Cannot run tear_down_script, file not found at '{tear_down_script}'") + + return eval_response_pb2.EvalResponse(response=response, session_id=session_id) + + except EmptyEvalResultError as e: + logging.warning(f"Eval produced no results: {e}") + context.set_code(grpc.StatusCode.FAILED_PRECONDITION) + context.set_details(str(e)) + return eval_response_pb2.EvalResponse(session_id=session_id) + + except Exception as e: + display_config = "Unknown" + # Attempt retrieval of configuration details if successfully loaded + try: + loaded_config = SESSIONMANAGER.get_session( + rpc_id_var.get()).get("config", {}) + cand = loaded_config.get("dataset_config", "Unknown") + g3_idx = cand.find("google3/") + display_config = cand[g3_idx:] if g3_idx != -1 else cand + except Exception as e_ctx: + # Best effort retrieval of metadata for tracing. Do not mask original fault. + logging.debug( + f"Unable to determine active dataset path for log context: {e_ctx}") + + logging.exception( + f"gRPC Eval failed for config/dataset '{display_config}': {e}") + raise + + async def Interact( + self, + request_iterator: AsyncIterator[eval_request_pb2.EvalInputRequest], + context: grpc.ServicerContext, + ) -> AsyncGenerator[eval_request_pb2.EvalInputRequest, None]: + """Bidirectional stream linking Google3 Agents to Evalbench Orchestrators.""" + session_id = rpc_id_var.get() session = SESSIONMANAGER.get_session(session_id) - config, db_configs, model_config, setup_config = load_session_configs(session) + config, db_configs, model_config, setup_config = load_session_configs( + session) + if config is None: context.set_code(grpc.StatusCode.FAILED_PRECONDITION) context.set_details("Session not configured") - return eval_response_pb2.EvalResponse() + return + + is_bidirectional = session.get( + "bidirectional_stream", False) if session else False + + if not is_bidirectional: + error_msg = ( + "Interact must be used with bidirectional streaming" + ) + logging.error(error_msg) + context.set_code(grpc.StatusCode.INVALID_ARGUMENT) + context.set_details(error_msg) + return + logging.info("Starting a bidirectional Interact stream...") config["session_id"] = session_id - streaming_eval = session.get("streaming_eval", False) if session else False + # Create thread-safe queues + in_queue = {} # Google3 -> Evalbench (mapped by conversation_id) + out_queue = queue.Queue() # Evalbench -> Google3 + + config["grpc_in_queues"] = in_queue + config["grpc_out_queue"] = out_queue + logging.info(f"CONFIG: {config}") + generator = model_config.get("generator") + + if generator != "grpc_proxy": + error_msg = ( + "Interactive evaluation failed: must use 'grpc_proxy' generator" + ) + logging.error(error_msg) + context.set_code(grpc.StatusCode.INVALID_ARGUMENT) + context.set_details(error_msg) + return + + # Load dataset and instantiate the Orchestrator. dataset_config is + # optional (datasetless orchestrators drive from the run config). + dataset_config_json = config.get("dataset_config") + dataset_dict = load_dataset_from_json(dataset_config_json, config) + + dataset = [] + for _, item_list in dataset_dict.items(): + dataset.extend(item_list) + + num_evals = config.get("num_evals_to_run") + if num_evals and int(num_evals) > 0: + dataset = dataset[:int(num_evals)] + + orchestrator = get_orchestrator( + config, db_configs, setup_config, report_progress=True) loop = asyncio.get_event_loop() + ctx = contextvars.copy_context() + + try: + PROXY_QUEUES[session_id] = (in_queue, out_queue) - if streaming_eval: - evaluator = get_streaming_orchestrator( - config, db_configs, setup_config, report_progress=True + def _cleanup_on_drop(ctx): + if session_id in PROXY_QUEUES: + PROXY_QUEUES.pop(session_id, None) + logging.info( + f"Cleaned up proxy queues for session {session_id} via disconnect callback") + + context.add_done_callback(_cleanup_on_drop) + + eval_task = loop.run_in_executor( + None, ctx.run, orchestrator.evaluate, dataset ) + + async def read_from_client(): + """Reads messages from the Google3 client stream.""" + async for response in request_iterator: + conv_id = str( + getattr(response, "conversation_id", getattr(response, "id", ""))) + logging.debug( + "Server-Inbound: Received from Google3 for conv_id %s", conv_id) + + if conv_id in in_queue: + logging.info( + f"[TRACE] Server-Inbound: Matched {conv_id} to active thread. Unblocking...") + in_queue[conv_id].put(response) + else: + logging.error( + "Server-Inbound: Orphaned reply! conv_id: '%s' not in active queues. Active keys: %s", + conv_id, list(in_queue.keys()) + ) + read_task = asyncio.create_task(read_from_client()) + + # Yield Loop: Read from out_queue and yield to Google3 + while True: + if eval_task.done(): + logging.info( + "Evaluator task finished for session %s.", session_id) + try: + eval_task.result() # Propagate exceptions + except Exception as e: + logging.error( + "Orchestrator/Evaluator task failed: %s", e, exc_info=True) + break + + if SESSIONMANAGER.get_session(session_id) is None: + logging.warning( + f"Session {session_id} deleted. Terminating stream.") + context.set_code(grpc.StatusCode.NOT_FOUND) + context.set_details("Session deleted") + return + + try: + out_request: eval_request_pb2.EvalInputRequest = await asyncio.to_thread(out_queue.get, True, 1.0) + + logging.debug( + "Server-Outbound: Yielding to Google3 for conv_id %s", out_request.conversation_id) + yield out_request + + except queue.Empty: + continue # Loop back and check if eval_task is done + except Exception as e: + import traceback + logging.error( + "Server-Outbound: Yield Loop error: %s", e, exc_info=True) + continue + + read_task.cancel() + try: + await read_task + except asyncio.CancelledError: + logging.debug("Read task cancelled as expected.") + + # Process final scoring and reporting + job_id, run_time, results_tf, scores_tf = orchestrator.process() + reporters = get_reporters(config.get( + "reporting") or {}, job_id, run_time) + logging.info( - "Streaming eval mode: evaluating items as they arrive..." - ) - tasks = [] - async for request in request_iterator: - eval_input = evalinput.EvalInputRequest.init_from_proto( - request - ) - ctx = contextvars.copy_context() + "Offloading interactive results processing to thread pool...") - task = loop.run_in_executor( - None, ctx.run, evaluator.evaluate_item, eval_input - ) - tasks.append(task) - await asyncio.gather(*tasks) - else: - dataset = await get_dataset_from_request(request_iterator) - evaluator = get_orchestrator( - config, db_configs, setup_config, report_progress=True - ) - logging.info("Batch eval mode: evaluating all items together...") - ctx = contextvars.copy_context() - await loop.run_in_executor( - None, ctx.run, evaluator.evaluate, dataset + summary = await loop.run_in_executor( + None, + ctx.run, + _process_results, + reporters, + job_id, + run_time, + results_tf, + scores_tf, + None, # Added None for multi_trial_scores_tf + config, + model_config, + db_configs, ) + logging.info( + f"Finished Interactive Job ID {job_id}. Summary: {summary}") - job_id, run_time, results_tf, scores_tf = evaluator.process() - # Fallback to empty dict if reporting is present but null in YAML - reporters = get_reporters( - config.get("reporting") or {}, job_id, run_time - ) + # Send the final payload back to the client to close the stream cleanly. + final_request = eval_request_pb2.EvalInputRequest() + final_request.payload = json.dumps( + {"job_id": job_id, "summary": summary}) - # Offload blocking results processing to a thread pool - logging.info("Offloading results processing to thread pool...") - ctx = contextvars.copy_context() - summary = await loop.run_in_executor( - None, - ctx.run, - _process_results, - reporters, - job_id, - run_time, - results_tf, - scores_tf, - config, - model_config, - db_configs, - ) + if dataset: + first_item = dataset[0] + conv_id = str(getattr(first_item, "id", "")) + if conv_id: + final_request.conversation_id = conv_id + logging.info(f"Yielding final summary payload: {final_request}") + yield final_request - logging.info( - f"Finished Job ID {job_id} Thread count:{threading.active_count()}" - ) - - if config.get("summary_in_response"): - response = json.dumps({"job_id": job_id, "summary": summary}) - else: - response = f"{job_id}" - return eval_response_pb2.EvalResponse(response=response, session_id=session_id) + finally: + # Clean up the global registry to prevent memory leaks. + PROXY_QUEUES.pop(session_id, None) + logging.info(f"Cleaned up proxy queues for session {session_id}") def _process_results( - reporters, job_id, run_time, results_tf, scores_tf, config, model_config, db_configs + reporters, job_id, run_time, results_tf, scores_tf, multi_trial_scores_tf, config, model_config, db_configs ): config_df = config_to_df( job_id, @@ -212,10 +466,24 @@ def _process_results( ) results = load_json(results_tf) results_df = report.get_dataframe(results) - assert not results_df.empty, "There were no matching evals in this run." + if results_df.empty: + raise EmptyEvalResultError( + "No matching evals were produced for this run. Check that the dataset, " + "dialect filters, and database configuration line up." + ) report.quick_summary(results_df) scores = load_json(scores_tf) - scores_df, summary_scores_df = analyzer.analyze_result(scores, config) + if multi_trial_scores_tf: + multi_trial_scores = load_json(multi_trial_scores_tf) + if multi_trial_scores: + scores.extend(multi_trial_scores) + + num_prompts = len(set(r.get("prompt_id") + for r in results if r.get("prompt_id"))) + num_trials = config.get("num_trials", 1) + scores_df, summary_scores_df = analyzer.analyze_result( + scores, config, num_prompts=num_prompts, num_trials=num_trials + ) summary_scores_df["job_id"] = job_id summary_scores_df["run_time"] = run_time @@ -229,6 +497,8 @@ def _process_results( # k8s emptyDir /tmp does not auto cleanup, so we explicitly delete pathlib.Path(results_tf).unlink() pathlib.Path(scores_tf).unlink() + if multi_trial_scores_tf: + pathlib.Path(multi_trial_scores_tf).unlink() # Build summary dict from summary_scores_df summary = {"total": 0, "scores": {}} diff --git a/evalbench/evalbench.py b/evalbench/evalbench.py index 5d509309..07813cc8 100644 --- a/evalbench/evalbench.py +++ b/evalbench/evalbench.py @@ -1,22 +1,23 @@ """EvalBench is a framework to measure the quality of a generative AI (GenAI) workflow.""" from collections.abc import Sequence +import logging +import multiprocessing +import os +import sys from absl import app from absl import flags -from reporting import get_reporters -from util.config import load_yaml_config, config_to_df -from dataset.dataset import load_json, load_dataset_from_json, flatten_dataset +from dataset.dataset import flatten_dataset, load_dataset_from_json, load_json from evaluator import get_orchestrator -import reporting.report as report +from reporting import get_reporters import reporting.analyzer as analyzer -import logging +import reporting.report as report +from util.config import config_to_df, load_yaml_config from util.config import set_session_configs -from util.service import load_session_configs from util.flags import EXPERIMENT_CONFIG -import os -import sys +from util.scriptrunner import run_script +from util.service import load_session_configs import yaml -import multiprocessing try: import google.colab # type: ignore @@ -28,14 +29,27 @@ logging.getLogger().setLevel(logging.INFO) +_SCENARIOS = flags.DEFINE_list( + "scenarios", + [], + "List of scenario IDs to run. Defaults to empty (runs all scenarios).", +) +_SCENARIO_PATTERN = flags.DEFINE_string( + "scenario_pattern", + None, + "Glob pattern of scenario IDs to run. Defaults to None (runs all scenarios).", +) _SUITE_CONFIG = flags.DEFINE_string( "suite_config", None, "Path to a suite configuration file to run multiple experiments.", ) +flags.declare_key_flag('experiment_config') def eval(experiment_config: str): + config = None + session_dir = None try: logging.info("EvalBench v1.0.0") logging.getLogger("google_genai.models").setLevel(logging.WARNING) @@ -44,14 +58,36 @@ def eval(experiment_config: str): session: dict = {} parsed_config = load_yaml_config(experiment_config) + + # Helper logic to generate clean display path + display_config = experiment_config + g3_idx = display_config.find("google3/") + if g3_idx != -1: + display_config = display_config[g3_idx:] + if parsed_config == "": - logging.error("No Eval Config Found.") + logging.error(f"No Eval Config Found for '{display_config}'.") return + # 1. Merge Environment Variables (overrides YAML) + env_scenarios = os.environ.get("EVAL_SCENARIOS") + if env_scenarios: + parsed_config["scenarios"] = [s.strip() for s in env_scenarios.split(",") if s.strip()] + + env_pattern = os.environ.get("EVAL_SCENARIO_PATTERN") + if env_pattern: + parsed_config["scenario_pattern"] = env_pattern + + # 2. Merge CLI Flags (overrides Environment Variables and YAML) + if flags.FLAGS.is_parsed(): + if _SCENARIOS.value: + parsed_config["scenarios"] = _SCENARIOS.value + if _SCENARIO_PATTERN.value: + parsed_config["scenario_pattern"] = _SCENARIO_PATTERN.value + set_session_configs(session, parsed_config) # Load the configs - config, db_configs, model_config, setup_config = load_session_configs( - session) + config, db_configs, model_config, setup_config = load_session_configs(session) logging.info("Loaded Configurations in %s", experiment_config) # Load the dataset @@ -61,27 +97,56 @@ def eval(experiment_config: str): config, db_configs, setup_config, report_progress=True ) + # Resolve session directory for local standalone logs + reporting_config = config.get("reporting") or {} + csv_config = reporting_config.get("csv") or {} + base_output_dir = csv_config.get("output_directory", "results") + session_dir = os.path.abspath(os.path.join(base_output_dir, evaluator.job_id)) + + set_up_script = config.get("set_up_script") + if set_up_script: + if os.path.exists(set_up_script): + logging.info("Executing set_up_script '%s'", set_up_script) + run_script( + set_up_script, + session_dir, + "setup", + env_updates=config.get("env") + ) + else: + logging.error( + "Cannot run set_up_script, file not found at '%s'", set_up_script + ) + # Run evaluations evaluator.evaluate(flatten_dataset(dataset)) - job_id, run_time, results_tf, scores_tf = evaluator.process() + job_id, run_time, results_tf, scores_tf, multi_trial_scores_tf = ( + evaluator.process() + ) # Create Dataframes for reporting if results_tf is not None and scores_tf is not None: - reporters = get_reporters( - parsed_config.get("reporting"), job_id, run_time) - config_df = config_to_df( - job_id, run_time, config, model_config, db_configs) + reporters = get_reporters(parsed_config.get("reporting"), job_id, run_time) + config_df = config_to_df(job_id, run_time, config, model_config, db_configs) results = load_json(results_tf) results_df = report.get_dataframe(results) report.quick_summary(results_df) scores = load_json(scores_tf) + if multi_trial_scores_tf: + multi_trial_scores = load_json(multi_trial_scores_tf) + if multi_trial_scores: + scores.extend(multi_trial_scores) + + num_prompts = len(flatten_dataset(dataset)) + num_trials = config.get("num_trials", 1) scores_df, summary_scores_df = analyzer.analyze_result( - scores, config) + scores, config, num_prompts=num_prompts, num_trials=num_trials + ) summary_scores_df["job_id"] = job_id summary_scores_df["run_time"] = run_time else: logging.warning( - "There were no matching evals in this run. Returning empty set." + f"There were no matching evals in run for config '{display_config}'. Returning empty set." ) reporters = [] config_df = None @@ -98,14 +163,31 @@ def eval(experiment_config: str): reporter.print_dashboard_links() print(f"Finished Job ID {job_id}") + return True except Exception as e: - logging.exception(e) + display_config = experiment_config + g3_idx = display_config.find("google3/") + if g3_idx != -1: + display_config = display_config[g3_idx:] + logging.exception(f"Evaluation failed for config '{display_config}': {e}") return False + finally: + if config and session_dir: + tear_down_script = config.get("tear_down_script") + if tear_down_script: + if os.path.exists(tear_down_script): + logging.info("Executing tear_down_script '%s'", tear_down_script) + run_script(tear_down_script, session_dir, "teardown") + else: + logging.error( + "Cannot run tear_down_script, file not found at '%s'", + tear_down_script, + ) def run_suite(suite_config_path: str) -> bool: - with open(suite_config_path, 'r') as f: + with open(suite_config_path, "r") as f: suite_conf = yaml.safe_load(f) runs = suite_conf.get("runs", []) @@ -113,8 +195,7 @@ def run_suite(suite_config_path: str) -> bool: logging.error("No runs defined in suite config.") return False - logging.info( - f"Starting EvalBench Suite: {suite_conf.get('name', 'Unnamed Suite')}") + logging.info(f"Starting EvalBench Suite: {suite_conf.get('name', 'Unnamed Suite')}") logging.info(f"Total runs scheduled: {len(runs)}") results = [] @@ -123,13 +204,14 @@ def run_suite(suite_config_path: str) -> bool: config_path = run.get("config_path") if not config_path: - logging.error( - f"Run '{run_name}' is missing 'config_path'. Skipping.") + logging.error(f"Run '{run_name}' is missing 'config_path'. Skipping.") results.append((run_name, False)) continue logging.info( - f"\n{'=' * 50}\nExecuting Suite Run {i + 1}/{len(runs)}: {run_name}\nConfig: {config_path}\n{'=' * 50}") + f"\n{'=' * 50}\nExecuting Suite Run {i + 1}/{len(runs)}:" + f" {run_name}\nConfig: {config_path}\n{'=' * 50}" + ) success = eval(config_path) results.append((run_name, success)) @@ -159,6 +241,24 @@ def main(argv: Sequence[str]): return os._exit(exit_code) +def run(): + """Starting function for the uvx package entrypoint.""" + # Fix absl help output when run via uvx/launcher + if '__main__' in sys.modules: + main_module = sys.modules['__main__'] + if main_module.__doc__ and 'exec' in main_module.__doc__: + main_module.__doc__ = sys.modules[__name__].__doc__ + # Clean up sys.argv[0] to hide the full temporary path + sys.argv[0] = os.path.basename(sys.argv[0]) + # Register key flags for __main__ and sys.argv[0] so they show up in launcher's short help + flags.FLAGS.register_key_flag_for_module('__main__', flags.FLAGS['experiment_config']) + flags.FLAGS.register_key_flag_for_module('__main__', flags.FLAGS['suite_config']) + flags.FLAGS.register_key_flag_for_module(sys.argv[0], flags.FLAGS['experiment_config']) + flags.FLAGS.register_key_flag_for_module(sys.argv[0], flags.FLAGS['suite_config']) + app.run(main) + + if __name__ == "__main__": - multiprocessing.freeze_support() # Required for PyInstaller multiprocessing support + # Required for PyInstaller multiprocessing support + multiprocessing.freeze_support() app.run(main) diff --git a/evalbench/evalproto/eval_connect.proto b/evalbench/evalproto/eval_connect.proto index 390c757b..6319359a 100644 --- a/evalbench/evalproto/eval_connect.proto +++ b/evalbench/evalproto/eval_connect.proto @@ -7,4 +7,5 @@ option java_multiple_files = true; message EvalConnectRequest { string client_id = 1; bool streaming_eval = 2; + bool bidirectional_stream = 3; } diff --git a/evalbench/evalproto/eval_request.proto b/evalbench/evalproto/eval_request.proto index 3aec80bf..872c983e 100644 --- a/evalbench/evalproto/eval_request.proto +++ b/evalbench/evalproto/eval_request.proto @@ -32,6 +32,8 @@ message EvalInputRequest { string job_id = 15; string trace_id = 16; string payload = 17; + string conversation_id = 18; + string generated_nl_response = 19; } message UserAction { diff --git a/evalbench/evalproto/eval_service.proto b/evalbench/evalproto/eval_service.proto index 60066053..490ce6d0 100644 --- a/evalbench/evalproto/eval_service.proto +++ b/evalbench/evalproto/eval_service.proto @@ -35,6 +35,11 @@ service EvalService { // option deadline = 1800; } + // MultiTurnEval. Bidirectional stream. + rpc Interact(stream EvalInputRequest) returns (stream EvalInputRequest) { + // option deadline = 1800; + } + // PrepareCodeEvalInputs for NL2Code Evaluation rpc PrepareCodeEvalInputs(EvalCodeInputRequest) returns (stream EvalCodeInputRequest) { // option deadline = 1800; diff --git a/evalbench/evaluator/__init__.py b/evalbench/evaluator/__init__.py index 21ba34e8..0374aaa6 100644 --- a/evalbench/evaluator/__init__.py +++ b/evalbench/evaluator/__init__.py @@ -1,15 +1,19 @@ from evaluator.orchestrator import Orchestrator from evaluator.oneshotorchestrator import OneShotOrchestrator +from evaluator.cortadoorchestrator import CortadoOrchestrator from evaluator.interactorchestrator import InteractOrchestrator from evaluator.dataagentorchestrator import DataAgentOrchestrator from evaluator.agentorchestrator import AgentOrchestrator from evaluator.streamingorchestrator import StreamingOrchestrator +from evaluator.dataengineeringagentorchestrator import ( + DataEngineeringAgentOrchestrator, +) +from evaluator.mcp_readability import McpReadabilityOrchestrator import logging def get_orchestrator(config, db_configs, setup_config, report_progress=False): orchestrator_type = config.get("orchestrator", "oneshot") - logging.info(f"Orchestrator Type: {orchestrator_type}") if orchestrator_type == "oneshot": return OneShotOrchestrator(config, db_configs, setup_config, report_progress) elif orchestrator_type == "interact": @@ -18,6 +22,16 @@ def get_orchestrator(config, db_configs, setup_config, report_progress=False): return DataAgentOrchestrator(config, db_configs, setup_config, report_progress) elif orchestrator_type in ("geminicli", "agent"): return AgentOrchestrator(config, db_configs, setup_config, report_progress) + elif orchestrator_type == "cortado": + return CortadoOrchestrator(config, db_configs, setup_config, report_progress) + elif orchestrator_type == "dea": + return DataEngineeringAgentOrchestrator( + config, db_configs, setup_config, report_progress + ) + elif orchestrator_type == "mcp_readability": + return McpReadabilityOrchestrator( + config, db_configs, setup_config, report_progress + ) else: return Orchestrator(config, db_configs, setup_config, report_progress) diff --git a/evalbench/evaluator/agentevaluator.py b/evalbench/evaluator/agentevaluator.py index 47399669..f464a83b 100644 --- a/evalbench/evaluator/agentevaluator.py +++ b/evalbench/evaluator/agentevaluator.py @@ -2,11 +2,13 @@ import datetime import concurrent.futures import logging +import os +import shutil +import threading from dataset.evalgeminicliinput import EvalGeminiCliRequest -from generators.models.gemini_cli import GeminiCliGenerator -from generators.models.claude_code import ClaudeCodeGenerator -from util.config import load_yaml_config +from generators.models import get_generator +from generators.models.agent_cli import AgentCliGenerator from mp import mprunner from work.agentgenwork import AgentGenWork from evaluator.simulateduser import SimulatedUser @@ -23,26 +25,23 @@ def __init__( ): self.config = config - # Load model config if provided - model_config = config - if "model_config" in config and isinstance(config["model_config"], str): - loaded_config = load_yaml_config(config["model_config"]) - # Merge main config into loaded config, giving precedence to main config - model_config = loaded_config.copy() - model_config.update(config) - - generator_type = model_config.get("generator") - if generator_type == "gemini_cli": - self.agent_version = model_config.get( - "gemini_cli_version", config.get("gemini_cli_version")) - self.generator = GeminiCliGenerator(model_config) - elif generator_type == "claude_code": - self.agent_version = model_config.get( - "claude_code_version", config.get("claude_code_version", "claude")) - self.generator = ClaudeCodeGenerator(model_config) - else: + model_config_path = config.get("model_config") + if not isinstance(model_config_path, str): raise ValueError( - f"Unsupported generator type for AgentEvaluator: {generator_type}") + "AgentEvaluator requires `model_config` to be a path to a model YAML") + + global_models = { + "lock": threading.Lock(), + "registered_models": {}, + } + self.generator = get_generator(global_models, model_config_path) + + if not isinstance(self.generator, AgentCliGenerator): + raise ValueError( + f"AgentEvaluator only supports agent CLI generators " + f"(gemini_cli, claude_code, codex_cli, agy_cli), got " + f"{type(self.generator).__name__}") + self.agent_version = self.generator.version runner_config = self.config.get("runners", {}) self.agent_runners = runner_config.get("agent_runners", 10) @@ -54,11 +53,11 @@ def evaluate( job_id: str, run_time: datetime.datetime, ): - if isinstance(self.generator, (GeminiCliGenerator, ClaudeCodeGenerator)): + if isinstance(self.generator, AgentCliGenerator): return self._evaluate_agent_cli(dataset, job_id, run_time) else: raise NotImplementedError( - "This evaluator currently only supports GeminiCliGenerator and ClaudeCodeGenerator") + "This evaluator currently only supports GeminiCliGenerator, ClaudeCodeGenerator, CodexCliGenerator and AgyCliGenerator") def _evaluate_agent_cli( self, @@ -116,31 +115,48 @@ def process_scenario( conversation_plan = scenario.get("conversation_plan", "") conversation_history = [] accumulated_tools = [] + accumulated_skills = [] last_result = None + resolved_work_dir = scenario.get("resolved_work_dir") + if resolved_work_dir: + os.makedirs(resolved_work_dir, exist_ok=True) + + # Copy declared env_files to fake_home + fake_home = getattr(self.generator, "fake_home", None) + if fake_home: + session_dir = os.path.dirname(fake_home) + env_files = scenario.get("env_files", []) + for env_file in env_files: + src_path = os.path.join(session_dir, "env_files", env_file) + dest_path = os.path.join(fake_home, env_file) + if os.path.exists(src_path): + os.makedirs(os.path.dirname(dest_path), exist_ok=True) + shutil.copy2(src_path, dest_path) + logging.info( + "Natively copied env file to sandbox: %s", dest_path + ) + else: + logging.warning( + "Declared env file not found in session: %s", src_path + ) + session_id = None for turn in range(max_turns): logging.info( f"Turn {turn + 1}/{max_turns} - Prompt: {current_prompt}") - if isinstance(self.generator, (GeminiCliGenerator, ClaudeCodeGenerator)): - if isinstance(self.generator, ClaudeCodeGenerator): - cli_cmd = self.generator.create_command( - cli=self.agent_version, - prompt=current_prompt, - env=env, - resume=(turn > 0), - session_id=session_id - ) - else: - cli_cmd = self.generator.create_command( - cli=self.agent_version, - prompt=current_prompt, - env=env, - resume=(turn > 0) - ) + if isinstance(self.generator, AgentCliGenerator): + cli_cmd = self.generator.create_command( + cli=self.agent_version, + prompt=current_prompt, + env=env, + resume=(turn > 0), + session_id=session_id, + cwd=resolved_work_dir, + ) try: result = self.generator.safe_generate(cli_cmd) - if isinstance(self.generator, ClaudeCodeGenerator) and result.stdout: + if result.stdout: parsed = self.generator.parse_response(result.stdout) if parsed.get("session_id"): session_id = parsed["session_id"] @@ -161,10 +177,15 @@ def process_scenario( self._log_cli_result(turn, max_turns, result) tools = [] - if isinstance(self.generator, (GeminiCliGenerator, ClaudeCodeGenerator)): + if isinstance(self.generator, AgentCliGenerator): tools = self.generator.extract_tools(result.stdout) accumulated_tools.extend(tools) + # Extract skills from generator output + if isinstance(self.generator, AgentCliGenerator): + skills = self.generator.extract_skills(result.stdout) + accumulated_skills.extend(skills) + conversation_history.append({ "user": current_prompt, "agent": result.stdout @@ -190,6 +211,7 @@ def process_scenario( last_result, conversation_history, accumulated_tools, + accumulated_skills, eval_result, job_id, metadata @@ -210,6 +232,7 @@ def _finalize_scenario( last_result: subprocess.CompletedProcess, conversation_history: List[Dict[str, str]], accumulated_tools: List[str], + accumulated_skills: List[str], eval_result: Any, job_id: str, metadata: Dict[str, Any] @@ -230,8 +253,10 @@ def _finalize_scenario( "conversation_history": json.dumps(conversation_history, indent=2), "scenario": scenario, "accumulated_tools": accumulated_tools, + "accumulated_skills": accumulated_skills, "job_id": job_id, - "metadata": metadata + "metadata": metadata, + "fake_home": self.generator.fake_home if hasattr(self.generator, "fake_home") else None } score_work = AgentScoreWork( diff --git a/evalbench/evaluator/agentorchestrator.py b/evalbench/evaluator/agentorchestrator.py index 972e4917..54b5c9d1 100644 --- a/evalbench/evaluator/agentorchestrator.py +++ b/evalbench/evaluator/agentorchestrator.py @@ -27,7 +27,7 @@ def __init__( self.report_progress = report_progress def evaluate(self, dataset: list[EvalGeminiCliRequest]): - logging.info("Starting Gemini CLI evaluation") + logging.info("Starting agent CLI evaluation") evaluator = AgentEvaluator(self.config) eval_outputs, scoring_results = evaluator.evaluate( dataset, self.job_id, self.run_time @@ -51,4 +51,5 @@ def process(self): self.run_time, results_tf, scores_tf, + None, ) diff --git a/evalbench/evaluator/cortadoevaluator.py b/evalbench/evaluator/cortadoevaluator.py new file mode 100644 index 00000000..64ceb803 --- /dev/null +++ b/evalbench/evaluator/cortadoevaluator.py @@ -0,0 +1,184 @@ +# cortadoevaluator.py + +from typing import Any, List, Dict +import datetime +import concurrent.futures +import logging +import json + +from dataset.cortadoinput import EvalCortadoRequest +from generators.models.grpc_proxy import GrpcProxyModel +from util.config import load_yaml_config +from mp import mprunner +from work.agentgenwork import AgentGenWork +from evaluator.simulateduser import SimulatedUser +from work.agentscorework import AgentScoreWork + + +class CortadoEvaluator: + def __init__(self, config): + self.config = config + + # Load model config + model_config = config + if "model_config" in config and isinstance(config["model_config"], str): + loaded_config = load_yaml_config(config["model_config"]) + model_config = loaded_config.copy() + model_config.update(config) + + generator_type = model_config.get("generator") + if generator_type == "grpc_proxy": + self.generator = GrpcProxyModel(model_config) + else: + raise ValueError( + f"CortadoEvaluator requires 'grpc_proxy' generator, got {generator_type}") + + runner_config = self.config.get("runners", {}) + self.agent_runners = runner_config.get("agent_runners", 10) + self.agentrunner = mprunner.MPRunner(self.agent_runners) + + def evaluate(self, dataset: List[EvalCortadoRequest], job_id: str, run_time: datetime.datetime): + eval_outputs: List[Any] = [] + scoring_results: List[Any] = [] + logging.info("Running Cortado gRPC evaluation") + + self.agentrunner.futures.clear() + + metadata = { + "dialects": self.config.get("dialects", []), + "database": self.config.get("database", "unknown"), + "scorers": self.config.get("scorers", {}), + } + + # Spin up threads for concurrent conversation processing + for item in dataset: + simulated_user = SimulatedUser(self.config) + work = AgentGenWork( + processor=self.process_scenario, + eval_result=item, + job_id=job_id, + metadata=metadata, + simulated_user=simulated_user + ) + self.agentrunner.execute_work(work) + + for future in concurrent.futures.as_completed(self.agentrunner.futures): + try: + # This now contains the returned object from process_scenario + modified_item = future.result() + if hasattr(modified_item, "agent_results"): + eval_outputs.extend(modified_item.agent_results) + if hasattr(modified_item, "scoring_results"): + scoring_results.extend(modified_item.scoring_results) + except Exception as e: + logging.error( + f"Error getting result from future: {e}", exc_info=True) + + return eval_outputs, scoring_results + + def process_scenario( + self, scenario: Dict[str, Any], eval_result: Any, job_id: str, + metadata: Dict[str, Any], simulated_user: Any = None + ) -> Any: + """Communication between Cortado and the Simulated User.""" + + current_prompt = scenario.get("starting_prompt", "") + max_turns = scenario.get("max_turns", 1) + conversation_plan = scenario.get("conversation_plan", []) + conversation_history = [] + last_agent_text = "" + last_sql_reply = "" + + # Parity tracking lists + accumulated_tools = [] + accumulated_skills = [] + + for turn in range(max_turns): + logging.info( + f"Turn {turn + 1}/{max_turns} - Prompt: {current_prompt}") + + # Inject the current prompt into the object + eval_result.nl_prompt = current_prompt + + # Hand it to the gRPC Proxy (blocks until client replies) + agent_text = "" + try: + self.generator.generate(eval_result) + + nl_reply = getattr(eval_result, "generated_nl_response", "") + sql_reply = getattr(eval_result, "generated_sql", "") + last_sql_reply = sql_reply + agent_text = nl_reply + + except Exception as e: + logging.error(f'gRPC generation failed: {e}', exc_info=True) + agent_text = f"Error: {e}" + last_sql_reply = "" + + last_agent_text = agent_text + logging.info( + f"Turn {turn + 1}/{max_turns} - Agent Reply to Simulated User: {agent_text}") + + # Log history + conversation_history.append({ + "user": current_prompt, + "agent": agent_text + }) + + # Invoke Simulated User to check plan and generate next turn + if turn < max_turns - 1 and simulated_user: + next_response = simulated_user.get_next_response( + conversation_plan, conversation_history, agent_text + ) + if "TERMINATE" in next_response: + logging.info( + "Simulated user met the goal and terminated the conversation.") + break + current_prompt = next_response + else: + break + + # Finalize and Score + self._finalize_scenario( + scenario, last_agent_text, conversation_history, + accumulated_tools, accumulated_skills, + eval_result, job_id, metadata, + last_sql_reply + ) + return eval_result + + def _finalize_scenario( + self, scenario: Dict[str, Any], last_response: str, + conversation_history: List[Dict[str, str]], + accumulated_tools: List[str], accumulated_skills: List[str], + eval_result: Any, job_id: str, metadata: Dict[str, Any], + last_sql: str + ): + """Packages the conversation and sends it to the scoring engine.""" + + eval_output_data = { + "eval_id": scenario["id"], + "stdout": last_response, # This is the text seen by the simulated user + "stderr": "", + "returncode": 0 if not last_response.startswith("Error") else 1, + "prompt_generator_error": None, + "generated_error": None, + "sql_generator_error": None, + "golden_error": None, + "generated_sql": last_sql, + "prompt": scenario["starting_prompt"], + "conversation_history": json.dumps(conversation_history, indent=2), + "scenario": scenario, + "accumulated_tools": accumulated_tools, # Passes empty list for now + "accumulated_skills": accumulated_skills, # Passes empty list for now + "job_id": job_id, + "metadata": metadata + } + + score_work = AgentScoreWork( + config=self.config, + eval_output=eval_output_data, + scoring_results=eval_result.scoring_results + ) + score_work.run() + eval_result.agent_results.append(eval_output_data) diff --git a/evalbench/evaluator/cortadoorchestrator.py b/evalbench/evaluator/cortadoorchestrator.py new file mode 100644 index 00000000..968cd933 --- /dev/null +++ b/evalbench/evaluator/cortadoorchestrator.py @@ -0,0 +1,37 @@ +from evaluator.orchestrator import Orchestrator +import uuid +import datetime +import tempfile +import json +from dataset.cortadoinput import EvalCortadoRequest +from evaluator.cortadoevaluator import CortadoEvaluator + + +class CortadoOrchestrator(Orchestrator): + def __init__(self, config, db_configs, setup_config, report_progress=False): + self.config = config + self.db_configs = db_configs + self.setup_config = setup_config + self.job_id = f"{uuid.uuid4()}" + self.run_time = datetime.datetime.now() + self.total_eval_outputs = [] + self.total_scoring_results = [] + + def evaluate(self, dataset: list[EvalCortadoRequest]): + evaluator = CortadoEvaluator(self.config) + eval_outputs, scoring_results = evaluator.evaluate( + dataset, self.job_id, self.run_time + ) + self.total_eval_outputs.extend(eval_outputs) + self.total_scoring_results.extend(scoring_results) + + def process(self): + with tempfile.NamedTemporaryFile(mode="w", delete=False, suffix=".json") as f: + json.dump(self.total_eval_outputs, f, + sort_keys=True, indent=4, default=str) + results_tf = f.name + with tempfile.NamedTemporaryFile(mode="w", delete=False, suffix=".json") as f: + json.dump(self.total_scoring_results, f, + sort_keys=True, indent=4, default=str) + scores_tf = f.name + return self.job_id, self.run_time, results_tf, scores_tf diff --git a/evalbench/evaluator/dataagentorchestrator.py b/evalbench/evaluator/dataagentorchestrator.py index 58b607f1..880941fa 100644 --- a/evalbench/evaluator/dataagentorchestrator.py +++ b/evalbench/evaluator/dataagentorchestrator.py @@ -174,6 +174,7 @@ def evaluate_sub_dataset( except Exception as e: logging.error( f"Skipping {query_type} queries as DB {database} " + + f"(dataset: {self.get_display_dataset_config()}) " + f"could not be setup properly in {dialect} due to {e}." ) skip_database( @@ -223,4 +224,5 @@ def process(self): self.run_time, results_tf, scores_tf, + None, ) diff --git a/evalbench/evaluator/dataengineeringagentevaluator.py b/evalbench/evaluator/dataengineeringagentevaluator.py new file mode 100644 index 00000000..f2b97371 --- /dev/null +++ b/evalbench/evaluator/dataengineeringagentevaluator.py @@ -0,0 +1,366 @@ +import concurrent.futures +import datetime +import json +import logging +import os +import re +from typing import Any, List + +from dataset.dataengineeringagentinput import EvalDeaRequest +from generators.models.gcp_data_engineering_agent import ( + DataEngineeringAgentGenerator, +) +from google.api_core import exceptions as api_exceptions +from google.cloud import storage +from mp import mprunner +from work.agentgenwork import AgentGenWork +from evaluator.simulateduser import SimulatedUser +from work.agentscorework import AgentScoreWork +from util.config import load_yaml_config +from util.dataform_workspace import DataformWorkspaceManager + +_WORKSPACE_RE = re.compile( + r"^projects/[^/]+/locations/[^/]+/repositories/([^/]+)/workspaces/([^/]+)$" +) + +# Module-level logger +logger = logging.getLogger(__name__) + + +class DataEngineeringAgentEvaluator: + """Evaluator designed specifically for pure conversational DEA evaluations. + + Coordinates turn-by-turn natural language dialogue between the Simulated + User and the generator, completely bypassing all SQL execution and + database dependencies. + """ + + generator: DataEngineeringAgentGenerator + agentrunner: mprunner.MPRunner + + def __init__(self, config: dict[str, Any]) -> None: + self.config = config + + # Resolve and parse model config + model_config = config + if ( + "model_config" in config + and isinstance(config["model_config"], str) + ): + loaded_config = load_yaml_config(config["model_config"]) + model_config = loaded_config.copy() + model_config.update(config) + + self.generator = DataEngineeringAgentGenerator(model_config) + + runner_config = self.config.get("runners", {}) + self.agent_runners = runner_config.get("agent_runners", 10) + self.agentrunner = mprunner.MPRunner(self.agent_runners) + + def _get_session_dir(self, job_id: str) -> str: + """Resolves the session directory path for a given job ID.""" + reporting_config = self.config.get("reporting") or {} + csv_config = reporting_config.get("csv") or {} + base_output_dir = csv_config.get("output_directory", "results") + return os.path.abspath(os.path.join(base_output_dir, job_id)) + + def evaluate( + self, + dataset: List[EvalDeaRequest], + job_id: str, + run_time: datetime.datetime, + ): + """Runs the conversational scenarios in a parallel thread pool.""" + eval_outputs: List[dict[str, Any]] = [] + scoring_results: List[dict[str, Any]] = [] + logger.info("Running pure conversational DEA evaluation") + + session_dir = self._get_session_dir(job_id) + state_file = os.path.join(session_dir, "target_workspace.txt") + workspace_uri = "" + + repo_id = self.config.get("dataform_repository") + ws_id = self.config.get("dataform_workspace") + project_id = self.config.get("gcp_project_id") or self.config.get("project_id") + region = self.config.get("gcp_region") or self.config.get("region") + + if os.path.exists(state_file): + with open(state_file, "r", encoding="utf-8") as sf: + workspace_uri = sf.read().strip() + logger.info( + "Redirecting DEA evaluation to dynamic workspace: %s", + workspace_uri, + ) + match = _WORKSPACE_RE.match(workspace_uri) + if match: + repo_id = match.group(1) + ws_id = match.group(2) + for item in dataset: + item.gcp_resource_id = workspace_uri + item.dataform_repository = repo_id + item.dataform_workspace = ws_id + else: + for item in dataset: + item.gcp_resource_id = workspace_uri + elif repo_id and ws_id and project_id and region: + workspace_uri = f"projects/{project_id}/locations/{region}/repositories/{repo_id}/workspaces/{ws_id}" + logger.info( + "Using configured static workspace: %s", + workspace_uri, + ) + for item in dataset: + item.gcp_resource_id = workspace_uri + item.dataform_repository = repo_id + item.dataform_workspace = ws_id + else: + raise ValueError( + "No workspace configured. Either provide setup/teardown scripts " + "for dynamic sandbox, or configure static 'dataform_repository', " + "'dataform_workspace', 'gcp_project_id', and 'gcp_region' in your run config." + ) + + self.agentrunner.futures.clear() + + metadata = { + "dialects": self.config.get("dialects", []), + "database": self.config.get("database", "unknown"), + "scorers": self.config.get("scorers", {}), + } + + try: + for item in dataset: + simulated_user = SimulatedUser(self.config) + work = AgentGenWork( + processor=self.process_scenario, + eval_result=item, + job_id=job_id, + metadata=metadata, + simulated_user=simulated_user, + ) + self.agentrunner.execute_work(work) + + futures = self.agentrunner.futures + for future in concurrent.futures.as_completed(futures): + try: + modified_item = future.result() + if hasattr(modified_item, "agent_results"): + eval_outputs.extend(modified_item.agent_results) + if hasattr(modified_item, "scoring_results"): + scoring_results.extend(modified_item.scoring_results) + except Exception as e: + logger.exception(f"Error getting result from future: {e}") + finally: + self._archive_workspace_to_gcs(workspace_uri, job_id, dataset) + + return eval_outputs, scoring_results + + def process_scenario( + self, + scenario: dict[str, Any], + eval_result: EvalDeaRequest, + job_id: str, + metadata: dict[str, Any], + simulated_user: SimulatedUser | None = None, + ) -> EvalDeaRequest: + """Manages the multi-turn conversational dialogue turn-by-turn.""" + + current_prompt = scenario.get("starting_prompt", "") + max_turns = scenario.get("max_turns", 1) + conversation_plan = scenario.get("conversation_plan", []) + conversation_history: List[dict[str, str]] = [] + last_agent_text = "" + + for turn in range(max_turns): + logger.info( + "Turn %d/%d - Prompt: %s", + turn + 1, + max_turns, + current_prompt + ) + + # Pass prompt to programmatic request object + eval_result.nl_prompt = current_prompt + + agent_text = "" + try: + # Native API call: mutates eval_result in-place + self.generator.generate(eval_result) + agent_text = getattr(eval_result, "generated_nl_response", "") + except Exception as e: + logger.exception( + "A2A SDK generation failed: %s", type(e).__name__ + ) + agent_text = f"Error: {e}" + + last_agent_text = agent_text + logger.info( + "Turn %d/%d - Agent Reply: %s", + turn + 1, + max_turns, + agent_text + ) + + this_turn_tools = getattr( + eval_result, "this_turn_tool_details", [] + ) + + tools_by_name = {} + for t in this_turn_tools: + name = t["name"] + params = t["params"] + output = t.get("output", {}) + fail = t.get("fail", 0) + + tool_data = tools_by_name.setdefault( + name, {"parameters": [], "outputs": [], "fail": 0} + ) + tool_data["parameters"].append(params) + tool_data["outputs"].append(output) + if fail: + tool_data["fail"] = 1 + + conversation_history.append({ + "user": current_prompt, + "agent": agent_text, + "agent_stats": { + "tools": { + "byName": tools_by_name + } + }, + }) + + # Simulated User checks conversation plan and generates next prompt + if turn < max_turns - 1 and simulated_user: + next_response = simulated_user.get_next_response( + conversation_plan, conversation_history, agent_text + ) + if "TERMINATE" in next_response: + logger.info("Simulated user met the goal and terminated.") + break + current_prompt = next_response + else: + break + + self._finalize_scenario( + scenario, + last_agent_text, + conversation_history, + eval_result, + job_id, + metadata, + ) + return eval_result + + def _archive_workspace_to_gcs( + self, workspace_uri: str, job_id: str, dataset: list + ) -> None: + """Archives the Dataform workspace to GCS if configured.""" + archive_config = self.config.get("dataform_workspace_gcs_archive") + if not archive_config or not workspace_uri: + return + + try: + bucket_name = archive_config.get("bucket") + prefix = archive_config.get("path_prefix", "workspaces") + project_id = archive_config.get("gcp_project_id") + location = archive_config.get("gcp_region") + if not project_id: + raise ValueError( + "gcp_project_id must be specified in " + "dataform_workspace_gcs_archive config." + ) + if not location: + raise ValueError( + "gcp_region must be specified in " + "dataform_workspace_gcs_archive config." + ) + manager = DataformWorkspaceManager(project_id, location) + zip_bytes = manager.download_and_zip(workspace_uri) + gcs_client = storage.Client(project=project_id) + + try: + bucket = gcs_client.get_bucket(bucket_name) + except api_exceptions.NotFound: + logger.info("Bucket %s not found. Creating...", bucket_name) + bucket = gcs_client.create_bucket( + bucket_name, location=location + ) + + scenario_id = self.config.get("env", {}).get( + "SCENARIO_ID", "default" + ) + + blob_path = f"{prefix}/{job_id}/{scenario_id}_debug.zip" + blob = bucket.blob(blob_path) + + blob.upload_from_string( + zip_bytes, content_type="application/zip" + ) + + gcs_uri = f"gs://{bucket_name}/{blob_path}" + logger.info("Exported workspace archive to %s", gcs_uri) + + for item in dataset: + item.gcs_debug_archive = gcs_uri + if not getattr(item, "dataform_repository", None): + item.dataform_repository = f"evalbench-{job_id}" + if not getattr(item, "dataform_workspace", None): + item.dataform_workspace = "default" + + # Propagate GCS archive link and workspace coordinates + # to finalized results in-place. + for res in getattr(item, "agent_results", []): + res["gcs_debug_archive"] = gcs_uri + res["dataform_repository"] = item.dataform_repository + res["dataform_workspace"] = item.dataform_workspace + except Exception as e: + logger.exception(f"Error archiving workspace to GCS: {e}") + + def _finalize_scenario( + self, + scenario: dict[str, Any], + last_response: str, + conversation_history: List[dict[str, str]], + eval_result: EvalDeaRequest, + job_id: str, + metadata: dict[str, Any], + ) -> None: + """Packages conversation and invokes scoring engine.""" + eval_output_data = { + "eval_id": scenario["id"], + "stdout": last_response, + "stderr": "", + "returncode": 0 if not last_response.startswith("Error") else 1, + "prompt_generator_error": None, + "generated_error": None, + "sql_generator_error": None, + "golden_error": None, + # Non-SQL conversational runs skip SQL evaluation + "generated_sql": "skipped", + "prompt": scenario["starting_prompt"], + "conversation_history": json.dumps(conversation_history, indent=2), + "scenario": scenario, + "accumulated_tools": getattr( + eval_result, "accumulated_tools", [] + ), + "accumulated_skills": [], + "job_id": job_id, + "metadata": metadata, + "dataform_repository": getattr( + eval_result, "dataform_repository", "" + ), + "dataform_workspace": getattr( + eval_result, "dataform_workspace", "" + ), + "gcs_debug_archive": getattr( + eval_result, "gcs_debug_archive", "" + ), + } + + score_work = AgentScoreWork( + config=self.config, + eval_output=eval_output_data, + scoring_results=eval_result.scoring_results, + ) + score_work.run() + eval_result.agent_results.append(eval_output_data) diff --git a/evalbench/evaluator/dataengineeringagentorchestrator.py b/evalbench/evaluator/dataengineeringagentorchestrator.py new file mode 100644 index 00000000..b26a206e --- /dev/null +++ b/evalbench/evaluator/dataengineeringagentorchestrator.py @@ -0,0 +1,69 @@ +import datetime +import json +import tempfile +import uuid +from typing import Any, List + +from dataset.dataengineeringagentinput import EvalDeaRequest +from evaluator.orchestrator import Orchestrator +from evaluator.dataengineeringagentevaluator import ( + DataEngineeringAgentEvaluator, +) + + +class DataEngineeringAgentOrchestrator(Orchestrator): + """Orchestrator designed for pure conversational non-db DEA evaluations. + + Bypasses all legacy database connection handshakes, dialect checks, + and connection pool setups. + """ + + def __init__( + self, + config: dict[str, Any], + db_configs: dict[str, Any] | None = None, + setup_config: dict[str, Any] | None = None, + report_progress: bool = False, + ): + super().__init__(config, db_configs, setup_config, report_progress) + self.job_id = f"dea-job-{uuid.uuid4()}" + + def evaluate(self, dataset: List[EvalDeaRequest]) -> None: + """Orchestrates pure conversational evaluations. + + Delegates straight to DataEngineeringAgentEvaluator. + """ + evaluator = DataEngineeringAgentEvaluator(self.config) + eval_outputs, scoring_results = evaluator.evaluate( + dataset, self.job_id, self.run_time + ) + self.total_eval_outputs.extend(eval_outputs) + self.total_scoring_results.extend(scoring_results) + + def process(self): + """Packages and writes final scores and transcripts to JSON files.""" + with tempfile.NamedTemporaryFile( + mode="w", delete=False, suffix=".json" + ) as f: + json.dump( + self.total_eval_outputs, + f, + sort_keys=True, + indent=4, + default=str + ) + results_tf = f.name + + with tempfile.NamedTemporaryFile( + mode="w", delete=False, suffix=".json" + ) as f: + json.dump( + self.total_scoring_results, + f, + sort_keys=True, + indent=4, + default=str + ) + scores_tf = f.name + + return self.job_id, self.run_time, results_tf, scores_tf, None diff --git a/evalbench/evaluator/db_manager.py b/evalbench/evaluator/db_manager.py index 27a953a4..015a81b8 100644 --- a/evalbench/evaluator/db_manager.py +++ b/evalbench/evaluator/db_manager.py @@ -5,11 +5,13 @@ from concurrent.futures import ThreadPoolExecutor from functools import partial from typing import Optional +import logging def build_db_queue( core_db: DB, db_name, db_config, setup_config, query_type: str, num_dbs: int ): + logging.info(f"Building DB queue (query_type='{query_type}') with {num_dbs} pools for {db_name}...") if query_type == "dql": return _prepare_db_queue_for_dql( core_db, db_name, db_config, setup_config, num_dbs @@ -22,6 +24,8 @@ def build_db_queue( return _prepare_db_queue_for_ddl( core_db, db_name, db_config, setup_config, num_dbs ) + + logging.info(f"Finished building DB queue for query_type '{query_type}' on {db_name}") return Queue[DB]() @@ -96,12 +100,15 @@ def _create_ddl_tmp_db(tmp_db, db_config, setup_scripts): def _get_setup_values(setup_config, db_name: str, db_type: str): try: - setup_scripts = load_setup_scripts( - setup_config["setup_directory"] + "/" + db_name + "/" + db_type - ) - data = load_db_data_from_csvs( - setup_config["setup_directory"] + "/" + db_name + "/data" - ) + scripts_path = setup_config["setup_directory"] + "/" + db_name + "/" + db_type + data_path = setup_config["setup_directory"] + "/" + db_name + "/data" + + logging.info(f"Loading DB setup files from location: {scripts_path}") + setup_scripts = load_setup_scripts(scripts_path) + + logging.info(f"Loading data populate files from location: {data_path}") + data = load_db_data_from_csvs(data_path) + return setup_scripts, data except Exception as e: raise FileNotFoundError( diff --git a/evalbench/evaluator/evaluator.py b/evalbench/evaluator/evaluator.py index 4d1a7ea5..8d6b9de0 100644 --- a/evalbench/evaluator/evaluator.py +++ b/evalbench/evaluator/evaluator.py @@ -1,30 +1,32 @@ +import collections +import concurrent.futures +import datetime import logging +import queue +from queue import Queue import time from typing import Any, List -import datetime -from util import truncateExecutionOutputs -from work import promptgenwork -from work import sqlgenwork -from work import sqlexecwork -from work import scorework -from mp import mprunner -import concurrent.futures +from databases import DB from dataset.evalinput import EvalInputRequest from dataset.evaloutput import EvalOutput from evaluator.progress_reporter import ( record_successful_prompt_gen, - record_successful_sql_gen, - record_successful_sql_exec, record_successful_scoring, + record_successful_sql_exec, + record_successful_sql_gen, ) -from queue import Queue -from databases import DB +from mp import mprunner +from util import truncateExecutionOutputs +from work import multi_trial_scorework +from work import promptgenwork +from work import scorework +from work import sqlexecwork +from work import sqlgenwork def _process_futures_with_timeout( - futures_to_process, - future_to_eval_map, - timeout=600): + futures_to_process, future_to_eval_map, timeout=600 +): """Yields (future, eval_output, timed_out) ensuring we never hang forever on deadlocked tasks.""" uncompleted = set(futures_to_process) # The timeout resets whenever AT LEAST ONE future completes. @@ -36,7 +38,8 @@ def _process_futures_with_timeout( if elapsed_since_last > timeout: logging.error( - f"Abandoning {len(uncompleted)} hung futures after {timeout}s timeout." + f"Abandoning {len(uncompleted)} hung futures after {timeout}s" + " timeout." ) for f in list(uncompleted): uncompleted.remove(f) @@ -44,9 +47,7 @@ def _process_futures_with_timeout( break done, not_done = concurrent.futures.wait( - uncompleted, - timeout=10, - return_when=concurrent.futures.FIRST_COMPLETED + uncompleted, timeout=10, return_when=concurrent.futures.FIRST_COMPLETED ) if done: @@ -58,6 +59,8 @@ def _process_futures_with_timeout( class Evaluator: + """Orchestrates the evaluation pipeline.""" + def __init__( self, config, @@ -70,6 +73,7 @@ def __init__( self.scoring_runners = runner_config.get("scoring_runners", 10) self.task_timeout_seconds = runner_config.get( "task_timeout_seconds", 600) + self.num_trials = self.config.get("num_trials", 1) def evaluate( self, @@ -85,6 +89,7 @@ def evaluate( ): eval_outputs: List[Any] = [] scoring_results: List[Any] = [] + multi_trial_scoring_results: List[Any] = [] self.promptrunner = mprunner.MPRunner(self.promptgen_runners) self.genrunner = mprunner.MPRunner(self.sqlgen_runners) @@ -98,6 +103,7 @@ def evaluate( self.scoringrunner.futures.clear() prompt_future_to_eval = {} + prompt_future_to_input = {} for eval_input in dataset: eval_output = EvalOutput(eval_input) eval_output["job_id"] = job_id @@ -106,12 +112,18 @@ def evaluate( prompt_generator, eval_output) self.promptrunner.execute_work(work) prompt_future_to_eval[self.promptrunner.futures[-1]] = eval_output + prompt_future_to_input[self.promptrunner.futures[-1]] = eval_input gen_future_to_eval = {} for future, eval_output, timed_out in _process_futures_with_timeout( - self.promptrunner.futures, prompt_future_to_eval, timeout=self.task_timeout_seconds): + self.promptrunner.futures, + prompt_future_to_eval, + timeout=self.task_timeout_seconds, + ): if timed_out: - eval_output["prompt_generator_error"] = "TimeoutError: Task hung for too long." + eval_output["prompt_generator_error"] = ( + "TimeoutError: Task hung for too long." + ) else: try: future.result() @@ -121,16 +133,36 @@ def evaluate( eval_output["prompt_generator_error"] = str(e) record_successful_prompt_gen(progress_reporting) - work = sqlgenwork.SQLGenWork(model_generator, eval_output) - self.genrunner.execute_work(work) - gen_future_to_eval[self.genrunner.futures[-1]] = eval_output + + eval_input = prompt_future_to_input[future] + + query_type = eval_output.get("query_type", "dql").lower() + trials_to_run = self.num_trials if query_type == "dql" else 1 + for trial_idx in range(trials_to_run): + trial_output = EvalOutput(eval_input) + trial_output["job_id"] = job_id + trial_output["run_time"] = run_time + trial_output.update(eval_output) + + trial_output["prompt_id"] = eval_output["id"] + trial_output["trial_index"] = trial_idx + trial_output["id"] = f"{eval_output['id']}_trial_{trial_idx}" + + work = sqlgenwork.SQLGenWork(model_generator, trial_output) + self.genrunner.execute_work(work) + gen_future_to_eval[self.genrunner.futures[-1]] = trial_output exec_future_to_eval = {} score_future_to_eval = {} for future, eval_output, timed_out in _process_futures_with_timeout( - self.genrunner.futures, gen_future_to_eval, timeout=self.task_timeout_seconds): + self.genrunner.futures, + gen_future_to_eval, + timeout=self.task_timeout_seconds, + ): if timed_out: - eval_output["sql_generator_error"] = "TimeoutError: Task hung for too long." + eval_output["sql_generator_error"] = ( + "TimeoutError: Task hung for too long." + ) else: try: future.result() @@ -142,17 +174,31 @@ def evaluate( record_successful_sql_gen(progress_reporting) try: - db_conn = db_queue.get(timeout=60) + db_conn = db_queue.get(timeout=180) work = sqlexecwork.SQLExecWork( db_conn, self.config, eval_output, db_queue ) self.sqlrunner.execute_work(work) exec_future_to_eval[self.sqlrunner.futures[-1]] = eval_output - except Exception as e: + except queue.Empty: + error_msg = f"Timeout Error: Waited too long (queue.Empty) for database '{eval_output.get('database', 'unknown')}'" + logging.error(error_msg) + eval_output["generated_error"] = error_msg + record_successful_sql_exec(progress_reporting) + work = scorework.ScorerWork( + self.config, eval_output, scoring_results, global_models + ) + self.scoringrunner.execute_work(work) + score_future_to_eval[self.scoringrunner.futures[-1] + ] = eval_output + except Exception as e: + exc_msg = str(e) or type(e).__name__ logging.error( - f"Failed to acquire DB connection from queue for database '{eval_output.get('database')}': {e}") - eval_output["generated_error"] = f"Failed to acquire DB connection: {e}" + "Failed to acquire DB connection from queue for database" + f" '{eval_output.get('database')}': {exc_msg}" + ) + eval_output["generated_error"] = f"Failed to acquire DB connection: {exc_msg}" record_successful_sql_exec(progress_reporting) work = scorework.ScorerWork( self.config, eval_output, scoring_results, global_models @@ -162,7 +208,10 @@ def evaluate( ] = eval_output for future, eval_output, timed_out in _process_futures_with_timeout( - self.sqlrunner.futures, exec_future_to_eval, timeout=self.task_timeout_seconds): + self.sqlrunner.futures, + exec_future_to_eval, + timeout=self.task_timeout_seconds, + ): if timed_out: eval_output["generated_error"] = "TimeoutError: Task hung for too long." else: @@ -181,7 +230,10 @@ def evaluate( score_future_to_eval[self.scoringrunner.futures[-1]] = eval_output for future, eval_output, timed_out in _process_futures_with_timeout( - self.scoringrunner.futures, score_future_to_eval, timeout=self.task_timeout_seconds): + self.scoringrunner.futures, + score_future_to_eval, + timeout=self.task_timeout_seconds, + ): if timed_out: eval_output["scoring_error"] = "TimeoutError: Task hung for too long." else: @@ -203,8 +255,42 @@ def evaluate( logging.error(f"Truncation error: {e}") eval_outputs.append(eval_output) + if self.num_trials > 1: + grouped_trials = collections.defaultdict(list) + for eo in eval_outputs: + prompt_id = eo.get("prompt_id") + if prompt_id: + grouped_trials[prompt_id].append(eo) + + multi_trial_futures = {} + for prompt_id, trials in grouped_trials.items(): + nl_prompt = trials[0].get("nl_prompt", "") + work = multi_trial_scorework.MultiTrialScorerWork( + prompt_id, + nl_prompt, + trials, + self.config, + multi_trial_scoring_results, + global_models, + progress_reporting, + ) + self.scoringrunner.execute_work(work) + multi_trial_futures[self.scoringrunner.futures[-1]] = trials[0] + + for future, _, timed_out in _process_futures_with_timeout( + list(multi_trial_futures.keys()), + multi_trial_futures, + timeout=self.task_timeout_seconds, + ): + if timed_out: + logging.error("Multi-trial scoring timed out.") + else: + try: + future.result() + except Exception as e: + logging.error(f"Multi-trial scoring future error: {e}") + if close_connections and db_queue: - import queue while True: try: db = db_queue.get(block=False) @@ -214,4 +300,4 @@ def evaluate( except Exception: break - return eval_outputs, scoring_results + return eval_outputs, scoring_results, multi_trial_scoring_results diff --git a/evalbench/evaluator/interactorchestrator.py b/evalbench/evaluator/interactorchestrator.py index 6bef89a8..b16c4abe 100644 --- a/evalbench/evaluator/interactorchestrator.py +++ b/evalbench/evaluator/interactorchestrator.py @@ -174,6 +174,7 @@ def evaluate_sub_dataset( except Exception as e: logging.error( f"Skipping {query_type} queries as DB {database} " + + f"(dataset: {self.get_display_dataset_config()}) " + f"could not be setup properly in {dialect} due to {e}." ) skip_database( @@ -224,4 +225,5 @@ def process(self): self.run_time, results_tf, scores_tf, + None, ) diff --git a/evalbench/evaluator/mcp_readability/__init__.py b/evalbench/evaluator/mcp_readability/__init__.py new file mode 100644 index 00000000..bff0cfa9 --- /dev/null +++ b/evalbench/evaluator/mcp_readability/__init__.py @@ -0,0 +1,3 @@ +from evaluator.mcp_readability.orchestrator import McpReadabilityOrchestrator + +__all__ = ["McpReadabilityOrchestrator"] diff --git a/evalbench/evaluator/mcp_readability/exceptions.py b/evalbench/evaluator/mcp_readability/exceptions.py new file mode 100644 index 00000000..49ed8b0a --- /dev/null +++ b/evalbench/evaluator/mcp_readability/exceptions.py @@ -0,0 +1,71 @@ +"""Loading and matching of per-endpoint style-rule exceptions (waivers). + +An exceptions file lets an operator waive a specific style requirement for a +specific endpoint when it legitimately cannot comply. Waived rules are passed to +the scorer so they are excluded from the P0/P1/P2 counts and surfaced separately +in the feedback. + +File schema (see ``datasets/mcp_readability/exceptions.yaml``):: + + exceptions: + - product_name: "Cloud SQL" # any of product_name / endpoint_type + rule_id: "tool-names" + reason: "..." + - endpoint_type: AUTOPUSH # "*" or omitted field = match-all + rule_id: "use-enums" + reason: "..." +""" + +import logging +from util.config import load_yaml_config + + +def load_exceptions(path: str) -> list[dict]: + """Load the exceptions list from a YAML file. Missing/empty -> [].""" + if not path: + return [] + parsed = load_yaml_config(path) + if not parsed: + return [] + exceptions = parsed.get("exceptions") or [] + if not isinstance(exceptions, list): + logging.warning( + "mcp_readability: 'exceptions' in %s is not a list; ignoring.", path + ) + return [] + return exceptions + + +def _matches(field_value, exception_value) -> bool: + """A matcher field matches when it is absent, '*', or equal (case-insensitive).""" + if exception_value is None or exception_value == "*": + return True + if field_value is None: + return False + return str(field_value).strip().lower() == str(exception_value).strip().lower() + + +def applicable_exceptions(endpoint: dict, all_exceptions: list[dict]) -> list[dict]: + """Return exceptions whose matchers all apply to ``endpoint``. + + Matchers considered: ``product_name`` and ``endpoint_type`` (the endpoint's + identity in #469's ``endpoints.yaml``). Each exception keeps its ``rule_id`` + and ``reason`` for the scorer prompt. + """ + matched = [] + for exc in all_exceptions: + if not isinstance(exc, dict): + continue + if not exc.get("rule_id"): + continue + if ( + _matches(endpoint.get("product_name"), exc.get("product_name")) + and _matches(endpoint.get("endpoint_type"), exc.get("endpoint_type")) + ): + matched.append( + { + "rule_id": exc.get("rule_id"), + "reason": exc.get("reason", ""), + } + ) + return matched diff --git a/evalbench/evaluator/mcp_readability/orchestrator.py b/evalbench/evaluator/mcp_readability/orchestrator.py new file mode 100644 index 00000000..97b084b5 --- /dev/null +++ b/evalbench/evaluator/mcp_readability/orchestrator.py @@ -0,0 +1,318 @@ +"""Orchestrator for the MCP style-guide readability check. + +Flow per endpoint: + 1. The tools generator fetches the endpoint's tools and renders the man-page + markup. + 2. Applicable exceptions (waivers) are gathered. + 3. Every scorer declared under ``scorers:`` in the run config is invoked with + the shared per-endpoint context; each contributes result-row columns and a + binary summary score (see ``scorers.mcp_readability_scoring``). + 4. A result row is assembled from the base identity columns plus every + scorer's contribution. + +The orchestrator is scorer-agnostic: it instantiates whatever scorers the run +config declares (via ``SCORER_REGISTRY``) and merges their contributions. Adding +a new scorer (e.g. conformance testing) needs only a registry entry and a +run-config block -- no changes here. + +Failure handling is fail-fast: if any endpoint cannot be fetched or scored, the +exception propagates and the whole job aborts with nothing persisted. There is no +per-endpoint status; the failure surfaces in the run log. Isolate flaky or +experimental endpoints with a separate run config rather than per-row state. + +Like the standard EvalBench orchestrators, :meth:`process` dumps the result rows +to a temp JSON and returns it as ``results_tf``; ``evalbench.py`` then hands that +to the shared reporters (CSV / BigQuery). The orchestrator performs no direct +CSV or BigQuery writes itself. +""" + +import concurrent.futures +import datetime +import json +import logging +import tempfile +import threading + +from evaluator.orchestrator import Orchestrator +from generators.models import get_generator +from scorers.mcp_readability_scoring import EndpointContext +from scorers.mcp_style_readability import McpStyleReadabilityScorer +from scorers.mcp_tool_metrics import McpToolMetricsScorer +from util.config import load_yaml_config + +from evaluator.mcp_readability import exceptions as exceptions_mod + + +# Registered mcp_readability scorers, keyed by the name used under ``scorers:`` in +# the run config (also each scorer's ``comparator`` in the summary). Add a new +# scorer here plus a run-config block to extend the check -- the orchestrator +# itself stays unchanged. +SCORER_REGISTRY = { + "mcp_tool_metrics": McpToolMetricsScorer, + "mcp_style_readability": McpStyleReadabilityScorer, +} + + +# Allowed endpoint_type values (deployment channel / dashboard categorization). +# Validated at load time; an unknown value fails the run fast. +ALLOWED_ENDPOINT_TYPES = ("PROD", "AUTOPUSH", "STAGING", "DEV") + + +# Base identity columns present on every result row (``job_id`` is the shared +# framework column). Each configured scorer appends its own COLUMNS; the full, +# canonical schema for a run is ``self.columns``. Columns are prefixed +# ``mcp_readability_`` so rows coexist with other eval types in the shared +# ``.evalbench.results`` table (``mcp_readability_score`` non-null is the +# readability-row discriminator). No per-endpoint status column: a fetch/scoring +# failure aborts the whole job (fail-fast), so every persisted row is successful. +BASE_COLUMNS = [ + "mcp_readability_product_name", + "mcp_readability_source_url", + "mcp_readability_endpoint_type", + "mcp_readability_check_timestamp", + "job_id", +] + + +def _validate_endpoint_type(value) -> str: + """Normalize + validate an endpoint_type; raise on unknown (fail-fast).""" + if value is None: + raise ValueError("mcp_readability: endpoint_type is required") + name = str(value).strip().upper() + if name not in ALLOWED_ENDPOINT_TYPES: + raise ValueError( + f"mcp_readability: unknown endpoint_type {value!r}; " + f"allowed: {', '.join(ALLOWED_ENDPOINT_TYPES)}" + ) + return name + + +class McpReadabilityOrchestrator(Orchestrator): + """Checks a list of MCP endpoints against an MCP tool style guide.""" + + def __init__(self, config, db_configs, setup_config, report_progress=False): + super().__init__(config, db_configs, setup_config, report_progress) + + runner_config = config.get("runners", {}) or {} + self.endpoint_runners = runner_config.get("endpoint_runners", 4) + + # Load endpoints (a flat list; no shared defaults block). + endpoints_config = config.get("endpoints_config") + if not endpoints_config: + raise ValueError("mcp_readability requires 'endpoints_config'") + parsed = load_yaml_config(endpoints_config) or {} + self.endpoints = parsed.get("endpoints") or [] + if not self.endpoints: + logging.warning( + "mcp_readability: no endpoints found in %s", endpoints_config + ) + + # Exceptions (waivers). + self.all_exceptions = exceptions_mod.load_exceptions( + config.get("exceptions_config") + ) + + # Optional endpoint_type filter (validated against the allowed set). + type_filter = config.get("endpoint_types") or [] + self.endpoint_type_filter = { + _validate_endpoint_type(t) for t in type_filter + } + + # Shared model registry for get_generator (lock + cache): get_generator + # memoizes instantiated model clients here (guarded by the lock) so + # repeated calls -- e.g. across scorers and endpoint threads -- reuse the + # same client instead of re-instantiating one per call. + self.global_models = { + "lock": threading.Lock(), + "registered_models": {}, + } + + # Tools generator (the "system under test" fetcher). + tools_generator_config = config.get("tools_generator_config") + if not tools_generator_config: + raise ValueError("mcp_readability requires 'tools_generator_config'") + self.tools_generator = get_generator( + self.global_models, tools_generator_config + ) + + # Plug-and-play scorers: instantiate exactly what the run config declares. + # Each scorer owns and validates its own config block (token_budget for + # metrics; model_config + style_guide for the LLM judge). + scorers_config = config.get("scorers") or {} + if not scorers_config: + raise ValueError("mcp_readability requires a 'scorers' block") + self.scorers = [] + for name, scorer_config in scorers_config.items(): + scorer_cls = SCORER_REGISTRY.get(name) + if scorer_cls is None: + raise ValueError( + f"mcp_readability: unknown scorer {name!r}; " + f"known: {', '.join(sorted(SCORER_REGISTRY))}" + ) + self.scorers.append( + scorer_cls(scorer_config or {}, self.global_models) + ) + + # Full canonical result schema for this run: base identity columns plus + # every configured scorer's columns. + self.columns = list(BASE_COLUMNS) + for scorer in self.scorers: + self.columns.extend(scorer.COLUMNS) + + self.rows = [] + self.score_rows = [] + + # ------------------------------------------------------------------ + # Public lifecycle + # ------------------------------------------------------------------ + def evaluate(self, dataset=None): + """Run the readability check for every (filtered) endpoint.""" + endpoints = self._filtered_endpoints() + if not endpoints: + logging.warning( + "mcp_readability: no endpoints to check after filtering." + ) + self.rows = [] + self.score_rows = [] + return + + workers = max(1, int(self.endpoint_runners)) + results = [] # list[(row, score_rows)] + with concurrent.futures.ThreadPoolExecutor(max_workers=workers) as pool: + futures = { + pool.submit(self._check_endpoint, ep): ep for ep in endpoints + } + # fail-fast: _check_endpoint lets exceptions propagate, so the first + # failed future re-raises here and aborts the run. + for future in concurrent.futures.as_completed(futures): + results.append(future.result()) + + # Deterministic ordering (by product_name then source url). + results.sort( + key=lambda rs: ( + rs[0].get("mcp_readability_product_name", ""), + rs[0].get("mcp_readability_source_url", ""), + ) + ) + self.rows = [row for row, _ in results] + self.score_rows = [s for _, score_rows in results for s in score_rows] + if self.report_progress: + logging.info( + "mcp_readability: checked %d endpoints.", len(self.rows) + ) + + def process(self): + """Dump result + score rows to temp JSONs and return them. + + Matches the standard orchestrator contract so ``evalbench.py`` + the + shared reporters handle the CSV / BigQuery writes with no special casing: + + - ``results_tf`` holds the full per-endpoint readability rows (drives the + evals output). + - ``scores_tf`` holds one score row per (endpoint, scorer) so the shared + analyzer aggregates a pass rate for each scorer key declared under + ``scorers:`` (drives the scores / summary output). + """ + results_tf = self._dump_temp_json(self.rows) + scores_tf = self._dump_temp_json(self.score_rows) + return (self.job_id, self.run_time, results_tf, scores_tf, None) + + # ------------------------------------------------------------------ + # Per-endpoint work + # ------------------------------------------------------------------ + def _check_endpoint(self, endpoint: dict): + """Fetch + score one endpoint. Returns ``(row, score_rows)``. + + Any failure propagates (fail-fast); the run aborts and nothing persists. + """ + product_name = endpoint.get("product_name", "") + endpoint_type = _validate_endpoint_type(endpoint.get("endpoint_type")) + endpoint_url = self._endpoint_ref(endpoint) + + row = self._base_row(product_name, endpoint_url, endpoint_type) + row["job_id"] = self.job_id + + try: + # 1. Fetch tools + render man-page markup. + tools, man_page = self.tools_generator.fetch_tools(endpoint) + + # 2. Exceptions (waivers) for this endpoint. + applicable = exceptions_mod.applicable_exceptions( + endpoint, self.all_exceptions + ) + + # 3. Run every configured scorer against the shared context. + context = EndpointContext( + product_name=product_name, + endpoint=endpoint, + tools=tools, + man_page=man_page, + exceptions=applicable, + ) + score_rows = [] + for scorer in self.scorers: + contribution = scorer.run(context) + row.update(contribution.row_fields) + score_rows.append( + { + "id": product_name or endpoint_url, + "comparator": scorer.name, + "score": contribution.score, + "comparison_logs": contribution.logs, + "comparison_error": None, + } + ) + except Exception: + logging.exception( + "mcp_readability: check failed for %s (%s); aborting run.", + product_name, + endpoint_url, + ) + raise + + return row, score_rows + + # ------------------------------------------------------------------ + # Helpers + # ------------------------------------------------------------------ + @staticmethod + def _endpoint_ref(endpoint: dict) -> str: + """A stable human-readable reference for an endpoint's tool source. + + Reported as ``mcp_readability_source_url``, and used as the sort key and + the score-row id. Falls back to the file ``path`` for offline sources + (which have no ``url``) so distinct endpoints don't collapse to "". + """ + src = endpoint.get("tools_source") or {} + return src.get("url") or src.get("path") or "" + + @staticmethod + def _dump_temp_json(rows) -> str: + with tempfile.NamedTemporaryFile( + mode="w", delete=False, suffix=".json" + ) as f: + json.dump(rows, f, sort_keys=True, indent=4, default=str) + return f.name + + def _filtered_endpoints(self) -> list[dict]: + if not self.endpoint_type_filter: + return list(self.endpoints) + kept = [] + for ep in self.endpoints: + if _validate_endpoint_type(ep.get("endpoint_type")) in ( + self.endpoint_type_filter + ): + kept.append(ep) + return kept + + @staticmethod + def _base_row(product_name, endpoint_url, endpoint_type) -> dict: + return { + "mcp_readability_product_name": product_name, + "mcp_readability_source_url": endpoint_url, + "mcp_readability_endpoint_type": endpoint_type, + "mcp_readability_check_timestamp": ( + datetime.datetime.now().isoformat() + ), + "job_id": "", + } diff --git a/evalbench/evaluator/oneshotorchestrator.py b/evalbench/evaluator/oneshotorchestrator.py index 9ca99505..84d9e624 100644 --- a/evalbench/evaluator/oneshotorchestrator.py +++ b/evalbench/evaluator/oneshotorchestrator.py @@ -1,20 +1,17 @@ -import logging - import concurrent.futures import datetime import json - +import logging +from multiprocessing import get_context import tempfile import threading import uuid -from multiprocessing import get_context import databases -import generators.models as models -import generators.prompts as prompts from dataset.evalinput import EvalInputRequest, breakdown_datasets from evaluator.db_manager import build_db_queue from evaluator.evaluator import Evaluator +from evaluator.orchestrator import Orchestrator from evaluator.progress_reporter import ( cleanup_progress_reporting, record_successful_setup, @@ -22,10 +19,12 @@ skip_database, skip_dialect, ) -from evaluator.orchestrator import Orchestrator +import generators.models as models +import generators.prompts as prompts class OneShotOrchestrator(Orchestrator): + def __init__( self, config, @@ -40,18 +39,22 @@ def __init__( self.run_time = datetime.datetime.now() self.total_eval_outputs = [] self.total_scoring_results = [] + self.total_multi_trial_scoring_results = [] self.reporting_total_evals_done = 0 self.report_progress = report_progress runner_config = self.config.get("runners", {}) self.eval_runners = runner_config.get("eval_runners", 4) self.sqlexec_runners = runner_config.get("sqlexec_runners", 10) + self.num_trials = self.config.get("num_trials", 1) def evaluate(self, dataset: list[EvalInputRequest]): - """This wrapper breaks down evaluations by category of evaluations. (dql, dml, ddl). - This allows the module to prepare the correct database connections as DDL queries - require setting up and tearing down the databsae and DML queries require prevention - of unintended consequences. Additionally, DQLs are run under a read-only user. + """This wrapper breaks down evaluations by category of evaluations. + + (dql, dml, ddl). This allows the module to prepare the correct database + connections as DDL queries require setting up and tearing down the databsae + and DML queries require prevention of unintended consequences. Additionally, + DQLs are run under a read-only user. """ progress_reporting_thread = None progress_reporting_finished = None @@ -59,9 +62,10 @@ def evaluate(self, dataset: list[EvalInputRequest]): tmp_buffer = None colab_progress_report = None - with get_context('spawn').Manager() as manager: + with get_context("spawn").Manager() as manager: sub_datasets, total_dataset_len, total_db_len = breakdown_datasets( - dataset) + dataset + ) try: if self.report_progress: ( @@ -71,7 +75,7 @@ def evaluate(self, dataset: list[EvalInputRequest]): tmp_buffer, colab_progress_report, ) = setup_progress_reporting( - manager, total_dataset_len, total_db_len + manager, total_dataset_len, total_db_len, self.num_trials ) global_models = {"registered_models": {}, @@ -85,8 +89,9 @@ def evaluate(self, dataset: list[EvalInputRequest]): db_configs = self.db_configs.get(dialect) if not db_configs: logging.info( - f"Skipping queries for {dialect} as no applicable db_config" + - " was found.") + f"Skipping queries for {dialect} as no applicable db_config" + + " was found." + ) skip_dialect( sub_datasets[dialect], progress_reporting) continue @@ -107,15 +112,22 @@ def evaluate(self, dataset: list[EvalInputRequest]): # 24 hour timeout on the entire evaluator thread # execution by default timeout_seconds = self.config.get( - "orchestrator_timeout_seconds", 86400) - eval_outputs, scoring_results = future.result( - timeout=timeout_seconds) + "orchestrator_timeout_seconds", 86400 + ) + eval_outputs, scoring_results, multi_trial_scoring_results = ( + future.result(timeout=timeout_seconds) + ) self.total_eval_outputs.extend(eval_outputs) self.total_scoring_results.extend(scoring_results) + self.total_multi_trial_scoring_results.extend( + multi_trial_scoring_results + ) except concurrent.futures.TimeoutError: logging.error( - f"A runner thread timed out and failed to complete within {timeout_seconds} seconds.") + "A runner thread timed out and failed to complete within" + f" {timeout_seconds} seconds." + ) except Exception as e: logging.error(f"A runner thread failed: {e}") @@ -146,6 +158,7 @@ def evaluate_sub_dataset( ): total_eval_outputs = [] total_scoring_results = [] + total_multi_trial_scoring_results = [] # Map database name if config provides mappings actual_db_name = database @@ -165,7 +178,9 @@ def evaluate_sub_dataset( skip_database(sub_datasets[dialect] [database], progress_reporting, None) logging.error( - f"Could not connect to database {actual_db_name} (from {database}) on {dialect}; due to {e}") + f"Could not connect to database {actual_db_name} (from {database}) on" + f" {dialect}; due to {e}" + ) return [], [] prompt_generator = prompts.get_generator(core_db, self.config) @@ -192,32 +207,37 @@ def evaluate_sub_dataset( except Exception as e: logging.error( f"Skipping {query_type} queries as DB {database} " + + f"(dataset: {self.get_display_dataset_config()}) " + f"could not be setup properly in {dialect} due to {e}." ) skip_database( - sub_datasets[dialect][database], - progress_reporting, - query_type) + sub_datasets[dialect][database], progress_reporting, query_type + ) continue evaluator = Evaluator(self.config) try: - eval_outputs, scoring_results = evaluator.evaluate( - sub_dataset, - db_queue, - prompt_generator, - model_generator, - self.job_id, - self.run_time, - progress_reporting, - global_models, + eval_outputs, scoring_results, multi_trial_scoring_results = ( + evaluator.evaluate( + sub_dataset, + db_queue, + prompt_generator, + model_generator, + self.job_id, + self.run_time, + progress_reporting, + global_models, + ) ) total_eval_outputs.extend(eval_outputs) total_scoring_results.extend(scoring_results) + total_multi_trial_scoring_results.extend( + multi_trial_scoring_results) except Exception as e: logging.error( - f"Failed to evaluate {sub_dataset_len} {query_type} queries " + - f"on DB {database} on {dialect}. Due to {e}") + f"Failed to evaluate {sub_dataset_len} {query_type} queries " + + f"on DB {database} on {dialect}. Due to {e}" + ) # Cleanup all the tmp creations that were built from the core # connection @@ -225,24 +245,42 @@ def evaluate_sub_dataset( core_db.clean_tmp_creations() core_db.close_connections() - return total_eval_outputs, total_scoring_results + return ( + total_eval_outputs, + total_scoring_results, + total_multi_trial_scoring_results, + ) def process(self): - with tempfile.NamedTemporaryFile(mode="w", delete=False, suffix=".json") as f: - json.dump(self.total_eval_outputs, f, - sort_keys=True, indent=4, default=str) + with tempfile.NamedTemporaryFile( + mode="w", delete=False, suffix=".json" + ) as f: + json.dump( + self.total_eval_outputs, f, sort_keys=True, indent=4, default=str + ) results_tf = f.name - with tempfile.NamedTemporaryFile(mode="w", delete=False, suffix=".json") as f: + with tempfile.NamedTemporaryFile( + mode="w", delete=False, suffix=".json" + ) as f: json.dump( - self.total_scoring_results, + self.total_scoring_results, f, sort_keys=True, indent=4, default=str + ) + scores_tf = f.name + with tempfile.NamedTemporaryFile( + mode="w", delete=False, suffix=".json" + ) as f: + json.dump( + self.total_multi_trial_scoring_results, f, sort_keys=True, indent=4, - default=str) - scores_tf = f.name + default=str, + ) + multi_trial_scores_tf = f.name return ( self.job_id, self.run_time, results_tf, scores_tf, + multi_trial_scores_tf, ) diff --git a/evalbench/evaluator/orchestrator.py b/evalbench/evaluator/orchestrator.py index 9308f038..0b5a3230 100644 --- a/evalbench/evaluator/orchestrator.py +++ b/evalbench/evaluator/orchestrator.py @@ -59,4 +59,13 @@ def process(self): self.run_time, None, None, + None, ) + + def get_display_dataset_config(self) -> str: + """Returns a sanitized configuration name suitable for safe log outputs.""" + cand = self.config.get("dataset_config", "Unknown config") + if not isinstance(cand, str): + return "Unknown config" + g3_idx = cand.find("google3/") + return cand[g3_idx:] if g3_idx != -1 else cand diff --git a/evalbench/evaluator/progress_reporter.py b/evalbench/evaluator/progress_reporter.py index 173a8ed4..28e3d53f 100644 --- a/evalbench/evaluator/progress_reporter.py +++ b/evalbench/evaluator/progress_reporter.py @@ -1,13 +1,11 @@ -import time +from io import StringIO import logging -import os - - from multiprocessing.managers import SyncManager +import os import sys import threading -from io import StringIO import threading +import time _ORIGINAL_STDOUT = sys.stdout _ORIGINAL_STDERR = sys.stderr @@ -24,7 +22,10 @@ def setup_progress_reporting( - manager: SyncManager, total_dataset_len: int, total_dbs: int + manager: SyncManager, + total_dataset_len: int, + total_dbs: int, + num_trials: int = 1, ): if sys.argv[0].endswith("eval_server.py"): return None, None, None, None, None @@ -39,6 +40,11 @@ def setup_progress_reporting( "exec_i": manager.Value("i", 0), "score_i": manager.Value("i", 0), "total": total_dataset_len, + "total_trials": total_dataset_len * num_trials, + "total_scoring": ( + total_dataset_len * num_trials + + (total_dataset_len if num_trials > 1 else 0) + ), "total_dbs": total_dbs, } if _IN_COLAB: @@ -83,10 +89,11 @@ def _setup_stdout_reporting(): def _report( - progress_reporting, - progress_reporting_finished, - tmp_buffer, - colab_progress_report): + progress_reporting, + progress_reporting_finished, + tmp_buffer, + colab_progress_report, +): last_change_time = time.time() last_counts = {} @@ -105,7 +112,12 @@ def _report( last_counts = current_counts last_change_time = time.time() elif time.time() - last_change_time > warn_seconds: - msg = f"\nWARNING: No progress observed for {warn_seconds} seconds. Currently at: Prompt {current_counts['prompt']}, Gen {current_counts['gen']}, Exec {current_counts['exec']}, Score {current_counts['score']} / {progress_reporting['total']}\n" + msg = ( + f"\nWARNING: No progress observed for {warn_seconds} seconds." + f" Currently at: Prompt {current_counts['prompt']}, Gen" + f" {current_counts['gen']}, Exec {current_counts['exec']}, Score" + f" {current_counts['score']} / {progress_reporting['total']}\n" + ) if tmp_buffer: _ORIGINAL_STDOUT.write(msg) _ORIGINAL_STDOUT.flush() @@ -124,17 +136,20 @@ def _report( def _colab_progress(progress_reporting): setup_done = ( - progress_reporting["setup_i"].value / progress_reporting["total_dbs"] + progress_reporting["setup_i"].value / (progress_reporting["total_dbs"] or 1) ) * 100 prompt_done = ( progress_reporting["prompt_i"].value / progress_reporting["total"] ) * 100 - gen_done = (progress_reporting["gen_i"].value / - progress_reporting["total"]) * 100 - exec_done = (progress_reporting["exec_i"].value / - progress_reporting["total"]) * 100 + gen_done = ( + progress_reporting["gen_i"].value / progress_reporting["total_trials"] + ) * 100 + exec_done = ( + progress_reporting["exec_i"].value / progress_reporting["total_trials"] + ) * 100 score_done = ( - progress_reporting["score_i"].value / progress_reporting["total"] + progress_reporting["score_i"].value / + progress_reporting["total_scoring"] ) * 100 return HTML( # type: ignore """ @@ -187,6 +202,8 @@ def _print_report(progress_reporting, tmp_buffer): exec_i = progress_reporting["exec_i"].value score_i = progress_reporting["score_i"].value dataset_len = progress_reporting["total"] + total_trials = progress_reporting["total_trials"] + total_scoring = progress_reporting["total_scoring"] databases = progress_reporting["total_dbs"] if tmp_buffer: @@ -204,19 +221,16 @@ def _print_report(progress_reporting, tmp_buffer): setup_i, databases, prefix="DBs Setup:", suffix="Complete", length=50 ) report_progress( - prompt_i, - dataset_len, - prefix="Prompts: ", - suffix="Complete", - length=50) + prompt_i, dataset_len, prefix="Prompts: ", suffix="Complete", length=50 + ) report_progress( - gen_i, dataset_len, prefix="SQLGen: ", suffix="Complete", length=50 + gen_i, total_trials, prefix="SQLGen: ", suffix="Complete", length=50 ) report_progress( - exec_i, dataset_len, prefix="SQLExec: ", suffix="Complete", length=50 + exec_i, total_trials, prefix="SQLExec: ", suffix="Complete", length=50 ) report_progress( - score_i, dataset_len, prefix="Scoring: ", suffix="Complete", length=50 + score_i, total_scoring, prefix="Scoring: ", suffix="Complete", length=50 ) _ORIGINAL_STDOUT.flush() @@ -275,9 +289,8 @@ def record_successful_setup(progress_reporting): def cleanup_progress_reporting( - progress_report, - tmp_buffer, - colab_progress_report): + progress_report, tmp_buffer, colab_progress_report +): if not progress_report: return if _IN_COLAB: @@ -305,14 +318,15 @@ def report_progress( fill="█", printEnd="\n", ): - """ - Call in a loop to create terminal progress bar + """Call in a loop to create terminal progress bar + @params: iteration - Required : current iteration (Int) total - Required : total iterations (Int) prefix - Optional : prefix string (Str) suffix - Optional : suffix string (Str) - decimals - Optional : positive number of decimals in percent complete (Int) + decimals - Optional : positive number of decimals in percent complete + (Int) length - Optional : character length of bar (Int) fill - Optional : bar fill character (Str) printEnd - Optional : end character (e.g. "\r", "\r\n") (Str) diff --git a/evalbench/evaluator/simulateduser.py b/evalbench/evaluator/simulateduser.py index a24ec158..35d3c2ad 100644 --- a/evalbench/evaluator/simulateduser.py +++ b/evalbench/evaluator/simulateduser.py @@ -1,7 +1,6 @@ import generators.models as models import generators.prompts as prompts import threading -import logging class SimulatedUser: @@ -13,8 +12,11 @@ def __init__(self, config): "registered_models": {} } - # Expect 'simulated_user_model_config' path in config model_config_path = config.get("simulated_user_model_config") + if not model_config_path: + raise ValueError( + "SimulatedUser requires 'simulated_user_model_config' in the run config; " + "without it every scenario terminates on turn 1.") self.prompt_generator = prompts.get_generator( None, @@ -22,24 +24,16 @@ def __init__(self, config): "SimulatedUserPromptGenerator" ) - self.model_generator = None - if model_config_path: - try: - self.model_generator = models.get_generator( - global_models, model_config_path, None - ) - except Exception as e: - logging.warning( - f"Failed to load simulated user model from {model_config_path}: {e}") - else: - logging.warning( - "No 'simulated_user_model_config' provided. SimulatedUser will not be able to generate responses.") + try: + self.model_generator = models.get_generator( + global_models, model_config_path, None + ) + except Exception as e: + raise RuntimeError( + f"Failed to load simulated user model from {model_config_path}: {e}" + ) from e def get_next_response(self, conversation_plan: str, history: list, last_agent_reply: str) -> str: - if not self.model_generator: - logging.error("Model generator not initialized.") - return "TERMINATE" - payload = { "conversation_plan": conversation_plan, "history": history, diff --git a/evalbench/evaluator/streamingorchestrator.py b/evalbench/evaluator/streamingorchestrator.py index afd8605d..263db112 100644 --- a/evalbench/evaluator/streamingorchestrator.py +++ b/evalbench/evaluator/streamingorchestrator.py @@ -32,6 +32,7 @@ def __init__( self.run_time = datetime.datetime.now() self.total_eval_outputs = [] self.total_scoring_results = [] + self.total_multi_trial_scoring_results = [] self.reporting_total_evals_done = 0 self.report_progress = report_progress @@ -46,7 +47,8 @@ def __init__( # Cache keyed by (dialect, database, query_type) self._db_queue_cache = {} - self._global_models = {"registered_models": {}, "lock": threading.Lock()} + self._global_models = { + "registered_models": {}, "lock": threading.Lock()} self._cache_lock = threading.Lock() self._results_lock = threading.Lock() @@ -119,7 +121,7 @@ def _evaluate_single(self, eval_input: EvalInputRequest, db_config, dialect): return evaluator = Evaluator(self.config) - eval_outputs, scoring_results = evaluator.evaluate( + eval_outputs, scoring_results, multi_trial_scoring_results = evaluator.evaluate( [eval_input], db_queue, prompt_gen, @@ -133,6 +135,8 @@ def _evaluate_single(self, eval_input: EvalInputRequest, db_config, dialect): with self._results_lock: self.total_eval_outputs.extend(eval_outputs) self.total_scoring_results.extend(scoring_results) + self.total_multi_trial_scoring_results.extend( + multi_trial_scoring_results) def _resolve_db_name(self, dialect, database): db_name_overrides = self.config.get("db_name_overrides", {}) @@ -181,9 +185,21 @@ def process(self): self.total_scoring_results, f, sort_keys=True, indent=4, default=str ) scores_tf = f.name + with tempfile.NamedTemporaryFile( + mode="w", delete=False, suffix=".json" + ) as f: + json.dump( + self.total_multi_trial_scoring_results, + f, + sort_keys=True, + indent=4, + default=str, + ) + multi_trial_scores_tf = f.name return ( self.job_id, self.run_time, results_tf, scores_tf, + multi_trial_scores_tf, ) diff --git a/evalbench/generators/models/__init__.py b/evalbench/generators/models/__init__.py index 7be8a7ec..fa6a1459 100644 --- a/evalbench/generators/models/__init__.py +++ b/evalbench/generators/models/__init__.py @@ -3,9 +3,16 @@ from generators.models.generator import QueryGenerator from .gemini import GeminiGenerator from .passthrough import NOOPGenerator +from .grpc_proxy import GrpcProxyModel from .claude import ClaudeGenerator from .querydata import QueryData from .query_data_api import QueryDataAPIGenerator +from .gemini_cli import GeminiCliGenerator +from .claude_code import ClaudeCodeGenerator +from .codex_cli import CodexCliGenerator +from .gcp_data_engineering_agent import DataEngineeringAgentGenerator +from .agy_cli import AgyCliGenerator +from .mcp_tools import McpToolsGenerator from util.config import load_yaml_config @@ -17,21 +24,25 @@ def get_generator(global_models, model_config_path: str, db: DB = None): config = load_yaml_config(model_config_path) # Create a new model_config - model: QueryGenerator | None = None - if config["generator"] == "gcp_vertex_gemini": - model = GeminiGenerator(config) - if config["generator"] == "gcp_vertex_claude": - model = ClaudeGenerator(config) - if config["generator"] == "noop": - model = NOOPGenerator(config) - if config["generator"] == "alloydb_ai_nl": - model = AlloyDBGenerator(db, config) - if config["generator"] == "querydata": - model = QueryData(config) - if config["generator"] == "query_data_api": - model = QueryDataAPIGenerator(config) - if not model: - raise ValueError(f"Unknown Generator {config['generator']}") + generators = { + "gcp_vertex_gemini": lambda: GeminiGenerator(config), + "gcp_vertex_claude": lambda: ClaudeGenerator(config), + "noop": lambda: NOOPGenerator(config), + "alloydb_ai_nl": lambda: AlloyDBGenerator(db, config), + "querydata": lambda: QueryData(config), + "query_data_api": lambda: QueryDataAPIGenerator(config), + "grpc_proxy": lambda: GrpcProxyModel(config), + "gemini_cli": lambda: GeminiCliGenerator(config), + "claude_code": lambda: ClaudeCodeGenerator(config), + "codex_cli": lambda: CodexCliGenerator(config), + "data_engineering_agent": lambda: DataEngineeringAgentGenerator(config), + "agy_cli": lambda: AgyCliGenerator(config), + "mcp_tools": lambda: McpToolsGenerator(config), + } + generator = config["generator"] + if generator not in generators: + raise ValueError(f"Unknown Generator {generator}") + model = generators[generator]() global_model_configs[model_config_path] = model return model diff --git a/evalbench/generators/models/agent_cli.py b/evalbench/generators/models/agent_cli.py new file mode 100644 index 00000000..0d518a70 --- /dev/null +++ b/evalbench/generators/models/agent_cli.py @@ -0,0 +1,73 @@ +from abc import abstractmethod + +from .generator import QueryGenerator +import os +import shutil + + +class AgentCliGenerator(QueryGenerator): + """Shared base for CLI-driven agent generators (gemini_cli, claude_code, + codex_cli, agy_cli). + + The evaluator treats every subclass uniformly: build a command with + ``create_command``, run it with ``safe_generate``, then read structured + data with ``parse_response`` / ``extract_tools`` / ``extract_skills``. The + reported agent version label is exposed via the ``version`` property. + Membership in this class is what ``AgentEvaluator`` keys off of, so a new + CLI generator only needs to subclass this -- no evaluator changes. + """ + + def _setup_gcloud_credentials(self, env: dict, real_home: str, fake_home: str): + """Sets up gcloud credentials in the fake home directory.""" + adc_path = env.get("GOOGLE_APPLICATION_CREDENTIALS") + if not adc_path: + adc_path = os.path.join( + real_home, + ".config", + "gcloud", + "application_default_credentials.json", + ) + if os.path.exists(adc_path): + env["GOOGLE_APPLICATION_CREDENTIALS"] = adc_path + + if adc_path and os.path.exists(adc_path): + fake_gcloud_dir = os.path.join(fake_home, ".config", "gcloud") + os.makedirs(fake_gcloud_dir, exist_ok=True) + fake_adc_path = os.path.join( + fake_gcloud_dir, "application_default_credentials.json" + ) + if os.path.abspath(adc_path) != os.path.abspath(fake_adc_path): + shutil.copy2(adc_path, fake_adc_path) + + if "CLOUDSDK_CONFIG" not in env: + env["CLOUDSDK_CONFIG"] = os.path.join( + real_home, ".config", "gcloud" + ) + + @property + @abstractmethod + def version(self) -> str: + raise NotImplementedError("Subclasses must implement this property") + + @abstractmethod + def create_command( + self, cli: str, prompt: str, env: dict = None, resume: bool = False, + session_id: str = None, cwd: str = None, + ): + raise NotImplementedError("Subclasses must implement this method") + + @abstractmethod + def safe_generate(self, cli_cmd): + raise NotImplementedError("Subclasses must implement this method") + + @abstractmethod + def parse_response(self, stdout: str) -> dict: + raise NotImplementedError("Subclasses must implement this method") + + @abstractmethod + def extract_tools(self, stdout: str) -> list: + raise NotImplementedError("Subclasses must implement this method") + + @abstractmethod + def extract_skills(self, stdout: str) -> list: + raise NotImplementedError("Subclasses must implement this method") diff --git a/evalbench/generators/models/agy_cli.py b/evalbench/generators/models/agy_cli.py new file mode 100644 index 00000000..f2b3d72c --- /dev/null +++ b/evalbench/generators/models/agy_cli.py @@ -0,0 +1,1146 @@ +from .agent_cli import AgentCliGenerator +from .tool_naming import canonicalize_agy_tool_name, parse_agy_mcp_tool_call +import subprocess +import os +import json +import logging +import re +import shutil +import sys +from util.context import rpc_id_var + +# Bare command name. agy's installer exposes no version pinning and the binary +# self-updates in the background, so there is nothing to configure. This is the +# reported agent_version label only -- the binary actually launched is the +# per-session install at self.agy_bin (see _ensure_agy_installed). +AGY_CLI = "agy" + +# Upstream one-line installer. Honors --dir (and $HOME) for the install +# location and skips the download when the binary already exists at the target. +AGY_INSTALL_URL = "https://antigravity.google/cli/install.sh" + + +class CLICommand: + def __init__(self, cli, prompt, env=None, resume=False, cwd=None): + self.cli = cli + self.prompt = prompt + self.env = env if env else {} + self.resume = resume + self.cwd = cwd + + +class AgyCliGenerator(AgentCliGenerator): + """Generator that queries via the Antigravity CLI (``agy``). + + The eval turn runs ``agy -p --dangerously-skip-permissions + --output-format stream-json [--model