feat(dea): add autopush option to model_env and update documentation

.dockerignore

@@ -2,4 +2,7 @@
 .git
 __pycache__
 *.pyc
+.pytest_cache
 results
+.jetskicli
+

.github/CODEOWNERS

@@ -5,4 +5,4 @@
 # the repo. Unless a later match takes precedence,
 # @global-owner1 and @global-owner2 will be requested for
 # review when someone opens a pull request.
-*       @ismailmehdi
+*       @ismailmehdi @helloeve @prernakakkar-google

.github/labels.yml

@@ -0,0 +1,105 @@
+# Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+- name: duplicate
+  color: ededed
+  description: ""
+
+- name: 'type: bug'
+  color: db4437
+  description: Error or flaw in code with unintended results or allowing sub-optimal usage patterns.
+
+- name: 'type: cleanup'
+  color: c5def5
+  description: An internal cleanup or hygiene concern.
+
+- name: 'type: docs'
+  color: 0000A0
+  description: Improvement to the documentation for an API.
+
+- name: 'type: feature request'
+  color: c5def5
+  description: "'Nice-to-have' improvement, new feature or different behavior or design."
+
+- name: 'type: process'
+  color: c5def5
+  description: A process-related concern. May include testing, release, or the like.
+
+- name: 'type: question'
+  color: c5def5
+  description: Request for information or clarification.
+
+- name: 'priority: p0'
+  color: b60205
+  description: Highest priority. Critical issue. P0 implies highest priority.
+
+- name: 'priority: p1'
+  color: ffa03e
+  description: Important issue which blocks shipping the next release. Will be fixed prior to next release.
+
+- name: 'priority: p2'
+  color: fef2c0
+  description: Moderately-important priority. Fix may not be included in next release.
+
+- name: 'priority: p3'
+  color: ffffc7
+  description: Desirable enhancement or fix. May not be included in next release.
+
+- name: 'do not merge'
+  color: d93f0b
+  description: Indicates a pull request not ready for merge, due to either quality or timing.
+
+- name: 'autorelease: pending'
+  color: ededed
+  description: Release please needs to do its work on this.
+
+- name: 'autorelease: triggered'
+  color: ededed
+  description: Release please has triggered a release for this.
+
+- name: 'autorelease: tagged'
+  color: ededed
+  description: Release please has completed a release for this.
+
+- name: 'status: help wanted'
+  color: 8befd7
+  description: 'Status: Unplanned work open to contributions from the community.'
+
+- name: 'status: feedback wanted'
+  color: 8befd7
+  description: 'Status: waiting for feedback from community or issue author.'
+
+- name: 'status: waiting for response'
+  color: 8befd7
+  description: 'Status: reviewer is awaiting feedback or responses from the author before proceeding.'
+
+- name: good first issue
+  color: 7057ff
+  description: Good for newcomers
+
+- name: generator
+  color: bfd4f2
+  description: Related to model/CLI generators (Jetski, Gemini CLI, Claude Code)
+
+- name: scorer
+  color: 7057ff
+  description: Related to evaluation scoring metrics
+
+- name: orchestrator
+  color: 008672
+  description: Related to orchestrators and dataset execution
+
+- name: database
+  color: e4e669
+  description: Database driver or connection pooling logic

.github/workflows/pytest.yml

@@ -0,0 +1,31 @@
+name: run-pytest
+
+on:
+  pull_request:
+    branches:
+      - main
+  push:
+    branches:
+      - main
+
+jobs:
+  test:
+    runs-on: [self-hosted, bastion]
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v4
+    - name: Setup Python
+      uses: actions/setup-python@v5
+      with:
+        python-version: "3.11"
+    - name: Install uv
+      run: |
+        curl -LsSf https://astral.sh/uv/install.sh | sh
+        echo "$HOME/.local/bin" >> $GITHUB_PATH
+    - name: Generate Proto Files
+      run: uv run make proto
+    - name: Run tests
+      run: |
+        uv run --with pytest pytest evalbench/test -vv
+      env:
+        PYTHONPATH: ${{ github.workspace }}/evalbench:${{ github.workspace }}

.gitignore

@@ -176,6 +176,7 @@ results/*
 evalbench/results/*
 db_connections/bat/db_blog.db
 datasets/bird-interact/livesqlbench-base-lite-dumps
+datasets/bird-interact/bird-interact-lite/
 datasets/bird/datasets/bird/prompts.json
 datasets/bird/db_connections/bird/california_schools.sqlite
 datasets/bird/db_connections/bird/card_games.sqlite
@@ -192,3 +193,14 @@ evalbench/db_connections/bat/db_blog.db
 
 # Autogenerated version file
 viewer/version.txt
+.jetskicli/project.json
+
+# Local testing datasets/configs (do not commit)
+datasets/bqml/
+datasets/model_configs/gemini_cli_test_model.yaml
+datasets/model_configs/gemini_2.5_pro_test_model.yaml
+datasets/model_configs/gemini_3_flash_test_model.yaml
+datasets/model_configs/codex_cli_dak_model.yaml
+
+
+

.release-please-manifest.json

@@ -1,3 +1,3 @@
 {
-  ".": "1.5.0"
+  ".": "1.13.0"
 }

AGENTS.md

@@ -52,7 +52,7 @@ graph TD
 ### Core Components
 - **Orchestrators**: Manages dataset breakdown and parallel execution stages (`OneShotOrchestrator`, `InteractOrchestrator`, `AgentOrchestrator`, `DataAgentOrchestrator`).
 - **Evaluators**: Executes the test scenario lifecycles for a given dialect or database (`Evaluator`, `InteractEvaluator`, `AgentEvaluator`, `DataAgentEvaluator`).
-- **Generators**: Acts as the driver for the tested model or CLI (`GeminiCliGenerator`, `ClaudeCodeGenerator`, `QueryData`).
+- **Generators**: Acts as the driver for the tested model or CLI (`GeminiCliGenerator`, `ClaudeCodeGenerator`, `CodexCliGenerator`, `AgyCliGenerator`, `QueryData`).
 - **Simulated Users**: Drives conversations autonomously by translating conversation plans into user messages.
 - **Scorers**: Computes correctness metrics (Exact Match, LLM-Rater, Trajectory Matcher, Behavioral Metrics).
 
@@ -81,7 +81,9 @@ evalbench/
 ├── generators/       # Tested system adapters
 │   ├── models/
 │   │   ├── claude_code.py  # Claude Code driver
-│   │   └── gemini_cli.py   # Gemini CLI driver
+│   │   ├── codex_cli.py    # Codex CLI driver
+│   │   ├── gemini_cli.py   # Gemini CLI driver
+│   │   └── agy_cli.py      # Antigravity (agy) CLI driver
 │   └── prompts/
 ├── mp/               # Multi-processing / multi-threading runners
 │   └── mprunner.py
@@ -105,9 +107,9 @@ When evaluating agentic frameworks that leverage external tools (e.g., Gemini CL
 
 | Paradigm | Supported Generators | How it Works |
 |---|---|---|
-| **MCP Servers** | `gemini_cli`, `claude_code` | Remote HTTP/SSE or local stdio-based Model Context Protocol servers |
-| **Extensions** | `gemini_cli` | GitHub-hosted plugin packages installed idempotently via CLI |
-| **Skills** | `gemini_cli` | Local or registry skill packages enabled/linked in the sandboxed environment |
+| **MCP Servers** | `gemini_cli`, `claude_code`, `codex_cli`, `agy_cli` | Remote HTTP/SSE or local stdio-based Model Context Protocol servers |
+| **Extensions** | `gemini_cli`, `claude_code`, `codex_cli` | GitHub-hosted plugin packages installed idempotently via CLI |
+| **Skills** | `gemini_cli`, `claude_code`, `codex_cli`, `agy_cli` | Skill packages installed into the sandboxed environment via each CLI's native mechanism (e.g. link/enable for gemini, `agy plugin install` for agy) |
 
 ---
 
@@ -207,7 +209,7 @@ Contains the test cases.
       "id": "list-instances-01",
       "starting_prompt": "List all Cloud SQL instances in project my-evaluation-project",
       "conversation_plan": "Ensure the agent accurately calls list_instances. Verify the output is returned correctly.",
-      "expected_trajectory": ["list_instances"],
+      "expected_trajectory": ["cloud-sql__list_instances"],
       "env": { "GOOGLE_CLOUD_PROJECT": "my-evaluation-project" },
       "max_turns": 4
     }
@@ -222,6 +224,7 @@ To upload metrics directly to Google BigQuery and create Looker Studio dashboard
 reporting:
   bigquery:
     gcp_project_id: 'your-gcp-project-id'
+    dataset_name: 'evalbench' # Optional, defaults to 'evalbench'
     dataset_location: 'US' # Optional, defaults to 'US'
 ```
 
@@ -231,10 +234,12 @@ reporting:
 
 ### Setup Python Virtual Environment
 
+Using `uv` to manage the environment:
+
 ```bash
-python3 -m venv venv
-source venv/bin/activate
-pip install -r requirements.txt
+uv venv
+source .venv/bin/activate
+uv sync
 ```
 
 ### GCP Authentication
@@ -271,6 +276,7 @@ If settings from a previous run conflict or cause issues, clear out the generato
 ```bash
 rm -rf .venv/fake_home
 rm -rf .venv/fake_home_claude
+rm -rf .venv/fake_home_agy
 ```
 
 ### Forcing Sequential Runs