diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index bd7e5d552..1d322453e 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -190,68 +190,6 @@ jobs: ARTIFACT_NAME: integration-full-${{ matrix.agent }} steps: *live-steps - managed: - name: Managed config · ${{ matrix.agent == 'claude' && 'Claude' || 'Codex' }} - if: ${{ inputs.suite != 'installation' && inputs.suite != 'smoke' && inputs.suite != 'tui' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) }} - runs-on: ubuntu-22.04 - # Kept non-blocking for now: the managed workspace (E2E_ADMIN_WORKSPACE) is repointed to - # runner-reachable ca-central, but leave these lanes for signal until the managed-config apply - # path is proven stable. Then drop this and add `managed` to the required set. - continue-on-error: true - timeout-minutes: 30 - strategy: - fail-fast: false - max-parallel: 2 - matrix: - agent: [claude, codex] - env: - DEPENDENCY: ${{ inputs.dependency }} - AGENT: ${{ matrix.agent }} - ARTIFACT_NAME: integration-managed-${{ matrix.agent }} - steps: - - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - with: - fetch-depth: 0 - - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4 - with: - node-version: 22.19.0 - - uses: astral-sh/setup-uv@08807647e7069bb48b6ef5acd8ec9567f424441b # v8.1.0 - with: - version: 0.9.8 - - uses: databricks/setup-cli@bdb89f81c11a5bd647fd55b585b7c396ec68a25a # v1.0.0 - - name: Run the managed cases against the managed e2e workspace - shell: bash - env: - # The managed workspace authenticates as a service principal; the runner mints a - # short-lived token from these standard Databricks client-credential variables. - UCODE_TEST_WORKSPACE: ${{ secrets.E2E_ADMIN_WORKSPACE }} - DATABRICKS_CLIENT_ID: ${{ secrets.E2E_ADMIN_SP_CLIENT_ID }} - DATABRICKS_CLIENT_SECRET: ${{ secrets.E2E_ADMIN_SP_CLIENT_SECRET }} - run: | - # A managed config enables both agents and `ug configure` applies it to every enabled - # agent, so both CLIs must be installed even though this lane asserts only one agent. - args=(--claude-version "$CLAUDE_VERSION" --codex-version "$CODEX_VERSION") - if [[ -n "$DEPENDENCY" ]]; then - args+=(--dependency "$DEPENDENCY") - fi - uv run --no-project --python 3.12 python scripts/run_integration.py \ - --python 3.12 --ug-version "$UG_VERSION" --entry-point "$ENTRY_POINT" \ - --default-index "$PACKAGE_INDEX" --output "$RUNNER_TEMP/ug-integration" \ - "${args[@]}" -- -m "(managed or managed_fixture) and $AGENT" - - name: Upload managed test evidence - if: ${{ !cancelled() }} - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 - with: - name: ${{ env.ARTIFACT_NAME }} - include-hidden-files: true - path: | - ${{ runner.temp }}/ug-integration/*.json - ${{ runner.temp }}/ug-integration/*.txt - ${{ runner.temp }}/ug-integration/*.xml - ${{ runner.temp }}/ug-integration/*.log - ${{ runner.temp }}/ug-integration/artifacts/ - ${{ runner.temp }}/ug-integration/wheels/ - cujs: name: All integration tests if: ${{ !cancelled() && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) }} @@ -276,7 +214,6 @@ jobs: required.append("smoke") if suite != "smoke": required.append("full") - # `managed` is intentionally omitted while it is non-blocking (see its job comment). failed = [job for job in required if results[job]["result"] != "success"] if failed: raise SystemExit("Integration jobs did not pass: " + ", ".join(failed)) diff --git a/.github/workflows/managed-integration.yml b/.github/workflows/managed-integration.yml new file mode 100644 index 000000000..c3cf80101 --- /dev/null +++ b/.github/workflows/managed-integration.yml @@ -0,0 +1,92 @@ +name: Managed integration (signal) + +on: + pull_request: + workflow_dispatch: + +permissions: + contents: read + +# This workflow is intentionally independent from ci.yml. A hosted-runner shutdown here must not +# cancel the reusable Integration workflow or the repository's required `e2e` check. +concurrency: + group: managed-integration-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +env: + UG_VERSION: checkout + ENTRY_POINT: ug + CLAUDE_VERSION: 2.1.268 + CODEX_VERSION: 0.154.0 + PACKAGE_INDEX: https://pypi.org/simple + +jobs: + managed: + name: Managed config · ${{ matrix.label }} + if: ${{ github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository }} + runs-on: ubuntu-22.04 + timeout-minutes: 25 + strategy: + fail-fast: false + max-parallel: 2 + matrix: + include: + - label: Claude + marker: (managed or managed_fixture) and claude + keyword: '' + artifact: integration-managed-claude + - label: Codex · workspace + marker: managed and codex + keyword: '' + artifact: integration-managed-codex-workspace + - label: Codex · discovery policy + marker: managed_fixture and codex + keyword: ug_codex_managed_model_discovery + artifact: integration-managed-codex-discovery + - label: Codex · remaining fixtures + marker: managed_fixture and codex + keyword: not ug_codex_managed_model_discovery + artifact: integration-managed-codex-fixtures + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + fetch-depth: 0 + - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4 + with: + node-version: 22.19.0 + - uses: astral-sh/setup-uv@08807647e7069bb48b6ef5acd8ec9567f424441b # v8.1.0 + with: + version: 0.9.8 + - uses: databricks/setup-cli@bdb89f81c11a5bd647fd55b585b7c396ec68a25a # v1.0.0 + - name: Run the managed shard against the managed e2e workspace + shell: bash + env: + UCODE_TEST_WORKSPACE: ${{ secrets.E2E_ADMIN_WORKSPACE }} + DATABRICKS_CLIENT_ID: ${{ secrets.E2E_ADMIN_SP_CLIENT_ID }} + DATABRICKS_CLIENT_SECRET: ${{ secrets.E2E_ADMIN_SP_CLIENT_SECRET }} + TEST_MARKER: ${{ matrix.marker }} + TEST_KEYWORD: ${{ matrix.keyword }} + run: | + # A managed config applies to every enabled agent, so both CLIs are required by each shard. + pytest_args=(-m "$TEST_MARKER") + if [[ -n "$TEST_KEYWORD" ]]; then + pytest_args+=(-k "$TEST_KEYWORD") + fi + uv run --no-project --python 3.12 python scripts/run_integration.py \ + --python 3.12 --ug-version "$UG_VERSION" --entry-point "$ENTRY_POINT" \ + --claude-version "$CLAUDE_VERSION" --codex-version "$CODEX_VERSION" \ + --default-index "$PACKAGE_INDEX" --output "$RUNNER_TEMP/ug-integration" \ + -- "${pytest_args[@]}" + - name: Upload managed test evidence + if: ${{ !cancelled() }} + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: ${{ matrix.artifact }} + include-hidden-files: true + path: | + ${{ runner.temp }}/ug-integration/*.json + ${{ runner.temp }}/ug-integration/*.txt + ${{ runner.temp }}/ug-integration/*.xml + ${{ runner.temp }}/ug-integration/*.log + ${{ runner.temp }}/ug-integration/artifacts/ + ${{ runner.temp }}/ug-integration/wheels/ diff --git a/tests/integration/README.md b/tests/integration/README.md index 8d4fc73e2..50fde8fd8 100644 --- a/tests/integration/README.md +++ b/tests/integration/README.md @@ -272,17 +272,16 @@ No test retries or assertion changes compensate for capacity failures. Both matrices use `fail-fast: false` and upload uniquely named evidence even when the other agent fails. The **All integration tests** check requires installation, workspace validation, smoke, and -both full lanes to pass. The **Managed config** lanes run for signal but are temporarily -non-blocking (`continue-on-error`): the managed workspace is now runner-reachable, but the lanes -stay non-blocking until the managed-config apply path is proven stable. They neither fail the -workflow nor gate merges until then. The -existing required `e2e` context also waits for the complete integration workflow, so integration -cannot still be running when that gate passes. Full coverage on PRs needs no label or opt-in. +both full lanes to pass. Managed-config coverage runs in the independent +`managed-integration.yml` signal workflow. A runner shutdown there cannot cancel this required +workflow or gate merges. Codex is divided into published-workspace, discovery-policy, and +remaining-fixture shards so each runner has a shorter exposure window and a stalled scenario is +immediately identifiable. Full required coverage on PRs needs no label or opt-in. ### Managed-workspace journeys -`test_ug_configure_managed.py` (marker `managed`, not `live`) runs in its own per-agent -**Managed config** jobs against a second workspace that publishes an admin CodingAgentConfig, +`test_ug_configure_managed.py` (marker `managed`, not `live`) runs in the independent +**Managed integration (signal)** workflow against a second workspace that publishes an admin CodingAgentConfig, which the shared `live` workspace deliberately does not. `ug configure` applies the admin config with no agent selector, and each agent's generated config exposes exactly the admin's static `model_services` (Claude's `availableModels`/`modelPicker`, Codex's model catalog). @@ -336,7 +335,7 @@ policies prevented Codex's bubblewrap tool from reading even the test file in th first run. The agent sandbox is not disabled or bypassed. The workflow consumes the stored bearer; it does not mint or refresh credentials. -For a manual run, use **Actions → Integration → Run workflow**, select the branch, +For a manual required-suite run, use **Actions → Integration → Run workflow**, select the branch, and choose `full` (default), `smoke`, `tui`, or `installation`. `live` remains an alias for `full`. Manual subsets are explicit: `smoke` runs just the six smoke cases; `tui` adds `and tui` to each agent lane's marker and runs all six TUI cases. Installation @@ -356,6 +355,12 @@ credentials or a workspace mismatch fail the workspace job. Expired or invalid credentials fail the actual workspace calls. Those failures do not count as live test passes. +Run **Managed integration (signal)** separately to reproduce the managed shards. Its four jobs +use distinct artifact names. Codex's discovery-policy shard owns +`test_ug_codex_managed_model_discovery.py`; the remaining-fixture shard selects the other Codex +`managed_fixture` cases; the workspace shard selects the un-stubbed `managed` cases. These +selectors are disjoint and together preserve the previous Codex managed coverage. + ## Reproduce and debug a CI failure locally Use the same runner and the failing job's artifacts. A new developer machine