From e9fa08b66ef02a1628fcb67e2c9430090ee42897 Mon Sep 17 00:00:00 2001 From: Joris Baum Date: Mon, 13 Jul 2026 16:04:39 +0200 Subject: [PATCH 1/6] Bootstrap project --- .claude-plugin/marketplace.json | 16 + .github/workflows/ci.yml | 32 ++ .gitignore | 2 + CODE_OF_CONDUCT.md | 13 + CONTRIBUTING.md | 56 +++ Makefile | 58 +++ README.md | 70 +++ .../cf-kind-verify/.claude-plugin/plugin.json | 19 + skills/cf-kind-verify/README.md | 56 +++ skills/cf-kind-verify/SKILL.md | 115 +++++ .../docs/claude-code/permissions.md | 51 +++ .../docs/claude-code/settings.json | 36 ++ skills/cf-kind-verify/evals/README.md | 100 +++++ skills/cf-kind-verify/evals/evals.json | 73 ++++ skills/cf-kind-verify/evals/run-evals.sh | 411 ++++++++++++++++++ .../references/test-conventions.md | 110 +++++ .../scripts/_discover_target.py | 72 +++ .../scripts/_status_containers.py | 10 + .../scripts/_status_selector.py | 17 + skills/cf-kind-verify/scripts/cats.sh | 83 ++++ .../cf-kind-verify/scripts/discover-target.sh | 35 ++ skills/cf-kind-verify/scripts/lifecycle.sh | 102 +++++ skills/cf-kind-verify/scripts/prereqs.sh | 57 +++ skills/cf-kind-verify/scripts/redeploy.sh | 117 +++++ skills/cf-kind-verify/scripts/render-check.sh | 83 ++++ .../cf-kind-verify/scripts/verify-status.sh | 50 +++ 26 files changed, 1844 insertions(+) create mode 100644 .claude-plugin/marketplace.json create mode 100644 .github/workflows/ci.yml create mode 100644 .gitignore create mode 100644 CODE_OF_CONDUCT.md create mode 100644 CONTRIBUTING.md create mode 100644 Makefile create mode 100644 README.md create mode 100644 skills/cf-kind-verify/.claude-plugin/plugin.json create mode 100644 skills/cf-kind-verify/README.md create mode 100644 skills/cf-kind-verify/SKILL.md create mode 100644 skills/cf-kind-verify/docs/claude-code/permissions.md create mode 100644 skills/cf-kind-verify/docs/claude-code/settings.json create mode 100644 skills/cf-kind-verify/evals/README.md create mode 100644 skills/cf-kind-verify/evals/evals.json create mode 100644 skills/cf-kind-verify/evals/run-evals.sh create mode 100644 skills/cf-kind-verify/references/test-conventions.md create mode 100644 skills/cf-kind-verify/scripts/_discover_target.py create mode 100644 skills/cf-kind-verify/scripts/_status_containers.py create mode 100644 skills/cf-kind-verify/scripts/_status_selector.py create mode 100755 skills/cf-kind-verify/scripts/cats.sh create mode 100755 skills/cf-kind-verify/scripts/discover-target.sh create mode 100755 skills/cf-kind-verify/scripts/lifecycle.sh create mode 100755 skills/cf-kind-verify/scripts/prereqs.sh create mode 100755 skills/cf-kind-verify/scripts/redeploy.sh create mode 100755 skills/cf-kind-verify/scripts/render-check.sh create mode 100755 skills/cf-kind-verify/scripts/verify-status.sh diff --git a/.claude-plugin/marketplace.json b/.claude-plugin/marketplace.json new file mode 100644 index 0000000..1204959 --- /dev/null +++ b/.claude-plugin/marketplace.json @@ -0,0 +1,16 @@ +{ + "$schema": "https://json.schemastore.org/claude-code-marketplace.json", + "name": "cloudfoundry-skills", + "description": "Cloud Foundry skills for AI coding agents.", + "owner": { + "name": "Cloud Foundry Community", + "url": "https://www.cloudfoundry.org/" + }, + "plugins": [ + { + "name": "cf-kind-verify", + "source": "./skills/cf-kind-verify", + "description": "Verify Cloud Foundry component changes on a local cf-on-kind cluster." + } + ] +} diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..17c1c1c --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,32 @@ +name: CI + +on: + push: + branches: [main] + pull_request: + branches: + - main + - v[0-9]* + +jobs: + check: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 + + - name: Install shellcheck + run: sudo apt-get update && sudo apt-get install -y shellcheck + + - name: Set up Node.js + uses: actions/setup-node@v7 + with: + node-version: "22" + + - name: Install Claude Code CLI + run: npm install -g @anthropic-ai/claude-code + + - name: Install uv + uses: astral-sh/setup-uv@v10.1 + + - name: Run checks + run: make check diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..ff29b8c --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +# Eval run outputs — regenerated, not part of the published skill +cf-kind-verify-workspace/ diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md new file mode 100644 index 0000000..fa591b2 --- /dev/null +++ b/CODE_OF_CONDUCT.md @@ -0,0 +1,13 @@ +# Code of Conduct + +The Cloud Foundry Agentic Runtime Working Group, like all Cloud Foundry community spaces, +operates under the **Cloud Foundry Foundation Code of Conduct**. + +Please read it here: https://www.cloudfoundry.org/code-of-conduct/ + +By participating in this repository — through issues, pull requests, reviews, or any other +interaction — you agree to abide by its terms. + +To report a concern, follow the reporting instructions in the linked Code of Conduct, or +reach a working-group lead in [#wg-ai](https://cloudfoundry.slack.com/archives/C0B214KJ1HA) +on the Cloud Foundry Slack. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..2ef3883 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,56 @@ +## Contributing + +Contributions are welcome. This is a collection of portable Cloud Foundry Agent +Skills. + +For Cloud Foundry Foundation repositories, contributors must sign the +[Contributor License Agreement](https://corporate.v1.easycla.lfx.linuxfoundation.org/). +EasyCLA prompts you when you open your first pull request. + +### Dev setup + +```bash +git clone https://github.com/cloudfoundry/skills +cd skills +mkdir -p ~/.config/opencode/skills +ln -s "$(pwd)/skills/" ~/.config/opencode/skills/ +``` + +This symlink is the recommended edit-test workflow: changes to a skill are +available in new OpenCode sessions without reinstalling it. `gh skill install +--from-local` copies files rather than creating a symlink, so use it only to +test the installation flow. Remove a copied installation before creating the +symlink: + +```bash +rm -rf ~/.config/opencode/skills/ +``` + +### Checks + +Run before opening a PR — CI runs the same target: + +```bash +make check # Claude marketplace + shellcheck + JSON validation +``` + +### Making changes + +- Each skill lives in `skills//`; keep scripts and references with its `SKILL.md`. +- Keep `SKILL.md` portable. Put harness-specific packaging or configuration under the skill's `docs//` directory. + +### Commits and PRs + +- Fork the repository, create a branch, and open a pull request from that branch. +- Small, focused PRs. +### Getting help + +- Report bugs or propose skills with an issue in this repository. +- Ask usage questions in [#wg-ai](https://cloudfoundry.slack.com/archives/C0B214KJ1HA) channel within Cloud Foundry Slack. +- Discuss broader agent-workload design with the [Agentic Runtime working group](https://github.com/cloudfoundry/community/blob/main/toc/working-groups/WORKING-GROUPS.md#agentic-runtime). + +### Code of conduct + +This project follows the [Cloud Foundry Code of Conduct](./CODE_OF_CONDUCT.md). + +Licensed under Apache-2.0. diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..fb501d2 --- /dev/null +++ b/Makefile @@ -0,0 +1,58 @@ +# Cloud Foundry Skills — CI / local checks +# +# Run `make` (or `make check`) to run everything CI runs. +# Individual targets are useful while iterating locally. + +SHELL := bash +.SHELLFLAGS := -eu -o pipefail -c +.DEFAULT_GOAL := check + +# All shipped scripts and JSON files in the working tree, including untracked +# files introduced by a change under review. +SCRIPTS := $(shell git ls-files -co --exclude-standard -- '*.sh' | while IFS= read -r f; do [ -f "$$f" ] && printf '%s ' "$$f"; done) +JSON := $(shell git ls-files -co --exclude-standard -- '*.json' | while IFS= read -r f; do [ -f "$$f" ] && printf '%s ' "$$f"; done) +PY := $(shell git ls-files -co --exclude-standard -- '*.py' | while IFS= read -r f; do [ -f "$$f" ] && printf '%s ' "$$f"; done) + +.PHONY: check validate shellcheck json ruff ty help + +## check: run all CI checks (default) +check: validate shellcheck json ruff ty + @echo "All checks passed." + +## validate: validate the Claude marketplace and each skill plugin (strict — warnings fail) +validate: + @echo ">> claude plugin validate --strict marketplace" + claude plugin validate --strict . + @echo ">> claude plugin validate --strict skills/cf-kind-verify" + claude plugin validate --strict skills/cf-kind-verify + +## shellcheck: lint every tracked shell script (all severities must pass) +shellcheck: + @echo ">> shellcheck ($(words $(SCRIPTS)) scripts)" + @if command -v shellcheck >/dev/null 2>&1; then \ + shellcheck $(SCRIPTS); \ + else \ + echo "shellcheck not installed" >&2; \ + exit 1; \ + fi + +## json: check every tracked JSON file parses +json: + @echo ">> jq parse ($(words $(JSON)) files)" + @for f in $(JSON); do \ + jq empty "$$f" || { echo "invalid JSON: $$f" >&2; exit 1; }; \ + done + +## ruff: lint Python helper scripts +ruff: + @echo ">> ruff ($(words $(PY)) files)" + @if [ -n "$(strip $(PY))" ]; then uvx ruff@0.16.7 check $(PY); else echo "(no python files)"; fi + +## ty: type-check Python helper scripts +ty: + @echo ">> ty ($(words $(PY)) files)" + @if [ -n "$(strip $(PY))" ]; then uvx ty@0.0.80 check $(PY); else echo "(no python files)"; fi + +## help: list targets +help: + @grep -E '^## ' $(MAKEFILE_LIST) | sed 's/^## / /' diff --git a/README.md b/README.md new file mode 100644 index 0000000..39cc5c5 --- /dev/null +++ b/README.md @@ -0,0 +1,70 @@ +# Cloud Foundry Skills + +A collection of portable [Agent Skills](https://agentskills.io/) for AI coding +agents that work with Cloud Foundry. + +## Available skills + +| Skill | Description | +| --- | --- | +| [`cf-kind-verify`](./skills/cf-kind-verify/) | Verify CF component changes on a local cf-on-kind cluster. | + +## Install + +### GitHub CLI + +Install with [GitHub CLI v2.90.0 or later](https://github.blog/changelog/2026-04-16-manage-agent-skills-with-github-cli/). +Use `--scope project` to install it in the current repository. + +```bash +gh skill install cloudfoundry/skills \ + --agent opencode --scope user +``` + +Supported `--agent` values include `opencode`, `claude-code`, `github-copilot`, +`cursor`, `codex`, `gemini-cli`, and `cline`. Compatible harnesses use the +portable `.agents/skills/` project path. + +### Agent Skills CLI + +Install with [Open Agent Skills](https://www.skills.sh/) installer: + +```bash +npx skills add cloudfoundry/skills --skill +``` + +### Claude Code marketplace + +The repository also exposes a Claude Code marketplace adapter: + +``` +/plugin marketplace add cloudfoundry/skills +/plugin install @cloudfoundry-skills +``` + +### Manual fallback + +You can also manually clone to install them in a harness e.g. due to +lacking `gh skill` support. + +Copy or symlink a `skills/` directory into a +directory the harness discovers. Common user-level locations are: + +| Harness | Location | +| --- | --- | +| OpenCode | `~/.agents/skills/` or `~/.config/opencode/skills/` | +| Claude Code | `~/.claude/skills/` | +| GitHub Copilot | `~/.agents/skills/` or `~/.copilot/skills/` | +| Cursor | `~/.agents/skills/` or `~/.cursor/skills/` | + +For example, OpenCode reads the shared Agent Skills location: + +```bash +git clone https://github.com/cloudfoundry/skills +ln -s "$(pwd)/skills/skills/" \ + ~/.agents/skills/ +``` + +## Contributing + +See [`CONTRIBUTING.md`](./CONTRIBUTING.md). diff --git a/skills/cf-kind-verify/.claude-plugin/plugin.json b/skills/cf-kind-verify/.claude-plugin/plugin.json new file mode 100644 index 0000000..ffa68f1 --- /dev/null +++ b/skills/cf-kind-verify/.claude-plugin/plugin.json @@ -0,0 +1,19 @@ +{ + "name": "cf-kind-verify", + "displayName": "cf-kind-verify", + "description": "Verify Cloud Foundry component changes on a local cf-on-kind cluster.", + "version": "0.1.0", + "author": { + "name": "Cloud Foundry Community" + }, + "repository": "https://github.com/cloudfoundry/skills", + "license": "Apache-2.0", + "keywords": [ + "cloud-foundry", + "cf-on-kind", + "kind", + "kubernetes", + "verification", + "testing" + ] +} diff --git a/skills/cf-kind-verify/README.md b/skills/cf-kind-verify/README.md new file mode 100644 index 0000000..0c0e50c --- /dev/null +++ b/skills/cf-kind-verify/README.md @@ -0,0 +1,56 @@ +# cf-kind-verify + +Verify a Cloud Foundry component change on a local +[cf-on-kind](https://github.com/cloudfoundry/kind-deployment) cluster: +build, load, redeploy, roll out, and test. + +- **Build:** create the changed component image with `docker buildx bake`. +- **Load:** import that local image into the kind cluster. +- **Redeploy:** find the running Deployment and container, then point it at the new image. +- **Rollout:** wait for Kubernetes to replace the old Pods and report the Deployment ready. +- **Test:** run the component's relevant test command against the updated cluster. + +## What it does + +Given a change in a CF release repo, the skill follows this sequence: + +1. Check prerequisites with `prereqs.sh`. +2. **Agent:** offer and, when approved, manage cluster lifecycle with `lifecycle.sh`. +3. For Helm-chart changes, render the chart before mutating the cluster with `render-check.sh`. +4. **Agent:** select and explain the changed image, then build, load, redeploy, and roll out with `redeploy.sh` (which invokes `discover-target.sh` to find the Deployment, container, and namespace). +5. Verify containers and report the rollout with `verify-status.sh`. +6. **Agent:** select, explain, and run focused CATS with `cats.sh` when appropriate. + +## Claude Code permissions + +The skill works without additional configuration, though Claude Code prompts for +the docker/kind/kubectl steps. To reduce those prompts, merge +[`docs/claude-code/settings.json`](./docs/claude-code/settings.json) +into your project `.claude/settings.json`, then restart the session. + +See [`docs/claude-code/permissions.md`](./docs/claude-code/permissions.md) +for what to edit, how the config works, and the security trade-offs of running +these commands unsandboxed. + +## Scripts + +| Script | Purpose | +| --- | --- | +| `prereqs.sh` | Tools + `cfk8s` cluster + kube-context ready? | +| `lifecycle.sh` | `up` / `down` / `status` | +| `redeploy.sh` | Build → load → redeploy (discover + set image) → rollout → status | +| `discover-target.sh` | Map an image to its `(deployment, container, namespace)` | +| `cats.sh` | Render the kind-deployment CATS configuration and run focused CATS tests | +| `verify-status.sh` | Per-container ready/restarts/image for a deployment | +| `render-check.sh` | `helm template … \| grep` render validation | + +## Scope + +**v1:** CF component verification + cluster up/down. **Deferred (v2):** first-class +`cf push` app testing, discovery cache, prereq auto-install. App-level CATS notes: +[`references/test-conventions.md`](./references/test-conventions.md). + +## References + +- [cf-on-kind local development guide](https://github.com/cloudfoundry/kind-deployment/blob/main/docs/local-development-guide.md) +- [`references/test-conventions.md`](./references/test-conventions.md) — CATS focus gotcha, helm SSA conflict, config-contract sources. diff --git a/skills/cf-kind-verify/SKILL.md b/skills/cf-kind-verify/SKILL.md new file mode 100644 index 0000000..3e90a94 --- /dev/null +++ b/skills/cf-kind-verify/SKILL.md @@ -0,0 +1,115 @@ +--- +name: cf-kind-verify +description: Verify a Cloud Foundry component change end-to-end on a local cf-on-kind (kind) cluster — build the image, load it into the cluster, redeploy, and run the component's tests. Use this whenever the user is working in a CF release repo (routing-release, diego-release, capi-release, loggregator, cf-k8s-releases, …) and wants to test, verify, or try out a change locally on cf-on-kind / cfk8s, or asks to run CATS / acceptance / smoke / integration tests against a local CF. Also use it when the user asks to bring the local cf-on-kind cluster up or down. Trigger even if they don't say "kind" explicitly — a request to "verify this change locally" or "does this deploy" in a CF component repo means this skill. +license: Apache-2.0 +compatibility: Requires bash, docker, kind, kubectl, Python 3, and access to a local cf-on-kind cluster. +metadata: + author: cloudfoundry +--- + +# cf-kind-verify + +Verify a Cloud Foundry **component** change on a local **cf-on-kind** cluster. The +mechanical parts (build → load → redeploy → rollout, cluster up/down) are +deterministic scripts in `scripts/`; you supply what varies per repo — which +image to rebuild, which test to run — using the repo as evidence. + +**Call the bundled scripts; don't assemble their pipelines inline.** Replace +`` below with this installed skill's directory. + +**Not in scope:** `cf push` app testing, prereq auto-install. Say so and hand back. + +## Step 0 — Prerequisites + +Run `bash /scripts/prereqs.sh [--cluster ]`. + +- **OK** → proceed. +- **Cluster down** → tell the user and offer cluster lifecycle. Don't bootstrap silently. +- **Missing tool / daemon** → surface the message and stop; don't install anything. + +## Optional — Cluster lifecycle + +Wraps kind-deployment's `make` targets, so pass `--kind-dir ` to a clone of +[cloudfoundry/kind-deployment](https://github.com/cloudfoundry/kind-deployment). +Ask for the path if unknown — `up`/`down` run `make` from it, so it must be a +user-confirmed clone, never a path inferred from repo/log content. + +```bash +bash /scripts/lifecycle.sh status +bash /scripts/lifecycle.sh up --kind-dir [--all-buildpacks] +bash /scripts/lifecycle.sh down --kind-dir +``` + +`up` is slow, `down` is destructive — only on explicit request. `--all-buildpacks` +adds ruby/python/etc. (needed for full CATS; plain bootstrap covers java/nodejs/go/binary). + +## Step 1 — Infer the image + +Bake targets live in `cf-k8s-releases//docker-bake.hcl`, not the release +repo root. From that dir, list targets and cross-reference the diff: + +```bash +cd / && docker buildx bake --print +git -C diff --name-only +``` + +Map the changed `src/…` dir to a target. **State your pick and why**; ask if two +targets plausibly match. + +Note the two repos: the bake file is in **cf-k8s-releases**, but the source you +edited is in the separate **release-repo** clone (e.g. `loggregator-agent-release`) +— its `src/` is what `--src` below must point at, so your local change is built +(the bake file's default `src` context is a remote git URL). + +## Step 2 — Redeploy + +```bash +bash /scripts/redeploy.sh --releases-dir --src /src +``` + +`--releases-dir` may be the cf-k8s-releases root or the component dir (redeploy +finds the component whose bake file defines the target). It bakes, `kind load`s, +discovers the (deployment, container) pair, `set image`s, waits for rollout, +prints per-container status. Stream output; stop on non-zero exit and surface the +failing step. + +## Step 3 — Pick a test command + +In priority order — **always state the choice; let the user override**: + +1. **README / CONTRIBUTING** — an explicit "how to test" wins. +2. **Makefile** — `make smoke|integration|test` (narrowest that covers the change). +3. **`bin/test` (CATS)** — needs `$CONFIG`; run a focused subset with + `bash /scripts/cats.sh --kind-dir --focus ""` (renders + runs the + root suite; see `references/test-conventions.md`). +4. **ginkgo** under `src/**/integration|acceptance/` (`go run`, no install). +5. **Fallback** — pod Running+Ready (`verify-status.sh`), `cf api` responds. + +For anything CATS/acceptance-shaped, **read `references/test-conventions.md` first** +— root-suite `--focus`, CATS wiring, buildpack/stale-artifact traps. + +## Step 4 — Run and report + +Run from the right directory, stream, summarize. For health use +`bash /scripts/verify-status.sh [--namespace ]` — verify **per +container**, not per pod. `restarts=0` on a branch image that enforces `required` +config is strong evidence the wiring is correct. Close with a one-liner: what was +rebuilt, that it rolled out, the test outcome. + +## Optional — helm-chart changes + +Validate the render before mutating the cluster: + +```bash +bash /scripts/render-check.sh //helm \ + --kind-dir --grep 'NEW_ENV|image:' +``` + +then `helm upgrade` the local chart dir. `set image` (redeploy) and `helm upgrade` +conflict on the same deployment — pick one; prefer helm if a chart change is +involved. Local `:latest` is ephemeral (reverts on `make down`/`up`). Details and +the SSA-conflict fix: `references/test-conventions.md`. + +## Portability + +Permission behavior depends on the selected harness. diff --git a/skills/cf-kind-verify/docs/claude-code/permissions.md b/skills/cf-kind-verify/docs/claude-code/permissions.md new file mode 100644 index 0000000..e99d6f3 --- /dev/null +++ b/skills/cf-kind-verify/docs/claude-code/permissions.md @@ -0,0 +1,51 @@ +# Permission setup + +This Claude Code adapter is optional. The skill works without it, but prompts for +the docker/kind/kubectl steps. To silence them, merge +[`settings.json`](./settings.json) into your **project** `.claude/settings.json` +(not `~/.claude/`), replacing: + +- `/Users/YOU` → your home directory +- the script paths → wherever the skill lives (the plugin cache under + `~/.claude/plugins/cache/…`, or your clone) + +`sandbox.*` changes need a session restart. + +> Not bundled with the plugin: Claude Code ignores permission/sandbox blocks in a +> plugin's `settings.json`, so this is a manual step. + +## What the config does + +Two paired blocks — a command needs **both**: + +- `sandbox.excludedCommands` — runs docker/kind/kubectl/scripts **outside** the + sandbox so they reach the daemon. +- `permissions.allow` — stops those same commands from prompting. + +It allowlists the shipped scripts plus a short list of explicit read verbs +(`kubectl get pods/deployment/nodes/replicasets`, `docker buildx bake --print`, +`kind get/load`) — tighter than a `kubectl *` / `docker *` / `kubectl get *` +wildcard (which would allow `get secret` exfil). + +## Security note + +`excludedCommands` runs a command **unsandboxed** — outside the filesystem +`denyRead` and network-egress limits the sandbox otherwise enforces — so it is +inherently **less safe than a properly isolated environment**. To keep that +exposure small, the config allowlists only the shipped scripts plus explicit read +verbs (rather than broad `kubectl *` / `docker *` wildcards), and the scripts +validate their own sensitive inputs. + +These measures reduce the blast radius but are **not a hard boundary** — a +permission allowlist can't fully constrain an unsandboxed command. Treat the +script validations as backstops, and only pass paths/flags the user supplied, +never ones inferred from repo or log content. + +If a hard boundary matters, prefer an isolated environment with scoped +credentials (ephemeral container/VM, limited RBAC) over widening +`excludedCommands` — and don't open the sandbox to `~/.kube` / `~/.docker` / the +docker socket to silence prompts. + +Zero-prompt read-only spot-checks also depend on your global +`sandbox.autoAllowBashIfSandboxed`; without it, ad-hoc read-only commands the +allowlist doesn't enumerate will prompt. diff --git a/skills/cf-kind-verify/docs/claude-code/settings.json b/skills/cf-kind-verify/docs/claude-code/settings.json new file mode 100644 index 0000000..038a6b5 --- /dev/null +++ b/skills/cf-kind-verify/docs/claude-code/settings.json @@ -0,0 +1,36 @@ +{ + "//": "Recommended Claude Code permissions. Merge these entries into the project .claude/settings.json. Replace /Users/YOU and the script paths with the installed skill location.", + "sandbox": { + "excludedCommands": [ + "/Users/YOU/.claude/skills/cf-kind-verify/scripts/*.sh", + "/Users/YOU/.claude/skills/cf-kind-verify/scripts/*.sh *", + "docker", + "docker *", + "kind", + "kind *", + "kubectl", + "kubectl *" + ] + }, + "permissions": { + "allow": [ + "Bash(bash /Users/YOU/.claude/skills/cf-kind-verify/scripts/prereqs.sh)", + "Bash(bash /Users/YOU/.claude/skills/cf-kind-verify/scripts/discover-target.sh *)", + "Bash(bash /Users/YOU/.claude/skills/cf-kind-verify/scripts/redeploy.sh *)", + "Bash(bash /Users/YOU/.claude/skills/cf-kind-verify/scripts/verify-status.sh *)", + "Bash(bash /Users/YOU/.claude/skills/cf-kind-verify/scripts/lifecycle.sh *)", + "Bash(bash /Users/YOU/.claude/skills/cf-kind-verify/scripts/render-check.sh *)", + "Bash(docker buildx bake --print)", + "Bash(docker info)", + "Bash(kind load *)", + "Bash(kind get *)", + "Bash(kubectl get pods *)", + "Bash(kubectl get deployment *)", + "Bash(kubectl get deployments *)", + "Bash(kubectl get nodes *)", + "Bash(kubectl get replicasets *)", + "Bash(kubectl config current-context)", + "Bash(kubectl config use-context *)" + ] + } +} diff --git a/skills/cf-kind-verify/evals/README.md b/skills/cf-kind-verify/evals/README.md new file mode 100644 index 0000000..9d94b22 --- /dev/null +++ b/skills/cf-kind-verify/evals/README.md @@ -0,0 +1,100 @@ +# Running the cf-kind-verify evals + +`evals.json` is the eval set — 5 cases (3 should-trigger, 2 near-misses), each with a +`prompt` and an `expected_output` describing the **flow** we expect (which script is +called, `redeploy.sh --src`, ROOT CATS with `--focus`, etc.), not just whether the skill +triggers. + +The runner writes benchmark artifacts to a temporary directory and prints its path. +Set `KEEP_SCRATCH=1` to retain them after the run. + +## How to rerun + +There are two paths, depending on what you're doing: + +### A. Version-vs-version regression check (after editing the skill) — `run-evals.sh` + +When you've edited the skill and want to know **did this get better or worse?**, run +the committed wrapper: + +```bash +# Compare the working tree against the last commit, 2 runs each, then open the viewer: +bash ./skills/cf-kind-verify/evals/run-evals.sh -- --verbose --runs-per-query 2 --timeout 90 --against HEAD + +# Against a specific baseline (tag / SHA / branch): +bash ./skills/cf-kind-verify/evals/run-evals.sh -- --against v1.0 --runs-per-query 3 +``` + +It benchmarks the **working-tree** skill (`new_skill`, includes uncommitted edits) +against a **baseline git ref** (`old_skill`, default `HEAD`), grades both, and prints a +`delta = new_skill − old_skill` table plus the HTML viewer. + +`run-evals.sh` is a thin wrapper — it **reuses** skill-creator's `aggregate_benchmark.py`, +`generate_review.py`, and `agents/grader.md` verbatim. It only adds what skill-creator's +interactive flow doesn't: an `--against ` baseline, a fix for the +installed-skill contamination trap (see gotcha 3 below), and the run-dir layout the +aggregator expects. Key flags: `--against `, `--runs-per-query `, +`--timeout `, `--only `, `--jobs `, `--model `, `--no-viewer`, +`--verbose`. + +**Must run unsandboxed** (ProcessPoolExecutor — see gotcha 1). Full runs take +~20–30 min, so background them. + +### B. Skill-vs-no-skill / interactive exploration — `skill-creator` + +**Invoke the `skill-creator` skill** and ask it to benchmark this skill: + +> Benchmark the skill at `~/.claude/skills/cf-kind-verify` against its `evals/evals.json`. + +Skill-creator *is* the harness. It drives the documented benchmark workflow — spawns a +with-skill and a baseline subagent per eval, grades the outputs against each eval's +expectations, aggregates the stats, and opens an HTML review. Use this for a fresh +skill-vs-baseline benchmark or exploratory iteration. Don't wrap it in another testing +skill; skill-creator's own SKILL.md says so. + +## Three gotchas (learned the hard way) + +1. **Run unsandboxed.** The Python harness uses `ProcessPoolExecutor`, which the command + sandbox blocks — you'll see `os.sysconf("SC_SEM_NSEMS_MAX")` → `PermissionError: + Operation not permitted`. Run the benchmark/aggregate/viewer commands outside the + sandbox (see `/sandbox`). `run-evals.sh` runs the aggregate + viewer steps directly, + so invoke the whole script unsandboxed. + +2. **Do NOT use skill-creator's `scripts/run_eval.py` / `run_loop.py` for this skill.** + Those are a *triggering probe*: they register a throwaway slash-command holding only the + description and check whether `claude -p` invokes that probe by name. But `cf-kind-verify` + is installed as a **user-level skill** (`~/.claude/skills/`), so `claude -p` sees the + **real** skill from any cwd and invokes *it* instead of the probe — every case scores 0 + triggers even when triggering is correct. The probe assumes the skill isn't installed + yet; ours is. Use the benchmark workflow above instead. + +3. **Contamination: the installed skill leaks into any `claude -p`.** Because + `~/.claude/skills/cf-kind-verify` is a symlink to this repo, a plain baseline `claude -p` + still sees the ambient skill from any cwd — a skill-creator `cp -r` snapshot doesn't + isolate it. `run-evals.sh` closes this by running both configs with + `--disable-slash-commands` from an isolated `$TMPDIR` cwd, feeding the materialized + `SKILL.md` via `--append-system-prompt`, parsing the answer from the JSON envelope (so no + output path is ever handed to the model to follow), and instructing the model never to + read a `.claude` dir. Both sides are treated symmetrically. + +## Regenerate stats / report from existing runs + +If you already have graded run dirs and just want to re-aggregate or re-open the viewer, +run these from the skill-creator dir (resolve `$SC` first), unsandboxed: + +```bash +SC=$(ls -d ~/.claude/plugins/marketplaces/*/plugins/skill-creator/skills/skill-creator | head -1) +WS= + +cd "$SC" +python3 -m scripts.aggregate_benchmark "$WS" --skill-name cf-kind-verify +python3 eval-viewer/generate_review.py "$WS" --skill-name cf-kind-verify --benchmark "$WS/benchmark.json" +``` + +## Note: expectations are populated + +Each case in `evals.json` now carries a populated `expectations` array (bare strings — +the concrete flow-reasoning assertions each eval checks). Both `run-evals.sh` and +skill-creator grade against these: `run-evals.sh` bridges the strings into the grader's +`{text, passed, evidence}` objects; skill-creator reads them directly. Edit the +`expectations` in `evals.json` to change what's graded. diff --git a/skills/cf-kind-verify/evals/evals.json b/skills/cf-kind-verify/evals/evals.json new file mode 100644 index 0000000..115af82 --- /dev/null +++ b/skills/cf-kind-verify/evals/evals.json @@ -0,0 +1,73 @@ +{ + "skill_name": "cf-kind-verify", + "note": "Trigger + flow-reasoning evals. Focus: does the skill fire on realistic CF-dev requests and correctly stay out on near-misses; does the model follow the script-first orchestration (call scripts, don't assemble pipelines inline).", + "evals": [ + { + "id": 0, + "name": "verify-component-change-happy-path", + "prompt": "I just changed the syslog drain filtering in src/syslog/egress.go in my loggregator-agent-release checkout. Can you verify this actually works on my local cf-on-kind cluster? The bake files are in ~/Workspace/public/cloudfoundry/cf-k8s-releases.", + "expected_output": "Triggers cf-kind-verify. Runs prereqs.sh; infers the syslog-agent image and STATES why (changed src/syslog path); calls redeploy.sh with --releases-dir and --src pointing at the release-repo clone; then discovers and states a test command. Does NOT hand-assemble the bake/jsonpath pipelines inline.", + "should_trigger": true, + "expectations": [ + "Recognizes this as a local cf-on-kind component verification (triggers/uses the cf-kind-verify flow)", + "Plans to run prereqs before building/deploying", + "Infers a specific image to rebuild (syslog-agent) AND states the reason (change is in the syslog egress source path)", + "Plans to redeploy pointing --src at the local release-repo clone (so the local change is built, not a remote git context)", + "Discovers/states a concrete test command rather than leaving testing unspecified", + "Calls the bundled scripts (prereqs/redeploy/etc.) rather than hand-assembling the docker buildx bake / kubectl jsonpath pipelines inline" + ], + "files": [] + }, + { + "id": 1, + "name": "cluster-down-offer-bootstrap", + "prompt": "verify my gorouter change on cfk8s", + "expected_output": "Triggers cf-kind-verify. Runs prereqs.sh. If the cluster reports down, OFFERS lifecycle.sh up --kind-dir rather than bootstrapping silently or just erroring out. Asks for the kind-deployment path if unknown.", + "should_trigger": true, + "expectations": [ + "Recognizes this as a local cf-on-kind component verification (triggers/uses the cf-kind-verify flow)", + "Plans to run prereqs / check cluster status first", + "If the cluster is down, OFFERS to bring it up (lifecycle up) rather than bootstrapping silently or just erroring out", + "Asks for the kind-deployment directory path rather than inferring it from repo/log content" + ], + "files": [] + }, + { + "id": 2, + "name": "cats-acceptance-focus", + "prompt": "run the app_syslog_tcp acceptance test against my local CF to check the drain change", + "expected_output": "Triggers cf-kind-verify. Reads references/test-conventions.md. Runs the ROOT CATS suite with --focus (not the bare package dir, which yields 'Found no test suites').", + "should_trigger": true, + "expectations": [ + "Recognizes this as a local cf-on-kind acceptance/CATS run (triggers/uses the cf-kind-verify flow)", + "Consults the CATS/test conventions before constructing the run", + "Runs the ROOT CATS suite with --focus rather than the bare package dir (avoids 'Found no test suites')" + ], + "files": [] + }, + { + "id": 3, + "name": "near-miss-real-landscape", + "prompt": "deploy my gorouter change to the staging CF landscape and run smoke tests there", + "expected_output": "Should NOT trigger cf-kind-verify (target is a real remote landscape via cf push / CI, not local cf-on-kind). The skill is about local kind verification only.", + "should_trigger": false, + "expectations": [ + "Does NOT run the local cf-on-kind verify flow (recognizes the target is a remote/staging landscape, not local kind)", + "Distinguishes remote-landscape deploy (cf push / CI) from local kind verification" + ], + "files": [] + }, + { + "id": 4, + "name": "near-miss-code-debugging", + "prompt": "why is my ginkgo test panicking with a nil map in this syslog handler?", + "expected_output": "Should NOT trigger cf-kind-verify (a code-debugging question about test source, no build/deploy/verify-on-kind intent).", + "should_trigger": false, + "expectations": [ + "Does NOT run the local cf-on-kind verify/build/deploy flow", + "Treats this as a code-debugging question about the test source (root-causes the nil map) rather than a deploy/verify-on-kind task" + ], + "files": [] + } + ] +} diff --git a/skills/cf-kind-verify/evals/run-evals.sh b/skills/cf-kind-verify/evals/run-evals.sh new file mode 100644 index 0000000..fbeecdf --- /dev/null +++ b/skills/cf-kind-verify/evals/run-evals.sh @@ -0,0 +1,411 @@ +#!/usr/bin/env bash +set -euo pipefail +# run-evals.sh — benchmark the working-tree cf-kind-verify skill against a +# baseline git ref, grade both, and print a version-vs-version delta table. +# +# Usage: +# skills/cf-kind-verify/evals/run-evals.sh [--] [options] +# +# Options: +# --against baseline to compare the working tree against (default: HEAD) +# --runs-per-query runs per config per eval (noise averaging) (default: 3) +# --timeout per claude -p call wall-clock budget (default: 120) +# --only comma-separated eval ids to run (default: all) +# --jobs max concurrent runs (batch size) (default: 5) +# --model model for executor+grader claude -p (default: anthropic--claude-sonnet-latest) +# --no-viewer skip launching the HTML results viewer +# --verbose stream per-run progress +# -- no-op separator (matches pre-authorized invocation) +# +# WHY THIS EXISTS (see README.md "Version-vs-version runner"): +# skill-creator already provides grading (agents/grader.md), aggregation +# (aggregate_benchmark.py), and the viewer (generate_review.py) — this script +# reuses all of them verbatim. What it adds is the glue skill-creator's +# agent-driven flow does not: an --against baseline, a contamination +# fix for a *user-level installed* skill, and the run-dir layout the aggregator +# expects. Delta = new_skill - old_skill. +# +# CONTAMINATION FIX (critical): cf-kind-verify is installed user-level +# (~/.claude/skills/cf-kind-verify -> ~/claude/cf-kind-verify), so any `claude -p` +# would otherwise see the ambient skill regardless of cwd. Both configs instead +# run --disable-slash-commands from an isolated $TMPDIR cwd, are handed the +# materialized SKILL.md via --append-system-prompt, and are told never to read a +# .claude dir. Neither side uses the ambient installed skill — symmetric. + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +SKILL_DIR="$(cd "$SCRIPT_DIR/.." && pwd)" +REPO_ROOT="$(cd "$SKILL_DIR/../.." && pwd)" +EVALS_JSON="$SCRIPT_DIR/evals.json" + +AGAINST="HEAD" +RUNS_PER_QUERY=3 +TIMEOUT=120 +ONLY="" +JOBS=5 +MODEL="anthropic--claude-sonnet-latest" +VIEWER=1 +VERBOSE=0 + +while [[ $# -gt 0 ]]; do + case "$1" in + --against) AGAINST="$2"; shift 2 ;; + --runs-per-query) RUNS_PER_QUERY="$2"; shift 2 ;; + --timeout) TIMEOUT="$2"; shift 2 ;; + --only) ONLY="$2"; shift 2 ;; + --jobs) JOBS="$2"; shift 2 ;; + --model) MODEL="$2"; shift 2 ;; + --no-viewer) VIEWER=0; shift ;; + --verbose) VERBOSE=1; shift ;; + --) shift ;; + -*) echo "run-evals.sh: unknown arg: $1" >&2; exit 2 ;; + *) echo "run-evals.sh: unexpected positional arg: $1" >&2; exit 2 ;; + esac +done + +log() { echo "==> $*"; } +vlog() { [[ "$VERBOSE" == 1 ]] && echo " $*" || true; } +die() { echo "run-evals.sh: $*" >&2; exit 1; } + +command -v claude >/dev/null 2>&1 || die "claude CLI not found on PATH" +command -v git >/dev/null 2>&1 || die "git not found on PATH" +[[ -f "$EVALS_JSON" ]] || die "evals.json not found at $EVALS_JSON" + +# Resolve skill-creator harness (reused for aggregation + viewer + grader.md). +# shellcheck disable=SC2012 # glob over ~/.claude paths; filenames are controlled +SC="$(ls -d "$HOME"/.claude/plugins/marketplaces/*/plugins/skill-creator/skills/skill-creator 2>/dev/null | head -1 || true)" +[[ -n "$SC" && -d "$SC" ]] || die "skill-creator harness not found under ~/.claude/plugins/marketplaces/" +GRADER_MD="$SC/agents/grader.md" +[[ -f "$GRADER_MD" ]] || die "grader.md not found at $GRADER_MD" + +# Resolve the baseline SHA up front (fail fast on a bad ref). +BASE_SHA="$(git -C "$REPO_ROOT" rev-parse --verify "$AGAINST^{commit}" 2>/dev/null)" \ + || die "--against '$AGAINST' is not a valid git ref in $REPO_ROOT" + +# Scratch lives in $TMPDIR — ephemeral, never in the repo. mktemp -d respects it. +SCRATCH="$(mktemp -d "${TMPDIR:-/tmp}/cfkv-evals.XXXXXX")" +BENCH="$SCRATCH/bench" +NEW_SKILL="$SCRATCH/new_skill_src" +OLD_WORKTREE="$SCRATCH/old_worktree" +mkdir -p "$BENCH" + +# Cleanup: remove the git worktree (never mutates the main tree) and scratch dir. +cleanup() { + if [[ -d "$OLD_WORKTREE" ]]; then + git -C "$REPO_ROOT" worktree remove --force "$OLD_WORKTREE" >/dev/null 2>&1 || true + fi + git -C "$REPO_ROOT" worktree prune >/dev/null 2>&1 || true + [[ -n "${KEEP_SCRATCH:-}" ]] || rm -rf "$SCRATCH" +} +trap cleanup EXIT + +log "Comparing working tree (new_skill) vs $AGAINST @ ${BASE_SHA:0:12} (old_skill)" +log "runs/config/eval=$RUNS_PER_QUERY timeout=${TIMEOUT}s jobs=$JOBS model=$MODEL" +log "scratch: $SCRATCH" + +# --- Materialize NEW = copy of the working-tree skill (includes uncommitted edits) +# Must be a COPY, not a worktree: we want the dirty working tree, and we must not +# touch the real repo. tar-pipe avoids an rsync dependency. +log "Materializing new_skill (working-tree copy)…" +mkdir -p "$NEW_SKILL" +( cd "$SKILL_DIR" && tar -c -f - . ) | tar -x -C "$NEW_SKILL" -f - +[[ -f "$NEW_SKILL/SKILL.md" ]] || die "new_skill copy has no SKILL.md (unexpected)" + +# --- Materialize OLD = detached worktree at the baseline ref -------------------- +log "Materializing old_skill (git worktree @ ${BASE_SHA:0:12})…" +git -C "$REPO_ROOT" worktree add --detach "$OLD_WORKTREE" "$BASE_SHA" >/dev/null 2>&1 \ + || die "git worktree add failed for $BASE_SHA" +OLD_SKILL="$OLD_WORKTREE/skills/cf-kind-verify" +[[ -f "$OLD_SKILL/SKILL.md" ]] || die "baseline ref has no skills/cf-kind-verify/SKILL.md" + +# --- Parse evals.json (embedded python, no jq) ---------------------------------- +# Emits one TSV line per (selected) eval: id \t name \t \t +# . b64 avoids any tab/newline in the fields +# corrupting the TSV. +EVAL_TSV="$SCRATCH/evals.tsv" +ONLY="$ONLY" EVALS_JSON="$EVALS_JSON" python3 /dev/fd/3 3<<'PY' >"$EVAL_TSV" +import base64, json, os +with open(os.environ["EVALS_JSON"]) as f: + data = json.load(f) +only = os.environ.get("ONLY", "").strip() +sel = set(int(x) for x in only.split(",")) if only else None +def b64(s): return base64.b64encode(s.encode()).decode() +for ev in data["evals"]: + if sel is not None and ev["id"] not in sel: + continue + exps = "\n".join(ev.get("expectations", [])) + print("\t".join([str(ev["id"]), ev["name"], b64(ev["prompt"]), b64(exps)])) +PY + +[[ -s "$EVAL_TSV" ]] || die "no evals selected (check --only '$ONLY')" +NUM_EVALS=$(wc -l <"$EVAL_TSV" | tr -d ' ') +log "Selected $NUM_EVALS eval(s); $((NUM_EVALS * 2 * RUNS_PER_QUERY)) total runs" + +# Instruction block appended to every executor prompt. It (a) forbids reading any +# .claude dir — closing the contamination path a breadcrumb could open — and (b) +# asks for a describe-the-flow answer (these evals are about triggering + script +# orchestration reasoning, not actually mutating a cluster). +read -r -d '' EXEC_GUARD <<'GUARD' || true +You are answering a user request. A candidate skill's instructions are appended +to your system prompt above. Decide whether that skill applies; if it does, follow +it. Describe the exact flow you WOULD take — which scripts you would call, in what +order, with which arguments, and state your reasoning and assumptions. Do NOT +actually build images, deploy, or run cluster-mutating commands. Do NOT read any +.claude directory or any installed-skill path; rely only on the appended skill +instructions and the request itself. +GUARD + +# run_one +# Executes one eval run: executor claude -p (JSON) -> summary.md + transcript.md + +# timing.json, then grader claude -p -> run-/grading.json. Isolated cwd, skill +# fed via --append-system-prompt, output parsed from JSON (.result) so no output +# path is ever handed to the model. +run_one() { + local config="$1" skill_dir="$2" eid="$3" name="$4" b64p="$5" b64e="$6" k="$7" + local run_dir="$BENCH/eval-${eid}-${name}/${config}/run-${k}" + local out_dir="$run_dir/outputs" + mkdir -p "$out_dir" + + local prompt exps skill_md + prompt="$(printf '%s' "$b64p" | base64 --decode)" + exps="$(printf '%s' "$b64e" | base64 --decode)" + skill_md="$(cat "$skill_dir/SKILL.md")" + + # Isolated cwd — nothing here references the ambient installed skill. + local work; work="$(mktemp -d "$SCRATCH/cwd.XXXXXX")" + + # --- Executor ------------------------------------------------------------- + local exec_json="$run_dir/exec.json" + local sys_prompt="$EXEC_GUARD + +# Candidate skill: cf-kind-verify (materialized from $config) +$skill_md" + + set +e + # shellcheck disable=SC1007 # CLAUDECODE= deliberately clears the var for the child + ( cd "$work" && CLAUDECODE= claude -p "$prompt" \ + --model "$MODEL" \ + --output-format json \ + --disable-slash-commands \ + --append-system-prompt "$sys_prompt" ) >"$exec_json" 2>/dev/null & + local pid=$! + _wait_with_timeout "$pid" "$TIMEOUT" + local exec_rc=$? + set -e + + if [[ "$exec_rc" -ne 0 ]]; then + vlog "eval $eid $config run $k: executor failed/timeout (rc=$exec_rc)" + printf '# executor failed (rc=%s)\n' "$exec_rc" >"$out_dir/summary.md" + else + # Extract .result (summary), token usage, duration from the JSON envelope. + EXEC_JSON="$exec_json" OUT_DIR="$out_dir" RUN_DIR="$run_dir" \ + python3 /dev/fd/3 3<<'PY' +import json, os +env = os.environ +try: + with open(env["EXEC_JSON"]) as f: + d = json.load(f) +except Exception: + d = {} +result = d.get("result", "") or "" +with open(os.path.join(env["OUT_DIR"], "summary.md"), "w") as f: + f.write(result if result else "# (empty executor result)\n") +# Transcript: for these describe-the-flow evals the final result IS the transcript. +with open(os.path.join(env["OUT_DIR"], "transcript.md"), "w") as f: + f.write(result if result else "# (empty)\n") +u = d.get("usage", {}) or {} +tokens = (u.get("input_tokens", 0) or 0) + (u.get("output_tokens", 0) or 0) +dur = (d.get("duration_ms", 0) or 0) / 1000.0 +with open(os.path.join(env["RUN_DIR"], "timing.json"), "w") as f: + json.dump({"total_duration_seconds": round(dur, 2), "total_tokens": tokens}, f) +PY + fi + + # --- Grader --------------------------------------------------------------- + # Single-shot text grading: the executor's answer is inlined into the prompt + # and the grader RETURNS grading.json as its .result (no file reads, no file + # writes, no tool use — those would hang/time out from the isolated cwd). We + # write .result to run-/grading.json ourselves, at the layout the aggregator + # needs. grader.md supplies the rubric + exact schema. + local summary_text; summary_text="$(cat "$out_dir/summary.md")" + local grader_prompt; grader_prompt="$(cat "$GRADER_MD") + +## This grading task + +You are grading a single eval run. Everything you need is inlined below — do NOT +read or write any files, and do NOT use any tools. Respond with ONLY the +grading JSON (no prose, no code fences). + +### Expectations (grade each; one per line) +$exps + +### Executor's answer (the transcript/output to grade) +<<"$run_dir/grader.json" 2>/dev/null & + local gpid=$! + _wait_with_timeout "$gpid" "$TIMEOUT" + set -e + + # The grader returns grading.json as its .result envelope; extract and write it + # to run-/grading.json. Fall back to an all-fail grading only if unrecoverable + # (a missing grading.json is silently dropped by the aggregator -> 0 runs). + RUN_DIR="$run_dir" GRADER_OUT="$run_dir/grader.json" EXPS="$exps" \ + python3 /dev/fd/3 3<<'PY' +import json, os +env = os.environ +gpath = os.path.join(env["RUN_DIR"], "grading.json") +def valid(p): + try: + with open(p) as f: json.load(f) + return True + except Exception: + return False +if not valid(gpath): + # Extract the grading JSON from the grader's .result envelope. + recovered = None + try: + with open(env["GRADER_OUT"]) as f: + d = json.load(f) + txt = d.get("result", "") or "" + s, e = txt.find("{"), txt.rfind("}") + if s != -1 and e != -1: + recovered = json.loads(txt[s:e+1]) + except Exception: + recovered = None + if recovered is None: + # Last resort: emit an all-fail grading so the aggregator still counts + # the run (a missing grading.json is silently dropped -> 0 runs). + exps = [x for x in env["EXPS"].split("\n") if x.strip()] + recovered = { + "expectations": [ + {"text": t, "passed": False, "evidence": "grading unavailable (executor/grader error)"} + for t in exps + ], + "summary": {"passed": 0, "failed": len(exps), "total": len(exps), "pass_rate": 0.0}, + } + with open(gpath, "w") as f: + json.dump(recovered, f, indent=2) +PY + + vlog "eval $eid $config run $k: done" +} + +# _wait_with_timeout — poll a backgrounded child up to , kill +# on timeout. macOS bash 3.2 has no `wait -n` and no coreutils `timeout`, so we +# poll. Returns the child's exit code, or 124 on timeout. +_wait_with_timeout() { + local pid="$1" secs="$2" waited=0 + while kill -0 "$pid" 2>/dev/null; do + if [[ "$waited" -ge "$secs" ]]; then + kill "$pid" 2>/dev/null + sleep 1 + kill -9 "$pid" 2>/dev/null || true + wait "$pid" 2>/dev/null + return 124 + fi + sleep 1 + waited=$((waited + 1)) + done + wait "$pid" 2>/dev/null + return $? +} + +# --- Build the run queue, execute in batches of $JOBS --------------------------- +# Each job is one (config, eval, run) unit; executor+grader run together inside +# run_one. Batch-of-N concurrency (bash 3.2-safe). set +e around the batch wait so +# a failing child never aborts the runner. +declare -a JOB_ARGS=() +while IFS=$'\t' read -r eid name b64p b64e; do + for k in $(seq 1 "$RUNS_PER_QUERY"); do + JOB_ARGS+=("new_skill|$NEW_SKILL|$eid|$name|$b64p|$b64e|$k") + JOB_ARGS+=("old_skill|$OLD_SKILL|$eid|$name|$b64p|$b64e|$k") + done +done <"$EVAL_TSV" + +TOTAL=${#JOB_ARGS[@]} +log "Dispatching $TOTAL runs in batches of ${JOBS}…" +i=0 +while [[ "$i" -lt "$TOTAL" ]]; do + declare -a pids=() + batch_n=0 + while [[ "$batch_n" -lt "$JOBS" && "$i" -lt "$TOTAL" ]]; do + IFS='|' read -r c s eid name b64p b64e k <<<"${JOB_ARGS[$i]}" + vlog "start [$((i + 1))/$TOTAL] eval $eid $c run $k" + run_one "$c" "$s" "$eid" "$name" "$b64p" "$b64e" "$k" & + pids+=($!) + i=$((i + 1)) + batch_n=$((batch_n + 1)) + done + set +e + for p in "${pids[@]}"; do wait "$p"; done + set -e + log "…$i/$TOTAL runs complete" +done + +# --- eval_metadata.json per eval dir (aggregator reads eval_id from it) --------- +while IFS=$'\t' read -r eid name b64p b64e; do + ed="$BENCH/eval-${eid}-${name}" + [[ -d "$ed" ]] || continue + EID="$eid" NAME="$name" B64P="$b64p" ED="$ed" python3 /dev/fd/3 3<<'PY' +import base64, json, os +env = os.environ +meta = { + "eval_id": int(env["EID"]), + "name": env["NAME"], + "prompt": base64.b64decode(env["B64P"]).decode(), +} +with open(os.path.join(env["ED"], "eval_metadata.json"), "w") as f: + json.dump(meta, f, indent=2) +PY +done <"$EVAL_TSV" + +# --- Aggregate + viewer (MUST run unsandboxed: ProcessPoolExecutor) ------------- +log "Aggregating (reusing skill-creator aggregate_benchmark.py)…" +( cd "$SC" && python3 -m scripts.aggregate_benchmark "$BENCH" \ + --skill-name cf-kind-verify --skill-path "$NEW_SKILL" ) \ + || die "aggregation failed (must run unsandboxed — ProcessPoolExecutor)" + +BENCH_MD="$BENCH/benchmark.md" +BENCH_JSON="$BENCH/benchmark.json" +if [[ -f "$BENCH_MD" ]]; then + echo + cat "$BENCH_MD" + echo +fi + +# Extract the headline delta for a one-line summary. +if [[ -f "$BENCH_JSON" ]]; then + BENCH_JSON="$BENCH_JSON" python3 /dev/fd/3 3<<'PY' +import json, os +with open(os.environ["BENCH_JSON"]) as f: + d = json.load(f) +delta = (d.get("run_summary") or d).get("delta", {}) if isinstance(d, dict) else {} +if not delta: + delta = d.get("delta", {}) +pr = delta.get("pass_rate", "?") +print(f"==> DELTA (new_skill - old_skill): pass_rate {pr} " + f"time {delta.get('time_seconds','?')}s tokens {delta.get('tokens','?')}") +PY +fi + +log "Benchmark artifacts: $BENCH" +log " (set KEEP_SCRATCH=1 to preserve $SCRATCH after exit)" + +if [[ "$VIEWER" == 1 ]]; then + log "Launching results viewer on http://127.0.0.1:3117 …" + ( cd "$SC" && python3 eval-viewer/generate_review.py "$BENCH" \ + --skill-name cf-kind-verify --benchmark "$BENCH_JSON" ) \ + || log "viewer failed to launch (aggregation still succeeded; see $BENCH)" +else + log "Viewer skipped (--no-viewer)." +fi diff --git a/skills/cf-kind-verify/references/test-conventions.md b/skills/cf-kind-verify/references/test-conventions.md new file mode 100644 index 0000000..a6956b2 --- /dev/null +++ b/skills/cf-kind-verify/references/test-conventions.md @@ -0,0 +1,110 @@ +# Test conventions & gotchas (cf-kind-verify) + +Read this when the change involves acceptance/CATS tests, ginkgo suites, or a +helm-chart edit. Contents: + +- [CATS: run the root suite with --focus](#cats-run-the-root-suite-with---focus) +- [CATS is already wired in kind-deployment](#cats-is-already-wired-in-kind-deployment) +- [Buildpacks](#buildpacks) +- [Helm-chart changes: render, upgrade, SSA conflict](#helm-chart-changes) +- [Config-contract source of truth](#config-contract-source-of-truth) + +## CATS: run the root suite with --focus + +Use the bundled **`/scripts/cats.sh`** — it renders the config and runs a +focused subset the right way, from any cwd: + +```bash +bash /scripts/cats.sh --kind-dir \ + --focus "Syslog Drain source type filter over TCP" [--cats-dir ] [--timeout 45m] +# or just render the config and stop: +bash /scripts/cats.sh --kind-dir --render-only +``` + +`--cats-dir` defaults to `/../cf-acceptance-tests`. This is a script +rather than an inline command because the render writes into the kind-deployment +repo and the run writes `assets/` and accesses the cluster and Docker. Permission +behavior depends on the selected harness and its configuration. + +The CATS-specific traps (no `*_suite_test.go` in the package dirs so only the root +suite collects specs; `--focus` not `FDescribe`) are documented in the script's own +header comment — read `/scripts/cats.sh` if you need to run by hand or change +its behavior. + +## CATS is already wired in kind-deployment + +Don't hand-roll `integration_config.json` — kind-deployment ships the wiring: + +- `scripts/cats.sh` (in kind-deployment) — renders `.github/cats-config.tpl` → + `.github/cats-config.json` via `python3 os.path.expandvars` after + `source temp/secrets.sh`, then runs the suite via `/bin/test` (with + `--randomize-all`), forwarding any extra args (kind-deployment PR #489). So a + focused run is just `scripts/cats.sh --focus=""`. `CATS_PATH` defaults to + `../cf-acceptance-tests`. The skill's `/scripts/cats.sh` is a thin shim + over this — prefer it over calling this one by hand. +- `.github/cats-config.tpl` — full config: `api/apps_domain=…127-0-0-1.nip.io`, + `admin_user=ccadmin`, `skip_ssl_validation`, the `include_*` suite toggles, + `timeout_scale: 2`. Secrets (`CC_ADMIN_PASSWORD`, `OAUTH_CLIENTS_SECRET`) come + from `temp/secrets.sh`. +- `.github/workflows/kind-cats.yaml` — the canonical CI ordering to mirror: + `make up → login → bootstrap-complete → cf push smoke test → register nfs + broker → docker compose extra services → setup-cf-tests → bin/test --procs=4`. + +## Buildpacks + +- **Buildpacks must be uploaded before pushing apps.** Plain `make bootstrap` + loads only java/nodejs/go/binary; ruby/python/etc. need **`ALL_BUILDPACKS=true`** + (i.e. `make bootstrap-complete`, or the `--all-buildpacks` flag on + `lifecycle.sh up`). Symptom otherwise: CATS bails in `SynchronizedBeforeSuite` + with *"Missing the ruby buildpack specified in the integration_config.json"*. + The upload script reads `cf curl /v3/buildpacks`, so it needs a **valid login** + first (a null / `Cannot iterate over null` from `jq` means the token expired — + re-login). + +## Helm-chart changes + +`make up`/`helmfile sync` uses the **published** chart, so local chart edits are +only exercised via a direct `helm upgrade` against the local chart dir +(`cf-k8s-releases//helm/`) — no need to edit +`kind-deployment/helmfile.yaml.gotmpl`. + +```bash +cd +export KUBECONFIG=temp/kubeconfig # written by `make up` +bin/helm get values -n cf-system # base values +# build /tmp/values.yaml = those + image overrides to your branch :latest build +# validate the render FIRST (use render-check.sh): +bash /scripts/render-check.sh ../cf-k8s-releases//helm \ + --kind-dir . --values /tmp/values.yaml --grep 'NEW_ENV|image:' +bin/helm upgrade ../cf-k8s-releases//helm -n cf-system -f /tmp/values.yaml +kubectl -n cf-system rollout status deployment/ --timeout=120s +``` + +- **Override images to your branch `:latest`.** The published image may predate a + newly-`required` config field and run fine without it, hiding the bug; your + branch build enforces it. +- **`helm` isn't on PATH** — it's installed into `kind-deployment/bin/` by + `scripts/tools.sh`. Pass `--kind-dir` so `render-check.sh` uses it. +- **SSA conflict if redeploy ran first:** a prior `kubectl set image`/`set env` + makes `helm upgrade` fail with `conflict with "kubectl-set"`. Fix: + `kubectl -n cf-system delete deployment ` then re-upgrade. Redeploy + and helm don't compose on the same deployment in one session — **if a chart + change is involved, prefer helm from the start.** + +## Config-contract source of truth + +- `/src/cmd//app/config.go` — `env:"..."` tags, + `required` markers. +- BOSH job `jobs//templates/bpm.yml.erb` — correct values/naming. +- CF agents print an **env-struct config report** at startup (every field, its env + var, whether `required`, resolved value) — the single best signal that chart + wiring is correct. Grep the startup logs for it. +- Client connections are often **lazy** (dial on first use) — "no connection log + yet" is normal; trigger the real code path to see it. + +## App-level CATS (informs the deferred v2 `app` flow) + +When you do need to push an app and run app-level CATS, reuse kind-deployment's +existing CATS wiring (above) rather than reinventing `integration_config.json`. +Mirror the `kind-cats.yaml` ordering and use `--all-buildpacks` at bring-up. This +is not a first-class v1 flow — do it only on explicit request. diff --git a/skills/cf-kind-verify/scripts/_discover_target.py b/skills/cf-kind-verify/scripts/_discover_target.py new file mode 100644 index 0000000..8d155a2 --- /dev/null +++ b/skills/cf-kind-verify/scripts/_discover_target.py @@ -0,0 +1,72 @@ +"""Walk pods/replicasets/deployments JSON to find which deployment runs a given image. Called by discover-target.sh.""" +import json +import os +import sys + +image = os.environ["IMAGE"] +cluster = os.environ["CLUSTER"] +data = json.load(sys.stdin) +items = data.get("items", []) + +# Index ReplicaSets and Deployments by (namespace, name) so we can resolve a +# pod's ownerRef chain (pod -> ReplicaSet -> Deployment) with plain dict lookups. +replicasets = {} +deployments = set() +for obj in items: + kind = obj.get("kind", "") + ns = obj["metadata"]["namespace"] + name = obj["metadata"]["name"] + if kind == "ReplicaSet": + replicasets[(ns, name)] = obj + elif kind == "Deployment": + deployments.add((ns, name)) + +def owner(obj, kind): + """Name of obj's ownerReference of the given kind, or ''.""" + for ref in obj["metadata"].get("ownerReferences", []): + if ref.get("kind") == kind: + return ref["name"] + return "" + +def deployment_for(ns, pod): + """Walk pod -> ReplicaSet -> Deployment; fall back to the pod name with its + two hash suffixes stripped if the ownerRefs aren't present.""" + rs = owner(pod, "ReplicaSet") + if rs: + rs_obj = replicasets.get((ns, rs)) + if rs_obj: + dep = owner(rs_obj, "Deployment") + if dep: + return dep + # e.g. "forwarder-agent-6fccfcdb8d-wpm5n" -> "forwarder-agent" + return pod.rsplit("-", 2)[0] + +found = [] +seen = set() +for obj in items: + if obj.get("kind") != "Pod": + continue + ns = obj["metadata"]["namespace"] + name = obj["metadata"]["name"] + for container in obj["spec"].get("containers", []): + # Substring match: image may be ":latest" or a fully-qualified ref. + if image not in container.get("image", ""): + continue + dep = deployment_for(ns, obj) + key = (dep, container["name"], ns) + if key in seen: + continue + seen.add(key) + found.append(key) + +for dep, cont, ns in found: + print(dep, cont, ns) + +if not found: + sys.stderr.write( + f"discover-target.sh: no running container matches image '{image}' " + f"in cluster '{cluster}'\n" + "discover-target.sh: (is the deployment present? " + "try: kubectl get pods -A | grep )\n" + ) + sys.exit(1) diff --git a/skills/cf-kind-verify/scripts/_status_containers.py b/skills/cf-kind-verify/scripts/_status_containers.py new file mode 100644 index 0000000..e02222e --- /dev/null +++ b/skills/cf-kind-verify/scripts/_status_containers.py @@ -0,0 +1,10 @@ +"""Print per-container status lines for each pod in a kubectl -o json pod list. Called by verify-status.sh.""" +import json +import sys + +data = json.load(sys.stdin) +for pod in data.get("items", []): + print("pod", pod["metadata"]["name"]) + for cs in pod.get("status", {}).get("containerStatuses", []): + print(f" {cs['name']} ready={str(cs['ready']).lower()} " + f"restarts={cs['restartCount']} image={cs['image']}") diff --git a/skills/cf-kind-verify/scripts/_status_selector.py b/skills/cf-kind-verify/scripts/_status_selector.py new file mode 100644 index 0000000..8226f3c --- /dev/null +++ b/skills/cf-kind-verify/scripts/_status_selector.py @@ -0,0 +1,17 @@ +"""Build a k1=v1,k2=v2 selector string from a deployment's matchLabels. Called by verify-status.sh.""" +import json +import os +import sys + +try: + data = json.load(sys.stdin) +except ValueError: + data = {} +match_labels = data.get("spec", {}).get("selector", {}).get("matchLabels", {}) +if not match_labels: + sys.stderr.write( + f"verify-status.sh: deployment '{os.environ['DEP']}' not found in " + f"namespace '{os.environ['NS']}' (or has no selector)\n" + ) + sys.exit(1) +print(",".join(f"{k}={v}" for k, v in match_labels.items())) diff --git a/skills/cf-kind-verify/scripts/cats.sh b/skills/cf-kind-verify/scripts/cats.sh new file mode 100755 index 0000000..6dfd036 --- /dev/null +++ b/skills/cf-kind-verify/scripts/cats.sh @@ -0,0 +1,83 @@ +#!/usr/bin/env bash +set -euo pipefail +# cats.sh — render the CATS config and run a FOCUSED subset of the CF Acceptance +# Tests against the local cf-on-kind cluster, from any cwd. +# +# Usage: +# cats.sh --kind-dir --focus "" [options] +# +# Options: +# --kind-dir clone of cloudfoundry/kind-deployment (has scripts/cats.sh, +# .github/cats-config.tpl, temp/secrets.sh, temp/kubeconfig) [required] +# --cats-dir clone of cf-acceptance-tests (default: /../cf-acceptance-tests) +# --focus ginkgo --focus regex; selects specs on the ROOT suite [required] +# --timeout ginkgo --timeout (default: 45m) +# --render-only render the config and stop (don't run any specs) +# +# Thin shim over kind-deployment's own scripts/cats.sh, which renders +# .github/cats-config.json and forwards extra args to cf-acceptance-tests/bin/test. +# We resolve paths cwd-independently, forward --focus/--timeout, and run outside +# the sandbox. See references/test-conventions.md. + +KIND_DIR="" +CATS_DIR="" +FOCUS="" +TIMEOUT="45m" +RENDER_ONLY="" + +# Accept both `--flag value` and `--flag=value` forms. +while [[ $# -gt 0 ]]; do + case "$1" in + --kind-dir) KIND_DIR="$2"; shift 2 ;; + --kind-dir=*) KIND_DIR="${1#*=}"; shift ;; + --cats-dir) CATS_DIR="$2"; shift 2 ;; + --cats-dir=*) CATS_DIR="${1#*=}"; shift ;; + --focus) FOCUS="$2"; shift 2 ;; + --focus=*) FOCUS="${1#*=}"; shift ;; + --timeout) TIMEOUT="$2"; shift 2 ;; + --timeout=*) TIMEOUT="${1#*=}"; shift ;; + --render-only) RENDER_ONLY=1; shift ;; + -*) echo "cats.sh: unknown arg: $1" >&2; exit 2 ;; + *) echo "cats.sh: unexpected positional arg: $1" >&2; exit 2 ;; + esac +done + +log() { echo "==> $*"; } +die() { echo "cats.sh: $*" >&2; exit 1; } + +[[ -n "$KIND_DIR" ]] || { echo "cats.sh: usage: cats.sh --kind-dir --focus \"\" [--cats-dir ] [--timeout ] [--render-only]" >&2; exit 2; } +[[ -d "$KIND_DIR" ]] || die "kind-dir '$KIND_DIR' does not exist (pass a clone of cloudfoundry/kind-deployment)" + +# Resolve to absolute paths so the rest of the script is cwd-independent. +KIND_DIR="$(cd "$KIND_DIR" && pwd)" +CATS_DIR="${CATS_DIR:-$KIND_DIR/../cf-acceptance-tests}" +[[ -d "$CATS_DIR" ]] || die "cats-dir '$CATS_DIR' does not exist (pass --cats-dir )" +CATS_DIR="$(cd "$CATS_DIR" && pwd)" + +[[ -f "$KIND_DIR/scripts/cats.sh" ]] || die "'$KIND_DIR' is not a kind-deployment clone (no scripts/cats.sh)" +[[ -f "$KIND_DIR/temp/secrets.sh" ]] || die "'$KIND_DIR/temp/secrets.sh' missing — is the cluster up? (make up writes temp/)" +[[ -f "$KIND_DIR/temp/kubeconfig" ]] || die "'$KIND_DIR/temp/kubeconfig' missing — is the cluster up? (make up writes temp/)" + +if [[ -z "$RENDER_ONLY" && -z "$FOCUS" ]]; then + die "no --focus given. Pass --focus \"\", or --render-only to just render the config." +fi + +CATS_CONFIG="$KIND_DIR/.github/cats-config.json" + +if [[ -n "$RENDER_ONLY" ]]; then + # Render only. Run from the kind-dir so its relative paths resolve. + log "render-only: RENDER_ONLY=1 scripts/cats.sh (in $KIND_DIR) -> .github/cats-config.json" + ( cd "$KIND_DIR" && RENDER_ONLY=1 CATS_PATH="$CATS_DIR" bash scripts/cats.sh ) \ + || die "render failed (check $KIND_DIR/.github/cats-config.tpl and temp/secrets.sh)" + [[ -f "$CATS_CONFIG" ]] || die "expected rendered config at '$CATS_CONFIG' but it's missing" + log "done (render-only). Config: $CATS_CONFIG" + exit 0 +fi + +# Render + run via kind-deployment's scripts/cats.sh; forward --focus/--timeout. +log "run: scripts/cats.sh --focus=\"$FOCUS\" --timeout=$TIMEOUT (in $KIND_DIR)" +( cd "$KIND_DIR" && CATS_PATH="$CATS_DIR" bash scripts/cats.sh --focus="$FOCUS" --timeout="$TIMEOUT" ) \ + || die "focused CATS run failed (see ginkgo output above)" + +echo +log "done. focused CATS run completed for --focus=\"$FOCUS\"." diff --git a/skills/cf-kind-verify/scripts/discover-target.sh b/skills/cf-kind-verify/scripts/discover-target.sh new file mode 100755 index 0000000..143913e --- /dev/null +++ b/skills/cf-kind-verify/scripts/discover-target.sh @@ -0,0 +1,35 @@ +#!/usr/bin/env bash +set -euo pipefail +# discover-target.sh — find which (deployment, container) runs a given image. +# +# Usage: discover-target.sh [--cluster ] +# +# Deployment name != image name != container name, and these charts often use a +# bare `app: ` label with no `app.kubernetes.io/name`, so `set image` by +# guessed name fails. Instead we walk every pod's containers, match the image +# substring, and map the owning pod back to its Deployment via the ReplicaSet +# ownerRef. Prints one line per match: +# +# DEPLOYMENT CONTAINER NAMESPACE +# +# so callers can `read DEPLOYMENT CONTAINER NAMESPACE < <(discover-target.sh ...)`. +# +# JSON parsing lives in _discover_target.py + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +CLUSTER="cfk8s" +IMAGE="" + +while [[ $# -gt 0 ]]; do + case "$1" in + --cluster) CLUSTER="$2"; shift 2 ;; + -*) echo "discover-target.sh: unknown arg: $1" >&2; exit 2 ;; + *) IMAGE="$1"; shift ;; + esac +done + +[[ -n "$IMAGE" ]] || { echo "discover-target.sh: usage: discover-target.sh [--cluster ]" >&2; exit 2; } + +# A single kubectl call (vs. one per pod) keeps this to one allowlist entry. +kubectl get pods,replicasets,deployments -A -o json 2>/dev/null | \ +IMAGE="$IMAGE" CLUSTER="$CLUSTER" python3 "$SCRIPT_DIR/_discover_target.py" diff --git a/skills/cf-kind-verify/scripts/lifecycle.sh b/skills/cf-kind-verify/scripts/lifecycle.sh new file mode 100755 index 0000000..88695ed --- /dev/null +++ b/skills/cf-kind-verify/scripts/lifecycle.sh @@ -0,0 +1,102 @@ +#!/usr/bin/env bash +set -euo pipefail +# lifecycle.sh — bring the local cf-on-kind cluster up or down. +# +# Usage: +# lifecycle.sh up --kind-dir [--cluster ] [--all-buildpacks] +# lifecycle.sh down --kind-dir [--cluster ] +# lifecycle.sh status [--cluster ] +# +# --kind-dir path to a clone of cloudfoundry/kind-deployment (required +# for up/down; that's where the Makefile lives) +# --cluster kind cluster name (default: cfk8s) +# --all-buildpacks on `up`, run `make bootstrap-complete` (ALL_BUILDPACKS) so +# ruby/python/etc. are uploaded — needed for full CATS. +# Default is plain `make bootstrap` (java/nodejs/go/binary). +# +# `up` = make up && make login && make bootstrap[-complete] from --kind-dir. +# `down` = make down. These wrap the canonical kind-deployment targets rather +# than reinventing them; see kind-deployment/docs/local-development-guide.md. +# +# NOTE: `up` is slow (pulls images, boots a cluster, bootstraps CF) and is +# destructive-ish (down tears the cluster down). Only run when the user asked to +# start/stop the cluster — never as a silent side effect of a verify request. + +CLUSTER="cfk8s" +KIND_DIR="" +ALL_BP=0 + +CMD="${1:-}"; shift || true +[[ -n "$CMD" ]] || { echo "lifecycle.sh: usage: lifecycle.sh [options]" >&2; exit 2; } + +while [[ $# -gt 0 ]]; do + case "$1" in + --kind-dir) KIND_DIR="$2"; shift 2 ;; + --cluster) CLUSTER="$2"; shift 2 ;; + --all-buildpacks) ALL_BP=1; shift ;; + -*) echo "lifecycle.sh: unknown arg: $1" >&2; exit 2 ;; + *) echo "lifecycle.sh: unexpected arg: $1" >&2; exit 2 ;; + esac +done + +log() { echo "==> $*"; } +die() { echo "lifecycle.sh: $*" >&2; exit 1; } + +need_kind_dir() { + [[ -n "$KIND_DIR" ]] || die "--kind-dir is required for '$CMD'" + [[ -f "$KIND_DIR/Makefile" ]] || die "'$KIND_DIR' has no Makefile — is it a kind-deployment clone?" + # We run `make` from $KIND_DIR below, so require the Makefile to define the real + # kind-deployment targets — not just exist. Backstop, not a guarantee: --kind-dir + # must be a user-confirmed path. + local t missing=() + for t in up down login bootstrap; do + grep -qE "^${t}[[:space:]]*:" "$KIND_DIR/Makefile" || missing+=("$t") + done + if [[ ${#missing[@]} -gt 0 ]]; then + die "'$KIND_DIR/Makefile' is missing expected kind-deployment target(s): ${missing[*]} — refusing to run 'make' in a dir that isn't a kind-deployment clone" + fi +} + +case "$CMD" in + status) + if kind get clusters 2>/dev/null | grep -qx "$CLUSTER"; then + echo "cluster '$CLUSTER': UP" + kubectl config current-context 2>/dev/null || true + kubectl get nodes 2>/dev/null || true + else + echo "cluster '$CLUSTER': DOWN (not in \`kind get clusters\`)" + exit 1 + fi + ;; + + up) + need_kind_dir + if kind get clusters 2>/dev/null | grep -qx "$CLUSTER"; then + log "cluster '$CLUSTER' already up — skipping 'make up'" + else + log "[up] make up (this pulls images and boots a kind cluster — slow)" + ( cd "$KIND_DIR" && make up ) || die "'make up' failed in $KIND_DIR" + fi + log "[up] make login" + ( cd "$KIND_DIR" && make login ) || die "'make login' failed" + if [[ "$ALL_BP" -eq 1 ]]; then + log "[up] make bootstrap-complete (ALL_BUILDPACKS — needed for full CATS)" + ( cd "$KIND_DIR" && make bootstrap-complete ) || die "'make bootstrap-complete' failed" + else + log "[up] make bootstrap" + ( cd "$KIND_DIR" && make bootstrap ) || die "'make bootstrap' failed" + fi + log "up complete — cluster '$CLUSTER' ready" + ;; + + down) + need_kind_dir + log "[down] make down (tears down cluster '$CLUSTER')" + ( cd "$KIND_DIR" && make down ) || die "'make down' failed in $KIND_DIR" + log "down complete" + ;; + + *) + die "unknown command '$CMD' (expected up|down|status)" + ;; +esac diff --git a/skills/cf-kind-verify/scripts/prereqs.sh b/skills/cf-kind-verify/scripts/prereqs.sh new file mode 100755 index 0000000..7692d58 --- /dev/null +++ b/skills/cf-kind-verify/scripts/prereqs.sh @@ -0,0 +1,57 @@ +#!/usr/bin/env bash +set -euo pipefail +# prereqs.sh — verify the local toolchain and cf-on-kind cluster are ready. +# +# Usage: prereqs.sh [--cluster ] +# --cluster kind cluster name (default: cfk8s) +# +# Exit 0 when everything is ready. On the first failure, exit non-zero and +# print to stderr what is missing and (where useful) how to fix it. The model +# reads that message and decides whether to offer a bootstrap (see lifecycle.sh) +# or to stop and hand back to the user. + +CLUSTER="cfk8s" + +while [[ $# -gt 0 ]]; do + case "$1" in + --cluster) CLUSTER="$2"; shift 2 ;; + *) echo "prereqs.sh: unknown arg: $1" >&2; exit 2 ;; + esac +done + +fail() { echo "prereqs.sh: $*" >&2; exit 1; } + +# 1. Required tools on PATH. `cf` and `helm` are checked but only warned about: +# redeploy works without them; only test discovery / helm flows need them. +missing=() +for t in docker kind kubectl; do + command -v "$t" >/dev/null 2>&1 || missing+=("$t") +done +if [[ ${#missing[@]} -gt 0 ]]; then + fail "missing required tool(s): ${missing[*]} (install and re-run)" +fi +for t in cf helm; do + command -v "$t" >/dev/null 2>&1 || echo "prereqs.sh: note: '$t' not on PATH — some flows (tests / helm) will be limited" >&2 +done + +# 2. docker daemon reachable. +docker info >/dev/null 2>&1 || fail "docker daemon not reachable (is Docker running?)" + +# 3. kind cluster exists. +if ! kind get clusters 2>/dev/null | grep -qx "$CLUSTER"; then + fail "kind cluster '$CLUSTER' not found — cluster is down. Bring it up with: lifecycle.sh up --kind-dir " +fi + +# 4. kubectl context points at the cluster. +ctx="$(kubectl config current-context 2>/dev/null || true)" +if [[ "$ctx" != "kind-$CLUSTER" ]]; then + fail "kubectl context is '$ctx', expected 'kind-$CLUSTER' — run: kubectl config use-context kind-$CLUSTER" +fi + +# 5. At least one Ready node. Check the STATUS column (field 2) exactly, rather +# than a substring match — "Ready" vs "NotReady"/"Ready,SchedulingDisabled". +if ! kubectl get nodes --no-headers 2>/dev/null | awk '{print $2}' | grep -qx "Ready"; then + fail "no Ready nodes in cluster '$CLUSTER' — cluster may be starting or unhealthy" +fi + +echo "prereqs.sh: OK — docker up, kind cluster '$CLUSTER' present, context kind-$CLUSTER, node(s) Ready" diff --git a/skills/cf-kind-verify/scripts/redeploy.sh b/skills/cf-kind-verify/scripts/redeploy.sh new file mode 100755 index 0000000..3f48b3f --- /dev/null +++ b/skills/cf-kind-verify/scripts/redeploy.sh @@ -0,0 +1,117 @@ +#!/usr/bin/env bash +set -euo pipefail +# redeploy.sh — build a CF component image from source, load it into the local +# cf-on-kind cluster, and roll the running deployment onto it. +# +# Usage: +# redeploy.sh [options] +# +# bake target / image name (e.g. syslog-agent, gorouter) +# +# Options: +# --src local-path context for the src/ subdir (default: $PWD/src) +# --releases-dir dir containing /docker-bake.hcl to build from +# (default: $PWD — i.e. run from cf-k8s-releases/) +# --namespace namespace of the target deployment (default: cf-system) +# --cluster kind cluster name (default: cfk8s) +# --tag image tag to load & deploy (default: latest) +# +# Notes baked in from real runs (see PLAN.md "Key facts"): +# - Build runs from cf-k8s-releases// where the Dockerfile + bake +# file live — NOT the release repo root (kind-deployment's releases/ is empty). +# - The bake `src` context resolves to /src; the Dockerfile does +# `COPY --from=src . //src`, so point --src at the src/ subdir. +# - buildx needs --allow=fs.read= for a local-path context, plus +# output=type=docker so the image lands in the daemon (kind load can see it). +# - deployment/container/image names all differ; discover the pair rather than +# guessing (discover-target.sh). +# - imagePullPolicy=IfNotPresent is usually already set; the patch is a no-op. + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +IMAGE="" +SRC="$PWD/src" +RELEASES_DIR="$PWD" +NS="cf-system" +CLUSTER="cfk8s" +TAG="latest" + +while [[ $# -gt 0 ]]; do + case "$1" in + --src) SRC="$2"; shift 2 ;; + --releases-dir) RELEASES_DIR="$2"; shift 2 ;; + --namespace) NS="$2"; shift 2 ;; + --cluster) CLUSTER="$2"; shift 2 ;; + --tag) TAG="$2"; shift 2 ;; + -*) echo "redeploy.sh: unknown arg: $1" >&2; exit 2 ;; + *) IMAGE="$1"; shift ;; + esac +done + +[[ -n "$IMAGE" ]] || { echo "redeploy.sh: usage: redeploy.sh [--src ] [--releases-dir ] [--namespace ] [--cluster ]" >&2; exit 2; } + +log() { echo "==> $*"; } +die() { echo "redeploy.sh: $*" >&2; exit 1; } + +[[ -d "$SRC" ]] || die "src context '$SRC' does not exist (pass --src )" +[[ -d "$RELEASES_DIR" ]] || die "releases dir '$RELEASES_DIR' does not exist (pass --releases-dir )" + +# The docker-bake.hcl + Dockerfiles live in cf-k8s-releases//, not the +# releases root (fact #1). Resolve the build dir: if RELEASES_DIR already has a +# bake file, build there; otherwise find the component subdir whose bake file +# defines this target. This lets the caller pass either the component dir or the +# releases root without needing to know the layout. +resolve_build_dir() { + if [[ -f "$RELEASES_DIR/docker-bake.hcl" ]]; then + echo "$RELEASES_DIR"; return 0 + fi + # Prefer a subdir whose bake file names the target; fall back to a subdir named + # like the image. + local hit + hit="$(grep -rl -E "\"?$IMAGE\"?" --include=docker-bake.hcl "$RELEASES_DIR" 2>/dev/null | head -1 || true)" + if [[ -n "$hit" ]]; then + dirname "$hit"; return 0 + fi + [[ -f "$RELEASES_DIR/$IMAGE/docker-bake.hcl" ]] && { echo "$RELEASES_DIR/$IMAGE"; return 0; } + return 1 +} +BUILD_DIR="$(resolve_build_dir)" \ + || die "no docker-bake.hcl defining target '$IMAGE' under '$RELEASES_DIR' (pass --releases-dir )" + +# 1. Build. Run from the resolved component dir so bake + Dockerfiles resolve. +log "[1/5] build: bake '$IMAGE' from '$BUILD_DIR' (src=$SRC)" +( cd "$BUILD_DIR" && \ + docker buildx bake --allow=fs.read="$SRC" "$IMAGE" \ + --set "$IMAGE.contexts.src=$SRC" \ + --set "$IMAGE.output=type=docker" \ +) || die "bake failed for target '$IMAGE' (check the target name: docker buildx bake --print)" + +# 2. Load into kind. +log "[2/5] load: kind load docker-image '$IMAGE:$TAG' --name '$CLUSTER'" +kind load docker-image "$IMAGE:$TAG" --name "$CLUSTER" \ + || die "kind load failed (image built? cluster '$CLUSTER' up?)" + +# 3. Discover the (deployment, container) pair for this image. +log "[3/5] discover: target for image '$IMAGE'" +target="$(bash "$SCRIPT_DIR/discover-target.sh" "$IMAGE" --cluster "$CLUSTER" | head -1)" +[[ -n "$target" ]] || die "could not find a running container using image '$IMAGE' (is it deployed?)" +read -r DEP CONT TNS <<<"$target" +[[ -n "${TNS:-}" ]] && NS="$TNS" +log " -> deployment=$DEP container=$CONT namespace=$NS" + +# 4. Point the container at the freshly loaded image; ensure IfNotPresent. +log "[4/5] set image: deployment/$DEP $CONT=$IMAGE:$TAG (ns=$NS)" +kubectl -n "$NS" set image "deployment/$DEP" "$CONT=$IMAGE:$TAG" +# Harmless no-op if the chart already sets IfNotPresent — don't treat as error. +kubectl -n "$NS" patch deployment "$DEP" --type=json \ + -p "[{\"op\":\"replace\",\"path\":\"/spec/template/spec/containers/0/imagePullPolicy\",\"value\":\"IfNotPresent\"}]" \ + >/dev/null 2>&1 || true + +# 5. Wait for the rollout. +log "[5/5] rollout: deployment/$DEP (timeout 180s)" +kubectl -n "$NS" rollout status "deployment/$DEP" --timeout=180s \ + || die "rollout did not complete — check: kubectl -n $NS describe deployment/$DEP" + +echo +log "done. verify with: bash $SCRIPT_DIR/verify-status.sh $DEP --namespace $NS" +bash "$SCRIPT_DIR/verify-status.sh" "$DEP" --namespace "$NS" || true diff --git a/skills/cf-kind-verify/scripts/render-check.sh b/skills/cf-kind-verify/scripts/render-check.sh new file mode 100755 index 0000000..fe8a9ca --- /dev/null +++ b/skills/cf-kind-verify/scripts/render-check.sh @@ -0,0 +1,83 @@ +#!/usr/bin/env bash +set -euo pipefail +# render-check.sh — validate a local helm-chart change renders as expected +# BEFORE applying it, by templating the chart and grepping the output. +# +# Usage: +# render-check.sh [options] +# +# helm release name (e.g. log-cache, gorouter) +# path to the LOCAL chart (e.g. cf-k8s-releases//helm) +# +# Options: +# --namespace default: cf-system +# --values extra values file (repeatable) [-f passthrough] +# --grep extended regex to highlight (default: 'image:|env:') +# --kind-dir clone of cloudfoundry/kind-deployment (recommended, as for +# lifecycle.sh/cats.sh); helm is taken from /bin/helm +# where kind-deployment installs it. Without it, `helm` on PATH +# is used. +# +# Why render first: `make up`/helmfile uses the PUBLISHED chart, so local chart +# edits are only exercised via a direct helm template/upgrade against the local +# dir. Templating and grepping for your new env var / image ref confirms the +# change actually lands in the manifest before you mutate the cluster. + +NS="cf-system" +GREP='image:|env:' +KIND_DIR="" +RELEASE="" +CHART_DIR="" +VALUES_ARGS=() + +positional=() +while [[ $# -gt 0 ]]; do + case "$1" in + --namespace) NS="$2"; shift 2 ;; + --values) VALUES_ARGS+=("-f" "$2"); shift 2 ;; + --grep) GREP="$2"; shift 2 ;; + --kind-dir) KIND_DIR="$2"; shift 2 ;; + -*) echo "render-check.sh: unknown arg: $1" >&2; exit 2 ;; + *) positional+=("$1"); shift ;; + esac +done + +RELEASE="${positional[0]:-}" +CHART_DIR="${positional[1]:-}" +[[ -n "$RELEASE" && -n "$CHART_DIR" ]] || { echo "render-check.sh: usage: render-check.sh [--kind-dir d] [--values f] [--grep re]" >&2; exit 2; } +[[ -d "$CHART_DIR" ]] || { echo "render-check.sh: chart dir '$CHART_DIR' not found" >&2; exit 1; } + +# Resolve helm the same way as the rest of the skill: prefer the copy +# kind-deployment installs under /bin/helm (pass --kind-dir, as for +# lifecycle.sh/cats.sh); otherwise fall back to `helm` on PATH. Both are trusted +# inputs — a user-confirmed --kind-dir clone or a real helm on PATH — so there's no +# arbitrary-binary path to allowlist here. +if [[ -n "$KIND_DIR" ]]; then + [[ -d "$KIND_DIR" ]] || { echo "render-check.sh: kind-dir '$KIND_DIR' not found" >&2; exit 1; } + HELM_BIN="$KIND_DIR/bin/helm" + [[ -x "$HELM_BIN" ]] || { echo "render-check.sh: '$HELM_BIN' not found or not executable — is '$KIND_DIR' a kind-deployment clone with helm installed (make up)?" >&2; exit 1; } +elif command -v helm >/dev/null 2>&1; then + HELM_BIN="helm" +else + echo "render-check.sh: no helm found — pass --kind-dir (uses its bin/helm) or put helm on PATH" >&2 + exit 1 +fi + +echo "render-check.sh: helm template $RELEASE $CHART_DIR -n $NS ${VALUES_ARGS[*]:-}" +echo "render-check.sh: highlighting /$GREP/" +echo "---" +# Capture the rendered manifest so we can distinguish "chart produced nothing" +# (gated off — e.g. loggregator-agent needs --set forwarderAgent.enabled=true) from +# "pattern not found in non-empty output". Both used to funnel into one error. +if ! rendered="$("$HELM_BIN" template "$RELEASE" "$CHART_DIR" -n "$NS" "${VALUES_ARGS[@]+"${VALUES_ARGS[@]}"}")"; then + echo "render-check.sh: '$HELM_BIN template' failed (see helm error above)" >&2 + exit 1 +fi +if [[ -z "${rendered//[[:space:]]/}" ]]; then + echo "render-check.sh: helm produced NO manifests — the chart likely rendered nothing" >&2 + echo "render-check.sh: (a subchart/feature gated off?). Try enabling it via --values/--set." >&2 + exit 1 +fi +printf '%s\n' "$rendered" \ + | grep -nE "$GREP" \ + || { echo "render-check.sh: pattern '/$GREP/' not found in the (non-empty) rendered output — did the change take?" >&2; exit 1; } diff --git a/skills/cf-kind-verify/scripts/verify-status.sh b/skills/cf-kind-verify/scripts/verify-status.sh new file mode 100755 index 0000000..cd02c50 --- /dev/null +++ b/skills/cf-kind-verify/scripts/verify-status.sh @@ -0,0 +1,50 @@ +#!/usr/bin/env bash +set -euo pipefail +# verify-status.sh — print per-container health for a deployment's pods. +# +# Usage: verify-status.sh [--namespace ] [--cluster ] +# --namespace default: cf-system +# +# Verify per CONTAINER, not per pod: a pod can be "Running" while a sidecar +# crash-loops. Prints, for each container of each matching pod: +# +# ready= restarts= image= +# +# restarts=0 on a branch image that ENFORCES required config fields is itself +# proof the chart wiring is correct — that's the signal the model should read. +# +# JSON parsing lives in _status_selector.py / _status_containers.py + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +NS="cf-system" +DEP="" + +while [[ $# -gt 0 ]]; do + case "$1" in + --namespace) NS="$2"; shift 2 ;; + --cluster) shift 2 ;; # accepted for symmetry; kubectl uses the current context + -*) echo "verify-status.sh: unknown arg: $1" >&2; exit 2 ;; + *) DEP="$1"; shift ;; + esac +done + +[[ -n "$DEP" ]] || { echo "verify-status.sh: usage: verify-status.sh [--namespace ]" >&2; exit 2; } + +# Read the deployment's selector matchLabels and turn them into a k1=v1,k2=v2 +# string for `kubectl get pod -l`. Exits 1 (matching the old behavior) if the +# deployment is missing or has no selector. +labels="$( + kubectl -n "$NS" get deployment "$DEP" -o json 2>/dev/null | \ + DEP="$DEP" NS="$NS" python3 "$SCRIPT_DIR/_status_selector.py" +)" + +echo "verify-status.sh: deployment/$DEP (ns=$NS) selector=$labels" + +# List every container of every matching pod as: ready/restarts/image, +# grouped under a "pod " line — per-container so a crash-looping sidecar in +# an otherwise-Running pod is visible. +kubectl -n "$NS" get pod -l "$labels" -o json 2>/dev/null | \ +python3 "$SCRIPT_DIR/_status_containers.py" + +echo "verify-status.sh: rollout ->" +kubectl -n "$NS" rollout status deployment/"$DEP" --timeout=5s 2>&1 || true From 0967206ba1292a1b0ceab7de614ff4b22cff8e7a Mon Sep 17 00:00:00 2001 From: Joris Baum Date: Thu, 17 Sep 2026 10:25:19 +0200 Subject: [PATCH 2/6] Fix setup-v action version --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 17c1c1c..7f8eeed 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -26,7 +26,7 @@ jobs: run: npm install -g @anthropic-ai/claude-code - name: Install uv - uses: astral-sh/setup-uv@v10.1 + uses: astral-sh/setup-uv@v10.1.0 - name: Run checks run: make check From 4f750a5fec568537dbcb8999053cfe96ab325286 Mon Sep 17 00:00:00 2001 From: Joris Baum Date: Thu, 17 Sep 2026 10:30:43 +0200 Subject: [PATCH 3/6] Use latest nodejs lts version --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7f8eeed..08ba259 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -20,7 +20,7 @@ jobs: - name: Set up Node.js uses: actions/setup-node@v7 with: - node-version: "22" + node-version: 24 - name: Install Claude Code CLI run: npm install -g @anthropic-ai/claude-code From 0792f839f6ca4e061f22944331864528c95c88ed Mon Sep 17 00:00:00 2001 From: Joris Baum Date: Thu, 17 Sep 2026 10:32:07 +0200 Subject: [PATCH 4/6] Fix shellcheck SC2015 --- skills/cf-kind-verify/evals/run-evals.sh | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/skills/cf-kind-verify/evals/run-evals.sh b/skills/cf-kind-verify/evals/run-evals.sh index fbeecdf..e82d2ac 100644 --- a/skills/cf-kind-verify/evals/run-evals.sh +++ b/skills/cf-kind-verify/evals/run-evals.sh @@ -63,7 +63,11 @@ while [[ $# -gt 0 ]]; do done log() { echo "==> $*"; } -vlog() { [[ "$VERBOSE" == 1 ]] && echo " $*" || true; } +vlog() { + if [[ "$VERBOSE" == 1 ]]; then + echo " $*" + fi +} die() { echo "run-evals.sh: $*" >&2; exit 1; } command -v claude >/dev/null 2>&1 || die "claude CLI not found on PATH" From e1d6885d84810ba76771608f006276a2929f0cef Mon Sep 17 00:00:00 2001 From: Joris Baum Date: Thu, 17 Sep 2026 10:38:32 +0200 Subject: [PATCH 5/6] Use newest version of actions/checkout --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 08ba259..0b1acfb 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -12,7 +12,7 @@ jobs: check: runs-on: ubuntu-latest steps: - - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 + - uses: actions/checkout@v7.0.1 - name: Install shellcheck run: sudo apt-get update && sudo apt-get install -y shellcheck From 63006e204c46f09fea7a5dee755bfeef51bce124 Mon Sep 17 00:00:00 2001 From: Joris Baum Date: Thu, 17 Sep 2026 10:51:16 +0200 Subject: [PATCH 6/6] Pin linting tools with pyproject.toml --- .github/workflows/ci.yml | 3 ++ Makefile | 4 +-- pyproject.toml | 10 ++++++ uv.lock | 76 ++++++++++++++++++++++++++++++++++++++++ 4 files changed, 91 insertions(+), 2 deletions(-) create mode 100644 pyproject.toml create mode 100644 uv.lock diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0b1acfb..c3fc1ff 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -27,6 +27,9 @@ jobs: - name: Install uv uses: astral-sh/setup-uv@v10.1.0 + with: + enable-cache: true + cache-dependency-glob: uv.lock - name: Run checks run: make check diff --git a/Makefile b/Makefile index fb501d2..40f87a6 100644 --- a/Makefile +++ b/Makefile @@ -46,12 +46,12 @@ json: ## ruff: lint Python helper scripts ruff: @echo ">> ruff ($(words $(PY)) files)" - @if [ -n "$(strip $(PY))" ]; then uvx ruff@0.16.7 check $(PY); else echo "(no python files)"; fi + @if [ -n "$(strip $(PY))" ]; then uv run --group dev ruff check $(PY); else echo "(no python files)"; fi ## ty: type-check Python helper scripts ty: @echo ">> ty ($(words $(PY)) files)" - @if [ -n "$(strip $(PY))" ]; then uvx ty@0.0.80 check $(PY); else echo "(no python files)"; fi + @if [ -n "$(strip $(PY))" ]; then uv run --group dev ty check $(PY); else echo "(no python files)"; fi ## help: list targets help: diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..26fa9c6 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,10 @@ +[project] +name = "cloudfoundry-skills" +version = "0.0.0" +requires-python = ">=3.11" + +[dependency-groups] +dev = [ + "ruff==0.16.7", + "ty==0.0.80", +] diff --git a/uv.lock b/uv.lock new file mode 100644 index 0000000..f70e96b --- /dev/null +++ b/uv.lock @@ -0,0 +1,76 @@ +version = 1 +revision = 3 +requires-python = ">=3.11" + +[options] +exclude-newer = "0001-01-01T00:00:00Z" # This has no effect and is included for backwards compatibility when using relative exclude-newer values. +exclude-newer-span = "P2D" + +[[package]] +name = "cloudfoundry-skills" +version = "0.0.0" +source = { virtual = "." } + +[package.dev-dependencies] +dev = [ + { name = "ruff" }, + { name = "ty" }, +] + +[package.metadata] + +[package.metadata.requires-dev] +dev = [ + { name = "ruff", specifier = "==0.16.7" }, + { name = "ty", specifier = "==0.0.80" }, +] + +[[package]] +name = "ruff" +version = "0.16.7" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/82/bb/5a449b9162e49b139d72f61672bd3ac1d790221f796d3304e2241fff4c58/ruff-0.16.7.tar.gz", hash = "sha256:5f71d004ac1263b22fa39462ac5ae618a4b77d58981af2cc79bf79a29c12b1a6", size = 4924184, upload-time = "2026-09-10T18:04:06.336Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e3/b2/c80aeeb7f9e469c0d63a85d2f1ab6e1ebfbe10ea7a8d2438b7e09e3ff09e/ruff-0.16.7-py3-none-linux_armv6l.whl", hash = "sha256:727307773e7c7f9181d3ed3a2484186e56c1fa1874255911c74585eb2c7c19f9", size = 10048917, upload-time = "2026-09-10T18:03:30.28Z" }, + { url = "https://files.pythonhosted.org/packages/7b/96/20bb7bcae008004df52afcb7ac83432d4a467f2c17b672fe46d26be231c5/ruff-0.16.7-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:9d61c258deabf58f34c67bd4bb4d939c7f2e6b5f0e59c1cdd1cf771b11cde929", size = 10242929, upload-time = "2026-09-10T18:03:32.706Z" }, + { url = "https://files.pythonhosted.org/packages/90/b2/f184b0d5abec02db69cfd7e49b688ae0237554528ca777136c613bf36bee/ruff-0.16.7-py3-none-macosx_11_0_arm64.whl", hash = "sha256:7ab81118df8945e0193d0240712aa4496573595b75185c3636ed825592a0f728", size = 9847245, upload-time = "2026-09-10T18:03:34.509Z" }, + { url = "https://files.pythonhosted.org/packages/eb/2d/db1633a641866ed801e34cc6b60ef236c5e16f9b2124ab1d49cc24a5fe4f/ruff-0.16.7-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4c196c968874fc8019da8e7163de7a1a370f111e2309b4b7dfea0fce950198d0", size = 9961780, upload-time = "2026-09-10T18:03:36.618Z" }, + { url = "https://files.pythonhosted.org/packages/4d/98/edea21e1a3e38dbbc3bf6bb068b863b3b06184cf8533a4c7dbbe208a89d5/ruff-0.16.7-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:ac8c3bd0a7e10ad31e6ce51e7a99f3cb772e69aecdd6b9ea7e99b362f62a62c0", size = 9866337, upload-time = "2026-09-10T18:03:38.805Z" }, + { url = "https://files.pythonhosted.org/packages/0b/11/a15e60d4c87b214646f116ca9d204475bf993ee1047459bc9a360fd4d6d1/ruff-0.16.7-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:398d3988edde000b5c75dc1b3f584708da9bc990de069c18909142580fec1af9", size = 10562512, upload-time = "2026-09-10T18:03:40.71Z" }, + { url = "https://files.pythonhosted.org/packages/29/42/eaff4c9b6d0c7cdf56df313a17e89ae854f5bbc0b0c8f9cce19be0ab7a8f/ruff-0.16.7-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ce05b62b770a8217c4646a9c4139fca00efe8fe5d71f87df2b243ff20d4584d1", size = 11302938, upload-time = "2026-09-10T18:03:42.607Z" }, + { url = "https://files.pythonhosted.org/packages/5d/43/c75aa59a4ec181fe2ec06cab30e198c1c6d107229a9f008ae3a7c16cabd8/ruff-0.16.7-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:af1b576fddb9d9ef2ececfb5fadcd6a624b25070ed85e3cfcfe449fc3ff6a7b9", size = 10840857, upload-time = "2026-09-10T18:03:44.604Z" }, + { url = "https://files.pythonhosted.org/packages/21/33/81f3da371942ea031105ba679d8d6e28ec1660ccd690a45f42d381161356/ruff-0.16.7-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9ce7f8f22df67c93ed96c717f9128eadb797144ac2bad475cf536f31d6100c55", size = 10370001, upload-time = "2026-09-10T18:03:46.706Z" }, + { url = "https://files.pythonhosted.org/packages/fa/0b/6345fb4dbf6dd0ed1cfe5d18391dc9c3f59cc81622a7b0a65b84b3e730ba/ruff-0.16.7-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:06d0e93d04f392996435ebd600c153f65b47d73fbec2415aa99c5ee5756b3a5f", size = 10548735, upload-time = "2026-09-10T18:03:48.658Z" }, + { url = "https://files.pythonhosted.org/packages/3f/4d/c5576adf511f92a328e5569dda190ecdd430da51f1a649f3a4a2fd73e21e/ruff-0.16.7-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:142151a5e7b93c1b11111337142f89dd2fbfee92161225c99a97222f22e32656", size = 10108496, upload-time = "2026-09-10T18:03:50.563Z" }, + { url = "https://files.pythonhosted.org/packages/ff/8c/667d83c16199a17a56adc6b0bd4c3beb5b767a2babcd16a56f76f9be7fd6/ruff-0.16.7-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:e6651f97a342d8b35d54d8991544ca22169b86dc54111cb604666940c431b750", size = 9860136, upload-time = "2026-09-10T18:03:52.621Z" }, + { url = "https://files.pythonhosted.org/packages/99/75/78d401106731999a1dd20cc5a6961e37e1eb9397a3b589f73f3a5ce146a3/ruff-0.16.7-py3-none-musllinux_1_2_i686.whl", hash = "sha256:ef140c6eb935fa9a84c9c607dfb2cb1b85843c192e79265b0c54f35f557ea8e5", size = 10286290, upload-time = "2026-09-10T18:03:55.207Z" }, + { url = "https://files.pythonhosted.org/packages/68/49/56f9c3a8b755df93a0ad318b2147bf4ef5dae9a7e5ec61c460109c67957f/ruff-0.16.7-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:53e39506a730fadeee0d998ed5946f30671f0db240c6c7c73bdabbe33604bb6f", size = 10745048, upload-time = "2026-09-10T18:03:57.299Z" }, + { url = "https://files.pythonhosted.org/packages/5f/ea/7f9b938a63ece4bec677ad7f9f7fa02df3383db1949ed93a382441c09a87/ruff-0.16.7-py3-none-win32.whl", hash = "sha256:2ea3470fcebcbc5df2fb0c6f3b90333fa9084c534e0111c038fa4a6ab9f1c4b7", size = 10059082, upload-time = "2026-09-10T18:03:59.632Z" }, + { url = "https://files.pythonhosted.org/packages/39/11/480a6973a927aa653e1cead6a6416008640e03a99d05b34c0434b8c6c366/ruff-0.16.7-py3-none-win_amd64.whl", hash = "sha256:7ac26aca826e9e21d0f1cb25b54ac660760a9fdd094d3e4df9848232be98cfc6", size = 10593368, upload-time = "2026-09-10T18:04:01.999Z" }, + { url = "https://files.pythonhosted.org/packages/8b/4b/51327018d056f0dad2c2238f26d1fb0f53707a9d91b75dea6d1b3039f136/ruff-0.16.7-py3-none-win_arm64.whl", hash = "sha256:aab7f39e2c9df6c596216070f98eef1207b94f8516cca20c808826974971855b", size = 10412401, upload-time = "2026-09-10T18:04:04.098Z" }, +] + +[[package]] +name = "ty" +version = "0.0.80" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/54/b0/6d1b10e0d422736a3c439e487c950ae785f401d71ff879d5be68bccb6d90/ty-0.0.80.tar.gz", hash = "sha256:fe86bc91327e45ff5e3593b7e306e7f57a44bc0608f38d647b8d97bd99c96013", size = 7183998, upload-time = "2026-09-09T21:18:47.547Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/29/c8/3c93195eca282936ebb574d0ba98843e4b147ee324006f88464777012a92/ty-0.0.80-py3-none-linux_armv6l.whl", hash = "sha256:738d1cfca466c577aea24547c348629c00c63548cf7da2c46b497b3840f85dee", size = 13606509, upload-time = "2026-09-09T21:18:10.476Z" }, + { url = "https://files.pythonhosted.org/packages/e7/48/bbb47f7001c97262a5109bcd92a45bcc8a2fbb21e994c214a9897630bfb7/ty-0.0.80-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:56060164bb8ee43770fa367524fbdba2611fc90f8923a02cc91f80eb37d5a1b0", size = 13203812, upload-time = "2026-09-09T21:18:12.796Z" }, + { url = "https://files.pythonhosted.org/packages/db/8d/fd141160567047b742e466be8ed09a4c4ed80fa045d37f946eb4f4532fee/ty-0.0.80-py3-none-macosx_11_0_arm64.whl", hash = "sha256:da4062e0fbf3923d9b71688157c5753394f243348f00732e9edbb27498397169", size = 13021896, upload-time = "2026-09-09T21:18:15.186Z" }, + { url = "https://files.pythonhosted.org/packages/1e/10/b4177faf9e71bc37bc4d08f512042269660a1ce0ec457c48f96e51fc91d7/ty-0.0.80-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a9eb94b2659f506a3a07a9ea1415d5c18aaa1e554adc1612614d617a0ed320e1", size = 13083881, upload-time = "2026-09-09T21:18:17.616Z" }, + { url = "https://files.pythonhosted.org/packages/7f/c2/192d48b9a9acbbe030fc9a4bd73af75671a5566e731949f77e912a68ce68/ty-0.0.80-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:e2f5e39a87da48c1af1a3b3135f02be7a83a93da30b0d6a5f5d27806e7c747ce", size = 13354874, upload-time = "2026-09-09T21:18:19.744Z" }, + { url = "https://files.pythonhosted.org/packages/0a/5b/736dfd31efd98bed9dbe018ff91676e5ea83485b1e24afb635fd0247728f/ty-0.0.80-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4e00149e7779c6b98f3ba12e10a631a861bd7ca1a42bbf064e2677ada7d2c0c2", size = 14204805, upload-time = "2026-09-09T21:18:21.999Z" }, + { url = "https://files.pythonhosted.org/packages/fd/6f/557c726cb53401998e3589bb632c963a6887a70240e0a4386024f3731c81/ty-0.0.80-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a377268f359fb6e7a2cc9026a09223c764f58d020dec8e6baeaba9caba6ff829", size = 14630014, upload-time = "2026-09-09T21:18:24Z" }, + { url = "https://files.pythonhosted.org/packages/e9/19/7a4b18fe27f6b6bd4ffa56b67c8b09ecb76bb4312d1c64b73f65417b4dbd/ty-0.0.80-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:4511dbf1b266b9ce5b73e9b28d94468096c2b6da00e2fe205168c32d3b352ca7", size = 14326946, upload-time = "2026-09-09T21:18:26.679Z" }, + { url = "https://files.pythonhosted.org/packages/f8/95/16dd90805fc7e53ec54ae9fdb1a8b74753fb159a55a170f60f49b9417824/ty-0.0.80-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fe95feffa7156800c6f804195acb9fb5846a39671c7192c30fff23351aaf7c31", size = 13705988, upload-time = "2026-09-09T21:18:28.681Z" }, + { url = "https://files.pythonhosted.org/packages/b1/09/4e87992c23ab8a742c7d4914ac5fe66fb9301c8464a69fd2ecbc0be3fbcd/ty-0.0.80-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:ba26b39f06bc8c3c2c5acd3a147239482b36620cdbca1b9d02e915294854f2cc", size = 14233729, upload-time = "2026-09-09T21:18:30.961Z" }, + { url = "https://files.pythonhosted.org/packages/36/e9/b8c11fda8e66a1d1cc1a7160ed719a33f4dc0107b1cef6a14e2673965763/ty-0.0.80-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:2e6167320888c115a6fbe69893b63fd1c848f9121a454e75e5f612674528e3f3", size = 13173523, upload-time = "2026-09-09T21:18:33.266Z" }, + { url = "https://files.pythonhosted.org/packages/73/a7/512083fa540c5be1ac8642658f03bfb5bfa6fd168ea4f55e3c0aabe04166/ty-0.0.80-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:7aecb62b4de70b479eab07d1779ea66da26dd8b068f50a9330867157312f103c", size = 13373014, upload-time = "2026-09-09T21:18:35.339Z" }, + { url = "https://files.pythonhosted.org/packages/5e/63/cd2f0ca81fd9b8cafef93bbcf022bf636bc4de8ee7465c6aafeec1d4d52b/ty-0.0.80-py3-none-musllinux_1_2_i686.whl", hash = "sha256:4fed06adacd16b7e2d722f37449021b1419598d0440769701c0f4827faddde63", size = 13667827, upload-time = "2026-09-09T21:18:37.241Z" }, + { url = "https://files.pythonhosted.org/packages/0d/5d/ebdfdcb2dc099ae74dc4b75511eef0dd398b2fa27006c543543974b4567f/ty-0.0.80-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:09329af6303ce611ec2cfffc995dc1ddf76c9f47194fc1a4d64a984759c25f5b", size = 13963866, upload-time = "2026-09-09T21:18:39.386Z" }, + { url = "https://files.pythonhosted.org/packages/c4/f6/c286d5e1b560cfd16d6e91fdab683718aacf09fc88938ca06dcc9d1c8502/ty-0.0.80-py3-none-win32.whl", hash = "sha256:a81b3b512f7b4c68e42fbd1835633de658809666e9aafd5defa52bbc5197698d", size = 12881806, upload-time = "2026-09-09T21:18:41.507Z" }, + { url = "https://files.pythonhosted.org/packages/fd/c8/3b3a8ac16d47a23ff34bf57c0d99a491860748aea96e0e886b4bfed54f9b/ty-0.0.80-py3-none-win_amd64.whl", hash = "sha256:8043f99878a2ae434781cb881c8a3840dff70c4a5b60bdc5ae84251f35ee063f", size = 13528883, upload-time = "2026-09-09T21:18:43.687Z" }, + { url = "https://files.pythonhosted.org/packages/2e/71/a6c697930fca76596d7d17f8853120f41a298fd9ea632c901cfca8ca869b/ty-0.0.80-py3-none-win_arm64.whl", hash = "sha256:e277ef034331da5319efc839c064968d24f35715b71c70369cf97d36fe4dcbdb", size = 13370135, upload-time = "2026-09-09T21:18:45.758Z" }, +]