From 1c89c40130506a0f95b46b8667ccc436b3b19b93 Mon Sep 17 00:00:00 2001 From: Thomas Stetson Date: Thu, 20 Aug 2026 10:53:54 -0400 Subject: [PATCH 1/3] fix(deps): bump control-plane and add website fixture validation (FLPATH-4770) Bump the control-plane dependency so the CLI's generated types include the `Resources` field required by the multi-resource catalog item schema. - Add contract tests (TC-U154, TC-U155) validating documented tutorial YAMLs serialize correctly to the API - Add testdata/website/ fixtures mirroring the Getting Started examples - Add hack/check-website-fixtures.sh + CI workflow to detect drift between local fixtures and upstream docs - Add argument validation (exit 2) and curl retry/timeout resilience to check-website-fixtures.sh - Update README.md and CLAUDE.md examples to use multi-resource schema (spec.resources, user_values[].resource) Co-Authored-By: Cursor AI Signed-off-by: Thomas Stetson --- .github/workflows/check-website-fixtures.yaml | 18 +++ CLAUDE.md | 10 +- Makefile | 5 +- README.md | 62 +++++---- go.mod | 2 +- go.sum | 4 +- hack/check-website-fixtures.sh | 119 ++++++++++++++++ internal/commands/contract_test.go | 130 ++++++++++++++++++ testdata/website/my-vm.yaml | 14 ++ testdata/website/small-vm.yaml | 38 +++++ 10 files changed, 373 insertions(+), 29 deletions(-) create mode 100644 .github/workflows/check-website-fixtures.yaml create mode 100755 hack/check-website-fixtures.sh create mode 100644 internal/commands/contract_test.go create mode 100644 testdata/website/my-vm.yaml create mode 100644 testdata/website/small-vm.yaml diff --git a/.github/workflows/check-website-fixtures.yaml b/.github/workflows/check-website-fixtures.yaml new file mode 100644 index 0000000..8684b38 --- /dev/null +++ b/.github/workflows/check-website-fixtures.yaml @@ -0,0 +1,18 @@ +name: Check website fixtures + +on: + push: + branches: [main] + pull_request: + branches: [main] + schedule: + - cron: "0 8 * * 1" # Weekly Monday 08:00 UTC + +jobs: + check-fixtures: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Verify testdata/website/ matches upstream docs + run: hack/check-website-fixtures.sh diff --git a/CLAUDE.md b/CLAUDE.md index 276d2f5..f82292f 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -42,6 +42,9 @@ make tidy # Run E2E tests (requires live DCM stack) make test-e2e + +# Check website fixture drift +make check-fixtures ``` ## Architecture @@ -82,6 +85,11 @@ make test-e2e - **test/e2e/**: E2E tests with `e2e` build tag (`//go:build e2e`) +- **testdata/website/**: YAML fixtures mirroring the Getting Started tutorial examples from `dcm-project.github.io`. Used by contract tests to validate CLI serialization of documented schemas. + +- **hack/**: Development scripts + - `check-website-fixtures.sh`: Detects drift between local `testdata/website/` fixtures and upstream docs. Supports `--update` to refresh fixtures. Exit codes: 0 success, 1 drift/runtime error, 2 usage error. + - **tools.go**: Build tool dependencies (ginkgo) ## Testing @@ -102,7 +110,7 @@ E2E tests live under `test/e2e/` and use the `e2e` build tag (`//go:build e2e`). 4. **Output formatting**: All commands support `--output/-o` flag with `table` (default), `json`, and `yaml` formats. -5. **Input files**: Resource creation and updates use `--from-file` flag accepting YAML or JSON files. +5. **Input files**: Resource creation and updates use `--from-file` flag accepting YAML or JSON files. Catalog items use the multi-resource schema (`spec.resources[]` with named resources, each containing `service_type` and `fields`). Catalog instances reference resources via `user_values[].resource`. 6. **Error handling**: API errors follow RFC 7807 Problem Details format. Exit code 0 for success, 1 for runtime errors, 2 for usage errors. diff --git a/Makefile b/Makefile index ed9ae6b..b347c48 100644 --- a/Makefile +++ b/Makefile @@ -6,7 +6,7 @@ LDFLAGS = -X github.com/dcm-project/cli/internal/version.Version=$(VERSION) \ -X github.com/dcm-project/cli/internal/version.Commit=$(COMMIT) \ -X github.com/dcm-project/cli/internal/version.BuildTime=$(BUILD_TIME) -.PHONY: build test test-e2e fmt vet lint clean tidy +.PHONY: build test test-e2e fmt vet lint clean tidy check-fixtures build: tidy go build -ldflags "$(LDFLAGS)" -o bin/dcm ./cmd/dcm @@ -31,3 +31,6 @@ clean: tidy: go mod tidy + +check-fixtures: + hack/check-website-fixtures.sh diff --git a/README.md b/README.md index 77671e6..265fd9d 100644 --- a/README.md +++ b/README.md @@ -422,26 +422,33 @@ dcm catalog item create --from-file item.yaml dcm catalog item create --from-file item.yaml --id my-catalog-item ``` -Catalog item file format: +Catalog item file format (multi-resource schema): ```yaml api_version: v1alpha1 -display_name: "Small Container" +display_name: "Small VM" spec: - service_type: container - fields: - - path: spec.replicas - display_name: "Replica Count" - editable: true - default: "1" - validation_schema: - type: integer - minimum: 1 - maximum: 10 - - path: spec.container.image - display_name: "Container Image" - editable: true -``` + resources: + - name: main + service_type: vm + fields: + - path: metadata + editable: true + - path: vcpu.count + display_name: "CPU Count" + editable: true + default: 2 + validation_schema: + type: integer + minimum: 1 + maximum: 4 + - path: memory.size + display_name: "Memory (GB)" + editable: false + default: "2GB" +``` + +Each catalog item defines one or more named resources under `spec.resources`. Each resource specifies its `service_type` and the `fields` available for customization. Example output (table): @@ -505,19 +512,26 @@ Create a new catalog item instance. dcm catalog instance create --from-file instance.yaml ``` -Instance file format: +Instance file format (multi-resource schema): ```yaml api_version: v1alpha1 -display_name: "My App Instance" +display_name: "My Dev VM" spec: - catalog_item_id: my-catalog-item + catalog_item_id: small-vm user_values: - - path: spec.replicas - value: "3" - - path: spec.container.image - value: "nginx:latest" -``` + - resource: main + path: metadata + value: + name: "small-vm" + labels: + env: "dev" + - resource: main + path: vcpu.count + value: 1 +``` + +Each `user_values` entry includes a `resource` field that identifies which named resource (from the catalog item's `spec.resources`) the value applies to. Example output (table): diff --git a/go.mod b/go.mod index c2d0b33..3bd1d84 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,7 @@ go 1.25.5 require ( github.com/coreos/go-oidc/v3 v3.20.0 - github.com/dcm-project/control-plane v0.0.0-20260617094433-e4374fc25292 + github.com/dcm-project/control-plane v0.0.0-20260814135831-7c173134e932 github.com/onsi/ginkgo/v2 v2.29.0 github.com/onsi/gomega v1.41.0 github.com/spf13/cobra v1.10.2 diff --git a/go.sum b/go.sum index f9639f9..f31720e 100644 --- a/go.sum +++ b/go.sum @@ -14,8 +14,8 @@ github.com/danieljoos/wincred v1.2.2/go.mod h1:w7w4Utbrz8lqeMbDAK0lkNJUv5sAOkFi7 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/dcm-project/control-plane v0.0.0-20260617094433-e4374fc25292 h1:VnUyqtBqLrXTC1ClDSogbvVGDWvikhvmyxP5Znw1yX4= -github.com/dcm-project/control-plane v0.0.0-20260617094433-e4374fc25292/go.mod h1:dYV62wU2z+H1XUBdGoTMdh0V5j8AyJc4Lk55TLgPSBw= +github.com/dcm-project/control-plane v0.0.0-20260814135831-7c173134e932 h1:dLCvaQn2vZ/0+ByjhGcYoxLzCimFyNNm0kNMgBt/hQg= +github.com/dcm-project/control-plane v0.0.0-20260814135831-7c173134e932/go.mod h1:oLQKa1vW81Dmpit5iGjU0EQaRARadjrJ/BmCadwqir4= github.com/dlclark/regexp2 v1.11.5 h1:Q/sSnsKerHeCkc/jSTNq1oCm7KiVgUMZRDUoRu0JQZQ= github.com/dlclark/regexp2 v1.11.5/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8= github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= diff --git a/hack/check-website-fixtures.sh b/hack/check-website-fixtures.sh new file mode 100755 index 0000000..26b55aa --- /dev/null +++ b/hack/check-website-fixtures.sh @@ -0,0 +1,119 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Verifies that testdata/website/ fixtures match the YAML examples +# published in the dcm-project.github.io Getting Started guides. +# +# Usage: hack/check-website-fixtures.sh [--update] +# --update Overwrite local fixtures with upstream content (for refresh) +# +# Exit codes: 0 = success, 1 = runtime/drift error, 2 = usage error + +usage() { + echo "Usage: ${0##*/} [--update]" + echo " --update Overwrite local fixtures with upstream content" +} + +REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +FIXTURE_DIR="${REPO_ROOT}/testdata/website" +WEBSITE_REPO="dcm-project/dcm-project.github.io" +WEBSITE_BRANCH="main" +BASE_URL="https://raw.githubusercontent.com/${WEBSITE_REPO}/${WEBSITE_BRANCH}/content/docs/getting-started" + +UPDATE=false +case "${1:-}" in + "") ;; + --update) UPDATE=true ;; + *) + echo "Error: unknown argument: $1" >&2 + usage >&2 + exit 2 + ;; +esac +if [[ $# -gt 1 ]]; then + echo "Error: too many arguments" >&2 + usage >&2 + exit 2 +fi + +# Maps local fixture filenames to their source markdown files. +declare -A FIXTURE_SOURCES=( + ["small-vm.yaml"]="create-small-vm-catalog-item.md" + ["my-vm.yaml"]="create-small-vm-instance.md" +) + +extract_yaml_block() { + # Extracts the first ```yaml ... ``` fenced block from markdown on stdin. + awk ' + /^```yaml/ { capture=1; next } + /^```/ && capture { capture=0; next } + capture { print } + ' +} + +errors=0 +fetch_failures=0 + +for fixture in "${!FIXTURE_SOURCES[@]}"; do + source_file="${FIXTURE_SOURCES[$fixture]}" + local_path="${FIXTURE_DIR}/${fixture}" + url="${BASE_URL}/${source_file}" + + echo "Checking ${fixture} ← ${source_file}" + + if ! markdown=$(curl -sf --connect-timeout 10 --max-time 30 --retry 3 --retry-delay 2 --retry-all-errors "${url}"); then + echo " WARNING: Could not fetch ${url} (network error, 404, or rate limiting)" + fetch_failures=$((fetch_failures + 1)) + continue + fi + + upstream_yaml=$(echo "${markdown}" | extract_yaml_block) + if [[ -z "${upstream_yaml}" ]]; then + echo " ERROR: No YAML code block found in ${source_file}" + errors=$((errors + 1)) + continue + fi + + if [[ "${UPDATE}" == "true" ]]; then + echo "${upstream_yaml}" > "${local_path}" + echo " Updated ${local_path}" + continue + fi + + if [[ ! -f "${local_path}" ]]; then + echo " ERROR: Local fixture missing: ${local_path}" + echo " Run: hack/check-website-fixtures.sh --update" + errors=$((errors + 1)) + continue + fi + + if ! diff_output=$(diff -u "${local_path}" <(echo "${upstream_yaml}")); then + echo " DRIFT DETECTED: ${fixture} differs from upstream" + echo "${diff_output}" + echo "" + echo " To update: hack/check-website-fixtures.sh --update" + errors=$((errors + 1)) + else + echo " OK" + fi +done + +if [[ ${errors} -gt 0 ]]; then + echo "" + echo "FAIL: ${errors} fixture(s) out of sync with ${WEBSITE_REPO}" + exit 1 +fi + +if [[ ${fetch_failures} -gt 0 ]]; then + echo "" + if [[ "${GITHUB_EVENT_NAME:-}" == "schedule" ]]; then + echo "FAIL: ${fetch_failures} fixture(s) could not be fetched (scheduled run — treating as error)" + exit 1 + fi + echo "WARNING: ${fetch_failures} fixture(s) could not be fetched (skipped; network issue or rate limiting)" + echo "Re-run or check manually: hack/check-website-fixtures.sh" + exit 0 +fi + +echo "" +echo "All fixtures match upstream." diff --git a/internal/commands/contract_test.go b/internal/commands/contract_test.go new file mode 100644 index 0000000..6597ee8 --- /dev/null +++ b/internal/commands/contract_test.go @@ -0,0 +1,130 @@ +package commands_test + +import ( + "bytes" + "encoding/json" + "net/http" + "net/http/httptest" + "path/filepath" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/dcm-project/cli/internal/commands" +) + +func docsFixturePath(name string) string { + return filepath.Join("..", "..", "testdata", "website", name) +} + +var _ = Describe("Documentation Contract", func() { + var ( + server *httptest.Server + outBuf *bytes.Buffer + errBuf *bytes.Buffer + ) + + BeforeEach(func() { + clearDCMEnvVars() + }) + + AfterEach(func() { + if server != nil { + server.Close() + server = nil + } + }) + + executeCommand := func(args ...string) error { + cmd := commands.NewRootCommand() + outBuf = new(bytes.Buffer) + errBuf = new(bytes.Buffer) + cmd.SetOut(outBuf) + cmd.SetErr(errBuf) + + fullArgs := []string{ + "--config", nonexistentConfigPath(), + } + if server != nil { + fullArgs = append(fullArgs, "--control-plane-url", server.URL) + } + fullArgs = append(fullArgs, args...) + cmd.SetArgs(fullArgs) + + return cmd.Execute() + } + + Describe("Catalog Item YAML (small-vm.yaml)", func() { + // TC-U154: Documented catalog item YAML preserves spec.resources through CLI serialization + It("TC-U154: should serialize spec.resources with all required fields to the API", func() { + var receivedBody map[string]any + + server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + Expect(r.Method).To(Equal(http.MethodPost)) + Expect(r.URL.Path).To(Equal("/api/v1alpha1/catalog-items")) + + Expect(json.NewDecoder(r.Body).Decode(&receivedBody)).To(Succeed()) + + writeJSONResponse(w, http.StatusCreated, sampleCatalogItemResponse()) + })) + + err := executeCommand("catalog", "item", "create", "--from-file", docsFixturePath("small-vm.yaml")) + Expect(err).NotTo(HaveOccurred()) + + Expect(receivedBody).To(HaveKey("spec")) + spec, ok := receivedBody["spec"].(map[string]any) + Expect(ok).To(BeTrue(), "spec should be an object") + + Expect(spec).To(HaveKey("resources"), "spec.resources must not be silently dropped") + resources, ok := spec["resources"].([]any) + Expect(ok).To(BeTrue(), "spec.resources should be an array") + Expect(resources).To(HaveLen(1)) + + res0, ok := resources[0].(map[string]any) + Expect(ok).To(BeTrue()) + Expect(res0["name"]).To(Equal("main")) + Expect(res0["service_type"]).To(Equal("vm")) + Expect(res0).To(HaveKey("fields")) + + fields, ok := res0["fields"].([]any) + Expect(ok).To(BeTrue()) + Expect(len(fields)).To(BeNumerically(">=", 5), "all documented fields should be preserved") + }) + }) + + Describe("Catalog Item Instance YAML (my-vm.yaml)", func() { + // TC-U155: Documented instance YAML preserves user_values[].resource through CLI serialization + It("TC-U155: should serialize user_values with the resource field to the API", func() { + var receivedBody map[string]any + + server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + Expect(r.Method).To(Equal(http.MethodPost)) + Expect(r.URL.Path).To(Equal("/api/v1alpha1/catalog-item-instances")) + + Expect(json.NewDecoder(r.Body).Decode(&receivedBody)).To(Succeed()) + + writeJSONResponse(w, http.StatusCreated, sampleInstanceResponse()) + })) + + err := executeCommand("catalog", "instance", "create", "--from-file", docsFixturePath("my-vm.yaml")) + Expect(err).NotTo(HaveOccurred()) + + Expect(receivedBody).To(HaveKey("spec")) + spec, ok := receivedBody["spec"].(map[string]any) + Expect(ok).To(BeTrue(), "spec should be an object") + + Expect(spec).To(HaveKey("user_values"), "spec.user_values must not be silently dropped") + userValues, ok := spec["user_values"].([]any) + Expect(ok).To(BeTrue(), "spec.user_values should be an array") + Expect(userValues).To(HaveLen(2)) + + for i, uv := range userValues { + uvMap, ok := uv.(map[string]any) + Expect(ok).To(BeTrue()) + Expect(uvMap).To(HaveKey("resource"), + "user_values[%d].resource must not be silently dropped", i) + Expect(uvMap["resource"]).To(Equal("main")) + } + }) + }) +}) diff --git a/testdata/website/my-vm.yaml b/testdata/website/my-vm.yaml new file mode 100644 index 0000000..6ce5d6c --- /dev/null +++ b/testdata/website/my-vm.yaml @@ -0,0 +1,14 @@ +api_version: v1alpha1 +display_name: "My Dev VM" +spec: + catalog_item_id: small-vm + user_values: + - resource: main + path: metadata + value: + name: "small-vm" + labels: + env: "dev" + - resource: main + path: vcpu.count + value: 1 diff --git a/testdata/website/small-vm.yaml b/testdata/website/small-vm.yaml new file mode 100644 index 0000000..61075f6 --- /dev/null +++ b/testdata/website/small-vm.yaml @@ -0,0 +1,38 @@ +api_version: v1alpha1 +display_name: "Small VM" +spec: + resources: + - name: main + service_type: vm + fields: + - path: metadata + editable: true + - path: vcpu.count + display_name: "CPU Count" + editable: true + default: 2 + validation_schema: + type: integer + minimum: 1 + maximum: 4 + - path: memory.size + display_name: "Memory (GB)" + editable: false + default: "2GB" + - path: storage.disks + display_name: "Storage (GB)" + editable: false + default: + - name: boot + capacity: "20GB" + validation_schema: + type: array + - path: guest_os.type + display_name: "Guest OS" + editable: true + default: "rhel-10" + validation_schema: + type: string + enum: + - rhel-9 + - rhel-10 From f34b21c2dafc45fc821818e65ff430c281745ca3 Mon Sep 17 00:00:00 2001 From: Thomas Stetson Date: Thu, 3 Sep 2026 11:15:55 -0400 Subject: [PATCH 2/3] test(contract): use typed control-plane structs in website fixture tests Also tighten the `check-website-fixtures.yaml` workflow so it only runs on PRs touching `testdata/website/**` or `hack/check-website-fixtures.sh`, and clarify that the scheduled run hard-fails on fetch errors. Signed-off-by: Thomas Stetson --- .github/workflows/check-website-fixtures.yaml | 7 +-- internal/commands/contract_test.go | 46 ++++++------------- 2 files changed, 18 insertions(+), 35 deletions(-) diff --git a/.github/workflows/check-website-fixtures.yaml b/.github/workflows/check-website-fixtures.yaml index 8684b38..e7ead5e 100644 --- a/.github/workflows/check-website-fixtures.yaml +++ b/.github/workflows/check-website-fixtures.yaml @@ -1,12 +1,13 @@ name: Check website fixtures on: - push: - branches: [main] pull_request: branches: [main] + paths: + - testdata/website/** + - hack/check-website-fixtures.sh schedule: - - cron: "0 8 * * 1" # Weekly Monday 08:00 UTC + - cron: "0 8 * * 1" # Weekly Monday 08:00 UTC — hard-fails on fetch errors jobs: check-fixtures: diff --git a/internal/commands/contract_test.go b/internal/commands/contract_test.go index 6597ee8..d6dd47f 100644 --- a/internal/commands/contract_test.go +++ b/internal/commands/contract_test.go @@ -10,6 +10,8 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + v1alpha1 "github.com/dcm-project/control-plane/api/catalog/v1alpha1" + "github.com/dcm-project/cli/internal/commands" ) @@ -57,7 +59,7 @@ var _ = Describe("Documentation Contract", func() { Describe("Catalog Item YAML (small-vm.yaml)", func() { // TC-U154: Documented catalog item YAML preserves spec.resources through CLI serialization It("TC-U154: should serialize spec.resources with all required fields to the API", func() { - var receivedBody map[string]any + var receivedBody v1alpha1.CatalogItem server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { Expect(r.Method).To(Equal(http.MethodPost)) @@ -71,31 +73,21 @@ var _ = Describe("Documentation Contract", func() { err := executeCommand("catalog", "item", "create", "--from-file", docsFixturePath("small-vm.yaml")) Expect(err).NotTo(HaveOccurred()) - Expect(receivedBody).To(HaveKey("spec")) - spec, ok := receivedBody["spec"].(map[string]any) - Expect(ok).To(BeTrue(), "spec should be an object") - - Expect(spec).To(HaveKey("resources"), "spec.resources must not be silently dropped") - resources, ok := spec["resources"].([]any) - Expect(ok).To(BeTrue(), "spec.resources should be an array") - Expect(resources).To(HaveLen(1)) + Expect(receivedBody.Spec).NotTo(BeNil(), "spec must not be silently dropped") + Expect(receivedBody.Spec.Resources).To(HaveLen(1), "spec.resources must not be silently dropped") - res0, ok := resources[0].(map[string]any) - Expect(ok).To(BeTrue()) - Expect(res0["name"]).To(Equal("main")) - Expect(res0["service_type"]).To(Equal("vm")) - Expect(res0).To(HaveKey("fields")) - - fields, ok := res0["fields"].([]any) - Expect(ok).To(BeTrue()) - Expect(len(fields)).To(BeNumerically(">=", 5), "all documented fields should be preserved") + res0 := receivedBody.Spec.Resources[0] + Expect(res0.Name).To(Equal("main")) + Expect(res0.ServiceType).To(Equal("vm")) + Expect(res0.Fields).NotTo(BeNil(), "fields must not be silently dropped") + Expect(*res0.Fields).To(HaveLen(5), "all documented fields should be preserved") }) }) Describe("Catalog Item Instance YAML (my-vm.yaml)", func() { // TC-U155: Documented instance YAML preserves user_values[].resource through CLI serialization It("TC-U155: should serialize user_values with the resource field to the API", func() { - var receivedBody map[string]any + var receivedBody v1alpha1.CatalogItemInstance server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { Expect(r.Method).To(Equal(http.MethodPost)) @@ -109,21 +101,11 @@ var _ = Describe("Documentation Contract", func() { err := executeCommand("catalog", "instance", "create", "--from-file", docsFixturePath("my-vm.yaml")) Expect(err).NotTo(HaveOccurred()) - Expect(receivedBody).To(HaveKey("spec")) - spec, ok := receivedBody["spec"].(map[string]any) - Expect(ok).To(BeTrue(), "spec should be an object") - - Expect(spec).To(HaveKey("user_values"), "spec.user_values must not be silently dropped") - userValues, ok := spec["user_values"].([]any) - Expect(ok).To(BeTrue(), "spec.user_values should be an array") - Expect(userValues).To(HaveLen(2)) + Expect(receivedBody.Spec.UserValues).To(HaveLen(2), "user_values must not be silently dropped") - for i, uv := range userValues { - uvMap, ok := uv.(map[string]any) - Expect(ok).To(BeTrue()) - Expect(uvMap).To(HaveKey("resource"), + for i, uv := range receivedBody.Spec.UserValues { + Expect(uv.Resource).To(Equal("main"), "user_values[%d].resource must not be silently dropped", i) - Expect(uvMap["resource"]).To(Equal("main")) } }) }) From 7d9a1f1ad9cc187db7919f4e38078f45b6932eea Mon Sep 17 00:00:00 2001 From: Thomas Stetson Date: Thu, 3 Sep 2026 12:17:20 -0400 Subject: [PATCH 3/3] test(contract): align website fixture test IDs and harden drift check - Rename contract tests from TC-U154/TC-U155 to TC-U158/TC-U159 and document them in the unit test plan under Documentation Contract Tests - Update requirement traceability for REQ-CIT-010 and REQ-CIN-010 - Make hack/check-website-fixtures.sh fail when no fixtures could be compared, regardless of trigger event Signed-off-by: Thomas Stetson --- .ai/test-plans/dcm-cli-unit.test-plan.md | 31 +++++++++++++++++++++--- hack/check-website-fixtures.sh | 6 +++-- internal/commands/contract_test.go | 8 +++--- 3 files changed, 36 insertions(+), 9 deletions(-) diff --git a/.ai/test-plans/dcm-cli-unit.test-plan.md b/.ai/test-plans/dcm-cli-unit.test-plan.md index 45cca30..d0e31b6 100644 --- a/.ai/test-plans/dcm-cli-unit.test-plan.md +++ b/.ai/test-plans/dcm-cli-unit.test-plan.md @@ -1380,6 +1380,31 @@ test classes. Instead: --- +## 13 · Documentation Contract Tests + +> **Suggested Ginkgo structure:** `Describe("Documentation Contract")` in `internal/commands/contract_test.go`. +> These tests verify that the YAML examples published in the Getting Started guides round-trip through the CLI serialization layer without field loss. They use `testdata/website/` fixtures (kept in sync by `hack/check-website-fixtures.sh`) as the source of truth. + +### TC-U158: Catalog item YAML preserves spec.resources through CLI serialization + +- **Requirement:** REQ-CIT-010 +- **Acceptance Criteria:** AC-CIT-010 +- **Type:** Unit +- **Given:** A mock server accepting POST `/api/v1alpha1/catalog-items` AND `testdata/website/small-vm.yaml` containing the documented multi-resource schema (`spec.resources`) +- **When:** `dcm catalog item create --from-file testdata/website/small-vm.yaml` is executed +- **Then:** The request body received by the mock server contains `spec.resources` with one resource whose `name` is `main`, `service_type` is `vm`, and `fields` contains all 5 documented key/value pairs + +### TC-U159: Catalog instance YAML preserves user_values[].resource through CLI serialization + +- **Requirement:** REQ-CIN-010 +- **Acceptance Criteria:** AC-CIN-010 +- **Type:** Unit +- **Given:** A mock server accepting POST `/api/v1alpha1/catalog-item-instances` AND `testdata/website/my-vm.yaml` containing the documented instance schema with `user_values[].resource` +- **When:** `dcm catalog instance create --from-file testdata/website/my-vm.yaml` is executed +- **Then:** The request body received by the mock server contains `spec.user_values` with 2 entries, each having `resource: main` + +--- + ## Utility Test Case Index Utility and helper functions are tested **transitively** through the @@ -1625,7 +1650,7 @@ dedicated test class or `Describe` block. | REQ-CST-030 | TC-U044 | Covered | | REQ-CST-040 | TC-U045 | Covered | | REQ-CST-050 | TC-U067 (via TC-U042, TC-U044) | Covered | -| REQ-CIT-010 | TC-U046 | Covered | +| REQ-CIT-010 | TC-U046, TC-U158 | Covered | | REQ-CIT-020 | TC-U047 | Covered | | REQ-CIT-030 | TC-U046 | Covered | | REQ-CIT-040 | TC-U049, TC-U050 | Covered | @@ -1636,7 +1661,7 @@ dedicated test class or `Describe` block. | REQ-CIT-110 | TC-U067 (via TC-U046, TC-U049, TC-U051, TC-U055) | Covered | | REQ-CIT-120 | TC-U048 | Covered | | REQ-CIT-130 | TC-U052, TC-U056 | Covered | -| REQ-CIN-010 | TC-U058 | Covered | +| REQ-CIN-010 | TC-U058, TC-U159 | Covered | | REQ-CIN-020 | TC-U059 | Covered | | REQ-CIN-030 | TC-U058 | Covered | | REQ-CIN-040 | TC-U073, TC-U074 | Covered | @@ -1700,7 +1725,7 @@ dedicated test class or `Describe` block. | REQ-XC-TLS-070 | TC-U095, TC-U096 | Covered | | REQ-XC-TLS-080 | TC-U090, TC-U097 | Covered | -**Total:** 121 test case IDs — 95 in behavioural test classes, 26 in the utility +**Total:** 123 test case IDs — 97 in behavioural test classes, 26 in the utility index (tested transitively through higher-level behavioural tests). --- diff --git a/hack/check-website-fixtures.sh b/hack/check-website-fixtures.sh index 26b55aa..c2dbc9c 100755 --- a/hack/check-website-fixtures.sh +++ b/hack/check-website-fixtures.sh @@ -53,6 +53,7 @@ extract_yaml_block() { errors=0 fetch_failures=0 +compared=0 for fixture in "${!FIXTURE_SOURCES[@]}"; do source_file="${FIXTURE_SOURCES[$fixture]}" @@ -96,6 +97,7 @@ for fixture in "${!FIXTURE_SOURCES[@]}"; do else echo " OK" fi + compared=$((compared + 1)) done if [[ ${errors} -gt 0 ]]; then @@ -106,8 +108,8 @@ fi if [[ ${fetch_failures} -gt 0 ]]; then echo "" - if [[ "${GITHUB_EVENT_NAME:-}" == "schedule" ]]; then - echo "FAIL: ${fetch_failures} fixture(s) could not be fetched (scheduled run — treating as error)" + if [[ "${GITHUB_EVENT_NAME:-}" == "schedule" ]] || [[ ${compared} -eq 0 ]]; then + echo "FAIL: ${fetch_failures} fixture(s) could not be fetched — no fixtures were checked" exit 1 fi echo "WARNING: ${fetch_failures} fixture(s) could not be fetched (skipped; network issue or rate limiting)" diff --git a/internal/commands/contract_test.go b/internal/commands/contract_test.go index d6dd47f..8a92999 100644 --- a/internal/commands/contract_test.go +++ b/internal/commands/contract_test.go @@ -57,8 +57,8 @@ var _ = Describe("Documentation Contract", func() { } Describe("Catalog Item YAML (small-vm.yaml)", func() { - // TC-U154: Documented catalog item YAML preserves spec.resources through CLI serialization - It("TC-U154: should serialize spec.resources with all required fields to the API", func() { + // TC-U158: Documented catalog item YAML preserves spec.resources through CLI serialization + It("TC-U158: should serialize spec.resources with all required fields to the API", func() { var receivedBody v1alpha1.CatalogItem server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -85,8 +85,8 @@ var _ = Describe("Documentation Contract", func() { }) Describe("Catalog Item Instance YAML (my-vm.yaml)", func() { - // TC-U155: Documented instance YAML preserves user_values[].resource through CLI serialization - It("TC-U155: should serialize user_values with the resource field to the API", func() { + // TC-U159: Documented instance YAML preserves user_values[].resource through CLI serialization + It("TC-U159: should serialize user_values with the resource field to the API", func() { var receivedBody v1alpha1.CatalogItemInstance server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {