diff --git a/.ai/test-plans/dcm-cli-unit.test-plan.md b/.ai/test-plans/dcm-cli-unit.test-plan.md index 45cca30..d0e31b6 100644 --- a/.ai/test-plans/dcm-cli-unit.test-plan.md +++ b/.ai/test-plans/dcm-cli-unit.test-plan.md @@ -1380,6 +1380,31 @@ test classes. Instead: --- +## 13 · Documentation Contract Tests + +> **Suggested Ginkgo structure:** `Describe("Documentation Contract")` in `internal/commands/contract_test.go`. +> These tests verify that the YAML examples published in the Getting Started guides round-trip through the CLI serialization layer without field loss. They use `testdata/website/` fixtures (kept in sync by `hack/check-website-fixtures.sh`) as the source of truth. + +### TC-U158: Catalog item YAML preserves spec.resources through CLI serialization + +- **Requirement:** REQ-CIT-010 +- **Acceptance Criteria:** AC-CIT-010 +- **Type:** Unit +- **Given:** A mock server accepting POST `/api/v1alpha1/catalog-items` AND `testdata/website/small-vm.yaml` containing the documented multi-resource schema (`spec.resources`) +- **When:** `dcm catalog item create --from-file testdata/website/small-vm.yaml` is executed +- **Then:** The request body received by the mock server contains `spec.resources` with one resource whose `name` is `main`, `service_type` is `vm`, and `fields` contains all 5 documented key/value pairs + +### TC-U159: Catalog instance YAML preserves user_values[].resource through CLI serialization + +- **Requirement:** REQ-CIN-010 +- **Acceptance Criteria:** AC-CIN-010 +- **Type:** Unit +- **Given:** A mock server accepting POST `/api/v1alpha1/catalog-item-instances` AND `testdata/website/my-vm.yaml` containing the documented instance schema with `user_values[].resource` +- **When:** `dcm catalog instance create --from-file testdata/website/my-vm.yaml` is executed +- **Then:** The request body received by the mock server contains `spec.user_values` with 2 entries, each having `resource: main` + +--- + ## Utility Test Case Index Utility and helper functions are tested **transitively** through the @@ -1625,7 +1650,7 @@ dedicated test class or `Describe` block. | REQ-CST-030 | TC-U044 | Covered | | REQ-CST-040 | TC-U045 | Covered | | REQ-CST-050 | TC-U067 (via TC-U042, TC-U044) | Covered | -| REQ-CIT-010 | TC-U046 | Covered | +| REQ-CIT-010 | TC-U046, TC-U158 | Covered | | REQ-CIT-020 | TC-U047 | Covered | | REQ-CIT-030 | TC-U046 | Covered | | REQ-CIT-040 | TC-U049, TC-U050 | Covered | @@ -1636,7 +1661,7 @@ dedicated test class or `Describe` block. | REQ-CIT-110 | TC-U067 (via TC-U046, TC-U049, TC-U051, TC-U055) | Covered | | REQ-CIT-120 | TC-U048 | Covered | | REQ-CIT-130 | TC-U052, TC-U056 | Covered | -| REQ-CIN-010 | TC-U058 | Covered | +| REQ-CIN-010 | TC-U058, TC-U159 | Covered | | REQ-CIN-020 | TC-U059 | Covered | | REQ-CIN-030 | TC-U058 | Covered | | REQ-CIN-040 | TC-U073, TC-U074 | Covered | @@ -1700,7 +1725,7 @@ dedicated test class or `Describe` block. | REQ-XC-TLS-070 | TC-U095, TC-U096 | Covered | | REQ-XC-TLS-080 | TC-U090, TC-U097 | Covered | -**Total:** 121 test case IDs — 95 in behavioural test classes, 26 in the utility +**Total:** 123 test case IDs — 97 in behavioural test classes, 26 in the utility index (tested transitively through higher-level behavioural tests). --- diff --git a/.github/workflows/check-website-fixtures.yaml b/.github/workflows/check-website-fixtures.yaml new file mode 100644 index 0000000..e7ead5e --- /dev/null +++ b/.github/workflows/check-website-fixtures.yaml @@ -0,0 +1,19 @@ +name: Check website fixtures + +on: + pull_request: + branches: [main] + paths: + - testdata/website/** + - hack/check-website-fixtures.sh + schedule: + - cron: "0 8 * * 1" # Weekly Monday 08:00 UTC — hard-fails on fetch errors + +jobs: + check-fixtures: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Verify testdata/website/ matches upstream docs + run: hack/check-website-fixtures.sh diff --git a/CLAUDE.md b/CLAUDE.md index 276d2f5..f82292f 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -42,6 +42,9 @@ make tidy # Run E2E tests (requires live DCM stack) make test-e2e + +# Check website fixture drift +make check-fixtures ``` ## Architecture @@ -82,6 +85,11 @@ make test-e2e - **test/e2e/**: E2E tests with `e2e` build tag (`//go:build e2e`) +- **testdata/website/**: YAML fixtures mirroring the Getting Started tutorial examples from `dcm-project.github.io`. Used by contract tests to validate CLI serialization of documented schemas. + +- **hack/**: Development scripts + - `check-website-fixtures.sh`: Detects drift between local `testdata/website/` fixtures and upstream docs. Supports `--update` to refresh fixtures. Exit codes: 0 success, 1 drift/runtime error, 2 usage error. + - **tools.go**: Build tool dependencies (ginkgo) ## Testing @@ -102,7 +110,7 @@ E2E tests live under `test/e2e/` and use the `e2e` build tag (`//go:build e2e`). 4. **Output formatting**: All commands support `--output/-o` flag with `table` (default), `json`, and `yaml` formats. -5. **Input files**: Resource creation and updates use `--from-file` flag accepting YAML or JSON files. +5. **Input files**: Resource creation and updates use `--from-file` flag accepting YAML or JSON files. Catalog items use the multi-resource schema (`spec.resources[]` with named resources, each containing `service_type` and `fields`). Catalog instances reference resources via `user_values[].resource`. 6. **Error handling**: API errors follow RFC 7807 Problem Details format. Exit code 0 for success, 1 for runtime errors, 2 for usage errors. diff --git a/Makefile b/Makefile index ed9ae6b..b347c48 100644 --- a/Makefile +++ b/Makefile @@ -6,7 +6,7 @@ LDFLAGS = -X github.com/dcm-project/cli/internal/version.Version=$(VERSION) \ -X github.com/dcm-project/cli/internal/version.Commit=$(COMMIT) \ -X github.com/dcm-project/cli/internal/version.BuildTime=$(BUILD_TIME) -.PHONY: build test test-e2e fmt vet lint clean tidy +.PHONY: build test test-e2e fmt vet lint clean tidy check-fixtures build: tidy go build -ldflags "$(LDFLAGS)" -o bin/dcm ./cmd/dcm @@ -31,3 +31,6 @@ clean: tidy: go mod tidy + +check-fixtures: + hack/check-website-fixtures.sh diff --git a/README.md b/README.md index 77671e6..265fd9d 100644 --- a/README.md +++ b/README.md @@ -422,26 +422,33 @@ dcm catalog item create --from-file item.yaml dcm catalog item create --from-file item.yaml --id my-catalog-item ``` -Catalog item file format: +Catalog item file format (multi-resource schema): ```yaml api_version: v1alpha1 -display_name: "Small Container" +display_name: "Small VM" spec: - service_type: container - fields: - - path: spec.replicas - display_name: "Replica Count" - editable: true - default: "1" - validation_schema: - type: integer - minimum: 1 - maximum: 10 - - path: spec.container.image - display_name: "Container Image" - editable: true -``` + resources: + - name: main + service_type: vm + fields: + - path: metadata + editable: true + - path: vcpu.count + display_name: "CPU Count" + editable: true + default: 2 + validation_schema: + type: integer + minimum: 1 + maximum: 4 + - path: memory.size + display_name: "Memory (GB)" + editable: false + default: "2GB" +``` + +Each catalog item defines one or more named resources under `spec.resources`. Each resource specifies its `service_type` and the `fields` available for customization. Example output (table): @@ -505,19 +512,26 @@ Create a new catalog item instance. dcm catalog instance create --from-file instance.yaml ``` -Instance file format: +Instance file format (multi-resource schema): ```yaml api_version: v1alpha1 -display_name: "My App Instance" +display_name: "My Dev VM" spec: - catalog_item_id: my-catalog-item + catalog_item_id: small-vm user_values: - - path: spec.replicas - value: "3" - - path: spec.container.image - value: "nginx:latest" -``` + - resource: main + path: metadata + value: + name: "small-vm" + labels: + env: "dev" + - resource: main + path: vcpu.count + value: 1 +``` + +Each `user_values` entry includes a `resource` field that identifies which named resource (from the catalog item's `spec.resources`) the value applies to. Example output (table): diff --git a/go.mod b/go.mod index c2d0b33..3bd1d84 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,7 @@ go 1.25.5 require ( github.com/coreos/go-oidc/v3 v3.20.0 - github.com/dcm-project/control-plane v0.0.0-20260617094433-e4374fc25292 + github.com/dcm-project/control-plane v0.0.0-20260814135831-7c173134e932 github.com/onsi/ginkgo/v2 v2.29.0 github.com/onsi/gomega v1.41.0 github.com/spf13/cobra v1.10.2 diff --git a/go.sum b/go.sum index f9639f9..f31720e 100644 --- a/go.sum +++ b/go.sum @@ -14,8 +14,8 @@ github.com/danieljoos/wincred v1.2.2/go.mod h1:w7w4Utbrz8lqeMbDAK0lkNJUv5sAOkFi7 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/dcm-project/control-plane v0.0.0-20260617094433-e4374fc25292 h1:VnUyqtBqLrXTC1ClDSogbvVGDWvikhvmyxP5Znw1yX4= -github.com/dcm-project/control-plane v0.0.0-20260617094433-e4374fc25292/go.mod h1:dYV62wU2z+H1XUBdGoTMdh0V5j8AyJc4Lk55TLgPSBw= +github.com/dcm-project/control-plane v0.0.0-20260814135831-7c173134e932 h1:dLCvaQn2vZ/0+ByjhGcYoxLzCimFyNNm0kNMgBt/hQg= +github.com/dcm-project/control-plane v0.0.0-20260814135831-7c173134e932/go.mod h1:oLQKa1vW81Dmpit5iGjU0EQaRARadjrJ/BmCadwqir4= github.com/dlclark/regexp2 v1.11.5 h1:Q/sSnsKerHeCkc/jSTNq1oCm7KiVgUMZRDUoRu0JQZQ= github.com/dlclark/regexp2 v1.11.5/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8= github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= diff --git a/hack/check-website-fixtures.sh b/hack/check-website-fixtures.sh new file mode 100755 index 0000000..c2dbc9c --- /dev/null +++ b/hack/check-website-fixtures.sh @@ -0,0 +1,121 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Verifies that testdata/website/ fixtures match the YAML examples +# published in the dcm-project.github.io Getting Started guides. +# +# Usage: hack/check-website-fixtures.sh [--update] +# --update Overwrite local fixtures with upstream content (for refresh) +# +# Exit codes: 0 = success, 1 = runtime/drift error, 2 = usage error + +usage() { + echo "Usage: ${0##*/} [--update]" + echo " --update Overwrite local fixtures with upstream content" +} + +REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +FIXTURE_DIR="${REPO_ROOT}/testdata/website" +WEBSITE_REPO="dcm-project/dcm-project.github.io" +WEBSITE_BRANCH="main" +BASE_URL="https://raw.githubusercontent.com/${WEBSITE_REPO}/${WEBSITE_BRANCH}/content/docs/getting-started" + +UPDATE=false +case "${1:-}" in + "") ;; + --update) UPDATE=true ;; + *) + echo "Error: unknown argument: $1" >&2 + usage >&2 + exit 2 + ;; +esac +if [[ $# -gt 1 ]]; then + echo "Error: too many arguments" >&2 + usage >&2 + exit 2 +fi + +# Maps local fixture filenames to their source markdown files. +declare -A FIXTURE_SOURCES=( + ["small-vm.yaml"]="create-small-vm-catalog-item.md" + ["my-vm.yaml"]="create-small-vm-instance.md" +) + +extract_yaml_block() { + # Extracts the first ```yaml ... ``` fenced block from markdown on stdin. + awk ' + /^```yaml/ { capture=1; next } + /^```/ && capture { capture=0; next } + capture { print } + ' +} + +errors=0 +fetch_failures=0 +compared=0 + +for fixture in "${!FIXTURE_SOURCES[@]}"; do + source_file="${FIXTURE_SOURCES[$fixture]}" + local_path="${FIXTURE_DIR}/${fixture}" + url="${BASE_URL}/${source_file}" + + echo "Checking ${fixture} ← ${source_file}" + + if ! markdown=$(curl -sf --connect-timeout 10 --max-time 30 --retry 3 --retry-delay 2 --retry-all-errors "${url}"); then + echo " WARNING: Could not fetch ${url} (network error, 404, or rate limiting)" + fetch_failures=$((fetch_failures + 1)) + continue + fi + + upstream_yaml=$(echo "${markdown}" | extract_yaml_block) + if [[ -z "${upstream_yaml}" ]]; then + echo " ERROR: No YAML code block found in ${source_file}" + errors=$((errors + 1)) + continue + fi + + if [[ "${UPDATE}" == "true" ]]; then + echo "${upstream_yaml}" > "${local_path}" + echo " Updated ${local_path}" + continue + fi + + if [[ ! -f "${local_path}" ]]; then + echo " ERROR: Local fixture missing: ${local_path}" + echo " Run: hack/check-website-fixtures.sh --update" + errors=$((errors + 1)) + continue + fi + + if ! diff_output=$(diff -u "${local_path}" <(echo "${upstream_yaml}")); then + echo " DRIFT DETECTED: ${fixture} differs from upstream" + echo "${diff_output}" + echo "" + echo " To update: hack/check-website-fixtures.sh --update" + errors=$((errors + 1)) + else + echo " OK" + fi + compared=$((compared + 1)) +done + +if [[ ${errors} -gt 0 ]]; then + echo "" + echo "FAIL: ${errors} fixture(s) out of sync with ${WEBSITE_REPO}" + exit 1 +fi + +if [[ ${fetch_failures} -gt 0 ]]; then + echo "" + if [[ "${GITHUB_EVENT_NAME:-}" == "schedule" ]] || [[ ${compared} -eq 0 ]]; then + echo "FAIL: ${fetch_failures} fixture(s) could not be fetched — no fixtures were checked" + exit 1 + fi + echo "WARNING: ${fetch_failures} fixture(s) could not be fetched (skipped; network issue or rate limiting)" + echo "Re-run or check manually: hack/check-website-fixtures.sh" + exit 0 +fi + +echo "" +echo "All fixtures match upstream." diff --git a/internal/commands/contract_test.go b/internal/commands/contract_test.go new file mode 100644 index 0000000..8a92999 --- /dev/null +++ b/internal/commands/contract_test.go @@ -0,0 +1,112 @@ +package commands_test + +import ( + "bytes" + "encoding/json" + "net/http" + "net/http/httptest" + "path/filepath" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + v1alpha1 "github.com/dcm-project/control-plane/api/catalog/v1alpha1" + + "github.com/dcm-project/cli/internal/commands" +) + +func docsFixturePath(name string) string { + return filepath.Join("..", "..", "testdata", "website", name) +} + +var _ = Describe("Documentation Contract", func() { + var ( + server *httptest.Server + outBuf *bytes.Buffer + errBuf *bytes.Buffer + ) + + BeforeEach(func() { + clearDCMEnvVars() + }) + + AfterEach(func() { + if server != nil { + server.Close() + server = nil + } + }) + + executeCommand := func(args ...string) error { + cmd := commands.NewRootCommand() + outBuf = new(bytes.Buffer) + errBuf = new(bytes.Buffer) + cmd.SetOut(outBuf) + cmd.SetErr(errBuf) + + fullArgs := []string{ + "--config", nonexistentConfigPath(), + } + if server != nil { + fullArgs = append(fullArgs, "--control-plane-url", server.URL) + } + fullArgs = append(fullArgs, args...) + cmd.SetArgs(fullArgs) + + return cmd.Execute() + } + + Describe("Catalog Item YAML (small-vm.yaml)", func() { + // TC-U158: Documented catalog item YAML preserves spec.resources through CLI serialization + It("TC-U158: should serialize spec.resources with all required fields to the API", func() { + var receivedBody v1alpha1.CatalogItem + + server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + Expect(r.Method).To(Equal(http.MethodPost)) + Expect(r.URL.Path).To(Equal("/api/v1alpha1/catalog-items")) + + Expect(json.NewDecoder(r.Body).Decode(&receivedBody)).To(Succeed()) + + writeJSONResponse(w, http.StatusCreated, sampleCatalogItemResponse()) + })) + + err := executeCommand("catalog", "item", "create", "--from-file", docsFixturePath("small-vm.yaml")) + Expect(err).NotTo(HaveOccurred()) + + Expect(receivedBody.Spec).NotTo(BeNil(), "spec must not be silently dropped") + Expect(receivedBody.Spec.Resources).To(HaveLen(1), "spec.resources must not be silently dropped") + + res0 := receivedBody.Spec.Resources[0] + Expect(res0.Name).To(Equal("main")) + Expect(res0.ServiceType).To(Equal("vm")) + Expect(res0.Fields).NotTo(BeNil(), "fields must not be silently dropped") + Expect(*res0.Fields).To(HaveLen(5), "all documented fields should be preserved") + }) + }) + + Describe("Catalog Item Instance YAML (my-vm.yaml)", func() { + // TC-U159: Documented instance YAML preserves user_values[].resource through CLI serialization + It("TC-U159: should serialize user_values with the resource field to the API", func() { + var receivedBody v1alpha1.CatalogItemInstance + + server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + Expect(r.Method).To(Equal(http.MethodPost)) + Expect(r.URL.Path).To(Equal("/api/v1alpha1/catalog-item-instances")) + + Expect(json.NewDecoder(r.Body).Decode(&receivedBody)).To(Succeed()) + + writeJSONResponse(w, http.StatusCreated, sampleInstanceResponse()) + })) + + err := executeCommand("catalog", "instance", "create", "--from-file", docsFixturePath("my-vm.yaml")) + Expect(err).NotTo(HaveOccurred()) + + Expect(receivedBody.Spec.UserValues).To(HaveLen(2), "user_values must not be silently dropped") + + for i, uv := range receivedBody.Spec.UserValues { + Expect(uv.Resource).To(Equal("main"), + "user_values[%d].resource must not be silently dropped", i) + } + }) + }) +}) diff --git a/testdata/website/my-vm.yaml b/testdata/website/my-vm.yaml new file mode 100644 index 0000000..6ce5d6c --- /dev/null +++ b/testdata/website/my-vm.yaml @@ -0,0 +1,14 @@ +api_version: v1alpha1 +display_name: "My Dev VM" +spec: + catalog_item_id: small-vm + user_values: + - resource: main + path: metadata + value: + name: "small-vm" + labels: + env: "dev" + - resource: main + path: vcpu.count + value: 1 diff --git a/testdata/website/small-vm.yaml b/testdata/website/small-vm.yaml new file mode 100644 index 0000000..61075f6 --- /dev/null +++ b/testdata/website/small-vm.yaml @@ -0,0 +1,38 @@ +api_version: v1alpha1 +display_name: "Small VM" +spec: + resources: + - name: main + service_type: vm + fields: + - path: metadata + editable: true + - path: vcpu.count + display_name: "CPU Count" + editable: true + default: 2 + validation_schema: + type: integer + minimum: 1 + maximum: 4 + - path: memory.size + display_name: "Memory (GB)" + editable: false + default: "2GB" + - path: storage.disks + display_name: "Storage (GB)" + editable: false + default: + - name: boot + capacity: "20GB" + validation_schema: + type: array + - path: guest_os.type + display_name: "Guest OS" + editable: true + default: "rhel-10" + validation_schema: + type: string + enum: + - rhel-9 + - rhel-10