diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md index 74f0d95..354166d 100644 --- a/DEVELOPMENT.md +++ b/DEVELOPMENT.md @@ -137,6 +137,29 @@ The release workflow packages and publishes `dist/chart`. More information can be found via the [Kubebuilder Documentation](https://book.kubebuilder.io/introduction.html) +## File-based tests + +Unit and envtest cases live in YAML under a `testdata/` directory next to the package, one file per case. The Go tests load those files, stand up fixtures, and check the result. Kind-based end-to-end tests under `test/e2e` are unchanged. + +``` +internal//testdata//.yaml +``` + +The suite directory groups a table (`guards/hold`, `controller/metric-scaling`). The file name is the case name in kebab-case. A document has: + +- exactly one YAML mapping. An empty file, a comment-only file, `null`, a bare `---`, or a second document fails the load. A trailing `---` counts as a second document. +- `name`, optional. When set it must equal the file name without `.yaml`. +- `description`, the sentence `go test -v` and Ginkgo print. When it is empty, the file name is used. +- `inputs` and `expected`, decoded into a Go struct. An unknown field, a duplicate key, a key in the wrong case, or a value of the wrong type fails the load. Mapping keys must match the json name of the field, including case. + +Only `*.yaml` is loaded. A `*.yml` file fails the load, so two extensions cannot share a stem. Other names are ignored. The suffix check is case-sensitive. + +Durations are Go duration strings (`30s`, `1m1s`, `-2h`, `0s`). Only a pointer to a duration may be omitted or `null`. A non-pointer duration rejects `null` and treats an omission as zero. A pointer to zero is `"0s"`, not a bare `0`. Times relative to a fixed origin are offsets from that origin; the runner adds them. + +The loader is `internal/testutil/cases`. It reads every `*.yaml` under the directory, in path order, and rejects an empty directory. `cases.Run` starts one subtest per file. Ginkgo tables use `ginkgocases.Entries` in `internal/testutil/cases/ginkgocases`, so a test binary that does not import that package does not register Ginkgo flags. + +To add a case, copy a nearby file, change the inputs and the expected result, and run `make test`. + ## License Copyright 2026. diff --git a/internal/chart/repo_test.go b/internal/chart/repo_test.go index 00f06b8..3f75bbc 100644 --- a/internal/chart/repo_test.go +++ b/internal/chart/repo_test.go @@ -4,6 +4,7 @@ import ( "encoding/base64" "errors" "fmt" + "io" "net/http" "net/http/httptest" "os" @@ -11,181 +12,341 @@ import ( "reflect" "strings" "testing" + + "github.com/modelsphere/llm-operator/internal/testutil/cases" ) -func TestVersionsFromARepoIndex(t *testing.T) { - srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if r.URL.Path != "/charts/index.yaml" { - http.NotFound(w, r) - return - } - _, _ = fmt.Fprint(w, "apiVersion: v1\nentries:\n sglang:\n - version: 0.8.0\n - version: 0.7.1\n vllm:\n - version: 1.0.0\n") - })) - t.Cleanup(srv.Close) +// chartCall is one Versions or VersionsAuth read. +// repoKind is https, oci, or local. A local read writes Chart.yaml from +// chartYAML and passes localRepo through; the directory wins over that URL. +// {server}, {repo}, {indexURL}, and {basic} in expected errors are filled +// once the server exists. {basic} is "Basic " plus base64(username:password). +type chartCall struct { + Fn string `json:"fn"` + RepoKind string `json:"repoKind"` + RepoPath string `json:"repoPath,omitempty"` + Chart string `json:"chart"` + Username string `json:"username,omitempty"` + Password string `json:"password,omitempty"` + LocalRepo string `json:"localRepo,omitempty"` + ChartYAML string `json:"chartYAML,omitempty"` - got, err := Versions(t.Context(), srv.URL+"/charts/", "", "sglang") - if err != nil || !reflect.DeepEqual(got, []string{v080, v071}) { - t.Fatalf("got %v, %v", got, err) - } - if _, err := Versions(t.Context(), srv.URL+"/charts", "", "nope"); err == nil || !strings.Contains(err.Error(), `no chart "nope"`) { - t.Errorf("a chart the index lacks: %v", err) - } + Versions *[]string `json:"versions,omitempty"` + ErrorExact string `json:"errorExact,omitempty"` + ErrorContains []string `json:"errorContains,omitempty"` + ErrorNotContains []string `json:"errorNotContains,omitempty"` + NeedsLogin bool `json:"needsLogin,omitempty"` + CapturedAuth string `json:"capturedAuth,omitempty"` + CapturedEquals string `json:"capturedEquals,omitempty"` +} + +// chartRoute is one httptest response. The first matching route wins. +// A route with no path matches every path still unmatched. +// No match answers 404 with body "404 page not found\n". +// query selects the route. requireQuery on a selected route answers 400 +// "bad scope ..." when the query does not match, as the token endpoint does. +// requireAuth skips the route when Authorization differs. +// captureAuth records Authorization when the path and query match. +type chartRoute struct { + Path string `json:"path"` + Query map[string]string `json:"query,omitempty"` + Status int `json:"status,omitempty"` + Body string `json:"body,omitempty"` + Headers map[string]string `json:"headers,omitempty"` + Challenge string `json:"challenge,omitempty"` + RealmPath string `json:"realmPath,omitempty"` + RealmName string `json:"realmName,omitempty"` + Service string `json:"service,omitempty"` + Scope string `json:"scope,omitempty"` + RequireAuth string `json:"requireAuth,omitempty"` + RequireQuery map[string]string `json:"requireQuery,omitempty"` + CaptureAuth string `json:"captureAuth,omitempty"` +} + +type repoCase struct { + Name string `json:"name,omitempty"` + Description string `json:"description,omitempty"` + TLS bool `json:"tls,omitempty"` + ReplaceClient bool `json:"replaceClient,omitempty"` + Username string `json:"username,omitempty"` + Password string `json:"password,omitempty"` + Routes []chartRoute `json:"routes,omitempty"` + Calls []chartCall `json:"calls"` +} + +func TestVersionsFromARepoIndex(t *testing.T) { + cases.Run(t, "testdata/repo/versions-from-a-repo-index", runRepoCase) } -// A public registry refuses the first read with a Bearer challenge and hands -// out an anonymous token; tags come back a page at a time. func TestVersionsFromAnOCIRegistry(t *testing.T) { + cases.Run(t, "testdata/repo/versions-from-an-oci-registry", runRepoCase) +} + +func TestVersionsFromALocalChart(t *testing.T) { + cases.Run(t, "testdata/repo/versions-from-a-local-chart", runRepoCase) +} + +func TestARefusedReadSaysItNeedsALogin(t *testing.T) { + cases.Run(t, "testdata/repo/a-refused-read-says-it-needs-a-login", runRepoCase) +} + +func TestARegistryAskingForBasicAuthNeedsALogin(t *testing.T) { + cases.Run(t, "testdata/repo/a-registry-asking-for-basic-auth-needs-a-login", runRepoCase) +} + +func TestVersionsAuthSendsBasicAuth(t *testing.T) { + cases.Run(t, "testdata/repo/versions-auth-sends-basic-auth", runRepoCase) +} + +func TestAMissingIndexIsReportedByStatus(t *testing.T) { + cases.Run(t, "testdata/repo/a-missing-index-is-reported-by-status", runRepoCase) +} + +func runRepoCase(t *testing.T, tc repoCase) { + t.Helper() + if len(tc.Calls) == 0 { + t.Fatal("no calls") + } + if tc.ReplaceClient && !tc.TLS { + t.Fatal("replaceClient is only for a TLS server") + } + basic := "" + if tc.Username != "" || tc.Password != "" { + basic = basicHeader(tc.Username, tc.Password) + } + needsServer := false + for _, call := range tc.Calls { + if call.RepoKind == "https" || call.RepoKind == "oci" { + needsServer = true + } + } var srv *httptest.Server - srv = httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - switch { - case r.URL.Path == tokenPath: - if r.URL.Query().Get("scope") != "repository:org/charts/sglang:pull" { - http.Error(w, "bad scope "+r.URL.Query().Get("scope"), http.StatusBadRequest) - return - } - _, _ = fmt.Fprint(w, `{"token":"t"}`) - case r.Header.Get("Authorization") != "Bearer t": - w.Header().Set("WWW-Authenticate", `Bearer realm="`+srv.URL+`/token",service="reg",scope="repository:org/charts/sglang:pull"`) - w.WriteHeader(http.StatusUnauthorized) - case r.URL.Path == "/v2/org/charts/sglang/tags/list" && r.URL.Query().Get("last") == "": - w.Header().Set("Link", `; rel="next"`) - _, _ = fmt.Fprint(w, `{"tags":["0.7.0","0.7.1"]}`) - case r.URL.Path == "/v2/org/charts/sglang/tags/list": - _, _ = fmt.Fprint(w, `{"tags":["0.8.0_build.1","sha256-abc.sig"]}`) - default: - http.NotFound(w, r) - } - })) - t.Cleanup(srv.Close) + serverURL := "" + captured := map[string]string{} saved := client - client = srv.Client() - t.Cleanup(func() { client = saved }) - - repo := "oci://" + strings.TrimPrefix(srv.URL, "https://") + "/org/charts" - got, err := Versions(t.Context(), repo, "", "sglang") - want := []string{"0.7.0", v071, "0.8.0+build.1", "sha256-abc.sig"} - if err != nil || !reflect.DeepEqual(got, want) { - t.Fatalf("got %v, %v; want %v", got, err, want) + if needsServer { + handler := repoHandler(&serverURL, tc.Routes, captured, basic) + if tc.TLS { + srv = httptest.NewTLSServer(handler) + } else { + srv = httptest.NewServer(handler) + } + serverURL = srv.URL + t.Cleanup(srv.Close) + if tc.ReplaceClient { + client = srv.Client() + t.Cleanup(func() { client = saved }) + } + } + for _, call := range tc.Calls { + runChartCall(t, srv, serverURL, basic, captured, call) + } + if !tc.ReplaceClient && client != saved { + t.Errorf("package client changed") } } -func TestVersionsFromALocalChart(t *testing.T) { - dir := t.TempDir() - if err := os.MkdirAll(filepath.Join(dir, "sglang"), 0o755); err != nil { - t.Fatal(err) +func runChartCall(t *testing.T, srv *httptest.Server, serverURL, basic string, captured map[string]string, call chartCall) { + t.Helper() + repo, path := call.source(t, srv) + indexURL := "" + if call.RepoKind == "https" { + indexURL = strings.TrimSuffix(repo, "/") + "/index.yaml" } - if err := os.WriteFile(filepath.Join(dir, "sglang", "Chart.yaml"), []byte("name: sglang\nversion: 0.7.4\n"), 0o644); err != nil { - t.Fatal(err) + callBasic := basic + if call.Username != "" || call.Password != "" { + callBasic = basicHeader(call.Username, call.Password) + } + var ( + got []string + err error + ) + switch call.Fn { + case "versions": + if call.Username != "" || call.Password != "" { + t.Fatal("versions is anonymous; use versionsAuth") + } + got, err = Versions(t.Context(), repo, path, call.Chart) + case "versionsAuth": + got, err = VersionsAuth(t.Context(), repo, path, call.Chart, Credentials{ + Username: call.Username, + Password: call.Password, + }) + default: + t.Fatalf("fn %q", call.Fn) } - got, err := Versions(t.Context(), "https://ignored.example.com", dir, "sglang") - if err != nil || !reflect.DeepEqual(got, []string{"0.7.4"}) { - t.Fatalf("got %v, %v; want the chart directory's own version", got, err) + switch { + case call.Versions != nil: + wantAuth, auth := "", "" + badAuth := false + if call.CapturedAuth != "" { + auth = captured[call.CapturedAuth] + wantAuth = subst(call.CapturedEquals, serverURL, repo, indexURL, callBasic) + badAuth = auth != wantAuth + } + if err != nil || !reflect.DeepEqual(got, *call.Versions) || badAuth { + t.Fatalf("got %v, %v; want %v; auth %q want %q", got, err, *call.Versions, auth, wantAuth) + } + case call.NeedsLogin || call.ErrorExact != "" || len(call.ErrorContains) > 0 || len(call.ErrorNotContains) > 0: + if call.NeedsLogin { + var re *RepoError + if err == nil || !errors.As(err, &re) || !re.NeedsLogin() { + t.Fatalf("got %v, want a RepoError needing a login", err) + } + } + if call.ErrorExact != "" { + want := subst(call.ErrorExact, serverURL, repo, indexURL, callBasic) + if err == nil || err.Error() != want { + t.Errorf("got %v", err) + } + } + checkContained(t, err, call.ErrorContains, serverURL, repo, indexURL, callBasic) + checkAbsent(t, err, call.ErrorNotContains, serverURL, repo, indexURL, callBasic) + default: + t.Fatal("call has no expectation") } } -// A private registry, or a chart that is not there: the token endpoint refuses, -// and the error says so in words, with what the registry said. -func TestARefusedReadSaysItNeedsALogin(t *testing.T) { - var srv *httptest.Server - srv = httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if r.URL.Path == tokenPath { - w.WriteHeader(http.StatusForbidden) - _, _ = fmt.Fprint(w, `{"errors":[{"code":"DENIED","message":"requested access to the resource is denied"}]}`) - return +// checkContained fails when err does not include every substring. +// An empty substring matches every error, so it is rejected on its own. +func checkContained(t *testing.T, err error, subs []string, serverURL, repo, indexURL, callBasic string) { + t.Helper() + for _, sub := range subs { + if sub == "" { + t.Fatal("errorContains entry is empty") } - w.Header().Set("WWW-Authenticate", `Bearer realm="`+srv.URL+`/token",service="reg",scope="repository:org/charts/sglang:pull"`) - w.WriteHeader(http.StatusUnauthorized) - })) - t.Cleanup(srv.Close) - saved := client - client = srv.Client() - t.Cleanup(func() { client = saved }) - - repo := "oci://" + strings.TrimPrefix(srv.URL, "https://") + "/org/charts" - _, err := Versions(t.Context(), repo, "", "sglang") - var re *RepoError - if !errors.As(err, &re) || !re.NeedsLogin() { - t.Fatalf("got %v, want a RepoError needing a login", err) - } - for _, want := range []string{repo + "/sglang", "needs a login", "DENIED: requested access to the resource is denied"} { - if !strings.Contains(err.Error(), want) { + want := subst(sub, serverURL, repo, indexURL, callBasic) + if err == nil || !strings.Contains(err.Error(), want) { t.Errorf("%q does not say %q", err, want) } } - if strings.Contains(err.Error(), tokenPath) { - t.Errorf("%q names the token endpoint rather than the repository", err) - } } -func TestARegistryAskingForBasicAuthNeedsALogin(t *testing.T) { - srv := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - w.Header().Set("WWW-Authenticate", `Basic realm="harbor"`) - w.WriteHeader(http.StatusUnauthorized) - })) - t.Cleanup(srv.Close) - saved := client - client = srv.Client() - t.Cleanup(func() { client = saved }) +// checkAbsent fails when err includes a substring that must stay out of it. +// An empty substring matches every error, so it is rejected on its own. +func checkAbsent(t *testing.T, err error, subs []string, serverURL, repo, indexURL, callBasic string) { + t.Helper() + for _, sub := range subs { + if sub == "" { + t.Fatal("errorNotContains entry is empty") + } + want := subst(sub, serverURL, repo, indexURL, callBasic) + if err != nil && strings.Contains(err.Error(), want) { + t.Errorf("%q names the token endpoint rather than the repository", err) + } + } +} - _, err := Versions(t.Context(), "oci://"+strings.TrimPrefix(srv.URL, "https://")+"/org", "", "sglang") - var re *RepoError - if !errors.As(err, &re) || !re.NeedsLogin() || !strings.Contains(err.Error(), "Basic auth") { - t.Fatalf("got %v", err) +func (call chartCall) source(t *testing.T, srv *httptest.Server) (repo, path string) { + t.Helper() + switch call.RepoKind { + case "https": + if srv == nil { + t.Fatal("https repo needs a server") + } + return srv.URL + call.RepoPath, "" + case "oci": + if srv == nil { + t.Fatal("oci repo needs a server") + } + return "oci://" + strings.TrimPrefix(srv.URL, "https://") + call.RepoPath, "" + case "local": + if call.ChartYAML == "" { + t.Fatal("local chart needs chartYAML") + } + root := t.TempDir() + dir := filepath.Join(root, call.Chart) + if err := os.MkdirAll(dir, 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(dir, "Chart.yaml"), []byte(call.ChartYAML), 0o644); err != nil { + t.Fatal(err) + } + return call.LocalRepo, root + default: + t.Fatalf("repoKind %q", call.RepoKind) + return "", "" } } -func TestVersionsAuthSendsBasicAuth(t *testing.T) { - want := "Basic " + base64.StdEncoding.EncodeToString([]byte("user:s3cret")) - var tokenAuth string - var srv *httptest.Server - srv = httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - switch r.URL.Path { - case tokenPath: - tokenAuth = r.Header.Get("Authorization") - if tokenAuth != want { - http.Error(w, "no basic", http.StatusUnauthorized) - return +func repoHandler(serverURL *string, routes []chartRoute, captured map[string]string, basic string) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + for i := range routes { + route := routes[i] + if route.Path != "" && r.URL.Path != route.Path { + continue } - _, _ = fmt.Fprint(w, `{"token":"t"}`) - case "/v2/org/sglang/tags/list": - if r.Header.Get("Authorization") != "Bearer t" { - w.Header().Set("WWW-Authenticate", `Bearer realm="`+srv.URL+`/token",service="reg",scope="repository:org/sglang:pull"`) - w.WriteHeader(http.StatusUnauthorized) - return + if !queryMatch(r, route.Query) { + continue } - _, _ = fmt.Fprint(w, `{"tags":["0.8.0"]}`) - case "/charts/index.yaml": - if r.Header.Get("Authorization") != want { - http.Error(w, "no basic", http.StatusUnauthorized) + if len(route.RequireQuery) > 0 && !queryMatch(r, route.RequireQuery) { + http.Error(w, "bad scope "+r.URL.Query().Get("scope"), http.StatusBadRequest) return } - _, _ = fmt.Fprint(w, "apiVersion: v1\nentries:\n sglang:\n - version: 0.8.0\n") - default: - http.NotFound(w, r) + if route.CaptureAuth != "" { + captured[route.CaptureAuth] = r.Header.Get("Authorization") + } + wantAuth := subst(route.RequireAuth, *serverURL, "", "", basic) + if route.RequireAuth != "" && r.Header.Get("Authorization") != wantAuth { + continue + } + writeRoute(w, route, *serverURL) + return } - })) - t.Cleanup(srv.Close) - saved := client - client = srv.Client() - t.Cleanup(func() { client = saved }) + http.NotFound(w, r) + }) +} - creds := Credentials{Username: "user", Password: "s3cret"} - idx, err := VersionsAuth(t.Context(), "https://"+strings.TrimPrefix(srv.URL, "https://")+"/charts", "", "sglang", creds) - if err != nil || !reflect.DeepEqual(idx, []string{v080}) { - t.Fatalf("index: got %v, %v", idx, err) +func writeRoute(w http.ResponseWriter, route chartRoute, serverURL string) { + if route.Challenge != "" { + w.Header().Set("WWW-Authenticate", challengeHeader(route, serverURL)) } - repo := "oci://" + strings.TrimPrefix(srv.URL, "https://") + "/org" - got, err := VersionsAuth(t.Context(), repo, "", "sglang", creds) - if err != nil || !reflect.DeepEqual(got, []string{v080}) || tokenAuth != want { - t.Fatalf("oci: got %v, %v; token auth %q", got, err, tokenAuth) + for k, v := range route.Headers { + w.Header().Set(k, v) + } + status := route.Status + if status == 0 { + if route.Challenge != "" { + status = http.StatusUnauthorized + } else { + status = http.StatusOK + } + } + w.WriteHeader(status) + if route.Body != "" { + _, _ = io.WriteString(w, route.Body) } } -func TestAMissingIndexIsReportedByStatus(t *testing.T) { - srv := httptest.NewServer(http.NotFoundHandler()) - t.Cleanup(srv.Close) - _, err := Versions(t.Context(), srv.URL+"/charts", "", "sglang") - if err == nil || err.Error() != srv.URL+"/charts/index.yaml: 404 Not Found -- the repository says: 404 page not found" { - t.Errorf("got %v", err) +func challengeHeader(route chartRoute, serverURL string) string { + switch route.Challenge { + case "bearer": + return fmt.Sprintf(`Bearer realm="%s%s",service="%s",scope="%s"`, serverURL, route.RealmPath, route.Service, route.Scope) + case "basic": + return fmt.Sprintf(`Basic realm="%s"`, route.RealmName) + default: + return route.Challenge } } + +func queryMatch(r *http.Request, q map[string]string) bool { + for k, v := range q { + if r.URL.Query().Get(k) != v { + return false + } + } + return true +} + +func basicHeader(user, pass string) string { + return "Basic " + base64.StdEncoding.EncodeToString([]byte(user+":"+pass)) +} + +func subst(s, server, repo, indexURL, basic string) string { + return strings.NewReplacer( + "{server}", server, + "{repo}", repo, + "{indexURL}", indexURL, + "{basic}", basic, + ).Replace(s) +} diff --git a/internal/chart/testdata/repo/a-missing-index-is-reported-by-status/a-missing-index-is-reported-by-status.yaml b/internal/chart/testdata/repo/a-missing-index-is-reported-by-status/a-missing-index-is-reported-by-status.yaml new file mode 100644 index 0000000..f3970e3 --- /dev/null +++ b/internal/chart/testdata/repo/a-missing-index-is-reported-by-status/a-missing-index-is-reported-by-status.yaml @@ -0,0 +1,7 @@ +description: missing index is reported by status +calls: + - fn: versions + repoKind: https + repoPath: /charts + chart: sglang + errorExact: "{indexURL}: 404 Not Found -- the repository says: 404 page not found" diff --git a/internal/chart/testdata/repo/a-refused-read-says-it-needs-a-login/a-refused-read-says-it-needs-a-login.yaml b/internal/chart/testdata/repo/a-refused-read-says-it-needs-a-login/a-refused-read-says-it-needs-a-login.yaml new file mode 100644 index 0000000..a57557f --- /dev/null +++ b/internal/chart/testdata/repo/a-refused-read-says-it-needs-a-login/a-refused-read-says-it-needs-a-login.yaml @@ -0,0 +1,23 @@ +description: refused read needs a login and does not name the token path +tls: true +replaceClient: true +routes: + - path: /token + status: 403 + body: '{"errors":[{"code":"DENIED","message":"requested access to the resource is denied"}]}' + - challenge: bearer + realmPath: /token + service: reg + scope: "repository:org/charts/sglang:pull" +calls: + - fn: versions + repoKind: oci + repoPath: /org/charts + chart: sglang + needsLogin: true + errorContains: + - "{repo}/sglang" + - "needs a login" + - "DENIED: requested access to the resource is denied" + errorNotContains: + - /token diff --git a/internal/chart/testdata/repo/a-registry-asking-for-basic-auth-needs-a-login/a-registry-asking-for-basic-auth-needs-a-login.yaml b/internal/chart/testdata/repo/a-registry-asking-for-basic-auth-needs-a-login/a-registry-asking-for-basic-auth-needs-a-login.yaml new file mode 100644 index 0000000..3587ce9 --- /dev/null +++ b/internal/chart/testdata/repo/a-registry-asking-for-basic-auth-needs-a-login/a-registry-asking-for-basic-auth-needs-a-login.yaml @@ -0,0 +1,14 @@ +description: basic challenge needs a login +tls: true +replaceClient: true +routes: + - challenge: basic + realmName: harbor +calls: + - fn: versions + repoKind: oci + repoPath: /org + chart: sglang + needsLogin: true + errorContains: + - "Basic auth" diff --git a/internal/chart/testdata/repo/versions-auth-sends-basic-auth/versions-auth-sends-basic-auth.yaml b/internal/chart/testdata/repo/versions-auth-sends-basic-auth/versions-auth-sends-basic-auth.yaml new file mode 100644 index 0000000..d892728 --- /dev/null +++ b/internal/chart/testdata/repo/versions-auth-sends-basic-auth/versions-auth-sends-basic-auth.yaml @@ -0,0 +1,50 @@ +description: index and oci send basic auth +tls: true +replaceClient: true +username: user +password: s3cret +routes: + - path: /token + requireAuth: "{basic}" + captureAuth: token + body: '{"token":"t"}' + - path: /token + status: 401 + body: "no basic\n" + - path: /v2/org/sglang/tags/list + requireAuth: "Bearer t" + body: '{"tags":["0.8.0"]}' + - path: /v2/org/sglang/tags/list + challenge: bearer + realmPath: /token + service: reg + scope: "repository:org/sglang:pull" + - path: /charts/index.yaml + requireAuth: "{basic}" + body: | + apiVersion: v1 + entries: + sglang: + - version: 0.8.0 + - path: /charts/index.yaml + status: 401 + body: "no basic\n" +calls: + - fn: versionsAuth + repoKind: https + repoPath: /charts + chart: sglang + username: user + password: s3cret + versions: + - "0.8.0" + - fn: versionsAuth + repoKind: oci + repoPath: /org + chart: sglang + username: user + password: s3cret + versions: + - "0.8.0" + capturedAuth: token + capturedEquals: "{basic}" diff --git a/internal/chart/testdata/repo/versions-from-a-local-chart/versions-from-a-local-chart.yaml b/internal/chart/testdata/repo/versions-from-a-local-chart/versions-from-a-local-chart.yaml new file mode 100644 index 0000000..16ad794 --- /dev/null +++ b/internal/chart/testdata/repo/versions-from-a-local-chart/versions-from-a-local-chart.yaml @@ -0,0 +1,11 @@ +description: local Chart.yaml version, repo URL ignored +calls: + - fn: versions + repoKind: local + localRepo: https://ignored.example.com + chart: sglang + chartYAML: | + name: sglang + version: 0.7.4 + versions: + - "0.7.4" diff --git a/internal/chart/testdata/repo/versions-from-a-repo-index/versions-from-a-repo-index.yaml b/internal/chart/testdata/repo/versions-from-a-repo-index/versions-from-a-repo-index.yaml new file mode 100644 index 0000000..f895ac6 --- /dev/null +++ b/internal/chart/testdata/repo/versions-from-a-repo-index/versions-from-a-repo-index.yaml @@ -0,0 +1,25 @@ +description: index lists sglang and reports a missing chart +routes: + - path: /charts/index.yaml + body: | + apiVersion: v1 + entries: + sglang: + - version: 0.8.0 + - version: 0.7.1 + vllm: + - version: 1.0.0 +calls: + - fn: versions + repoKind: https + repoPath: /charts/ + chart: sglang + versions: + - "0.8.0" + - "0.7.1" + - fn: versions + repoKind: https + repoPath: /charts + chart: nope + errorContains: + - 'no chart "nope"' diff --git a/internal/chart/testdata/repo/versions-from-an-oci-registry/versions-from-an-oci-registry.yaml b/internal/chart/testdata/repo/versions-from-an-oci-registry/versions-from-an-oci-registry.yaml new file mode 100644 index 0000000..171b44f --- /dev/null +++ b/internal/chart/testdata/repo/versions-from-an-oci-registry/versions-from-an-oci-registry.yaml @@ -0,0 +1,32 @@ +description: bearer challenge, anonymous token, and paginated tags +tls: true +replaceClient: true +routes: + - path: /token + requireQuery: + scope: "repository:org/charts/sglang:pull" + body: '{"token":"t"}' + - path: /v2/org/charts/sglang/tags/list + query: + last: "" + requireAuth: "Bearer t" + headers: + Link: '; rel="next"' + body: '{"tags":["0.7.0","0.7.1"]}' + - path: /v2/org/charts/sglang/tags/list + requireAuth: "Bearer t" + body: '{"tags":["0.8.0_build.1","sha256-abc.sig"]}' + - challenge: bearer + realmPath: /token + service: reg + scope: "repository:org/charts/sglang:pull" +calls: + - fn: versions + repoKind: oci + repoPath: /org/charts + chart: sglang + versions: + - "0.7.0" + - "0.7.1" + - "0.8.0+build.1" + - sha256-abc.sig diff --git a/internal/chart/testdata/version/bare/exact.yaml b/internal/chart/testdata/version/bare/exact.yaml new file mode 100644 index 0000000..4297528 --- /dev/null +++ b/internal/chart/testdata/version/bare/exact.yaml @@ -0,0 +1,7 @@ +description: a bare version allows only itself +spec: "0.7.1" +exact: "0.7.1" +allows: + - "0.7.1" +refuses: + - "0.7.2" diff --git a/internal/chart/testdata/version/bare/range.yaml b/internal/chart/testdata/version/bare/range.yaml new file mode 100644 index 0000000..bc7a98f --- /dev/null +++ b/internal/chart/testdata/version/bare/range.yaml @@ -0,0 +1,3 @@ +description: a range names no one version +spec: ">=0.7.1" +exact: "" diff --git a/internal/chart/testdata/version/match/caret.yaml b/internal/chart/testdata/version/match/caret.yaml new file mode 100644 index 0000000..4bb1976 --- /dev/null +++ b/internal/chart/testdata/version/match/caret.yaml @@ -0,0 +1,9 @@ +description: "^0.7.1 want 0.7.x only" +spec: "^0.7.1" +have: + - "0.7.1" + - "0.7.9" + - "0.8.0" +want: + - "0.7.9" + - "0.7.1" diff --git a/internal/chart/testdata/version/match/newest-first.yaml b/internal/chart/testdata/version/match/newest-first.yaml new file mode 100644 index 0000000..19d3bfe --- /dev/null +++ b/internal/chart/testdata/version/match/newest-first.yaml @@ -0,0 +1,14 @@ +description: newest first, no prerelease, no duplicates +spec: ">=0.7.1" +have: + - "0.7.0" + - "0.7.1" + - "0.10.0" + - latest + - "0.8.0" + - "0.9.0-rc1" + - "0.8.0" +want: + - "0.10.0" + - "0.8.0" + - "0.7.1" diff --git a/internal/chart/testdata/version/parse/banana.yaml b/internal/chart/testdata/version/parse/banana.yaml new file mode 100644 index 0000000..2f3012e --- /dev/null +++ b/internal/chart/testdata/version/parse/banana.yaml @@ -0,0 +1,2 @@ +description: ">=banana" +spec: ">=banana" diff --git a/internal/chart/testdata/version/parse/empty.yaml b/internal/chart/testdata/version/parse/empty.yaml new file mode 100644 index 0000000..aea354b --- /dev/null +++ b/internal/chart/testdata/version/parse/empty.yaml @@ -0,0 +1,2 @@ +description: empty +spec: "" diff --git a/internal/chart/testdata/version/parse/latest.yaml b/internal/chart/testdata/version/parse/latest.yaml new file mode 100644 index 0000000..dde4415 --- /dev/null +++ b/internal/chart/testdata/version/parse/latest.yaml @@ -0,0 +1,2 @@ +description: latest +spec: latest diff --git a/internal/chart/testdata/version/resolve-refuses/named-out-of-range.yaml b/internal/chart/testdata/version/resolve-refuses/named-out-of-range.yaml new file mode 100644 index 0000000..ebba9a5 --- /dev/null +++ b/internal/chart/testdata/version/resolve-refuses/named-out-of-range.yaml @@ -0,0 +1,5 @@ +description: a named version out of range +spec: ">=0.7.1" +want: "0.6.0" +noList: true +errorContains: "not in the catalog's" diff --git a/internal/chart/testdata/version/resolve-refuses/nil-lister.yaml b/internal/chart/testdata/version/resolve-refuses/nil-lister.yaml new file mode 100644 index 0000000..86dfdca --- /dev/null +++ b/internal/chart/testdata/version/resolve-refuses/nil-lister.yaml @@ -0,0 +1,4 @@ +description: a range with nowhere to list +spec: ">=0.7.1" +listNil: true +errorContains: "Name a chart version" diff --git a/internal/chart/testdata/version/resolve-refuses/nothing-in-range.yaml b/internal/chart/testdata/version/resolve-refuses/nothing-in-range.yaml new file mode 100644 index 0000000..23f9164 --- /dev/null +++ b/internal/chart/testdata/version/resolve-refuses/nothing-in-range.yaml @@ -0,0 +1,5 @@ +description: nothing in range +spec: ">=0.7.1" +list: + - "0.6.0" +errorContains: "no chart version satisfies" diff --git a/internal/chart/testdata/version/resolve/exact-moves-running-off.yaml b/internal/chart/testdata/version/resolve/exact-moves-running-off.yaml new file mode 100644 index 0000000..752ef0c --- /dev/null +++ b/internal/chart/testdata/version/resolve/exact-moves-running-off.yaml @@ -0,0 +1,5 @@ +description: exact moves a running one off +spec: "0.7.2" +keep: "0.7.1" +got: "0.7.2" +noList: true diff --git a/internal/chart/testdata/version/resolve/exact-needs-no-registry.yaml b/internal/chart/testdata/version/resolve/exact-needs-no-registry.yaml new file mode 100644 index 0000000..b23c5a6 --- /dev/null +++ b/internal/chart/testdata/version/resolve/exact-needs-no-registry.yaml @@ -0,0 +1,4 @@ +description: exact needs no registry +spec: "0.7.1" +got: "0.7.1" +noList: true diff --git a/internal/chart/testdata/version/resolve/named-in-range.yaml b/internal/chart/testdata/version/resolve/named-in-range.yaml new file mode 100644 index 0000000..0be71f4 --- /dev/null +++ b/internal/chart/testdata/version/resolve/named-in-range.yaml @@ -0,0 +1,5 @@ +description: named, in range +spec: ">=0.7.1" +want: "0.7.3" +got: "0.7.3" +noList: true diff --git a/internal/chart/testdata/version/resolve/newest-in-range.yaml b/internal/chart/testdata/version/resolve/newest-in-range.yaml new file mode 100644 index 0000000..37c6763 --- /dev/null +++ b/internal/chart/testdata/version/resolve/newest-in-range.yaml @@ -0,0 +1,7 @@ +description: newest in range +spec: ">=0.7.1 <0.9.0" +got: "0.8.1" +list: + - "0.7.1" + - "0.8.1" + - "0.9.0" diff --git a/internal/chart/testdata/version/resolve/running-kept-in-range.yaml b/internal/chart/testdata/version/resolve/running-kept-in-range.yaml new file mode 100644 index 0000000..477ea4e --- /dev/null +++ b/internal/chart/testdata/version/resolve/running-kept-in-range.yaml @@ -0,0 +1,5 @@ +description: running one kept while in range +spec: ">=0.7.1" +keep: "0.7.2" +got: "0.7.2" +noList: true diff --git a/internal/chart/testdata/version/resolve/running-out-of-range.yaml b/internal/chart/testdata/version/resolve/running-out-of-range.yaml new file mode 100644 index 0000000..a7f015f --- /dev/null +++ b/internal/chart/testdata/version/resolve/running-out-of-range.yaml @@ -0,0 +1,8 @@ +description: running one out of range +spec: ">=0.8.0" +keep: "0.7.2" +got: "0.9.0" +list: + - "0.7.2" + - "0.8.0" + - "0.9.0" diff --git a/internal/chart/version_test.go b/internal/chart/version_test.go index 1892948..b50c627 100644 --- a/internal/chart/version_test.go +++ b/internal/chart/version_test.go @@ -6,13 +6,8 @@ import ( "reflect" "strings" "testing" -) -const ( - v071 = "0.7.1" - v072 = "0.7.2" - v080 = "0.8.0" - tokenPath = "/token" + "github.com/modelsphere/llm-operator/internal/testutil/cases" ) func spec(t *testing.T, s string) Spec { @@ -24,79 +19,145 @@ func spec(t *testing.T, s string) Spec { return sp } -// A bare version is that version and nothing else, as before ranges. +// bareCase is one exact-version check. Exact is a pointer so a range can +// expect the empty string and an omitted field stays distinct from that. +type bareCase struct { + Name string `json:"name,omitempty"` + Description string `json:"description,omitempty"` + Spec string `json:"spec"` + Exact *string `json:"exact,omitempty"` + Allows []string `json:"allows,omitempty"` + Refuses []string `json:"refuses,omitempty"` +} + func TestABareVersionIsExact(t *testing.T) { - sp := spec(t, v071) - if sp.Exact() != v071 { - t.Errorf("Exact = %q, want 0.7.1", sp.Exact()) - } - if sp.Allows(v072) || !sp.Allows(v071) { - t.Error("0.7.1 must allow 0.7.1 and only it") - } - if spec(t, ">=0.7.1").Exact() != "" { - t.Error("a range names no one version") - } + cases.Run(t, "testdata/version/bare", func(t *testing.T, tc bareCase) { + sp := spec(t, tc.Spec) + if tc.Exact == nil { + t.Fatal("exact is omitted; set it to the one version or to an empty string") + } + if sp.Exact() != *tc.Exact { + if *tc.Exact == "" { + t.Error("a range names no one version") + } else { + t.Errorf("Exact = %q, want %s", sp.Exact(), *tc.Exact) + } + } + bad := false + for _, v := range tc.Allows { + if !sp.Allows(v) { + bad = true + } + } + for _, v := range tc.Refuses { + if sp.Allows(v) { + bad = true + } + } + if bad { + t.Errorf("%s must allow %v and not %v", tc.Spec, tc.Allows, tc.Refuses) + } + }) +} + +type parseCase struct { + Name string `json:"name,omitempty"` + Description string `json:"description,omitempty"` + Spec string `json:"spec"` } func TestParseSpecRefusesWhatIsNotAVersion(t *testing.T) { - for _, s := range []string{"", "latest", ">=banana"} { - if _, err := ParseSpec(s); err == nil { - t.Errorf("%q: want an error", s) + cases.Run(t, "testdata/version/parse", func(t *testing.T, tc parseCase) { + if _, err := ParseSpec(tc.Spec); err == nil { + t.Errorf("%q: want an error", tc.Spec) } - } + }) } -func TestMatchIsNewestFirstAndSkipsNonVersions(t *testing.T) { - got := spec(t, ">=0.7.1").Match([]string{"0.7.0", v071, "0.10.0", "latest", v080, "0.9.0-rc1", v080}) - want := []string{"0.10.0", v080, v071} - if !reflect.DeepEqual(got, want) { - t.Errorf("got %v, want %v -- newest first, no prerelease, no duplicates", got, want) - } - if got := spec(t, "^0.7.1").Match([]string{v071, "0.7.9", v080}); !reflect.DeepEqual(got, []string{"0.7.9", v071}) { - t.Errorf("^0.7.1 matched %v, want 0.7.x only", got) - } +type matchCase struct { + Name string `json:"name,omitempty"` + Description string `json:"description,omitempty"` + Spec string `json:"spec"` + Have []string `json:"have"` + Want []string `json:"want"` } -func list(vs ...string) Lister { - return func(context.Context) ([]string, error) { return vs, nil } +func TestMatchIsNewestFirstAndSkipsNonVersions(t *testing.T) { + cases.Run(t, "testdata/version/match", func(t *testing.T, tc matchCase) { + got := spec(t, tc.Spec).Match(tc.Have) + if !reflect.DeepEqual(got, tc.Want) { + t.Errorf("got %v, want %v -- %s", got, tc.Want, tc.Description) + } + }) } -func noList(t *testing.T) Lister { - return func(context.Context) ([]string, error) { - t.Error("the repository must not be read") - return nil, errors.New("read") - } +// resolveCase is one Resolve call. +// Exactly one of noList, listNil, and list is set. +// noList fails the test if the repository is read. +// listNil passes a nil Lister. +// list, including an empty array, is returned as the catalog. +type resolveCase struct { + Name string `json:"name,omitempty"` + Description string `json:"description,omitempty"` + Spec string `json:"spec"` + Want string `json:"want,omitempty"` + Keep string `json:"keep,omitempty"` + Got string `json:"got,omitempty"` + NoList bool `json:"noList,omitempty"` + ListNil bool `json:"listNil,omitempty"` + List *[]string `json:"list,omitempty"` + ErrorContains string `json:"errorContains,omitempty"` } -func TestResolve(t *testing.T) { - ctx := t.Context() - for _, tc := range []struct { - name, spec, want, keep, got string - list Lister - }{ - {name: "named, in range", spec: ">=0.7.1", want: "0.7.3", got: "0.7.3", list: noList(t)}, - {name: "exact needs no registry", spec: v071, got: v071, list: noList(t)}, - {name: "exact moves a running one off", spec: v072, keep: v071, got: v072, list: noList(t)}, - {name: "running one kept while in range", spec: ">=0.7.1", keep: v072, got: v072, list: noList(t)}, - {name: "running one out of range", spec: ">=0.8.0", keep: v072, got: "0.9.0", list: list(v072, v080, "0.9.0")}, - {name: "newest in range", spec: ">=0.7.1 <0.9.0", got: "0.8.1", list: list(v071, "0.8.1", "0.9.0")}, - } { - got, err := Resolve(ctx, spec(t, tc.spec), tc.want, tc.keep, tc.list) - if err != nil || got != tc.got { - t.Errorf("%s: got %q, %v; want %q", tc.name, got, err, tc.got) +func (tc resolveCase) lister(t *testing.T) Lister { + t.Helper() + n := 0 + if tc.NoList { + n++ + } + if tc.ListNil { + n++ + } + if tc.List != nil { + n++ + } + if n != 1 { + t.Fatal("set exactly one of noList, listNil, and list") + } + if tc.NoList { + return func(context.Context) ([]string, error) { + t.Error("the repository must not be read") + return nil, errors.New("read") } } + if tc.ListNil { + return nil + } + vs := *tc.List + return func(context.Context) ([]string, error) { return vs, nil } } -func TestResolveRefuses(t *testing.T) { - ctx := t.Context() - if _, err := Resolve(ctx, spec(t, ">=0.7.1"), "0.6.0", "", noList(t)); err == nil || !strings.Contains(err.Error(), "not in the catalog's") { - t.Errorf("a named version out of range: %v", err) +func runResolve(t *testing.T, tc resolveCase) { + t.Helper() + if tc.ErrorContains == "" && tc.Got == "" { + t.Fatal("set got or errorContains") } - if _, err := Resolve(ctx, spec(t, ">=0.7.1"), "", "", list("0.6.0")); err == nil || !strings.Contains(err.Error(), "no chart version satisfies") { - t.Errorf("nothing in range: %v", err) + got, err := Resolve(t.Context(), spec(t, tc.Spec), tc.Want, tc.Keep, tc.lister(t)) + if tc.ErrorContains != "" { + if err == nil || !strings.Contains(err.Error(), tc.ErrorContains) { + t.Errorf("%s: got %q, %v; want an error with %q", tc.Description, got, err, tc.ErrorContains) + } + return } - if _, err := Resolve(ctx, spec(t, ">=0.7.1"), "", "", nil); err == nil || !strings.Contains(err.Error(), "Name a chart version") { - t.Errorf("a range with nowhere to list: %v", err) + if err != nil || got != tc.Got { + t.Errorf("%s: got %q, %v; want %q", tc.Description, got, err, tc.Got) } } + +func TestResolve(t *testing.T) { + cases.Run(t, "testdata/version/resolve", runResolve) +} + +func TestResolveRefuses(t *testing.T) { + cases.Run(t, "testdata/version/resolve-refuses", runResolve) +} diff --git a/internal/controller/autoscaling/customprovider_test.go b/internal/controller/autoscaling/customprovider_test.go index 386d0cd..fc16894 100644 --- a/internal/controller/autoscaling/customprovider_test.go +++ b/internal/controller/autoscaling/customprovider_test.go @@ -27,237 +27,219 @@ import ( "k8s.io/apimachinery/pkg/types" autoscalingv1alpha1 "github.com/modelsphere/llm-operator/api/autoscaling/v1alpha1" + "github.com/modelsphere/llm-operator/internal/testutil/cases" ) const ( // sampleServiceID is the serviceId from the decision server's own example. sampleServiceID = "fallback-modelforge-01" - - // otherNamespace stands in for a second namespace reporting the same - // serviceId, which is the case customProvider.namespace exists to resolve. - otherNamespace = "staging" ) -// decisionBody wraps decisions in the envelope the server sends, so each test -// only has to spell out the part it is about. Format verbs in decisions are -// filled from args. -func decisionBody(decisions string, args ...any) string { - return fmt.Sprintf(`{"apiVersion":%q,"decisions":[%s]}`, - decisionsAPIVersion, fmt.Sprintf(decisions, args...)) +type decisionFile struct { + Name string `json:"name,omitempty"` + Description string `json:"description,omitempty"` + Inputs struct { + ServiceID string `json:"serviceId"` + Namespace string `json:"namespace"` + Body string `json:"body"` + Status int `json:"status"` + PinAPIVersion bool `json:"pinAPIVersion"` + } `json:"inputs"` + Expected struct { + Replicas int32 `json:"replicas"` + ErrorContains string `json:"errorContains"` + } `json:"expected"` } -// TestQueryDecisionReplicas covers how a decision payload is resolved to a -// replica count: the schema must be one we know, exactly one decision must -// address us, and the count must be present and sane. Everything else is an -// error the caller skips the sync on rather than a zero. func TestQueryDecisionReplicas(t *testing.T) { - tests := []struct { - name string - namespace string - body string - want int32 - wantErr string - }{ - { - name: "a matching decision resolves to its active count", - body: decisionBody(`{"namespace":%q,"serviceId":"svc","replicas":{"active":15}}`, decisionNamespace), - want: 15, - }, - { - name: "zero is a decision, not a missing one", - body: decisionBody(`{"namespace":%q,"serviceId":"svc","replicas":{"active":0}}`, decisionNamespace), - want: 0, - }, - { - name: "a decision with no active count is not a zero", - body: decisionBody(`{"namespace":%q,"serviceId":"svc","replicas":{}}`, decisionNamespace), - wantErr: "carries no replicas.active", - }, - { - name: "a negative count is refused", - body: decisionBody(`{"serviceId":"svc","replicas":{"active":-1}}`), - wantErr: "out of range", - }, - { - name: "a count past int32 is refused rather than wrapped", - body: decisionBody(`{"serviceId":"svc","replicas":{"active":4294967296}}`), - wantErr: "out of range", - }, - { - name: "an unknown schema is refused, not parsed hopefully", - body: `{"apiVersion":"llmscaling.inference.x-k8s.io/v2","decisions":[{"serviceId":"svc","replicas":{"active":3}}]}`, - wantErr: "unsupported decision schema", - }, - { - name: "a payload with no apiVersion is refused", - body: `{"decisions":[{"serviceId":"svc","replicas":{"active":3}}]}`, - wantErr: "unsupported decision schema", - }, - { - name: "an empty decision list is not a zero", - body: decisionBody(``), - wantErr: `no decision for serviceId "svc"`, - }, - { - // The server answers with what it likes; a decision for someone else - // must not drive this scaler just because it came back. - name: "a decision for another service does not match", - body: decisionBody(`{"namespace":%q,"serviceId":"other","replicas":{"active":9}}`, decisionNamespace), - wantErr: `no decision for serviceId "svc"`, - }, - { - name: "the namespace filter picks between same-named services", - body: decisionBody(`{"namespace":%q,"serviceId":"svc","replicas":{"active":2}}, - {"namespace":%q,"serviceId":"svc","replicas":{"active":7}}`, - otherNamespace, decisionNamespace), - namespace: decisionNamespace, - want: 7, - }, - { - name: "duplicates are an error, not a pick-first", - body: decisionBody(`{"namespace":%q,"serviceId":"svc","replicas":{"active":2}}, - {"namespace":%q,"serviceId":"svc","replicas":{"active":7}}`, - otherNamespace, decisionNamespace), - wantErr: "2 decisions match", - }, - { - name: "the namespace filter is reported when nothing matches", - body: decisionBody(`{"namespace":%q,"serviceId":"svc","replicas":{"active":2}}`, otherNamespace), - namespace: decisionNamespace, - wantErr: fmt.Sprintf("in namespace %q", decisionNamespace), - }, - { - name: "a body that is not JSON is an error", - body: `502 Bad Gateway`, - wantErr: "failed to decode", - }, - } - - for _, tt := range tests { - srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { - _, _ = w.Write([]byte(tt.body)) - })) - - got, err := queryDecisionReplicas(srv.URL, "", "svc", tt.namespace, nil) - switch { - case tt.wantErr == "" && err != nil: - t.Errorf("%s: unexpected error: %v", tt.name, err) - case tt.wantErr == "" && got != tt.want: - t.Errorf("%s: got %d, want %d", tt.name, got, tt.want) - case tt.wantErr != "" && err == nil: - t.Errorf("%s: expected an error containing %q, got value %d", tt.name, tt.wantErr, got) - case tt.wantErr != "" && !strings.Contains(err.Error(), tt.wantErr): - t.Errorf("%s: error = %q, want it to contain %q", tt.name, err, tt.wantErr) - } - srv.Close() - } + cases.Run(t, "testdata/helpers/decision-replicas", func(t *testing.T, tc decisionFile) { + assertDecision(t, tc) + }) } // TestQueryDecisionReplicasSamplePayload runs a real reply from the decision // server through the client verbatim, so the parser is pinned against the wire // format and not only against the fixtures the table above builds. func TestQueryDecisionReplicasSamplePayload(t *testing.T) { + cases.Run(t, "testdata/helpers/decision-sample", func(t *testing.T, tc decisionFile) { + if tc.Inputs.PinAPIVersion && !strings.Contains(tc.Inputs.Body, decisionsAPIVersion) { + t.Fatalf("sample body must carry apiVersion %s", decisionsAPIVersion) + } + assertDecision(t, tc) + }) +} + +func assertDecision(t *testing.T, tc decisionFile) { + t.Helper() + // Zero is an omitted status, not HTTP 200. WriteHeader panics on it. + if tc.Inputs.Status < 100 || tc.Inputs.Status > 999 { + t.Fatalf("status is required, got %d", tc.Inputs.Status) + } srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { - _, _ = w.Write([]byte(`{ - "apiVersion": "llmscaling.inference.x-k8s.io/v1alpha1", - "decisions": [ - { - "namespace": "modelforge", - "serviceId": "fallback-modelforge-01", - "replicas": { - "active": 15 - } - } - ] -}`)) + w.WriteHeader(tc.Inputs.Status) + _, _ = w.Write([]byte(tc.Inputs.Body)) })) defer srv.Close() - got, err := queryDecisionReplicas(srv.URL, "", sampleServiceID, "", nil) - if err != nil { - t.Fatalf("unexpected error: %v", err) + got, err := queryDecisionReplicas(srv.URL, "", tc.Inputs.ServiceID, tc.Inputs.Namespace, nil) + // An empty errorContains means the call succeeds. A case that expects a + // failure names the substring; an empty string would match every error. + if tc.Expected.ErrorContains == "" { + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if got != tc.Expected.Replicas { + t.Fatalf("got %d, want %d", got, tc.Expected.Replicas) + } + return } - if got != 15 { - t.Errorf("got %d replicas, want 15", got) + if err == nil || !strings.Contains(err.Error(), tc.Expected.ErrorContains) { + t.Fatalf("error = %v, want it to contain %q", err, tc.Expected.ErrorContains) } } +type decisionRequestFile struct { + Name string `json:"name,omitempty"` + Description string `json:"description,omitempty"` + Inputs struct { + ServiceID string `json:"serviceId"` + Path string `json:"path"` + TrailingSlash bool `json:"trailingSlash"` + Headers map[string]string `json:"headers"` + } `json:"inputs"` + Expected struct { + Replicas int32 `json:"replicas"` + Path string `json:"path"` + ServiceID string `json:"serviceId"` + Authorization string `json:"authorization"` + PinDefaultPath bool `json:"pinDefaultPath"` + } `json:"expected"` +} + // TestQueryDecisionReplicasRequest pins the request the server actually sees: // the documented path and query parameter, plus any configured headers. func TestQueryDecisionReplicasRequest(t *testing.T) { - tests := []struct { - name string - path string - trailingSlash bool - wantPath string - }{ - {name: "an empty path falls back to the default", wantPath: defaultDecisionsPath}, - {name: "a configured path is used", path: "/v2/decisions", wantPath: "/v2/decisions"}, - {name: "a path without a leading slash still forms one", path: "decisions", wantPath: defaultDecisionsPath}, - {name: "a trailing slash on serverAddress is not doubled", path: defaultDecisionsPath, trailingSlash: true, wantPath: defaultDecisionsPath}, - } - - for _, tt := range tests { + cases.Run(t, "testdata/helpers/decision-request", func(t *testing.T, tc decisionRequestFile) { + if tc.Expected.PinDefaultPath && tc.Expected.Path != defaultDecisionsPath { + t.Fatalf("path %q, want default %q", tc.Expected.Path, defaultDecisionsPath) + } var gotPath, gotServiceID, gotAuth string srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { gotPath = req.URL.Path gotServiceID = req.URL.Query().Get("serviceId") gotAuth = req.Header.Get("Authorization") - _, _ = w.Write([]byte(decisionBody(`{"serviceId":%q,"replicas":{"active":4}}`, sampleServiceID))) + _, _ = fmt.Fprintf(w, `{"apiVersion":%q,"decisions":[{"serviceId":%q,"replicas":{"active":%d}}]}`, + decisionsAPIVersion, tc.Inputs.ServiceID, tc.Expected.Replicas) })) + defer srv.Close() address := srv.URL - if tt.trailingSlash { + if tc.Inputs.TrailingSlash { address += "/" } - got, err := queryDecisionReplicas(address, tt.path, sampleServiceID, "", - map[string]string{"Authorization": "Bearer t"}) + got, err := queryDecisionReplicas(address, tc.Inputs.Path, tc.Inputs.ServiceID, "", tc.Inputs.Headers) if err != nil { - t.Errorf("%s: unexpected error: %v", tt.name, err) + t.Fatalf("unexpected error: %v", err) } - if got != 4 { - t.Errorf("%s: got %d replicas, want 4", tt.name, got) + if got != tc.Expected.Replicas { + t.Fatalf("got %d replicas, want %d", got, tc.Expected.Replicas) } - if gotPath != tt.wantPath { - t.Errorf("%s: path = %q, want %q", tt.name, gotPath, tt.wantPath) + if gotPath != tc.Expected.Path { + t.Fatalf("path = %q, want %q", gotPath, tc.Expected.Path) } - if gotServiceID != sampleServiceID { - t.Errorf("%s: serviceId = %q, want %q", tt.name, gotServiceID, sampleServiceID) + if gotServiceID != tc.Expected.ServiceID { + t.Fatalf("serviceId = %q, want %q", gotServiceID, tc.Expected.ServiceID) } - if gotAuth != "Bearer t" { - t.Errorf("%s: Authorization = %q, want %q", tt.name, gotAuth, "Bearer t") + if gotAuth != tc.Expected.Authorization { + t.Fatalf("Authorization = %q, want %q", gotAuth, tc.Expected.Authorization) } - srv.Close() - } + }) +} + +type decisionNon200File struct { + Name string `json:"name,omitempty"` + Description string `json:"description,omitempty"` + Inputs struct { + ServiceID string `json:"serviceId"` + Status int `json:"status"` + Body string `json:"body"` + } `json:"inputs"` + Expected struct { + ErrorContains string `json:"errorContains"` + } `json:"expected"` } // TestQueryDecisionReplicasNon200 checks that a non-OK reply is an error even // when its body happens to be a well-formed decision payload. func TestQueryDecisionReplicasNon200(t *testing.T) { - srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { - w.WriteHeader(http.StatusServiceUnavailable) - _, _ = w.Write([]byte(decisionBody(`{"serviceId":"svc","replicas":{"active":3}}`))) - })) - defer srv.Close() + cases.Run(t, "testdata/helpers/decision-non-200", func(t *testing.T, tc decisionNon200File) { + // Zero is an omitted status, and WriteHeader panics on it. + if tc.Inputs.Status < 100 || tc.Inputs.Status > 999 { + t.Fatalf("status is required, got %d", tc.Inputs.Status) + } + // An empty substring matches every error. + if tc.Expected.ErrorContains == "" { + t.Fatal("errorContains is required") + } + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(tc.Inputs.Status) + _, _ = w.Write([]byte(tc.Inputs.Body)) + })) + defer srv.Close() - _, err := queryDecisionReplicas(srv.URL, "", "svc", "", nil) - if err == nil || !strings.Contains(err.Error(), "status 503") { - t.Errorf("error = %v, want it to mention status 503", err) - } + _, err := queryDecisionReplicas(srv.URL, "", tc.Inputs.ServiceID, "", nil) + if err == nil || !strings.Contains(err.Error(), tc.Expected.ErrorContains) { + t.Errorf("error = %v, want it to contain %q", err, tc.Expected.ErrorContains) + } + }) +} + +type decisionMisconfigFile struct { + Name string `json:"name,omitempty"` + Description string `json:"description,omitempty"` + Inputs struct { + ServerAddress string `json:"serverAddress"` + ServiceID string `json:"serviceId"` + } `json:"inputs"` + Expected struct { + ErrorContains string `json:"errorContains"` + } `json:"expected"` } // TestQueryDecisionReplicasMisconfigured covers the two settings that make a // request impossible to build. They are caught before the round trip so the // error names the field rather than the URL. func TestQueryDecisionReplicasMisconfigured(t *testing.T) { - if _, err := queryDecisionReplicas("", "", "svc", "", nil); err == nil || - !strings.Contains(err.Error(), "serverAddress") { - t.Errorf("empty serverAddress: error = %v, want it to name serverAddress", err) - } - if _, err := queryDecisionReplicas("http://example.invalid", "", "", "", nil); err == nil || - !strings.Contains(err.Error(), "serviceId") { - t.Errorf("empty serviceId: error = %v, want it to name serviceId", err) - } + cases.Run(t, "testdata/helpers/decision-misconfigured", func(t *testing.T, tc decisionMisconfigFile) { + // An empty substring matches every error. + if tc.Expected.ErrorContains == "" { + t.Fatal("errorContains is required") + } + _, err := queryDecisionReplicas(tc.Inputs.ServerAddress, "", tc.Inputs.ServiceID, "", nil) + if err == nil || !strings.Contains(err.Error(), tc.Expected.ErrorContains) { + t.Errorf("error = %v, want it to contain %q", err, tc.Expected.ErrorContains) + } + }) +} + +type recommendCustomFile struct { + Name string `json:"name,omitempty"` + Description string `json:"description,omitempty"` + Inputs struct { + Active string `json:"active"` + MinReplicas int32 `json:"minReplicas"` + MaxReplicas int32 `json:"maxReplicas"` + ReadyReplicas int64 `json:"readyReplicas"` + SpecReplicas int64 `json:"specReplicas"` + ServiceID string `json:"serviceId"` + Namespace string `json:"namespace"` + APIVersion string `json:"apiVersion"` + PinAPIVersion bool `json:"pinAPIVersion"` + AbsentCustomProvider bool `json:"absentCustomProvider"` + } `json:"inputs"` + Expected struct { + Replicas int32 `json:"replicas"` + } `json:"expected"` } // TestRecommendReplicasCustomProvider pins the semantic that separates the two @@ -266,75 +248,63 @@ func TestQueryDecisionReplicasMisconfigured(t *testing.T) { // Prometheus sample is. The CRD bounds still apply on top of it — the server // recommends, this operator keeps the rail. func TestRecommendReplicasCustomProvider(t *testing.T) { - tests := []struct { - name string - active string - min, max int32 - readyReplicas int64 - want int32 - unknownAPIVersion bool - absentCustomConfig bool - }{ - { - name: "the decision is used as-is, not scaled by readyReplicas", - // 8 ready replicas would turn a per-replica metric into a much larger - // number; a decision of 15 is already the answer. - active: "15", min: 1, max: 20, readyReplicas: 8, want: 15, - }, - { - name: "a decision above maxReplicas is clamped", - active: "15", min: 1, max: 5, readyReplicas: 8, want: 5, - }, - { - name: "a decision of zero clamps up to minReplicas", - active: "0", min: 2, max: 20, readyReplicas: 8, want: 2, - }, - { - name: "an unreadable decision holds the fleet where it is", - active: "15", min: 1, max: 20, readyReplicas: 8, want: 8, - unknownAPIVersion: true, - }, - { - name: "a missing customProvider holds the fleet where it is", - active: "15", min: 1, max: 20, readyReplicas: 8, want: 8, - absentCustomConfig: true, - }, - } - - for _, tt := range tests { - apiVersion := decisionsAPIVersion - if tt.unknownAPIVersion { - apiVersion = "some.other.api/v1" + cases.Run(t, "testdata/helpers/recommend-custom", func(t *testing.T, tc recommendCustomFile) { + in := tc.Inputs + if in.PinAPIVersion && in.APIVersion != decisionsAPIVersion { + t.Fatalf("apiVersion %q, want %q", in.APIVersion, decisionsAPIVersion) } srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { - _, _ = w.Write(fmt.Appendf(nil, - `{"apiVersion":%q,"decisions":[{"namespace":%q,"serviceId":"svc","replicas":{"active":%s}}]}`, - apiVersion, decisionNamespace, tt.active)) + _, _ = fmt.Fprintf(w, + `{"apiVersion":%q,"decisions":[{"namespace":%q,"serviceId":%q,"replicas":{"active":%s}}]}`, + in.APIVersion, in.Namespace, in.ServiceID, in.Active) })) + defer srv.Close() scaler := &autoscalingv1alpha1.LLMScaler{ Spec: autoscalingv1alpha1.LLMScalerSpec{ MetricProvider: autoscalingv1alpha1.MetricProviderCustom, ServerAddress: srv.URL, - CustomProvider: &autoscalingv1alpha1.CustomProviderSpec{ServiceID: "svc"}, - MinReplicas: tt.min, - MaxReplicas: tt.max, + CustomProvider: &autoscalingv1alpha1.CustomProviderSpec{ServiceID: in.ServiceID}, + MinReplicas: in.MinReplicas, + MaxReplicas: in.MaxReplicas, }, } - if tt.absentCustomConfig { + if in.AbsentCustomProvider { scaler.Spec.CustomProvider = nil } r := &LLMScalerReconciler{} key := types.NamespacedName{Namespace: "ns", Name: "s"} - // specReplicas is what "hold where it is" means, so it is set to the same - // figure as readyReplicas to keep the failure cases unambiguous. - got := r.recommendReplicas(context.Background(), scaler, key, tt.readyReplicas, tt.readyReplicas) - if got != tt.want { - t.Errorf("%s: recommendReplicas = %d, want %d", tt.name, got, tt.want) + // specReplicas is what "hold where it is" means. The files set it to the + // same figure as readyReplicas so a hold is not confused with a clamp. + got := r.recommendReplicas(context.Background(), scaler, key, in.ReadyReplicas, in.SpecReplicas) + if got != tc.Expected.Replicas { + t.Fatalf("recommendReplicas = %d, want %d", got, tc.Expected.Replicas) } - srv.Close() - } + }) +} + +type metricInput struct { + Name string `json:"name"` + Query string `json:"query"` + Target string `json:"target"` +} + +type recommendPromFile struct { + Name string `json:"name,omitempty"` + Description string `json:"description,omitempty"` + Inputs struct { + Sample string `json:"sample"` + ReadyReplicas int64 `json:"readyReplicas"` + SpecReplicas int64 `json:"specReplicas"` + MinReplicas int32 `json:"minReplicas"` + MaxReplicas int32 `json:"maxReplicas"` + Metrics []metricInput `json:"metrics"` + } `json:"inputs"` + Expected struct { + Replicas int32 `json:"replicas"` + Path string `json:"path"` + } `json:"expected"` } // TestRecommendReplicasDefaultProviderIsPrometheus checks that an LLMScaler with @@ -342,31 +312,35 @@ func TestRecommendReplicasCustomProvider(t *testing.T) { // Prometheus at admission, but objects reaching the controller from a test or an // older CRD may carry it empty. func TestRecommendReplicasDefaultProviderIsPrometheus(t *testing.T) { - var gotPath string - srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { - gotPath = req.URL.Path - _, _ = w.Write([]byte(`{"status":"success","data":{"result":[{"metric":{},"value":[0,"1.6"]}]}}`)) - })) - defer srv.Close() + cases.Run(t, "testdata/helpers/recommend-default-prometheus", func(t *testing.T, tc recommendPromFile) { + var gotPath string + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + gotPath = req.URL.Path + _, _ = fmt.Fprintf(w, `{"status":"success","data":{"result":[{"metric":{},"value":[0,"%s"]}]}}`, tc.Inputs.Sample) + })) + defer srv.Close() - scaler := &autoscalingv1alpha1.LLMScaler{ - Spec: autoscalingv1alpha1.LLMScalerSpec{ - ServerAddress: srv.URL, - MinReplicas: 1, - MaxReplicas: 20, - Metrics: []autoscalingv1alpha1.MetricSpec{ - {Name: "kv-cache", Query: "avg(kv)", Target: "0.8"}, + metrics := make([]autoscalingv1alpha1.MetricSpec, len(tc.Inputs.Metrics)) + for i, m := range tc.Inputs.Metrics { + metrics[i] = autoscalingv1alpha1.MetricSpec{Name: m.Name, Query: m.Query, Target: m.Target} + } + scaler := &autoscalingv1alpha1.LLMScaler{ + Spec: autoscalingv1alpha1.LLMScalerSpec{ + ServerAddress: srv.URL, + MinReplicas: tc.Inputs.MinReplicas, + MaxReplicas: tc.Inputs.MaxReplicas, + Metrics: metrics, }, - }, - } + } - r := &LLMScalerReconciler{} - key := types.NamespacedName{Namespace: "ns", Name: "s"} - // ceil(4 * 1.6/0.8) = 8, i.e. the per-replica ratio the Custom path skips. - if got := r.recommendReplicas(context.Background(), scaler, key, 4, 4); got != 8 { - t.Errorf("recommendReplicas = %d, want 8", got) - } - if gotPath != "/api/v1/query" { - t.Errorf("path = %q, want the Prometheus query API", gotPath) - } + r := &LLMScalerReconciler{} + key := types.NamespacedName{Namespace: "ns", Name: "s"} + got := r.recommendReplicas(context.Background(), scaler, key, tc.Inputs.ReadyReplicas, tc.Inputs.SpecReplicas) + if got != tc.Expected.Replicas { + t.Errorf("recommendReplicas = %d, want %d", got, tc.Expected.Replicas) + } + if gotPath != tc.Expected.Path { + t.Errorf("path = %q, want %q", gotPath, tc.Expected.Path) + } + }) } diff --git a/internal/controller/autoscaling/guards_cases_test.go b/internal/controller/autoscaling/guards_cases_test.go deleted file mode 100644 index bf9d5e2..0000000 --- a/internal/controller/autoscaling/guards_cases_test.go +++ /dev/null @@ -1,404 +0,0 @@ -/* -Copyright 2026. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package controller - -import ( - autoscalingv1alpha1 "github.com/modelsphere/llm-operator/api/autoscaling/v1alpha1" -) - -// guardHoldCases run one guard over a WorkloadState and a ScaleIntent. -// checkTerminating is written onto the package flag for the row and restored. -// Rollout rows cover each in-progress signal, a finished rollout, a -// non-Deployment, and a hold in each direction. Settle rows cover direction, -// a ready count that has not caught up (including 0), and the terminating -// gate on and off. -var guardHoldCases = []guardHoldCase{ - { - name: "rollout: generation not observed holds a scale-up", - guard: "rollout", - state: WorkloadState{ - Spec: 5, Total: 5, Updated: 5, Ready: 5, - Generation: 4, ObservedGeneration: 3, - TracksRollout: true, - }, - intent: ScaleIntent{From: 5, To: 7}, - wantHeld: true, - wantReason: autoscalingv1alpha1.ReasonRolloutInProgress, - wantWhy: msgRolloutInProgress, - }, - { - name: "rollout: updated replicas behind spec holds a scale-down", - guard: "rollout", - state: WorkloadState{ - Spec: 5, Total: 4, Updated: 4, Ready: 4, - Generation: 3, ObservedGeneration: 3, - TracksRollout: true, - }, - intent: ScaleIntent{From: 5, To: 3}, - wantHeld: true, - wantReason: autoscalingv1alpha1.ReasonRolloutInProgress, - wantWhy: msgRolloutInProgress, - }, - { - name: "rollout: old-template pods hold a no-op", - guard: "rollout", - state: WorkloadState{ - Spec: 5, Total: 6, Updated: 5, Ready: 5, - Generation: 3, ObservedGeneration: 3, - TracksRollout: true, - }, - intent: ScaleIntent{From: 5, To: 5}, - wantHeld: true, - wantReason: autoscalingv1alpha1.ReasonRolloutInProgress, - wantWhy: msgRolloutInProgress, - }, - { - name: "rollout: fully rolled out does not hold", - guard: "rollout", - state: WorkloadState{ - Spec: 5, Total: 5, Updated: 5, Ready: 5, - Generation: 3, ObservedGeneration: 3, - TracksRollout: true, - }, - intent: ScaleIntent{From: 5, To: 4}, - wantReason: autoscalingv1alpha1.ReasonRolloutInProgress, - }, - { - name: "rollout: non-Deployment is never held", - guard: "rollout", - state: WorkloadState{ - Spec: 5, Total: 6, Updated: 0, Ready: 5, - Generation: 4, ObservedGeneration: 1, - }, - intent: ScaleIntent{From: 5, To: 7}, - wantReason: autoscalingv1alpha1.ReasonRolloutInProgress, - }, - { - name: "settle: scale-up is not held", - guard: "settle", - state: WorkloadState{Spec: 5, Ready: 2}, - intent: ScaleIntent{From: 5, To: 7}, - checkTerminating: false, - wantReason: autoscalingv1alpha1.ReasonScaleDownSettling, - }, - { - name: "settle: no-op is not held", - guard: "settle", - state: WorkloadState{Spec: 5, Ready: 2}, - intent: ScaleIntent{From: 5, To: 5}, - checkTerminating: false, - wantReason: autoscalingv1alpha1.ReasonScaleDownSettling, - }, - { - name: "settle: ready differs from spec", - guard: "settle", - state: WorkloadState{Spec: 5, Ready: 3}, - intent: ScaleIntent{From: 5, To: 4}, - checkTerminating: false, - wantHeld: true, - wantReason: autoscalingv1alpha1.ReasonScaleDownSettling, - wantWhy: msgScaleDownNotSettled, - }, - { - // A ready count of 0 is the raw missing-field value. Treating it as - // spec would skip the guard. - name: "settle: ready count of 0 is not settled", - guard: "settle", - state: WorkloadState{Spec: 5, Ready: 0}, - intent: ScaleIntent{From: 5, To: 4}, - checkTerminating: false, - wantHeld: true, - wantReason: autoscalingv1alpha1.ReasonScaleDownSettling, - wantWhy: msgScaleDownNotSettled, - }, - { - name: "settle: ready back at spec", - guard: "settle", - state: WorkloadState{Spec: 5, Ready: 5}, - intent: ScaleIntent{From: 5, To: 4}, - checkTerminating: false, - wantReason: autoscalingv1alpha1.ReasonScaleDownSettling, - }, - { - name: "settle: terminating pods with the gate off", - guard: "settle", - state: WorkloadState{ - Spec: 5, Ready: 5, Terminating: 2, TerminatingReported: true, - }, - intent: ScaleIntent{From: 5, To: 4}, - checkTerminating: false, - wantReason: autoscalingv1alpha1.ReasonScaleDownSettling, - }, - { - name: "settle: terminating pods with the gate on", - guard: "settle", - state: WorkloadState{ - Spec: 5, Ready: 5, Terminating: 2, TerminatingReported: true, - }, - intent: ScaleIntent{From: 5, To: 4}, - checkTerminating: true, - wantHeld: true, - wantReason: autoscalingv1alpha1.ReasonScaleDownSettling, - wantWhy: msgScaleDownNotSettled, - }, - { - name: "settle: terminating count of 0 with the gate on", - guard: "settle", - state: WorkloadState{ - Spec: 5, Ready: 5, Terminating: 0, TerminatingReported: true, - }, - intent: ScaleIntent{From: 5, To: 4}, - checkTerminating: true, - wantReason: autoscalingv1alpha1.ReasonScaleDownSettling, - }, - { - // The count is ignored when the field was not reported. A non-zero - // Terminating here would still be settled; the parser stores 0 in - // that case, so this row is what makes the flag observable. - name: "settle: terminating not reported with the gate on", - guard: "settle", - state: WorkloadState{ - Spec: 5, Ready: 5, Terminating: 4, TerminatingReported: false, - }, - intent: ScaleIntent{From: 5, To: 4}, - checkTerminating: true, - wantReason: autoscalingv1alpha1.ReasonScaleDownSettling, - }, -} - -// guardOrderCases walk the guards slice. The first hold wins: a rollout is -// reported ahead of a scale-down that has not settled. -var guardOrderCases = []guardOrderCase{ - { - name: "rollout wins over an unsettled scale-down", - state: WorkloadState{ - Spec: 5, Total: 5, Updated: 5, Ready: 2, - Generation: 4, ObservedGeneration: 3, - TracksRollout: true, - }, - intent: ScaleIntent{From: 5, To: 3}, - wantReason: autoscalingv1alpha1.ReasonRolloutInProgress, - wantMessage: msgRolloutInProgress, - }, - { - name: "settle holds once the rollout is done", - state: WorkloadState{ - Spec: 5, Total: 5, Updated: 5, Ready: 3, - Generation: 3, ObservedGeneration: 3, - TracksRollout: true, - }, - intent: ScaleIntent{From: 5, To: 4}, - wantReason: autoscalingv1alpha1.ReasonScaleDownSettling, - wantMessage: msgScaleDownNotSettled, - }, - { - name: "no guard holds a settled scale-down", - state: WorkloadState{ - Spec: 5, Total: 5, Updated: 5, Ready: 5, - Generation: 3, ObservedGeneration: 3, - TracksRollout: true, - }, - intent: ScaleIntent{From: 5, To: 4}, - }, -} - -// workloadStateCases are readWorkloadState over each kind, then the missing -// and invalid-field defaults. A nil field is omitted. The four full-status -// rows share one set of numbers so the only per-kind difference is -// TracksRollout. -var workloadStateCases = []workloadStateCase{ - { - name: "Deployment with full status", - kind: deploymentKind, - fields: workloadObjectFields{ - spec: 4, ready: 3, total: 6, updated: 2, terminating: 1, - generation: 9, observed: 8, - }, - want: WorkloadState{ - Spec: 4, Total: 6, Updated: 2, Ready: 3, - Terminating: 1, TerminatingReported: true, - Generation: 9, ObservedGeneration: 8, - TracksRollout: true, - }, - }, - { - name: "StatefulSet with full status", - kind: "StatefulSet", - fields: workloadObjectFields{ - spec: 4, ready: 3, total: 6, updated: 2, terminating: 1, - generation: 9, observed: 8, - }, - want: WorkloadState{ - Spec: 4, Total: 6, Updated: 2, Ready: 3, - Terminating: 1, TerminatingReported: true, - Generation: 9, ObservedGeneration: 8, - }, - }, - { - name: "LeaderWorkerSet with full status", - kind: "LeaderWorkerSet", - fields: workloadObjectFields{ - spec: 4, ready: 3, total: 6, updated: 2, terminating: 1, - generation: 9, observed: 8, - }, - want: WorkloadState{ - Spec: 4, Total: 6, Updated: 2, Ready: 3, - Terminating: 1, TerminatingReported: true, - Generation: 9, ObservedGeneration: 8, - }, - }, - { - name: "generic kind with full status", - kind: "ReplicaSet", - fields: workloadObjectFields{ - spec: 4, ready: 3, total: 6, updated: 2, terminating: 1, - generation: 9, observed: 8, - }, - want: WorkloadState{ - Spec: 4, Total: 6, Updated: 2, Ready: 3, - Terminating: 1, TerminatingReported: true, - Generation: 9, ObservedGeneration: 8, - }, - }, - { - name: "missing spec.replicas reads as 1", - kind: deploymentKind, - fields: workloadObjectFields{ - ready: 3, total: 6, updated: 2, terminating: 1, - generation: 9, observed: 8, - }, - want: WorkloadState{ - Spec: 1, Total: 6, Updated: 2, Ready: 3, - Terminating: 1, TerminatingReported: true, - Generation: 9, ObservedGeneration: 8, - TracksRollout: true, - }, - }, - { - name: "spec.replicas of 0 is kept", - kind: deploymentKind, - fields: workloadObjectFields{ - spec: 0, ready: 0, total: 0, updated: 0, - generation: 2, observed: 2, - }, - want: WorkloadState{ - Spec: 0, Generation: 2, ObservedGeneration: 2, - TracksRollout: true, - }, - }, - { - name: "missing readyReplicas reads as 0", - kind: deploymentKind, - fields: workloadObjectFields{ - spec: 4, total: 6, updated: 2, terminating: 1, - generation: 9, observed: 8, - }, - want: WorkloadState{ - Spec: 4, Total: 6, Updated: 2, Ready: 0, - Terminating: 1, TerminatingReported: true, - Generation: 9, ObservedGeneration: 8, - TracksRollout: true, - }, - }, - { - name: "missing terminatingReplicas is not reported", - kind: deploymentKind, - fields: workloadObjectFields{ - spec: 4, ready: 3, total: 6, updated: 2, - generation: 9, observed: 8, - }, - want: WorkloadState{ - Spec: 4, Total: 6, Updated: 2, Ready: 3, - Generation: 9, ObservedGeneration: 8, - TracksRollout: true, - }, - }, - { - name: "missing generation, observedGeneration, updated, and total read as 0", - kind: deploymentKind, - fields: workloadObjectFields{ - spec: 4, ready: 3, terminating: 1, - }, - want: WorkloadState{ - Spec: 4, Ready: 3, - Terminating: 1, TerminatingReported: true, - TracksRollout: true, - }, - }, - { - name: "invalid-typed fields behave like missing", - kind: deploymentKind, - fields: workloadObjectFields{ - spec: "nope", ready: "nope", total: "nope", updated: "nope", - terminating: "nope", generation: "nope", observed: "nope", - }, - want: WorkloadState{Spec: 1, TracksRollout: true}, - }, -} - -// targetGuardCases are the guard result the controller turns into a -// condition, starting from a target object rather than a WorkloadState. -// readyReplicas is the raw count: 0 does not count as settled. -// gen/observed/updated/spec/ready build the object. status.replicas stays -// at 5 inside the builder. The runner parses that object and evaluates guards. -var targetGuardCases = []targetGuardCase{ - { - name: "rollout holds a scale-up", - gen: 4, observed: 3, updated: 5, - kind: deploymentKind, spec: 5, ready: 5, target: 7, - wantReason: autoscalingv1alpha1.ReasonRolloutInProgress, - wantMessage: msgRolloutInProgress, - }, - { - name: "rollout wins over an unsettled scale-down", - gen: 4, observed: 3, updated: 5, - kind: deploymentKind, spec: 5, ready: 2, target: 3, - wantReason: autoscalingv1alpha1.ReasonRolloutInProgress, - wantMessage: msgRolloutInProgress, - }, - { - name: "unsettled scale-down", - gen: 3, observed: 3, updated: 5, // fully rolled out - kind: deploymentKind, spec: 5, ready: 3, target: 4, - wantReason: autoscalingv1alpha1.ReasonScaleDownSettling, - wantMessage: msgScaleDownNotSettled, - }, - { - // readyReplicas 0 is also what a missing field reads as. Treating it - // as spec would skip the guard. - name: "ready count of 0 is not settled", - gen: 3, observed: 3, updated: 5, - kind: deploymentKind, spec: 5, ready: 0, target: 4, - wantReason: autoscalingv1alpha1.ReasonScaleDownSettling, - wantMessage: msgScaleDownNotSettled, - }, - { - name: "settled scale-down may proceed", - gen: 3, observed: 3, updated: 5, - kind: deploymentKind, spec: 5, ready: 5, target: 4, - }, - { - name: "scale-up is not gated on settle", - gen: 3, observed: 3, updated: 5, - kind: deploymentKind, spec: 5, ready: 2, target: 6, - }, - { - name: "non-Deployment ignores the rollout guard", - gen: 4, observed: 1, updated: 0, - kind: "StatefulSet", spec: 5, ready: 5, target: 7, - }, -} diff --git a/internal/controller/autoscaling/guards_test.go b/internal/controller/autoscaling/guards_test.go index 0316ca6..13919d0 100644 --- a/internal/controller/autoscaling/guards_test.go +++ b/internal/controller/autoscaling/guards_test.go @@ -17,39 +17,68 @@ limitations under the License. package controller import ( + "math" "testing" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + + "github.com/modelsphere/llm-operator/internal/testutil/cases" ) -type guardHoldCase struct { - name string - guard string - state WorkloadState - intent ScaleIntent - checkTerminating bool - wantHeld bool - wantReason string - wantWhy string +// stateInput is the WorkloadState a guard case feeds Hold or evaluateGuards. +// Zero fields are left at the Go zero, which is what an omitted YAML key means. +type stateInput struct { + Spec int32 `json:"spec"` + Total int32 `json:"total"` + Updated int32 `json:"updated"` + Ready int32 `json:"ready"` + Terminating int32 `json:"terminating"` + TerminatingReported bool `json:"terminatingReported"` + Generation int64 `json:"generation"` + ObservedGeneration int64 `json:"observedGeneration"` + TracksRollout bool `json:"tracksRollout"` +} + +func (s stateInput) state() WorkloadState { return WorkloadState(s) } + +type intentInput struct { + From int32 `json:"from"` + To int32 `json:"to"` +} + +func (in intentInput) intent() ScaleIntent { return ScaleIntent(in) } + +type guardHoldFile struct { + Name string `json:"name,omitempty"` + Description string `json:"description,omitempty"` + Inputs struct { + Guard string `json:"guard"` + State stateInput `json:"state"` + Intent intentInput `json:"intent"` + CheckTerminating bool `json:"checkTerminating"` + } `json:"inputs"` + Expected struct { + Held bool `json:"held"` + Reason string `json:"reason"` + Why string `json:"why"` + } `json:"expected"` } func TestGuardHold(t *testing.T) { - for _, tc := range guardHoldCases { - t.Run(tc.name, func(t *testing.T) { - setTerminatingGate(t, tc.checkTerminating) - g, ok := guardByName(tc.guard) - if !ok { - t.Fatalf("unknown guard %q", tc.guard) - } - if g.Reason() != tc.wantReason { - t.Fatalf("Reason() = %q, want %q", g.Reason(), tc.wantReason) - } - held, why := g.Hold(tc.state, tc.intent) - if held != tc.wantHeld || why != tc.wantWhy { - t.Fatalf("Hold() = (%v, %q), want (%v, %q)", held, why, tc.wantHeld, tc.wantWhy) - } - }) - } + cases.Run(t, "testdata/guards/hold", func(t *testing.T, tc guardHoldFile) { + setTerminatingGate(t, tc.Inputs.CheckTerminating) + g, ok := guardByName(tc.Inputs.Guard) + if !ok { + t.Fatalf("unknown guard %q", tc.Inputs.Guard) + } + if g.Reason() != tc.Expected.Reason { + t.Fatalf("Reason() = %q, want %q", g.Reason(), tc.Expected.Reason) + } + held, why := g.Hold(tc.Inputs.State.state(), tc.Inputs.Intent.intent()) + if held != tc.Expected.Held || why != tc.Expected.Why { + t.Fatalf("Hold() = (%v, %q), want (%v, %q)", held, why, tc.Expected.Held, tc.Expected.Why) + } + }) } func guardByName(name string) (Guard, bool) { @@ -63,26 +92,29 @@ func guardByName(name string) (Guard, bool) { } } -type guardOrderCase struct { - name string - state WorkloadState - intent ScaleIntent - wantReason string - wantMessage string +type guardOrderFile struct { + Name string `json:"name,omitempty"` + Description string `json:"description,omitempty"` + Inputs struct { + State stateInput `json:"state"` + Intent intentInput `json:"intent"` + } `json:"inputs"` + Expected struct { + Reason string `json:"reason"` + Message string `json:"message"` + } `json:"expected"` } // TestGuardOrder evaluates the guards slice. The flag is forced off so a // row does not depend on whichever case ran before it. func TestGuardOrder(t *testing.T) { - for _, tc := range guardOrderCases { - t.Run(tc.name, func(t *testing.T) { - setTerminatingGate(t, false) - got := evaluateGuards(tc.state, tc.intent) - if got.reason != tc.wantReason || got.message != tc.wantMessage { - t.Fatalf("got {%q %q}, want {%q %q}", got.reason, got.message, tc.wantReason, tc.wantMessage) - } - }) - } + cases.Run(t, "testdata/guards/order", func(t *testing.T, tc guardOrderFile) { + setTerminatingGate(t, false) + got := evaluateGuards(tc.Inputs.State.state(), tc.Inputs.Intent.intent()) + if got.reason != tc.Expected.Reason || got.message != tc.Expected.Message { + t.Fatalf("got {%q %q}, want {%q %q}", got.reason, got.message, tc.Expected.Reason, tc.Expected.Message) + } + }) } func setTerminatingGate(t *testing.T, on bool) { @@ -92,125 +124,99 @@ func setTerminatingGate(t *testing.T, on bool) { t.Cleanup(func() { checkTerminatingReplicas = prev }) } -// workloadObjectFields is one target object as data. A nil field is omitted. -// An int, int32, or int64 is stored as int64, which is what NestedInt64 reads. -// Any other type, such as a string standing in for an invalid field, is stored -// as itself. -type workloadObjectFields struct { - spec any - ready any - total any - updated any - terminating any - generation any - observed any +type workloadStateFile struct { + Name string `json:"name,omitempty"` + Description string `json:"description,omitempty"` + Inputs struct { + Kind string `json:"kind"` + Object *unstructured.Unstructured `json:"object"` + } `json:"inputs"` + Expected stateInput `json:"expected"` } -type workloadStateCase struct { - name string - kind string - fields workloadObjectFields - want WorkloadState +func TestReadWorkloadState(t *testing.T) { + cases.Run(t, "testdata/guards/workload-state", func(t *testing.T, tc workloadStateFile) { + normalizeUnstructured(tc.Inputs.Object) + got := readWorkloadState(tc.Inputs.Object, tc.Inputs.Kind) + if got != tc.Expected.state() { + t.Fatalf("got %+v, want %+v", got, tc.Expected.state()) + } + }) } -func TestReadWorkloadState(t *testing.T) { - for _, tc := range workloadStateCases { - t.Run(tc.name, func(t *testing.T) { - got := readWorkloadState(workloadObject(t, tc.fields), tc.kind) - if got != tc.want { - t.Fatalf("got %+v, want %+v", got, tc.want) - } - }) - } +// targetGuardFile is one object-level guard result. The object is a +// Deployment-shaped document; kind selects the parse. status.replicas stays +// at 5 in every file. readyReplicas 0 is the raw count and is not settled. +type targetGuardFile struct { + Name string `json:"name,omitempty"` + Description string `json:"description,omitempty"` + Inputs struct { + Kind string `json:"kind"` + Object *unstructured.Unstructured `json:"object"` + Target int32 `json:"target"` + } `json:"inputs"` + Expected struct { + Spec int32 `json:"spec"` + Ready int32 `json:"ready"` + Total int32 `json:"total"` + Updated int32 `json:"updated"` + Generation int64 `json:"generation"` + ObservedGeneration int64 `json:"observedGeneration"` + Reason string `json:"reason"` + Message string `json:"message"` + } `json:"expected"` } -func workloadObject(t *testing.T, f workloadObjectFields) *unstructured.Unstructured { - t.Helper() - obj := map[string]any{} - set := func(v any, fields ...string) { - t.Helper() - stored, ok := nestedJSONValue(v) - if !ok { - return +// TestGuardsOnTarget parses each file's object and evaluates the guards. +func TestGuardsOnTarget(t *testing.T) { + cases.Run(t, "testdata/guards/target", func(t *testing.T, tc targetGuardFile) { + normalizeUnstructured(tc.Inputs.Object) + state := readWorkloadState(tc.Inputs.Object, tc.Inputs.Kind) + exp := tc.Expected + if state.Spec != exp.Spec || state.Ready != exp.Ready { + t.Fatalf("parsed spec/ready = %d/%d, want %d/%d", state.Spec, state.Ready, exp.Spec, exp.Ready) } - if err := unstructured.SetNestedField(obj, stored, fields...); err != nil { - t.Fatalf("set %v: %v", fields, err) + if state.Total != exp.Total || state.Updated != exp.Updated || + state.Generation != exp.Generation || state.ObservedGeneration != exp.ObservedGeneration { + t.Fatalf("parsed rollout fields = total %d updated %d gen %d observed %d", + state.Total, state.Updated, state.Generation, state.ObservedGeneration) } - } - set(f.spec, "spec", "replicas") - set(f.ready, "status", "readyReplicas") - set(f.total, "status", "replicas") - set(f.updated, "status", "updatedReplicas") - set(f.terminating, "status", "terminatingReplicas") - set(f.generation, "metadata", "generation") - set(f.observed, "status", "observedGeneration") - return &unstructured.Unstructured{Object: obj} + got := evaluateGuards(state, ScaleIntent{From: state.Spec, To: tc.Inputs.Target}) + if got.reason != exp.Reason || got.message != exp.Message { + t.Fatalf("got {%q %q}, want {%q %q}", got.reason, got.message, exp.Reason, exp.Message) + } + }) } -// nestedJSONValue converts a case-table number to the int64 NestedInt64 -// accepts. nil means the field is omitted. A non-number is returned unchanged -// so an invalid type can be written on purpose. -func nestedJSONValue(v any) (any, bool) { - switch n := v.(type) { - case nil: - return nil, false - case int: - return int64(n), true - case int32: - return int64(n), true - case int64: - return n, true - default: - return v, true +// normalizeUnstructured turns whole-number float64 values into int64. +// NestedInt64 only accepts int64. A string standing in for an invalid field +// is left alone so it still reads as missing. +func normalizeUnstructured(obj *unstructured.Unstructured) { + if obj == nil || obj.Object == nil { + return } + obj.Object = normalizeJSON(obj.Object).(map[string]any) } -// targetGuardCase is one object-level guard result: the fields build a -// Deployment-shaped object, kind selects the parse. -type targetGuardCase struct { - name string - gen int64 - observed int64 - updated int64 - kind string - spec int64 - ready int64 - target int32 - wantReason string - wantMessage string -} - -// deploymentObject builds the target a targetGuardCase describes. -// status.replicas stays at 5; the cases vary generation, updatedReplicas, -// spec.replicas, and readyReplicas. -func deploymentObject(gen, observed, updated, spec, ready int64) *unstructured.Unstructured { - return &unstructured.Unstructured{Object: map[string]any{ - "metadata": map[string]any{"generation": gen}, - "spec": map[string]any{"replicas": spec}, - "status": map[string]any{ - "observedGeneration": observed, - "updatedReplicas": updated, - "replicas": int64(5), - "readyReplicas": ready, - }, - }} -} - -// TestGuardsOnTarget parses each row's object and evaluates the guards. -func TestGuardsOnTarget(t *testing.T) { - for _, tc := range targetGuardCases { - t.Run(tc.name, func(t *testing.T) { - state := readWorkloadState(deploymentObject(tc.gen, tc.observed, tc.updated, tc.spec, tc.ready), tc.kind) - if state.Spec != int32(tc.spec) || state.Ready != int32(tc.ready) { - t.Fatalf("parsed spec/ready = %d/%d, want %d/%d", state.Spec, state.Ready, tc.spec, tc.ready) - } - if state.Total != 5 || state.Updated != int32(tc.updated) || state.Generation != tc.gen || state.ObservedGeneration != tc.observed { - t.Fatalf("parsed rollout fields = total %d updated %d gen %d observed %d", state.Total, state.Updated, state.Generation, state.ObservedGeneration) - } - got := evaluateGuards(state, ScaleIntent{From: state.Spec, To: tc.target}) - if got.reason != tc.wantReason || got.message != tc.wantMessage { - t.Fatalf("got {%q %q}, want {%q %q}", got.reason, got.message, tc.wantReason, tc.wantMessage) - } - }) +func normalizeJSON(v any) any { + switch n := v.(type) { + case float64: + if !math.IsNaN(n) && !math.IsInf(n, 0) && n == math.Trunc(n) && + n >= math.MinInt64 && n <= math.MaxInt64 { + return int64(n) + } + return n + case map[string]any: + for k, child := range n { + n[k] = normalizeJSON(child) + } + return n + case []any: + for i := range n { + n[i] = normalizeJSON(n[i]) + } + return n + default: + return v } } diff --git a/internal/controller/autoscaling/llmscaler_casefile_test.go b/internal/controller/autoscaling/llmscaler_casefile_test.go new file mode 100644 index 0000000..65c7d22 --- /dev/null +++ b/internal/controller/autoscaling/llmscaler_casefile_test.go @@ -0,0 +1,403 @@ +/* +Copyright 2026. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controller + +import ( + "encoding/json" + "fmt" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + autoscalingv1alpha1 "github.com/modelsphere/llm-operator/api/autoscaling/v1alpha1" + "github.com/modelsphere/llm-operator/internal/testutil/cases" +) + +// sinceCheck is how one step reads status.deferredSince. +// Omitted means do not check. JSON null means the field was cleared. +// "seed" is the instant written by seedDeferredSince. Any other string is a +// Go duration offset from deferralStart, including "0s". +type sinceCheck struct { + kind sinceKind + offset time.Duration +} + +func (s *sinceCheck) UnmarshalJSON(b []byte) error { + if string(b) == "null" { + s.kind = sinceNone + return nil + } + var raw string + if err := json.Unmarshal(b, &raw); err != nil { + return fmt.Errorf("deferredSince: %w", err) + } + if raw == "seed" { + s.kind = sinceSeeded + return nil + } + d, err := time.ParseDuration(raw) + if err != nil { + return fmt.Errorf("deferredSince %q: %w", raw, err) + } + s.kind = sinceOffset + s.offset = d + return nil +} + +func (s sinceCheck) expect() sinceExpect { return sinceExpect(s) } + +type specPatchInput struct { + MinReplicas *int32 `json:"minReplicas"` + MaxStepReplicas *int32 `json:"maxStepReplicas"` + StabilizationWindowSeconds *int32 `json:"stabilizationWindowSeconds"` + DeferTimeoutSeconds *int32 `json:"deferTimeoutSeconds"` + SyncPeriodSeconds *int32 `json:"syncPeriodSeconds"` +} + +func (p specPatchInput) patch() scalerSpecPatch { + return scalerSpecPatch{ + minReplicas: p.MinReplicas, + maxStepReplicas: p.MaxStepReplicas, + stabilizationWindowSeconds: p.StabilizationWindowSeconds, + deferTimeoutSeconds: p.DeferTimeoutSeconds, + syncPeriodSeconds: p.SyncPeriodSeconds, + } +} + +type deployInput struct { + Spec *int32 `json:"spec"` + Replicas int32 `json:"replicas"` + Updated int32 `json:"updated"` + Ready int32 `json:"ready"` + Available int32 `json:"available"` + Terminating *int32 `json:"terminating"` +} + +func (d deployInput) state() deployState { + return deployState{ + spec: d.Spec, + replicas: d.Replicas, + updated: d.Updated, + ready: d.Ready, + available: d.Available, + terminating: d.Terminating, + } +} + +type stepInput struct { + By string `json:"by"` + At cases.Duration `json:"at"` + Metric string `json:"metric"` + Deploy *deployInput `json:"deploy"` + ObserveSpec *int32 `json:"observeSpec"` + Rollout bool `json:"rollout"` + RestoreStatus bool `json:"restoreStatus"` + SkipReconcile bool `json:"skipReconcile"` + SnapshotStatus bool `json:"snapshotStatus"` +} + +type condInput struct { + Status string `json:"status"` + Reason string `json:"reason"` + MessageContains []string `json:"messageContains"` + ObservedGenerationCurrent bool `json:"observedGenerationCurrent"` +} + +func (c condInput) expect() condExpect { + return condExpect{ + status: metav1.ConditionStatus(c.Status), + reason: c.Reason, + messageContains: c.MessageContains, + observedGenerationCurrent: c.ObservedGenerationCurrent, + } +} + +type eventInput struct { + Prefix string `json:"prefix"` + Contains []string `json:"contains"` +} + +func (e eventInput) expect() eventExpect { + return eventExpect{prefix: e.Prefix, contains: e.Contains} +} + +type stepExpected struct { + Replicas *int32 `json:"replicas"` + Desired *int32 `json:"desired"` + Applied *int32 `json:"applied"` + Current *int32 `json:"current"` + DeferredSince sinceCheck `json:"deferredSince"` + Condition *condInput `json:"condition"` + RequeueAfter *cases.Duration `json:"requeueAfter"` + Events []eventInput `json:"events"` + NoMoreEvents bool `json:"noMoreEvents"` + ForcedWrite *bool `json:"forcedWrite"` + ForcedWriteAt *cases.Duration `json:"forcedWriteAt"` +} + +func (e stepExpected) expect() stepExpect { + out := stepExpect{ + replicas: e.Replicas, + desired: e.Desired, + applied: e.Applied, + current: e.Current, + deferredSince: e.DeferredSince.expect(), + requeueAfter: durPtr(e.RequeueAfter), + noMoreEvents: e.NoMoreEvents, + forcedWrite: e.ForcedWrite, + forcedWriteAt: durPtr(e.ForcedWriteAt), + } + if e.Condition != nil { + cond := e.Condition.expect() + out.condition = &cond + } + if len(e.Events) > 0 { + out.events = make([]eventExpect, len(e.Events)) + for i, ev := range e.Events { + out.events[i] = ev.expect() + } + } + return out +} + +func (s stepInput) step(exp stepExpected) scenarioStep { + out := scenarioStep{ + by: s.By, + at: s.At.Std(), + metric: s.Metric, + observeSpec: s.ObserveSpec, + rollout: s.Rollout, + restoreStatus: s.RestoreStatus, + skipReconcile: s.SkipReconcile, + snapshotStatus: s.SnapshotStatus, + expect: exp.expect(), + } + if s.Deploy != nil { + st := s.Deploy.state() + out.deploy = &st + } + return out +} + +func durPtr(d *cases.Duration) *time.Duration { + if d == nil { + return nil + } + v := d.Std() + return &v +} + +type scalerInputs struct { + Prelude string `json:"prelude"` + Spec specPatchInput `json:"spec"` + Deploy *deployInput `json:"deploy"` + Rollout bool `json:"rollout"` + Metric string `json:"metric"` + SeedDeferredSince *cases.Duration `json:"seedDeferredSince"` + CheckTerminating *bool `json:"checkTerminating"` + PlainReconciler bool `json:"plainReconciler"` + ExpectNilRecommendations bool `json:"expectNilRecommendations"` + Steps []stepInput `json:"steps"` +} + +type scalerExpected struct { + Steps []stepExpected `json:"steps"` +} + +// scalerCase is one Prometheus-provider reconcile spec. +// inputs.steps and expected.steps are zipped by index. +type scalerCase struct { + Name string `json:"name,omitempty"` + Description string `json:"description,omitempty"` + Inputs scalerInputs `json:"inputs"` + Expected scalerExpected `json:"expected"` +} + +func (c scalerCase) scenario() scalerScenario { + in := c.Inputs + if len(in.Steps) != len(c.Expected.Steps) { + Fail(fmt.Sprintf("%s: %d input steps and %d expected steps", c.Description, len(in.Steps), len(c.Expected.Steps))) + } + steps := make([]scenarioStep, len(in.Steps)) + for i := range in.Steps { + steps[i] = in.Steps[i].step(c.Expected.Steps[i]) + } + sc := scalerScenario{ + prelude: in.Prelude, + spec: in.Spec.patch(), + rollout: in.Rollout, + metric: in.Metric, + seedDeferredSince: durPtr(in.SeedDeferredSince), + checkTerminating: in.CheckTerminating, + plainReconciler: in.PlainReconciler, + expectNilRecommendations: in.ExpectNilRecommendations, + steps: steps, + } + if in.Deploy != nil { + st := in.Deploy.state() + sc.deploy = &st + } + return sc +} + +func runScalerCase(c scalerCase) { + runScalerScenario(c.scenario()) +} + +type customReconcileFile struct { + Name string `json:"name,omitempty"` + Description string `json:"description,omitempty"` + Inputs struct { + CreateBy string `json:"createBy"` + Deploy *deployInput `json:"deploy"` + DeployBy string `json:"deployBy"` + CloseServer bool `json:"closeServer"` + CloseBy string `json:"closeBy"` + ReconcileBy string `json:"reconcileBy"` + ReplicasBy string `json:"replicasBy"` + StatusBy string `json:"statusBy"` + } `json:"inputs"` + Expected struct { + Replicas int32 `json:"replicas"` + Desired *int32 `json:"desired"` + DecisionStatus string `json:"decisionStatus"` + DecisionReason string `json:"decisionReason"` + DecisionMessageNonEmpty bool `json:"decisionMessageNonEmpty"` + } `json:"expected"` +} + +func runCustomReconcileFile(c customReconcileFile) { + exp := c.Expected + tc := customReconcileCase{ + createBy: c.Inputs.CreateBy, + deployBy: c.Inputs.DeployBy, + closeServer: c.Inputs.CloseServer, + closeBy: c.Inputs.CloseBy, + reconcileBy: c.Inputs.ReconcileBy, + replicasBy: c.Inputs.ReplicasBy, + wantReplicas: exp.Replicas, + statusBy: c.Inputs.StatusBy, + wantDesired: exp.Desired, + decisionStatus: metav1.ConditionStatus(exp.DecisionStatus), + decisionReason: exp.DecisionReason, + decisionMessageNonEmpty: exp.DecisionMessageNonEmpty, + } + if c.Inputs.Deploy != nil { + st := c.Inputs.Deploy.state() + tc.deploy = &st + } + runCustomProviderReconcile(tc) +} + +type customAdmissionFile struct { + Name string `json:"name,omitempty"` + Description string `json:"description,omitempty"` + Inputs struct { + MetricProvider string `json:"metricProvider"` + OmitCustomProvider bool `json:"omitCustomProvider"` + PinProvider bool `json:"pinProvider"` + } `json:"inputs"` + Expected struct { + ErrorContains string `json:"errorContains"` + } `json:"expected"` +} + +func runCustomAdmissionFile(c customAdmissionFile) { + if c.Inputs.PinProvider && c.Inputs.MetricProvider != autoscalingv1alpha1.MetricProviderPrometheus { + Fail(fmt.Sprintf("metricProvider %q, want %q", c.Inputs.MetricProvider, autoscalingv1alpha1.MetricProviderPrometheus)) + } + // An empty substring matches every error. + if c.Expected.ErrorContains == "" { + Fail(c.Description + ": errorContains is required") + } + runCustomProviderAdmission(customAdmissionCase{ + metricProvider: c.Inputs.MetricProvider, + omitCustomProvider: c.Inputs.OmitCustomProvider, + wantErr: c.Expected.ErrorContains, + }) +} + +type deferDefaultFile struct { + Name string `json:"name,omitempty"` + Description string `json:"description,omitempty"` + Inputs struct { + } `json:"inputs"` + Expected struct { + Seconds int32 `json:"seconds"` + } `json:"expected"` +} + +func runDeferDefault(tc deferDefaultFile) { + if tc.Expected.Seconds != autoscalingv1alpha1.DefaultDeferTimeoutSeconds { + Fail(fmt.Sprintf("seconds %d, want DefaultDeferTimeoutSeconds %d", + tc.Expected.Seconds, autoscalingv1alpha1.DefaultDeferTimeoutSeconds)) + } + scaler := &autoscalingv1alpha1.LLMScaler{} + Expect(k8sClient.Get(promFix.ctx, promFix.scaler, scaler)).To(Succeed()) + // The fixture leaves the field unset; the CRD default fills it. + Expect(scaler.Spec.DeferTimeoutSeconds).NotTo(BeNil()) + Expect(*scaler.Spec.DeferTimeoutSeconds).To(Equal(tc.Expected.Seconds)) +} + +type customDefaultFile struct { + Name string `json:"name,omitempty"` + Description string `json:"description,omitempty"` + Inputs struct { + CreateBy string `json:"createBy"` + SwitchBy string `json:"switchBy"` + Query string `json:"query"` + Target string `json:"target"` + ServiceID string `json:"serviceId"` + } `json:"inputs"` + Expected struct { + MetricProvider string `json:"metricProvider"` + Path string `json:"path"` + } `json:"expected"` +} + +func runCustomDefault(tc customDefaultFile) { + if tc.Expected.MetricProvider != autoscalingv1alpha1.MetricProviderPrometheus { + Fail(fmt.Sprintf("metricProvider %q, want %q", tc.Expected.MetricProvider, autoscalingv1alpha1.MetricProviderPrometheus)) + } + if tc.Expected.Path != defaultDecisionsPath { + Fail(fmt.Sprintf("path %q, want %q", tc.Expected.Path, defaultDecisionsPath)) + } + if tc.Inputs.ServiceID != sampleServiceID { + Fail(fmt.Sprintf("serviceId %q, want %q", tc.Inputs.ServiceID, sampleServiceID)) + } + + By(tc.Inputs.CreateBy) + scaler := customFix.newScaler() + scaler.Spec.MetricProvider = "" + scaler.Spec.CustomProvider = nil + scaler.Spec.Metrics = []autoscalingv1alpha1.MetricSpec{{Query: tc.Inputs.Query, Target: tc.Inputs.Target}} + Expect(k8sClient.Create(customFix.ctx, scaler)).To(Succeed()) + + stored := &autoscalingv1alpha1.LLMScaler{} + Expect(k8sClient.Get(customFix.ctx, customFix.scaler, stored)).To(Succeed()) + Expect(stored.Spec.MetricProvider).To(Equal(tc.Expected.MetricProvider)) + + By(tc.Inputs.SwitchBy) + stored.Spec.MetricProvider = autoscalingv1alpha1.MetricProviderCustom + stored.Spec.CustomProvider = &autoscalingv1alpha1.CustomProviderSpec{ServiceID: tc.Inputs.ServiceID} + Expect(k8sClient.Update(customFix.ctx, stored)).To(Succeed()) + + Expect(k8sClient.Get(customFix.ctx, customFix.scaler, stored)).To(Succeed()) + Expect(stored.Spec.CustomProvider.Path).To(Equal(tc.Expected.Path)) +} diff --git a/internal/controller/autoscaling/llmscaler_controller_cases_test.go b/internal/controller/autoscaling/llmscaler_controller_cases_test.go deleted file mode 100644 index f379025..0000000 --- a/internal/controller/autoscaling/llmscaler_controller_cases_test.go +++ /dev/null @@ -1,722 +0,0 @@ -/* -Copyright 2026. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package controller - -import ( - "time" - - . "github.com/onsi/ginkgo/v2" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - - autoscalingv1alpha1 "github.com/modelsphere/llm-operator/api/autoscaling/v1alpha1" -) - -// metricScalingCases are plain metric-driven writes: scale up, down to min, to zero, wake from zero, and the ready-replica ratio fallback. -var metricScalingCases = []TableEntry{ - // ceil(1 * 0.85/0.5) = ceil(1.7) = 2. The mock sample is 0.85 and the target is 0.5. - Entry("should successfully scale the deployment based on metrics", scalerScenario{ - plainReconciler: true, - steps: []scenarioStep{ - {by: "running the Reconciler"}, - {by: "checking if the Deployment replicas increased", skipReconcile: true, - expect: stepExpect{replicas: ptr(int32(2))}}, - }, - }), - - // desired = ceil(5 * 0/0.5) = 0, clamped up to minReplicas (1). - Entry("should scale down to minReplicas when the metric is zero", scalerScenario{ - plainReconciler: true, - steps: []scenarioStep{ - {by: "setting the Deployment to 5 replicas and the metric to 0 (idle)", skipReconcile: true, - metric: "0", // empty queue - deploy: &deployState{spec: ptr(int32(5)), replicas: 5, updated: 5, ready: 5, available: 5}}, - {by: "running the Reconciler"}, - {by: "checking it scaled down to minReplicas", skipReconcile: true, - expect: stepExpect{replicas: ptr(int32(1))}}, - }, - }), - - // desired = ceil(2 * 0/0.5) = 0, clamped to minReplicas (0). - Entry("should scale down to zero when minReplicas is 0", scalerScenario{ - plainReconciler: true, - prelude: "lowering the scaler's minReplicas to 0", - spec: scalerSpecPatch{minReplicas: ptr(int32(0))}, - steps: []scenarioStep{ - {by: "setting the Deployment to 2 replicas, fully rolled out", skipReconcile: true, - metric: "0", // idle -> wants zero - deploy: &deployState{spec: ptr(int32(2)), replicas: 2, updated: 2, ready: 2, available: 2}}, - {by: "running the Reconciler"}, - {by: "checking it scaled down to zero", skipReconcile: true, - expect: stepExpect{replicas: ptr(int32(0))}}, - }, - }), - - // desired = ceil(1 * 0.85/0.5) = 2 against the single-replica baseline, - // not ceil(0 * 0.85/0.5) = 0. - Entry("should wake the fleet from zero when the metric turns positive", scalerScenario{ - plainReconciler: true, - prelude: "lowering the scaler's minReplicas to 0 and parking the Deployment at 0", - spec: scalerSpecPatch{minReplicas: ptr(int32(0))}, - // Load arrives while the fleet is at zero. mockMetricValue is already the fixture sample. - deploy: &deployState{spec: ptr(int32(0)), replicas: 0, updated: 0, ready: 0, available: 0}, - steps: []scenarioStep{ - {by: "running the Reconciler"}, - {by: "checking the fleet woke up", skipReconcile: true, - expect: stepExpect{replicas: ptr(int32(2))}}, - }, - }), - - // readyReplicas is 0, so the ratio falls back to spec (3): - // ceil(3 * 0.85/0.5) = 6, clamped to maxReplicas 5. Without that - // fallback the single-replica baseline would recommend 2. - // status.currentReplicas stays 0. ready 0 is omitted and reads as missing, - // so available must be 0 too or the apiserver rejects the status. - Entry("should keep the ready-replica fallback for the scaling ratio only", scalerScenario{ - plainReconciler: true, - steps: []scenarioStep{ - {deploy: &deployState{spec: ptr(int32(3)), replicas: 3, updated: 3, ready: 0, available: 0}, - expect: stepExpect{ - replicas: ptr(int32(5)), - current: ptr(int32(0)), - desired: ptr(int32(5)), - applied: ptr(int32(5)), - deferredSince: sinceNil(), - condition: &condExpect{ - status: metav1.ConditionFalse, - reason: autoscalingv1alpha1.ReasonNotDeferred, - }, - }}, - }, - }), -} - -// rolloutGuardCases (G8) defer scaling while the Deployment is mid-rollout. -var rolloutGuardCases = []TableEntry{ - // The metric would compute desired=2, but the rollout guard defers - // scaling, so replicas stay at 1. - Entry("should defer scaling while the Deployment is rolling out", scalerScenario{ - plainReconciler: true, - steps: []scenarioStep{ - {by: "marking the Deployment as mid-rollout", rollout: true, skipReconcile: true}, - {by: "running the Reconciler"}, - {by: "checking the Deployment was NOT scaled", skipReconcile: true, - expect: stepExpect{replicas: ptr(int32(1))}}, - }, - }), -} - -// settleGuardCases (G7) defer scale-down until the fleet is back at spec, and do not gate scale-up. -var settleGuardCases = []TableEntry{ - Entry("should defer a scale-down until readyReplicas is back at specReplicas", scalerScenario{ - plainReconciler: true, - metric: "0", // empty queue -> wants minReplicas - steps: []scenarioStep{ - {by: "setting the Deployment to 5 replicas with only 3 ready", - deploy: &deployState{spec: ptr(int32(5)), replicas: 5, updated: 5, ready: 3, available: 3}}, - {by: "checking the scale-down was deferred", skipReconcile: true, - expect: stepExpect{replicas: ptr(int32(5))}}, - {by: "letting the fleet converge", - deploy: &deployState{replicas: 5, updated: 5, ready: 5, available: 5}}, - {by: "checking the scale-down now proceeds", skipReconcile: true, - expect: stepExpect{replicas: ptr(int32(1))}}, - }, - }), - - // The fallback path. readyReplicas excludes terminating pods, so the - // fleet reads as settled the moment the previous step's pods are marked - // for deletion — the descent is then paced by syncPeriodSeconds, not by - // the drain. - Entry("should ignore still-draining pods while checkTerminatingReplicas is off", scalerScenario{ - plainReconciler: true, - checkTerminating: ptr(false), - metric: "0", - steps: []scenarioStep{ - {by: "setting 5 ready with 2 still draining", - deploy: &deployState{spec: ptr(int32(5)), replicas: 5, updated: 5, ready: 5, available: 5, terminating: ptr(int32(2))}}, - {by: "checking the scale-down was not held back by the drain", skipReconcile: true, - expect: stepExpect{replicas: ptr(int32(1))}}, - }, - }), - - Entry("should also wait for the drain when checkTerminatingReplicas is on", scalerScenario{ - plainReconciler: true, - checkTerminating: ptr(true), - metric: "0", - steps: []scenarioStep{ - {by: "setting 5 ready with 2 still draining", - deploy: &deployState{spec: ptr(int32(5)), replicas: 5, updated: 5, ready: 5, available: 5, terminating: ptr(int32(2))}}, - {by: "checking it is deferred; readyReplicas alone would have said settled", skipReconcile: true, - expect: stepExpect{replicas: ptr(int32(5))}}, - {by: "letting the drain finish", - deploy: &deployState{replicas: 5, updated: 5, ready: 5, available: 5, terminating: ptr(int32(0))}}, - {by: "checking the scale-down now proceeds", skipReconcile: true, - expect: stepExpect{replicas: ptr(int32(1))}}, - }, - }), - - // ceil(2 * 0.85/0.5) = 4; the settle guard only gates scale-down. - Entry("should not gate scale-up on the previous scale-down settling", scalerScenario{ - plainReconciler: true, - steps: []scenarioStep{ - {by: "setting 3 replicas with 2 ready and 1 still terminating, under load", - // Over the 0.5 target -> wants more replicas. - metric: mockMetricValue, - deploy: &deployState{spec: ptr(int32(3)), replicas: 3, updated: 3, ready: 2, available: 2, terminating: ptr(int32(1))}}, - {by: "checking capacity was added despite the unsettled fleet", skipReconcile: true, - expect: stepExpect{replicas: ptr(int32(4))}}, - }, - }), -} - -// stabilizationAndStepCases hold scale-down for the stabilization window and walk down one maxStepReplicas at a time. -var stabilizationAndStepCases = []TableEntry{ - Entry("should hold for the stabilization window between scale-down steps", scalerScenario{ - plainReconciler: true, - prelude: "enabling a 300s window and a 2-replica step cap", - spec: scalerSpecPatch{ - stabilizationWindowSeconds: ptr(int32(300)), - maxStepReplicas: ptr(int32(2)), - }, - metric: "0", - steps: []scenarioStep{ - {by: "starting settled at 5 with an idle metric", - deploy: &deployState{spec: ptr(int32(5)), replicas: 5, updated: 5, ready: 5, available: 5}}, - {by: "checking the first step landed", skipReconcile: true, - expect: stepExpect{replicas: ptr(int32(3))}}, - {by: "settling the fleet at 3 so neither the rollout nor the settle guard fires", - observeSpec: ptr(int32(3))}, - {by: "checking the next step is held; without the pin it would drop to 1", skipReconcile: true, - expect: stepExpect{replicas: ptr(int32(3))}}, - }, - }), - - Entry("should walk down one step at a time when maxStepReplicas is set", scalerScenario{ - plainReconciler: true, - prelude: "capping the scale-down step at 2 replicas", - spec: scalerSpecPatch{maxStepReplicas: ptr(int32(2))}, - steps: []scenarioStep{ - {by: "setting the Deployment to 5 replicas and the metric to 0 (idle)", skipReconcile: true, - metric: "0", // empty queue - deploy: &deployState{spec: ptr(int32(5)), replicas: 5, updated: 5, ready: 5, available: 5}}, - // desired = 1 (clamped to minReplicas), capped to 5 - 2 = 3. - {by: "running the Reconciler; raw recommendation is minReplicas but the step is capped", - expect: stepExpect{replicas: ptr(int32(3))}}, - // The cap rate-limits the write; the recommendation is still 1. - {by: "checking status reports the uncapped recommendation, not the step", skipReconcile: true, - expect: stepExpect{desired: ptr(int32(1))}}, - // 3 - 2 = 1, which is minReplicas, so it lands there rather than below. - {by: "running the Reconciler again; it takes the next step down", - deploy: &deployState{spec: ptr(int32(3)), replicas: 3, updated: 3, ready: 3, available: 3}, - expect: stepExpect{replicas: ptr(int32(1))}}, - }, - }), -} - -// deferralTimeoutCases force a held write once the deferral times out, including a restarted reconciler and a timeout of 0. -var deferralTimeoutCases = []TableEntry{ - Entry("should force a scale-up once a stuck rollout exceeds the deferral timeout", scalerScenario{ - spec: scalerSpecPatch{deferTimeoutSeconds: ptr(int32(60))}, - rollout: true, - steps: []scenarioStep{ - // Deadline is 60s out and the default sync period is 15s, so the - // wait is the sync period. The recommendation is recorded even - // though nothing was written. - {by: "deferring the first sync and stamping deferredSince", at: 0, - expect: stepExpect{ - requeueAfter: ptr(15 * time.Second), - replicas: ptr(int32(1)), - deferredSince: sinceAt(0), - desired: ptr(int32(2)), - applied: ptr(int32(1)), - condition: &condExpect{ - status: metav1.ConditionTrue, - reason: autoscalingv1alpha1.ReasonRolloutInProgress, - observedGenerationCurrent: true, - messageContains: []string{ - "holding 1 replicas (wanted 2)", - msgRolloutInProgress, - deferralStart.Format(time.RFC3339), - "will apply after 60s", - }, - }, - events: []eventExpect{{ - prefix: "Normal " + eventReasonScalingDeferred, - contains: []string{"holding 1 replicas (wanted 2)"}, - }}, - }}, - // One second remains until the deadline, under the sync period. - {by: "reconciling again before the timeout, without restamping or rewriting", - at: 59 * time.Second, - expect: stepExpect{ - requeueAfter: ptr(time.Second), - replicas: ptr(int32(1)), - deferredSince: sinceAt(0), - applied: ptr(int32(1)), - noMoreEvents: true, - }}, - // The rollout guard still matches after the write, so the clock - // stays where the deferral started. The forced write itself is not - // a hold, so the next sync is a full period. - {by: "advancing past the timeout and forcing the held scale-up", - at: 60 * time.Second, - expect: stepExpect{ - replicas: ptr(int32(2)), - deferredSince: sinceAt(0), - desired: ptr(int32(2)), - applied: ptr(int32(2)), - requeueAfter: ptr(15 * time.Second), - condition: &condExpect{ - status: metav1.ConditionFalse, - reason: autoscalingv1alpha1.ReasonDeferralTimedOut, - observedGenerationCurrent: true, - messageContains: []string{"scaling 1 -> 2 anyway"}, - }, - events: []eventExpect{{ - prefix: "Warning " + eventReasonDeferralTimedOut, - contains: []string{msgRolloutInProgress, "1 -> 2"}, - }}, - noMoreEvents: true, - }}, - }, - }), - - // readyReplicas == 0 is omitted from status, so the controller sees the - // field as missing. A ready==0 -> spec fallback in the guard would have made - // this look settled and scaled immediately; the settle guard now uses - // the raw count. The ratio still falls back to spec, which is why an - // idle metric recommends minReplicas rather than staying at 5. - // available must be 0 too: the apiserver rejects availableReplicas > readyReplicas, - // and 0 is omitted, so the controller sees that field as missing as well. - Entry("should force a scale-down of an unready fleet once the deferral times out", scalerScenario{ - spec: scalerSpecPatch{ - deferTimeoutSeconds: ptr(int32(60)), - maxStepReplicas: ptr(int32(2)), - }, - prelude: "capping the step and parking 5 replicas with none ready", - deploy: &deployState{spec: ptr(int32(5)), replicas: 5, updated: 5, ready: 0, available: 0}, - metric: "0", - steps: []scenarioStep{ - // Uncapped recommendation is minReplicas; the held write is the step. - {at: 0, - expect: stepExpect{ - replicas: ptr(int32(5)), - deferredSince: sinceAt(0), - desired: ptr(int32(1)), - current: ptr(int32(0)), - applied: ptr(int32(5)), - condition: &condExpect{ - status: metav1.ConditionTrue, - reason: autoscalingv1alpha1.ReasonScaleDownSettling, - messageContains: []string{ - "holding 5 replicas (wanted 3)", - "will apply after 60s", - }, - }, - }}, - {by: "keeping the same deferral until the timeout", at: 59 * time.Second, - expect: stepExpect{ - replicas: ptr(int32(5)), - deferredSince: sinceAt(0), - }}, - // None of the pods are ready, so the settle guard still matches and - // the clock is not restarted for the next step. - {by: "forcing the capped step, not the uncapped recommendation", at: 60 * time.Second, - expect: stepExpect{ - replicas: ptr(int32(3)), - deferredSince: sinceAt(0), - desired: ptr(int32(1)), - applied: ptr(int32(3)), - current: ptr(int32(0)), - condition: &condExpect{ - status: metav1.ConditionFalse, - reason: autoscalingv1alpha1.ReasonDeferralTimedOut, - messageContains: []string{"scaling 5 -> 3 anyway"}, - }, - // The entering deferral event, then the timeout warning. - events: []eventExpect{ - {prefix: "Normal " + eventReasonScalingDeferred}, - {prefix: "Warning " + eventReasonDeferralTimedOut, contains: []string{msgScaleDownNotSettled, "5 -> 3"}}, - }, - }}, - }, - }), - - // deferredSince lives in status, so a process restart (a new reconciler, - // with an empty recommendation map) still enforces the deadline. - // The rollout is still in progress, so the deadline carried in status is - // what forced the write and it stays put. - Entry("should force a deferral that already exceeded the timeout on a fresh reconciler", scalerScenario{ - spec: scalerSpecPatch{deferTimeoutSeconds: ptr(int32(60))}, - rollout: true, - seedDeferredSince: ptr(-2 * time.Hour), - expectNilRecommendations: true, - steps: []scenarioStep{ - {at: 0, expect: stepExpect{ - replicas: ptr(int32(2)), - deferredSince: sinceSeed(), - applied: ptr(int32(2)), - condition: &condExpect{ - status: metav1.ConditionFalse, - reason: autoscalingv1alpha1.ReasonDeferralTimedOut, - }, - events: []eventExpect{{ - prefix: "Warning " + eventReasonDeferralTimedOut, - contains: []string{"1 -> 2"}, - }}, - }}, - }, - }), - - // Timeout 0 never forces, so the wait stays the sync period even though - // the recorded deferral is already a day old. - Entry("should never force a deferral when deferTimeoutSeconds is 0", scalerScenario{ - spec: scalerSpecPatch{deferTimeoutSeconds: ptr(int32(0))}, - rollout: true, - seedDeferredSince: ptr(-24 * time.Hour), - steps: []scenarioStep{ - {at: 0, expect: stepExpect{ - requeueAfter: ptr(15 * time.Second), - replicas: ptr(int32(1)), - deferredSince: sinceSeed(), - condition: &condExpect{ - status: metav1.ConditionTrue, - reason: autoscalingv1alpha1.ReasonRolloutInProgress, - // Split so this is not a third copy of the controller's limit phrase. - messageContains: []string{"never forced (" + "deferTimeoutSeconds is 0)"}, - }, - events: []eventExpect{{prefix: "Normal " + eventReasonScalingDeferred}}, - noMoreEvents: true, - }}, - }, - }), -} - -// deferralStatusCases cover the condition, status, and event lifecycle of one deferral. -var deferralStatusCases = []TableEntry{ - // 5 replicas, 3 ready, but not yet rolled out: the rollout guard wins - // over the settle guard. An idle metric wants to scale down. - Entry("should keep one deferral when the guard reason changes", scalerScenario{ - spec: scalerSpecPatch{deferTimeoutSeconds: ptr(int32(60))}, - metric: "0", - deploy: &deployState{spec: ptr(int32(5)), replicas: 5, updated: 5, ready: 3, available: 3}, - rollout: true, - steps: []scenarioStep{ - {at: 0, expect: stepExpect{ - deferredSince: sinceAt(0), - condition: &condExpect{reason: autoscalingv1alpha1.ReasonRolloutInProgress}, - }}, - {by: "finishing the rollout while the fleet is still short of ready", - at: 10 * time.Second, - deploy: &deployState{replicas: 5, updated: 5, ready: 3, available: 3}, - expect: stepExpect{ - replicas: ptr(int32(5)), - deferredSince: sinceAt(0), - condition: &condExpect{reason: autoscalingv1alpha1.ReasonScaleDownSettling}, - events: []eventExpect{ - {contains: []string{msgRolloutInProgress}}, - {contains: []string{msgScaleDownNotSettled}}, - }, - }}, - {by: "not emitting again while the same guard still defers", - at: 10 * time.Second, - expect: stepExpect{ - deferredSince: sinceAt(0), - noMoreEvents: true, - }}, - }, - }), - - // ceil(1 * 0.5/0.5) = 1, which is already spec, and the rollout is done, - // so no guard matches. The no-op ends the deferral. - Entry("should clear a deferral on a no-op once no guard matches", scalerScenario{ - spec: scalerSpecPatch{deferTimeoutSeconds: ptr(int32(60))}, - rollout: true, - steps: []scenarioStep{ - {at: 0, expect: stepExpect{deferredSince: sinceAt(0)}}, - {by: "finishing the rollout at the replica count the metric already wants", - at: 10 * time.Second, - metric: "0.5", - observeSpec: ptr(int32(1)), - expect: stepExpect{ - replicas: ptr(int32(1)), - deferredSince: sinceNil(), - applied: ptr(int32(1)), - forcedWrite: ptr(false), - condition: &condExpect{ - status: metav1.ConditionFalse, - reason: autoscalingv1alpha1.ReasonNotDeferred, - messageContains: []string{"already at the step target"}, - }, - }}, - }, - }), - - Entry("should keep deferredSince across no-op syncs while a rollout guard still matches", scalerScenario{ - spec: scalerSpecPatch{deferTimeoutSeconds: ptr(int32(60))}, - rollout: true, - steps: []scenarioStep{ - // ceil(1 * 0.5/0.5) = 1, already spec. The guard matches, but there - // is nothing to hold, so deferredSince stays nil. - {by: "not starting a deferral on a no-op, even though the rollout guard matches", - at: 0, metric: "0.5", - expect: stepExpect{ - deferredSince: sinceNil(), - replicas: ptr(int32(1)), - condition: &condExpect{ - status: metav1.ConditionFalse, - reason: autoscalingv1alpha1.ReasonNotDeferred, - }, - }}, - // ceil(1 * 0.85/0.5) = 2. - {by: "stamping deferredSince on the first sync that actually wants to write", - at: 10 * time.Second, metric: "0.85", - expect: stepExpect{ - deferredSince: sinceAt(10 * time.Second), - replicas: ptr(int32(1)), - condition: &condExpect{ - status: metav1.ConditionTrue, - reason: autoscalingv1alpha1.ReasonRolloutInProgress, - }, - }}, - // The no-ops must not move the clock. The stamp is deferralStart+10s. - {by: "alternating a no-op with a held scale-up; the no-ops must not move the clock", - at: 20 * time.Second, metric: "0.5", - expect: stepExpect{ - replicas: ptr(int32(1)), - deferredSince: sinceAt(10 * time.Second), - condition: &condExpect{ - status: metav1.ConditionTrue, - reason: autoscalingv1alpha1.ReasonRolloutInProgress, - messageContains: []string{ - "nothing to write this sync", - "guard is still active", - "deferral clock keeps running", - deferralStart.Add(10 * time.Second).Format(time.RFC3339), - msgRolloutInProgress, - }, - }, - }}, - {at: 30 * time.Second, metric: "0.85", - expect: stepExpect{ - replicas: ptr(int32(1)), - deferredSince: sinceAt(10 * time.Second), - condition: &condExpect{ - status: metav1.ConditionTrue, - reason: autoscalingv1alpha1.ReasonRolloutInProgress, - }, - }}, - {at: 40 * time.Second, metric: "0.5", - expect: stepExpect{ - replicas: ptr(int32(1)), - deferredSince: sinceAt(10 * time.Second), - condition: &condExpect{ - status: metav1.ConditionTrue, - reason: autoscalingv1alpha1.ReasonRolloutInProgress, - messageContains: []string{ - "nothing to write this sync", - "guard is still active", - "deferral clock keeps running", - deferralStart.Add(10 * time.Second).Format(time.RFC3339), - msgRolloutInProgress, - }, - }, - }}, - {by: "forcing the held scale-up once the original deadline passes", - at: 70 * time.Second, metric: "0.85", - expect: stepExpect{ - replicas: ptr(int32(2)), - deferredSince: sinceAt(10 * time.Second), - condition: &condExpect{ - status: metav1.ConditionFalse, - reason: autoscalingv1alpha1.ReasonDeferralTimedOut, - messageContains: []string{"scaling 1 -> 2 anyway"}, - }, - }}, - }, - }), -} - -// forcedWritePacingCases pace a second forced step to one per sync period, and clear the pace when the guard lifts. -var forcedWritePacingCases = []TableEntry{ - Entry("should force the next stuck scale-down step after one sync period", scalerScenario{ - spec: scalerSpecPatch{ - deferTimeoutSeconds: ptr(int32(60)), - syncPeriodSeconds: ptr(int32(15)), - maxStepReplicas: ptr(int32(2)), - }, - prelude: "capping the step and parking 5 replicas with none ready", - deploy: &deployState{spec: ptr(int32(5)), replicas: 5, updated: 5, ready: 0, available: 0}, - metric: "0", - steps: []scenarioStep{ - {at: 0, - expect: stepExpect{ - replicas: ptr(int32(5)), - deferredSince: sinceAt(0), - condition: &condExpect{reason: autoscalingv1alpha1.ReasonScaleDownSettling}, - }}, - {by: "forcing the first step once the timeout elapses", at: 60 * time.Second, - expect: stepExpect{ - replicas: ptr(int32(3)), - deferredSince: sinceAt(0), - forcedWrite: ptr(true), - forcedWriteAt: ptr(60 * time.Second), - }}, - // The spec write bumped generation. Observe it with nothing ready - // so the settle guard, not a phantom rollout, is what still matches. - {by: "keeping the fleet unready and taking the next step one sync period later, without another timeout", - at: 75 * time.Second, - observeSpec: ptr(int32(0)), - expect: stepExpect{ - replicas: ptr(int32(1)), - deferredSince: sinceAt(0), - applied: ptr(int32(1)), - condition: &condExpect{ - status: metav1.ConditionFalse, - reason: autoscalingv1alpha1.ReasonDeferralTimedOut, - messageContains: []string{ - "scaling 3 -> 1 anyway", - msgScaleDownNotSettled, - }, - }, - }}, - }, - }), - - Entry("should not take a second forced step until a sync period has elapsed", scalerScenario{ - spec: scalerSpecPatch{ - deferTimeoutSeconds: ptr(int32(60)), - syncPeriodSeconds: ptr(int32(15)), - maxStepReplicas: ptr(int32(2)), - }, - metric: "0", - deploy: &deployState{spec: ptr(int32(5)), replicas: 5, updated: 5, ready: 0, available: 0}, - steps: []scenarioStep{ - // Snapshot the pre-write status. The next step is the forced write - // this By describes. - {by: "forcing one scale-down step of an unready fleet", at: 0, snapshotStatus: true}, - {at: 60 * time.Second, expect: stepExpect{replicas: ptr(int32(3))}}, - // The target write stuck at 3. The status write that would have - // recorded it did not, so a retry still sees the deadline passed. - {by: "putting the status back to its pre-write state, as if that update had conflicted", - restoreStatus: true, skipReconcile: true}, - {by: "reconciling again with the clock unchanged, which must not take the next step", - at: 60 * time.Second, - expect: stepExpect{ - replicas: ptr(int32(3)), - requeueAfter: ptr(15 * time.Second), - deferredSince: sinceAt(0), - applied: ptr(int32(3)), - condition: &condExpect{ - status: metav1.ConditionTrue, - messageContains: []string{"forced write paced to one per sync period"}, - }, - }}, - {by: "still holding one second short of the sync period", - at: 74 * time.Second, - expect: stepExpect{ - replicas: ptr(int32(3)), - requeueAfter: ptr(time.Second), - deferredSince: sinceAt(0), - }}, - {by: "forcing the next step once the sync period has elapsed", - at: 75 * time.Second, - expect: stepExpect{ - replicas: ptr(int32(1)), - deferredSince: sinceAt(0), - applied: ptr(int32(1)), - }}, - }, - }), - - Entry("should clear deferredSince and the forced-write pace when no guard matches", scalerScenario{ - spec: scalerSpecPatch{ - deferTimeoutSeconds: ptr(int32(60)), - maxStepReplicas: ptr(int32(2)), - }, - metric: "0", - deploy: &deployState{spec: ptr(int32(5)), replicas: 5, updated: 5, ready: 0, available: 0}, - steps: []scenarioStep{ - {at: 0}, - {at: 60 * time.Second, expect: stepExpect{ - replicas: ptr(int32(3)), - forcedWrite: ptr(true), - deferredSince: sinceAt(0), - }}, - // Settled and fully observed: the next scale-down is an ordinary - // write, not a forced one, even though the pace has not elapsed. - {by: "letting the fleet become ready at the replica count just written", - at: 60 * time.Second, - observeSpec: ptr(int32(3)), - expect: stepExpect{ - replicas: ptr(int32(1)), - deferredSince: sinceNil(), - forcedWrite: ptr(false), - condition: &condExpect{ - status: metav1.ConditionFalse, - reason: autoscalingv1alpha1.ReasonNotDeferred, - messageContains: []string{"scaled 3 -> 1"}, - }, - }}, - }, - }), -} - -// customProviderReconcileCases scale from a decision server, and hold replicas when that server cannot be reached. -var customProviderReconcileCases = []TableEntry{ - // spec.metrics is required only under the Prometheus provider, so a - // Custom scaler must be accepted without it. active 4 is an absolute - // count, so 1 ready replica scales straight to 4 rather than through - // the per-replica HPA ratio. - Entry("should scale the Deployment to the count the decision server reports", customReconcileCase{ - createBy: "creating an LLMScaler with no metrics at all", - reconcileBy: "running the Reconciler", - replicasBy: "checking the Deployment landed on the decision, not on a ratio of it", - wantReplicas: 4, - statusBy: "checking the status reports the same recommendation", - wantDesired: ptr(int32(4)), - decisionStatus: metav1.ConditionTrue, - }), - - // Above minReplicas on purpose: at 1 replica "held" and "collapsed to - // minReplicas" are the same number, so the test would pass either way. - Entry("should hold replicas when the decision server is unreachable", customReconcileCase{ - deploy: &deployState{spec: ptr(int32(3)), replicas: 3, updated: 3, ready: 3, available: 3}, - deployBy: "putting the Deployment at 3 replicas, above minReplicas", - closeServer: true, - closeBy: "taking the decision server down", - reconcileBy: "running the Reconciler", - replicasBy: "checking the Deployment was left where it was, not collapsed to minReplicas", - wantReplicas: 3, - statusBy: "checking the failure is reported on the status", - decisionStatus: metav1.ConditionFalse, - decisionReason: autoscalingv1alpha1.ReasonDecisionUnavailable, - decisionMessageNonEmpty: true, - }), -} - -// customProviderAdmissionCases are the CRD union rules for the metric provider. -var customProviderAdmissionCases = []TableEntry{ - Entry("should reject a Custom scaler with no customProvider", customAdmissionCase{ - omitCustomProvider: true, - wantErr: "spec.customProvider is required", - }), - Entry("should reject a Prometheus scaler with no metrics", customAdmissionCase{ - metricProvider: autoscalingv1alpha1.MetricProviderPrometheus, - omitCustomProvider: true, - wantErr: "spec.metrics must be non-empty", - }), -} diff --git a/internal/controller/autoscaling/llmscaler_controller_test.go b/internal/controller/autoscaling/llmscaler_controller_test.go index 212a83a..fc0d495 100644 --- a/internal/controller/autoscaling/llmscaler_controller_test.go +++ b/internal/controller/autoscaling/llmscaler_controller_test.go @@ -18,15 +18,15 @@ package controller import ( . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - autoscalingv1alpha1 "github.com/modelsphere/llm-operator/api/autoscaling/v1alpha1" + "github.com/modelsphere/llm-operator/internal/testutil/cases" + "github.com/modelsphere/llm-operator/internal/testutil/cases/ginkgocases" ) -// The reconcile specs are tables of scalerScenario values. The runner and the -// fixtures live in llmscaler_scenario_test.go; the rows live in -// llmscaler_controller_cases_test.go. Helper tests that do not need envtest -// are plain testing funcs in llmscaler_helpers_test.go. +// The reconcile specs are one YAML file each under testdata/controller. +// The runner and the fixtures live in llmscaler_scenario_test.go. +// Helper tests that do not need envtest are plain testing funcs in +// llmscaler_helpers_test.go. var _ = Describe("LLMScaler Controller", func() { Context("When reconciling a resource", func() { BeforeEach(func() { @@ -37,21 +37,25 @@ var _ = Describe("LLMScaler Controller", func() { promFix = nil }) - DescribeTable("metric scaling", runScalerScenario, metricScalingCases) - DescribeTable("rollout guard (G8)", runScalerScenario, rolloutGuardCases) - DescribeTable("settle guard (G7)", runScalerScenario, settleGuardCases) - DescribeTable("stabilization and max step", runScalerScenario, stabilizationAndStepCases) - DescribeTable("deferral timeout", runScalerScenario, deferralTimeoutCases) - DescribeTable("deferral status", runScalerScenario, deferralStatusCases) - DescribeTable("forced-write pacing", runScalerScenario, forcedWritePacingCases) + DescribeTable("metric scaling", runScalerCase, + ginkgocases.Entries("testdata/controller/metric-scaling", scalerDesc)) + DescribeTable("rollout guard (G8)", runScalerCase, + ginkgocases.Entries("testdata/controller/rollout-guard", scalerDesc)) + DescribeTable("settle guard (G7)", runScalerCase, + ginkgocases.Entries("testdata/controller/settle-guard", scalerDesc)) + DescribeTable("stabilization and max step", runScalerCase, + ginkgocases.Entries("testdata/controller/stabilization-and-step", scalerDesc)) + DescribeTable("deferral timeout", runScalerCase, + ginkgocases.Entries("testdata/controller/deferral-timeout", scalerDesc)) + DescribeTable("deferral status", runScalerCase, + ginkgocases.Entries("testdata/controller/deferral-status", scalerDesc)) + DescribeTable("forced-write pacing", runScalerCase, + ginkgocases.Entries("testdata/controller/forced-write-pacing", scalerDesc)) - It("should default deferTimeoutSeconds to 600", func() { - scaler := &autoscalingv1alpha1.LLMScaler{} - Expect(k8sClient.Get(promFix.ctx, promFix.scaler, scaler)).To(Succeed()) - // The fixture leaves the field unset; the CRD default fills it. - Expect(scaler.Spec.DeferTimeoutSeconds).NotTo(BeNil()) - Expect(*scaler.Spec.DeferTimeoutSeconds).To(Equal(autoscalingv1alpha1.DefaultDeferTimeoutSeconds)) - }) + for _, f := range cases.MustLoad[deferDefaultFile]("testdata/controller/crd-default") { + tc := f.Case + It(tc.Description, func() { runDeferDefault(tc) }) + } }) // These run against the envtest apiserver so the CRD's own union rules are @@ -65,28 +69,20 @@ var _ = Describe("LLMScaler Controller", func() { customFix = nil }) - DescribeTable("decision server", runCustomProviderReconcile, customProviderReconcileCases) - DescribeTable("admission", runCustomProviderAdmission, customProviderAdmissionCases) + DescribeTable("decision server", runCustomReconcileFile, + ginkgocases.Entries("testdata/controller/custom-reconcile", customReconcileDesc)) + DescribeTable("admission", runCustomAdmissionFile, + ginkgocases.Entries("testdata/controller/custom-admission", customAdmissionDesc)) - It("should default metricProvider to Prometheus and customProvider.path to /decisions", func() { - By("creating a scaler that leaves both unset") - scaler := customFix.newScaler() - scaler.Spec.MetricProvider = "" - scaler.Spec.CustomProvider = nil - scaler.Spec.Metrics = []autoscalingv1alpha1.MetricSpec{{Query: "avg(x)", Target: "1"}} - Expect(k8sClient.Create(customFix.ctx, scaler)).To(Succeed()) + for _, f := range cases.MustLoad[customDefaultFile]("testdata/controller/custom-defaults") { + tc := f.Case + It(tc.Description, func() { runCustomDefault(tc) }) + } + }) +}) - stored := &autoscalingv1alpha1.LLMScaler{} - Expect(k8sClient.Get(customFix.ctx, customFix.scaler, stored)).To(Succeed()) - Expect(stored.Spec.MetricProvider).To(Equal(autoscalingv1alpha1.MetricProviderPrometheus)) +func scalerDesc(c scalerCase) string { return c.Description } - By("switching it to the Custom provider without naming a path") - stored.Spec.MetricProvider = autoscalingv1alpha1.MetricProviderCustom - stored.Spec.CustomProvider = &autoscalingv1alpha1.CustomProviderSpec{ServiceID: sampleServiceID} - Expect(k8sClient.Update(customFix.ctx, stored)).To(Succeed()) +func customReconcileDesc(c customReconcileFile) string { return c.Description } - Expect(k8sClient.Get(customFix.ctx, customFix.scaler, stored)).To(Succeed()) - Expect(stored.Spec.CustomProvider.Path).To(Equal(defaultDecisionsPath)) - }) - }) -}) +func customAdmissionDesc(c customAdmissionFile) string { return c.Description } diff --git a/internal/controller/autoscaling/llmscaler_helpers_cases_test.go b/internal/controller/autoscaling/llmscaler_helpers_cases_test.go deleted file mode 100644 index 5aeb195..0000000 --- a/internal/controller/autoscaling/llmscaler_helpers_cases_test.go +++ /dev/null @@ -1,199 +0,0 @@ -/* -Copyright 2026. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package controller - -import ( - "time" - - autoscalingv1alpha1 "github.com/modelsphere/llm-operator/api/autoscaling/v1alpha1" -) - -// deferTimeoutCases cover a nil spec (CRD default / in-memory object) and an -// explicit 0, which must stay distinguishable so "never force" is opt-in. -var deferTimeoutCases = []deferTimeoutCase{ - {name: "nil spec", want: time.Duration(autoscalingv1alpha1.DefaultDeferTimeoutSeconds) * time.Second}, - {name: "explicit 0", deferTimeout: ptr(int32(0)), want: 0}, - {name: "explicit 30", deferTimeout: ptr(int32(30)), want: 30 * time.Second}, -} - -// pastDeferTimeoutCases pin the deadline: equal to the timeout forces, short of -// it does not, and 0 never does. Elapsed is added to the runner's fixed base. -var pastDeferTimeoutCases = []pastDeferTimeoutCase{ - {name: "nil deferredSince", nilSince: true, timeout: time.Minute, elapsed: time.Hour, want: false}, - {name: "timeout 0", timeout: 0, elapsed: 24 * time.Hour, want: false}, - {name: "59s of a 60s timeout", timeout: time.Minute, elapsed: 59 * time.Second, want: false}, - {name: "exactly the timeout", timeout: time.Minute, elapsed: time.Minute, want: true}, - {name: "past the timeout", timeout: time.Minute, elapsed: time.Minute + time.Second, want: true}, -} - -// deferralRequeueAfterCases is how long a deferred sync waits. The runner passes -// a 15s sync period, the controller default; rows whose want is 15s are the ones -// that stay on that period. -var deferralRequeueAfterCases = []deferralRequeueAfterCase{ - {name: "deadline sooner than sync period", holding: true, timeout: time.Minute, until: 5 * time.Second, want: 5 * time.Second}, - {name: "deadline later than sync period", holding: true, timeout: time.Minute, until: 30 * time.Second, want: 15 * time.Second}, - {name: "deadline already passed", holding: true, timeout: time.Minute, until: -time.Second, want: time.Second}, - {name: "deadline due this instant", holding: true, timeout: time.Minute, until: 0, want: time.Second}, - {name: "deadline under the 1s floor", holding: true, timeout: time.Minute, until: 200 * time.Millisecond, want: time.Second}, - {name: "timeout 0 stays on sync period", holding: true, timeout: 0, until: 5 * time.Second, want: 15 * time.Second}, - {name: "not holding stays on sync period", holding: false, timeout: time.Minute, until: 5 * time.Second, want: 15 * time.Second}, - {name: "pacing sooner than sync period", holding: true, timeout: time.Minute, until: -time.Second, pacing: 4 * time.Second, want: 4 * time.Second}, - {name: "pacing later than sync period", holding: true, timeout: time.Minute, until: -time.Second, pacing: 30 * time.Second, want: 15 * time.Second}, - {name: "pacing under the 1s floor", holding: true, timeout: time.Minute, until: -time.Second, pacing: 100 * time.Millisecond, want: time.Second}, -} - -// forcedWritePaceCases cover the per-process gap between forced writes. -// wantRemaining nil leaves the remaining duration unchecked. -var forcedWritePaceCases = []forcedWritePaceCase{ - {name: "no record", recorded: false, sync: 15 * time.Second, at: 0, wantAllowed: true, wantRemaining: ptr(time.Duration(0))}, - {name: "zero sync period", recorded: true, sync: 0, at: time.Hour, wantAllowed: true, wantRemaining: ptr(time.Duration(0))}, - {name: "exactly one sync period", recorded: true, sync: 15 * time.Second, at: 15 * time.Second, wantAllowed: true}, - {name: "just recorded", recorded: true, sync: 15 * time.Second, at: 0, wantAllowed: false}, - {name: "14s into a 15s period", recorded: true, sync: 15 * time.Second, at: 14 * time.Second, wantAllowed: false, wantRemaining: ptr(time.Second)}, -} - -// stabilizationSequenceCases run an ordered list of recommend/hold ops on one -// fresh reconciler per case. at is an offset from the runner's fixed base, -// not the wall clock. TestStabilizeDesired and TestHoldAfterScaleDown are both -// rows here, plus the window-0 reconciler. -var stabilizationSequenceCases = []stabilizationCase{ - { - // First recommendation sticks, a dip inside the window holds the peak, - // the peak expires after the window, and a scale-up is immediate. - name: "peak holds inside the window then expires", - window: 60 * time.Second, - ops: []stabilizationOp{ - {at: 0, recommend: ptr(int32(5)), want: ptr(int32(5))}, - // Metric dipped to 1, but the peak of 5 is still within the window. - {at: 10 * time.Second, recommend: ptr(int32(1)), want: ptr(int32(5))}, - // After the window the peak has expired. - {at: 61 * time.Second, recommend: ptr(int32(1)), want: ptr(int32(1))}, - // Scale-up is immediate regardless of history. - {at: 61 * time.Second, recommend: ptr(int32(9)), want: ptr(int32(9))}, - }, - }, - { - // Pinning the written count after a scale-down paces the next one a - // full window later, rather than letting the descent resolve off one - // collapsed reading. - name: "pin after a scale-down paces the next step", - window: 300 * time.Second, - ops: []stabilizationOp{ - // Fleet of 8, metric collapses; the peak is still in the window. - {at: 0, recommend: ptr(int32(8)), want: ptr(int32(8))}, - // A window later the peak has expired and the low recommendation - // wins. The controller then writes 6 (an 8 -> 1 recommendation - // capped to a 2-replica step) and pins it. - {at: 301 * time.Second, recommend: ptr(int32(1)), want: ptr(int32(1))}, - {at: 301 * time.Second, holdAt: ptr(int32(6))}, - // Within the window the fleet is held at 6 even though the metric - // still reads idle. Before the pin these would each have shrunk it. - {at: 311 * time.Second, recommend: ptr(int32(1)), want: ptr(int32(6))}, - {at: 400 * time.Second, recommend: ptr(int32(1)), want: ptr(int32(6))}, - {at: 600 * time.Second, recommend: ptr(int32(1)), want: ptr(int32(6))}, - // Once the pin ages out, the next step is allowed. - {at: 602 * time.Second, recommend: ptr(int32(1)), want: ptr(int32(1))}, - // Scale-up is never blocked by the pin. - {at: 700 * time.Second, holdAt: ptr(int32(6))}, - {at: 710 * time.Second, recommend: ptr(int32(9)), want: ptr(int32(9))}, - }, - }, - { - // With stabilization off there is nothing to pace with, so the pin is a no-op. - name: "window 0 makes the pin a no-op", - window: 0, - ops: []stabilizationOp{ - {at: 0, holdAt: ptr(int32(6))}, - {at: 0, recommend: ptr(int32(1)), want: ptr(int32(1))}, - }, - }, -} - -// parseTargetValueCases cover the target parser. It is the denominator of the -// scaling ratio and must never yield zero, a negative, or a non-finite value. -var parseTargetValueCases = []parseTargetValueCase{ - {name: "plain integer", in: "5", want: 5}, - {name: "ratio", in: "0.8", want: 0.8}, - {name: "surrounding whitespace", in: " 2.5 ", want: 2.5}, - {name: "percent suffix is rejected, not stripped", in: "80%", wantErr: true}, - {name: "zero would make the ratio infinite", in: "0", wantErr: true}, - {name: "negative would invert scaling", in: "-1", wantErr: true}, - {name: "NaN parses as a float but is not usable", in: "NaN", wantErr: true}, - {name: "Inf parses as a float but is not usable", in: "+Inf", wantErr: true}, - {name: "not a number", in: "abc", wantErr: true}, - {name: "empty", in: "", wantErr: true}, -} - -// queryPrometheusScalarCases resolve an instant-query vector: exactly one series -// is a value, and anything else is an error the caller skips the metric on. -var queryPrometheusScalarCases = []queryPrometheusScalarCase{ - { - name: "single series resolves", - result: `[{"metric":{},"value":[0,"1.5"]}]`, - want: 1.5, - }, - { - name: "empty vector is not zero", - result: `[]`, - wantErrContains: []string{"returned no data"}, - }, - { - name: "an under-aggregated query is rejected, naming the series", - result: `[{"metric":{"__name__":"m","model":"qwen","backend":"10.0.0.1:8000"},"value":[0,"1"]}, - {"metric":{"__name__":"m","model":"qwen","backend":"10.0.0.2:8000"},"value":[0,"9"]}]`, - wantErrContains: []string{"returned 2 series"}, - }, - { - // The multi-series error has to point at the label to aggregate away, - // so it quotes both series with their labels sorted. - name: "multi-series error names both label sets", - result: `[ - {"metric":{"model":"qwen","backend":"10.0.0.1:8000"},"value":[0,"1"]}, - {"metric":{"model":"qwen","backend":"10.0.0.2:8000"},"value":[0,"9"]}]`, - wantErrContains: []string{ - `{backend="10.0.0.1:8000",model="qwen"}`, - `{backend="10.0.0.2:8000",model="qwen"}`, - "avg(...)", - }, - }, -} - -// computeDesiredFromMetricsCases cover a non-finite sample (not a measurement) -// and the scale-from-zero baseline. A NaN or Inf sample must not set haveMetric: -// histogram_quantile over an empty window returns NaN, and treating that as zero -// would read as "scale to minReplicas". With no ready replicas a positive signal -// is evaluated against a single-replica baseline, so the fleet wakes; a zero -// signal still recommends 0. -var computeDesiredFromMetricsCases = []computeDesiredCase{ - {name: "NaN", sample: "NaN", ready: 4, metricName: "guardrail", query: "histogram_quantile(0.95, x)", target: "2"}, - {name: "+Inf", sample: "+Inf", ready: 4, metricName: "guardrail", query: "histogram_quantile(0.95, x)", target: "2"}, - {name: "-Inf", sample: "-Inf", ready: 4, metricName: "guardrail", query: "histogram_quantile(0.95, x)", target: "2"}, - // ceil(1 * 0.85/0.5) = 2, not ceil(0 * 0.85/0.5) = 0. - {name: "positive signal wakes the fleet", sample: mockMetricValue, ready: 0, minReplicas: 0, maxReplicas: 5, metricName: "queue", query: "sum(queue_depth)", target: "0.5", wantHave: true, wantDesired: 2}, - // A zero signal still recommends 0. - {name: "zero signal recommends zero", sample: "0", ready: 0, minReplicas: 0, maxReplicas: 5, metricName: "queue", query: "sum(queue_depth)", target: "0.5", wantHave: true, wantDesired: 0}, -} - -// scaleGuardCases are the scale-down rate limiter. Scale-up and a no-op pass through. -var scaleGuardCases = []scaleGuardCase{ - {name: "unlimited when maxStep is 0", current: 10, desired: 1, maxStep: 0, want: 1}, - {name: "caps a burst scale-down", current: 10, desired: 1, maxStep: 2, want: 8}, - {name: "passes through a step within the cap", current: 10, desired: 9, maxStep: 2, want: 9}, - {name: "passes through a step exactly at the cap", current: 10, desired: 8, maxStep: 2, want: 8}, - {name: "leaves scale-up untouched", current: 2, desired: 10, maxStep: 1, want: 10}, - {name: "leaves a no-op untouched", current: 5, desired: 5, maxStep: 1, want: 5}, -} diff --git a/internal/controller/autoscaling/llmscaler_helpers_test.go b/internal/controller/autoscaling/llmscaler_helpers_test.go index 6f46897..8f8916c 100644 --- a/internal/controller/autoscaling/llmscaler_helpers_test.go +++ b/internal/controller/autoscaling/llmscaler_helpers_test.go @@ -29,6 +29,7 @@ import ( "k8s.io/apimachinery/pkg/types" autoscalingv1alpha1 "github.com/modelsphere/llm-operator/api/autoscaling/v1alpha1" + "github.com/modelsphere/llm-operator/internal/testutil/cases" ) // pastDeferBase is the deferredSince instant for TestPastDeferTimeout. @@ -47,288 +48,369 @@ var stabilizationKey = types.NamespacedName{Namespace: "ns", Name: "s"} // deferralRequeueSync is the sync period passed to every deferralRequeueAfter row. const deferralRequeueSync = 15 * time.Second -type deferTimeoutCase struct { - name string - deferTimeout *int32 - want time.Duration +type deferTimeoutFile struct { + Name string `json:"name,omitempty"` + Description string `json:"description,omitempty"` + Inputs struct { + DeferTimeoutSeconds *int32 `json:"deferTimeoutSeconds"` + } `json:"inputs"` + Expected struct { + Duration cases.Duration `json:"duration"` + EqualsDefault bool `json:"equalsDefault"` + } `json:"expected"` } func TestDeferTimeout(t *testing.T) { - for _, tc := range deferTimeoutCases { - t.Run(tc.name, func(t *testing.T) { - scaler := &autoscalingv1alpha1.LLMScaler{} - scaler.Spec.DeferTimeoutSeconds = tc.deferTimeout - got := deferTimeout(scaler) - if got != tc.want { - t.Fatalf("got %s, want %s", got, tc.want) + cases.Run(t, "testdata/helpers/defer-timeout", func(t *testing.T, tc deferTimeoutFile) { + scaler := &autoscalingv1alpha1.LLMScaler{} + scaler.Spec.DeferTimeoutSeconds = tc.Inputs.DeferTimeoutSeconds + got := deferTimeout(scaler) + if got != tc.Expected.Duration.Std() { + t.Fatalf("got %s, want %s", got, tc.Expected.Duration.Std()) + } + if tc.Expected.EqualsDefault { + want := time.Duration(autoscalingv1alpha1.DefaultDeferTimeoutSeconds) * time.Second + if got != want { + t.Fatalf("got %s, want default %s", got, want) } - }) - } + } + }) } -type pastDeferTimeoutCase struct { - name string - nilSince bool - timeout time.Duration - elapsed time.Duration - want bool +type pastDeferFile struct { + Name string `json:"name,omitempty"` + Description string `json:"description,omitempty"` + Inputs struct { + NilSince bool `json:"nilSince"` + Timeout cases.Duration `json:"timeout"` + Elapsed cases.Duration `json:"elapsed"` + } `json:"inputs"` + Expected struct { + Past bool `json:"past"` + } `json:"expected"` } func TestPastDeferTimeout(t *testing.T) { - for _, tc := range pastDeferTimeoutCases { - t.Run(tc.name, func(t *testing.T) { - var since *metav1.Time - if !tc.nilSince { - stamped := metav1.NewTime(pastDeferBase) - since = &stamped - } - got := pastDeferTimeout(since, tc.timeout, pastDeferBase.Add(tc.elapsed)) - if got != tc.want { - t.Fatalf("got %v, want %v", got, tc.want) - } - }) - } + cases.Run(t, "testdata/helpers/past-defer-timeout", func(t *testing.T, tc pastDeferFile) { + var since *metav1.Time + if !tc.Inputs.NilSince { + stamped := metav1.NewTime(pastDeferBase) + since = &stamped + } + now := pastDeferBase.Add(tc.Inputs.Elapsed.Std()) + got := pastDeferTimeout(since, tc.Inputs.Timeout.Std(), now) + if got != tc.Expected.Past { + t.Fatalf("got %v, want %v", got, tc.Expected.Past) + } + }) } -type deferralRequeueAfterCase struct { - name string - holding bool - timeout time.Duration - until time.Duration - pacing time.Duration - want time.Duration +type requeueFile struct { + Name string `json:"name,omitempty"` + Description string `json:"description,omitempty"` + Inputs struct { + Holding bool `json:"holding"` + Timeout cases.Duration `json:"timeout"` + Until cases.Duration `json:"until"` + Pacing cases.Duration `json:"pacing"` + } `json:"inputs"` + Expected struct { + After cases.Duration `json:"after"` + } `json:"expected"` } func TestDeferralRequeueAfter(t *testing.T) { - for _, tc := range deferralRequeueAfterCases { - t.Run(tc.name, func(t *testing.T) { - got := deferralRequeueAfter(deferralRequeueSync, tc.holding, tc.timeout, tc.until, tc.pacing) - if got != tc.want { - t.Fatalf("got %s, want %s", got, tc.want) - } - }) - } + cases.Run(t, "testdata/helpers/deferral-requeue-after", func(t *testing.T, tc requeueFile) { + in := tc.Inputs + got := deferralRequeueAfter(deferralRequeueSync, in.Holding, in.Timeout.Std(), in.Until.Std(), in.Pacing.Std()) + if got != tc.Expected.After.Std() { + t.Fatalf("got %s, want %s", got, tc.Expected.After.Std()) + } + }) } -type forcedWritePaceCase struct { - name string - recorded bool - sync time.Duration - at time.Duration - wantAllowed bool - wantRemaining *time.Duration +type forcedWriteFile struct { + Name string `json:"name,omitempty"` + Description string `json:"description,omitempty"` + Inputs struct { + Recorded bool `json:"recorded"` + Sync cases.Duration `json:"sync"` + At cases.Duration `json:"at"` + } `json:"inputs"` + Expected struct { + Allowed bool `json:"allowed"` + Remaining *cases.Duration `json:"remaining"` + } `json:"expected"` } func TestForcedWritePace(t *testing.T) { - for _, tc := range forcedWritePaceCases { - t.Run(tc.name, func(t *testing.T) { - allowed, remaining := forcedWritePace(forcedWriteBase, tc.recorded, tc.sync, forcedWriteBase.Add(tc.at)) - if allowed != tc.wantAllowed { - t.Fatalf("allowed=%v, want %v", allowed, tc.wantAllowed) - } - if tc.wantRemaining != nil && remaining != *tc.wantRemaining { - t.Fatalf("remaining=%s, want %s", remaining, *tc.wantRemaining) - } - }) - } + cases.Run(t, "testdata/helpers/forced-write-pace", func(t *testing.T, tc forcedWriteFile) { + at := forcedWriteBase.Add(tc.Inputs.At.Std()) + allowed, remaining := forcedWritePace(forcedWriteBase, tc.Inputs.Recorded, tc.Inputs.Sync.Std(), at) + if allowed != tc.Expected.Allowed { + t.Fatalf("allowed=%v, want %v", allowed, tc.Expected.Allowed) + } + if tc.Expected.Remaining != nil && remaining != tc.Expected.Remaining.Std() { + t.Fatalf("remaining=%s, want %s", remaining, tc.Expected.Remaining.Std()) + } + }) +} + +type memoryStep struct { + Op string `json:"op"` +} + +type memoryFile struct { + Name string `json:"name,omitempty"` + Description string `json:"description,omitempty"` + Inputs struct { + Namespace string `json:"namespace"` + Scaler string `json:"scaler"` + At string `json:"at"` + Steps []memoryStep `json:"steps"` + } `json:"inputs"` + Expected struct{} `json:"expected"` } // TestDeferralMemoryClearedOnEndAndDelete checks the two ways the in-memory // pace record goes away: the deferral ending, and the scaler being deleted. func TestDeferralMemoryClearedOnEndAndDelete(t *testing.T) { - r := &LLMScalerReconciler{} - key := types.NamespacedName{Namespace: "ns", Name: "s"} - now := time.Date(2026, 3, 1, 0, 0, 0, 0, time.UTC) - r.recordForcedWrite(key, now) - scaler := &autoscalingv1alpha1.LLMScaler{} - stamped := metav1.NewTime(now) - scaler.Status.DeferredSince = &stamped - - r.endDeferral(key, scaler) - if scaler.Status.DeferredSince != nil { - t.Fatal("endDeferral left deferredSince set") - } - if _, ok := r.lastForcedWriteAt(key); ok { - t.Fatal("endDeferral left the forced-write record") - } - - r.recordForcedWrite(key, now) - r.forgetRecommendations(key) - if _, ok := r.lastForcedWriteAt(key); ok { - t.Fatal("forgetRecommendations left the forced-write record") - } + cases.Run(t, "testdata/helpers/deferral-memory", func(t *testing.T, tc memoryFile) { + r := &LLMScalerReconciler{} + key := types.NamespacedName{Namespace: tc.Inputs.Namespace, Name: tc.Inputs.Scaler} + now, err := time.Parse(time.RFC3339, tc.Inputs.At) + if err != nil { + t.Fatalf("at: %v", err) + } + scaler := &autoscalingv1alpha1.LLMScaler{} + for _, step := range tc.Inputs.Steps { + switch step.Op { + case "record": + r.recordForcedWrite(key, now) + case "setSince": + stamped := metav1.NewTime(now) + scaler.Status.DeferredSince = &stamped + case "end": + r.endDeferral(key, scaler) + case "forget": + r.forgetRecommendations(key) + case "expectSinceCleared": + if scaler.Status.DeferredSince != nil { + t.Fatal("endDeferral left deferredSince set") + } + case "expectRecordCleared": + if _, ok := r.lastForcedWriteAt(key); ok { + t.Fatal("forced-write record still set") + } + default: + t.Fatalf("unknown step %q", step.Op) + } + } + }) +} + +type reasonFile struct { + Name string `json:"name,omitempty"` + Description string `json:"description,omitempty"` + Inputs struct { + } `json:"inputs"` + Expected struct { + Reason string `json:"reason"` + } `json:"expected"` } // TestReasonDeferralTimedOut pins the published condition reason, which users // match on in alerts and kubectl output. func TestReasonDeferralTimedOut(t *testing.T) { - if autoscalingv1alpha1.ReasonDeferralTimedOut != "DeferralTimedOut" { - t.Fatalf("ReasonDeferralTimedOut = %q, want DeferralTimedOut", autoscalingv1alpha1.ReasonDeferralTimedOut) - } + cases.Run(t, "testdata/helpers/reason-deferral-timed-out", func(t *testing.T, tc reasonFile) { + if autoscalingv1alpha1.ReasonDeferralTimedOut != tc.Expected.Reason { + t.Fatalf("ReasonDeferralTimedOut = %q, want %q", autoscalingv1alpha1.ReasonDeferralTimedOut, tc.Expected.Reason) + } + }) } -type stabilizationOp struct { - at time.Duration - recommend *int32 - holdAt *int32 - want *int32 +type stabilizationOpFile struct { + At cases.Duration `json:"at"` + Recommend *int32 `json:"recommend"` + HoldAt *int32 `json:"holdAt"` + Want *int32 `json:"want"` } -type stabilizationCase struct { - name string - window time.Duration - ops []stabilizationOp +type stabilizationFile struct { + Name string `json:"name,omitempty"` + Description string `json:"description,omitempty"` + Inputs struct { + Window cases.Duration `json:"window"` + Ops []stabilizationOpFile `json:"ops"` + } `json:"inputs"` + Expected struct{} `json:"expected"` } // TestStabilizeDesired runs each stabilization sequence on its own reconciler. -// TestHoldAfterScaleDown's steps, including the window-0 reconciler, are rows -// of the same table. +// The window-0 pin is a row of the same table. func TestStabilizeDesired(t *testing.T) { - for _, tc := range stabilizationSequenceCases { - t.Run(tc.name, func(t *testing.T) { - r := &LLMScalerReconciler{} - for _, op := range tc.ops { - now := stabilizationBase.Add(op.at) - switch { - case op.holdAt != nil && op.recommend != nil: - t.Fatalf("op at %s sets both holdAt and recommend", op.at) - case op.holdAt != nil: - r.holdAfterScaleDown(stabilizationKey, *op.holdAt, tc.window, now) - case op.recommend != nil: - if op.want == nil { - t.Fatalf("recommend at %s has no want", op.at) - } - got := r.stabilizeDesired(stabilizationKey, *op.recommend, tc.window, now) - if got != *op.want { - t.Fatalf("at %s: got %d, want %d", op.at, got, *op.want) - } - default: - t.Fatalf("op at %s has neither holdAt nor recommend", op.at) + cases.Run(t, "testdata/helpers/stabilization", func(t *testing.T, tc stabilizationFile) { + r := &LLMScalerReconciler{} + window := tc.Inputs.Window.Std() + for _, op := range tc.Inputs.Ops { + now := stabilizationBase.Add(op.At.Std()) + switch { + case op.HoldAt != nil && op.Recommend != nil: + t.Fatalf("op at %s sets both holdAt and recommend", op.At.Std()) + case op.HoldAt != nil: + r.holdAfterScaleDown(stabilizationKey, *op.HoldAt, window, now) + case op.Recommend != nil: + if op.Want == nil { + t.Fatalf("recommend at %s has no want", op.At.Std()) + } + got := r.stabilizeDesired(stabilizationKey, *op.Recommend, window, now) + if got != *op.Want { + t.Fatalf("at %s: got %d, want %d", op.At.Std(), got, *op.Want) } + default: + t.Fatalf("op at %s has neither holdAt nor recommend", op.At.Std()) } - }) - } + } + }) } -type parseTargetValueCase struct { - name string - in string - want float64 - wantErr bool +type parseTargetFile struct { + Name string `json:"name,omitempty"` + Description string `json:"description,omitempty"` + Inputs struct { + Value string `json:"value"` + } `json:"inputs"` + Expected struct { + Value float64 `json:"value"` + Error bool `json:"error"` + } `json:"expected"` } func TestParseTargetValue(t *testing.T) { - for _, tc := range parseTargetValueCases { - t.Run(tc.name, func(t *testing.T) { - got, err := parseTargetValue(tc.in) - if (err != nil) != tc.wantErr { - t.Fatalf("parseTargetValue(%q) error = %v, wantErr %v", tc.in, err, tc.wantErr) - } - if err == nil && got != tc.want { - t.Fatalf("parseTargetValue(%q) = %v, want %v", tc.in, got, tc.want) - } - }) - } + cases.Run(t, "testdata/helpers/parse-target-value", func(t *testing.T, tc parseTargetFile) { + got, err := parseTargetValue(tc.Inputs.Value) + if (err != nil) != tc.Expected.Error { + t.Fatalf("parseTargetValue(%q) error = %v, wantErr %v", tc.Inputs.Value, err, tc.Expected.Error) + } + if err == nil && got != tc.Expected.Value { + t.Fatalf("parseTargetValue(%q) = %v, want %v", tc.Inputs.Value, got, tc.Expected.Value) + } + }) } -type queryPrometheusScalarCase struct { - name string - result string - want float64 - wantErrContains []string +type promScalarFile struct { + Name string `json:"name,omitempty"` + Description string `json:"description,omitempty"` + Inputs struct { + Result string `json:"result"` + } `json:"inputs"` + Expected struct { + Value float64 `json:"value"` + ErrorContains []string `json:"errorContains"` + } `json:"expected"` } func TestQueryPrometheusScalar(t *testing.T) { - for _, tc := range queryPrometheusScalarCases { - t.Run(tc.name, func(t *testing.T) { - srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { - _, _ = fmt.Fprintf(w, `{"status":"success","data":{"result":%s}}`, tc.result) - })) - defer srv.Close() - - got, err := queryPrometheusScalar(srv.URL, "some_query", nil) - if len(tc.wantErrContains) == 0 { - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - if got != tc.want { - t.Fatalf("got %v, want %v", got, tc.want) - } - return + cases.Run(t, "testdata/helpers/query-prometheus-scalar", func(t *testing.T, tc promScalarFile) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + _, _ = fmt.Fprintf(w, `{"status":"success","data":{"result":%s}}`, tc.Inputs.Result) + })) + defer srv.Close() + + got, err := queryPrometheusScalar(srv.URL, "some_query", nil) + if len(tc.Expected.ErrorContains) == 0 { + if err != nil { + t.Fatalf("unexpected error: %v", err) } - if err == nil { - t.Fatalf("expected error containing %q, got value %v", tc.wantErrContains, got) + if got != tc.Expected.Value { + t.Fatalf("got %v, want %v", got, tc.Expected.Value) } - for _, want := range tc.wantErrContains { - if !strings.Contains(err.Error(), want) { - t.Errorf("error = %q, want it to contain %q", err, want) - } + return + } + if err == nil { + t.Fatalf("expected error containing %q, got value %v", tc.Expected.ErrorContains, got) + } + for _, want := range tc.Expected.ErrorContains { + // An empty substring matches every error. + if want == "" { + t.Fatal("errorContains entry is empty") + } + if !strings.Contains(err.Error(), want) { + t.Errorf("error = %q, want it to contain %q", err, want) } - }) - } + } + }) } -type computeDesiredCase struct { - name string - sample string - ready int64 - minReplicas int32 - maxReplicas int32 - metricName string - query string - target string - wantHave bool - wantDesired int32 +type computeDesiredFile struct { + Name string `json:"name,omitempty"` + Description string `json:"description,omitempty"` + Inputs struct { + Sample string `json:"sample"` + Ready int64 `json:"ready"` + MinReplicas int32 `json:"minReplicas"` + MaxReplicas int32 `json:"maxReplicas"` + MetricName string `json:"metricName"` + Query string `json:"query"` + Target string `json:"target"` + } `json:"inputs"` + Expected struct { + Have bool `json:"have"` + Desired int32 `json:"desired"` + } `json:"expected"` } // TestComputeDesiredFromMetrics covers non-finite samples and scale-from-zero. func TestComputeDesiredFromMetrics(t *testing.T) { - for _, tc := range computeDesiredFromMetricsCases { - t.Run(tc.name, func(t *testing.T) { - srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { - _, _ = fmt.Fprintf(w, - `{"status":"success","data":{"result":[{"metric":{},"value":[0,"%s"]}]}}`, tc.sample) - })) - defer srv.Close() - - scaler := &autoscalingv1alpha1.LLMScaler{ - Spec: autoscalingv1alpha1.LLMScalerSpec{ - ServerAddress: srv.URL, - MinReplicas: tc.minReplicas, - MaxReplicas: tc.maxReplicas, - Metrics: []autoscalingv1alpha1.MetricSpec{{ - Name: tc.metricName, - Query: tc.query, - Target: tc.target, - }}, - }, - } - desired, haveMetric := (&LLMScalerReconciler{}).computeDesiredFromMetrics(context.Background(), scaler, tc.ready) - if haveMetric != tc.wantHave { - t.Errorf("haveMetric = %v, want %v", haveMetric, tc.wantHave) - } - if desired != tc.wantDesired { - t.Errorf("desired = %d, want %d", desired, tc.wantDesired) - } - }) - } + cases.Run(t, "testdata/helpers/compute-desired", func(t *testing.T, tc computeDesiredFile) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + _, _ = fmt.Fprintf(w, + `{"status":"success","data":{"result":[{"metric":{},"value":[0,"%s"]}]}}`, tc.Inputs.Sample) + })) + defer srv.Close() + + scaler := &autoscalingv1alpha1.LLMScaler{ + Spec: autoscalingv1alpha1.LLMScalerSpec{ + ServerAddress: srv.URL, + MinReplicas: tc.Inputs.MinReplicas, + MaxReplicas: tc.Inputs.MaxReplicas, + Metrics: []autoscalingv1alpha1.MetricSpec{{ + Name: tc.Inputs.MetricName, + Query: tc.Inputs.Query, + Target: tc.Inputs.Target, + }}, + }, + } + desired, haveMetric := (&LLMScalerReconciler{}).computeDesiredFromMetrics( + context.Background(), scaler, tc.Inputs.Ready) + if haveMetric != tc.Expected.Have { + t.Errorf("haveMetric = %v, want %v", haveMetric, tc.Expected.Have) + } + if desired != tc.Expected.Desired { + t.Errorf("desired = %d, want %d", desired, tc.Expected.Desired) + } + }) } -type scaleGuardCase struct { - name string - current int32 - desired int32 - maxStep int32 - want int32 +type scaleGuardFile struct { + Name string `json:"name,omitempty"` + Description string `json:"description,omitempty"` + Inputs struct { + Current int32 `json:"current"` + Desired int32 `json:"desired"` + MaxStep int32 `json:"maxStep"` + } `json:"inputs"` + Expected struct { + Want int32 `json:"want"` + } `json:"expected"` } func TestScaleGuard(t *testing.T) { - for _, tc := range scaleGuardCases { - t.Run(tc.name, func(t *testing.T) { - got := scaleGuard(tc.current, tc.desired, tc.maxStep) - if got != tc.want { - t.Fatalf("scaleGuard(%d, %d, %d) = %d, want %d", tc.current, tc.desired, tc.maxStep, got, tc.want) - } - }) - } + cases.Run(t, "testdata/helpers/scale-guard", func(t *testing.T, tc scaleGuardFile) { + in := tc.Inputs + got := scaleGuard(in.Current, in.Desired, in.MaxStep) + if got != tc.Expected.Want { + t.Fatalf("scaleGuard(%d, %d, %d) = %d, want %d", in.Current, in.Desired, in.MaxStep, got, tc.Expected.Want) + } + }) } diff --git a/internal/controller/autoscaling/llmscaler_scenario_test.go b/internal/controller/autoscaling/llmscaler_scenario_test.go index 8521afa..df4fc37 100644 --- a/internal/controller/autoscaling/llmscaler_scenario_test.go +++ b/internal/controller/autoscaling/llmscaler_scenario_test.go @@ -93,14 +93,6 @@ type sinceExpect struct { offset time.Duration } -func sinceAt(offset time.Duration) sinceExpect { - return sinceExpect{kind: sinceOffset, offset: offset} -} - -func sinceNil() sinceExpect { return sinceExpect{kind: sinceNone} } - -func sinceSeed() sinceExpect { return sinceExpect{kind: sinceSeeded} } - // scalerSpecPatch is applied to the fixture LLMScaler before the steps. // A nil field is left at the fixture value. DeferTimeoutSeconds is a pointer // on the API, so a pointer to 0 is "never force" and nil means "don't patch". @@ -635,7 +627,11 @@ func assertCondition(scaler *autoscalingv1alpha1.LLMScaler, exp condExpect) { if exp.reason != "" { Expect(cond.Reason).To(Equal(exp.reason)) } + // An empty substring matches every message, so it is rejected on its own. for _, sub := range exp.messageContains { + if sub == "" { + Fail("messageContains entry is empty") + } Expect(cond.Message).To(ContainSubstring(sub)) } if exp.observedGenerationCurrent { @@ -649,6 +645,12 @@ func assertEvents(run *scenarioRun, exp stepExpect) { } Expect(run.rec).NotTo(BeNil()) for _, want := range exp.events { + // An empty substring matches every event, so it is rejected on its own. + for _, sub := range want.contains { + if sub == "" { + Fail("events.contains entry is empty") + } + } var got string Expect(run.rec.Events).To(Receive(&got)) if want.prefix != "" { diff --git a/internal/controller/autoscaling/testdata/controller/crd-default/should-default-defertimeoutseconds-to-600.yaml b/internal/controller/autoscaling/testdata/controller/crd-default/should-default-defertimeoutseconds-to-600.yaml new file mode 100644 index 0000000..41cf7bb --- /dev/null +++ b/internal/controller/autoscaling/testdata/controller/crd-default/should-default-defertimeoutseconds-to-600.yaml @@ -0,0 +1,6 @@ +# The fixture leaves the field unset; the CRD default fills it. +"name": "should-default-defertimeoutseconds-to-600" +"description": "should default deferTimeoutSeconds to 600" +"inputs": {} +"expected": + "seconds": 600 diff --git a/internal/controller/autoscaling/testdata/controller/custom-admission/should-reject-a-custom-scaler-with-no-customprovider.yaml b/internal/controller/autoscaling/testdata/controller/custom-admission/should-reject-a-custom-scaler-with-no-customprovider.yaml new file mode 100644 index 0000000..a83874c --- /dev/null +++ b/internal/controller/autoscaling/testdata/controller/custom-admission/should-reject-a-custom-scaler-with-no-customprovider.yaml @@ -0,0 +1,6 @@ +"name": "should-reject-a-custom-scaler-with-no-customprovider" +"description": "should reject a Custom scaler with no customProvider" +"inputs": + "omitCustomProvider": true +"expected": + "errorContains": "spec.customProvider is required" diff --git a/internal/controller/autoscaling/testdata/controller/custom-admission/should-reject-a-prometheus-scaler-with-no-metrics.yaml b/internal/controller/autoscaling/testdata/controller/custom-admission/should-reject-a-prometheus-scaler-with-no-metrics.yaml new file mode 100644 index 0000000..2dff606 --- /dev/null +++ b/internal/controller/autoscaling/testdata/controller/custom-admission/should-reject-a-prometheus-scaler-with-no-metrics.yaml @@ -0,0 +1,8 @@ +"name": "should-reject-a-prometheus-scaler-with-no-metrics" +"description": "should reject a Prometheus scaler with no metrics" +"inputs": + "metricProvider": "Prometheus" + "omitCustomProvider": true + "pinProvider": true +"expected": + "errorContains": "spec.metrics must be non-empty" diff --git a/internal/controller/autoscaling/testdata/controller/custom-defaults/should-default-metricprovider-to-prometheus-and-customprovider-path-to-decisions.yaml b/internal/controller/autoscaling/testdata/controller/custom-defaults/should-default-metricprovider-to-prometheus-and-customprovider-path-to-decisions.yaml new file mode 100644 index 0000000..2f73a20 --- /dev/null +++ b/internal/controller/autoscaling/testdata/controller/custom-defaults/should-default-metricprovider-to-prometheus-and-customprovider-path-to-decisions.yaml @@ -0,0 +1,11 @@ +"name": "should-default-metricprovider-to-prometheus-and-customprovider-path-to-decisions" +"description": "should default metricProvider to Prometheus and customProvider.path to /decisions" +"inputs": + "createBy": "creating a scaler that leaves both unset" + "switchBy": "switching it to the Custom provider without naming a path" + "query": "avg(x)" + "target": "1" + "serviceId": "fallback-modelforge-01" +"expected": + "metricProvider": "Prometheus" + "path": "/decisions" diff --git a/internal/controller/autoscaling/testdata/controller/custom-reconcile/should-hold-replicas-when-the-decision-server-is-unreachable.yaml b/internal/controller/autoscaling/testdata/controller/custom-reconcile/should-hold-replicas-when-the-decision-server-is-unreachable.yaml new file mode 100644 index 0000000..6537faa --- /dev/null +++ b/internal/controller/autoscaling/testdata/controller/custom-reconcile/should-hold-replicas-when-the-decision-server-is-unreachable.yaml @@ -0,0 +1,20 @@ +"name": "should-hold-replicas-when-the-decision-server-is-unreachable" +"description": "should hold replicas when the decision server is unreachable" +"inputs": + "deploy": + "replicas": 3 + "updated": 3 + "ready": 3 + "available": 3 + "spec": 3 + "deployBy": "putting the Deployment at 3 replicas, above minReplicas" + "closeServer": true + "closeBy": "taking the decision server down" + "reconcileBy": "running the Reconciler" + "replicasBy": "checking the Deployment was left where it was, not collapsed to minReplicas" + "statusBy": "checking the failure is reported on the status" +"expected": + "replicas": 3 + "decisionStatus": "False" + "decisionReason": "DecisionUnavailable" + "decisionMessageNonEmpty": true diff --git a/internal/controller/autoscaling/testdata/controller/custom-reconcile/should-scale-the-deployment-to-the-count-the-decision-server-reports.yaml b/internal/controller/autoscaling/testdata/controller/custom-reconcile/should-scale-the-deployment-to-the-count-the-decision-server-reports.yaml new file mode 100644 index 0000000..734a302 --- /dev/null +++ b/internal/controller/autoscaling/testdata/controller/custom-reconcile/should-scale-the-deployment-to-the-count-the-decision-server-reports.yaml @@ -0,0 +1,12 @@ +# active 4 is an absolute count, so 1 ready replica scales straight to 4. +"name": "should-scale-the-deployment-to-the-count-the-decision-server-reports" +"description": "should scale the Deployment to the count the decision server reports" +"inputs": + "createBy": "creating an LLMScaler with no metrics at all" + "reconcileBy": "running the Reconciler" + "replicasBy": "checking the Deployment landed on the decision, not on a ratio of it" + "statusBy": "checking the status reports the same recommendation" +"expected": + "replicas": 4 + "desired": 4 + "decisionStatus": "True" diff --git a/internal/controller/autoscaling/testdata/controller/deferral-status/should-clear-a-deferral-on-a-no-op-once-no-guard-matches.yaml b/internal/controller/autoscaling/testdata/controller/deferral-status/should-clear-a-deferral-on-a-no-op-once-no-guard-matches.yaml new file mode 100644 index 0000000..2a3182d --- /dev/null +++ b/internal/controller/autoscaling/testdata/controller/deferral-status/should-clear-a-deferral-on-a-no-op-once-no-guard-matches.yaml @@ -0,0 +1,25 @@ +# ceil(1 * 0.5/0.5) = 1, which is already spec, and the rollout is done, so no guard matches. +"name": "should-clear-a-deferral-on-a-no-op-once-no-guard-matches" +"description": "should clear a deferral on a no-op once no guard matches" +"inputs": + "spec": + "deferTimeoutSeconds": 60 + "rollout": true + "steps": + - "at": "0s" + - "by": "finishing the rollout at the replica count the metric already wants" + "at": "10s" + "metric": "0.5" + "observeSpec": 1 +"expected": + "steps": + - "deferredSince": "0s" + - "replicas": 1 + "deferredSince": null + "applied": 1 + "forcedWrite": false + "condition": + "status": "False" + "reason": "NotDeferred" + "messageContains": + - "already at the step target" diff --git a/internal/controller/autoscaling/testdata/controller/deferral-status/should-keep-deferredsince-across-no-op-syncs-while-a-rollout-guard-still-matches.yaml b/internal/controller/autoscaling/testdata/controller/deferral-status/should-keep-deferredsince-across-no-op-syncs-while-a-rollout-guard-still-matches.yaml new file mode 100644 index 0000000..b23401e --- /dev/null +++ b/internal/controller/autoscaling/testdata/controller/deferral-status/should-keep-deferredsince-across-no-op-syncs-while-a-rollout-guard-still-matches.yaml @@ -0,0 +1,64 @@ +"name": "should-keep-deferredsince-across-no-op-syncs-while-a-rollout-guard-still-matches" +"description": "should keep deferredSince across no-op syncs while a rollout guard still matches" +"inputs": + "spec": + "deferTimeoutSeconds": 60 + "rollout": true + "steps": + - "by": "not starting a deferral on a no-op, even though the rollout guard matches" + "at": "0s" + "metric": "0.5" + - "by": "stamping deferredSince on the first sync that actually wants to write" + "at": "10s" + "metric": "0.85" + - "by": "alternating a no-op with a held scale-up; the no-ops must not move the clock" + "at": "20s" + "metric": "0.5" + - "at": "30s" + "metric": "0.85" + - "at": "40s" + "metric": "0.5" + - "by": "forcing the held scale-up once the original deadline passes" + "at": "70s" + "metric": "0.85" +"expected": + "steps": + - "deferredSince": null + "replicas": 1 + "condition": + "status": "False" + "reason": "NotDeferred" + - "deferredSince": "10s" + "replicas": 1 + "condition": + "status": "True" + "reason": "RolloutInProgress" + - "replicas": 1 + "deferredSince": "10s" + "condition": + "status": "True" + "reason": "RolloutInProgress" + "messageContains": &id001 + - "nothing to write this sync" + - "guard is still active" + - "deferral clock keeps running" + - "2026-03-01T12:00:10Z" + - "Deployment rollout in progress" + - "replicas": 1 + "deferredSince": "10s" + "condition": + "status": "True" + "reason": "RolloutInProgress" + - "replicas": 1 + "deferredSince": "10s" + "condition": + "status": "True" + "reason": "RolloutInProgress" + "messageContains": *id001 + - "replicas": 2 + "deferredSince": "10s" + "condition": + "status": "False" + "reason": "DeferralTimedOut" + "messageContains": + - "scaling 1 -> 2 anyway" diff --git a/internal/controller/autoscaling/testdata/controller/deferral-status/should-keep-one-deferral-when-the-guard-reason-changes.yaml b/internal/controller/autoscaling/testdata/controller/deferral-status/should-keep-one-deferral-when-the-guard-reason-changes.yaml new file mode 100644 index 0000000..4fce1e1 --- /dev/null +++ b/internal/controller/autoscaling/testdata/controller/deferral-status/should-keep-one-deferral-when-the-guard-reason-changes.yaml @@ -0,0 +1,40 @@ +"name": "should-keep-one-deferral-when-the-guard-reason-changes" +"description": "should keep one deferral when the guard reason changes" +"inputs": + "spec": + "deferTimeoutSeconds": 60 + "metric": "0" + "deploy": + "replicas": 5 + "updated": 5 + "ready": 3 + "available": 3 + "spec": 5 + "rollout": true + "steps": + - "at": "0s" + - "by": "finishing the rollout while the fleet is still short of ready" + "at": "10s" + "deploy": + "replicas": 5 + "updated": 5 + "ready": 3 + "available": 3 + - "by": "not emitting again while the same guard still defers" + "at": "10s" +"expected": + "steps": + - "deferredSince": "0s" + "condition": + "reason": "RolloutInProgress" + - "replicas": 5 + "deferredSince": "0s" + "condition": + "reason": "ScaleDownSettling" + "events": + - "contains": + - "Deployment rollout in progress" + - "contains": + - "previous scale-down has not settled" + - "deferredSince": "0s" + "noMoreEvents": true diff --git a/internal/controller/autoscaling/testdata/controller/deferral-timeout/should-force-a-deferral-that-already-exceeded-the-timeout-on-a-fresh-reconciler.yaml b/internal/controller/autoscaling/testdata/controller/deferral-timeout/should-force-a-deferral-that-already-exceeded-the-timeout-on-a-fresh-reconciler.yaml new file mode 100644 index 0000000..273eea2 --- /dev/null +++ b/internal/controller/autoscaling/testdata/controller/deferral-timeout/should-force-a-deferral-that-already-exceeded-the-timeout-on-a-fresh-reconciler.yaml @@ -0,0 +1,23 @@ +# deferredSince lives in status, so a fresh reconciler still enforces the deadline. +"name": "should-force-a-deferral-that-already-exceeded-the-timeout-on-a-fresh-reconciler" +"description": "should force a deferral that already exceeded the timeout on a fresh reconciler" +"inputs": + "spec": + "deferTimeoutSeconds": 60 + "rollout": true + "seedDeferredSince": "-2h" + "expectNilRecommendations": true + "steps": + - "at": "0s" +"expected": + "steps": + - "replicas": 2 + "deferredSince": "seed" + "applied": 2 + "condition": + "status": "False" + "reason": "DeferralTimedOut" + "events": + - "prefix": "Warning DeferralTimedOut" + "contains": + - "1 -> 2" diff --git a/internal/controller/autoscaling/testdata/controller/deferral-timeout/should-force-a-scale-down-of-an-unready-fleet-once-the-deferral-times-out.yaml b/internal/controller/autoscaling/testdata/controller/deferral-timeout/should-force-a-scale-down-of-an-unready-fleet-once-the-deferral-times-out.yaml new file mode 100644 index 0000000..d108f09 --- /dev/null +++ b/internal/controller/autoscaling/testdata/controller/deferral-timeout/should-force-a-scale-down-of-an-unready-fleet-once-the-deferral-times-out.yaml @@ -0,0 +1,52 @@ +# readyReplicas 0 is omitted from status and reads as missing. available is 0 too, because availableReplicas cannot exceed readyReplicas. +"name": "should-force-a-scale-down-of-an-unready-fleet-once-the-deferral-times-out" +"description": "should force a scale-down of an unready fleet once the deferral times out" +"inputs": + "spec": + "deferTimeoutSeconds": 60 + "maxStepReplicas": 2 + "prelude": "capping the step and parking 5 replicas with none ready" + "deploy": + "replicas": 5 + "updated": 5 + "ready": 0 + "available": 0 + "spec": 5 + "metric": "0" + "steps": + - "at": "0s" + - "by": "keeping the same deferral until the timeout" + "at": "59s" + - "by": "forcing the capped step, not the uncapped recommendation" + "at": "60s" +"expected": + "steps": + - "replicas": 5 + "deferredSince": "0s" + "desired": 1 + "current": 0 + "applied": 5 + "condition": + "status": "True" + "reason": "ScaleDownSettling" + "messageContains": + - "holding 5 replicas (wanted 3)" + - "will apply after 60s" + - "replicas": 5 + "deferredSince": "0s" + - "replicas": 3 + "deferredSince": "0s" + "desired": 1 + "applied": 3 + "current": 0 + "condition": + "status": "False" + "reason": "DeferralTimedOut" + "messageContains": + - "scaling 5 -> 3 anyway" + "events": + - "prefix": "Normal ScalingDeferred" + - "prefix": "Warning DeferralTimedOut" + "contains": + - "previous scale-down has not settled" + - "5 -> 3" diff --git a/internal/controller/autoscaling/testdata/controller/deferral-timeout/should-force-a-scale-up-once-a-stuck-rollout-exceeds-the-deferral-timeout.yaml b/internal/controller/autoscaling/testdata/controller/deferral-timeout/should-force-a-scale-up-once-a-stuck-rollout-exceeds-the-deferral-timeout.yaml new file mode 100644 index 0000000..a97d880 --- /dev/null +++ b/internal/controller/autoscaling/testdata/controller/deferral-timeout/should-force-a-scale-up-once-a-stuck-rollout-exceeds-the-deferral-timeout.yaml @@ -0,0 +1,55 @@ +"name": "should-force-a-scale-up-once-a-stuck-rollout-exceeds-the-deferral-timeout" +"description": "should force a scale-up once a stuck rollout exceeds the deferral timeout" +"inputs": + "spec": + "deferTimeoutSeconds": 60 + "rollout": true + "steps": + - "by": "deferring the first sync and stamping deferredSince" + "at": "0s" + - "by": "reconciling again before the timeout, without restamping or rewriting" + "at": "59s" + - "by": "advancing past the timeout and forcing the held scale-up" + "at": "60s" +"expected": + "steps": + - "requeueAfter": "15s" + "replicas": 1 + "deferredSince": "0s" + "desired": 2 + "applied": 1 + "condition": + "status": "True" + "reason": "RolloutInProgress" + "messageContains": + - "holding 1 replicas (wanted 2)" + - "Deployment rollout in progress" + - "2026-03-01T12:00:00Z" + - "will apply after 60s" + "observedGenerationCurrent": true + "events": + - "prefix": "Normal ScalingDeferred" + "contains": + - "holding 1 replicas (wanted 2)" + - "requeueAfter": "1s" + "replicas": 1 + "deferredSince": "0s" + "applied": 1 + "noMoreEvents": true + - "replicas": 2 + "deferredSince": "0s" + "desired": 2 + "applied": 2 + "requeueAfter": "15s" + "condition": + "status": "False" + "reason": "DeferralTimedOut" + "messageContains": + - "scaling 1 -> 2 anyway" + "observedGenerationCurrent": true + "events": + - "prefix": "Warning DeferralTimedOut" + "contains": + - "Deployment rollout in progress" + - "1 -> 2" + "noMoreEvents": true diff --git a/internal/controller/autoscaling/testdata/controller/deferral-timeout/should-never-force-a-deferral-when-defertimeoutseconds-is-0.yaml b/internal/controller/autoscaling/testdata/controller/deferral-timeout/should-never-force-a-deferral-when-defertimeoutseconds-is-0.yaml new file mode 100644 index 0000000..6cfde4a --- /dev/null +++ b/internal/controller/autoscaling/testdata/controller/deferral-timeout/should-never-force-a-deferral-when-defertimeoutseconds-is-0.yaml @@ -0,0 +1,22 @@ +"name": "should-never-force-a-deferral-when-defertimeoutseconds-is-0" +"description": "should never force a deferral when deferTimeoutSeconds is 0" +"inputs": + "spec": + "deferTimeoutSeconds": 0 + "rollout": true + "seedDeferredSince": "-24h" + "steps": + - "at": "0s" +"expected": + "steps": + - "requeueAfter": "15s" + "replicas": 1 + "deferredSince": "seed" + "condition": + "status": "True" + "reason": "RolloutInProgress" + "messageContains": + - "never forced (deferTimeoutSeconds is 0)" + "events": + - "prefix": "Normal ScalingDeferred" + "noMoreEvents": true diff --git a/internal/controller/autoscaling/testdata/controller/forced-write-pacing/should-clear-deferredsince-and-the-forced-write-pace-when-no-guard-matches.yaml b/internal/controller/autoscaling/testdata/controller/forced-write-pacing/should-clear-deferredsince-and-the-forced-write-pace-when-no-guard-matches.yaml new file mode 100644 index 0000000..e69bfe5 --- /dev/null +++ b/internal/controller/autoscaling/testdata/controller/forced-write-pacing/should-clear-deferredsince-and-the-forced-write-pace-when-no-guard-matches.yaml @@ -0,0 +1,33 @@ +"name": "should-clear-deferredsince-and-the-forced-write-pace-when-no-guard-matches" +"description": "should clear deferredSince and the forced-write pace when no guard matches" +"inputs": + "spec": + "deferTimeoutSeconds": 60 + "maxStepReplicas": 2 + "metric": "0" + "deploy": + "replicas": 5 + "updated": 5 + "ready": 0 + "available": 0 + "spec": 5 + "steps": + - "at": "0s" + - "at": "60s" + - "by": "letting the fleet become ready at the replica count just written" + "at": "60s" + "observeSpec": 3 +"expected": + "steps": + - {} + - "replicas": 3 + "forcedWrite": true + "deferredSince": "0s" + - "replicas": 1 + "deferredSince": null + "forcedWrite": false + "condition": + "status": "False" + "reason": "NotDeferred" + "messageContains": + - "scaled 3 -> 1" diff --git a/internal/controller/autoscaling/testdata/controller/forced-write-pacing/should-force-the-next-stuck-scale-down-step-after-one-sync-period.yaml b/internal/controller/autoscaling/testdata/controller/forced-write-pacing/should-force-the-next-stuck-scale-down-step-after-one-sync-period.yaml new file mode 100644 index 0000000..5a52f47 --- /dev/null +++ b/internal/controller/autoscaling/testdata/controller/forced-write-pacing/should-force-the-next-stuck-scale-down-step-after-one-sync-period.yaml @@ -0,0 +1,41 @@ +"name": "should-force-the-next-stuck-scale-down-step-after-one-sync-period" +"description": "should force the next stuck scale-down step after one sync period" +"inputs": + "spec": + "deferTimeoutSeconds": 60 + "syncPeriodSeconds": 15 + "maxStepReplicas": 2 + "prelude": "capping the step and parking 5 replicas with none ready" + "deploy": + "replicas": 5 + "updated": 5 + "ready": 0 + "available": 0 + "spec": 5 + "metric": "0" + "steps": + - "at": "0s" + - "by": "forcing the first step once the timeout elapses" + "at": "60s" + - "by": "keeping the fleet unready and taking the next step one sync period later, without another timeout" + "at": "75s" + "observeSpec": 0 +"expected": + "steps": + - "replicas": 5 + "deferredSince": "0s" + "condition": + "reason": "ScaleDownSettling" + - "replicas": 3 + "deferredSince": "0s" + "forcedWrite": true + "forcedWriteAt": "60s" + - "replicas": 1 + "deferredSince": "0s" + "applied": 1 + "condition": + "status": "False" + "reason": "DeferralTimedOut" + "messageContains": + - "scaling 3 -> 1 anyway" + - "previous scale-down has not settled" diff --git a/internal/controller/autoscaling/testdata/controller/forced-write-pacing/should-not-take-a-second-forced-step-until-a-sync-period-has-elapsed.yaml b/internal/controller/autoscaling/testdata/controller/forced-write-pacing/should-not-take-a-second-forced-step-until-a-sync-period-has-elapsed.yaml new file mode 100644 index 0000000..6e348a5 --- /dev/null +++ b/internal/controller/autoscaling/testdata/controller/forced-write-pacing/should-not-take-a-second-forced-step-until-a-sync-period-has-elapsed.yaml @@ -0,0 +1,47 @@ +"name": "should-not-take-a-second-forced-step-until-a-sync-period-has-elapsed" +"description": "should not take a second forced step until a sync period has elapsed" +"inputs": + "spec": + "deferTimeoutSeconds": 60 + "syncPeriodSeconds": 15 + "maxStepReplicas": 2 + "metric": "0" + "deploy": + "replicas": 5 + "updated": 5 + "ready": 0 + "available": 0 + "spec": 5 + "steps": + - "by": "forcing one scale-down step of an unready fleet" + "at": "0s" + "snapshotStatus": true + - "at": "60s" + - "by": "putting the status back to its pre-write state, as if that update had conflicted" + "restoreStatus": true + "skipReconcile": true + - "by": "reconciling again with the clock unchanged, which must not take the next step" + "at": "60s" + - "by": "still holding one second short of the sync period" + "at": "74s" + - "by": "forcing the next step once the sync period has elapsed" + "at": "75s" +"expected": + "steps": + - {} + - "replicas": 3 + - {} + - "replicas": 3 + "requeueAfter": "15s" + "deferredSince": "0s" + "applied": 3 + "condition": + "status": "True" + "messageContains": + - "forced write paced to one per sync period" + - "replicas": 3 + "requeueAfter": "1s" + "deferredSince": "0s" + - "replicas": 1 + "deferredSince": "0s" + "applied": 1 diff --git a/internal/controller/autoscaling/testdata/controller/metric-scaling/should-keep-the-ready-replica-fallback-for-the-scaling-ratio-only.yaml b/internal/controller/autoscaling/testdata/controller/metric-scaling/should-keep-the-ready-replica-fallback-for-the-scaling-ratio-only.yaml new file mode 100644 index 0000000..c182d5a --- /dev/null +++ b/internal/controller/autoscaling/testdata/controller/metric-scaling/should-keep-the-ready-replica-fallback-for-the-scaling-ratio-only.yaml @@ -0,0 +1,22 @@ +# readyReplicas is 0, so the ratio falls back to spec (3): ceil(3 * 0.85/0.5) = 6, clamped to maxReplicas 5. status.currentReplicas stays 0. +"name": "should-keep-the-ready-replica-fallback-for-the-scaling-ratio-only" +"description": "should keep the ready-replica fallback for the scaling ratio only" +"inputs": + "plainReconciler": true + "steps": + - "deploy": + "replicas": 3 + "updated": 3 + "ready": 0 + "available": 0 + "spec": 3 +"expected": + "steps": + - "replicas": 5 + "current": 0 + "desired": 5 + "applied": 5 + "deferredSince": null + "condition": + "status": "False" + "reason": "NotDeferred" diff --git a/internal/controller/autoscaling/testdata/controller/metric-scaling/should-scale-down-to-minreplicas-when-the-metric-is-zero.yaml b/internal/controller/autoscaling/testdata/controller/metric-scaling/should-scale-down-to-minreplicas-when-the-metric-is-zero.yaml new file mode 100644 index 0000000..2713d3e --- /dev/null +++ b/internal/controller/autoscaling/testdata/controller/metric-scaling/should-scale-down-to-minreplicas-when-the-metric-is-zero.yaml @@ -0,0 +1,23 @@ +# desired = ceil(5 * 0/0.5) = 0, clamped up to minReplicas (1). +"name": "should-scale-down-to-minreplicas-when-the-metric-is-zero" +"description": "should scale down to minReplicas when the metric is zero" +"inputs": + "plainReconciler": true + "steps": + - "by": "setting the Deployment to 5 replicas and the metric to 0 (idle)" + "skipReconcile": true + "metric": "0" + "deploy": + "replicas": 5 + "updated": 5 + "ready": 5 + "available": 5 + "spec": 5 + - "by": "running the Reconciler" + - "by": "checking it scaled down to minReplicas" + "skipReconcile": true +"expected": + "steps": + - {} + - {} + - "replicas": 1 diff --git a/internal/controller/autoscaling/testdata/controller/metric-scaling/should-scale-down-to-zero-when-minreplicas-is-0.yaml b/internal/controller/autoscaling/testdata/controller/metric-scaling/should-scale-down-to-zero-when-minreplicas-is-0.yaml new file mode 100644 index 0000000..a12d78c --- /dev/null +++ b/internal/controller/autoscaling/testdata/controller/metric-scaling/should-scale-down-to-zero-when-minreplicas-is-0.yaml @@ -0,0 +1,26 @@ +# desired = ceil(2 * 0/0.5) = 0, clamped to minReplicas (0). +"name": "should-scale-down-to-zero-when-minreplicas-is-0" +"description": "should scale down to zero when minReplicas is 0" +"inputs": + "plainReconciler": true + "prelude": "lowering the scaler's minReplicas to 0" + "spec": + "minReplicas": 0 + "steps": + - "by": "setting the Deployment to 2 replicas, fully rolled out" + "skipReconcile": true + "metric": "0" + "deploy": + "replicas": 2 + "updated": 2 + "ready": 2 + "available": 2 + "spec": 2 + - "by": "running the Reconciler" + - "by": "checking it scaled down to zero" + "skipReconcile": true +"expected": + "steps": + - {} + - {} + - "replicas": 0 diff --git a/internal/controller/autoscaling/testdata/controller/metric-scaling/should-successfully-scale-the-deployment-based-on-metrics.yaml b/internal/controller/autoscaling/testdata/controller/metric-scaling/should-successfully-scale-the-deployment-based-on-metrics.yaml new file mode 100644 index 0000000..f8e2fc6 --- /dev/null +++ b/internal/controller/autoscaling/testdata/controller/metric-scaling/should-successfully-scale-the-deployment-based-on-metrics.yaml @@ -0,0 +1,13 @@ +# ceil(1 * 0.85/0.5) = ceil(1.7) = 2. The mock sample is 0.85 and the target is 0.5. +"name": "should-successfully-scale-the-deployment-based-on-metrics" +"description": "should successfully scale the deployment based on metrics" +"inputs": + "plainReconciler": true + "steps": + - "by": "running the Reconciler" + - "by": "checking if the Deployment replicas increased" + "skipReconcile": true +"expected": + "steps": + - {} + - "replicas": 2 diff --git a/internal/controller/autoscaling/testdata/controller/metric-scaling/should-wake-the-fleet-from-zero-when-the-metric-turns-positive.yaml b/internal/controller/autoscaling/testdata/controller/metric-scaling/should-wake-the-fleet-from-zero-when-the-metric-turns-positive.yaml new file mode 100644 index 0000000..63bc655 --- /dev/null +++ b/internal/controller/autoscaling/testdata/controller/metric-scaling/should-wake-the-fleet-from-zero-when-the-metric-turns-positive.yaml @@ -0,0 +1,22 @@ +# desired = ceil(1 * 0.85/0.5) = 2 against the single-replica baseline, not ceil(0 * 0.85/0.5) = 0. +"name": "should-wake-the-fleet-from-zero-when-the-metric-turns-positive" +"description": "should wake the fleet from zero when the metric turns positive" +"inputs": + "plainReconciler": true + "prelude": "lowering the scaler's minReplicas to 0 and parking the Deployment at 0" + "spec": + "minReplicas": 0 + "deploy": + "replicas": 0 + "updated": 0 + "ready": 0 + "available": 0 + "spec": 0 + "steps": + - "by": "running the Reconciler" + - "by": "checking the fleet woke up" + "skipReconcile": true +"expected": + "steps": + - {} + - "replicas": 2 diff --git a/internal/controller/autoscaling/testdata/controller/rollout-guard/should-defer-scaling-while-the-deployment-is-rolling-out.yaml b/internal/controller/autoscaling/testdata/controller/rollout-guard/should-defer-scaling-while-the-deployment-is-rolling-out.yaml new file mode 100644 index 0000000..b6e13d2 --- /dev/null +++ b/internal/controller/autoscaling/testdata/controller/rollout-guard/should-defer-scaling-while-the-deployment-is-rolling-out.yaml @@ -0,0 +1,17 @@ +# The metric would compute desired=2, but the rollout guard defers scaling, so replicas stay at 1. +"name": "should-defer-scaling-while-the-deployment-is-rolling-out" +"description": "should defer scaling while the Deployment is rolling out" +"inputs": + "plainReconciler": true + "steps": + - "by": "marking the Deployment as mid-rollout" + "rollout": true + "skipReconcile": true + - "by": "running the Reconciler" + - "by": "checking the Deployment was NOT scaled" + "skipReconcile": true +"expected": + "steps": + - {} + - {} + - "replicas": 1 diff --git a/internal/controller/autoscaling/testdata/controller/settle-guard/should-also-wait-for-the-drain-when-checkterminatingreplicas-is-on.yaml b/internal/controller/autoscaling/testdata/controller/settle-guard/should-also-wait-for-the-drain-when-checkterminatingreplicas-is-on.yaml new file mode 100644 index 0000000..a8d6dc7 --- /dev/null +++ b/internal/controller/autoscaling/testdata/controller/settle-guard/should-also-wait-for-the-drain-when-checkterminatingreplicas-is-on.yaml @@ -0,0 +1,32 @@ +"name": "should-also-wait-for-the-drain-when-checkterminatingreplicas-is-on" +"description": "should also wait for the drain when checkTerminatingReplicas is on" +"inputs": + "plainReconciler": true + "checkTerminating": true + "metric": "0" + "steps": + - "by": "setting 5 ready with 2 still draining" + "deploy": + "replicas": 5 + "updated": 5 + "ready": 5 + "available": 5 + "spec": 5 + "terminating": 2 + - "by": "checking it is deferred; readyReplicas alone would have said settled" + "skipReconcile": true + - "by": "letting the drain finish" + "deploy": + "replicas": 5 + "updated": 5 + "ready": 5 + "available": 5 + "terminating": 0 + - "by": "checking the scale-down now proceeds" + "skipReconcile": true +"expected": + "steps": + - {} + - "replicas": 5 + - {} + - "replicas": 1 diff --git a/internal/controller/autoscaling/testdata/controller/settle-guard/should-defer-a-scale-down-until-readyreplicas-is-back-at-specreplicas.yaml b/internal/controller/autoscaling/testdata/controller/settle-guard/should-defer-a-scale-down-until-readyreplicas-is-back-at-specreplicas.yaml new file mode 100644 index 0000000..c4c06d9 --- /dev/null +++ b/internal/controller/autoscaling/testdata/controller/settle-guard/should-defer-a-scale-down-until-readyreplicas-is-back-at-specreplicas.yaml @@ -0,0 +1,29 @@ +"name": "should-defer-a-scale-down-until-readyreplicas-is-back-at-specreplicas" +"description": "should defer a scale-down until readyReplicas is back at specReplicas" +"inputs": + "plainReconciler": true + "metric": "0" + "steps": + - "by": "setting the Deployment to 5 replicas with only 3 ready" + "deploy": + "replicas": 5 + "updated": 5 + "ready": 3 + "available": 3 + "spec": 5 + - "by": "checking the scale-down was deferred" + "skipReconcile": true + - "by": "letting the fleet converge" + "deploy": + "replicas": 5 + "updated": 5 + "ready": 5 + "available": 5 + - "by": "checking the scale-down now proceeds" + "skipReconcile": true +"expected": + "steps": + - {} + - "replicas": 5 + - {} + - "replicas": 1 diff --git a/internal/controller/autoscaling/testdata/controller/settle-guard/should-ignore-still-draining-pods-while-checkterminatingreplicas-is-off.yaml b/internal/controller/autoscaling/testdata/controller/settle-guard/should-ignore-still-draining-pods-while-checkterminatingreplicas-is-off.yaml new file mode 100644 index 0000000..23c2572 --- /dev/null +++ b/internal/controller/autoscaling/testdata/controller/settle-guard/should-ignore-still-draining-pods-while-checkterminatingreplicas-is-off.yaml @@ -0,0 +1,22 @@ +# readyReplicas excludes terminating pods, so the fleet reads as settled the moment the previous step's pods are marked for deletion. +"name": "should-ignore-still-draining-pods-while-checkterminatingreplicas-is-off" +"description": "should ignore still-draining pods while checkTerminatingReplicas is off" +"inputs": + "plainReconciler": true + "checkTerminating": false + "metric": "0" + "steps": + - "by": "setting 5 ready with 2 still draining" + "deploy": + "replicas": 5 + "updated": 5 + "ready": 5 + "available": 5 + "spec": 5 + "terminating": 2 + - "by": "checking the scale-down was not held back by the drain" + "skipReconcile": true +"expected": + "steps": + - {} + - "replicas": 1 diff --git a/internal/controller/autoscaling/testdata/controller/settle-guard/should-not-gate-scale-up-on-the-previous-scale-down-settling.yaml b/internal/controller/autoscaling/testdata/controller/settle-guard/should-not-gate-scale-up-on-the-previous-scale-down-settling.yaml new file mode 100644 index 0000000..8b68273 --- /dev/null +++ b/internal/controller/autoscaling/testdata/controller/settle-guard/should-not-gate-scale-up-on-the-previous-scale-down-settling.yaml @@ -0,0 +1,21 @@ +# ceil(2 * 0.85/0.5) = 4; the settle guard only gates scale-down. +"name": "should-not-gate-scale-up-on-the-previous-scale-down-settling" +"description": "should not gate scale-up on the previous scale-down settling" +"inputs": + "plainReconciler": true + "steps": + - "by": "setting 3 replicas with 2 ready and 1 still terminating, under load" + "metric": "0.85" + "deploy": + "replicas": 3 + "updated": 3 + "ready": 2 + "available": 2 + "spec": 3 + "terminating": 1 + - "by": "checking capacity was added despite the unsettled fleet" + "skipReconcile": true +"expected": + "steps": + - {} + - "replicas": 4 diff --git a/internal/controller/autoscaling/testdata/controller/stabilization-and-step/should-hold-for-the-stabilization-window-between-scale-down-steps.yaml b/internal/controller/autoscaling/testdata/controller/stabilization-and-step/should-hold-for-the-stabilization-window-between-scale-down-steps.yaml new file mode 100644 index 0000000..0c78180 --- /dev/null +++ b/internal/controller/autoscaling/testdata/controller/stabilization-and-step/should-hold-for-the-stabilization-window-between-scale-down-steps.yaml @@ -0,0 +1,29 @@ +"name": "should-hold-for-the-stabilization-window-between-scale-down-steps" +"description": "should hold for the stabilization window between scale-down steps" +"inputs": + "plainReconciler": true + "prelude": "enabling a 300s window and a 2-replica step cap" + "spec": + "stabilizationWindowSeconds": 300 + "maxStepReplicas": 2 + "metric": "0" + "steps": + - "by": "starting settled at 5 with an idle metric" + "deploy": + "replicas": 5 + "updated": 5 + "ready": 5 + "available": 5 + "spec": 5 + - "by": "checking the first step landed" + "skipReconcile": true + - "by": "settling the fleet at 3 so neither the rollout nor the settle guard fires" + "observeSpec": 3 + - "by": "checking the next step is held; without the pin it would drop to 1" + "skipReconcile": true +"expected": + "steps": + - {} + - "replicas": 3 + - {} + - "replicas": 3 diff --git a/internal/controller/autoscaling/testdata/controller/stabilization-and-step/should-walk-down-one-step-at-a-time-when-maxstepreplicas-is-set.yaml b/internal/controller/autoscaling/testdata/controller/stabilization-and-step/should-walk-down-one-step-at-a-time-when-maxstepreplicas-is-set.yaml new file mode 100644 index 0000000..3bfcab2 --- /dev/null +++ b/internal/controller/autoscaling/testdata/controller/stabilization-and-step/should-walk-down-one-step-at-a-time-when-maxstepreplicas-is-set.yaml @@ -0,0 +1,33 @@ +"name": "should-walk-down-one-step-at-a-time-when-maxstepreplicas-is-set" +"description": "should walk down one step at a time when maxStepReplicas is set" +"inputs": + "plainReconciler": true + "prelude": "capping the scale-down step at 2 replicas" + "spec": + "maxStepReplicas": 2 + "steps": + - "by": "setting the Deployment to 5 replicas and the metric to 0 (idle)" + "skipReconcile": true + "metric": "0" + "deploy": + "replicas": 5 + "updated": 5 + "ready": 5 + "available": 5 + "spec": 5 + - "by": "running the Reconciler; raw recommendation is minReplicas but the step is capped" + - "by": "checking status reports the uncapped recommendation, not the step" + "skipReconcile": true + - "by": "running the Reconciler again; it takes the next step down" + "deploy": + "replicas": 3 + "updated": 3 + "ready": 3 + "available": 3 + "spec": 3 +"expected": + "steps": + - {} + - "replicas": 3 + - "desired": 1 + - "replicas": 1 diff --git a/internal/controller/autoscaling/testdata/guards/hold/rollout-fully-rolled-out-does-not-hold.yaml b/internal/controller/autoscaling/testdata/guards/hold/rollout-fully-rolled-out-does-not-hold.yaml new file mode 100644 index 0000000..e45635c --- /dev/null +++ b/internal/controller/autoscaling/testdata/guards/hold/rollout-fully-rolled-out-does-not-hold.yaml @@ -0,0 +1,18 @@ +"name": "rollout-fully-rolled-out-does-not-hold" +"description": "rollout: fully rolled out does not hold" +"inputs": + "guard": "rollout" + "state": + "spec": 5 + "total": 5 + "updated": 5 + "ready": 5 + "generation": 3 + "observedGeneration": 3 + "tracksRollout": true + "intent": + "from": 5 + "to": 4 +"expected": + "held": false + "reason": "RolloutInProgress" diff --git a/internal/controller/autoscaling/testdata/guards/hold/rollout-generation-not-observed-holds-a-scale-up.yaml b/internal/controller/autoscaling/testdata/guards/hold/rollout-generation-not-observed-holds-a-scale-up.yaml new file mode 100644 index 0000000..a6ef99e --- /dev/null +++ b/internal/controller/autoscaling/testdata/guards/hold/rollout-generation-not-observed-holds-a-scale-up.yaml @@ -0,0 +1,19 @@ +"name": "rollout-generation-not-observed-holds-a-scale-up" +"description": "rollout: generation not observed holds a scale-up" +"inputs": + "guard": "rollout" + "state": + "spec": 5 + "total": 5 + "updated": 5 + "ready": 5 + "generation": 4 + "observedGeneration": 3 + "tracksRollout": true + "intent": + "from": 5 + "to": 7 +"expected": + "held": true + "reason": "RolloutInProgress" + "why": "Deployment rollout in progress" diff --git a/internal/controller/autoscaling/testdata/guards/hold/rollout-non-deployment-is-never-held.yaml b/internal/controller/autoscaling/testdata/guards/hold/rollout-non-deployment-is-never-held.yaml new file mode 100644 index 0000000..2e899fc --- /dev/null +++ b/internal/controller/autoscaling/testdata/guards/hold/rollout-non-deployment-is-never-held.yaml @@ -0,0 +1,17 @@ +"name": "rollout-non-deployment-is-never-held" +"description": "rollout: non-Deployment is never held" +"inputs": + "guard": "rollout" + "state": + "spec": 5 + "total": 6 + "updated": 0 + "ready": 5 + "generation": 4 + "observedGeneration": 1 + "intent": + "from": 5 + "to": 7 +"expected": + "held": false + "reason": "RolloutInProgress" diff --git a/internal/controller/autoscaling/testdata/guards/hold/rollout-old-template-pods-hold-a-no-op.yaml b/internal/controller/autoscaling/testdata/guards/hold/rollout-old-template-pods-hold-a-no-op.yaml new file mode 100644 index 0000000..e1517f2 --- /dev/null +++ b/internal/controller/autoscaling/testdata/guards/hold/rollout-old-template-pods-hold-a-no-op.yaml @@ -0,0 +1,19 @@ +"name": "rollout-old-template-pods-hold-a-no-op" +"description": "rollout: old-template pods hold a no-op" +"inputs": + "guard": "rollout" + "state": + "spec": 5 + "total": 6 + "updated": 5 + "ready": 5 + "generation": 3 + "observedGeneration": 3 + "tracksRollout": true + "intent": + "from": 5 + "to": 5 +"expected": + "held": true + "reason": "RolloutInProgress" + "why": "Deployment rollout in progress" diff --git a/internal/controller/autoscaling/testdata/guards/hold/rollout-updated-replicas-behind-spec-holds-a-scale-down.yaml b/internal/controller/autoscaling/testdata/guards/hold/rollout-updated-replicas-behind-spec-holds-a-scale-down.yaml new file mode 100644 index 0000000..21f415c --- /dev/null +++ b/internal/controller/autoscaling/testdata/guards/hold/rollout-updated-replicas-behind-spec-holds-a-scale-down.yaml @@ -0,0 +1,19 @@ +"name": "rollout-updated-replicas-behind-spec-holds-a-scale-down" +"description": "rollout: updated replicas behind spec holds a scale-down" +"inputs": + "guard": "rollout" + "state": + "spec": 5 + "total": 4 + "updated": 4 + "ready": 4 + "generation": 3 + "observedGeneration": 3 + "tracksRollout": true + "intent": + "from": 5 + "to": 3 +"expected": + "held": true + "reason": "RolloutInProgress" + "why": "Deployment rollout in progress" diff --git a/internal/controller/autoscaling/testdata/guards/hold/settle-no-op-is-not-held.yaml b/internal/controller/autoscaling/testdata/guards/hold/settle-no-op-is-not-held.yaml new file mode 100644 index 0000000..17c8008 --- /dev/null +++ b/internal/controller/autoscaling/testdata/guards/hold/settle-no-op-is-not-held.yaml @@ -0,0 +1,13 @@ +"name": "settle-no-op-is-not-held" +"description": "settle: no-op is not held" +"inputs": + "guard": "settle" + "state": + "spec": 5 + "ready": 2 + "intent": + "from": 5 + "to": 5 +"expected": + "held": false + "reason": "ScaleDownSettling" diff --git a/internal/controller/autoscaling/testdata/guards/hold/settle-ready-back-at-spec.yaml b/internal/controller/autoscaling/testdata/guards/hold/settle-ready-back-at-spec.yaml new file mode 100644 index 0000000..b103b60 --- /dev/null +++ b/internal/controller/autoscaling/testdata/guards/hold/settle-ready-back-at-spec.yaml @@ -0,0 +1,13 @@ +"name": "settle-ready-back-at-spec" +"description": "settle: ready back at spec" +"inputs": + "guard": "settle" + "state": + "spec": 5 + "ready": 5 + "intent": + "from": 5 + "to": 4 +"expected": + "held": false + "reason": "ScaleDownSettling" diff --git a/internal/controller/autoscaling/testdata/guards/hold/settle-ready-count-of-0-is-not-settled.yaml b/internal/controller/autoscaling/testdata/guards/hold/settle-ready-count-of-0-is-not-settled.yaml new file mode 100644 index 0000000..3dab245 --- /dev/null +++ b/internal/controller/autoscaling/testdata/guards/hold/settle-ready-count-of-0-is-not-settled.yaml @@ -0,0 +1,15 @@ +# A ready count of 0 is the raw missing-field value. Treating it as spec would skip the guard. +"name": "settle-ready-count-of-0-is-not-settled" +"description": "settle: ready count of 0 is not settled" +"inputs": + "guard": "settle" + "state": + "spec": 5 + "ready": 0 + "intent": + "from": 5 + "to": 4 +"expected": + "held": true + "reason": "ScaleDownSettling" + "why": "previous scale-down has not settled" diff --git a/internal/controller/autoscaling/testdata/guards/hold/settle-ready-differs-from-spec.yaml b/internal/controller/autoscaling/testdata/guards/hold/settle-ready-differs-from-spec.yaml new file mode 100644 index 0000000..4167edc --- /dev/null +++ b/internal/controller/autoscaling/testdata/guards/hold/settle-ready-differs-from-spec.yaml @@ -0,0 +1,14 @@ +"name": "settle-ready-differs-from-spec" +"description": "settle: ready differs from spec" +"inputs": + "guard": "settle" + "state": + "spec": 5 + "ready": 3 + "intent": + "from": 5 + "to": 4 +"expected": + "held": true + "reason": "ScaleDownSettling" + "why": "previous scale-down has not settled" diff --git a/internal/controller/autoscaling/testdata/guards/hold/settle-scale-up-is-not-held.yaml b/internal/controller/autoscaling/testdata/guards/hold/settle-scale-up-is-not-held.yaml new file mode 100644 index 0000000..39d5508 --- /dev/null +++ b/internal/controller/autoscaling/testdata/guards/hold/settle-scale-up-is-not-held.yaml @@ -0,0 +1,13 @@ +"name": "settle-scale-up-is-not-held" +"description": "settle: scale-up is not held" +"inputs": + "guard": "settle" + "state": + "spec": 5 + "ready": 2 + "intent": + "from": 5 + "to": 7 +"expected": + "held": false + "reason": "ScaleDownSettling" diff --git a/internal/controller/autoscaling/testdata/guards/hold/settle-terminating-count-of-0-with-the-gate-on.yaml b/internal/controller/autoscaling/testdata/guards/hold/settle-terminating-count-of-0-with-the-gate-on.yaml new file mode 100644 index 0000000..903eac8 --- /dev/null +++ b/internal/controller/autoscaling/testdata/guards/hold/settle-terminating-count-of-0-with-the-gate-on.yaml @@ -0,0 +1,16 @@ +"name": "settle-terminating-count-of-0-with-the-gate-on" +"description": "settle: terminating count of 0 with the gate on" +"inputs": + "guard": "settle" + "state": + "spec": 5 + "ready": 5 + "terminating": 0 + "terminatingReported": true + "intent": + "from": 5 + "to": 4 + "checkTerminating": true +"expected": + "held": false + "reason": "ScaleDownSettling" diff --git a/internal/controller/autoscaling/testdata/guards/hold/settle-terminating-not-reported-with-the-gate-on.yaml b/internal/controller/autoscaling/testdata/guards/hold/settle-terminating-not-reported-with-the-gate-on.yaml new file mode 100644 index 0000000..ac3ef3f --- /dev/null +++ b/internal/controller/autoscaling/testdata/guards/hold/settle-terminating-not-reported-with-the-gate-on.yaml @@ -0,0 +1,17 @@ +# The count is ignored when the field was not reported. A non-zero Terminating here would still be settled; the parser stores 0 in that case, so this row is what makes the flag observable. +"name": "settle-terminating-not-reported-with-the-gate-on" +"description": "settle: terminating not reported with the gate on" +"inputs": + "guard": "settle" + "state": + "spec": 5 + "ready": 5 + "terminating": 4 + "terminatingReported": false + "intent": + "from": 5 + "to": 4 + "checkTerminating": true +"expected": + "held": false + "reason": "ScaleDownSettling" diff --git a/internal/controller/autoscaling/testdata/guards/hold/settle-terminating-pods-with-the-gate-off.yaml b/internal/controller/autoscaling/testdata/guards/hold/settle-terminating-pods-with-the-gate-off.yaml new file mode 100644 index 0000000..3238c7e --- /dev/null +++ b/internal/controller/autoscaling/testdata/guards/hold/settle-terminating-pods-with-the-gate-off.yaml @@ -0,0 +1,15 @@ +"name": "settle-terminating-pods-with-the-gate-off" +"description": "settle: terminating pods with the gate off" +"inputs": + "guard": "settle" + "state": + "spec": 5 + "ready": 5 + "terminating": 2 + "terminatingReported": true + "intent": + "from": 5 + "to": 4 +"expected": + "held": false + "reason": "ScaleDownSettling" diff --git a/internal/controller/autoscaling/testdata/guards/hold/settle-terminating-pods-with-the-gate-on.yaml b/internal/controller/autoscaling/testdata/guards/hold/settle-terminating-pods-with-the-gate-on.yaml new file mode 100644 index 0000000..49b1bb8 --- /dev/null +++ b/internal/controller/autoscaling/testdata/guards/hold/settle-terminating-pods-with-the-gate-on.yaml @@ -0,0 +1,17 @@ +"name": "settle-terminating-pods-with-the-gate-on" +"description": "settle: terminating pods with the gate on" +"inputs": + "guard": "settle" + "state": + "spec": 5 + "ready": 5 + "terminating": 2 + "terminatingReported": true + "intent": + "from": 5 + "to": 4 + "checkTerminating": true +"expected": + "held": true + "reason": "ScaleDownSettling" + "why": "previous scale-down has not settled" diff --git a/internal/controller/autoscaling/testdata/guards/order/no-guard-holds-a-settled-scale-down.yaml b/internal/controller/autoscaling/testdata/guards/order/no-guard-holds-a-settled-scale-down.yaml new file mode 100644 index 0000000..1410df5 --- /dev/null +++ b/internal/controller/autoscaling/testdata/guards/order/no-guard-holds-a-settled-scale-down.yaml @@ -0,0 +1,15 @@ +"name": "no-guard-holds-a-settled-scale-down" +"description": "no guard holds a settled scale-down" +"inputs": + "state": + "spec": 5 + "total": 5 + "updated": 5 + "ready": 5 + "generation": 3 + "observedGeneration": 3 + "tracksRollout": true + "intent": + "from": 5 + "to": 4 +"expected": {} diff --git a/internal/controller/autoscaling/testdata/guards/order/rollout-wins-over-an-unsettled-scale-down.yaml b/internal/controller/autoscaling/testdata/guards/order/rollout-wins-over-an-unsettled-scale-down.yaml new file mode 100644 index 0000000..17c79d9 --- /dev/null +++ b/internal/controller/autoscaling/testdata/guards/order/rollout-wins-over-an-unsettled-scale-down.yaml @@ -0,0 +1,17 @@ +"name": "rollout-wins-over-an-unsettled-scale-down" +"description": "rollout wins over an unsettled scale-down" +"inputs": + "state": + "spec": 5 + "total": 5 + "updated": 5 + "ready": 2 + "generation": 4 + "observedGeneration": 3 + "tracksRollout": true + "intent": + "from": 5 + "to": 3 +"expected": + "reason": "RolloutInProgress" + "message": "Deployment rollout in progress" diff --git a/internal/controller/autoscaling/testdata/guards/order/settle-holds-once-the-rollout-is-done.yaml b/internal/controller/autoscaling/testdata/guards/order/settle-holds-once-the-rollout-is-done.yaml new file mode 100644 index 0000000..2f84b7e --- /dev/null +++ b/internal/controller/autoscaling/testdata/guards/order/settle-holds-once-the-rollout-is-done.yaml @@ -0,0 +1,17 @@ +"name": "settle-holds-once-the-rollout-is-done" +"description": "settle holds once the rollout is done" +"inputs": + "state": + "spec": 5 + "total": 5 + "updated": 5 + "ready": 3 + "generation": 3 + "observedGeneration": 3 + "tracksRollout": true + "intent": + "from": 5 + "to": 4 +"expected": + "reason": "ScaleDownSettling" + "message": "previous scale-down has not settled" diff --git a/internal/controller/autoscaling/testdata/guards/target/non-deployment-ignores-the-rollout-guard.yaml b/internal/controller/autoscaling/testdata/guards/target/non-deployment-ignores-the-rollout-guard.yaml new file mode 100644 index 0000000..5d8db59 --- /dev/null +++ b/internal/controller/autoscaling/testdata/guards/target/non-deployment-ignores-the-rollout-guard.yaml @@ -0,0 +1,24 @@ +"name": "non-deployment-ignores-the-rollout-guard" +"description": "non-Deployment ignores the rollout guard" +"inputs": + "kind": "StatefulSet" + "object": + "apiVersion": "apps/v1" + "kind": "StatefulSet" + "metadata": + "generation": 4 + "spec": + "replicas": 5 + "status": + "observedGeneration": 1 + "updatedReplicas": 0 + "replicas": 5 + "readyReplicas": 5 + "target": 7 +"expected": + "spec": 5 + "ready": 5 + "total": 5 + "updated": 0 + "generation": 4 + "observedGeneration": 1 diff --git a/internal/controller/autoscaling/testdata/guards/target/ready-count-of-0-is-not-settled.yaml b/internal/controller/autoscaling/testdata/guards/target/ready-count-of-0-is-not-settled.yaml new file mode 100644 index 0000000..0b397f6 --- /dev/null +++ b/internal/controller/autoscaling/testdata/guards/target/ready-count-of-0-is-not-settled.yaml @@ -0,0 +1,27 @@ +# readyReplicas 0 is also what a missing field reads as. Treating it as spec would skip the guard. +"name": "ready-count-of-0-is-not-settled" +"description": "ready count of 0 is not settled" +"inputs": + "kind": "Deployment" + "object": + "apiVersion": "apps/v1" + "kind": "Deployment" + "metadata": + "generation": 3 + "spec": + "replicas": 5 + "status": + "observedGeneration": 3 + "updatedReplicas": 5 + "replicas": 5 + "readyReplicas": 0 + "target": 4 +"expected": + "spec": 5 + "ready": 0 + "total": 5 + "updated": 5 + "generation": 3 + "observedGeneration": 3 + "reason": "ScaleDownSettling" + "message": "previous scale-down has not settled" diff --git a/internal/controller/autoscaling/testdata/guards/target/rollout-holds-a-scale-up.yaml b/internal/controller/autoscaling/testdata/guards/target/rollout-holds-a-scale-up.yaml new file mode 100644 index 0000000..a90dc43 --- /dev/null +++ b/internal/controller/autoscaling/testdata/guards/target/rollout-holds-a-scale-up.yaml @@ -0,0 +1,26 @@ +"name": "rollout-holds-a-scale-up" +"description": "rollout holds a scale-up" +"inputs": + "kind": "Deployment" + "object": + "apiVersion": "apps/v1" + "kind": "Deployment" + "metadata": + "generation": 4 + "spec": + "replicas": 5 + "status": + "observedGeneration": 3 + "updatedReplicas": 5 + "replicas": 5 + "readyReplicas": 5 + "target": 7 +"expected": + "spec": 5 + "ready": 5 + "total": 5 + "updated": 5 + "generation": 4 + "observedGeneration": 3 + "reason": "RolloutInProgress" + "message": "Deployment rollout in progress" diff --git a/internal/controller/autoscaling/testdata/guards/target/rollout-wins-over-an-unsettled-scale-down.yaml b/internal/controller/autoscaling/testdata/guards/target/rollout-wins-over-an-unsettled-scale-down.yaml new file mode 100644 index 0000000..d5b13b5 --- /dev/null +++ b/internal/controller/autoscaling/testdata/guards/target/rollout-wins-over-an-unsettled-scale-down.yaml @@ -0,0 +1,26 @@ +"name": "rollout-wins-over-an-unsettled-scale-down" +"description": "rollout wins over an unsettled scale-down" +"inputs": + "kind": "Deployment" + "object": + "apiVersion": "apps/v1" + "kind": "Deployment" + "metadata": + "generation": 4 + "spec": + "replicas": 5 + "status": + "observedGeneration": 3 + "updatedReplicas": 5 + "replicas": 5 + "readyReplicas": 2 + "target": 3 +"expected": + "spec": 5 + "ready": 2 + "total": 5 + "updated": 5 + "generation": 4 + "observedGeneration": 3 + "reason": "RolloutInProgress" + "message": "Deployment rollout in progress" diff --git a/internal/controller/autoscaling/testdata/guards/target/scale-up-is-not-gated-on-settle.yaml b/internal/controller/autoscaling/testdata/guards/target/scale-up-is-not-gated-on-settle.yaml new file mode 100644 index 0000000..2e8152e --- /dev/null +++ b/internal/controller/autoscaling/testdata/guards/target/scale-up-is-not-gated-on-settle.yaml @@ -0,0 +1,24 @@ +"name": "scale-up-is-not-gated-on-settle" +"description": "scale-up is not gated on settle" +"inputs": + "kind": "Deployment" + "object": + "apiVersion": "apps/v1" + "kind": "Deployment" + "metadata": + "generation": 3 + "spec": + "replicas": 5 + "status": + "observedGeneration": 3 + "updatedReplicas": 5 + "replicas": 5 + "readyReplicas": 2 + "target": 6 +"expected": + "spec": 5 + "ready": 2 + "total": 5 + "updated": 5 + "generation": 3 + "observedGeneration": 3 diff --git a/internal/controller/autoscaling/testdata/guards/target/settled-scale-down-may-proceed.yaml b/internal/controller/autoscaling/testdata/guards/target/settled-scale-down-may-proceed.yaml new file mode 100644 index 0000000..1ecb526 --- /dev/null +++ b/internal/controller/autoscaling/testdata/guards/target/settled-scale-down-may-proceed.yaml @@ -0,0 +1,24 @@ +"name": "settled-scale-down-may-proceed" +"description": "settled scale-down may proceed" +"inputs": + "kind": "Deployment" + "object": + "apiVersion": "apps/v1" + "kind": "Deployment" + "metadata": + "generation": 3 + "spec": + "replicas": 5 + "status": + "observedGeneration": 3 + "updatedReplicas": 5 + "replicas": 5 + "readyReplicas": 5 + "target": 4 +"expected": + "spec": 5 + "ready": 5 + "total": 5 + "updated": 5 + "generation": 3 + "observedGeneration": 3 diff --git a/internal/controller/autoscaling/testdata/guards/target/unsettled-scale-down.yaml b/internal/controller/autoscaling/testdata/guards/target/unsettled-scale-down.yaml new file mode 100644 index 0000000..e4b4f25 --- /dev/null +++ b/internal/controller/autoscaling/testdata/guards/target/unsettled-scale-down.yaml @@ -0,0 +1,27 @@ +# Fully rolled out. readyReplicas is the raw count. +"name": "unsettled-scale-down" +"description": "unsettled scale-down" +"inputs": + "kind": "Deployment" + "object": + "apiVersion": "apps/v1" + "kind": "Deployment" + "metadata": + "generation": 3 + "spec": + "replicas": 5 + "status": + "observedGeneration": 3 + "updatedReplicas": 5 + "replicas": 5 + "readyReplicas": 3 + "target": 4 +"expected": + "spec": 5 + "ready": 3 + "total": 5 + "updated": 5 + "generation": 3 + "observedGeneration": 3 + "reason": "ScaleDownSettling" + "message": "previous scale-down has not settled" diff --git a/internal/controller/autoscaling/testdata/guards/workload-state/deployment-with-full-status.yaml b/internal/controller/autoscaling/testdata/guards/workload-state/deployment-with-full-status.yaml new file mode 100644 index 0000000..3d5cc10 --- /dev/null +++ b/internal/controller/autoscaling/testdata/guards/workload-state/deployment-with-full-status.yaml @@ -0,0 +1,27 @@ +"name": "deployment-with-full-status" +"description": "Deployment with full status" +"inputs": + "kind": "Deployment" + "object": + "apiVersion": "apps/v1" + "kind": "Deployment" + "metadata": + "generation": 9 + "spec": + "replicas": 4 + "status": + "readyReplicas": 3 + "replicas": 6 + "updatedReplicas": 2 + "terminatingReplicas": 1 + "observedGeneration": 8 +"expected": + "spec": 4 + "total": 6 + "updated": 2 + "ready": 3 + "terminating": 1 + "terminatingReported": true + "generation": 9 + "observedGeneration": 8 + "tracksRollout": true diff --git a/internal/controller/autoscaling/testdata/guards/workload-state/generic-kind-with-full-status.yaml b/internal/controller/autoscaling/testdata/guards/workload-state/generic-kind-with-full-status.yaml new file mode 100644 index 0000000..0e7e41a --- /dev/null +++ b/internal/controller/autoscaling/testdata/guards/workload-state/generic-kind-with-full-status.yaml @@ -0,0 +1,26 @@ +"name": "generic-kind-with-full-status" +"description": "generic kind with full status" +"inputs": + "kind": "ReplicaSet" + "object": + "apiVersion": "apps/v1" + "kind": "ReplicaSet" + "metadata": + "generation": 9 + "spec": + "replicas": 4 + "status": + "readyReplicas": 3 + "replicas": 6 + "updatedReplicas": 2 + "terminatingReplicas": 1 + "observedGeneration": 8 +"expected": + "spec": 4 + "total": 6 + "updated": 2 + "ready": 3 + "terminating": 1 + "terminatingReported": true + "generation": 9 + "observedGeneration": 8 diff --git a/internal/controller/autoscaling/testdata/guards/workload-state/invalid-typed-fields-behave-like-missing.yaml b/internal/controller/autoscaling/testdata/guards/workload-state/invalid-typed-fields-behave-like-missing.yaml new file mode 100644 index 0000000..821d145 --- /dev/null +++ b/internal/controller/autoscaling/testdata/guards/workload-state/invalid-typed-fields-behave-like-missing.yaml @@ -0,0 +1,20 @@ +"name": "invalid-typed-fields-behave-like-missing" +"description": "invalid-typed fields behave like missing" +"inputs": + "kind": "Deployment" + "object": + "apiVersion": "apps/v1" + "kind": "Deployment" + "metadata": + "generation": "nope" + "spec": + "replicas": "nope" + "status": + "readyReplicas": "nope" + "replicas": "nope" + "updatedReplicas": "nope" + "terminatingReplicas": "nope" + "observedGeneration": "nope" +"expected": + "spec": 1 + "tracksRollout": true diff --git a/internal/controller/autoscaling/testdata/guards/workload-state/leaderworkerset-with-full-status.yaml b/internal/controller/autoscaling/testdata/guards/workload-state/leaderworkerset-with-full-status.yaml new file mode 100644 index 0000000..b728ca1 --- /dev/null +++ b/internal/controller/autoscaling/testdata/guards/workload-state/leaderworkerset-with-full-status.yaml @@ -0,0 +1,26 @@ +"name": "leaderworkerset-with-full-status" +"description": "LeaderWorkerSet with full status" +"inputs": + "kind": "LeaderWorkerSet" + "object": + "apiVersion": "leaderworkerset.x-k8s.io/v1" + "kind": "LeaderWorkerSet" + "metadata": + "generation": 9 + "spec": + "replicas": 4 + "status": + "readyReplicas": 3 + "replicas": 6 + "updatedReplicas": 2 + "terminatingReplicas": 1 + "observedGeneration": 8 +"expected": + "spec": 4 + "total": 6 + "updated": 2 + "ready": 3 + "terminating": 1 + "terminatingReported": true + "generation": 9 + "observedGeneration": 8 diff --git a/internal/controller/autoscaling/testdata/guards/workload-state/missing-generation-observedgeneration-updated-and-total-read-as-0.yaml b/internal/controller/autoscaling/testdata/guards/workload-state/missing-generation-observedgeneration-updated-and-total-read-as-0.yaml new file mode 100644 index 0000000..cc9c211 --- /dev/null +++ b/internal/controller/autoscaling/testdata/guards/workload-state/missing-generation-observedgeneration-updated-and-total-read-as-0.yaml @@ -0,0 +1,18 @@ +"name": "missing-generation-observedgeneration-updated-and-total-read-as-0" +"description": "missing generation, observedGeneration, updated, and total read as 0" +"inputs": + "kind": "Deployment" + "object": + "apiVersion": "apps/v1" + "kind": "Deployment" + "spec": + "replicas": 4 + "status": + "readyReplicas": 3 + "terminatingReplicas": 1 +"expected": + "spec": 4 + "ready": 3 + "terminating": 1 + "terminatingReported": true + "tracksRollout": true diff --git a/internal/controller/autoscaling/testdata/guards/workload-state/missing-readyreplicas-reads-as-0.yaml b/internal/controller/autoscaling/testdata/guards/workload-state/missing-readyreplicas-reads-as-0.yaml new file mode 100644 index 0000000..c40d6ea --- /dev/null +++ b/internal/controller/autoscaling/testdata/guards/workload-state/missing-readyreplicas-reads-as-0.yaml @@ -0,0 +1,26 @@ +"name": "missing-readyreplicas-reads-as-0" +"description": "missing readyReplicas reads as 0" +"inputs": + "kind": "Deployment" + "object": + "apiVersion": "apps/v1" + "kind": "Deployment" + "metadata": + "generation": 9 + "spec": + "replicas": 4 + "status": + "replicas": 6 + "updatedReplicas": 2 + "terminatingReplicas": 1 + "observedGeneration": 8 +"expected": + "spec": 4 + "total": 6 + "updated": 2 + "ready": 0 + "terminating": 1 + "terminatingReported": true + "generation": 9 + "observedGeneration": 8 + "tracksRollout": true diff --git a/internal/controller/autoscaling/testdata/guards/workload-state/missing-spec-replicas-reads-as-1.yaml b/internal/controller/autoscaling/testdata/guards/workload-state/missing-spec-replicas-reads-as-1.yaml new file mode 100644 index 0000000..338ffca --- /dev/null +++ b/internal/controller/autoscaling/testdata/guards/workload-state/missing-spec-replicas-reads-as-1.yaml @@ -0,0 +1,25 @@ +"name": "missing-spec-replicas-reads-as-1" +"description": "missing spec.replicas reads as 1" +"inputs": + "kind": "Deployment" + "object": + "apiVersion": "apps/v1" + "kind": "Deployment" + "metadata": + "generation": 9 + "status": + "readyReplicas": 3 + "replicas": 6 + "updatedReplicas": 2 + "terminatingReplicas": 1 + "observedGeneration": 8 +"expected": + "spec": 1 + "total": 6 + "updated": 2 + "ready": 3 + "terminating": 1 + "terminatingReported": true + "generation": 9 + "observedGeneration": 8 + "tracksRollout": true diff --git a/internal/controller/autoscaling/testdata/guards/workload-state/missing-terminatingreplicas-is-not-reported.yaml b/internal/controller/autoscaling/testdata/guards/workload-state/missing-terminatingreplicas-is-not-reported.yaml new file mode 100644 index 0000000..4782f37 --- /dev/null +++ b/internal/controller/autoscaling/testdata/guards/workload-state/missing-terminatingreplicas-is-not-reported.yaml @@ -0,0 +1,24 @@ +"name": "missing-terminatingreplicas-is-not-reported" +"description": "missing terminatingReplicas is not reported" +"inputs": + "kind": "Deployment" + "object": + "apiVersion": "apps/v1" + "kind": "Deployment" + "metadata": + "generation": 9 + "spec": + "replicas": 4 + "status": + "readyReplicas": 3 + "replicas": 6 + "updatedReplicas": 2 + "observedGeneration": 8 +"expected": + "spec": 4 + "total": 6 + "updated": 2 + "ready": 3 + "generation": 9 + "observedGeneration": 8 + "tracksRollout": true diff --git a/internal/controller/autoscaling/testdata/guards/workload-state/spec-replicas-of-0-is-kept.yaml b/internal/controller/autoscaling/testdata/guards/workload-state/spec-replicas-of-0-is-kept.yaml new file mode 100644 index 0000000..8477632 --- /dev/null +++ b/internal/controller/autoscaling/testdata/guards/workload-state/spec-replicas-of-0-is-kept.yaml @@ -0,0 +1,21 @@ +"name": "spec-replicas-of-0-is-kept" +"description": "spec.replicas of 0 is kept" +"inputs": + "kind": "Deployment" + "object": + "apiVersion": "apps/v1" + "kind": "Deployment" + "metadata": + "generation": 2 + "spec": + "replicas": 0 + "status": + "readyReplicas": 0 + "replicas": 0 + "updatedReplicas": 0 + "observedGeneration": 2 +"expected": + "spec": 0 + "generation": 2 + "observedGeneration": 2 + "tracksRollout": true diff --git a/internal/controller/autoscaling/testdata/guards/workload-state/statefulset-with-full-status.yaml b/internal/controller/autoscaling/testdata/guards/workload-state/statefulset-with-full-status.yaml new file mode 100644 index 0000000..2ba4e30 --- /dev/null +++ b/internal/controller/autoscaling/testdata/guards/workload-state/statefulset-with-full-status.yaml @@ -0,0 +1,26 @@ +"name": "statefulset-with-full-status" +"description": "StatefulSet with full status" +"inputs": + "kind": "StatefulSet" + "object": + "apiVersion": "apps/v1" + "kind": "StatefulSet" + "metadata": + "generation": 9 + "spec": + "replicas": 4 + "status": + "readyReplicas": 3 + "replicas": 6 + "updatedReplicas": 2 + "terminatingReplicas": 1 + "observedGeneration": 8 +"expected": + "spec": 4 + "total": 6 + "updated": 2 + "ready": 3 + "terminating": 1 + "terminatingReported": true + "generation": 9 + "observedGeneration": 8 diff --git a/internal/controller/autoscaling/testdata/helpers/compute-desired/minus-inf.yaml b/internal/controller/autoscaling/testdata/helpers/compute-desired/minus-inf.yaml new file mode 100644 index 0000000..8423bed --- /dev/null +++ b/internal/controller/autoscaling/testdata/helpers/compute-desired/minus-inf.yaml @@ -0,0 +1,9 @@ +"name": "minus-inf" +"description": "-Inf" +"inputs": + "sample": "-Inf" + "ready": 4 + "metricName": "guardrail" + "query": "histogram_quantile(0.95, x)" + "target": "2" +"expected": {} diff --git a/internal/controller/autoscaling/testdata/helpers/compute-desired/nan.yaml b/internal/controller/autoscaling/testdata/helpers/compute-desired/nan.yaml new file mode 100644 index 0000000..1d50b3f --- /dev/null +++ b/internal/controller/autoscaling/testdata/helpers/compute-desired/nan.yaml @@ -0,0 +1,9 @@ +"name": "nan" +"description": "NaN" +"inputs": + "sample": "NaN" + "ready": 4 + "metricName": "guardrail" + "query": "histogram_quantile(0.95, x)" + "target": "2" +"expected": {} diff --git a/internal/controller/autoscaling/testdata/helpers/compute-desired/plus-inf.yaml b/internal/controller/autoscaling/testdata/helpers/compute-desired/plus-inf.yaml new file mode 100644 index 0000000..5882327 --- /dev/null +++ b/internal/controller/autoscaling/testdata/helpers/compute-desired/plus-inf.yaml @@ -0,0 +1,9 @@ +"name": "plus-inf" +"description": "+Inf" +"inputs": + "sample": "+Inf" + "ready": 4 + "metricName": "guardrail" + "query": "histogram_quantile(0.95, x)" + "target": "2" +"expected": {} diff --git a/internal/controller/autoscaling/testdata/helpers/compute-desired/positive-signal-wakes-the-fleet.yaml b/internal/controller/autoscaling/testdata/helpers/compute-desired/positive-signal-wakes-the-fleet.yaml new file mode 100644 index 0000000..acd1ecf --- /dev/null +++ b/internal/controller/autoscaling/testdata/helpers/compute-desired/positive-signal-wakes-the-fleet.yaml @@ -0,0 +1,14 @@ +# ceil(1 * 0.85/0.5) = 2, not ceil(0 * 0.85/0.5) = 0. +"name": "positive-signal-wakes-the-fleet" +"description": "positive signal wakes the fleet" +"inputs": + "sample": "0.85" + "ready": 0 + "minReplicas": 0 + "maxReplicas": 5 + "metricName": "queue" + "query": "sum(queue_depth)" + "target": "0.5" +"expected": + "have": true + "desired": 2 diff --git a/internal/controller/autoscaling/testdata/helpers/compute-desired/zero-signal-recommends-zero.yaml b/internal/controller/autoscaling/testdata/helpers/compute-desired/zero-signal-recommends-zero.yaml new file mode 100644 index 0000000..d77be3c --- /dev/null +++ b/internal/controller/autoscaling/testdata/helpers/compute-desired/zero-signal-recommends-zero.yaml @@ -0,0 +1,14 @@ +# A zero signal still recommends 0. +"name": "zero-signal-recommends-zero" +"description": "zero signal recommends zero" +"inputs": + "sample": "0" + "ready": 0 + "minReplicas": 0 + "maxReplicas": 5 + "metricName": "queue" + "query": "sum(queue_depth)" + "target": "0.5" +"expected": + "have": true + "desired": 0 diff --git a/internal/controller/autoscaling/testdata/helpers/decision-misconfigured/empty-serveraddress.yaml b/internal/controller/autoscaling/testdata/helpers/decision-misconfigured/empty-serveraddress.yaml new file mode 100644 index 0000000..f700327 --- /dev/null +++ b/internal/controller/autoscaling/testdata/helpers/decision-misconfigured/empty-serveraddress.yaml @@ -0,0 +1,7 @@ +"name": "empty-serveraddress" +"description": "empty serverAddress" +"inputs": + "serverAddress": "" + "serviceId": "svc" +"expected": + "errorContains": "serverAddress" diff --git a/internal/controller/autoscaling/testdata/helpers/decision-misconfigured/empty-serviceid.yaml b/internal/controller/autoscaling/testdata/helpers/decision-misconfigured/empty-serviceid.yaml new file mode 100644 index 0000000..46befec --- /dev/null +++ b/internal/controller/autoscaling/testdata/helpers/decision-misconfigured/empty-serviceid.yaml @@ -0,0 +1,7 @@ +"name": "empty-serviceid" +"description": "empty serviceId" +"inputs": + "serverAddress": "http://example.invalid" + "serviceId": "" +"expected": + "errorContains": "serviceId" diff --git a/internal/controller/autoscaling/testdata/helpers/decision-non-200/a-non-ok-reply-is-an-error-even-when-the-body-is-a-decision.yaml b/internal/controller/autoscaling/testdata/helpers/decision-non-200/a-non-ok-reply-is-an-error-even-when-the-body-is-a-decision.yaml new file mode 100644 index 0000000..9f12d80 --- /dev/null +++ b/internal/controller/autoscaling/testdata/helpers/decision-non-200/a-non-ok-reply-is-an-error-even-when-the-body-is-a-decision.yaml @@ -0,0 +1,8 @@ +"name": "a-non-ok-reply-is-an-error-even-when-the-body-is-a-decision" +"description": "a non-OK reply is an error even when the body is a decision" +"inputs": + "serviceId": "svc" + "status": 503 + "body": "{\"apiVersion\":\"llmscaling.inference.x-k8s.io/v1alpha1\",\"decisions\":[{\"serviceId\":\"svc\",\"replicas\":{\"active\":3}}]}" +"expected": + "errorContains": "status 503" diff --git a/internal/controller/autoscaling/testdata/helpers/decision-replicas/a-body-that-is-not-json-is-an-error.yaml b/internal/controller/autoscaling/testdata/helpers/decision-replicas/a-body-that-is-not-json-is-an-error.yaml new file mode 100644 index 0000000..d5db537 --- /dev/null +++ b/internal/controller/autoscaling/testdata/helpers/decision-replicas/a-body-that-is-not-json-is-an-error.yaml @@ -0,0 +1,8 @@ +"name": "a-body-that-is-not-json-is-an-error" +"description": "a body that is not JSON is an error" +"inputs": + "status": 200 + "serviceId": "svc" + "body": "502 Bad Gateway" +"expected": + "errorContains": "failed to decode" diff --git a/internal/controller/autoscaling/testdata/helpers/decision-replicas/a-count-past-int32-is-refused-rather-than-wrapped.yaml b/internal/controller/autoscaling/testdata/helpers/decision-replicas/a-count-past-int32-is-refused-rather-than-wrapped.yaml new file mode 100644 index 0000000..08bafea --- /dev/null +++ b/internal/controller/autoscaling/testdata/helpers/decision-replicas/a-count-past-int32-is-refused-rather-than-wrapped.yaml @@ -0,0 +1,8 @@ +"name": "a-count-past-int32-is-refused-rather-than-wrapped" +"description": "a count past int32 is refused rather than wrapped" +"inputs": + "status": 200 + "serviceId": "svc" + "body": "{\"apiVersion\":\"llmscaling.inference.x-k8s.io/v1alpha1\",\"decisions\":[{\"serviceId\":\"svc\",\"replicas\":{\"active\":4294967296}}]}" +"expected": + "errorContains": "out of range" diff --git a/internal/controller/autoscaling/testdata/helpers/decision-replicas/a-decision-for-another-service-does-not-match.yaml b/internal/controller/autoscaling/testdata/helpers/decision-replicas/a-decision-for-another-service-does-not-match.yaml new file mode 100644 index 0000000..5284bd1 --- /dev/null +++ b/internal/controller/autoscaling/testdata/helpers/decision-replicas/a-decision-for-another-service-does-not-match.yaml @@ -0,0 +1,9 @@ +# A decision for someone else must not drive this scaler just because it came back. +"name": "a-decision-for-another-service-does-not-match" +"description": "a decision for another service does not match" +"inputs": + "status": 200 + "serviceId": "svc" + "body": "{\"apiVersion\":\"llmscaling.inference.x-k8s.io/v1alpha1\",\"decisions\":[{\"namespace\":\"modelforge\",\"serviceId\":\"other\",\"replicas\":{\"active\":9}}]}" +"expected": + "errorContains": "no decision for serviceId \"svc\"" diff --git a/internal/controller/autoscaling/testdata/helpers/decision-replicas/a-decision-with-no-active-count-is-not-a-zero.yaml b/internal/controller/autoscaling/testdata/helpers/decision-replicas/a-decision-with-no-active-count-is-not-a-zero.yaml new file mode 100644 index 0000000..4dfeaec --- /dev/null +++ b/internal/controller/autoscaling/testdata/helpers/decision-replicas/a-decision-with-no-active-count-is-not-a-zero.yaml @@ -0,0 +1,8 @@ +"name": "a-decision-with-no-active-count-is-not-a-zero" +"description": "a decision with no active count is not a zero" +"inputs": + "status": 200 + "serviceId": "svc" + "body": "{\"apiVersion\":\"llmscaling.inference.x-k8s.io/v1alpha1\",\"decisions\":[{\"namespace\":\"modelforge\",\"serviceId\":\"svc\",\"replicas\":{}}]}" +"expected": + "errorContains": "carries no replicas.active" diff --git a/internal/controller/autoscaling/testdata/helpers/decision-replicas/a-matching-decision-resolves-to-its-active-count.yaml b/internal/controller/autoscaling/testdata/helpers/decision-replicas/a-matching-decision-resolves-to-its-active-count.yaml new file mode 100644 index 0000000..f886bff --- /dev/null +++ b/internal/controller/autoscaling/testdata/helpers/decision-replicas/a-matching-decision-resolves-to-its-active-count.yaml @@ -0,0 +1,8 @@ +"name": "a-matching-decision-resolves-to-its-active-count" +"description": "a matching decision resolves to its active count" +"inputs": + "status": 200 + "serviceId": "svc" + "body": "{\"apiVersion\":\"llmscaling.inference.x-k8s.io/v1alpha1\",\"decisions\":[{\"namespace\":\"modelforge\",\"serviceId\":\"svc\",\"replicas\":{\"active\":15}}]}" +"expected": + "replicas": 15 diff --git a/internal/controller/autoscaling/testdata/helpers/decision-replicas/a-negative-count-is-refused.yaml b/internal/controller/autoscaling/testdata/helpers/decision-replicas/a-negative-count-is-refused.yaml new file mode 100644 index 0000000..2b1ebf3 --- /dev/null +++ b/internal/controller/autoscaling/testdata/helpers/decision-replicas/a-negative-count-is-refused.yaml @@ -0,0 +1,8 @@ +"name": "a-negative-count-is-refused" +"description": "a negative count is refused" +"inputs": + "status": 200 + "serviceId": "svc" + "body": "{\"apiVersion\":\"llmscaling.inference.x-k8s.io/v1alpha1\",\"decisions\":[{\"serviceId\":\"svc\",\"replicas\":{\"active\":-1}}]}" +"expected": + "errorContains": "out of range" diff --git a/internal/controller/autoscaling/testdata/helpers/decision-replicas/a-payload-with-no-apiversion-is-refused.yaml b/internal/controller/autoscaling/testdata/helpers/decision-replicas/a-payload-with-no-apiversion-is-refused.yaml new file mode 100644 index 0000000..fb9855b --- /dev/null +++ b/internal/controller/autoscaling/testdata/helpers/decision-replicas/a-payload-with-no-apiversion-is-refused.yaml @@ -0,0 +1,8 @@ +"name": "a-payload-with-no-apiversion-is-refused" +"description": "a payload with no apiVersion is refused" +"inputs": + "status": 200 + "serviceId": "svc" + "body": "{\"decisions\":[{\"serviceId\":\"svc\",\"replicas\":{\"active\":3}}]}" +"expected": + "errorContains": "unsupported decision schema" diff --git a/internal/controller/autoscaling/testdata/helpers/decision-replicas/an-empty-decision-list-is-not-a-zero.yaml b/internal/controller/autoscaling/testdata/helpers/decision-replicas/an-empty-decision-list-is-not-a-zero.yaml new file mode 100644 index 0000000..826f0d7 --- /dev/null +++ b/internal/controller/autoscaling/testdata/helpers/decision-replicas/an-empty-decision-list-is-not-a-zero.yaml @@ -0,0 +1,8 @@ +"name": "an-empty-decision-list-is-not-a-zero" +"description": "an empty decision list is not a zero" +"inputs": + "status": 200 + "serviceId": "svc" + "body": "{\"apiVersion\":\"llmscaling.inference.x-k8s.io/v1alpha1\",\"decisions\":[]}" +"expected": + "errorContains": "no decision for serviceId \"svc\"" diff --git a/internal/controller/autoscaling/testdata/helpers/decision-replicas/an-unknown-schema-is-refused-not-parsed-hopefully.yaml b/internal/controller/autoscaling/testdata/helpers/decision-replicas/an-unknown-schema-is-refused-not-parsed-hopefully.yaml new file mode 100644 index 0000000..ecd39ec --- /dev/null +++ b/internal/controller/autoscaling/testdata/helpers/decision-replicas/an-unknown-schema-is-refused-not-parsed-hopefully.yaml @@ -0,0 +1,8 @@ +"name": "an-unknown-schema-is-refused-not-parsed-hopefully" +"description": "an unknown schema is refused, not parsed hopefully" +"inputs": + "status": 200 + "serviceId": "svc" + "body": "{\"apiVersion\":\"llmscaling.inference.x-k8s.io/v2\",\"decisions\":[{\"serviceId\":\"svc\",\"replicas\":{\"active\":3}}]}" +"expected": + "errorContains": "unsupported decision schema" diff --git a/internal/controller/autoscaling/testdata/helpers/decision-replicas/duplicates-are-an-error-not-a-pick-first.yaml b/internal/controller/autoscaling/testdata/helpers/decision-replicas/duplicates-are-an-error-not-a-pick-first.yaml new file mode 100644 index 0000000..e2c6abc --- /dev/null +++ b/internal/controller/autoscaling/testdata/helpers/decision-replicas/duplicates-are-an-error-not-a-pick-first.yaml @@ -0,0 +1,8 @@ +"name": "duplicates-are-an-error-not-a-pick-first" +"description": "duplicates are an error, not a pick-first" +"inputs": + "status": 200 + "serviceId": "svc" + "body": "{\"apiVersion\":\"llmscaling.inference.x-k8s.io/v1alpha1\",\"decisions\":[{\"namespace\":\"staging\",\"serviceId\":\"svc\",\"replicas\":{\"active\":2}},{\"namespace\":\"modelforge\",\"serviceId\":\"svc\",\"replicas\":{\"active\":7}}]}" +"expected": + "errorContains": "2 decisions match" diff --git a/internal/controller/autoscaling/testdata/helpers/decision-replicas/the-namespace-filter-is-reported-when-nothing-matches.yaml b/internal/controller/autoscaling/testdata/helpers/decision-replicas/the-namespace-filter-is-reported-when-nothing-matches.yaml new file mode 100644 index 0000000..1b1b354 --- /dev/null +++ b/internal/controller/autoscaling/testdata/helpers/decision-replicas/the-namespace-filter-is-reported-when-nothing-matches.yaml @@ -0,0 +1,9 @@ +"name": "the-namespace-filter-is-reported-when-nothing-matches" +"description": "the namespace filter is reported when nothing matches" +"inputs": + "status": 200 + "serviceId": "svc" + "namespace": "modelforge" + "body": "{\"apiVersion\":\"llmscaling.inference.x-k8s.io/v1alpha1\",\"decisions\":[{\"namespace\":\"staging\",\"serviceId\":\"svc\",\"replicas\":{\"active\":2}}]}" +"expected": + "errorContains": "in namespace \"modelforge\"" diff --git a/internal/controller/autoscaling/testdata/helpers/decision-replicas/the-namespace-filter-picks-between-same-named-services.yaml b/internal/controller/autoscaling/testdata/helpers/decision-replicas/the-namespace-filter-picks-between-same-named-services.yaml new file mode 100644 index 0000000..6948158 --- /dev/null +++ b/internal/controller/autoscaling/testdata/helpers/decision-replicas/the-namespace-filter-picks-between-same-named-services.yaml @@ -0,0 +1,9 @@ +"name": "the-namespace-filter-picks-between-same-named-services" +"description": "the namespace filter picks between same-named services" +"inputs": + "status": 200 + "serviceId": "svc" + "namespace": "modelforge" + "body": "{\"apiVersion\":\"llmscaling.inference.x-k8s.io/v1alpha1\",\"decisions\":[{\"namespace\":\"staging\",\"serviceId\":\"svc\",\"replicas\":{\"active\":2}},{\"namespace\":\"modelforge\",\"serviceId\":\"svc\",\"replicas\":{\"active\":7}}]}" +"expected": + "replicas": 7 diff --git a/internal/controller/autoscaling/testdata/helpers/decision-replicas/zero-is-a-decision-not-a-missing-one.yaml b/internal/controller/autoscaling/testdata/helpers/decision-replicas/zero-is-a-decision-not-a-missing-one.yaml new file mode 100644 index 0000000..c556e46 --- /dev/null +++ b/internal/controller/autoscaling/testdata/helpers/decision-replicas/zero-is-a-decision-not-a-missing-one.yaml @@ -0,0 +1,8 @@ +"name": "zero-is-a-decision-not-a-missing-one" +"description": "zero is a decision, not a missing one" +"inputs": + "status": 200 + "serviceId": "svc" + "body": "{\"apiVersion\":\"llmscaling.inference.x-k8s.io/v1alpha1\",\"decisions\":[{\"namespace\":\"modelforge\",\"serviceId\":\"svc\",\"replicas\":{\"active\":0}}]}" +"expected": + "replicas": 0 diff --git a/internal/controller/autoscaling/testdata/helpers/decision-request/a-configured-path-is-used.yaml b/internal/controller/autoscaling/testdata/helpers/decision-request/a-configured-path-is-used.yaml new file mode 100644 index 0000000..a1e8db1 --- /dev/null +++ b/internal/controller/autoscaling/testdata/helpers/decision-request/a-configured-path-is-used.yaml @@ -0,0 +1,12 @@ +"name": "a-configured-path-is-used" +"description": "a configured path is used" +"inputs": + "serviceId": "fallback-modelforge-01" + "path": "/v2/decisions" + "headers": + "Authorization": "Bearer t" +"expected": + "replicas": 4 + "path": "/v2/decisions" + "serviceId": "fallback-modelforge-01" + "authorization": "Bearer t" diff --git a/internal/controller/autoscaling/testdata/helpers/decision-request/a-path-without-a-leading-slash-still-forms-one.yaml b/internal/controller/autoscaling/testdata/helpers/decision-request/a-path-without-a-leading-slash-still-forms-one.yaml new file mode 100644 index 0000000..55e3bb5 --- /dev/null +++ b/internal/controller/autoscaling/testdata/helpers/decision-request/a-path-without-a-leading-slash-still-forms-one.yaml @@ -0,0 +1,13 @@ +"name": "a-path-without-a-leading-slash-still-forms-one" +"description": "a path without a leading slash still forms one" +"inputs": + "serviceId": "fallback-modelforge-01" + "path": "decisions" + "headers": + "Authorization": "Bearer t" +"expected": + "replicas": 4 + "path": "/decisions" + "serviceId": "fallback-modelforge-01" + "authorization": "Bearer t" + "pinDefaultPath": true diff --git a/internal/controller/autoscaling/testdata/helpers/decision-request/a-trailing-slash-on-serveraddress-is-not-doubled.yaml b/internal/controller/autoscaling/testdata/helpers/decision-request/a-trailing-slash-on-serveraddress-is-not-doubled.yaml new file mode 100644 index 0000000..ed99eb0 --- /dev/null +++ b/internal/controller/autoscaling/testdata/helpers/decision-request/a-trailing-slash-on-serveraddress-is-not-doubled.yaml @@ -0,0 +1,14 @@ +"name": "a-trailing-slash-on-serveraddress-is-not-doubled" +"description": "a trailing slash on serverAddress is not doubled" +"inputs": + "serviceId": "fallback-modelforge-01" + "path": "/decisions" + "trailingSlash": true + "headers": + "Authorization": "Bearer t" +"expected": + "replicas": 4 + "path": "/decisions" + "serviceId": "fallback-modelforge-01" + "authorization": "Bearer t" + "pinDefaultPath": true diff --git a/internal/controller/autoscaling/testdata/helpers/decision-request/an-empty-path-falls-back-to-the-default.yaml b/internal/controller/autoscaling/testdata/helpers/decision-request/an-empty-path-falls-back-to-the-default.yaml new file mode 100644 index 0000000..560f815 --- /dev/null +++ b/internal/controller/autoscaling/testdata/helpers/decision-request/an-empty-path-falls-back-to-the-default.yaml @@ -0,0 +1,12 @@ +"name": "an-empty-path-falls-back-to-the-default" +"description": "an empty path falls back to the default" +"inputs": + "serviceId": "fallback-modelforge-01" + "headers": + "Authorization": "Bearer t" +"expected": + "replicas": 4 + "path": "/decisions" + "serviceId": "fallback-modelforge-01" + "authorization": "Bearer t" + "pinDefaultPath": true diff --git a/internal/controller/autoscaling/testdata/helpers/decision-sample/a-verbatim-decision-server-payload-resolves-to-15.yaml b/internal/controller/autoscaling/testdata/helpers/decision-sample/a-verbatim-decision-server-payload-resolves-to-15.yaml new file mode 100644 index 0000000..34d7f3a --- /dev/null +++ b/internal/controller/autoscaling/testdata/helpers/decision-sample/a-verbatim-decision-server-payload-resolves-to-15.yaml @@ -0,0 +1,22 @@ +# Wire format from the decision server, not a fixture built by the table. +"name": "a-verbatim-decision-server-payload-resolves-to-15" +"description": "a verbatim decision-server payload resolves to 15" +"inputs": + "status": 200 + "serviceId": "fallback-modelforge-01" + "body": |- + { + "apiVersion": "llmscaling.inference.x-k8s.io/v1alpha1", + "decisions": [ + { + "namespace": "modelforge", + "serviceId": "fallback-modelforge-01", + "replicas": { + "active": 15 + } + } + ] + } + "pinAPIVersion": true +"expected": + "replicas": 15 diff --git a/internal/controller/autoscaling/testdata/helpers/defer-timeout/explicit-0.yaml b/internal/controller/autoscaling/testdata/helpers/defer-timeout/explicit-0.yaml new file mode 100644 index 0000000..2499429 --- /dev/null +++ b/internal/controller/autoscaling/testdata/helpers/defer-timeout/explicit-0.yaml @@ -0,0 +1,6 @@ +"name": "explicit-0" +"description": "explicit 0" +"inputs": + "deferTimeoutSeconds": 0 +"expected": + "duration": "0s" diff --git a/internal/controller/autoscaling/testdata/helpers/defer-timeout/explicit-30.yaml b/internal/controller/autoscaling/testdata/helpers/defer-timeout/explicit-30.yaml new file mode 100644 index 0000000..bed9777 --- /dev/null +++ b/internal/controller/autoscaling/testdata/helpers/defer-timeout/explicit-30.yaml @@ -0,0 +1,6 @@ +"name": "explicit-30" +"description": "explicit 30" +"inputs": + "deferTimeoutSeconds": 30 +"expected": + "duration": "30s" diff --git a/internal/controller/autoscaling/testdata/helpers/defer-timeout/nil-spec.yaml b/internal/controller/autoscaling/testdata/helpers/defer-timeout/nil-spec.yaml new file mode 100644 index 0000000..a5ed99c --- /dev/null +++ b/internal/controller/autoscaling/testdata/helpers/defer-timeout/nil-spec.yaml @@ -0,0 +1,6 @@ +"name": "nil-spec" +"description": "nil spec" +"inputs": {} +"expected": + "duration": "600s" + "equalsDefault": true diff --git a/internal/controller/autoscaling/testdata/helpers/deferral-memory/enddeferral-and-forgetrecommendations-clear-the-forced-write-record.yaml b/internal/controller/autoscaling/testdata/helpers/deferral-memory/enddeferral-and-forgetrecommendations-clear-the-forced-write-record.yaml new file mode 100644 index 0000000..739f16f --- /dev/null +++ b/internal/controller/autoscaling/testdata/helpers/deferral-memory/enddeferral-and-forgetrecommendations-clear-the-forced-write-record.yaml @@ -0,0 +1,16 @@ +"name": "enddeferral-and-forgetrecommendations-clear-the-forced-write-record" +"description": "endDeferral and forgetRecommendations clear the forced-write record" +"inputs": + "namespace": "ns" + "scaler": "s" + "at": "2026-03-01T00:00:00Z" + "steps": + - "op": "record" + - "op": "setSince" + - "op": "end" + - "op": "expectSinceCleared" + - "op": "expectRecordCleared" + - "op": "record" + - "op": "forget" + - "op": "expectRecordCleared" +"expected": {} diff --git a/internal/controller/autoscaling/testdata/helpers/deferral-requeue-after/deadline-already-passed.yaml b/internal/controller/autoscaling/testdata/helpers/deferral-requeue-after/deadline-already-passed.yaml new file mode 100644 index 0000000..85d0778 --- /dev/null +++ b/internal/controller/autoscaling/testdata/helpers/deferral-requeue-after/deadline-already-passed.yaml @@ -0,0 +1,8 @@ +"name": "deadline-already-passed" +"description": "deadline already passed" +"inputs": + "holding": true + "timeout": "1m" + "until": "-1s" +"expected": + "after": "1s" diff --git a/internal/controller/autoscaling/testdata/helpers/deferral-requeue-after/deadline-due-this-instant.yaml b/internal/controller/autoscaling/testdata/helpers/deferral-requeue-after/deadline-due-this-instant.yaml new file mode 100644 index 0000000..25e3bda --- /dev/null +++ b/internal/controller/autoscaling/testdata/helpers/deferral-requeue-after/deadline-due-this-instant.yaml @@ -0,0 +1,8 @@ +"name": "deadline-due-this-instant" +"description": "deadline due this instant" +"inputs": + "holding": true + "timeout": "1m" + "until": "0s" +"expected": + "after": "1s" diff --git a/internal/controller/autoscaling/testdata/helpers/deferral-requeue-after/deadline-later-than-sync-period.yaml b/internal/controller/autoscaling/testdata/helpers/deferral-requeue-after/deadline-later-than-sync-period.yaml new file mode 100644 index 0000000..1a90ff3 --- /dev/null +++ b/internal/controller/autoscaling/testdata/helpers/deferral-requeue-after/deadline-later-than-sync-period.yaml @@ -0,0 +1,8 @@ +"name": "deadline-later-than-sync-period" +"description": "deadline later than sync period" +"inputs": + "holding": true + "timeout": "1m" + "until": "30s" +"expected": + "after": "15s" diff --git a/internal/controller/autoscaling/testdata/helpers/deferral-requeue-after/deadline-sooner-than-sync-period.yaml b/internal/controller/autoscaling/testdata/helpers/deferral-requeue-after/deadline-sooner-than-sync-period.yaml new file mode 100644 index 0000000..6dda718 --- /dev/null +++ b/internal/controller/autoscaling/testdata/helpers/deferral-requeue-after/deadline-sooner-than-sync-period.yaml @@ -0,0 +1,8 @@ +"name": "deadline-sooner-than-sync-period" +"description": "deadline sooner than sync period" +"inputs": + "holding": true + "timeout": "1m" + "until": "5s" +"expected": + "after": "5s" diff --git a/internal/controller/autoscaling/testdata/helpers/deferral-requeue-after/deadline-under-the-1s-floor.yaml b/internal/controller/autoscaling/testdata/helpers/deferral-requeue-after/deadline-under-the-1s-floor.yaml new file mode 100644 index 0000000..e76800b --- /dev/null +++ b/internal/controller/autoscaling/testdata/helpers/deferral-requeue-after/deadline-under-the-1s-floor.yaml @@ -0,0 +1,8 @@ +"name": "deadline-under-the-1s-floor" +"description": "deadline under the 1s floor" +"inputs": + "holding": true + "timeout": "1m" + "until": "200ms" +"expected": + "after": "1s" diff --git a/internal/controller/autoscaling/testdata/helpers/deferral-requeue-after/not-holding-stays-on-sync-period.yaml b/internal/controller/autoscaling/testdata/helpers/deferral-requeue-after/not-holding-stays-on-sync-period.yaml new file mode 100644 index 0000000..1f065b6 --- /dev/null +++ b/internal/controller/autoscaling/testdata/helpers/deferral-requeue-after/not-holding-stays-on-sync-period.yaml @@ -0,0 +1,8 @@ +"name": "not-holding-stays-on-sync-period" +"description": "not holding stays on sync period" +"inputs": + "holding": false + "timeout": "1m" + "until": "5s" +"expected": + "after": "15s" diff --git a/internal/controller/autoscaling/testdata/helpers/deferral-requeue-after/pacing-later-than-sync-period.yaml b/internal/controller/autoscaling/testdata/helpers/deferral-requeue-after/pacing-later-than-sync-period.yaml new file mode 100644 index 0000000..f8d1353 --- /dev/null +++ b/internal/controller/autoscaling/testdata/helpers/deferral-requeue-after/pacing-later-than-sync-period.yaml @@ -0,0 +1,9 @@ +"name": "pacing-later-than-sync-period" +"description": "pacing later than sync period" +"inputs": + "holding": true + "timeout": "1m" + "until": "-1s" + "pacing": "30s" +"expected": + "after": "15s" diff --git a/internal/controller/autoscaling/testdata/helpers/deferral-requeue-after/pacing-sooner-than-sync-period.yaml b/internal/controller/autoscaling/testdata/helpers/deferral-requeue-after/pacing-sooner-than-sync-period.yaml new file mode 100644 index 0000000..e43f746 --- /dev/null +++ b/internal/controller/autoscaling/testdata/helpers/deferral-requeue-after/pacing-sooner-than-sync-period.yaml @@ -0,0 +1,9 @@ +"name": "pacing-sooner-than-sync-period" +"description": "pacing sooner than sync period" +"inputs": + "holding": true + "timeout": "1m" + "until": "-1s" + "pacing": "4s" +"expected": + "after": "4s" diff --git a/internal/controller/autoscaling/testdata/helpers/deferral-requeue-after/pacing-under-the-1s-floor.yaml b/internal/controller/autoscaling/testdata/helpers/deferral-requeue-after/pacing-under-the-1s-floor.yaml new file mode 100644 index 0000000..a777abc --- /dev/null +++ b/internal/controller/autoscaling/testdata/helpers/deferral-requeue-after/pacing-under-the-1s-floor.yaml @@ -0,0 +1,9 @@ +"name": "pacing-under-the-1s-floor" +"description": "pacing under the 1s floor" +"inputs": + "holding": true + "timeout": "1m" + "until": "-1s" + "pacing": "100ms" +"expected": + "after": "1s" diff --git a/internal/controller/autoscaling/testdata/helpers/deferral-requeue-after/timeout-0-stays-on-sync-period.yaml b/internal/controller/autoscaling/testdata/helpers/deferral-requeue-after/timeout-0-stays-on-sync-period.yaml new file mode 100644 index 0000000..3c171bb --- /dev/null +++ b/internal/controller/autoscaling/testdata/helpers/deferral-requeue-after/timeout-0-stays-on-sync-period.yaml @@ -0,0 +1,8 @@ +"name": "timeout-0-stays-on-sync-period" +"description": "timeout 0 stays on sync period" +"inputs": + "holding": true + "timeout": "0s" + "until": "5s" +"expected": + "after": "15s" diff --git a/internal/controller/autoscaling/testdata/helpers/forced-write-pace/14s-into-a-15s-period.yaml b/internal/controller/autoscaling/testdata/helpers/forced-write-pace/14s-into-a-15s-period.yaml new file mode 100644 index 0000000..6825126 --- /dev/null +++ b/internal/controller/autoscaling/testdata/helpers/forced-write-pace/14s-into-a-15s-period.yaml @@ -0,0 +1,9 @@ +"name": "14s-into-a-15s-period" +"description": "14s into a 15s period" +"inputs": + "recorded": true + "sync": "15s" + "at": "14s" +"expected": + "allowed": false + "remaining": "1s" diff --git a/internal/controller/autoscaling/testdata/helpers/forced-write-pace/exactly-one-sync-period.yaml b/internal/controller/autoscaling/testdata/helpers/forced-write-pace/exactly-one-sync-period.yaml new file mode 100644 index 0000000..69cbb8b --- /dev/null +++ b/internal/controller/autoscaling/testdata/helpers/forced-write-pace/exactly-one-sync-period.yaml @@ -0,0 +1,8 @@ +"name": "exactly-one-sync-period" +"description": "exactly one sync period" +"inputs": + "recorded": true + "sync": "15s" + "at": "15s" +"expected": + "allowed": true diff --git a/internal/controller/autoscaling/testdata/helpers/forced-write-pace/just-recorded.yaml b/internal/controller/autoscaling/testdata/helpers/forced-write-pace/just-recorded.yaml new file mode 100644 index 0000000..57ed154 --- /dev/null +++ b/internal/controller/autoscaling/testdata/helpers/forced-write-pace/just-recorded.yaml @@ -0,0 +1,8 @@ +"name": "just-recorded" +"description": "just recorded" +"inputs": + "recorded": true + "sync": "15s" + "at": "0s" +"expected": + "allowed": false diff --git a/internal/controller/autoscaling/testdata/helpers/forced-write-pace/no-record.yaml b/internal/controller/autoscaling/testdata/helpers/forced-write-pace/no-record.yaml new file mode 100644 index 0000000..e4af893 --- /dev/null +++ b/internal/controller/autoscaling/testdata/helpers/forced-write-pace/no-record.yaml @@ -0,0 +1,9 @@ +"name": "no-record" +"description": "no record" +"inputs": + "recorded": false + "sync": "15s" + "at": "0s" +"expected": + "allowed": true + "remaining": "0s" diff --git a/internal/controller/autoscaling/testdata/helpers/forced-write-pace/zero-sync-period.yaml b/internal/controller/autoscaling/testdata/helpers/forced-write-pace/zero-sync-period.yaml new file mode 100644 index 0000000..4bfe8d3 --- /dev/null +++ b/internal/controller/autoscaling/testdata/helpers/forced-write-pace/zero-sync-period.yaml @@ -0,0 +1,9 @@ +"name": "zero-sync-period" +"description": "zero sync period" +"inputs": + "recorded": true + "sync": "0s" + "at": "1h" +"expected": + "allowed": true + "remaining": "0s" diff --git a/internal/controller/autoscaling/testdata/helpers/parse-target-value/empty.yaml b/internal/controller/autoscaling/testdata/helpers/parse-target-value/empty.yaml new file mode 100644 index 0000000..ac8035d --- /dev/null +++ b/internal/controller/autoscaling/testdata/helpers/parse-target-value/empty.yaml @@ -0,0 +1,6 @@ +"name": "empty" +"description": "empty" +"inputs": + "value": "" +"expected": + "error": true diff --git a/internal/controller/autoscaling/testdata/helpers/parse-target-value/inf-parses-as-a-float-but-is-not-usable.yaml b/internal/controller/autoscaling/testdata/helpers/parse-target-value/inf-parses-as-a-float-but-is-not-usable.yaml new file mode 100644 index 0000000..c14be6e --- /dev/null +++ b/internal/controller/autoscaling/testdata/helpers/parse-target-value/inf-parses-as-a-float-but-is-not-usable.yaml @@ -0,0 +1,6 @@ +"name": "inf-parses-as-a-float-but-is-not-usable" +"description": "Inf parses as a float but is not usable" +"inputs": + "value": "+Inf" +"expected": + "error": true diff --git a/internal/controller/autoscaling/testdata/helpers/parse-target-value/nan-parses-as-a-float-but-is-not-usable.yaml b/internal/controller/autoscaling/testdata/helpers/parse-target-value/nan-parses-as-a-float-but-is-not-usable.yaml new file mode 100644 index 0000000..664966d --- /dev/null +++ b/internal/controller/autoscaling/testdata/helpers/parse-target-value/nan-parses-as-a-float-but-is-not-usable.yaml @@ -0,0 +1,6 @@ +"name": "nan-parses-as-a-float-but-is-not-usable" +"description": "NaN parses as a float but is not usable" +"inputs": + "value": "NaN" +"expected": + "error": true diff --git a/internal/controller/autoscaling/testdata/helpers/parse-target-value/negative-would-invert-scaling.yaml b/internal/controller/autoscaling/testdata/helpers/parse-target-value/negative-would-invert-scaling.yaml new file mode 100644 index 0000000..e2084dc --- /dev/null +++ b/internal/controller/autoscaling/testdata/helpers/parse-target-value/negative-would-invert-scaling.yaml @@ -0,0 +1,6 @@ +"name": "negative-would-invert-scaling" +"description": "negative would invert scaling" +"inputs": + "value": "-1" +"expected": + "error": true diff --git a/internal/controller/autoscaling/testdata/helpers/parse-target-value/not-a-number.yaml b/internal/controller/autoscaling/testdata/helpers/parse-target-value/not-a-number.yaml new file mode 100644 index 0000000..83b3203 --- /dev/null +++ b/internal/controller/autoscaling/testdata/helpers/parse-target-value/not-a-number.yaml @@ -0,0 +1,6 @@ +"name": "not-a-number" +"description": "not a number" +"inputs": + "value": "abc" +"expected": + "error": true diff --git a/internal/controller/autoscaling/testdata/helpers/parse-target-value/percent-suffix-is-rejected-not-stripped.yaml b/internal/controller/autoscaling/testdata/helpers/parse-target-value/percent-suffix-is-rejected-not-stripped.yaml new file mode 100644 index 0000000..fc811f3 --- /dev/null +++ b/internal/controller/autoscaling/testdata/helpers/parse-target-value/percent-suffix-is-rejected-not-stripped.yaml @@ -0,0 +1,6 @@ +"name": "percent-suffix-is-rejected-not-stripped" +"description": "percent suffix is rejected, not stripped" +"inputs": + "value": "80%" +"expected": + "error": true diff --git a/internal/controller/autoscaling/testdata/helpers/parse-target-value/plain-integer.yaml b/internal/controller/autoscaling/testdata/helpers/parse-target-value/plain-integer.yaml new file mode 100644 index 0000000..d4a875f --- /dev/null +++ b/internal/controller/autoscaling/testdata/helpers/parse-target-value/plain-integer.yaml @@ -0,0 +1,6 @@ +"name": "plain-integer" +"description": "plain integer" +"inputs": + "value": "5" +"expected": + "value": 5 diff --git a/internal/controller/autoscaling/testdata/helpers/parse-target-value/ratio.yaml b/internal/controller/autoscaling/testdata/helpers/parse-target-value/ratio.yaml new file mode 100644 index 0000000..9999cf6 --- /dev/null +++ b/internal/controller/autoscaling/testdata/helpers/parse-target-value/ratio.yaml @@ -0,0 +1,6 @@ +"name": "ratio" +"description": "ratio" +"inputs": + "value": "0.8" +"expected": + "value": 0.8 diff --git a/internal/controller/autoscaling/testdata/helpers/parse-target-value/surrounding-whitespace.yaml b/internal/controller/autoscaling/testdata/helpers/parse-target-value/surrounding-whitespace.yaml new file mode 100644 index 0000000..168c2bb --- /dev/null +++ b/internal/controller/autoscaling/testdata/helpers/parse-target-value/surrounding-whitespace.yaml @@ -0,0 +1,6 @@ +"name": "surrounding-whitespace" +"description": "surrounding whitespace" +"inputs": + "value": " 2.5 " +"expected": + "value": 2.5 diff --git a/internal/controller/autoscaling/testdata/helpers/parse-target-value/zero-would-make-the-ratio-infinite.yaml b/internal/controller/autoscaling/testdata/helpers/parse-target-value/zero-would-make-the-ratio-infinite.yaml new file mode 100644 index 0000000..5291ffe --- /dev/null +++ b/internal/controller/autoscaling/testdata/helpers/parse-target-value/zero-would-make-the-ratio-infinite.yaml @@ -0,0 +1,6 @@ +"name": "zero-would-make-the-ratio-infinite" +"description": "zero would make the ratio infinite" +"inputs": + "value": "0" +"expected": + "error": true diff --git a/internal/controller/autoscaling/testdata/helpers/past-defer-timeout/59s-of-a-60s-timeout.yaml b/internal/controller/autoscaling/testdata/helpers/past-defer-timeout/59s-of-a-60s-timeout.yaml new file mode 100644 index 0000000..cf07d0c --- /dev/null +++ b/internal/controller/autoscaling/testdata/helpers/past-defer-timeout/59s-of-a-60s-timeout.yaml @@ -0,0 +1,7 @@ +"name": "59s-of-a-60s-timeout" +"description": "59s of a 60s timeout" +"inputs": + "timeout": "1m" + "elapsed": "59s" +"expected": + "past": false diff --git a/internal/controller/autoscaling/testdata/helpers/past-defer-timeout/exactly-the-timeout.yaml b/internal/controller/autoscaling/testdata/helpers/past-defer-timeout/exactly-the-timeout.yaml new file mode 100644 index 0000000..0cac796 --- /dev/null +++ b/internal/controller/autoscaling/testdata/helpers/past-defer-timeout/exactly-the-timeout.yaml @@ -0,0 +1,7 @@ +"name": "exactly-the-timeout" +"description": "exactly the timeout" +"inputs": + "timeout": "1m" + "elapsed": "1m" +"expected": + "past": true diff --git a/internal/controller/autoscaling/testdata/helpers/past-defer-timeout/nil-deferredsince.yaml b/internal/controller/autoscaling/testdata/helpers/past-defer-timeout/nil-deferredsince.yaml new file mode 100644 index 0000000..4a53360 --- /dev/null +++ b/internal/controller/autoscaling/testdata/helpers/past-defer-timeout/nil-deferredsince.yaml @@ -0,0 +1,8 @@ +"name": "nil-deferredsince" +"description": "nil deferredSince" +"inputs": + "nilSince": true + "timeout": "1m" + "elapsed": "1h" +"expected": + "past": false diff --git a/internal/controller/autoscaling/testdata/helpers/past-defer-timeout/past-the-timeout.yaml b/internal/controller/autoscaling/testdata/helpers/past-defer-timeout/past-the-timeout.yaml new file mode 100644 index 0000000..57f71e9 --- /dev/null +++ b/internal/controller/autoscaling/testdata/helpers/past-defer-timeout/past-the-timeout.yaml @@ -0,0 +1,7 @@ +"name": "past-the-timeout" +"description": "past the timeout" +"inputs": + "timeout": "1m" + "elapsed": "1m1s" +"expected": + "past": true diff --git a/internal/controller/autoscaling/testdata/helpers/past-defer-timeout/timeout-0.yaml b/internal/controller/autoscaling/testdata/helpers/past-defer-timeout/timeout-0.yaml new file mode 100644 index 0000000..b5c583c --- /dev/null +++ b/internal/controller/autoscaling/testdata/helpers/past-defer-timeout/timeout-0.yaml @@ -0,0 +1,7 @@ +"name": "timeout-0" +"description": "timeout 0" +"inputs": + "timeout": "0s" + "elapsed": "24h" +"expected": + "past": false diff --git a/internal/controller/autoscaling/testdata/helpers/query-prometheus-scalar/an-under-aggregated-query-is-rejected-naming-the-series.yaml b/internal/controller/autoscaling/testdata/helpers/query-prometheus-scalar/an-under-aggregated-query-is-rejected-naming-the-series.yaml new file mode 100644 index 0000000..469d0c9 --- /dev/null +++ b/internal/controller/autoscaling/testdata/helpers/query-prometheus-scalar/an-under-aggregated-query-is-rejected-naming-the-series.yaml @@ -0,0 +1,9 @@ +"name": "an-under-aggregated-query-is-rejected-naming-the-series" +"description": "an under-aggregated query is rejected, naming the series" +"inputs": + "result": |- + [{"metric":{"__name__":"m","model":"qwen","backend":"10.0.0.1:8000"},"value":[0,"1"]}, + {"metric":{"__name__":"m","model":"qwen","backend":"10.0.0.2:8000"},"value":[0,"9"]}] +"expected": + "errorContains": + - "returned 2 series" diff --git a/internal/controller/autoscaling/testdata/helpers/query-prometheus-scalar/empty-vector-is-not-zero.yaml b/internal/controller/autoscaling/testdata/helpers/query-prometheus-scalar/empty-vector-is-not-zero.yaml new file mode 100644 index 0000000..a1adcf8 --- /dev/null +++ b/internal/controller/autoscaling/testdata/helpers/query-prometheus-scalar/empty-vector-is-not-zero.yaml @@ -0,0 +1,7 @@ +"name": "empty-vector-is-not-zero" +"description": "empty vector is not zero" +"inputs": + "result": "[]" +"expected": + "errorContains": + - "returned no data" diff --git a/internal/controller/autoscaling/testdata/helpers/query-prometheus-scalar/multi-series-error-names-both-label-sets.yaml b/internal/controller/autoscaling/testdata/helpers/query-prometheus-scalar/multi-series-error-names-both-label-sets.yaml new file mode 100644 index 0000000..5a27b8c --- /dev/null +++ b/internal/controller/autoscaling/testdata/helpers/query-prometheus-scalar/multi-series-error-names-both-label-sets.yaml @@ -0,0 +1,10 @@ +# The multi-series error quotes both series with their labels sorted. +"name": "multi-series-error-names-both-label-sets" +"description": "multi-series error names both label sets" +"inputs": + "result": "[\n\t\t\t{\"metric\":{\"model\":\"qwen\",\"backend\":\"10.0.0.1:8000\"},\"value\":[0,\"1\"]},\n\t\t\t{\"metric\":{\"model\":\"qwen\",\"backend\":\"10.0.0.2:8000\"},\"value\":[0,\"9\"]}]" +"expected": + "errorContains": + - "{backend=\"10.0.0.1:8000\",model=\"qwen\"}" + - "{backend=\"10.0.0.2:8000\",model=\"qwen\"}" + - "avg(...)" diff --git a/internal/controller/autoscaling/testdata/helpers/query-prometheus-scalar/single-series-resolves.yaml b/internal/controller/autoscaling/testdata/helpers/query-prometheus-scalar/single-series-resolves.yaml new file mode 100644 index 0000000..a9c4bb8 --- /dev/null +++ b/internal/controller/autoscaling/testdata/helpers/query-prometheus-scalar/single-series-resolves.yaml @@ -0,0 +1,6 @@ +"name": "single-series-resolves" +"description": "single series resolves" +"inputs": + "result": "[{\"metric\":{},\"value\":[0,\"1.5\"]}]" +"expected": + "value": 1.5 diff --git a/internal/controller/autoscaling/testdata/helpers/reason-deferral-timed-out/pins-the-published-condition-reason.yaml b/internal/controller/autoscaling/testdata/helpers/reason-deferral-timed-out/pins-the-published-condition-reason.yaml new file mode 100644 index 0000000..0fd98c4 --- /dev/null +++ b/internal/controller/autoscaling/testdata/helpers/reason-deferral-timed-out/pins-the-published-condition-reason.yaml @@ -0,0 +1,5 @@ +"name": "pins-the-published-condition-reason" +"description": "pins the published condition reason" +"inputs": {} +"expected": + "reason": "DeferralTimedOut" diff --git a/internal/controller/autoscaling/testdata/helpers/recommend-custom/a-decision-above-maxreplicas-is-clamped.yaml b/internal/controller/autoscaling/testdata/helpers/recommend-custom/a-decision-above-maxreplicas-is-clamped.yaml new file mode 100644 index 0000000..34f4b8a --- /dev/null +++ b/internal/controller/autoscaling/testdata/helpers/recommend-custom/a-decision-above-maxreplicas-is-clamped.yaml @@ -0,0 +1,14 @@ +"name": "a-decision-above-maxreplicas-is-clamped" +"description": "a decision above maxReplicas is clamped" +"inputs": + "active": "15" + "minReplicas": 1 + "maxReplicas": 5 + "readyReplicas": 8 + "specReplicas": 8 + "serviceId": "svc" + "namespace": "modelforge" + "apiVersion": "llmscaling.inference.x-k8s.io/v1alpha1" + "pinAPIVersion": true +"expected": + "replicas": 5 diff --git a/internal/controller/autoscaling/testdata/helpers/recommend-custom/a-decision-of-zero-clamps-up-to-minreplicas.yaml b/internal/controller/autoscaling/testdata/helpers/recommend-custom/a-decision-of-zero-clamps-up-to-minreplicas.yaml new file mode 100644 index 0000000..13800e5 --- /dev/null +++ b/internal/controller/autoscaling/testdata/helpers/recommend-custom/a-decision-of-zero-clamps-up-to-minreplicas.yaml @@ -0,0 +1,14 @@ +"name": "a-decision-of-zero-clamps-up-to-minreplicas" +"description": "a decision of zero clamps up to minReplicas" +"inputs": + "active": "0" + "minReplicas": 2 + "maxReplicas": 20 + "readyReplicas": 8 + "specReplicas": 8 + "serviceId": "svc" + "namespace": "modelforge" + "apiVersion": "llmscaling.inference.x-k8s.io/v1alpha1" + "pinAPIVersion": true +"expected": + "replicas": 2 diff --git a/internal/controller/autoscaling/testdata/helpers/recommend-custom/a-missing-customprovider-holds-the-fleet-where-it-is.yaml b/internal/controller/autoscaling/testdata/helpers/recommend-custom/a-missing-customprovider-holds-the-fleet-where-it-is.yaml new file mode 100644 index 0000000..33ecae8 --- /dev/null +++ b/internal/controller/autoscaling/testdata/helpers/recommend-custom/a-missing-customprovider-holds-the-fleet-where-it-is.yaml @@ -0,0 +1,15 @@ +"name": "a-missing-customprovider-holds-the-fleet-where-it-is" +"description": "a missing customProvider holds the fleet where it is" +"inputs": + "active": "15" + "minReplicas": 1 + "maxReplicas": 20 + "readyReplicas": 8 + "specReplicas": 8 + "serviceId": "svc" + "namespace": "modelforge" + "apiVersion": "llmscaling.inference.x-k8s.io/v1alpha1" + "absentCustomProvider": true + "pinAPIVersion": true +"expected": + "replicas": 8 diff --git a/internal/controller/autoscaling/testdata/helpers/recommend-custom/an-unreadable-decision-holds-the-fleet-where-it-is.yaml b/internal/controller/autoscaling/testdata/helpers/recommend-custom/an-unreadable-decision-holds-the-fleet-where-it-is.yaml new file mode 100644 index 0000000..26c0199 --- /dev/null +++ b/internal/controller/autoscaling/testdata/helpers/recommend-custom/an-unreadable-decision-holds-the-fleet-where-it-is.yaml @@ -0,0 +1,13 @@ +"name": "an-unreadable-decision-holds-the-fleet-where-it-is" +"description": "an unreadable decision holds the fleet where it is" +"inputs": + "active": "15" + "minReplicas": 1 + "maxReplicas": 20 + "readyReplicas": 8 + "specReplicas": 8 + "serviceId": "svc" + "namespace": "modelforge" + "apiVersion": "some.other.api/v1" +"expected": + "replicas": 8 diff --git a/internal/controller/autoscaling/testdata/helpers/recommend-custom/the-decision-is-used-as-is-not-scaled-by-readyreplicas.yaml b/internal/controller/autoscaling/testdata/helpers/recommend-custom/the-decision-is-used-as-is-not-scaled-by-readyreplicas.yaml new file mode 100644 index 0000000..b8bb5a7 --- /dev/null +++ b/internal/controller/autoscaling/testdata/helpers/recommend-custom/the-decision-is-used-as-is-not-scaled-by-readyreplicas.yaml @@ -0,0 +1,15 @@ +# 8 ready replicas would turn a per-replica metric into a much larger number; a decision of 15 is already the answer. +"name": "the-decision-is-used-as-is-not-scaled-by-readyreplicas" +"description": "the decision is used as-is, not scaled by readyReplicas" +"inputs": + "active": "15" + "minReplicas": 1 + "maxReplicas": 20 + "readyReplicas": 8 + "specReplicas": 8 + "serviceId": "svc" + "namespace": "modelforge" + "apiVersion": "llmscaling.inference.x-k8s.io/v1alpha1" + "pinAPIVersion": true +"expected": + "replicas": 15 diff --git a/internal/controller/autoscaling/testdata/helpers/recommend-default-prometheus/an-empty-metricprovider-still-queries-prometheus.yaml b/internal/controller/autoscaling/testdata/helpers/recommend-default-prometheus/an-empty-metricprovider-still-queries-prometheus.yaml new file mode 100644 index 0000000..1e19cba --- /dev/null +++ b/internal/controller/autoscaling/testdata/helpers/recommend-default-prometheus/an-empty-metricprovider-still-queries-prometheus.yaml @@ -0,0 +1,16 @@ +# ceil(4 * 1.6/0.8) = 8, the per-replica ratio the Custom path skips. +"name": "an-empty-metricprovider-still-queries-prometheus" +"description": "an empty metricProvider still queries Prometheus" +"inputs": + "sample": "1.6" + "readyReplicas": 4 + "specReplicas": 4 + "minReplicas": 1 + "maxReplicas": 20 + "metrics": + - "name": "kv-cache" + "query": "avg(kv)" + "target": "0.8" +"expected": + "replicas": 8 + "path": "/api/v1/query" diff --git a/internal/controller/autoscaling/testdata/helpers/scale-guard/caps-a-burst-scale-down.yaml b/internal/controller/autoscaling/testdata/helpers/scale-guard/caps-a-burst-scale-down.yaml new file mode 100644 index 0000000..b03f424 --- /dev/null +++ b/internal/controller/autoscaling/testdata/helpers/scale-guard/caps-a-burst-scale-down.yaml @@ -0,0 +1,8 @@ +"name": "caps-a-burst-scale-down" +"description": "caps a burst scale-down" +"inputs": + "current": 10 + "desired": 1 + "maxStep": 2 +"expected": + "want": 8 diff --git a/internal/controller/autoscaling/testdata/helpers/scale-guard/leaves-a-no-op-untouched.yaml b/internal/controller/autoscaling/testdata/helpers/scale-guard/leaves-a-no-op-untouched.yaml new file mode 100644 index 0000000..f5c8774 --- /dev/null +++ b/internal/controller/autoscaling/testdata/helpers/scale-guard/leaves-a-no-op-untouched.yaml @@ -0,0 +1,8 @@ +"name": "leaves-a-no-op-untouched" +"description": "leaves a no-op untouched" +"inputs": + "current": 5 + "desired": 5 + "maxStep": 1 +"expected": + "want": 5 diff --git a/internal/controller/autoscaling/testdata/helpers/scale-guard/leaves-scale-up-untouched.yaml b/internal/controller/autoscaling/testdata/helpers/scale-guard/leaves-scale-up-untouched.yaml new file mode 100644 index 0000000..39eef98 --- /dev/null +++ b/internal/controller/autoscaling/testdata/helpers/scale-guard/leaves-scale-up-untouched.yaml @@ -0,0 +1,8 @@ +"name": "leaves-scale-up-untouched" +"description": "leaves scale-up untouched" +"inputs": + "current": 2 + "desired": 10 + "maxStep": 1 +"expected": + "want": 10 diff --git a/internal/controller/autoscaling/testdata/helpers/scale-guard/passes-through-a-step-exactly-at-the-cap.yaml b/internal/controller/autoscaling/testdata/helpers/scale-guard/passes-through-a-step-exactly-at-the-cap.yaml new file mode 100644 index 0000000..72d38df --- /dev/null +++ b/internal/controller/autoscaling/testdata/helpers/scale-guard/passes-through-a-step-exactly-at-the-cap.yaml @@ -0,0 +1,8 @@ +"name": "passes-through-a-step-exactly-at-the-cap" +"description": "passes through a step exactly at the cap" +"inputs": + "current": 10 + "desired": 8 + "maxStep": 2 +"expected": + "want": 8 diff --git a/internal/controller/autoscaling/testdata/helpers/scale-guard/passes-through-a-step-within-the-cap.yaml b/internal/controller/autoscaling/testdata/helpers/scale-guard/passes-through-a-step-within-the-cap.yaml new file mode 100644 index 0000000..e6123bd --- /dev/null +++ b/internal/controller/autoscaling/testdata/helpers/scale-guard/passes-through-a-step-within-the-cap.yaml @@ -0,0 +1,8 @@ +"name": "passes-through-a-step-within-the-cap" +"description": "passes through a step within the cap" +"inputs": + "current": 10 + "desired": 9 + "maxStep": 2 +"expected": + "want": 9 diff --git a/internal/controller/autoscaling/testdata/helpers/scale-guard/unlimited-when-maxstep-is-0.yaml b/internal/controller/autoscaling/testdata/helpers/scale-guard/unlimited-when-maxstep-is-0.yaml new file mode 100644 index 0000000..a1aa396 --- /dev/null +++ b/internal/controller/autoscaling/testdata/helpers/scale-guard/unlimited-when-maxstep-is-0.yaml @@ -0,0 +1,8 @@ +"name": "unlimited-when-maxstep-is-0" +"description": "unlimited when maxStep is 0" +"inputs": + "current": 10 + "desired": 1 + "maxStep": 0 +"expected": + "want": 1 diff --git a/internal/controller/autoscaling/testdata/helpers/stabilization/peak-holds-inside-the-window-then-expires.yaml b/internal/controller/autoscaling/testdata/helpers/stabilization/peak-holds-inside-the-window-then-expires.yaml new file mode 100644 index 0000000..472901c --- /dev/null +++ b/internal/controller/autoscaling/testdata/helpers/stabilization/peak-holds-inside-the-window-then-expires.yaml @@ -0,0 +1,19 @@ +# First recommendation sticks, a dip inside the window holds the peak, the peak expires after the window, and a scale-up is immediate. +"name": "peak-holds-inside-the-window-then-expires" +"description": "peak holds inside the window then expires" +"inputs": + "window": "60s" + "ops": + - "at": "0s" + "recommend": 5 + "want": 5 + - "at": "10s" + "recommend": 1 + "want": 5 + - "at": "61s" + "recommend": 1 + "want": 1 + - "at": "61s" + "recommend": 9 + "want": 9 +"expected": {} diff --git a/internal/controller/autoscaling/testdata/helpers/stabilization/pin-after-a-scale-down-paces-the-next-step.yaml b/internal/controller/autoscaling/testdata/helpers/stabilization/pin-after-a-scale-down-paces-the-next-step.yaml new file mode 100644 index 0000000..d6a4768 --- /dev/null +++ b/internal/controller/autoscaling/testdata/helpers/stabilization/pin-after-a-scale-down-paces-the-next-step.yaml @@ -0,0 +1,32 @@ +# Pinning the written count after a scale-down paces the next one a full window later. +"name": "pin-after-a-scale-down-paces-the-next-step" +"description": "pin after a scale-down paces the next step" +"inputs": + "window": "300s" + "ops": + - "at": "0s" + "recommend": 8 + "want": 8 + - "at": "301s" + "recommend": 1 + "want": 1 + - "at": "301s" + "holdAt": 6 + - "at": "311s" + "recommend": 1 + "want": 6 + - "at": "400s" + "recommend": 1 + "want": 6 + - "at": "600s" + "recommend": 1 + "want": 6 + - "at": "602s" + "recommend": 1 + "want": 1 + - "at": "700s" + "holdAt": 6 + - "at": "710s" + "recommend": 9 + "want": 9 +"expected": {} diff --git a/internal/controller/autoscaling/testdata/helpers/stabilization/window-0-makes-the-pin-a-no-op.yaml b/internal/controller/autoscaling/testdata/helpers/stabilization/window-0-makes-the-pin-a-no-op.yaml new file mode 100644 index 0000000..2354b7e --- /dev/null +++ b/internal/controller/autoscaling/testdata/helpers/stabilization/window-0-makes-the-pin-a-no-op.yaml @@ -0,0 +1,12 @@ +# With stabilization off there is nothing to pace with, so the pin is a no-op. +"name": "window-0-makes-the-pin-a-no-op" +"description": "window 0 makes the pin a no-op" +"inputs": + "window": "0s" + "ops": + - "at": "0s" + "holdAt": 6 + - "at": "0s" + "recommend": 1 + "want": 1 +"expected": {} diff --git a/internal/controller/serving/llmservice_cases_test.go b/internal/controller/serving/llmservice_cases_test.go new file mode 100644 index 0000000..da7988c --- /dev/null +++ b/internal/controller/serving/llmservice_cases_test.go @@ -0,0 +1,901 @@ +/* +Copyright 2026. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package serving + +import ( + "encoding/json" + "errors" + "fmt" + "maps" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + appsv1 "k8s.io/api/apps/v1" + apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/tools/events" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + + servingv1alpha1 "github.com/modelsphere/llm-operator/api/serving/v1alpha1" + "github.com/modelsphere/llm-operator/internal/helm" + "github.com/modelsphere/llm-operator/internal/testutil/cases" +) + +// serviceCase is one LLMService controller spec. HistoryLimit defaults to 10. +// PendingTimeout zero (omitted) leaves the controller default. +type serviceCase struct { + Name string `json:"name,omitempty"` + Description string `json:"description,omitempty"` + HistoryLimit int `json:"historyLimit,omitempty"` + PendingTimeout *cases.Duration `json:"pendingTimeout,omitempty"` + Steps []serviceStep `json:"steps"` +} + +type serviceStep struct { + Action string `json:"action"` + FreshNamespace bool `json:"freshNamespace,omitempty"` + Name string `json:"name,omitempty"` + Version string `json:"version,omitempty"` + Annotations map[string]string `json:"annotations,omitempty"` + Finalizers []string `json:"finalizers,omitempty"` + ChartVersion *string `json:"chartVersion,omitempty"` + Suspend *bool `json:"suspend,omitempty"` + Layers []layerSpec `json:"layers,omitempty"` + Release *releaseSpec `json:"release,omitempty"` + Entries []releaseSpec `json:"entries,omitempty"` + Put string `json:"put,omitempty"` + Versions []string `json:"versions,omitempty"` + ListErr *string `json:"listErr,omitempty"` + InstallErr *string `json:"installErr,omitempty"` + UpgradeErr *string `json:"upgradeErr,omitempty"` + GetErr *string `json:"getErr,omitempty"` + KeepFailed *bool `json:"keepFailed,omitempty"` + Repos []string `json:"repos,omitempty"` + Status *statusSpec `json:"status,omitempty"` + Capture []string `json:"capture,omitempty"` + Expect *expectSpec `json:"expect,omitempty"` +} + +type layerSpec struct { + Name string `json:"name"` + Values string `json:"values,omitempty"` +} + +type releaseSpec struct { + Revision int `json:"revision,omitempty"` + Status string `json:"status,omitempty"` + ChartName string `json:"chartName,omitempty"` + ChartVersion string `json:"chartVersion,omitempty"` + Config map[string]any `json:"config,omitempty"` + Labels map[string]string `json:"labels,omitempty"` + Owned bool `json:"owned,omitempty"` + Slot bool `json:"slot,omitempty"` + Updated *cases.Duration `json:"updated,omitempty"` +} + +type statusSpec struct { + Phase string `json:"phase,omitempty"` + Conditions []conditionSpec `json:"conditions,omitempty"` +} + +type conditionSpec struct { + Type string `json:"type"` + Status string `json:"status,omitempty"` + Reason string `json:"reason,omitempty"` + Message string `json:"message,omitempty"` + ObservedGeneration string `json:"observedGeneration,omitempty"` + Absent bool `json:"absent,omitempty"` +} + +type expectSpec struct { + Mid *expectSpec `json:"mid,omitempty"` + Error *bool `json:"error,omitempty"` + ErrorContains string `json:"errorContains,omitempty"` + Installs *int `json:"installs,omitempty"` + Upgrades *int `json:"upgrades,omitempty"` + Gets *int `json:"gets,omitempty"` + Uninstalls *int `json:"uninstalls,omitempty"` + Rollbacks *int `json:"rollbacks,omitempty"` + LastRollback *int `json:"lastRollback,omitempty"` + HelmTouched *int `json:"helmTouched,omitempty"` + ListCalls *int `json:"listCalls,omitempty"` + LastForce *bool `json:"lastForce,omitempty"` + GetsUnchanged *bool `json:"getsUnchanged,omitempty"` + UpgradesUnchanged *bool `json:"upgradesUnchanged,omitempty"` + Phase string `json:"phase,omitempty"` + ChartVersion string `json:"chartVersion,omitempty"` + ReResolve *string `json:"reResolve,omitempty"` + AppliedHash *hashExpect `json:"appliedHash,omitempty"` + HelmRevision *int `json:"helmRevision,omitempty"` + ObservedGeneration string `json:"observedGeneration,omitempty"` + Generation *int64 `json:"generation,omitempty"` + History *historyExpect `json:"history,omitempty"` + Conditions []conditionSpec `json:"conditions,omitempty"` + ReleaseUID string `json:"releaseUID,omitempty"` + Release *releaseExpect `json:"release,omitempty"` + NotFound *bool `json:"notFound,omitempty"` + Message *messageExpect `json:"message,omitempty"` + DistinctRevisions *int `json:"distinctRevisions,omitempty"` + Snapshots []snapshotExpect `json:"snapshots,omitempty"` + Requeue *requeueExpect `json:"requeue,omitempty"` + EventContains string `json:"eventContains,omitempty"` + Finalizer string `json:"finalizer,omitempty"` + PlantedRevisionUsed *bool `json:"plantedRevisionUsed,omitempty"` + CapturedRevisionGone *bool `json:"capturedRevisionGone,omitempty"` + LatestRevisionExists *bool `json:"latestRevisionExists,omitempty"` +} + +type hashExpect struct { + Prefix string `json:"prefix,omitempty"` + Empty *bool `json:"empty,omitempty"` + Captured *bool `json:"captured,omitempty"` +} + +type historyExpect struct { + Length *int `json:"length,omitempty"` + Entries []historyEntryExpect `json:"entries,omitempty"` +} + +type historyEntryExpect struct { + Index int `json:"index"` + Action string `json:"action,omitempty"` + Revision *int `json:"revision,omitempty"` + AnnotationsPresent []string `json:"annotationsPresent,omitempty"` + AnnotationsAbsent []string `json:"annotationsAbsent,omitempty"` + ReResolved *bool `json:"reResolved,omitempty"` + ForceConflicts *bool `json:"forceConflicts,omitempty"` + HashEqualsApplied bool `json:"hashEqualsApplied,omitempty"` + HashEqualsIndex *int `json:"hashEqualsIndex,omitempty"` +} + +type releaseExpect struct { + Exists *bool `json:"exists,omitempty"` + Status string `json:"status,omitempty"` + Labels map[string]string `json:"labels,omitempty"` + LabelsAbsent []string `json:"labelsAbsent,omitempty"` +} + +type messageExpect struct { + Contains []string `json:"contains,omitempty"` + Equals string `json:"equals,omitempty"` + NotContains []string `json:"notContains,omitempty"` +} + +type snapshotExpect struct { + Index int `json:"index"` + Annotations map[string]string `json:"annotations,omitempty"` + AnnotationsAbsent []string `json:"annotationsAbsent,omitempty"` + ChartName string `json:"chartName,omitempty"` + Labels map[string]string `json:"labels,omitempty"` + ControllerOwner *bool `json:"controllerOwner,omitempty"` +} + +type requeueExpect struct { + Exact *cases.Duration `json:"exact,omitempty"` + AtLeast *cases.Duration `json:"atLeast,omitempty"` + Above *cases.Duration `json:"above,omitempty"` + AtMost *cases.Duration `json:"atMost,omitempty"` +} + +type helmSnap struct { + gets int + upgrades int +} + +type caseRun struct { + tc serviceCase + ns string + name string + h *fakeHelm + l *versionList + r *LLMServiceReconciler + events chan string + capturedRevision string + appliedHash string + observedGeneration int64 + planted string +} + +func runServiceCase(tc serviceCase) { + rec := events.NewFakeRecorder(32) + h := &fakeHelm{} + l := &versionList{versions: []string{ver086}} + limit := tc.HistoryLimit + if limit == 0 { + limit = 10 + } + r := newReconciler(h, l, limit) + r.Recorder = rec + if tc.PendingTimeout != nil { + r.PendingTimeout = tc.PendingTimeout.Std() + } + run := &caseRun{ + tc: tc, + ns: takeNS(), + name: "qwen", + h: h, + l: l, + r: r, + events: rec.Events, + } + Expect(tc.Steps).NotTo(BeEmpty(), tc.Name) + for i, step := range tc.Steps { + run.step(i, step) + } +} + +func (c *caseRun) step(i int, step serviceStep) { + if step.FreshNamespace { + c.ns = takeNS() + } + if step.Name != "" { + c.name = step.Name + } + if step.Action == "" { + Fail(fmt.Sprintf("%s: step %d has no action", c.tc.Name, i+1)) + } + if stepRequiresExpect(step.Action) && step.Expect == nil { + Fail(fmt.Sprintf("%s: step %d (%s) missing expect", c.tc.Name, i+1, step.Action)) + } + switch step.Action { + case "create": + c.create(step) + case "reconcile": + c.reconcile(step.Expect) + case "update": + c.update(step) + case "delete": + c.deleteObj() + case "putRelease": + c.putRelease(step) + case "setVersions": + c.setVersions(step) + case "setHelm": + c.setHelm(step) + case "setAllowed": + // An omitted list is nil, and a nil allowlist permits every repository. + if step.Repos == nil { + Fail(fmt.Sprintf("%s: step %d setAllowed requires repos", c.tc.Name, i+1)) + } + c.r.AllowedChartRepos = append([]string(nil), step.Repos...) + case "deleteRelease": + delete(c.h.releases, c.h.slot(c.ns, c.name)) + case "setHistory": + c.setHistory(step) + case "plantStale": + c.plantStale() + case "reconcileHeld": + c.reconcileHeld(step.Expect) + case "capture": + c.capture(step) + case "setStatus": + c.setStatus(step) + case "setStatusReleaseUID": + c.setStatusReleaseUID() + default: + Fail(fmt.Sprintf("%s: step %d unknown action %q", c.tc.Name, i+1, step.Action)) + } + if step.Expect != nil && step.Action != "reconcile" && step.Action != "reconcileHeld" { + c.check(step.Expect, reconcile.Result{}, nil, helmSnap{}, false) + } +} + +func stepRequiresExpect(action string) bool { + return action == "reconcile" || action == "reconcileHeld" +} + +func (c *caseRun) create(step serviceStep) { + Expect(step.Version).NotTo(BeEmpty()) + obj := newService(c.ns, c.name, step.Version) + maps.Copy(obj.Annotations, step.Annotations) + if step.Finalizers != nil { + obj.Finalizers = append([]string(nil), step.Finalizers...) + } + Expect(k8sClient.Create(ctx, obj)).To(Succeed()) +} + +func (c *caseRun) update(step serviceStep) { + obj := fetch(c.ns, c.name) + if step.Annotations != nil { + if obj.Annotations == nil { + obj.Annotations = map[string]string{} + } + maps.Copy(obj.Annotations, step.Annotations) + } + if step.ChartVersion != nil { + obj.Spec.Chart.Version = *step.ChartVersion + } + if step.Suspend != nil { + obj.Spec.Suspend = *step.Suspend + } + if step.Layers != nil { + obj.Spec.Layers = layersFrom(step.Layers) + } + Expect(k8sClient.Update(ctx, obj)).To(Succeed()) +} + +func layersFrom(in []layerSpec) []servingv1alpha1.Layer { + out := make([]servingv1alpha1.Layer, len(in)) + for i, l := range in { + out[i] = servingv1alpha1.Layer{Name: l.Name} + if l.Values != "" { + out[i].Values = &apiextensionsv1.JSON{Raw: []byte(l.Values)} + } + } + return out +} + +func (c *caseRun) deleteObj() { + Expect(k8sClient.Delete(ctx, fetch(c.ns, c.name))).To(Succeed()) +} + +func (c *caseRun) reconcile(exp *expectSpec) { + before := helmSnap{gets: c.h.gets, upgrades: c.h.upgrades} + result, err := doReconcile(c.r, c.ns, c.name) + c.check(exp, result, err, before, true) +} + +// reconcileHeld runs one reconcile that blocks inside helm, checks the +// mid-apply status, then lets the install finish. +func (c *caseRun) reconcileHeld(exp *expectSpec) { + Expect(exp).NotTo(BeNil()) + Expect(exp.Mid).NotTo(BeNil()) + c.h.hold = make(chan struct{}) + c.h.entered = make(chan struct{}, 1) + released := false + release := func() { + if !released { + close(c.h.hold) + released = true + } + } + defer release() + + done := make(chan error, 1) + go func() { + defer GinkgoRecover() + _, recErr := doReconcile(c.r, c.ns, c.name) + done <- recErr + }() + Eventually(c.h.entered).WithTimeout(5 * time.Second).Should(Receive()) + c.check(exp.Mid, reconcile.Result{}, nil, helmSnap{}, false) + release() + var recErr error + Eventually(done).WithTimeout(5 * time.Second).Should(Receive(&recErr)) + Expect(recErr).NotTo(HaveOccurred()) + c.check(exp, reconcile.Result{}, recErr, helmSnap{}, true) +} + +func (c *caseRun) putRelease(step serviceStep) { + Expect(step.Release).NotTo(BeNil()) + rel := c.releaseFrom(*step.Release) + c.h.put(c.ns, c.name, &rel) +} + +func (c *caseRun) setHistory(step serviceStep) { + Expect(step.Entries).NotTo(BeEmpty()) + hist := make([]helm.Release, len(step.Entries)) + for i, e := range step.Entries { + hist[i] = c.releaseFrom(e) + } + c.h.histories = map[string][]helm.Release{c.h.slot(c.ns, c.name): hist} + if step.Put == "" { + return + } + Expect(step.Put).To(Equal("last")) + last := hist[len(hist)-1] + c.h.put(c.ns, c.name, &last) +} + +func (c *caseRun) releaseFrom(in releaseSpec) helm.Release { + if in.Slot { + cur := c.h.releases[c.h.slot(c.ns, c.name)] + Expect(cur).NotTo(BeNil()) + return *cur + } + if in.Owned { + updated := time.Now() + if in.Updated != nil { + updated = time.Now().Add(in.Updated.Std()) + } + return *ownedRel(fetch(c.ns, c.name), in.Revision, in.Status, updated) + } + rel := helm.Release{ + Revision: in.Revision, + Status: in.Status, + ChartName: in.ChartName, + ChartVersion: in.ChartVersion, + Config: in.Config, + Labels: in.Labels, + } + if in.Updated != nil { + rel.Updated = time.Now().Add(in.Updated.Std()) + } + return rel +} + +func (c *caseRun) setVersions(step serviceStep) { + if step.Versions != nil { + c.l.versions = append([]string(nil), step.Versions...) + } + if step.ListErr != nil { + c.l.err = errFrom(*step.ListErr) + } +} + +func (c *caseRun) setHelm(step serviceStep) { + if step.InstallErr != nil { + c.h.installErr = errFrom(*step.InstallErr) + } + if step.UpgradeErr != nil { + c.h.upgradeErr = errFrom(*step.UpgradeErr) + } + if step.GetErr != nil { + c.h.getErr = errFrom(*step.GetErr) + } + if step.KeepFailed != nil { + c.h.keepFailed = *step.KeepFailed + } +} + +func errFrom(msg string) error { + if msg == "" { + return nil + } + return errors.New(msg) +} + +func (c *caseRun) setStatus(step serviceStep) { + Expect(step.Status).NotTo(BeNil()) + obj := fetch(c.ns, c.name) + if step.Status.Phase != "" { + obj.Status.Phase = servingv1alpha1.Phase(step.Status.Phase) + } + for _, cond := range step.Status.Conditions { + meta.SetStatusCondition(&obj.Status.Conditions, conditionFrom(obj, cond)) + } + Expect(k8sClient.Status().Update(ctx, obj)).To(Succeed()) +} + +func conditionFrom(obj *servingv1alpha1.LLMService, cond conditionSpec) metav1.Condition { + out := metav1.Condition{ + Type: cond.Type, + Status: metav1.ConditionStatus(cond.Status), + Reason: cond.Reason, + Message: cond.Message, + } + switch cond.ObservedGeneration { + case "": + case "current": + out.ObservedGeneration = obj.Generation + default: + Fail("unsupported observedGeneration " + cond.ObservedGeneration) + } + return out +} + +func (c *caseRun) setStatusReleaseUID() { + obj := fetch(c.ns, c.name) + obj.Status.ReleaseUID = string(obj.UID) + Expect(k8sClient.Status().Update(ctx, obj)).To(Succeed()) +} + +func (c *caseRun) capture(step serviceStep) { + Expect(step.Capture).NotTo(BeEmpty()) + obj := fetch(c.ns, c.name) + for _, what := range step.Capture { + switch what { + case "observedGeneration": + c.observedGeneration = obj.Status.ObservedGeneration + case "appliedHash": + c.appliedHash = obj.Status.AppliedHash + case "revision": + Expect(obj.Status.History).NotTo(BeEmpty()) + c.capturedRevision = obj.Status.History[0].ControllerRevision + default: + Fail("unknown capture " + what) + } + } +} + +// plantStale leaves a ControllerRevision under the name the next install +// reuses, owned by the service, so the reconcile replaces that snapshot. +func (c *caseRun) plantStale() { + obj := fetch(c.ns, c.name) + values, err := mergeLayers(obj.Spec.Layers) + Expect(err).NotTo(HaveOccurred()) + hash, err := appliedHash(chartName, ver086, values) + Expect(err).NotTo(HaveOccurred()) + stale := &appsv1.ControllerRevision{ + ObjectMeta: metav1.ObjectMeta{ + Name: controllerRevisionName(obj.Name, hash, 1), + Namespace: c.ns, + }, + Data: runtime.RawExtension{Raw: []byte(`{"annotations":{"swiss.modelsphere.dev/action":"old"}}`)}, + Revision: 1, + } + Expect(controllerutil.SetControllerReference(obj, stale, k8sClient.Scheme())).To(Succeed()) + Expect(k8sClient.Create(ctx, stale)).To(Succeed()) + c.planted = stale.Name +} + +func (c *caseRun) check(exp *expectSpec, result reconcile.Result, err error, before helmSnap, reconciled bool) { + if exp == nil { + return + } + c.checkErr(exp, err, reconciled) + c.checkRequeue(exp.Requeue, result) + c.checkHelm(exp, before) + if exp.EventContains != "" { + Expect(c.events).To(Receive(ContainSubstring(exp.EventContains))) + } + if exp.NotFound != nil && *exp.NotFound { + c.checkMissing() + } + var got *servingv1alpha1.LLMService + if exp.wantsObject() { + got = fetch(c.ns, c.name) + c.checkObject(exp, got) + } + if exp.Release != nil { + c.checkRelease(exp.Release, got) + } +} + +func (c *caseRun) checkErr(exp *expectSpec, err error, reconciled bool) { + if exp.ErrorContains != "" { + Expect(err).To(MatchError(ContainSubstring(exp.ErrorContains))) + return + } + if exp.Error != nil { + if *exp.Error { + Expect(err).To(HaveOccurred()) + } else { + Expect(err).NotTo(HaveOccurred()) + } + return + } + if reconciled { + Expect(err).NotTo(HaveOccurred()) + } +} + +func (c *caseRun) checkRequeue(exp *requeueExpect, result reconcile.Result) { + if exp == nil { + return + } + if exp.Exact != nil { + Expect(result.RequeueAfter).To(Equal(exp.Exact.Std())) + } + if exp.AtLeast != nil { + Expect(result.RequeueAfter).To(BeNumerically(">=", exp.AtLeast.Std())) + } + if exp.Above != nil { + Expect(result.RequeueAfter).To(BeNumerically(">", exp.Above.Std())) + } + if exp.AtMost != nil { + Expect(result.RequeueAfter).To(BeNumerically("<=", exp.AtMost.Std())) + } +} + +func (c *caseRun) checkHelm(exp *expectSpec, before helmSnap) { + expectInt(c.h.installs, exp.Installs) + expectInt(c.h.upgrades, exp.Upgrades) + expectInt(c.h.gets, exp.Gets) + expectInt(c.h.uninstalls, exp.Uninstalls) + expectInt(c.h.rollbacks, exp.Rollbacks) + expectInt(c.h.lastRollback, exp.LastRollback) + expectInt(c.l.calls, exp.ListCalls) + if exp.HelmTouched != nil { + Expect(c.h.gets + c.h.installs + c.h.upgrades).To(Equal(*exp.HelmTouched)) + } + expectBool(c.h.lastForce, exp.LastForce) + if exp.GetsUnchanged != nil && *exp.GetsUnchanged { + Expect(c.h.gets).To(Equal(before.gets)) + } + if exp.UpgradesUnchanged != nil && *exp.UpgradesUnchanged { + Expect(c.h.upgrades).To(Equal(before.upgrades)) + } +} + +func expectInt(got int, want *int) { + if want != nil { + Expect(got).To(Equal(*want)) + } +} + +func expectBool(got bool, want *bool) { + if want == nil { + return + } + if *want { + Expect(got).To(BeTrue()) + } else { + Expect(got).To(BeFalse()) + } +} + +func (e *expectSpec) wantsObject() bool { + if e.Phase != "" || e.ChartVersion != "" || e.ReResolve != nil || e.AppliedHash != nil { + return true + } + if e.HelmRevision != nil || e.ObservedGeneration != "" || e.Generation != nil || e.History != nil { + return true + } + if len(e.Conditions) > 0 || e.ReleaseUID != "" || e.Message != nil || e.Finalizer != "" { + return true + } + if e.DistinctRevisions != nil || len(e.Snapshots) > 0 || e.PlantedRevisionUsed != nil { + return true + } + return e.CapturedRevisionGone != nil || e.LatestRevisionExists != nil +} + +func (c *caseRun) checkObject(exp *expectSpec, got *servingv1alpha1.LLMService) { + if exp.Phase != "" { + Expect(got.Status.Phase).To(Equal(servingv1alpha1.Phase(exp.Phase))) + } + if exp.ChartVersion != "" { + Expect(got.Status.Chart.Version).To(Equal(exp.ChartVersion)) + } + if exp.ReResolve != nil { + Expect(got.Status.Chart.ReResolve).To(Equal(*exp.ReResolve)) + } + c.checkHash(exp.AppliedHash, got) + if exp.HelmRevision != nil { + Expect(got.Status.Helm.Revision).To(Equal(*exp.HelmRevision)) + } + c.checkGeneration(exp, got) + c.checkHistory(exp, got) + c.checkConditions(exp.Conditions, got) + c.checkReleaseUID(exp.ReleaseUID, got) + c.checkMessage(exp.Message, got) + c.checkFinalizer(exp.Finalizer, got) + c.checkRevisions(exp, got) +} + +func (c *caseRun) checkHash(exp *hashExpect, got *servingv1alpha1.LLMService) { + if exp == nil { + return + } + if exp.Empty != nil && *exp.Empty { + Expect(got.Status.AppliedHash).To(BeEmpty()) + } + if exp.Prefix != "" { + Expect(got.Status.AppliedHash).To(HavePrefix(exp.Prefix)) + } + if exp.Captured != nil && *exp.Captured { + Expect(got.Status.AppliedHash).To(Equal(c.appliedHash)) + } +} + +func (c *caseRun) checkGeneration(exp *expectSpec, got *servingv1alpha1.LLMService) { + if exp.Generation != nil { + Expect(got.Generation).To(Equal(*exp.Generation)) + } + switch exp.ObservedGeneration { + case "": + case "current": + Expect(got.Status.ObservedGeneration).To(Equal(got.Generation)) + case "captured": + Expect(got.Status.ObservedGeneration).To(Equal(c.observedGeneration)) + case "nonzero": + Expect(got.Status.ObservedGeneration).NotTo(BeZero()) + default: + Fail("unsupported observedGeneration " + exp.ObservedGeneration) + } +} + +func (c *caseRun) checkHistory(exp *expectSpec, got *servingv1alpha1.LLMService) { + if exp.History == nil { + return + } + if exp.History.Length != nil { + Expect(got.Status.History).To(HaveLen(*exp.History.Length)) + } + for _, e := range exp.History.Entries { + Expect(e.Index).To(BeNumerically(">=", 0)) + Expect(e.Index).To(BeNumerically("<", len(got.Status.History))) + entry := got.Status.History[e.Index] + if e.Action != "" { + Expect(entry.Action).To(Equal(e.Action)) + } + if e.Revision != nil { + Expect(entry.Revision).To(Equal(*e.Revision)) + } + for _, k := range e.AnnotationsPresent { + Expect(entry.Annotations).To(HaveKey(k)) + } + for _, k := range e.AnnotationsAbsent { + Expect(entry.Annotations).NotTo(HaveKey(k)) + } + expectBool(entry.ReResolved, e.ReResolved) + expectBool(entry.ForceConflicts, e.ForceConflicts) + if e.HashEqualsApplied { + Expect(entry.Hash).To(Equal(got.Status.AppliedHash)) + } + if e.HashEqualsIndex != nil { + Expect(entry.Hash).To(Equal(got.Status.History[*e.HashEqualsIndex].Hash)) + } + } + if exp.DistinctRevisions != nil { + names := map[string]struct{}{} + for _, entry := range got.Status.History { + names[entry.ControllerRevision] = struct{}{} + } + Expect(names).To(HaveLen(*exp.DistinctRevisions)) + } +} + +func (c *caseRun) checkConditions(conds []conditionSpec, got *servingv1alpha1.LLMService) { + for _, want := range conds { + gotCond := condition(got, want.Type) + if want.Absent { + Expect(gotCond).To(BeNil()) + continue + } + Expect(gotCond).NotTo(BeNil()) + if want.Status != "" { + Expect(gotCond.Status).To(Equal(metav1.ConditionStatus(want.Status))) + } + if want.Reason != "" { + Expect(gotCond.Reason).To(Equal(want.Reason)) + } + if want.Message != "" { + Expect(gotCond.Message).To(Equal(want.Message)) + } + switch want.ObservedGeneration { + case "": + case "current": + Expect(gotCond.ObservedGeneration).To(Equal(got.Generation)) + default: + Fail("unsupported observedGeneration " + want.ObservedGeneration) + } + } +} + +func (c *caseRun) checkReleaseUID(want string, got *servingv1alpha1.LLMService) { + switch want { + case "": + case "self": + Expect(got.Status.ReleaseUID).To(Equal(string(got.UID))) + case "empty": + Expect(got.Status.ReleaseUID).To(BeEmpty()) + default: + Fail("unsupported releaseUID " + want) + } +} + +func (c *caseRun) checkMessage(exp *messageExpect, got *servingv1alpha1.LLMService) { + if exp == nil { + return + } + if exp.Equals != "" { + Expect(got.Status.Message).To(Equal(exp.Equals)) + } + // An empty substring matches every message, so it is rejected on its own. + for _, sub := range exp.Contains { + if sub == "" { + Fail("message.contains entry is empty") + } + Expect(got.Status.Message).To(ContainSubstring(sub)) + } + for _, sub := range exp.NotContains { + if sub == "" { + Fail("message.notContains entry is empty") + } + Expect(got.Status.Message).NotTo(ContainSubstring(sub)) + } +} + +func (c *caseRun) checkFinalizer(want string, got *servingv1alpha1.LLMService) { + switch want { + case "": + case "present": + Expect(got.Finalizers).To(ContainElement(servingv1alpha1.Finalizer)) + case "absent": + Expect(got.Finalizers).NotTo(ContainElement(servingv1alpha1.Finalizer)) + default: + Fail("unsupported finalizer " + want) + } +} + +func (c *caseRun) checkRevisions(exp *expectSpec, got *servingv1alpha1.LLMService) { + if exp.PlantedRevisionUsed != nil && *exp.PlantedRevisionUsed { + Expect(got.Status.History).NotTo(BeEmpty()) + Expect(got.Status.History[0].ControllerRevision).To(Equal(c.planted)) + } + if exp.CapturedRevisionGone != nil && *exp.CapturedRevisionGone { + err := k8sClient.Get(ctx, types.NamespacedName{Namespace: c.ns, Name: c.capturedRevision}, &appsv1.ControllerRevision{}) + Expect(apierrors.IsNotFound(err)).To(BeTrue()) + } + if exp.LatestRevisionExists != nil && *exp.LatestRevisionExists { + Expect(got.Status.History).NotTo(BeEmpty()) + Expect(k8sClient.Get(ctx, types.NamespacedName{Namespace: c.ns, Name: got.Status.History[0].ControllerRevision}, &appsv1.ControllerRevision{})).To(Succeed()) + } + for _, sn := range exp.Snapshots { + Expect(sn.Index).To(BeNumerically("<", len(got.Status.History))) + c.checkSnapshot(got.Status.History[sn.Index].ControllerRevision, sn) + } +} + +func (c *caseRun) checkSnapshot(name string, sn snapshotExpect) { + var cr appsv1.ControllerRevision + Expect(k8sClient.Get(ctx, types.NamespacedName{Namespace: c.ns, Name: name}, &cr)).To(Succeed()) + if sn.ControllerOwner != nil && *sn.ControllerOwner { + Expect(cr.OwnerReferences).NotTo(BeEmpty()) + Expect(cr.OwnerReferences[0].Controller).NotTo(BeNil()) + Expect(*cr.OwnerReferences[0].Controller).To(BeTrue()) + } + for k, v := range sn.Labels { + Expect(cr.Labels).To(HaveKeyWithValue(k, v)) + } + var snap revisionSnapshot + Expect(json.Unmarshal(cr.Data.Raw, &snap)).To(Succeed()) + for k, v := range sn.Annotations { + Expect(snap.Annotations).To(HaveKeyWithValue(k, v)) + } + for _, k := range sn.AnnotationsAbsent { + Expect(snap.Annotations).NotTo(HaveKey(k)) + } + if sn.ChartName != "" { + Expect(snap.Spec.Chart.Name).To(Equal(sn.ChartName)) + } +} + +func (c *caseRun) checkRelease(exp *releaseExpect, got *servingv1alpha1.LLMService) { + slot := c.h.slot(c.ns, c.name) + rel := c.h.releases[slot] + if exp.Exists != nil { + if *exp.Exists { + Expect(c.h.releases).To(HaveKey(slot)) + } else { + Expect(c.h.releases).NotTo(HaveKey(slot)) + } + } + if exp.Status != "" { + Expect(rel).NotTo(BeNil()) + Expect(rel.Status).To(Equal(exp.Status)) + } + if len(exp.Labels) == 0 && len(exp.LabelsAbsent) == 0 { + return + } + Expect(rel).NotTo(BeNil()) + for k, v := range exp.Labels { + if v == "self" { + if got == nil { + got = fetch(c.ns, c.name) + } + v = string(got.UID) + } + Expect(rel.Labels).To(HaveKeyWithValue(k, v)) + } + for _, k := range exp.LabelsAbsent { + Expect(rel.Labels).NotTo(HaveKey(k)) + } +} + +func (c *caseRun) checkMissing() { + err := k8sClient.Get(ctx, types.NamespacedName{Namespace: c.ns, Name: c.name}, &servingv1alpha1.LLMService{}) + Expect(apierrors.IsNotFound(err)).To(BeTrue()) +} diff --git a/internal/controller/serving/llmservice_controller_test.go b/internal/controller/serving/llmservice_controller_test.go index b1eb8fe..c69e874 100644 --- a/internal/controller/serving/llmservice_controller_test.go +++ b/internal/controller/serving/llmservice_controller_test.go @@ -19,28 +19,23 @@ package serving import ( "cmp" "context" - "encoding/json" - "errors" "fmt" "slices" "time" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" - appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" - apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/tools/events" - "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" "sigs.k8s.io/controller-runtime/pkg/reconcile" servingv1alpha1 "github.com/modelsphere/llm-operator/api/serving/v1alpha1" "github.com/modelsphere/llm-operator/internal/helm" + "github.com/modelsphere/llm-operator/internal/testutil/cases" ) const ( @@ -270,917 +265,12 @@ func condition(obj *servingv1alpha1.LLMService, typ string) *metav1.Condition { } var _ = Describe("LLMService controller", func() { - It("installs and records a ControllerRevision", func() { - ns := takeNS() - h := &fakeHelm{} - l := &versionList{versions: []string{ver086}} - obj := newService(ns, "qwen", ">=0.8.0") - Expect(k8sClient.Create(ctx, obj)).To(Succeed()) - - _, err := doReconcile(newReconciler(h, l, 10), ns, "qwen") - Expect(err).NotTo(HaveOccurred()) - Expect(h.installs).To(Equal(1)) - Expect(h.upgrades).To(Equal(0)) - - got := fetch(ns, "qwen") - Expect(got.Finalizers).To(ContainElement(servingv1alpha1.Finalizer)) - Expect(got.Status.Phase).To(Equal(servingv1alpha1.PhaseApplied)) - Expect(got.Status.Chart.Version).To(Equal(ver086)) - Expect(got.Status.AppliedHash).To(HavePrefix("sha256:")) - Expect(got.Status.Helm.Revision).To(Equal(1)) - Expect(got.Status.ObservedGeneration).To(Equal(got.Generation)) - Expect(got.Status.History).To(HaveLen(1)) - Expect(got.Status.History[0].Action).To(Equal("install")) - Expect(got.Status.History[0].Annotations).To(HaveKey("swiss.modelsphere.dev/note")) - Expect(got.Status.History[0].Annotations).NotTo(HaveKey("unrelated")) - Expect(condition(got, servingv1alpha1.ConditionApplied).Status).To(Equal(metav1.ConditionTrue)) - Expect(condition(got, servingv1alpha1.ConditionApplied).Reason).To(Equal(servingv1alpha1.ReasonInstalled)) - Expect(got.Status.ReleaseUID).To(Equal(string(got.UID))) - Expect(h.releases[h.slot(ns, "qwen")].Labels).To(HaveKeyWithValue(llmserviceUIDLabel, string(got.UID))) - Expect(h.releases[h.slot(ns, "qwen")].Labels).To(HaveKeyWithValue(llmserviceLabel, "qwen")) - - var cr appsv1.ControllerRevision - Expect(k8sClient.Get(ctx, types.NamespacedName{Namespace: ns, Name: got.Status.History[0].ControllerRevision}, &cr)).To(Succeed()) - Expect(cr.Labels).To(HaveKeyWithValue(llmserviceLabel, "qwen")) - Expect(cr.OwnerReferences).NotTo(BeEmpty()) - Expect(cr.OwnerReferences[0].Controller).NotTo(BeNil()) - Expect(*cr.OwnerReferences[0].Controller).To(BeTrue()) - var snap revisionSnapshot - Expect(json.Unmarshal(cr.Data.Raw, &snap)).To(Succeed()) - Expect(snap.Annotations).To(HaveKeyWithValue("swiss.modelsphere.dev/action", "install")) - Expect(snap.Annotations).NotTo(HaveKey("unrelated")) - Expect(snap.Spec.Chart.Name).To(Equal("sglang")) - }) - - It("does not call helm when the spec is unchanged", func() { - ns := takeNS() - h := &fakeHelm{} - l := &versionList{versions: []string{ver086}} - r := newReconciler(h, l, 10) - Expect(k8sClient.Create(ctx, newService(ns, "qwen", ver086))).To(Succeed()) - _, err := doReconcile(r, ns, "qwen") - Expect(err).NotTo(HaveOccurred()) - - got := fetch(ns, "qwen") - got.Annotations["swiss.modelsphere.dev/note"] = "later" - Expect(k8sClient.Update(ctx, got)).To(Succeed()) - _, err = doReconcile(r, ns, "qwen") - Expect(err).NotTo(HaveOccurred()) - Expect(h.installs).To(Equal(1)) - Expect(h.upgrades).To(Equal(0)) - Expect(h.gets).To(Equal(1)) - }) - - It("upgrades and bounds history", func() { - ns := takeNS() - h := &fakeHelm{} - l := &versionList{versions: []string{ver086}} - r := newReconciler(h, l, 2) - Expect(k8sClient.Create(ctx, newService(ns, "qwen", ver086))).To(Succeed()) - _, err := doReconcile(r, ns, "qwen") - Expect(err).NotTo(HaveOccurred()) - first := fetch(ns, "qwen").Status.History[0].ControllerRevision - - for _, body := range []string{`{"replicaCount":2}`, `{"replicaCount":3}`} { - got := fetch(ns, "qwen") - got.Spec.Layers = []servingv1alpha1.Layer{layer(body)} - Expect(k8sClient.Update(ctx, got)).To(Succeed()) - _, err = doReconcile(r, ns, "qwen") - Expect(err).NotTo(HaveOccurred()) - } - Expect(h.upgrades).To(Equal(2)) - got := fetch(ns, "qwen") - Expect(got.Status.History).To(HaveLen(2)) - Expect(got.Status.History[0].Revision).To(Equal(3)) - Expect(got.Status.History[1].Revision).To(Equal(2)) - Expect(got.Status.Helm.Revision).To(Equal(3)) - err = k8sClient.Get(ctx, types.NamespacedName{Namespace: ns, Name: first}, &appsv1.ControllerRevision{}) - Expect(apierrors.IsNotFound(err)).To(BeTrue()) - Expect(k8sClient.Get(ctx, types.NamespacedName{Namespace: ns, Name: got.Status.History[0].ControllerRevision}, &appsv1.ControllerRevision{})).To(Succeed()) - }) - - It("keeps a resolved version while the range allows it", func() { - ns := takeNS() - h := &fakeHelm{} - l := &versionList{versions: []string{ver086}} - r := newReconciler(h, l, 10) - Expect(k8sClient.Create(ctx, newService(ns, "qwen", ">=0.8.0"))).To(Succeed()) - _, err := doReconcile(r, ns, "qwen") - Expect(err).NotTo(HaveOccurred()) - Expect(fetch(ns, "qwen").Status.Chart.Version).To(Equal(ver086)) - Expect(l.calls).To(Equal(1)) - - l.versions = []string{ver086, "0.9.0"} - _, err = doReconcile(r, ns, "qwen") - Expect(err).NotTo(HaveOccurred()) - Expect(fetch(ns, "qwen").Status.Chart.Version).To(Equal(ver086)) - Expect(l.calls).To(Equal(1)) - Expect(h.upgrades).To(Equal(0)) - - got := fetch(ns, "qwen") - got.Spec.Chart.Version = ">=0.9.0" - Expect(k8sClient.Update(ctx, got)).To(Succeed()) - _, err = doReconcile(r, ns, "qwen") - Expect(err).NotTo(HaveOccurred()) - Expect(fetch(ns, "qwen").Status.Chart.Version).To(Equal("0.9.0")) - Expect(l.calls).To(Equal(2)) - Expect(h.upgrades).To(Equal(1)) - }) - - It("re-resolves once per new re-resolve value", func() { - ns := takeNS() - h := &fakeHelm{} - l := &versionList{versions: []string{ver086}} - r := newReconciler(h, l, 10) - Expect(k8sClient.Create(ctx, newService(ns, "qwen", ">=0.8.0"))).To(Succeed()) - _, err := doReconcile(r, ns, "qwen") - Expect(err).NotTo(HaveOccurred()) - reResolve := func(v string) { - got := fetch(ns, "qwen") - got.Annotations[servingv1alpha1.AnnotationReResolve] = v - Expect(k8sClient.Update(ctx, got)).To(Succeed()) - } - - // Nothing newer: honored without an upgrade. - reResolve("1") - _, err = doReconcile(r, ns, "qwen") - Expect(err).NotTo(HaveOccurred()) - got := fetch(ns, "qwen") - Expect(got.Status.Chart.Version).To(Equal(ver086)) - Expect(got.Status.Chart.ReResolve).To(Equal("1")) - Expect(l.calls).To(Equal(2)) - Expect(h.upgrades).To(Equal(0)) - - // A newer chart under an honored value does not move the release. - l.versions = []string{ver086, "0.9.0"} - _, err = doReconcile(r, ns, "qwen") - Expect(err).NotTo(HaveOccurred()) - Expect(fetch(ns, "qwen").Status.Chart.Version).To(Equal(ver086)) - Expect(l.calls).To(Equal(2)) - - // A failed upgrade leaves the value unhonored, so the retry re-resolves. - h.upgradeErr = errors.New("boom") - reResolve("2") - _, err = doReconcile(r, ns, "qwen") - Expect(err).To(HaveOccurred()) - got = fetch(ns, "qwen") - Expect(got.Status.Chart.Version).To(Equal(ver086)) - Expect(got.Status.Chart.ReResolve).To(Equal("1")) - - h.upgradeErr = nil - _, err = doReconcile(r, ns, "qwen") - Expect(err).NotTo(HaveOccurred()) - got = fetch(ns, "qwen") - Expect(got.Status.Chart.Version).To(Equal("0.9.0")) - Expect(got.Status.Chart.ReResolve).To(Equal("2")) - Expect(got.Status.History[0].ReResolved).To(BeTrue()) - Expect(h.upgrades).To(Equal(2)) - - l.versions = []string{ver086, "0.9.0", "0.9.1"} - _, err = doReconcile(r, ns, "qwen") - Expect(err).NotTo(HaveOccurred()) - Expect(fetch(ns, "qwen").Status.Chart.Version).To(Equal("0.9.0")) - Expect(h.upgrades).To(Equal(2)) - }) - - It("adopts a matching release and refuses drift", func() { - ns := takeNS() - h := &fakeHelm{} - h.put(ns, "qwen", &helm.Release{ - Revision: 7, Status: deployed, ChartName: chartName, ChartVersion: ver086, - Config: map[string]any{replicasKey: float64(1)}, - }) - l := &versionList{versions: []string{ver086}} - r := newReconciler(h, l, 10) - Expect(k8sClient.Create(ctx, newService(ns, "qwen", ">=0.8.0"))).To(Succeed()) - _, err := doReconcile(r, ns, "qwen") - Expect(err).NotTo(HaveOccurred()) - Expect(h.installs).To(Equal(0)) - Expect(h.upgrades).To(Equal(0)) - got := fetch(ns, "qwen") - Expect(got.Status.Phase).To(Equal(servingv1alpha1.PhaseApplied)) - Expect(condition(got, servingv1alpha1.ConditionAdopted).Status).To(Equal(metav1.ConditionTrue)) - Expect(condition(got, servingv1alpha1.ConditionAdopted).Reason).To(Equal(servingv1alpha1.ReasonAdopted)) - Expect(condition(got, servingv1alpha1.ConditionApplied).Reason).To(Equal(servingv1alpha1.ReasonAdopted)) - Expect(got.Status.History[0].Revision).To(Equal(7)) - Expect(got.Status.ReleaseUID).To(Equal(string(got.UID))) - - ns2 := takeNS() - h.put(ns2, "qwen", &helm.Release{ - Revision: 3, Status: deployed, ChartName: "other", ChartVersion: "0.1.0", - Config: map[string]any{replicasKey: float64(9)}, - }) - Expect(k8sClient.Create(ctx, newService(ns2, "qwen", ">=0.8.0"))).To(Succeed()) - _, err = doReconcile(r, ns2, "qwen") - Expect(err).NotTo(HaveOccurred()) - _, err = doReconcile(r, ns2, "qwen") - Expect(err).NotTo(HaveOccurred()) - Expect(h.installs).To(Equal(0)) - Expect(h.upgrades).To(Equal(0)) - drifted := fetch(ns2, "qwen") - Expect(drifted.Status.Phase).To(Equal(servingv1alpha1.PhaseFailed)) - Expect(drifted.Status.ObservedGeneration).To(Equal(drifted.Generation)) - Expect(condition(drifted, servingv1alpha1.ConditionAdopted).Status).To(Equal(metav1.ConditionFalse)) - Expect(condition(drifted, servingv1alpha1.ConditionAdopted).Reason).To(Equal(servingv1alpha1.ReasonDrift)) - Expect(drifted.Status.Message).To(ContainSubstring("chart")) - Expect(drifted.Status.Message).To(ContainSubstring("values")) - Expect(drifted.Status.ReleaseUID).To(BeEmpty()) - }) - - It("orphans a delete and uninstalls otherwise", func() { - ns := takeNS() - h := &fakeHelm{} - l := &versionList{versions: []string{ver086}} - r := newReconciler(h, l, 10) - Expect(k8sClient.Create(ctx, newService(ns, "keep", ver086))).To(Succeed()) - _, err := doReconcile(r, ns, "keep") - Expect(err).NotTo(HaveOccurred()) - got := fetch(ns, "keep") - got.Annotations[servingv1alpha1.AnnotationDeletionPolicy] = servingv1alpha1.DeletionPolicyOrphan - Expect(k8sClient.Update(ctx, got)).To(Succeed()) - Expect(k8sClient.Delete(ctx, got)).To(Succeed()) - gets := h.gets - _, err = doReconcile(r, ns, "keep") - Expect(err).NotTo(HaveOccurred()) - Expect(h.uninstalls).To(Equal(0)) - Expect(h.gets).To(Equal(gets)) - err = k8sClient.Get(ctx, types.NamespacedName{Namespace: ns, Name: "keep"}, &servingv1alpha1.LLMService{}) - Expect(apierrors.IsNotFound(err)).To(BeTrue()) - - Expect(k8sClient.Create(ctx, newService(ns, "drop", ver086))).To(Succeed()) - _, err = doReconcile(r, ns, "drop") - Expect(err).NotTo(HaveOccurred()) - drop := fetch(ns, "drop") - Expect(k8sClient.Delete(ctx, drop)).To(Succeed()) - _, err = doReconcile(r, ns, "drop") - Expect(err).NotTo(HaveOccurred()) - Expect(h.uninstalls).To(Equal(1)) - err = k8sClient.Get(ctx, types.NamespacedName{Namespace: ns, Name: "drop"}, &servingv1alpha1.LLMService{}) - Expect(apierrors.IsNotFound(err)).To(BeTrue()) - }) - - It("drops the finalizer when the release is already absent", func() { - ns := takeNS() - h := &fakeHelm{} - l := &versionList{versions: []string{ver086}} - obj := newService(ns, "gone", ver086) - obj.Finalizers = []string{servingv1alpha1.Finalizer} - Expect(k8sClient.Create(ctx, obj)).To(Succeed()) - Expect(k8sClient.Delete(ctx, obj)).To(Succeed()) - _, err := doReconcile(newReconciler(h, l, 10), ns, "gone") - Expect(err).NotTo(HaveOccurred()) - Expect(h.gets).To(Equal(1)) - Expect(h.installs).To(Equal(0)) - Expect(h.upgrades).To(Equal(0)) - Expect(h.uninstalls).To(Equal(0)) - err = k8sClient.Get(ctx, types.NamespacedName{Namespace: ns, Name: "gone"}, &servingv1alpha1.LLMService{}) - Expect(apierrors.IsNotFound(err)).To(BeTrue()) - }) - - It("keeps a release it never installed or adopted on delete", func() { - ns := takeNS() - h := &fakeHelm{} - l := &versionList{versions: []string{ver086}} - r := newReconciler(h, l, 10) - - // A foreign release with other values: adoption is refused as drift. - foreign := &helm.Release{ - Revision: 3, Status: deployed, ChartName: chartName, ChartVersion: ver086, - Config: map[string]any{replicasKey: float64(9)}, - } - h.put(ns, "qwen", foreign) - Expect(k8sClient.Create(ctx, newService(ns, "qwen", ver086))).To(Succeed()) - _, err := doReconcile(r, ns, "qwen") - Expect(err).NotTo(HaveOccurred()) - Expect(condition(fetch(ns, "qwen"), servingv1alpha1.ConditionAdopted).Reason).To(Equal(servingv1alpha1.ReasonDrift)) - - Expect(k8sClient.Delete(ctx, fetch(ns, "qwen"))).To(Succeed()) - _, err = doReconcile(r, ns, "qwen") - Expect(err).NotTo(HaveOccurred()) - Expect(h.uninstalls).To(Equal(0)) - Expect(h.releases).To(HaveKey(h.slot(ns, "qwen"))) - err = k8sClient.Get(ctx, types.NamespacedName{Namespace: ns, Name: "qwen"}, &servingv1alpha1.LLMService{}) - Expect(apierrors.IsNotFound(err)).To(BeTrue()) - - // status.releaseUID claimed before a first install that never stored a - // release, and a release since installed by hand: not this object's. - obj := newService(ns, "claimed", ver086) - obj.Finalizers = []string{servingv1alpha1.Finalizer} - Expect(k8sClient.Create(ctx, obj)).To(Succeed()) - obj = fetch(ns, "claimed") - obj.Status.ReleaseUID = string(obj.UID) - Expect(k8sClient.Status().Update(ctx, obj)).To(Succeed()) - h.put(ns, "claimed", &helm.Release{Revision: 1, Status: deployed, ChartName: chartName, ChartVersion: ver086}) - Expect(k8sClient.Delete(ctx, obj)).To(Succeed()) - _, err = doReconcile(r, ns, "claimed") - Expect(err).NotTo(HaveOccurred()) - Expect(h.uninstalls).To(Equal(0)) - Expect(h.releases).To(HaveKey(h.slot(ns, "claimed"))) - - // A release labelled for another LLMService of the same name. - obj = newService(ns, "other", ver086) - obj.Finalizers = []string{servingv1alpha1.Finalizer} - Expect(k8sClient.Create(ctx, obj)).To(Succeed()) - h.put(ns, "other", &helm.Release{ - Revision: 2, Status: deployed, ChartName: chartName, ChartVersion: ver086, - Labels: map[string]string{llmserviceLabel: "other", llmserviceUIDLabel: "some-other-uid"}, - }) - Expect(k8sClient.Delete(ctx, fetch(ns, "other"))).To(Succeed()) - _, err = doReconcile(r, ns, "other") - Expect(err).NotTo(HaveOccurred()) - Expect(h.uninstalls).To(Equal(0)) - }) - - It("uninstalls an adopted release on delete", func() { - ns := takeNS() - h := &fakeHelm{} - l := &versionList{versions: []string{ver086}} - r := newReconciler(h, l, 10) - h.put(ns, "qwen", &helm.Release{ - Revision: 7, Status: deployed, ChartName: chartName, ChartVersion: ver086, - Config: map[string]any{replicasKey: float64(1)}, - }) - Expect(k8sClient.Create(ctx, newService(ns, "qwen", ver086))).To(Succeed()) - _, err := doReconcile(r, ns, "qwen") - Expect(err).NotTo(HaveOccurred()) - Expect(condition(fetch(ns, "qwen"), servingv1alpha1.ConditionAdopted).Status).To(Equal(metav1.ConditionTrue)) - Expect(h.releases[h.slot(ns, "qwen")].Labels).NotTo(HaveKey(llmserviceUIDLabel)) - - Expect(k8sClient.Delete(ctx, fetch(ns, "qwen"))).To(Succeed()) - _, err = doReconcile(r, ns, "qwen") - Expect(err).NotTo(HaveOccurred()) - Expect(h.uninstalls).To(Equal(1)) - }) - - It("refuses a chart repository it was not started to install from", func() { - ns := takeNS() - h := &fakeHelm{} - l := &versionList{versions: []string{ver086}} - r := newReconciler(h, l, 10) - r.AllowedChartRepos = []string{"oci://ghcr.io/other"} - Expect(k8sClient.Create(ctx, newService(ns, "qwen", ver086))).To(Succeed()) - _, err := doReconcile(r, ns, "qwen") - Expect(err).NotTo(HaveOccurred()) - Expect(h.gets + h.installs + h.upgrades).To(Equal(0)) - Expect(l.calls).To(Equal(0)) - got := fetch(ns, "qwen") - Expect(got.Status.Phase).To(Equal(servingv1alpha1.PhaseFailed)) - Expect(condition(got, servingv1alpha1.ConditionApplied).Reason).To(Equal(servingv1alpha1.ReasonChartRepoNotAllowed)) - Expect(got.Status.Message).To(ContainSubstring("oci://ghcr.io/modelsphere/charts")) - - r.AllowedChartRepos = []string{modelsphereOCI} - _, err = doReconcile(r, ns, "qwen") - Expect(err).NotTo(HaveOccurred()) - Expect(h.installs).To(Equal(1)) - Expect(fetch(ns, "qwen").Status.Phase).To(Equal(servingv1alpha1.PhaseApplied)) - }) - - It("leaves status unchanged while suspended", func() { - ns := takeNS() - h := &fakeHelm{} - l := &versionList{versions: []string{ver086}} - r := newReconciler(h, l, 10) - Expect(k8sClient.Create(ctx, newService(ns, "qwen", ver086))).To(Succeed()) - _, err := doReconcile(r, ns, "qwen") - Expect(err).NotTo(HaveOccurred()) - before := fetch(ns, "qwen") - before.Spec.Suspend = true - before.Spec.Layers = []servingv1alpha1.Layer{layer(`{"replicaCount":9}`)} - Expect(k8sClient.Update(ctx, before)).To(Succeed()) - _, err = doReconcile(r, ns, "qwen") - Expect(err).NotTo(HaveOccurred()) - Expect(h.installs).To(Equal(1)) - Expect(h.upgrades).To(Equal(0)) - got := fetch(ns, "qwen") - Expect(got.Status.Phase).To(Equal(servingv1alpha1.PhaseApplied)) - Expect(got.Status.AppliedHash).To(Equal(before.Status.AppliedHash)) - Expect(got.Status.Helm.Revision).To(Equal(1)) - Expect(got.Status.ObservedGeneration).To(Equal(before.Status.ObservedGeneration)) - }) - - It("honors force-conflicts only for the current generation", func() { - ns := takeNS() - h := &fakeHelm{} - l := &versionList{versions: []string{ver086}} - r := newReconciler(h, l, 10) - obj := newService(ns, "qwen", ver086) - obj.Annotations[servingv1alpha1.AnnotationForceConflicts] = "1" - Expect(k8sClient.Create(ctx, obj)).To(Succeed()) - _, err := doReconcile(r, ns, "qwen") - Expect(err).NotTo(HaveOccurred()) - Expect(h.lastForce).To(BeTrue()) - Expect(fetch(ns, "qwen").Status.History[0].ForceConflicts).To(BeTrue()) - - got := fetch(ns, "qwen") - got.Spec.Layers = []servingv1alpha1.Layer{layer(`{"replicaCount":2}`)} - Expect(k8sClient.Update(ctx, got)).To(Succeed()) - Expect(fetch(ns, "qwen").Generation).To(Equal(int64(2))) - _, err = doReconcile(r, ns, "qwen") - Expect(err).NotTo(HaveOccurred()) - Expect(h.lastForce).To(BeFalse()) - Expect(fetch(ns, "qwen").Status.History[0].ForceConflicts).To(BeFalse()) - }) - - It("records a helm failure", func() { - ns := takeNS() - h := &fakeHelm{installErr: errors.New("boom")} - l := &versionList{versions: []string{ver086}} - Expect(k8sClient.Create(ctx, newService(ns, "qwen", ver086))).To(Succeed()) - _, err := doReconcile(newReconciler(h, l, 10), ns, "qwen") - Expect(err).To(MatchError(ContainSubstring("boom"))) - got := fetch(ns, "qwen") - Expect(got.Status.Phase).To(Equal(servingv1alpha1.PhaseFailed)) - Expect(got.Status.Message).To(ContainSubstring("boom")) - Expect(got.Status.ObservedGeneration).To(Equal(got.Generation)) - Expect(condition(got, servingv1alpha1.ConditionApplied).Reason).To(Equal(servingv1alpha1.ReasonApplyFailed)) - Expect(condition(got, servingv1alpha1.ConditionApplied).Status).To(Equal(metav1.ConditionFalse)) - Expect(got.Status.ReleaseUID).To(Equal(string(got.UID))) - }) - - It("installs again after a failed install and a transient chart list error", func() { - ns := takeNS() - h := &fakeHelm{installErr: errors.New("boom"), keepFailed: true} - l := &versionList{versions: []string{ver086}} - r := newReconciler(h, l, 10) - Expect(k8sClient.Create(ctx, newService(ns, "qwen", ">=0.8.0"))).To(Succeed()) - _, err := doReconcile(r, ns, "qwen") - Expect(err).To(MatchError(ContainSubstring("boom"))) - got := fetch(ns, "qwen") - Expect(got.Status.AppliedHash).To(BeEmpty()) - Expect(got.Status.History).To(BeEmpty()) - Expect(got.Status.ReleaseUID).To(Equal(string(got.UID))) - Expect(h.releases[h.slot(ns, "qwen")].Labels).To(HaveKeyWithValue(llmserviceUIDLabel, string(got.UID))) - Expect(h.releases[h.slot(ns, "qwen")].Status).To(Equal(statusFailed)) - Expect(condition(got, servingv1alpha1.ConditionApplied).Reason).To(Equal(servingv1alpha1.ReasonApplyFailed)) - - l.err = errors.New("registry down") - _, err = doReconcile(r, ns, "qwen") - Expect(err).To(MatchError(ContainSubstring("registry down"))) - got = fetch(ns, "qwen") - Expect(got.Status.Phase).To(Equal(servingv1alpha1.PhaseFailed)) - Expect(condition(got, servingv1alpha1.ConditionApplied).Reason).To(Equal(servingv1alpha1.ReasonChartResolveFailed)) - Expect(condition(got, servingv1alpha1.ConditionAdopted)).To(BeNil()) - Expect(got.Status.ReleaseUID).To(Equal(string(got.UID))) - Expect(h.installs).To(Equal(1)) - Expect(h.upgrades).To(Equal(0)) - Expect(h.uninstalls).To(Equal(0)) - - l.err = nil - h.installErr = nil - _, err = doReconcile(r, ns, "qwen") - Expect(err).NotTo(HaveOccurred()) - Expect(h.installs).To(Equal(2)) - Expect(h.upgrades).To(Equal(0)) - Expect(h.uninstalls).To(Equal(1)) - Expect(h.rollbacks).To(Equal(0)) - got = fetch(ns, "qwen") - Expect(got.Status.Phase).To(Equal(servingv1alpha1.PhaseApplied)) - Expect(condition(got, servingv1alpha1.ConditionApplied).Reason).To(Equal(servingv1alpha1.ReasonInstalled)) - Expect(condition(got, servingv1alpha1.ConditionAdopted)).To(BeNil()) - Expect(got.Status.Message).NotTo(ContainSubstring("Drift")) - }) - - It("requeues a helm get error instead of drifting a failed release", func() { - ns := takeNS() - h := &fakeHelm{installErr: errors.New("boom"), keepFailed: true} - l := &versionList{versions: []string{ver086}} - r := newReconciler(h, l, 10) - Expect(k8sClient.Create(ctx, newService(ns, "qwen", ver086))).To(Succeed()) - _, err := doReconcile(r, ns, "qwen") - Expect(err).To(MatchError(ContainSubstring("boom"))) - - h.installErr = nil - h.getErr = errors.New("apiserver timeout") - _, err = doReconcile(r, ns, "qwen") - Expect(err).To(MatchError(ContainSubstring("apiserver timeout"))) - got := fetch(ns, "qwen") - Expect(condition(got, servingv1alpha1.ConditionApplied).Reason).To(Equal(servingv1alpha1.ReasonApplyFailed)) - Expect(condition(got, servingv1alpha1.ConditionAdopted)).To(BeNil()) - Expect(h.uninstalls).To(Equal(0)) - Expect(h.installs).To(Equal(1)) - - h.getErr = nil - _, err = doReconcile(r, ns, "qwen") - Expect(err).NotTo(HaveOccurred()) - Expect(h.uninstalls).To(Equal(1)) - Expect(h.installs).To(Equal(2)) - Expect(h.upgrades).To(Equal(0)) - got = fetch(ns, "qwen") - Expect(got.Status.Phase).To(Equal(servingv1alpha1.PhaseApplied)) - Expect(condition(got, servingv1alpha1.ConditionApplied).Reason).To(Equal(servingv1alpha1.ReasonInstalled)) - }) - - It("upgrades an owned failed release that still has a deployed revision", func() { - ns := takeNS() - h := &fakeHelm{} - l := &versionList{versions: []string{ver086}} - r := newReconciler(h, l, 10) - Expect(k8sClient.Create(ctx, newService(ns, "qwen", ver086))).To(Succeed()) - _, err := doReconcile(r, ns, "qwen") - Expect(err).NotTo(HaveOccurred()) - got := fetch(ns, "qwen") - slot := h.slot(ns, "qwen") - failed := ownedRel(got, 2, statusFailed, time.Now()) - h.histories = map[string][]helm.Release{ - slot: {*h.releases[slot], *failed}, - } - h.put(ns, "qwen", failed) - got.Spec.Layers = []servingv1alpha1.Layer{layer(`{"replicaCount":2}`)} - Expect(k8sClient.Update(ctx, got)).To(Succeed()) - - _, err = doReconcile(r, ns, "qwen") - Expect(err).NotTo(HaveOccurred()) - Expect(h.upgrades).To(Equal(1)) - Expect(h.uninstalls).To(Equal(0)) - Expect(h.installs).To(Equal(1)) - Expect(fetch(ns, "qwen").Status.Phase).To(Equal(servingv1alpha1.PhaseApplied)) - Expect(condition(fetch(ns, "qwen"), servingv1alpha1.ConditionApplied).Reason).To(Equal(servingv1alpha1.ReasonUpgraded)) - }) - - It("does not adopt a release that is not deployed", func() { - ns := takeNS() - h := &fakeHelm{} - h.put(ns, "qwen", &helm.Release{ - Revision: 1, Status: statusFailed, ChartName: chartName, ChartVersion: ver086, - Config: map[string]any{replicasKey: float64(1)}, - }) - l := &versionList{versions: []string{ver086}} - r := newReconciler(h, l, 10) - Expect(k8sClient.Create(ctx, newService(ns, "qwen", ver086))).To(Succeed()) - _, err := doReconcile(r, ns, "qwen") - Expect(err).NotTo(HaveOccurred()) - _, err = doReconcile(r, ns, "qwen") - Expect(err).NotTo(HaveOccurred()) - Expect(h.installs).To(Equal(0)) - Expect(h.upgrades).To(Equal(0)) - got := fetch(ns, "qwen") - Expect(got.Status.Phase).To(Equal(servingv1alpha1.PhaseFailed)) - Expect(got.Status.Message).To(Equal("release is failed, not deployed")) - Expect(got.Status.ObservedGeneration).To(Equal(got.Generation)) - Expect(condition(got, servingv1alpha1.ConditionAdopted).Status).To(Equal(metav1.ConditionFalse)) - Expect(condition(got, servingv1alpha1.ConditionAdopted).Reason).To(Equal(servingv1alpha1.ReasonDrift)) - Expect(got.Status.ReleaseUID).To(BeEmpty()) - }) - - It("still drifts a foreign failed release when Applied is ApplyFailed", func() { - ns := takeNS() - h := &fakeHelm{} - l := &versionList{versions: []string{ver086}} - r := newReconciler(h, l, 10) - Expect(k8sClient.Create(ctx, newService(ns, "qwen", ver086))).To(Succeed()) - got := fetch(ns, "qwen") - got.Status.Phase = servingv1alpha1.PhaseFailed - meta.SetStatusCondition(&got.Status.Conditions, metav1.Condition{ - Type: servingv1alpha1.ConditionApplied, - Status: metav1.ConditionFalse, - Reason: servingv1alpha1.ReasonApplyFailed, - Message: "boom", - ObservedGeneration: got.Generation, - }) - Expect(k8sClient.Status().Update(ctx, got)).To(Succeed()) - h.put(ns, "qwen", &helm.Release{ - Revision: 1, Status: statusFailed, ChartName: chartName, ChartVersion: ver086, - Config: map[string]any{replicasKey: float64(1)}, - }) - _, err := doReconcile(r, ns, "qwen") - Expect(err).NotTo(HaveOccurred()) - Expect(h.installs).To(Equal(0)) - Expect(h.upgrades).To(Equal(0)) - Expect(h.uninstalls).To(Equal(0)) - got = fetch(ns, "qwen") - Expect(got.Status.Phase).To(Equal(servingv1alpha1.PhaseFailed)) - Expect(condition(got, servingv1alpha1.ConditionAdopted).Reason).To(Equal(servingv1alpha1.ReasonDrift)) - Expect(condition(got, servingv1alpha1.ConditionApplied).Reason).To(Equal(servingv1alpha1.ReasonDrift)) - }) - - It("keeps observedGeneration while install is in progress", func() { - ns := takeNS() - h := &fakeHelm{} - l := &versionList{versions: []string{ver086}} - r := newReconciler(h, l, 10) - Expect(k8sClient.Create(ctx, newService(ns, "qwen", ver086))).To(Succeed()) - _, err := doReconcile(r, ns, "qwen") - Expect(err).NotTo(HaveOccurred()) - prev := fetch(ns, "qwen").Status.ObservedGeneration - Expect(prev).NotTo(BeZero()) - - delete(h.releases, h.slot(ns, "qwen")) - got := fetch(ns, "qwen") - got.Spec.Layers = []servingv1alpha1.Layer{layer(`{"replicaCount":4}`)} - Expect(k8sClient.Update(ctx, got)).To(Succeed()) - - h.hold = make(chan struct{}) - h.entered = make(chan struct{}, 1) - done := make(chan error, 1) - go func() { - _, recErr := doReconcile(r, ns, "qwen") - done <- recErr - }() - Eventually(h.entered).WithTimeout(5 * time.Second).Should(Receive()) - mid := fetch(ns, "qwen") - Expect(mid.Status.Phase).To(Equal(servingv1alpha1.PhaseApplying)) - Expect(mid.Status.ObservedGeneration).To(Equal(prev)) - close(h.hold) - Eventually(done).WithTimeout(5 * time.Second).Should(Receive(BeNil())) - Expect(h.installs).To(Equal(2)) - Expect(h.upgrades).To(Equal(0)) - doneObj := fetch(ns, "qwen") - Expect(doneObj.Status.Phase).To(Equal(servingv1alpha1.PhaseApplied)) - Expect(doneObj.Status.ObservedGeneration).To(Equal(doneObj.Generation)) - }) - - It("records a rollback to earlier values as a ControllerRevision of its own", func() { - ns := takeNS() - h := &fakeHelm{} - l := &versionList{versions: []string{ver086}} - r := newReconciler(h, l, 10) - Expect(k8sClient.Create(ctx, newService(ns, "qwen", ver086))).To(Succeed()) - _, err := doReconcile(r, ns, "qwen") - Expect(err).NotTo(HaveOccurred()) - - got := fetch(ns, "qwen") - got.Spec.Layers = []servingv1alpha1.Layer{layer(`{"replicaCount":2}`)} - Expect(k8sClient.Update(ctx, got)).To(Succeed()) - _, err = doReconcile(r, ns, "qwen") - Expect(err).NotTo(HaveOccurred()) - - // What swiss sends for a rollback: the earlier values, with its own - // annotations, which the snapshot records -- the data a reused - // ControllerRevision name would have had to change. - got = fetch(ns, "qwen") - got.Spec.Layers = []servingv1alpha1.Layer{layer(`{"replicaCount":1}`)} - got.Annotations["swiss.modelsphere.dev/action"] = "rollback" - got.Annotations["swiss.modelsphere.dev/plan-hash"] = "p3" - Expect(k8sClient.Update(ctx, got)).To(Succeed()) - _, err = doReconcile(r, ns, "qwen") - Expect(err).NotTo(HaveOccurred()) - - got = fetch(ns, "qwen") - Expect(got.Status.History).To(HaveLen(3)) - Expect(got.Status.AppliedHash).To(Equal(got.Status.History[0].Hash)) - Expect(got.Status.History[0].Hash).To(Equal(got.Status.History[2].Hash)) - Expect(got.Status.History[0].Action).To(Equal("rollback")) - names := map[string]bool{} - for _, e := range got.Status.History { - names[e.ControllerRevision] = true - } - Expect(names).To(HaveLen(3)) - Expect(got.Status.History[0].Revision).To(Equal(3)) - - snapshot := func(name string) revisionSnapshot { - var cr appsv1.ControllerRevision - Expect(k8sClient.Get(ctx, types.NamespacedName{Namespace: ns, Name: name}, &cr)).To(Succeed()) - var snap revisionSnapshot - Expect(json.Unmarshal(cr.Data.Raw, &snap)).To(Succeed()) - return snap - } - Expect(snapshot(got.Status.History[0].ControllerRevision).Annotations).To(HaveKeyWithValue("swiss.modelsphere.dev/action", "rollback")) - Expect(snapshot(got.Status.History[2].ControllerRevision).Annotations).To(HaveKeyWithValue("swiss.modelsphere.dev/action", "install")) - - // Applied: the next reconcile has nothing to do. - upgrades := h.upgrades - _, err = doReconcile(r, ns, "qwen") - Expect(err).NotTo(HaveOccurred()) - Expect(h.upgrades).To(Equal(upgrades)) - }) - - It("replaces its own stale snapshot under a reused revision name", func() { - ns := takeNS() - h := &fakeHelm{} - l := &versionList{versions: []string{ver086}} - r := newReconciler(h, l, 10) - Expect(k8sClient.Create(ctx, newService(ns, "qwen", ver086))).To(Succeed()) - obj := fetch(ns, "qwen") - - // What an uninstall leaves when helm's numbering restarts: a snapshot - // of this object's, under the name the next install will use. - values, err := mergeLayers(obj.Spec.Layers) - Expect(err).NotTo(HaveOccurred()) - hash, err := appliedHash(chartName, ver086, values) - Expect(err).NotTo(HaveOccurred()) - stale := &appsv1.ControllerRevision{ - ObjectMeta: metav1.ObjectMeta{Name: controllerRevisionName("qwen", hash, 1), Namespace: ns}, - Data: runtime.RawExtension{Raw: []byte(`{"annotations":{"swiss.modelsphere.dev/action":"old"}}`)}, - Revision: 1, - } - Expect(controllerutil.SetControllerReference(obj, stale, k8sClient.Scheme())).To(Succeed()) - Expect(k8sClient.Create(ctx, stale)).To(Succeed()) - - _, err = doReconcile(r, ns, "qwen") - Expect(err).NotTo(HaveOccurred()) - got := fetch(ns, "qwen") - Expect(got.Status.Phase).To(Equal(servingv1alpha1.PhaseApplied)) - Expect(got.Status.History[0].ControllerRevision).To(Equal(stale.Name)) - var cr appsv1.ControllerRevision - Expect(k8sClient.Get(ctx, types.NamespacedName{Namespace: ns, Name: stale.Name}, &cr)).To(Succeed()) - var snap revisionSnapshot - Expect(json.Unmarshal(cr.Data.Raw, &snap)).To(Succeed()) - Expect(snap.Annotations).To(HaveKeyWithValue("swiss.modelsphere.dev/action", "install")) - }) - - It("fails when the chart version cannot be resolved", func() { - ns := takeNS() - h := &fakeHelm{} - l := &versionList{err: errors.New("registry down")} - Expect(k8sClient.Create(ctx, newService(ns, "qwen", ">=0.8.0"))).To(Succeed()) - _, err := doReconcile(newReconciler(h, l, 10), ns, "qwen") - Expect(err).To(HaveOccurred()) - got := fetch(ns, "qwen") - Expect(got.Status.Phase).To(Equal(servingv1alpha1.PhaseFailed)) - Expect(got.Status.Message).To(ContainSubstring("registry down")) - Expect(got.Status.ObservedGeneration).To(Equal(got.Generation)) - Expect(condition(got, servingv1alpha1.ConditionApplied).Reason).To(Equal(servingv1alpha1.ReasonChartResolveFailed)) - Expect(h.installs).To(Equal(0)) - }) - - It("recovers a stale owned pending-install and installs", func() { - ns := takeNS() - h := &fakeHelm{} - l := &versionList{versions: []string{ver086}} - rec := events.NewFakeRecorder(8) - r := newReconciler(h, l, 10) - r.Recorder = rec - Expect(k8sClient.Create(ctx, newService(ns, "qwen", ver086))).To(Succeed()) - obj := fetch(ns, "qwen") - h.put(ns, "qwen", ownedRel(obj, 1, statusPendingInstall, time.Now().Add(-time.Hour))) - - _, err := doReconcile(r, ns, "qwen") - Expect(err).NotTo(HaveOccurred()) - Expect(h.uninstalls).To(Equal(1)) - Expect(h.installs).To(Equal(1)) - Expect(h.upgrades).To(Equal(0)) - Expect(h.rollbacks).To(Equal(0)) - got := fetch(ns, "qwen") - Expect(got.Status.Phase).To(Equal(servingv1alpha1.PhaseApplied)) - Expect(condition(got, servingv1alpha1.ConditionApplied).Reason).To(Equal(servingv1alpha1.ReasonInstalled)) - Expect(got.Status.ReleaseUID).To(Equal(string(got.UID))) - Expect(rec.Events).To(Receive(ContainSubstring(servingv1alpha1.ReasonRecoveredPendingRelease))) - }) - - It("waits out a fresh owned pending-upgrade", func() { - ns := takeNS() - h := &fakeHelm{} - l := &versionList{versions: []string{ver086}} - r := newReconciler(h, l, 10) - r.PendingTimeout = 2 * time.Minute - Expect(k8sClient.Create(ctx, newService(ns, "qwen", ver086))).To(Succeed()) - obj := fetch(ns, "qwen") - h.put(ns, "qwen", ownedRel(obj, 2, statusPendingUpgrade, time.Now())) - h.histories = map[string][]helm.Release{ - h.slot(ns, "qwen"): {ownedRelVal(obj, 1, deployed, time.Now().Add(-time.Hour)), *h.releases[h.slot(ns, "qwen")]}, - } - - result, err := doReconcile(r, ns, "qwen") - Expect(err).NotTo(HaveOccurred()) - Expect(result.RequeueAfter).To(BeNumerically(">=", minPendingRequeue)) - Expect(result.RequeueAfter).To(BeNumerically(">", time.Minute)) - Expect(result.RequeueAfter).To(BeNumerically("<=", 2*time.Minute)) - Expect(h.installs).To(Equal(0)) - Expect(h.upgrades).To(Equal(0)) - Expect(h.uninstalls).To(Equal(0)) - Expect(h.rollbacks).To(Equal(0)) - got := fetch(ns, "qwen") - Expect(got.Status.Phase).To(Equal(servingv1alpha1.PhaseApplying)) - Expect(condition(got, servingv1alpha1.ConditionApplied).Reason).To(Equal(servingv1alpha1.ReasonOperationInProgress)) - Expect(got.Status.Message).To(ContainSubstring(statusPendingUpgrade)) - - result, err = doReconcile(r, ns, "qwen") - Expect(err).NotTo(HaveOccurred()) - Expect(result.RequeueAfter).To(BeNumerically(">=", minPendingRequeue)) - Expect(h.rollbacks).To(Equal(0)) - Expect(h.upgrades).To(Equal(0)) - }) - - It("clamps a short pending wait to 15s", func() { - ns := takeNS() - h := &fakeHelm{} - l := &versionList{versions: []string{ver086}} - r := newReconciler(h, l, 10) - r.PendingTimeout = 10 * time.Second - Expect(k8sClient.Create(ctx, newService(ns, "qwen", ver086))).To(Succeed()) - obj := fetch(ns, "qwen") - h.put(ns, "qwen", ownedRel(obj, 1, statusPendingRollback, time.Now().Add(-8*time.Second))) - - result, err := doReconcile(r, ns, "qwen") - Expect(err).NotTo(HaveOccurred()) - Expect(result.RequeueAfter).To(Equal(minPendingRequeue)) - Expect(h.rollbacks).To(Equal(0)) - Expect(h.uninstalls).To(Equal(0)) - Expect(condition(fetch(ns, "qwen"), servingv1alpha1.ConditionApplied).Reason).To(Equal(servingv1alpha1.ReasonOperationInProgress)) - }) - - It("rolls back a stale owned pending-upgrade and upgrades", func() { - ns := takeNS() - h := &fakeHelm{} - l := &versionList{versions: []string{ver086}} - rec := events.NewFakeRecorder(8) - r := newReconciler(h, l, 10) - r.Recorder = rec - Expect(k8sClient.Create(ctx, newService(ns, "qwen", ver086))).To(Succeed()) - obj := fetch(ns, "qwen") - deployedRel := ownedRelVal(obj, 4, deployed, time.Now().Add(-2*time.Hour)) - pending := ownedRel(obj, 5, statusPendingUpgrade, time.Now().Add(-time.Hour)) - h.put(ns, "qwen", pending) - h.histories = map[string][]helm.Release{ - h.slot(ns, "qwen"): { - ownedRelVal(obj, 2, deployed, time.Now().Add(-3*time.Hour)), - deployedRel, - *pending, - }, - } - - _, err := doReconcile(r, ns, "qwen") - Expect(err).NotTo(HaveOccurred()) - Expect(h.rollbacks).To(Equal(1)) - Expect(h.lastRollback).To(Equal(4)) - Expect(h.upgrades).To(Equal(1)) - Expect(h.uninstalls).To(Equal(0)) - Expect(h.installs).To(Equal(0)) - got := fetch(ns, "qwen") - Expect(got.Status.Phase).To(Equal(servingv1alpha1.PhaseApplied)) - Expect(condition(got, servingv1alpha1.ConditionApplied).Reason).To(Equal(servingv1alpha1.ReasonUpgraded)) - Expect(rec.Events).To(Receive(ContainSubstring(servingv1alpha1.ReasonRecoveredPendingRelease))) - }) - - It("uninstalls a stale owned pending-rollback that has no deployed revision", func() { - ns := takeNS() - h := &fakeHelm{} - l := &versionList{versions: []string{ver086}} - r := newReconciler(h, l, 10) - Expect(k8sClient.Create(ctx, newService(ns, "qwen", ver086))).To(Succeed()) - obj := fetch(ns, "qwen") - pending := ownedRel(obj, 2, statusPendingRollback, time.Now().Add(-time.Hour)) - h.put(ns, "qwen", pending) - h.histories = map[string][]helm.Release{ - h.slot(ns, "qwen"): {ownedRelVal(obj, 1, statusFailed, time.Now().Add(-2*time.Hour)), *pending}, - } - - _, err := doReconcile(r, ns, "qwen") - Expect(err).NotTo(HaveOccurred()) - Expect(h.rollbacks).To(Equal(0)) - Expect(h.uninstalls).To(Equal(1)) - Expect(h.installs).To(Equal(1)) - Expect(h.upgrades).To(Equal(0)) - got := fetch(ns, "qwen") - Expect(got.Status.Phase).To(Equal(servingv1alpha1.PhaseApplied)) - Expect(condition(got, servingv1alpha1.ConditionApplied).Reason).To(Equal(servingv1alpha1.ReasonInstalled)) - }) - - It("does not write a pending release it does not own", func() { - ns := takeNS() - h := &fakeHelm{} - l := &versionList{versions: []string{ver086}} - r := newReconciler(h, l, 10) - Expect(k8sClient.Create(ctx, newService(ns, "qwen", ver086))).To(Succeed()) - h.put(ns, "qwen", &helm.Release{ - Revision: 3, Status: statusPendingUpgrade, ChartName: chartName, ChartVersion: ver086, - Updated: time.Now().Add(-time.Hour), + for _, f := range cases.MustLoad[serviceCase]("testdata/controller") { + tc := f.Case + It(tc.Description, func() { + runServiceCase(tc) }) - - result, err := doReconcile(r, ns, "qwen") - Expect(err).NotTo(HaveOccurred()) - Expect(result.RequeueAfter).To(Equal(time.Minute)) - Expect(h.installs).To(Equal(0)) - Expect(h.upgrades).To(Equal(0)) - Expect(h.uninstalls).To(Equal(0)) - Expect(h.rollbacks).To(Equal(0)) - got := fetch(ns, "qwen") - Expect(got.Status.Phase).To(Equal(servingv1alpha1.PhaseFailed)) - Expect(got.Status.Message).To(Equal("release is pending-upgrade; another client is operating on it")) - Expect(condition(got, servingv1alpha1.ConditionApplied).Status).To(Equal(metav1.ConditionFalse)) - Expect(condition(got, servingv1alpha1.ConditionApplied).Reason).To(Equal(servingv1alpha1.ReasonPendingRelease)) - Expect(got.Status.ObservedGeneration).To(Equal(got.Generation)) - - result, err = doReconcile(r, ns, "qwen") - Expect(err).NotTo(HaveOccurred()) - Expect(result.RequeueAfter).To(Equal(time.Minute)) - Expect(h.installs).To(Equal(0)) - Expect(h.upgrades).To(Equal(0)) - Expect(h.uninstalls).To(Equal(0)) - Expect(h.rollbacks).To(Equal(0)) - }) - - It("upgrades an owned deployed release whose status was lost", func() { - ns := takeNS() - h := &fakeHelm{} - l := &versionList{versions: []string{ver086}} - r := newReconciler(h, l, 10) - Expect(k8sClient.Create(ctx, newService(ns, "qwen", ver086))).To(Succeed()) - obj := fetch(ns, "qwen") - h.put(ns, "qwen", ownedRel(obj, 4, deployed, time.Now().Add(-time.Hour))) - - _, err := doReconcile(r, ns, "qwen") - Expect(err).NotTo(HaveOccurred()) - Expect(h.upgrades).To(Equal(1)) - Expect(h.installs).To(Equal(0)) - Expect(h.uninstalls).To(Equal(0)) - got := fetch(ns, "qwen") - Expect(got.Status.Phase).To(Equal(servingv1alpha1.PhaseApplied)) - Expect(condition(got, servingv1alpha1.ConditionApplied).Reason).To(Equal(servingv1alpha1.ReasonUpgraded)) - Expect(condition(got, servingv1alpha1.ConditionAdopted)).To(BeNil()) - Expect(got.Status.ReleaseUID).To(Equal(string(got.UID))) - Expect(got.Status.Helm.Revision).To(Equal(5)) - }) + } }) func ownedRel(obj *servingv1alpha1.LLMService, rev int, status string, updated time.Time) *helm.Release { diff --git a/internal/controller/serving/predicate_test.go b/internal/controller/serving/predicate_test.go index 673ded5..7c558e2 100644 --- a/internal/controller/serving/predicate_test.go +++ b/internal/controller/serving/predicate_test.go @@ -7,39 +7,55 @@ import ( "sigs.k8s.io/controller-runtime/pkg/event" servingv1alpha1 "github.com/modelsphere/llm-operator/api/serving/v1alpha1" + "github.com/modelsphere/llm-operator/internal/testutil/cases" ) -func TestLLMServicePredicate(t *testing.T) { - p := llmServicePredicate{} - base := &servingv1alpha1.LLMService{} - base.Generation = 1 - base.Annotations = map[string]string{"a": "1"} - - same := base.DeepCopy() - same.Status.Phase = servingv1alpha1.PhaseApplied - if p.Update(event.UpdateEvent{ObjectOld: base, ObjectNew: same}) { - t.Fatal("status-only update reconciled") - } - - gen := base.DeepCopy() - gen.Generation = 2 - if !p.Update(event.UpdateEvent{ObjectOld: base, ObjectNew: gen}) { - t.Fatal("generation change ignored") - } - - ann := base.DeepCopy() - ann.Annotations = map[string]string{"a": "2"} - if !p.Update(event.UpdateEvent{ObjectOld: base, ObjectNew: ann}) { - t.Fatal("annotation change ignored") - } +// predicateCase is one llmServicePredicate input. The base object is generation +// 1 with annotation a=1; the case describes what changes on the new object. +type predicateCase struct { + Name string `json:"name,omitempty"` + Description string `json:"description,omitempty"` + Op string `json:"op"` + Phase string `json:"phase,omitempty"` + Generation int64 `json:"generation,omitempty"` + Annotations map[string]string `json:"annotations,omitempty"` + Deleting bool `json:"deleting,omitempty"` + Want bool `json:"want"` +} - now := metav1.Now() - del := base.DeepCopy() - del.DeletionTimestamp = &now - if !p.Update(event.UpdateEvent{ObjectOld: base, ObjectNew: del}) { - t.Fatal("deletion ignored") - } - if !p.Create(event.CreateEvent{Object: base}) || !p.Delete(event.DeleteEvent{Object: base}) { - t.Fatal("create or delete ignored") - } +func TestLLMServicePredicate(t *testing.T) { + cases.Run(t, "testdata/predicate", func(t *testing.T, tc predicateCase) { + p := llmServicePredicate{} + base := &servingv1alpha1.LLMService{} + base.Generation = 1 + base.Annotations = map[string]string{"a": "1"} + var got bool + switch tc.Op { + case "update": + next := base.DeepCopy() + if tc.Phase != "" { + next.Status.Phase = servingv1alpha1.Phase(tc.Phase) + } + if tc.Generation != 0 { + next.Generation = tc.Generation + } + if tc.Annotations != nil { + next.Annotations = tc.Annotations + } + if tc.Deleting { + now := metav1.Now() + next.DeletionTimestamp = &now + } + got = p.Update(event.UpdateEvent{ObjectOld: base, ObjectNew: next}) + case "create": + got = p.Create(event.CreateEvent{Object: base}) + case "delete": + got = p.Delete(event.DeleteEvent{Object: base}) + default: + t.Fatalf("%s: unknown op %q", tc.Name, tc.Op) + } + if got != tc.Want { + t.Fatalf("%s: reconcile = %v, want %v", tc.Description, got, tc.Want) + } + }) } diff --git a/internal/controller/serving/repo_test.go b/internal/controller/serving/repo_test.go index 978354f..e174238 100644 --- a/internal/controller/serving/repo_test.go +++ b/internal/controller/serving/repo_test.go @@ -1,26 +1,36 @@ package serving -import "testing" +import ( + "testing" -const modelsphereOCI = "oci://ghcr.io/modelsphere" + "github.com/modelsphere/llm-operator/internal/testutil/cases" +) + +// repoCase is one chartRepoAllowed input. +// allowed is the allowlist and must be set, possibly as an empty list. +// An empty list and a nil list both permit every repository; nilAllowlist +// is how a case passes nil explicitly. Omitting both is an error, because +// a forgotten allowlist would otherwise permit every repository. +type repoCase struct { + Name string `json:"name,omitempty"` + Description string `json:"description,omitempty"` + Allowed []string `json:"allowed,omitempty"` + NilAllowlist bool `json:"nilAllowlist,omitempty"` + Repo string `json:"repo"` + Want bool `json:"want"` +} func TestChartRepoAllowed(t *testing.T) { - allowed := []string{modelsphereOCI, "https://modelsphere.github.io/helm-charts/"} - for repo, want := range map[string]bool{ - modelsphereOCI: true, - "oci://ghcr.io/modelsphere/charts": true, - "oci://ghcr.io/modelsphere/charts/": true, - "https://modelsphere.github.io/helm-charts": true, - "oci://ghcr.io/modelsphere-x/charts": false, - "oci://ghcr.io/other/charts": false, - "https://modelsphere.github.io": false, - "": false, - } { - if got := chartRepoAllowed(allowed, repo); got != want { - t.Errorf("%q: got %v, want %v", repo, got, want) + cases.Run(t, "testdata/repo", func(t *testing.T, tc repoCase) { + if !tc.NilAllowlist && tc.Allowed == nil { + t.Fatal("set allowed (an empty list allows every repository) or nilAllowlist") + } + var allowed []string + if !tc.NilAllowlist { + allowed = tc.Allowed + } + if got := chartRepoAllowed(allowed, tc.Repo); got != tc.Want { + t.Errorf("%s: %q: got %v, want %v", tc.Description, tc.Repo, got, tc.Want) } - } - if !chartRepoAllowed(nil, "https://anything.example.com/charts") { - t.Error("no allowlist allows any repository") - } + }) } diff --git a/internal/controller/serving/testdata/controller/adopts-a-matching-release-and-refuses-drift.yaml b/internal/controller/serving/testdata/controller/adopts-a-matching-release-and-refuses-drift.yaml new file mode 100644 index 0000000..a4e070c --- /dev/null +++ b/internal/controller/serving/testdata/controller/adopts-a-matching-release-and-refuses-drift.yaml @@ -0,0 +1,57 @@ +name: adopts-a-matching-release-and-refuses-drift +description: adopts a matching release and refuses drift +steps: + - action: putRelease + release: + revision: 7 + status: deployed + chartName: sglang + chartVersion: "0.8.6" + config: + replicaCount: 1 + - action: create + version: ">=0.8.0" + - action: reconcile + expect: + installs: 0 + upgrades: 0 + phase: Applied + conditions: + - type: Adopted + status: "True" + reason: Adopted + - type: Applied + reason: Adopted + history: + entries: + - index: 0 + revision: 7 + releaseUID: self + - action: putRelease + freshNamespace: true + release: + revision: 3 + status: deployed + chartName: other + chartVersion: "0.1.0" + config: + replicaCount: 9 + - action: create + version: ">=0.8.0" + - action: reconcile + expect: {} + - action: reconcile + expect: + installs: 0 + upgrades: 0 + phase: Failed + observedGeneration: current + conditions: + - type: Adopted + status: "False" + reason: Drift + message: + contains: + - chart + - values + releaseUID: empty diff --git a/internal/controller/serving/testdata/controller/clamps-a-short-pending-wait-to-15s.yaml b/internal/controller/serving/testdata/controller/clamps-a-short-pending-wait-to-15s.yaml new file mode 100644 index 0000000..88615e6 --- /dev/null +++ b/internal/controller/serving/testdata/controller/clamps-a-short-pending-wait-to-15s.yaml @@ -0,0 +1,21 @@ +name: clamps-a-short-pending-wait-to-15s +description: clamps a short pending wait to 15s +pendingTimeout: "10s" +steps: + - action: create + version: "0.8.6" + - action: putRelease + release: + owned: true + revision: 1 + status: pending-rollback + updated: "-8s" + - action: reconcile + expect: + requeue: + exact: "15s" + rollbacks: 0 + uninstalls: 0 + conditions: + - type: Applied + reason: OperationInProgress diff --git a/internal/controller/serving/testdata/controller/does-not-adopt-a-release-that-is-not-deployed.yaml b/internal/controller/serving/testdata/controller/does-not-adopt-a-release-that-is-not-deployed.yaml new file mode 100644 index 0000000..a2f447c --- /dev/null +++ b/internal/controller/serving/testdata/controller/does-not-adopt-a-release-that-is-not-deployed.yaml @@ -0,0 +1,28 @@ +name: does-not-adopt-a-release-that-is-not-deployed +description: does not adopt a release that is not deployed +steps: + - action: putRelease + release: + revision: 1 + status: failed + chartName: sglang + chartVersion: "0.8.6" + config: + replicaCount: 1 + - action: create + version: "0.8.6" + - action: reconcile + expect: {} + - action: reconcile + expect: + installs: 0 + upgrades: 0 + phase: Failed + message: + equals: release is failed, not deployed + observedGeneration: current + conditions: + - type: Adopted + status: "False" + reason: Drift + releaseUID: empty diff --git a/internal/controller/serving/testdata/controller/does-not-call-helm-when-the-spec-is-unchanged.yaml b/internal/controller/serving/testdata/controller/does-not-call-helm-when-the-spec-is-unchanged.yaml new file mode 100644 index 0000000..07f0891 --- /dev/null +++ b/internal/controller/serving/testdata/controller/does-not-call-helm-when-the-spec-is-unchanged.yaml @@ -0,0 +1,15 @@ +name: does-not-call-helm-when-the-spec-is-unchanged +description: does not call helm when the spec is unchanged +steps: + - action: create + version: "0.8.6" + - action: reconcile + expect: {} + - action: update + annotations: + swiss.modelsphere.dev/note: later + - action: reconcile + expect: + installs: 1 + upgrades: 0 + gets: 1 diff --git a/internal/controller/serving/testdata/controller/does-not-write-a-pending-release-it-does-not-own.yaml b/internal/controller/serving/testdata/controller/does-not-write-a-pending-release-it-does-not-own.yaml new file mode 100644 index 0000000..43e8d2d --- /dev/null +++ b/internal/controller/serving/testdata/controller/does-not-write-a-pending-release-it-does-not-own.yaml @@ -0,0 +1,36 @@ +name: does-not-write-a-pending-release-it-does-not-own +description: does not write a pending release it does not own +steps: + - action: create + version: "0.8.6" + - action: putRelease + release: + revision: 3 + status: pending-upgrade + chartName: sglang + chartVersion: "0.8.6" + updated: "-1h" + - action: reconcile + expect: + requeue: + exact: "1m" + installs: 0 + upgrades: 0 + uninstalls: 0 + rollbacks: 0 + phase: Failed + message: + equals: "release is pending-upgrade; another client is operating on it" + conditions: + - type: Applied + status: "False" + reason: PendingRelease + observedGeneration: current + - action: reconcile + expect: + requeue: + exact: "1m" + installs: 0 + upgrades: 0 + uninstalls: 0 + rollbacks: 0 diff --git a/internal/controller/serving/testdata/controller/drops-the-finalizer-when-the-release-is-already-absent.yaml b/internal/controller/serving/testdata/controller/drops-the-finalizer-when-the-release-is-already-absent.yaml new file mode 100644 index 0000000..6c46727 --- /dev/null +++ b/internal/controller/serving/testdata/controller/drops-the-finalizer-when-the-release-is-already-absent.yaml @@ -0,0 +1,16 @@ +name: drops-the-finalizer-when-the-release-is-already-absent +description: drops the finalizer when the release is already absent +steps: + - action: create + name: gone + version: "0.8.6" + finalizers: + - serving.modelsphere.dev/finalizer + - action: delete + - action: reconcile + expect: + gets: 1 + installs: 0 + upgrades: 0 + uninstalls: 0 + notFound: true diff --git a/internal/controller/serving/testdata/controller/fails-when-the-chart-version-cannot-be-resolved.yaml b/internal/controller/serving/testdata/controller/fails-when-the-chart-version-cannot-be-resolved.yaml new file mode 100644 index 0000000..157a185 --- /dev/null +++ b/internal/controller/serving/testdata/controller/fails-when-the-chart-version-cannot-be-resolved.yaml @@ -0,0 +1,19 @@ +name: fails-when-the-chart-version-cannot-be-resolved +description: fails when the chart version cannot be resolved +steps: + - action: setVersions + listErr: registry down + - action: create + version: ">=0.8.0" + - action: reconcile + expect: + error: true + phase: Failed + message: + contains: + - registry down + observedGeneration: current + conditions: + - type: Applied + reason: ChartResolveFailed + installs: 0 diff --git a/internal/controller/serving/testdata/controller/honors-force-conflicts-only-for-the-current-generation.yaml b/internal/controller/serving/testdata/controller/honors-force-conflicts-only-for-the-current-generation.yaml new file mode 100644 index 0000000..c75d686 --- /dev/null +++ b/internal/controller/serving/testdata/controller/honors-force-conflicts-only-for-the-current-generation.yaml @@ -0,0 +1,27 @@ +name: honors-force-conflicts-only-for-the-current-generation +description: honors force-conflicts only for the current generation +steps: + - action: create + version: "0.8.6" + annotations: + serving.modelsphere.dev/force-conflicts: "1" + - action: reconcile + expect: + lastForce: true + history: + entries: + - index: 0 + forceConflicts: true + - action: update + layers: + - name: site + values: '{"replicaCount":2}' + expect: + generation: 2 + - action: reconcile + expect: + lastForce: false + history: + entries: + - index: 0 + forceConflicts: false diff --git a/internal/controller/serving/testdata/controller/installs-again-after-a-failed-install-and-a-transient-chart-list-error.yaml b/internal/controller/serving/testdata/controller/installs-again-after-a-failed-install-and-a-transient-chart-list-error.yaml new file mode 100644 index 0000000..77cc169 --- /dev/null +++ b/internal/controller/serving/testdata/controller/installs-again-after-a-failed-install-and-a-transient-chart-list-error.yaml @@ -0,0 +1,57 @@ +name: installs-again-after-a-failed-install-and-a-transient-chart-list-error +description: installs again after a failed install and a transient chart list error +steps: + - action: setHelm + installErr: boom + keepFailed: true + - action: create + version: ">=0.8.0" + - action: reconcile + expect: + errorContains: boom + appliedHash: + empty: true + history: + length: 0 + releaseUID: self + release: + labels: + serving.modelsphere.dev/llmservice-uid: self + status: failed + conditions: + - type: Applied + reason: ApplyFailed + - action: setVersions + listErr: registry down + - action: reconcile + expect: + errorContains: registry down + phase: Failed + conditions: + - type: Applied + reason: ChartResolveFailed + - type: Adopted + absent: true + releaseUID: self + installs: 1 + upgrades: 0 + uninstalls: 0 + - action: setVersions + listErr: "" + - action: setHelm + installErr: "" + - action: reconcile + expect: + installs: 2 + upgrades: 0 + uninstalls: 1 + rollbacks: 0 + phase: Applied + conditions: + - type: Applied + reason: Installed + - type: Adopted + absent: true + message: + notContains: + - Drift diff --git a/internal/controller/serving/testdata/controller/installs-and-records-a-controllerrevision.yaml b/internal/controller/serving/testdata/controller/installs-and-records-a-controllerrevision.yaml new file mode 100644 index 0000000..48dffcc --- /dev/null +++ b/internal/controller/serving/testdata/controller/installs-and-records-a-controllerrevision.yaml @@ -0,0 +1,44 @@ +name: installs-and-records-a-controllerrevision +description: installs and records a ControllerRevision +steps: + - action: create + version: ">=0.8.0" + - action: reconcile + expect: + installs: 1 + upgrades: 0 + finalizer: present + phase: Applied + chartVersion: "0.8.6" + appliedHash: + prefix: "sha256:" + helmRevision: 1 + observedGeneration: current + history: + length: 1 + entries: + - index: 0 + action: install + annotationsPresent: + - swiss.modelsphere.dev/note + annotationsAbsent: + - unrelated + conditions: + - type: Applied + status: "True" + reason: Installed + releaseUID: self + release: + labels: + serving.modelsphere.dev/llmservice-uid: self + serving.modelsphere.dev/llmservice: qwen + snapshots: + - index: 0 + labels: + serving.modelsphere.dev/llmservice: qwen + controllerOwner: true + annotations: + swiss.modelsphere.dev/action: install + annotationsAbsent: + - unrelated + chartName: sglang diff --git a/internal/controller/serving/testdata/controller/keeps-a-release-it-never-installed-or-adopted-on-delete.yaml b/internal/controller/serving/testdata/controller/keeps-a-release-it-never-installed-or-adopted-on-delete.yaml new file mode 100644 index 0000000..4f761d9 --- /dev/null +++ b/internal/controller/serving/testdata/controller/keeps-a-release-it-never-installed-or-adopted-on-delete.yaml @@ -0,0 +1,61 @@ +name: keeps-a-release-it-never-installed-or-adopted-on-delete +description: keeps a release it never installed or adopted on delete +steps: + - action: putRelease + release: + revision: 3 + status: deployed + chartName: sglang + chartVersion: "0.8.6" + config: + replicaCount: 9 + - action: create + version: "0.8.6" + - action: reconcile + expect: + conditions: + - type: Adopted + reason: Drift + - action: delete + - action: reconcile + expect: + uninstalls: 0 + release: + exists: true + notFound: true + - action: create + name: claimed + version: "0.8.6" + finalizers: + - serving.modelsphere.dev/finalizer + - action: setStatusReleaseUID + - action: putRelease + release: + revision: 1 + status: deployed + chartName: sglang + chartVersion: "0.8.6" + - action: delete + - action: reconcile + expect: + uninstalls: 0 + release: + exists: true + - action: create + name: other + version: "0.8.6" + finalizers: + - serving.modelsphere.dev/finalizer + - action: putRelease + release: + revision: 2 + status: deployed + chartName: sglang + chartVersion: "0.8.6" + labels: + serving.modelsphere.dev/llmservice: other + serving.modelsphere.dev/llmservice-uid: some-other-uid + - action: delete + - action: reconcile + expect: + uninstalls: 0 diff --git a/internal/controller/serving/testdata/controller/keeps-a-resolved-version-while-the-range-allows-it.yaml b/internal/controller/serving/testdata/controller/keeps-a-resolved-version-while-the-range-allows-it.yaml new file mode 100644 index 0000000..17f928a --- /dev/null +++ b/internal/controller/serving/testdata/controller/keeps-a-resolved-version-while-the-range-allows-it.yaml @@ -0,0 +1,25 @@ +name: keeps-a-resolved-version-while-the-range-allows-it +description: keeps a resolved version while the range allows it +steps: + - action: create + version: ">=0.8.0" + - action: reconcile + expect: + chartVersion: "0.8.6" + listCalls: 1 + - action: setVersions + versions: + - "0.8.6" + - "0.9.0" + - action: reconcile + expect: + chartVersion: "0.8.6" + listCalls: 1 + upgrades: 0 + - action: update + chartVersion: ">=0.9.0" + - action: reconcile + expect: + chartVersion: "0.9.0" + listCalls: 2 + upgrades: 1 diff --git a/internal/controller/serving/testdata/controller/keeps-observedgeneration-while-install-is-in-progress.yaml b/internal/controller/serving/testdata/controller/keeps-observedgeneration-while-install-is-in-progress.yaml new file mode 100644 index 0000000..d588c1b --- /dev/null +++ b/internal/controller/serving/testdata/controller/keeps-observedgeneration-while-install-is-in-progress.yaml @@ -0,0 +1,26 @@ +name: keeps-observedgeneration-while-install-is-in-progress +description: keeps observedGeneration while install is in progress +steps: + - action: create + version: "0.8.6" + - action: reconcile + expect: {} + - action: capture + capture: + - observedGeneration + expect: + observedGeneration: nonzero + - action: deleteRelease + - action: update + layers: + - name: site + values: '{"replicaCount":4}' + - action: reconcileHeld + expect: + mid: + phase: Applying + observedGeneration: captured + installs: 2 + upgrades: 0 + phase: Applied + observedGeneration: current diff --git a/internal/controller/serving/testdata/controller/leaves-status-unchanged-while-suspended.yaml b/internal/controller/serving/testdata/controller/leaves-status-unchanged-while-suspended.yaml new file mode 100644 index 0000000..e3472a5 --- /dev/null +++ b/internal/controller/serving/testdata/controller/leaves-status-unchanged-while-suspended.yaml @@ -0,0 +1,25 @@ +name: leaves-status-unchanged-while-suspended +description: leaves status unchanged while suspended +steps: + - action: create + version: "0.8.6" + - action: reconcile + expect: {} + - action: capture + capture: + - appliedHash + - observedGeneration + - action: update + suspend: true + layers: + - name: site + values: '{"replicaCount":9}' + - action: reconcile + expect: + installs: 1 + upgrades: 0 + phase: Applied + appliedHash: + captured: true + helmRevision: 1 + observedGeneration: captured diff --git a/internal/controller/serving/testdata/controller/orphans-a-delete-and-uninstalls-otherwise.yaml b/internal/controller/serving/testdata/controller/orphans-a-delete-and-uninstalls-otherwise.yaml new file mode 100644 index 0000000..104599b --- /dev/null +++ b/internal/controller/serving/testdata/controller/orphans-a-delete-and-uninstalls-otherwise.yaml @@ -0,0 +1,27 @@ +name: orphans-a-delete-and-uninstalls-otherwise +description: orphans a delete and uninstalls otherwise +steps: + - action: create + name: keep + version: "0.8.6" + - action: reconcile + expect: {} + - action: update + annotations: + serving.modelsphere.dev/deletion-policy: Orphan + - action: delete + - action: reconcile + expect: + uninstalls: 0 + getsUnchanged: true + notFound: true + - action: create + name: drop + version: "0.8.6" + - action: reconcile + expect: {} + - action: delete + - action: reconcile + expect: + uninstalls: 1 + notFound: true diff --git a/internal/controller/serving/testdata/controller/re-resolves-once-per-new-re-resolve-value.yaml b/internal/controller/serving/testdata/controller/re-resolves-once-per-new-re-resolve-value.yaml new file mode 100644 index 0000000..d0eb4c1 --- /dev/null +++ b/internal/controller/serving/testdata/controller/re-resolves-once-per-new-re-resolve-value.yaml @@ -0,0 +1,54 @@ +name: re-resolves-once-per-new-re-resolve-value +description: re-resolves once per new re-resolve value +steps: + - action: create + version: ">=0.8.0" + - action: reconcile + expect: {} + - action: update + annotations: + serving.modelsphere.dev/re-resolve: "1" + - action: reconcile + expect: + chartVersion: "0.8.6" + reResolve: "1" + listCalls: 2 + upgrades: 0 + - action: setVersions + versions: + - "0.8.6" + - "0.9.0" + - action: reconcile + expect: + chartVersion: "0.8.6" + listCalls: 2 + - action: setHelm + upgradeErr: boom + - action: update + annotations: + serving.modelsphere.dev/re-resolve: "2" + - action: reconcile + expect: + errorContains: boom + chartVersion: "0.8.6" + reResolve: "1" + - action: setHelm + upgradeErr: "" + - action: reconcile + expect: + chartVersion: "0.9.0" + reResolve: "2" + history: + entries: + - index: 0 + reResolved: true + upgrades: 2 + - action: setVersions + versions: + - "0.8.6" + - "0.9.0" + - "0.9.1" + - action: reconcile + expect: + chartVersion: "0.9.0" + upgrades: 2 diff --git a/internal/controller/serving/testdata/controller/records-a-helm-failure.yaml b/internal/controller/serving/testdata/controller/records-a-helm-failure.yaml new file mode 100644 index 0000000..b997750 --- /dev/null +++ b/internal/controller/serving/testdata/controller/records-a-helm-failure.yaml @@ -0,0 +1,20 @@ +name: records-a-helm-failure +description: records a helm failure +steps: + - action: setHelm + installErr: boom + - action: create + version: "0.8.6" + - action: reconcile + expect: + errorContains: boom + phase: Failed + message: + contains: + - boom + observedGeneration: current + conditions: + - type: Applied + reason: ApplyFailed + status: "False" + releaseUID: self diff --git a/internal/controller/serving/testdata/controller/records-a-rollback-to-earlier-values-as-a-controllerrevision-of-its-own.yaml b/internal/controller/serving/testdata/controller/records-a-rollback-to-earlier-values-as-a-controllerrevision-of-its-own.yaml new file mode 100644 index 0000000..e5ac402 --- /dev/null +++ b/internal/controller/serving/testdata/controller/records-a-rollback-to-earlier-values-as-a-controllerrevision-of-its-own.yaml @@ -0,0 +1,41 @@ +name: records-a-rollback-to-earlier-values-as-a-controllerrevision-of-its-own +description: records a rollback to earlier values as a ControllerRevision of its own +steps: + - action: create + version: "0.8.6" + - action: reconcile + expect: {} + - action: update + layers: + - name: site + values: '{"replicaCount":2}' + - action: reconcile + expect: {} + - action: update + layers: + - name: site + values: '{"replicaCount":1}' + annotations: + swiss.modelsphere.dev/action: rollback + swiss.modelsphere.dev/plan-hash: p3 + - action: reconcile + expect: + history: + length: 3 + entries: + - index: 0 + action: rollback + revision: 3 + hashEqualsApplied: true + hashEqualsIndex: 2 + distinctRevisions: 3 + snapshots: + - index: 0 + annotations: + swiss.modelsphere.dev/action: rollback + - index: 2 + annotations: + swiss.modelsphere.dev/action: install + - action: reconcile + expect: + upgradesUnchanged: true diff --git a/internal/controller/serving/testdata/controller/recovers-a-stale-owned-pending-install-and-installs.yaml b/internal/controller/serving/testdata/controller/recovers-a-stale-owned-pending-install-and-installs.yaml new file mode 100644 index 0000000..487b71f --- /dev/null +++ b/internal/controller/serving/testdata/controller/recovers-a-stale-owned-pending-install-and-installs.yaml @@ -0,0 +1,23 @@ +name: recovers-a-stale-owned-pending-install-and-installs +description: recovers a stale owned pending-install and installs +steps: + - action: create + version: "0.8.6" + - action: putRelease + release: + owned: true + revision: 1 + status: pending-install + updated: "-1h" + - action: reconcile + expect: + uninstalls: 1 + installs: 1 + upgrades: 0 + rollbacks: 0 + phase: Applied + conditions: + - type: Applied + reason: Installed + releaseUID: self + eventContains: RecoveredPendingRelease diff --git a/internal/controller/serving/testdata/controller/refuses-a-chart-repository-it-was-not-started-to-install-from.yaml b/internal/controller/serving/testdata/controller/refuses-a-chart-repository-it-was-not-started-to-install-from.yaml new file mode 100644 index 0000000..64075d2 --- /dev/null +++ b/internal/controller/serving/testdata/controller/refuses-a-chart-repository-it-was-not-started-to-install-from.yaml @@ -0,0 +1,26 @@ +name: refuses-a-chart-repository-it-was-not-started-to-install-from +description: refuses a chart repository it was not started to install from +steps: + - action: setAllowed + repos: + - oci://ghcr.io/other + - action: create + version: "0.8.6" + - action: reconcile + expect: + helmTouched: 0 + listCalls: 0 + phase: Failed + conditions: + - type: Applied + reason: ChartRepoNotAllowed + message: + contains: + - oci://ghcr.io/modelsphere/charts + - action: setAllowed + repos: + - oci://ghcr.io/modelsphere + - action: reconcile + expect: + installs: 1 + phase: Applied diff --git a/internal/controller/serving/testdata/controller/replaces-its-own-stale-snapshot-under-a-reused-revision-name.yaml b/internal/controller/serving/testdata/controller/replaces-its-own-stale-snapshot-under-a-reused-revision-name.yaml new file mode 100644 index 0000000..652fd73 --- /dev/null +++ b/internal/controller/serving/testdata/controller/replaces-its-own-stale-snapshot-under-a-reused-revision-name.yaml @@ -0,0 +1,14 @@ +name: replaces-its-own-stale-snapshot-under-a-reused-revision-name +description: replaces its own stale snapshot under a reused revision name +steps: + - action: create + version: "0.8.6" + - action: plantStale + - action: reconcile + expect: + phase: Applied + plantedRevisionUsed: true + snapshots: + - index: 0 + annotations: + swiss.modelsphere.dev/action: install diff --git a/internal/controller/serving/testdata/controller/requeues-a-helm-get-error-instead-of-drifting-a-failed-release.yaml b/internal/controller/serving/testdata/controller/requeues-a-helm-get-error-instead-of-drifting-a-failed-release.yaml new file mode 100644 index 0000000..0c469a0 --- /dev/null +++ b/internal/controller/serving/testdata/controller/requeues-a-helm-get-error-instead-of-drifting-a-failed-release.yaml @@ -0,0 +1,35 @@ +name: requeues-a-helm-get-error-instead-of-drifting-a-failed-release +description: requeues a helm get error instead of drifting a failed release +steps: + - action: setHelm + installErr: boom + keepFailed: true + - action: create + version: "0.8.6" + - action: reconcile + expect: + errorContains: boom + - action: setHelm + installErr: "" + getErr: apiserver timeout + - action: reconcile + expect: + errorContains: apiserver timeout + conditions: + - type: Applied + reason: ApplyFailed + - type: Adopted + absent: true + uninstalls: 0 + installs: 1 + - action: setHelm + getErr: "" + - action: reconcile + expect: + uninstalls: 1 + installs: 2 + upgrades: 0 + phase: Applied + conditions: + - type: Applied + reason: Installed diff --git a/internal/controller/serving/testdata/controller/rolls-back-a-stale-owned-pending-upgrade-and-upgrades.yaml b/internal/controller/serving/testdata/controller/rolls-back-a-stale-owned-pending-upgrade-and-upgrades.yaml new file mode 100644 index 0000000..c32f519 --- /dev/null +++ b/internal/controller/serving/testdata/controller/rolls-back-a-stale-owned-pending-upgrade-and-upgrades.yaml @@ -0,0 +1,34 @@ +name: rolls-back-a-stale-owned-pending-upgrade-and-upgrades +description: rolls back a stale owned pending-upgrade and upgrades +steps: + - action: create + version: "0.8.6" + - action: putRelease + release: + owned: true + revision: 5 + status: pending-upgrade + updated: "-1h" + - action: setHistory + entries: + - owned: true + revision: 2 + status: deployed + updated: "-3h" + - owned: true + revision: 4 + status: deployed + updated: "-2h" + - slot: true + - action: reconcile + expect: + rollbacks: 1 + lastRollback: 4 + upgrades: 1 + uninstalls: 0 + installs: 0 + phase: Applied + conditions: + - type: Applied + reason: Upgraded + eventContains: RecoveredPendingRelease diff --git a/internal/controller/serving/testdata/controller/still-drifts-a-foreign-failed-release-when-applied-is-applyfailed.yaml b/internal/controller/serving/testdata/controller/still-drifts-a-foreign-failed-release-when-applied-is-applyfailed.yaml new file mode 100644 index 0000000..cc1d5d1 --- /dev/null +++ b/internal/controller/serving/testdata/controller/still-drifts-a-foreign-failed-release-when-applied-is-applyfailed.yaml @@ -0,0 +1,33 @@ +name: still-drifts-a-foreign-failed-release-when-applied-is-applyfailed +description: still drifts a foreign failed release when Applied is ApplyFailed +steps: + - action: create + version: "0.8.6" + - action: setStatus + status: + phase: Failed + conditions: + - type: Applied + status: "False" + reason: ApplyFailed + message: boom + observedGeneration: current + - action: putRelease + release: + revision: 1 + status: failed + chartName: sglang + chartVersion: "0.8.6" + config: + replicaCount: 1 + - action: reconcile + expect: + installs: 0 + upgrades: 0 + uninstalls: 0 + phase: Failed + conditions: + - type: Adopted + reason: Drift + - type: Applied + reason: Drift diff --git a/internal/controller/serving/testdata/controller/uninstalls-a-stale-owned-pending-rollback-that-has-no-deployed-revision.yaml b/internal/controller/serving/testdata/controller/uninstalls-a-stale-owned-pending-rollback-that-has-no-deployed-revision.yaml new file mode 100644 index 0000000..e0787c7 --- /dev/null +++ b/internal/controller/serving/testdata/controller/uninstalls-a-stale-owned-pending-rollback-that-has-no-deployed-revision.yaml @@ -0,0 +1,28 @@ +name: uninstalls-a-stale-owned-pending-rollback-that-has-no-deployed-revision +description: uninstalls a stale owned pending-rollback that has no deployed revision +steps: + - action: create + version: "0.8.6" + - action: putRelease + release: + owned: true + revision: 2 + status: pending-rollback + updated: "-1h" + - action: setHistory + entries: + - owned: true + revision: 1 + status: failed + updated: "-2h" + - slot: true + - action: reconcile + expect: + rollbacks: 0 + uninstalls: 1 + installs: 1 + upgrades: 0 + phase: Applied + conditions: + - type: Applied + reason: Installed diff --git a/internal/controller/serving/testdata/controller/uninstalls-an-adopted-release-on-delete.yaml b/internal/controller/serving/testdata/controller/uninstalls-an-adopted-release-on-delete.yaml new file mode 100644 index 0000000..5e9fc95 --- /dev/null +++ b/internal/controller/serving/testdata/controller/uninstalls-an-adopted-release-on-delete.yaml @@ -0,0 +1,25 @@ +name: uninstalls-an-adopted-release-on-delete +description: uninstalls an adopted release on delete +steps: + - action: putRelease + release: + revision: 7 + status: deployed + chartName: sglang + chartVersion: "0.8.6" + config: + replicaCount: 1 + - action: create + version: "0.8.6" + - action: reconcile + expect: + conditions: + - type: Adopted + status: "True" + release: + labelsAbsent: + - serving.modelsphere.dev/llmservice-uid + - action: delete + - action: reconcile + expect: + uninstalls: 1 diff --git a/internal/controller/serving/testdata/controller/upgrades-an-owned-deployed-release-whose-status-was-lost.yaml b/internal/controller/serving/testdata/controller/upgrades-an-owned-deployed-release-whose-status-was-lost.yaml new file mode 100644 index 0000000..d909358 --- /dev/null +++ b/internal/controller/serving/testdata/controller/upgrades-an-owned-deployed-release-whose-status-was-lost.yaml @@ -0,0 +1,24 @@ +name: upgrades-an-owned-deployed-release-whose-status-was-lost +description: upgrades an owned deployed release whose status was lost +steps: + - action: create + version: "0.8.6" + - action: putRelease + release: + owned: true + revision: 4 + status: deployed + updated: "-1h" + - action: reconcile + expect: + upgrades: 1 + installs: 0 + uninstalls: 0 + phase: Applied + conditions: + - type: Applied + reason: Upgraded + - type: Adopted + absent: true + releaseUID: self + helmRevision: 5 diff --git a/internal/controller/serving/testdata/controller/upgrades-an-owned-failed-release-that-still-has-a-deployed-revision.yaml b/internal/controller/serving/testdata/controller/upgrades-an-owned-failed-release-that-still-has-a-deployed-revision.yaml new file mode 100644 index 0000000..138122d --- /dev/null +++ b/internal/controller/serving/testdata/controller/upgrades-an-owned-failed-release-that-still-has-a-deployed-revision.yaml @@ -0,0 +1,28 @@ +name: upgrades-an-owned-failed-release-that-still-has-a-deployed-revision +description: upgrades an owned failed release that still has a deployed revision +steps: + - action: create + version: "0.8.6" + - action: reconcile + expect: {} + - action: setHistory + entries: + - slot: true + - owned: true + revision: 2 + status: failed + updated: "0s" + put: last + - action: update + layers: + - name: site + values: '{"replicaCount":2}' + - action: reconcile + expect: + upgrades: 1 + uninstalls: 0 + installs: 1 + phase: Applied + conditions: + - type: Applied + reason: Upgraded diff --git a/internal/controller/serving/testdata/controller/upgrades-and-bounds-history.yaml b/internal/controller/serving/testdata/controller/upgrades-and-bounds-history.yaml new file mode 100644 index 0000000..6be49e4 --- /dev/null +++ b/internal/controller/serving/testdata/controller/upgrades-and-bounds-history.yaml @@ -0,0 +1,34 @@ +name: upgrades-and-bounds-history +description: upgrades and bounds history +historyLimit: 2 +steps: + - action: create + version: "0.8.6" + - action: reconcile + expect: {} + - action: capture + capture: + - revision + - action: update + layers: + - name: site + values: '{"replicaCount":2}' + - action: reconcile + expect: {} + - action: update + layers: + - name: site + values: '{"replicaCount":3}' + - action: reconcile + expect: + upgrades: 2 + history: + length: 2 + entries: + - index: 0 + revision: 3 + - index: 1 + revision: 2 + helmRevision: 3 + capturedRevisionGone: true + latestRevisionExists: true diff --git a/internal/controller/serving/testdata/controller/waits-out-a-fresh-owned-pending-upgrade.yaml b/internal/controller/serving/testdata/controller/waits-out-a-fresh-owned-pending-upgrade.yaml new file mode 100644 index 0000000..0b8a08a --- /dev/null +++ b/internal/controller/serving/testdata/controller/waits-out-a-fresh-owned-pending-upgrade.yaml @@ -0,0 +1,42 @@ +name: waits-out-a-fresh-owned-pending-upgrade +description: waits out a fresh owned pending-upgrade +pendingTimeout: "2m" +steps: + - action: create + version: "0.8.6" + - action: putRelease + release: + owned: true + revision: 2 + status: pending-upgrade + updated: "0s" + - action: setHistory + entries: + - owned: true + revision: 1 + status: deployed + updated: "-1h" + - slot: true + - action: reconcile + expect: + requeue: + atLeast: "15s" + above: "1m" + atMost: "2m" + installs: 0 + upgrades: 0 + uninstalls: 0 + rollbacks: 0 + phase: Applying + conditions: + - type: Applied + reason: OperationInProgress + message: + contains: + - pending-upgrade + - action: reconcile + expect: + requeue: + atLeast: "15s" + rollbacks: 0 + upgrades: 0 diff --git a/internal/controller/serving/testdata/predicate/annotation-change.yaml b/internal/controller/serving/testdata/predicate/annotation-change.yaml new file mode 100644 index 0000000..6fb070d --- /dev/null +++ b/internal/controller/serving/testdata/predicate/annotation-change.yaml @@ -0,0 +1,6 @@ +name: annotation-change +description: annotation change +op: update +annotations: + a: "2" +want: true diff --git a/internal/controller/serving/testdata/predicate/create.yaml b/internal/controller/serving/testdata/predicate/create.yaml new file mode 100644 index 0000000..81a4441 --- /dev/null +++ b/internal/controller/serving/testdata/predicate/create.yaml @@ -0,0 +1,4 @@ +name: create +description: create reconciles +op: create +want: true diff --git a/internal/controller/serving/testdata/predicate/delete.yaml b/internal/controller/serving/testdata/predicate/delete.yaml new file mode 100644 index 0000000..c81d4aa --- /dev/null +++ b/internal/controller/serving/testdata/predicate/delete.yaml @@ -0,0 +1,4 @@ +name: delete +description: delete reconciles +op: delete +want: true diff --git a/internal/controller/serving/testdata/predicate/deletion-timestamp.yaml b/internal/controller/serving/testdata/predicate/deletion-timestamp.yaml new file mode 100644 index 0000000..78803dc --- /dev/null +++ b/internal/controller/serving/testdata/predicate/deletion-timestamp.yaml @@ -0,0 +1,5 @@ +name: deletion-timestamp +description: deletion timestamp +op: update +deleting: true +want: true diff --git a/internal/controller/serving/testdata/predicate/generation-change.yaml b/internal/controller/serving/testdata/predicate/generation-change.yaml new file mode 100644 index 0000000..53b8bd0 --- /dev/null +++ b/internal/controller/serving/testdata/predicate/generation-change.yaml @@ -0,0 +1,5 @@ +name: generation-change +description: generation change +op: update +generation: 2 +want: true diff --git a/internal/controller/serving/testdata/predicate/status-only-ignored.yaml b/internal/controller/serving/testdata/predicate/status-only-ignored.yaml new file mode 100644 index 0000000..8504500 --- /dev/null +++ b/internal/controller/serving/testdata/predicate/status-only-ignored.yaml @@ -0,0 +1,5 @@ +name: status-only-ignored +description: status-only ignored +op: update +phase: Applied +want: false diff --git a/internal/controller/serving/testdata/repo/empty.yaml b/internal/controller/serving/testdata/repo/empty.yaml new file mode 100644 index 0000000..cec8688 --- /dev/null +++ b/internal/controller/serving/testdata/repo/empty.yaml @@ -0,0 +1,7 @@ +name: empty +description: empty repository is refused +allowed: + - oci://ghcr.io/modelsphere + - https://modelsphere.github.io/helm-charts/ +repo: "" +want: false diff --git a/internal/controller/serving/testdata/repo/https-modelsphere-github-io-helm-charts.yaml b/internal/controller/serving/testdata/repo/https-modelsphere-github-io-helm-charts.yaml new file mode 100644 index 0000000..c912e68 --- /dev/null +++ b/internal/controller/serving/testdata/repo/https-modelsphere-github-io-helm-charts.yaml @@ -0,0 +1,7 @@ +name: https-modelsphere-github-io-helm-charts +description: "https://modelsphere.github.io/helm-charts" +allowed: + - oci://ghcr.io/modelsphere + - https://modelsphere.github.io/helm-charts/ +repo: "https://modelsphere.github.io/helm-charts" +want: true diff --git a/internal/controller/serving/testdata/repo/https-modelsphere-github-io.yaml b/internal/controller/serving/testdata/repo/https-modelsphere-github-io.yaml new file mode 100644 index 0000000..025bfdd --- /dev/null +++ b/internal/controller/serving/testdata/repo/https-modelsphere-github-io.yaml @@ -0,0 +1,7 @@ +name: https-modelsphere-github-io +description: "https://modelsphere.github.io" +allowed: + - oci://ghcr.io/modelsphere + - https://modelsphere.github.io/helm-charts/ +repo: "https://modelsphere.github.io" +want: false diff --git a/internal/controller/serving/testdata/repo/nil-allowlist.yaml b/internal/controller/serving/testdata/repo/nil-allowlist.yaml new file mode 100644 index 0000000..9a153b2 --- /dev/null +++ b/internal/controller/serving/testdata/repo/nil-allowlist.yaml @@ -0,0 +1,5 @@ +name: nil-allowlist +description: no allowlist allows any repository +nilAllowlist: true +repo: https://anything.example.com/charts +want: true diff --git a/internal/controller/serving/testdata/repo/oci-ghcr-io-modelsphere-charts-slash.yaml b/internal/controller/serving/testdata/repo/oci-ghcr-io-modelsphere-charts-slash.yaml new file mode 100644 index 0000000..b6185ce --- /dev/null +++ b/internal/controller/serving/testdata/repo/oci-ghcr-io-modelsphere-charts-slash.yaml @@ -0,0 +1,7 @@ +name: oci-ghcr-io-modelsphere-charts-slash +description: "oci://ghcr.io/modelsphere/charts/" +allowed: + - oci://ghcr.io/modelsphere + - https://modelsphere.github.io/helm-charts/ +repo: "oci://ghcr.io/modelsphere/charts/" +want: true diff --git a/internal/controller/serving/testdata/repo/oci-ghcr-io-modelsphere-charts.yaml b/internal/controller/serving/testdata/repo/oci-ghcr-io-modelsphere-charts.yaml new file mode 100644 index 0000000..1257aa8 --- /dev/null +++ b/internal/controller/serving/testdata/repo/oci-ghcr-io-modelsphere-charts.yaml @@ -0,0 +1,7 @@ +name: oci-ghcr-io-modelsphere-charts +description: "oci://ghcr.io/modelsphere/charts" +allowed: + - oci://ghcr.io/modelsphere + - https://modelsphere.github.io/helm-charts/ +repo: "oci://ghcr.io/modelsphere/charts" +want: true diff --git a/internal/controller/serving/testdata/repo/oci-ghcr-io-modelsphere-x-charts.yaml b/internal/controller/serving/testdata/repo/oci-ghcr-io-modelsphere-x-charts.yaml new file mode 100644 index 0000000..d1896fd --- /dev/null +++ b/internal/controller/serving/testdata/repo/oci-ghcr-io-modelsphere-x-charts.yaml @@ -0,0 +1,7 @@ +name: oci-ghcr-io-modelsphere-x-charts +description: "oci://ghcr.io/modelsphere-x/charts" +allowed: + - oci://ghcr.io/modelsphere + - https://modelsphere.github.io/helm-charts/ +repo: "oci://ghcr.io/modelsphere-x/charts" +want: false diff --git a/internal/controller/serving/testdata/repo/oci-ghcr-io-modelsphere.yaml b/internal/controller/serving/testdata/repo/oci-ghcr-io-modelsphere.yaml new file mode 100644 index 0000000..8af55ed --- /dev/null +++ b/internal/controller/serving/testdata/repo/oci-ghcr-io-modelsphere.yaml @@ -0,0 +1,7 @@ +name: oci-ghcr-io-modelsphere +description: "oci://ghcr.io/modelsphere" +allowed: + - oci://ghcr.io/modelsphere + - https://modelsphere.github.io/helm-charts/ +repo: "oci://ghcr.io/modelsphere" +want: true diff --git a/internal/controller/serving/testdata/repo/oci-ghcr-io-other-charts.yaml b/internal/controller/serving/testdata/repo/oci-ghcr-io-other-charts.yaml new file mode 100644 index 0000000..ffde8a6 --- /dev/null +++ b/internal/controller/serving/testdata/repo/oci-ghcr-io-other-charts.yaml @@ -0,0 +1,7 @@ +name: oci-ghcr-io-other-charts +description: "oci://ghcr.io/other/charts" +allowed: + - oci://ghcr.io/modelsphere + - https://modelsphere.github.io/helm-charts/ +repo: "oci://ghcr.io/other/charts" +want: false diff --git a/internal/controller/serving/testdata/values/hash/stable-and-sensitive.yaml b/internal/controller/serving/testdata/values/hash/stable-and-sensitive.yaml new file mode 100644 index 0000000..212e663 --- /dev/null +++ b/internal/controller/serving/testdata/values/hash/stable-and-sensitive.yaml @@ -0,0 +1,32 @@ +name: stable-and-sensitive +description: order-stable sha256 and sensitive to version and values +checks: + - kind: stable + chart: sglang + version: "0.8.6" + prefix: "sha256:" + hashLen: 71 + left: + b: 1 + a: + d: 1 + c: 2 + right: + a: + c: 2 + d: 1 + b: 1 + - kind: nilEqualsEmpty + - kind: version + chart: sglang + version: "0.8.7" + values: + b: 1 + a: + d: 1 + c: 2 + - kind: values + chart: sglang + version: "0.8.6" + values: + b: 2 diff --git a/internal/controller/serving/testdata/values/merge/flat.yaml b/internal/controller/serving/testdata/values/merge/flat.yaml new file mode 100644 index 0000000..ebe0575 --- /dev/null +++ b/internal/controller/serving/testdata/values/merge/flat.yaml @@ -0,0 +1,23 @@ +name: flat +description: flat merge of four layers +layers: + - name: a + values: '{"x":1,"m":{"a":1,"b":2},"s":[1],"keep":true}' + - name: b + values: '{"x":2,"m":{"b":null,"c":3},"s":[2,3],"n":null}' + - name: empty + - name: scalar + values: '{"m":"replaced"}' +checks: + - path: ["x"] + equals: 2 + - path: ["n"] + absent: true + - path: ["keep"] + equals: true + - path: ["m"] + equals: replaced + - path: ["s"] + equals: + - 2 + - 3 diff --git a/internal/controller/serving/testdata/values/merge/nested.yaml b/internal/controller/serving/testdata/values/merge/nested.yaml new file mode 100644 index 0000000..8c7291e --- /dev/null +++ b/internal/controller/serving/testdata/values/merge/nested.yaml @@ -0,0 +1,12 @@ +name: nested +description: nested maps merge recursively +layers: + - name: a + values: '{"m":{"a":1,"b":{"c":1}}}' + - name: b + values: '{"m":{"b":{"d":2}}}' +checks: + - path: ["m", "b", "c"] + equals: 1 + - path: ["m", "b", "d"] + equals: 2 diff --git a/internal/controller/serving/values_test.go b/internal/controller/serving/values_test.go index 82b7f64..4980a58 100644 --- a/internal/controller/serving/values_test.go +++ b/internal/controller/serving/values_test.go @@ -1,80 +1,125 @@ package serving import ( + "reflect" "strings" "testing" - apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" - - servingv1alpha1 "github.com/modelsphere/llm-operator/api/serving/v1alpha1" + "github.com/modelsphere/llm-operator/internal/testutil/cases" ) -func raw(s string) *apiextensionsv1.JSON { - return &apiextensionsv1.JSON{Raw: []byte(s)} +// mergeCase is one mergeLayers call and the values it must produce. +// JSON numbers in a layer are float64, and checks compare with the same type. +type mergeCase struct { + Name string `json:"name,omitempty"` + Description string `json:"description,omitempty"` + Layers []layerSpec `json:"layers"` + Checks []mergeCheck `json:"checks"` +} + +type mergeCheck struct { + Path []string `json:"path"` + Absent bool `json:"absent,omitempty"` + Equals any `json:"equals,omitempty"` } func TestMergeLayers(t *testing.T) { - got, err := mergeLayers([]servingv1alpha1.Layer{ - {Name: "a", Values: raw(`{"x":1,"m":{"a":1,"b":2},"s":[1],"keep":true}`)}, - {Name: "b", Values: raw(`{"x":2,"m":{"b":null,"c":3},"s":[2,3],"n":null}`)}, - {Name: "empty"}, - {Name: "scalar", Values: raw(`{"m":"replaced"}`)}, + cases.Run(t, "testdata/values/merge", func(t *testing.T, tc mergeCase) { + got, err := mergeLayers(layersFrom(tc.Layers)) + if err != nil { + t.Fatal(err) + } + for _, check := range tc.Checks { + val, ok := lookupPath(got, check.Path) + if check.Absent { + if ok { + t.Errorf("%s: %v was kept: %#v", tc.Description, check.Path, val) + } + continue + } + if !ok || !reflect.DeepEqual(val, check.Equals) { + t.Errorf("%s: %v = %#v, want %#v", tc.Description, check.Path, val, check.Equals) + } + } }) - if err != nil { - t.Fatal(err) - } - if got["x"] != float64(2) { - t.Errorf("x = %#v", got["x"]) - } - if _, ok := got["n"]; ok { - t.Errorf("null key n was kept: %#v", got["n"]) - } - if got["keep"] != true { - t.Errorf("keep = %#v", got["keep"]) - } - m, ok := got["m"].(string) - if !ok || m != "replaced" { - t.Errorf("map replaced by scalar = %#v", got["m"]) - } - s, ok := got["s"].([]any) - if !ok || len(s) != 2 || s[0] != float64(2) || s[1] != float64(3) { - t.Errorf("slice = %#v", got["s"]) - } +} - nested, err := mergeLayers([]servingv1alpha1.Layer{ - {Name: "a", Values: raw(`{"m":{"a":1,"b":{"c":1}}}`)}, - {Name: "b", Values: raw(`{"m":{"b":{"d":2}}}`)}, - }) - if err != nil { - t.Fatal(err) - } - inner := nested["m"].(map[string]any)["b"].(map[string]any) - if inner["c"] != float64(1) || inner["d"] != float64(2) { - t.Errorf("nested merge = %#v", nested["m"]) +func lookupPath(root map[string]any, path []string) (any, bool) { + var cur any = root + for _, p := range path { + m, ok := cur.(map[string]any) + if !ok { + return nil, false + } + cur, ok = m[p] + if !ok { + return nil, false + } } + return cur, true +} + +// hashCase runs ordered appliedHash checks. The stable check records the +// digest the later checks compare against, so it has to come first. +type hashCase struct { + Name string `json:"name,omitempty"` + Description string `json:"description,omitempty"` + Checks []hashCheck `json:"checks"` +} + +type hashCheck struct { + Kind string `json:"kind"` + Chart string `json:"chart,omitempty"` + Version string `json:"version,omitempty"` + Left map[string]any `json:"left,omitempty"` + Right map[string]any `json:"right,omitempty"` + Values map[string]any `json:"values,omitempty"` + Prefix string `json:"prefix,omitempty"` + HashLen int `json:"hashLen,omitempty"` } func TestAppliedHashIsStableAndSensitive(t *testing.T) { - a, err := appliedHash("sglang", "0.8.6", map[string]any{"b": 1, "a": map[string]any{"d": 1, "c": 2}}) - if err != nil { - t.Fatal(err) - } - b, err := appliedHash("sglang", "0.8.6", map[string]any{"a": map[string]any{"c": 2, "d": 1}, "b": 1}) - if err != nil { - t.Fatal(err) - } - if a != b || !strings.HasPrefix(a, "sha256:") || len(a) != len("sha256:")+64 { - t.Fatalf("hash = %s and %s", a, b) - } - if same, err := jsonEqual(nil, map[string]any{}); err != nil || !same { - t.Fatalf("nil and empty map: %v %v", same, err) - } - changed, err := appliedHash("sglang", "0.8.7", map[string]any{"b": 1, "a": map[string]any{"d": 1, "c": 2}}) - if err != nil || changed == a { - t.Fatalf("version change did not move the hash: %s", changed) - } - changed, err = appliedHash("sglang", "0.8.6", map[string]any{"b": 2}) - if err != nil || changed == a { - t.Fatalf("values change did not move the hash: %s", changed) - } + cases.Run(t, "testdata/values/hash", func(t *testing.T, tc hashCase) { + var base string + for _, check := range tc.Checks { + switch check.Kind { + case "stable": + a, err := appliedHash(check.Chart, check.Version, check.Left) + if err != nil { + t.Fatal(err) + } + b, err := appliedHash(check.Chart, check.Version, check.Right) + if err != nil { + t.Fatal(err) + } + if a != b || !strings.HasPrefix(a, check.Prefix) || len(a) != check.HashLen { + t.Fatalf("hash = %s and %s", a, b) + } + base = a + case "nilEqualsEmpty": + same, err := jsonEqual(nil, map[string]any{}) + if err != nil || !same { + t.Fatalf("nil and empty map: %v %v", same, err) + } + case "version": + if base == "" { + t.Fatal("stable hash was not recorded") + } + changed, err := appliedHash(check.Chart, check.Version, check.Values) + if err != nil || changed == base { + t.Fatalf("version change did not move the hash: %s", changed) + } + case "values": + if base == "" { + t.Fatal("stable hash was not recorded") + } + changed, err := appliedHash(check.Chart, check.Version, check.Values) + if err != nil || changed == base { + t.Fatalf("values change did not move the hash: %s", changed) + } + default: + t.Fatalf("%s: unknown check %q", tc.Name, check.Kind) + } + } + }) } diff --git a/internal/helm/guard_test.go b/internal/helm/guard_test.go index db45f51..4702947 100644 --- a/internal/helm/guard_test.go +++ b/internal/helm/guard_test.go @@ -6,73 +6,111 @@ import ( "path/filepath" "strings" "testing" + + "github.com/modelsphere/llm-operator/internal/testutil/cases" ) -const clusterScoped = "cluster-scoped" +// namespaceCase is one post-render. An empty refused string means the +// manifest is returned unchanged and must equal the input bytes. +type namespaceCase struct { + Name string `json:"name,omitempty"` + Description string `json:"description,omitempty"` + Manifest string `json:"manifest"` + Refused string `json:"refused,omitempty"` +} func TestNamespaceGuard(t *testing.T) { - g := namespaceGuard{namespace: "models", mapper: testMapper()} - for name, tc := range map[string]struct { - manifest, refused string - }{ - "own namespace": {"apiVersion: v1\nkind: ConfigMap\nmetadata:\n name: a\n namespace: models\n", ""}, - "no namespace": {"apiVersion: v1\nkind: ConfigMap\nmetadata:\n name: a\n", ""}, - "comments only": {"# Source: x/templates/empty.yaml\n---\n", ""}, - "another namespace": {"apiVersion: v1\nkind: ConfigMap\nmetadata:\n name: a\n namespace: kube-system\n", "into namespace kube-system"}, - clusterScoped: {"apiVersion: rbac.authorization.k8s.io/v1\nkind: ClusterRole\nmetadata:\n name: a\n", "ClusterRole \"a\", which is cluster-scoped"}, - "namespace object": {"apiVersion: v1\nkind: Namespace\nmetadata:\n name: a\n", clusterScoped}, - "inside a list": {"apiVersion: v1\nkind: List\nitems:\n- apiVersion: v1\n kind: ConfigMap\n metadata:\n name: a\n" + - "- apiVersion: rbac.authorization.k8s.io/v1\n kind: Role\n metadata:\n name: b\n namespace: kube-system\n", "Role \"b\" into namespace kube-system"}, - "after a good one": {"apiVersion: v1\nkind: ConfigMap\nmetadata:\n name: a\n---\napiVersion: rbac.authorization.k8s.io/v1\nkind: ClusterRole\nmetadata:\n name: b\n", clusterScoped}, - "unknown kind": {"apiVersion: example.com/v1\nkind: Widget\nmetadata:\n name: a\n", "cannot be told"}, - "no kind": {"replicaCount: 1\n", "an object with no kind"}, - } { - out, err := g.Run(bytes.NewBufferString(tc.manifest)) - switch { - case tc.refused == "" && err != nil: - t.Errorf("%s: refused: %v", name, err) - case tc.refused == "" && out.String() != tc.manifest: - t.Errorf("%s: manifests changed:\n%s", name, out) - case tc.refused != "" && (err == nil || !strings.Contains(err.Error(), tc.refused)): - t.Errorf("%s: got %v, want an error with %q", name, err, tc.refused) + cases.Run(t, "testdata/guard/namespace", func(t *testing.T, tc namespaceCase) { + g := namespaceGuard{namespace: "models", mapper: testMapper()} + out, err := g.Run(bytes.NewBufferString(tc.Manifest)) + if tc.Refused == "" { + if err != nil { + t.Fatalf("refused: %v", err) + } + if out == nil || out.String() != tc.Manifest { + got := "" + if out != nil { + got = out.String() + } + t.Fatalf("manifests changed:\ngot %q\nwant %q", got, tc.Manifest) + } + return } - } + if err == nil || !strings.Contains(err.Error(), tc.Refused) { + t.Fatalf("got %v, want an error with %q", err, tc.Refused) + } + }) +} + +// applyCase is one install or upgrade sequence on a fresh client. +// The refused template and hook installs do not share the later upgrade. +type applyCase struct { + Name string `json:"name,omitempty"` + Description string `json:"description,omitempty"` + Steps []applyStep `json:"steps"` +} + +type applyStep struct { + Write string `json:"write,omitempty"` + Op string `json:"op,omitempty"` + ErrorContains string `json:"errorContains,omitempty"` + Absent bool `json:"absent,omitempty"` } -// Install and Upgrade run the guard, hooks included, and store nothing a -// refused chart rendered. func TestApplyRefusesAnotherNamespace(t *testing.T) { - for name, tpl := range map[string]string{ - "template": "apiVersion: v1\nkind: ConfigMap\nmetadata:\n name: escape\n namespace: kube-system\n", - "hook": "apiVersion: v1\nkind: ConfigMap\nmetadata:\n name: escape\n namespace: kube-system\n" + - " annotations:\n helm.sh/hook: pre-install,pre-upgrade\n", - } { - dir := writeTinyChart(t) - if err := os.WriteFile(filepath.Join(dir, "templates", "escape.yaml"), []byte(tpl), 0o644); err != nil { - t.Fatal(err) - } - c, _ := testClient(t, dir) - ref := ChartRef{Repo: testRepo, Name: tinyChartName, Version: tinyVersion} - if _, err := c.Install(t.Context(), testNamespace, tinyChartName, ref, nil, ApplyOpts{}); err == nil || !strings.Contains(err.Error(), "kube-system") { - t.Errorf("%s: install: got %v, want a refusal", name, err) - } - if got, err := c.Get(t.Context(), testNamespace, tinyChartName); err != nil || got != nil { - t.Errorf("%s: a refused install stored %#v, %v", name, got, err) - } - } + cases.Run(t, "testdata/guard/apply", runApplyCase) +} - // A chart that changes to write elsewhere is refused on upgrade. +func runApplyCase(t *testing.T, tc applyCase) { + t.Helper() dir := writeTinyChart(t) c, _ := testClient(t, dir) ref := ChartRef{Repo: testRepo, Name: tinyChartName, Version: tinyVersion} - if _, err := c.Install(t.Context(), testNamespace, tinyChartName, ref, nil, ApplyOpts{}); err != nil { - t.Fatal(err) - } - cluster := "apiVersion: rbac.authorization.k8s.io/v1\nkind: ClusterRole\nmetadata:\n name: escape\n" - if err := os.WriteFile(filepath.Join(dir, "templates", "escape.yaml"), []byte(cluster), 0o644); err != nil { - t.Fatal(err) - } - if _, err := c.Upgrade(t.Context(), testNamespace, tinyChartName, ref, nil, ApplyOpts{}); err == nil || !strings.Contains(err.Error(), clusterScoped) { - t.Errorf("upgrade: got %v, want a refusal", err) + ctx := t.Context() + for _, step := range tc.Steps { + if step.Write != "" { + err := os.WriteFile(filepath.Join(dir, "templates", "escape.yaml"), []byte(step.Write), 0o644) + if err != nil { + t.Fatal(err) + } + } + switch step.Op { + case "": + if step.Write == "" { + t.Fatal("step has no write and no op") + } + case "install": + _, err := c.Install(ctx, testNamespace, tinyChartName, ref, nil, ApplyOpts{}) + if step.ErrorContains == "" { + if err != nil { + t.Fatal(err) + } + continue + } + if err == nil || !strings.Contains(err.Error(), step.ErrorContains) { + t.Errorf("install: got %v, want a refusal", err) + } + case "upgrade": + _, err := c.Upgrade(ctx, testNamespace, tinyChartName, ref, nil, ApplyOpts{}) + if step.ErrorContains == "" { + if err != nil { + t.Fatal(err) + } + continue + } + if err == nil || !strings.Contains(err.Error(), step.ErrorContains) { + t.Errorf("upgrade: got %v, want a refusal", err) + } + case "get": + got, err := c.Get(ctx, testNamespace, tinyChartName) + if !step.Absent { + t.Fatal("get step expects absent") + } + if err != nil || got != nil { + t.Errorf("a refused install stored %#v, %v", got, err) + } + default: + t.Fatalf("op %q", step.Op) + } } } diff --git a/internal/helm/helm_test.go b/internal/helm/helm_test.go index 061754c..a375383 100644 --- a/internal/helm/helm_test.go +++ b/internal/helm/helm_test.go @@ -19,16 +19,15 @@ import ( "helm.sh/helm/v4/pkg/storage/driver" "k8s.io/apimachinery/pkg/api/meta" "k8s.io/apimachinery/pkg/runtime/schema" + + "github.com/modelsphere/llm-operator/internal/testutil/cases" ) const ( - tinyVersion = "0.1.0" - tinyChartName = "tiny" - testNamespace = "default" - testReleaseUID = "uid-1" - uidLabelKey = "serving.modelsphere.dev/llmservice-uid" - nameLabelKey = "serving.modelsphere.dev/llmservice" - testRepo = "oci://example.com/charts" + tinyVersion = "0.1.0" + tinyChartName = "tiny" + testNamespace = "default" + testRepo = "oci://example.com/charts" ) func testClient(t *testing.T, chartDir string) (*Client, *action.Configuration) { @@ -80,196 +79,227 @@ func writeTinyChart(t *testing.T) string { return dir } -func TestGetInstallUpgradeUninstall(t *testing.T) { - const valueOne = "one" - c, _ := testClient(t, writeTinyChart(t)) - ctx := t.Context() - const ns, name = testNamespace, tinyChartName - ref := ChartRef{Repo: testRepo, Name: name, Version: tinyVersion} - labels := testLabels(name) - before := time.Now().Add(-time.Second) - - got, err := c.Get(ctx, ns, name) - if err != nil || got != nil { - t.Fatalf("absent get = %#v, %v", got, err) +func storeRelease(t *testing.T, cfg *action.Configuration, rel *releasev1.Release) { + t.Helper() + if err := cfg.Releases.Create(rel); err != nil { + t.Fatal(err) } +} - installed, err := c.Install(ctx, ns, name, ref, map[string]any{"v": valueOne}, ApplyOpts{ForceConflicts: true, Labels: labels}) - if err != nil { - t.Fatalf("install: %v", err) - } - if installed.Revision != 1 || installed.Status != "deployed" || installed.ChartName != name || installed.ChartVersion != tinyVersion { - t.Fatalf("install release = %#v", installed) - } - if installed.Config["v"] != valueOne { - t.Fatalf("install config = %#v", installed.Config) - } - if installed.Labels[uidLabelKey] != testReleaseUID || installed.Updated.Before(before) { - t.Fatalf("install labels/updated = %#v", installed) +func storedRelease(name string, rev int, status common.Status, updated time.Time, labels map[string]string) *releasev1.Release { + return &releasev1.Release{ + Name: name, + Namespace: testNamespace, + Version: rev, + Labels: labels, + Config: map[string]any{"v": "one"}, + Manifest: "apiVersion: v1\nkind: ConfigMap\nmetadata:\n name: " + tinyChartName + "\n", + Chart: &chartv2.Chart{Metadata: &chartv2.Metadata{Name: tinyChartName, Version: tinyVersion}}, + Info: &releasev1.Info{ + Status: status, + LastDeployed: updated, + FirstDeployed: updated, + }, } +} - got, err = c.Get(ctx, ns, name) - if err != nil || got == nil || got.Revision != 1 || got.Config["v"] != valueOne { - t.Fatalf("get after install = %#v, %v", got, err) - } - if got.Labels[uidLabelKey] != testReleaseUID || got.Updated.Before(before) { - t.Fatalf("get labels/updated = %#v", got) - } +// clientCase is one client sequence. Seed times are offsets from a base +// captured at the start, stored as base.Add(offset).UTC(). +// updatedAfterStart mirrors !Updated.Before(base.Add(-time.Second)). +type clientCase struct { + Name string `json:"name,omitempty"` + Description string `json:"description,omitempty"` + Namespace string `json:"namespace,omitempty"` + ChartRepo string `json:"chartRepo,omitempty"` + ChartName string `json:"chartName,omitempty"` + ChartVersion string `json:"chartVersion,omitempty"` + Seed []seedRelease `json:"seed,omitempty"` + Steps []clientStep `json:"steps"` +} - upgraded, err := c.Upgrade(ctx, ns, name, ref, map[string]any{"v": "two"}, ApplyOpts{Labels: labels}) - if err != nil { - t.Fatalf("upgrade: %v", err) - } - if upgraded.Revision != 2 || upgraded.Config["v"] != "two" || upgraded.Status != "deployed" { - t.Fatalf("upgrade release = %#v", upgraded) - } - if upgraded.Labels[uidLabelKey] != testReleaseUID || upgraded.Updated.Before(before) { - t.Fatalf("upgrade labels/updated = %#v", upgraded) - } +type seedRelease struct { + Name string `json:"name"` + Revision int `json:"revision"` + Status string `json:"status"` + Updated cases.Duration `json:"updated"` + Labels map[string]string `json:"labels,omitempty"` +} - if err := c.Uninstall(ctx, ns, name); err != nil { - t.Fatalf("uninstall: %v", err) - } - got, err = c.Get(ctx, ns, name) - if err != nil || got != nil { - t.Fatalf("get after uninstall = %#v, %v", got, err) - } - if err := c.Uninstall(ctx, ns, name); err != nil { - t.Fatalf("uninstall absent: %v", err) - } +type clientStep struct { + Op string `json:"op"` + Release string `json:"release"` + Values map[string]any `json:"values,omitempty"` + ForceConflicts bool `json:"forceConflicts,omitempty"` + Labels map[string]string `json:"labels,omitempty"` + Revision int `json:"revision,omitempty"` + Absent bool `json:"absent,omitempty"` + Expect *releaseExpect `json:"expect,omitempty"` + History []releaseExpect `json:"history,omitempty"` +} + +type releaseExpect struct { + Revision *int `json:"revision,omitempty"` + Status string `json:"status,omitempty"` + ChartName string `json:"chartName,omitempty"` + ChartVersion string `json:"chartVersion,omitempty"` + Config map[string]string `json:"config,omitempty"` + Labels map[string]string `json:"labels,omitempty"` + Updated *cases.Duration `json:"updated,omitempty"` + UpdatedAfterStart bool `json:"updatedAfterStart,omitempty"` +} + +func TestGetInstallUpgradeUninstall(t *testing.T) { + cases.Run(t, "testdata/client/get-install-upgrade-uninstall", runClientCase) } func TestHistoryNewestFirstAndRollbackPendingUpgrade(t *testing.T) { - c, cfg := testClient(t, writeTinyChart(t)) - ctx := t.Context() - const ns, name = testNamespace, tinyChartName - labels := testLabels(name) - deployedAt := time.Now().Add(-2 * time.Hour).UTC() - failedAt := time.Now().Add(-time.Hour).UTC() - pendingAt := time.Now().Add(-30 * time.Minute).UTC() - storeRelease(t, cfg, storedRelease(name, 1, common.StatusDeployed, deployedAt, labels)) - storeRelease(t, cfg, storedRelease(name, 2, common.StatusFailed, failedAt, labels)) - storeRelease(t, cfg, storedRelease(name, 3, common.StatusPendingUpgrade, pendingAt, labels)) + cases.Run(t, "testdata/client/history-newest-first", runClientCase) +} - got, err := c.Get(ctx, ns, name) - if err != nil { - t.Fatalf("get: %v", err) - } - if got.Revision != 3 || got.Status != string(common.StatusPendingUpgrade) || !got.Updated.Equal(pendingAt) { - t.Fatalf("get pending = %#v", got) - } - if got.Labels[uidLabelKey] != testReleaseUID { - t.Fatalf("get labels = %#v", got.Labels) - } +func TestPendingInstallFailedAndAbsentHistory(t *testing.T) { + cases.Run(t, "testdata/client/pending-install-failed", runClientCase) +} + +func TestRollbackPendingRollback(t *testing.T) { + cases.Run(t, "testdata/client/rollback-pending-rollback", runClientCase) +} - hist, err := c.History(ctx, ns, name) - if err != nil { - t.Fatalf("history: %v", err) +func runClientCase(t *testing.T, tc clientCase) { + t.Helper() + base := time.Now() + ns := tc.Namespace + if ns == "" { + ns = testNamespace } - if len(hist) != 3 || hist[0].Revision != 3 || hist[1].Revision != 2 || hist[2].Revision != 1 { - t.Fatalf("history = %#v", hist) + c, cfg := testClient(t, writeTinyChart(t)) + for _, seed := range tc.Seed { + at := base.Add(seed.Updated.Std()).UTC() + rel := storedRelease(seed.Name, seed.Revision, common.Status(seed.Status), at, seed.Labels) + storeRelease(t, cfg, rel) } - if hist[0].Status != string(common.StatusPendingUpgrade) || hist[1].Status != string(common.StatusFailed) || hist[2].Status != string(common.StatusDeployed) { - t.Fatalf("history status = %#v", hist) + ref := ChartRef{Repo: tc.ChartRepo, Name: tc.ChartName, Version: tc.ChartVersion} + if ref.Repo == "" { + ref.Repo = testRepo } - if !hist[2].Updated.Equal(deployedAt) || hist[2].Labels[uidLabelKey] != testReleaseUID { - t.Fatalf("deployed history = %#v", hist[2]) + if ref.Name == "" { + ref.Name = tinyChartName } - - if err := c.Rollback(ctx, ns, name, 1); err != nil { - t.Fatalf("rollback: %v", err) + if ref.Version == "" { + ref.Version = tinyVersion } - got, err = c.Get(ctx, ns, name) - if err != nil { - t.Fatalf("get after rollback: %v", err) - } - if got.Status != string(common.StatusDeployed) || got.Revision != 4 || got.Labels[uidLabelKey] != testReleaseUID || got.ChartName != name { - t.Fatalf("rolled back = %#v", got) + ctx := t.Context() + for _, step := range tc.Steps { + runClientStep(t, ctx, c, ns, ref, base, step) } } -func TestPendingInstallFailedAndAbsentHistory(t *testing.T) { - c, cfg := testClient(t, writeTinyChart(t)) - ctx := t.Context() - const ns = testNamespace - labels := testLabels("fresh") - failedAt := time.Now().Add(-time.Hour).UTC() - installAt := time.Now().Add(-3 * time.Hour).UTC() - storeRelease(t, cfg, storedRelease("fresh", 1, common.StatusPendingInstall, installAt, labels)) - storeRelease(t, cfg, storedRelease("broke", 1, common.StatusFailed, failedAt, nil)) +func runClientStep(t *testing.T, ctx context.Context, c *Client, ns string, ref ChartRef, base time.Time, step clientStep) { + t.Helper() + if step.Release == "" { + t.Fatal("step has no release") + } + switch step.Op { + case "get": + got, err := c.Get(ctx, ns, step.Release) + if step.Absent { + if err != nil || got != nil { + t.Fatalf("absent get = %#v, %v", got, err) + } + return + } + if err != nil { + t.Fatalf("get: %v", err) + } + checkRelease(t, "get", got, step.Expect, base) + case "install": + got, err := c.Install(ctx, ns, step.Release, ref, step.Values, ApplyOpts{ + ForceConflicts: step.ForceConflicts, + Labels: step.Labels, + }) + if err != nil { + t.Fatalf("install: %v", err) + } + checkRelease(t, "install", got, step.Expect, base) + case "upgrade": + got, err := c.Upgrade(ctx, ns, step.Release, ref, step.Values, ApplyOpts{ + ForceConflicts: step.ForceConflicts, + Labels: step.Labels, + }) + if err != nil { + t.Fatalf("upgrade: %v", err) + } + checkRelease(t, "upgrade", got, step.Expect, base) + case "uninstall": + if err := c.Uninstall(ctx, ns, step.Release); err != nil { + t.Fatalf("uninstall: %v", err) + } + case "history": + hist, err := c.History(ctx, ns, step.Release) + if err != nil { + t.Fatalf("history: %v", err) + } + if step.Absent { + if hist != nil { + t.Fatalf("absent history = %#v, %v", hist, err) + } + return + } + if len(hist) != len(step.History) { + t.Fatalf("history = %#v", hist) + } + for i := range step.History { + checkRelease(t, "history", &hist[i], &step.History[i], base) + } + case "rollback": + if step.Revision == 0 { + t.Fatal("rollback has no revision") + } + if err := c.Rollback(ctx, ns, step.Release, step.Revision); err != nil { + t.Fatalf("rollback: %v", err) + } + default: + t.Fatalf("op %q", step.Op) + } +} - hist, err := c.History(ctx, ns, "fresh") - if err != nil || len(hist) != 1 || hist[0].Status != string(common.StatusPendingInstall) || !hist[0].Updated.Equal(installAt) { - t.Fatalf("pending-install history = %#v, %v", hist, err) +func checkRelease(t *testing.T, what string, rel *Release, exp *releaseExpect, base time.Time) { + t.Helper() + if rel == nil { + t.Fatalf("%s = nil", what) } - if err := c.Uninstall(ctx, ns, "fresh"); err != nil { - t.Fatalf("uninstall pending-install: %v", err) + if exp == nil { + t.Fatalf("%s has no expect", what) } - got, err := c.Get(ctx, ns, "fresh") - if err != nil || got != nil { - t.Fatalf("get after uninstall pending-install = %#v, %v", got, err) + if exp.Revision != nil && rel.Revision != *exp.Revision { + t.Fatalf("%s revision = %d, want %d", what, rel.Revision, *exp.Revision) } - - got, err = c.Get(ctx, ns, "broke") - if err != nil || got == nil || got.Status != string(common.StatusFailed) || !got.Updated.Equal(failedAt) { - t.Fatalf("failed get = %#v, %v", got, err) + if exp.Status != "" && rel.Status != exp.Status { + t.Fatalf("%s status = %q, want %q", what, rel.Status, exp.Status) } - hist, err = c.History(ctx, ns, "broke") - if err != nil || len(hist) != 1 || hist[0].Status != string(common.StatusFailed) || hist[0].Revision != 1 { - t.Fatalf("failed history = %#v, %v", hist, err) + if exp.ChartName != "" && rel.ChartName != exp.ChartName { + t.Fatalf("%s chart = %q, want %q", what, rel.ChartName, exp.ChartName) } - - missing, err := c.History(ctx, ns, "absent") - if err != nil || missing != nil { - t.Fatalf("absent history = %#v, %v", missing, err) + if exp.ChartVersion != "" && rel.ChartVersion != exp.ChartVersion { + t.Fatalf("%s chart version = %q, want %q", what, rel.ChartVersion, exp.ChartVersion) } -} - -func TestRollbackPendingRollback(t *testing.T) { - c, cfg := testClient(t, writeTinyChart(t)) - ctx := t.Context() - const ns, name = testNamespace, "back" - deployedAt := time.Now().Add(-2 * time.Hour).UTC() - pendingAt := time.Now().Add(-30 * time.Minute).UTC() - storeRelease(t, cfg, storedRelease(name, 1, common.StatusDeployed, deployedAt, testLabels(name))) - storeRelease(t, cfg, storedRelease(name, 2, common.StatusPendingRollback, pendingAt, testLabels(name))) - - if err := c.Rollback(ctx, ns, name, 1); err != nil { - t.Fatalf("rollback pending-rollback: %v", err) + for k, want := range exp.Config { + got, ok := rel.Config[k].(string) + if !ok || got != want { + t.Fatalf("%s config[%s] = %#v, want %q", what, k, rel.Config[k], want) + } } - got, err := c.Get(ctx, ns, name) - if err != nil || got == nil || got.Status != string(common.StatusDeployed) || got.Revision != 3 { - t.Fatalf("pending-rollback recovered = %#v, %v", got, err) + for k, want := range exp.Labels { + if rel.Labels[k] != want { + t.Fatalf("%s label %s = %q, want %q", what, k, rel.Labels[k], want) + } } -} - -func testLabels(name string) map[string]string { - return map[string]string{ - uidLabelKey: testReleaseUID, - nameLabelKey: name, - } -} - -func storeRelease(t *testing.T, cfg *action.Configuration, rel *releasev1.Release) { - t.Helper() - if err := cfg.Releases.Create(rel); err != nil { - t.Fatal(err) + if exp.Updated != nil { + want := base.Add(exp.Updated.Std()).UTC() + if !rel.Updated.Equal(want) { + t.Fatalf("%s updated = %s, want %s", what, rel.Updated, want) + } } -} - -func storedRelease(name string, rev int, status common.Status, updated time.Time, labels map[string]string) *releasev1.Release { - return &releasev1.Release{ - Name: name, - Namespace: testNamespace, - Version: rev, - Labels: labels, - Config: map[string]any{"v": "one"}, - Manifest: "apiVersion: v1\nkind: ConfigMap\nmetadata:\n name: " + tinyChartName + "\n", - Chart: &chartv2.Chart{Metadata: &chartv2.Metadata{Name: tinyChartName, Version: tinyVersion}}, - Info: &releasev1.Info{ - Status: status, - LastDeployed: updated, - FirstDeployed: updated, - }, + if exp.UpdatedAfterStart && rel.Updated.Before(base.Add(-time.Second)) { + t.Fatalf("%s updated = %s, want after %s", what, rel.Updated, base.Add(-time.Second)) } } diff --git a/internal/helm/testdata/client/get-install-upgrade-uninstall/get-install-upgrade-uninstall.yaml b/internal/helm/testdata/client/get-install-upgrade-uninstall/get-install-upgrade-uninstall.yaml new file mode 100644 index 0000000..057a679 --- /dev/null +++ b/internal/helm/testdata/client/get-install-upgrade-uninstall/get-install-upgrade-uninstall.yaml @@ -0,0 +1,58 @@ +description: get, install, upgrade, and uninstall +namespace: default +chartRepo: oci://example.com/charts +chartName: tiny +chartVersion: "0.1.0" +steps: + - op: get + release: tiny + absent: true + - op: install + release: tiny + forceConflicts: true + values: + v: one + labels: + serving.modelsphere.dev/llmservice-uid: uid-1 + serving.modelsphere.dev/llmservice: tiny + expect: + revision: 1 + status: deployed + chartName: tiny + chartVersion: "0.1.0" + config: + v: one + labels: + serving.modelsphere.dev/llmservice-uid: uid-1 + updatedAfterStart: true + - op: get + release: tiny + expect: + revision: 1 + config: + v: one + labels: + serving.modelsphere.dev/llmservice-uid: uid-1 + updatedAfterStart: true + - op: upgrade + release: tiny + values: + v: two + labels: + serving.modelsphere.dev/llmservice-uid: uid-1 + serving.modelsphere.dev/llmservice: tiny + expect: + revision: 2 + status: deployed + config: + v: two + labels: + serving.modelsphere.dev/llmservice-uid: uid-1 + updatedAfterStart: true + - op: uninstall + release: tiny + - op: get + release: tiny + absent: true + - op: uninstall + release: tiny diff --git a/internal/helm/testdata/client/history-newest-first/history-newest-first.yaml b/internal/helm/testdata/client/history-newest-first/history-newest-first.yaml new file mode 100644 index 0000000..012f654 --- /dev/null +++ b/internal/helm/testdata/client/history-newest-first/history-newest-first.yaml @@ -0,0 +1,56 @@ +description: history newest first and rollback of a pending upgrade +namespace: default +seed: + - name: tiny + revision: 1 + status: deployed + updated: "-2h" + labels: + serving.modelsphere.dev/llmservice-uid: uid-1 + serving.modelsphere.dev/llmservice: tiny + - name: tiny + revision: 2 + status: failed + updated: "-1h" + labels: + serving.modelsphere.dev/llmservice-uid: uid-1 + serving.modelsphere.dev/llmservice: tiny + - name: tiny + revision: 3 + status: pending-upgrade + updated: "-30m" + labels: + serving.modelsphere.dev/llmservice-uid: uid-1 + serving.modelsphere.dev/llmservice: tiny +steps: + - op: get + release: tiny + expect: + revision: 3 + status: pending-upgrade + updated: "-30m" + labels: + serving.modelsphere.dev/llmservice-uid: uid-1 + - op: history + release: tiny + history: + - revision: 3 + status: pending-upgrade + - revision: 2 + status: failed + - revision: 1 + status: deployed + updated: "-2h" + labels: + serving.modelsphere.dev/llmservice-uid: uid-1 + - op: rollback + release: tiny + revision: 1 + - op: get + release: tiny + expect: + revision: 4 + status: deployed + chartName: tiny + labels: + serving.modelsphere.dev/llmservice-uid: uid-1 diff --git a/internal/helm/testdata/client/pending-install-failed/pending-install-failed.yaml b/internal/helm/testdata/client/pending-install-failed/pending-install-failed.yaml new file mode 100644 index 0000000..31731c6 --- /dev/null +++ b/internal/helm/testdata/client/pending-install-failed/pending-install-failed.yaml @@ -0,0 +1,38 @@ +description: pending install, a failed release, and absent history +namespace: default +seed: + - name: fresh + revision: 1 + status: pending-install + updated: "-3h" + labels: + serving.modelsphere.dev/llmservice-uid: uid-1 + serving.modelsphere.dev/llmservice: fresh + - name: broke + revision: 1 + status: failed + updated: "-1h" +steps: + - op: history + release: fresh + history: + - status: pending-install + updated: "-3h" + - op: uninstall + release: fresh + - op: get + release: fresh + absent: true + - op: get + release: broke + expect: + status: failed + updated: "-1h" + - op: history + release: broke + history: + - revision: 1 + status: failed + - op: history + release: absent + absent: true diff --git a/internal/helm/testdata/client/rollback-pending-rollback/rollback-pending-rollback.yaml b/internal/helm/testdata/client/rollback-pending-rollback/rollback-pending-rollback.yaml new file mode 100644 index 0000000..2e6327f --- /dev/null +++ b/internal/helm/testdata/client/rollback-pending-rollback/rollback-pending-rollback.yaml @@ -0,0 +1,26 @@ +description: rollback of a pending rollback +namespace: default +seed: + - name: back + revision: 1 + status: deployed + updated: "-2h" + labels: + serving.modelsphere.dev/llmservice-uid: uid-1 + serving.modelsphere.dev/llmservice: back + - name: back + revision: 2 + status: pending-rollback + updated: "-30m" + labels: + serving.modelsphere.dev/llmservice-uid: uid-1 + serving.modelsphere.dev/llmservice: back +steps: + - op: rollback + release: back + revision: 1 + - op: get + release: back + expect: + revision: 3 + status: deployed diff --git a/internal/helm/testdata/guard/apply/hook.yaml b/internal/helm/testdata/guard/apply/hook.yaml new file mode 100644 index 0000000..aca78ab --- /dev/null +++ b/internal/helm/testdata/guard/apply/hook.yaml @@ -0,0 +1,14 @@ +description: hook install refused +steps: + - write: | + apiVersion: v1 + kind: ConfigMap + metadata: + name: escape + namespace: kube-system + annotations: + helm.sh/hook: pre-install,pre-upgrade + - op: install + errorContains: kube-system + - op: get + absent: true diff --git a/internal/helm/testdata/guard/apply/template.yaml b/internal/helm/testdata/guard/apply/template.yaml new file mode 100644 index 0000000..2a06b43 --- /dev/null +++ b/internal/helm/testdata/guard/apply/template.yaml @@ -0,0 +1,12 @@ +description: template install refused +steps: + - write: | + apiVersion: v1 + kind: ConfigMap + metadata: + name: escape + namespace: kube-system + - op: install + errorContains: kube-system + - op: get + absent: true diff --git a/internal/helm/testdata/guard/apply/upgrade.yaml b/internal/helm/testdata/guard/apply/upgrade.yaml new file mode 100644 index 0000000..d7ff57a --- /dev/null +++ b/internal/helm/testdata/guard/apply/upgrade.yaml @@ -0,0 +1,10 @@ +description: upgrade refused when the chart becomes cluster-scoped +steps: + - op: install + - write: | + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + name: escape + - op: upgrade + errorContains: cluster-scoped diff --git a/internal/helm/testdata/guard/namespace/after-a-good-one.yaml b/internal/helm/testdata/guard/namespace/after-a-good-one.yaml new file mode 100644 index 0000000..8c938ab --- /dev/null +++ b/internal/helm/testdata/guard/namespace/after-a-good-one.yaml @@ -0,0 +1,12 @@ +description: after a good one +manifest: | + apiVersion: v1 + kind: ConfigMap + metadata: + name: a + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + name: b +refused: cluster-scoped diff --git a/internal/helm/testdata/guard/namespace/another-namespace.yaml b/internal/helm/testdata/guard/namespace/another-namespace.yaml new file mode 100644 index 0000000..c82c7f1 --- /dev/null +++ b/internal/helm/testdata/guard/namespace/another-namespace.yaml @@ -0,0 +1,8 @@ +description: another namespace +manifest: | + apiVersion: v1 + kind: ConfigMap + metadata: + name: a + namespace: kube-system +refused: into namespace kube-system diff --git a/internal/helm/testdata/guard/namespace/cluster-scoped.yaml b/internal/helm/testdata/guard/namespace/cluster-scoped.yaml new file mode 100644 index 0000000..373d6dc --- /dev/null +++ b/internal/helm/testdata/guard/namespace/cluster-scoped.yaml @@ -0,0 +1,7 @@ +description: cluster-scoped +manifest: | + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + name: a +refused: 'ClusterRole "a", which is cluster-scoped' diff --git a/internal/helm/testdata/guard/namespace/comments-only.yaml b/internal/helm/testdata/guard/namespace/comments-only.yaml new file mode 100644 index 0000000..00eee77 --- /dev/null +++ b/internal/helm/testdata/guard/namespace/comments-only.yaml @@ -0,0 +1,5 @@ +description: comments only +manifest: | + # Source: x/templates/empty.yaml + --- +refused: "" diff --git a/internal/helm/testdata/guard/namespace/inside-a-list.yaml b/internal/helm/testdata/guard/namespace/inside-a-list.yaml new file mode 100644 index 0000000..1360362 --- /dev/null +++ b/internal/helm/testdata/guard/namespace/inside-a-list.yaml @@ -0,0 +1,15 @@ +description: inside a list +manifest: | + apiVersion: v1 + kind: List + items: + - apiVersion: v1 + kind: ConfigMap + metadata: + name: a + - apiVersion: rbac.authorization.k8s.io/v1 + kind: Role + metadata: + name: b + namespace: kube-system +refused: 'Role "b" into namespace kube-system' diff --git a/internal/helm/testdata/guard/namespace/namespace-object.yaml b/internal/helm/testdata/guard/namespace/namespace-object.yaml new file mode 100644 index 0000000..741d335 --- /dev/null +++ b/internal/helm/testdata/guard/namespace/namespace-object.yaml @@ -0,0 +1,7 @@ +description: namespace object +manifest: | + apiVersion: v1 + kind: Namespace + metadata: + name: a +refused: cluster-scoped diff --git a/internal/helm/testdata/guard/namespace/no-kind.yaml b/internal/helm/testdata/guard/namespace/no-kind.yaml new file mode 100644 index 0000000..5a2f967 --- /dev/null +++ b/internal/helm/testdata/guard/namespace/no-kind.yaml @@ -0,0 +1,4 @@ +description: no kind +manifest: | + replicaCount: 1 +refused: an object with no kind diff --git a/internal/helm/testdata/guard/namespace/no-namespace.yaml b/internal/helm/testdata/guard/namespace/no-namespace.yaml new file mode 100644 index 0000000..772e290 --- /dev/null +++ b/internal/helm/testdata/guard/namespace/no-namespace.yaml @@ -0,0 +1,7 @@ +description: no namespace +manifest: | + apiVersion: v1 + kind: ConfigMap + metadata: + name: a +refused: "" diff --git a/internal/helm/testdata/guard/namespace/own-namespace.yaml b/internal/helm/testdata/guard/namespace/own-namespace.yaml new file mode 100644 index 0000000..9022b29 --- /dev/null +++ b/internal/helm/testdata/guard/namespace/own-namespace.yaml @@ -0,0 +1,8 @@ +description: own namespace +manifest: | + apiVersion: v1 + kind: ConfigMap + metadata: + name: a + namespace: models +refused: "" diff --git a/internal/helm/testdata/guard/namespace/unknown-kind.yaml b/internal/helm/testdata/guard/namespace/unknown-kind.yaml new file mode 100644 index 0000000..9b0d1fe --- /dev/null +++ b/internal/helm/testdata/guard/namespace/unknown-kind.yaml @@ -0,0 +1,7 @@ +description: unknown kind +manifest: | + apiVersion: example.com/v1 + kind: Widget + metadata: + name: a +refused: cannot be told diff --git a/internal/testutil/cases/cases.go b/internal/testutil/cases/cases.go new file mode 100644 index 0000000..b90a81a --- /dev/null +++ b/internal/testutil/cases/cases.go @@ -0,0 +1,300 @@ +/* +Copyright 2026. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package cases loads one YAML document per test case. +// +// Files are discovered recursively under a directory. A name ending in +// ".yaml" is a case. A name ending in ".yml" is an error, so a second +// extension cannot hide a case or make two files share a stem. Any other +// name (a README, a note) is ignored. The suffix check is case-sensitive, +// matching the ".yaml" files already in the tree. The result is ordered by +// the slash-separated path relative to that directory, so a rename or a new +// file changes order only by that path. +// +// Each file must contain exactly one mapping. An empty file, a comment-only +// file, a null document (`null`, `~`, or a bare `---`), or a second document +// fails the load. A trailing `---` is a second, empty document and fails too. +// The mapping is decoded with sigs.k8s.io/yaml in strict mode: an unknown +// field, a duplicate key, or a value of the wrong type fails the load. +// Mapping keys are case-sensitive. A key must equal the json name of a field +// (or the field name when the struct has no json name). An anonymous embedded +// struct contributes its fields, as encoding/json does. A type with +// UnmarshalJSON, including Duration and Kubernetes types that decode +// themselves, is a leaf; other structs, slices, maps, and pointers are +// walked. An empty directory, a missing directory, and an unreadable file +// fail as well. +// +// A decoded case may carry Name and Description. Name, when set, must equal the +// file stem (the base name without ".yaml"). Description is the sentence +// runners use as the subtest text; when it is empty, the stem is used +// instead. Ginkgo table entries are built by package ginkgocases, so this +// package does not import Ginkgo. +// +// Durations in a case are Go duration strings ("30s", "1h", "-2h", "0s") +// decoded into Duration. Only a pointer to Duration may be omitted or null. +// A non-pointer Duration rejects null and treats an omission as zero. A +// pointer to zero is the string "0s", not a bare 0. +// +// Keys are decoded with YAML 1.1 rules. An unquoted n, y, yes, no, on, or off +// is a boolean, not a string, and strict decoding then reports it as an +// unknown field. Quote any such key. +package cases + +import ( + "bytes" + "encoding/json" + "fmt" + "io" + "os" + "path/filepath" + "reflect" + "slices" + "strings" + "testing" + "time" + + yamlv3 "gopkg.in/yaml.v3" + "sigs.k8s.io/yaml" +) + +// Duration is a time.Duration stored in YAML as a Go duration string. +type Duration time.Duration + +// Std returns the duration as a time.Duration. +func (d Duration) Std() time.Duration { return time.Duration(d) } + +// UnmarshalJSON accepts a JSON string in Go duration syntax. +// Null is rejected: only a pointer to Duration can represent an omitted value. +func (d *Duration) UnmarshalJSON(b []byte) error { + var s string + if err := json.Unmarshal(b, &s); err != nil { + return fmt.Errorf("duration: %w", err) + } + parsed, err := time.ParseDuration(s) + if err != nil { + return fmt.Errorf("duration %q: %w", s, err) + } + *d = Duration(parsed) + return nil +} + +// File is one decoded case and where it was read from. +type File[T any] struct { + // Rel is the path relative to the loaded directory, with forward slashes. + Rel string + // Stem is the file name without the .yaml suffix. + Stem string + // Case is the decoded document. + Case T +} + +// Load reads and decodes every case file under dir. +func Load[T any](dir string) ([]T, error) { + files, err := LoadFiles[T](dir) + if err != nil { + return nil, err + } + out := make([]T, len(files)) + for i, f := range files { + out[i] = f.Case + } + return out, nil +} + +// LoadFiles is Load with the relative path and stem of each file. +func LoadFiles[T any](dir string) ([]File[T], error) { + rels, err := yamlRels(dir) + if err != nil { + return nil, err + } + out := make([]File[T], 0, len(rels)) + for _, rel := range rels { + path := filepath.Join(dir, filepath.FromSlash(rel)) + body, err := os.ReadFile(path) + if err != nil { + return nil, fmt.Errorf("cases: read %s: %w", rel, err) + } + tc, err := decodeCase[T](rel, body) + if err != nil { + return nil, err + } + stem := strings.TrimSuffix(filepath.Base(rel), ".yaml") + if err := checkName(tc, stem, rel); err != nil { + return nil, err + } + out = append(out, File[T]{Rel: rel, Stem: stem, Case: tc}) + } + return out, nil +} + +// MustLoad is LoadFiles for a test container that has no testing.T yet. +// A bad directory panics so the suite fails instead of running zero cases. +func MustLoad[T any](dir string) []File[T] { + files, err := LoadFiles[T](dir) + if err != nil { + panic(err) + } + return files +} + +// Run loads dir and calls fn as one subtest per file. +// The subtest name is the case Description when that field is set, and the +// file stem otherwise, so an existing sentence stays the name go test prints. +func Run[T any](t *testing.T, dir string, fn func(t *testing.T, tc T)) { + t.Helper() + files, err := LoadFiles[T](dir) + if err != nil { + t.Fatal(err) + } + for _, f := range files { + t.Run(SpecName(f.Case, f.Stem), func(t *testing.T) { + fn(t, f.Case) + }) + } +} + +// SpecName is the subtest text for a case: its Description, or stem. +func SpecName[T any](tc T, stem string) string { + if d, ok := stringField(tc, "Description"); ok && d != "" { + return d + } + return stem +} + +func decodeCase[T any](rel string, body []byte) (T, error) { + var zero T + if err := oneMapping(body); err != nil { + return zero, fmt.Errorf("cases: decode %s: %w", rel, err) + } + var tc T + if err := yaml.UnmarshalStrict(body, &tc); err != nil { + return zero, fmt.Errorf("cases: decode %s: %w", rel, err) + } + var tree any + if err := yaml.Unmarshal(body, &tree); err != nil { + return zero, fmt.Errorf("cases: decode %s: %w", rel, err) + } + if err := checkKeys(tree, reflect.TypeOf(tc)); err != nil { + return zero, fmt.Errorf("cases: decode %s: %w", rel, err) + } + return tc, nil +} + +// oneMapping requires exactly one YAML document, and that document to be a +// non-null mapping. yaml.v3 counts documents; sigs.k8s.io/yaml then decodes +// the bytes, so YAML 1.1 values are unchanged. +func oneMapping(body []byte) error { + dec := yamlv3.NewDecoder(bytes.NewReader(body)) + var ( + count int + doc yamlv3.Node + ) + for { + var next yamlv3.Node + err := dec.Decode(&next) + if err == io.EOF { + break + } + if err != nil { + return err + } + count++ + doc = next + } + if count == 0 { + return fmt.Errorf("no document") + } + if count != 1 { + return fmt.Errorf("%d documents, want 1", count) + } + if doc.Kind != yamlv3.DocumentNode || len(doc.Content) != 1 { + return fmt.Errorf("document is empty") + } + n := doc.Content[0] + if n.Tag == "!!null" { + return fmt.Errorf("document is null") + } + if n.Kind != yamlv3.MappingNode { + return fmt.Errorf("document must be a mapping") + } + return nil +} + +func yamlRels(dir string) ([]string, error) { + var rels []string + err := filepath.WalkDir(dir, func(path string, d os.DirEntry, err error) error { + if err != nil { + return err + } + if d.IsDir() { + return nil + } + rel, err := filepath.Rel(dir, path) + if err != nil { + return err + } + rel = filepath.ToSlash(rel) + // .yml is an error rather than a skip: one extension keeps the stem + // unambiguous, and a mis-named file must not vanish from the run. + if strings.HasSuffix(d.Name(), ".yml") { + return fmt.Errorf("%s: .yml files are not loaded; rename it to .yaml", rel) + } + if !strings.HasSuffix(d.Name(), ".yaml") { + return nil + } + rels = append(rels, rel) + return nil + }) + if err != nil { + return nil, fmt.Errorf("cases: read %s: %w", dir, err) + } + if len(rels) == 0 { + return nil, fmt.Errorf("cases: %s has no yaml files", dir) + } + slices.Sort(rels) + return rels, nil +} + +func checkName(tc any, stem, rel string) error { + name, ok := stringField(tc, "Name") + if !ok || name == "" { + return nil + } + if name != stem { + return fmt.Errorf("cases: %s: name %q must equal the file stem %q", rel, name, stem) + } + return nil +} + +// stringField reads an exported string field. ok is false when the case has +// no such field, so a case type is not required to carry Name or Description. +func stringField(tc any, field string) (string, bool) { + v := reflect.ValueOf(tc) + if v.Kind() == reflect.Pointer { + if v.IsNil() { + return "", false + } + v = v.Elem() + } + if v.Kind() != reflect.Struct { + return "", false + } + f := v.FieldByName(field) + if !f.IsValid() || f.Kind() != reflect.String || !f.CanInterface() { + return "", false + } + return f.String(), true +} diff --git a/internal/testutil/cases/cases_test.go b/internal/testutil/cases/cases_test.go new file mode 100644 index 0000000..7873563 --- /dev/null +++ b/internal/testutil/cases/cases_test.go @@ -0,0 +1,280 @@ +/* +Copyright 2026. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package cases + +import ( + "os" + "path/filepath" + "strings" + "testing" + "time" +) + +// sample is the document shape under testdata/. The happy-path files are the +// cases; the other directories are documents this loader must reject. +type sample struct { + Name string `json:"name,omitempty"` + Description string `json:"description,omitempty"` + Inputs struct { + Value int `json:"value"` + Wait Duration `json:"wait,omitempty"` + Maybe *Duration `json:"maybe,omitempty"` + Empty *Duration `json:"empty,omitempty"` + } `json:"inputs"` + Expected struct { + Value int `json:"value"` + } `json:"expected"` +} + +func TestLoadOrdersByRelativePath(t *testing.T) { + files, err := LoadFiles[sample]("testdata/ok") + if err != nil { + t.Fatal(err) + } + if len(files) != 2 { + t.Fatalf("got %d files", len(files)) + } + if files[0].Rel != "a/c.yaml" || files[1].Rel != "b.yaml" { + t.Fatalf("order = %q, %q", files[0].Rel, files[1].Rel) + } + if files[0].Case.Inputs.Value != 1 || files[1].Case.Expected.Value != 2 { + t.Fatalf("decoded %+v %+v", files[0].Case, files[1].Case) + } + if files[0].Case.Inputs.Wait.Std() != 30*time.Second { + t.Fatalf("wait = %s", files[0].Case.Inputs.Wait.Std()) + } + if files[0].Case.Inputs.Maybe == nil || files[0].Case.Inputs.Maybe.Std() != time.Second { + t.Fatalf("maybe = %v", files[0].Case.Inputs.Maybe) + } + if files[0].Case.Inputs.Empty != nil { + t.Fatal("null duration decoded as set") + } + + got, err := Load[sample]("testdata/ok") + if err != nil || len(got) != 2 || got[0].Description != "first" { + t.Fatalf("Load = %+v, %v", got, err) + } +} + +func TestRunUsesDescription(t *testing.T) { + var seen []string + Run(t, "testdata/ok", func(t *testing.T, tc sample) { + seen = append(seen, tc.Description) + if !strings.HasSuffix(t.Name(), "/"+tc.Description) { + t.Errorf("subtest %q does not end with %q", t.Name(), tc.Description) + } + }) + if strings.Join(seen, ",") != "first,second" { + t.Fatalf("ran %v", seen) + } +} + +func TestRejectsBadDocuments(t *testing.T) { + checks := []struct { + dir string + want string + }{ + {"testdata/unknown-field", "unknown field"}, + {"testdata/name-mismatch", "must equal the file stem"}, + {"testdata/bad-yaml", "decode"}, + {"testdata/no-yaml", "has no yaml files"}, + {"testdata/missing", "read"}, + } + for _, tc := range checks { + t.Run(tc.dir, func(t *testing.T) { + _, err := Load[sample](tc.dir) + if err == nil || !strings.Contains(err.Error(), tc.want) { + t.Fatalf("got %v, want it to contain %q", err, tc.want) + } + }) + } +} + +// An empty directory cannot be stored, so this one failure mode is built in a +// temp dir. The other failures live as files under testdata/. +func TestEmptyDirFails(t *testing.T) { + dir := t.TempDir() + if _, err := Load[sample](dir); err == nil || !strings.Contains(err.Error(), "has no yaml files") { + t.Fatalf("got %v", err) + } +} + +func TestMustLoadPanics(t *testing.T) { + defer func() { + if recover() == nil { + t.Fatal("MustLoad returned") + } + }() + MustLoad[sample](filepath.Join(t.TempDir(), "absent")) +} + +func TestRejectsDocumentsThatAreNotOneMapping(t *testing.T) { + checks := []struct { + name string + body string + want string + }{ + {"empty.yaml", "", "no document"}, + {"comment.yaml", "# nothing to run\n", "no document"}, + {"blank.yaml", "\n\n", "no document"}, + {"null.yaml", "null\n", "document is null"}, + {"tilde.yaml", "~\n", "document is null"}, + {"sep.yaml", "---\n", "document is null"}, + {"seq.yaml", "[]\n", "must be a mapping"}, + {"multi.yaml", "name: multi\ninputs:\n value: 1\nexpected:\n value: 1\n---\nother: 2\n", "2 documents"}, + {"later.yaml", "name: later\ninputs:\n value: 1\nexpected:\n value: 1\n---\nbogus: 2\n", "2 documents"}, + } + for _, tc := range checks { + t.Run(tc.name, func(t *testing.T) { + dir := writeYAML(t, tc.name, tc.body) + _, err := Load[sample](dir) + if err == nil || !strings.Contains(err.Error(), tc.want) || !strings.Contains(err.Error(), tc.name) { + t.Fatalf("got %v, want it to name %s and contain %q", err, tc.name, tc.want) + } + }) + } +} + +func TestLeadingSeparatorIsOneDocument(t *testing.T) { + dir := writeYAML(t, "lead.yaml", "---\nname: lead\ninputs:\n value: 3\nexpected:\n value: 3\n") + got, err := Load[sample](dir) + if err != nil || len(got) != 1 || got[0].Inputs.Value != 3 { + t.Fatalf("got %+v, %v", got, err) + } +} + +func TestEmptyMappingIsOneDocument(t *testing.T) { + dir := writeYAML(t, "empty-map.yaml", "{}\n") + got, err := Load[sample](dir) + if err != nil || len(got) != 1 { + t.Fatalf("got %+v, %v", got, err) + } +} + +func TestRejectsYMLExtension(t *testing.T) { + dir := t.TempDir() + if err := os.WriteFile(filepath.Join(dir, "kept.yaml"), []byte("name: kept\ninputs:\n value: 1\nexpected:\n value: 1\n"), 0o644); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(dir, "skipped.yml"), []byte("name: skipped\n"), 0o644); err != nil { + t.Fatal(err) + } + _, err := Load[sample](dir) + if err == nil || !strings.Contains(err.Error(), "skipped.yml") || !strings.Contains(err.Error(), ".yml") { + t.Fatalf("got %v", err) + } +} + +func TestRejectsCaseMismatchedKey(t *testing.T) { + body := "name: case\ninputs:\n Value: 1\nexpected:\n value: 1\n" + dir := writeYAML(t, "case.yaml", body) + _, err := Load[sample](dir) + if err == nil || !strings.Contains(err.Error(), "case.yaml") || !strings.Contains(err.Error(), "Value") { + t.Fatalf("got %v", err) + } +} + +func TestRejectsCaseMismatchInSliceAndMap(t *testing.T) { + dir := writeYAML(t, "slice.yaml", "items:\n - Kind: pod\n") + if _, err := Load[nest](dir); err == nil || !strings.Contains(err.Error(), "Kind") { + t.Fatalf("slice: %v", err) + } + dir = writeYAML(t, "map.yaml", "bags:\n x:\n N: 1\n") + if _, err := Load[nest](dir); err == nil || !strings.Contains(err.Error(), "N") { + t.Fatalf("map: %v", err) + } + // "n" is quoted: an unquoted n is a YAML 1.1 boolean, not a key. + dir = writeYAML(t, "ok.yaml", "items:\n - kind: pod\nbags:\n x:\n \"n\": 1\n") + got, err := Load[nest](dir) + if err != nil || len(got) != 1 || got[0].Items[0].Kind != "pod" || got[0].Bags["x"].N != 1 { + t.Fatalf("got %+v, %v", got, err) + } +} + +func TestNonPointerDurationRejectsNull(t *testing.T) { + dir := writeYAML(t, "wait.yaml", "name: wait\ninputs:\n value: 1\n wait: null\nexpected:\n value: 1\n") + _, err := Load[sample](dir) + if err == nil || !strings.Contains(err.Error(), "wait.yaml") || !strings.Contains(err.Error(), "duration") { + t.Fatalf("got %v", err) + } + dir = writeYAML(t, "zero.yaml", "name: zero\ninputs:\n value: 1\n wait: \"0s\"\nexpected:\n value: 1\n") + got, err := Load[sample](dir) + if err != nil || got[0].Inputs.Wait.Std() != 0 { + t.Fatalf("got %+v, %v", got, err) + } + dir = writeYAML(t, "bare.yaml", "name: bare\ninputs:\n value: 1\n wait: 0\nexpected:\n value: 1\n") + if _, err := Load[sample](dir); err == nil { + t.Fatal("bare 0 decoded as a duration") + } +} + +func TestPromotedEmbeddedFieldsAreCaseSensitive(t *testing.T) { + dir := writeYAML(t, "ok.yaml", "a: 1\nb: 2\n") + got, err := Load[outer](dir) + if err != nil || got[0].A != 1 || got[0].B != 2 { + t.Fatalf("got %+v, %v", got, err) + } + dir = writeYAML(t, "bad.yaml", "A: 1\nb: 2\n") + if _, err := Load[outer](dir); err == nil || !strings.Contains(err.Error(), "A") { + t.Fatalf("got %v", err) + } +} + +type nest struct { + Items []struct { + Kind string `json:"kind"` + } `json:"items"` + Bags map[string]struct { + N int `json:"n"` + } `json:"bags"` +} + +type inner struct { + A int `json:"a"` +} + +type outer struct { + inner + B int `json:"b"` +} + +func writeYAML(t *testing.T, name, body string) string { + t.Helper() + dir := t.TempDir() + if err := os.WriteFile(filepath.Join(dir, name), []byte(body), 0o644); err != nil { + t.Fatal(err) + } + return dir +} + +func TestUnreadableFileFails(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "gone.yaml") + if err := os.WriteFile(path, []byte("name: gone\n"), 0o644); err != nil { + t.Fatal(err) + } + if err := os.Chmod(path, 0); err != nil { + t.Fatal(err) + } + // Root ignores mode bits. Skip when the file is still readable. + if _, err := os.ReadFile(path); err == nil { + t.Skip("process can read a mode-0 file") + } + if _, err := Load[sample](dir); err == nil || !strings.Contains(err.Error(), "read") { + t.Fatalf("got %v", err) + } +} diff --git a/internal/testutil/cases/ginkgocases/entries.go b/internal/testutil/cases/ginkgocases/entries.go new file mode 100644 index 0000000..a875eee --- /dev/null +++ b/internal/testutil/cases/ginkgocases/entries.go @@ -0,0 +1,49 @@ +/* +Copyright 2026. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package ginkgocases builds Ginkgo table entries from case files. +// +// It lives apart from package cases so a test binary that only calls +// cases.Run or cases.Load does not import Ginkgo and does not register +// ginkgo flags. +package ginkgocases + +import ( + "github.com/onsi/ginkgo/v2" + + "github.com/modelsphere/llm-operator/internal/testutil/cases" +) + +// Entries loads dir as Ginkgo table entries, one Entry per file. +// description supplies the entry text; a nil description, or a result of "", +// uses the file stem. A load error panics so suite setup fails loudly. +func Entries[T any](dir string, description func(T) string) []ginkgo.TableEntry { + files, err := cases.LoadFiles[T](dir) + if err != nil { + panic(err) + } + entries := make([]ginkgo.TableEntry, len(files)) + for i, f := range files { + text := f.Stem + if description != nil { + if d := description(f.Case); d != "" { + text = d + } + } + entries[i] = ginkgo.Entry(text, f.Case) + } + return entries +} diff --git a/internal/testutil/cases/ginkgocases/entries_test.go b/internal/testutil/cases/ginkgocases/entries_test.go new file mode 100644 index 0000000..9261ccf --- /dev/null +++ b/internal/testutil/cases/ginkgocases/entries_test.go @@ -0,0 +1,49 @@ +/* +Copyright 2026. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package ginkgocases + +import ( + "testing" + + "github.com/modelsphere/llm-operator/internal/testutil/cases" +) + +// sample matches the documents under ../testdata/ok. +type sample struct { + Name string `json:"name,omitempty"` + Description string `json:"description,omitempty"` + Inputs struct { + Value int `json:"value"` + Wait cases.Duration `json:"wait,omitempty"` + Maybe *cases.Duration `json:"maybe,omitempty"` + Empty *cases.Duration `json:"empty,omitempty"` + } `json:"inputs"` + Expected struct { + Value int `json:"value"` + } `json:"expected"` +} + +func TestEntriesFollowsDescription(t *testing.T) { + entries := Entries("../testdata/ok", func(s sample) string { return s.Description }) + if len(entries) != 2 { + t.Fatalf("got %d entries", len(entries)) + } + entries = Entries[sample]("../testdata/ok", nil) + if len(entries) != 2 { + t.Fatalf("nil description: got %d entries", len(entries)) + } +} diff --git a/internal/testutil/cases/keys.go b/internal/testutil/cases/keys.go new file mode 100644 index 0000000..d91e20a --- /dev/null +++ b/internal/testutil/cases/keys.go @@ -0,0 +1,133 @@ +/* +Copyright 2026. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package cases + +import ( + "encoding/json" + "fmt" + "reflect" + "strings" +) + +// checkKeys walks a decoded YAML tree beside t and rejects a mapping key +// that does not exactly equal a json name. encoding/json, which +// sigs.k8s.io/yaml uses, matches those names without regard to case. +// Types that implement json.Unmarshaler are leaves: their own decoder owns +// the value, and Kubernetes types that decode themselves fall in that set. +func checkKeys(tree any, t reflect.Type) error { + if t == nil || tree == nil { + return nil + } + for t.Kind() == reflect.Pointer { + t = t.Elem() + } + if hasUnmarshalJSON(t) { + return nil + } + switch t.Kind() { + case reflect.Struct: + m, ok := tree.(map[string]any) + if !ok { + return fmt.Errorf("mapping for %s has type %T", typeName(t), tree) + } + fields := jsonFields(t) + for k, v := range m { + ft, ok := fields[k] + if !ok { + return fmt.Errorf("key %q does not match a field (names are case-sensitive)", k) + } + if err := checkKeys(v, ft); err != nil { + return fmt.Errorf("%s: %w", k, err) + } + } + case reflect.Slice, reflect.Array: + items, ok := tree.([]any) + if !ok { + return fmt.Errorf("sequence for %s has type %T", typeName(t), tree) + } + for i, item := range items { + if err := checkKeys(item, t.Elem()); err != nil { + return fmt.Errorf("[%d]: %w", i, err) + } + } + case reflect.Map: + if t.Key().Kind() != reflect.String { + return nil + } + m, ok := tree.(map[string]any) + if !ok { + return fmt.Errorf("mapping for %s has type %T", typeName(t), tree) + } + for k, v := range m { + if err := checkKeys(v, t.Elem()); err != nil { + return fmt.Errorf("[%s]: %w", k, err) + } + } + default: + return nil + } + return nil +} + +func jsonFields(t reflect.Type) map[string]reflect.Type { + out := map[string]reflect.Type{} + collectFields(out, t) + return out +} + +// collectFields records the json names encoding/json would accept, including +// fields promoted from an anonymous embedded struct. +func collectFields(out map[string]reflect.Type, t reflect.Type) { + for f := range t.Fields() { + if f.PkgPath != "" && !f.Anonymous { + continue + } + tag := f.Tag.Get("json") + if tag == "-" { + continue + } + name, _, _ := strings.Cut(tag, ",") + // encoding/json promotes an anonymous struct when the tag names no + // field of its own. json:",inline" is that case: the name is empty. + if f.Anonymous && name == "" { + ft := f.Type + for ft.Kind() == reflect.Pointer { + ft = ft.Elem() + } + if ft.Kind() == reflect.Struct && !hasUnmarshalJSON(ft) { + collectFields(out, ft) + continue + } + } + if name == "" { + name = f.Name + } + out[name] = f.Type + } +} + +func hasUnmarshalJSON(t reflect.Type) bool { + u := reflect.TypeFor[json.Unmarshaler]() + return t.Implements(u) || reflect.PointerTo(t).Implements(u) +} + +func typeName(t reflect.Type) string { + if t.Name() != "" { + return t.Name() + } + return t.String() +} diff --git a/internal/testutil/cases/testdata/bad-yaml/x.yaml b/internal/testutil/cases/testdata/bad-yaml/x.yaml new file mode 100644 index 0000000..89a1e6b --- /dev/null +++ b/internal/testutil/cases/testdata/bad-yaml/x.yaml @@ -0,0 +1 @@ +name: [unterminated diff --git a/internal/testutil/cases/testdata/name-mismatch/nope.yaml b/internal/testutil/cases/testdata/name-mismatch/nope.yaml new file mode 100644 index 0000000..2121d0e --- /dev/null +++ b/internal/testutil/cases/testdata/name-mismatch/nope.yaml @@ -0,0 +1,6 @@ +name: other +description: name does not match the file +inputs: + value: 1 +expected: + value: 1 diff --git a/internal/testutil/cases/testdata/no-yaml/note.txt b/internal/testutil/cases/testdata/no-yaml/note.txt new file mode 100644 index 0000000..0622888 --- /dev/null +++ b/internal/testutil/cases/testdata/no-yaml/note.txt @@ -0,0 +1 @@ +not a case diff --git a/internal/testutil/cases/testdata/ok/a/c.yaml b/internal/testutil/cases/testdata/ok/a/c.yaml new file mode 100644 index 0000000..9269483 --- /dev/null +++ b/internal/testutil/cases/testdata/ok/a/c.yaml @@ -0,0 +1,9 @@ +name: c +description: first +inputs: + value: 1 + wait: "30s" + maybe: "1s" + empty: null +expected: + value: 1 diff --git a/internal/testutil/cases/testdata/ok/b.yaml b/internal/testutil/cases/testdata/ok/b.yaml new file mode 100644 index 0000000..ee3db7b --- /dev/null +++ b/internal/testutil/cases/testdata/ok/b.yaml @@ -0,0 +1,6 @@ +name: b +description: second +inputs: + value: 2 +expected: + value: 2 diff --git a/internal/testutil/cases/testdata/unknown-field/bad.yaml b/internal/testutil/cases/testdata/unknown-field/bad.yaml new file mode 100644 index 0000000..c0a7637 --- /dev/null +++ b/internal/testutil/cases/testdata/unknown-field/bad.yaml @@ -0,0 +1,7 @@ +name: bad +description: unknown field +inputs: + value: 1 +expected: + value: 1 + extra: true