From 9f41659a4b2d0afbd1563bb84429182e84ab01ba Mon Sep 17 00:00:00 2001 From: Imani Manyara Date: Wed, 4 Nov 2026 15:36:42 -0500 Subject: [PATCH 1/4] feat: export RedactError for adapters outside the root package An adapter in its own package has to return transport and provider errors through the one redaction rule, and its callers still match ErrEgressDenied and ErrDenied with errors.Is. The unexported redactErr already did both; RedactError exposes it, and redactErr now delegates. --- src/client.go | 13 ++++++++++--- src/cloudkit_test.go | 22 ++++++++++++++++++++++ 2 files changed, 32 insertions(+), 3 deletions(-) diff --git a/src/client.go b/src/client.go index 2b2a8b4..de873db 100644 --- a/src/client.go +++ b/src/client.go @@ -74,11 +74,18 @@ type redactedError struct { func (e *redactedError) Error() string { return e.msg } func (e *redactedError) Unwrap() error { return e.err } -// redactErr is how an error from outside the module (a provider, an adapter, -// a credential source) leaves it. -func redactErr(err error) error { +// RedactError returns err with its message passed through [Redact], while +// errors.Is and errors.As still see the original chain. An adapter returns +// every error it did not build itself (a transport failure, a provider's +// response) through it, so a credential a proxy or a provider echoed back +// never leaves the module. It returns nil for a nil error. +func RedactError(err error) error { if err == nil { return nil } return &redactedError{msg: Redact(err.Error()), err: err} } + +// redactErr is how an error from outside the module (a provider, an adapter, +// a credential source) leaves it. +func redactErr(err error) error { return RedactError(err) } diff --git a/src/cloudkit_test.go b/src/cloudkit_test.go index a32ec01..38e62de 100644 --- a/src/cloudkit_test.go +++ b/src/cloudkit_test.go @@ -269,3 +269,25 @@ func TestEgressBoundsRedirectChains(t *testing.T) { t.Fatalf("followed %d hops", len(rt.seen)) } } + +// RedactError strips credentials from the message and keeps the chain, so an +// adapter can redact a transport error and its caller still matches +// ErrEgressDenied. +func TestRedactErrorKeepsTheChain(t *testing.T) { + if RedactError(nil) != nil { + t.Fatal("RedactError(nil) is not nil") + } + tok := rand.Text() + cause := &EgressError{Host: "elsewhere.test", Reason: "proxy said Authorization: Bearer " + tok} + err := RedactError(fmt.Errorf("probe: %w", cause)) + if strings.Contains(err.Error(), tok) { + t.Errorf("the token survived: %q", err.Error()) + } + if !errors.Is(err, ErrEgressDenied) { + t.Error("errors.Is no longer reaches ErrEgressDenied") + } + var eg *EgressError + if !errors.As(err, &eg) || eg.Host != "elsewhere.test" { + t.Errorf("errors.As lost the EgressError: %v", eg) + } +} From 5f1fc87b73567b24f94a1f589760e6af9635022a Mon Sep 17 00:00:00 2001 From: Imani Manyara Date: Wed, 4 Nov 2026 15:36:42 -0500 Subject: [PATCH 2/4] feat: add cloudkittest, the contract kit Run and Check run nine contract cases against any Provider: transport only, nil transport, egress, read-only token, plan drift, destroy, redaction, cloud-init and SSH. Each case builds its own registry over a replaying Recorder or a fault-injecting transport, with a token generated for the run, so nothing reaches the network. Recorder replays one cassette, answers each interaction once and in order, and refuses any request it does not hold. Recordings load only through ParseRecordings, which checks every recorded and synthetic response against the provider's OpenAPI document and refuses any recording carrying an Authorization header or a credential value. The validator implements a stated JSON Schema subset and refuses every other keyword when the document loads. Synthetic responses come from the document's examples and are marked as such. The core tests' fake provider is written again against a hand-written spec under testdata; it passes the suite, and one planted defect per case fails that case. --- src/cloudkittest/cases.go | 705 ++++++++++++++++++ src/cloudkittest/doc.go | 66 ++ src/cloudkittest/provider_test.go | 327 ++++++++ src/cloudkittest/recorder.go | 189 +++++ src/cloudkittest/recordings.go | 243 ++++++ src/cloudkittest/recordings_test.go | 215 ++++++ src/cloudkittest/schema.go | 390 ++++++++++ src/cloudkittest/spec.go | 411 ++++++++++ src/cloudkittest/spec_test.go | 201 +++++ src/cloudkittest/suite.go | 333 +++++++++ src/cloudkittest/suite_test.go | 132 ++++ .../testdata/broken-schema.recordings.json | 24 + src/cloudkittest/testdata/fake.openapi.json | 123 +++ .../testdata/fake.recordings.json | 322 ++++++++ 14 files changed, 3681 insertions(+) create mode 100644 src/cloudkittest/cases.go create mode 100644 src/cloudkittest/doc.go create mode 100644 src/cloudkittest/provider_test.go create mode 100644 src/cloudkittest/recorder.go create mode 100644 src/cloudkittest/recordings.go create mode 100644 src/cloudkittest/recordings_test.go create mode 100644 src/cloudkittest/schema.go create mode 100644 src/cloudkittest/spec.go create mode 100644 src/cloudkittest/spec_test.go create mode 100644 src/cloudkittest/suite.go create mode 100644 src/cloudkittest/suite_test.go create mode 100644 src/cloudkittest/testdata/broken-schema.recordings.json create mode 100644 src/cloudkittest/testdata/fake.openapi.json create mode 100644 src/cloudkittest/testdata/fake.recordings.json diff --git a/src/cloudkittest/cases.go b/src/cloudkittest/cases.go new file mode 100644 index 0000000..f48955c --- /dev/null +++ b/src/cloudkittest/cases.go @@ -0,0 +1,705 @@ +package cloudkittest + +import ( + "bytes" + "context" + "crypto/rand" + "errors" + "fmt" + "io" + "log/slog" + "maps" + "net" + "net/http" + "slices" + "strings" + "sync" + + "github.com/simtabi/cloudkit/src" +) + +// contractAccount is the account name every case opens. +const contractAccount = "contract" + +// The hosts the egress case uses. .invalid never resolves (RFC 2606), so even +// a request that escaped every guard could not reach anything. +const ( + outsideHost = "outside.cloudkit.invalid" + redirectHost = "redirect.cloudkit.invalid" +) + +var errTransportDown = errors.New("cloudkittest: the injected transport is down") + +type runner struct { + ctx context.Context + s Suite + token string +} + +// tokenSource hands out the run's token for every account. +type tokenSource struct{ v string } + +func (t tokenSource) Token(context.Context, string) (cloudkit.Secret, error) { + return cloudkit.NewSecret([]byte(t.v)), nil +} + +func refuseDial(context.Context, string, string) (net.Conn, error) { + return nil, errors.New("cloudkittest: the contract suite never dials") +} + +// lockedBuffer collects log records written from any goroutine. +type lockedBuffer struct { + mu sync.Mutex + buf bytes.Buffer +} + +func (b *lockedBuffer) Write(p []byte) (int, error) { + b.mu.Lock() + defer b.mu.Unlock() + return b.buf.Write(p) +} + +func (b *lockedBuffer) String() string { + b.mu.Lock() + defer b.mu.Unlock() + return b.buf.String() +} + +// env is one registry a case opens the adapter through. +type env struct { + reg *cloudkit.Registry + logs *lockedBuffer + + mu sync.Mutex + accounts []cloudkit.Account +} + +func (e *env) lastAccount() (cloudkit.Account, bool) { + e.mu.Lock() + defer e.mu.Unlock() + if len(e.accounts) == 0 { + return cloudkit.Account{}, false + } + return e.accounts[len(e.accounts)-1], true +} + +// open builds a registry over rt and opens the adapter. It records the +// Account the factory received, so a case can use the very client the +// adapter was given. +func (r *runner) open(c *caseT, rt http.RoundTripper) (*env, cloudkit.Provider, bool) { + e := &env{logs: &lockedBuffer{}} + f := r.s.Factory + inner := f.New + f.New = func(ctx context.Context, a cloudkit.Account) (cloudkit.Provider, error) { + e.mu.Lock() + e.accounts = append(e.accounts, a) + e.mu.Unlock() + return inner(ctx, a) + } + reg, err := cloudkit.NewRegistry(cloudkit.Options{ + Transport: cloudkit.Transport{HTTP: rt, Dial: refuseDial}, + Credentials: tokenSource{r.token}, + Logger: slog.New(slog.NewJSONHandler(e.logs, &slog.HandlerOptions{Level: slog.LevelDebug})), + Retry: cloudkit.RetryPolicy{MaxAttempts: 1}, // a cassette records each call once + UserAgent: "cloudkittest", + }, f) + if err != nil { + c.failf("NewRegistry refused the suite's factory: %v", err) + return nil, nil, false + } + e.reg = reg + p, err := reg.Open(r.ctx, f.Descriptor.ID, contractAccount) + if err != nil { + c.failf("Open failed: %v", err) + return nil, nil, false + } + return e, p, true +} + +func (r *runner) recorder(c *caseT, cassette string) (*Recorder, bool) { + rec, err := NewRecorder(r.s.Recordings, cassette) + if err != nil { + c.failf("%v", err) + return nil, false + } + return rec, true +} + +// cassetteDone fails the case for every request the cassette did not hold +// and every interaction the adapter never asked for. +func cassetteDone(c *caseT, cassette string, rec *Recorder) { + for _, u := range rec.Unexpected() { + c.failf("cassette %s does not hold %s %s", cassette, u.Method, u.URL) + } + for _, u := range rec.Unused() { + c.failf("the adapter never sent the recorded %s %s (cassette %s)", u.Method, u.URL, cassette) + } +} + +// faultRT answers every request itself and keeps each one, with its raw URL +// and headers, for the case to inspect. +type faultRT struct { + answer func(*http.Request) (*http.Response, error) + + mu sync.Mutex + seen []*http.Request +} + +func (f *faultRT) RoundTrip(req *http.Request) (*http.Response, error) { + if req.Body != nil { + _, _ = io.Copy(io.Discard, req.Body) + _ = req.Body.Close() + } + f.mu.Lock() + f.seen = append(f.seen, req) + f.mu.Unlock() + return f.answer(req) +} + +func (f *faultRT) requests() []*http.Request { + f.mu.Lock() + defer f.mu.Unlock() + return slices.Clone(f.seen) +} + +// spyRT passes requests on and keeps their raw URLs. +type spyRT struct { + next http.RoundTripper + + mu sync.Mutex + urls []string +} + +func (s *spyRT) RoundTrip(req *http.Request) (*http.Response, error) { + s.mu.Lock() + s.urls = append(s.urls, req.URL.String()) + s.mu.Unlock() + return s.next.RoundTrip(req) +} + +func reply(req *http.Request, code int, header http.Header, body string) *http.Response { + if header == nil { + header = http.Header{} + } + return &http.Response{ + Status: fmt.Sprintf("%d %s", code, http.StatusText(code)), + StatusCode: code, + Proto: "HTTP/1.1", + ProtoMajor: 1, + ProtoMinor: 1, + Header: header, + Body: io.NopCloser(strings.NewReader(body)), + ContentLength: int64(len(body)), + Request: req, + } +} + +// call is one thing a case asks the adapter to do. +type call struct { + name string + run func(ctx context.Context) error +} + +// everyCall is every call the suite knows how to make: Probe, Inventory and +// each plan. +func (r *runner) everyCall(p cloudkit.Provider) []call { + calls := []call{ + {"Probe", func(ctx context.Context) error { _, err := p.Probe(ctx); return err }}, + {"Inventory", func(ctx context.Context) error { _, err := p.Inventory(ctx, cloudkit.Filter{}); return err }}, + } + plan := func(name string, f PlanFunc) call { + return call{name, func(ctx context.Context) error { _, err := f(ctx, p); return err }} + } + for _, w := range r.s.Writes { + calls = append(calls, plan(w.Name, w.Plan)) + } + if r.s.Drift != nil { + calls = append(calls, plan("Drift", r.s.Drift)) + } + if r.s.Destroy != nil && r.s.Destroy.Plan != nil { + calls = append(calls, plan("Destroy", r.s.Destroy.Plan)) + } + return calls +} + +// guarded runs f, turning a panic into its printed value. +func guarded(f func() error) (recovered string, err error) { + defer func() { + if p := recover(); p != nil { + recovered = fmt.Sprint(p) + } + }() + return "", f() +} + +func isWriteMethod(m string) bool { + return m != http.MethodGet && m != http.MethodHead && m != http.MethodOptions +} + +// 1. Transport only: every request goes through the injected transport, and +// a failing transport fails every call. +func (r *runner) transportOnly(c *caseT) { + if rec, ok := r.recorder(c, CassetteReadWrite); ok { + if _, p, ok := r.open(c, rec); ok { + n := len(rec.Requests()) + caps, err := p.Probe(r.ctx) + switch { + case err != nil: + c.failf("Probe over the read-write cassette: %v", err) + case len(rec.Requests()) == n: + c.failf("Probe sent nothing through the injected transport") + } + for _, cp := range caps.Capabilities() { + if cp.IsWrite() && caps.Get(cp) == cloudkit.Allowed && + !slices.ContainsFunc(r.s.Writes, func(w Write) bool { return w.Capability == cp }) { + c.failf("Probe allows %s, and the suite has no Write for it", cp) + } + } + n = len(rec.Requests()) + if _, err := p.Inventory(r.ctx, cloudkit.Filter{}); err != nil { + c.failf("Inventory over the read-write cassette: %v", err) + } else if len(rec.Requests()) == n { + c.failf("Inventory sent nothing through the injected transport") + } + cassetteDone(c, CassetteReadWrite, rec) + } + } + down := &faultRT{answer: func(*http.Request) (*http.Response, error) { return nil, errTransportDown }} + _, p, ok := r.open(c, down) + if !ok { + return + } + for _, cl := range r.everyCall(p) { + rec, err := guarded(func() error { return cl.run(r.ctx) }) + switch { + case rec != "": + c.failf("%s panicked with a failing transport: %s", cl.name, rec) + case err == nil: + c.failf("%s succeeded although the injected transport fails every request", cl.name) + } + } + if len(down.requests()) == 0 { + c.failf("no request reached the injected transport") + } +} + +// 2. Nil transport: NewRegistry refuses it, and the factory refuses an +// account with no guarded client rather than falling back to the default one. +func (r *runner) nilTransport(c *caseT) { + creds := tokenSource{r.token} + discard := slog.New(slog.DiscardHandler) + transports := []struct { + name string + t cloudkit.Transport + }{ + {"a nil HTTP round tripper", cloudkit.Transport{Dial: refuseDial}}, + {"a nil Dial", cloudkit.Transport{HTTP: &faultRT{answer: func(*http.Request) (*http.Response, error) { return nil, errTransportDown }}}}, + } + for _, tc := range transports { + _, err := cloudkit.NewRegistry(cloudkit.Options{Transport: tc.t, Credentials: creds, Logger: discard}, r.s.Factory) + if !errors.Is(err, cloudkit.ErrInvalidTransport) { + c.failf("NewRegistry with %s: got %v, want ErrInvalidTransport", tc.name, err) + } + } + clients := []struct { + name string + c *http.Client + }{ + {"a nil client", nil}, + {"a client with no Transport, which would use the default one", &http.Client{}}, + } + for _, tc := range clients { + var prov cloudkit.Provider + rec, err := guarded(func() error { + var err error + prov, err = r.s.Factory.New(r.ctx, cloudkit.Account{ + Provider: r.s.Factory.Descriptor.ID, Name: contractAccount, HTTP: tc.c, Logger: discard, + }) + return err + }) + switch { + case rec != "": + c.failf("Factory.New panicked on %s; it must return an error: %s", tc.name, rec) + case err == nil || prov != nil: + c.failf("Factory.New accepted %s; it must refuse it, not fall back (C2)", tc.name) + } + } +} + +// 3. Egress: a request to a host outside APIHosts is refused before the +// transport sees it, and a redirect to an unlisted host is refused. +func (r *runner) egress(c *caseT) { + d := r.s.Factory.Descriptor + redirector := &faultRT{answer: func(req *http.Request) (*http.Response, error) { + h := http.Header{} + h.Set("Location", "https://"+redirectHost+"/elsewhere") + return reply(req, http.StatusFound, h, ""), nil + }} + e, p, ok := r.open(c, redirector) + if !ok { + return + } + got := p.Descriptor() + if !sameHosts(got.APIHosts, d.APIHosts) || !sameHosts(got.RedirectHosts, d.RedirectHosts) { + c.failf("the provider describes hosts %v/%v, its factory %v/%v", got.APIHosts, got.RedirectHosts, d.APIHosts, d.RedirectHosts) + } + if acct, ok := e.lastAccount(); ok && acct.HTTP != nil { + req, err := http.NewRequestWithContext(r.ctx, http.MethodGet, "https://"+outsideHost+"/", nil) + if err != nil { + c.failf("%v", err) + return + } + resp, err := acct.HTTP.Do(req) + if resp != nil { + _ = resp.Body.Close() + } + if !errors.Is(err, cloudkit.ErrEgressDenied) { + c.failf("the adapter's client sent a request to %s: got %v, want ErrEgressDenied", outsideHost, err) + } + } else { + c.failf("the factory was never handed an account with a client") + } + for _, cl := range r.everyCall(p)[:2] { // Probe and Inventory + rec, err := guarded(func() error { return cl.run(r.ctx) }) + switch { + case rec != "": + c.failf("%s panicked on a redirect: %s", cl.name, rec) + case !errors.Is(err, cloudkit.ErrEgressDenied): + c.failf("%s, redirected to an unlisted host: got %v, want an error matching ErrEgressDenied", cl.name, err) + } + } + for _, req := range redirector.requests() { + if h := strings.ToLower(req.URL.Hostname()); !slices.Contains(lower(d.APIHosts), h) { + c.failf("a request to %s reached the transport", h) + } + } +} + +func lower(hs []string) []string { + out := make([]string, len(hs)) + for i, h := range hs { + out[i] = strings.TrimSuffix(strings.ToLower(h), ".") + } + return out +} + +func sameHosts(a, b []string) bool { + x, y := lower(a), lower(b) + slices.Sort(x) + slices.Sort(y) + return slices.Equal(x, y) +} + +// 4. Read-only token: Probe reports every write as Denied, and every write +// plan returns ErrDenied without sending a write. +func (r *runner) readOnly(c *caseT) { + rec, ok := r.recorder(c, CassetteReadOnly) + if !ok { + return + } + _, p, ok := r.open(c, rec) + if !ok { + return + } + caps, err := p.Probe(r.ctx) + if err != nil { + c.failf("Probe with a read-only token: %v", err) + return + } + for _, cp := range caps.Capabilities() { + if cp.IsWrite() && caps.Get(cp) != cloudkit.Denied { + c.failf("Probe reports %s as %s with a read-only token, want denied", cp, caps.Get(cp)) + } + } + for _, w := range r.s.Writes { + if _, said := caps[w.Capability]; !said { + c.failf("Probe with a read-only token does not report %s (needed by %s), want denied", w.Capability, w.Name) + } + } + for _, w := range r.s.Writes { + rec, err := guarded(func() error { _, err := w.Plan(r.ctx, p); return err }) + switch { + case rec != "": + c.failf("%s panicked with a read-only token: %s", w.Name, rec) + case !errors.Is(err, cloudkit.ErrDenied): + c.failf("%s with a read-only token: got %v, want ErrDenied", w.Name, err) + } + } + for _, q := range rec.Requests() { + if isWriteMethod(q.Method) { + c.failf("a read-only token sent %s %s", q.Method, q.URL) + } + } + cassetteDone(c, CassetteReadOnly, rec) +} + +// 5. Plan drift: Apply of a plan whose live state changed returns ErrConflict +// and makes no write. +func (r *runner) planDrift(c *caseT) { + if len(r.s.Writes) == 0 { + c.skipf("the adapter plans no writes") + return + } + rec, ok := r.recorder(c, CassetteDrift) + if !ok { + return + } + e, p, ok := r.open(c, rec) + if !ok { + return + } + plan, err := r.s.Drift(r.ctx, p) + if err != nil { + c.failf("Drift plan: %v", err) + return + } + _, err = e.reg.Apply(r.ctx, plan, cloudkit.Confirmation{Typed: plan.ConfirmText()}) + if !errors.Is(err, cloudkit.ErrConflict) { + c.failf("Apply after the live state moved: got %v, want ErrConflict", err) + } + for _, q := range rec.Requests() { + if isWriteMethod(q.Method) { + c.failf("Apply sent %s %s although the live state moved", q.Method, q.URL) + } + } + cassetteDone(c, CassetteDrift, rec) +} + +// 6. Destroy: without the typed name Apply refuses; the plan lists every +// attached volume and IP; with the name typed back it goes through, so the +// refusals were the name and nothing else. +func (r *runner) destroy(c *caseT) { + if len(r.s.Writes) == 0 { + c.skipf("the adapter plans no writes") + return + } + rec, ok := r.recorder(c, CassetteDestroy) + if !ok { + return + } + e, p, ok := r.open(c, rec) + if !ok { + return + } + plan, err := r.s.Destroy.Plan(r.ctx, p) + if err != nil { + c.failf("Destroy plan: %v", err) + return + } + if !plan.IsDestroy() { + c.failf("the Destroy plan deletes nothing") + return + } + for _, want := range r.s.Destroy.Loses { + if !slices.ContainsFunc(plan.Loses, func(l cloudkit.Resource) bool { return l.Kind == want.Kind && l.ID == want.ID }) { + c.failf("the destroy plan does not list %s %s in Loses", want.Kind, want.ID) + } + } + n := len(rec.Requests()) + for _, typed := range []string{"", "not " + plan.ConfirmText()} { + if _, err := e.reg.Apply(r.ctx, plan, cloudkit.Confirmation{Typed: typed}); !errors.Is(err, cloudkit.ErrConfirmationRequired) { + c.failf("Apply with %q typed: got %v, want ErrConfirmationRequired", typed, err) + } + } + if len(rec.Requests()) != n { + c.failf("a refused destroy sent %d request(s)", len(rec.Requests())-n) + } + if _, err := e.reg.Apply(r.ctx, plan, cloudkit.Confirmation{Typed: plan.ConfirmText()}); err != nil { + c.failf("Apply with the names typed back failed, so the refusals prove nothing: %v", err) + } + cassetteDone(c, CassetteDestroy, rec) +} + +// 7. Redaction: no token, Authorization header or credential query string +// appears in any error, log record or panic, including when a proxy or the +// provider echoes the request back. +func (r *runner) redaction(c *caseT) { + tok := r.token + echo := &faultRT{answer: func(req *http.Request) (*http.Response, error) { + return nil, fmt.Errorf("cloudkittest: proxy refused %s?access_token=%s: Authorization: %s", req.URL.Path, tok, req.Header.Get("Authorization")) + }} + reflect := &faultRT{answer: func(req *http.Request) (*http.Response, error) { + h := http.Header{} + h.Set("Content-Type", "application/json") + body := fmt.Sprintf(`{"message":"refused Authorization: Bearer %s","request":"%s?access_token=%s"}`, tok, req.URL.Path, tok) + return reply(req, http.StatusUnauthorized, h, body), nil + }} + var outputs []string // "where: text" + var urls []string + var logs []*lockedBuffer + for _, f := range []struct { + name string + rt *faultRT + }{{"a proxy that echoes the request", echo}, {"a provider that echoes the token", reflect}} { + e, p, ok := r.open(c, f.rt) + if !ok { + return + } + logs = append(logs, e.logs) + for _, cl := range r.everyCall(p) { + rec, err := guarded(func() error { return cl.run(r.ctx) }) + if rec != "" { + c.failf("%s panicked behind %s", cl.name, f.name) + outputs = append(outputs, cl.name+" panic: "+rec) + } + if err != nil { + outputs = append(outputs, cl.name+" error behind "+f.name+": "+err.Error()) + } + } + for _, q := range f.rt.requests() { + urls = append(urls, q.URL.String()) + } + } + if rec, ok := r.recorder(c, CassetteReadWrite); ok { + spy := &spyRT{next: rec} + if e, p, ok := r.open(c, spy); ok { + logs = append(logs, e.logs) + for _, cl := range r.everyCall(p)[:2] { + if err := cl.run(r.ctx); err != nil { + outputs = append(outputs, cl.name+" error: "+err.Error()) + } + } + spy.mu.Lock() + urls = append(urls, spy.urls...) + spy.mu.Unlock() + cassetteDone(c, CassetteReadWrite, rec) + } + } + for _, o := range outputs { + if strings.Contains(o, tok) { + where, _, _ := strings.Cut(o, ":") + c.failf("the token appears in %s", where) + } + } + for _, l := range logs { + if strings.Contains(l.String(), tok) { + c.failf("a log record carries the token") + } + } + for _, u := range urls { + if strings.Contains(u, tok) { + c.failf("the token was sent in a URL: %s", u) + } + } +} + +// 8. cloud-init: a secret in any field but OneTimeToken fails the render, and +// output above UserDataLimit fails. +func (r *runner) cloudInit(c *caseT) { + limit := r.s.Factory.Descriptor.UserDataLimit + ci := r.s.CloudInit + if ci == nil { + if limit == 0 { + c.skipf("the provider accepts no user data (UserDataLimit 0)") + return + } + c.failf("the descriptor accepts %d bytes of user data, and the suite has no CloudInit to check", limit) + return + } + if !slices.Contains(ci.Fields, OneTimeTokenField) { + c.failf("CloudInit.Fields must include %s", OneTimeTokenField) + return + } + render := func(fields map[string]string) (string, error) { + var out []byte + rec, err := guarded(func() error { + var err error + out, err = ci.Render(r.ctx, fields, limit) + return err + }) + if rec != "" { + return "", fmt.Errorf("panicked: %s", rec) + } + return string(out), err + } + base := map[string]string{} + for _, f := range ci.Fields { + base[f] = "cloudkit-contract" + } + out, err := render(maps.Clone(base)) + if limit == 0 { + if err == nil { + c.failf("Render produced user data for a provider that accepts none") + } + return + } + if err != nil { + c.failf("Render refused plain user data: %v", err) + return + } + if len(out) > limit { + c.failf("plain user data rendered to %d bytes, above the provider's limit of %d", len(out), limit) + } + others := slices.DeleteFunc(slices.Sorted(maps.Keys(base)), func(f string) bool { return f == OneTimeTokenField }) + for _, f := range others { + for _, sh := range secretShapes() { + m := maps.Clone(base) + m[f] = sh.value + if _, err := render(m); err == nil { + c.failf("%s in %s was rendered; only %s may hold a secret", sh.name, f, OneTimeTokenField) + } + } + } + m := maps.Clone(base) + m[OneTimeTokenField] = rand.Text() + if _, err := render(m); err != nil { + c.failf("a one-time token in %s was refused: %v", OneTimeTokenField, err) + } + if len(others) > 0 { + m := maps.Clone(base) + m[others[0]] = strings.Repeat("a", limit+1) + if _, err := render(m); err == nil { + c.failf("user data above the provider's limit of %d bytes was rendered", limit) + } + } else { + c.failf("CloudInit.Fields has no field but %s, so the size limit cannot be checked", OneTimeTokenField) + } +} + +// secretShapes are values any secret lint must refuse, generated per run so +// no credential-shaped literal sits in the source. +func secretShapes() []struct{ name, value string } { + pem := "PRIVATE KEY-----" + return []struct{ name, value string }{ + {"a private key", "-----BEGIN OPENSSH " + pem + "\n" + rand.Text() + "\n-----END OPENSSH " + pem}, + {"a bearer token", "Bearer " + rand.Text()}, + {"a password", "password=" + rand.Text()}, + } +} + +// 9. SSH: an unknown host key fails under every policy except AcceptNew, and +// password auth is never offered. +func (r *runner) ssh(c *caseT) { + s := r.s.SSH + if s == nil { + c.skipf("Suite.SSH is nil: no SSH client to check") + return + } + for _, want := range []string{PolicyKnownHosts, PolicyPinned, PolicyAcceptNew} { + if !slices.Contains(s.Policies, want) { + c.failf("SSH.Policies lacks %s", want) + } + } + for _, pol := range s.Policies { + var offered []string + rec, err := guarded(func() error { + var err error + offered, err = s.Handshake(r.ctx, pol) + return err + }) + switch { + case rec != "": + c.failf("Handshake under %s panicked: %s", pol, rec) + continue + case pol == PolicyAcceptNew && err != nil: + c.failf("under %s, a new host key was refused: %v", pol, err) + case pol != PolicyAcceptNew && err == nil: + c.failf("under %s, a host key never seen before was accepted", pol) + } + for _, m := range offered { + if m == "password" || m == "keyboard-interactive" { + c.failf("under %s, the client offered %s authentication", pol, m) + } + } + } +} diff --git a/src/cloudkittest/doc.go b/src/cloudkittest/doc.go new file mode 100644 index 0000000..bd63f6e --- /dev/null +++ b/src/cloudkittest/doc.go @@ -0,0 +1,66 @@ +// Package cloudkittest is the contract kit every cloudkit adapter passes: nine +// contract cases, run against any [cloudkit.Provider] over spec-checked +// recordings, with nothing reaching the network. +// +// # Running the suite +// +// [Run] runs every case as a subtest; [Check] runs them and returns a +// [Report] without a *testing.T, for a loader that must refuse an adapter +// (or a REST descriptor) whose recordings fail. A [Suite] names the +// adapter's [cloudkit.Factory], its [Recordings], and how to reach its writes +// ([Write], [PlanFunc], [Destroy]), its user-data renderer ([CloudInit]) and +// its SSH client ([SSH]). +// +// spec, err := cloudkittest.LoadSpec(os.DirFS("testdata"), "openapi.json") +// recs, err := cloudkittest.LoadRecordings(os.DirFS("testdata"), "recordings.json", spec) +// cloudkittest.Run(t, cloudkittest.Suite{Factory: hetzner.Factory(), Recordings: recs, ...}) +// +// # Recordings and replay +// +// [Recorder] is an http.RoundTripper that replays one cassette of +// [Recordings] and refuses any request the cassette does not hold. Each +// interaction answers once and in order, so a cassette records how many times +// a call is made and what state each identical call sees. +// +// [ParseRecordings] is the only way to build Recordings, and it checks every +// response against the provider's OpenAPI document ([Spec]): an undescribed +// path or status fails with [ErrNotInSpec], and a body that breaks its schema +// with [ErrSchemaViolation], so a stale or hand-edited fixture cannot load. +// A write path with no recording yet uses a synthetic response, generated +// from the document's examples by [Spec.Synthesize] and marked Synthetic. A +// recording that carries an Authorization header, a cookie, an API-key header +// or any value [cloudkit.Redact] would change is refused whole with +// [ErrCredentialInRecording]. +// +// # The schema subset +// +// The validator implements the part of JSON Schema, as OpenAPI 3.0 spells it, +// that provider responses use. Assertions, all checked: +// +// - type: "object", "array", "string", "number", "integer" or "boolean", +// as a string or an array of them ("null" in an array means nullable). +// "integer" is a number with no fractional part. +// - nullable: true admits null. Without it, null fails whatever the type. +// - required: property names an object must have. +// - properties: a schema per property name. +// - additionalProperties: true, false, or a schema for every property not +// in properties. Absent, any extra property is allowed. +// - items: one schema every array element must match. +// - enum: the value must equal one of the listed JSON values; numbers +// compare by value. +// - $ref: a JSON pointer within the same document ("#/..."), with no +// sibling keyword but description. +// +// Annotations, accepted and not checked: description, title, example, +// default, format, deprecated, readOnly, writeOnly. +// +// Any other keyword (allOf, oneOf, anyOf, not, pattern, minimum, maxLength, +// and the rest) fails [ParseSpec] with [ErrUnsupportedKeyword], naming the +// keyword and where it sits. It is never ignored: a schema is either checked +// in full or not loaded, and the validator grows when a provider's document +// needs it to. +// +// The package uses the standard library only and, like cloudkit itself, never +// touches the standard library's default HTTP client or transport: the +// policy tests in internal/policy scan it with the rest of the module. +package cloudkittest diff --git a/src/cloudkittest/provider_test.go b/src/cloudkittest/provider_test.go new file mode 100644 index 0000000..018940c --- /dev/null +++ b/src/cloudkittest/provider_test.go @@ -0,0 +1,327 @@ +package cloudkittest_test + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "os" + "regexp" + "slices" + "sort" + "strings" + "testing" + + "github.com/simtabi/cloudkit/src" + "github.com/simtabi/cloudkit/src/cloudkittest" +) + +// The test provider is the core tests' fake provider (src/fake_test.go) written +// again against the replayed API in testdata/fake.openapi.json: Probe reads +// GET /token, Inventory GET /resources, and Apply deletes through +// DELETE /resources/{id}. Each field of defects plants the defect one +// contract case exists to catch. + +const fakeHost = "api.fake.test" + +type defects struct { + bypassTransport bool // 1: talks to its own canned transport, not Account.HTTP + acceptNilClient bool // 2: builds itself over a nil or default client + flattenErrors bool // 3: drops the error chain of a failed request + probeIgnoresToken bool // 4: reports writes allowed whatever the token says + staleLiveState bool // 5: re-uses the state it planned against + dropAttached bool // 6: lists only the target in a destroy's Loses + leakErrors bool // 7: returns transport errors unredacted + renderIgnoresLimit bool // 8: renders user data of any size + offerPassword bool // 9: offers password auth over SSH +} + +func fakeDescriptor() cloudkit.Descriptor { + return cloudkit.Descriptor{ + ID: "fake", + Title: "Fake Cloud", + DocsURL: "https://docs.fake.test/api", + APIHosts: []string{fakeHost}, + RedirectHosts: []string{"objects.fake.test"}, + TokenModel: cloudkit.ReadOrReadWrite, + UserDataLimit: 4 << 10, + } +} + +type testProvider struct { + acct cloudkit.Account + client *http.Client + defects defects + cache *planCache // shared by every provider one factory builds +} + +// planCache is where the stale defect keeps the state it planned against. +// It lives in the factory, as an adapter-wide cache would, so it survives the +// fresh provider Registry.Apply opens. +type planCache struct{ live []cloudkit.Resource } + +func fakeFactory(d defects) cloudkit.Factory { + cache := &planCache{} + return cloudkit.Factory{Descriptor: fakeDescriptor(), New: func(_ context.Context, a cloudkit.Account) (cloudkit.Provider, error) { + if !d.acceptNilClient && (a.HTTP == nil || a.HTTP.Transport == nil) { + return nil, errors.New("fake: the account has no guarded client") + } + p := &testProvider{acct: a, client: a.HTTP, defects: d, cache: cache} + if d.bypassTransport { + p.client = &http.Client{Transport: cannedRT{}} + } + return p, nil + }} +} + +// cannedRT answers like the provider without touching any transport the +// caller gave: what an SDK that builds its own client does. +type cannedRT struct{} + +func (cannedRT) RoundTrip(req *http.Request) (*http.Response, error) { + body := `[]` + if strings.HasSuffix(req.URL.Path, "/token") { + body = `{"write":true}` + } + return &http.Response{StatusCode: http.StatusOK, Header: http.Header{}, Body: io.NopCloser(strings.NewReader(body)), Request: req}, nil +} + +func (p *testProvider) Descriptor() cloudkit.Descriptor { return fakeDescriptor() } + +func (p *testProvider) Probe(ctx context.Context) (cloudkit.CapSet, error) { + var tok struct{ Write bool } + if err := p.call(ctx, http.MethodGet, "/v1/token", &tok); err != nil { + return nil, err + } + write := cloudkit.Denied + if tok.Write || p.defects.probeIgnoresToken { + write = cloudkit.Allowed + } + return cloudkit.CapSet{ + cloudkit.CapInventory: cloudkit.Allowed, + cloudkit.CapServerCreate: write, + cloudkit.CapServerDestroy: write, + }, nil +} + +func (p *testProvider) Inventory(ctx context.Context, f cloudkit.Filter) ([]cloudkit.Resource, error) { + var all []cloudkit.Resource + if err := p.call(ctx, http.MethodGet, "/v1/resources", &all); err != nil { + return nil, err + } + return slices.DeleteFunc(all, func(r cloudkit.Resource) bool { return !f.Matches(r) }), nil +} + +func (p *testProvider) writable(ctx context.Context) error { + caps, err := p.Probe(ctx) + if err != nil { + return err + } + if !caps.Allowed(cloudkit.CapServerCreate) { + return fmt.Errorf("%w: the token is read only", cloudkit.ErrDenied) + } + return nil +} + +func (p *testProvider) PlanCreate(ctx context.Context, r cloudkit.Resource) (cloudkit.Plan, error) { + if err := p.writable(ctx); err != nil { + return cloudkit.Plan{}, err + } + live, err := p.Inventory(ctx, cloudkit.Filter{}) + if err != nil { + return cloudkit.Plan{}, err + } + p.cache.live = live + return cloudkit.NewPlan(p.acct.Provider, p.acct.Name, []cloudkit.Change{{Action: cloudkit.Create, After: &r}}, nil, live) +} + +func (p *testProvider) PlanDestroy(ctx context.Context, id string) (cloudkit.Plan, error) { + if err := p.writable(ctx); err != nil { + return cloudkit.Plan{}, err + } + live, err := p.Inventory(ctx, cloudkit.Filter{}) + if err != nil { + return cloudkit.Plan{}, err + } + p.cache.live = live + i := slices.IndexFunc(live, func(r cloudkit.Resource) bool { return r.ID == id }) + if i < 0 { + return cloudkit.Plan{}, fmt.Errorf("%w: %s", cloudkit.ErrNotFound, id) + } + target := live[i] + loses := []cloudkit.Resource{target} + if !p.defects.dropAttached { + for _, r := range live { + if r.Attrs["attached_to"] == id { + loses = append(loses, r) + } + } + } + return cloudkit.NewPlan(p.acct.Provider, p.acct.Name, []cloudkit.Change{{Action: cloudkit.Delete, Before: &target}}, loses, live) +} + +func (p *testProvider) LiveState(ctx context.Context, _ cloudkit.Plan) ([]cloudkit.Resource, error) { + if p.defects.staleLiveState && p.cache.live != nil { + return p.cache.live, nil + } + return p.Inventory(ctx, cloudkit.Filter{}) +} + +func (p *testProvider) Execute(ctx context.Context, pl cloudkit.Plan) (cloudkit.Result, error) { + var res cloudkit.Result + for _, c := range pl.Changes { + switch c.Action { + case cloudkit.Create: + var r cloudkit.Resource + if err := p.send(ctx, http.MethodPost, "/v1/resources", c.After, &r); err != nil { + return cloudkit.Result{}, err + } + res.Resources = append(res.Resources, r) + case cloudkit.Delete: + var out struct{ Action struct{ Status string } } + if err := p.send(ctx, http.MethodDelete, "/v1/resources/"+c.Before.ID, nil, &out); err != nil { + return cloudkit.Result{}, err + } + if out.Action.Status != "success" { + return cloudkit.Result{}, fmt.Errorf("fake: delete of %s ended %q", c.Before.ID, out.Action.Status) + } + default: + return cloudkit.Result{}, fmt.Errorf("%w: %s", cloudkit.ErrUnsupported, c.Action) + } + } + return res, nil +} + +func (p *testProvider) call(ctx context.Context, method, path string, out any) error { + return p.send(ctx, method, path, nil, out) +} + +func (p *testProvider) send(ctx context.Context, method, path string, in, out any) error { + var body io.Reader + if in != nil { + b, err := json.Marshal(in) + if err != nil { + return err + } + body = bytes.NewReader(b) + } + req, err := http.NewRequestWithContext(ctx, method, "https://"+fakeHost+path, body) + if err != nil { + return err + } + resp, err := p.client.Do(req) + if err != nil { + switch { + case p.defects.leakErrors: + return fmt.Errorf("fake: %s %s: %w", method, path, err) + case p.defects.flattenErrors: + return errors.New(cloudkit.Redact(err.Error())) + } + return cloudkit.RedactError(fmt.Errorf("fake: %s %s: %w", method, path, err)) + } + defer func() { _ = resp.Body.Close() }() + b, _ := io.ReadAll(io.LimitReader(resp.Body, 1<<20)) + p.acct.Logger.DebugContext(ctx, "fake: call", "method", method, "path", path, "status", resp.StatusCode) + switch { + case resp.StatusCode == http.StatusForbidden: + return fmt.Errorf("%w: %s %s", cloudkit.ErrDenied, method, path) + case resp.StatusCode == http.StatusNotFound: + return fmt.Errorf("%w: %s %s", cloudkit.ErrNotFound, method, path) + case resp.StatusCode >= 300: + return fmt.Errorf("fake: %s %s: %d %s", method, path, resp.StatusCode, cloudkit.RedactBody(b)) + } + if out != nil && len(b) > 0 { + return json.Unmarshal(b, out) + } + return nil +} + +// A stand-in for the cloud-init renderer, which does not exist yet: enough to show that the cloud-init +// case catches a renderer that lets a secret or an oversized file through. +var reSecretValue = regexp.MustCompile(`(?i)PRIVATE KEY-----|\bbearer\s+\S{8,}|\b(password|passwd|secret)\s*[=:]`) + +func standInRender(ignoreLimit bool) func(context.Context, map[string]string, int) ([]byte, error) { + return func(_ context.Context, fields map[string]string, limit int) ([]byte, error) { + names := make([]string, 0, len(fields)) + for k := range fields { + names = append(names, k) + } + sort.Strings(names) + var b strings.Builder + b.WriteString("#cloud-config\n") + for _, k := range names { + if k != cloudkittest.OneTimeTokenField && reSecretValue.MatchString(fields[k]) { + return nil, fmt.Errorf("user data field %s looks like a secret", k) + } + fmt.Fprintf(&b, "# %s: %q\n", k, fields[k]) + } + if !ignoreLimit && b.Len() > limit { + return nil, fmt.Errorf("user data is %d bytes, above the provider's %d", b.Len(), limit) + } + return []byte(b.String()), nil + } +} + +// A stand-in for the SSH client, for the same reason. +func standInHandshake(offerPassword bool) func(context.Context, string) ([]string, error) { + return func(_ context.Context, policy string) ([]string, error) { + offered := []string{"publickey"} + if offerPassword { + offered = append(offered, "password") + } + if policy != cloudkittest.PolicyAcceptNew { + return offered, fmt.Errorf("ssh: host key not known under %s", policy) + } + return offered, nil + } +} + +func loadFixtures(t *testing.T) (*cloudkittest.Spec, *cloudkittest.Recordings) { + t.Helper() + fsys := os.DirFS("testdata") + spec, err := cloudkittest.LoadSpec(fsys, "fake.openapi.json") + if err != nil { + t.Fatal(err) + } + recs, err := cloudkittest.LoadRecordings(fsys, "fake.recordings.json", spec) + if err != nil { + t.Fatal(err) + } + return spec, recs +} + +// suite is the contract suite for the test provider with the given defects. +func suite(t *testing.T, d defects) cloudkittest.Suite { + t.Helper() + _, recs := loadFixtures(t) + asFake := func(p cloudkit.Provider) *testProvider { return p.(*testProvider) } + create := func(ctx context.Context, p cloudkit.Provider) (cloudkit.Plan, error) { + return asFake(p).PlanCreate(ctx, cloudkit.Resource{Kind: "server", Name: "web-2", Region: "fsn1"}) + } + destroy := func(ctx context.Context, p cloudkit.Provider) (cloudkit.Plan, error) { + return asFake(p).PlanDestroy(ctx, "s1") + } + return cloudkittest.Suite{ + Factory: fakeFactory(d), + Recordings: recs, + Writes: []cloudkittest.Write{ + {Name: "PlanCreate", Capability: cloudkit.CapServerCreate, Plan: create}, + {Name: "PlanDestroy", Capability: cloudkit.CapServerDestroy, Plan: destroy}, + }, + Drift: create, + Destroy: &cloudkittest.Destroy{Plan: destroy, Loses: []cloudkit.Resource{ + {Kind: "server", ID: "s1"}, {Kind: "volume", ID: "v1"}, {Kind: "ipv4", ID: "ip1"}, + }}, + CloudInit: &cloudkittest.CloudInit{ + Render: standInRender(d.renderIgnoresLimit), + Fields: []string{"Hostname", "SSHAuthorizedKey", "Steps", cloudkittest.OneTimeTokenField}, + }, + SSH: &cloudkittest.SSH{ + Policies: []string{cloudkittest.PolicyKnownHosts, cloudkittest.PolicyPinned, cloudkittest.PolicyFromProvider, cloudkittest.PolicyAcceptNew}, + Handshake: standInHandshake(d.offerPassword), + }, + } +} diff --git a/src/cloudkittest/recorder.go b/src/cloudkittest/recorder.go new file mode 100644 index 0000000..f3dca6f --- /dev/null +++ b/src/cloudkittest/recorder.go @@ -0,0 +1,189 @@ +package cloudkittest + +import ( + "bytes" + "errors" + "fmt" + "io" + "net/http" + "net/url" + "strings" + "sync" + + "github.com/simtabi/cloudkit/src" +) + +// ErrUnexpectedRequest means a [Recorder] received a request its cassette +// does not hold, or holds fewer times than it was sent. +var ErrUnexpectedRequest = errors.New("cloudkittest: unexpected request") + +// UnexpectedRequestError names a request a [Recorder] refused. It matches +// [ErrUnexpectedRequest]. The URL has been through [cloudkit.Redact]. +type UnexpectedRequestError struct { + Cassette string + Method, URL string +} + +func (e *UnexpectedRequestError) Error() string { + return fmt.Sprintf("cloudkittest: unexpected request in cassette %s: %s %s", e.Cassette, e.Method, e.URL) +} + +// Unwrap lets errors.Is match [ErrUnexpectedRequest]. +func (e *UnexpectedRequestError) Unwrap() error { return ErrUnexpectedRequest } + +// Request is a request a [Recorder] saw, with its URL redacted. +type Request struct { + Method, URL string + Unexpected bool // the cassette did not hold it, so it was refused +} + +// Recorder is an http.RoundTripper that replays one cassette. A request is +// answered by the first interaction not yet used whose method, URL (scheme, +// host, path, and query in any order) and JSON body match; each interaction +// answers once, so a cassette records how many times a call is made, and in +// what order identical calls see different state. Any other request gets an +// [*UnexpectedRequestError] and is never sent anywhere. Headers are not +// matched, so the Authorization header cloudkit adds is ignored and never +// stored. It is safe for concurrent use. +// +// Recording new responses from a live provider is the job of a scripted +// recording run, not of this type. +type Recorder struct { + cassette string + list []interaction + + mu sync.Mutex + used []bool + seen []Request +} + +// NewRecorder returns a recorder over one cassette of recs. +func NewRecorder(recs *Recordings, cassette string) (*Recorder, error) { + if recs == nil { + return nil, errors.New("cloudkittest: NewRecorder: recordings are nil") + } + list, ok := recs.cassettes[cassette] + if !ok { + return nil, fmt.Errorf("cloudkittest: recordings for %s have no cassette %q", recs.provider, cassette) + } + return &Recorder{cassette: cassette, list: list, used: make([]bool, len(list))}, nil +} + +// RoundTrip implements http.RoundTripper. +func (r *Recorder) RoundTrip(req *http.Request) (*http.Response, error) { + var body []byte + if req.Body != nil { + b, err := io.ReadAll(req.Body) + _ = req.Body.Close() + if err != nil { + return nil, err + } + body = b + } + if err := req.Context().Err(); err != nil { + return nil, err + } + shown := cloudkit.Redact(req.URL.String()) + r.mu.Lock() + defer r.mu.Unlock() + for i, in := range r.list { + if r.used[i] || !in.matches(req, body) { + continue + } + r.used[i] = true + r.seen = append(r.seen, Request{Method: req.Method, URL: shown}) + return in.response(req), nil + } + r.seen = append(r.seen, Request{Method: req.Method, URL: shown, Unexpected: true}) + return nil, &UnexpectedRequestError{Cassette: r.cassette, Method: req.Method, URL: shown} +} + +// Requests returns every request the recorder saw, in order, the refused +// ones included. +func (r *Recorder) Requests() []Request { + r.mu.Lock() + defer r.mu.Unlock() + return append([]Request(nil), r.seen...) +} + +// Unexpected returns the requests the cassette did not hold. +func (r *Recorder) Unexpected() []Request { + r.mu.Lock() + defer r.mu.Unlock() + var out []Request + for _, s := range r.seen { + if s.Unexpected { + out = append(out, s) + } + } + return out +} + +// Unused returns the cassette's interactions no request has used yet. A +// passing contract case uses every one, so a leftover means the adapter +// skipped a call it was recorded making. +func (r *Recorder) Unused() []Request { + r.mu.Lock() + defer r.mu.Unlock() + var out []Request + for i, in := range r.list { + if !r.used[i] { + out = append(out, Request{Method: in.method, URL: in.url.String()}) + } + } + return out +} + +// Err returns every unexpected request as one error, or nil. +func (r *Recorder) Err() error { + var errs []error + for _, u := range r.Unexpected() { + errs = append(errs, &UnexpectedRequestError{Cassette: r.cassette, Method: u.Method, URL: u.URL}) + } + return errors.Join(errs...) +} + +func (in interaction) matches(req *http.Request, body []byte) bool { + if req.Method != in.method || !sameURL(req.URL, in.url) { + return false + } + if len(body) == 0 { + return !in.hasBody + } + if !in.hasBody { + return false + } + v, err := decodeJSON(body) + return err == nil && equalJSON(v, in.body) +} + +func sameURL(a, b *url.URL) bool { + if a.Scheme != b.Scheme || !strings.EqualFold(a.Host, b.Host) || a.EscapedPath() != b.EscapedPath() { + return false + } + qa, qb := a.Query(), b.Query() + if len(qa) != len(qb) { + return false + } + for k, va := range qa { + vb, ok := qb[k] + if !ok || strings.Join(va, "\x00") != strings.Join(vb, "\x00") { + return false + } + } + return true +} + +func (in interaction) response(req *http.Request) *http.Response { + return &http.Response{ + Status: fmt.Sprintf("%d %s", in.resp.Status, http.StatusText(in.resp.Status)), + StatusCode: in.resp.Status, + Proto: "HTTP/1.1", + ProtoMajor: 1, + ProtoMinor: 1, + Header: in.resp.Header.Clone(), + Body: io.NopCloser(bytes.NewReader(in.resp.Body)), + ContentLength: int64(len(in.resp.Body)), + Request: req, + } +} diff --git a/src/cloudkittest/recordings.go b/src/cloudkittest/recordings.go new file mode 100644 index 0000000..d8f5f8b --- /dev/null +++ b/src/cloudkittest/recordings.go @@ -0,0 +1,243 @@ +package cloudkittest + +import ( + "bytes" + "encoding/json" + "errors" + "fmt" + "io" + "io/fs" + "net/http" + "net/url" + "regexp" + "slices" + + "github.com/simtabi/cloudkit/src" +) + +// ErrCredentialInRecording means a recording carries an Authorization header, +// a cookie, an API-key header, or a value [cloudkit.Redact] would change (a +// bearer token, a credential query parameter, URL userinfo). Recordings are +// committed to a public repository, so such a file is refused whole. +var ErrCredentialInRecording = errors.New("cloudkittest: recording carries a credential") + +// credentialHeaders are refused in a recording, in a request or a response, +// whatever their value. +var credentialHeaders = []string{ + "Authorization", "Proxy-Authorization", "Cookie", "Set-Cookie", + "X-Auth-Token", "X-Api-Key", "Api-Key", +} + +var reCassette = regexp.MustCompile(`^[a-z][a-z0-9]*(-[a-z0-9]+)*$`) + +// Recordings are one provider's recorded responses, as named cassettes, each +// an ordered list of request and response pairs. Build them with +// [ParseRecordings], which checks every response, recorded or synthetic, +// against the provider's [Spec]; there is no other way to make one, so every +// response a [Recorder] replays has been checked. +type Recordings struct { + provider string + cassettes map[string][]interaction +} + +type interaction struct { + method string + url *url.URL + body any // decoded request body; nil with hasBody false means none + hasBody bool + resp Response +} + +// The file format, in JSON: +// +// { +// "provider": "fake", +// "cassettes": { +// "read-write": [ +// {"request": {"method": "GET", "url": "https://api.fake.test/v1/token"}, +// "response": {"status": 200, "headers": {"Content-Type": ["application/json"]}, "body": {"write": true}}}, +// {"request": {"method": "DELETE", "url": "https://api.fake.test/v1/resources/s1"}, +// "synthetic": {"status": 200, "example": "deleted"}} +// ] +// } +// } +type fileRecordings struct { + Provider string `json:"provider"` + Cassettes map[string][]fileInteraction `json:"cassettes"` +} + +type fileInteraction struct { + Request fileRequest `json:"request"` + Response *fileResponse `json:"response,omitempty"` + Synthetic *fileSynthetic `json:"synthetic,omitempty"` +} + +type fileRequest struct { + Method string `json:"method"` + URL string `json:"url"` + Header http.Header `json:"headers,omitempty"` + Body json.RawMessage `json:"body,omitempty"` +} + +type fileResponse struct { + Status int `json:"status"` + Header http.Header `json:"headers,omitempty"` + Body json.RawMessage `json:"body,omitempty"` +} + +type fileSynthetic struct { + Status int `json:"status"` + Example string `json:"example,omitempty"` +} + +// ParseRecordings reads a recordings file and checks it against spec. It +// refuses unknown fields, a credential anywhere ([ErrCredentialInRecording]), +// an interaction with both or neither of a response and a synthetic entry, +// and any response the spec does not describe ([ErrNotInSpec]) or whose body +// breaks its schema ([ErrSchemaViolation]). A synthetic entry is generated +// with [Spec.Synthesize] and marked Synthetic. +func ParseRecordings(b []byte, spec *Spec) (*Recordings, error) { + if spec == nil { + return nil, errors.New("cloudkittest: recordings need the provider's spec; every response is checked against it") + } + dec := json.NewDecoder(bytes.NewReader(b)) + dec.DisallowUnknownFields() + var f fileRecordings + if err := dec.Decode(&f); err != nil { + return nil, fmt.Errorf("cloudkittest: recordings: %w", err) + } + if err := dec.Decode(&struct{}{}); !errors.Is(err, io.EOF) { + return nil, errors.New("cloudkittest: recordings: trailing data after the JSON object") + } + if !reCassette.MatchString(f.Provider) { + return nil, fmt.Errorf("cloudkittest: recordings name no valid provider (%q)", f.Provider) + } + if len(f.Cassettes) == 0 { + return nil, errors.New("cloudkittest: recordings hold no cassettes") + } + recs := &Recordings{provider: f.Provider, cassettes: map[string][]interaction{}} + for name, list := range f.Cassettes { + if !reCassette.MatchString(name) { + return nil, fmt.Errorf("cloudkittest: cassette name %q must be lower-case words joined by hyphens", name) + } + if len(list) == 0 { + return nil, fmt.Errorf("cloudkittest: cassette %s is empty", name) + } + for i, fi := range list { + in, err := fi.resolve(spec) + if err != nil { + return nil, fmt.Errorf("cloudkittest: cassette %s, interaction %d: %w", name, i, err) + } + recs.cassettes[name] = append(recs.cassettes[name], in) + } + } + return recs, nil +} + +// LoadRecordings reads and parses a recordings file from fsys. +func LoadRecordings(fsys fs.FS, name string, spec *Spec) (*Recordings, error) { + b, err := fs.ReadFile(fsys, name) + if err != nil { + return nil, err + } + r, err := ParseRecordings(b, spec) + if err != nil { + return nil, fmt.Errorf("%s: %w", name, err) + } + return r, nil +} + +// Provider returns the provider ID the recordings were made for. +func (r *Recordings) Provider() string { return r.provider } + +// Cassettes returns the cassette names, sorted. +func (r *Recordings) Cassettes() []string { + out := make([]string, 0, len(r.cassettes)) + for k := range r.cassettes { + out = append(out, k) + } + slices.Sort(out) + return out +} + +// Len returns the number of interactions in a cassette, 0 if it is absent. +func (r *Recordings) Len(cassette string) int { return len(r.cassettes[cassette]) } + +func (fi fileInteraction) resolve(spec *Spec) (interaction, error) { + req := fi.Request + if err := noCredentials("request", req.URL, req.Header, req.Body); err != nil { + return interaction{}, err + } + switch req.Method { + case http.MethodGet, http.MethodHead, http.MethodPost, http.MethodPut, http.MethodPatch, http.MethodDelete, http.MethodOptions: + default: + return interaction{}, fmt.Errorf("request method %q is not an upper-case HTTP method", req.Method) + } + u, err := url.Parse(req.URL) + if err != nil || u.Scheme != "https" || u.Host == "" { + return interaction{}, fmt.Errorf("request URL %q must be an absolute https URL", req.URL) + } + in := interaction{method: req.Method, url: u} + if len(req.Body) > 0 { + v, err := decodeJSON(req.Body) + if err != nil { + return interaction{}, fmt.Errorf("request body is not JSON: %w", err) + } + in.body, in.hasBody = v, true + } + switch { + case fi.Response != nil && fi.Synthetic != nil: + return interaction{}, errors.New("an interaction has a response or a synthetic entry, not both") + case fi.Response != nil: + fr := fi.Response + if err := noCredentials("response", "", fr.Header, fr.Body); err != nil { + return interaction{}, err + } + h := fr.Header.Clone() + if h == nil { + h = http.Header{} + } + body := []byte(fr.Body) + if len(body) > 0 && h.Get("Content-Type") == "" { + h.Set("Content-Type", "application/json") + } + if err := spec.ValidateResponse(req.Method, req.URL, fr.Status, h, body); err != nil { + return interaction{}, err + } + in.resp = Response{Status: fr.Status, Header: h, Body: body} + case fi.Synthetic != nil: + resp, err := spec.Synthesize(req.Method, req.URL, fi.Synthetic.Status, fi.Synthetic.Example) + if err != nil { + return interaction{}, err + } + if err := noCredentials("synthetic response", "", resp.Header, resp.Body); err != nil { + return interaction{}, err + } + in.resp = resp + default: + return interaction{}, errors.New("an interaction needs a response or a synthetic entry") + } + return in, nil +} + +// noCredentials refuses a credential header by name, and any URL, header +// value or body that cloudkit.Redact would change. +func noCredentials(where, rawURL string, h http.Header, body []byte) error { + for name, vals := range h { + if slices.Contains(credentialHeaders, http.CanonicalHeaderKey(name)) { + return fmt.Errorf("%w: the %s has a %s header", ErrCredentialInRecording, where, http.CanonicalHeaderKey(name)) + } + for _, v := range vals { + if cloudkit.Redact(v) != v { + return fmt.Errorf("%w: the %s's %s header", ErrCredentialInRecording, where, http.CanonicalHeaderKey(name)) + } + } + } + if rawURL != "" && cloudkit.Redact(rawURL) != rawURL { + return fmt.Errorf("%w: the %s URL", ErrCredentialInRecording, where) + } + if s := string(body); cloudkit.Redact(s) != s { + return fmt.Errorf("%w: the %s body", ErrCredentialInRecording, where) + } + return nil +} diff --git a/src/cloudkittest/recordings_test.go b/src/cloudkittest/recordings_test.go new file mode 100644 index 0000000..06e4bf4 --- /dev/null +++ b/src/cloudkittest/recordings_test.go @@ -0,0 +1,215 @@ +package cloudkittest_test + +import ( + "context" + "crypto/rand" + "errors" + "io" + "net/http" + "os" + "strings" + "testing" + + "github.com/simtabi/cloudkit/src/cloudkittest" +) + +func fakeSpec(t *testing.T) *cloudkittest.Spec { + t.Helper() + spec, _ := loadFixtures(t) + return spec +} + +// one is a recordings file with one cassette holding one interaction. +func one(interaction string) []byte { + return []byte(`{"provider":"fake","cassettes":{"read-write":[` + interaction + `]}}`) +} + +const tokenOK = `{"request":{"method":"GET","url":"https://api.fake.test/v1/token"}, + "response":{"status":200,"headers":{"Content-Type":["application/json"]},"body":{"write":true}}}` + +func TestTheFixturesLoad(t *testing.T) { + _, recs := loadFixtures(t) + if got := strings.Join(recs.Cassettes(), ","); got != "destroy,drift,read-only,read-write" { + t.Errorf("cassettes %s", got) + } + if recs.Provider() != "fake" || recs.Len(cloudkittest.CassetteDestroy) != 4 || recs.Len("absent") != 0 { + t.Errorf("provider %q, destroy %d", recs.Provider(), recs.Len(cloudkittest.CassetteDestroy)) + } +} + +// A fixture whose response breaks the provider's schema does not load. +func TestAFixtureThatBreaksTheSchemaFails(t *testing.T) { + _, err := cloudkittest.LoadRecordings(os.DirFS("testdata"), "broken-schema.recordings.json", fakeSpec(t)) + if !errors.Is(err, cloudkittest.ErrSchemaViolation) { + t.Fatalf("got %v, want ErrSchemaViolation", err) + } + if !strings.Contains(err.Error(), "$.write") { + t.Errorf("the error does not name the field: %v", err) + } +} + +// A recording never carries an Authorization header or a token: a request +// or response header by name, or a credential anywhere Redact finds one, and +// the file is refused whole. +func TestRecordingsRefuseCredentials(t *testing.T) { + tok := rand.Text() + cases := map[string]string{ + "request Authorization": `{"request":{"method":"GET","url":"https://api.fake.test/v1/token","headers":{"Authorization":["Bearer ` + tok + `"]}}, + "response":{"status":200,"body":{"write":true}}}`, + "lower-case authorization": `{"request":{"method":"GET","url":"https://api.fake.test/v1/token","headers":{"authorization":["x"]}}, + "response":{"status":200,"body":{"write":true}}}`, + "response Set-Cookie": `{"request":{"method":"GET","url":"https://api.fake.test/v1/token"}, + "response":{"status":200,"headers":{"Set-Cookie":["s=1"]},"body":{"write":true}}}`, + "X-Api-Key": `{"request":{"method":"GET","url":"https://api.fake.test/v1/token","headers":{"X-Api-Key":["k"]}}, + "response":{"status":200,"body":{"write":true}}}`, + "token in a query string": `{"request":{"method":"GET","url":"https://api.fake.test/v1/token?access_token=` + tok + `"}, + "response":{"status":200,"body":{"write":true}}}`, + "bearer token in a body": `{"request":{"method":"GET","url":"https://api.fake.test/v1/token"}, + "response":{"status":401,"body":{"message":"Bearer ` + tok + ` is not valid"}}}`, + "token in a request body": `{"request":{"method":"POST","url":"https://api.fake.test/v1/resources","body":{"note":"Authorization: Bearer ` + tok + `"}}, + "response":{"status":201,"body":{"kind":"server","id":"s2"}}}`, + "userinfo in the URL": `{"request":{"method":"GET","url":"https://admin:` + tok + `@api.fake.test/v1/token"}, + "response":{"status":200,"body":{"write":true}}}`, + } + spec := fakeSpec(t) + for name, in := range cases { + _, err := cloudkittest.ParseRecordings(one(in), spec) + if !errors.Is(err, cloudkittest.ErrCredentialInRecording) { + t.Errorf("%s: got %v, want ErrCredentialInRecording", name, err) + } + if err != nil && strings.Contains(err.Error(), tok) { + t.Errorf("%s: the refusal repeats the token", name) + } + } +} + +func TestParseRecordingsRefusesMalformedFiles(t *testing.T) { + spec := fakeSpec(t) + cases := map[string][]byte{ + "no spec": nil, + "unknown field": []byte(`{"provider":"fake","cassettes":{"read-write":[` + tokenOK + `]},"extra":1}`), + "no provider": []byte(`{"cassettes":{"read-write":[` + tokenOK + `]}}`), + "no cassettes": []byte(`{"provider":"fake","cassettes":{}}`), + "empty cassette": []byte(`{"provider":"fake","cassettes":{"read-write":[]}}`), + "bad cassette": []byte(`{"provider":"fake","cassettes":{"Read Write":[` + tokenOK + `]}}`), + "trailing data": []byte(`{"provider":"fake","cassettes":{"read-write":[` + tokenOK + `]}} {}`), + "lower-case verb": one(`{"request":{"method":"get","url":"https://api.fake.test/v1/token"},"response":{"status":200,"body":{"write":true}}}`), + "plain http": one(`{"request":{"method":"GET","url":"http://api.fake.test/v1/token"},"response":{"status":200,"body":{"write":true}}}`), + "both": one(`{"request":{"method":"GET","url":"https://api.fake.test/v1/token"},"response":{"status":200,"body":{"write":true}},"synthetic":{"status":200}}`), + "neither": one(`{"request":{"method":"GET","url":"https://api.fake.test/v1/token"}}`), + "undescribed path": one(`{"request":{"method":"GET","url":"https://api.fake.test/v1/secrets"},"response":{"status":200,"body":{}}}`), + "undescribed code": one(`{"request":{"method":"DELETE","url":"https://api.fake.test/v1/resources/s1"},"response":{"status":204}}`), + "bad synthetic": one(`{"request":{"method":"DELETE","url":"https://api.fake.test/v1/resources/s1"},"synthetic":{"status":200,"example":"nope"}}`), + } + for name, b := range cases { + s := spec + if name == "no spec" { + s, b = nil, one(tokenOK) + } + if _, err := cloudkittest.ParseRecordings(b, s); err == nil { + t.Errorf("%s: accepted", name) + } + } + if _, err := cloudkittest.ParseRecordings(one(tokenOK), spec); err != nil { + t.Errorf("a valid file refused: %v", err) + } +} + +// get sends one GET through rt and returns the body it was answered with. +func get(t *testing.T, rt http.RoundTripper, url string) (string, error) { + t.Helper() + req, err := http.NewRequestWithContext(context.Background(), http.MethodGet, url, nil) + if err != nil { + t.Fatal(err) + } + resp, err := rt.RoundTrip(req) + if err != nil { + return "", err + } + defer func() { _ = resp.Body.Close() }() + b, err := io.ReadAll(resp.Body) + return string(b), err +} + +// The recorder fails any request its cassette does not hold, answers each +// interaction once and in order, and reports what was left unused. +func TestRecorderFailsUnexpectedRequests(t *testing.T) { + _, recs := loadFixtures(t) + rec, err := cloudkittest.NewRecorder(recs, cloudkittest.CassetteDrift) + if err != nil { + t.Fatal(err) + } + if _, err := get(t, rec, "https://api.fake.test/v1/elsewhere"); !errors.Is(err, cloudkittest.ErrUnexpectedRequest) { + t.Fatalf("an unrecorded request: got %v, want ErrUnexpectedRequest", err) + } + var bodies []string + for range 2 { + b, err := get(t, rec, "https://api.fake.test/v1/resources") + if err != nil { + t.Fatal(err) + } + bodies = append(bodies, b) + } + if !strings.Contains(bodies[0], `"small"`) || !strings.Contains(bodies[1], `"large"`) { + t.Errorf("identical requests were not answered in recorded order: %v", bodies) + } + if _, err := get(t, rec, "https://api.fake.test/v1/resources"); !errors.Is(err, cloudkittest.ErrUnexpectedRequest) { + t.Errorf("a third identical request: got %v, want ErrUnexpectedRequest (each interaction answers once)", err) + } + if n := len(rec.Unexpected()); n != 2 { + t.Errorf("%d unexpected requests recorded, want 2", n) + } + if !errors.Is(rec.Err(), cloudkittest.ErrUnexpectedRequest) { + t.Errorf("Err: %v", rec.Err()) + } + if u := rec.Unused(); len(u) != 1 || !strings.HasSuffix(u[0].URL, "/v1/token") { + t.Errorf("unused %v, want the token interaction", u) + } + if n := len(rec.Requests()); n != 4 { + t.Errorf("%d requests seen, want 4", n) + } + if _, err := cloudkittest.NewRecorder(recs, "absent"); err == nil { + t.Error("an absent cassette: accepted") + } +} + +// Matching ignores query order and headers (the Authorization cloudkit adds +// is never compared, so never needs storing), and compares JSON bodies by +// value. +func TestRecorderMatching(t *testing.T) { + spec := fakeSpec(t) + recs, err := cloudkittest.ParseRecordings([]byte(`{"provider":"fake","cassettes":{"read-write":[ + {"request":{"method":"GET","url":"https://api.fake.test/v1/resources?a=1&b=2"},"response":{"status":200,"body":[]}}, + {"request":{"method":"POST","url":"https://api.fake.test/v1/resources","body":{"kind":"server","name":"web"}}, + "synthetic":{"status":201}}]}}`), spec) + if err != nil { + t.Fatal(err) + } + rec, _ := cloudkittest.NewRecorder(recs, cloudkittest.CassetteReadWrite) + req, _ := http.NewRequestWithContext(context.Background(), http.MethodGet, "https://API.fake.test/v1/resources?b=2&a=1", nil) + req.Header.Set("Authorization", "Bearer "+rand.Text()) + resp, err := rec.RoundTrip(req) + if err != nil { + t.Fatalf("query in another order: %v", err) + } + _ = resp.Body.Close() + post := func(body string) error { + req, _ := http.NewRequestWithContext(context.Background(), http.MethodPost, "https://api.fake.test/v1/resources", strings.NewReader(body)) + resp, err := rec.RoundTrip(req) + if resp != nil { + _ = resp.Body.Close() + } + return err + } + if err := post(`{"name":"other","kind":"server"}`); !errors.Is(err, cloudkittest.ErrUnexpectedRequest) { + t.Errorf("another body matched: %v", err) + } + if err := post(`{"name": "web", "kind": "server"}`); err != nil { + t.Errorf("the same body, re-spaced and reordered: %v", err) + } + for _, q := range rec.Requests() { + if strings.Contains(q.URL, "Bearer") { + t.Errorf("a seen request carries a credential: %v", q) + } + } +} diff --git a/src/cloudkittest/schema.go b/src/cloudkittest/schema.go new file mode 100644 index 0000000..ff66242 --- /dev/null +++ b/src/cloudkittest/schema.go @@ -0,0 +1,390 @@ +package cloudkittest + +import ( + "encoding/json" + "errors" + "fmt" + "io" + "math/big" + "slices" + "strings" +) + +// ErrSchemaViolation means a response body does not match the schema the +// provider's OpenAPI document gives for it. +var ErrSchemaViolation = errors.New("cloudkittest: response does not match the provider's schema") + +// ErrUnsupportedKeyword means a schema uses a keyword this validator does not +// implement. It is refused, never ignored, so a schema is either checked in +// full or not loaded at all. +var ErrUnsupportedKeyword = errors.New("cloudkittest: unsupported schema keyword") + +// The keywords the validator knows; the package documentation says what each +// one checks. Anything else is refused with ErrUnsupportedKeyword when the +// document loads. +var ( + assertionKeywords = []string{"type", "nullable", "required", "properties", "additionalProperties", "items", "enum", "$ref"} + annotationKeywords = []string{"description", "title", "example", "default", "format", "deprecated", "readOnly", "writeOnly"} + schemaTypes = []string{"object", "array", "string", "number", "integer", "boolean"} +) + +// schema is one compiled schema. A $ref schema has target set and nothing +// else. +type schema struct { + ref string + target *schema + types []string + nullable bool + required []string + properties map[string]*schema + // additional is the schema for properties not in properties; nil with + // noAdditional false means any. + additional *schema + noAdditional bool + items *schema + enum []any + hasEnum bool +} + +// compiler turns decoded schema objects into compiled ones, resolving $ref +// against the whole document and compiling each target once, so a schema +// that refers to itself terminates. +type compiler struct { + doc map[string]any + refs map[string]*schema +} + +func (c *compiler) compile(v any, at string) (*schema, error) { + m, ok := v.(map[string]any) + if !ok { + return nil, fmt.Errorf("cloudkittest: %s: a schema must be an object", at) + } + for k := range m { + if !slices.Contains(assertionKeywords, k) && !slices.Contains(annotationKeywords, k) { + return nil, fmt.Errorf("%w %q at %s; extend the validator rather than ignore it", ErrUnsupportedKeyword, k, at) + } + } + if ref, ok := m["$ref"]; ok { + for k := range m { + if k != "$ref" && k != "description" { + return nil, fmt.Errorf("%w %q beside $ref at %s", ErrUnsupportedKeyword, k, at) + } + } + s, ok := ref.(string) + if !ok { + return nil, fmt.Errorf("cloudkittest: %s: $ref must be a string", at) + } + return c.resolve(s, at) + } + s := &schema{} + if t, ok := m["type"]; ok { + switch tv := t.(type) { + case string: + s.types = []string{tv} + case []any: + for _, e := range tv { + name, ok := e.(string) + if !ok { + return nil, fmt.Errorf("cloudkittest: %s: type must be a string or strings", at) + } + s.types = append(s.types, name) + } + default: + return nil, fmt.Errorf("cloudkittest: %s: type must be a string or strings", at) + } + for _, name := range s.types { + if name == "null" { // OpenAPI 3.1 spelling of nullable + s.nullable = true + continue + } + if !slices.Contains(schemaTypes, name) { + return nil, fmt.Errorf("cloudkittest: %s: unknown type %q", at, name) + } + } + s.types = slices.DeleteFunc(s.types, func(n string) bool { return n == "null" }) + } + if n, ok := m["nullable"]; ok { + b, ok := n.(bool) + if !ok { + return nil, fmt.Errorf("cloudkittest: %s: nullable must be a boolean", at) + } + s.nullable = s.nullable || b + } + if r, ok := m["required"]; ok { + list, ok := r.([]any) + if !ok { + return nil, fmt.Errorf("cloudkittest: %s: required must be an array of names", at) + } + for _, e := range list { + name, ok := e.(string) + if !ok { + return nil, fmt.Errorf("cloudkittest: %s: required must be an array of names", at) + } + s.required = append(s.required, name) + } + } + if p, ok := m["properties"]; ok { + pm, ok := p.(map[string]any) + if !ok { + return nil, fmt.Errorf("cloudkittest: %s: properties must be an object", at) + } + s.properties = make(map[string]*schema, len(pm)) + for name, ps := range pm { + cs, err := c.compile(ps, at+"/properties/"+name) + if err != nil { + return nil, err + } + s.properties[name] = cs + } + } + if a, ok := m["additionalProperties"]; ok { + switch av := a.(type) { + case bool: + s.noAdditional = !av + default: + cs, err := c.compile(av, at+"/additionalProperties") + if err != nil { + return nil, err + } + s.additional = cs + } + } + if it, ok := m["items"]; ok { + cs, err := c.compile(it, at+"/items") + if err != nil { + return nil, err + } + s.items = cs + } + if e, ok := m["enum"]; ok { + list, ok := e.([]any) + if !ok || len(list) == 0 { + return nil, fmt.Errorf("cloudkittest: %s: enum must be a non-empty array", at) + } + s.enum, s.hasEnum = list, true + } + return s, nil +} + +// resolve compiles the schema a local $ref points at, once. +func (c *compiler) resolve(ref, at string) (*schema, error) { + if s, ok := c.refs[ref]; ok { + return s, nil + } + if !strings.HasPrefix(ref, "#/") { + return nil, fmt.Errorf("%w: $ref %q at %s: only references within the document (#/...) are supported", ErrUnsupportedKeyword, ref, at) + } + v, err := pointer(c.doc, ref) + if err != nil { + return nil, fmt.Errorf("cloudkittest: %s: %w", at, err) + } + s := &schema{ref: ref} + c.refs[ref] = s // before compiling, so a cycle finds it + t, err := c.compile(v, ref) + if err != nil { + return nil, err + } + s.target = t + return s, nil +} + +// pointer follows a "#/a/b" JSON pointer (RFC 6901) through a decoded document. +func pointer(doc map[string]any, ref string) (any, error) { + var cur any = doc + for _, tok := range strings.Split(strings.TrimPrefix(ref, "#/"), "/") { + tok = strings.ReplaceAll(strings.ReplaceAll(tok, "~1", "/"), "~0", "~") + m, ok := cur.(map[string]any) + if !ok { + return nil, fmt.Errorf("$ref %q does not resolve", ref) + } + if cur, ok = m[tok]; !ok { + return nil, fmt.Errorf("$ref %q does not resolve", ref) + } + } + return cur, nil +} + +// validate checks v, decoded with json.Decoder.UseNumber, against s. It +// returns the first mismatch, naming its JSON path. +func (s *schema) validate(v any, at string) error { + for s.target != nil { + s = s.target + } + if v == nil { + if s.nullable { + return nil + } + return violation(at, "is null, and the schema is not nullable") + } + if len(s.types) > 0 && !slices.ContainsFunc(s.types, func(t string) bool { return hasType(v, t) }) { + return violation(at, fmt.Sprintf("is %s, want %s", typeOf(v), strings.Join(s.types, " or "))) + } + if s.hasEnum && !slices.ContainsFunc(s.enum, func(e any) bool { return equalJSON(e, v) }) { + return violation(at, fmt.Sprintf("is %s, not one of the schema's enum values", short(v))) + } + switch tv := v.(type) { + case map[string]any: + for _, name := range s.required { + if _, ok := tv[name]; !ok { + return violation(at, fmt.Sprintf("lacks the required property %q", name)) + } + } + for _, name := range sortedKeys(tv) { + ps, listed := s.properties[name] + switch { + case listed: + case s.noAdditional: + return violation(at, fmt.Sprintf("has the property %q, which the schema does not allow", name)) + case s.additional != nil: + ps = s.additional + default: + continue + } + if err := ps.validate(tv[name], at+"."+name); err != nil { + return err + } + } + case []any: + if s.items != nil { + for i, e := range tv { + if err := s.items.validate(e, fmt.Sprintf("%s[%d]", at, i)); err != nil { + return err + } + } + } + } + return nil +} + +// SchemaError says where a response broke its schema. It matches +// [ErrSchemaViolation]. +type SchemaError struct { + Path string // a JSON path into the body, "$" for the root + Reason string +} + +func (e *SchemaError) Error() string { + return fmt.Sprintf("cloudkittest: schema violation at %s: %s", e.Path, e.Reason) +} + +// Unwrap lets errors.Is match [ErrSchemaViolation]. +func (e *SchemaError) Unwrap() error { return ErrSchemaViolation } + +func violation(at, reason string) error { return &SchemaError{Path: at, Reason: reason} } + +func hasType(v any, t string) bool { + switch t { + case "object": + _, ok := v.(map[string]any) + return ok + case "array": + _, ok := v.([]any) + return ok + case "string": + _, ok := v.(string) + return ok + case "boolean": + _, ok := v.(bool) + return ok + case "number": + _, ok := v.(json.Number) + return ok + case "integer": + n, ok := v.(json.Number) + if !ok { + return false + } + f, _, err := big.ParseFloat(n.String(), 10, 256, big.ToNearestEven) + return err == nil && f.IsInt() + } + return false +} + +func typeOf(v any) string { + switch v.(type) { + case map[string]any: + return "an object" + case []any: + return "an array" + case string: + return "a string" + case bool: + return "a boolean" + case json.Number: + return "a number" + } + return fmt.Sprintf("%T", v) +} + +// equalJSON compares two decoded JSON values; numbers compare by value. +func equalJSON(a, b any) bool { + switch av := a.(type) { + case json.Number: + bv, ok := b.(json.Number) + if !ok { + return false + } + x, _, err1 := big.ParseFloat(av.String(), 10, 256, big.ToNearestEven) + y, _, err2 := big.ParseFloat(bv.String(), 10, 256, big.ToNearestEven) + return err1 == nil && err2 == nil && x.Cmp(y) == 0 + case map[string]any: + bv, ok := b.(map[string]any) + if !ok || len(av) != len(bv) { + return false + } + for k, e := range av { + if f, ok := bv[k]; !ok || !equalJSON(e, f) { + return false + } + } + return true + case []any: + bv, ok := b.([]any) + if !ok || len(av) != len(bv) { + return false + } + for i := range av { + if !equalJSON(av[i], bv[i]) { + return false + } + } + return true + } + return a == b +} + +func sortedKeys(m map[string]any) []string { + keys := make([]string, 0, len(m)) + for k := range m { + keys = append(keys, k) + } + slices.Sort(keys) + return keys +} + +// short renders a value for a message, bounded. +func short(v any) string { + b, err := json.Marshal(v) + if err != nil { + return typeOf(v) + } + if len(b) > 64 { + return string(b[:64]) + "…" + } + return string(b) +} + +// decodeJSON decodes b keeping numbers exact, and refuses trailing data. +func decodeJSON(b []byte) (any, error) { + dec := json.NewDecoder(strings.NewReader(string(b))) + dec.UseNumber() + var v any + if err := dec.Decode(&v); err != nil { + return nil, err + } + var rest any + if err := dec.Decode(&rest); !errors.Is(err, io.EOF) { + return nil, errors.New("trailing data after the JSON value") + } + return v, nil +} diff --git a/src/cloudkittest/spec.go b/src/cloudkittest/spec.go new file mode 100644 index 0000000..310832e --- /dev/null +++ b/src/cloudkittest/spec.go @@ -0,0 +1,411 @@ +package cloudkittest + +import ( + "encoding/json" + "errors" + "fmt" + "io/fs" + "mime" + "net/http" + "net/url" + "slices" + "strconv" + "strings" +) + +// ErrNotInSpec means a request or response the provider's OpenAPI document +// does not describe: an unknown server, path, method or status code. +var ErrNotInSpec = errors.New("cloudkittest: not described by the provider's OpenAPI document") + +// Spec is a provider's OpenAPI 3 document in JSON, reduced to what the +// contract kit checks: its servers, its paths and methods, and each +// response's JSON schema and examples. Build one with [ParseSpec]. +// +// Every schema reachable from a response is compiled when the document loads, +// in the JSON Schema subset the package documentation lists: type, nullable, +// required, properties, additionalProperties, items, enum and local $ref, +// plus annotations that are not checked. A keyword outside that subset fails +// ParseSpec with [ErrUnsupportedKeyword], so no part of a response is +// silently left unchecked. Request bodies and parameters are not read. Only +// JSON media types are supported, and a response object may not be a $ref. +type Spec struct { + servers []*url.URL + paths []specPath +} + +type specPath struct { + template string + segments []string + ops map[string]*specOperation // by upper-case method +} + +type specOperation struct { + responses map[string]*specResponse // "200", "2XX", "default" +} + +type specResponse struct { + content map[string]*specMedia // empty: the response has no body +} + +type specMedia struct { + schema *schema + example any + hasExample bool + examples map[string]any +} + +var methods = []string{"get", "put", "post", "delete", "patch", "head", "options", "trace"} + +// ParseSpec reads an OpenAPI 3 document in JSON. +func ParseSpec(b []byte) (*Spec, error) { + v, err := decodeJSON(b) + if err != nil { + return nil, fmt.Errorf("cloudkittest: spec is not JSON: %w", err) + } + doc, ok := v.(map[string]any) + if !ok { + return nil, errors.New("cloudkittest: spec is not a JSON object") + } + if ver, _ := doc["openapi"].(string); !strings.HasPrefix(ver, "3.") { + return nil, fmt.Errorf("cloudkittest: spec: openapi %q is not 3.x", ver) + } + s := &Spec{} + servers, _ := doc["servers"].([]any) + for i, e := range servers { + m, _ := e.(map[string]any) + raw, _ := m["url"].(string) + u, err := url.Parse(raw) + if err != nil || u.Scheme != "https" || u.Host == "" { + return nil, fmt.Errorf("cloudkittest: spec: servers[%d] %q must be an absolute https URL", i, raw) + } + u.Path = strings.TrimSuffix(u.Path, "/") + s.servers = append(s.servers, u) + } + if len(s.servers) == 0 { + return nil, errors.New("cloudkittest: spec names no servers") + } + paths, ok := doc["paths"].(map[string]any) + if !ok || len(paths) == 0 { + return nil, errors.New("cloudkittest: spec has no paths") + } + c := &compiler{doc: doc, refs: map[string]*schema{}} + for _, tmpl := range sortedKeys(paths) { + if !strings.HasPrefix(tmpl, "/") { + return nil, fmt.Errorf("cloudkittest: spec: path %q must start with /", tmpl) + } + item, ok := paths[tmpl].(map[string]any) + if !ok { + return nil, fmt.Errorf("cloudkittest: spec: path %q is not an object", tmpl) + } + p := specPath{template: tmpl, segments: split(tmpl), ops: map[string]*specOperation{}} + for _, m := range methods { + raw, ok := item[m] + if !ok { + continue + } + op, err := c.operation(raw, "paths."+tmpl+"."+m) + if err != nil { + return nil, err + } + p.ops[strings.ToUpper(m)] = op + } + s.paths = append(s.paths, p) + } + return s, nil +} + +// LoadSpec reads and parses an OpenAPI document from fsys. +func LoadSpec(fsys fs.FS, name string) (*Spec, error) { + b, err := fs.ReadFile(fsys, name) + if err != nil { + return nil, err + } + s, err := ParseSpec(b) + if err != nil { + return nil, fmt.Errorf("%s: %w", name, err) + } + return s, nil +} + +func (c *compiler) operation(raw any, at string) (*specOperation, error) { + m, ok := raw.(map[string]any) + if !ok { + return nil, fmt.Errorf("cloudkittest: spec: %s is not an object", at) + } + rs, ok := m["responses"].(map[string]any) + if !ok || len(rs) == 0 { + return nil, fmt.Errorf("cloudkittest: spec: %s has no responses", at) + } + op := &specOperation{responses: map[string]*specResponse{}} + for _, code := range sortedKeys(rs) { + if !validStatusKey(code) { + return nil, fmt.Errorf("cloudkittest: spec: %s: response key %q is not a status code, a range like 2XX, or default", at, code) + } + r, err := c.response(rs[code], at+".responses."+code) + if err != nil { + return nil, err + } + op.responses[strings.ToUpper(code)] = r + } + return op, nil +} + +func (c *compiler) response(raw any, at string) (*specResponse, error) { + m, ok := raw.(map[string]any) + if !ok { + return nil, fmt.Errorf("cloudkittest: spec: %s is not an object", at) + } + if _, ok := m["$ref"]; ok { + return nil, fmt.Errorf("%w: $ref on a response object at %s; inline the response", ErrUnsupportedKeyword, at) + } + r := &specResponse{content: map[string]*specMedia{}} + content, _ := m["content"].(map[string]any) + for _, mt := range sortedKeys(content) { + if !isJSONMedia(mt) { + return nil, fmt.Errorf("cloudkittest: spec: %s: media type %q is not JSON; only JSON bodies are checked", at, mt) + } + mm, ok := content[mt].(map[string]any) + if !ok { + return nil, fmt.Errorf("cloudkittest: spec: %s.content.%s is not an object", at, mt) + } + raw, ok := mm["schema"] + if !ok { + return nil, fmt.Errorf("cloudkittest: spec: %s.content.%s has no schema, so its body cannot be checked", at, mt) + } + sc, err := c.compile(raw, at+".content."+mt+".schema") + if err != nil { + return nil, err + } + media := &specMedia{schema: sc} + media.example, media.hasExample = mm["example"] + if ex, ok := mm["examples"].(map[string]any); ok { + media.examples = map[string]any{} + for name, e := range ex { + em, _ := e.(map[string]any) + val, ok := em["value"] + if !ok { + return nil, fmt.Errorf("%w: example %q at %s has no inline value ($ref and externalValue are not supported)", ErrUnsupportedKeyword, name, at) + } + media.examples[name] = val + } + } + r.content[mediaKey(mt)] = media + } + return r, nil +} + +func validStatusKey(k string) bool { + if k == "default" { + return true + } + if len(k) != 3 || k[0] < '1' || k[0] > '5' { + return false + } + if strings.EqualFold(k[1:], "XX") { + return true + } + _, err := strconv.Atoi(k) + return err == nil +} + +func isJSONMedia(mt string) bool { + base, _, err := mime.ParseMediaType(mt) + return err == nil && (base == "application/json" || strings.HasSuffix(base, "+json")) +} + +func mediaKey(mt string) string { + base, _, err := mime.ParseMediaType(mt) + if err != nil { + return mt + } + return base +} + +func split(p string) []string { + p = strings.Trim(p, "/") + if p == "" { + return nil + } + return strings.Split(p, "/") +} + +// operation finds the operation a request goes to: a server whose scheme, +// host and base path prefix the URL, then the path template that matches the +// rest with the most literal segments. +func (s *Spec) operation(method, rawURL string) (*specOperation, string, error) { + u, err := url.Parse(rawURL) + if err != nil { + return nil, "", fmt.Errorf("cloudkittest: %q is not a URL: %w", rawURL, err) + } + var rest string + found := false + for _, srv := range s.servers { + if u.Scheme != srv.Scheme || !strings.EqualFold(u.Host, srv.Host) { + continue + } + p := strings.TrimSuffix(u.Path, "/") + if p == srv.Path || strings.HasPrefix(p, srv.Path+"/") { + rest, found = strings.TrimPrefix(p, srv.Path), true + break + } + } + if !found { + return nil, "", fmt.Errorf("%w: %s %s matches none of the spec's servers", ErrNotInSpec, method, u.Redacted()) + } + segs := split(rest) + best, bestLiterals := -1, -1 + for i, p := range s.paths { + if lit, ok := matchSegments(p.segments, segs); ok && lit > bestLiterals { + best, bestLiterals = i, lit + } + } + if best < 0 { + return nil, "", fmt.Errorf("%w: no path matches %s %s", ErrNotInSpec, method, rest) + } + p := s.paths[best] + op, ok := p.ops[strings.ToUpper(method)] + if !ok { + return nil, "", fmt.Errorf("%w: %s has no %s operation", ErrNotInSpec, p.template, strings.ToUpper(method)) + } + return op, strings.ToUpper(method) + " " + p.template, nil +} + +func matchSegments(tmpl, segs []string) (literals int, ok bool) { + if len(tmpl) != len(segs) { + return 0, false + } + for i, t := range tmpl { + if strings.HasPrefix(t, "{") && strings.HasSuffix(t, "}") { + if segs[i] == "" { + return 0, false + } + continue + } + if t != segs[i] { + return 0, false + } + literals++ + } + return literals, true +} + +// response picks the response for a status: the exact code, then its range +// (2XX), then default. +func (op *specOperation) response(status int) (*specResponse, bool) { + code := strconv.Itoa(status) + for _, k := range []string{code, code[:1] + "XX", "DEFAULT"} { + if r, ok := op.responses[k]; ok { + return r, true + } + } + return nil, false +} + +// ValidateResponse checks a response against the document: the request's +// server, path and method must be described, the status must have a response, +// and the body must match that response's schema, or be empty when the +// response has no content. A mismatch matches [ErrSchemaViolation]; an +// undescribed request or status matches [ErrNotInSpec]. +func (s *Spec) ValidateResponse(method, rawURL string, status int, header http.Header, body []byte) error { + op, name, err := s.operation(method, rawURL) + if err != nil { + return err + } + r, ok := op.response(status) + if !ok { + return fmt.Errorf("%w: %s has no response for status %d", ErrNotInSpec, name, status) + } + if len(r.content) == 0 { + if len(body) > 0 { + return fmt.Errorf("%s %d: %w", name, status, violation("$", "the spec gives this response no body, and it has one")) + } + return nil + } + if len(body) == 0 { + return fmt.Errorf("%s %d: %w", name, status, violation("$", "the spec gives this response a body, and it has none")) + } + ct := header.Get("Content-Type") + media, ok := r.content[mediaKey(ct)] + if !ok { + return fmt.Errorf("%w: %s %d has no content for Content-Type %q", ErrNotInSpec, name, status, ct) + } + v, err := decodeJSON(body) + if err != nil { + return fmt.Errorf("%s %d: %w", name, status, violation("$", "is not JSON: "+err.Error())) + } + if err := media.schema.validate(v, "$"); err != nil { + return fmt.Errorf("%s %d: %w", name, status, err) + } + return nil +} + +// Response is a provider response the contract kit replays. Synthetic is true +// when it was generated from the document's examples rather than recorded +// from the provider. +type Response struct { + Status int + Header http.Header + Body []byte + Synthetic bool +} + +// Synthesize builds a response from the document's examples, for a write path +// with no recording yet. example names an entry of the media +// type's examples; empty means its single example (the example field, or the +// only entry of examples). A response with no content needs no example. The +// result is marked Synthetic and has passed [Spec.ValidateResponse], so an +// example that breaks its own schema is refused. +func (s *Spec) Synthesize(method, rawURL string, status int, example string) (Response, error) { + op, name, err := s.operation(method, rawURL) + if err != nil { + return Response{}, err + } + r, ok := op.response(status) + if !ok { + return Response{}, fmt.Errorf("%w: %s has no response for status %d", ErrNotInSpec, name, status) + } + resp := Response{Status: status, Header: http.Header{}, Synthetic: true} + if len(r.content) == 0 { + if example != "" { + return Response{}, fmt.Errorf("cloudkittest: %s %d has no content, so no example %q", name, status, example) + } + return resp, nil + } + mt := "application/json" + if _, ok := r.content[mt]; !ok { + keys := make([]string, 0, len(r.content)) + for k := range r.content { + keys = append(keys, k) + } + slices.Sort(keys) + mt = keys[0] + } + media := r.content[mt] + var val any + switch { + case example != "": + v, ok := media.examples[example] + if !ok { + return Response{}, fmt.Errorf("cloudkittest: %s %d has no example %q", name, status, example) + } + val = v + case media.hasExample: + val = media.example + case len(media.examples) == 1: + for _, v := range media.examples { + val = v + } + default: + return Response{}, fmt.Errorf("cloudkittest: %s %d: name one of its examples", name, status) + } + b, err := json.Marshal(val) + if err != nil { + return Response{}, err + } + resp.Header.Set("Content-Type", mt) + resp.Body = b + if err := s.ValidateResponse(method, rawURL, status, resp.Header, resp.Body); err != nil { + return Response{}, fmt.Errorf("cloudkittest: the spec's own example breaks its schema: %w", err) + } + return resp, nil +} diff --git a/src/cloudkittest/spec_test.go b/src/cloudkittest/spec_test.go new file mode 100644 index 0000000..f270745 --- /dev/null +++ b/src/cloudkittest/spec_test.go @@ -0,0 +1,201 @@ +package cloudkittest_test + +import ( + "errors" + "net/http" + "strings" + "testing" + + "github.com/simtabi/cloudkit/src/cloudkittest" +) + +// specWith is a one-path document whose 200 response has the given schema. +func specWith(schema string) string { + return `{"openapi":"3.0.3","info":{"title":"t","version":"1"}, + "servers":[{"url":"https://api.example.test/v2"}], + "paths":{"/things/{id}":{"get":{"responses":{"200":{"description":"d", + "content":{"application/json":{"schema":` + schema + `}}}}}}}, + "components":{"schemas":{ + "Node":{"type":"object","required":["name"],"properties":{"name":{"type":"string"},"children":{"type":"array","items":{"$ref":"#/components/schemas/Node"}}}}}}}` +} + +const thingURL = "https://api.example.test/v2/things/7" + +func validate(t *testing.T, schema, body string) error { + t.Helper() + s, err := cloudkittest.ParseSpec([]byte(specWith(schema))) + if err != nil { + t.Fatalf("spec: %v", err) + } + h := http.Header{"Content-Type": {"application/json"}} + return s.ValidateResponse(http.MethodGet, thingURL, 200, h, []byte(body)) +} + +// Each assertion of the subset, accepting a value and refusing one. +func TestSchemaSubset(t *testing.T) { + cases := []struct { + name, schema, good, bad string + }{ + {"string", `{"type":"string"}`, `"x"`, `1`}, + {"integer", `{"type":"integer"}`, `3`, `3.5`}, + {"integer written with an exponent", `{"type":"integer"}`, `1e3`, `true`}, + {"number", `{"type":"number"}`, `3.5`, `"3.5"`}, + {"boolean", `{"type":"boolean"}`, `false`, `"false"`}, + {"array of types", `{"type":["string","integer"]}`, `4`, `[]`}, + {"null without nullable", `{"type":"string"}`, `"x"`, `null`}, + {"nullable", `{"type":"string","nullable":true}`, `null`, `5`}, + {"3.1 null type", `{"type":["string","null"]}`, `null`, `5`}, + {"required", `{"type":"object","required":["id"]}`, `{"id":1}`, `{"name":"a"}`}, + {"properties", `{"type":"object","properties":{"id":{"type":"string"}}}`, `{"id":"a","extra":1}`, `{"id":1}`}, + {"no additional properties", `{"type":"object","properties":{"id":{}},"additionalProperties":false}`, `{"id":1}`, `{"id":1,"x":2}`}, + {"additional properties schema", `{"type":"object","additionalProperties":{"type":"string"}}`, `{"a":"b"}`, `{"a":1}`}, + {"items", `{"type":"array","items":{"type":"integer"}}`, `[1,2]`, `[1,"2"]`}, + {"enum", `{"type":"string","enum":["on","off"]}`, `"on"`, `"maybe"`}, + {"numeric enum by value", `{"enum":[1,2]}`, `1.0`, `3`}, + {"ref", `{"$ref":"#/components/schemas/Node"}`, `{"name":"a"}`, `{"children":[]}`}, + {"recursive ref", `{"$ref":"#/components/schemas/Node"}`, `{"name":"a","children":[{"name":"b"}]}`, `{"name":"a","children":[{"name":2}]}`}, + {"annotations are not checked", `{"type":"string","format":"uuid","description":"d","example":"x"}`, `"not-a-uuid"`, `7`}, + } + for _, c := range cases { + if err := validate(t, c.schema, c.good); err != nil { + t.Errorf("%s: %s refused: %v", c.name, c.good, err) + } + err := validate(t, c.schema, c.bad) + if !errors.Is(err, cloudkittest.ErrSchemaViolation) { + t.Errorf("%s: %s: got %v, want ErrSchemaViolation", c.name, c.bad, err) + } + var se *cloudkittest.SchemaError + if err != nil && (!errors.As(err, &se) || !strings.HasPrefix(se.Path, "$")) { + t.Errorf("%s: the violation does not name a JSON path: %v", c.name, err) + } + } +} + +// An unknown keyword is refused when the document loads, wherever it sits, +// never ignored. +func TestUnknownSchemaKeywordsAreRefused(t *testing.T) { + for _, schema := range []string{ + `{"type":"string","pattern":"^a"}`, + `{"type":"integer","minimum":1}`, + `{"oneOf":[{"type":"string"}]}`, + `{"type":"object","properties":{"deep":{"type":"array","items":{"type":"string","maxLength":3}}}}`, + `{"$ref":"#/components/schemas/Node","type":"object"}`, + `{"$ref":"https://elsewhere.test/schema.json"}`, + } { + _, err := cloudkittest.ParseSpec([]byte(specWith(schema))) + if !errors.Is(err, cloudkittest.ErrUnsupportedKeyword) { + t.Errorf("%s: got %v, want ErrUnsupportedKeyword", schema, err) + } + } + // A keyword in a schema no response reaches is not compiled, and so not refused. + doc := strings.Replace(specWith(`{"type":"string"}`), `"Node":`, `"Unused":{"pattern":"x"},"Node":`, 1) + if _, err := cloudkittest.ParseSpec([]byte(doc)); err != nil { + t.Errorf("an unreferenced schema was compiled: %v", err) + } +} + +func TestParseSpecRefusesBadDocuments(t *testing.T) { + for name, doc := range map[string]string{ + "not JSON": `{`, + "swagger 2": `{"swagger":"2.0","paths":{}}`, + "no servers": strings.Replace(specWith(`{}`), `"servers":[{"url":"https://api.example.test/v2"}],`, ``, 1), + "http server": strings.Replace(specWith(`{}`), `https://api.example.test`, `http://api.example.test`, 1), + "dangling ref": specWith(`{"$ref":"#/components/schemas/Missing"}`), + "non-JSON media": strings.Replace(specWith(`{}`), `application/json`, `text/plain`, 1), + "media, no schema": strings.Replace(specWith(`{}`), `"schema":{}`, `"example":1`, 1), + "bad status key": strings.Replace(specWith(`{}`), `"200":`, `"ok":`, 1), + "trailing data": specWith(`{}`) + `{}`, + "unknown type name": specWith(`{"type":"date"}`), + } { + if _, err := cloudkittest.ParseSpec([]byte(doc)); err == nil { + t.Errorf("%s: accepted", name) + } + } +} + +// A request the document does not describe is not a schema pass. +func TestValidateResponseNeedsTheRequestInTheSpec(t *testing.T) { + s, err := cloudkittest.ParseSpec([]byte(specWith(`{"type":"string"}`))) + if err != nil { + t.Fatal(err) + } + h := http.Header{"Content-Type": {"application/json"}} + for name, tc := range map[string]struct { + method, url string + status int + }{ + "other host": {http.MethodGet, "https://api.other.test/v2/things/7", 200}, + "outside base": {http.MethodGet, "https://api.example.test/v1/things/7", 200}, + "unknown path": {http.MethodGet, "https://api.example.test/v2/others/7", 200}, + "missing param": {http.MethodGet, "https://api.example.test/v2/things/", 200}, + "unknown method": {http.MethodPost, thingURL, 200}, + "unknown status": {http.MethodGet, thingURL, 500}, + } { + if err := s.ValidateResponse(tc.method, tc.url, tc.status, h, []byte(`"x"`)); !errors.Is(err, cloudkittest.ErrNotInSpec) { + t.Errorf("%s: got %v, want ErrNotInSpec", name, err) + } + } + if err := s.ValidateResponse(http.MethodGet, thingURL, 200, h, nil); !errors.Is(err, cloudkittest.ErrSchemaViolation) { + t.Errorf("an empty body where the spec gives one: got %v", err) + } + if err := s.ValidateResponse(http.MethodGet, thingURL, 200, http.Header{"Content-Type": {"text/html"}}, []byte(`"x"`)); !errors.Is(err, cloudkittest.ErrNotInSpec) { + t.Errorf("a content type the spec does not give: got %v", err) + } +} + +// A literal path segment wins over a template, and a range or default +// response answers a status with no exact entry. +func TestOperationAndResponseLookup(t *testing.T) { + doc := `{"openapi":"3.0.3","servers":[{"url":"https://api.example.test"}],"paths":{ + "/things/{id}":{"get":{"responses":{"200":{"description":"d","content":{"application/json":{"schema":{"type":"integer"}}}}}}}, + "/things/search":{"get":{"responses":{"2XX":{"description":"d","content":{"application/json":{"schema":{"type":"string"}}}}, + "default":{"description":"e","content":{"application/json":{"schema":{"type":"boolean"}}}}}}}}}` + s, err := cloudkittest.ParseSpec([]byte(doc)) + if err != nil { + t.Fatal(err) + } + h := http.Header{"Content-Type": {"application/json; charset=utf-8"}} + if err := s.ValidateResponse(http.MethodGet, "https://api.example.test/things/search", 202, h, []byte(`"found"`)); err != nil { + t.Errorf("2XX for the literal path: %v", err) + } + if err := s.ValidateResponse(http.MethodGet, "https://api.example.test/things/search", 503, h, []byte(`true`)); err != nil { + t.Errorf("default: %v", err) + } + if err := s.ValidateResponse(http.MethodGet, "https://api.example.test/things/9", 200, h, []byte(`9`)); err != nil { + t.Errorf("templated path: %v", err) + } +} + +// Synthetic responses come from the spec's examples, are marked, and are +// refused when the example breaks its own schema. +func TestSynthesize(t *testing.T) { + spec, _ := loadFixtures(t) + del := "https://api.fake.test/v1/resources/s1" + r, err := spec.Synthesize(http.MethodDelete, del, 200, "deleted") + if err != nil { + t.Fatal(err) + } + if !r.Synthetic || r.Status != 200 || !strings.Contains(string(r.Body), `"success"`) || r.Header.Get("Content-Type") != "application/json" { + t.Errorf("deleted: %+v %s", r, r.Body) + } + if _, err := spec.Synthesize(http.MethodDelete, del, 200, ""); err == nil { + t.Error("two named examples and none chosen: accepted") + } + if _, err := spec.Synthesize(http.MethodDelete, del, 200, "missing"); err == nil { + t.Error("an unknown example name: accepted") + } + if r, err := spec.Synthesize(http.MethodPost, "https://api.fake.test/v1/resources", 201, ""); err != nil || !r.Synthetic { + t.Errorf("the single example field: %v", err) + } + if _, err := spec.Synthesize(http.MethodGet, "https://api.fake.test/v1/elsewhere", 200, ""); !errors.Is(err, cloudkittest.ErrNotInSpec) { + t.Errorf("an undescribed path: %v", err) + } + bad := strings.Replace(specWith(`{"type":"integer"}`), `"schema":{"type":"integer"}`, `"schema":{"type":"integer"},"example":"seven"`, 1) + s, err := cloudkittest.ParseSpec([]byte(bad)) + if err != nil { + t.Fatal(err) + } + if _, err := s.Synthesize(http.MethodGet, thingURL, 200, ""); !errors.Is(err, cloudkittest.ErrSchemaViolation) { + t.Errorf("an example that breaks its schema: got %v, want ErrSchemaViolation", err) + } +} diff --git a/src/cloudkittest/suite.go b/src/cloudkittest/suite.go new file mode 100644 index 0000000..c25d6c2 --- /dev/null +++ b/src/cloudkittest/suite.go @@ -0,0 +1,333 @@ +package cloudkittest + +import ( + "context" + "crypto/rand" + "errors" + "fmt" + "strings" + "testing" + + "github.com/simtabi/cloudkit/src" +) + +// The nine contract cases, in the order they run. +const ( + CaseTransport = "transport-only" + CaseNilTransport = "nil-transport" + CaseEgress = "egress" + CaseReadOnly = "read-only-token" + CasePlanDrift = "plan-drift" + CaseDestroy = "destroy" + CaseRedaction = "redaction" + CaseCloudInit = "cloud-init" + CaseSSH = "ssh" +) + +// Cases returns the case names in the order [Check] runs them. +func Cases() []string { + return []string{ + CaseTransport, CaseNilTransport, CaseEgress, CaseReadOnly, CasePlanDrift, + CaseDestroy, CaseRedaction, CaseCloudInit, CaseSSH, + } +} + +// The cassettes a suite's recordings must hold. Each case replays a fresh +// [Recorder] over one of them and expects the adapter to use every +// interaction, exactly once. +const ( + // CassetteReadWrite holds what Probe and then Inventory send with a token + // that may write. + CassetteReadWrite = "read-write" + // CassetteReadOnly holds what Probe sends with a read-only token, then + // what each of the suite's Writes sends, in order, before it is refused. + CassetteReadOnly = "read-only" + // CassetteDrift holds what the Drift plan sends, then the live-state read + // Apply makes, which must return a different state. Needed when the + // suite has Writes. + CassetteDrift = "drift" + // CassetteDestroy holds what the Destroy plan sends, then the live-state + // read and the delete calls of applying it with the names typed back. + // Needed when the suite has Writes. + CassetteDestroy = "destroy" +) + +// PlanFunc plans one write on an opened provider. It is how the suite reaches +// an adapter's Plan* methods, whose shapes differ by capability. +type PlanFunc func(ctx context.Context, p cloudkit.Provider) (cloudkit.Plan, error) + +// Write is one write an adapter offers. +type Write struct { + Name string // e.g. "PlanCreate" + Capability cloudkit.Capability // the capability the probe reports for it; must be a write + Plan PlanFunc +} + +// Destroy describes the destroy the destroy case plans. +type Destroy struct { + Plan PlanFunc + // Loses names, by kind and ID, the resource the plan deletes and + // everything attached to it in the destroy cassette (volumes, IPs). + // The plan's Loses must list every one. At least two entries: the + // target and one attachment, or the case proves nothing. + Loses []cloudkit.Resource +} + +// OneTimeTokenField is the only user-data field that may hold a secret: a +// single-use, short-lived enrolment token. +const OneTimeTokenField = "OneTimeToken" + +// CloudInit is how the cloud-init case reaches the user-data renderer an +// adapter's creates use. +type CloudInit struct { + // Render renders user data with the named fields set, for a provider that + // accepts at most limit bytes. It must refuse a secret in any field but + // OneTimeTokenField, and output above limit. + Render func(ctx context.Context, fields map[string]string, limit int) ([]byte, error) + // Fields names every field Render accepts; it must include + // OneTimeTokenField. + Fields []string +} + +// The host-key policies the SSH case checks. +const ( + PolicyKnownHosts = "known-hosts" + PolicyPinned = "pinned" + PolicyFromProvider = "from-provider" + PolicyAcceptNew = "accept-new" +) + +// SSH is how the SSH case reaches the adapter's SSH client. +type SSH struct { + // Policies names every host-key policy the client offers. It must + // include PolicyKnownHosts, PolicyPinned and PolicyAcceptNew. + Policies []string + // Handshake connects, under the named policy, to a host presenting a key + // that policy has never seen, and returns the authentication methods the + // client offered (as SSH names them: "publickey", "password", + // "keyboard-interactive") and the handshake's error. + Handshake func(ctx context.Context, policy string) (offered []string, err error) +} + +// Suite is one adapter under contract. +type Suite struct { + // Factory builds the adapter, exactly as a consumer registers it. + Factory cloudkit.Factory + // Recordings are the adapter's spec-checked recordings; they must be for + // the factory's provider ID and hold the cassettes named above. + Recordings *Recordings + // Writes is every write the adapter plans. Any capability the read-write + // probe reports Allowed must have one. Empty for a read-only adapter, + // which skips the drift and destroy cases. + Writes []Write + // Drift plans the write the plan-drift case applies. Required with Writes. + Drift PlanFunc + // Destroy is the destroy the destroy case plans. Required with Writes. + Destroy *Destroy + // CloudInit is required when the descriptor's UserDataLimit is above 0. + CloudInit *CloudInit + // SSH is optional until the SSH path exists; without it the SSH case is + // skipped, and says so. + SSH *SSH +} + +// Status is a case's outcome. +type Status int + +// The three outcomes. Skipped always carries a reason. +const ( + Passed Status = iota + 1 + Failed + Skipped +) + +func (s Status) String() string { + switch s { + case Passed: + return "passed" + case Failed: + return "failed" + case Skipped: + return "skipped" + } + return fmt.Sprintf("Status(%d)", int(s)) +} + +// CaseResult is one case's outcome. +type CaseResult struct { + Name string + Status Status + Failures []string // why it failed; the run's token never appears in them + Reason string // why it was skipped +} + +// Report is the outcome of a [Check]. Err is set when the suite itself is +// unusable, and then no case ran. +type Report struct { + Err error + Cases []CaseResult +} + +// Failed reports whether the suite was unusable or any case failed. +func (r Report) Failed() bool { + if r.Err != nil { + return true + } + for _, c := range r.Cases { + if c.Status == Failed { + return true + } + } + return false +} + +// Case returns the result of the named case. +func (r Report) Case(name string) (CaseResult, bool) { + for _, c := range r.Cases { + if c.Name == name { + return c, true + } + } + return CaseResult{}, false +} + +// Run runs every contract case against the suite's adapter, one subtest per +// case, and fails t on any failure. A skipped case is reported with +// t.Skip and its reason. +func Run(t *testing.T, s Suite) { + t.Helper() + rep := Check(t.Context(), s) + if rep.Err != nil { + t.Fatalf("cloudkittest: %v", rep.Err) + } + for _, c := range rep.Cases { + t.Run(c.Name, func(t *testing.T) { + switch c.Status { + case Skipped: + t.Skip(c.Reason) + case Failed: + for _, f := range c.Failures { + t.Error(f) + } + } + }) + } +} + +// Check runs every contract case and returns the outcome without a +// *testing.T, so a loader (a descriptor that must pass the suite before it +// loads) or a test of the suite itself can read it. +// +// Each case builds its own [cloudkit.Registry] over a fresh transport, a +// token generated for this run, no retries, and a logger that keeps every +// record for the redaction case. Nothing reaches the network: requests are +// answered by a [Recorder] or by a transport that injects a fault, and +// dialing is refused. +func Check(ctx context.Context, s Suite) Report { + if err := s.validate(); err != nil { + return Report{Err: err} + } + r := &runner{ctx: ctx, s: s, token: rand.Text()} + cases := []struct { + name string + run func(*caseT) + }{ + {CaseTransport, r.transportOnly}, + {CaseNilTransport, r.nilTransport}, + {CaseEgress, r.egress}, + {CaseReadOnly, r.readOnly}, + {CasePlanDrift, r.planDrift}, + {CaseDestroy, r.destroy}, + {CaseRedaction, r.redaction}, + {CaseCloudInit, r.cloudInit}, + {CaseSSH, r.ssh}, + } + rep := Report{} + for _, c := range cases { + ct := &caseT{token: r.token} + func() { + defer func() { + if p := recover(); p != nil { + ct.failf("panicked: %v", p) + } + }() + c.run(ct) + }() + rep.Cases = append(rep.Cases, ct.result(c.name)) + } + return rep +} + +func (s Suite) validate() error { + if err := s.Factory.Descriptor.Validate(); err != nil { + return fmt.Errorf("suite: %w", err) + } + if s.Factory.New == nil { + return errors.New("suite: the factory has no New function") + } + if s.Recordings == nil { + return errors.New("suite: no recordings; build them with ParseRecordings") + } + if got, want := s.Recordings.Provider(), s.Factory.Descriptor.ID; got != want { + return fmt.Errorf("suite: the recordings are for %q, the factory is %q", got, want) + } + need := []string{CassetteReadWrite, CassetteReadOnly} + if len(s.Writes) > 0 { + need = append(need, CassetteDrift, CassetteDestroy) + if s.Drift == nil { + return errors.New("suite: Writes are set but Drift is nil") + } + if s.Destroy == nil || s.Destroy.Plan == nil { + return errors.New("suite: Writes are set but Destroy has no Plan") + } + if len(s.Destroy.Loses) < 2 { + return errors.New("suite: Destroy.Loses must name the target and at least one attached resource") + } + } + for _, c := range need { + if s.Recordings.Len(c) == 0 { + return fmt.Errorf("suite: the recordings have no %q cassette", c) + } + } + for _, w := range s.Writes { + if w.Name == "" || w.Plan == nil { + return errors.New("suite: every Write needs a Name and a Plan") + } + if !w.Capability.IsWrite() { + return fmt.Errorf("suite: Write %s names %q, which is not a write capability", w.Name, w.Capability) + } + } + if c := s.CloudInit; c != nil && c.Render == nil { + return errors.New("suite: CloudInit has no Render") + } + if c := s.SSH; c != nil && c.Handshake == nil { + return errors.New("suite: SSH has no Handshake") + } + return nil +} + +// caseT collects one case's failures, with the run's token masked out of +// every message, so a report never prints the token even when the adapter +// leaked it. +type caseT struct { + token string + failures []string + skip string +} + +func (c *caseT) failf(format string, args ...any) { + msg := fmt.Sprintf(format, args...) + c.failures = append(c.failures, strings.ReplaceAll(msg, c.token, "[TOKEN]")) +} + +func (c *caseT) skipf(format string, args ...any) { c.skip = fmt.Sprintf(format, args...) } + +func (c *caseT) result(name string) CaseResult { + switch { + case len(c.failures) > 0: + return CaseResult{Name: name, Status: Failed, Failures: c.failures} + case c.skip != "": + return CaseResult{Name: name, Status: Skipped, Reason: c.skip} + } + return CaseResult{Name: name, Status: Passed} +} diff --git a/src/cloudkittest/suite_test.go b/src/cloudkittest/suite_test.go new file mode 100644 index 0000000..56e4790 --- /dev/null +++ b/src/cloudkittest/suite_test.go @@ -0,0 +1,132 @@ +package cloudkittest_test + +import ( + "strings" + "testing" + + "github.com/simtabi/cloudkit/src/cloudkittest" +) + +// The healthy test provider passes all nine cases, through Run itself. +func TestTheTestProviderPassesTheSuite(t *testing.T) { + cloudkittest.Run(t, suite(t, defects{})) +} + +// Check reports all nine cases, passed, and none skipped: a skip would hide a +// case the healthy provider is meant to exercise. +func TestCheckReportsEveryCase(t *testing.T) { + rep := cloudkittest.Check(t.Context(), suite(t, defects{})) + if rep.Err != nil { + t.Fatal(rep.Err) + } + if len(rep.Cases) != 9 || len(cloudkittest.Cases()) != 9 { + t.Fatalf("got %d cases (Cases lists %d), want nine", len(rep.Cases), len(cloudkittest.Cases())) + } + for i, c := range rep.Cases { + if c.Name != cloudkittest.Cases()[i] { + t.Errorf("case %d is %s, want %s", i, c.Name, cloudkittest.Cases()[i]) + } + if c.Status != cloudkittest.Passed { + t.Errorf("%s: %s %v %s", c.Name, c.Status, c.Failures, c.Reason) + } + } +} + +// One deliberately broken provider per case: each must fail the case written +// to catch it. A case that cannot fail is not known to work. +func TestEachCaseCatchesItsDefect(t *testing.T) { + broken := []struct { + name string + d defects + want string // the failure must say this + }{ + {cloudkittest.CaseTransport, defects{bypassTransport: true}, "sent nothing through the injected transport"}, + {cloudkittest.CaseNilTransport, defects{acceptNilClient: true}, "Factory.New accepted a nil client"}, + {cloudkittest.CaseEgress, defects{flattenErrors: true}, "want an error matching ErrEgressDenied"}, + {cloudkittest.CaseReadOnly, defects{probeIgnoresToken: true}, "with a read-only token, want denied"}, + {cloudkittest.CasePlanDrift, defects{staleLiveState: true}, "want ErrConflict"}, + {cloudkittest.CaseDestroy, defects{dropAttached: true}, "does not list volume v1 in Loses"}, + {cloudkittest.CaseRedaction, defects{leakErrors: true}, "the token appears in"}, + {cloudkittest.CaseCloudInit, defects{renderIgnoresLimit: true}, "above the provider's limit"}, + {cloudkittest.CaseSSH, defects{offerPassword: true}, "offered password authentication"}, + } + if len(broken) != len(cloudkittest.Cases()) { + t.Fatalf("%d broken variants for %d cases", len(broken), len(cloudkittest.Cases())) + } + for _, b := range broken { + t.Run(b.name, func(t *testing.T) { + rep := cloudkittest.Check(t.Context(), suite(t, b.d)) + if rep.Err != nil { + t.Fatal(rep.Err) + } + c, ok := rep.Case(b.name) + if !ok { + t.Fatalf("no result for %s", b.name) + } + if c.Status != cloudkittest.Failed { + t.Fatalf("%s %s with its defect planted; want failed", b.name, c.Status) + } + if !strings.Contains(strings.Join(c.Failures, "\n"), b.want) { + t.Errorf("%s failed, but not for its defect:\n%s", b.name, strings.Join(c.Failures, "\n")) + } + if !rep.Failed() { + t.Error("Report.Failed is false with a failed case") + } + }) + } +} + +// The redaction case's own failure messages never print the token, even when +// the adapter leaked it. +func TestFailuresNeverPrintTheToken(t *testing.T) { + rep := cloudkittest.Check(t.Context(), suite(t, defects{leakErrors: true})) + c, _ := rep.Case(cloudkittest.CaseRedaction) + for _, f := range c.Failures { + if strings.Contains(f, "Bearer ") && !strings.Contains(f, "[TOKEN]") && !strings.Contains(f, "[REDACTED]") { + t.Errorf("a failure may carry the token: %q", f) + } + } + if len(c.Failures) == 0 { + t.Fatal("the leaking provider passed the redaction case") + } +} + +// An unusable suite is reported as such, and no case runs. +func TestCheckRefusesAnUnusableSuite(t *testing.T) { + mutations := map[string]func(*cloudkittest.Suite){ + "no recordings": func(s *cloudkittest.Suite) { s.Recordings = nil }, + "no New": func(s *cloudkittest.Suite) { s.Factory.New = nil }, + "wrong provider": func(s *cloudkittest.Suite) { s.Factory.Descriptor.ID = "other" }, + "writes but no drift": func(s *cloudkittest.Suite) { s.Drift = nil }, + "writes, no destroy": func(s *cloudkittest.Suite) { s.Destroy = nil }, + "vacuous loses": func(s *cloudkittest.Suite) { s.Destroy.Loses = s.Destroy.Loses[:1] }, + "read as write": func(s *cloudkittest.Suite) { s.Writes[0].Capability = "inventory.read" }, + "cloud-init, no fn": func(s *cloudkittest.Suite) { s.CloudInit.Render = nil }, + "ssh, no fn": func(s *cloudkittest.Suite) { s.SSH.Handshake = nil }, + } + for name, mutate := range mutations { + s := suite(t, defects{}) + mutate(&s) + rep := cloudkittest.Check(t.Context(), s) + if rep.Err == nil || len(rep.Cases) != 0 || !rep.Failed() { + t.Errorf("%s: err %v, %d cases ran", name, rep.Err, len(rep.Cases)) + } + } +} + +// Without an SSH client the SSH case is skipped and says why; without a +// renderer for a provider that takes user data, the cloud-init case fails. +func TestOptionalCases(t *testing.T) { + s := suite(t, defects{}) + s.SSH = nil + rep := cloudkittest.Check(t.Context(), s) + if c, _ := rep.Case(cloudkittest.CaseSSH); c.Status != cloudkittest.Skipped || c.Reason == "" { + t.Errorf("no SSH hook: %s %q, want skipped with a reason", c.Status, c.Reason) + } + s = suite(t, defects{}) + s.CloudInit = nil + rep = cloudkittest.Check(t.Context(), s) + if c, _ := rep.Case(cloudkittest.CaseCloudInit); c.Status != cloudkittest.Failed { + t.Errorf("no renderer, UserDataLimit > 0: %s, want failed", c.Status) + } +} diff --git a/src/cloudkittest/testdata/broken-schema.recordings.json b/src/cloudkittest/testdata/broken-schema.recordings.json new file mode 100644 index 0000000..fe6d5e3 --- /dev/null +++ b/src/cloudkittest/testdata/broken-schema.recordings.json @@ -0,0 +1,24 @@ +{ + "provider": "fake", + "cassettes": { + "read-write": [ + { + "request": { + "method": "GET", + "url": "https://api.fake.test/v1/token" + }, + "response": { + "status": 200, + "headers": { + "Content-Type": [ + "application/json" + ] + }, + "body": { + "write": "yes" + } + } + } + ] + } +} diff --git a/src/cloudkittest/testdata/fake.openapi.json b/src/cloudkittest/testdata/fake.openapi.json new file mode 100644 index 0000000..e39c926 --- /dev/null +++ b/src/cloudkittest/testdata/fake.openapi.json @@ -0,0 +1,123 @@ +{ + "openapi": "3.0.3", + "info": { + "title": "Fake Cloud API", + "version": "1.0.0", + "description": "A hand-written document for cloudkittest's own tests. It describes no real provider." + }, + "servers": [{"url": "https://api.fake.test/v1"}], + "paths": { + "/token": { + "get": { + "operationId": "getToken", + "summary": "What the token may do", + "responses": { + "200": { + "description": "The token's scope", + "content": {"application/json": {"schema": {"$ref": "#/components/schemas/Token"}}} + }, + "default": { + "description": "An error", + "content": {"application/json": {"schema": {"$ref": "#/components/schemas/Error"}}} + } + } + } + }, + "/resources": { + "get": { + "operationId": "listResources", + "responses": { + "200": { + "description": "Every resource in the account", + "content": {"application/json": {"schema": {"type": "array", "items": {"$ref": "#/components/schemas/Resource"}}}} + }, + "403": { + "description": "The token may not list", + "content": {"application/json": {"schema": {"$ref": "#/components/schemas/Error"}}} + } + } + }, + "post": { + "operationId": "createResource", + "responses": { + "201": { + "description": "The created resource", + "content": { + "application/json": { + "schema": {"$ref": "#/components/schemas/Resource"}, + "example": {"kind": "server", "id": "s9", "name": "web-2", "region": "fsn1", "attrs": {"size": "small"}} + } + } + }, + "403": { + "description": "The token may not write", + "content": {"application/json": {"schema": {"$ref": "#/components/schemas/Error"}}} + } + } + } + }, + "/resources/{id}": { + "delete": { + "operationId": "deleteResource", + "responses": { + "200": { + "description": "The action that deletes the resource and what is attached to it", + "content": { + "application/json": { + "schema": {"$ref": "#/components/schemas/ActionEnvelope"}, + "examples": { + "deleted": {"value": {"action": {"id": "a1", "command": "delete_resource", "status": "success"}}}, + "running": {"value": {"action": {"id": "a2", "command": "delete_resource", "status": "running"}}} + } + } + } + }, + "404": { + "description": "No such resource", + "content": {"application/json": {"schema": {"$ref": "#/components/schemas/Error"}}} + } + } + } + } + }, + "components": { + "schemas": { + "Token": { + "type": "object", + "required": ["write"], + "properties": {"write": {"type": "boolean", "description": "true when the token may write"}} + }, + "Resource": { + "type": "object", + "required": ["kind", "id"], + "properties": { + "kind": {"type": "string", "enum": ["server", "volume", "ipv4"]}, + "id": {"type": "string"}, + "name": {"type": "string"}, + "region": {"type": "string", "nullable": true}, + "labels": {"type": "object", "additionalProperties": {"type": "string"}}, + "attrs": {"type": "object", "additionalProperties": {"type": "string"}} + } + }, + "Action": { + "type": "object", + "required": ["id", "status"], + "properties": { + "id": {"type": "string"}, + "command": {"type": "string"}, + "status": {"type": "string", "enum": ["running", "success", "error"]} + } + }, + "ActionEnvelope": { + "type": "object", + "required": ["action"], + "properties": {"action": {"$ref": "#/components/schemas/Action"}} + }, + "Error": { + "type": "object", + "required": ["message"], + "properties": {"message": {"type": "string"}} + } + } + } +} diff --git a/src/cloudkittest/testdata/fake.recordings.json b/src/cloudkittest/testdata/fake.recordings.json new file mode 100644 index 0000000..a8f8c88 --- /dev/null +++ b/src/cloudkittest/testdata/fake.recordings.json @@ -0,0 +1,322 @@ +{ + "provider": "fake", + "cassettes": { + "read-write": [ + { + "request": { + "method": "GET", + "url": "https://api.fake.test/v1/token" + }, + "response": { + "status": 200, + "headers": { + "Content-Type": [ + "application/json" + ] + }, + "body": { + "write": true + } + } + }, + { + "request": { + "method": "GET", + "url": "https://api.fake.test/v1/resources" + }, + "response": { + "status": 200, + "headers": { + "Content-Type": [ + "application/json" + ] + }, + "body": [ + { + "kind": "server", + "id": "s1", + "name": "web-1", + "region": "fsn1", + "labels": { + "env": "production" + }, + "attrs": { + "size": "small" + } + }, + { + "kind": "volume", + "id": "v1", + "name": "web-1-data", + "region": "fsn1", + "attrs": { + "attached_to": "s1", + "size_gb": "10" + } + }, + { + "kind": "ipv4", + "id": "ip1", + "name": "web-1-ip", + "region": "fsn1", + "attrs": { + "attached_to": "s1" + } + } + ] + } + } + ], + "read-only": [ + { + "request": { + "method": "GET", + "url": "https://api.fake.test/v1/token" + }, + "response": { + "status": 200, + "headers": { + "Content-Type": [ + "application/json" + ] + }, + "body": { + "write": false + } + } + }, + { + "request": { + "method": "GET", + "url": "https://api.fake.test/v1/token" + }, + "response": { + "status": 200, + "headers": { + "Content-Type": [ + "application/json" + ] + }, + "body": { + "write": false + } + } + }, + { + "request": { + "method": "GET", + "url": "https://api.fake.test/v1/token" + }, + "response": { + "status": 200, + "headers": { + "Content-Type": [ + "application/json" + ] + }, + "body": { + "write": false + } + } + } + ], + "drift": [ + { + "request": { + "method": "GET", + "url": "https://api.fake.test/v1/token" + }, + "response": { + "status": 200, + "headers": { + "Content-Type": [ + "application/json" + ] + }, + "body": { + "write": true + } + } + }, + { + "request": { + "method": "GET", + "url": "https://api.fake.test/v1/resources" + }, + "response": { + "status": 200, + "headers": { + "Content-Type": [ + "application/json" + ] + }, + "body": [ + { + "kind": "server", + "id": "s1", + "name": "web-1", + "region": "fsn1", + "labels": { + "env": "production" + }, + "attrs": { + "size": "small" + } + } + ] + } + }, + { + "request": { + "method": "GET", + "url": "https://api.fake.test/v1/resources" + }, + "response": { + "status": 200, + "headers": { + "Content-Type": [ + "application/json" + ] + }, + "body": [ + { + "kind": "server", + "id": "s1", + "name": "web-1", + "region": "fsn1", + "labels": { + "env": "production" + }, + "attrs": { + "size": "large" + } + } + ] + } + } + ], + "destroy": [ + { + "request": { + "method": "GET", + "url": "https://api.fake.test/v1/token" + }, + "response": { + "status": 200, + "headers": { + "Content-Type": [ + "application/json" + ] + }, + "body": { + "write": true + } + } + }, + { + "request": { + "method": "GET", + "url": "https://api.fake.test/v1/resources" + }, + "response": { + "status": 200, + "headers": { + "Content-Type": [ + "application/json" + ] + }, + "body": [ + { + "kind": "server", + "id": "s1", + "name": "web-1", + "region": "fsn1", + "labels": { + "env": "production" + }, + "attrs": { + "size": "small" + } + }, + { + "kind": "volume", + "id": "v1", + "name": "web-1-data", + "region": "fsn1", + "attrs": { + "attached_to": "s1", + "size_gb": "10" + } + }, + { + "kind": "ipv4", + "id": "ip1", + "name": "web-1-ip", + "region": "fsn1", + "attrs": { + "attached_to": "s1" + } + } + ] + } + }, + { + "request": { + "method": "GET", + "url": "https://api.fake.test/v1/resources" + }, + "response": { + "status": 200, + "headers": { + "Content-Type": [ + "application/json" + ] + }, + "body": [ + { + "kind": "server", + "id": "s1", + "name": "web-1", + "region": "fsn1", + "labels": { + "env": "production" + }, + "attrs": { + "size": "small" + } + }, + { + "kind": "volume", + "id": "v1", + "name": "web-1-data", + "region": "fsn1", + "attrs": { + "attached_to": "s1", + "size_gb": "10" + } + }, + { + "kind": "ipv4", + "id": "ip1", + "name": "web-1-ip", + "region": "fsn1", + "attrs": { + "attached_to": "s1" + } + } + ] + } + }, + { + "request": { + "method": "DELETE", + "url": "https://api.fake.test/v1/resources/s1" + }, + "synthetic": { + "status": 200, + "example": "deleted" + } + } + ] + } +} From 8d91625faec14d357d92751cc3e915f4dfe12d0e Mon Sep 17 00:00:00 2001 From: Imani Manyara Date: Wed, 4 Nov 2026 15:36:43 -0500 Subject: [PATCH 3/4] test: extend the policy guards to cloudkittest The file floor rises to the exact 28 the scan reads. Context-first now also checks exported func types and reads cloudkittest's hooks, with the floor at the measured 14 signatures. The shipped-surface guard covers cloudkittest's exported names and RedactError. --- src/internal/policy/policy_test.go | 81 ++++++++++++++++++++++++------ 1 file changed, 67 insertions(+), 14 deletions(-) diff --git a/src/internal/policy/policy_test.go b/src/internal/policy/policy_test.go index 2845b1c..fb0f0da 100644 --- a/src/internal/policy/policy_test.go +++ b/src/internal/policy/policy_test.go @@ -18,7 +18,7 @@ import ( // minGoFiles is a floor on how many Go files the scan reads. A walk that // stops matching would otherwise report a clean module. Raise it as the // module grows; never lower it to make a failure go away. -const minGoFiles = 17 +const minGoFiles = 28 // forbidden are the symbols that would let a request or a session bypass the // caller's transport, egress controls or host-key checks, or that would give @@ -192,12 +192,30 @@ func TestActionPins(t *testing.T) { } } +// shippedPackages are the packages whose syntax the AST guards read: the +// directory under the module root, the package name, and a floor on how many +// non-test files they must find. +var shippedPackages = []struct { + dir, name string + minFiles int +}{ + {".", "cloudkit", 11}, + {"cloudkittest", "cloudkittest", 7}, +} + // parsePackage parses the non-test files of package cloudkit, at the module // root, and fails below a floor so a parse that finds nothing cannot pass. func parsePackage(t *testing.T) []*ast.File { + t.Helper() + p := shippedPackages[0] + return parseDir(t, p.dir, p.name, p.minFiles) +} + +// parseDir parses the non-test files of one package and fails below a floor. +func parseDir(t *testing.T, dir, name string, minSourceFiles int) []*ast.File { t.Helper() root := moduleRoot(t) - matches, err := filepath.Glob(filepath.Join(root, "*.go")) + matches, err := filepath.Glob(filepath.Join(root, dir, "*.go")) if err != nil { t.Fatal(err) } @@ -211,20 +229,20 @@ func parsePackage(t *testing.T) []*ast.File { if err != nil { t.Fatal(err) } - if f.Name.Name == "cloudkit" { + if f.Name.Name == name { files = append(files, f) } } - const minSourceFiles = 11 if len(files) < minSourceFiles { - t.Fatalf("parsed %d source files of package cloudkit, below the floor of %d", len(files), minSourceFiles) + t.Fatalf("parsed %d source files of package %s, below the floor of %d", len(files), name, minSourceFiles) } return files } -// C10: every method of an exported interface, every method of *Registry and -// every exported func-typed field takes a context.Context first, so the -// caller owns cancellation. The exemptions are pure accessors that reach +// C10: every method of an exported interface, every method of *Registry, +// every exported func type and every exported func-typed field takes a +// context.Context first, so the caller owns cancellation. It reads every +// shipped package, cloudkittest's hooks included. The exemptions are pure accessors that reach // nothing, each with its reason. var contextExempt = map[string]string{ "Provider.Descriptor": "returns a constant description; no I/O", @@ -243,7 +261,11 @@ func TestContextFirst(t *testing.T) { t.Errorf("%s does not take context.Context first (C10)", name) } } - for _, f := range parsePackage(t) { + var files []*ast.File + for _, p := range shippedPackages { + files = append(files, parseDir(t, p.dir, p.name, p.minFiles)...) + } + for _, f := range files { for _, decl := range f.Decls { switch d := decl.(type) { case *ast.FuncDecl: @@ -257,6 +279,8 @@ func TestContextFirst(t *testing.T) { continue } switch tt := ts.Type.(type) { + case *ast.FuncType: + check(ts.Name.Name, tt) case *ast.InterfaceType: for _, m := range tt.Methods.List { if ft, ok := m.Type.(*ast.FuncType); ok { @@ -280,10 +304,11 @@ func TestContextFirst(t *testing.T) { } } } - // Measured 2026-11-03: Registry.{Open,Apply,Descriptors}, Provider's 3, - // Applier's 2, CredentialSource.Token, Factory.New and Transport.Dial. - if checked < 10 { - t.Fatalf("checked %d signatures, below the floor of 10", checked) + // Measured 2026-11-04: Registry.{Open,Apply,Descriptors}, Provider's 3, + // Applier's 2, CredentialSource.Token, Factory.New and Transport.Dial; + // cloudkittest's PlanFunc, CloudInit.Render and SSH.Handshake. + if checked < 14 { + t.Fatalf("checked %d signatures, below the floor of 14", checked) } for name := range contextExempt { if !strings.Contains(name, ".") { @@ -341,11 +366,39 @@ var shipped = []string{ "UnsupportedError", "UnsupportedError.Error", "UnsupportedError.Unwrap", "RetryPolicy", "RetryPolicy.Validate", "RetryPolicy.Backoff", "RateLimitError", "RateLimitError.Error", "RateLimitError.Unwrap", "DefaultMaxAttempts", "DefaultBaseDelay", "DefaultMaxDelay", + // Phase 2, contract kit. + "RedactError", +} + +// shippedKit is every exported name package cloudkittest has carried, under +// the same rule as shipped. Append, never delete. +var shippedKit = []string{ + // Phase 2, contract kit. + "Run", "Check", "Cases", "Suite", "Write", "PlanFunc", "Destroy", "CloudInit", "SSH", + "OneTimeTokenField", "PolicyKnownHosts", "PolicyPinned", "PolicyFromProvider", "PolicyAcceptNew", + "CaseTransport", "CaseNilTransport", "CaseEgress", "CaseReadOnly", "CasePlanDrift", "CaseDestroy", + "CaseRedaction", "CaseCloudInit", "CaseSSH", "CassetteReadWrite", "CassetteReadOnly", + "CassetteDrift", "CassetteDestroy", "Status", "Status.String", "Passed", "Failed", "Skipped", + "CaseResult", "Report", "Report.Failed", "Report.Case", + "Spec", "ParseSpec", "LoadSpec", "Spec.ValidateResponse", "Spec.Synthesize", "Response", + "Recordings", "ParseRecordings", "LoadRecordings", "Recordings.Provider", "Recordings.Cassettes", + "Recordings.Len", "Recorder", "NewRecorder", "Recorder.RoundTrip", "Recorder.Requests", + "Recorder.Unexpected", "Recorder.Unused", "Recorder.Err", "Request", "UnexpectedRequestError", + "UnexpectedRequestError.Error", "UnexpectedRequestError.Unwrap", "SchemaError", "SchemaError.Error", + "SchemaError.Unwrap", "ErrSchemaViolation", "ErrUnsupportedKeyword", "ErrNotInSpec", + "ErrCredentialInRecording", "ErrUnexpectedRequest", } func TestShippedSurfaceIsPreserved(t *testing.T) { + checkSurface(t, parsePackage(t), shipped) + kit := shippedPackages[1] + checkSurface(t, parseDir(t, kit.dir, kit.name, kit.minFiles), shippedKit) +} + +func checkSurface(t *testing.T, files []*ast.File, shipped []string) { + t.Helper() have := map[string]bool{} - for _, f := range parsePackage(t) { + for _, f := range files { for _, decl := range f.Decls { switch d := decl.(type) { case *ast.FuncDecl: From 7dee8a4777e258bd7dbb5f3467539d3e1a97400a Mon Sep 17 00:00:00 2001 From: Imani Manyara Date: Wed, 4 Nov 2026 15:36:43 -0500 Subject: [PATCH 4/4] docs: document the contract kit A reference page for cloudkittest (the nine cases, cassettes, recordings, Recorder, the validator's subset, errors), the reasons for spec-checked replay in the architecture guide, and the changelog. --- CHANGELOG.md | 21 +++++ README.md | 4 + docs/architecture.md | 43 +++++++++-- docs/tools/cloudkittest.md | 152 +++++++++++++++++++++++++++++++++++++ 4 files changed, 213 insertions(+), 7 deletions(-) create mode 100644 docs/tools/cloudkittest.md diff --git a/CHANGELOG.md b/CHANGELOG.md index a03e252..961ab83 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -38,6 +38,27 @@ All notable changes to this project are documented here. The format follows - `UnsupportedError`, matching `ErrUnsupported`. - Policy tests: context first on every interface and registry method (C10), no package-level client or registry (C9), and no shipped exported name removed. +- `RedactError`, which redacts an error's message and keeps its chain, so an adapter can return a + transport or provider error without leaking a credential and its caller still matches + `ErrEgressDenied` or `ErrDenied`. +- Package `cloudkittest`, the contract kit. `Run` and `Check` run nine contract cases against any + `Provider` (transport only, nil transport, egress, read-only token, plan drift, destroy, redaction, + cloud-init and SSH), each over its own registry, a token generated for the run and no network; + `Report` gives each case's result. +- `cloudkittest.Recorder`, an `http.RoundTripper` that replays one cassette, answers each interaction + once and in order, refuses any request the cassette does not hold (`ErrUnexpectedRequest`), and + reports interactions nothing used. +- `cloudkittest.ParseRecordings` and `LoadRecordings`: every recorded and synthetic response is + checked against the provider's OpenAPI document, and a recording carrying an `Authorization` + header, a cookie, an API-key header or a value `Redact` would change is refused + (`ErrCredentialInRecording`). +- `cloudkittest.ParseSpec`, `LoadSpec`, `Spec.ValidateResponse` and `Spec.Synthesize`: an OpenAPI 3 + validator over a stated JSON Schema subset (`type`, `nullable`, `required`, `properties`, + `additionalProperties`, `items`, `enum`, local `$ref`) that refuses any other keyword + (`ErrUnsupportedKeyword`), and synthetic responses generated from the document's examples, marked + `Synthetic`. +- Policy tests read `cloudkittest` too: no default HTTP client or transport, context first on its + hook signatures, and its shipped names preserved. ### Changed diff --git a/README.md b/README.md index 7f7f403..a37438e 100644 --- a/README.md +++ b/README.md @@ -66,6 +66,10 @@ Hosted at . - [Architecture](docs/architecture.md) — the rules every adapter follows, and why - [Release](docs/release.md) — how a version is tagged and released +### Reference + +- [`cloudkittest`](docs/tools/cloudkittest.md) — the contract suite every adapter passes, the replaying recorder and the OpenAPI validator + ## Contributing and security See [CONTRIBUTING.md](CONTRIBUTING.md). Report vulnerabilities privately, as diff --git a/docs/architecture.md b/docs/architecture.md index 9ae4af8..03b8085 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -36,11 +36,15 @@ its own engine adapters. | C10 | `context.Context` first on every call; the logger is injected | C1 to C7, C9 and C10 are implemented in this release; C8 arrives with the SSH path. A policy test -in `src/internal/policy` scans every Go file for the symbols that would break C1, C4, C8, C9 and -C10, with an exemption list that records the exact count it allows. Three more policy tests read -the package's syntax: every interface method, `*Registry` method and exported function field takes -a `context.Context` first (C10); no package-level client or registry exists (C9); and no exported -name the package has shipped disappears. +in `src/internal/policy` scans every Go file of the module, `cloudkittest` included, for the +symbols that would break C1, C4, C8, C9 and C10, with an exemption list that records the exact count +it allows. Three more policy tests read the syntax of `cloudkit` and `cloudkittest`: every interface +method, `*Registry` method, exported function type and exported function field takes a +`context.Context` first (C10); no package-level client or registry exists (C9); and no exported +name either package has shipped disappears. + +Every adapter must also pass the contract suite in `cloudkittest`: nine cases, one per rule a +provider could break, run over recorded responses. See [`cloudkittest`](tools/cloudkittest.md). ## Why the caller owns the transport @@ -75,6 +79,30 @@ token for the provider's object store. Retries never resend a non-idempotent req ambiguous failure, so a create is not sent twice; a 429 is retried because the provider did not act on it. +## Why the contract suite replays spec-checked recordings + +An adapter is proven against what the provider actually sends, without spending money or needing a +token on every pull request. Each case replays a cassette through a `Recorder`, which refuses any +request the cassette does not hold, so an adapter cannot reach the network and cannot make a call +nobody recorded. A case also fails when a recorded request goes unused, because the defect is often +the call an adapter skipped, such as the second live-state read that catches drift. + +Recordings go stale, and hand-edited ones lie. So every response, recorded or synthetic, is checked +against the provider's published OpenAPI document when the recordings load, and a fixture that no +longer matches the schema does not load at all. A write path with no recording yet replays a +response generated from the document's own examples, marked synthetic. The validator implements a +small JSON Schema subset and refuses any keyword outside it, rather than skipping it: a schema +check that quietly ignores part of the schema reports a pass it did not earn. + +A recording is committed to a public repository, so the loader refuses one that carries an +`Authorization` header, a cookie, an API-key header, or any value the module's redactor would +change. The recorder never compares or stores request headers, so the token cloudkit adds to each +request has nowhere to land. + +Each case runs with a token generated for that run. The redaction case puts the token where real +failures put it, in a proxy error that echoes the request and in a provider body that echoes the +token, and then searches every error, log record, panic and URL for it. + ## Why it is a library without a CLI framework Each consumer has its own command tree and configuration loader. A library that brought cobra or a @@ -97,11 +125,12 @@ before it loads. | Path | Holds | |---|---| | `src/` | package `cloudkit`: transport, egress policy, secrets, errors, redaction, the registry, capabilities, plans and `Apply`, retries | +| `src/cloudkittest/` | the contract suite (`Run`, `Check`), the replaying `Recorder`, recordings and the OpenAPI validator | | `src/internal/policy/` | repository-wide tests: forbidden symbols, workflow guards | Planned packages are added when they have code: `providers/` (wave 1: Hetzner, DigitalOcean, -Linode, Vultr; plus `generic` for any SSH host), `ssh/`, `cloudinit/`, `transport/`, `descriptor/` -and `cloudkittest/`. +Linode, Vultr; plus `generic` for any SSH host), `ssh/`, `cloudinit/`, `transport/` and +`descriptor/`. --- diff --git a/docs/tools/cloudkittest.md b/docs/tools/cloudkittest.md new file mode 100644 index 0000000..13d6b01 --- /dev/null +++ b/docs/tools/cloudkittest.md @@ -0,0 +1,152 @@ +# `cloudkittest` + +`cloudkittest` (`github.com/simtabi/cloudkit/src/cloudkittest`) is the contract kit: it runs the nine +contract cases every adapter must pass against any `cloudkit.Provider`, over recordings that have been +checked against the provider's OpenAPI document, with nothing reaching the network. + +## Running the suite + +`cloudkittest.Run(t, suite)` runs each case as a subtest. `cloudkittest.Check(ctx, suite)` runs them +and returns a `Report` without a `*testing.T`, for code that must refuse an adapter whose recordings +fail, such as the descriptor loader. + +```go +func TestHetznerContract(t *testing.T) { + fsys := os.DirFS("testdata") + spec, err := cloudkittest.LoadSpec(fsys, "hetzner.openapi.json") + if err != nil { + t.Fatal(err) + } + recs, err := cloudkittest.LoadRecordings(fsys, "hetzner.recordings.json", spec) + if err != nil { + t.Fatal(err) + } + cloudkittest.Run(t, cloudkittest.Suite{ + Factory: hetzner.Factory(), + Recordings: recs, + Writes: []cloudkittest.Write{ + {Name: "PlanCreate", Capability: cloudkit.CapServerCreate, Plan: planCreate}, + {Name: "PlanDestroy", Capability: cloudkit.CapServerDestroy, Plan: planDestroy}, + }, + Drift: planCreate, + Destroy: &cloudkittest.Destroy{Plan: planDestroy, Loses: []cloudkit.Resource{ + {Kind: "server", ID: "42"}, {Kind: "volume", ID: "7"}, {Kind: "ipv4", ID: "9"}, + }}, + CloudInit: &cloudkittest.CloudInit{Render: render, Fields: fields}, + }) +} +``` + +Each case builds its own `cloudkit.Registry` with a token generated for the run, no retries, dialing +refused, and a logger that keeps every record. A case's failure messages never print that token, even +when the adapter leaked it. + +## The nine cases + +| Case | Passes when | +|---|---| +| `transport-only` | Probe and Inventory reach the injected transport; with a transport that fails every request, every call fails; every write the probe allows has a `Write` in the suite | +| `nil-transport` | `NewRegistry` refuses a nil round tripper and a nil dialer; `Factory.New` refuses an account with a nil client, or a client with no `Transport` (which would use the default one) | +| `egress` | the adapter's client refuses a host outside `APIHosts` before the transport sees it; Probe and Inventory, redirected to an unlisted host, return an error matching `ErrEgressDenied`; the provider describes the same hosts as its factory | +| `read-only-token` | with the read-only cassette, Probe reports every write as `Denied` and every `Write` returns `ErrDenied`, with no write request sent | +| `plan-drift` | `Apply` of the `Drift` plan, after the live state moved, returns `ErrConflict` and sends no write | +| `destroy` | the plan lists every resource in `Destroy.Loses`; `Apply` with nothing or the wrong name typed refuses without a request; with the names typed back it goes through | +| `redaction` | behind a proxy that echoes the request and a provider that echoes the token, no error, log record or panic carries the token, and no URL does | +| `cloud-init` | plain user data renders within `UserDataLimit`; a private key, bearer token or password in any field but `OneTimeToken` is refused; output above the limit is refused | +| `ssh` | under every host-key policy but `accept-new`, a key never seen fails; no policy offers `password` or `keyboard-interactive` | + +A read-only adapter (no `Writes`) skips `plan-drift` and `destroy`. `cloud-init` is skipped only when +`UserDataLimit` is 0, and fails when the provider takes user data but the suite has no `CloudInit`. +`ssh` is skipped, with its reason, while `Suite.SSH` is nil. Every cassette a case replays must be +used in full: a recorded request the adapter never sent fails the case, because skipping it (a +live-state read, say) is often the defect. + +## Recordings + +A recordings file holds named cassettes, each an ordered list of request and response pairs: + +```json +{ + "provider": "fake", + "cassettes": { + "read-write": [ + {"request": {"method": "GET", "url": "https://api.fake.test/v1/token"}, + "response": {"status": 200, "headers": {"Content-Type": ["application/json"]}, "body": {"write": true}}} + ], + "destroy": [ + {"request": {"method": "DELETE", "url": "https://api.fake.test/v1/resources/s1"}, + "synthetic": {"status": 200, "example": "deleted"}} + ] + } +} +``` + +| Cassette | Holds | +|---|---| +| `read-write` | Probe, then Inventory, with a token that may write | +| `read-only` | Probe with a read-only token, then what each `Write` sends before it is refused | +| `drift` | what the `Drift` plan sends, then the live-state read `Apply` makes, returning a different state | +| `destroy` | what the `Destroy` plan sends, then the live-state read and the deletes of applying it | + +`ParseRecordings` is the only way to build `Recordings`, and it refuses: + +| Refused | Error | +|---|---| +| an `Authorization`, `Proxy-Authorization`, `Cookie`, `Set-Cookie`, `X-Auth-Token`, `X-Api-Key` or `Api-Key` header, in a request or a response | `ErrCredentialInRecording` | +| a URL, header value or body that `cloudkit.Redact` would change (a bearer token, a credential query parameter, URL userinfo) | `ErrCredentialInRecording` | +| a request or status the provider's document does not describe | `ErrNotInSpec` | +| a body that breaks its response schema | `ErrSchemaViolation` | +| unknown fields, an interaction with both or neither of `response` and `synthetic` | an error naming the cassette and position | + +A `synthetic` entry is generated by `Spec.Synthesize` from the document's examples and marked +`Synthetic`, for a write path with no recording yet. The example is checked against its own +schema, so a wrong example is refused like a wrong recording. + +## `Recorder` + +`NewRecorder(recs, cassette)` returns an `http.RoundTripper` over one cassette. A request is answered by +the first unused interaction with the same method, URL (scheme, host, path, and query in any order) and +JSON body; each interaction answers once, in order. Any other request returns an +`*UnexpectedRequestError`, matching `ErrUnexpectedRequest`, and is sent nowhere. Headers are not +compared, so the `Authorization` header cloudkit adds is ignored and never stored. + +| Method | Returns | +|---|---| +| `Requests()` | every request seen, URLs redacted, refused ones flagged `Unexpected` | +| `Unexpected()` | the requests the cassette did not hold | +| `Unused()` | the interactions nothing asked for | +| `Err()` | every unexpected request as one error, or nil | + +Recording new responses from a live provider is the job of a scripted recording run, which arrives with +the first provider adapter. + +## The OpenAPI validator + +`ParseSpec` reads an OpenAPI 3 document in JSON and compiles every schema a response reaches. +`Spec.ValidateResponse(method, url, status, header, body)` finds the server (scheme, host and base +path), the path template (a literal segment beats a `{param}`), the method and the response (the exact +status, then `2XX`, then `default`), and checks the body. + +| Keywords | Treatment | +|---|---| +| `type`, `nullable`, `required`, `properties`, `additionalProperties`, `items`, `enum`, `$ref` (within the document) | checked | +| `description`, `title`, `example`, `default`, `format`, `deprecated`, `readOnly`, `writeOnly` | accepted, not checked | +| anything else (`allOf`, `oneOf`, `pattern`, `minimum`, ...) | the document is refused with `ErrUnsupportedKeyword` | + +Refusing an unknown keyword, rather than skipping it, means a response is either checked in full or not +loaded at all. When a provider's document needs a keyword, the validator grows to check it. Only JSON +media types are read, and a response object may not itself be a `$ref`. + +## Errors + +| Error | Meaning | +|---|---| +| `ErrUnexpectedRequest` | a recorder was sent a request its cassette does not hold | +| `ErrCredentialInRecording` | a recording carries a credential header or value | +| `ErrSchemaViolation` | a body breaks its schema; `*SchemaError` names the JSON path | +| `ErrNotInSpec` | a server, path, method, status or content type the document does not describe | +| `ErrUnsupportedKeyword` | a schema keyword outside the subset | + +--- + +[← Docs index](../../README.md#documentation)