From dcbf1ff7440f238b97ae2cdff93ac76786e1025e Mon Sep 17 00:00:00 2001 From: uzi-agent Date: Wed, 11 Nov 2026 08:29:43 +0000 Subject: [PATCH 01/17] issuetracker: driver seam, redactor, no-redirect client and BaseFake (PRD #4173 M3) The package an issue tracker driver lives in (D6): the Driver interface (forge.IssueTracker plus verify, list projects, list states), the kind registry, a redactor that scrubs every encoded form of the token and strips control and bidi characters, an HTTP client that never follows a redirect, bounded body reads and paging backstops (S3), and the loud ErrNotStubbed BaseFake (D7). Co-Authored-By: Claude Opus 5.5 --- api/internal/issuetracker/bounded.go | 30 +++ api/internal/issuetracker/bounded_test.go | 63 +++++++ api/internal/issuetracker/doc.go | 13 ++ api/internal/issuetracker/driver.go | 43 +++++ api/internal/issuetracker/driver_test.go | 35 ++++ api/internal/issuetracker/errors.go | 19 ++ api/internal/issuetracker/httpclient.go | 42 +++++ api/internal/issuetracker/httpclient_test.go | 69 +++++++ .../issuetracker/issuetrackertest/basefake.go | 96 ++++++++++ .../issuetrackertest/basefake_test.go | 56 ++++++ api/internal/issuetracker/kind.go | 35 ++++ api/internal/issuetracker/kind_test.go | 37 ++++ api/internal/issuetracker/options.go | 27 +++ api/internal/issuetracker/pagination.go | 39 ++++ api/internal/issuetracker/pagination_test.go | 77 ++++++++ api/internal/issuetracker/redact.go | 173 +++++++++++++++++ api/internal/issuetracker/redact_test.go | 174 ++++++++++++++++++ api/internal/issuetracker/registry.go | 19 ++ api/internal/issuetracker/registry_test.go | 16 ++ 19 files changed, 1063 insertions(+) create mode 100644 api/internal/issuetracker/bounded.go create mode 100644 api/internal/issuetracker/bounded_test.go create mode 100644 api/internal/issuetracker/doc.go create mode 100644 api/internal/issuetracker/driver.go create mode 100644 api/internal/issuetracker/driver_test.go create mode 100644 api/internal/issuetracker/errors.go create mode 100644 api/internal/issuetracker/httpclient.go create mode 100644 api/internal/issuetracker/httpclient_test.go create mode 100644 api/internal/issuetracker/issuetrackertest/basefake.go create mode 100644 api/internal/issuetracker/issuetrackertest/basefake_test.go create mode 100644 api/internal/issuetracker/kind.go create mode 100644 api/internal/issuetracker/kind_test.go create mode 100644 api/internal/issuetracker/options.go create mode 100644 api/internal/issuetracker/pagination.go create mode 100644 api/internal/issuetracker/pagination_test.go create mode 100644 api/internal/issuetracker/redact.go create mode 100644 api/internal/issuetracker/redact_test.go create mode 100644 api/internal/issuetracker/registry.go create mode 100644 api/internal/issuetracker/registry_test.go diff --git a/api/internal/issuetracker/bounded.go b/api/internal/issuetracker/bounded.go new file mode 100644 index 0000000000..c8568db667 --- /dev/null +++ b/api/internal/issuetracker/bounded.go @@ -0,0 +1,30 @@ +package issuetracker + +import ( + "io" + + "github.com/vtmocanu/uzi/api/internal/forge" +) + +const ( + // listPageBytes caps one page of a list response. + listPageBytes int64 = 8 << 20 + // objectBytes caps a single-object response. + objectBytes int64 = 2 << 20 + // errorBodyBytes caps the body read from an error response. + errorBodyBytes int64 = 4 << 10 +) + +// readBounded reads at most limit bytes of r. A longer body returns a +// *forge.ResponseTooLargeError carrying prov and no remote content. The limit +// counts bytes after transparent decompression, which is what r yields. +func readBounded(r io.Reader, limit int64, prov forge.ResponseProvenance) ([]byte, error) { + body, err := io.ReadAll(io.LimitReader(r, limit+1)) + if err != nil { + return nil, err + } + if int64(len(body)) > limit { + return nil, &forge.ResponseTooLargeError{Provenance: prov, Outcome: forge.ErrorClassOther} + } + return body, nil +} diff --git a/api/internal/issuetracker/bounded_test.go b/api/internal/issuetracker/bounded_test.go new file mode 100644 index 0000000000..3046b03e9c --- /dev/null +++ b/api/internal/issuetracker/bounded_test.go @@ -0,0 +1,63 @@ +package issuetracker + +import ( + "bytes" + "compress/gzip" + "errors" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/vtmocanu/uzi/api/internal/forge" +) + +func TestReadBounded(t *testing.T) { + got, err := readBounded(strings.NewReader("hello"), 5, forge.ResponseSuccess) + if err != nil || string(got) != "hello" { + t.Fatalf("at limit: %q, %v", got, err) + } + _, err = readBounded(strings.NewReader("hello!"), 5, forge.ResponseIssueSuccess) + if !errors.Is(err, forge.ErrResponseTooLarge) { + t.Fatalf("err = %v, want ErrResponseTooLarge", err) + } + if !forge.IsIssueResponseTooLarge(err) { + t.Fatal("provenance not issue success") + } + _, err = readBounded(strings.NewReader("hello!"), 5, forge.ResponseError) + if forge.IsIssueResponseTooLarge(err) { + t.Fatal("error provenance reported as issue success") + } +} + +func TestReadBoundedCapsDecompressedBytes(t *testing.T) { + var gz bytes.Buffer + zw := gzip.NewWriter(&gz) + if _, err := zw.Write(bytes.Repeat([]byte("a"), 64<<10)); err != nil { + t.Fatal(err) + } + if err := zw.Close(); err != nil { + t.Fatal(err) + } + if gz.Len() > 1<<10 { + t.Fatalf("fixture not small: %d", gz.Len()) + } + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Encoding", "gzip") + _, _ = w.Write(gz.Bytes()) + })) + defer srv.Close() + + resp, err := newHTTPClient(0).Get(srv.URL) + if err != nil { + t.Fatal(err) + } + defer resp.Body.Close() + if resp.ContentLength >= 64<<10 && resp.ContentLength != -1 { + t.Fatalf("declared length %d", resp.ContentLength) + } + _, err = readBounded(resp.Body, 1<<10, forge.ResponseIssueSuccess) + if !forge.IsIssueResponseTooLarge(err) { + t.Fatalf("decompressed overflow not capped: %v", err) + } +} diff --git a/api/internal/issuetracker/doc.go b/api/internal/issuetracker/doc.go new file mode 100644 index 0000000000..98d0580d80 --- /dev/null +++ b/api/internal/issuetracker/doc.go @@ -0,0 +1,13 @@ +// Package issuetracker holds drivers for external issue trackers: systems that +// keep issues but host no code (PRD #4173 D6). +// +// A Driver implements forge.IssueTracker and never forge.CodeHost (PRD #4173 +// S1): an issue tracker cannot answer repository, merge request or pipeline +// questions, so the type system refuses to let one be used as a code host. +// Each driver is constructed through New by Kind. +// +// Credentials are handled per PRD #4173 S3: the token is scrubbed from every +// error a driver returns (see redactor), responses are size-bounded, redirects +// are refused so the token never follows a Location header, and the HTTP +// client has a finite timeout. +package issuetracker diff --git a/api/internal/issuetracker/driver.go b/api/internal/issuetracker/driver.go new file mode 100644 index 0000000000..0356fa1bc7 --- /dev/null +++ b/api/internal/issuetracker/driver.go @@ -0,0 +1,43 @@ +package issuetracker + +import ( + "context" + + "github.com/vtmocanu/uzi/api/internal/forge" +) + +// Account is the identity a token authenticates as. +type Account struct { + ID string + Login string +} + +// Project is an issue container on the issue tracker. ID is the opaque +// server-side identifier; ShortName is the human-facing key (for example the +// prefix of an issue id). +type Project struct { + ID string + ShortName string + Name string +} + +// State is one value of the issue state field. +type State struct { + ID string + Name string + Resolved bool // true when the state counts as closed + Ordinal int +} + +// Driver is an external issue tracker connection. It embeds forge.IssueTracker +// only, never forge.CodeHost (PRD #4173 S1). The extra methods serve the +// connection flow: verify the token, then discover projects and states. +type Driver interface { + forge.IssueTracker + // VerifyToken confirms the token works and returns the account it belongs to. + VerifyToken(ctx context.Context) (Account, error) + // ListProjects returns the projects the token can see. + ListProjects(ctx context.Context) ([]Project, error) + // ListStates returns the issue states available in a project. + ListStates(ctx context.Context, projectID string) ([]State, error) +} diff --git a/api/internal/issuetracker/driver_test.go b/api/internal/issuetracker/driver_test.go new file mode 100644 index 0000000000..ed2736c522 --- /dev/null +++ b/api/internal/issuetracker/driver_test.go @@ -0,0 +1,35 @@ +package issuetracker + +import ( + "reflect" + "testing" + + "github.com/vtmocanu/uzi/api/internal/forge" +) + +// TestDriverIsNotCodeHost pins PRD #4173 S1: the Driver method set must not +// satisfy forge.CodeHost, nor even share its code-host methods. +func TestDriverIsNotCodeHost(t *testing.T) { + driver := reflect.TypeFor[Driver]() + codeHost := reflect.TypeFor[forge.CodeHost]() + if driver.Implements(codeHost) { + t.Fatal("Driver implements forge.CodeHost") + } + // VerifyToken and ListProjects legitimately exist on both with different + // signatures; every other CodeHost method must be absent from Driver. + shared := map[string]bool{"VerifyToken": true, "ListProjects": true} + for m := range codeHost.Methods() { + if shared[m.Name] { + continue + } + if _, ok := driver.MethodByName(m.Name); ok { + t.Errorf("Driver exposes code-host method %s", m.Name) + } + } +} + +func TestDriverEmbedsIssueTracker(t *testing.T) { + if !reflect.TypeFor[Driver]().Implements(reflect.TypeFor[forge.IssueTracker]()) { + t.Fatal("Driver does not implement forge.IssueTracker") + } +} diff --git a/api/internal/issuetracker/errors.go b/api/internal/issuetracker/errors.go new file mode 100644 index 0000000000..2d76aaf3ba --- /dev/null +++ b/api/internal/issuetracker/errors.go @@ -0,0 +1,19 @@ +package issuetracker + +import "errors" + +var ( + // ErrUnsupportedKind is returned by New for a Kind with no driver. + ErrUnsupportedKind = errors.New("issuetracker: unsupported kind") + // ErrProjectNotBound is returned when a project-scoped call names a project + // other than the one the connection is bound to. + ErrProjectNotBound = errors.New("issuetracker: project not bound to this connection") + // ErrStateMappingUnset is returned when open/closed state mapping is needed + // but the connection has not configured it. + ErrStateMappingUnset = errors.New("issuetracker: state mapping not configured") + // ErrInvalidLabelName is returned for a label name the issue tracker cannot carry. + ErrInvalidLabelName = errors.New("issuetracker: invalid label name") + // ErrRedirectRefused is returned when the server answers with a redirect. + // The client never follows one, so the token cannot be forwarded. + ErrRedirectRefused = errors.New("issuetracker: redirect refused") +) diff --git a/api/internal/issuetracker/httpclient.go b/api/internal/issuetracker/httpclient.go new file mode 100644 index 0000000000..f964f1dc24 --- /dev/null +++ b/api/internal/issuetracker/httpclient.go @@ -0,0 +1,42 @@ +package issuetracker + +import ( + "net/http" + "time" + + "github.com/vtmocanu/uzi/api/internal/httptransport" +) + +// defaultTimeout bounds one request when Options.Timeout is zero. +const defaultTimeout = 15 * time.Second + +// testTransport, when non-nil, replaces the production transport. It is nil +// outside tests: export_test.go sets it to trust an httptest server's roots, +// which Options deliberately cannot express. +var testTransport http.RoundTripper + +// newHTTPClient builds the client a driver uses: an independent transport, a +// finite timeout, and no redirect following, so a Location header cannot carry +// the token to another host. +func newHTTPClient(timeout time.Duration) *http.Client { + if timeout <= 0 { + timeout = defaultTimeout + } + var rt http.RoundTripper = httptransport.New(http.DefaultTransport) + if testTransport != nil { + rt = testTransport + } + return &http.Client{ + Transport: rt, + Timeout: timeout, + CheckRedirect: func(*http.Request, []*http.Request) error { + return http.ErrUseLastResponse + }, + } +} + +// isRedirect reports whether resp is any 3xx, including 304 and a response with +// no Location. None is ever followed or treated as success. +func isRedirect(resp *http.Response) bool { + return resp.StatusCode >= 300 && resp.StatusCode < 400 +} diff --git a/api/internal/issuetracker/httpclient_test.go b/api/internal/issuetracker/httpclient_test.go new file mode 100644 index 0000000000..f8f5a11770 --- /dev/null +++ b/api/internal/issuetracker/httpclient_test.go @@ -0,0 +1,69 @@ +package issuetracker + +import ( + "net/http" + "net/http/httptest" + "sync/atomic" + "testing" + "time" +) + +func TestClientDoesNotFollowRedirect(t *testing.T) { + var hits atomic.Int32 + second := httptest.NewServer(http.HandlerFunc(func(http.ResponseWriter, *http.Request) { + hits.Add(1) + })) + defer second.Close() + first := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + http.Redirect(w, r, second.URL, http.StatusFound) + })) + defer first.Close() + + resp, err := newHTTPClient(0).Get(first.URL) + if err != nil { + t.Fatal(err) + } + defer resp.Body.Close() + if !isRedirect(resp) { + t.Fatalf("status %d not detected as redirect", resp.StatusCode) + } + if n := hits.Load(); n != 0 { + t.Fatalf("redirect target got %d hits", n) + } +} + +func TestIsRedirect(t *testing.T) { + for code, want := range map[int]bool{ + 200: false, 204: false, 299: false, + 300: true, 301: true, 302: true, 304: true, 399: true, + 400: false, 500: false, + } { + if got := isRedirect(&http.Response{StatusCode: code}); got != want { + t.Errorf("isRedirect(%d) = %v, want %v", code, got, want) + } + } +} + +func TestClientNotModifiedWithoutLocation(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusNotModified) + })) + defer srv.Close() + resp, err := newHTTPClient(0).Get(srv.URL) + if err != nil { + t.Fatal(err) + } + defer resp.Body.Close() + if !isRedirect(resp) { + t.Fatal("304 not detected") + } +} + +func TestClientTimeout(t *testing.T) { + if got := newHTTPClient(0).Timeout; got != defaultTimeout { + t.Fatalf("default timeout = %v", got) + } + if got := newHTTPClient(3 * time.Second).Timeout; got != 3*time.Second { + t.Fatalf("timeout = %v", got) + } +} diff --git a/api/internal/issuetracker/issuetrackertest/basefake.go b/api/internal/issuetracker/issuetrackertest/basefake.go new file mode 100644 index 0000000000..c22d8f3fd7 --- /dev/null +++ b/api/internal/issuetracker/issuetrackertest/basefake.go @@ -0,0 +1,96 @@ +// Package issuetrackertest provides BaseFake, an embeddable implementation of +// every issuetracker.Driver method that fails loudly, so a test fake embeds it +// and overrides only what it exercises. An unstubbed method returns an error +// wrapping ErrNotStubbed that names the method, never a silent zero success. +package issuetrackertest + +import ( + "context" + "errors" + "fmt" + + "github.com/vtmocanu/uzi/api/internal/forge" + "github.com/vtmocanu/uzi/api/internal/issuetracker" +) + +// ErrNotStubbed is the sentinel every unstubbed BaseFake method wraps. +var ErrNotStubbed = errors.New("issuetrackertest: method not stubbed") + +func notStubbed(method string) error { + return fmt.Errorf("issuetrackertest: method %s not stubbed: %w", method, ErrNotStubbed) +} + +// BaseFake implements issuetracker.Driver with not-stubbed defaults. +type BaseFake struct{} + +var _ issuetracker.Driver = (*BaseFake)(nil) + +// ListLabels implements issuetracker.Driver. +func (*BaseFake) ListLabels(context.Context, int64) ([]forge.Label, error) { + return nil, notStubbed("ListLabels") +} + +// EnsureLabels implements issuetracker.Driver. +func (*BaseFake) EnsureLabels(context.Context, int64, []forge.Label) error { + return notStubbed("EnsureLabels") +} + +// ListIssues implements issuetracker.Driver. +func (*BaseFake) ListIssues(context.Context, int64, forge.ListIssuesOptions) ([]forge.Issue, error) { + return nil, notStubbed("ListIssues") +} + +// GetIssue implements issuetracker.Driver. +func (*BaseFake) GetIssue(context.Context, int64, int64) (forge.Issue, error) { + return forge.Issue{}, notStubbed("GetIssue") +} + +// CreateIssue implements issuetracker.Driver. +func (*BaseFake) CreateIssue(context.Context, int64, string, string, []string) (forge.Issue, error) { + return forge.Issue{}, notStubbed("CreateIssue") +} + +// UpdateIssueLabels implements issuetracker.Driver. +func (*BaseFake) UpdateIssueLabels(context.Context, int64, int64, []string, []string) error { + return notStubbed("UpdateIssueLabels") +} + +// UpdateIssueDescription implements issuetracker.Driver. +func (*BaseFake) UpdateIssueDescription(context.Context, int64, int64, string) error { + return notStubbed("UpdateIssueDescription") +} + +// SetIssueState implements issuetracker.Driver. +func (*BaseFake) SetIssueState(context.Context, int64, int64, forge.IssueState) error { + return notStubbed("SetIssueState") +} + +// ListIssueLabelEvents implements issuetracker.Driver. +func (*BaseFake) ListIssueLabelEvents(context.Context, int64, int64) ([]forge.LabelEvent, error) { + return nil, notStubbed("ListIssueLabelEvents") +} + +// ListIssueComments implements issuetracker.Driver. +func (*BaseFake) ListIssueComments(context.Context, int64, int64) ([]forge.IssueComment, error) { + return nil, notStubbed("ListIssueComments") +} + +// CreateIssueNote implements issuetracker.Driver. +func (*BaseFake) CreateIssueNote(context.Context, int64, int64, string) (forge.IssueNote, error) { + return forge.IssueNote{}, notStubbed("CreateIssueNote") +} + +// VerifyToken implements issuetracker.Driver. +func (*BaseFake) VerifyToken(context.Context) (issuetracker.Account, error) { + return issuetracker.Account{}, notStubbed("VerifyToken") +} + +// ListProjects implements issuetracker.Driver. +func (*BaseFake) ListProjects(context.Context) ([]issuetracker.Project, error) { + return nil, notStubbed("ListProjects") +} + +// ListStates implements issuetracker.Driver. +func (*BaseFake) ListStates(context.Context, string) ([]issuetracker.State, error) { + return nil, notStubbed("ListStates") +} diff --git a/api/internal/issuetracker/issuetrackertest/basefake_test.go b/api/internal/issuetracker/issuetrackertest/basefake_test.go new file mode 100644 index 0000000000..4b9a0a3659 --- /dev/null +++ b/api/internal/issuetracker/issuetrackertest/basefake_test.go @@ -0,0 +1,56 @@ +package issuetrackertest + +import ( + "context" + "errors" + "strings" + "testing" + + "github.com/vtmocanu/uzi/api/internal/forge" +) + +// Calling every method keeps each one reachable for `deadcode -test`, since the +// compile-time assertion in basefake.go creates no reachability. +func TestBaseFakeMethodsNotStubbed(t *testing.T) { + ctx := context.Background() + methods := []struct { + name string + call func(*BaseFake) error + }{ + {"ListLabels", func(b *BaseFake) error { _, err := b.ListLabels(ctx, 1); return err }}, + {"EnsureLabels", func(b *BaseFake) error { return b.EnsureLabels(ctx, 1, nil) }}, + {"ListIssues", func(b *BaseFake) error { + _, err := b.ListIssues(ctx, 1, forge.ListIssuesOptions{}) + return err + }}, + {"GetIssue", func(b *BaseFake) error { _, err := b.GetIssue(ctx, 1, 2); return err }}, + {"CreateIssue", func(b *BaseFake) error { _, err := b.CreateIssue(ctx, 1, "t", "d", nil); return err }}, + {"UpdateIssueLabels", func(b *BaseFake) error { return b.UpdateIssueLabels(ctx, 1, 2, nil, nil) }}, + {"UpdateIssueDescription", func(b *BaseFake) error { return b.UpdateIssueDescription(ctx, 1, 2, "d") }}, + {"SetIssueState", func(b *BaseFake) error { return b.SetIssueState(ctx, 1, 2, forge.StateClosed) }}, + {"ListIssueLabelEvents", func(b *BaseFake) error { _, err := b.ListIssueLabelEvents(ctx, 1, 2); return err }}, + {"ListIssueComments", func(b *BaseFake) error { _, err := b.ListIssueComments(ctx, 1, 2); return err }}, + {"CreateIssueNote", func(b *BaseFake) error { _, err := b.CreateIssueNote(ctx, 1, 2, "b"); return err }}, + {"VerifyToken", func(b *BaseFake) error { _, err := b.VerifyToken(ctx); return err }}, + {"ListProjects", func(b *BaseFake) error { _, err := b.ListProjects(ctx); return err }}, + {"ListStates", func(b *BaseFake) error { _, err := b.ListStates(ctx, "p"); return err }}, + } + if len(methods) != 14 { + t.Fatalf("expected 14 methods, got %d", len(methods)) + } + b := &BaseFake{} + for _, m := range methods { + t.Run(m.name, func(t *testing.T) { + err := m.call(b) + if err == nil { + t.Fatal("expected an error, got nil") + } + if !errors.Is(err, ErrNotStubbed) { + t.Fatalf("error does not wrap ErrNotStubbed: %v", err) + } + if !strings.Contains(err.Error(), m.name) { + t.Fatalf("error %q does not name the method", err.Error()) + } + }) + } +} diff --git a/api/internal/issuetracker/kind.go b/api/internal/issuetracker/kind.go new file mode 100644 index 0000000000..e4e63260a4 --- /dev/null +++ b/api/internal/issuetracker/kind.go @@ -0,0 +1,35 @@ +package issuetracker + +import "strings" + +// Kind names an issue tracker driver. The string is the stored value. +type Kind string + +// KindYouTrack is the JetBrains YouTrack driver. +const KindYouTrack Kind = "youtrack" + +// SupportedKinds lists every driver, in the order a connect UI would offer them. +func SupportedKinds() []Kind { + return []Kind{KindYouTrack} +} + +// ParseKind returns the driver named by s, or false when s names none. +// Matching is exact. +func ParseKind(s string) (Kind, bool) { + for _, k := range SupportedKinds() { + if string(k) == s { + return k, true + } + } + return "", false +} + +// SupportedKindsList renders SupportedKinds for an error message. +func SupportedKindsList() string { + kinds := SupportedKinds() + names := make([]string, 0, len(kinds)) + for _, k := range kinds { + names = append(names, string(k)) + } + return strings.Join(names, ", ") +} diff --git a/api/internal/issuetracker/kind_test.go b/api/internal/issuetracker/kind_test.go new file mode 100644 index 0000000000..ba5704ae17 --- /dev/null +++ b/api/internal/issuetracker/kind_test.go @@ -0,0 +1,37 @@ +package issuetracker + +import "testing" + +func TestParseKind(t *testing.T) { + for _, tc := range []struct { + in string + want Kind + ok bool + }{ + {"youtrack", KindYouTrack, true}, + {"YouTrack", "", false}, + {" youtrack", "", false}, + {"", "", false}, + {"gitlab", "", false}, + } { + got, ok := ParseKind(tc.in) + if got != tc.want || ok != tc.ok { + t.Errorf("ParseKind(%q) = %q, %v; want %q, %v", tc.in, got, ok, tc.want, tc.ok) + } + } +} + +func TestSupportedKindsRoundTrip(t *testing.T) { + kinds := SupportedKinds() + if len(kinds) == 0 { + t.Fatal("no supported kinds") + } + for _, k := range kinds { + if got, ok := ParseKind(string(k)); !ok || got != k { + t.Errorf("ParseKind(%q) = %q, %v", k, got, ok) + } + } + if got := SupportedKindsList(); got != "youtrack" { + t.Errorf("SupportedKindsList() = %q", got) + } +} diff --git a/api/internal/issuetracker/options.go b/api/internal/issuetracker/options.go new file mode 100644 index 0000000000..57dbc27604 --- /dev/null +++ b/api/internal/issuetracker/options.go @@ -0,0 +1,27 @@ +package issuetracker + +import "time" + +// ProjectRef identifies the project a connection is bound to. +type ProjectRef struct { + ID string + ShortName string +} + +// Options configures a driver. There is deliberately no TLS field: a driver +// trusts the system roots and nothing else (PRD #4173 S3). +type Options struct { + // BaseURL is the issue tracker's root URL; the caller has already validated it. + BaseURL string + // Token is the permanent token. It never appears in a returned error. + Token string + // Project is the project the connection is bound to. + Project ProjectRef + // StateField, OpenState and ClosedState map neutral open/closed onto the + // issue tracker's own state field and values. + StateField string + OpenState string + ClosedState string + // Timeout bounds one HTTP request; zero selects defaultTimeout. + Timeout time.Duration +} diff --git a/api/internal/issuetracker/pagination.go b/api/internal/issuetracker/pagination.go new file mode 100644 index 0000000000..5e27ef1433 --- /dev/null +++ b/api/internal/issuetracker/pagination.go @@ -0,0 +1,39 @@ +package issuetracker + +import "errors" + +// pageSize is the $top of every list request. +const pageSize = 100 + +// maxItems and maxPages are sanity ceilings against a server that never ends a +// listing. Vars so tests can lower them. +var ( + maxItems = 1_000_000 + maxPages = 20_000 +) + +// paginate calls fetch with a growing skip until a short page. A positive limit +// stops once that many items are collected and truncates to it; zero means no +// cap. Exceeding maxItems or maxPages is an error carrying no remote content. +func paginate[T any](fetch func(skip, top int) ([]T, error), limit int) ([]T, error) { + var all []T + for page := 0; ; page++ { + if page >= maxPages { + return nil, errors.New("issuetracker: listing exceeds the page ceiling") + } + items, err := fetch(len(all), pageSize) + if err != nil { + return nil, err + } + all = append(all, items...) + if limit > 0 && len(all) >= limit { + return all[:limit], nil + } + if len(all) > maxItems { + return nil, errors.New("issuetracker: listing exceeds the item ceiling") + } + if len(items) < pageSize { + return all, nil + } + } +} diff --git a/api/internal/issuetracker/pagination_test.go b/api/internal/issuetracker/pagination_test.go new file mode 100644 index 0000000000..ce5769d93a --- /dev/null +++ b/api/internal/issuetracker/pagination_test.go @@ -0,0 +1,77 @@ +package issuetracker + +import ( + "errors" + "testing" +) + +// serve returns a fetch over total synthetic items, recording each call. +func serve(total int, calls *[][2]int) func(skip, top int) ([]int, error) { + return func(skip, top int) ([]int, error) { + *calls = append(*calls, [2]int{skip, top}) + var out []int + for i := skip; i < skip+top && i < total; i++ { + out = append(out, i) + } + return out, nil + } +} + +func TestPaginateCollectsAll(t *testing.T) { + var calls [][2]int + got, err := paginate(serve(250, &calls), 0) + if err != nil || len(got) != 250 { + t.Fatalf("len %d, err %v", len(got), err) + } + if len(calls) != 3 || calls[2] != [2]int{200, pageSize} { + t.Fatalf("calls = %v", calls) + } +} + +func TestPaginateExactMultipleFetchesEmptyPage(t *testing.T) { + var calls [][2]int + got, err := paginate(serve(200, &calls), 0) + if err != nil || len(got) != 200 || len(calls) != 3 { + t.Fatalf("len %d, calls %d, err %v", len(got), len(calls), err) + } +} + +func TestPaginateLimit(t *testing.T) { + var calls [][2]int + got, err := paginate(serve(1000, &calls), 150) + if err != nil || len(got) != 150 || len(calls) != 2 { + t.Fatalf("len %d, calls %d, err %v", len(got), len(calls), err) + } +} + +func TestPaginateFetchError(t *testing.T) { + want := errors.New("boom") + _, err := paginate(func(int, int) ([]int, error) { return nil, want }, 0) + if !errors.Is(err, want) { + t.Fatalf("err = %v", err) + } +} + +func TestPaginateCaps(t *testing.T) { + t.Run("items", func(t *testing.T) { + old := maxItems + maxItems = 250 + t.Cleanup(func() { maxItems = old }) + var calls [][2]int + if _, err := paginate(serve(10_000, &calls), 0); err == nil { + t.Fatal("expected item ceiling error") + } + }) + t.Run("pages", func(t *testing.T) { + old := maxPages + maxPages = 3 + t.Cleanup(func() { maxPages = old }) + var calls [][2]int + if _, err := paginate(serve(10_000, &calls), 0); err == nil { + t.Fatal("expected page ceiling error") + } + if len(calls) != 3 { + t.Fatalf("calls = %d", len(calls)) + } + }) +} diff --git a/api/internal/issuetracker/redact.go b/api/internal/issuetracker/redact.go new file mode 100644 index 0000000000..d1bf08b5cd --- /dev/null +++ b/api/internal/issuetracker/redact.go @@ -0,0 +1,173 @@ +package issuetracker + +import ( + "context" + "encoding/json" + "errors" + "net" + "net/http" + "net/url" + "strings" + "unicode/utf8" + + "github.com/vtmocanu/uzi/api/internal/forge" +) + +const ( + // redactPlaceholder replaces any occurrence of the token in a message. + redactPlaceholder = "[REDACTED]" + // maxMessageBytes bounds a returned error message. + maxMessageBytes = 512 + // minTokenLen is the shortest token a constructor accepts. A shorter secret + // could not be scrubbed without mangling unrelated text. + minTokenLen = 16 + // minSegmentLen is the shortest dot-separated token segment scrubbed alone. + minSegmentLen = 8 +) + +// validateToken rejects a token too short to redact safely. The message never +// includes the token. +func validateToken(token string) error { + if len(token) < minTokenLen { + return errors.New("issuetracker: token shorter than the minimum length") + } + return nil +} + +// redactor scrubs one token out of every error a driver returns. Build it from +// the token once, in the constructor. +type redactor struct { + secrets []string +} + +// newRedactor lists the token's raw, query-escaped, path-escaped and +// JSON-escaped forms, plus its final '.'-separated segment when long enough +// (a permanent token has the shape prefix:id.secret). +func newRedactor(token string) redactor { + var secrets []string + add := func(s string) { + if s != "" && len(s) >= minSegmentLen { + secrets = append(secrets, s) + } + } + add(token) + add(url.QueryEscape(token)) + add(url.PathEscape(token)) + if b, err := json.Marshal(token); err == nil && len(b) >= 2 { + add(string(b[1 : len(b)-1])) + } + if i := strings.LastIndexByte(token, '.'); i >= 0 { + add(token[i+1:]) + } + return redactor{secrets: secrets} +} + +// string scrubs secrets, strips control and bidi characters, and truncates. +func (r redactor) string(s string) string { + for _, secret := range r.secrets { + s = strings.ReplaceAll(s, secret, redactPlaceholder) + } + return truncate(sanitize(s), maxMessageBytes) +} + +// sanitize replaces C0/C1 control characters and bidi overrides with a space. +func sanitize(s string) string { + return strings.Map(func(c rune) rune { + switch { + case c < 0x20, c >= 0x7f && c <= 0x9f, + c >= 0x202a && c <= 0x202e, + c >= 0x2066 && c <= 0x2069: + return ' ' + } + return c + }, s) +} + +// truncate cuts s to at most limit bytes on a UTF-8 boundary. +func truncate(s string, limit int) string { + if len(s) <= limit { + return s + } + s = s[:limit] + for len(s) > 0 && !utf8.ValidString(s) { + s = s[:len(s)-1] + } + return s +} + +// classifiedError carries a scrubbed message and a normalized class. It has no +// Unwrap on purpose: the original error may contain the token. +type classifiedError struct { + message string + class forge.ErrorClass +} + +func (e *classifiedError) Error() string { return e.message } +func (e *classifiedError) Class() forge.ErrorClass { return e.class } + +// sentinelError is a redacted error that stays identifiable: errors.Is matches +// the fixed sentinel and nothing else, because Unwrap returns only the +// sentinel. Use it only for a sentinel that carries no secret. +type sentinelError struct { + classifiedError + sentinel error +} + +func (e *sentinelError) Unwrap() error { return e.sentinel } + +// error returns err as a scrubbed, classified error, or nil for nil. A forge +// oversize error passes through as a fresh copy so its provenance survives. +func (r redactor) error(err error) error { + return r.errorStatus(err, 0) +} + +// errorStatus is error with the HTTP status of the response, when known. +func (r redactor) errorStatus(err error, status int) error { + if err == nil { + return nil + } + var large *forge.ResponseTooLargeError + if errors.As(err, &large) { + return &forge.ResponseTooLargeError{Provenance: large.Provenance, Outcome: large.Class()} + } + return &classifiedError{message: r.string(err.Error()), class: classify(err, status)} +} + +// sentinel returns a redacted error for sentinel with extra context in its +// message. errors.Is(result, sentinel) holds; the chain exposes nothing else. +func (r redactor) sentinel(sentinel error, status int, detail string) error { + msg := sentinel.Error() + if detail != "" { + msg += ": " + detail + } + return &sentinelError{ + classifiedError: classifiedError{message: r.string(msg), class: classifyStatus(status)}, + sentinel: sentinel, + } +} + +func classify(err error, status int) forge.ErrorClass { + if errors.Is(err, context.DeadlineExceeded) { + return forge.ErrorClassTimeout + } + var netErr net.Error + if errors.As(err, &netErr) && netErr.Timeout() { + return forge.ErrorClassTimeout + } + if c := classifyStatus(status); c != forge.ErrorClassOther { + return c + } + return forge.Class(err) +} + +func classifyStatus(status int) forge.ErrorClass { + switch { + case status == http.StatusUnauthorized || status == http.StatusForbidden: + return forge.ErrorClassAuth + case status == http.StatusTooManyRequests: + return forge.ErrorClassRateLimited + case status >= 500: + return forge.ErrorClassServerError + } + return forge.ErrorClassOther +} diff --git a/api/internal/issuetracker/redact_test.go b/api/internal/issuetracker/redact_test.go new file mode 100644 index 0000000000..fe52eb00d2 --- /dev/null +++ b/api/internal/issuetracker/redact_test.go @@ -0,0 +1,174 @@ +package issuetracker + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "net/url" + "strings" + "testing" + + "github.com/vtmocanu/uzi/api/internal/forge" +) + +// fixtureCredential assembles a credential-shaped value at runtime so no +// complete token literal sits in source. It contains characters that each +// escaping form rewrites (space, slash, quote). +func fixtureCredential() string { + return strings.Join([]string{"perm", ":", "ab c/d\"e", ".", "Zq9xLm2VnB7wKp"}, "") +} + +type timeoutErr struct{} + +func (timeoutErr) Error() string { return "i/o timeout" } +func (timeoutErr) Timeout() bool { return true } +func (timeoutErr) Temporary() bool { return false } + +func TestRedactorScrubsEveryEncoding(t *testing.T) { + cred := fixtureCredential() + r := newRedactor(cred) + segment := cred[strings.LastIndexByte(cred, '.')+1:] + jsonForm := strings.Trim(mustJSON(t, cred), `"`) + for name, form := range map[string]string{ + "raw": cred, + "query": url.QueryEscape(cred), + "path": url.PathEscape(cred), + "json": jsonForm, + "segment": segment, + } { + t.Run(name, func(t *testing.T) { + got := r.error(fmt.Errorf("boom %s end", form)).Error() + if strings.Contains(got, form) { + t.Fatalf("message still contains %s form: %q", name, got) + } + if !strings.Contains(got, redactPlaceholder) { + t.Fatalf("no placeholder in %q", got) + } + }) + } +} + +func mustJSON(t *testing.T, s string) string { + t.Helper() + b, err := json.Marshal(s) + if err != nil { + t.Fatal(err) + } + return string(b) +} + +func TestRedactorShortSegmentKept(t *testing.T) { + r := newRedactor(strings.Join([]string{"abcdefghijklmnop", ".", "xyz"}, "")) + if got := r.string("see xyz here"); got != "see xyz here" { + t.Fatalf("short segment scrubbed: %q", got) + } +} + +func TestRedactorStripsControlAndBidi(t *testing.T) { + r := newRedactor(fixtureCredential()) + got := r.string("a\x00b\x1bc\u0085d‮e⁦f⁩g\th") + for _, c := range got { + if c < 0x20 || (c >= 0x7f && c <= 0x9f) || (c >= 0x202a && c <= 0x202e) || (c >= 0x2066 && c <= 0x2069) { + t.Fatalf("unsafe rune %U in %q", c, got) + } + } + if !strings.Contains(got, "a") || !strings.Contains(got, "h") { + t.Fatalf("printable text lost: %q", got) + } +} + +func TestRedactorTruncatesOnRuneBoundary(t *testing.T) { + r := newRedactor(fixtureCredential()) + in := strings.Repeat("a", maxMessageBytes-1) + "éé" + got := r.string(in) + if len(got) > maxMessageBytes { + t.Fatalf("len = %d", len(got)) + } + if got != strings.Repeat("a", maxMessageBytes-1) { + t.Fatalf("split rune kept: len %d", len(got)) + } +} + +func TestRedactorErrorHasNoUnwrap(t *testing.T) { + r := newRedactor(fixtureCredential()) + orig := fmt.Errorf("wrap: %w", context.Canceled) + err := r.error(orig) + if errors.Is(err, context.Canceled) { + t.Fatal("redacted error exposes the original chain") + } + if errors.Unwrap(err) != nil { + t.Fatal("redacted error has Unwrap") + } + if r.error(nil) != nil { + t.Fatal("nil must stay nil") + } +} + +func TestRedactorClass(t *testing.T) { + r := newRedactor(fixtureCredential()) + for _, tc := range []struct { + name string + err error + status int + want forge.ErrorClass + }{ + {"deadline", context.DeadlineExceeded, 0, forge.ErrorClassTimeout}, + {"net timeout", timeoutErr{}, 0, forge.ErrorClassTimeout}, + {"401", errors.New("x"), 401, forge.ErrorClassAuth}, + {"403", errors.New("x"), 403, forge.ErrorClassAuth}, + {"429", errors.New("x"), 429, forge.ErrorClassRateLimited}, + {"500", errors.New("x"), 500, forge.ErrorClassServerError}, + {"503", errors.New("x"), 503, forge.ErrorClassServerError}, + {"404", errors.New("x"), 404, forge.ErrorClassOther}, + {"plain", errors.New("x"), 0, forge.ErrorClassOther}, + } { + t.Run(tc.name, func(t *testing.T) { + if got := forge.Class(r.errorStatus(tc.err, tc.status)); got != tc.want { + t.Fatalf("Class = %q, want %q", got, tc.want) + } + }) + } +} + +func TestRedactorKeepsOversizeProvenance(t *testing.T) { + r := newRedactor(fixtureCredential()) + err := r.error(&forge.ResponseTooLargeError{Provenance: forge.ResponseIssueSuccess}) + if !forge.IsIssueResponseTooLarge(err) || !errors.Is(err, forge.ErrResponseTooLarge) { + t.Fatalf("oversize identity lost: %v", err) + } +} + +func TestRedactorSentinelIdentifiableAndRedacted(t *testing.T) { + cred := fixtureCredential() + r := newRedactor(cred) + err := r.sentinel(ErrRedirectRefused, 302, "to "+cred) + if !errors.Is(err, ErrRedirectRefused) { + t.Fatal("sentinel not identifiable") + } + if strings.Contains(err.Error(), cred) { + t.Fatalf("token leaked: %q", err.Error()) + } + if errors.Is(err, ErrProjectNotBound) { + t.Fatal("matches an unrelated sentinel") + } + if got := errors.Unwrap(err); got != ErrRedirectRefused { + t.Fatalf("Unwrap = %v, want only the sentinel", got) + } + if forge.Class(err) != forge.ErrorClassOther { + t.Fatalf("class = %q", forge.Class(err)) + } +} + +func TestValidateToken(t *testing.T) { + if validateToken(strings.Repeat("a", minTokenLen-1)) == nil { + t.Fatal("short token accepted") + } + if err := validateToken(strings.Repeat("a", minTokenLen)); err != nil { + t.Fatal(err) + } + short := strings.Repeat("q", minTokenLen-1) + if strings.Contains(validateToken(short).Error(), short) { + t.Fatal("error leaks token") + } +} diff --git a/api/internal/issuetracker/registry.go b/api/internal/issuetracker/registry.go new file mode 100644 index 0000000000..6d3b1dcd6d --- /dev/null +++ b/api/internal/issuetracker/registry.go @@ -0,0 +1,19 @@ +package issuetracker + +import "fmt" + +// Constructor builds a driver from options. +type Constructor func(Options) (Driver, error) + +// constructors maps each Kind to its driver constructor. A driver adds one +// entry here; there is no init-time registration. +var constructors = map[Kind]Constructor{} + +// New constructs the driver for kind. An unknown kind wraps ErrUnsupportedKind. +func New(kind Kind, opts Options) (Driver, error) { + ctor, ok := constructors[kind] + if !ok { + return nil, fmt.Errorf("%w: %q (supported: %s)", ErrUnsupportedKind, string(kind), SupportedKindsList()) + } + return ctor(opts) +} diff --git a/api/internal/issuetracker/registry_test.go b/api/internal/issuetracker/registry_test.go new file mode 100644 index 0000000000..7fc53eddb8 --- /dev/null +++ b/api/internal/issuetracker/registry_test.go @@ -0,0 +1,16 @@ +package issuetracker + +import ( + "errors" + "testing" +) + +func TestNewUnknownKind(t *testing.T) { + d, err := New(Kind("nope"), Options{}) + if d != nil { + t.Fatalf("driver = %v, want nil", d) + } + if !errors.Is(err, ErrUnsupportedKind) { + t.Fatalf("err = %v, want ErrUnsupportedKind", err) + } +} From 07bc4f646897287276058f3a6789826280b30524 Mon Sep 17 00:00:00 2001 From: uzi-agent Date: Wed, 11 Nov 2026 08:38:43 +0000 Subject: [PATCH 02/17] issuetracker: close the m1 review findings Total byte budget across pages, the full control and format character predicate, every encoded form of the token and its secret segment, status class on oversize errors, no package-level test transport, and a redaction test that fails when any one form is dropped. Co-Authored-By: Claude Opus 5.5 --- api/internal/issuetracker/bounded.go | 16 ++- api/internal/issuetracker/bounded_test.go | 37 +++++- api/internal/issuetracker/errors.go | 4 +- api/internal/issuetracker/httpclient.go | 21 ++-- api/internal/issuetracker/httpclient_test.go | 8 +- api/internal/issuetracker/pagination.go | 22 ++-- api/internal/issuetracker/pagination_test.go | 37 +++++- api/internal/issuetracker/redact.go | 110 +++++++++++++---- api/internal/issuetracker/redact_test.go | 121 ++++++++++++++----- api/internal/issuetracker/registry_test.go | 21 ++++ 10 files changed, 300 insertions(+), 97 deletions(-) diff --git a/api/internal/issuetracker/bounded.go b/api/internal/issuetracker/bounded.go index c8568db667..5960c8d8e8 100644 --- a/api/internal/issuetracker/bounded.go +++ b/api/internal/issuetracker/bounded.go @@ -16,15 +16,19 @@ const ( ) // readBounded reads at most limit bytes of r. A longer body returns a -// *forge.ResponseTooLargeError carrying prov and no remote content. The limit -// counts bytes after transparent decompression, which is what r yields. -func readBounded(r io.Reader, limit int64, prov forge.ResponseProvenance) ([]byte, error) { +// *forge.ResponseTooLargeError carrying prov, an Outcome classified from the +// HTTP status of the response, and no remote content. Overflow is checked +// before the read error so an oversize stays identifiable. The caller closes +// the body; readBounded does not. The limit counts bytes after transparent +// decompression, which is what r yields, deliberately (forge instead forces +// identity encoding). +func readBounded(r io.Reader, limit int64, prov forge.ResponseProvenance, status int) ([]byte, error) { body, err := io.ReadAll(io.LimitReader(r, limit+1)) + if int64(len(body)) > limit { + return nil, &forge.ResponseTooLargeError{Provenance: prov, Outcome: classifyStatus(status)} + } if err != nil { return nil, err } - if int64(len(body)) > limit { - return nil, &forge.ResponseTooLargeError{Provenance: prov, Outcome: forge.ErrorClassOther} - } return body, nil } diff --git a/api/internal/issuetracker/bounded_test.go b/api/internal/issuetracker/bounded_test.go index 3046b03e9c..82d818a1b8 100644 --- a/api/internal/issuetracker/bounded_test.go +++ b/api/internal/issuetracker/bounded_test.go @@ -4,27 +4,30 @@ import ( "bytes" "compress/gzip" "errors" + "fmt" + "io" "net/http" "net/http/httptest" "strings" "testing" + "testing/iotest" "github.com/vtmocanu/uzi/api/internal/forge" ) func TestReadBounded(t *testing.T) { - got, err := readBounded(strings.NewReader("hello"), 5, forge.ResponseSuccess) + got, err := readBounded(strings.NewReader("hello"), 5, forge.ResponseSuccess, 200) if err != nil || string(got) != "hello" { t.Fatalf("at limit: %q, %v", got, err) } - _, err = readBounded(strings.NewReader("hello!"), 5, forge.ResponseIssueSuccess) + _, err = readBounded(strings.NewReader("hello!"), 5, forge.ResponseIssueSuccess, 200) if !errors.Is(err, forge.ErrResponseTooLarge) { t.Fatalf("err = %v, want ErrResponseTooLarge", err) } if !forge.IsIssueResponseTooLarge(err) { t.Fatal("provenance not issue success") } - _, err = readBounded(strings.NewReader("hello!"), 5, forge.ResponseError) + _, err = readBounded(strings.NewReader("hello!"), 5, forge.ResponseError, 200) if forge.IsIssueResponseTooLarge(err) { t.Fatal("error provenance reported as issue success") } @@ -48,16 +51,38 @@ func TestReadBoundedCapsDecompressedBytes(t *testing.T) { })) defer srv.Close() - resp, err := newHTTPClient(0).Get(srv.URL) + resp, err := newHTTPClient(0, nil).Get(srv.URL) if err != nil { t.Fatal(err) } defer resp.Body.Close() - if resp.ContentLength >= 64<<10 && resp.ContentLength != -1 { + if resp.ContentLength != -1 { t.Fatalf("declared length %d", resp.ContentLength) } - _, err = readBounded(resp.Body, 1<<10, forge.ResponseIssueSuccess) + _, err = readBounded(resp.Body, 1<<10, forge.ResponseIssueSuccess, 200) if !forge.IsIssueResponseTooLarge(err) { t.Fatalf("decompressed overflow not capped: %v", err) } } + +func TestReadBoundedOversizeClassFromStatus(t *testing.T) { + _, err := readBounded(strings.NewReader("hello!"), 5, forge.ResponseError, 503) + if got := forge.Class(err); got != forge.ErrorClassServerError { + t.Fatalf("Class = %q, want server_error", got) + } + r := newRedactor(fixtureCredential()) + if got := forge.Class(r.errorStatus(&forge.ResponseTooLargeError{Provenance: forge.ResponseError}, 503)); got != forge.ErrorClassServerError { + t.Fatalf("redacted Class = %q, want server_error", got) + } + bare := fmt.Errorf("x: %w", forge.ErrResponseTooLarge) + if got := r.errorStatus(bare, 429); !errors.Is(got, forge.ErrResponseTooLarge) || forge.Class(got) != forge.ErrorClassRateLimited { + t.Fatalf("bare oversize: %v class %q", got, forge.Class(got)) + } +} + +func TestReadBoundedOverflowBeatsReadError(t *testing.T) { + r := io.MultiReader(strings.NewReader("hello!"), iotest.ErrReader(errors.New("late"))) + if _, err := readBounded(r, 5, forge.ResponseIssueSuccess, 200); !forge.IsIssueResponseTooLarge(err) { + t.Fatalf("err = %v", err) + } +} diff --git a/api/internal/issuetracker/errors.go b/api/internal/issuetracker/errors.go index 2d76aaf3ba..1f28a8f9ab 100644 --- a/api/internal/issuetracker/errors.go +++ b/api/internal/issuetracker/errors.go @@ -5,8 +5,8 @@ import "errors" var ( // ErrUnsupportedKind is returned by New for a Kind with no driver. ErrUnsupportedKind = errors.New("issuetracker: unsupported kind") - // ErrProjectNotBound is returned when a project-scoped call names a project - // other than the one the connection is bound to. + // ErrProjectNotBound is returned by an IssueTracker method on a driver built + // with no bound project (Options.Project empty). ErrProjectNotBound = errors.New("issuetracker: project not bound to this connection") // ErrStateMappingUnset is returned when open/closed state mapping is needed // but the connection has not configured it. diff --git a/api/internal/issuetracker/httpclient.go b/api/internal/issuetracker/httpclient.go index f964f1dc24..b381991375 100644 --- a/api/internal/issuetracker/httpclient.go +++ b/api/internal/issuetracker/httpclient.go @@ -10,24 +10,19 @@ import ( // defaultTimeout bounds one request when Options.Timeout is zero. const defaultTimeout = 15 * time.Second -// testTransport, when non-nil, replaces the production transport. It is nil -// outside tests: export_test.go sets it to trust an httptest server's roots, -// which Options deliberately cannot express. -var testTransport http.RoundTripper - -// newHTTPClient builds the client a driver uses: an independent transport, a -// finite timeout, and no redirect following, so a Location header cannot carry -// the token to another host. -func newHTTPClient(timeout time.Duration) *http.Client { +// newHTTPClient builds the client a driver uses: a finite timeout and no +// redirect following, so a Location header cannot carry the token to another +// host. A nil base means an independent httptransport.New(http.DefaultTransport); +// a driver sets base only through a test-only constructor helper. +func newHTTPClient(timeout time.Duration, base http.RoundTripper) *http.Client { if timeout <= 0 { timeout = defaultTimeout } - var rt http.RoundTripper = httptransport.New(http.DefaultTransport) - if testTransport != nil { - rt = testTransport + if base == nil { + base = httptransport.New(http.DefaultTransport) } return &http.Client{ - Transport: rt, + Transport: base, Timeout: timeout, CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse diff --git a/api/internal/issuetracker/httpclient_test.go b/api/internal/issuetracker/httpclient_test.go index f8f5a11770..a6383993ea 100644 --- a/api/internal/issuetracker/httpclient_test.go +++ b/api/internal/issuetracker/httpclient_test.go @@ -19,7 +19,7 @@ func TestClientDoesNotFollowRedirect(t *testing.T) { })) defer first.Close() - resp, err := newHTTPClient(0).Get(first.URL) + resp, err := newHTTPClient(0, nil).Get(first.URL) if err != nil { t.Fatal(err) } @@ -49,7 +49,7 @@ func TestClientNotModifiedWithoutLocation(t *testing.T) { w.WriteHeader(http.StatusNotModified) })) defer srv.Close() - resp, err := newHTTPClient(0).Get(srv.URL) + resp, err := newHTTPClient(0, nil).Get(srv.URL) if err != nil { t.Fatal(err) } @@ -60,10 +60,10 @@ func TestClientNotModifiedWithoutLocation(t *testing.T) { } func TestClientTimeout(t *testing.T) { - if got := newHTTPClient(0).Timeout; got != defaultTimeout { + if got := newHTTPClient(0, nil).Timeout; got != defaultTimeout { t.Fatalf("default timeout = %v", got) } - if got := newHTTPClient(3 * time.Second).Timeout; got != 3*time.Second { + if got := newHTTPClient(3*time.Second, nil).Timeout; got != 3*time.Second { t.Fatalf("timeout = %v", got) } } diff --git a/api/internal/issuetracker/pagination.go b/api/internal/issuetracker/pagination.go index 5e27ef1433..64e0445ad4 100644 --- a/api/internal/issuetracker/pagination.go +++ b/api/internal/issuetracker/pagination.go @@ -5,26 +5,34 @@ import "errors" // pageSize is the $top of every list request. const pageSize = 100 -// maxItems and maxPages are sanity ceilings against a server that never ends a -// listing. Vars so tests can lower them. +// maxItems, maxPages and maxTotalBytes are sanity ceilings against a server +// that never ends a listing. Vars so tests can lower them. var ( - maxItems = 1_000_000 - maxPages = 20_000 + maxItems = 100_000 + maxPages = 2_000 + maxTotalBytes = 64 << 20 ) // paginate calls fetch with a growing skip until a short page. A positive limit // stops once that many items are collected and truncates to it; zero means no -// cap. Exceeding maxItems or maxPages is an error carrying no remote content. -func paginate[T any](fetch func(skip, top int) ([]T, error), limit int) ([]T, error) { +// cap. fetch returns the page and its raw byte count; the counts are summed +// across pages. Exceeding maxItems, maxPages or maxTotalBytes is an error +// carrying no remote content. +func paginate[T any](fetch func(skip, top int) ([]T, int, error), limit int) ([]T, error) { var all []T + total := 0 for page := 0; ; page++ { if page >= maxPages { return nil, errors.New("issuetracker: listing exceeds the page ceiling") } - items, err := fetch(len(all), pageSize) + items, n, err := fetch(len(all), pageSize) if err != nil { return nil, err } + total += n + if total > maxTotalBytes { + return nil, errors.New("issuetracker: listing exceeds the byte ceiling") + } all = append(all, items...) if limit > 0 && len(all) >= limit { return all[:limit], nil diff --git a/api/internal/issuetracker/pagination_test.go b/api/internal/issuetracker/pagination_test.go index ce5769d93a..57e97cccd5 100644 --- a/api/internal/issuetracker/pagination_test.go +++ b/api/internal/issuetracker/pagination_test.go @@ -6,14 +6,14 @@ import ( ) // serve returns a fetch over total synthetic items, recording each call. -func serve(total int, calls *[][2]int) func(skip, top int) ([]int, error) { - return func(skip, top int) ([]int, error) { +func serve(total int, calls *[][2]int) func(skip, top int) ([]int, int, error) { + return func(skip, top int) ([]int, int, error) { *calls = append(*calls, [2]int{skip, top}) var out []int for i := skip; i < skip+top && i < total; i++ { out = append(out, i) } - return out, nil + return out, len(out), nil } } @@ -46,7 +46,7 @@ func TestPaginateLimit(t *testing.T) { func TestPaginateFetchError(t *testing.T) { want := errors.New("boom") - _, err := paginate(func(int, int) ([]int, error) { return nil, want }, 0) + _, err := paginate(func(int, int) ([]int, int, error) { return nil, 0, want }, 0) if !errors.Is(err, want) { t.Fatalf("err = %v", err) } @@ -74,4 +74,33 @@ func TestPaginateCaps(t *testing.T) { t.Fatalf("calls = %d", len(calls)) } }) + t.Run("bytes", func(t *testing.T) { + old := maxTotalBytes + maxTotalBytes = 1000 + t.Cleanup(func() { maxTotalBytes = old }) + calls := 0 + full := func(skip, top int) ([]int, int, error) { + calls++ + return make([]int, top), 400, nil + } + if _, err := paginate(full, 0); err == nil { + t.Fatal("expected byte ceiling error") + } + if calls != 3 { + t.Fatalf("calls = %d, want 3 (stop once the budget is crossed)", calls) + } + }) + t.Run("server ignoring skip", func(t *testing.T) { + calls := 0 + same := func(skip, top int) ([]int, int, error) { + calls++ + return make([]int, top), 1 << 20, nil + } + if _, err := paginate(same, 0); err == nil { + t.Fatal("expected an error") + } + if calls > maxPages { + t.Fatalf("calls = %d", calls) + } + }) } diff --git a/api/internal/issuetracker/redact.go b/api/internal/issuetracker/redact.go index d1bf08b5cd..b84949a161 100644 --- a/api/internal/issuetracker/redact.go +++ b/api/internal/issuetracker/redact.go @@ -8,6 +8,7 @@ import ( "net/http" "net/url" "strings" + "unicode" "unicode/utf8" "github.com/vtmocanu/uzi/api/internal/forge" @@ -41,42 +42,80 @@ type redactor struct { } // newRedactor lists the token's raw, query-escaped, path-escaped and -// JSON-escaped forms, plus its final '.'-separated segment when long enough -// (a permanent token has the shape prefix:id.secret). +// JSON-escaped forms (percent-hex in both cases, JSON with and without "\/"), +// plus the same escaped forms of its final '.'-separated segment when long +// enough (a permanent token has the shape prefix:id.secret). The token's own +// forms are always listed whatever its length; minSegmentLen applies to the +// segment only. The list has no duplicates. func newRedactor(token string) redactor { var secrets []string + seen := map[string]bool{} add := func(s string) { - if s != "" && len(s) >= minSegmentLen { + if s != "" && !seen[s] { + seen[s] = true secrets = append(secrets, s) } } - add(token) - add(url.QueryEscape(token)) - add(url.PathEscape(token)) - if b, err := json.Marshal(token); err == nil && len(b) >= 2 { - add(string(b[1 : len(b)-1])) + addForms := func(v string) { + add(v) + q, p := url.QueryEscape(v), url.PathEscape(v) + add(q) + add(lowerPercent(q)) + add(p) + add(lowerPercent(p)) + if b, err := json.Marshal(v); err == nil && len(b) >= 2 { + j := string(b[1 : len(b)-1]) + add(j) + add(strings.ReplaceAll(j, "/", `\/`)) + } } - if i := strings.LastIndexByte(token, '.'); i >= 0 { - add(token[i+1:]) + addForms(token) + if i := strings.LastIndexByte(token, '.'); i >= 0 && len(token)-i-1 >= minSegmentLen { + addForms(token[i+1:]) } return redactor{secrets: secrets} } -// string scrubs secrets, strips control and bidi characters, and truncates. -func (r redactor) string(s string) string { +// lowerPercent lower-cases the hex digits of every %XX escape in s. +func lowerPercent(s string) string { + b := []byte(s) + for i := 0; i+2 < len(b); i++ { + if b[i] == '%' { + b[i+1] = lowerHex(b[i+1]) + b[i+2] = lowerHex(b[i+2]) + i += 2 + } + } + return string(b) +} + +func lowerHex(c byte) byte { + if c >= 'A' && c <= 'F' { + return c + ('a' - 'A') + } + return c +} + +// scrub replaces every listed secret form in s. +func (r redactor) scrub(s string) string { for _, secret := range r.secrets { s = strings.ReplaceAll(s, secret, redactPlaceholder) } - return truncate(sanitize(s), maxMessageBytes) + return s +} + +// string scrubs secrets, strips control and format characters, scrubs again +// (stripping can join fragments of a secret), and truncates. +func (r redactor) string(s string) string { + return truncate(r.scrub(sanitize(r.scrub(s))), maxMessageBytes) } -// sanitize replaces C0/C1 control characters and bidi overrides with a space. +// sanitize replaces control characters, Unicode format characters (bidi +// controls, zero-width characters, BOM) and the line and paragraph separators +// with a space. It uses the predicate of issueinput.AuthorName. func sanitize(s string) string { return strings.Map(func(c rune) rune { - switch { - case c < 0x20, c >= 0x7f && c <= 0x9f, - c >= 0x202a && c <= 0x202e, - c >= 0x2066 && c <= 0x2069: + if unicode.IsControl(c) || unicode.Is(unicode.Cf, c) || c == '\u2028' || c == '\u2029' { return ' ' } return c @@ -128,7 +167,14 @@ func (r redactor) errorStatus(err error, status int) error { } var large *forge.ResponseTooLargeError if errors.As(err, &large) { - return &forge.ResponseTooLargeError{Provenance: large.Provenance, Outcome: large.Class()} + outcome := large.Class() + if outcome == forge.ErrorClassOther && status != 0 { + outcome = classifyStatus(status) + } + return &forge.ResponseTooLargeError{Provenance: large.Provenance, Outcome: outcome} + } + if errors.Is(err, forge.ErrResponseTooLarge) { + return &forge.ResponseTooLargeError{Outcome: classifyStatus(status)} } return &classifiedError{message: r.string(err.Error()), class: classify(err, status)} } @@ -136,16 +182,34 @@ func (r redactor) errorStatus(err error, status int) error { // sentinel returns a redacted error for sentinel with extra context in its // message. errors.Is(result, sentinel) holds; the chain exposes nothing else. func (r redactor) sentinel(sentinel error, status int, detail string) error { - msg := sentinel.Error() - if detail != "" { - msg += ": " + detail + if !isPackageSentinel(sentinel) { + return &classifiedError{message: r.string(sentinel.Error() + detailSuffix(detail)), class: classifyStatus(status)} } + msg := sentinel.Error() + detailSuffix(detail) return &sentinelError{ classifiedError: classifiedError{message: r.string(msg), class: classifyStatus(status)}, sentinel: sentinel, } } +func detailSuffix(detail string) string { + if detail == "" { + return "" + } + return ": " + detail +} + +// isPackageSentinel reports whether err is one of this package's fixed +// sentinels; only those may stay in an error chain. +func isPackageSentinel(err error) bool { + for _, s := range []error{ErrUnsupportedKind, ErrProjectNotBound, ErrStateMappingUnset, ErrInvalidLabelName, ErrRedirectRefused} { + if err == s { + return true + } + } + return false +} + func classify(err error, status int) forge.ErrorClass { if errors.Is(err, context.DeadlineExceeded) { return forge.ErrorClassTimeout @@ -166,7 +230,7 @@ func classifyStatus(status int) forge.ErrorClass { return forge.ErrorClassAuth case status == http.StatusTooManyRequests: return forge.ErrorClassRateLimited - case status >= 500: + case status >= 500 && status < 600: return forge.ErrorClassServerError } return forge.ErrorClassOther diff --git a/api/internal/issuetracker/redact_test.go b/api/internal/issuetracker/redact_test.go index fe52eb00d2..b8b0338dca 100644 --- a/api/internal/issuetracker/redact_test.go +++ b/api/internal/issuetracker/redact_test.go @@ -2,10 +2,8 @@ package issuetracker import ( "context" - "encoding/json" "errors" "fmt" - "net/url" "strings" "testing" @@ -25,37 +23,62 @@ func (timeoutErr) Error() string { return "i/o timeout" } func (timeoutErr) Timeout() bool { return true } func (timeoutErr) Temporary() bool { return false } +// scrubsTo asserts r replaces form in a message with exactly one placeholder +// and nothing else, so removing any single encoding from newRedactor fails. +func scrubsTo(t *testing.T, r redactor, form string) { + t.Helper() + if got, want := r.error(fmt.Errorf("boom %s end", form)).Error(), "boom [REDACTED] end"; got != want { + t.Fatalf("got %q, want %q", got, want) + } +} + +// TestRedactorScrubsEveryEncoding uses a token without '.', so no segment +// scrub can mask a missing whole-token form. func TestRedactorScrubsEveryEncoding(t *testing.T) { - cred := fixtureCredential() + cred := strings.Join([]string{"perm", ":", "ab c/d\"e", "Zq9xLm2VnB7wKp"}, "") r := newRedactor(cred) - segment := cred[strings.LastIndexByte(cred, '.')+1:] - jsonForm := strings.Trim(mustJSON(t, cred), `"`) for name, form := range map[string]string{ - "raw": cred, - "query": url.QueryEscape(cred), - "path": url.PathEscape(cred), - "json": jsonForm, - "segment": segment, + "raw": cred, + "query": "perm%3Aab+c%2Fd%22eZq9xLm2VnB7wKp", + "query lower": "perm%3aab+c%2fd%22eZq9xLm2VnB7wKp", + "path": "perm:ab%20c%2Fd%22eZq9xLm2VnB7wKp", + "path lower": "perm:ab%20c%2fd%22eZq9xLm2VnB7wKp", + "json": `perm:ab c/d\"eZq9xLm2VnB7wKp`, + "json slash": `perm:ab c\/d\"eZq9xLm2VnB7wKp`, } { - t.Run(name, func(t *testing.T) { - got := r.error(fmt.Errorf("boom %s end", form)).Error() - if strings.Contains(got, form) { - t.Fatalf("message still contains %s form: %q", name, got) - } - if !strings.Contains(got, redactPlaceholder) { - t.Fatalf("no placeholder in %q", got) - } - }) + t.Run(name, func(t *testing.T) { scrubsTo(t, r, form) }) } } -func mustJSON(t *testing.T, s string) string { - t.Helper() - b, err := json.Marshal(s) - if err != nil { - t.Fatal(err) +// TestRedactorScrubsEverySegmentEncoding covers the final '.'-separated +// segment in each form; the segment holds characters each escaper rewrites. +func TestRedactorScrubsEverySegmentEncoding(t *testing.T) { + r := newRedactor(strings.Join([]string{"perm:id", ".", "Zq9/xLm 2VnB7w\"Kp"}, "")) + for name, form := range map[string]string{ + "raw": "Zq9/xLm 2VnB7w\"Kp", + "query": "Zq9%2FxLm+2VnB7w%22Kp", + "query lower": "Zq9%2fxLm+2VnB7w%22Kp", + "path": "Zq9%2FxLm%202VnB7w%22Kp", + "path lower": "Zq9%2fxLm%202VnB7w%22Kp", + "json": `Zq9/xLm 2VnB7w\"Kp`, + "json slash": `Zq9\/xLm 2VnB7w\"Kp`, + } { + t.Run(name, func(t *testing.T) { scrubsTo(t, r, form) }) + } +} + +func TestRedactorShortTokenStillScrubbed(t *testing.T) { + r := newRedactor("abcdef") // below minSegmentLen: the raw token is still listed + if got := r.string("x abcdef y"); got != "x [REDACTED] y" { + t.Fatalf("got %q", got) + } +} + +func TestRedactorSecretsDeduped(t *testing.T) { + r := newRedactor(strings.Repeat("a", 20)) + if len(r.secrets) != 1 { + t.Fatalf("secrets = %q, want one entry", r.secrets) } - return string(b) } func TestRedactorShortSegmentKept(t *testing.T) { @@ -65,16 +88,27 @@ func TestRedactorShortSegmentKept(t *testing.T) { } } -func TestRedactorStripsControlAndBidi(t *testing.T) { +func TestRedactorStripsControlAndFormatRunes(t *testing.T) { r := newRedactor(fixtureCredential()) - got := r.string("a\x00b\x1bc\u0085d‮e⁦f⁩g\th") - for _, c := range got { - if c < 0x20 || (c >= 0x7f && c <= 0x9f) || (c >= 0x202a && c <= 0x202e) || (c >= 0x2066 && c <= 0x2069) { - t.Fatalf("unsafe rune %U in %q", c, got) + for _, c := range []rune{ + 0x00, 0x1b, 0x09, 0x7f, 0x85, 0x9f, + 0x200b, 0x200e, 0x200f, 0x061c, 0xfeff, + 0x202a, 0x202b, 0x202c, 0x202d, 0x202e, + 0x2066, 0x2067, 0x2068, 0x2069, 0x2028, 0x2029, + } { + got := r.string("a" + string(c) + "b") + if got != "a b" { + t.Errorf("U+%04X: got %q, want %q", c, got, "a b") } } - if !strings.Contains(got, "a") || !strings.Contains(got, "h") { - t.Fatalf("printable text lost: %q", got) +} + +func TestRedactorScrubsTokenSplitByControlRune(t *testing.T) { + cred := fixtureCredential() + r := newRedactor(cred) + // A raw token has no control runes, so it is scrubbed whole. + if got := r.string("x " + cred + " y"); strings.Contains(got, cred) { + t.Fatalf("leaked: %q", got) } } @@ -160,6 +194,29 @@ func TestRedactorSentinelIdentifiableAndRedacted(t *testing.T) { } } +func TestRedactorSentinelRestrictedToPackageSentinels(t *testing.T) { + r := newRedactor(fixtureCredential()) + foreign := errors.New("foreign") + err := r.sentinel(foreign, 0, "d") + if errors.Is(err, foreign) || errors.Unwrap(err) != nil { + t.Fatalf("foreign error stayed in the chain: %v", err) + } + if err.Error() != "foreign: d" { + t.Fatalf("message = %q", err.Error()) + } +} + +func TestClassifyStatusServerErrorRange(t *testing.T) { + for status, want := range map[int]forge.ErrorClass{ + 499: forge.ErrorClassOther, 500: forge.ErrorClassServerError, + 599: forge.ErrorClassServerError, 600: forge.ErrorClassOther, + } { + if got := classifyStatus(status); got != want { + t.Errorf("classifyStatus(%d) = %q, want %q", status, got, want) + } + } +} + func TestValidateToken(t *testing.T) { if validateToken(strings.Repeat("a", minTokenLen-1)) == nil { t.Fatal("short token accepted") diff --git a/api/internal/issuetracker/registry_test.go b/api/internal/issuetracker/registry_test.go index 7fc53eddb8..106a664b14 100644 --- a/api/internal/issuetracker/registry_test.go +++ b/api/internal/issuetracker/registry_test.go @@ -14,3 +14,24 @@ func TestNewUnknownKind(t *testing.T) { t.Fatalf("err = %v, want ErrUnsupportedKind", err) } } + +// Every constructor key must be a supported kind; the converse holds once a +// driver registers (m2 registers YouTrack), so it is checked only then. +func TestConstructorsMatchSupportedKinds(t *testing.T) { + supported := map[Kind]bool{} + for _, k := range SupportedKinds() { + supported[k] = true + } + for k := range constructors { + if !supported[k] { + t.Errorf("constructor for %q is not in SupportedKinds", k) + } + } + if len(constructors) > 0 { + for k := range supported { + if constructors[k] == nil { + t.Errorf("SupportedKinds entry %q has no constructor", k) + } + } + } +} From 4266f0e318dd11d4c49e1322c7264a9a4caefaac Mon Sep 17 00:00:00 2001 From: uzi-agent Date: Wed, 11 Nov 2026 09:16:43 +0000 Subject: [PATCH 03/17] issuetracker: YouTrack driver, its fake and the contract suite (PRD #4173 M3) The YouTrack driver implements every forge.IssueTracker method plus verify token, list projects and list states, bound to one YouTrack project. Every per-issue call checks the issue is still in that project, so a moved issue reads as gone and is never written to. youtracktest is an in-process HTTPS YouTrack fake that fails the test on any route it does not model, and issuetrackertest.RunContract is the one contract suite every IssueTracker implementation runs. Co-Authored-By: Claude Opus 5.5 --- api/internal/issuetracker/errors.go | 4 + api/internal/issuetracker/export_test.go | 21 + api/internal/issuetracker/httpclient.go | 2 +- .../issuetracker/issuetrackertest/contract.go | 344 +++++++++++++ api/internal/issuetracker/redact.go | 37 +- api/internal/issuetracker/redact_test.go | 57 ++ api/internal/issuetracker/registry.go | 2 +- api/internal/issuetracker/youtrack.go | 117 +++++ api/internal/issuetracker/youtrack_auth.go | 25 + .../issuetracker/youtrack_comments.go | 72 +++ .../issuetracker/youtrack_contract_test.go | 44 ++ api/internal/issuetracker/youtrack_events.go | 51 ++ .../issuetracker/youtrack_issue_resolve.go | 68 +++ api/internal/issuetracker/youtrack_issues.go | 129 +++++ api/internal/issuetracker/youtrack_labels.go | 178 +++++++ .../issuetracker/youtrack_projects.go | 73 +++ api/internal/issuetracker/youtrack_query.go | 92 ++++ .../issuetracker/youtrack_query_test.go | 188 +++++++ api/internal/issuetracker/youtrack_request.go | 151 ++++++ .../issuetracker/youtrack_security_test.go | 209 ++++++++ api/internal/issuetracker/youtrack_state.go | 47 ++ api/internal/issuetracker/youtrack_test.go | 55 ++ api/internal/issuetracker/youtrack_wire.go | 120 +++++ .../issuetracker/youtracktest/fields.go | 82 +++ .../issuetracker/youtracktest/fields_test.go | 31 ++ .../issuetracker/youtracktest/routes.go | 485 ++++++++++++++++++ .../issuetracker/youtracktest/seed.go | 93 ++++ .../issuetracker/youtracktest/server.go | 107 ++++ .../issuetracker/youtracktest/state.go | 271 ++++++++++ 29 files changed, 3150 insertions(+), 5 deletions(-) create mode 100644 api/internal/issuetracker/export_test.go create mode 100644 api/internal/issuetracker/issuetrackertest/contract.go create mode 100644 api/internal/issuetracker/youtrack.go create mode 100644 api/internal/issuetracker/youtrack_auth.go create mode 100644 api/internal/issuetracker/youtrack_comments.go create mode 100644 api/internal/issuetracker/youtrack_contract_test.go create mode 100644 api/internal/issuetracker/youtrack_events.go create mode 100644 api/internal/issuetracker/youtrack_issue_resolve.go create mode 100644 api/internal/issuetracker/youtrack_issues.go create mode 100644 api/internal/issuetracker/youtrack_labels.go create mode 100644 api/internal/issuetracker/youtrack_projects.go create mode 100644 api/internal/issuetracker/youtrack_query.go create mode 100644 api/internal/issuetracker/youtrack_query_test.go create mode 100644 api/internal/issuetracker/youtrack_request.go create mode 100644 api/internal/issuetracker/youtrack_security_test.go create mode 100644 api/internal/issuetracker/youtrack_state.go create mode 100644 api/internal/issuetracker/youtrack_test.go create mode 100644 api/internal/issuetracker/youtrack_wire.go create mode 100644 api/internal/issuetracker/youtracktest/fields.go create mode 100644 api/internal/issuetracker/youtracktest/fields_test.go create mode 100644 api/internal/issuetracker/youtracktest/routes.go create mode 100644 api/internal/issuetracker/youtracktest/seed.go create mode 100644 api/internal/issuetracker/youtracktest/server.go create mode 100644 api/internal/issuetracker/youtracktest/state.go diff --git a/api/internal/issuetracker/errors.go b/api/internal/issuetracker/errors.go index 1f28a8f9ab..e5fc387f12 100644 --- a/api/internal/issuetracker/errors.go +++ b/api/internal/issuetracker/errors.go @@ -17,3 +17,7 @@ var ( // The client never follows one, so the token cannot be forwarded. ErrRedirectRefused = errors.New("issuetracker: redirect refused") ) + +// packageSentinels lists every sentinel above. redactor.sentinel keeps only +// these in an error chain, so a new sentinel must be added here too. +var packageSentinels = []error{ErrUnsupportedKind, ErrProjectNotBound, ErrStateMappingUnset, ErrInvalidLabelName, ErrRedirectRefused} diff --git a/api/internal/issuetracker/export_test.go b/api/internal/issuetracker/export_test.go new file mode 100644 index 0000000000..ec19b9553f --- /dev/null +++ b/api/internal/issuetracker/export_test.go @@ -0,0 +1,21 @@ +package issuetracker + +import "net/http" + +// newYouTrackOver builds a YouTrack driver whose HTTP transport is base, so a +// test can trust the self-signed certificate of a youtracktest server or an +// httptest TLS server. Options carries no TLS field by design. +func newYouTrackOver(opts Options, base http.RoundTripper) (*youTrack, error) { + return buildYouTrack(opts, base) +} + +// NewYouTrackOver is newYouTrackOver for the external test package +// issuetracker_test, which must be external because issuetrackertest imports +// issuetracker. +func NewYouTrackOver(opts Options, base http.RoundTripper) (Driver, error) { + y, err := buildYouTrack(opts, base) + if err != nil { + return nil, err + } + return y, nil +} diff --git a/api/internal/issuetracker/httpclient.go b/api/internal/issuetracker/httpclient.go index b381991375..dbe63ed12d 100644 --- a/api/internal/issuetracker/httpclient.go +++ b/api/internal/issuetracker/httpclient.go @@ -13,7 +13,7 @@ const defaultTimeout = 15 * time.Second // newHTTPClient builds the client a driver uses: a finite timeout and no // redirect following, so a Location header cannot carry the token to another // host. A nil base means an independent httptransport.New(http.DefaultTransport); -// a driver sets base only through a test-only constructor helper. +// only the test-only constructor in export_test.go sets base. func newHTTPClient(timeout time.Duration, base http.RoundTripper) *http.Client { if timeout <= 0 { timeout = defaultTimeout diff --git a/api/internal/issuetracker/issuetrackertest/contract.go b/api/internal/issuetracker/issuetrackertest/contract.go new file mode 100644 index 0000000000..782b0bde72 --- /dev/null +++ b/api/internal/issuetracker/issuetrackertest/contract.go @@ -0,0 +1,344 @@ +package issuetrackertest + +import ( + "context" + "errors" + "fmt" + "slices" + "sync/atomic" + "testing" + + "github.com/vtmocanu/uzi/api/internal/forge" +) + +// Subject is one implementation of forge.IssueTracker under contract test, +// with the seams the suite needs to put state in it that the interface cannot. +type Subject struct { + IssueTracker forge.IssueTracker + // ProjectID is passed as the projectID argument of every call. + ProjectID int64 + // SeedHumanComment adds a comment by username on issue iid, bypassing the + // IssueTracker under test. + SeedHumanComment func(t *testing.T, iid int64, username, body string) + // SeedLabelEvent records that username added ("add") or removed ("remove") + // label on issue iid, bypassing the IssueTracker under test. + SeedLabelEvent func(t *testing.T, iid int64, action, label, username string) + // MoveIssue moves issue iid to another project. Nil means issues do not + // move on this implementation. + MoveIssue func(t *testing.T, iid int64) +} + +// uniqueSeq makes every name the suite creates unique within the process, so +// a subject whose backing store is shared never sees a collision. +var uniqueSeq atomic.Int64 + +func unique(prefix string) string { + return fmt.Sprintf("%s-%d", prefix, uniqueSeq.Add(1)) +} + +// RunContract runs the shared behaviour of every issue tracker implementation. +// newSubject is called once per case, so a case starts from the state its +// constructor gives. Assertions look for the data the case created and never +// count everything the subject holds, because a subject may hold other data. +func RunContract(t *testing.T, newSubject func(t *testing.T) Subject) { + cases := []struct { + name string + run func(t *testing.T, c *contract) + }{ + {"EnsureLabelsIsIdempotent", caseEnsureLabels}, + {"CreateIssueRoundTrips", caseCreateIssue}, + {"ListIssuesFilters", caseListIssues}, + {"UpdateLabelsAndDescription", caseUpdate}, + {"SetIssueStateClosesAndReopens", caseSetState}, + {"CommentsOldestFirst", caseComments}, + {"LabelEventsOldestFirst", caseLabelEvents}, + {"MissingIssueIsNotFound", caseMissing}, + {"MovedIssueIsRefused", caseMoved}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + s := newSubject(t) + tc.run(t, &contract{Subject: s, ctx: t.Context()}) + }) + } +} + +type contract struct { + Subject + ctx context.Context +} + +func (c *contract) ensure(t *testing.T, names ...string) { + t.Helper() + labels := make([]forge.Label, 0, len(names)) + for _, n := range names { + labels = append(labels, forge.Label{Name: n, Color: "#428bca"}) + } + if err := c.IssueTracker.EnsureLabels(c.ctx, c.ProjectID, labels); err != nil { + t.Fatalf("EnsureLabels(%v): %v", names, err) + } +} + +func (c *contract) create(t *testing.T, description string, labels ...string) forge.Issue { + t.Helper() + issue, err := c.IssueTracker.CreateIssue(c.ctx, c.ProjectID, unique("contract issue"), description, labels) + if err != nil { + t.Fatalf("CreateIssue: %v", err) + } + if issue.IID <= 0 { + t.Fatalf("CreateIssue returned IID %d", issue.IID) + } + return issue +} + +func (c *contract) get(t *testing.T, iid int64) forge.Issue { + t.Helper() + issue, err := c.IssueTracker.GetIssue(c.ctx, c.ProjectID, iid) + if err != nil { + t.Fatalf("GetIssue(%d): %v", iid, err) + } + return issue +} + +// listIIDs returns the sorted IIDs ListIssues finds for opts. +func (c *contract) listIIDs(t *testing.T, opts forge.ListIssuesOptions) []int64 { + t.Helper() + issues, err := c.IssueTracker.ListIssues(c.ctx, c.ProjectID, opts) + if err != nil { + t.Fatalf("ListIssues(%+v): %v", opts, err) + } + iids := make([]int64, 0, len(issues)) + for _, i := range issues { + iids = append(iids, i.IID) + } + slices.Sort(iids) + return iids +} + +func wantIIDs(t *testing.T, got []int64, want ...int64) { + t.Helper() + slices.Sort(want) + if !slices.Equal(got, want) { + t.Fatalf("ListIssues IIDs = %v, want %v", got, want) + } +} + +func sameSet(a, b []string) bool { + a, b = slices.Clone(a), slices.Clone(b) + slices.Sort(a) + slices.Sort(b) + return slices.Equal(a, b) +} + +func caseEnsureLabels(t *testing.T, c *contract) { + a, b := unique("contract-label"), unique("contract-label") + c.ensure(t, a, b) + c.ensure(t, a, b) // idempotent + labels, err := c.IssueTracker.ListLabels(c.ctx, c.ProjectID) + if err != nil { + t.Fatalf("ListLabels: %v", err) + } + count := map[string]int{} + for _, l := range labels { + count[l.Name]++ + } + for _, name := range []string{a, b} { + if count[name] != 1 { + t.Errorf("label %q listed %d times, want once", name, count[name]) + } + } +} + +func caseCreateIssue(t *testing.T, c *contract) { + l1, l2 := unique("contract-label"), unique("contract-label") + c.ensure(t, l1, l2) + created := c.create(t, "the description", l1, l2) + got := c.get(t, created.IID) + if got.IID != created.IID { + t.Errorf("IID = %d, want %d", got.IID, created.IID) + } + if got.Title != created.Title || got.Title == "" { + t.Errorf("Title = %q, created as %q", got.Title, created.Title) + } + if got.Description != "the description" { + t.Errorf("Description = %q", got.Description) + } + if !sameSet(got.Labels, []string{l1, l2}) { + t.Errorf("Labels = %v, want %v and %v", got.Labels, l1, l2) + } + if got.State != string(forge.StateOpened) { + t.Errorf("State = %q, want opened", got.State) + } + if got.WebURL == "" || got.WebURL != created.WebURL { + t.Errorf("WebURL = %q, created as %q", got.WebURL, created.WebURL) + } + if got.Assignees == nil { + t.Error("Assignees is nil, want an empty slice") + } +} + +func caseListIssues(t *testing.T, c *contract) { + l1, l2 := unique("contract-label"), unique("contract-label") + c.ensure(t, l1, l2) + both := c.create(t, "both", l1, l2) + onlyOne := c.create(t, "one", l1) + other := c.create(t, "two", l2) + + wantIIDs(t, c.listIIDs(t, forge.ListIssuesOptions{Labels: []string{l1, l2}}), both.IID) + wantIIDs(t, c.listIIDs(t, forge.ListIssuesOptions{Labels: []string{l1}}), both.IID, onlyOne.IID) + wantIIDs(t, c.listIIDs(t, forge.ListIssuesOptions{Labels: []string{l2}}), both.IID, other.IID) + + if err := c.IssueTracker.SetIssueState(c.ctx, c.ProjectID, onlyOne.IID, forge.StateClosed); err != nil { + t.Fatalf("SetIssueState: %v", err) + } + wantIIDs(t, c.listIIDs(t, forge.ListIssuesOptions{Labels: []string{l1}, State: forge.StateOpened}), both.IID) + wantIIDs(t, c.listIIDs(t, forge.ListIssuesOptions{Labels: []string{l1}, State: forge.StateClosed}), onlyOne.IID) + wantIIDs(t, c.listIIDs(t, forge.ListIssuesOptions{Labels: []string{l1}, State: forge.StateAll}), both.IID, onlyOne.IID) + + limited := c.listIIDs(t, forge.ListIssuesOptions{Labels: []string{l1}, Limit: 1}) + if len(limited) != 1 { + t.Fatalf("Limit 1 returned %d issues", len(limited)) + } +} + +func caseUpdate(t *testing.T, c *contract) { + l1, l2 := unique("contract-label"), unique("contract-label") + c.ensure(t, l1, l2) + issue := c.create(t, "before", l1) + if err := c.IssueTracker.UpdateIssueLabels(c.ctx, c.ProjectID, issue.IID, []string{l2}, []string{l1}); err != nil { + t.Fatalf("UpdateIssueLabels: %v", err) + } + if err := c.IssueTracker.UpdateIssueDescription(c.ctx, c.ProjectID, issue.IID, "after"); err != nil { + t.Fatalf("UpdateIssueDescription: %v", err) + } + got := c.get(t, issue.IID) + if !sameSet(got.Labels, []string{l2}) { + t.Errorf("Labels = %v, want only %v", got.Labels, l2) + } + if got.Description != "after" { + t.Errorf("Description = %q, want after", got.Description) + } +} + +func caseSetState(t *testing.T, c *contract) { + label := unique("contract-label") + c.ensure(t, label) + issue := c.create(t, "state", label) + inList := func(state forge.IssueState) bool { + return slices.Contains(c.listIIDs(t, forge.ListIssuesOptions{Labels: []string{label}, State: state}), issue.IID) + } + if err := c.IssueTracker.SetIssueState(c.ctx, c.ProjectID, issue.IID, forge.StateClosed); err != nil { + t.Fatalf("close: %v", err) + } + if got := c.get(t, issue.IID).State; got != string(forge.StateClosed) { + t.Errorf("State after close = %q", got) + } + if !inList(forge.StateClosed) || inList(forge.StateOpened) { + t.Error("ListIssues state filters disagree with a closed issue") + } + if err := c.IssueTracker.SetIssueState(c.ctx, c.ProjectID, issue.IID, forge.StateOpened); err != nil { + t.Fatalf("reopen: %v", err) + } + if got := c.get(t, issue.IID).State; got != string(forge.StateOpened) { + t.Errorf("State after reopen = %q", got) + } + if inList(forge.StateClosed) || !inList(forge.StateOpened) { + t.Error("ListIssues state filters disagree with a reopened issue") + } +} + +func caseComments(t *testing.T, c *contract) { + issue := c.create(t, "comments") + human, note := unique("human comment"), unique("bot note") + c.SeedHumanComment(t, issue.IID, "alice", human) + created, err := c.IssueTracker.CreateIssueNote(c.ctx, c.ProjectID, issue.IID, note) + if err != nil { + t.Fatalf("CreateIssueNote: %v", err) + } + if created.ID == 0 || created.Body != note { + t.Errorf("CreateIssueNote = %+v, want a non-zero ID and body %q", created, note) + } + comments, err := c.IssueTracker.ListIssueComments(c.ctx, c.ProjectID, issue.IID) + if err != nil { + t.Fatalf("ListIssueComments: %v", err) + } + var humanAt, noteAt = -1, -1 + for i, cm := range comments { + switch cm.Body { + case human: + humanAt = i + if cm.AuthorUsername != "alice" { + t.Errorf("human comment author = %q, want alice", cm.AuthorUsername) + } + case note: + noteAt = i + } + } + if humanAt < 0 || noteAt < 0 { + t.Fatalf("comments %+v are missing the seeded comment or the note", comments) + } + if humanAt > noteAt { + t.Errorf("human comment at %d after the note at %d, want oldest first", humanAt, noteAt) + } +} + +func caseLabelEvents(t *testing.T, c *contract) { + label := unique("contract-label") + c.ensure(t, label) + issue := c.create(t, "events") + c.SeedLabelEvent(t, issue.IID, "add", label, "alice") + c.SeedLabelEvent(t, issue.IID, "remove", label, "bob") + events, err := c.IssueTracker.ListIssueLabelEvents(c.ctx, c.ProjectID, issue.IID) + if err != nil { + t.Fatalf("ListIssueLabelEvents: %v", err) + } + var mine []forge.LabelEvent + for _, e := range events { + if e.LabelName == label { + mine = append(mine, e) + } + } + if len(mine) != 2 { + t.Fatalf("events for %q = %+v, want an add then a remove", label, mine) + } + if mine[0].Action != "add" || mine[0].Username != "alice" { + t.Errorf("first event = %+v, want add by alice", mine[0]) + } + if mine[1].Action != "remove" || mine[1].Username != "bob" { + t.Errorf("second event = %+v, want remove by bob", mine[1]) + } +} + +func caseMissing(t *testing.T, c *contract) { + _, err := c.IssueTracker.GetIssue(c.ctx, c.ProjectID, 987654321) + var notFound *forge.IssueNotFoundError + if !errors.As(err, ¬Found) { + t.Fatalf("GetIssue of a missing IID = %v, want *forge.IssueNotFoundError", err) + } +} + +func caseMoved(t *testing.T, c *contract) { + if c.MoveIssue == nil { + t.Skip("issues do not move on this implementation") + } + issue := c.create(t, "will move") + c.MoveIssue(t, issue.IID) + it, pid, iid := c.IssueTracker, c.ProjectID, issue.IID + calls := map[string]func() error{ + "GetIssue": func() error { _, err := it.GetIssue(c.ctx, pid, iid); return err }, + "UpdateIssueLabels": func() error { + return it.UpdateIssueLabels(c.ctx, pid, iid, []string{unique("contract-label")}, nil) + }, + "UpdateIssueDescription": func() error { return it.UpdateIssueDescription(c.ctx, pid, iid, "x") }, + "SetIssueState": func() error { return it.SetIssueState(c.ctx, pid, iid, forge.StateClosed) }, + "ListIssueLabelEvents": func() error { _, err := it.ListIssueLabelEvents(c.ctx, pid, iid); return err }, + "ListIssueComments": func() error { _, err := it.ListIssueComments(c.ctx, pid, iid); return err }, + "CreateIssueNote": func() error { _, err := it.CreateIssueNote(c.ctx, pid, iid, "x"); return err }, + } + for name, call := range calls { + var notFound *forge.IssueNotFoundError + if err := call(); !errors.As(err, ¬Found) { + t.Errorf("%s on a moved issue = %v, want *forge.IssueNotFoundError", name, err) + } + } +} diff --git a/api/internal/issuetracker/redact.go b/api/internal/issuetracker/redact.go index b84949a161..3389e75070 100644 --- a/api/internal/issuetracker/redact.go +++ b/api/internal/issuetracker/redact.go @@ -89,6 +89,26 @@ func lowerPercent(s string) string { return string(b) } +// upperPercent upper-cases the hex digits of every %XX escape in s. +func upperPercent(s string) string { + b := []byte(s) + for i := 0; i+2 < len(b); i++ { + if b[i] == '%' { + b[i+1] = upperHex(b[i+1]) + b[i+2] = upperHex(b[i+2]) + i += 2 + } + } + return string(b) +} + +func upperHex(c byte) byte { + if c >= 'a' && c <= 'f' { + return c - ('a' - 'A') + } + return c +} + func lowerHex(c byte) byte { if c >= 'A' && c <= 'F' { return c + ('a' - 'A') @@ -96,8 +116,19 @@ func lowerHex(c byte) byte { return c } -// scrub replaces every listed secret form in s. +// scrub replaces every listed secret form in s. A second pass runs over a copy +// whose %XX escapes are upper-cased, so a mixed-case escape such as one that +// reads "%2f...%2F" is caught too; the copy is used only when it matched. func (r redactor) scrub(s string) string { + out := r.scrubForms(s) + canon := upperPercent(out) + if c := r.scrubForms(canon); c != canon { + return c + } + return out +} + +func (r redactor) scrubForms(s string) string { for _, secret := range r.secrets { s = strings.ReplaceAll(s, secret, redactPlaceholder) } @@ -168,7 +199,7 @@ func (r redactor) errorStatus(err error, status int) error { var large *forge.ResponseTooLargeError if errors.As(err, &large) { outcome := large.Class() - if outcome == forge.ErrorClassOther && status != 0 { + if outcome == forge.ErrorClassOther { outcome = classifyStatus(status) } return &forge.ResponseTooLargeError{Provenance: large.Provenance, Outcome: outcome} @@ -202,7 +233,7 @@ func detailSuffix(detail string) string { // isPackageSentinel reports whether err is one of this package's fixed // sentinels; only those may stay in an error chain. func isPackageSentinel(err error) bool { - for _, s := range []error{ErrUnsupportedKind, ErrProjectNotBound, ErrStateMappingUnset, ErrInvalidLabelName, ErrRedirectRefused} { + for _, s := range packageSentinels { if err == s { return true } diff --git a/api/internal/issuetracker/redact_test.go b/api/internal/issuetracker/redact_test.go index b8b0338dca..1d8446869c 100644 --- a/api/internal/issuetracker/redact_test.go +++ b/api/internal/issuetracker/redact_test.go @@ -4,6 +4,9 @@ import ( "context" "errors" "fmt" + "go/ast" + "go/parser" + gotoken "go/token" "strings" "testing" @@ -229,3 +232,57 @@ func TestValidateToken(t *testing.T) { t.Fatal("error leaks token") } } + +// TestRedactorScrubsMixedCaseEscapes covers a percent-escaped token whose hex +// digits are upper case in one escape and lower case in another. +func TestRedactorScrubsMixedCaseEscapes(t *testing.T) { + cred := strings.Join([]string{"perm", ":", "a/b/c", "Zq9xLm2VnB7wKp"}, "") + r := newRedactor(cred) + scrubsTo(t, r, "perm%3Aa%2fb%2Fc"+"Zq9xLm2VnB7wKp") + scrubsTo(t, r, "perm%3aa%2Fb%2fc"+"Zq9xLm2VnB7wKp") +} + +// TestRedactorSecondScrubAfterSanitize pins the scrub that follows sanitize: a +// dotless token holding a space is split by a TAB in the message, which only +// sanitize turns back into the token. +func TestRedactorSecondScrubAfterSanitize(t *testing.T) { + cred := strings.Join([]string{"ab cd", "-efghijklmnop"}, "") + r := newRedactor(cred) + got := r.error(errors.New("x ab\tcd-efghijklmnop y")).Error() + if got != "x [REDACTED] y" { + t.Fatalf("got %q", got) + } +} + +// TestPackageSentinelsMatchErrorsFile fails when errors.go declares an Err +// variable that packageSentinels omits, which would stop errors.Is matching it +// through redactor.sentinel. +func TestPackageSentinelsMatchErrorsFile(t *testing.T) { + file, err := parser.ParseFile(gotoken.NewFileSet(), "errors.go", nil, 0) + if err != nil { + t.Fatal(err) + } + declared := 0 + for _, d := range file.Decls { + gd, ok := d.(*ast.GenDecl) + if !ok || gd.Tok != gotoken.VAR { + continue + } + for _, spec := range gd.Specs { + for _, n := range spec.(*ast.ValueSpec).Names { + if strings.HasPrefix(n.Name, "Err") { + declared++ + } + } + } + } + if declared != len(packageSentinels) { + t.Fatalf("errors.go declares %d Err variables, packageSentinels lists %d", declared, len(packageSentinels)) + } + r := newRedactor(fixtureCredential()) + for _, s := range packageSentinels { + if !errors.Is(r.sentinel(s, 0, "x"), s) { + t.Errorf("sentinel %v lost through redactor.sentinel", s) + } + } +} diff --git a/api/internal/issuetracker/registry.go b/api/internal/issuetracker/registry.go index 6d3b1dcd6d..9bd737517c 100644 --- a/api/internal/issuetracker/registry.go +++ b/api/internal/issuetracker/registry.go @@ -7,7 +7,7 @@ type Constructor func(Options) (Driver, error) // constructors maps each Kind to its driver constructor. A driver adds one // entry here; there is no init-time registration. -var constructors = map[Kind]Constructor{} +var constructors = map[Kind]Constructor{KindYouTrack: newYouTrack} // New constructs the driver for kind. An unknown kind wraps ErrUnsupportedKind. func New(kind Kind, opts Options) (Driver, error) { diff --git a/api/internal/issuetracker/youtrack.go b/api/internal/issuetracker/youtrack.go new file mode 100644 index 0000000000..033ae02dff --- /dev/null +++ b/api/internal/issuetracker/youtrack.go @@ -0,0 +1,117 @@ +package issuetracker + +import ( + "errors" + "fmt" + "net/http" + "net/url" + "regexp" + "strings" +) + +// defaultStateField is the YouTrack custom field that holds the issue state. +const defaultStateField = "State" + +var ( + shortNameRE = regexp.MustCompile(`^[A-Za-z][A-Za-z0-9_]*$`) + // entityIDRE matches a YouTrack entity id such as "2-15". Every id taken + // from a response and placed in a path is checked against it. + entityIDRE = regexp.MustCompile(`^\d+-\d+$`) +) + +// youTrack is the JetBrains YouTrack driver. It is bound to one project when +// built (see ProjectRef): the projectID argument of every forge.IssueTracker +// method is ignored, and a driver with no project bound refuses them with +// ErrProjectNotBound. VerifyToken, ListProjects and ListStates need no binding. +type youTrack struct { + baseURL string // scheme and host, no trailing slash + token string + project ProjectRef + stateField string + openState string + closedState string + client *http.Client + red redactor +} + +var _ Driver = (*youTrack)(nil) + +// newYouTrack is the registry constructor: system roots, default transport. +func newYouTrack(opts Options) (Driver, error) { + return buildYouTrack(opts, nil) +} + +// buildYouTrack validates opts and builds the driver over base, which is nil +// outside tests. +func buildYouTrack(opts Options, base http.RoundTripper) (*youTrack, error) { + baseURL, err := strictBaseURL(opts.BaseURL) + if err != nil { + return nil, err + } + if err := validateToken(opts.Token); err != nil { + return nil, err + } + if opts.Project != (ProjectRef{}) { + if !shortNameRE.MatchString(opts.Project.ShortName) { + return nil, errors.New("issuetracker: invalid project short name") + } + if !entityIDRE.MatchString(opts.Project.ID) { + return nil, errors.New("issuetracker: invalid project id") + } + } + stateField := opts.StateField + if stateField == "" { + stateField = defaultStateField + } + return &youTrack{ + baseURL: baseURL, + token: opts.Token, + project: opts.Project, + stateField: stateField, + openState: opts.OpenState, + closedState: opts.ClosedState, + client: newHTTPClient(opts.Timeout, base), + red: newRedactor(opts.Token), + }, nil +} + +// strictBaseURL accepts only https://host[:port] with an optional single +// trailing slash. A path, userinfo, query or fragment is rejected rather than +// stripped, so the URL the token is sent to is exactly the one that was +// validated. +func strictBaseURL(raw string) (string, error) { + bad := func(why string) (string, error) { + return "", fmt.Errorf("issuetracker: base URL %s", why) + } + if strings.ContainsAny(raw, "?#") { + return bad("must not carry a query or fragment") + } + u, err := url.Parse(raw) + if err != nil { + return bad("is not a valid URL") + } + switch { + case u.Scheme != "https": + return bad("must use https") + case u.Opaque != "" || u.Hostname() == "": + return bad("must name a host") + case u.User != nil: + return bad("must not carry credentials") + case u.Path != "" && u.Path != "/": + return bad("must not carry a path") + } + return "https://" + u.Host, nil +} + +// requireProject returns the bound project or the ErrProjectNotBound sentinel. +func (y *youTrack) requireProject() (ProjectRef, error) { + if y.project == (ProjectRef{}) { + return ProjectRef{}, y.red.sentinel(ErrProjectNotBound, 0, "") + } + return y.project, nil +} + +// fail returns a redacted error built from a format. +func (y *youTrack) fail(format string, args ...any) error { + return y.red.error(fmt.Errorf(format, args...)) +} diff --git a/api/internal/issuetracker/youtrack_auth.go b/api/internal/issuetracker/youtrack_auth.go new file mode 100644 index 0000000000..a7cc3ce45c --- /dev/null +++ b/api/internal/issuetracker/youtrack_auth.go @@ -0,0 +1,25 @@ +package issuetracker + +import ( + "context" + "net/http" + "net/url" + + "github.com/vtmocanu/uzi/api/internal/forge" +) + +// VerifyToken implements Driver. It asks YouTrack who the token belongs to. +func (y *youTrack) VerifyToken(ctx context.Context) (Account, error) { + var u ytUser + _, _, err := y.do(ctx, ytCall{ + method: http.MethodGet, + path: []string{"users", "me"}, + query: url.Values{"fields": {"id,login"}}, + limit: objectBytes, + prov: forge.ResponseSuccess, + }, &u) + if err != nil { + return Account{}, err + } + return Account{ID: u.ID, Login: u.Login}, nil +} diff --git a/api/internal/issuetracker/youtrack_comments.go b/api/internal/issuetracker/youtrack_comments.go new file mode 100644 index 0000000000..c2723c3793 --- /dev/null +++ b/api/internal/issuetracker/youtrack_comments.go @@ -0,0 +1,72 @@ +package issuetracker + +import ( + "context" + "net/http" + "net/url" + "slices" + "time" + + "github.com/vtmocanu/uzi/api/internal/forge" +) + +// ListIssueComments implements forge.IssueTracker: deleted comments are +// dropped and the rest come back oldest first. Author ids are 0 (unresolved), +// because YouTrack account ids are strings. +func (y *youTrack) ListIssueComments(ctx context.Context, _, issueIID int64) ([]forge.IssueComment, error) { + w, err := y.resolveIssue(ctx, issueIID) + if err != nil { + return nil, err + } + comments, err := ytList[ytComment](ctx, y, []string{"issues", w.ID, "comments"}, url.Values{"fields": {commentFields}}) + if err != nil { + return nil, err + } + slices.SortStableFunc(comments, func(a, b ytComment) int { + switch { + case a.Created < b.Created: + return -1 + case a.Created > b.Created: + return 1 + } + return 0 + }) + out := []forge.IssueComment{} + for _, c := range comments { + if c.Deleted { + continue + } + out = append(out, forge.IssueComment{ + AuthorUsername: login(c.Author), + Body: c.Text, + CreatedAt: time.UnixMilli(c.Created).UTC(), + }) + } + return out, nil +} + +// CreateIssueNote implements forge.IssueTracker. ID is the numeric suffix of +// the comment's entity id. +func (y *youTrack) CreateIssueNote(ctx context.Context, _, issueIID int64, body string) (forge.IssueNote, error) { + w, err := y.resolveIssue(ctx, issueIID) + if err != nil { + return forge.IssueNote{}, err + } + var c ytComment + _, _, err = y.do(ctx, ytCall{ + method: http.MethodPost, + path: []string{"issues", w.ID, "comments"}, + query: url.Values{"fields": {"id,text,created,author(id,login)"}}, + body: map[string]string{"text": body}, + limit: objectBytes, + prov: forge.ResponseSuccess, + }, &c) + if err != nil { + return forge.IssueNote{}, err + } + id, err := noteID(c.ID) + if err != nil { + return forge.IssueNote{}, y.red.error(err) + } + return forge.IssueNote{ID: id, Body: c.Text}, nil +} diff --git a/api/internal/issuetracker/youtrack_contract_test.go b/api/internal/issuetracker/youtrack_contract_test.go new file mode 100644 index 0000000000..5388780a94 --- /dev/null +++ b/api/internal/issuetracker/youtrack_contract_test.go @@ -0,0 +1,44 @@ +package issuetracker_test + +import ( + "testing" + + "github.com/vtmocanu/uzi/api/internal/issuetracker" + "github.com/vtmocanu/uzi/api/internal/issuetracker/issuetrackertest" + "github.com/vtmocanu/uzi/api/internal/issuetracker/youtracktest" +) + +// TestYouTrackContract runs the shared issue tracker contract against the +// YouTrack driver over the in-process fake. +func TestYouTrackContract(t *testing.T) { + issuetrackertest.RunContract(t, func(t *testing.T) issuetrackertest.Subject { + srv := youtracktest.NewServer(t) + y, err := issuetracker.NewYouTrackOver(issuetracker.Options{ + BaseURL: srv.URL, + Token: srv.Token, + Project: issuetracker.ProjectRef{ID: youtracktest.ProjectID, ShortName: youtracktest.ProjectShortName}, + OpenState: "Open", + ClosedState: "Fixed", + }, srv.Client().Transport) + if err != nil { + t.Fatal(err) + } + return issuetrackertest.Subject{ + IssueTracker: y, + ProjectID: 1, + SeedHumanComment: func(t *testing.T, iid int64, username, body string) { + srv.SeedComment(iid, username, body) + }, + SeedLabelEvent: func(t *testing.T, iid int64, action, label, username string) { + if action == "add" { + srv.SeedTagActivity(iid, username, []string{label}, nil) + } else { + srv.SeedTagActivity(iid, username, nil, []string{label}) + } + }, + MoveIssue: func(t *testing.T, iid int64) { + srv.MoveIssue(iid, youtracktest.OtherProjectID) + }, + } + }) +} diff --git a/api/internal/issuetracker/youtrack_events.go b/api/internal/issuetracker/youtrack_events.go new file mode 100644 index 0000000000..603edd03b8 --- /dev/null +++ b/api/internal/issuetracker/youtrack_events.go @@ -0,0 +1,51 @@ +package issuetracker + +import ( + "context" + "net/url" + "slices" + "time" + + "github.com/vtmocanu/uzi/api/internal/forge" +) + +// ListIssueLabelEvents implements forge.IssueTracker from the issue's tag +// activities. Events are oldest first, one per added or removed tag; ID is the +// 1-based position in that order. +func (y *youTrack) ListIssueLabelEvents(ctx context.Context, _, issueIID int64) ([]forge.LabelEvent, error) { + w, err := y.resolveIssue(ctx, issueIID) + if err != nil { + return nil, err + } + acts, err := ytList[ytActivity](ctx, y, []string{"issues", w.ID, "activities"}, + url.Values{"categories": {"TagsCategory"}, "fields": {activityFields}}) + if err != nil { + return nil, err + } + slices.SortStableFunc(acts, func(a, b ytActivity) int { + switch { + case a.Timestamp < b.Timestamp: + return -1 + case a.Timestamp > b.Timestamp: + return 1 + } + return 0 + }) + out := []forge.LabelEvent{} + emit := func(a ytActivity, action string, tags []ytTag) { + for _, t := range tags { + out = append(out, forge.LabelEvent{ + ID: int64(len(out) + 1), + Action: action, + LabelName: t.Name, + Username: login(a.Author), + CreatedAt: time.UnixMilli(a.Timestamp).UTC(), + }) + } + } + for _, a := range acts { + emit(a, "add", a.Added) + emit(a, "remove", a.Removed) + } + return out, nil +} diff --git a/api/internal/issuetracker/youtrack_issue_resolve.go b/api/internal/issuetracker/youtrack_issue_resolve.go new file mode 100644 index 0000000000..ae1c6b482c --- /dev/null +++ b/api/internal/issuetracker/youtrack_issue_resolve.go @@ -0,0 +1,68 @@ +package issuetracker + +import ( + "context" + "net/http" + "net/url" + "strconv" + + "github.com/vtmocanu/uzi/api/internal/forge" +) + +// resolveIssue is the one step every per-issue method starts with. issueIID is +// YouTrack's numberInProject. The issue is read as SHORT-n and must sit in the +// bound project with that number; otherwise it is reported as +// *forge.IssueNotFoundError. YouTrack keeps answering an old SHORT-n after the +// issue moves to another project, so this check is what makes a moved issue +// read as gone. Follow-up calls address the issue by the returned entity id. +func (y *youTrack) resolveIssue(ctx context.Context, issueIID int64) (ytIssue, error) { + p, err := y.requireProject() + if err != nil { + return ytIssue{}, err + } + if issueIID <= 0 { + return ytIssue{}, &forge.IssueNotFoundError{Err: y.fail("youtrack: issue number %d is out of range", issueIID)} + } + readable := p.ShortName + "-" + strconv.FormatInt(issueIID, 10) + var w ytIssue + status, _, err := y.do(ctx, ytCall{ + method: http.MethodGet, + path: []string{"issues", readable}, + query: url.Values{"fields": {issueFields}}, + limit: objectBytes, + prov: forge.ResponseIssueSuccess, + }, &w) + if err != nil { + if status == http.StatusNotFound { + return ytIssue{}, &forge.IssueNotFoundError{Err: err} + } + return ytIssue{}, err + } + if !y.inBoundProject(w) || w.NumberInProject != issueIID { + return ytIssue{}, &forge.IssueNotFoundError{Err: y.fail("youtrack: issue %s is not in the bound project", readable)} + } + if !entityIDRE.MatchString(w.ID) || !idReadableRE.MatchString(w.IDReadable) { + return ytIssue{}, y.fail("youtrack: issue with a malformed id in the response") + } + return w, nil +} + +// issueByEntity reads an issue by entity id, as CreateIssue does after it has +// written to it. The issue must be in the bound project. +func (y *youTrack) issueByEntity(ctx context.Context, id string) (ytIssue, error) { + var w ytIssue + _, _, err := y.do(ctx, ytCall{ + method: http.MethodGet, + path: []string{"issues", id}, + query: url.Values{"fields": {issueFields}}, + limit: objectBytes, + prov: forge.ResponseIssueSuccess, + }, &w) + if err != nil { + return ytIssue{}, err + } + if !y.inBoundProject(w) || w.ID != id { + return ytIssue{}, y.fail("youtrack: created issue is not in the bound project") + } + return w, nil +} diff --git a/api/internal/issuetracker/youtrack_issues.go b/api/internal/issuetracker/youtrack_issues.go new file mode 100644 index 0000000000..3e09d2942a --- /dev/null +++ b/api/internal/issuetracker/youtrack_issues.go @@ -0,0 +1,129 @@ +package issuetracker + +import ( + "context" + "net/http" + "net/url" + + "github.com/vtmocanu/uzi/api/internal/forge" +) + +// ListIssues implements forge.IssueTracker. The query narrows by project, +// labels, state and update date; the driver then filters again on exact tag +// names, the exact update instant and the bound project, and honours +// opts.Limit after that filter. +func (y *youTrack) ListIssues(ctx context.Context, _ int64, opts forge.ListIssuesOptions) ([]forge.Issue, error) { + p, err := y.requireProject() + if err != nil { + return nil, err + } + if err := y.checkLabelNames(opts.Labels); err != nil { + return nil, err + } + query := url.Values{"query": {issueQuery(p.ShortName, opts)}, "fields": {issueFields}} + out := []forge.Issue{} + var mapErr error + err = ytPages(ctx, y, []string{"issues"}, query, func(page []ytIssue) bool { + for _, w := range page { + if !y.matchesIssue(w, opts) { + continue + } + issue, err := y.toIssue(w) + if err != nil { + mapErr = err + return true + } + out = append(out, issue) + if opts.Limit > 0 && len(out) >= opts.Limit { + return true + } + } + return false + }) + if err != nil { + return nil, err + } + if mapErr != nil { + return nil, mapErr + } + return out, nil +} + +// GetIssue implements forge.IssueTracker. issueIID is the number in the project. +func (y *youTrack) GetIssue(ctx context.Context, _, issueIID int64) (forge.Issue, error) { + w, err := y.resolveIssue(ctx, issueIID) + if err != nil { + return forge.Issue{}, err + } + return y.toIssue(w) +} + +// CreateIssue implements forge.IssueTracker. The issue is created first, then +// each label is attached as a tag (created when missing), and the result is +// read back, so it has the shape GetIssue returns. +func (y *youTrack) CreateIssue(ctx context.Context, _ int64, title, description string, labels []string) (forge.Issue, error) { + p, err := y.requireProject() + if err != nil { + return forge.Issue{}, err + } + if err := y.checkLabelNames(labels); err != nil { + return forge.Issue{}, err + } + var created ytIssue + _, _, err = y.do(ctx, ytCall{ + method: http.MethodPost, + path: []string{"issues"}, + query: url.Values{"fields": {issueFields}}, + body: map[string]any{ + "project": map[string]string{"id": p.ID}, + "summary": title, + "description": description, + }, + limit: objectBytes, + prov: forge.ResponseIssueSuccess, + }, &created) + if err != nil { + return forge.Issue{}, err + } + if !entityIDRE.MatchString(created.ID) { + return forge.Issue{}, y.fail("youtrack: issue with a malformed id in the response") + } + if len(labels) > 0 { + ids, err := y.tagIDsFor(ctx, labels) + if err != nil { + return forge.Issue{}, err + } + attached := map[string]bool{} + for _, n := range labels { + if attached[n] { + continue + } + attached[n] = true + if err := y.attachTag(ctx, created.ID, ids[n]); err != nil { + return forge.Issue{}, err + } + } + } + w, err := y.issueByEntity(ctx, created.ID) + if err != nil { + return forge.Issue{}, err + } + return y.toIssue(w) +} + +// UpdateIssueDescription implements forge.IssueTracker. +func (y *youTrack) UpdateIssueDescription(ctx context.Context, _, issueIID int64, description string) error { + w, err := y.resolveIssue(ctx, issueIID) + if err != nil { + return err + } + _, _, err = y.do(ctx, ytCall{ + method: http.MethodPost, + path: []string{"issues", w.ID}, + query: url.Values{"fields": {"id"}}, + body: map[string]string{"description": description}, + limit: objectBytes, + prov: forge.ResponseSuccess, + }, nil) + return err +} diff --git a/api/internal/issuetracker/youtrack_labels.go b/api/internal/issuetracker/youtrack_labels.go new file mode 100644 index 0000000000..e4c6bf4059 --- /dev/null +++ b/api/internal/issuetracker/youtrack_labels.go @@ -0,0 +1,178 @@ +package issuetracker + +import ( + "context" + "net/http" + "net/url" + + "github.com/vtmocanu/uzi/api/internal/forge" +) + +// listTags returns the tags the token can see, keyed by name. +func (y *youTrack) listTags(ctx context.Context) (map[string]ytTag, []ytTag, error) { + tags, err := ytList[ytTag](ctx, y, []string{"tags"}, url.Values{"fields": {tagFields}}) + if err != nil { + return nil, nil, err + } + byName := make(map[string]ytTag, len(tags)) + for _, t := range tags { + if !entityIDRE.MatchString(t.ID) { + return nil, nil, y.fail("youtrack: tag with a malformed id in the response") + } + if _, dup := byName[t.Name]; !dup { + byName[t.Name] = t + } + } + return byName, tags, nil +} + +// createTag creates a tag by name and returns it. +func (y *youTrack) createTag(ctx context.Context, name string) (ytTag, error) { + var t ytTag + _, _, err := y.do(ctx, ytCall{ + method: http.MethodPost, + path: []string{"tags"}, + query: url.Values{"fields": {tagFields}}, + body: map[string]string{"name": name}, + limit: objectBytes, + prov: forge.ResponseSuccess, + }, &t) + if err != nil { + return ytTag{}, err + } + if !entityIDRE.MatchString(t.ID) { + return ytTag{}, y.fail("youtrack: tag with a malformed id in the response") + } + return t, nil +} + +// tagIDsFor maps names to the ids of their tags, creating a tag that does not +// exist yet. Duplicate names are created once. +func (y *youTrack) tagIDsFor(ctx context.Context, names []string) (map[string]string, error) { + byName, _, err := y.listTags(ctx) + if err != nil { + return nil, err + } + ids := make(map[string]string, len(names)) + for _, n := range names { + t, ok := byName[n] + if !ok { + if t, err = y.createTag(ctx, n); err != nil { + return nil, err + } + byName[n] = t + } + ids[n] = t.ID + } + return ids, nil +} + +// ListLabels implements forge.IssueTracker. Labels are YouTrack tags, global +// to the account; projectID is ignored. +func (y *youTrack) ListLabels(ctx context.Context, _ int64) ([]forge.Label, error) { + if _, err := y.requireProject(); err != nil { + return nil, err + } + _, tags, err := y.listTags(ctx) + if err != nil { + return nil, err + } + out := make([]forge.Label, 0, len(tags)) + for _, t := range tags { + out = append(out, forge.Label{Name: t.Name}) + } + return out, nil +} + +// EnsureLabels implements forge.IssueTracker: it creates the tags that do not +// exist by name. Label.Color is ignored, because YouTrack colours are palette +// ids. +func (y *youTrack) EnsureLabels(ctx context.Context, _ int64, labels []forge.Label) error { + if _, err := y.requireProject(); err != nil { + return err + } + names := make([]string, 0, len(labels)) + for _, l := range labels { + names = append(names, l.Name) + } + if err := y.checkLabelNames(names); err != nil { + return err + } + _, err := y.tagIDsFor(ctx, names) + return err +} + +// UpdateIssueLabels implements forge.IssueTracker. It makes one REST call per +// tag, adds first and then removes, so it is not atomic. The commands API is +// not used, so a label name never enters a command query. A tag to add that +// does not exist is created; one to remove that is not on the issue is skipped. +func (y *youTrack) UpdateIssueLabels(ctx context.Context, _, issueIID int64, add, remove []string) error { + if _, err := y.requireProject(); err != nil { + return err + } + if err := y.checkLabelNames(add, remove); err != nil { + return err + } + w, err := y.resolveIssue(ctx, issueIID) + if err != nil { + return err + } + present := make(map[string]string, len(w.Tags)) + for _, t := range w.Tags { + present[t.Name] = t.ID + } + var missing []string + for _, n := range add { + if _, ok := present[n]; !ok { + missing = append(missing, n) + } + } + var ids map[string]string + if len(missing) > 0 { + if ids, err = y.tagIDsFor(ctx, missing); err != nil { + return err + } + } + for _, n := range missing { + if _, ok := present[n]; ok { + continue + } + if err := y.attachTag(ctx, w.ID, ids[n]); err != nil { + return err + } + present[n] = ids[n] + } + for _, n := range remove { + id, ok := present[n] + if !ok { + continue + } + if err := y.detachTag(ctx, w.ID, id); err != nil { + return err + } + delete(present, n) + } + return nil +} + +func (y *youTrack) attachTag(ctx context.Context, issueID, tagID string) error { + _, _, err := y.do(ctx, ytCall{ + method: http.MethodPost, + path: []string{"issues", issueID, "tags"}, + query: url.Values{"fields": {tagFields}}, + body: map[string]string{"id": tagID}, + limit: objectBytes, + prov: forge.ResponseSuccess, + }, nil) + return err +} + +func (y *youTrack) detachTag(ctx context.Context, issueID, tagID string) error { + _, _, err := y.do(ctx, ytCall{ + method: http.MethodDelete, + path: []string{"issues", issueID, "tags", tagID}, + limit: objectBytes, + prov: forge.ResponseSuccess, + }, nil) + return err +} diff --git a/api/internal/issuetracker/youtrack_projects.go b/api/internal/issuetracker/youtrack_projects.go new file mode 100644 index 0000000000..e8fface8a5 --- /dev/null +++ b/api/internal/issuetracker/youtrack_projects.go @@ -0,0 +1,73 @@ +package issuetracker + +import ( + "context" + "net/url" +) + +// ListProjects implements Driver. +func (y *youTrack) ListProjects(ctx context.Context) ([]Project, error) { + wire, err := ytList[ytProject](ctx, y, []string{"admin", "projects"}, url.Values{"fields": {"id,shortName,name"}}) + if err != nil { + return nil, err + } + out := make([]Project, 0, len(wire)) + for _, p := range wire { + if !entityIDRE.MatchString(p.ID) || !shortNameRE.MatchString(p.ShortName) { + return nil, y.fail("youtrack: project with a malformed id in the response") + } + out = append(out, Project{ID: p.ID, ShortName: p.ShortName, Name: p.Name}) + } + return out, nil +} + +type ytProjectField struct { + ID string `json:"id"` + Type string `json:"$type"` + Field *struct { + Name string `json:"name"` + } `json:"field"` +} + +type ytStateValue struct { + ID string `json:"id"` + Name string `json:"name"` + IsResolved bool `json:"isResolved"` + Ordinal int `json:"ordinal"` +} + +// ListStates implements Driver. It finds the project's State field (named by +// Options.StateField, "State" by default) and lists the values of its bundle. +func (y *youTrack) ListStates(ctx context.Context, projectID string) ([]State, error) { + if !entityIDRE.MatchString(projectID) { + return nil, y.fail("youtrack: invalid project id") + } + fields, err := ytList[ytProjectField](ctx, y, []string{"admin", "projects", projectID, "customFields"}, + url.Values{"fields": {"id,field(name),$type,bundle(id)"}}) + if err != nil { + return nil, err + } + var fieldID string + for _, f := range fields { + if f.Type == "StateProjectCustomField" && f.Field != nil && f.Field.Name == y.stateField { + fieldID = f.ID + break + } + } + if fieldID == "" { + return nil, y.fail("youtrack: the project has no %q state field", y.stateField) + } + if !entityIDRE.MatchString(fieldID) { + return nil, y.fail("youtrack: state field with a malformed id in the response") + } + values, err := ytList[ytStateValue](ctx, y, []string{"admin", "projects", projectID, "customFields", fieldID, "bundle", "values"}, + url.Values{"fields": {"id,name,isResolved,ordinal"}}) + if err != nil { + return nil, err + } + out := make([]State, 0, len(values)) + for _, v := range values { + out = append(out, State{ID: v.ID, Name: v.Name, Resolved: v.IsResolved, Ordinal: v.Ordinal}) + } + return out, nil +} diff --git a/api/internal/issuetracker/youtrack_query.go b/api/internal/issuetracker/youtrack_query.go new file mode 100644 index 0000000000..49ca0450af --- /dev/null +++ b/api/internal/issuetracker/youtrack_query.go @@ -0,0 +1,92 @@ +package issuetracker + +import ( + "strings" + "time" + "unicode" + + "github.com/vtmocanu/uzi/api/internal/forge" +) + +// maxLabelNameLen bounds a tag name the driver will send. +const maxLabelNameLen = 200 + +// checkLabelName refuses a name that could change the meaning of a query or +// that carries invisible characters: braces (the query's quoting), control and +// format characters, and an empty name. The error never echoes the name. +func (y *youTrack) checkLabelName(name string) error { + bad := strings.TrimSpace(name) == "" || len(name) > maxLabelNameLen + for _, c := range name { + if c == '{' || c == '}' || unicode.IsControl(c) || unicode.Is(unicode.Cf, c) || c == '
' || c == '
' { + bad = true + } + } + if bad { + return y.red.sentinel(ErrInvalidLabelName, 0, "name is empty, too long, or has a brace or control character") + } + return nil +} + +func (y *youTrack) checkLabelNames(names ...[]string) error { + for _, list := range names { + for _, n := range list { + if err := y.checkLabelName(n); err != nil { + return err + } + } + } + return nil +} + +// issueQuery builds the YouTrack search query for ListIssues. Every label name +// has passed checkLabelName, so none can close its brace group. The date floor +// is one UTC day early: the query's granularity is a day, and the driver +// filters again on the exact millisecond. +func issueQuery(shortName string, opts forge.ListIssuesOptions) string { + parts := []string{"project: {" + shortName + "}"} + for _, l := range opts.Labels { + parts = append(parts, "tag: {"+l+"}") + } + switch opts.State { + case forge.StateOpened: + parts = append(parts, "#Unresolved") + case forge.StateClosed: + parts = append(parts, "#Resolved") + } + if opts.UpdatedAfter != nil { + parts = append(parts, "updated: "+opts.UpdatedAfter.UTC().AddDate(0, 0, -1).Format(time.DateOnly)+" .. *") + } + parts = append(parts, "sort by: updated asc") + return strings.Join(parts, " ") +} + +// matchesIssue is the client-side filter that backs the query: exact tag +// names, exact state, the exact update instant, and the bound project. +func (y *youTrack) matchesIssue(w ytIssue, opts forge.ListIssuesOptions) bool { + if !y.inBoundProject(w) { + return false + } + for _, l := range opts.Labels { + found := false + for _, t := range w.Tags { + if t.Name == l { + found = true + break + } + } + if !found { + return false + } + } + switch opts.State { + case forge.StateOpened: + if w.Resolved != nil { + return false + } + case forge.StateClosed: + if w.Resolved == nil { + return false + } + } + return opts.UpdatedAfter == nil || w.Updated >= opts.UpdatedAfter.UnixMilli() +} diff --git a/api/internal/issuetracker/youtrack_query_test.go b/api/internal/issuetracker/youtrack_query_test.go new file mode 100644 index 0000000000..88e10fb87f --- /dev/null +++ b/api/internal/issuetracker/youtrack_query_test.go @@ -0,0 +1,188 @@ +package issuetracker + +import ( + "context" + "errors" + "testing" + "time" + + "github.com/vtmocanu/uzi/api/internal/forge" +) + +func TestIssueQuery(t *testing.T) { + after := time.Date(2026, 3, 1, 23, 30, 0, 0, time.UTC) + got := issueQuery("UZI", forge.ListIssuesOptions{ + Labels: []string{"a b", "c"}, State: forge.StateOpened, UpdatedAfter: &after, + }) + want := "project: {UZI} tag: {a b} tag: {c} #Unresolved updated: 2026-02-28 .. * sort by: updated asc" + if got != want { + t.Fatalf("got %q, want %q", got, want) + } + if got := issueQuery("UZI", forge.ListIssuesOptions{State: forge.StateClosed}); got != "project: {UZI} #Resolved sort by: updated asc" { + t.Fatalf("got %q", got) + } +} + +func TestLabelNameGuard(t *testing.T) { + y, _ := newFakeDriver(t) + ctx := context.Background() + for _, name := range []string{"", " ", "a}b", "a{b", "a\nb", "a‮b"} { + if err := y.EnsureLabels(ctx, 1, []forge.Label{{Name: name}}); !errors.Is(err, ErrInvalidLabelName) { + t.Errorf("EnsureLabels(%q) = %v, want ErrInvalidLabelName", name, err) + } + if _, err := y.ListIssues(ctx, 1, forge.ListIssuesOptions{Labels: []string{name}}); !errors.Is(err, ErrInvalidLabelName) { + t.Errorf("ListIssues(%q) = %v, want ErrInvalidLabelName", name, err) + } + } +} + +func TestListIssuesUpdatedAfterAndLimit(t *testing.T) { + y, srv := newFakeDriver(t) + ctx := context.Background() + first := srv.SeedIssue("one", "") + second := srv.SeedIssue("two", "") + all, err := y.ListIssues(ctx, 1, forge.ListIssuesOptions{}) + if err != nil || len(all) != 2 || all[0].IID != first { + t.Fatalf("ListIssues = %+v, %v", all, err) + } + // The fake clock is one second per event; the day-granular query keeps + // both, so only the driver's exact filter can drop the first. + after := all[1].UpdatedAt + got, err := y.ListIssues(ctx, 1, forge.ListIssuesOptions{UpdatedAfter: &after}) + if err != nil || len(got) != 1 || got[0].IID != second { + t.Fatalf("UpdatedAfter = %+v, %v", got, err) + } + got, err = y.ListIssues(ctx, 1, forge.ListIssuesOptions{Limit: 1}) + if err != nil || len(got) != 1 { + t.Fatalf("Limit = %+v, %v", got, err) + } +} + +func TestIssueMapping(t *testing.T) { + y, srv := newFakeDriver(t) + n := srv.SeedIssue("title", "body", "x", "y") + got, err := y.GetIssue(context.Background(), 99, n) + if err != nil { + t.Fatal(err) + } + if got.IID != n || got.Title != "title" || got.Description != "body" || got.State != "opened" || + got.Author != youtracktestBot || got.AuthorForgeUserID != 0 || got.Assignees == nil || len(got.Assignees) != 0 || + got.WebURL != srv.URL+"/issue/UZI-1" || len(got.Labels) != 2 || got.UpdatedAt.IsZero() { + t.Fatalf("mapped issue = %+v", got) + } +} + +func TestPagingWalksEveryPage(t *testing.T) { + y, srv := newFakeDriver(t) + for i := 0; i < pageSize+5; i++ { + srv.SeedIssue("i", "") + } + got, err := y.ListIssues(context.Background(), 1, forge.ListIssuesOptions{}) + if err != nil || len(got) != pageSize+5 { + t.Fatalf("got %d issues, %v", len(got), err) + } +} + +func TestStatesAndProjects(t *testing.T) { + y, _ := newFakeDriver(t) + ctx := context.Background() + acct, err := y.VerifyToken(ctx) + if err != nil || acct.Login != youtracktestBot { + t.Fatalf("VerifyToken = %+v, %v", acct, err) + } + projects, err := y.ListProjects(ctx) + if err != nil || len(projects) != 2 { + t.Fatalf("ListProjects = %+v, %v", projects, err) + } + states, err := y.ListStates(ctx, "0-1") + if err != nil || len(states) != 3 || states[0].Resolved || !states[1].Resolved || states[2].Name != "Won't fix" { + t.Fatalf("ListStates = %+v, %v", states, err) + } + if _, err := y.ListStates(ctx, "../x"); err == nil { + t.Fatal("ListStates accepted a malformed project id") + } +} + +func TestStateFieldOption(t *testing.T) { + srv, y := newFakeDriverWithField(t, "Stage") + _ = srv + ctx := context.Background() + if _, err := y.ListStates(ctx, "0-1"); err != nil { + t.Fatal(err) + } +} + +func TestUnboundDriver(t *testing.T) { + _, srv := newFakeDriver(t) + opts := fakeOptions(srv) + opts.Project = ProjectRef{} + y, err := newYouTrackOver(opts, srv.Client().Transport) + if err != nil { + t.Fatal(err) + } + ctx := context.Background() + if _, err := y.ListLabels(ctx, 1); !errors.Is(err, ErrProjectNotBound) { + t.Errorf("ListLabels = %v", err) + } + if _, err := y.GetIssue(ctx, 1, 1); !errors.Is(err, ErrProjectNotBound) { + t.Errorf("GetIssue = %v", err) + } + if _, err := y.ListIssues(ctx, 1, forge.ListIssuesOptions{}); !errors.Is(err, ErrProjectNotBound) { + t.Errorf("ListIssues = %v", err) + } + if _, err := y.ListProjects(ctx); err != nil { + t.Errorf("ListProjects on an unbound driver: %v", err) + } +} + +func TestStateMappingUnset(t *testing.T) { + _, srv := newFakeDriver(t) + opts := fakeOptions(srv) + opts.OpenState, opts.ClosedState = "", "" + y, err := newYouTrackOver(opts, srv.Client().Transport) + if err != nil { + t.Fatal(err) + } + n := srv.SeedIssue("t", "") + for _, s := range []forge.IssueState{forge.StateOpened, forge.StateClosed} { + if err := y.SetIssueState(context.Background(), 1, n, s); !errors.Is(err, ErrStateMappingUnset) { + t.Errorf("SetIssueState(%s) = %v", s, err) + } + } + y2, _ := newFakeDriver(t) + if err := y2.SetIssueState(context.Background(), 1, 1, forge.StateAll); err == nil { + t.Error("SetIssueState accepted StateAll") + } +} + +func TestConstructorValidation(t *testing.T) { + good := Options{BaseURL: "https://yt.example.com", Token: credential(), Project: ProjectRef{ID: "0-1", ShortName: "UZI"}} + mutate := func(f func(*Options)) Options { o := good; f(&o); return o } + if _, err := New(KindYouTrack, good); err != nil { + t.Fatalf("good options: %v", err) + } + if _, err := New(KindYouTrack, mutate(func(o *Options) { o.Project = ProjectRef{} })); err != nil { + t.Fatalf("unbound options: %v", err) + } + bad := map[string]Options{ + "http": mutate(func(o *Options) { o.BaseURL = "http://yt.example.com" }), + "path": mutate(func(o *Options) { o.BaseURL = "https://yt.example.com/youtrack" }), + "userinfo": mutate(func(o *Options) { o.BaseURL = "https://u:p@yt.example.com" }), + "query": mutate(func(o *Options) { o.BaseURL = "https://yt.example.com?x=1" }), + "fragment": mutate(func(o *Options) { o.BaseURL = "https://yt.example.com#x" }), + "no host": mutate(func(o *Options) { o.BaseURL = "https://" }), + "empty": mutate(func(o *Options) { o.BaseURL = "" }), + "short token": mutate(func(o *Options) { o.Token = "short" }), + "bad short": mutate(func(o *Options) { o.Project.ShortName = "1X" }), + "bad id": mutate(func(o *Options) { o.Project.ID = "abc" }), + "half bound": mutate(func(o *Options) { o.Project.ID = "" }), + } + for name, o := range bad { + if _, err := New(KindYouTrack, o); err == nil { + t.Errorf("%s: constructor accepted %+v", name, o) + } + } + if _, err := New(KindYouTrack, mutate(func(o *Options) { o.BaseURL = "https://yt.example.com/" })); err != nil { + t.Errorf("trailing slash rejected: %v", err) + } +} diff --git a/api/internal/issuetracker/youtrack_request.go b/api/internal/issuetracker/youtrack_request.go new file mode 100644 index 0000000000..423bea7be0 --- /dev/null +++ b/api/internal/issuetracker/youtrack_request.go @@ -0,0 +1,151 @@ +package issuetracker + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "net/http" + "net/url" + "strconv" + "strings" + + "github.com/vtmocanu/uzi/api/internal/forge" +) + +// ytCall describes one request. path holds unescaped segments; each is +// escaped when the URL is built. +type ytCall struct { + method string + path []string + query url.Values + body any + limit int64 + prov forge.ResponseProvenance +} + +// escapedPath joins the segments, escaping every one. +func escapedPath(segments []string) string { + var b strings.Builder + for _, s := range segments { + b.WriteByte('/') + b.WriteString(url.PathEscape(s)) + } + return b.String() +} + +// do sends one request and decodes a successful JSON answer into out (nil to +// discard). It returns the HTTP status and the number of body bytes actually +// read. Every error is already redacted; a redirect is refused before any body +// is read; the response is read through a cap and decoded only from the capped +// buffer. There is no retry. +func (y *youTrack) do(ctx context.Context, c ytCall, out any) (status, n int, err error) { + var reqBody *bytes.Reader + if c.body != nil { + b, err := json.Marshal(c.body) + if err != nil { + return 0, 0, y.red.error(err) + } + reqBody = bytes.NewReader(b) + } else { + reqBody = bytes.NewReader(nil) + } + path := escapedPath(append([]string{"api"}, c.path...)) + target := y.baseURL + path + if len(c.query) > 0 { + target += "?" + c.query.Encode() + } + req, err := http.NewRequestWithContext(ctx, c.method, target, reqBody) + if err != nil { + return 0, 0, y.red.error(err) + } + req.Header.Set("Authorization", "Bearer "+y.token) + req.Header.Set("Accept", "application/json") + if c.body != nil { + req.Header.Set("Content-Type", "application/json") + } + resp, err := y.client.Do(req) + if err != nil { + return 0, 0, y.red.error(err) + } + defer resp.Body.Close() + status = resp.StatusCode + if isRedirect(resp) { + return status, 0, y.red.sentinel(ErrRedirectRefused, status, fmt.Sprintf("%s %s answered HTTP %d", c.method, path, status)) + } + if status < 200 || status > 299 { + body, err := readBounded(resp.Body, errorBodyBytes, forge.ResponseError, status) + if err != nil { + return status, 0, y.red.errorStatus(err, status) + } + return status, len(body), y.red.errorStatus(fmt.Errorf("youtrack: %s %s: HTTP %d: %s", c.method, path, status, errorText(body)), status) + } + body, err := readBounded(resp.Body, c.limit, c.prov, status) + if err != nil { + return status, 0, y.red.errorStatus(err, status) + } + if out != nil { + if err := json.Unmarshal(body, out); err != nil { + return status, len(body), y.red.errorStatus(fmt.Errorf("youtrack: %s %s: decode response: %w", c.method, path, err), status) + } + } + return status, len(body), nil +} + +// errorText renders a YouTrack error body: its error and error_description +// members, or a bounded prefix of the raw text. The caller's redactor scrubs +// and truncates the result. +func errorText(body []byte) string { + var e struct { + Error string `json:"error"` + Description string `json:"error_description"` + } + if json.Unmarshal(body, &e) == nil && (e.Error != "" || e.Description != "") { + return strings.TrimSpace(e.Error + ": " + e.Description) + } + const rawLimit = 200 + if len(body) > rawLimit { + body = body[:rawLimit] + } + return string(body) +} + +// ytPages walks every page of a list endpoint under paginate's ceilings and +// calls visit with each page. visit returns true to stop early. Errors come +// back redacted. +func ytPages[T any](ctx context.Context, y *youTrack, path []string, query url.Values, visit func([]T) bool) error { + var fetchErr error + _, err := paginate(func(skip, top int) ([]T, int, error) { + q := url.Values{} + for k, v := range query { + q[k] = v + } + q.Set("$top", strconv.Itoa(top)) + q.Set("$skip", strconv.Itoa(skip)) + var items []T + _, n, err := y.do(ctx, ytCall{method: http.MethodGet, path: path, query: q, limit: listPageBytes, prov: forge.ResponseSuccess}, &items) + if err != nil { + fetchErr = err + return nil, 0, err + } + if visit(items) { + // A short page ends the walk; the items were already consumed. + return nil, n, nil + } + return items, n, nil + }, 0) + if fetchErr != nil { + return fetchErr + } + return y.red.error(err) +} + +// ytList collects every item of a list endpoint. +func ytList[T any](ctx context.Context, y *youTrack, path []string, query url.Values) ([]T, error) { + var all []T + err := ytPages(ctx, y, path, query, func(page []T) bool { + all = append(all, page...) + return false + }) + return all, err +} diff --git a/api/internal/issuetracker/youtrack_security_test.go b/api/internal/issuetracker/youtrack_security_test.go new file mode 100644 index 0000000000..c4cec94706 --- /dev/null +++ b/api/internal/issuetracker/youtrack_security_test.go @@ -0,0 +1,209 @@ +package issuetracker + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "net/http" + "net/http/httptest" + "net/url" + "strings" + "sync/atomic" + "testing" + "unicode" + + "github.com/vtmocanu/uzi/api/internal/forge" + "github.com/vtmocanu/uzi/api/internal/issuetracker/youtracktest" +) + +// tlsDriver builds a bound driver over a TLS test server running h. +func tlsDriver(t *testing.T, h http.Handler) (*youTrack, *httptest.Server) { + t.Helper() + srv := httptest.NewTLSServer(h) + t.Cleanup(srv.Close) + y, err := newYouTrackOver(Options{ + BaseURL: srv.URL, + Token: credential(), + Project: ProjectRef{ID: "0-1", ShortName: "UZI"}, + OpenState: "Open", + ClosedState: "Fixed", + }, srv.Client().Transport) + if err != nil { + t.Fatal(err) + } + return y, srv +} + +func tokenForms() []string { + c := credential() + j, _ := json.Marshal(c) + return []string{c, url.QueryEscape(c), url.PathEscape(c), string(j[1 : len(j)-1])} +} + +func assertScrubbed(t *testing.T, err error) { + t.Helper() + if err == nil { + t.Fatal("want an error") + } + msg := err.Error() + for _, f := range tokenForms() { + if strings.Contains(msg, f) { + t.Errorf("error contains a token form %q: %q", f, msg) + } + } + for _, r := range msg { + if unicode.IsControl(r) || unicode.Is(unicode.Cf, r) { + t.Errorf("error contains control or format character %U: %q", r, msg) + break + } + } +} + +func TestRedirectsAreRefused(t *testing.T) { + for _, code := range []int{301, 302, 307, 308} { + t.Run(fmt.Sprint(code), func(t *testing.T) { + var hits atomic.Int64 + other := httptest.NewTLSServer(http.HandlerFunc(func(http.ResponseWriter, *http.Request) { hits.Add(1) })) + t.Cleanup(other.Close) + y, _ := tlsDriver(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + http.Redirect(w, r, other.URL+r.URL.Path, code) + })) + ctx := context.Background() + calls := map[string]func() error{ + "GetIssue": func() error { _, err := y.GetIssue(ctx, 1, 1); return err }, + "VerifyToken": func() error { _, err := y.VerifyToken(ctx); return err }, + } + if code == 307 || code == 308 { + calls["CreateIssue"] = func() error { _, err := y.CreateIssue(ctx, 1, "t", "d", nil); return err } + } + for name, call := range calls { + err := call() + if !errors.Is(err, ErrRedirectRefused) { + t.Errorf("%s: %v, want ErrRedirectRefused", name, err) + } + assertScrubbed(t, err) + } + if n := hits.Load(); n != 0 { + t.Errorf("redirect target received %d requests", n) + } + }) + } +} + +func TestOversizedResponses(t *testing.T) { + pad := func(n int) []byte { return append([]byte("["), bytes.Repeat([]byte(" "), n)...) } + t.Run("list page", func(t *testing.T) { + y, _ := tlsDriver(t, http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + _, _ = w.Write(pad(int(listPageBytes) + 10)) + })) + if _, err := y.ListProjects(context.Background()); !errors.Is(err, forge.ErrResponseTooLarge) { + t.Fatalf("got %v, want ErrResponseTooLarge", err) + } + }) + t.Run("GetIssue", func(t *testing.T) { + y, _ := tlsDriver(t, http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + _, _ = w.Write(pad(int(objectBytes) + 10)) + })) + _, err := y.GetIssue(context.Background(), 1, 1) + if !errors.Is(err, forge.ErrResponseTooLarge) || !forge.IsIssueResponseTooLarge(err) { + t.Fatalf("got %v, want an issue-provenance ErrResponseTooLarge", err) + } + }) +} + +func TestErrorBodiesAreScrubbed(t *testing.T) { + c := credential() + nasty := "\r\n\x1b[31m‮" + j, _ := json.Marshal(c) + bodies := map[string][]byte{ + "raw": []byte("echo " + c + nasty), + "query": []byte("echo " + url.QueryEscape(c) + nasty), + "json": []byte(`{"error":"bad","error_description":"echo ` + string(j[1:len(j)-1]) + `\r\n\u001b[31m‮"}`), + "marshal": mustJSON(t, map[string]string{"error": c + nasty, "error_description": url.QueryEscape(c) + nasty}), + } + for name, body := range bodies { + t.Run(name, func(t *testing.T) { + y, _ := tlsDriver(t, http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusBadRequest) + _, _ = w.Write(body) + })) + _, err := y.VerifyToken(context.Background()) + assertScrubbed(t, err) + }) + } +} + +func mustJSON(t *testing.T, v any) []byte { + t.Helper() + b, err := json.Marshal(v) + if err != nil { + t.Fatal(err) + } + return b +} + +func TestEveryMethodScrubsItsErrors(t *testing.T) { + y, _ := tlsDriver(t, http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusInternalServerError) + _, _ = w.Write(mustJSON(t, map[string]string{"error": "boom", "error_description": "token " + credential() + " ‮"})) + })) + ctx := context.Background() + calls := map[string]func() error{ + "ListLabels": func() error { _, err := y.ListLabels(ctx, 1); return err }, + "EnsureLabels": func() error { return y.EnsureLabels(ctx, 1, []forge.Label{{Name: "a"}}) }, + "ListIssues": func() error { _, err := y.ListIssues(ctx, 1, forge.ListIssuesOptions{}); return err }, + "GetIssue": func() error { _, err := y.GetIssue(ctx, 1, 1); return err }, + "CreateIssue": func() error { _, err := y.CreateIssue(ctx, 1, "t", "d", nil); return err }, + "UpdateIssueLabels": func() error { return y.UpdateIssueLabels(ctx, 1, 1, []string{"a"}, nil) }, + "UpdateIssueDescription": func() error { return y.UpdateIssueDescription(ctx, 1, 1, "d") }, + "SetIssueState": func() error { return y.SetIssueState(ctx, 1, 1, forge.StateClosed) }, + "ListIssueLabelEvents": func() error { _, err := y.ListIssueLabelEvents(ctx, 1, 1); return err }, + "ListIssueComments": func() error { _, err := y.ListIssueComments(ctx, 1, 1); return err }, + "CreateIssueNote": func() error { _, err := y.CreateIssueNote(ctx, 1, 1, "n"); return err }, + "VerifyToken": func() error { _, err := y.VerifyToken(ctx); return err }, + "ListProjects": func() error { _, err := y.ListProjects(ctx); return err }, + "ListStates": func() error { _, err := y.ListStates(ctx, "0-1"); return err }, + } + if len(calls) != 14 { + t.Fatalf("table covers %d methods, want 14", len(calls)) + } + for name, call := range calls { + t.Run(name, func(t *testing.T) { + err := call() + assertScrubbed(t, err) + if forge.Class(err) != forge.ErrorClassServerError { + t.Errorf("class = %v, want server error", forge.Class(err)) + } + }) + } +} + +func TestMovedIssueIsNeverWritten(t *testing.T) { + y, srv := newFakeDriver(t) + ctx := context.Background() + n := srv.SeedIssue("moves", "d") + srv.MoveIssue(n, youtracktest.OtherProjectID) + srv.ResetRequests() + calls := map[string]func() error{ + "GetIssue": func() error { _, err := y.GetIssue(ctx, 1, n); return err }, + "UpdateIssueLabels": func() error { return y.UpdateIssueLabels(ctx, 1, n, []string{"a"}, []string{"b"}) }, + "UpdateIssueDescription": func() error { return y.UpdateIssueDescription(ctx, 1, n, "d") }, + "SetIssueState": func() error { return y.SetIssueState(ctx, 1, n, forge.StateClosed) }, + "ListIssueLabelEvents": func() error { _, err := y.ListIssueLabelEvents(ctx, 1, n); return err }, + "ListIssueComments": func() error { _, err := y.ListIssueComments(ctx, 1, n); return err }, + "CreateIssueNote": func() error { _, err := y.CreateIssueNote(ctx, 1, n, "x"); return err }, + } + for name, call := range calls { + var nf *forge.IssueNotFoundError + if err := call(); !errors.As(err, &nf) { + t.Errorf("%s = %v, want *forge.IssueNotFoundError", name, err) + } + } + for _, r := range srv.Requests() { + if r.Method != http.MethodGet { + t.Errorf("write after move: %s %s", r.Method, r.Path) + } + } +} diff --git a/api/internal/issuetracker/youtrack_state.go b/api/internal/issuetracker/youtrack_state.go new file mode 100644 index 0000000000..5b81f9de75 --- /dev/null +++ b/api/internal/issuetracker/youtrack_state.go @@ -0,0 +1,47 @@ +package issuetracker + +import ( + "context" + "net/http" + "net/url" + + "github.com/vtmocanu/uzi/api/internal/forge" +) + +// SetIssueState implements forge.IssueTracker. It writes the explicit state +// value Options names for the requested state and never guesses one; with that +// value unset it returns ErrStateMappingUnset. +func (y *youTrack) SetIssueState(ctx context.Context, _, issueIID int64, state forge.IssueState) error { + if _, err := y.requireProject(); err != nil { + return err + } + var value string + switch state { + case forge.StateOpened: + value = y.openState + case forge.StateClosed: + value = y.closedState + default: + return y.fail("youtrack: state must be opened or closed") + } + if value == "" { + return y.red.sentinel(ErrStateMappingUnset, 0, "no value configured for "+string(state)) + } + w, err := y.resolveIssue(ctx, issueIID) + if err != nil { + return err + } + _, _, err = y.do(ctx, ytCall{ + method: http.MethodPost, + path: []string{"issues", w.ID}, + query: url.Values{"fields": {"id"}}, + body: map[string]any{"customFields": []any{map[string]any{ + "name": y.stateField, + "$type": "StateIssueCustomField", + "value": map[string]string{"name": value}, + }}}, + limit: objectBytes, + prov: forge.ResponseSuccess, + }, nil) + return err +} diff --git a/api/internal/issuetracker/youtrack_test.go b/api/internal/issuetracker/youtrack_test.go new file mode 100644 index 0000000000..d064d7d994 --- /dev/null +++ b/api/internal/issuetracker/youtrack_test.go @@ -0,0 +1,55 @@ +package issuetracker + +import ( + "strings" + "testing" + + "github.com/vtmocanu/uzi/api/internal/issuetracker/youtracktest" +) + +// fakeOptions are the Options that bind a driver to the fake's main project +// with the state mapping its State bundle supports. +func fakeOptions(srv *youtracktest.Server) Options { + return Options{ + BaseURL: srv.URL, + Token: srv.Token, + Project: ProjectRef{ID: youtracktest.ProjectID, ShortName: youtracktest.ProjectShortName}, + OpenState: "Open", + ClosedState: "Fixed", + } +} + +// newFakeDriver starts the fake and builds a driver over it. +func newFakeDriver(t *testing.T) (*youTrack, *youtracktest.Server) { + t.Helper() + srv := youtracktest.NewServer(t) + y, err := newYouTrackOver(fakeOptions(srv), srv.Client().Transport) + if err != nil { + t.Fatal(err) + } + return y, srv +} + +// credential assembles a token-shaped value at runtime, with characters that +// each escaping form rewrites. +func credential() string { + return strings.Join([]string{"perm", ":", "ab c/d\"e", "-", "Zq9xLm2VnB7wKp3T"}, "") +} + +// youtracktestBot is the login the fake's bearer token authenticates as. +const youtracktestBot = youtracktest.BotLogin + +// newFakeDriverWithField is newFakeDriver with the State field renamed and the +// driver told the new name. +func newFakeDriverWithField(t *testing.T, field string) (*youtracktest.Server, *youTrack) { + t.Helper() + srv := youtracktest.NewServer(t) + srv.SetStateField(field) + opts := fakeOptions(srv) + opts.StateField = field + y, err := newYouTrackOver(opts, srv.Client().Transport) + if err != nil { + t.Fatal(err) + } + return srv, y +} diff --git a/api/internal/issuetracker/youtrack_wire.go b/api/internal/issuetracker/youtrack_wire.go new file mode 100644 index 0000000000..49528d33ad --- /dev/null +++ b/api/internal/issuetracker/youtrack_wire.go @@ -0,0 +1,120 @@ +package issuetracker + +import ( + "fmt" + "regexp" + "strconv" + "strings" + "time" + + "github.com/vtmocanu/uzi/api/internal/forge" +) + +// fields parameters. YouTrack returns only what is asked for. +const ( + issueFields = "id,idReadable,numberInProject,summary,description,created,updated,resolved,reporter(id,login),tags(id,name),project(id,shortName)" + tagFields = "id,name" + commentFields = "id,text,created,deleted,author(id,login)" + activityFields = "id,timestamp,author(id,login),added(id,name),removed(id,name)" +) + +// idReadableRE matches an id-readable such as "UZI-15"; it ends up in WebURL. +var idReadableRE = regexp.MustCompile(`^[A-Za-z][A-Za-z0-9_]*-\d+$`) + +type ytUser struct { + ID string `json:"id"` + Login string `json:"login"` +} + +type ytTag struct { + ID string `json:"id"` + Name string `json:"name"` +} + +type ytProject struct { + ID string `json:"id"` + ShortName string `json:"shortName"` + Name string `json:"name"` +} + +type ytIssue struct { + ID string `json:"id"` + IDReadable string `json:"idReadable"` + NumberInProject int64 `json:"numberInProject"` + Summary string `json:"summary"` + Description string `json:"description"` + Created int64 `json:"created"` + Updated int64 `json:"updated"` + Resolved *int64 `json:"resolved"` + Reporter *ytUser `json:"reporter"` + Tags []ytTag `json:"tags"` + Project *ytProject `json:"project"` +} + +type ytComment struct { + ID string `json:"id"` + Text string `json:"text"` + Created int64 `json:"created"` + Deleted bool `json:"deleted"` + Author *ytUser `json:"author"` +} + +type ytActivity struct { + ID string `json:"id"` + Timestamp int64 `json:"timestamp"` + Author *ytUser `json:"author"` + Added []ytTag `json:"added"` + Removed []ytTag `json:"removed"` +} + +// toIssue maps a wire issue to the neutral type. The issue must carry a valid +// entity id and id-readable. +func (y *youTrack) toIssue(w ytIssue) (forge.Issue, error) { + if !entityIDRE.MatchString(w.ID) || !idReadableRE.MatchString(w.IDReadable) { + return forge.Issue{}, y.fail("youtrack: issue with a malformed id in the response") + } + issue := forge.Issue{ + IID: w.NumberInProject, + Title: w.Summary, + State: string(forge.StateOpened), + Labels: make([]string, 0, len(w.Tags)), + Description: w.Description, + WebURL: y.baseURL + "/issue/" + w.IDReadable, + UpdatedAt: time.UnixMilli(w.Updated).UTC(), + Assignees: []int64{}, + } + if w.Resolved != nil { + issue.State = string(forge.StateClosed) + } + if w.Reporter != nil { + issue.Author = w.Reporter.Login + } + for _, t := range w.Tags { + issue.Labels = append(issue.Labels, t.Name) + } + return issue, nil +} + +// inBoundProject reports whether the issue sits in the bound project. +func (y *youTrack) inBoundProject(w ytIssue) bool { + return w.Project != nil && w.Project.ID == y.project.ID && w.NumberInProject > 0 +} + +// noteID is the numeric suffix of a comment entity id such as "4-17". +func noteID(id string) (int64, error) { + if !entityIDRE.MatchString(id) { + return 0, fmt.Errorf("youtrack: comment with a malformed id in the response") + } + n, err := strconv.ParseInt(id[strings.IndexByte(id, '-')+1:], 10, 64) + if err != nil { + return 0, fmt.Errorf("youtrack: comment with a malformed id in the response") + } + return n, nil +} + +func login(u *ytUser) string { + if u == nil { + return "" + } + return u.Login +} diff --git a/api/internal/issuetracker/youtracktest/fields.go b/api/internal/issuetracker/youtracktest/fields.go new file mode 100644 index 0000000000..6e6b9b73a3 --- /dev/null +++ b/api/internal/issuetracker/youtracktest/fields.go @@ -0,0 +1,82 @@ +package youtracktest + +import ( + "strings" +) + +// fieldSpec is a parsed YouTrack "fields" query: a leaf selects a whole +// value, a node selects named members of an object or of each array element. +type fieldSpec map[string]fieldSpec + +// parseFields parses "a,b(c,d(e)),$type". It reports false for malformed +// input (unbalanced parentheses, empty names). +func parseFields(s string) (fieldSpec, bool) { + spec, rest, ok := parseFieldList(s) + if !ok || rest != "" || len(spec) == 0 { + return nil, false + } + return spec, true +} + +func parseFieldList(s string) (fieldSpec, string, bool) { + spec := fieldSpec{} + for { + end := strings.IndexAny(s, ",()") + if end < 0 { + end = len(s) + } + name := s[:end] + if name == "" { + return nil, "", false + } + s = s[end:] + var child fieldSpec + if strings.HasPrefix(s, "(") { + var ok bool + child, s, ok = parseFieldList(s[1:]) + if !ok || !strings.HasPrefix(s, ")") { + return nil, "", false + } + s = s[1:] + } + spec[name] = child + if !strings.HasPrefix(s, ",") { + return spec, s, true + } + s = s[1:] + } +} + +// project returns v restricted to spec, so a driver that forgets to request a +// field sees it missing, as it would on a real server. +func (spec fieldSpec) project(v any) any { + switch x := v.(type) { + case map[string]any: + out := map[string]any{} + for name, child := range spec { + member, ok := x[name] + if !ok { + continue + } + if child == nil { + out[name] = member + } else { + out[name] = child.project(member) + } + } + return out + case []map[string]any: + out := make([]any, 0, len(x)) + for _, e := range x { + out = append(out, spec.project(e)) + } + return out + case []any: + out := make([]any, 0, len(x)) + for _, e := range x { + out = append(out, spec.project(e)) + } + return out + } + return v +} diff --git a/api/internal/issuetracker/youtracktest/fields_test.go b/api/internal/issuetracker/youtracktest/fields_test.go new file mode 100644 index 0000000000..8f44887f85 --- /dev/null +++ b/api/internal/issuetracker/youtracktest/fields_test.go @@ -0,0 +1,31 @@ +package youtracktest + +import ( + "reflect" + "testing" +) + +func TestFieldsProjection(t *testing.T) { + spec, ok := parseFields("id,author(id,login),tags(name),$type") + if !ok { + t.Fatal("parse failed") + } + in := map[string]any{ + "id": "1-1", "extra": 1, "$type": "Issue", + "author": map[string]any{"id": "2", "login": "a", "x": 1}, + "tags": []map[string]any{{"name": "n", "id": "9"}}, + } + want := map[string]any{ + "id": "1-1", "$type": "Issue", + "author": map[string]any{"id": "2", "login": "a"}, + "tags": []any{map[string]any{"name": "n"}}, + } + if got := spec.project(in); !reflect.DeepEqual(got, want) { + t.Fatalf("got %#v", got) + } + for _, bad := range []string{"", "a(", "a,,b", "a(b", "(a)"} { + if _, ok := parseFields(bad); ok { + t.Errorf("parseFields(%q) accepted", bad) + } + } +} diff --git a/api/internal/issuetracker/youtracktest/routes.go b/api/internal/issuetracker/youtracktest/routes.go new file mode 100644 index 0000000000..3cf5010001 --- /dev/null +++ b/api/internal/issuetracker/youtracktest/routes.go @@ -0,0 +1,485 @@ +package youtracktest + +import ( + "encoding/json" + "net/http" + "regexp" + "slices" + "strconv" + "strings" +) + +// defaultTop is YouTrack's page size when $top is absent, so a driver that +// forgets to page sees a truncated list here as it would there. +const defaultTop = 42 + +// routes builds the mux. Every modelled path dispatches on the method inside, +// so a known path with an unmodelled method is also loud (501), not 405. +func (s *Server) routes() *http.ServeMux { + mux := http.NewServeMux() + route := func(pattern string, handlers map[string]http.HandlerFunc) { + mux.HandleFunc(pattern, func(w http.ResponseWriter, r *http.Request) { + if h, ok := handlers[r.Method]; ok { + h(w, r) + return + } + s.unmodelledRoute(w, r) + }) + } + route("/api/users/me", map[string]http.HandlerFunc{http.MethodGet: s.getMe}) + route("/api/admin/projects", map[string]http.HandlerFunc{http.MethodGet: s.listProjects}) + route("/api/admin/projects/{pid}/customFields", map[string]http.HandlerFunc{http.MethodGet: s.listProjectFields}) + route("/api/admin/projects/{pid}/customFields/{cf}/bundle/values", map[string]http.HandlerFunc{http.MethodGet: s.listBundleValues}) + route("/api/tags", map[string]http.HandlerFunc{http.MethodGet: s.listTags, http.MethodPost: s.createTag}) + route("/api/issues", map[string]http.HandlerFunc{http.MethodGet: s.listIssues, http.MethodPost: s.createIssue}) + route("/api/issues/{id}", map[string]http.HandlerFunc{http.MethodGet: s.getIssue, http.MethodPost: s.updateIssue}) + route("/api/issues/{id}/tags", map[string]http.HandlerFunc{http.MethodPost: s.addIssueTag}) + route("/api/issues/{id}/tags/{tag}", map[string]http.HandlerFunc{http.MethodDelete: s.removeIssueTag}) + route("/api/issues/{id}/activities", map[string]http.HandlerFunc{http.MethodGet: s.listActivities}) + route("/api/issues/{id}/comments", map[string]http.HandlerFunc{http.MethodGet: s.listComments, http.MethodPost: s.createComment}) + mux.HandleFunc("/", s.unmodelledRoute) + return mux +} + +// fields parses the mandatory fields parameter; it answers 400 and reports +// false when it is missing or malformed. +func fields(w http.ResponseWriter, r *http.Request) (fieldSpec, bool) { + spec, ok := parseFields(r.URL.Query().Get("fields")) + if !ok { + writeError(w, http.StatusBadRequest, "Bad Request", "a valid fields parameter is required") + } + return spec, ok +} + +// page applies $top and $skip to items. +func page[T any](r *http.Request, items []T) []T { + top, skip := defaultTop, 0 + if v := r.URL.Query().Get("$top"); v != "" { + if n, err := strconv.Atoi(v); err == nil && n >= 0 { + top = n + } + } + if v := r.URL.Query().Get("$skip"); v != "" { + if n, err := strconv.Atoi(v); err == nil && n >= 0 { + skip = n + } + } + if skip > len(items) { + skip = len(items) + } + items = items[skip:] + if top < len(items) { + items = items[:top] + } + return items +} + +// writePage projects, pages and writes a list. The items are projected after +// paging so a driver sees exactly the requested window. +func writePage(w http.ResponseWriter, r *http.Request, spec fieldSpec, items []map[string]any) { + window := page(r, items) + out := make([]any, 0, len(window)) + for _, it := range window { + out = append(out, spec.project(it)) + } + writeJSON(w, http.StatusOK, out) +} + +func notFound(w http.ResponseWriter, what string) { + writeError(w, http.StatusNotFound, "Not Found", "Entity not found: "+what) +} + +// decodeBody reads a strict JSON body; unknown members are a 400, so a driver +// cannot send fields the API ignores. +func decodeBody(w http.ResponseWriter, r *http.Request, v any) bool { + dec := json.NewDecoder(http.MaxBytesReader(w, r.Body, 1<<20)) + dec.DisallowUnknownFields() + if err := dec.Decode(v); err != nil { + writeError(w, http.StatusBadRequest, "Bad Request", "invalid request body: "+err.Error()) + return false + } + return true +} + +func (s *Server) getMe(w http.ResponseWriter, r *http.Request) { + spec, ok := fields(w, r) + if !ok { + return + } + writeJSON(w, http.StatusOK, spec.project(userJSON(BotLogin))) +} + +func (s *Server) listProjects(w http.ResponseWriter, r *http.Request) { + spec, ok := fields(w, r) + if !ok { + return + } + var items []map[string]any + for _, p := range s.m.projects { + items = append(items, map[string]any{"$type": "Project", "id": p.id, "shortName": p.shortName, "name": p.name}) + } + writePage(w, r, spec, items) +} + +// stateFieldID is the id of the State custom field of project pid. +func stateFieldID(pid string) string { return "60-" + strings.TrimPrefix(pid, "0-") } + +func (s *Server) listProjectFields(w http.ResponseWriter, r *http.Request) { + spec, ok := fields(w, r) + if !ok { + return + } + pid := r.PathValue("pid") + if s.m.projectByID(pid) == nil { + notFound(w, pid) + return + } + items := []map[string]any{ + {"$type": "EnumProjectCustomField", "id": "61-" + strings.TrimPrefix(pid, "0-"), "field": map[string]any{"name": "Priority"}, "bundle": map[string]any{"id": "41-1"}}, + {"$type": "StateProjectCustomField", "id": stateFieldID(pid), "field": map[string]any{"name": s.m.stateField}, "bundle": map[string]any{"id": "40-1"}}, + } + writePage(w, r, spec, items) +} + +func (s *Server) listBundleValues(w http.ResponseWriter, r *http.Request) { + spec, ok := fields(w, r) + if !ok { + return + } + pid, cf := r.PathValue("pid"), r.PathValue("cf") + if s.m.projectByID(pid) == nil || cf != stateFieldID(pid) { + notFound(w, pid+"/"+cf) + return + } + var items []map[string]any + for _, v := range s.m.states { + items = append(items, map[string]any{"$type": "StateBundleElement", "id": v.id, "name": v.name, "isResolved": v.resolved, "ordinal": v.ordinal}) + } + writePage(w, r, spec, items) +} + +func (s *Server) listTags(w http.ResponseWriter, r *http.Request) { + spec, ok := fields(w, r) + if !ok { + return + } + writePage(w, r, spec, s.m.tagsJSON(tagIDs(s.m.tags))) +} + +func tagIDs(tags []*tag) []string { + ids := make([]string, 0, len(tags)) + for _, t := range tags { + ids = append(ids, t.id) + } + return ids +} + +func (s *Server) createTag(w http.ResponseWriter, r *http.Request) { + spec, ok := fields(w, r) + if !ok { + return + } + var body struct { + Name string `json:"name"` + } + if !decodeBody(w, r, &body) { + return + } + if strings.TrimSpace(body.Name) == "" { + writeError(w, http.StatusBadRequest, "Bad Request", "tag name is required") + return + } + for _, t := range s.m.tags { + if t.name == body.Name { + writeError(w, http.StatusConflict, "Conflict", "tag already exists") + return + } + } + t := s.m.ensureTag(body.Name) + writeJSON(w, http.StatusOK, spec.project(s.m.tagsJSON([]string{t.id})[0])) +} + +var ( + queryProject = regexp.MustCompile(`project: \{([^{}]*)\}`) + queryTag = regexp.MustCompile(`tag: \{([^{}]*)\}`) + queryUpdated = regexp.MustCompile(`updated: (\d{4}-\d{2}-\d{2}) \.\. \*`) + queryUnresolved = regexp.MustCompile(`#Unresolved`) + queryResolved = regexp.MustCompile(`#Resolved`) + querySort = regexp.MustCompile(`sort by: updated asc`) +) + +// listIssues understands exactly the query grammar the driver builds and +// answers 400 for anything else. +func (s *Server) listIssues(w http.ResponseWriter, r *http.Request) { + spec, ok := fields(w, r) + if !ok { + return + } + q := r.URL.Query().Get("query") + var tagNames []string + for _, m := range queryTag.FindAllStringSubmatch(q, -1) { + tagNames = append(tagNames, m[1]) + } + q = queryTag.ReplaceAllString(q, "") + proj := queryProject.FindStringSubmatch(q) + q = queryProject.ReplaceAllString(q, "") + upd := queryUpdated.FindStringSubmatch(q) + q = queryUpdated.ReplaceAllString(q, "") + unresolved, resolved := queryUnresolved.MatchString(q), queryResolved.MatchString(q) + q = queryResolved.ReplaceAllString(queryUnresolved.ReplaceAllString(querySort.ReplaceAllString(q, ""), ""), "") + if strings.TrimSpace(q) != "" || proj == nil || (unresolved && resolved) { + writeError(w, http.StatusBadRequest, "Bad Request", "unsupported query") + return + } + p := s.m.projectByShort(proj[1]) + var floor int64 + if upd != nil { + d, err := parseDay(upd[1]) + if err != nil { + writeError(w, http.StatusBadRequest, "Bad Request", "bad date") + return + } + floor = d + } + var matched []*issue + for _, i := range s.m.issues { + if i.project != p || i.updated < floor || (unresolved && i.resolved != nil) || (resolved && i.resolved == nil) { + continue + } + if !s.hasAllTags(i, tagNames) { + continue + } + matched = append(matched, i) + } + slices.SortStableFunc(matched, func(a, b *issue) int { return int(a.updated - b.updated) }) + items := make([]map[string]any, 0, len(matched)) + for _, i := range matched { + items = append(items, s.m.issueJSON(i)) + } + writePage(w, r, spec, items) +} + +// hasAllTags matches tag names case-insensitively, as YouTrack's query does; +// the driver filters again on the exact name. +func (s *Server) hasAllTags(i *issue, names []string) bool { + for _, name := range names { + found := false + for _, id := range i.tags { + if t := s.m.tagByID(id); t != nil && strings.EqualFold(t.name, name) { + found = true + } + } + if !found { + return false + } + } + return true +} + +func (s *Server) createIssue(w http.ResponseWriter, r *http.Request) { + spec, ok := fields(w, r) + if !ok { + return + } + var body struct { + Project struct { + ID string `json:"id"` + } `json:"project"` + Summary string `json:"summary"` + Description string `json:"description"` + } + if !decodeBody(w, r, &body) { + return + } + p := s.m.projectByID(body.Project.ID) + if p == nil { + writeError(w, http.StatusBadRequest, "Bad Request", "unknown project") + return + } + if strings.TrimSpace(body.Summary) == "" { + writeError(w, http.StatusBadRequest, "Bad Request", "summary is required") + return + } + i := s.m.newIssue(p, body.Summary, body.Description) + writeJSON(w, http.StatusOK, spec.project(s.m.issueJSON(i))) +} + +func (s *Server) getIssue(w http.ResponseWriter, r *http.Request) { + spec, ok := fields(w, r) + if !ok { + return + } + i := s.m.lookup(r.PathValue("id")) + if i == nil { + notFound(w, r.PathValue("id")) + return + } + writeJSON(w, http.StatusOK, spec.project(s.m.issueJSON(i))) +} + +// updateIssue handles the two writes the driver makes: description, and the +// State custom field. +func (s *Server) updateIssue(w http.ResponseWriter, r *http.Request) { + spec, ok := fields(w, r) + if !ok { + return + } + i := s.m.lookup(r.PathValue("id")) + if i == nil { + notFound(w, r.PathValue("id")) + return + } + var body struct { + Description *string `json:"description"` + CustomFields []struct { + Name string `json:"name"` + Type string `json:"$type"` + Value struct { + Name string `json:"name"` + } `json:"value"` + } `json:"customFields"` + } + if !decodeBody(w, r, &body) { + return + } + if body.Description != nil { + i.description = *body.Description + i.updated = s.m.now() + } + for _, cf := range body.CustomFields { + v := s.m.stateByName(cf.Value.Name) + if cf.Name != s.m.stateField || cf.Type != "StateIssueCustomField" || v == nil { + writeError(w, http.StatusBadRequest, "Bad Request", "unsupported custom field update") + return + } + ts := s.m.now() + i.updated = ts + if v.resolved { + if i.resolved == nil { + i.resolved = &ts + } + } else { + i.resolved = nil + } + } + writeJSON(w, http.StatusOK, spec.project(s.m.issueJSON(i))) +} + +func (s *Server) addIssueTag(w http.ResponseWriter, r *http.Request) { + spec, ok := fields(w, r) + if !ok { + return + } + i := s.m.lookup(r.PathValue("id")) + if i == nil { + notFound(w, r.PathValue("id")) + return + } + var body struct { + ID string `json:"id"` + } + if !decodeBody(w, r, &body) { + return + } + t := s.m.tagByID(body.ID) + if t == nil { + writeError(w, http.StatusBadRequest, "Bad Request", "unknown tag") + return + } + if i.hasTag(t.id) { + writeError(w, http.StatusConflict, "Conflict", "tag already on the issue") + return + } + s.m.addTag(i, t, BotLogin) + writeJSON(w, http.StatusOK, spec.project(s.m.tagsJSON([]string{t.id})[0])) +} + +func (s *Server) removeIssueTag(w http.ResponseWriter, r *http.Request) { + i := s.m.lookup(r.PathValue("id")) + if i == nil { + notFound(w, r.PathValue("id")) + return + } + t := s.m.tagByID(r.PathValue("tag")) + if t == nil || !i.hasTag(t.id) { + notFound(w, r.PathValue("tag")) + return + } + s.m.removeTag(i, t, BotLogin) + w.WriteHeader(http.StatusNoContent) +} + +func (s *Server) listActivities(w http.ResponseWriter, r *http.Request) { + spec, ok := fields(w, r) + if !ok { + return + } + if r.URL.Query().Get("categories") != "TagsCategory" { + writeError(w, http.StatusBadRequest, "Bad Request", "categories must be TagsCategory") + return + } + i := s.m.lookup(r.PathValue("id")) + if i == nil { + notFound(w, r.PathValue("id")) + return + } + // Newest first, so a driver that does not sort is caught. + var items []map[string]any + for k := len(i.activities) - 1; k >= 0; k-- { + a := i.activities[k] + items = append(items, map[string]any{ + "$type": "TagsActivityItem", "id": a.id, "timestamp": a.timestamp, "author": userJSON(a.author), + "added": s.m.tagsJSON(a.added), "removed": s.m.tagsJSON(a.removed), + }) + } + writePage(w, r, spec, items) +} + +func commentJSON(c *comment) map[string]any { + return map[string]any{"$type": "IssueComment", "id": c.id, "text": c.text, "created": c.created, "deleted": c.deleted, "author": userJSON(c.author)} +} + +func (s *Server) listComments(w http.ResponseWriter, r *http.Request) { + spec, ok := fields(w, r) + if !ok { + return + } + i := s.m.lookup(r.PathValue("id")) + if i == nil { + notFound(w, r.PathValue("id")) + return + } + // Newest first, so a driver that does not sort is caught. + var items []map[string]any + for k := len(i.comments) - 1; k >= 0; k-- { + items = append(items, commentJSON(i.comments[k])) + } + writePage(w, r, spec, items) +} + +func (s *Server) createComment(w http.ResponseWriter, r *http.Request) { + spec, ok := fields(w, r) + if !ok { + return + } + i := s.m.lookup(r.PathValue("id")) + if i == nil { + notFound(w, r.PathValue("id")) + return + } + var body struct { + Text string `json:"text"` + } + if !decodeBody(w, r, &body) { + return + } + if strings.TrimSpace(body.Text) == "" { + writeError(w, http.StatusBadRequest, "Bad Request", "text is required") + return + } + ts := s.m.now() + c := &comment{id: s.m.nextID(4), text: body.Text, created: ts, author: BotLogin} + i.comments = append(i.comments, c) + i.updated = ts + writeJSON(w, http.StatusOK, spec.project(commentJSON(c))) +} diff --git a/api/internal/issuetracker/youtracktest/seed.go b/api/internal/issuetracker/youtracktest/seed.go new file mode 100644 index 0000000000..4573240c0a --- /dev/null +++ b/api/internal/issuetracker/youtracktest/seed.go @@ -0,0 +1,93 @@ +package youtracktest + +// SeedIssue creates an issue in the main project and returns its number. Tags +// are created on demand and added without recording an activity. +func (s *Server) SeedIssue(summary, description string, tags ...string) int64 { + s.mu.Lock() + defer s.mu.Unlock() + i := s.m.newIssue(s.m.projectByID(ProjectID), summary, description) + for _, name := range tags { + i.tags = append(i.tags, s.m.ensureTag(name).id) + } + return i.number +} + +// SeedComment adds a comment by login on issue number n. +func (s *Server) SeedComment(n int64, login, text string) { + s.seedComment(n, login, text, false) +} + +// SeedDeletedComment adds a comment that the API reports as deleted. +func (s *Server) SeedDeletedComment(n int64, login, text string) { + s.seedComment(n, login, text, true) +} + +func (s *Server) seedComment(n int64, login, text string, deleted bool) { + s.t.Helper() + s.mu.Lock() + defer s.mu.Unlock() + i := s.m.byNumber(n) + if i == nil { + s.t.Fatalf("youtracktest: no issue %d", n) + } + ts := s.m.now() + i.updated = ts + i.comments = append(i.comments, &comment{id: s.m.nextID(4), text: text, created: ts, deleted: deleted, author: login}) +} + +// SeedTagActivity records that login added and removed the named tags on issue +// number n. Tags are created on demand; the issue's own tag list is left +// alone, so a test can seed history that no longer matches the present. +func (s *Server) SeedTagActivity(n int64, login string, added, removed []string) { + s.t.Helper() + s.mu.Lock() + defer s.mu.Unlock() + i := s.m.byNumber(n) + if i == nil { + s.t.Fatalf("youtracktest: no issue %d", n) + } + a := &activity{id: s.m.nextID(3), timestamp: s.m.now(), author: login} + for _, name := range added { + a.added = append(a.added, s.m.ensureTag(name).id) + } + for _, name := range removed { + a.removed = append(a.removed, s.m.ensureTag(name).id) + } + i.activities = append(i.activities, a) +} + +// MoveIssue moves issue number n to the project toProjectID. Like YouTrack it +// takes a new number there, and the old id-readable keeps resolving to it. It +// returns the new id-readable. +func (s *Server) MoveIssue(n int64, toProjectID string) string { + s.t.Helper() + s.mu.Lock() + defer s.mu.Unlock() + i := s.m.byNumber(n) + to := s.m.projectByID(toProjectID) + if i == nil || to == nil { + s.t.Fatalf("youtracktest: cannot move issue %d to project %q", n, toProjectID) + } + to.next++ + i.project, i.number = to, to.next + i.updated = s.m.now() + s.m.aliases[i.readable()] = i + return i.readable() +} + +// IssueTags returns the names of the tags currently on issue number n in the +// main project, for assertions on server-side state. +func (s *Server) IssueTags(n int64) []string { + s.t.Helper() + s.mu.Lock() + defer s.mu.Unlock() + i := s.m.byNumber(n) + if i == nil { + s.t.Fatalf("youtracktest: no issue %d", n) + } + var names []string + for _, id := range i.tags { + names = append(names, s.m.tagByID(id).name) + } + return names +} diff --git a/api/internal/issuetracker/youtracktest/server.go b/api/internal/issuetracker/youtracktest/server.go new file mode 100644 index 0000000000..042792eb0f --- /dev/null +++ b/api/internal/issuetracker/youtracktest/server.go @@ -0,0 +1,107 @@ +// Package youtracktest is an in-process HTTPS fake of the YouTrack REST API, +// modelling only the routes the issuetracker YouTrack driver uses. A route it +// does not model answers 501 and fails the test at cleanup, so an unmodelled +// call is loud rather than silently green. +package youtracktest + +import ( + "encoding/json" + "net/http" + "net/http/httptest" + "slices" + "strings" + "sync" + "testing" +) + +// Request is one request the server received. +type Request struct { + Method string + // Path is the escaped request path, without the query. + Path string + RawQuery string +} + +// Server is the fake. Build it with NewServer; the embedded httptest.Server +// provides URL and Client (which trusts the server's certificate). +type Server struct { + *httptest.Server + // Token is the bearer token the server accepts. + Token string + + t testing.TB + mu sync.Mutex // guards every field below + m *model + // requests logs every request in arrival order. + requests []Request + unmodelled []string +} + +// NewServer starts the fake over TLS and registers its cleanup. The +// cleanup fails the test when a request reached an unmodelled route. +func NewServer(t testing.TB) *Server { + t.Helper() + s := &Server{ + t: t, + m: newModel(), + Token: strings.Join([]string{"perm", ":", "fixture-", "id.", "Yk3mQ8vR2xTn5Lw7"}, ""), + } + s.Server = httptest.NewTLSServer(http.HandlerFunc(s.serve)) + t.Cleanup(func() { + s.Server.Close() + s.mu.Lock() + defer s.mu.Unlock() + if len(s.unmodelled) > 0 { + t.Errorf("youtracktest: requests reached routes the fake does not model:\n %s", strings.Join(s.unmodelled, "\n ")) + } + }) + return s +} + +// Requests returns a copy of the request log. +func (s *Server) Requests() []Request { + s.mu.Lock() + defer s.mu.Unlock() + return slices.Clone(s.requests) +} + +// ResetRequests empties the request log. +func (s *Server) ResetRequests() { + s.mu.Lock() + defer s.mu.Unlock() + s.requests = nil +} + +// SetStateField renames the State custom field, so a test can prove the driver +// honours Options.StateField. +func (s *Server) SetStateField(name string) { + s.mu.Lock() + defer s.mu.Unlock() + s.m.stateField = name +} + +func (s *Server) serve(w http.ResponseWriter, r *http.Request) { + s.mu.Lock() + defer s.mu.Unlock() + s.requests = append(s.requests, Request{Method: r.Method, Path: r.URL.EscapedPath(), RawQuery: r.URL.RawQuery}) + if r.Header.Get("Authorization") != "Bearer "+s.Token { + writeError(w, http.StatusUnauthorized, "Unauthorized", "invalid or missing bearer token") + return + } + s.routes().ServeHTTP(w, r) +} + +func (s *Server) unmodelledRoute(w http.ResponseWriter, r *http.Request) { + s.unmodelled = append(s.unmodelled, r.Method+" "+r.URL.EscapedPath()) + writeError(w, http.StatusNotImplemented, "Not Implemented", "route not modelled by youtracktest") +} + +func writeError(w http.ResponseWriter, status int, kind, description string) { + writeJSON(w, status, map[string]any{"error": kind, "error_description": description}) +} + +func writeJSON(w http.ResponseWriter, status int, v any) { + w.Header().Set("Content-Type", "application/json;charset=UTF-8") + w.WriteHeader(status) + _ = json.NewEncoder(w).Encode(v) +} diff --git a/api/internal/issuetracker/youtracktest/state.go b/api/internal/issuetracker/youtracktest/state.go new file mode 100644 index 0000000000..2042dbd360 --- /dev/null +++ b/api/internal/issuetracker/youtracktest/state.go @@ -0,0 +1,271 @@ +package youtracktest + +import ( + "fmt" + "strings" + "time" +) + +// Fixed identities of the fake's data. +const ( + // ProjectID and ProjectShortName name the project issues are created in. + ProjectID = "0-1" + ProjectShortName = "UZI" + // OtherProjectID and OtherProjectShortName name a second project, the + // target of MoveIssue. + OtherProjectID = "0-2" + OtherProjectShortName = "OTH" + // BotLogin is the account the bearer token authenticates as. + BotLogin = "uzi-bot" + botID = "1-1" + // DefaultStateField is the name of the State custom field. + DefaultStateField = "State" +) + +type project struct { + id, shortName, name string + next int64 // last number handed out in this project +} + +type tag struct { + id, name string +} + +type comment struct { + id string + text string + created int64 + deleted bool + author string +} + +type activity struct { + id string + timestamp int64 + author string + added []string // tag ids + removed []string +} + +type stateValue struct { + id, name string + resolved bool + ordinal int +} + +type issue struct { + id string + project *project + number int64 + summary string + description string + created int64 + updated int64 + resolved *int64 + reporter string + tags []string // tag ids, oldest first + comments []*comment + activities []*activity +} + +func (i *issue) readable() string { return fmt.Sprintf("%s-%d", i.project.shortName, i.number) } + +// model is the in-memory YouTrack state. Server.mu guards every field. +type model struct { + projects []*project + tags []*tag + issues []*issue + aliases map[string]*issue // every id-readable an issue has answered to + states []stateValue + stateField string + clock int64 + seq int64 +} + +func newModel() *model { + return &model{ + projects: []*project{ + {id: ProjectID, shortName: ProjectShortName, name: "Uzi"}, + {id: OtherProjectID, shortName: OtherProjectShortName, name: "Other"}, + }, + aliases: map[string]*issue{}, + states: []stateValue{ + {id: "10-1", name: "Open", resolved: false, ordinal: 0}, + {id: "10-2", name: "Fixed", resolved: true, ordinal: 1}, + {id: "10-3", name: "Won't fix", resolved: true, ordinal: 2}, + }, + stateField: DefaultStateField, + clock: time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC).UnixMilli(), + } +} + +// now returns a strictly increasing epoch-millisecond clock, one second per +// call, so ordering assertions never depend on wall time. +func (m *model) now() int64 { + m.clock += 1000 + return m.clock +} + +func (m *model) nextID(prefix int) string { + m.seq++ + return fmt.Sprintf("%d-%d", prefix, m.seq) +} + +func (m *model) projectByID(id string) *project { + for _, p := range m.projects { + if p.id == id { + return p + } + } + return nil +} + +func (m *model) projectByShort(short string) *project { + for _, p := range m.projects { + if strings.EqualFold(p.shortName, short) { + return p + } + } + return nil +} + +func (m *model) tagByID(id string) *tag { + for _, t := range m.tags { + if t.id == id { + return t + } + } + return nil +} + +// tagByName returns the tag called name, creating it when missing. +func (m *model) ensureTag(name string) *tag { + for _, t := range m.tags { + if t.name == name { + return t + } + } + t := &tag{id: m.nextID(5), name: name} + m.tags = append(m.tags, t) + return t +} + +func (m *model) newIssue(p *project, summary, description string) *issue { + p.next++ + ts := m.now() + i := &issue{ + id: m.nextID(2), project: p, number: p.next, summary: summary, description: description, + created: ts, updated: ts, reporter: BotLogin, + } + m.issues = append(m.issues, i) + m.aliases[i.readable()] = i + return i +} + +// lookup resolves an entity id or any id-readable the issue has had. +func (m *model) lookup(id string) *issue { + for _, i := range m.issues { + if i.id == id { + return i + } + } + return m.aliases[id] +} + +func (m *model) byNumber(n int64) *issue { + return m.aliases[fmt.Sprintf("%s-%d", ProjectShortName, n)] +} + +func (i *issue) hasTag(id string) bool { + for _, t := range i.tags { + if t == id { + return true + } + } + return false +} + +// addTag adds a tag and records the activity, unless it is already present. +func (m *model) addTag(i *issue, t *tag, author string) { + if i.hasTag(t.id) { + return + } + ts := m.now() + i.tags = append(i.tags, t.id) + i.updated = ts + i.activities = append(i.activities, &activity{id: m.nextID(3), timestamp: ts, author: author, added: []string{t.id}}) +} + +func (m *model) removeTag(i *issue, t *tag, author string) { + if !i.hasTag(t.id) { + return + } + ts := m.now() + kept := i.tags[:0:0] + for _, id := range i.tags { + if id != t.id { + kept = append(kept, id) + } + } + i.tags = kept + i.updated = ts + i.activities = append(i.activities, &activity{id: m.nextID(3), timestamp: ts, author: author, removed: []string{t.id}}) +} + +func (m *model) stateByName(name string) *stateValue { + for k := range m.states { + if m.states[k].name == name { + return &m.states[k] + } + } + return nil +} + +// issueJSON is the full wire object of an issue; callers project it through a +// fields spec. +func (m *model) issueJSON(i *issue) map[string]any { + tags := make([]map[string]any, 0, len(i.tags)) + for _, id := range i.tags { + if t := m.tagByID(id); t != nil { + tags = append(tags, map[string]any{"id": t.id, "name": t.name, "$type": "Tag"}) + } + } + var resolved any + if i.resolved != nil { + resolved = *i.resolved + } + return map[string]any{ + "$type": "Issue", + "id": i.id, + "idReadable": i.readable(), + "numberInProject": i.number, + "summary": i.summary, + "description": i.description, + "created": i.created, + "updated": i.updated, + "resolved": resolved, + "reporter": userJSON(i.reporter), + "tags": tags, + "project": map[string]any{"id": i.project.id, "shortName": i.project.shortName, "$type": "Project"}, + } +} + +func userJSON(login string) map[string]any { + return map[string]any{"$type": "User", "id": botID, "login": login} +} + +func (m *model) tagsJSON(ids []string) []map[string]any { + out := make([]map[string]any, 0, len(ids)) + for _, id := range ids { + if t := m.tagByID(id); t != nil { + out = append(out, map[string]any{"id": t.id, "name": t.name, "$type": "Tag"}) + } + } + return out +} + +// parseDay returns the epoch milliseconds of the start of the UTC day. +func parseDay(s string) (int64, error) { + t, err := time.Parse("2006-01-02", s) + return t.UnixMilli(), err +} From 0d0976dec938c38502dd5169ffab676db281b2f3 Mon Sep 17 00:00:00 2001 From: uzi-agent Date: Wed, 11 Nov 2026 09:16:43 +0000 Subject: [PATCH 04/17] issuetrackersvc: issue tracker connections (PRD #4173 M3) The issue_tracker_connections table (migration 00424), its owner-scoped queries and the service: https-only base URL on the separate ISSUE_TRACKER_ALLOWED_BASE_URLS allowlist (empty by default, so no connection can be created), token verified then sealed in secretbox bound to user, kind and base URL, no field that can reveal it, and the allowlist re-checked before a stored token is opened. Co-Authored-By: Claude Opus 5.5 --- api/internal/config/config.go | 31 ++ .../config/issue_tracker_allowlist_test.go | 62 +++ api/internal/issuetrackersvc/account.go | 37 ++ api/internal/issuetrackersvc/baseurl.go | 35 ++ api/internal/issuetrackersvc/connect.go | 69 +++ api/internal/issuetrackersvc/connection.go | 36 ++ api/internal/issuetrackersvc/doc.go | 11 + api/internal/issuetrackersvc/driver.go | 77 +++ api/internal/issuetrackersvc/errors.go | 29 ++ api/internal/issuetrackersvc/read.go | 60 +++ api/internal/issuetrackersvc/seal.go | 34 ++ api/internal/issuetrackersvc/service.go | 47 ++ api/internal/issuetrackersvc/service_test.go | 437 ++++++++++++++++++ .../store/issue_tracker_connections.sql.go | 139 ++++++ .../issue_tracker_connections_livedb_test.go | 150 ++++++ .../00424_issue_tracker_connections.sql | 21 + api/internal/store/models.go | 12 + .../queries/issue_tracker_connections.sql | 24 + 18 files changed, 1311 insertions(+) create mode 100644 api/internal/config/issue_tracker_allowlist_test.go create mode 100644 api/internal/issuetrackersvc/account.go create mode 100644 api/internal/issuetrackersvc/baseurl.go create mode 100644 api/internal/issuetrackersvc/connect.go create mode 100644 api/internal/issuetrackersvc/connection.go create mode 100644 api/internal/issuetrackersvc/doc.go create mode 100644 api/internal/issuetrackersvc/driver.go create mode 100644 api/internal/issuetrackersvc/errors.go create mode 100644 api/internal/issuetrackersvc/read.go create mode 100644 api/internal/issuetrackersvc/seal.go create mode 100644 api/internal/issuetrackersvc/service.go create mode 100644 api/internal/issuetrackersvc/service_test.go create mode 100644 api/internal/store/issue_tracker_connections.sql.go create mode 100644 api/internal/store/issue_tracker_connections_livedb_test.go create mode 100644 api/internal/store/migrations/00424_issue_tracker_connections.sql create mode 100644 api/internal/store/queries/issue_tracker_connections.sql diff --git a/api/internal/config/config.go b/api/internal/config/config.go index 927809fdb8..b08308c660 100644 --- a/api/internal/config/config.go +++ b/api/internal/config/config.go @@ -151,6 +151,13 @@ type Config struct { // scheme+host, https only) but a DISTINCT list, so enabling one source never widens the // other. EMPTY means the feature is off: ProductSkillsBaseURLAllowed is false for every URL. ProductSkillsAllowedBaseURLs []string + // IssueTrackerAllowedBaseURLs is the SEPARATE SSRF allowlist for external issue tracker + // connections (PRD #4173 D8, ISSUE_TRACKER_ALLOWED_BASE_URLS): the only base URLs a + // connection may target. Same shape as the other allowlists (normalized scheme+host, https + // only) but a DISTINCT list, never FORGE_ALLOWED_BASE_URLS, so a forge host is not an issue + // tracker host. EMPTY by default, which means no connection can be created: + // IssueTrackerBaseURLAllowed is false for every URL. + IssueTrackerAllowedBaseURLs []string // ForgePollInterval is the per-enabled-repo incremental poll cadence. ForgePollInterval time.Duration // ForgeReconcileEvery is the number of incremental polls between full @@ -994,6 +1001,12 @@ func Load() (Config, error) { return Config{}, err } cfg.ProductSkillsAllowedBaseURLs = productSkillsAllowed + // PRD #4173 D8: the issue tracker allowlist, default EMPTY (no connection can be created, instance still boots). + issueTrackerAllowed, err := parseBaseURLAllowlist("ISSUE_TRACKER_ALLOWED_BASE_URLS", getenv("ISSUE_TRACKER_ALLOWED_BASE_URLS", "")) + if err != nil { + return Config{}, err + } + cfg.IssueTrackerAllowedBaseURLs = issueTrackerAllowed cfg.ForgePollInterval = parseDuration("FORGE_POLL_INTERVAL", time.Minute) cfg.ForgeReconcileEvery = parseInt("FORGE_RECONCILE_EVERY", 10) cfg.ForgeHTTPTimeout = parseDuration("FORGE_HTTP_TIMEOUT", 15*time.Second) @@ -2231,6 +2244,24 @@ func (c Config) AgentSourceBaseURLAllowed(raw string) bool { return false } +// IssueTrackerBaseURLAllowed reports whether raw normalizes to an allowlisted issue tracker base +// URL (PRD #4173 D8). It checks the SEPARATE IssueTrackerAllowedBaseURLs list and is false when +// that list is empty. NormalizeForgeBaseURL keeps only scheme and host, so it does NOT reject +// userinfo, a path, a query or a fragment: the issue tracker service applies its own strict check +// first and passes this method the already-normalized value. +func (c Config) IssueTrackerBaseURLAllowed(raw string) bool { + norm, err := NormalizeForgeBaseURL(raw) + if err != nil { + return false + } + for _, a := range c.IssueTrackerAllowedBaseURLs { + if a == norm { + return true + } + } + return false +} + // validateSecret enforces the boot guard: reject missing, empty, placeholder, // or too-short secrets. func validateSecret(name, raw string) ([]byte, error) { diff --git a/api/internal/config/issue_tracker_allowlist_test.go b/api/internal/config/issue_tracker_allowlist_test.go new file mode 100644 index 0000000000..adfd683d9f --- /dev/null +++ b/api/internal/config/issue_tracker_allowlist_test.go @@ -0,0 +1,62 @@ +package config + +import ( + "strings" + "testing" +) + +// TestLoadReadsIssueTrackerAllowlist (PRD #4173 D8): the variable reaches Config, defaults to an +// empty list while the instance still boots, and a non-https entry fails the boot naming the variable. +func TestLoadReadsIssueTrackerAllowlist(t *testing.T) { + autoselectEnv(t) + const env = "ISSUE_TRACKER_ALLOWED_BASE_URLS" + + t.Setenv(env, "") + cfg, err := Load() + if err != nil { + t.Fatalf("Load with the variable empty: %v", err) + } + if cfg.IssueTrackerAllowedBaseURLs != nil { + t.Errorf("empty variable gave %v, want nil", cfg.IssueTrackerAllowedBaseURLs) + } + + t.Setenv(env, "https://Issues.Example.com/, https://issues.example.com") + cfg, err = Load() + if err != nil { + t.Fatalf("Load: %v", err) + } + if got := cfg.IssueTrackerAllowedBaseURLs; len(got) != 1 || got[0] != "https://issues.example.com" { + t.Errorf("allowlist = %v, want one normalized entry", got) + } + + t.Setenv(env, "http://issues.example.com") + _, err = Load() + if err == nil { + t.Fatal("Load accepted an http entry; want a boot error") + } + if !strings.Contains(err.Error(), env) { + t.Errorf("boot error %q does not name %s", err, env) + } +} + +// TestIssueTrackerBaseURLAllowed: an empty list refuses everything, a listed host is allowed, and +// the list is separate from the forge list. +func TestIssueTrackerBaseURLAllowed(t *testing.T) { + if (Config{}).IssueTrackerBaseURLAllowed("https://issues.example.com") { + t.Error("an empty allowlist allowed a URL") + } + c := Config{ + ForgeAllowedBaseURLs: []string{"https://gitlab.example.com"}, + IssueTrackerAllowedBaseURLs: []string{"https://issues.example.com"}, + } + for _, ok := range []string{"https://issues.example.com", "https://ISSUES.example.com/"} { + if !c.IssueTrackerBaseURLAllowed(ok) { + t.Errorf("IssueTrackerBaseURLAllowed(%q) = false, want true", ok) + } + } + for _, no := range []string{"https://evil.example.com", "http://issues.example.com", "https://issues.example.com:8443", "https://gitlab.example.com", "not a url", ""} { + if c.IssueTrackerBaseURLAllowed(no) { + t.Errorf("IssueTrackerBaseURLAllowed(%q) = true, want false", no) + } + } +} diff --git a/api/internal/issuetrackersvc/account.go b/api/internal/issuetrackersvc/account.go new file mode 100644 index 0000000000..ce6c5e8ccc --- /dev/null +++ b/api/internal/issuetrackersvc/account.go @@ -0,0 +1,37 @@ +package issuetrackersvc + +import ( + "errors" + "unicode" + + "github.com/vtmocanu/uzi/api/internal/issuetracker" +) + +// maxAccountField bounds the id and login a driver may report, in bytes. +const maxAccountField = 255 + +// validateAccount refuses an account whose id or login is empty, too long, or +// holds a control, format (Cf) or line/paragraph separator character. The +// values come from a remote server and are stored and shown to the user, so a +// hostile one must not smuggle in invisible or bidirectional text. +func validateAccount(a issuetracker.Account) error { + if err := validateAccountField(a.ID); err != nil { + return errors.New("issue tracker returned an invalid account id") + } + if err := validateAccountField(a.Login); err != nil { + return errors.New("issue tracker returned an invalid account login") + } + return nil +} + +func validateAccountField(s string) error { + if s == "" || len(s) > maxAccountField { + return errors.New("empty or too long") + } + for _, r := range s { + if unicode.IsControl(r) || unicode.Is(unicode.Cf, r) || r == '
' || r == '
' { + return errors.New("forbidden character") + } + } + return nil +} diff --git a/api/internal/issuetrackersvc/baseurl.go b/api/internal/issuetrackersvc/baseurl.go new file mode 100644 index 0000000000..1173812d15 --- /dev/null +++ b/api/internal/issuetrackersvc/baseurl.go @@ -0,0 +1,35 @@ +package issuetrackersvc + +import ( + "errors" + "net/url" + "strings" + + "github.com/vtmocanu/uzi/api/internal/config" +) + +// strictBaseURL validates raw as an https scheme-and-host base URL and returns +// config.NormalizeForgeBaseURL's canonical form. Unlike that function it +// refuses, rather than strips, userinfo, an opaque part, a path other than +// empty or "/", a query and a fragment: stripping would let +// "https://allowed@evil" or "https://host/x" mean something other than what was +// checked. The returned string is the one value that is checked against the +// allowlist, stored, and bound into the token's AAD. +func strictBaseURL(raw string) (string, error) { + raw = strings.TrimSpace(raw) + u, err := url.Parse(raw) + if err != nil { + return "", errors.New("invalid base URL") + } + // "?" and "#" with nothing after them parse to empty RawQuery/Fragment, so + // they are refused on the text too. + if u.User != nil || u.Opaque != "" || (u.Path != "" && u.Path != "/") || + u.RawQuery != "" || u.ForceQuery || u.Fragment != "" || strings.ContainsAny(raw, "?#") { + return "", errors.New("invalid base URL: use https scheme and host only, with no credentials, path, query or fragment") + } + norm, err := config.NormalizeForgeBaseURL(raw) + if err != nil { + return "", errors.New("invalid base URL: it must be an https URL with a host") + } + return norm, nil +} diff --git a/api/internal/issuetrackersvc/connect.go b/api/internal/issuetrackersvc/connect.go new file mode 100644 index 0000000000..eef5599603 --- /dev/null +++ b/api/internal/issuetrackersvc/connect.go @@ -0,0 +1,69 @@ +package issuetrackersvc + +import ( + "context" + "strings" + + "github.com/google/uuid" + + "github.com/vtmocanu/uzi/api/internal/issuetracker" + "github.com/vtmocanu/uzi/api/internal/store" +) + +// Connect verifies a token against the issue tracker and stores it sealed for +// the user, creating the connection or rotating the existing one for the same +// (user, kind, base URL). The order is fixed: kind, token presence, strict base +// URL, allowlist, driver build, token verification, account check, seal, +// upsert. A refused base URL never reaches the driver factory, so no outbound +// request or token use happens for it. The token is trimmed of surrounding +// whitespace, and it appears in no returned error or log line. +func (s *Service) Connect(ctx context.Context, userID uuid.UUID, kind, baseURL, token string) (Connection, error) { + parsedKind, ok := issuetracker.ParseKind(strings.TrimSpace(kind)) + if !ok { + return Connection{}, &Error{Kind: KindBadRequest, Message: "unsupported issue tracker kind; supported values: " + issuetracker.SupportedKindsList()} + } + token = strings.TrimSpace(token) + if token == "" { + return Connection{}, &Error{Kind: KindBadRequest, Message: "a token is required"} + } + norm, err := strictBaseURL(baseURL) + if err != nil { + return Connection{}, &Error{Kind: KindBadRequest, Message: err.Error()} + } + if !s.allowed(norm) { + return Connection{}, &Error{Kind: KindBadRequest, Message: "base URL is not in the allowed issue tracker list"} + } + + d, err := s.newDriver(parsedKind, issuetracker.Options{BaseURL: norm, Token: token}) + if err != nil { + s.log.Error("build issue tracker driver", "kind", string(parsedKind), "error", err) + return Connection{}, &Error{Kind: KindBadRequest, Message: "could not initialize issue tracker client"} + } + account, err := d.VerifyToken(ctx) + if err != nil { + // err is already token-redacted by the driver. + return Connection{}, &Error{Kind: KindBadGateway, Message: "token verification failed: " + err.Error()} + } + if err := validateAccount(account); err != nil { + return Connection{}, &Error{Kind: KindBadGateway, Message: err.Error()} + } + + sealed, err := sealToken(s.box, userID, parsedKind, norm, token) + if err != nil { + s.log.Error("seal issue tracker token", "error", err) + return Connection{}, internal() + } + row, err := s.q.UpsertIssueTrackerConnection(ctx, store.UpsertIssueTrackerConnectionParams{ + UserID: userID, + Kind: string(parsedKind), + BaseUrl: norm, + AccountID: account.ID, + AccountLogin: account.Login, + TokenCiphertext: sealed, + }) + if err != nil { + s.log.Error("upsert issue tracker connection", "error", err) + return Connection{}, internal() + } + return toConnection(row), nil +} diff --git a/api/internal/issuetrackersvc/connection.go b/api/internal/issuetrackersvc/connection.go new file mode 100644 index 0000000000..5f27853152 --- /dev/null +++ b/api/internal/issuetrackersvc/connection.go @@ -0,0 +1,36 @@ +package issuetrackersvc + +import ( + "time" + + "github.com/google/uuid" + + "github.com/vtmocanu/uzi/api/internal/store" +) + +// Connection is the client-safe view of a stored connection. It deliberately +// has no token or ciphertext field: nothing the service returns can reveal the +// secret. +type Connection struct { + ID uuid.UUID + UserID uuid.UUID + Kind string + BaseURL string + AccountID string + AccountLogin string + CreatedAt time.Time + LastVerifiedAt time.Time +} + +func toConnection(r store.IssueTrackerConnection) Connection { + return Connection{ + ID: r.ID, + UserID: r.UserID, + Kind: r.Kind, + BaseURL: r.BaseUrl, + AccountID: r.AccountID, + AccountLogin: r.AccountLogin, + CreatedAt: r.CreatedAt.Time, + LastVerifiedAt: r.LastVerifiedAt.Time, + } +} diff --git a/api/internal/issuetrackersvc/doc.go b/api/internal/issuetrackersvc/doc.go new file mode 100644 index 0000000000..650ce66553 --- /dev/null +++ b/api/internal/issuetrackersvc/doc.go @@ -0,0 +1,11 @@ +// Package issuetrackersvc holds the decision logic of external issue tracker +// connections (PRD #4173): connect (verify and store a token), list, read and +// delete a connection, and open the stored token to build a driver for project +// and state discovery. +// +// The rules live here so every caller shares them: the base URL must be https +// scheme and host only and on the issue tracker allowlist, the token is sealed +// with the user, kind and base URL bound in as additional authenticated data, +// and no method ever returns or logs the token. Failures are typed *Error; +// each caller maps them to its own wire format. +package issuetrackersvc diff --git a/api/internal/issuetrackersvc/driver.go b/api/internal/issuetrackersvc/driver.go new file mode 100644 index 0000000000..496c3fbd83 --- /dev/null +++ b/api/internal/issuetrackersvc/driver.go @@ -0,0 +1,77 @@ +package issuetrackersvc + +import ( + "context" + + "github.com/google/uuid" + + "github.com/vtmocanu/uzi/api/internal/issuetracker" +) + +// DriverFor builds a driver for a stored connection. It re-checks the strict +// base URL form and the allowlist against the stored row BEFORE opening the +// token, so removing a host from the allowlist stops use of existing rows +// (fail closed). The token is opened with the row's user, kind and base URL as +// AAD, used only to build the driver, and never returned or logged. +func (s *Service) DriverFor(ctx context.Context, userID, connID uuid.UUID, project issuetracker.ProjectRef, stateField, open, closed string) (issuetracker.Driver, error) { + row, err := s.getRow(ctx, userID, connID) + if err != nil { + return nil, err + } + kind, ok := issuetracker.ParseKind(row.Kind) + if !ok { + s.log.Error("stored issue tracker connection has an unknown kind", "connection", connID.String(), "kind", row.Kind) + return nil, internal() + } + norm, err := strictBaseURL(row.BaseUrl) + if err != nil || norm != row.BaseUrl { + return nil, &Error{Kind: KindBadRequest, Message: "base URL is not valid for an issue tracker connection"} + } + if !s.allowed(norm) { + return nil, &Error{Kind: KindBadRequest, Message: "base URL is not in the allowed issue tracker list"} + } + token, err := openToken(s.box, userID, kind, norm, row.TokenCiphertext) + if err != nil { + s.log.Error("open issue tracker token", "connection", connID.String(), "error", err) + return nil, internal() + } + d, err := s.newDriver(kind, issuetracker.Options{ + BaseURL: norm, + Token: token, + Project: project, + StateField: stateField, + OpenState: open, + ClosedState: closed, + }) + if err != nil { + s.log.Error("build issue tracker driver", "kind", row.Kind, "error", err) + return nil, internal() + } + return d, nil +} + +// ListProjects lists the projects the connection's token can see. +func (s *Service) ListProjects(ctx context.Context, userID, connID uuid.UUID) ([]issuetracker.Project, error) { + d, err := s.DriverFor(ctx, userID, connID, issuetracker.ProjectRef{}, "", "", "") + if err != nil { + return nil, err + } + projects, err := d.ListProjects(ctx) + if err != nil { + return nil, &Error{Kind: KindBadGateway, Message: "could not list projects: " + err.Error()} + } + return projects, nil +} + +// ListStates lists the issue states of one project of the connection. +func (s *Service) ListStates(ctx context.Context, userID, connID uuid.UUID, projectID string) ([]issuetracker.State, error) { + d, err := s.DriverFor(ctx, userID, connID, issuetracker.ProjectRef{ID: projectID}, "", "", "") + if err != nil { + return nil, err + } + states, err := d.ListStates(ctx, projectID) + if err != nil { + return nil, &Error{Kind: KindBadGateway, Message: "could not list states: " + err.Error()} + } + return states, nil +} diff --git a/api/internal/issuetrackersvc/errors.go b/api/internal/issuetrackersvc/errors.go new file mode 100644 index 0000000000..3cab1d6534 --- /dev/null +++ b/api/internal/issuetrackersvc/errors.go @@ -0,0 +1,29 @@ +package issuetrackersvc + +// Kind classifies a failure so callers can pick a status code. +type Kind int + +const ( + // KindBadRequest: the input is invalid or refused (HTTP 400). + KindBadRequest Kind = iota + // KindNotFound: the connection is absent or not owned by the user (404). + KindNotFound + // KindBadGateway: the issue tracker refused or failed (502). + KindBadGateway + // KindInternal: a database or crypto failure, already logged (500). + KindInternal +) + +// Error is the typed failure of every Service method. Message is safe to show +// a client: issue tracker text is token-redacted by the driver, and an +// internal failure carries only a fixed message. +type Error struct { + Kind Kind + Message string +} + +func (e *Error) Error() string { return e.Message } + +func internal() *Error { return &Error{Kind: KindInternal, Message: "internal error"} } + +func notFound() *Error { return &Error{Kind: KindNotFound, Message: "connection not found"} } diff --git a/api/internal/issuetrackersvc/read.go b/api/internal/issuetrackersvc/read.go new file mode 100644 index 0000000000..649988afc2 --- /dev/null +++ b/api/internal/issuetrackersvc/read.go @@ -0,0 +1,60 @@ +package issuetrackersvc + +import ( + "context" + "errors" + + "github.com/google/uuid" + "github.com/jackc/pgx/v5" + + "github.com/vtmocanu/uzi/api/internal/store" +) + +// List returns the user's connections, oldest first. +func (s *Service) List(ctx context.Context, userID uuid.UUID) ([]Connection, error) { + rows, err := s.q.ListIssueTrackerConnectionsByUser(ctx, userID) + if err != nil { + s.log.Error("list issue tracker connections", "error", err) + return nil, internal() + } + out := make([]Connection, 0, len(rows)) + for _, r := range rows { + out = append(out, toConnection(r)) + } + return out, nil +} + +// Get returns one of the user's connections; a missing or foreign id is NotFound. +func (s *Service) Get(ctx context.Context, userID, connID uuid.UUID) (Connection, error) { + row, err := s.getRow(ctx, userID, connID) + if err != nil { + return Connection{}, err + } + return toConnection(row), nil +} + +// Delete removes one of the user's connections; a missing or foreign id is NotFound. +func (s *Service) Delete(ctx context.Context, userID, connID uuid.UUID) error { + n, err := s.q.DeleteIssueTrackerConnectionForUser(ctx, store.DeleteIssueTrackerConnectionForUserParams{ID: connID, UserID: userID}) + if err != nil { + s.log.Error("delete issue tracker connection", "error", err) + return internal() + } + if n == 0 { + return notFound() + } + return nil +} + +// getRow loads the stored row, ciphertext included, for in-package use only. +func (s *Service) getRow(ctx context.Context, userID, connID uuid.UUID) (store.IssueTrackerConnection, error) { + row, err := s.q.GetIssueTrackerConnectionForUser(ctx, store.GetIssueTrackerConnectionForUserParams{ID: connID, UserID: userID}) + if errors.Is(err, pgx.ErrNoRows) { + return store.IssueTrackerConnection{}, notFound() + } + if err != nil { + s.log.Error("get issue tracker connection", "error", err) + return store.IssueTrackerConnection{}, internal() + } + return row, nil +} diff --git a/api/internal/issuetrackersvc/seal.go b/api/internal/issuetrackersvc/seal.go new file mode 100644 index 0000000000..f788de3c58 --- /dev/null +++ b/api/internal/issuetrackersvc/seal.go @@ -0,0 +1,34 @@ +package issuetrackersvc + +import ( + "github.com/google/uuid" + + "github.com/vtmocanu/uzi/api/internal/issuetracker" + "github.com/vtmocanu/uzi/api/internal/secretbox" +) + +// tokenAAD is the additional authenticated data bound into a sealed token: +// +// "uzi/issue-tracker-token/v1|" + userID + "|" + kind + "|" + baseURL +// +// with userID in canonical lowercase hyphenated form, kind the stored kind +// string, and baseURL the value strictBaseURL returns. The format is frozen: +// changing any part makes every stored ciphertext unreadable, so a new format +// takes a new version prefix and a re-seal migration. Binding the identity in +// means a ciphertext copied to another user's row, or another base URL, fails +// to open instead of sending one user's token to another host. +func tokenAAD(userID uuid.UUID, kind issuetracker.Kind, baseURL string) []byte { + return []byte("uzi/issue-tracker-token/v1|" + userID.String() + "|" + string(kind) + "|" + baseURL) +} + +func sealToken(box *secretbox.Box, userID uuid.UUID, kind issuetracker.Kind, baseURL, token string) ([]byte, error) { + return box.SealWithAAD([]byte(token), tokenAAD(userID, kind, baseURL)) +} + +func openToken(box *secretbox.Box, userID uuid.UUID, kind issuetracker.Kind, baseURL string, sealed []byte) (string, error) { + plain, err := box.OpenWithAAD(sealed, tokenAAD(userID, kind, baseURL)) + if err != nil { + return "", err + } + return string(plain), nil +} diff --git a/api/internal/issuetrackersvc/service.go b/api/internal/issuetrackersvc/service.go new file mode 100644 index 0000000000..8ed6b44e3a --- /dev/null +++ b/api/internal/issuetrackersvc/service.go @@ -0,0 +1,47 @@ +package issuetrackersvc + +import ( + "context" + "github.com/google/uuid" + "log/slog" + + "github.com/vtmocanu/uzi/api/internal/issuetracker" + "github.com/vtmocanu/uzi/api/internal/secretbox" + "github.com/vtmocanu/uzi/api/internal/store" +) + +// Querier is the slice of *store.Queries the service uses. +type Querier interface { + UpsertIssueTrackerConnection(ctx context.Context, arg store.UpsertIssueTrackerConnectionParams) (store.IssueTrackerConnection, error) + ListIssueTrackerConnectionsByUser(ctx context.Context, userID uuid.UUID) ([]store.IssueTrackerConnection, error) + GetIssueTrackerConnectionForUser(ctx context.Context, arg store.GetIssueTrackerConnectionForUserParams) (store.IssueTrackerConnection, error) + DeleteIssueTrackerConnectionForUser(ctx context.Context, arg store.DeleteIssueTrackerConnectionForUserParams) (int64, error) +} + +var _ Querier = (*store.Queries)(nil) + +// NewDriverFunc builds a driver; issuetracker.New is the production value. +type NewDriverFunc func(issuetracker.Kind, issuetracker.Options) (issuetracker.Driver, error) + +// Service owns the connection rules. It is safe for concurrent use when its +// Querier is. +type Service struct { + q Querier + box *secretbox.Box + allowed func(string) bool + newDriver NewDriverFunc + log *slog.Logger +} + +// New builds a Service. allowed reports whether a normalized base URL is on the +// issue tracker allowlist (config.Config.IssueTrackerBaseURLAllowed); a nil +// allowed refuses everything. A nil newDriver selects issuetracker.New. +func New(q Querier, box *secretbox.Box, allowed func(string) bool, newDriver NewDriverFunc) *Service { + if newDriver == nil { + newDriver = issuetracker.New + } + if allowed == nil { + allowed = func(string) bool { return false } + } + return &Service{q: q, box: box, allowed: allowed, newDriver: newDriver, log: slog.Default()} +} diff --git a/api/internal/issuetrackersvc/service_test.go b/api/internal/issuetrackersvc/service_test.go new file mode 100644 index 0000000000..7c756976e2 --- /dev/null +++ b/api/internal/issuetrackersvc/service_test.go @@ -0,0 +1,437 @@ +package issuetrackersvc + +import ( + "context" + "errors" + "io" + "log/slog" + "reflect" + "strings" + "sync" + "testing" + "time" + + "github.com/google/uuid" + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgtype" + + "github.com/vtmocanu/uzi/api/internal/issuetracker" + "github.com/vtmocanu/uzi/api/internal/issuetracker/issuetrackertest" + "github.com/vtmocanu/uzi/api/internal/secretbox" + "github.com/vtmocanu/uzi/api/internal/store" +) + +const testHost = "https://issues.example.com" + +// fakeQuerier is an in-memory Querier keyed like the real table. +type fakeQuerier struct { + mu sync.Mutex + rows []store.IssueTrackerConnection +} + +func (f *fakeQuerier) UpsertIssueTrackerConnection(_ context.Context, a store.UpsertIssueTrackerConnectionParams) (store.IssueTrackerConnection, error) { + f.mu.Lock() + defer f.mu.Unlock() + now := pgtype.Timestamptz{Time: time.Now(), Valid: true} + for i, r := range f.rows { + if r.UserID == a.UserID && r.Kind == a.Kind && r.BaseUrl == a.BaseUrl { + f.rows[i].AccountID, f.rows[i].AccountLogin = a.AccountID, a.AccountLogin + f.rows[i].TokenCiphertext, f.rows[i].LastVerifiedAt = a.TokenCiphertext, now + return f.rows[i], nil + } + } + r := store.IssueTrackerConnection{ + ID: uuid.New(), UserID: a.UserID, Kind: a.Kind, BaseUrl: a.BaseUrl, + AccountID: a.AccountID, AccountLogin: a.AccountLogin, TokenCiphertext: a.TokenCiphertext, + CreatedAt: now, LastVerifiedAt: now, + } + f.rows = append(f.rows, r) + return r, nil +} + +func (f *fakeQuerier) ListIssueTrackerConnectionsByUser(_ context.Context, u uuid.UUID) ([]store.IssueTrackerConnection, error) { + f.mu.Lock() + defer f.mu.Unlock() + var out []store.IssueTrackerConnection + for _, r := range f.rows { + if r.UserID == u { + out = append(out, r) + } + } + return out, nil +} + +func (f *fakeQuerier) GetIssueTrackerConnectionForUser(_ context.Context, a store.GetIssueTrackerConnectionForUserParams) (store.IssueTrackerConnection, error) { + f.mu.Lock() + defer f.mu.Unlock() + for _, r := range f.rows { + if r.ID == a.ID && r.UserID == a.UserID { + return r, nil + } + } + return store.IssueTrackerConnection{}, pgx.ErrNoRows +} + +func (f *fakeQuerier) DeleteIssueTrackerConnectionForUser(_ context.Context, a store.DeleteIssueTrackerConnectionForUserParams) (int64, error) { + f.mu.Lock() + defer f.mu.Unlock() + for i, r := range f.rows { + if r.ID == a.ID && r.UserID == a.UserID { + f.rows = append(f.rows[:i], f.rows[i+1:]...) + return 1, nil + } + } + return 0, nil +} + +// fakeDriver embeds BaseFake and overrides only what the service calls. +type fakeDriver struct { + issuetrackertest.BaseFake + opts issuetracker.Options + account issuetracker.Account + verifyErr error + projects []issuetracker.Project + projectErr error +} + +func (d *fakeDriver) VerifyToken(context.Context) (issuetracker.Account, error) { + return d.account, d.verifyErr +} + +func (d *fakeDriver) ListProjects(context.Context) ([]issuetracker.Project, error) { + return d.projects, d.projectErr +} + +// harness wires a Service to the fakes and records every driver build. +type harness struct { + svc *Service + q *fakeQuerier + box *secretbox.Box + allow map[string]bool + built []issuetracker.Options + template fakeDriver +} + +func newHarness(t *testing.T) *harness { + t.Helper() + key := make([]byte, secretbox.KeySize) + for i := range key { + key[i] = byte(i*7 + 1) + } + box, err := secretbox.New(key) + if err != nil { + t.Fatal(err) + } + h := &harness{q: &fakeQuerier{}, box: box, allow: map[string]bool{testHost: true}} + h.template.account = issuetracker.Account{ID: "1-1", Login: "bot"} + h.svc = New(h.q, box, func(u string) bool { return h.allow[u] }, func(_ issuetracker.Kind, o issuetracker.Options) (issuetracker.Driver, error) { + h.built = append(h.built, o) + d := h.template + d.opts = o + return &d, nil + }) + h.svc.log = slog.New(slog.NewTextHandler(io.Discard, nil)) + return h +} + +// testToken assembles a token-shaped value at runtime. +func testToken(parts ...string) string { return strings.Join(append([]string{"perm"}, parts...), "-") } + +func wantKind(t *testing.T, err error, want Kind) *Error { + t.Helper() + var e *Error + if !errors.As(err, &e) || e.Kind != want { + t.Fatalf("err = %v, want *Error of kind %d", err, want) + } + return e +} + +func TestConnectRefusesWhenAllowlistEmpty(t *testing.T) { + h := newHarness(t) + h.allow = map[string]bool{} + _, err := h.svc.Connect(context.Background(), uuid.New(), "youtrack", testHost, testToken("a", "b")) + wantKind(t, err, KindBadRequest) + if len(h.built) != 0 { + t.Fatalf("driver factory called %d times for a refused host", len(h.built)) + } + if len(h.q.rows) != 0 { + t.Fatal("a row was stored for a refused host") + } +} + +func TestConnectRefusesBadInput(t *testing.T) { + cases := []struct{ name, kind, url, token string }{ + {"unknown kind", "jira", testHost, "tok"}, + {"empty kind", "", testHost, "tok"}, + {"empty token", "youtrack", testHost, " "}, + {"http", "youtrack", "http://issues.example.com", "tok"}, + {"path", "youtrack", testHost + "/youtrack", "tok"}, + {"userinfo", "youtrack", "https://issues.example.com@evil.example.com", "tok"}, + {"userinfo with password", "youtrack", "https://u:p@issues.example.com", "tok"}, + {"query", "youtrack", testHost + "?a=b", "tok"}, + {"bare question mark", "youtrack", testHost + "?", "tok"}, + {"fragment", "youtrack", testHost + "#x", "tok"}, + {"no host", "youtrack", "https://", "tok"}, + {"not allowlisted", "youtrack", "https://evil.example.com", "tok"}, + {"explicit default port is a different origin string", "youtrack", testHost + ":443", "tok"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + h := newHarness(t) + _, err := h.svc.Connect(context.Background(), uuid.New(), tc.kind, tc.url, tc.token) + wantKind(t, err, KindBadRequest) + if len(h.built) != 0 { + t.Error("driver built for refused input") + } + }) + } +} + +func TestConnectNormalizesBeforeAllowlist(t *testing.T) { + h := newHarness(t) + c, err := h.svc.Connect(context.Background(), uuid.New(), "youtrack", " HTTPS://Issues.Example.com/ ", testToken("x")) + if err != nil { + t.Fatal(err) + } + if c.BaseURL != testHost { + t.Errorf("BaseURL = %q, want %q", c.BaseURL, testHost) + } + if len(h.built) != 1 || h.built[0].BaseURL != testHost { + t.Errorf("driver built with %+v, want the normalized URL", h.built) + } +} + +func TestConnectVerifyFailureIsBadGatewayWithoutToken(t *testing.T) { + h := newHarness(t) + tok := testToken("secret", "value") + h.template.verifyErr = errors.New("401 unauthorized") + _, err := h.svc.Connect(context.Background(), uuid.New(), "youtrack", testHost, tok) + e := wantKind(t, err, KindBadGateway) + if strings.Contains(e.Message, tok) || !strings.HasPrefix(e.Message, "token verification failed: ") { + t.Errorf("message = %q", e.Message) + } + if len(h.q.rows) != 0 { + t.Error("a row was stored after a failed verification") + } +} + +func TestConnectRefusesBadAccount(t *testing.T) { + bad := map[string]issuetracker.Account{ + "empty id": {ID: "", Login: "bot"}, + "empty login": {ID: "1", Login: ""}, + "control in id": {ID: "1\x00", Login: "bot"}, + "newline login": {ID: "1", Login: "bo\nt"}, + "bidi override": {ID: "1", Login: "bot‮"}, + "zero width": {ID: "1", Login: "bo​t"}, + "line separator": {ID: "1", Login: "bo
t"}, + "para separator": {ID: "1", Login: "bo
t"}, + "login too long": {ID: "1", Login: strings.Repeat("a", 256)}, + "id too long": {ID: strings.Repeat("1", 256), Login: "bot"}, + "control in both": {ID: "\x07", Login: "\x07"}, + } + for name, acct := range bad { + t.Run(name, func(t *testing.T) { + h := newHarness(t) + h.template.account = acct + _, err := h.svc.Connect(context.Background(), uuid.New(), "youtrack", testHost, testToken("t")) + wantKind(t, err, KindBadGateway) + if len(h.q.rows) != 0 { + t.Error("row stored for a bad account") + } + }) + } + h := newHarness(t) + h.template.account = issuetracker.Account{ID: strings.Repeat("1", 255), Login: "Zoë"} + if _, err := h.svc.Connect(context.Background(), uuid.New(), "youtrack", testHost, testToken("t")); err != nil { + t.Errorf("a 255-byte id and a non-ASCII login must pass: %v", err) + } +} + +func TestConnectStoresSealedTokenBoundToIdentity(t *testing.T) { + h := newHarness(t) + user := uuid.New() + tok := testToken("alpha", "beta") + c, err := h.svc.Connect(context.Background(), user, "youtrack", testHost, tok) + if err != nil { + t.Fatal(err) + } + if c.UserID != user || c.Kind != "youtrack" || c.AccountID != "1-1" || c.AccountLogin != "bot" { + t.Errorf("connection = %+v", c) + } + row := h.q.rows[0] + if strings.Contains(string(row.TokenCiphertext), tok) || len(row.TokenCiphertext) == 0 { + t.Fatal("ciphertext is empty or holds the token in the clear") + } + if h.built[0].Token != tok { + t.Error("the driver was not given the token") + } + + got, err := openToken(h.box, user, issuetracker.KindYouTrack, testHost, row.TokenCiphertext) + if err != nil || got != tok { + t.Fatalf("open with the right AAD = %q, %v", got, err) + } + if _, err := openToken(h.box, uuid.New(), issuetracker.KindYouTrack, testHost, row.TokenCiphertext); err == nil { + t.Error("opened with another user's AAD") + } + if _, err := openToken(h.box, user, issuetracker.KindYouTrack, "https://other.example.com", row.TokenCiphertext); err == nil { + t.Error("opened with another base URL's AAD") + } + if _, err := openToken(h.box, user, issuetracker.Kind("other"), testHost, row.TokenCiphertext); err == nil { + t.Error("opened with another kind's AAD") + } +} + +func TestConnectRotatesInPlace(t *testing.T) { + h := newHarness(t) + user := uuid.New() + a, err := h.svc.Connect(context.Background(), user, "youtrack", testHost, testToken("one")) + if err != nil { + t.Fatal(err) + } + first := string(h.q.rows[0].TokenCiphertext) + b, err := h.svc.Connect(context.Background(), user, "youtrack", testHost, testToken("two")) + if err != nil { + t.Fatal(err) + } + if a.ID != b.ID || len(h.q.rows) != 1 || string(h.q.rows[0].TokenCiphertext) == first { + t.Error("reconnect must rotate the single row in place") + } +} + +func TestConnectionHasNoSecretField(t *testing.T) { + typ := reflect.TypeOf(Connection{}) + for i := 0; i < typ.NumField(); i++ { + n := strings.ToLower(typ.Field(i).Name) + if strings.Contains(n, "token") || strings.Contains(n, "cipher") { + t.Errorf("Connection field %s could carry the secret", typ.Field(i).Name) + } + } +} + +func TestReadsAreOwnerScoped(t *testing.T) { + h := newHarness(t) + ctx := context.Background() + owner, other := uuid.New(), uuid.New() + c, err := h.svc.Connect(ctx, owner, "youtrack", testHost, testToken("t")) + if err != nil { + t.Fatal(err) + } + if got, err := h.svc.Get(ctx, owner, c.ID); err != nil || got.ID != c.ID { + t.Fatalf("owner get = %v, %v", got, err) + } + _, err = h.svc.Get(ctx, other, c.ID) + wantKind(t, err, KindNotFound) + if list, err := h.svc.List(ctx, other); err != nil || len(list) != 0 { + t.Errorf("foreign list = %v, %v", list, err) + } + if list, err := h.svc.List(ctx, owner); err != nil || len(list) != 1 { + t.Errorf("owner list = %v, %v", list, err) + } + wantKind(t, h.svc.Delete(ctx, other, c.ID), KindNotFound) + if _, err := h.svc.Get(ctx, owner, c.ID); err != nil { + t.Fatalf("foreign delete removed the row: %v", err) + } + if err := h.svc.Delete(ctx, owner, c.ID); err != nil { + t.Fatal(err) + } + wantKind(t, h.svc.Delete(ctx, owner, c.ID), KindNotFound) +} + +func TestDriverForOpensTokenAndPassesOptions(t *testing.T) { + h := newHarness(t) + ctx := context.Background() + user := uuid.New() + tok := testToken("open", "me") + c, err := h.svc.Connect(ctx, user, "youtrack", testHost, tok) + if err != nil { + t.Fatal(err) + } + h.built = nil + ref := issuetracker.ProjectRef{ID: "0-1", ShortName: "UZI"} + if _, err := h.svc.DriverFor(ctx, user, c.ID, ref, "State", "Open", "Done"); err != nil { + t.Fatal(err) + } + want := issuetracker.Options{BaseURL: testHost, Token: tok, Project: ref, StateField: "State", OpenState: "Open", ClosedState: "Done"} + if len(h.built) != 1 || h.built[0] != want { + t.Errorf("built = %+v, want %+v", h.built, want) + } + _, err = h.svc.DriverFor(ctx, uuid.New(), c.ID, ref, "", "", "") + wantKind(t, err, KindNotFound) +} + +func TestDriverForFailsClosedWhenAllowlistEmptied(t *testing.T) { + h := newHarness(t) + ctx := context.Background() + user := uuid.New() + c, err := h.svc.Connect(ctx, user, "youtrack", testHost, testToken("t")) + if err != nil { + t.Fatal(err) + } + h.built = nil + h.allow = map[string]bool{} + _, err = h.svc.DriverFor(ctx, user, c.ID, issuetracker.ProjectRef{}, "", "", "") + wantKind(t, err, KindBadRequest) + if len(h.built) != 0 { + t.Error("driver built after the allowlist was emptied") + } + // Discovery goes through the same gate. + _, err = h.svc.ListProjects(ctx, user, c.ID) + wantKind(t, err, KindBadRequest) +} + +func TestDriverForRefusesTamperedRow(t *testing.T) { + h := newHarness(t) + ctx := context.Background() + user := uuid.New() + c, err := h.svc.Connect(ctx, user, "youtrack", testHost, testToken("t")) + if err != nil { + t.Fatal(err) + } + h.built = nil + // A row whose base URL was rewritten no longer opens: the AAD differs. + h.allow["https://evil.example.com"] = true + h.q.rows[0].BaseUrl = "https://evil.example.com" + _, err = h.svc.DriverFor(ctx, user, c.ID, issuetracker.ProjectRef{}, "", "", "") + e := wantKind(t, err, KindInternal) + if e.Message != "internal error" || len(h.built) != 0 { + t.Errorf("message = %q, built = %d", e.Message, len(h.built)) + } + // A row with a non-strict URL is refused before opening. + h.allow["https://evil.example.com/x"] = true + h.q.rows[0].BaseUrl = "https://evil.example.com/x" + _, err = h.svc.DriverFor(ctx, user, c.ID, issuetracker.ProjectRef{}, "", "", "") + wantKind(t, err, KindBadRequest) +} + +func TestListProjectsAndStates(t *testing.T) { + h := newHarness(t) + ctx := context.Background() + user := uuid.New() + c, err := h.svc.Connect(ctx, user, "youtrack", testHost, testToken("t")) + if err != nil { + t.Fatal(err) + } + h.template.projects = []issuetracker.Project{{ID: "0-1", ShortName: "UZI", Name: "Uzi"}} + got, err := h.svc.ListProjects(ctx, user, c.ID) + if err != nil || len(got) != 1 || got[0].ShortName != "UZI" { + t.Fatalf("ListProjects = %v, %v", got, err) + } + h.template.projectErr = errors.New("boom") + _, err = h.svc.ListProjects(ctx, user, c.ID) + wantKind(t, err, KindBadGateway) + // ListStates is not stubbed on the double: its error surfaces as a bad gateway, not a panic. + _, err = h.svc.ListStates(ctx, user, c.ID, "0-1") + wantKind(t, err, KindBadGateway) + _, err = h.svc.ListStates(ctx, uuid.New(), c.ID, "0-1") + wantKind(t, err, KindNotFound) +} + +func TestNewDefaultsToRegistry(t *testing.T) { + s := New(&fakeQuerier{}, nil, nil, nil) + if s.newDriver == nil { + t.Fatal("nil newDriver must default to issuetracker.New") + } + if _, err := s.Connect(context.Background(), uuid.New(), "youtrack", testHost, "tok"); err == nil { + t.Fatal("a nil allowlist must refuse") + } +} diff --git a/api/internal/store/issue_tracker_connections.sql.go b/api/internal/store/issue_tracker_connections.sql.go new file mode 100644 index 0000000000..ce742513dc --- /dev/null +++ b/api/internal/store/issue_tracker_connections.sql.go @@ -0,0 +1,139 @@ +// Code generated by sqlc. DO NOT EDIT. +// versions: +// sqlc v1.31.1 +// source: issue_tracker_connections.sql + +package store + +import ( + "context" + + "github.com/google/uuid" +) + +const deleteIssueTrackerConnectionForUser = `-- name: DeleteIssueTrackerConnectionForUser :execrows +DELETE FROM issue_tracker_connections WHERE id = $1 AND user_id = $2 +` + +type DeleteIssueTrackerConnectionForUserParams struct { + ID uuid.UUID `json:"id"` + UserID uuid.UUID `json:"user_id"` +} + +func (q *Queries) DeleteIssueTrackerConnectionForUser(ctx context.Context, arg DeleteIssueTrackerConnectionForUserParams) (int64, error) { + result, err := q.db.Exec(ctx, deleteIssueTrackerConnectionForUser, arg.ID, arg.UserID) + if err != nil { + return 0, err + } + return result.RowsAffected(), nil +} + +const getIssueTrackerConnectionForUser = `-- name: GetIssueTrackerConnectionForUser :one +SELECT id, user_id, kind, base_url, account_id, account_login, token_ciphertext, created_at, last_verified_at FROM issue_tracker_connections WHERE id = $1 AND user_id = $2 +` + +type GetIssueTrackerConnectionForUserParams struct { + ID uuid.UUID `json:"id"` + UserID uuid.UUID `json:"user_id"` +} + +func (q *Queries) GetIssueTrackerConnectionForUser(ctx context.Context, arg GetIssueTrackerConnectionForUserParams) (IssueTrackerConnection, error) { + row := q.db.QueryRow(ctx, getIssueTrackerConnectionForUser, arg.ID, arg.UserID) + var i IssueTrackerConnection + err := row.Scan( + &i.ID, + &i.UserID, + &i.Kind, + &i.BaseUrl, + &i.AccountID, + &i.AccountLogin, + &i.TokenCiphertext, + &i.CreatedAt, + &i.LastVerifiedAt, + ) + return i, err +} + +const listIssueTrackerConnectionsByUser = `-- name: ListIssueTrackerConnectionsByUser :many +SELECT id, user_id, kind, base_url, account_id, account_login, token_ciphertext, created_at, last_verified_at FROM issue_tracker_connections WHERE user_id = $1 ORDER BY created_at ASC, id ASC +` + +func (q *Queries) ListIssueTrackerConnectionsByUser(ctx context.Context, userID uuid.UUID) ([]IssueTrackerConnection, error) { + rows, err := q.db.Query(ctx, listIssueTrackerConnectionsByUser, userID) + if err != nil { + return nil, err + } + defer rows.Close() + items := []IssueTrackerConnection{} + for rows.Next() { + var i IssueTrackerConnection + if err := rows.Scan( + &i.ID, + &i.UserID, + &i.Kind, + &i.BaseUrl, + &i.AccountID, + &i.AccountLogin, + &i.TokenCiphertext, + &i.CreatedAt, + &i.LastVerifiedAt, + ); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const upsertIssueTrackerConnection = `-- name: UpsertIssueTrackerConnection :one + +INSERT INTO issue_tracker_connections ( + user_id, kind, base_url, account_id, account_login, token_ciphertext, last_verified_at +) VALUES ($1, $2, $3, $4, $5, $6, now()) +ON CONFLICT (user_id, kind, base_url) DO UPDATE +SET account_id = EXCLUDED.account_id, + account_login = EXCLUDED.account_login, + token_ciphertext = EXCLUDED.token_ciphertext, + last_verified_at = now() +RETURNING id, user_id, kind, base_url, account_id, account_login, token_ciphertext, created_at, last_verified_at +` + +type UpsertIssueTrackerConnectionParams struct { + UserID uuid.UUID `json:"user_id"` + Kind string `json:"kind"` + BaseUrl string `json:"base_url"` + AccountID string `json:"account_id"` + AccountLogin string `json:"account_login"` + TokenCiphertext []byte `json:"token_ciphertext"` +} + +// Issue tracker connections ------------------------------------------------ +// Connect or reconnect/rotate: one connection per (user, kind, base_url). +// Reconnecting with a fresh token updates the ciphertext and account identity +// in place, keeps the row id, and stamps last_verified_at. +func (q *Queries) UpsertIssueTrackerConnection(ctx context.Context, arg UpsertIssueTrackerConnectionParams) (IssueTrackerConnection, error) { + row := q.db.QueryRow(ctx, upsertIssueTrackerConnection, + arg.UserID, + arg.Kind, + arg.BaseUrl, + arg.AccountID, + arg.AccountLogin, + arg.TokenCiphertext, + ) + var i IssueTrackerConnection + err := row.Scan( + &i.ID, + &i.UserID, + &i.Kind, + &i.BaseUrl, + &i.AccountID, + &i.AccountLogin, + &i.TokenCiphertext, + &i.CreatedAt, + &i.LastVerifiedAt, + ) + return i, err +} diff --git a/api/internal/store/issue_tracker_connections_livedb_test.go b/api/internal/store/issue_tracker_connections_livedb_test.go new file mode 100644 index 0000000000..8965c7a2ce --- /dev/null +++ b/api/internal/store/issue_tracker_connections_livedb_test.go @@ -0,0 +1,150 @@ +package store_test + +import ( + "bytes" + "errors" + "testing" + + "github.com/google/uuid" + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgconn" + "github.com/vtmocanu/uzi/api/internal/store" +) + +// TestIssueTrackerConnectionLiveDB covers the issue_tracker_connections table and its queries +// (PRD #4173): upsert rotates in place, reads and deletes are owner-scoped, the unique key and the +// https CHECK hold, and deleting the user cascades. +func TestIssueTrackerConnectionLiveDB(t *testing.T) { + ctx, dsn := standIsolatedDB(t, "issue_tracker_conn_") + if err := store.Migrate(ctx, dsn); err != nil { + t.Fatalf("migrate: %v", err) + } + pool, err := store.OpenPool(ctx, dsn) + if err != nil { + t.Fatal(err) + } + defer pool.Close() + q := store.New(pool) + + owner, other := uuid.New(), uuid.New() + for _, id := range []uuid.UUID{owner, other} { + mustExec(ctx, t, pool, "INSERT INTO users(id,email,password_hash) VALUES($1,$2,'x')", id, id.String()+"@example.com") + } + params := func(user uuid.UUID, base string, ct string) store.UpsertIssueTrackerConnectionParams { + return store.UpsertIssueTrackerConnectionParams{ + UserID: user, Kind: "youtrack", BaseUrl: base, + AccountID: "1-1", AccountLogin: "bot", TokenCiphertext: []byte(ct), + } + } + + first, err := q.UpsertIssueTrackerConnection(ctx, params(owner, "https://issues.example.com", "cipher-one")) + if err != nil { + t.Fatalf("upsert: %v", err) + } + + t.Run("rotate keeps the row id and replaces the ciphertext", func(t *testing.T) { + p := params(owner, "https://issues.example.com", "cipher-two") + p.AccountLogin = "renamed" + second, err := q.UpsertIssueTrackerConnection(ctx, p) + if err != nil { + t.Fatalf("upsert again: %v", err) + } + if second.ID != first.ID { + t.Errorf("rotate changed the row id: %s -> %s", first.ID, second.ID) + } + if !bytes.Equal(second.TokenCiphertext, []byte("cipher-two")) || second.AccountLogin != "renamed" { + t.Errorf("rotate did not update the row: %+v", second) + } + if !second.CreatedAt.Time.Equal(first.CreatedAt.Time) { + t.Error("rotate changed created_at") + } + if second.LastVerifiedAt.Time.Before(first.LastVerifiedAt.Time) { + t.Error("rotate moved last_verified_at backwards") + } + list, err := q.ListIssueTrackerConnectionsByUser(ctx, owner) + if err != nil || len(list) != 1 { + t.Errorf("list = %d rows, %v; want exactly one", len(list), err) + } + }) + + t.Run("a different base URL is a second connection", func(t *testing.T) { + second, err := q.UpsertIssueTrackerConnection(ctx, params(owner, "https://other.example.com", "c")) + if err != nil { + t.Fatal(err) + } + if second.ID == first.ID { + t.Error("distinct base URL reused the row") + } + list, err := q.ListIssueTrackerConnectionsByUser(ctx, owner) + if err != nil || len(list) != 2 || list[0].ID != first.ID { + t.Errorf("list = %d rows, %v; want two ordered by creation", len(list), err) + } + }) + + t.Run("reads and deletes are owner scoped", func(t *testing.T) { + if _, err := q.GetIssueTrackerConnectionForUser(ctx, store.GetIssueTrackerConnectionForUserParams{ID: first.ID, UserID: other}); !errors.Is(err, pgx.ErrNoRows) { + t.Errorf("get as another user = %v, want ErrNoRows", err) + } + n, err := q.DeleteIssueTrackerConnectionForUser(ctx, store.DeleteIssueTrackerConnectionForUserParams{ID: first.ID, UserID: other}) + if err != nil || n != 0 { + t.Errorf("delete as another user = %d, %v; want 0 rows", n, err) + } + if _, err := q.GetIssueTrackerConnectionForUser(ctx, store.GetIssueTrackerConnectionForUserParams{ID: first.ID, UserID: owner}); err != nil { + t.Errorf("owner lost the row after a foreign delete: %v", err) + } + list, err := q.ListIssueTrackerConnectionsByUser(ctx, other) + if err != nil || len(list) != 0 { + t.Errorf("another user lists %d rows, %v", len(list), err) + } + }) + + t.Run("the same target for another user is its own row", func(t *testing.T) { + row, err := q.UpsertIssueTrackerConnection(ctx, params(other, "https://issues.example.com", "c")) + if err != nil || row.ID == first.ID { + t.Errorf("upsert for another user = %v, %v", row.ID, err) + } + }) + + t.Run("the unique key is enforced", func(t *testing.T) { + _, err := pool.Exec(ctx, `INSERT INTO issue_tracker_connections(user_id,kind,base_url,account_id,account_login,token_ciphertext) VALUES($1,'youtrack','https://issues.example.com','a','b','c')`, owner) + var pgErr *pgconn.PgError + if !errors.As(err, &pgErr) || pgErr.Code != "23505" { + t.Errorf("duplicate insert = %v, want unique violation", err) + } + }) + + t.Run("the https CHECK refuses other schemes", func(t *testing.T) { + for _, base := range []string{"http://issues.example.com", "ftp://x", "issues.example.com"} { + _, err := q.UpsertIssueTrackerConnection(ctx, params(owner, base, "c")) + var pgErr *pgconn.PgError + if !errors.As(err, &pgErr) || pgErr.Code != "23514" { + t.Errorf("base_url %q = %v, want check violation", base, err) + } + } + }) + + t.Run("the kind CHECK refuses unknown kinds", func(t *testing.T) { + p := params(owner, "https://k.example.com", "c") + p.Kind = "jira" + _, err := q.UpsertIssueTrackerConnection(ctx, p) + var pgErr *pgconn.PgError + if !errors.As(err, &pgErr) || pgErr.Code != "23514" { + t.Errorf("kind jira = %v, want check violation", err) + } + }) + + t.Run("owner delete removes the row", func(t *testing.T) { + n, err := q.DeleteIssueTrackerConnectionForUser(ctx, store.DeleteIssueTrackerConnectionForUserParams{ID: first.ID, UserID: owner}) + if err != nil || n != 1 { + t.Errorf("delete = %d, %v; want 1", n, err) + } + }) + + t.Run("deleting the user cascades", func(t *testing.T) { + mustExec(ctx, t, pool, "DELETE FROM users WHERE id=$1", other) + var n int + if err := pool.QueryRow(ctx, "SELECT count(*) FROM issue_tracker_connections WHERE user_id=$1", other).Scan(&n); err != nil || n != 0 { + t.Errorf("rows after user delete = %d, %v; want 0", n, err) + } + }) +} diff --git a/api/internal/store/migrations/00424_issue_tracker_connections.sql b/api/internal/store/migrations/00424_issue_tracker_connections.sql new file mode 100644 index 0000000000..aad40e3784 --- /dev/null +++ b/api/internal/store/migrations/00424_issue_tracker_connections.sql @@ -0,0 +1,21 @@ +-- +goose Up + +-- A user's connection to an external issue tracker (PRD #4173) via their own +-- permanent token. The token is stored encrypted (AES-256-GCM, with the user, +-- kind and base_url bound in as additional authenticated data); a DB dump +-- alone cannot recover it. One connection per (user, kind, base_url). +CREATE TABLE issue_tracker_connections ( + id uuid PRIMARY KEY DEFAULT gen_random_uuid(), + user_id uuid NOT NULL REFERENCES users ON DELETE CASCADE, + kind text NOT NULL CHECK (kind IN ('youtrack')), + base_url text NOT NULL CHECK (base_url LIKE 'https://%'), -- must be in ISSUE_TRACKER_ALLOWED_BASE_URLS + account_id text NOT NULL, + account_login text NOT NULL, + token_ciphertext bytea NOT NULL, -- AES-256-GCM(token), AAD-bound + created_at timestamptz NOT NULL DEFAULT now(), + last_verified_at timestamptz NOT NULL DEFAULT now(), + UNIQUE (user_id, kind, base_url) +); + +-- +goose Down +DROP TABLE issue_tracker_connections; diff --git a/api/internal/store/models.go b/api/internal/store/models.go index 96ba38dd38..15c5f9d29d 100644 --- a/api/internal/store/models.go +++ b/api/internal/store/models.go @@ -599,6 +599,18 @@ type IssueProposal struct { ConfirmingSince pgtype.Timestamptz `json:"confirming_since"` } +type IssueTrackerConnection struct { + ID uuid.UUID `json:"id"` + UserID uuid.UUID `json:"user_id"` + Kind string `json:"kind"` + BaseUrl string `json:"base_url"` + AccountID string `json:"account_id"` + AccountLogin string `json:"account_login"` + TokenCiphertext []byte `json:"token_ciphertext"` + CreatedAt pgtype.Timestamptz `json:"created_at"` + LastVerifiedAt pgtype.Timestamptz `json:"last_verified_at"` +} + type JobFile struct { ID uuid.UUID `json:"id"` UserID uuid.UUID `json:"user_id"` diff --git a/api/internal/store/queries/issue_tracker_connections.sql b/api/internal/store/queries/issue_tracker_connections.sql new file mode 100644 index 0000000000..625b01d4e6 --- /dev/null +++ b/api/internal/store/queries/issue_tracker_connections.sql @@ -0,0 +1,24 @@ +-- Issue tracker connections ------------------------------------------------ + +-- name: UpsertIssueTrackerConnection :one +-- Connect or reconnect/rotate: one connection per (user, kind, base_url). +-- Reconnecting with a fresh token updates the ciphertext and account identity +-- in place, keeps the row id, and stamps last_verified_at. +INSERT INTO issue_tracker_connections ( + user_id, kind, base_url, account_id, account_login, token_ciphertext, last_verified_at +) VALUES ($1, $2, $3, $4, $5, $6, now()) +ON CONFLICT (user_id, kind, base_url) DO UPDATE +SET account_id = EXCLUDED.account_id, + account_login = EXCLUDED.account_login, + token_ciphertext = EXCLUDED.token_ciphertext, + last_verified_at = now() +RETURNING *; + +-- name: ListIssueTrackerConnectionsByUser :many +SELECT * FROM issue_tracker_connections WHERE user_id = $1 ORDER BY created_at ASC, id ASC; + +-- name: GetIssueTrackerConnectionForUser :one +SELECT * FROM issue_tracker_connections WHERE id = $1 AND user_id = $2; + +-- name: DeleteIssueTrackerConnectionForUser :execrows +DELETE FROM issue_tracker_connections WHERE id = $1 AND user_id = $2; From e9d8bb09ff9ead120208f623913bd2fe991652c3 Mon Sep 17 00:00:00 2001 From: uzi-agent Date: Wed, 11 Nov 2026 09:30:48 +0000 Subject: [PATCH 05/17] issuetracker: drive Connect end to end through the YouTrack driver Co-Authored-By: Claude Opus 5.5 --- .../issuetracker/connection_service_test.go | 216 ++++++++++++++++++ 1 file changed, 216 insertions(+) create mode 100644 api/internal/issuetracker/connection_service_test.go diff --git a/api/internal/issuetracker/connection_service_test.go b/api/internal/issuetracker/connection_service_test.go new file mode 100644 index 0000000000..e63874ed92 --- /dev/null +++ b/api/internal/issuetracker/connection_service_test.go @@ -0,0 +1,216 @@ +package issuetracker_test + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "net/http" + "net/http/httptest" + "net/url" + "strings" + "sync" + "sync/atomic" + "testing" + "unicode" + + "github.com/google/uuid" + + "github.com/vtmocanu/uzi/api/internal/issuetracker" + "github.com/vtmocanu/uzi/api/internal/issuetracker/youtracktest" + "github.com/vtmocanu/uzi/api/internal/issuetrackersvc" + "github.com/vtmocanu/uzi/api/internal/secretbox" + "github.com/vtmocanu/uzi/api/internal/store" +) + +// memQuerier is a minimal in-memory issuetrackersvc.Querier: only the upsert +// Connect uses is real, the rest are unreachable from these tests. +type memQuerier struct { + mu sync.Mutex + rows []store.IssueTrackerConnection +} + +func (m *memQuerier) UpsertIssueTrackerConnection(_ context.Context, a store.UpsertIssueTrackerConnectionParams) (store.IssueTrackerConnection, error) { + m.mu.Lock() + defer m.mu.Unlock() + r := store.IssueTrackerConnection{ + ID: uuid.New(), UserID: a.UserID, Kind: a.Kind, BaseUrl: a.BaseUrl, + AccountID: a.AccountID, AccountLogin: a.AccountLogin, TokenCiphertext: a.TokenCiphertext, + } + m.rows = append(m.rows, r) + return r, nil +} + +func (m *memQuerier) ListIssueTrackerConnectionsByUser(context.Context, uuid.UUID) ([]store.IssueTrackerConnection, error) { + return nil, errors.New("not used") +} + +func (m *memQuerier) GetIssueTrackerConnectionForUser(context.Context, store.GetIssueTrackerConnectionForUserParams) (store.IssueTrackerConnection, error) { + return store.IssueTrackerConnection{}, errors.New("not used") +} + +func (m *memQuerier) DeleteIssueTrackerConnectionForUser(context.Context, store.DeleteIssueTrackerConnectionForUserParams) (int64, error) { + return 0, errors.New("not used") +} + +func (m *memQuerier) stored() []store.IssueTrackerConnection { + m.mu.Lock() + defer m.mu.Unlock() + return append([]store.IssueTrackerConnection(nil), m.rows...) +} + +type svcHarness struct { + svc *issuetrackersvc.Service + q *memQuerier + builds *atomic.Int64 +} + +// newSvcHarness wires a Service whose driver factory builds the real YouTrack +// driver over base, counting factory calls. +func newSvcHarness(t *testing.T, base http.RoundTripper, allowed func(string) bool) svcHarness { + t.Helper() + key := make([]byte, secretbox.KeySize) + for i := range key { + key[i] = byte(i*5 + 3) + } + box, err := secretbox.New(key) + if err != nil { + t.Fatal(err) + } + q := &memQuerier{} + builds := &atomic.Int64{} + svc := issuetrackersvc.New(q, box, allowed, func(_ issuetracker.Kind, o issuetracker.Options) (issuetracker.Driver, error) { + builds.Add(1) + return issuetracker.NewYouTrackOver(o, base) + }) + return svcHarness{svc: svc, q: q, builds: builds} +} + +func allowOnly(u string) func(string) bool { + return func(got string) bool { return got == u } +} + +func connKind(t *testing.T) string { + t.Helper() + ks := issuetracker.SupportedKindsList() + if !strings.Contains(ks, "youtrack") { + t.Fatalf("youtrack is not a supported kind: %q", ks) + } + return "youtrack" +} + +func svcCredential() string { + return strings.Join([]string{"perm", ":", "ab c/d\"e", "-", "Zq9xLm2VnB7wKp3T"}, "") +} + +func svcTokenForms() []string { + c := svcCredential() + j, _ := json.Marshal(c) + return []string{c, url.QueryEscape(c), url.PathEscape(c), string(j[1 : len(j)-1])} +} + +func assertSvcScrubbed(t *testing.T, err error) { + t.Helper() + var se *issuetrackersvc.Error + if !errors.As(err, &se) { + t.Fatalf("want *issuetrackersvc.Error, got %T: %v", err, err) + } + for _, f := range svcTokenForms() { + if strings.Contains(se.Message, f) { + t.Errorf("message contains token form %q: %q", f, se.Message) + } + } + for _, r := range se.Message { + if unicode.IsControl(r) || unicode.Is(unicode.Cf, r) { + t.Errorf("message contains control or format character %U: %q", r, se.Message) + break + } + } +} + +func TestConnectYouTrackHappyPath(t *testing.T) { + srv := youtracktest.NewServer(t) + h := newSvcHarness(t, srv.Client().Transport, allowOnly(srv.URL)) + + conn, err := h.svc.Connect(context.Background(), uuid.New(), connKind(t), srv.URL, srv.Token) + if err != nil { + t.Fatalf("Connect: %v", err) + } + if conn.AccountLogin != youtracktest.BotLogin { + t.Errorf("AccountLogin = %q, want %q", conn.AccountLogin, youtracktest.BotLogin) + } + rows := h.q.stored() + if len(rows) != 1 { + t.Fatalf("stored %d rows, want 1", len(rows)) + } + if len(rows[0].TokenCiphertext) == 0 || bytes.Contains(rows[0].TokenCiphertext, []byte(srv.Token)) { + t.Error("ciphertext is empty or contains the token") + } +} + +func TestConnectTokenEchoIsScrubbed(t *testing.T) { + c := svcCredential() + j, _ := json.Marshal(c) + echo := strings.Join([]string{ + c, url.QueryEscape(c), string(j[1 : len(j)-1]), + }, " | ") + "\r\n\x1b[31m‮" + srv := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Type", "text/plain") + w.WriteHeader(http.StatusUnauthorized) + _, _ = w.Write([]byte(echo)) + })) + t.Cleanup(srv.Close) + h := newSvcHarness(t, srv.Client().Transport, allowOnly(srv.URL)) + + _, err := h.svc.Connect(context.Background(), uuid.New(), connKind(t), srv.URL, c) + if err == nil { + t.Fatal("want an error") + } + assertSvcScrubbed(t, err) + if n := len(h.q.stored()); n != 0 { + t.Errorf("stored %d rows, want 0", n) + } +} + +func TestConnectRefusesRedirect(t *testing.T) { + var hits atomic.Int64 + other := httptest.NewTLSServer(http.HandlerFunc(func(http.ResponseWriter, *http.Request) { hits.Add(1) })) + t.Cleanup(other.Close) + srv := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + http.Redirect(w, r, other.URL+r.URL.Path, http.StatusFound) + })) + t.Cleanup(srv.Close) + h := newSvcHarness(t, srv.Client().Transport, allowOnly(srv.URL)) + + _, err := h.svc.Connect(context.Background(), uuid.New(), connKind(t), srv.URL, svcCredential()) + if err == nil { + t.Fatal("want an error") + } + assertSvcScrubbed(t, err) + if n := hits.Load(); n != 0 { + t.Errorf("redirect target got %d hits, want 0", n) + } + if n := len(h.q.stored()); n != 0 { + t.Errorf("stored %d rows, want 0", n) + } +} + +func TestConnectEmptyAllowlistRefusesBeforeDriver(t *testing.T) { + srv := youtracktest.NewServer(t) + h := newSvcHarness(t, srv.Client().Transport, func(string) bool { return false }) + + _, err := h.svc.Connect(context.Background(), uuid.New(), connKind(t), srv.URL, srv.Token) + var se *issuetrackersvc.Error + if !errors.As(err, &se) || se.Kind != issuetrackersvc.KindBadRequest { + t.Fatalf("want a bad-request *issuetrackersvc.Error, got %v", err) + } + if n := h.builds.Load(); n != 0 { + t.Errorf("driver factory called %d times, want 0", n) + } + if n := len(srv.Requests()); n != 0 { + t.Errorf("server got %d requests, want 0", n) + } + if n := len(h.q.stored()); n != 0 { + t.Errorf("stored %d rows, want 0", n) + } +} From e2792965d600c7a8eaffcaa01c28afd2074ea8c1 Mon Sep 17 00:00:00 2001 From: uzi-agent Date: Wed, 11 Nov 2026 09:35:55 +0000 Subject: [PATCH 06/17] issuetracker: close the m2 review findings Nested tags count toward the item ceiling and are capped per issue and activity; error bodies are scrubbed before they are cut; every id taken from a response is validated before it reaches a path; the token has an upper bound and no control characters; and tests now pin every claimed behaviour the mutation pass found unpinned. Co-Authored-By: Claude Opus 5.5 --- .../issuetracker/issuetrackertest/contract.go | 22 +++ api/internal/issuetracker/redact.go | 10 + api/internal/issuetracker/redact_test.go | 24 +++ .../issuetracker/youtrack_comments_test.go | 35 ++++ .../issuetracker/youtrack_contract_test.go | 3 + .../issuetracker/youtrack_events_test.go | 74 +++++++ .../issuetracker/youtrack_issue_resolve.go | 6 +- .../youtrack_issue_resolve_test.go | 78 ++++++++ api/internal/issuetracker/youtrack_issues.go | 4 +- .../issuetracker/youtrack_issues_test.go | 116 +++++++++++ api/internal/issuetracker/youtrack_labels.go | 12 +- .../issuetracker/youtrack_labels_test.go | 147 ++++++++++++++ .../issuetracker/youtrack_projects_test.go | 26 +++ api/internal/issuetracker/youtrack_query.go | 2 +- .../issuetracker/youtrack_query_test.go | 151 --------------- api/internal/issuetracker/youtrack_request.go | 51 ++++- .../issuetracker/youtrack_request_test.go | 135 +++++++++++++ .../issuetracker/youtrack_security_test.go | 8 +- .../issuetracker/youtrack_state_test.go | 51 +++++ api/internal/issuetracker/youtrack_test.go | 180 ++++++++++++++++++ api/internal/issuetracker/youtrack_wire.go | 26 ++- .../issuetracker/youtrack_wire_test.go | 20 ++ .../issuetracker/youtracktest/server.go | 10 +- .../issuetracker/youtracktest/server_test.go | 62 ++++++ 24 files changed, 1078 insertions(+), 175 deletions(-) create mode 100644 api/internal/issuetracker/youtrack_comments_test.go create mode 100644 api/internal/issuetracker/youtrack_events_test.go create mode 100644 api/internal/issuetracker/youtrack_issue_resolve_test.go create mode 100644 api/internal/issuetracker/youtrack_issues_test.go create mode 100644 api/internal/issuetracker/youtrack_labels_test.go create mode 100644 api/internal/issuetracker/youtrack_projects_test.go create mode 100644 api/internal/issuetracker/youtrack_request_test.go create mode 100644 api/internal/issuetracker/youtrack_state_test.go create mode 100644 api/internal/issuetracker/youtrack_wire_test.go create mode 100644 api/internal/issuetracker/youtracktest/server_test.go diff --git a/api/internal/issuetracker/issuetrackertest/contract.go b/api/internal/issuetracker/issuetrackertest/contract.go index 782b0bde72..abdc47c590 100644 --- a/api/internal/issuetracker/issuetrackertest/contract.go +++ b/api/internal/issuetracker/issuetrackertest/contract.go @@ -23,6 +23,9 @@ type Subject struct { // SeedLabelEvent records that username added ("add") or removed ("remove") // label on issue iid, bypassing the IssueTracker under test. SeedLabelEvent func(t *testing.T, iid int64, action, label, username string) + // SeedDeletedComment adds a comment that the issue tracker reports as + // deleted. Nil means the implementation has no such comment. + SeedDeletedComment func(t *testing.T, iid int64, username, body string) // MoveIssue moves issue iid to another project. Nil means issues do not // move on this implementation. MoveIssue func(t *testing.T, iid int64) @@ -199,6 +202,9 @@ func caseListIssues(t *testing.T, c *contract) { if len(limited) != 1 { t.Fatalf("Limit 1 returned %d issues", len(limited)) } + if limited[0] != both.IID && limited[0] != onlyOne.IID { + t.Errorf("Limit 1 returned %d, want one of the matches %d or %d", limited[0], both.IID, onlyOne.IID) + } } func caseUpdate(t *testing.T, c *contract) { @@ -274,6 +280,19 @@ func caseComments(t *testing.T, c *contract) { noteAt = i } } + if c.SeedDeletedComment != nil { + gone := unique("contract deleted") + c.SeedDeletedComment(t, issue.IID, "carol", gone) + again, err := c.IssueTracker.ListIssueComments(c.ctx, c.ProjectID, issue.IID) + if err != nil { + t.Fatalf("ListIssueComments: %v", err) + } + for _, cm := range again { + if cm.Body == gone { + t.Errorf("a deleted comment was listed: %+v", cm) + } + } + } if humanAt < 0 || noteAt < 0 { t.Fatalf("comments %+v are missing the seeded comment or the note", comments) } @@ -301,6 +320,9 @@ func caseLabelEvents(t *testing.T, c *contract) { if len(mine) != 2 { t.Fatalf("events for %q = %+v, want an add then a remove", label, mine) } + if mine[1].CreatedAt.Before(mine[0].CreatedAt) { + t.Errorf("events out of order: %v before %v", mine[1].CreatedAt, mine[0].CreatedAt) + } if mine[0].Action != "add" || mine[0].Username != "alice" { t.Errorf("first event = %+v, want add by alice", mine[0]) } diff --git a/api/internal/issuetracker/redact.go b/api/internal/issuetracker/redact.go index 3389e75070..e3c11b3a7e 100644 --- a/api/internal/issuetracker/redact.go +++ b/api/internal/issuetracker/redact.go @@ -22,6 +22,8 @@ const ( // minTokenLen is the shortest token a constructor accepts. A shorter secret // could not be scrubbed without mangling unrelated text. minTokenLen = 16 + // maxTokenLen is the longest token a constructor accepts. + maxTokenLen = 4096 // minSegmentLen is the shortest dot-separated token segment scrubbed alone. minSegmentLen = 8 ) @@ -32,6 +34,14 @@ func validateToken(token string) error { if len(token) < minTokenLen { return errors.New("issuetracker: token shorter than the minimum length") } + if len(token) > maxTokenLen { + return errors.New("issuetracker: token longer than the maximum length") + } + for _, c := range token { + if unicode.IsControl(c) || unicode.Is(unicode.Cf, c) { + return errors.New("issuetracker: token contains a control or format character") + } + } return nil } diff --git a/api/internal/issuetracker/redact_test.go b/api/internal/issuetracker/redact_test.go index 1d8446869c..0ea65e3367 100644 --- a/api/internal/issuetracker/redact_test.go +++ b/api/internal/issuetracker/redact_test.go @@ -233,6 +233,30 @@ func TestValidateToken(t *testing.T) { } } +func TestValidateTokenBoundsAndCharacters(t *testing.T) { + if err := validateToken(strings.Repeat("a", maxTokenLen)); err != nil { + t.Errorf("a token of the maximum length: %v", err) + } + cases := map[string]string{ + "too long": strings.Repeat("a", maxTokenLen+1), + "newline": strings.Repeat("a", 20) + "\n", + "nul": strings.Repeat("a", 20) + "\x00", + "zero width": strings.Repeat("a", 20) + "\u200b", + "bidi": strings.Repeat("a", 20) + "\u202e", + "delete": strings.Repeat("a", 20) + "\x7f", + } + for name, tok := range cases { + err := validateToken(tok) + if err == nil { + t.Errorf("%s: accepted", name) + continue + } + if strings.Contains(err.Error(), tok) { + t.Errorf("%s: error leaks the token", name) + } + } +} + // TestRedactorScrubsMixedCaseEscapes covers a percent-escaped token whose hex // digits are upper case in one escape and lower case in another. func TestRedactorScrubsMixedCaseEscapes(t *testing.T) { diff --git a/api/internal/issuetracker/youtrack_comments_test.go b/api/internal/issuetracker/youtrack_comments_test.go new file mode 100644 index 0000000000..65e3946714 --- /dev/null +++ b/api/internal/issuetracker/youtrack_comments_test.go @@ -0,0 +1,35 @@ +package issuetracker + +import ( + "context" + "net/http" + "testing" +) + +func TestDeletedCommentsAreDropped(t *testing.T) { + y, srv := newFakeDriver(t) + n := srv.SeedIssue("t", "") + srv.SeedComment(n, "alice", "first") + srv.SeedDeletedComment(n, "bob", "gone") + srv.SeedComment(n, "carol", "second") + got, err := y.ListIssueComments(context.Background(), 1, n) + if err != nil { + t.Fatal(err) + } + if len(got) != 2 || got[0].Body != "first" || got[1].Body != "second" { + t.Fatalf("comments = %+v, want first and second only", got) + } +} + +func TestCreateIssueNoteRefusesAMalformedCommentID(t *testing.T) { + y, _ := loggedDriver(t, func(w http.ResponseWriter, r *http.Request) { + if r.Method == http.MethodPost { + writeRaw(w, `{"id":"../x","text":"n"}`) + return + } + writeRaw(w, rawIssue("2-1", "UZI-1", 1, "0-1", "")) + }) + if _, err := y.CreateIssueNote(context.Background(), 1, 1, "n"); err == nil { + t.Fatal("CreateIssueNote accepted a malformed comment id") + } +} diff --git a/api/internal/issuetracker/youtrack_contract_test.go b/api/internal/issuetracker/youtrack_contract_test.go index 5388780a94..4b3eb58ef9 100644 --- a/api/internal/issuetracker/youtrack_contract_test.go +++ b/api/internal/issuetracker/youtrack_contract_test.go @@ -36,6 +36,9 @@ func TestYouTrackContract(t *testing.T) { srv.SeedTagActivity(iid, username, nil, []string{label}) } }, + SeedDeletedComment: func(t *testing.T, iid int64, username, body string) { + srv.SeedDeletedComment(iid, username, body) + }, MoveIssue: func(t *testing.T, iid int64) { srv.MoveIssue(iid, youtracktest.OtherProjectID) }, diff --git a/api/internal/issuetracker/youtrack_events_test.go b/api/internal/issuetracker/youtrack_events_test.go new file mode 100644 index 0000000000..dc587f75a3 --- /dev/null +++ b/api/internal/issuetracker/youtrack_events_test.go @@ -0,0 +1,74 @@ +package issuetracker + +import ( + "context" + "net/http" + "strings" + "testing" +) + +func TestLabelEventsOrdinalAndExpansion(t *testing.T) { + y, srv := newFakeDriver(t) + n := srv.SeedIssue("t", "") + srv.SeedTagActivity(n, "alice", []string{"a", "b"}, []string{"c"}) + srv.SeedTagActivity(n, "bob", []string{"d"}, nil) + got, err := y.ListIssueLabelEvents(context.Background(), 1, n) + if err != nil { + t.Fatal(err) + } + want := []struct { + id int64 + action, name string + user string + }{{1, "add", "a", "alice"}, {2, "add", "b", "alice"}, {3, "remove", "c", "alice"}, {4, "add", "d", "bob"}} + if len(got) != len(want) { + t.Fatalf("got %d events, want %d: %+v", len(got), len(want), got) + } + for i, w := range want { + e := got[i] + if e.ID != w.id || e.Action != w.action || e.LabelName != w.name || e.Username != w.user { + t.Errorf("event %d = %+v, want %+v", i, e, w) + } + } +} + +// activityHandler serves issue UZI-1 and the given activities body. +func activityHandler(activities string) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + if strings.HasSuffix(r.URL.Path, "/activities") { + writeRaw(w, activities) + return + } + writeRaw(w, rawIssue("2-1", "UZI-1", 1, "0-1", "")) + } +} + +func lowerMaxItems(t *testing.T, n int) { + t.Helper() + old := maxItems + maxItems = n + t.Cleanup(func() { maxItems = old }) +} + +func TestLabelEventsNestedTagBomb(t *testing.T) { + // 200k empty tags in one activity fit under listPageBytes. + body := `[{"id":"3-1","timestamp":1,"added":[` + strings.Repeat("{},", 199_999) + `{}]}]` + if int64(len(body)) >= listPageBytes { + t.Fatalf("fixture is %d bytes, not under the page cap", len(body)) + } + y, _ := loggedDriver(t, activityHandler(body)) + got, err := y.ListIssueLabelEvents(context.Background(), 1, 1) + if err == nil || got != nil { + t.Fatalf("got %d events, %v; want an error", len(got), err) + } +} + +func TestNestedTagsCountTowardTheItemCeiling(t *testing.T) { + lowerMaxItems(t, 5) + // Two activities, three tags each: two items, eight counting the tags. + act := `{"id":"3-1","timestamp":1,"added":[{"id":"5-1","name":"a"},{"id":"5-2","name":"b"}],"removed":[{"id":"5-3","name":"c"}]}` + y, _ := loggedDriver(t, activityHandler("["+act+","+act+"]")) + if _, err := y.ListIssueLabelEvents(context.Background(), 1, 1); err == nil { + t.Fatal("nested tags did not count toward the item ceiling") + } +} diff --git a/api/internal/issuetracker/youtrack_issue_resolve.go b/api/internal/issuetracker/youtrack_issue_resolve.go index ae1c6b482c..11204d61a9 100644 --- a/api/internal/issuetracker/youtrack_issue_resolve.go +++ b/api/internal/issuetracker/youtrack_issue_resolve.go @@ -15,6 +15,8 @@ import ( // *forge.IssueNotFoundError. YouTrack keeps answering an old SHORT-n after the // issue moves to another project, so this check is what makes a moved issue // read as gone. Follow-up calls address the issue by the returned entity id. +// The check is a read: YouTrack has no conditional write, so an issue moved +// after resolveIssue returns can still be written by the follow-up call. func (y *youTrack) resolveIssue(ctx context.Context, issueIID int64) (ytIssue, error) { p, err := y.requireProject() if err != nil { @@ -41,8 +43,8 @@ func (y *youTrack) resolveIssue(ctx context.Context, issueIID int64) (ytIssue, e if !y.inBoundProject(w) || w.NumberInProject != issueIID { return ytIssue{}, &forge.IssueNotFoundError{Err: y.fail("youtrack: issue %s is not in the bound project", readable)} } - if !entityIDRE.MatchString(w.ID) || !idReadableRE.MatchString(w.IDReadable) { - return ytIssue{}, y.fail("youtrack: issue with a malformed id in the response") + if err := y.checkIssue(w); err != nil { + return ytIssue{}, err } return w, nil } diff --git a/api/internal/issuetracker/youtrack_issue_resolve_test.go b/api/internal/issuetracker/youtrack_issue_resolve_test.go new file mode 100644 index 0000000000..0a651534d0 --- /dev/null +++ b/api/internal/issuetracker/youtrack_issue_resolve_test.go @@ -0,0 +1,78 @@ +package issuetracker + +import ( + "context" + "errors" + "net/http" + "testing" + + "github.com/vtmocanu/uzi/api/internal/forge" +) + +func isNotFound(err error) bool { + var nf *forge.IssueNotFoundError + return errors.As(err, &nf) +} + +func TestResolveIssueNumberMismatchIsNotFound(t *testing.T) { + y, _ := loggedDriver(t, func(w http.ResponseWriter, _ *http.Request) { + writeRaw(w, rawIssue("2-7", "UZI-7", 7, "0-1", "")) + }) + if _, err := y.GetIssue(context.Background(), 1, 5); !isNotFound(err) { + t.Fatalf("GetIssue = %v, want *forge.IssueNotFoundError", err) + } +} + +func TestResolveIssueOutOfRangeNumberSendsNothing(t *testing.T) { + y, log := loggedDriver(t, func(w http.ResponseWriter, _ *http.Request) { + writeRaw(w, rawIssue("2-1", "UZI-1", 1, "0-1", "")) + }) + for _, n := range []int64{0, -1} { + if _, err := y.GetIssue(context.Background(), 1, n); !isNotFound(err) { + t.Errorf("GetIssue(%d) = %v, want *forge.IssueNotFoundError", n, err) + } + } + if got := log.all(); len(got) != 0 { + t.Errorf("sent %v", got) + } +} + +// TestResolveIssueValidatesIDsItself uses a method that does not map the +// issue afterwards, so only resolveIssue stands between a bad id and a write. +func TestResolveIssueValidatesIDsItself(t *testing.T) { + for name, issue := range map[string]string{ + "entity id": rawIssue("../x", "UZI-1", 1, "0-1", ""), + "id-readable": rawIssue("2-1", "UZI-1/../x", 1, "0-1", ""), + } { + t.Run(name, func(t *testing.T) { + y, log := loggedDriver(t, func(w http.ResponseWriter, _ *http.Request) { writeRaw(w, issue) }) + err := y.UpdateIssueDescription(context.Background(), 1, 1, "d") + if err == nil || isNotFound(err) { + t.Fatalf("UpdateIssueDescription = %v, want a malformed-id error", err) + } + if got := log.writes(); len(got) != 0 { + t.Errorf("sent %v", got) + } + }) + } +} + +func TestCreateIssueReadBackMustBeTheCreatedIssueInTheBoundProject(t *testing.T) { + for name, readBack := range map[string]string{ + "other project": rawIssue("2-9", "UZI-9", 9, "0-2", ""), + "other id": rawIssue("2-8", "UZI-8", 8, "0-1", ""), + } { + t.Run(name, func(t *testing.T) { + y, _ := loggedDriver(t, func(w http.ResponseWriter, r *http.Request) { + if r.Method == http.MethodPost { + writeRaw(w, `{"id":"2-9"}`) + return + } + writeRaw(w, readBack) + }) + if _, err := y.CreateIssue(context.Background(), 1, "t", "d", nil); err == nil { + t.Fatal("CreateIssue returned an issue that failed the read-back check") + } + }) + } +} diff --git a/api/internal/issuetracker/youtrack_issues.go b/api/internal/issuetracker/youtrack_issues.go index 3e09d2942a..78f117b9ba 100644 --- a/api/internal/issuetracker/youtrack_issues.go +++ b/api/internal/issuetracker/youtrack_issues.go @@ -60,7 +60,9 @@ func (y *youTrack) GetIssue(ctx context.Context, _, issueIID int64) (forge.Issue // CreateIssue implements forge.IssueTracker. The issue is created first, then // each label is attached as a tag (created when missing), and the result is -// read back, so it has the shape GetIssue returns. +// read back, so it has the shape GetIssue returns. There is no rollback: when +// attaching a tag or the read-back fails, the error is returned (redacted) but +// the issue stays created. func (y *youTrack) CreateIssue(ctx context.Context, _ int64, title, description string, labels []string) (forge.Issue, error) { p, err := y.requireProject() if err != nil { diff --git a/api/internal/issuetracker/youtrack_issues_test.go b/api/internal/issuetracker/youtrack_issues_test.go new file mode 100644 index 0000000000..05c51b6779 --- /dev/null +++ b/api/internal/issuetracker/youtrack_issues_test.go @@ -0,0 +1,116 @@ +package issuetracker + +import ( + "context" + "net/http" + "slices" + "strings" + "testing" + "time" + + "github.com/vtmocanu/uzi/api/internal/forge" +) + +func TestListIssuesUpdatedAfterAndLimit(t *testing.T) { + y, srv := newFakeDriver(t) + ctx := context.Background() + first := srv.SeedIssue("one", "") + second := srv.SeedIssue("two", "") + all, err := y.ListIssues(ctx, 1, forge.ListIssuesOptions{}) + if err != nil || len(all) != 2 || all[0].IID != first { + t.Fatalf("ListIssues = %+v, %v", all, err) + } + // The fake clock is one second per event; the day-granular query keeps + // both, so only the driver's exact filter can drop the first. + after := all[1].UpdatedAt + got, err := y.ListIssues(ctx, 1, forge.ListIssuesOptions{UpdatedAfter: &after}) + if err != nil || len(got) != 1 || got[0].IID != second { + t.Fatalf("UpdatedAfter = %+v, %v", got, err) + } + got, err = y.ListIssues(ctx, 1, forge.ListIssuesOptions{Limit: 1}) + if err != nil || len(got) != 1 { + t.Fatalf("Limit = %+v, %v", got, err) + } +} + +func TestListIssuesNestedTagsCountTowardTheItemCeiling(t *testing.T) { + lowerMaxItems(t, 5) + tags := `{"id":"5-1","name":"a"},{"id":"5-2","name":"b"},{"id":"5-3","name":"c"}` + y, _ := loggedDriver(t, func(w http.ResponseWriter, _ *http.Request) { + writeRaw(w, "["+rawIssue("2-1", "UZI-1", 1, "0-1", tags)+","+rawIssue("2-2", "UZI-2", 2, "0-1", tags)+"]") + }) + if _, err := y.ListIssues(context.Background(), 1, forge.ListIssuesOptions{}); err == nil { + t.Fatal("nested tags did not count toward the item ceiling") + } +} + +func TestIssueTagCountIsCapped(t *testing.T) { + tags := strings.TrimSuffix(strings.Repeat(`{"id":"5-1","name":"a"},`, maxNestedItems+1), ",") + issue := rawIssue("2-1", "UZI-1", 1, "0-1", tags) + y, _ := loggedDriver(t, func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/api/issues" { + writeRaw(w, "["+issue+"]") + return + } + writeRaw(w, issue) + }) + ctx := context.Background() + if _, err := y.ListIssues(ctx, 1, forge.ListIssuesOptions{}); err == nil { + t.Error("ListIssues accepted an issue with too many tags") + } + if _, err := y.GetIssue(ctx, 1, 1); err == nil { + t.Error("GetIssue accepted an issue with too many tags") + } +} + +// rawListHandler serves one page of issues for every list request. +func rawListHandler(issues ...string) http.HandlerFunc { + return func(w http.ResponseWriter, _ *http.Request) { + writeRaw(w, "["+strings.Join(issues, ",")+"]") + } +} + +func TestListIssuesClientSideFilters(t *testing.T) { + bug := `{"id":"5-1","name":"bug"}` + good := rawIssue("2-1", "UZI-1", 1, "0-1", bug) + resolved := strings.Replace(rawIssue("2-5", "UZI-5", 5, "0-1", bug), `"updated":2000`, `"updated":2000,"resolved":3000`, 1) + old := strings.Replace(rawIssue("2-6", "UZI-6", 6, "0-1", bug), `"updated":2000`, `"updated":1`, 1) + issues := []string{ + good, + rawIssue("2-2", "UZI-2", 2, "0-1", `{"id":"5-2","name":"bugfix"}`), + rawIssue("2-3", "UZI-3", 3, "0-2", bug), + rawIssue("2-4", "UZI-4", 4, "0-1", `{"id":"5-3","name":"Bug"}`), + resolved, + old, + } + y, _ := loggedDriver(t, rawListHandler(issues...)) + ctx := context.Background() + iids := func(opts forge.ListIssuesOptions) []int64 { + t.Helper() + got, err := y.ListIssues(ctx, 1, opts) + if err != nil { + t.Fatal(err) + } + var out []int64 + for _, i := range got { + out = append(out, i.IID) + } + return out + } + if got := iids(forge.ListIssuesOptions{Labels: []string{"bug"}}); !slices.Equal(got, []int64{1, 5, 6}) { + t.Errorf("label filter = %v, want [1 5 6]", got) + } + if got := iids(forge.ListIssuesOptions{Labels: []string{"bug"}, State: forge.StateOpened}); !slices.Equal(got, []int64{1, 6}) { + t.Errorf("opened filter = %v, want [1 6]", got) + } + if got := iids(forge.ListIssuesOptions{Labels: []string{"bug"}, State: forge.StateClosed}); !slices.Equal(got, []int64{5}) { + t.Errorf("closed filter = %v, want [5]", got) + } + after := time.UnixMilli(2000) + if got := iids(forge.ListIssuesOptions{Labels: []string{"bug"}, UpdatedAfter: &after}); !slices.Equal(got, []int64{1, 5}) { + t.Errorf("updated-after filter = %v, want [1 5]", got) + } + if got := iids(forge.ListIssuesOptions{}); !slices.Equal(got, []int64{1, 2, 4, 5, 6}) { + t.Errorf("project filter = %v, want the other project's issue 3 dropped", got) + } +} diff --git a/api/internal/issuetracker/youtrack_labels.go b/api/internal/issuetracker/youtrack_labels.go index e4c6bf4059..57eeb218ec 100644 --- a/api/internal/issuetracker/youtrack_labels.go +++ b/api/internal/issuetracker/youtrack_labels.go @@ -68,17 +68,23 @@ func (y *youTrack) tagIDsFor(ctx context.Context, names []string) (map[string]st } // ListLabels implements forge.IssueTracker. Labels are YouTrack tags, global -// to the account; projectID is ignored. +// to the account; projectID is ignored. A name carried by several tags is +// listed once. func (y *youTrack) ListLabels(ctx context.Context, _ int64) ([]forge.Label, error) { if _, err := y.requireProject(); err != nil { return nil, err } - _, tags, err := y.listTags(ctx) + byName, tags, err := y.listTags(ctx) if err != nil { return nil, err } - out := make([]forge.Label, 0, len(tags)) + out := make([]forge.Label, 0, len(byName)) + seen := make(map[string]bool, len(byName)) for _, t := range tags { + if seen[t.Name] { + continue + } + seen[t.Name] = true out = append(out, forge.Label{Name: t.Name}) } return out, nil diff --git a/api/internal/issuetracker/youtrack_labels_test.go b/api/internal/issuetracker/youtrack_labels_test.go new file mode 100644 index 0000000000..23e3d687ae --- /dev/null +++ b/api/internal/issuetracker/youtrack_labels_test.go @@ -0,0 +1,147 @@ +package issuetracker + +import ( + "context" + "errors" + "net/http" + "slices" + "strings" + "testing" + + "github.com/vtmocanu/uzi/api/internal/forge" +) + +// TestLabelGuardOnEveryGuardedMethod sends a brace, and a name one byte over +// the length limit, through every method that takes label names. +func TestLabelGuardOnEveryGuardedMethod(t *testing.T) { + y, log := loggedDriver(t, func(w http.ResponseWriter, _ *http.Request) { writeRaw(w, "[]") }) + ctx := context.Background() + for _, name := range []string{"a}b", strings.Repeat("a", maxLabelNameLen+1)} { + calls := map[string]func() error{ + "EnsureLabels": func() error { return y.EnsureLabels(ctx, 1, []forge.Label{{Name: name}}) }, + "ListIssues": func() error { + _, err := y.ListIssues(ctx, 1, forge.ListIssuesOptions{Labels: []string{name}}) + return err + }, + "CreateIssue": func() error { _, err := y.CreateIssue(ctx, 1, "t", "d", []string{name}); return err }, + "UpdateIssueLabels add": func() error { return y.UpdateIssueLabels(ctx, 1, 1, []string{name}, nil) }, + "UpdateIssueLabels remove": func() error { return y.UpdateIssueLabels(ctx, 1, 1, nil, []string{name}) }, + } + for method, call := range calls { + if err := call(); !errors.Is(err, ErrInvalidLabelName) { + t.Errorf("%s(len %d) = %v, want ErrInvalidLabelName", method, len(name), err) + } + } + } + if got := log.all(); len(got) != 0 { + t.Errorf("guarded calls sent %v", got) + } +} + +func TestUpdateIssueLabelsSkipsAbsentTagRemoval(t *testing.T) { + y, srv := newFakeDriver(t) + n := srv.SeedIssue("t", "", "keep", "drop") + srv.ResetRequests() + if err := y.UpdateIssueLabels(context.Background(), 1, n, nil, []string{"never-there", "drop"}); err != nil { + t.Fatal(err) + } + var deletes int + for _, r := range srv.Requests() { + if r.Method == http.MethodDelete { + deletes++ + } + } + if deletes != 1 { + t.Errorf("sent %d DELETE requests, want 1 (only the tag on the issue)", deletes) + } + if got := srv.IssueTags(n); !slices.Equal(got, []string{"keep"}) { + t.Errorf("tags = %v, want [keep]", got) + } +} + +func TestTagIDsForTakesTheFirstTagOfADuplicatedName(t *testing.T) { + y, _ := loggedDriver(t, func(w http.ResponseWriter, _ *http.Request) { + writeRaw(w, `[{"id":"5-1","name":"dup"},{"id":"5-2","name":"dup"}]`) + }) + ids, err := y.tagIDsFor(context.Background(), []string{"dup"}) + if err != nil || ids["dup"] != "5-1" { + t.Fatalf("tagIDsFor = %v, %v; want the first tag, 5-1", ids, err) + } + labels, err := y.ListLabels(context.Background(), 1) + if err != nil || len(labels) != 1 || labels[0].Name != "dup" { + t.Fatalf("ListLabels = %+v, %v; want one dup", labels, err) + } +} + +func TestCreateIssueAttachesADuplicateLabelOnce(t *testing.T) { + y, srv := newFakeDriver(t) + srv.ResetRequests() + if _, err := y.CreateIssue(context.Background(), 1, "t", "d", []string{"x", "x"}); err != nil { + t.Fatal(err) + } + var attaches int + for _, r := range srv.Requests() { + if r.Method == http.MethodPost && strings.HasPrefix(r.Path, "/api/issues/") && strings.HasSuffix(r.Path, "/tags") { + attaches++ + } + } + if attaches != 1 { + t.Errorf("attached %d times, want once", attaches) + } +} + +// TestMalformedTagIDsNeverReachAPath feeds ids that would alter a path. +func TestMalformedTagIDsNeverReachAPath(t *testing.T) { + const bad = ".." + t.Run("on the issue", func(t *testing.T) { + y, log := loggedDriver(t, func(w http.ResponseWriter, _ *http.Request) { + writeRaw(w, rawIssue("2-1", "UZI-1", 1, "0-1", `{"id":"`+bad+`","name":"x"}`)) + }) + if err := y.UpdateIssueLabels(context.Background(), 1, 1, nil, []string{"x"}); err == nil { + t.Error("UpdateIssueLabels accepted a malformed tag id") + } + if got := log.writes(); len(got) != 0 { + t.Errorf("sent %v", got) + } + }) + t.Run("in the tag list", func(t *testing.T) { + y, log := loggedDriver(t, func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/api/tags" { + writeRaw(w, `[{"id":"`+bad+`","name":"x"}]`) + return + } + writeRaw(w, rawIssue("2-1", "UZI-1", 1, "0-1", "")) + }) + ctx := context.Background() + if _, err := y.ListLabels(ctx, 1); err == nil { + t.Error("ListLabels accepted a malformed tag id") + } + if err := y.UpdateIssueLabels(ctx, 1, 1, []string{"x"}, nil); err == nil { + t.Error("UpdateIssueLabels attached a malformed tag id") + } + if got := log.writes(); len(got) != 0 { + t.Errorf("sent %v", got) + } + }) + t.Run("in a created tag", func(t *testing.T) { + y, log := loggedDriver(t, func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/api/tags" && r.Method == http.MethodGet { + writeRaw(w, `[]`) + return + } + if r.URL.Path == "/api/tags" { + writeRaw(w, `{"id":"`+bad+`","name":"x"}`) + return + } + writeRaw(w, rawIssue("2-1", "UZI-1", 1, "0-1", "")) + }) + if err := y.UpdateIssueLabels(context.Background(), 1, 1, []string{"x"}, nil); err == nil { + t.Error("UpdateIssueLabels attached a malformed created tag id") + } + for _, w := range log.writes() { + if strings.Contains(w, "/issues/") { + t.Errorf("sent %v", w) + } + } + }) +} diff --git a/api/internal/issuetracker/youtrack_projects_test.go b/api/internal/issuetracker/youtrack_projects_test.go new file mode 100644 index 0000000000..4d8f229d8d --- /dev/null +++ b/api/internal/issuetracker/youtrack_projects_test.go @@ -0,0 +1,26 @@ +package issuetracker + +import ( + "context" + "testing" +) + +func TestStatesAndProjects(t *testing.T) { + y, _ := newFakeDriver(t) + ctx := context.Background() + acct, err := y.VerifyToken(ctx) + if err != nil || acct.Login != youtracktestBot { + t.Fatalf("VerifyToken = %+v, %v", acct, err) + } + projects, err := y.ListProjects(ctx) + if err != nil || len(projects) != 2 { + t.Fatalf("ListProjects = %+v, %v", projects, err) + } + states, err := y.ListStates(ctx, "0-1") + if err != nil || len(states) != 3 || states[0].Resolved || !states[1].Resolved || states[2].Name != "Won't fix" { + t.Fatalf("ListStates = %+v, %v", states, err) + } + if _, err := y.ListStates(ctx, "../x"); err == nil { + t.Fatal("ListStates accepted a malformed project id") + } +} diff --git a/api/internal/issuetracker/youtrack_query.go b/api/internal/issuetracker/youtrack_query.go index 49ca0450af..78bb033dd7 100644 --- a/api/internal/issuetracker/youtrack_query.go +++ b/api/internal/issuetracker/youtrack_query.go @@ -17,7 +17,7 @@ const maxLabelNameLen = 200 func (y *youTrack) checkLabelName(name string) error { bad := strings.TrimSpace(name) == "" || len(name) > maxLabelNameLen for _, c := range name { - if c == '{' || c == '}' || unicode.IsControl(c) || unicode.Is(unicode.Cf, c) || c == '
' || c == '
' { + if c == '{' || c == '}' || unicode.IsControl(c) || unicode.Is(unicode.Cf, c) || c == '\u2028' || c == '\u2029' { bad = true } } diff --git a/api/internal/issuetracker/youtrack_query_test.go b/api/internal/issuetracker/youtrack_query_test.go index 88e10fb87f..5d51e27fde 100644 --- a/api/internal/issuetracker/youtrack_query_test.go +++ b/api/internal/issuetracker/youtrack_query_test.go @@ -35,154 +35,3 @@ func TestLabelNameGuard(t *testing.T) { } } } - -func TestListIssuesUpdatedAfterAndLimit(t *testing.T) { - y, srv := newFakeDriver(t) - ctx := context.Background() - first := srv.SeedIssue("one", "") - second := srv.SeedIssue("two", "") - all, err := y.ListIssues(ctx, 1, forge.ListIssuesOptions{}) - if err != nil || len(all) != 2 || all[0].IID != first { - t.Fatalf("ListIssues = %+v, %v", all, err) - } - // The fake clock is one second per event; the day-granular query keeps - // both, so only the driver's exact filter can drop the first. - after := all[1].UpdatedAt - got, err := y.ListIssues(ctx, 1, forge.ListIssuesOptions{UpdatedAfter: &after}) - if err != nil || len(got) != 1 || got[0].IID != second { - t.Fatalf("UpdatedAfter = %+v, %v", got, err) - } - got, err = y.ListIssues(ctx, 1, forge.ListIssuesOptions{Limit: 1}) - if err != nil || len(got) != 1 { - t.Fatalf("Limit = %+v, %v", got, err) - } -} - -func TestIssueMapping(t *testing.T) { - y, srv := newFakeDriver(t) - n := srv.SeedIssue("title", "body", "x", "y") - got, err := y.GetIssue(context.Background(), 99, n) - if err != nil { - t.Fatal(err) - } - if got.IID != n || got.Title != "title" || got.Description != "body" || got.State != "opened" || - got.Author != youtracktestBot || got.AuthorForgeUserID != 0 || got.Assignees == nil || len(got.Assignees) != 0 || - got.WebURL != srv.URL+"/issue/UZI-1" || len(got.Labels) != 2 || got.UpdatedAt.IsZero() { - t.Fatalf("mapped issue = %+v", got) - } -} - -func TestPagingWalksEveryPage(t *testing.T) { - y, srv := newFakeDriver(t) - for i := 0; i < pageSize+5; i++ { - srv.SeedIssue("i", "") - } - got, err := y.ListIssues(context.Background(), 1, forge.ListIssuesOptions{}) - if err != nil || len(got) != pageSize+5 { - t.Fatalf("got %d issues, %v", len(got), err) - } -} - -func TestStatesAndProjects(t *testing.T) { - y, _ := newFakeDriver(t) - ctx := context.Background() - acct, err := y.VerifyToken(ctx) - if err != nil || acct.Login != youtracktestBot { - t.Fatalf("VerifyToken = %+v, %v", acct, err) - } - projects, err := y.ListProjects(ctx) - if err != nil || len(projects) != 2 { - t.Fatalf("ListProjects = %+v, %v", projects, err) - } - states, err := y.ListStates(ctx, "0-1") - if err != nil || len(states) != 3 || states[0].Resolved || !states[1].Resolved || states[2].Name != "Won't fix" { - t.Fatalf("ListStates = %+v, %v", states, err) - } - if _, err := y.ListStates(ctx, "../x"); err == nil { - t.Fatal("ListStates accepted a malformed project id") - } -} - -func TestStateFieldOption(t *testing.T) { - srv, y := newFakeDriverWithField(t, "Stage") - _ = srv - ctx := context.Background() - if _, err := y.ListStates(ctx, "0-1"); err != nil { - t.Fatal(err) - } -} - -func TestUnboundDriver(t *testing.T) { - _, srv := newFakeDriver(t) - opts := fakeOptions(srv) - opts.Project = ProjectRef{} - y, err := newYouTrackOver(opts, srv.Client().Transport) - if err != nil { - t.Fatal(err) - } - ctx := context.Background() - if _, err := y.ListLabels(ctx, 1); !errors.Is(err, ErrProjectNotBound) { - t.Errorf("ListLabels = %v", err) - } - if _, err := y.GetIssue(ctx, 1, 1); !errors.Is(err, ErrProjectNotBound) { - t.Errorf("GetIssue = %v", err) - } - if _, err := y.ListIssues(ctx, 1, forge.ListIssuesOptions{}); !errors.Is(err, ErrProjectNotBound) { - t.Errorf("ListIssues = %v", err) - } - if _, err := y.ListProjects(ctx); err != nil { - t.Errorf("ListProjects on an unbound driver: %v", err) - } -} - -func TestStateMappingUnset(t *testing.T) { - _, srv := newFakeDriver(t) - opts := fakeOptions(srv) - opts.OpenState, opts.ClosedState = "", "" - y, err := newYouTrackOver(opts, srv.Client().Transport) - if err != nil { - t.Fatal(err) - } - n := srv.SeedIssue("t", "") - for _, s := range []forge.IssueState{forge.StateOpened, forge.StateClosed} { - if err := y.SetIssueState(context.Background(), 1, n, s); !errors.Is(err, ErrStateMappingUnset) { - t.Errorf("SetIssueState(%s) = %v", s, err) - } - } - y2, _ := newFakeDriver(t) - if err := y2.SetIssueState(context.Background(), 1, 1, forge.StateAll); err == nil { - t.Error("SetIssueState accepted StateAll") - } -} - -func TestConstructorValidation(t *testing.T) { - good := Options{BaseURL: "https://yt.example.com", Token: credential(), Project: ProjectRef{ID: "0-1", ShortName: "UZI"}} - mutate := func(f func(*Options)) Options { o := good; f(&o); return o } - if _, err := New(KindYouTrack, good); err != nil { - t.Fatalf("good options: %v", err) - } - if _, err := New(KindYouTrack, mutate(func(o *Options) { o.Project = ProjectRef{} })); err != nil { - t.Fatalf("unbound options: %v", err) - } - bad := map[string]Options{ - "http": mutate(func(o *Options) { o.BaseURL = "http://yt.example.com" }), - "path": mutate(func(o *Options) { o.BaseURL = "https://yt.example.com/youtrack" }), - "userinfo": mutate(func(o *Options) { o.BaseURL = "https://u:p@yt.example.com" }), - "query": mutate(func(o *Options) { o.BaseURL = "https://yt.example.com?x=1" }), - "fragment": mutate(func(o *Options) { o.BaseURL = "https://yt.example.com#x" }), - "no host": mutate(func(o *Options) { o.BaseURL = "https://" }), - "empty": mutate(func(o *Options) { o.BaseURL = "" }), - "short token": mutate(func(o *Options) { o.Token = "short" }), - "bad short": mutate(func(o *Options) { o.Project.ShortName = "1X" }), - "bad id": mutate(func(o *Options) { o.Project.ID = "abc" }), - "half bound": mutate(func(o *Options) { o.Project.ID = "" }), - } - for name, o := range bad { - if _, err := New(KindYouTrack, o); err == nil { - t.Errorf("%s: constructor accepted %+v", name, o) - } - } - if _, err := New(KindYouTrack, mutate(func(o *Options) { o.BaseURL = "https://yt.example.com/" })); err != nil { - t.Errorf("trailing slash rejected: %v", err) - } -} diff --git a/api/internal/issuetracker/youtrack_request.go b/api/internal/issuetracker/youtrack_request.go index 423bea7be0..78db4f1cf5 100644 --- a/api/internal/issuetracker/youtrack_request.go +++ b/api/internal/issuetracker/youtrack_request.go @@ -4,6 +4,7 @@ import ( "bytes" "context" "encoding/json" + "errors" "fmt" "net/http" "net/url" @@ -78,7 +79,7 @@ func (y *youTrack) do(ctx context.Context, c ytCall, out any) (status, n int, er if err != nil { return status, 0, y.red.errorStatus(err, status) } - return status, len(body), y.red.errorStatus(fmt.Errorf("youtrack: %s %s: HTTP %d: %s", c.method, path, status, errorText(body)), status) + return status, len(body), y.red.errorStatus(fmt.Errorf("youtrack: %s %s: HTTP %d: %s", c.method, path, status, y.errorText(body)), status) } body, err := readBounded(resp.Body, c.limit, c.prov, status) if err != nil { @@ -93,28 +94,43 @@ func (y *youTrack) do(ctx context.Context, c ytCall, out any) (status, n int, er } // errorText renders a YouTrack error body: its error and error_description -// members, or a bounded prefix of the raw text. The caller's redactor scrubs -// and truncates the result. -func errorText(body []byte) string { +// members joined by ": ", or the raw text. The text is scrubbed whole before it +// is cut to rawLimit, so a token that straddles the cut cannot leave a prefix +// behind; the caller's redactor scrubs and truncates the result again. +func (y *youTrack) errorText(body []byte) string { var e struct { Error string `json:"error"` Description string `json:"error_description"` } if json.Unmarshal(body, &e) == nil && (e.Error != "" || e.Description != "") { - return strings.TrimSpace(e.Error + ": " + e.Description) + var parts []string + for _, p := range []string{e.Error, e.Description} { + if p != "" { + parts = append(parts, p) + } + } + return strings.Join(parts, ": ") } const rawLimit = 200 - if len(body) > rawLimit { - body = body[:rawLimit] - } - return string(body) + return truncate(y.red.scrub(string(body)), rawLimit) +} + +// maxNestedItems caps the tags of one issue or one activity. +const maxNestedItems = 1000 + +// nester is implemented by a wire type that carries nested arrays; ytPages +// counts them toward maxItems and caps them at maxNestedItems. +type nester interface { + nestedCount() int } // ytPages walks every page of a list endpoint under paginate's ceilings and -// calls visit with each page. visit returns true to stop early. Errors come +// calls visit with each page. visit returns true to stop early. Items that +// implement nester count their nested entries toward maxItems. Errors come // back redacted. func ytPages[T any](ctx context.Context, y *youTrack, path []string, query url.Values, visit func([]T) bool) error { var fetchErr error + seen := 0 _, err := paginate(func(skip, top int) ([]T, int, error) { q := url.Values{} for k, v := range query { @@ -128,6 +144,21 @@ func ytPages[T any](ctx context.Context, y *youTrack, path []string, query url.V fetchErr = err return nil, 0, err } + seen += len(items) + for _, it := range items { + if nn, ok := any(it).(nester); ok { + c := nn.nestedCount() + if c > maxNestedItems { + fetchErr = y.fail("youtrack: an item carries more than %d nested entries", maxNestedItems) + return nil, 0, fetchErr + } + seen += c + } + } + if seen > maxItems { + fetchErr = errors.New("issuetracker: listing exceeds the item ceiling") + return nil, 0, fetchErr + } if visit(items) { // A short page ends the walk; the items were already consumed. return nil, n, nil diff --git a/api/internal/issuetracker/youtrack_request_test.go b/api/internal/issuetracker/youtrack_request_test.go new file mode 100644 index 0000000000..149d56691a --- /dev/null +++ b/api/internal/issuetracker/youtrack_request_test.go @@ -0,0 +1,135 @@ +package issuetracker + +import ( + "context" + "errors" + "fmt" + "net/http" + "strings" + "testing" + + "github.com/vtmocanu/uzi/api/internal/forge" +) + +func TestPagingWalksEveryPage(t *testing.T) { + y, srv := newFakeDriver(t) + for i := 0; i < pageSize+5; i++ { + srv.SeedIssue("i", "") + } + got, err := y.ListIssues(context.Background(), 1, forge.ListIssuesOptions{}) + if err != nil || len(got) != pageSize+5 { + t.Fatalf("got %d issues, %v", len(got), err) + } +} + +// eightCharWindows lists every 8-byte substring of s. +func eightCharWindows(s string) []string { + var out []string + for i := 0; i+8 <= len(s); i++ { + out = append(out, s[i:i+8]) + } + return out +} + +func assertNoTokenWindow(t *testing.T, msg string) { + t.Helper() + for _, w := range eightCharWindows(credential()) { + if strings.Contains(msg, w) { + t.Fatalf("message carries the token fragment %q: %q", w, msg) + } + } +} + +// TestErrorTextScrubsBeforeCutting puts a non-JSON error body's token at +// offsets around and beyond the 200-byte cut. +func TestErrorTextScrubsBeforeCutting(t *testing.T) { + var offsets []int + for o := 0; o <= 230; o++ { + offsets = append(offsets, o) + } + offsets = append(offsets, 400, 1000, 3000) + var pad int + y, _ := tlsDriver(t, http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusBadRequest) + _, _ = w.Write([]byte(strings.Repeat("x", pad) + credential())) + })) + for _, o := range offsets { + pad = o + _, err := y.VerifyToken(context.Background()) + if err == nil { + t.Fatal("want an error") + } + assertNoTokenWindow(t, err.Error()) + } +} + +func TestErrorTextDescriptionOnly(t *testing.T) { + y, _ := tlsDriver(t, http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusBadRequest) + writeRaw(w, `{"error_description":"only this"}`) + })) + _, err := y.VerifyToken(context.Background()) + if err == nil || !strings.HasSuffix(err.Error(), "HTTP 400: only this") { + t.Fatalf("err = %v, want a message ending in \"HTTP 400: only this\"", err) + } +} + +type roundTripFunc func(*http.Request) (*http.Response, error) + +func (f roundTripFunc) RoundTrip(r *http.Request) (*http.Response, error) { return f(r) } + +func TestTransportErrorIsScrubbed(t *testing.T) { + y, err := newYouTrackOver(Options{BaseURL: "https://yt.example.com", Token: credential()}, + roundTripFunc(func(r *http.Request) (*http.Response, error) { + return nil, fmt.Errorf("dial failed with %s", credential()) + })) + if err != nil { + t.Fatal(err) + } + _, err = y.VerifyToken(context.Background()) + assertScrubbed(t, err) +} + +// TestDecodeErrorIsScrubbed uses a numeric token, because the JSON decoder +// echoes an offending number in its error. +func TestDecodeErrorIsScrubbed(t *testing.T) { + const num = "99999999999999999999" + y, _ := tlsDriverToken(t, http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + writeRaw(w, `{"id":`+num+`}`) + }), num) + _, err := y.VerifyToken(context.Background()) + if err == nil || !strings.Contains(err.Error(), "decode response") { + t.Fatalf("err = %v, want a decode error", err) + } + if strings.Contains(err.Error(), num) { + t.Fatalf("decode error carries the token: %v", err) + } +} + +func TestOversizedErrorBodyKeepsItsClass(t *testing.T) { + y, _ := tlsDriver(t, http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusInternalServerError) + _, _ = w.Write([]byte(credential() + strings.Repeat("x", int(errorBodyBytes)))) + })) + _, err := y.VerifyToken(context.Background()) + assertScrubbed(t, err) + if !errors.Is(err, forge.ErrResponseTooLarge) || forge.Class(err) != forge.ErrorClassServerError { + t.Fatalf("err = %v (class %v), want ErrResponseTooLarge, server error", err, forge.Class(err)) + } +} + +func TestListingByteCeilingCountsEveryPage(t *testing.T) { + old := maxTotalBytes + maxTotalBytes = 10_000 + t.Cleanup(func() { maxTotalBytes = old }) + project := `{"id":"0-1","shortName":"UZI","name":"Uzi"}` + page := "[" + strings.TrimSuffix(strings.Repeat(project+",", pageSize), ",") + "]" + if len(page) >= int(listPageBytes) || len(page) > maxTotalBytes { + t.Fatalf("fixture page is %d bytes", len(page)) + } + y, _ := tlsDriver(t, http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { writeRaw(w, page) })) + _, err := y.ListProjects(context.Background()) + if err == nil || !strings.Contains(err.Error(), "byte ceiling") { + t.Fatalf("err = %v, want the byte ceiling", err) + } +} diff --git a/api/internal/issuetracker/youtrack_security_test.go b/api/internal/issuetracker/youtrack_security_test.go index c4cec94706..93d3df05f8 100644 --- a/api/internal/issuetracker/youtrack_security_test.go +++ b/api/internal/issuetracker/youtrack_security_test.go @@ -20,12 +20,18 @@ import ( // tlsDriver builds a bound driver over a TLS test server running h. func tlsDriver(t *testing.T, h http.Handler) (*youTrack, *httptest.Server) { + t.Helper() + return tlsDriverToken(t, h, credential()) +} + +// tlsDriverToken is tlsDriver with a chosen token. +func tlsDriverToken(t *testing.T, h http.Handler, token string) (*youTrack, *httptest.Server) { t.Helper() srv := httptest.NewTLSServer(h) t.Cleanup(srv.Close) y, err := newYouTrackOver(Options{ BaseURL: srv.URL, - Token: credential(), + Token: token, Project: ProjectRef{ID: "0-1", ShortName: "UZI"}, OpenState: "Open", ClosedState: "Fixed", diff --git a/api/internal/issuetracker/youtrack_state_test.go b/api/internal/issuetracker/youtrack_state_test.go new file mode 100644 index 0000000000..601fa842b1 --- /dev/null +++ b/api/internal/issuetracker/youtrack_state_test.go @@ -0,0 +1,51 @@ +package issuetracker + +import ( + "context" + "errors" + "testing" + + "github.com/vtmocanu/uzi/api/internal/forge" +) + +func TestStateFieldOption(t *testing.T) { + srv, y := newFakeDriverWithField(t, "Stage") + _ = srv + ctx := context.Background() + if _, err := y.ListStates(ctx, "0-1"); err != nil { + t.Fatal(err) + } +} + +func TestStateMappingUnset(t *testing.T) { + _, srv := newFakeDriver(t) + opts := fakeOptions(srv) + opts.OpenState, opts.ClosedState = "", "" + y, err := newYouTrackOver(opts, srv.Client().Transport) + if err != nil { + t.Fatal(err) + } + n := srv.SeedIssue("t", "") + for _, s := range []forge.IssueState{forge.StateOpened, forge.StateClosed} { + if err := y.SetIssueState(context.Background(), 1, n, s); !errors.Is(err, ErrStateMappingUnset) { + t.Errorf("SetIssueState(%s) = %v", s, err) + } + } + y2, _ := newFakeDriver(t) + if err := y2.SetIssueState(context.Background(), 1, 1, forge.StateAll); err == nil { + t.Error("SetIssueState accepted StateAll") + } +} + +func TestSetIssueStateHonoursStateField(t *testing.T) { + srv, y := newFakeDriverWithField(t, "Stage") + n := srv.SeedIssue("t", "") + ctx := context.Background() + if err := y.SetIssueState(ctx, 1, n, forge.StateClosed); err != nil { + t.Fatalf("SetIssueState: %v", err) + } + got, err := y.GetIssue(ctx, 1, n) + if err != nil || got.State != string(forge.StateClosed) { + t.Fatalf("GetIssue = %+v, %v; want closed", got, err) + } +} diff --git a/api/internal/issuetracker/youtrack_test.go b/api/internal/issuetracker/youtrack_test.go index d064d7d994..e74776e5b3 100644 --- a/api/internal/issuetracker/youtrack_test.go +++ b/api/internal/issuetracker/youtrack_test.go @@ -1,9 +1,17 @@ package issuetracker import ( + "context" + "errors" + "fmt" + "net/http" + "net/http/httptest" + "slices" "strings" + "sync" "testing" + "github.com/vtmocanu/uzi/api/internal/forge" "github.com/vtmocanu/uzi/api/internal/issuetracker/youtracktest" ) @@ -53,3 +61,175 @@ func newFakeDriverWithField(t *testing.T, field string) (*youtracktest.Server, * } return srv, y } + +func TestConstructorValidation(t *testing.T) { + good := Options{BaseURL: "https://yt.example.com", Token: credential(), Project: ProjectRef{ID: "0-1", ShortName: "UZI"}} + mutate := func(f func(*Options)) Options { o := good; f(&o); return o } + if _, err := New(KindYouTrack, good); err != nil { + t.Fatalf("good options: %v", err) + } + if _, err := New(KindYouTrack, mutate(func(o *Options) { o.Project = ProjectRef{} })); err != nil { + t.Fatalf("unbound options: %v", err) + } + bad := map[string]Options{ + "http": mutate(func(o *Options) { o.BaseURL = "http://yt.example.com" }), + "path": mutate(func(o *Options) { o.BaseURL = "https://yt.example.com/youtrack" }), + "userinfo": mutate(func(o *Options) { o.BaseURL = "https://u:p@yt.example.com" }), + "query": mutate(func(o *Options) { o.BaseURL = "https://yt.example.com?x=1" }), + "fragment": mutate(func(o *Options) { o.BaseURL = "https://yt.example.com#x" }), + "no host": mutate(func(o *Options) { o.BaseURL = "https://" }), + "empty": mutate(func(o *Options) { o.BaseURL = "" }), + "short token": mutate(func(o *Options) { o.Token = "short" }), + "bad short": mutate(func(o *Options) { o.Project.ShortName = "1X" }), + "bad id": mutate(func(o *Options) { o.Project.ID = "abc" }), + "half bound": mutate(func(o *Options) { o.Project.ID = "" }), + } + for name, o := range bad { + if _, err := New(KindYouTrack, o); err == nil { + t.Errorf("%s: constructor accepted %+v", name, o) + } + } + if _, err := New(KindYouTrack, mutate(func(o *Options) { o.BaseURL = "https://yt.example.com/" })); err != nil { + t.Errorf("trailing slash rejected: %v", err) + } +} + +// requestLog counts the requests a raw handler receives, by method. +type requestLog struct { + mu sync.Mutex + calls []string +} + +func (l *requestLog) record(r *http.Request) { + l.mu.Lock() + defer l.mu.Unlock() + l.calls = append(l.calls, r.Method+" "+r.URL.EscapedPath()) +} + +func (l *requestLog) all() []string { + l.mu.Lock() + defer l.mu.Unlock() + return slices.Clone(l.calls) +} + +// writes returns the logged requests that are not GETs. +func (l *requestLog) writes() []string { + var out []string + for _, c := range l.all() { + if !strings.HasPrefix(c, "GET ") { + out = append(out, c) + } + } + return out +} + +// loggedDriver builds a bound driver over a raw handler and logs every request. +func loggedDriver(t *testing.T, h http.HandlerFunc) (*youTrack, *requestLog) { + t.Helper() + log := &requestLog{} + y, _ := tlsDriver(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + log.record(r) + h(w, r) + })) + return y, log +} + +// rawIssue renders an issue as the API would, in project projectID. +func rawIssue(id, readable string, number int64, projectID, tags string) string { + return fmt.Sprintf(`{"id":%q,"idReadable":%q,"numberInProject":%d,"summary":"s","created":1,"updated":2000,`+ + `"project":{"id":%q,"shortName":"UZI"},"tags":[%s]}`, id, readable, number, projectID, tags) +} + +func writeRaw(w http.ResponseWriter, body string) { + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(body)) +} + +// everyMethod lists the eleven forge.IssueTracker methods as closures over y. +func everyMethod(y *youTrack) map[string]func() error { + ctx := context.Background() + return map[string]func() error{ + "ListLabels": func() error { _, err := y.ListLabels(ctx, 1); return err }, + "EnsureLabels": func() error { return y.EnsureLabels(ctx, 1, []forge.Label{{Name: "a"}}) }, + "ListIssues": func() error { _, err := y.ListIssues(ctx, 1, forge.ListIssuesOptions{}); return err }, + "GetIssue": func() error { _, err := y.GetIssue(ctx, 1, 1); return err }, + "CreateIssue": func() error { _, err := y.CreateIssue(ctx, 1, "t", "d", []string{"a"}); return err }, + "UpdateIssueLabels": func() error { return y.UpdateIssueLabels(ctx, 1, 1, []string{"a"}, []string{"b"}) }, + "UpdateIssueDescription": func() error { return y.UpdateIssueDescription(ctx, 1, 1, "d") }, + "SetIssueState": func() error { return y.SetIssueState(ctx, 1, 1, forge.StateClosed) }, + "ListIssueLabelEvents": func() error { _, err := y.ListIssueLabelEvents(ctx, 1, 1); return err }, + "ListIssueComments": func() error { _, err := y.ListIssueComments(ctx, 1, 1); return err }, + "CreateIssueNote": func() error { _, err := y.CreateIssueNote(ctx, 1, 1, "n"); return err }, + } +} + +// TestUnboundDriverRefusesEveryMethod builds a driver with no project and +// checks that every IssueTracker method refuses before sending a request. +func TestUnboundDriverRefusesEveryMethod(t *testing.T) { + log := &requestLog{} + srv := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + log.record(r) + writeRaw(w, "[]") + })) + t.Cleanup(srv.Close) + y, err := newYouTrackOver(Options{BaseURL: srv.URL, Token: credential(), OpenState: "Open", ClosedState: "Fixed"}, srv.Client().Transport) + if err != nil { + t.Fatal(err) + } + calls := everyMethod(y) + if len(calls) != 11 { + t.Fatalf("table covers %d methods, want 11", len(calls)) + } + for name, call := range calls { + if err := call(); !errors.Is(err, ErrProjectNotBound) { + t.Errorf("%s = %v, want ErrProjectNotBound", name, err) + } + } + if got := log.all(); len(got) != 0 { + t.Errorf("unbound driver sent %v", got) + } + // ListProjects needs no binding. + if _, err := y.ListProjects(context.Background()); err != nil { + t.Errorf("ListProjects on an unbound driver: %v", err) + } +} + +// TestBaseURLTrailingSlashIsNormalised checks that a base URL with a trailing +// slash never yields a double slash in a request path. +func TestBaseURLTrailingSlashIsNormalised(t *testing.T) { + log := &requestLog{} + srv := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + log.record(r) + writeRaw(w, `{"id":"1-1","login":"u"}`) + })) + t.Cleanup(srv.Close) + y, err := newYouTrackOver(Options{BaseURL: srv.URL + "/", Token: credential()}, srv.Client().Transport) + if err != nil { + t.Fatal(err) + } + if strings.HasSuffix(y.baseURL, "/") { + t.Errorf("baseURL = %q, want no trailing slash", y.baseURL) + } + if _, err := y.VerifyToken(context.Background()); err != nil { + t.Fatal(err) + } + if got := log.all(); len(got) != 1 || got[0] != "GET /api/users/me" { + t.Errorf("requests = %v, want one GET /api/users/me", got) + } +} + +// TestPinnedLimits pins the literal byte limits. +func TestPinnedLimits(t *testing.T) { + for _, tc := range []struct { + name string + got, want int64 + }{ + {"listPageBytes", listPageBytes, 8 << 20}, + {"objectBytes", objectBytes, 2 << 20}, + {"errorBodyBytes", errorBodyBytes, 4 << 10}, + } { + if tc.got != tc.want { + t.Errorf("%s = %d, want %d", tc.name, tc.got, tc.want) + } + } +} diff --git a/api/internal/issuetracker/youtrack_wire.go b/api/internal/issuetracker/youtrack_wire.go index 49528d33ad..0b7fccc083 100644 --- a/api/internal/issuetracker/youtrack_wire.go +++ b/api/internal/issuetracker/youtrack_wire.go @@ -51,6 +51,10 @@ type ytIssue struct { Project *ytProject `json:"project"` } +func (w ytIssue) nestedCount() int { return len(w.Tags) } + +func (a ytActivity) nestedCount() int { return len(a.Added) + len(a.Removed) } + type ytComment struct { ID string `json:"id"` Text string `json:"text"` @@ -67,11 +71,29 @@ type ytActivity struct { Removed []ytTag `json:"removed"` } +// checkIssue refuses an issue whose entity id, id-readable or tag ids are not +// well formed, or whose tag count exceeds maxNestedItems: those ids end up in +// request paths and in WebURL. +func (y *youTrack) checkIssue(w ytIssue) error { + if !entityIDRE.MatchString(w.ID) || !idReadableRE.MatchString(w.IDReadable) { + return y.fail("youtrack: issue with a malformed id in the response") + } + if len(w.Tags) > maxNestedItems { + return y.fail("youtrack: issue carries more than %d tags", maxNestedItems) + } + for _, t := range w.Tags { + if !entityIDRE.MatchString(t.ID) { + return y.fail("youtrack: tag with a malformed id in the response") + } + } + return nil +} + // toIssue maps a wire issue to the neutral type. The issue must carry a valid // entity id and id-readable. func (y *youTrack) toIssue(w ytIssue) (forge.Issue, error) { - if !entityIDRE.MatchString(w.ID) || !idReadableRE.MatchString(w.IDReadable) { - return forge.Issue{}, y.fail("youtrack: issue with a malformed id in the response") + if err := y.checkIssue(w); err != nil { + return forge.Issue{}, err } issue := forge.Issue{ IID: w.NumberInProject, diff --git a/api/internal/issuetracker/youtrack_wire_test.go b/api/internal/issuetracker/youtrack_wire_test.go new file mode 100644 index 0000000000..df31fbac4b --- /dev/null +++ b/api/internal/issuetracker/youtrack_wire_test.go @@ -0,0 +1,20 @@ +package issuetracker + +import ( + "context" + "testing" +) + +func TestIssueMapping(t *testing.T) { + y, srv := newFakeDriver(t) + n := srv.SeedIssue("title", "body", "x", "y") + got, err := y.GetIssue(context.Background(), 99, n) + if err != nil { + t.Fatal(err) + } + if got.IID != n || got.Title != "title" || got.Description != "body" || got.State != "opened" || + got.Author != youtracktestBot || got.AuthorForgeUserID != 0 || got.Assignees == nil || len(got.Assignees) != 0 || + got.WebURL != srv.URL+"/issue/UZI-1" || len(got.Labels) != 2 || got.UpdatedAt.IsZero() { + t.Fatalf("mapped issue = %+v", got) + } +} diff --git a/api/internal/issuetracker/youtracktest/server.go b/api/internal/issuetracker/youtracktest/server.go index 042792eb0f..728817c9ad 100644 --- a/api/internal/issuetracker/youtracktest/server.go +++ b/api/internal/issuetracker/youtracktest/server.go @@ -29,9 +29,10 @@ type Server struct { // Token is the bearer token the server accepts. Token string - t testing.TB - mu sync.Mutex // guards every field below - m *model + t testing.TB + mux *http.ServeMux // built once by NewServer + mu sync.Mutex // guards every field below + m *model // requests logs every request in arrival order. requests []Request unmodelled []string @@ -46,6 +47,7 @@ func NewServer(t testing.TB) *Server { m: newModel(), Token: strings.Join([]string{"perm", ":", "fixture-", "id.", "Yk3mQ8vR2xTn5Lw7"}, ""), } + s.mux = s.routes() s.Server = httptest.NewTLSServer(http.HandlerFunc(s.serve)) t.Cleanup(func() { s.Server.Close() @@ -88,7 +90,7 @@ func (s *Server) serve(w http.ResponseWriter, r *http.Request) { writeError(w, http.StatusUnauthorized, "Unauthorized", "invalid or missing bearer token") return } - s.routes().ServeHTTP(w, r) + s.mux.ServeHTTP(w, r) } func (s *Server) unmodelledRoute(w http.ResponseWriter, r *http.Request) { diff --git a/api/internal/issuetracker/youtracktest/server_test.go b/api/internal/issuetracker/youtracktest/server_test.go new file mode 100644 index 0000000000..209cd79359 --- /dev/null +++ b/api/internal/issuetracker/youtracktest/server_test.go @@ -0,0 +1,62 @@ +package youtracktest + +import ( + "fmt" + "net/http" + "testing" +) + +// recorder is a testing.TB that records Errorf calls and holds cleanups so +// the test can run them by hand. Embedding the interface leaves every other +// method nil: calling one fails loudly. +type recorder struct { + testing.TB + errors []string + cleanups []func() +} + +func (r *recorder) Helper() {} +func (r *recorder) Cleanup(f func()) { r.cleanups = append(r.cleanups, f) } +func (r *recorder) Errorf(f string, a ...any) { r.errors = append(r.errors, fmt.Sprintf(f, a...)) } + +func TestUnmodelledRouteIsLoud(t *testing.T) { + rec := &recorder{TB: t} + srv := NewServer(rec) + req, err := http.NewRequest(http.MethodGet, srv.URL+"/api/nowhere", nil) + if err != nil { + t.Fatal(err) + } + req.Header.Set("Authorization", "Bearer "+srv.Token) + resp, err := srv.Client().Do(req) + if err != nil { + t.Fatal(err) + } + _ = resp.Body.Close() + if resp.StatusCode != http.StatusNotImplemented { + t.Errorf("status = %d, want 501", resp.StatusCode) + } + for i := len(rec.cleanups) - 1; i >= 0; i-- { + rec.cleanups[i]() + } + if len(rec.errors) != 1 { + t.Fatalf("cleanup reported %d errors, want 1: %v", len(rec.errors), rec.errors) + } +} + +func TestModelledRoutesAreQuietAtCleanup(t *testing.T) { + rec := &recorder{TB: t} + srv := NewServer(rec) + req, _ := http.NewRequest(http.MethodGet, srv.URL+"/api/users/me?fields=id", nil) + req.Header.Set("Authorization", "Bearer "+srv.Token) + resp, err := srv.Client().Do(req) + if err != nil { + t.Fatal(err) + } + _ = resp.Body.Close() + for i := len(rec.cleanups) - 1; i >= 0; i-- { + rec.cleanups[i]() + } + if len(rec.errors) != 0 { + t.Fatalf("cleanup reported %v", rec.errors) + } +} From 2afbaf02379569cf7f0f16d2b2b9cd7e991f604c Mon Sep 17 00:00:00 2001 From: uzi-agent Date: Wed, 11 Nov 2026 09:35:55 +0000 Subject: [PATCH 07/17] issuetrackersvc: close the m3 review findings ListStates binds no project, project references are validated as bad requests, the token is bounded and free of control characters, the strict base URL must be idempotent with a host, and tests pin the nil allowlist, bare query or fragment and non-canonical stored URL refusals. Co-Authored-By: Claude Opus 5.5 --- .../config/issue_tracker_allowlist_test.go | 14 +++ api/internal/issuetrackersvc/account.go | 2 +- api/internal/issuetrackersvc/baseurl.go | 15 ++- api/internal/issuetrackersvc/connect.go | 25 +++- api/internal/issuetrackersvc/driver.go | 19 ++- api/internal/issuetrackersvc/service.go | 3 +- api/internal/issuetrackersvc/service_test.go | 115 ++++++++++++++++++ 7 files changed, 188 insertions(+), 5 deletions(-) diff --git a/api/internal/config/issue_tracker_allowlist_test.go b/api/internal/config/issue_tracker_allowlist_test.go index adfd683d9f..78b9f0b12d 100644 --- a/api/internal/config/issue_tracker_allowlist_test.go +++ b/api/internal/config/issue_tracker_allowlist_test.go @@ -1,6 +1,7 @@ package config import ( + "os" "strings" "testing" ) @@ -20,6 +21,19 @@ func TestLoadReadsIssueTrackerAllowlist(t *testing.T) { t.Errorf("empty variable gave %v, want nil", cfg.IssueTrackerAllowedBaseURLs) } + // Truly unset, not just empty: t.Setenv registers the restore, then Unsetenv removes it. + t.Setenv(env, "x") + if err := os.Unsetenv(env); err != nil { + t.Fatal(err) + } + cfg, err = Load() + if err != nil { + t.Fatalf("Load with the variable unset: %v", err) + } + if cfg.IssueTrackerAllowedBaseURLs != nil { + t.Errorf("unset variable gave %v, want nil", cfg.IssueTrackerAllowedBaseURLs) + } + t.Setenv(env, "https://Issues.Example.com/, https://issues.example.com") cfg, err = Load() if err != nil { diff --git a/api/internal/issuetrackersvc/account.go b/api/internal/issuetrackersvc/account.go index ce6c5e8ccc..8b59ffc0a6 100644 --- a/api/internal/issuetrackersvc/account.go +++ b/api/internal/issuetrackersvc/account.go @@ -29,7 +29,7 @@ func validateAccountField(s string) error { return errors.New("empty or too long") } for _, r := range s { - if unicode.IsControl(r) || unicode.Is(unicode.Cf, r) || r == '
' || r == '
' { + if unicode.IsControl(r) || unicode.Is(unicode.Cf, r) || r == '\u2028' || r == '\u2029' { return errors.New("forbidden character") } } diff --git a/api/internal/issuetrackersvc/baseurl.go b/api/internal/issuetrackersvc/baseurl.go index 1173812d15..547358fc25 100644 --- a/api/internal/issuetrackersvc/baseurl.go +++ b/api/internal/issuetrackersvc/baseurl.go @@ -13,7 +13,10 @@ import ( // refuses, rather than strips, userinfo, an opaque part, a path other than // empty or "/", a query and a fragment: stripping would let // "https://allowed@evil" or "https://host/x" mean something other than what was -// checked. The returned string is the one value that is checked against the +// checked. The service keeps its own check, rather than trusting the driver's, +// because it must normalize to the stored and AAD form via +// config.NormalizeForgeBaseURL; the driver re-checks the URL independently as +// defence in depth. The returned string is the one value that is checked against the // allowlist, stored, and bound into the token's AAD. func strictBaseURL(raw string) (string, error) { raw = strings.TrimSpace(raw) @@ -31,5 +34,15 @@ func strictBaseURL(raw string) (string, error) { if err != nil { return "", errors.New("invalid base URL: it must be an https URL with a host") } + // The canonical form must be a fixed point with a real hostname, so an + // odd host (an escaped "%", an empty name before a port) cannot be stored + // in a form that normalizes to something else later. + again, err := config.NormalizeForgeBaseURL(norm) + if err != nil || again != norm { + return "", errors.New("invalid base URL: it must be an https URL with a host") + } + if nu, err := url.Parse(norm); err != nil || nu.Hostname() == "" { + return "", errors.New("invalid base URL: it must be an https URL with a host") + } return norm, nil } diff --git a/api/internal/issuetrackersvc/connect.go b/api/internal/issuetrackersvc/connect.go index eef5599603..ad4be85dff 100644 --- a/api/internal/issuetrackersvc/connect.go +++ b/api/internal/issuetrackersvc/connect.go @@ -2,7 +2,9 @@ package issuetrackersvc import ( "context" + "errors" "strings" + "unicode" "github.com/google/uuid" @@ -26,6 +28,9 @@ func (s *Service) Connect(ctx context.Context, userID uuid.UUID, kind, baseURL, if token == "" { return Connection{}, &Error{Kind: KindBadRequest, Message: "a token is required"} } + if err := validateTokenShape(token); err != nil { + return Connection{}, &Error{Kind: KindBadRequest, Message: err.Error()} + } norm, err := strictBaseURL(baseURL) if err != nil { return Connection{}, &Error{Kind: KindBadRequest, Message: err.Error()} @@ -37,7 +42,8 @@ func (s *Service) Connect(ctx context.Context, userID uuid.UUID, kind, baseURL, d, err := s.newDriver(parsedKind, issuetracker.Options{BaseURL: norm, Token: token}) if err != nil { s.log.Error("build issue tracker driver", "kind", string(parsedKind), "error", err) - return Connection{}, &Error{Kind: KindBadRequest, Message: "could not initialize issue tracker client"} + // The driver's build errors never carry the token. + return Connection{}, &Error{Kind: KindBadRequest, Message: "invalid token or connection settings: " + err.Error()} } account, err := d.VerifyToken(ctx) if err != nil { @@ -67,3 +73,20 @@ func (s *Service) Connect(ctx context.Context, userID uuid.UUID, kind, baseURL, } return toConnection(row), nil } + +// maxTokenLen bounds a token in bytes, before it is sealed or handed to a driver. +const maxTokenLen = 4096 + +// validateTokenShape refuses a token that is too long or holds a control or +// format (Cf) character. The message never includes the token. +func validateTokenShape(token string) error { + if len(token) > maxTokenLen { + return errors.New("token is too long") + } + for _, r := range token { + if unicode.IsControl(r) || unicode.Is(unicode.Cf, r) { + return errors.New("token holds a control or invisible character") + } + } + return nil +} diff --git a/api/internal/issuetrackersvc/driver.go b/api/internal/issuetrackersvc/driver.go index 496c3fbd83..9d610afa22 100644 --- a/api/internal/issuetrackersvc/driver.go +++ b/api/internal/issuetrackersvc/driver.go @@ -2,18 +2,32 @@ package issuetrackersvc import ( "context" + "regexp" "github.com/google/uuid" "github.com/vtmocanu/uzi/api/internal/issuetracker" ) +var ( + entityIDRE = regexp.MustCompile(`^\d+-\d+$`) + shortNameRE = regexp.MustCompile(`^[A-Za-z][A-Za-z0-9_]*$`) +) + +// validProjectRef reports whether p is empty or has a well-formed id and short name. +func validProjectRef(p issuetracker.ProjectRef) bool { + return p == (issuetracker.ProjectRef{}) || (entityIDRE.MatchString(p.ID) && shortNameRE.MatchString(p.ShortName)) +} + // DriverFor builds a driver for a stored connection. It re-checks the strict // base URL form and the allowlist against the stored row BEFORE opening the // token, so removing a host from the allowlist stops use of existing rows // (fail closed). The token is opened with the row's user, kind and base URL as // AAD, used only to build the driver, and never returned or logged. func (s *Service) DriverFor(ctx context.Context, userID, connID uuid.UUID, project issuetracker.ProjectRef, stateField, open, closed string) (issuetracker.Driver, error) { + if !validProjectRef(project) { + return nil, &Error{Kind: KindBadRequest, Message: "invalid project reference"} + } row, err := s.getRow(ctx, userID, connID) if err != nil { return nil, err @@ -65,7 +79,10 @@ func (s *Service) ListProjects(ctx context.Context, userID, connID uuid.UUID) ([ // ListStates lists the issue states of one project of the connection. func (s *Service) ListStates(ctx context.Context, userID, connID uuid.UUID, projectID string) ([]issuetracker.State, error) { - d, err := s.DriverFor(ctx, userID, connID, issuetracker.ProjectRef{ID: projectID}, "", "", "") + if !entityIDRE.MatchString(projectID) { + return nil, &Error{Kind: KindBadRequest, Message: "invalid project id"} + } + d, err := s.DriverFor(ctx, userID, connID, issuetracker.ProjectRef{}, "", "", "") if err != nil { return nil, err } diff --git a/api/internal/issuetrackersvc/service.go b/api/internal/issuetrackersvc/service.go index 8ed6b44e3a..98401ee21c 100644 --- a/api/internal/issuetrackersvc/service.go +++ b/api/internal/issuetrackersvc/service.go @@ -2,9 +2,10 @@ package issuetrackersvc import ( "context" - "github.com/google/uuid" "log/slog" + "github.com/google/uuid" + "github.com/vtmocanu/uzi/api/internal/issuetracker" "github.com/vtmocanu/uzi/api/internal/secretbox" "github.com/vtmocanu/uzi/api/internal/store" diff --git a/api/internal/issuetrackersvc/service_test.go b/api/internal/issuetrackersvc/service_test.go index 7c756976e2..6b35233fc2 100644 --- a/api/internal/issuetrackersvc/service_test.go +++ b/api/internal/issuetrackersvc/service_test.go @@ -171,6 +171,9 @@ func TestConnectRefusesBadInput(t *testing.T) { {"query", "youtrack", testHost + "?a=b", "tok"}, {"bare question mark", "youtrack", testHost + "?", "tok"}, {"fragment", "youtrack", testHost + "#x", "tok"}, + {"bare hash", "youtrack", testHost + "#", "tok"}, + {"escaped percent in host", "youtrack", "https://x%25y.example.com", "tok"}, + {"empty hostname with port", "youtrack", "https://:443", "tok"}, {"no host", "youtrack", "https://", "tok"}, {"not allowlisted", "youtrack", "https://evil.example.com", "tok"}, {"explicit default port is a different origin string", "youtrack", testHost + ":443", "tok"}, @@ -187,6 +190,57 @@ func TestConnectRefusesBadInput(t *testing.T) { } } +func TestConnectRefusesBadToken(t *testing.T) { + cases := map[string]string{ + "8 MiB": strings.Repeat("a", 8<<20), + "just too long": strings.Repeat("a", maxTokenLen+1), + "embedded CRLF": strings.Join([]string{"perm", "abc\r\nX-Evil: 1", "def"}, "-"), + "zero width": strings.Join([]string{"perm", "abc\u200bdef", "ghi"}, "-"), + "bidi override": strings.Join([]string{"perm", "abc\u202edef", "ghi"}, "-"), + "embedded NUL": strings.Join([]string{"perm", "abc\x00def", "ghi"}, "-"), + } + for name, tok := range cases { + t.Run(name, func(t *testing.T) { + h := newHarness(t) + _, err := h.svc.Connect(context.Background(), uuid.New(), "youtrack", testHost, tok) + wantKind(t, err, KindBadRequest) + if len(h.built) != 0 || len(h.q.rows) != 0 { + t.Error("driver built or row stored for a refused token") + } + }) + } + h := newHarness(t) + if _, err := h.svc.Connect(context.Background(), uuid.New(), "youtrack", testHost, strings.Repeat("a", maxTokenLen)); err != nil { + t.Errorf("a token of exactly the limit must pass: %v", err) + } +} + +func TestConnectNilAllowlistRefusesAll(t *testing.T) { + built := 0 + s := New(&fakeQuerier{}, nil, nil, func(issuetracker.Kind, issuetracker.Options) (issuetracker.Driver, error) { + built++ + return &fakeDriver{}, nil + }) + _, err := s.Connect(context.Background(), uuid.New(), "youtrack", testHost, strings.Repeat("a", 32)) + e := wantKind(t, err, KindBadRequest) + if !strings.Contains(e.Message, "allowed") || built != 0 { + t.Errorf("message = %q, built = %d", e.Message, built) + } +} + +func TestConnectDriverBuildFailureIsBadRequest(t *testing.T) { + h := newHarness(t) + h.svc.newDriver = func(issuetracker.Kind, issuetracker.Options) (issuetracker.Driver, error) { + return nil, errors.New("issuetracker: token shorter than the minimum length") + } + tok := testToken("s") + _, err := h.svc.Connect(context.Background(), uuid.New(), "youtrack", testHost, tok) + e := wantKind(t, err, KindBadRequest) + if !strings.Contains(e.Message, "invalid token") || strings.Contains(e.Message, tok) { + t.Errorf("message = %q", e.Message) + } +} + func TestConnectNormalizesBeforeAllowlist(t *testing.T) { h := newHarness(t) c, err := h.svc.Connect(context.Background(), uuid.New(), "youtrack", " HTTPS://Issues.Example.com/ ", testToken("x")) @@ -403,6 +457,67 @@ func TestDriverForRefusesTamperedRow(t *testing.T) { wantKind(t, err, KindBadRequest) } +func TestDriverForRefusesNonCanonicalStoredURL(t *testing.T) { + h := newHarness(t) + ctx := context.Background() + user := uuid.New() + c, err := h.svc.Connect(ctx, user, "youtrack", testHost, testToken("t")) + if err != nil { + t.Fatal(err) + } + h.built = nil + // Strict-valid and allowlisted, but not the canonical stored form. + h.allow["https://ISSUES.example.com"] = true + h.q.rows[0].BaseUrl = "https://ISSUES.example.com" + _, err = h.svc.DriverFor(ctx, user, c.ID, issuetracker.ProjectRef{}, "", "", "") + wantKind(t, err, KindBadRequest) + if len(h.built) != 0 { + t.Error("driver built for a non-canonical stored URL") + } +} + +func TestDriverForRefusesMalformedProjectRef(t *testing.T) { + h := newHarness(t) + ctx := context.Background() + user := uuid.New() + c, err := h.svc.Connect(ctx, user, "youtrack", testHost, testToken("t")) + if err != nil { + t.Fatal(err) + } + h.built = nil + for _, ref := range []issuetracker.ProjectRef{{ID: "0-1"}, {ShortName: "UZI"}, {ID: "x", ShortName: "UZI"}, {ID: "0-1", ShortName: "1bad"}} { + _, err = h.svc.DriverFor(ctx, user, c.ID, ref, "", "", "") + wantKind(t, err, KindBadRequest) + } + if len(h.built) != 0 { + t.Error("driver built for a malformed project reference") + } +} + +func TestListStatesUsesEmptyProjectRefAndValidatesID(t *testing.T) { + h := newHarness(t) + ctx := context.Background() + user := uuid.New() + c, err := h.svc.Connect(ctx, user, "youtrack", testHost, testToken("t")) + if err != nil { + t.Fatal(err) + } + h.built = nil + // The double's ListStates is unstubbed and errors; only the build matters here. + _, _ = h.svc.ListStates(ctx, user, c.ID, "0-1") + if len(h.built) != 1 || h.built[0].Project != (issuetracker.ProjectRef{}) { + t.Errorf("built = %+v, want one build with an empty Project", h.built) + } + h.built = nil + for _, bad := range []string{"", "abc", "0-1/../x", "1-", "0-1\n"} { + _, err = h.svc.ListStates(ctx, user, c.ID, bad) + wantKind(t, err, KindBadRequest) + } + if len(h.built) != 0 { + t.Error("driver built for a bad project id") + } +} + func TestListProjectsAndStates(t *testing.T) { h := newHarness(t) ctx := context.Background() From d4b592e6b54c13c403e0348e0cfc2f4a2685c868 Mon Sep 17 00:00:00 2001 From: uzi-agent Date: Wed, 11 Nov 2026 10:07:38 +0000 Subject: [PATCH 08/17] issuetracker: bound decoding before it allocates; pin the remaining guards List pages stream through a json.Decoder that refuses past the requested page size, and nested tag arrays stop at the per-item cap while decoding, so a hostile response cannot allocate far beyond its wire size. The service reuses the driver's exported token and project validators, and tests now pin every guard the mutation pass found unpinned. Co-Authored-By: Claude Opus 5.5 --- .../issuetracker/connection_service_test.go | 59 ++-- api/internal/issuetracker/export_test.go | 7 + api/internal/issuetracker/pagination.go | 25 +- api/internal/issuetracker/pagination_test.go | 24 +- api/internal/issuetracker/redact.go | 20 +- api/internal/issuetracker/redact_test.go | 12 +- api/internal/issuetracker/registry.go | 5 +- api/internal/issuetracker/validate.go | 52 +++ api/internal/issuetracker/youtrack.go | 14 +- api/internal/issuetracker/youtrack_decode.go | 82 +++++ .../issuetracker/youtrack_decode_test.go | 167 ++++++++++ .../issuetracker/youtrack_guards_test.go | 300 ++++++++++++++++++ api/internal/issuetracker/youtrack_query.go | 3 +- .../issuetracker/youtrack_query_test.go | 2 +- api/internal/issuetracker/youtrack_request.go | 44 +-- .../issuetracker/youtrack_security_test.go | 9 +- api/internal/issuetracker/youtrack_test.go | 12 +- api/internal/issuetracker/youtrack_wire.go | 13 +- .../issuetracker/youtracktest/server_test.go | 49 +++ api/internal/issuetrackersvc/baseurl.go | 11 +- api/internal/issuetrackersvc/connect.go | 28 +- api/internal/issuetrackersvc/driver.go | 15 +- api/internal/issuetrackersvc/service_test.go | 91 ++++-- 23 files changed, 851 insertions(+), 193 deletions(-) create mode 100644 api/internal/issuetracker/validate.go create mode 100644 api/internal/issuetracker/youtrack_decode.go create mode 100644 api/internal/issuetracker/youtrack_decode_test.go create mode 100644 api/internal/issuetracker/youtrack_guards_test.go diff --git a/api/internal/issuetracker/connection_service_test.go b/api/internal/issuetracker/connection_service_test.go index e63874ed92..c04b0aa045 100644 --- a/api/internal/issuetracker/connection_service_test.go +++ b/api/internal/issuetracker/connection_service_test.go @@ -16,6 +16,7 @@ import ( "github.com/google/uuid" + "github.com/vtmocanu/uzi/api/internal/config" "github.com/vtmocanu/uzi/api/internal/issuetracker" "github.com/vtmocanu/uzi/api/internal/issuetracker/youtracktest" "github.com/vtmocanu/uzi/api/internal/issuetrackersvc" @@ -90,32 +91,18 @@ func allowOnly(u string) func(string) bool { return func(got string) bool { return got == u } } -func connKind(t *testing.T) string { - t.Helper() - ks := issuetracker.SupportedKindsList() - if !strings.Contains(ks, "youtrack") { - t.Fatalf("youtrack is not a supported kind: %q", ks) - } - return "youtrack" -} - -func svcCredential() string { - return strings.Join([]string{"perm", ":", "ab c/d\"e", "-", "Zq9xLm2VnB7wKp3T"}, "") -} - -func svcTokenForms() []string { - c := svcCredential() - j, _ := json.Marshal(c) - return []string{c, url.QueryEscape(c), url.PathEscape(c), string(j[1 : len(j)-1])} -} - -func assertSvcScrubbed(t *testing.T, err error) { +// assertSvcScrubbed checks that err is a scrubbed *issuetrackersvc.Error of the +// given kind and returns it. +func assertSvcScrubbed(t *testing.T, err error, kind issuetrackersvc.Kind) { t.Helper() var se *issuetrackersvc.Error if !errors.As(err, &se) { t.Fatalf("want *issuetrackersvc.Error, got %T: %v", err, err) } - for _, f := range svcTokenForms() { + if se.Kind != kind { + t.Errorf("Kind = %d, want %d", se.Kind, kind) + } + for _, f := range issuetracker.TokenForms() { if strings.Contains(se.Message, f) { t.Errorf("message contains token form %q: %q", f, se.Message) } @@ -132,7 +119,7 @@ func TestConnectYouTrackHappyPath(t *testing.T) { srv := youtracktest.NewServer(t) h := newSvcHarness(t, srv.Client().Transport, allowOnly(srv.URL)) - conn, err := h.svc.Connect(context.Background(), uuid.New(), connKind(t), srv.URL, srv.Token) + conn, err := h.svc.Connect(context.Background(), uuid.New(), string(issuetracker.KindYouTrack), srv.URL, srv.Token) if err != nil { t.Fatalf("Connect: %v", err) } @@ -149,12 +136,14 @@ func TestConnectYouTrackHappyPath(t *testing.T) { } func TestConnectTokenEchoIsScrubbed(t *testing.T) { - c := svcCredential() + c := issuetracker.Credential() j, _ := json.Marshal(c) echo := strings.Join([]string{ c, url.QueryEscape(c), string(j[1 : len(j)-1]), - }, " | ") + "\r\n\x1b[31m‮" + }, " | ") + "\r\n\x1b[31m\u202e" + var hits atomic.Int64 srv := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + hits.Add(1) w.Header().Set("Content-Type", "text/plain") w.WriteHeader(http.StatusUnauthorized) _, _ = w.Write([]byte(echo)) @@ -162,31 +151,38 @@ func TestConnectTokenEchoIsScrubbed(t *testing.T) { t.Cleanup(srv.Close) h := newSvcHarness(t, srv.Client().Transport, allowOnly(srv.URL)) - _, err := h.svc.Connect(context.Background(), uuid.New(), connKind(t), srv.URL, c) + _, err := h.svc.Connect(context.Background(), uuid.New(), string(issuetracker.KindYouTrack), srv.URL, c) if err == nil { t.Fatal("want an error") } - assertSvcScrubbed(t, err) + assertSvcScrubbed(t, err, issuetrackersvc.KindBadGateway) + if n := hits.Load(); n != 1 { + t.Errorf("issue tracker got %d requests, want exactly the verification call", n) + } if n := len(h.q.stored()); n != 0 { t.Errorf("stored %d rows, want 0", n) } } func TestConnectRefusesRedirect(t *testing.T) { - var hits atomic.Int64 + var hits, primary atomic.Int64 other := httptest.NewTLSServer(http.HandlerFunc(func(http.ResponseWriter, *http.Request) { hits.Add(1) })) t.Cleanup(other.Close) srv := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + primary.Add(1) http.Redirect(w, r, other.URL+r.URL.Path, http.StatusFound) })) t.Cleanup(srv.Close) h := newSvcHarness(t, srv.Client().Transport, allowOnly(srv.URL)) - _, err := h.svc.Connect(context.Background(), uuid.New(), connKind(t), srv.URL, svcCredential()) + _, err := h.svc.Connect(context.Background(), uuid.New(), string(issuetracker.KindYouTrack), srv.URL, issuetracker.Credential()) if err == nil { t.Fatal("want an error") } - assertSvcScrubbed(t, err) + assertSvcScrubbed(t, err, issuetrackersvc.KindBadGateway) + if n := primary.Load(); n != 1 { + t.Errorf("primary server got %d requests, want exactly one", n) + } if n := hits.Load(); n != 0 { t.Errorf("redirect target got %d hits, want 0", n) } @@ -197,9 +193,10 @@ func TestConnectRefusesRedirect(t *testing.T) { func TestConnectEmptyAllowlistRefusesBeforeDriver(t *testing.T) { srv := youtracktest.NewServer(t) - h := newSvcHarness(t, srv.Client().Transport, func(string) bool { return false }) + // The real predicate with no allowlist configured. + h := newSvcHarness(t, srv.Client().Transport, config.Config{}.IssueTrackerBaseURLAllowed) - _, err := h.svc.Connect(context.Background(), uuid.New(), connKind(t), srv.URL, srv.Token) + _, err := h.svc.Connect(context.Background(), uuid.New(), string(issuetracker.KindYouTrack), srv.URL, srv.Token) var se *issuetrackersvc.Error if !errors.As(err, &se) || se.Kind != issuetrackersvc.KindBadRequest { t.Fatalf("want a bad-request *issuetrackersvc.Error, got %v", err) diff --git a/api/internal/issuetracker/export_test.go b/api/internal/issuetracker/export_test.go index ec19b9553f..8240b0e7d8 100644 --- a/api/internal/issuetracker/export_test.go +++ b/api/internal/issuetracker/export_test.go @@ -19,3 +19,10 @@ func NewYouTrackOver(opts Options, base http.RoundTripper) (Driver, error) { } return y, nil } + +// Credential and TokenForms share the internal tests' token fixtures with the +// external test package, so there is one copy of them. +var ( + Credential = credential + TokenForms = tokenForms +) diff --git a/api/internal/issuetracker/pagination.go b/api/internal/issuetracker/pagination.go index 64e0445ad4..34c6e64c58 100644 --- a/api/internal/issuetracker/pagination.go +++ b/api/internal/issuetracker/pagination.go @@ -5,20 +5,26 @@ import "errors" // pageSize is the $top of every list request. const pageSize = 100 +// errItemCeiling is returned when a listing carries more than the item budget +// (maxItems) or a page carries more than pageSize items. It carries no remote +// content. +var errItemCeiling = errors.New("issuetracker: listing exceeds the item ceiling") + // maxItems, maxPages and maxTotalBytes are sanity ceilings against a server -// that never ends a listing. Vars so tests can lower them. +// that never ends a listing. maxItems is one combined budget: it counts the +// top-level items of every page plus the nested entries (tags) they carry. Vars +// so tests can lower them. var ( maxItems = 100_000 maxPages = 2_000 maxTotalBytes = 64 << 20 ) -// paginate calls fetch with a growing skip until a short page. A positive limit -// stops once that many items are collected and truncates to it; zero means no -// cap. fetch returns the page and its raw byte count; the counts are summed -// across pages. Exceeding maxItems, maxPages or maxTotalBytes is an error -// carrying no remote content. -func paginate[T any](fetch func(skip, top int) ([]T, int, error), limit int) ([]T, error) { +// paginate calls fetch with a growing skip until a short page. fetch returns +// the page and its raw byte count; the counts are summed across pages. +// Exceeding maxItems, maxPages or maxTotalBytes is an error carrying no remote +// content. +func paginate[T any](fetch func(skip, top int) ([]T, int, error)) ([]T, error) { var all []T total := 0 for page := 0; ; page++ { @@ -34,11 +40,8 @@ func paginate[T any](fetch func(skip, top int) ([]T, int, error), limit int) ([] return nil, errors.New("issuetracker: listing exceeds the byte ceiling") } all = append(all, items...) - if limit > 0 && len(all) >= limit { - return all[:limit], nil - } if len(all) > maxItems { - return nil, errors.New("issuetracker: listing exceeds the item ceiling") + return nil, errItemCeiling } if len(items) < pageSize { return all, nil diff --git a/api/internal/issuetracker/pagination_test.go b/api/internal/issuetracker/pagination_test.go index 57e97cccd5..e9d5918cc3 100644 --- a/api/internal/issuetracker/pagination_test.go +++ b/api/internal/issuetracker/pagination_test.go @@ -19,7 +19,7 @@ func serve(total int, calls *[][2]int) func(skip, top int) ([]int, int, error) { func TestPaginateCollectsAll(t *testing.T) { var calls [][2]int - got, err := paginate(serve(250, &calls), 0) + got, err := paginate(serve(250, &calls)) if err != nil || len(got) != 250 { t.Fatalf("len %d, err %v", len(got), err) } @@ -30,23 +30,15 @@ func TestPaginateCollectsAll(t *testing.T) { func TestPaginateExactMultipleFetchesEmptyPage(t *testing.T) { var calls [][2]int - got, err := paginate(serve(200, &calls), 0) + got, err := paginate(serve(200, &calls)) if err != nil || len(got) != 200 || len(calls) != 3 { t.Fatalf("len %d, calls %d, err %v", len(got), len(calls), err) } } -func TestPaginateLimit(t *testing.T) { - var calls [][2]int - got, err := paginate(serve(1000, &calls), 150) - if err != nil || len(got) != 150 || len(calls) != 2 { - t.Fatalf("len %d, calls %d, err %v", len(got), len(calls), err) - } -} - func TestPaginateFetchError(t *testing.T) { want := errors.New("boom") - _, err := paginate(func(int, int) ([]int, int, error) { return nil, 0, want }, 0) + _, err := paginate(func(int, int) ([]int, int, error) { return nil, 0, want }) if !errors.Is(err, want) { t.Fatalf("err = %v", err) } @@ -58,8 +50,8 @@ func TestPaginateCaps(t *testing.T) { maxItems = 250 t.Cleanup(func() { maxItems = old }) var calls [][2]int - if _, err := paginate(serve(10_000, &calls), 0); err == nil { - t.Fatal("expected item ceiling error") + if _, err := paginate(serve(10_000, &calls)); !errors.Is(err, errItemCeiling) { + t.Fatalf("err = %v, want errItemCeiling", err) } }) t.Run("pages", func(t *testing.T) { @@ -67,7 +59,7 @@ func TestPaginateCaps(t *testing.T) { maxPages = 3 t.Cleanup(func() { maxPages = old }) var calls [][2]int - if _, err := paginate(serve(10_000, &calls), 0); err == nil { + if _, err := paginate(serve(10_000, &calls)); err == nil { t.Fatal("expected page ceiling error") } if len(calls) != 3 { @@ -83,7 +75,7 @@ func TestPaginateCaps(t *testing.T) { calls++ return make([]int, top), 400, nil } - if _, err := paginate(full, 0); err == nil { + if _, err := paginate(full); err == nil { t.Fatal("expected byte ceiling error") } if calls != 3 { @@ -96,7 +88,7 @@ func TestPaginateCaps(t *testing.T) { calls++ return make([]int, top), 1 << 20, nil } - if _, err := paginate(same, 0); err == nil { + if _, err := paginate(same); err == nil { t.Fatal("expected an error") } if calls > maxPages { diff --git a/api/internal/issuetracker/redact.go b/api/internal/issuetracker/redact.go index e3c11b3a7e..02ba781fab 100644 --- a/api/internal/issuetracker/redact.go +++ b/api/internal/issuetracker/redact.go @@ -8,7 +8,6 @@ import ( "net/http" "net/url" "strings" - "unicode" "unicode/utf8" "github.com/vtmocanu/uzi/api/internal/forge" @@ -28,23 +27,6 @@ const ( minSegmentLen = 8 ) -// validateToken rejects a token too short to redact safely. The message never -// includes the token. -func validateToken(token string) error { - if len(token) < minTokenLen { - return errors.New("issuetracker: token shorter than the minimum length") - } - if len(token) > maxTokenLen { - return errors.New("issuetracker: token longer than the maximum length") - } - for _, c := range token { - if unicode.IsControl(c) || unicode.Is(unicode.Cf, c) { - return errors.New("issuetracker: token contains a control or format character") - } - } - return nil -} - // redactor scrubs one token out of every error a driver returns. Build it from // the token once, in the constructor. type redactor struct { @@ -156,7 +138,7 @@ func (r redactor) string(s string) string { // with a space. It uses the predicate of issueinput.AuthorName. func sanitize(s string) string { return strings.Map(func(c rune) rune { - if unicode.IsControl(c) || unicode.Is(unicode.Cf, c) || c == '\u2028' || c == '\u2029' { + if invisible(c) { return ' ' } return c diff --git a/api/internal/issuetracker/redact_test.go b/api/internal/issuetracker/redact_test.go index 0ea65e3367..49904593dc 100644 --- a/api/internal/issuetracker/redact_test.go +++ b/api/internal/issuetracker/redact_test.go @@ -221,20 +221,20 @@ func TestClassifyStatusServerErrorRange(t *testing.T) { } func TestValidateToken(t *testing.T) { - if validateToken(strings.Repeat("a", minTokenLen-1)) == nil { + if ValidateToken(strings.Repeat("a", minTokenLen-1)) == nil { t.Fatal("short token accepted") } - if err := validateToken(strings.Repeat("a", minTokenLen)); err != nil { + if err := ValidateToken(strings.Repeat("a", minTokenLen)); err != nil { t.Fatal(err) } short := strings.Repeat("q", minTokenLen-1) - if strings.Contains(validateToken(short).Error(), short) { + if strings.Contains(ValidateToken(short).Error(), short) { t.Fatal("error leaks token") } } func TestValidateTokenBoundsAndCharacters(t *testing.T) { - if err := validateToken(strings.Repeat("a", maxTokenLen)); err != nil { + if err := ValidateToken(strings.Repeat("a", maxTokenLen)); err != nil { t.Errorf("a token of the maximum length: %v", err) } cases := map[string]string{ @@ -244,9 +244,11 @@ func TestValidateTokenBoundsAndCharacters(t *testing.T) { "zero width": strings.Repeat("a", 20) + "\u200b", "bidi": strings.Repeat("a", 20) + "\u202e", "delete": strings.Repeat("a", 20) + "\x7f", + "line sep": strings.Repeat("a", 20) + "\u2028", + "para sep": strings.Repeat("a", 20) + "\u2029", } for name, tok := range cases { - err := validateToken(tok) + err := ValidateToken(tok) if err == nil { t.Errorf("%s: accepted", name) continue diff --git a/api/internal/issuetracker/registry.go b/api/internal/issuetracker/registry.go index 9bd737517c..2e94af5eab 100644 --- a/api/internal/issuetracker/registry.go +++ b/api/internal/issuetracker/registry.go @@ -2,7 +2,9 @@ package issuetracker import "fmt" -// Constructor builds a driver from options. +// Constructor builds a driver from options. A build error never carries the +// token, a part of it or any other remote content, because callers show it to +// the user. type Constructor func(Options) (Driver, error) // constructors maps each Kind to its driver constructor. A driver adds one @@ -10,6 +12,7 @@ type Constructor func(Options) (Driver, error) var constructors = map[Kind]Constructor{KindYouTrack: newYouTrack} // New constructs the driver for kind. An unknown kind wraps ErrUnsupportedKind. +// Like every Constructor, it never puts the token in an error. func New(kind Kind, opts Options) (Driver, error) { ctor, ok := constructors[kind] if !ok { diff --git a/api/internal/issuetracker/validate.go b/api/internal/issuetracker/validate.go new file mode 100644 index 0000000000..2f2d70c987 --- /dev/null +++ b/api/internal/issuetracker/validate.go @@ -0,0 +1,52 @@ +package issuetracker + +import ( + "errors" + "unicode" +) + +// ValidateToken rejects a token a driver would not accept: shorter than +// minTokenLen (too short to redact safely), longer than maxTokenLen, or holding +// a control character, a Unicode format character or a line or paragraph +// separator. The message never includes the token. +func ValidateToken(token string) error { + if len(token) < minTokenLen { + return errors.New("issuetracker: token shorter than the minimum length") + } + if len(token) > maxTokenLen { + return errors.New("issuetracker: token longer than the maximum length") + } + for _, c := range token { + if invisible(c) { + return errors.New("issuetracker: token contains a control or format character") + } + } + return nil +} + +// invisible reports whether c is a control character, a Unicode format +// character (bidi controls, zero-width characters, BOM) or the line or +// paragraph separator. +func invisible(c rune) bool { + return unicode.IsControl(c) || unicode.Is(unicode.Cf, c) || c == '\u2028' || c == '\u2029' +} + +// ValidProjectID reports whether id is a well-formed project entity id such as +// "0-1". +func ValidProjectID(id string) bool { return entityIDRE.MatchString(id) } + +// ValidProjectRef accepts the empty ProjectRef (no project bound) and +// otherwise requires a well-formed id and short name. The message never +// includes the input. +func ValidProjectRef(ref ProjectRef) error { + if ref == (ProjectRef{}) { + return nil + } + if !shortNameRE.MatchString(ref.ShortName) { + return errors.New("issuetracker: invalid project short name") + } + if !ValidProjectID(ref.ID) { + return errors.New("issuetracker: invalid project id") + } + return nil +} diff --git a/api/internal/issuetracker/youtrack.go b/api/internal/issuetracker/youtrack.go index 033ae02dff..77a951795e 100644 --- a/api/internal/issuetracker/youtrack.go +++ b/api/internal/issuetracker/youtrack.go @@ -1,7 +1,6 @@ package issuetracker import ( - "errors" "fmt" "net/http" "net/url" @@ -42,22 +41,17 @@ func newYouTrack(opts Options) (Driver, error) { } // buildYouTrack validates opts and builds the driver over base, which is nil -// outside tests. +// outside tests. A build error never carries the token or any part of it. func buildYouTrack(opts Options, base http.RoundTripper) (*youTrack, error) { baseURL, err := strictBaseURL(opts.BaseURL) if err != nil { return nil, err } - if err := validateToken(opts.Token); err != nil { + if err := ValidateToken(opts.Token); err != nil { return nil, err } - if opts.Project != (ProjectRef{}) { - if !shortNameRE.MatchString(opts.Project.ShortName) { - return nil, errors.New("issuetracker: invalid project short name") - } - if !entityIDRE.MatchString(opts.Project.ID) { - return nil, errors.New("issuetracker: invalid project id") - } + if err := ValidProjectRef(opts.Project); err != nil { + return nil, err } stateField := opts.StateField if stateField == "" { diff --git a/api/internal/issuetracker/youtrack_decode.go b/api/internal/issuetracker/youtrack_decode.go new file mode 100644 index 0000000000..8ad4410636 --- /dev/null +++ b/api/internal/issuetracker/youtrack_decode.go @@ -0,0 +1,82 @@ +package issuetracker + +import ( + "bytes" + "encoding/json" + "errors" + "io" +) + +// maxNestedItems caps the tags of one issue or one activity. +const maxNestedItems = 1000 + +// errNestedCeiling is returned when one object carries more than maxNestedItems +// nested entries. +var errNestedCeiling = errors.New("issuetracker: an item carries more nested entries than the ceiling allows") + +// errNotArray is returned when a list response is not a JSON array. +var errNotArray = errors.New("issuetracker: response is not a JSON array") + +// errTrailingData is returned when a list response has data after its array. +var errTrailingData = errors.New("issuetracker: response has data after the JSON array") + +// ytTags is a nested tag array. Its UnmarshalJSON streams the elements and +// refuses once more than maxNestedItems have been read, so one object cannot +// make the decoder allocate a slice as large as the response. +type ytTags []ytTag + +// UnmarshalJSON implements json.Unmarshaler. +func (t *ytTags) UnmarshalJSON(b []byte) error { + items, err := decodeArray[ytTag](b, maxNestedItems, errNestedCeiling) + if err != nil { + return err + } + *t = items + return nil +} + +// decodeArray decodes a JSON array element by element and returns refuse as +// soon as it meets element number limit+1, before that element is decoded, so at +// most limit elements are ever allocated. A JSON null is an empty array. +func decodeArray[T any](b []byte, limit int, refuse error) ([]T, error) { + dec := json.NewDecoder(bytes.NewReader(b)) + tok, err := dec.Token() + if err != nil { + return nil, err + } + if tok == nil { + return nil, endOfInput(dec) + } + if d, ok := tok.(json.Delim); !ok || d != '[' { + return nil, errNotArray + } + var items []T + for dec.More() { + if len(items) >= limit { + return nil, refuse + } + var it T + if err := dec.Decode(&it); err != nil { + return nil, err + } + items = append(items, it) + } + if _, err := dec.Token(); err != nil { + return nil, err + } + return items, endOfInput(dec) +} + +// endOfInput reports errTrailingData unless dec has nothing left. +func endOfInput(dec *json.Decoder) error { + if _, err := dec.Token(); err != io.EOF { + return errTrailingData + } + return nil +} + +// decodePage decodes one list page. The driver asked for pageSize items, so a +// longer array is a protocol violation and is refused while decoding. +func decodePage[T any](body []byte) ([]T, error) { + return decodeArray[T](body, pageSize, errItemCeiling) +} diff --git a/api/internal/issuetracker/youtrack_decode_test.go b/api/internal/issuetracker/youtrack_decode_test.go new file mode 100644 index 0000000000..6d09c8627c --- /dev/null +++ b/api/internal/issuetracker/youtrack_decode_test.go @@ -0,0 +1,167 @@ +package issuetracker + +import ( + "context" + "net/http" + "runtime" + "strings" + "testing" + + "github.com/vtmocanu/uzi/api/internal/forge" +) + +// allocBound is the most a decode test lets the process allocate while it +// serves and decodes one oversized response. The response itself is read into a +// buffer of at most 8 MiB (grown by append, so several times that is allocated +// in total); the unbounded decode this guards against allocated about 1 GiB. +const allocBound = 64 << 20 + +// totalAlloc reports the bytes allocated while f runs. +func totalAlloc(f func()) uint64 { + runtime.GC() + var before, after runtime.MemStats + runtime.ReadMemStats(&before) + f() + runtime.ReadMemStats(&after) + return after.TotalAlloc - before.TotalAlloc +} + +// emptyObjects renders a JSON array of n empty objects. +func emptyObjects(n int) string { + return "[" + strings.Repeat("{},", n-1) + "{}]" +} + +func TestListPageOverPageSizeIsRefused(t *testing.T) { + project := `{"id":"0-1","shortName":"UZI","name":"Uzi"}` + page := func(n int) http.HandlerFunc { + body := "[" + strings.TrimSuffix(strings.Repeat(project+",", n), ",") + "]" + return func(w http.ResponseWriter, r *http.Request) { + if r.URL.Query().Get("$skip") != "0" { + writeRaw(w, "[]") + return + } + writeRaw(w, body) + } + } + y, _ := loggedDriver(t, page(pageSize)) + if got, err := y.ListProjects(context.Background()); err != nil || len(got) != pageSize { + t.Fatalf("a page of exactly pageSize items: %d items, %v", len(got), err) + } + y, _ = loggedDriver(t, page(pageSize+1)) + _, err := y.ListProjects(context.Background()) + if err == nil || !strings.Contains(err.Error(), "item ceiling") { + t.Fatalf("a page of pageSize+1 items: %v, want the item ceiling", err) + } +} + +func TestOversizedPageOfEmptyObjectsAllocatesLittle(t *testing.T) { + n := int(listPageBytes) / 3 + body := []byte(emptyObjects(n)) + if int64(len(body)) > listPageBytes { + t.Fatalf("fixture is %d bytes, over the page cap", len(body)) + } + y, _ := loggedDriver(t, func(w http.ResponseWriter, _ *http.Request) { _, _ = w.Write(body) }) + var err error + got := totalAlloc(func() { _, err = y.ListProjects(context.Background()) }) + if err == nil || !strings.Contains(err.Error(), "item ceiling") { + t.Fatalf("err = %v, want the item ceiling", err) + } + if got > allocBound { + t.Errorf("allocated %d MiB for a %d MiB page, want under %d MiB", got>>20, len(body)>>20, allocBound>>20) + } +} + +func TestTagBombAllocatesLittle(t *testing.T) { + prefix := rawIssue("2-1", "UZI-1", 1, "0-1", "") + prefix = strings.TrimSuffix(prefix, `"tags":[]}`) + `"tags":` + n := int(objectBytes-int64(len(prefix))-8) / 3 + body := []byte(prefix + emptyObjects(n) + "}") + if int64(len(body)) > objectBytes { + t.Fatalf("fixture is %d bytes, over the object cap", len(body)) + } + y, _ := loggedDriver(t, func(w http.ResponseWriter, _ *http.Request) { _, _ = w.Write(body) }) + var err error + got := totalAlloc(func() { _, err = y.GetIssue(context.Background(), 1, 1) }) + if err == nil { + t.Fatal("GetIssue accepted a tag bomb") + } + if got > allocBound { + t.Errorf("allocated %d MiB for a %d MiB issue, want under %d MiB", got>>20, len(body)>>20, allocBound>>20) + } +} + +func TestListDecodeShapes(t *testing.T) { + for name, tc := range map[string]struct { + body string + wantErr bool + }{ + "null is an empty list": {`null`, false}, + "object": {`{}`, true}, + "trailing data": {`[] []`, true}, + "truncated": {`[{"id":"0-1"`, true}, + } { + t.Run(name, func(t *testing.T) { + y, _ := loggedDriver(t, func(w http.ResponseWriter, _ *http.Request) { writeRaw(w, tc.body) }) + got, err := y.ListProjects(context.Background()) + if (err != nil) != tc.wantErr || (err == nil && len(got) != 0) { + t.Errorf("got %v, %v; wantErr %v", got, err, tc.wantErr) + } + }) + } +} + +func tagsJSON(n int) string { + return strings.TrimSuffix(strings.Repeat(`{"id":"5-1","name":"a"},`, n), ",") +} + +func TestNestedTagCeilingBoundary(t *testing.T) { + ctx := context.Background() + for _, tc := range []struct { + n int + wantErr bool + }{{maxNestedItems, false}, {maxNestedItems + 1, true}} { + issue := rawIssue("2-1", "UZI-1", 1, "0-1", tagsJSON(tc.n)) + y, _ := loggedDriver(t, func(w http.ResponseWriter, _ *http.Request) { writeRaw(w, issue) }) + got, err := y.GetIssue(ctx, 1, 1) + if (err != nil) != tc.wantErr || (err == nil && len(got.Labels) != tc.n) { + t.Errorf("%d tags: %d labels, %v; wantErr %v", tc.n, len(got.Labels), err, tc.wantErr) + } + } +} + +func TestActivityNestedCeilings(t *testing.T) { + ctx := context.Background() + for name, act := range map[string]string{ + "added": `[{"id":"3-1","timestamp":1,"added":[` + tagsJSON(maxNestedItems+1) + `]}]`, + "removed": `[{"id":"3-1","timestamp":1,"removed":[` + tagsJSON(maxNestedItems+1) + `]}]`, + } { + t.Run(name, func(t *testing.T) { + y, _ := loggedDriver(t, activityHandler(act)) + if got, err := y.ListIssueLabelEvents(ctx, 1, 1); err == nil { + t.Errorf("%d events accepted for more than %d nested entries", len(got), maxNestedItems) + } + }) + } + t.Run("removed only over the budget", func(t *testing.T) { + lowerMaxItems(t, 5) + act := `{"id":"3-1","timestamp":1,"removed":[{"id":"5-1","name":"a"},{"id":"5-2","name":"b"},{"id":"5-3","name":"c"}]}` + y, _ := loggedDriver(t, activityHandler("["+act+","+act+"]")) + if _, err := y.ListIssueLabelEvents(ctx, 1, 1); err == nil { + t.Error("removed tags did not count toward the item ceiling") + } + }) +} + +func TestTopLevelItemsAndNestedTagsShareOneBudget(t *testing.T) { + lowerMaxItems(t, 10) + one := rawIssue("2-1", "UZI-1", 1, "0-1", `{"id":"5-1","name":"a"}`) + y, _ := loggedDriver(t, rawListHandler(one, one, one, one, one, one)) + // Six items alone fit under 10, and so do six tags alone; together they do not. + if _, err := y.ListIssues(context.Background(), 1, forge.ListIssuesOptions{}); err == nil { + t.Error("6 items + 6 tags = 12 accepted under a ceiling of 10") + } + y, _ = loggedDriver(t, rawListHandler(one, one, one, one)) + if _, err := y.ListIssues(context.Background(), 1, forge.ListIssuesOptions{}); err != nil { + t.Errorf("4 items + 4 tags = 8 refused under a ceiling of 10: %v", err) + } +} diff --git a/api/internal/issuetracker/youtrack_guards_test.go b/api/internal/issuetracker/youtrack_guards_test.go new file mode 100644 index 0000000000..ae254a02a9 --- /dev/null +++ b/api/internal/issuetracker/youtrack_guards_test.go @@ -0,0 +1,300 @@ +package issuetracker + +import ( + "context" + "errors" + "net/http" + "strings" + "testing" + "time" + + "github.com/vtmocanu/uzi/api/internal/forge" +) + +func TestSetIssueStateRefusesAStateThatIsNeitherOpenedNorClosed(t *testing.T) { + y, srv := newFakeDriver(t) + n := srv.SeedIssue("t", "") + srv.ResetRequests() + if err := y.SetIssueState(context.Background(), 1, n, forge.IssueState("merged")); err == nil { + t.Fatal("SetIssueState accepted an unknown state") + } + for _, r := range srv.Requests() { + if r.Method != http.MethodGet { + t.Errorf("sent a write: %v", r) + } + } +} + +func TestSetIssueStateOnAnUnboundDriverWithNoMappingIsNotBound(t *testing.T) { + y, err := newYouTrackOver(Options{BaseURL: "https://yt.example.com", Token: credential()}, roundTripFunc( + func(*http.Request) (*http.Response, error) { return nil, errors.New("no request expected") })) + if err != nil { + t.Fatal(err) + } + err = y.SetIssueState(context.Background(), 1, 1, forge.StateClosed) + if !errors.Is(err, ErrProjectNotBound) { + t.Fatalf("err = %v, want ErrProjectNotBound (checked before the mapping)", err) + } +} + +func TestListIssuesRefusesMalformedIDs(t *testing.T) { + for name, issue := range map[string]string{ + "id-readable": rawIssue("2-1", "x y/../z", 1, "0-1", ""), + "entity id": rawIssue("../..", "UZI-1", 1, "0-1", ""), + "tag id": rawIssue("2-1", "UZI-1", 1, "0-1", `{"id":"..","name":"x"}`), + } { + t.Run(name, func(t *testing.T) { + y, _ := loggedDriver(t, rawListHandler(issue)) + if got, err := y.ListIssues(context.Background(), 1, forge.ListIssuesOptions{}); err == nil { + t.Errorf("ListIssues accepted a malformed %s: %+v", name, got) + } + }) + } +} + +func TestListIssuesLimitStopsAcrossPages(t *testing.T) { + y, srv := newFakeDriver(t) + for range 250 { + srv.SeedIssue("t", "") + } + srv.ResetRequests() + got, err := y.ListIssues(context.Background(), 1, forge.ListIssuesOptions{Limit: 5}) + if err != nil || len(got) != 5 { + t.Fatalf("got %d issues, %v; want 5", len(got), err) + } + reqs := srv.Requests() + if len(reqs) != 1 || reqs[0].Method != http.MethodGet || reqs[0].Path != "/api/issues" { + t.Errorf("requests = %v, want one GET /api/issues", reqs) + } +} + +func TestIssueWithoutANumberIsNotInTheProject(t *testing.T) { + zero := rawIssue("2-9", "UZI-9", 0, "0-1", "") + good := rawIssue("2-1", "UZI-1", 1, "0-1", "") + y, _ := loggedDriver(t, rawListHandler(zero, good)) + got, err := y.ListIssues(context.Background(), 1, forge.ListIssuesOptions{}) + if err != nil || len(got) != 1 || got[0].IID != 1 { + t.Fatalf("ListIssues = %+v, %v; want only issue 1", got, err) + } + // CreateIssue reads the created issue back; a number of 0 there is refused. + y, _ = loggedDriver(t, func(w http.ResponseWriter, r *http.Request) { + if r.Method == http.MethodPost || strings.HasSuffix(r.URL.Path, "/2-1") { + writeRaw(w, rawIssue("2-1", "UZI-1", 0, "0-1", "")) + return + } + writeRaw(w, "[]") + }) + if got, err := y.CreateIssue(context.Background(), 1, "t", "d", nil); err == nil { + t.Errorf("CreateIssue accepted a read-back with number 0: %+v", got) + } +} + +func TestCreateIssueRefusesAMalformedCreatedID(t *testing.T) { + y, log := loggedDriver(t, func(w http.ResponseWriter, r *http.Request) { + if r.Method == http.MethodPost { + writeRaw(w, rawIssue("..", "UZI-1", 1, "0-1", "")) + return + } + writeRaw(w, rawIssue("2-1", "UZI-1", 1, "0-1", "")) + }) + if _, err := y.CreateIssue(context.Background(), 1, "t", "d", []string{"a"}); err == nil { + t.Fatal("CreateIssue accepted a malformed created id") + } + if got := log.all(); len(got) != 1 { + t.Errorf("requests = %v, want only the POST", got) + } +} + +func TestUpdateIssueLabelsDeduplicates(t *testing.T) { + y, srv := newFakeDriver(t) + n := srv.SeedIssue("t", "") + ctx := context.Background() + count := func(method, suffix string) int { + c := 0 + for _, r := range srv.Requests() { + if r.Method == method && strings.HasSuffix(r.Path, suffix) && strings.Contains(r.Path, "/issues/") { + c++ + } + } + return c + } + srv.ResetRequests() + if err := y.UpdateIssueLabels(ctx, 1, n, []string{"x", "x"}, nil); err != nil { + t.Fatal(err) + } + if got := count(http.MethodPost, "/tags"); got != 1 { + t.Errorf("%d attach calls for [x x], want 1", got) + } + srv.ResetRequests() + if err := y.UpdateIssueLabels(ctx, 1, n, nil, []string{"x", "x"}); err != nil { + t.Fatal(err) + } + deletes := 0 + for _, r := range srv.Requests() { + if r.Method == http.MethodDelete { + deletes++ + } + } + if deletes != 1 { + t.Errorf("%d detach calls for [x x], want 1", deletes) + } +} + +func TestUpdateIssueLabelsSkipsTheTagListingForAPresentLabel(t *testing.T) { + y, srv := newFakeDriver(t) + n := srv.SeedIssue("t", "", "x") + srv.ResetRequests() + if err := y.UpdateIssueLabels(context.Background(), 1, n, []string{"x"}, nil); err != nil { + t.Fatal(err) + } + for _, r := range srv.Requests() { + if strings.HasPrefix(r.Path, "/api/tags") || r.Method != http.MethodGet { + t.Errorf("unexpected request for an already-present label: %v", r) + } + } +} + +func TestListedCommentsAndEventsCarryTheirTimestamps(t *testing.T) { + y, _ := loggedDriver(t, func(w http.ResponseWriter, r *http.Request) { + switch { + case strings.HasSuffix(r.URL.Path, "/comments"): + writeRaw(w, `[{"id":"4-1","text":"hi","created":1500,"author":{"id":"1-1","login":"alice"}}]`) + case strings.HasSuffix(r.URL.Path, "/activities"): + writeRaw(w, `[{"id":"3-1","timestamp":2500,"author":{"id":"1-1","login":"bob"},"added":[{"id":"5-1","name":"a"}]}]`) + default: + writeRaw(w, rawIssue("2-1", "UZI-1", 1, "0-1", "")) + } + }) + ctx := context.Background() + comments, err := y.ListIssueComments(ctx, 1, 1) + if err != nil || len(comments) != 1 || !comments[0].CreatedAt.Equal(time.UnixMilli(1500)) || comments[0].AuthorUsername != "alice" { + t.Errorf("comments = %+v, %v", comments, err) + } + events, err := y.ListIssueLabelEvents(ctx, 1, 1) + if err != nil || len(events) != 1 || !events[0].CreatedAt.Equal(time.UnixMilli(2500)) || events[0].Username != "bob" { + t.Errorf("events = %+v, %v", events, err) + } +} + +func TestErrorTextJoinsErrorAndDescription(t *testing.T) { + y, _ := loggedDriver(t, func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusBadRequest) + writeRaw(w, `{"error":"bad_request","error_description":"nope"}`) + }) + _, err := y.GetIssue(context.Background(), 1, 1) + if err == nil || !strings.Contains(err.Error(), "bad_request: nope") { + t.Errorf("err = %v, want a message with \"bad_request: nope\"", err) + } +} + +func TestRawErrorBodyIsCutToTwoHundredBytes(t *testing.T) { + y, _ := loggedDriver(t, func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusBadRequest) + writeRaw(w, strings.Repeat("x", 400)) + }) + _, err := y.GetIssue(context.Background(), 1, 1) + if err == nil || strings.Count(err.Error(), "x") != 200 { + t.Errorf("err = %v, want exactly 200 body bytes", err) + } +} + +func TestResponseFailuresAreClassifiedAndRedacted(t *testing.T) { + classified := func(t *testing.T, err error) *classifiedError { + t.Helper() + var ce *classifiedError + if !errors.As(err, &ce) { + t.Fatalf("err %T %v is not a redacted, classified error", err, err) + } + return ce + } + t.Run("decode error", func(t *testing.T) { + y, _ := loggedDriver(t, func(w http.ResponseWriter, _ *http.Request) { writeRaw(w, `{not json`) }) + _, err := y.GetIssue(context.Background(), 1, 1) + if ce := classified(t, err); !strings.Contains(ce.message, "decode response") { + t.Errorf("message = %q", ce.message) + } + }) + // A body shorter than its Content-Length ends in a read error, on both the + // success path and the error-body path. + truncated := func(status int) http.HandlerFunc { + return func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Length", "100") + w.WriteHeader(status) + _, _ = w.Write([]byte("0123456789")) + w.(http.Flusher).Flush() + panic(http.ErrAbortHandler) + } + } + t.Run("read error on success", func(t *testing.T) { + y, _ := loggedDriver(t, truncated(http.StatusOK)) + _, err := y.GetIssue(context.Background(), 1, 1) + classified(t, err) + }) + t.Run("read error on a server error", func(t *testing.T) { + y, _ := loggedDriver(t, truncated(http.StatusInternalServerError)) + _, err := y.GetIssue(context.Background(), 1, 1) + if ce := classified(t, err); ce.class != forge.ErrorClassServerError { + t.Errorf("class = %v, want server error", ce.class) + } + }) +} + +func TestRedactorScrubsBeforeItTruncates(t *testing.T) { + tok := fixtureCredential() + r := newRedactor(tok) + // The token starts before the cut at maxMessageBytes and ends after it, so + // truncating first would leave its prefix behind. + out := r.string(strings.Repeat("a", maxMessageBytes-7) + tok) + if strings.Contains(out, "perm") { + t.Errorf("a token that straddles the cut left a prefix: %q", out[len(out)-12:]) + } +} + +func TestValidProjectRef(t *testing.T) { + for ref, want := range map[ProjectRef]bool{ + {}: true, + {ID: "0-1", ShortName: "UZI"}: true, + {ID: "0-1"}: false, + {ShortName: "UZI"}: false, + {ID: "../x", ShortName: "UZI"}: false, + {ID: "0-1", ShortName: "1X"}: false, + {ID: "0-1", ShortName: "U Z I"}: false, + } { + if got := ValidProjectRef(ref) == nil; got != want { + t.Errorf("ValidProjectRef(%+v) ok = %v, want %v", ref, got, want) + } + } + if !ValidProjectID("12-3") || ValidProjectID("12") || ValidProjectID("") { + t.Error("ValidProjectID disagrees with the entity id shape") + } +} + +// TestBuildErrorsNeverCarryTheToken builds a driver with each kind of invalid +// option and checks that no message holds the token or a piece of it. +func TestBuildErrorsNeverCarryTheToken(t *testing.T) { + long := strings.Repeat("Qz7", maxTokenLen) + secret := strings.Join([]string{"perm", ":", "Zq9xLm2VnB7wKp3T", "-", "Yk3mQ8vR2xTn5Lw7"}, "") + good := Options{BaseURL: "https://yt.example.com", Token: secret} + for name, o := range map[string]Options{ + "bad URL": {BaseURL: "http://yt.example.com", Token: secret}, + "URL with token": {BaseURL: "https://" + secret + "@yt.example.com", Token: secret}, + "control char": {BaseURL: good.BaseURL, Token: secret + "\u001b"}, + "line separator": {BaseURL: good.BaseURL, Token: secret + "\u2028"}, + "too long": {BaseURL: good.BaseURL, Token: long}, + "bad project": {BaseURL: good.BaseURL, Token: secret, Project: ProjectRef{ID: "x", ShortName: "UZI"}}, + "bad short name": {BaseURL: good.BaseURL, Token: secret, Project: ProjectRef{ID: "0-1", ShortName: "1"}}, + } { + _, err := New(KindYouTrack, o) + if err == nil { + t.Errorf("%s: accepted", name) + continue + } + msg := err.Error() + for _, w := range eightCharWindows(o.Token) { + if strings.Contains(msg, w) { + t.Errorf("%s: the message carries the token fragment %q", name, w) + break + } + } + } +} diff --git a/api/internal/issuetracker/youtrack_query.go b/api/internal/issuetracker/youtrack_query.go index 78bb033dd7..15742e1c24 100644 --- a/api/internal/issuetracker/youtrack_query.go +++ b/api/internal/issuetracker/youtrack_query.go @@ -3,7 +3,6 @@ package issuetracker import ( "strings" "time" - "unicode" "github.com/vtmocanu/uzi/api/internal/forge" ) @@ -17,7 +16,7 @@ const maxLabelNameLen = 200 func (y *youTrack) checkLabelName(name string) error { bad := strings.TrimSpace(name) == "" || len(name) > maxLabelNameLen for _, c := range name { - if c == '{' || c == '}' || unicode.IsControl(c) || unicode.Is(unicode.Cf, c) || c == '\u2028' || c == '\u2029' { + if c == '{' || c == '}' || invisible(c) { bad = true } } diff --git a/api/internal/issuetracker/youtrack_query_test.go b/api/internal/issuetracker/youtrack_query_test.go index 5d51e27fde..38e2fc0db4 100644 --- a/api/internal/issuetracker/youtrack_query_test.go +++ b/api/internal/issuetracker/youtrack_query_test.go @@ -26,7 +26,7 @@ func TestIssueQuery(t *testing.T) { func TestLabelNameGuard(t *testing.T) { y, _ := newFakeDriver(t) ctx := context.Background() - for _, name := range []string{"", " ", "a}b", "a{b", "a\nb", "a‮b"} { + for _, name := range []string{"", " ", "a}b", "a{b", "a\nb", "a\u202eb", "a\u2028b", "a\u2029b"} { if err := y.EnsureLabels(ctx, 1, []forge.Label{{Name: name}}); !errors.Is(err, ErrInvalidLabelName) { t.Errorf("EnsureLabels(%q) = %v, want ErrInvalidLabelName", name, err) } diff --git a/api/internal/issuetracker/youtrack_request.go b/api/internal/issuetracker/youtrack_request.go index 78db4f1cf5..d3fbcd4c86 100644 --- a/api/internal/issuetracker/youtrack_request.go +++ b/api/internal/issuetracker/youtrack_request.go @@ -4,7 +4,6 @@ import ( "bytes" "context" "encoding/json" - "errors" "fmt" "net/http" "net/url" @@ -23,6 +22,9 @@ type ytCall struct { body any limit int64 prov forge.ResponseProvenance + // decode, when set, replaces json.Unmarshal into out; ytPages uses it to + // stream a list page. + decode func([]byte) error } // escapedPath joins the segments, escaping every one. @@ -85,10 +87,15 @@ func (y *youTrack) do(ctx context.Context, c ytCall, out any) (status, n int, er if err != nil { return status, 0, y.red.errorStatus(err, status) } - if out != nil { - if err := json.Unmarshal(body, out); err != nil { - return status, len(body), y.red.errorStatus(fmt.Errorf("youtrack: %s %s: decode response: %w", c.method, path, err), status) - } + var decodeErr error + switch { + case c.decode != nil: + decodeErr = c.decode(body) + case out != nil: + decodeErr = json.Unmarshal(body, out) + } + if decodeErr != nil { + return status, len(body), y.red.errorStatus(fmt.Errorf("youtrack: %s %s: decode response: %w", c.method, path, decodeErr), status) } return status, len(body), nil } @@ -115,19 +122,18 @@ func (y *youTrack) errorText(body []byte) string { return truncate(y.red.scrub(string(body)), rawLimit) } -// maxNestedItems caps the tags of one issue or one activity. -const maxNestedItems = 1000 - // nester is implemented by a wire type that carries nested arrays; ytPages -// counts them toward maxItems and caps them at maxNestedItems. +// counts them toward the maxItems budget. Each array is already capped at +// maxNestedItems while it is decoded. type nester interface { nestedCount() int } // ytPages walks every page of a list endpoint under paginate's ceilings and // calls visit with each page. visit returns true to stop early. Items that -// implement nester count their nested entries toward maxItems. Errors come -// back redacted. +// implement nester count their nested entries toward maxItems. A page is +// decoded as a stream and refused as soon as it holds more than pageSize +// items. Errors come back redacted. func ytPages[T any](ctx context.Context, y *youTrack, path []string, query url.Values, visit func([]T) bool) error { var fetchErr error seen := 0 @@ -139,7 +145,10 @@ func ytPages[T any](ctx context.Context, y *youTrack, path []string, query url.V q.Set("$top", strconv.Itoa(top)) q.Set("$skip", strconv.Itoa(skip)) var items []T - _, n, err := y.do(ctx, ytCall{method: http.MethodGet, path: path, query: q, limit: listPageBytes, prov: forge.ResponseSuccess}, &items) + _, n, err := y.do(ctx, ytCall{ + method: http.MethodGet, path: path, query: q, limit: listPageBytes, prov: forge.ResponseSuccess, + decode: func(b []byte) (err error) { items, err = decodePage[T](b); return err }, + }, nil) if err != nil { fetchErr = err return nil, 0, err @@ -147,16 +156,11 @@ func ytPages[T any](ctx context.Context, y *youTrack, path []string, query url.V seen += len(items) for _, it := range items { if nn, ok := any(it).(nester); ok { - c := nn.nestedCount() - if c > maxNestedItems { - fetchErr = y.fail("youtrack: an item carries more than %d nested entries", maxNestedItems) - return nil, 0, fetchErr - } - seen += c + seen += nn.nestedCount() } } if seen > maxItems { - fetchErr = errors.New("issuetracker: listing exceeds the item ceiling") + fetchErr = y.red.error(errItemCeiling) return nil, 0, fetchErr } if visit(items) { @@ -164,7 +168,7 @@ func ytPages[T any](ctx context.Context, y *youTrack, path []string, query url.V return nil, n, nil } return items, n, nil - }, 0) + }) if fetchErr != nil { return fetchErr } diff --git a/api/internal/issuetracker/youtrack_security_test.go b/api/internal/issuetracker/youtrack_security_test.go index 93d3df05f8..01c33973e4 100644 --- a/api/internal/issuetracker/youtrack_security_test.go +++ b/api/internal/issuetracker/youtrack_security_test.go @@ -89,6 +89,9 @@ func TestRedirectsAreRefused(t *testing.T) { if !errors.Is(err, ErrRedirectRefused) { t.Errorf("%s: %v, want ErrRedirectRefused", name, err) } + if err != nil && !strings.Contains(err.Error(), fmt.Sprintf("HTTP %d", code)) { + t.Errorf("%s: message %q does not name the status", name, err) + } assertScrubbed(t, err) } if n := hits.Load(); n != 0 { @@ -121,12 +124,12 @@ func TestOversizedResponses(t *testing.T) { func TestErrorBodiesAreScrubbed(t *testing.T) { c := credential() - nasty := "\r\n\x1b[31m‮" + nasty := "\r\n\x1b[31m\u202e" j, _ := json.Marshal(c) bodies := map[string][]byte{ "raw": []byte("echo " + c + nasty), "query": []byte("echo " + url.QueryEscape(c) + nasty), - "json": []byte(`{"error":"bad","error_description":"echo ` + string(j[1:len(j)-1]) + `\r\n\u001b[31m‮"}`), + "json": []byte(`{"error":"bad","error_description":"echo ` + string(j[1:len(j)-1]) + `\r\n\u001b[31m\u202e"}`), "marshal": mustJSON(t, map[string]string{"error": c + nasty, "error_description": url.QueryEscape(c) + nasty}), } for name, body := range bodies { @@ -153,7 +156,7 @@ func mustJSON(t *testing.T, v any) []byte { func TestEveryMethodScrubsItsErrors(t *testing.T) { y, _ := tlsDriver(t, http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusInternalServerError) - _, _ = w.Write(mustJSON(t, map[string]string{"error": "boom", "error_description": "token " + credential() + " ‮"})) + _, _ = w.Write(mustJSON(t, map[string]string{"error": "boom", "error_description": "token " + credential() + " \u202e"})) })) ctx := context.Background() calls := map[string]func() error{ diff --git a/api/internal/issuetracker/youtrack_test.go b/api/internal/issuetracker/youtrack_test.go index e74776e5b3..e028147ac7 100644 --- a/api/internal/issuetracker/youtrack_test.go +++ b/api/internal/issuetracker/youtrack_test.go @@ -218,7 +218,7 @@ func TestBaseURLTrailingSlashIsNormalised(t *testing.T) { } } -// TestPinnedLimits pins the literal byte limits. +// TestPinnedLimits pins the literal limits. func TestPinnedLimits(t *testing.T) { for _, tc := range []struct { name string @@ -227,6 +227,16 @@ func TestPinnedLimits(t *testing.T) { {"listPageBytes", listPageBytes, 8 << 20}, {"objectBytes", objectBytes, 2 << 20}, {"errorBodyBytes", errorBodyBytes, 4 << 10}, + {"pageSize", pageSize, 100}, + {"maxItems", int64(maxItems), 100000}, + {"maxPages", int64(maxPages), 2000}, + {"maxTotalBytes", int64(maxTotalBytes), 64 << 20}, + {"maxNestedItems", maxNestedItems, 1000}, + {"maxLabelNameLen", maxLabelNameLen, 200}, + {"minTokenLen", minTokenLen, 16}, + {"maxTokenLen", maxTokenLen, 4096}, + {"maxMessageBytes", maxMessageBytes, 512}, + {"minSegmentLen", minSegmentLen, 8}, } { if tc.got != tc.want { t.Errorf("%s = %d, want %d", tc.name, tc.got, tc.want) diff --git a/api/internal/issuetracker/youtrack_wire.go b/api/internal/issuetracker/youtrack_wire.go index 0b7fccc083..ea813e10a4 100644 --- a/api/internal/issuetracker/youtrack_wire.go +++ b/api/internal/issuetracker/youtrack_wire.go @@ -47,7 +47,7 @@ type ytIssue struct { Updated int64 `json:"updated"` Resolved *int64 `json:"resolved"` Reporter *ytUser `json:"reporter"` - Tags []ytTag `json:"tags"` + Tags ytTags `json:"tags"` Project *ytProject `json:"project"` } @@ -67,20 +67,17 @@ type ytActivity struct { ID string `json:"id"` Timestamp int64 `json:"timestamp"` Author *ytUser `json:"author"` - Added []ytTag `json:"added"` - Removed []ytTag `json:"removed"` + Added ytTags `json:"added"` + Removed ytTags `json:"removed"` } // checkIssue refuses an issue whose entity id, id-readable or tag ids are not -// well formed, or whose tag count exceeds maxNestedItems: those ids end up in -// request paths and in WebURL. +// well formed: those ids end up in request paths and in WebURL. The tag count is +// capped while decoding (ytTags). func (y *youTrack) checkIssue(w ytIssue) error { if !entityIDRE.MatchString(w.ID) || !idReadableRE.MatchString(w.IDReadable) { return y.fail("youtrack: issue with a malformed id in the response") } - if len(w.Tags) > maxNestedItems { - return y.fail("youtrack: issue carries more than %d tags", maxNestedItems) - } for _, t := range w.Tags { if !entityIDRE.MatchString(t.ID) { return y.fail("youtrack: tag with a malformed id in the response") diff --git a/api/internal/issuetracker/youtracktest/server_test.go b/api/internal/issuetracker/youtracktest/server_test.go index 209cd79359..cd5492aa0d 100644 --- a/api/internal/issuetracker/youtracktest/server_test.go +++ b/api/internal/issuetracker/youtracktest/server_test.go @@ -60,3 +60,52 @@ func TestModelledRoutesAreQuietAtCleanup(t *testing.T) { t.Fatalf("cleanup reported %v", rec.errors) } } + +func TestKnownPathWithAnUnmodelledMethodIsLoud(t *testing.T) { + rec := &recorder{TB: t} + srv := NewServer(rec) + req, err := http.NewRequest(http.MethodDelete, srv.URL+"/api/tags", nil) + if err != nil { + t.Fatal(err) + } + req.Header.Set("Authorization", "Bearer "+srv.Token) + resp, err := srv.Client().Do(req) + if err != nil { + t.Fatal(err) + } + _ = resp.Body.Close() + if resp.StatusCode != http.StatusNotImplemented { + t.Errorf("status = %d, want 501 rather than a silent 405", resp.StatusCode) + } + for i := len(rec.cleanups) - 1; i >= 0; i-- { + rec.cleanups[i]() + } + if len(rec.errors) != 1 { + t.Fatalf("cleanup reported %d errors, want 1: %v", len(rec.errors), rec.errors) + } +} + +func TestWrongBearerTokenIsRefused(t *testing.T) { + srv := NewServer(t) + for name, auth := range map[string]string{ + "wrong token": "Bearer " + srv.Token + "x", + "no scheme": srv.Token, + "absent": "", + } { + req, err := http.NewRequest(http.MethodGet, srv.URL+"/api/users/me?fields=id", nil) + if err != nil { + t.Fatal(err) + } + if auth != "" { + req.Header.Set("Authorization", auth) + } + resp, err := srv.Client().Do(req) + if err != nil { + t.Fatal(err) + } + _ = resp.Body.Close() + if resp.StatusCode != http.StatusUnauthorized { + t.Errorf("%s: status = %d, want 401", name, resp.StatusCode) + } + } +} diff --git a/api/internal/issuetrackersvc/baseurl.go b/api/internal/issuetrackersvc/baseurl.go index 547358fc25..e088a215dd 100644 --- a/api/internal/issuetrackersvc/baseurl.go +++ b/api/internal/issuetrackersvc/baseurl.go @@ -4,6 +4,7 @@ import ( "errors" "net/url" "strings" + "unicode" "github.com/vtmocanu/uzi/api/internal/config" ) @@ -41,8 +42,16 @@ func strictBaseURL(raw string) (string, error) { if err != nil || again != norm { return "", errors.New("invalid base URL: it must be an https URL with a host") } - if nu, err := url.Parse(norm); err != nil || nu.Hostname() == "" { + nu, err := url.Parse(norm) + if err != nil || nu.Hostname() == "" { return "", errors.New("invalid base URL: it must be an https URL with a host") } + // A control or format character in the host (a zero-width or bidi + // character) would look like another host once it is shown. + for _, r := range nu.Host { + if unicode.IsControl(r) || unicode.Is(unicode.Cf, r) || r == '\u2028' || r == '\u2029' { + return "", errors.New("invalid base URL: the host holds a control or invisible character") + } + } return norm, nil } diff --git a/api/internal/issuetrackersvc/connect.go b/api/internal/issuetrackersvc/connect.go index ad4be85dff..96e12797e5 100644 --- a/api/internal/issuetrackersvc/connect.go +++ b/api/internal/issuetrackersvc/connect.go @@ -2,9 +2,7 @@ package issuetrackersvc import ( "context" - "errors" "strings" - "unicode" "github.com/google/uuid" @@ -14,8 +12,8 @@ import ( // Connect verifies a token against the issue tracker and stores it sealed for // the user, creating the connection or rotating the existing one for the same -// (user, kind, base URL). The order is fixed: kind, token presence, strict base -// URL, allowlist, driver build, token verification, account check, seal, +// (user, kind, base URL). The order is fixed: kind, token presence, token shape +// (issuetracker.ValidateToken), strict base URL, allowlist, driver build, token verification, account check, seal, // upsert. A refused base URL never reaches the driver factory, so no outbound // request or token use happens for it. The token is trimmed of surrounding // whitespace, and it appears in no returned error or log line. @@ -28,8 +26,9 @@ func (s *Service) Connect(ctx context.Context, userID uuid.UUID, kind, baseURL, if token == "" { return Connection{}, &Error{Kind: KindBadRequest, Message: "a token is required"} } - if err := validateTokenShape(token); err != nil { - return Connection{}, &Error{Kind: KindBadRequest, Message: err.Error()} + if err := issuetracker.ValidateToken(token); err != nil { + // The driver's message names the rule, never the token; the client gets a fixed one. + return Connection{}, &Error{Kind: KindBadRequest, Message: "invalid token"} } norm, err := strictBaseURL(baseURL) if err != nil { @@ -73,20 +72,3 @@ func (s *Service) Connect(ctx context.Context, userID uuid.UUID, kind, baseURL, } return toConnection(row), nil } - -// maxTokenLen bounds a token in bytes, before it is sealed or handed to a driver. -const maxTokenLen = 4096 - -// validateTokenShape refuses a token that is too long or holds a control or -// format (Cf) character. The message never includes the token. -func validateTokenShape(token string) error { - if len(token) > maxTokenLen { - return errors.New("token is too long") - } - for _, r := range token { - if unicode.IsControl(r) || unicode.Is(unicode.Cf, r) { - return errors.New("token holds a control or invisible character") - } - } - return nil -} diff --git a/api/internal/issuetrackersvc/driver.go b/api/internal/issuetrackersvc/driver.go index 9d610afa22..df6ec6d8fd 100644 --- a/api/internal/issuetrackersvc/driver.go +++ b/api/internal/issuetrackersvc/driver.go @@ -2,30 +2,19 @@ package issuetrackersvc import ( "context" - "regexp" "github.com/google/uuid" "github.com/vtmocanu/uzi/api/internal/issuetracker" ) -var ( - entityIDRE = regexp.MustCompile(`^\d+-\d+$`) - shortNameRE = regexp.MustCompile(`^[A-Za-z][A-Za-z0-9_]*$`) -) - -// validProjectRef reports whether p is empty or has a well-formed id and short name. -func validProjectRef(p issuetracker.ProjectRef) bool { - return p == (issuetracker.ProjectRef{}) || (entityIDRE.MatchString(p.ID) && shortNameRE.MatchString(p.ShortName)) -} - // DriverFor builds a driver for a stored connection. It re-checks the strict // base URL form and the allowlist against the stored row BEFORE opening the // token, so removing a host from the allowlist stops use of existing rows // (fail closed). The token is opened with the row's user, kind and base URL as // AAD, used only to build the driver, and never returned or logged. func (s *Service) DriverFor(ctx context.Context, userID, connID uuid.UUID, project issuetracker.ProjectRef, stateField, open, closed string) (issuetracker.Driver, error) { - if !validProjectRef(project) { + if issuetracker.ValidProjectRef(project) != nil { return nil, &Error{Kind: KindBadRequest, Message: "invalid project reference"} } row, err := s.getRow(ctx, userID, connID) @@ -79,7 +68,7 @@ func (s *Service) ListProjects(ctx context.Context, userID, connID uuid.UUID) ([ // ListStates lists the issue states of one project of the connection. func (s *Service) ListStates(ctx context.Context, userID, connID uuid.UUID, projectID string) ([]issuetracker.State, error) { - if !entityIDRE.MatchString(projectID) { + if !issuetracker.ValidProjectID(projectID) { return nil, &Error{Kind: KindBadRequest, Message: "invalid project id"} } d, err := s.DriverFor(ctx, userID, connID, issuetracker.ProjectRef{}, "", "", "") diff --git a/api/internal/issuetrackersvc/service_test.go b/api/internal/issuetrackersvc/service_test.go index 6b35233fc2..168c0778e8 100644 --- a/api/internal/issuetrackersvc/service_test.go +++ b/api/internal/issuetrackersvc/service_test.go @@ -135,7 +135,13 @@ func newHarness(t *testing.T) *harness { } // testToken assembles a token-shaped value at runtime. -func testToken(parts ...string) string { return strings.Join(append([]string{"perm"}, parts...), "-") } +// The fixed tail keeps it above the driver's minimum token length. +func testToken(parts ...string) string { + return strings.Join(append([]string{"perm"}, parts...), "-") + "-0123456789abcdef" +} + +// testMaxTokenLen is the longest token the service accepts. +const testMaxTokenLen = 4096 func wantKind(t *testing.T, err error, want Kind) *Error { t.Helper() @@ -160,23 +166,26 @@ func TestConnectRefusesWhenAllowlistEmpty(t *testing.T) { } func TestConnectRefusesBadInput(t *testing.T) { + validToken := testToken("v") cases := []struct{ name, kind, url, token string }{ - {"unknown kind", "jira", testHost, "tok"}, - {"empty kind", "", testHost, "tok"}, + {"unknown kind", "jira", testHost, validToken}, + {"empty kind", "", testHost, validToken}, {"empty token", "youtrack", testHost, " "}, - {"http", "youtrack", "http://issues.example.com", "tok"}, - {"path", "youtrack", testHost + "/youtrack", "tok"}, - {"userinfo", "youtrack", "https://issues.example.com@evil.example.com", "tok"}, - {"userinfo with password", "youtrack", "https://u:p@issues.example.com", "tok"}, - {"query", "youtrack", testHost + "?a=b", "tok"}, - {"bare question mark", "youtrack", testHost + "?", "tok"}, - {"fragment", "youtrack", testHost + "#x", "tok"}, - {"bare hash", "youtrack", testHost + "#", "tok"}, - {"escaped percent in host", "youtrack", "https://x%25y.example.com", "tok"}, - {"empty hostname with port", "youtrack", "https://:443", "tok"}, - {"no host", "youtrack", "https://", "tok"}, - {"not allowlisted", "youtrack", "https://evil.example.com", "tok"}, - {"explicit default port is a different origin string", "youtrack", testHost + ":443", "tok"}, + {"http", "youtrack", "http://issues.example.com", validToken}, + {"path", "youtrack", testHost + "/youtrack", validToken}, + {"userinfo", "youtrack", "https://issues.example.com@evil.example.com", validToken}, + {"userinfo with password", "youtrack", "https://u:p@issues.example.com", validToken}, + {"query", "youtrack", testHost + "?a=b", validToken}, + {"bare question mark", "youtrack", testHost + "?", validToken}, + {"fragment", "youtrack", testHost + "#x", validToken}, + {"bare hash", "youtrack", testHost + "#", validToken}, + {"escaped percent in host", "youtrack", "https://x%25y.example.com", validToken}, + {"empty hostname with port", "youtrack", "https://:443", validToken}, + {"zero-width character in host", "youtrack", "https://issues.exam\u200bple.com", validToken}, + {"bidi override in host", "youtrack", "https://issues.example.com\u202e", validToken}, + {"no host", "youtrack", "https://", validToken}, + {"not allowlisted", "youtrack", "https://evil.example.com", validToken}, + {"explicit default port is a different origin string", "youtrack", testHost + ":443", validToken}, } for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { @@ -192,25 +201,30 @@ func TestConnectRefusesBadInput(t *testing.T) { func TestConnectRefusesBadToken(t *testing.T) { cases := map[string]string{ - "8 MiB": strings.Repeat("a", 8<<20), - "just too long": strings.Repeat("a", maxTokenLen+1), - "embedded CRLF": strings.Join([]string{"perm", "abc\r\nX-Evil: 1", "def"}, "-"), - "zero width": strings.Join([]string{"perm", "abc\u200bdef", "ghi"}, "-"), - "bidi override": strings.Join([]string{"perm", "abc\u202edef", "ghi"}, "-"), - "embedded NUL": strings.Join([]string{"perm", "abc\x00def", "ghi"}, "-"), + "8 MiB": strings.Repeat("a", 8<<20), + "just too long": strings.Repeat("a", testMaxTokenLen+1), + "embedded CRLF": strings.Join([]string{"perm", "abc\r\nX-Evil: 1", "def"}, "-"), + "zero width": strings.Join([]string{"perm", "abc\u200bdef", "ghi"}, "-"), + "bidi override": strings.Join([]string{"perm", "abc\u202edef", "ghi"}, "-"), + "embedded NUL": strings.Join([]string{"perm", "abc\x00def", "ghi"}, "-"), + "line separator": strings.Join([]string{"perm", "abc\u2028def", "ghi"}, "-"), + "too short": "tok", } for name, tok := range cases { t.Run(name, func(t *testing.T) { h := newHarness(t) _, err := h.svc.Connect(context.Background(), uuid.New(), "youtrack", testHost, tok) - wantKind(t, err, KindBadRequest) + e := wantKind(t, err, KindBadRequest) + if e.Message != "invalid token" { + t.Errorf("message = %q, want the fixed invalid token message", e.Message) + } if len(h.built) != 0 || len(h.q.rows) != 0 { t.Error("driver built or row stored for a refused token") } }) } h := newHarness(t) - if _, err := h.svc.Connect(context.Background(), uuid.New(), "youtrack", testHost, strings.Repeat("a", maxTokenLen)); err != nil { + if _, err := h.svc.Connect(context.Background(), uuid.New(), "youtrack", testHost, strings.Repeat("a", testMaxTokenLen)); err != nil { t.Errorf("a token of exactly the limit must pass: %v", err) } } @@ -275,10 +289,10 @@ func TestConnectRefusesBadAccount(t *testing.T) { "empty login": {ID: "1", Login: ""}, "control in id": {ID: "1\x00", Login: "bot"}, "newline login": {ID: "1", Login: "bo\nt"}, - "bidi override": {ID: "1", Login: "bot‮"}, - "zero width": {ID: "1", Login: "bo​t"}, - "line separator": {ID: "1", Login: "bo
t"}, - "para separator": {ID: "1", Login: "bo
t"}, + "bidi override": {ID: "1", Login: "bot\u202e"}, + "zero width": {ID: "1", Login: "bo\u200bt"}, + "line separator": {ID: "1", Login: "bo\u2028t"}, + "para separator": {ID: "1", Login: "bo\u2029t"}, "login too long": {ID: "1", Login: strings.Repeat("a", 256)}, "id too long": {ID: strings.Repeat("1", 256), Login: "bot"}, "control in both": {ID: "\x07", Login: "\x07"}, @@ -550,3 +564,24 @@ func TestNewDefaultsToRegistry(t *testing.T) { t.Fatal("a nil allowlist must refuse") } } + +// TestConnectRefusesAnInvisibleHostCharacterBeforeTheAllowlist uses an +// allowlist that accepts anything, so only the base URL check can refuse. +func TestConnectRefusesAnInvisibleHostCharacterBeforeTheAllowlist(t *testing.T) { + for name, u := range map[string]string{ + "zero width": "https://issues.exam​ple.com", + "bidi override": "https://issues.example.com‮", + "line separator": "https://issues.exam
ple.com", + } { + t.Run(name, func(t *testing.T) { + h := newHarness(t) + asked := 0 + h.svc.allowed = func(string) bool { asked++; return true } + _, err := h.svc.Connect(context.Background(), uuid.New(), "youtrack", u, testToken("v")) + wantKind(t, err, KindBadRequest) + if asked != 0 || len(h.built) != 0 { + t.Errorf("allowlist asked %d times, driver built %d times", asked, len(h.built)) + } + }) + } +} From 9b54c1544bc5fa5c082ffb98180ea847b730ec0b Mon Sep 17 00:00:00 2001 From: uzi-agent Date: Wed, 11 Nov 2026 10:16:38 +0000 Subject: [PATCH 09/17] issuetracker: count before unmarshal so repeated keys cannot churn Nested tag arrays and list pages are counted with a byte scan and then unmarshalled once into the existing slice, so a response that repeats the tags key allocates nothing extra. One exported invisible-character predicate now guards tokens, labels, messages and hosts in both the driver and the service, and a refused token says which rule it broke. Co-Authored-By: Claude Opus 5.5 --- api/internal/issuetracker/redact.go | 2 +- api/internal/issuetracker/validate.go | 6 +- api/internal/issuetracker/youtrack.go | 8 ++ api/internal/issuetracker/youtrack_decode.go | 131 ++++++++++++------ .../issuetracker/youtrack_decode_test.go | 69 +++++++++ .../issuetracker/youtrack_guards_test.go | 2 +- api/internal/issuetracker/youtrack_query.go | 2 +- api/internal/issuetracker/youtrack_test.go | 4 + api/internal/issuetrackersvc/baseurl.go | 9 +- api/internal/issuetrackersvc/connect.go | 9 +- api/internal/issuetrackersvc/service_test.go | 21 ++- 11 files changed, 201 insertions(+), 62 deletions(-) diff --git a/api/internal/issuetracker/redact.go b/api/internal/issuetracker/redact.go index 02ba781fab..e572bddd2d 100644 --- a/api/internal/issuetracker/redact.go +++ b/api/internal/issuetracker/redact.go @@ -138,7 +138,7 @@ func (r redactor) string(s string) string { // with a space. It uses the predicate of issueinput.AuthorName. func sanitize(s string) string { return strings.Map(func(c rune) rune { - if invisible(c) { + if InvisibleRune(c) { return ' ' } return c diff --git a/api/internal/issuetracker/validate.go b/api/internal/issuetracker/validate.go index 2f2d70c987..361d71a4b4 100644 --- a/api/internal/issuetracker/validate.go +++ b/api/internal/issuetracker/validate.go @@ -17,17 +17,17 @@ func ValidateToken(token string) error { return errors.New("issuetracker: token longer than the maximum length") } for _, c := range token { - if invisible(c) { + if InvisibleRune(c) { return errors.New("issuetracker: token contains a control or format character") } } return nil } -// invisible reports whether c is a control character, a Unicode format +// InvisibleRune reports whether c is a control character, a Unicode format // character (bidi controls, zero-width characters, BOM) or the line or // paragraph separator. -func invisible(c rune) bool { +func InvisibleRune(c rune) bool { return unicode.IsControl(c) || unicode.Is(unicode.Cf, c) || c == '\u2028' || c == '\u2029' } diff --git a/api/internal/issuetracker/youtrack.go b/api/internal/issuetracker/youtrack.go index 77a951795e..aafa89de7e 100644 --- a/api/internal/issuetracker/youtrack.go +++ b/api/internal/issuetracker/youtrack.go @@ -6,6 +6,7 @@ import ( "net/url" "regexp" "strings" + "unicode" ) // defaultStateField is the YouTrack custom field that holds the issue state. @@ -94,6 +95,13 @@ func strictBaseURL(raw string) (string, error) { case u.Path != "" && u.Path != "/": return bad("must not carry a path") } + // An invisible or non-ASCII host character would look like another host + // once shown; an internationalised name arrives as "xn--" punycode. + for _, c := range u.Host { + if InvisibleRune(c) || c > unicode.MaxASCII { + return bad("host must be plain ASCII with no control or invisible character") + } + } return "https://" + u.Host, nil } diff --git a/api/internal/issuetracker/youtrack_decode.go b/api/internal/issuetracker/youtrack_decode.go index 8ad4410636..f73711b009 100644 --- a/api/internal/issuetracker/youtrack_decode.go +++ b/api/internal/issuetracker/youtrack_decode.go @@ -20,63 +20,108 @@ var errNotArray = errors.New("issuetracker: response is not a JSON array") // errTrailingData is returned when a list response has data after its array. var errTrailingData = errors.New("issuetracker: response has data after the JSON array") -// ytTags is a nested tag array. Its UnmarshalJSON streams the elements and -// refuses once more than maxNestedItems have been read, so one object cannot -// make the decoder allocate a slice as large as the response. +// ytTags is a nested tag array. Its UnmarshalJSON counts the elements with a +// byte scan and refuses above maxNestedItems before anything is decoded, so one +// object cannot make the decoder allocate a slice as large as the response. It +// builds no json.Decoder: encoding/json calls it once per repeated "tags" key, +// and a per-call Decoder would multiply across a response of repeated keys. type ytTags []ytTag -// UnmarshalJSON implements json.Unmarshaler. +// UnmarshalJSON implements json.Unmarshaler. b is already-validated JSON. func (t *ytTags) UnmarshalJSON(b []byte) error { - items, err := decodeArray[ytTag](b, maxNestedItems, errNestedCeiling) - if err != nil { - return err + if n, _ := topLevelCount(b); n > maxNestedItems { + return errNestedCeiling } - *t = items - return nil + return json.Unmarshal(b, (*[]ytTag)(t)) } -// decodeArray decodes a JSON array element by element and returns refuse as -// soon as it meets element number limit+1, before that element is decoded, so at -// most limit elements are ever allocated. A JSON null is an empty array. -func decodeArray[T any](b []byte, limit int, refuse error) ([]T, error) { - dec := json.NewDecoder(bytes.NewReader(b)) - tok, err := dec.Token() - if err != nil { - return nil, err - } - if tok == nil { - return nil, endOfInput(dec) +// topLevelCount scans b, a JSON value, and returns the number of elements of +// the array it starts with (0 when it does not start with an array) and the +// offset just past that array (0 when it does not start with one, or never +// closes). It tracks bracket depth and string state only, so it neither +// validates nor allocates; a caller that needs validity gets it from +// json.Unmarshal afterwards. +func topLevelCount(b []byte) (count, end int) { + i := 0 + for i < len(b) && isJSONSpace(b[i]) { + i++ } - if d, ok := tok.(json.Delim); !ok || d != '[' { - return nil, errNotArray + if i >= len(b) || b[i] != '[' { + return 0, 0 } - var items []T - for dec.More() { - if len(items) >= limit { - return nil, refuse + depth, seen, inStr, esc := 0, false, false, false + for ; i < len(b); i++ { + c := b[i] + if inStr { + switch { + case esc: + esc = false + case c == '\\': + esc = true + case c == '"': + inStr = false + } + continue } - var it T - if err := dec.Decode(&it); err != nil { - return nil, err + switch c { + case '[', '{': + if depth == 1 { + seen = true + } + depth++ + case ']', '}': + depth-- + if depth == 0 { + if seen { + count++ + } + return count, i + 1 + } + case ',': + if depth == 1 { + count++ + } + case '"': + if depth == 1 { + seen = true + } + inStr = true + default: + if depth == 1 && !isJSONSpace(c) { + seen = true + } } - items = append(items, it) } - if _, err := dec.Token(); err != nil { - return nil, err - } - return items, endOfInput(dec) + return count, 0 } -// endOfInput reports errTrailingData unless dec has nothing left. -func endOfInput(dec *json.Decoder) error { - if _, err := dec.Token(); err != io.EOF { - return errTrailingData - } - return nil -} +func isJSONSpace(c byte) bool { return c == ' ' || c == '\t' || c == '\n' || c == '\r' } // decodePage decodes one list page. The driver asked for pageSize items, so a -// longer array is a protocol violation and is refused while decoding. +// longer array is a protocol violation: the elements are counted before any is +// decoded, so an oversized page allocates nothing for its items. A JSON null is +// an empty page. func decodePage[T any](body []byte) ([]T, error) { - return decodeArray[T](body, pageSize, errItemCeiling) + trimmed := bytes.TrimSpace(body) + if string(trimmed) == "null" { + return nil, nil + } + n, end := topLevelCount(body) + if end == 0 { + if len(trimmed) > 0 && trimmed[0] == '[' { + return nil, io.ErrUnexpectedEOF + } + return nil, errNotArray + } + if n > pageSize { + return nil, errItemCeiling + } + if len(bytes.TrimSpace(body[end:])) > 0 { + return nil, errTrailingData + } + var items []T + if err := json.Unmarshal(body, &items); err != nil { + return nil, err + } + return items, nil } diff --git a/api/internal/issuetracker/youtrack_decode_test.go b/api/internal/issuetracker/youtrack_decode_test.go index 6d09c8627c..29e05e1f69 100644 --- a/api/internal/issuetracker/youtrack_decode_test.go +++ b/api/internal/issuetracker/youtrack_decode_test.go @@ -10,6 +10,10 @@ import ( "github.com/vtmocanu/uzi/api/internal/forge" ) +// The allocation tests below read the process-wide runtime.MemStats +// TotalAlloc, so they must not run in parallel with each other or with any +// other test: do not add t.Parallel to them. + // allocBound is the most a decode test lets the process allocate while it // serves and decodes one oversized response. The response itself is read into a // buffer of at most 8 MiB (grown by append, so several times that is allocated @@ -90,6 +94,52 @@ func TestTagBombAllocatesLittle(t *testing.T) { } } +// TestDuplicateTagsKeysAllocateLittle pins that encoding/json calling +// ytTags.UnmarshalJSON once per repeated "tags" key costs no per-call Decoder: +// the unfixed decoder allocated about 380 MiB for the page fixture. The fixed +// decoder allocates about 30 MiB without -race and about 100 MiB with it (the +// race build makes sync.Pool drop entries, so encoding/json's own pooled state +// is reallocated), so the bound sits between the two. +func TestDuplicateTagsKeysAllocateLittle(t *testing.T) { + const bound = 192 << 20 + t.Run("page", func(t *testing.T) { + issue := strings.TrimSuffix(rawIssue("2-1", "UZI-1", 1, "0-1", ""), `"tags":[]}`) + dup := `"tags":[{}],` + body := "[" + issue + strings.Repeat(dup, int(listPageBytes-int64(len(issue))-64)/len(dup)) + `"tags":[]}]` + if int64(len(body)) > listPageBytes { + t.Fatalf("fixture is %d bytes, over the page cap", len(body)) + } + y, _ := loggedDriver(t, func(w http.ResponseWriter, r *http.Request) { + if r.URL.Query().Get("$skip") != "0" { + writeRaw(w, "[]") + return + } + _, _ = w.Write([]byte(body)) + }) + var err error + got := totalAlloc(func() { _, err = y.ListIssues(context.Background(), 1, forge.ListIssuesOptions{}) }) + if err != nil { + t.Logf("ListIssues: %v", err) + } + if got > bound { + t.Errorf("allocated %d MiB for a %d MiB page, want under %d MiB", got>>20, len(body)>>20, bound>>20) + } + }) + t.Run("issue", func(t *testing.T) { + issue := strings.TrimSuffix(rawIssue("2-1", "UZI-1", 1, "0-1", ""), `"tags":[]}`) + dup := `"tags":[{}],` + body := []byte(issue + strings.Repeat(dup, int(objectBytes-int64(len(issue))-64)/len(dup)) + `"tags":[]}`) + if int64(len(body)) > objectBytes { + t.Fatalf("fixture is %d bytes, over the object cap", len(body)) + } + y, _ := loggedDriver(t, func(w http.ResponseWriter, _ *http.Request) { _, _ = w.Write(body) }) + got := totalAlloc(func() { _, _ = y.GetIssue(context.Background(), 1, 1) }) + if got > bound { + t.Errorf("allocated %d MiB for a %d MiB issue, want under %d MiB", got>>20, len(body)>>20, bound>>20) + } + }) +} + func TestListDecodeShapes(t *testing.T) { for name, tc := range map[string]struct { body string @@ -165,3 +215,22 @@ func TestTopLevelItemsAndNestedTagsShareOneBudget(t *testing.T) { t.Errorf("4 items + 4 tags = 8 refused under a ceiling of 10: %v", err) } } + +func TestTopLevelCountIgnoresBracketsInStrings(t *testing.T) { + for name, tc := range map[string]struct { + in string + count int + }{ + "empty": {`[]`, 0}, + "spaced empty": {` [ ] `, 0}, + "scalars": {`[1, 2,3]`, 3}, + "nested": {`[[1,2],{"a":[3,4]},5]`, 3}, + "string with ,]": {`["a,]","b\"],"]`, 2}, + "escaped backslash": {`["\\",1]`, 2}, + "not an array": {`{"a":[1,2]}`, 0}, + } { + if got, _ := topLevelCount([]byte(tc.in)); got != tc.count { + t.Errorf("%s: count %d, want %d", name, got, tc.count) + } + } +} diff --git a/api/internal/issuetracker/youtrack_guards_test.go b/api/internal/issuetracker/youtrack_guards_test.go index ae254a02a9..1aaaec27bb 100644 --- a/api/internal/issuetracker/youtrack_guards_test.go +++ b/api/internal/issuetracker/youtrack_guards_test.go @@ -52,7 +52,7 @@ func TestListIssuesRefusesMalformedIDs(t *testing.T) { } } -func TestListIssuesLimitStopsAcrossPages(t *testing.T) { +func TestListIssuesLimitWithinOnePageStopsAfterOneRequest(t *testing.T) { y, srv := newFakeDriver(t) for range 250 { srv.SeedIssue("t", "") diff --git a/api/internal/issuetracker/youtrack_query.go b/api/internal/issuetracker/youtrack_query.go index 15742e1c24..6c685f0cff 100644 --- a/api/internal/issuetracker/youtrack_query.go +++ b/api/internal/issuetracker/youtrack_query.go @@ -16,7 +16,7 @@ const maxLabelNameLen = 200 func (y *youTrack) checkLabelName(name string) error { bad := strings.TrimSpace(name) == "" || len(name) > maxLabelNameLen for _, c := range name { - if c == '{' || c == '}' || invisible(c) { + if c == '{' || c == '}' || InvisibleRune(c) { bad = true } } diff --git a/api/internal/issuetracker/youtrack_test.go b/api/internal/issuetracker/youtrack_test.go index e028147ac7..1fdb354bfb 100644 --- a/api/internal/issuetracker/youtrack_test.go +++ b/api/internal/issuetracker/youtrack_test.go @@ -78,6 +78,10 @@ func TestConstructorValidation(t *testing.T) { "query": mutate(func(o *Options) { o.BaseURL = "https://yt.example.com?x=1" }), "fragment": mutate(func(o *Options) { o.BaseURL = "https://yt.example.com#x" }), "no host": mutate(func(o *Options) { o.BaseURL = "https://" }), + "zero width": mutate(func(o *Options) { o.BaseURL = "https://yt.exam\u200bple.com" }), + "bidi": mutate(func(o *Options) { o.BaseURL = "https://yt.example.com\u202e" }), + "hangul fill": mutate(func(o *Options) { o.BaseURL = "https://yt.exam\u3164ple.com" }), + "unicode idn": mutate(func(o *Options) { o.BaseURL = "https://yt.exampl\u00e9.com" }), "empty": mutate(func(o *Options) { o.BaseURL = "" }), "short token": mutate(func(o *Options) { o.Token = "short" }), "bad short": mutate(func(o *Options) { o.Project.ShortName = "1X" }), diff --git a/api/internal/issuetrackersvc/baseurl.go b/api/internal/issuetrackersvc/baseurl.go index e088a215dd..4ba1b11ca9 100644 --- a/api/internal/issuetrackersvc/baseurl.go +++ b/api/internal/issuetrackersvc/baseurl.go @@ -7,6 +7,7 @@ import ( "unicode" "github.com/vtmocanu/uzi/api/internal/config" + "github.com/vtmocanu/uzi/api/internal/issuetracker" ) // strictBaseURL validates raw as an https scheme-and-host base URL and returns @@ -46,12 +47,16 @@ func strictBaseURL(raw string) (string, error) { if err != nil || nu.Hostname() == "" { return "", errors.New("invalid base URL: it must be an https URL with a host") } - // A control or format character in the host (a zero-width or bidi + // A control, format or non-ASCII character in the host (a zero-width or bidi // character) would look like another host once it is shown. for _, r := range nu.Host { - if unicode.IsControl(r) || unicode.Is(unicode.Cf, r) || r == '\u2028' || r == '\u2029' { + if issuetracker.InvisibleRune(r) { return "", errors.New("invalid base URL: the host holds a control or invisible character") } + // An internationalised name arrives as "xn--" punycode. + if r > unicode.MaxASCII { + return "", errors.New("invalid base URL: the host must be ASCII; use the xn-- form of an internationalised name") + } } return norm, nil } diff --git a/api/internal/issuetrackersvc/connect.go b/api/internal/issuetrackersvc/connect.go index 96e12797e5..30ca821ecc 100644 --- a/api/internal/issuetrackersvc/connect.go +++ b/api/internal/issuetrackersvc/connect.go @@ -13,8 +13,8 @@ import ( // Connect verifies a token against the issue tracker and stores it sealed for // the user, creating the connection or rotating the existing one for the same // (user, kind, base URL). The order is fixed: kind, token presence, token shape -// (issuetracker.ValidateToken), strict base URL, allowlist, driver build, token verification, account check, seal, -// upsert. A refused base URL never reaches the driver factory, so no outbound +// (issuetracker.ValidateToken), strict base URL, allowlist, driver build, token +// verification, account check, seal, upsert. A refused base URL never reaches the driver factory, so no outbound // request or token use happens for it. The token is trimmed of surrounding // whitespace, and it appears in no returned error or log line. func (s *Service) Connect(ctx context.Context, userID uuid.UUID, kind, baseURL, token string) (Connection, error) { @@ -27,8 +27,9 @@ func (s *Service) Connect(ctx context.Context, userID uuid.UUID, kind, baseURL, return Connection{}, &Error{Kind: KindBadRequest, Message: "a token is required"} } if err := issuetracker.ValidateToken(token); err != nil { - // The driver's message names the rule, never the token; the client gets a fixed one. - return Connection{}, &Error{Kind: KindBadRequest, Message: "invalid token"} + // ValidateToken's message names the rule that failed and never + // includes the token, so it is passed through. + return Connection{}, &Error{Kind: KindBadRequest, Message: "invalid token: " + err.Error()} } norm, err := strictBaseURL(baseURL) if err != nil { diff --git a/api/internal/issuetrackersvc/service_test.go b/api/internal/issuetrackersvc/service_test.go index 168c0778e8..68e4e65c21 100644 --- a/api/internal/issuetrackersvc/service_test.go +++ b/api/internal/issuetrackersvc/service_test.go @@ -181,8 +181,6 @@ func TestConnectRefusesBadInput(t *testing.T) { {"bare hash", "youtrack", testHost + "#", validToken}, {"escaped percent in host", "youtrack", "https://x%25y.example.com", validToken}, {"empty hostname with port", "youtrack", "https://:443", validToken}, - {"zero-width character in host", "youtrack", "https://issues.exam\u200bple.com", validToken}, - {"bidi override in host", "youtrack", "https://issues.example.com\u202e", validToken}, {"no host", "youtrack", "https://", validToken}, {"not allowlisted", "youtrack", "https://evil.example.com", validToken}, {"explicit default port is a different origin string", "youtrack", testHost + ":443", validToken}, @@ -215,8 +213,15 @@ func TestConnectRefusesBadToken(t *testing.T) { h := newHarness(t) _, err := h.svc.Connect(context.Background(), uuid.New(), "youtrack", testHost, tok) e := wantKind(t, err, KindBadRequest) - if e.Message != "invalid token" { - t.Errorf("message = %q, want the fixed invalid token message", e.Message) + if !strings.HasPrefix(e.Message, "invalid token: ") { + t.Errorf("message = %q, want the validator's reason after \"invalid token: \"", e.Message) + } + // The reason must never echo the token: no 8-byte window of it appears. + for i := 0; i+8 <= len(tok) && i < 64; i++ { + if strings.Contains(e.Message, tok[i:i+8]) { + t.Errorf("message %q contains the token window %q", e.Message, tok[i:i+8]) + break + } } if len(h.built) != 0 || len(h.q.rows) != 0 { t.Error("driver built or row stored for a refused token") @@ -569,9 +574,11 @@ func TestNewDefaultsToRegistry(t *testing.T) { // allowlist that accepts anything, so only the base URL check can refuse. func TestConnectRefusesAnInvisibleHostCharacterBeforeTheAllowlist(t *testing.T) { for name, u := range map[string]string{ - "zero width": "https://issues.exam​ple.com", - "bidi override": "https://issues.example.com‮", - "line separator": "https://issues.exam
ple.com", + "zero width": "https://issues.exam\u200bple.com", + "bidi override": "https://issues.example.com\u202e", + "line separator": "https://issues.exam\u2028ple.com", + "hangul filler": "https://issues.exam\u3164ple.com", + "unicode idn": "https://issues.exampl\u00e9.com", } { t.Run(name, func(t *testing.T) { h := newHarness(t) From 89e7002fce450d3220d24ee8139411cf5815b878 Mon Sep 17 00:00:00 2001 From: uzi-agent Date: Wed, 11 Nov 2026 10:28:04 +0000 Subject: [PATCH 10/17] issuetracker: bound every JSON value before unmarshal A byte scan counts values at every depth and refuses a page or object over its structural budget before json.Unmarshal runs, and tag elements must be objects, so repeated wrong-typed values cannot amplify into per-element error allocations. Page whitespace follows JSON's own rules and each decode sentinel is pinned. Co-Authored-By: Claude Opus 5.5 --- api/internal/issuetracker/youtrack_decode.go | 145 +++++++++++++++++- .../issuetracker/youtrack_decode_test.go | 123 +++++++++++++-- api/internal/issuetracker/youtrack_request.go | 10 +- api/internal/issuetrackersvc/account.go | 3 +- 4 files changed, 257 insertions(+), 24 deletions(-) diff --git a/api/internal/issuetracker/youtrack_decode.go b/api/internal/issuetracker/youtrack_decode.go index f73711b009..89c8d8bfa9 100644 --- a/api/internal/issuetracker/youtrack_decode.go +++ b/api/internal/issuetracker/youtrack_decode.go @@ -1,7 +1,6 @@ package issuetracker import ( - "bytes" "encoding/json" "errors" "io" @@ -14,12 +13,37 @@ const maxNestedItems = 1000 // nested entries. var errNestedCeiling = errors.New("issuetracker: an item carries more nested entries than the ceiling allows") +// Structural budgets, applied by a byte scan before any response is handed to +// json.Unmarshal. A value is counted as one comma or one opening bracket +// outside a string, an upper bound on the number of JSON values. A tag with an +// id and a name costs 3 (its brace, the comma between its members and the comma +// after it), and an activity carries up to maxNestedItems tags in each of +// "added" and "removed", so one item is allowed 6*maxNestedItems+64 and a +// single object (an issue holds one tags array) 4*maxNestedItems+256. The +// budget exists because encoding/json keeps decoding after a type error and +// builds an error per bad value, so a response of many bad values would +// otherwise allocate in proportion to its bytes, not to what it legitimately +// holds. +const ( + itemValueBudget = 6*maxNestedItems + 64 + objectValueBudget = 4*maxNestedItems + 256 + pageValueBudget = pageSize * itemValueBudget +) + +// errValueCeiling is returned when a response holds more JSON values than its +// structural budget allows. +var errValueCeiling = errors.New("issuetracker: response holds more values than the budget allows") + // errNotArray is returned when a list response is not a JSON array. var errNotArray = errors.New("issuetracker: response is not a JSON array") // errTrailingData is returned when a list response has data after its array. var errTrailingData = errors.New("issuetracker: response has data after the JSON array") +// errTagNotObject is returned when a tags array holds an element that is not an +// object, refused before decoding so a bad element costs no error value. +var errTagNotObject = errors.New("issuetracker: a tag is not a JSON object") + // ytTags is a nested tag array. Its UnmarshalJSON counts the elements with a // byte scan and refuses above maxNestedItems before anything is decoded, so one // object cannot make the decoder allocate a slice as large as the response. It @@ -32,6 +56,9 @@ func (t *ytTags) UnmarshalJSON(b []byte) error { if n, _ := topLevelCount(b); n > maxNestedItems { return errNestedCeiling } + if !onlyObjects(b) { + return errTagNotObject + } return json.Unmarshal(b, (*[]ytTag)(t)) } @@ -95,16 +122,115 @@ func topLevelCount(b []byte) (count, end int) { return count, 0 } +// onlyObjects reports whether b, a JSON value, is null or an array whose every +// element is an object or null. It looks only at the first byte of each +// element, so it is meaningful on already-validated JSON; any other first byte +// (a number, string, array, true, false) is refused. +func onlyObjects(b []byte) bool { + i := 0 + for i < len(b) && isJSONSpace(b[i]) { + i++ + } + if i >= len(b) { + return true + } + if b[i] == 'n' { + return true + } + if b[i] != '[' { + return false + } + depth, inStr, esc, want := 0, false, false, false + for ; i < len(b); i++ { + c := b[i] + if inStr { + switch { + case esc: + esc = false + case c == '\\': + esc = true + case c == '"': + inStr = false + } + continue + } + if want && !isJSONSpace(c) { + if c != '{' && c != 'n' && c != ']' { + return false + } + want = false + } + switch c { + case '[', '{': + depth++ + want = depth == 1 + case ']', '}': + depth-- + if depth == 0 { + return true + } + case ',': + want = depth == 1 + case '"': + inStr = true + } + } + return true +} + +// valueCount returns the number of commas and opening brackets outside strings +// in b, at every depth: an upper bound on the number of JSON values. +func valueCount(b []byte) int { + n, inStr, esc := 0, false, false + for _, c := range b { + if inStr { + switch { + case esc: + esc = false + case c == '\\': + esc = true + case c == '"': + inStr = false + } + continue + } + switch c { + case '"': + inStr = true + case ',', '[', '{': + n++ + } + } + return n +} + +// trimJSONSpace trims JSON's own whitespace (space, tab, CR, LF) from both ends. +func trimJSONSpace(b []byte) []byte { + for len(b) > 0 && isJSONSpace(b[0]) { + b = b[1:] + } + for len(b) > 0 && isJSONSpace(b[len(b)-1]) { + b = b[:len(b)-1] + } + return b +} + func isJSONSpace(c byte) bool { return c == ' ' || c == '\t' || c == '\n' || c == '\r' } // decodePage decodes one list page. The driver asked for pageSize items, so a -// longer array is a protocol violation: the elements are counted before any is -// decoded, so an oversized page allocates nothing for its items. A JSON null is -// an empty page. +// longer array is a protocol violation: the elements and the structural values +// are counted by byte scans before any element is decoded, so an oversized page +// allocates nothing for its items. A JSON null is an empty page. Whitespace is +// JSON's own, so a form feed is a syntax error rather than padding. func decodePage[T any](body []byte) ([]T, error) { - trimmed := bytes.TrimSpace(body) - if string(trimmed) == "null" { - return nil, nil + trimmed := trimJSONSpace(body) + if len(trimmed) >= 4 && string(trimmed[:4]) == "null" { + switch { + case len(trimmed) == 4: + return nil, nil + case isJSONSpace(trimmed[4]): + return nil, errTrailingData + } } n, end := topLevelCount(body) if end == 0 { @@ -116,9 +242,12 @@ func decodePage[T any](body []byte) ([]T, error) { if n > pageSize { return nil, errItemCeiling } - if len(bytes.TrimSpace(body[end:])) > 0 { + if len(trimJSONSpace(body[end:])) > 0 { return nil, errTrailingData } + if valueCount(body) > pageValueBudget { + return nil, errValueCeiling + } var items []T if err := json.Unmarshal(body, &items); err != nil { return nil, err diff --git a/api/internal/issuetracker/youtrack_decode_test.go b/api/internal/issuetracker/youtrack_decode_test.go index 29e05e1f69..913e3439a7 100644 --- a/api/internal/issuetracker/youtrack_decode_test.go +++ b/api/internal/issuetracker/youtrack_decode_test.go @@ -2,6 +2,8 @@ package issuetracker import ( "context" + "errors" + "io" "net/http" "runtime" "strings" @@ -102,6 +104,9 @@ func TestTagBombAllocatesLittle(t *testing.T) { // is reallocated), so the bound sits between the two. func TestDuplicateTagsKeysAllocateLittle(t *testing.T) { const bound = 192 << 20 + // The fixed single-issue decode allocates about 4 MiB (25 MiB with -race); + // the unfixed one allocated about 99 MiB (120 MiB with -race). + const issueBound = 48 << 20 t.Run("page", func(t *testing.T) { issue := strings.TrimSuffix(rawIssue("2-1", "UZI-1", 1, "0-1", ""), `"tags":[]}`) dup := `"tags":[{}],` @@ -134,8 +139,8 @@ func TestDuplicateTagsKeysAllocateLittle(t *testing.T) { } y, _ := loggedDriver(t, func(w http.ResponseWriter, _ *http.Request) { _, _ = w.Write(body) }) got := totalAlloc(func() { _, _ = y.GetIssue(context.Background(), 1, 1) }) - if got > bound { - t.Errorf("allocated %d MiB for a %d MiB issue, want under %d MiB", got>>20, len(body)>>20, bound>>20) + if got > issueBound { + t.Errorf("allocated %d MiB for a %d MiB issue, want under %d MiB", got>>20, len(body)>>20, issueBound>>20) } }) } @@ -143,23 +148,119 @@ func TestDuplicateTagsKeysAllocateLittle(t *testing.T) { func TestListDecodeShapes(t *testing.T) { for name, tc := range map[string]struct { body string - wantErr bool + want error + wantLen int + }{ + "null is an empty list": {`null`, nil, 0}, + "null padded with JSON space": {" \t\r\nnull\n ", nil, 0}, + "form feed is not JSON space": {"\fnull", errNotArray, 0}, + "data after null": {`null x`, errTrailingData, 0}, + "object": {`{}`, errNotArray, 0}, + "trailing data": {`[] []`, errTrailingData, 0}, + "truncated": {`[{"id":"0-1"`, io.ErrUnexpectedEOF, 0}, + "over the item ceiling": {emptyObjects(pageSize + 1), errItemCeiling, 0}, + "exactly the item ceiling": {emptyObjects(pageSize), nil, pageSize}, + "repeated bad values": {`[{` + strings.Repeat(`"a":"",`, pageValueBudget) + `"a":""}]`, errValueCeiling, 0}, + "form feed after the array": {"[]\f", errTrailingData, 0}, + "JSON space after the array": {"[] \n", nil, 0}, + } { + t.Run(name, func(t *testing.T) { + got, err := decodePage[map[string]any]([]byte(tc.body)) + if !errors.Is(err, tc.want) || len(got) != tc.wantLen { + t.Errorf("got %d items, %v; want %d items, %v", len(got), err, tc.wantLen, tc.want) + } + }) + } +} + +// TestRepeatedBadValuesAllocateLittle pins that a response of many values that +// fail to decode is refused by the structural budget before encoding/json +// builds an error per value (the unfixed decoder allocated about 510 MiB for the +// repeated-tags page; the fixed one allocates the response buffer only). +func TestRepeatedBadValuesAllocateLittle(t *testing.T) { + issue := strings.TrimSuffix(rawIssue("2-1", "UZI-1", 1, "0-1", ""), `"tags":[]}`) + fill := func(dup string, limit int64, head string) string { + return head + strings.Repeat(dup, int(limit-int64(len(head))-64)/len(dup)) + } + badTags := `"tags":[` + strings.Repeat("1,", 999) + `1],` + for name, tc := range map[string]struct { + dup string + page bool }{ - "null is an empty list": {`null`, false}, - "object": {`{}`, true}, - "trailing data": {`[] []`, true}, - "truncated": {`[{"id":"0-1"`, true}, + "page of bad tags": {badTags, true}, + "issue of bad tags": {badTags, false}, + "page of bad created": {`"created":"",`, true}, + "issue of bad created": {`"created":"",`, false}, } { t.Run(name, func(t *testing.T) { - y, _ := loggedDriver(t, func(w http.ResponseWriter, _ *http.Request) { writeRaw(w, tc.body) }) - got, err := y.ListProjects(context.Background()) - if (err != nil) != tc.wantErr || (err == nil && len(got) != 0) { - t.Errorf("got %v, %v; wantErr %v", got, err, tc.wantErr) + var body string + if tc.page { + body = "[" + fill(tc.dup, listPageBytes-2, issue) + `"tags":[]}]` + } else { + body = fill(tc.dup, objectBytes, issue) + `"tags":[]}` + } + y, _ := loggedDriver(t, func(w http.ResponseWriter, r *http.Request) { + if r.URL.Query().Get("$skip") != "0" && tc.page { + writeRaw(w, "[]") + return + } + _, _ = w.Write([]byte(body)) + }) + var err error + got := totalAlloc(func() { + if tc.page { + _, err = y.ListIssues(context.Background(), 1, forge.ListIssuesOptions{}) + } else { + _, err = y.GetIssue(context.Background(), 1, 1) + } + }) + if err == nil { + t.Error("a response of repeated bad values was accepted") + } + bound := uint64(repeatedIssueBound) + if tc.page { + bound = repeatedPageBound + } + if got > bound { + t.Errorf("allocated %d MiB for a %d MiB response, want under %d MiB", got>>20, len(body)>>20, bound>>20) } }) } } +// Bounds for TestRepeatedBadValuesAllocateLittle, in bytes allocated. Measured +// without and with -race: fixed page 24 and 41 MiB (the response buffer), fixed +// issue 6 and 11 MiB; unbudgeted page 118 to 615 MiB, unbudgeted issue of bad +// tags 134 and 155 MiB (the unbudgeted issue of bad created values allocates +// 26 and 33 MiB, so that case is a plain regression check, not a discriminator). +const ( + repeatedPageBound = 64 << 20 + repeatedIssueBound = 20 << 20 +) + +func TestTagElementsMustBeObjects(t *testing.T) { + for name, tc := range map[string]struct { + in string + ok bool + }{ + "empty": {`[]`, true}, + "objects": {`[{"id":"a"} , {}]`, true}, + "null": {`null`, true}, + "null element": {`[null]`, true}, + "number": {`[1]`, false}, + "later number": {`[{},1]`, false}, + "string": {`["{"]`, false}, + "nested array": {`[[{}]]`, false}, + "brace in text": {`[{"a":"1,2"},{"b":[1,2]}]`, true}, + } { + var tags ytTags + err := tags.UnmarshalJSON([]byte(tc.in)) + if (err == nil) != tc.ok { + t.Errorf("%s: err = %v, want ok=%v", name, err, tc.ok) + } + } +} + func tagsJSON(n int) string { return strings.TrimSuffix(strings.Repeat(`{"id":"5-1","name":"a"},`, n), ",") } diff --git a/api/internal/issuetracker/youtrack_request.go b/api/internal/issuetracker/youtrack_request.go index d3fbcd4c86..871eeb00e7 100644 --- a/api/internal/issuetracker/youtrack_request.go +++ b/api/internal/issuetracker/youtrack_request.go @@ -92,7 +92,11 @@ func (y *youTrack) do(ctx context.Context, c ytCall, out any) (status, n int, er case c.decode != nil: decodeErr = c.decode(body) case out != nil: - decodeErr = json.Unmarshal(body, out) + if valueCount(body) > objectValueBudget { + decodeErr = errValueCeiling + } else { + decodeErr = json.Unmarshal(body, out) + } } if decodeErr != nil { return status, len(body), y.red.errorStatus(fmt.Errorf("youtrack: %s %s: decode response: %w", c.method, path, decodeErr), status) @@ -132,8 +136,8 @@ type nester interface { // ytPages walks every page of a list endpoint under paginate's ceilings and // calls visit with each page. visit returns true to stop early. Items that // implement nester count their nested entries toward maxItems. A page is -// decoded as a stream and refused as soon as it holds more than pageSize -// items. Errors come back redacted. +// counted by a byte scan and refused above pageSize items or the structural +// value budget before any element is decoded. Errors come back redacted. func ytPages[T any](ctx context.Context, y *youTrack, path []string, query url.Values, visit func([]T) bool) error { var fetchErr error seen := 0 diff --git a/api/internal/issuetrackersvc/account.go b/api/internal/issuetrackersvc/account.go index 8b59ffc0a6..eecffeb1e3 100644 --- a/api/internal/issuetrackersvc/account.go +++ b/api/internal/issuetrackersvc/account.go @@ -2,7 +2,6 @@ package issuetrackersvc import ( "errors" - "unicode" "github.com/vtmocanu/uzi/api/internal/issuetracker" ) @@ -29,7 +28,7 @@ func validateAccountField(s string) error { return errors.New("empty or too long") } for _, r := range s { - if unicode.IsControl(r) || unicode.Is(unicode.Cf, r) || r == '\u2028' || r == '\u2029' { + if issuetracker.InvisibleRune(r) { return errors.New("forbidden character") } } From 493803cf27616dd8bcbf0b7446f44590ece1d346 Mon Sep 17 00:00:00 2001 From: uzi-agent Date: Wed, 11 Nov 2026 10:42:54 +0000 Subject: [PATCH 11/17] issuetracker: decode list pages one element at a time A page is validated whole, then each element is budgeted and unmarshalled on its own, stopping at the first error, so wrong-typed scalars cost at most one element's worth of error values. The item budget leaves room for real YouTrack activities with $type tags at the nested ceiling, and the duplicate-key and tag-shape tests now reach the code they pin. Co-Authored-By: Claude Opus 5.5 --- api/internal/issuetracker/youtrack_decode.go | 111 +++++++++++++-- .../issuetracker/youtrack_decode_test.go | 129 +++++++++++++++--- 2 files changed, 209 insertions(+), 31 deletions(-) diff --git a/api/internal/issuetracker/youtrack_decode.go b/api/internal/issuetracker/youtrack_decode.go index 89c8d8bfa9..8da75bbcd8 100644 --- a/api/internal/issuetracker/youtrack_decode.go +++ b/api/internal/issuetracker/youtrack_decode.go @@ -15,18 +15,25 @@ var errNestedCeiling = errors.New("issuetracker: an item carries more nested ent // Structural budgets, applied by a byte scan before any response is handed to // json.Unmarshal. A value is counted as one comma or one opening bracket -// outside a string, an upper bound on the number of JSON values. A tag with an -// id and a name costs 3 (its brace, the comma between its members and the comma -// after it), and an activity carries up to maxNestedItems tags in each of -// "added" and "removed", so one item is allowed 6*maxNestedItems+64 and a -// single object (an issue holds one tags array) 4*maxNestedItems+256. The -// budget exists because encoding/json keeps decoding after a type error and -// builds an error per bad value, so a response of many bad values would -// otherwise allocate in proportion to its bytes, not to what it legitimately -// holds. +// outside a string, an upper bound on the number of JSON values. Real YouTrack +// adds a "$type" member to every entity, so a tag with $type, id and name costs +// 4 (its brace, the commas between its three members and the comma after it). +// An activity carries up to maxNestedItems such tags in each of "added" and +// "removed" (8*maxNestedItems values), so one item is allowed +// 10*maxNestedItems+256, which leaves headroom for one more member per tag; a single object (an +// issue holds one tags array, plus a few members of its own) is allowed +// 6*maxNestedItems+256, which leaves headroom for $type and one more member per +// tag. Three totals are enforced: pageValueBudget for a whole list page, +// itemValueBudget for each element of it and objectValueBudget for one +// single-object response. The budgets bound the structure, not the cost of a +// type error: 13 bytes of bad scalar fit many times inside a page budget, and +// encoding/json keeps decoding after a type error, building one error (about +// 150 bytes) per bad value. So decodePage also decodes element by element and +// stops at the first error, which leaves one element, at most itemValueBudget +// values, to amplify. const ( - itemValueBudget = 6*maxNestedItems + 64 - objectValueBudget = 4*maxNestedItems + 256 + itemValueBudget = 10*maxNestedItems + 256 + objectValueBudget = 6*maxNestedItems + 256 pageValueBudget = pageSize * itemValueBudget ) @@ -221,7 +228,11 @@ func isJSONSpace(c byte) bool { return c == ' ' || c == '\t' || c == '\n' || c = // longer array is a protocol violation: the elements and the structural values // are counted by byte scans before any element is decoded, so an oversized page // allocates nothing for its items. A JSON null is an empty page. Whitespace is -// JSON's own, so a form feed is a syntax error rather than padding. +// JSON's own, so a form feed is refused (errNotArray, or errTrailingData after +// the array) rather than skipped as padding. The whole page is validated first, +// then each element is decoded on its own and the first error ends the decode: +// encoding/json would otherwise go on past a type error and build an error per +// bad value, allocating in proportion to the page's bytes. func decodePage[T any](body []byte) ([]T, error) { trimmed := trimJSONSpace(body) if len(trimmed) >= 4 && string(trimmed[:4]) == "null" { @@ -248,9 +259,81 @@ func decodePage[T any](body []byte) ([]T, error) { if valueCount(body) > pageValueBudget { return nil, errValueCeiling } + if !json.Valid(body) { + // Unmarshal reports the syntax error; it decodes nothing on invalid input. + var discard []T + return nil, json.Unmarshal(body, &discard) + } var items []T - if err := json.Unmarshal(body, &items); err != nil { - return nil, err + var elemErr error + forEachElement(body, func(elem []byte) bool { + if valueCount(elem) > itemValueBudget { + elemErr = errValueCeiling + return false + } + var item T + if elemErr = json.Unmarshal(elem, &item); elemErr != nil { + return false + } + items = append(items, item) + return true + }) + if elemErr != nil { + return nil, elemErr } return items, nil } + +// forEachElement calls f with each top-level element of b, a valid JSON array +// (possibly padded with JSON space), stopping when f returns false. It tracks +// bracket depth and string state only and allocates nothing. +func forEachElement(b []byte, f func(elem []byte) bool) { + depth, start, inStr, esc := 0, -1, false, false + emit := func(end int) bool { + if start < 0 { + return true + } + elem := trimJSONSpace(b[start:end]) + start = -1 + return len(elem) == 0 || f(elem) + } + for i, c := range b { + if inStr { + switch { + case esc: + esc = false + case c == '\\': + esc = true + case c == '"': + inStr = false + } + continue + } + switch c { + case '[', '{': + depth++ + if depth == 2 && start < 0 { + start = i + } + case ']', '}': + depth-- + if depth == 0 { + emit(i) + return + } + case ',': + if depth == 1 && !emit(i) { + return + } + case '"': + inStr = true + if depth == 1 && start < 0 { + start = i + } + default: + if depth == 1 && start < 0 && !isJSONSpace(c) { + start = i + } + } + } +} diff --git a/api/internal/issuetracker/youtrack_decode_test.go b/api/internal/issuetracker/youtrack_decode_test.go index 913e3439a7..d46514fa33 100644 --- a/api/internal/issuetracker/youtrack_decode_test.go +++ b/api/internal/issuetracker/youtrack_decode_test.go @@ -97,20 +97,23 @@ func TestTagBombAllocatesLittle(t *testing.T) { } // TestDuplicateTagsKeysAllocateLittle pins that encoding/json calling -// ytTags.UnmarshalJSON once per repeated "tags" key costs no per-call Decoder: -// the unfixed decoder allocated about 380 MiB for the page fixture. The fixed -// decoder allocates about 30 MiB without -race and about 100 MiB with it (the -// race build makes sync.Pool drop entries, so encoding/json's own pooled state -// is reallocated), so the bound sits between the two. +// ytTags.UnmarshalJSON once per repeated "tags" key costs no per-call Decoder. +// The fixtures sit just under pageValueBudget (and itemValueBudget per issue) and +// objectValueBudget, so every repeated key reaches ytTags. Measured without and +// with -race: the page allocates about 8 and 31 MiB, and about 101 and 125 MiB +// with a json.Decoder built per call; the issue allocates about 279 and 646 KiB, +// and about 1282 and 1593 KiB with one. func TestDuplicateTagsKeysAllocateLittle(t *testing.T) { - const bound = 192 << 20 - // The fixed single-issue decode allocates about 4 MiB (25 MiB with -race); - // the unfixed one allocated about 99 MiB (120 MiB with -race). - const issueBound = 48 << 20 + const bound = 64 << 20 + const issueBound = 960 << 10 t.Run("page", func(t *testing.T) { issue := strings.TrimSuffix(rawIssue("2-1", "UZI-1", 1, "0-1", ""), `"tags":[]}`) dup := `"tags":[{}],` - body := "[" + issue + strings.Repeat(dup, int(listPageBytes-int64(len(issue))-64)/len(dup)) + `"tags":[]}]` + // Each repetition is 3 values: pageSize issues just under itemValueBudget + // each keep the page under pageValueBudget and ytTags runs once per + // repeated key. + elem := issue + strings.Repeat(dup, (itemValueBudget-64)/3) + `"tags":[]}` + body := "[" + strings.TrimSuffix(strings.Repeat(elem+",", pageSize), ",") + "]" if int64(len(body)) > listPageBytes { t.Fatalf("fixture is %d bytes, over the page cap", len(body)) } @@ -124,23 +127,24 @@ func TestDuplicateTagsKeysAllocateLittle(t *testing.T) { var err error got := totalAlloc(func() { _, err = y.ListIssues(context.Background(), 1, forge.ListIssuesOptions{}) }) if err != nil { - t.Logf("ListIssues: %v", err) + t.Errorf("ListIssues: %v", err) } if got > bound { - t.Errorf("allocated %d MiB for a %d MiB page, want under %d MiB", got>>20, len(body)>>20, bound>>20) + t.Errorf("allocated %d KiB for a %d KiB page, want under %d KiB", got>>10, len(body)>>10, bound>>10) } }) t.Run("issue", func(t *testing.T) { issue := strings.TrimSuffix(rawIssue("2-1", "UZI-1", 1, "0-1", ""), `"tags":[]}`) dup := `"tags":[{}],` - body := []byte(issue + strings.Repeat(dup, int(objectBytes-int64(len(issue))-64)/len(dup)) + `"tags":[]}`) + // Just under objectValueBudget, so ytTags runs once per repeated key. + body := []byte(issue + strings.Repeat(dup, (objectValueBudget-64)/3) + `"tags":[]}`) if int64(len(body)) > objectBytes { t.Fatalf("fixture is %d bytes, over the object cap", len(body)) } y, _ := loggedDriver(t, func(w http.ResponseWriter, _ *http.Request) { _, _ = w.Write(body) }) got := totalAlloc(func() { _, _ = y.GetIssue(context.Background(), 1, 1) }) if got > issueBound { - t.Errorf("allocated %d MiB for a %d MiB issue, want under %d MiB", got>>20, len(body)>>20, issueBound>>20) + t.Errorf("allocated %d KiB for a %d KiB issue, want under %d KiB", got>>10, len(body)>>10, issueBound>>10) } }) } @@ -160,6 +164,7 @@ func TestListDecodeShapes(t *testing.T) { "truncated": {`[{"id":"0-1"`, io.ErrUnexpectedEOF, 0}, "over the item ceiling": {emptyObjects(pageSize + 1), errItemCeiling, 0}, "exactly the item ceiling": {emptyObjects(pageSize), nil, pageSize}, + "one item over its budget": {`[{` + strings.Repeat(`"a":"",`, itemValueBudget) + `"a":""}]`, errValueCeiling, 0}, "repeated bad values": {`[{` + strings.Repeat(`"a":"",`, pageValueBudget) + `"a":""}]`, errValueCeiling, 0}, "form feed after the array": {"[]\f", errTrailingData, 0}, "JSON space after the array": {"[] \n", nil, 0}, @@ -231,11 +236,13 @@ func TestRepeatedBadValuesAllocateLittle(t *testing.T) { // Bounds for TestRepeatedBadValuesAllocateLittle, in bytes allocated. Measured // without and with -race: fixed page 24 and 41 MiB (the response buffer), fixed // issue 6 and 11 MiB; unbudgeted page 118 to 615 MiB, unbudgeted issue of bad -// tags 134 and 155 MiB (the unbudgeted issue of bad created values allocates -// 26 and 33 MiB, so that case is a plain regression check, not a discriminator). +// tags 134 and 155 MiB, unbudgeted issue of bad created values 26 and 33 MiB. +// The issue of bad created values is therefore the discriminator for the +// budget that do() applies to a single object: it is the one case whose +// unbudgeted allocation (26 MiB) is above repeatedIssueBound. const ( repeatedPageBound = 64 << 20 - repeatedIssueBound = 20 << 20 + repeatedIssueBound = 18 << 20 ) func TestTagElementsMustBeObjects(t *testing.T) { @@ -258,6 +265,9 @@ func TestTagElementsMustBeObjects(t *testing.T) { if (err == nil) != tc.ok { t.Errorf("%s: err = %v, want ok=%v", name, err, tc.ok) } + if !tc.ok && !errors.Is(err, errTagNotObject) { + t.Errorf("%s: err = %v, want errTagNotObject", name, err) + } } } @@ -335,3 +345,88 @@ func TestTopLevelCountIgnoresBracketsInStrings(t *testing.T) { } } } + +// TestScalarTypeErrorsStopAtTheFirstElement pins that a page whose values all +// fail to decode but stay under pageValueBudget allocates little: decodePage +// decodes element by element and stops at the first error, where one +// json.Unmarshal of the whole page keeps going and builds an error per bad +// value (about 150 bytes each). +func TestScalarTypeErrorsStopAtTheFirstElement(t *testing.T) { + // The fixed decode allocates the response buffer only: about 24 MiB, 40 MiB + // with -race. The unfixed one allocated about 120 MiB more. + const bound = 64 << 20 + issue := strings.TrimSuffix(rawIssue("2-1", "UZI-1", 1, "0-1", ""), `"tags":[]}`) + for name, dup := range map[string]string{ + "created is a string": `"created":"",`, + "summary is a number": `"summary":1,`, + } { + t.Run(name, func(t *testing.T) { + // pageSize issues of 6000 bad values each: every issue under + // itemValueBudget, the page under pageValueBudget and under + // listPageBytes, while every one of its values is bad. + elem := issue + strings.Repeat(dup, 6000) + `"tags":[]}` + body := "[" + strings.TrimSuffix(strings.Repeat(elem+",", pageSize), ",") + "]" + if int64(len(body)) > listPageBytes { + t.Fatalf("fixture is %d bytes, over the page cap", len(body)) + } + if n := valueCount([]byte(body)); n > pageValueBudget { + t.Fatalf("fixture holds %d values, over the page budget %d", n, pageValueBudget) + } + y, _ := loggedDriver(t, func(w http.ResponseWriter, r *http.Request) { + if r.URL.Query().Get("$skip") != "0" { + writeRaw(w, "[]") + return + } + _, _ = w.Write([]byte(body)) + }) + var err error + got := totalAlloc(func() { _, err = y.ListIssues(context.Background(), 1, forge.ListIssuesOptions{}) }) + // The driver redacts errors into text, so the type error is matched + // by message: the budget sentinel would read "more values". + if err == nil || !strings.Contains(err.Error(), "cannot unmarshal") { + t.Fatalf("err = %v, want a type error", err) + } + if got > bound { + t.Errorf("allocated %d MiB for a %d MiB page, want under %d MiB", got>>20, len(body)>>20, bound>>20) + } + }) + } +} + +// TestRealShapedIssueIsAccepted decodes an issue shaped like YouTrack's own +// output (a $type on every entity, 1000 tags) through GetIssue, so the object +// budget keeps headroom for the shape the real server sends. +func TestRealShapedIssueIsAccepted(t *testing.T) { + tag := `{"$type":"Tag","id":"6-1","name":"x"}` + issue := `{"$type":"Issue","id":"2-1","idReadable":"UZI-1","numberInProject":1,"summary":"s","created":1,"updated":2000,` + + `"project":{"$type":"Project","id":"0-1","shortName":"UZI"},` + + `"tags":[` + strings.TrimSuffix(strings.Repeat(tag+",", maxNestedItems), ",") + `]}` + if n := valueCount([]byte(issue)); n > objectValueBudget || n < objectValueBudget/2 { + t.Fatalf("fixture holds %d values, want at least half of the %d budget without exceeding it", n, objectValueBudget) + } + y, _ := loggedDriver(t, func(w http.ResponseWriter, _ *http.Request) { writeRaw(w, issue) }) + got, err := y.GetIssue(context.Background(), 1, 1) + if err != nil || len(got.Labels) != maxNestedItems { + t.Fatalf("%d labels, %v; want %d labels", len(got.Labels), err, maxNestedItems) + } +} + +// TestRealShapedActivityIsAccepted decodes a page of activities shaped like +// YouTrack's own output (a $type on every entity) at the nested ceiling in both +// "added" and "removed", so the per-item budget keeps headroom for it. +func TestRealShapedActivityIsAccepted(t *testing.T) { + tag := `{"$type":"Tag","id":"6-1","name":"x"}` + tags := "[" + strings.TrimSuffix(strings.Repeat(tag+",", maxNestedItems), ",") + "]" + act := `{"$type":"TagsActivityItem","id":"a-1","timestamp":1,"author":{"$type":"User","id":"1-1","login":"bot"},"added":` + tags + `,"removed":` + tags + `}` + body := "[" + strings.TrimSuffix(strings.Repeat(act+",", 3), ",") + "]" + if n := valueCount([]byte(act)); n <= 6*maxNestedItems+64 || n > itemValueBudget { + t.Fatalf("fixture holds %d values per activity, want between the old and the current item budget", n) + } + got, err := decodePage[ytActivity]([]byte(body)) + if err != nil { + t.Fatalf("decodePage: %v", err) + } + if len(got) != 3 || len(got[0].Added) != maxNestedItems || len(got[0].Removed) != maxNestedItems { + t.Fatalf("decoded %d activities, want 3 with %d tags each way", len(got), maxNestedItems) + } +} From 698ef1877bea928d1584e66c8025ca336114bb07 Mon Sep 17 00:00:00 2001 From: uzi-agent Date: Wed, 11 Nov 2026 10:55:05 +0000 Subject: [PATCH 12/17] issuetracker: pin string-aware element walking and real-shaped budgets A null tags value keeps the backing array, the real-shaped fixtures carry one more member per tag so the documented headroom is tested, and the decode comments and allocation figures match what was measured. Co-Authored-By: Claude Opus 5.5 --- api/internal/issuetracker/youtrack_decode.go | 44 +++++++++++-------- .../issuetracker/youtrack_decode_test.go | 39 +++++++++++----- 2 files changed, 55 insertions(+), 28 deletions(-) diff --git a/api/internal/issuetracker/youtrack_decode.go b/api/internal/issuetracker/youtrack_decode.go index 8da75bbcd8..338974ca0d 100644 --- a/api/internal/issuetracker/youtrack_decode.go +++ b/api/internal/issuetracker/youtrack_decode.go @@ -15,22 +15,22 @@ var errNestedCeiling = errors.New("issuetracker: an item carries more nested ent // Structural budgets, applied by a byte scan before any response is handed to // json.Unmarshal. A value is counted as one comma or one opening bracket -// outside a string, an upper bound on the number of JSON values. Real YouTrack -// adds a "$type" member to every entity, so a tag with $type, id and name costs -// 4 (its brace, the commas between its three members and the comma after it). -// An activity carries up to maxNestedItems such tags in each of "added" and -// "removed" (8*maxNestedItems values), so one item is allowed -// 10*maxNestedItems+256, which leaves headroom for one more member per tag; a single object (an -// issue holds one tags array, plus a few members of its own) is allowed -// 6*maxNestedItems+256, which leaves headroom for $type and one more member per -// tag. Three totals are enforced: pageValueBudget for a whole list page, -// itemValueBudget for each element of it and objectValueBudget for one -// single-object response. The budgets bound the structure, not the cost of a -// type error: 13 bytes of bad scalar fit many times inside a page budget, and -// encoding/json keeps decoding after a type error, building one error (about -// 150 bytes) per bad value. So decodePage also decodes element by element and -// stops at the first error, which leaves one element, at most itemValueBudget -// values, to amplify. +// outside a string, which approximates the number of JSON values (at most +// 2*count+1). Real YouTrack adds a "$type" member to every entity, so a tag +// with $type, id, name and one more member such as "color" costs 5 (its brace, +// the commas between its four members and the comma after it). An activity +// carries up to maxNestedItems such tags in each of "added" and "removed" +// (10*maxNestedItems values), so one item is allowed 10*maxNestedItems+256. A +// single object (an issue holds one tags array, plus a few members of its own) +// is allowed 6*maxNestedItems+256, which holds 5*maxNestedItems for the tags. +// The real-shaped tests pin that headroom. Three totals are enforced: +// pageValueBudget for a whole list page, itemValueBudget for each element of +// it and objectValueBudget for one single-object response. The budgets bound +// the structure, not the cost of a type error: 13 bytes of bad scalar fit many +// times inside a page budget, and encoding/json keeps decoding after a type +// error, building one error (about 150 bytes) per bad value. So decodePage +// also decodes element by element and stops at the first error, which leaves +// one element, at most itemValueBudget values, to amplify. const ( itemValueBudget = 10*maxNestedItems + 256 objectValueBudget = 6*maxNestedItems + 256 @@ -60,6 +60,12 @@ type ytTags []ytTag // UnmarshalJSON implements json.Unmarshaler. b is already-validated JSON. func (t *ytTags) UnmarshalJSON(b []byte) error { + if trimmed := trimJSONSpace(b); string(trimmed) == "null" { + // null is an empty list; keep the backing array so a repeated key does + // not force a reallocation per call. + *t = (*t)[:0] + return nil + } if n, _ := topLevelCount(b); n > maxNestedItems { return errNestedCeiling } @@ -186,7 +192,8 @@ func onlyObjects(b []byte) bool { } // valueCount returns the number of commas and opening brackets outside strings -// in b, at every depth: an upper bound on the number of JSON values. +// in b, at every depth: approximately the number of JSON values (at most +// 2*count+1). func valueCount(b []byte) int { n, inStr, esc := 0, false, false for _, c := range b { @@ -229,7 +236,8 @@ func isJSONSpace(c byte) bool { return c == ' ' || c == '\t' || c == '\n' || c = // are counted by byte scans before any element is decoded, so an oversized page // allocates nothing for its items. A JSON null is an empty page. Whitespace is // JSON's own, so a form feed is refused (errNotArray, or errTrailingData after -// the array) rather than skipped as padding. The whole page is validated first, +// the array) rather than skipped as padding, and a form feed inside the array +// is a syntax error. The whole page is validated first, // then each element is decoded on its own and the first error ends the decode: // encoding/json would otherwise go on past a type error and build an error per // bad value, allocating in proportion to the page's bytes. diff --git a/api/internal/issuetracker/youtrack_decode_test.go b/api/internal/issuetracker/youtrack_decode_test.go index d46514fa33..42b1db1706 100644 --- a/api/internal/issuetracker/youtrack_decode_test.go +++ b/api/internal/issuetracker/youtrack_decode_test.go @@ -100,11 +100,11 @@ func TestTagBombAllocatesLittle(t *testing.T) { // ytTags.UnmarshalJSON once per repeated "tags" key costs no per-call Decoder. // The fixtures sit just under pageValueBudget (and itemValueBudget per issue) and // objectValueBudget, so every repeated key reaches ytTags. Measured without and -// with -race: the page allocates about 8 and 31 MiB, and about 101 and 125 MiB +// with -race: the page allocates about 14 and 54 MiB, and about 172 and 215 MiB // with a json.Decoder built per call; the issue allocates about 279 and 646 KiB, // and about 1282 and 1593 KiB with one. func TestDuplicateTagsKeysAllocateLittle(t *testing.T) { - const bound = 64 << 20 + const bound = 112 << 20 const issueBound = 960 << 10 t.Run("page", func(t *testing.T) { issue := strings.TrimSuffix(rawIssue("2-1", "UZI-1", 1, "0-1", ""), `"tags":[]}`) @@ -236,7 +236,8 @@ func TestRepeatedBadValuesAllocateLittle(t *testing.T) { // Bounds for TestRepeatedBadValuesAllocateLittle, in bytes allocated. Measured // without and with -race: fixed page 24 and 41 MiB (the response buffer), fixed // issue 6 and 11 MiB; unbudgeted page 118 to 615 MiB, unbudgeted issue of bad -// tags 134 and 155 MiB, unbudgeted issue of bad created values 26 and 33 MiB. +// tags 134 and 155 MiB only with both the object budget and onlyObjects +// removed, unbudgeted issue of bad created values 26 and 33 MiB. // The issue of bad created values is therefore the discriminator for the // budget that do() applies to a single object: it is the one case whose // unbudgeted allocation (26 MiB) is above repeatedIssueBound. @@ -327,6 +328,23 @@ func TestTopLevelItemsAndNestedTagsShareOneBudget(t *testing.T) { } } +func TestDecodePageIgnoresBracketsInStrings(t *testing.T) { + type item struct { + Summary string `json:"summary"` + } + got, err := decodePage[item]([]byte(`[{"summary":"a}],{"}, {"summary":"\\\"]"}]`)) + if err != nil || len(got) != 2 || got[0].Summary != `a}],{` || got[1].Summary != `\"]` { + t.Fatalf("got %+v, %v; want 2 items with the bracketed summaries", got, err) + } +} + +func TestNullTagsKeepsEmptyListSemantics(t *testing.T) { + tags := ytTags{{}, {}} + if err := tags.UnmarshalJSON([]byte(" null ")); err != nil || len(tags) != 0 { + t.Errorf("null: %d tags, %v; want an empty list", len(tags), err) + } +} + func TestTopLevelCountIgnoresBracketsInStrings(t *testing.T) { for name, tc := range map[string]struct { in string @@ -353,7 +371,8 @@ func TestTopLevelCountIgnoresBracketsInStrings(t *testing.T) { // value (about 150 bytes each). func TestScalarTypeErrorsStopAtTheFirstElement(t *testing.T) { // The fixed decode allocates the response buffer only: about 24 MiB, 40 MiB - // with -race. The unfixed one allocated about 120 MiB more. + // with -race. Decoding the whole page at once measured 110 and 131 MiB, about + // 86 to 91 MiB more. const bound = 64 << 20 issue := strings.TrimSuffix(rawIssue("2-1", "UZI-1", 1, "0-1", ""), `"tags":[]}`) for name, dup := range map[string]string{ @@ -397,12 +416,12 @@ func TestScalarTypeErrorsStopAtTheFirstElement(t *testing.T) { // output (a $type on every entity, 1000 tags) through GetIssue, so the object // budget keeps headroom for the shape the real server sends. func TestRealShapedIssueIsAccepted(t *testing.T) { - tag := `{"$type":"Tag","id":"6-1","name":"x"}` + tag := `{"$type":"Tag","id":"6-1","name":"x","color":null}` issue := `{"$type":"Issue","id":"2-1","idReadable":"UZI-1","numberInProject":1,"summary":"s","created":1,"updated":2000,` + `"project":{"$type":"Project","id":"0-1","shortName":"UZI"},` + `"tags":[` + strings.TrimSuffix(strings.Repeat(tag+",", maxNestedItems), ",") + `]}` - if n := valueCount([]byte(issue)); n > objectValueBudget || n < objectValueBudget/2 { - t.Fatalf("fixture holds %d values, want at least half of the %d budget without exceeding it", n, objectValueBudget) + if n := valueCount([]byte(issue)); n > objectValueBudget || n <= 4*maxNestedItems+256 { + t.Fatalf("fixture holds %d values, want more than %d without exceeding the %d budget", n, 4*maxNestedItems+256, objectValueBudget) } y, _ := loggedDriver(t, func(w http.ResponseWriter, _ *http.Request) { writeRaw(w, issue) }) got, err := y.GetIssue(context.Background(), 1, 1) @@ -415,12 +434,12 @@ func TestRealShapedIssueIsAccepted(t *testing.T) { // YouTrack's own output (a $type on every entity) at the nested ceiling in both // "added" and "removed", so the per-item budget keeps headroom for it. func TestRealShapedActivityIsAccepted(t *testing.T) { - tag := `{"$type":"Tag","id":"6-1","name":"x"}` + tag := `{"$type":"Tag","id":"6-1","name":"x","color":null}` tags := "[" + strings.TrimSuffix(strings.Repeat(tag+",", maxNestedItems), ",") + "]" act := `{"$type":"TagsActivityItem","id":"a-1","timestamp":1,"author":{"$type":"User","id":"1-1","login":"bot"},"added":` + tags + `,"removed":` + tags + `}` body := "[" + strings.TrimSuffix(strings.Repeat(act+",", 3), ",") + "]" - if n := valueCount([]byte(act)); n <= 6*maxNestedItems+64 || n > itemValueBudget { - t.Fatalf("fixture holds %d values per activity, want between the old and the current item budget", n) + if n := valueCount([]byte(act)); n <= 8*maxNestedItems+64 || n > itemValueBudget { + t.Fatalf("fixture holds %d values per activity, want between %d and the item budget %d", n, 8*maxNestedItems+64, itemValueBudget) } got, err := decodePage[ytActivity]([]byte(body)) if err != nil { From fc627cfcd84565ee05c2045934834eb78a284fd7 Mon Sep 17 00:00:00 2001 From: uzi-agent Date: Wed, 11 Nov 2026 10:59:18 +0000 Subject: [PATCH 13/17] issuetracker: clear reused tag elements; pin the null-key allocation encoding/json reuses slice elements without zeroing them, so a repeated tags key could inherit an id an earlier key set. Reused elements are now cleared, and tests pin both that and the null branch's allocation bound (race-aware). Co-Authored-By: Claude Opus 5.5 --- api/internal/issuetracker/race_off_test.go | 7 ++++ api/internal/issuetracker/race_on_test.go | 7 ++++ api/internal/issuetracker/youtrack_decode.go | 8 +++- .../issuetracker/youtrack_decode_test.go | 40 +++++++++++++++++++ 4 files changed, 60 insertions(+), 2 deletions(-) create mode 100644 api/internal/issuetracker/race_off_test.go create mode 100644 api/internal/issuetracker/race_on_test.go diff --git a/api/internal/issuetracker/race_off_test.go b/api/internal/issuetracker/race_off_test.go new file mode 100644 index 0000000000..b36e484431 --- /dev/null +++ b/api/internal/issuetracker/race_off_test.go @@ -0,0 +1,7 @@ +//go:build !race + +package issuetracker + +// raceEnabled reports whether the race detector is on; it changes allocation +// counts (sync.Pool drops entries), so allocation bounds read it. +const raceEnabled = false diff --git a/api/internal/issuetracker/race_on_test.go b/api/internal/issuetracker/race_on_test.go new file mode 100644 index 0000000000..81d1c5d81a --- /dev/null +++ b/api/internal/issuetracker/race_on_test.go @@ -0,0 +1,7 @@ +//go:build race + +package issuetracker + +// raceEnabled reports whether the race detector is on; it changes allocation +// counts (sync.Pool drops entries), so allocation bounds read it. +const raceEnabled = true diff --git a/api/internal/issuetracker/youtrack_decode.go b/api/internal/issuetracker/youtrack_decode.go index 338974ca0d..5ef1834f73 100644 --- a/api/internal/issuetracker/youtrack_decode.go +++ b/api/internal/issuetracker/youtrack_decode.go @@ -60,9 +60,13 @@ type ytTags []ytTag // UnmarshalJSON implements json.Unmarshaler. b is already-validated JSON. func (t *ytTags) UnmarshalJSON(b []byte) error { + // A repeated "tags" key reuses the backing array, so a later key never + // forces a reallocation per call. encoding/json does not zero reused + // elements, so clear them first: a field a later key omits must not keep + // the value an earlier key gave it. + clear((*t)[:cap(*t)]) if trimmed := trimJSONSpace(b); string(trimmed) == "null" { - // null is an empty list; keep the backing array so a repeated key does - // not force a reallocation per call. + // null is an empty list. *t = (*t)[:0] return nil } diff --git a/api/internal/issuetracker/youtrack_decode_test.go b/api/internal/issuetracker/youtrack_decode_test.go index 42b1db1706..9b86f5a9a8 100644 --- a/api/internal/issuetracker/youtrack_decode_test.go +++ b/api/internal/issuetracker/youtrack_decode_test.go @@ -2,6 +2,7 @@ package issuetracker import ( "context" + "encoding/json" "errors" "io" "net/http" @@ -345,6 +346,45 @@ func TestNullTagsKeepsEmptyListSemantics(t *testing.T) { } } +// TestRepeatedTagsKeysLeaveNoStaleFields pins that a later "tags" key never +// inherits a field an earlier key set: the reused elements are cleared. +func TestRepeatedTagsKeysLeaveNoStaleFields(t *testing.T) { + for name, in := range map[string]string{ + "null between": `{"tags":[{"id":"6-1","name":"a"},{"id":"6-2","name":"b"}],"tags":null,"tags":[{"name":"c"}]}`, + "no null": `{"tags":[{"id":"6-1","name":"a"},{"id":"6-2","name":"b"}],"tags":[{"name":"c"}]}`, + } { + var w ytIssue + if err := json.Unmarshal([]byte(in), &w); err != nil { + t.Fatalf("%s: %v", name, err) + } + if len(w.Tags) != 1 || w.Tags[0].ID != "" || w.Tags[0].Name != "c" { + t.Errorf("%s: tags = %+v, want one tag named c with no id", name, w.Tags) + } + } +} + +// TestNullBetweenRepeatedTagsAllocatesLittle pins that a null between repeated +// "tags" keys keeps the backing array. Measured under 2 MiB (22 MiB with +// -race), against about 7.5 MiB (49 MiB with -race) when null is handed to +// json.Unmarshal and the slice is rebuilt per key. +func TestNullBetweenRepeatedTagsAllocatesLittle(t *testing.T) { + bound := uint64(4 << 20) + if raceEnabled { + bound = 36 << 20 + } + issue := strings.TrimSuffix(rawIssue("2-1", "UZI-1", 1, "0-1", ""), `"tags":[]}`) + elem := issue + strings.Repeat(`"tags":[{}],"tags":null,`, 2400) + `"tags":[]}` + body := []byte("[" + strings.TrimSuffix(strings.Repeat(elem+",", pageSize), ",") + "]") + var err error + got := totalAlloc(func() { _, err = decodePage[ytIssue](body) }) + if err != nil { + t.Fatalf("decodePage: %v", err) + } + if got > bound { + t.Errorf("allocated %d KiB, want under %d KiB", got>>10, bound>>10) + } +} + func TestTopLevelCountIgnoresBracketsInStrings(t *testing.T) { for name, tc := range map[string]struct { in string From d07aef4bbcce0ace277fb6ce15577433ab0a77a2 Mon Sep 17 00:00:00 2001 From: uzi-agent Date: Wed, 11 Nov 2026 11:05:44 +0000 Subject: [PATCH 14/17] issuetracker: run the contract suite on the three forge drivers TestIssueTrackerForgeFakeContracts runs issuetrackertest.RunContract against the real GitLab, GitHub and Forgejo drivers over the Node forge fake, launched by the new gate:repo task test:issue-tracker-forge-fake-contract. The fake's extra issue routes and label catalogs answer only under FORGE_FAKE_ISSUE_CONTRACT=1, so the e2e harness sees the fake exactly as before. Co-Authored-By: Claude Opus 5.5 --- Taskfile.yml | 12 ++ .../issuetracker/forgefake_contract_test.go | 90 +++++++++++ e2e/forge-fake/forge-fake.mjs | 89 ++++++++++- .../issue-tracker-contracts.test.mjs | 142 ++++++++++++++++++ 4 files changed, 330 insertions(+), 3 deletions(-) create mode 100644 api/internal/issuetracker/forgefake_contract_test.go create mode 100644 e2e/forge-fake/issue-tracker-contracts.test.mjs diff --git a/Taskfile.yml b/Taskfile.yml index 29d1835543..41ebc200a1 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -352,6 +352,7 @@ tasks: - task: test:renovate-schedule-groups - task: test:forge-fake - task: test:forge-fake-contract + - task: test:issue-tracker-forge-fake-contract # Hermetic bot-provisioning regressions (issues #2010 and #2016): fake glab, # no network, covering host selection, JSON, PAT headers and safe diagnostics. - task: test:create-gitlab-bot @@ -1213,6 +1214,17 @@ tasks: # Anchored run selects only this external-package contract, not the suite. - env GIT_CONFIG_GLOBAL=/dev/null GIT_CONFIG_SYSTEM=/dev/null node e2e/forge-fake/operation-contracts.test.mjs -- go test -buildvcs=false -count=1 -v -timeout=60s -run='^TestForgeFakeOperationContracts$' ./internal/forge + test:issue-tracker-forge-fake-contract: + desc: shared issue tracker contract against the real GitLab, GitHub and Forgejo drivers over the Node HTTPS fake + preconditions: + - sh: command -v node + msg: the issue tracker forge fake contract requires Node + cmds: + # Same shape as test:forge-fake-contract: the launcher starts the fake with + # FORGE_FAKE_ISSUE_CONTRACT=1 and a self-signed cert, then runs this exact Go + # argv from api. It allows only the YouTrack-only subtests to skip. + - env GIT_CONFIG_GLOBAL=/dev/null GIT_CONFIG_SYSTEM=/dev/null node e2e/forge-fake/issue-tracker-contracts.test.mjs -- go test -buildvcs=false -count=1 -v -timeout=60s -run='^TestIssueTrackerForgeFakeContracts$' ./internal/issuetracker + test:create-gitlab-bot: desc: hermetic bot-provisioning contract for create-gitlab-bot.sh (fake glab, no network), tally-guarded cmds: diff --git a/api/internal/issuetracker/forgefake_contract_test.go b/api/internal/issuetracker/forgefake_contract_test.go new file mode 100644 index 0000000000..7066ade4e9 --- /dev/null +++ b/api/internal/issuetracker/forgefake_contract_test.go @@ -0,0 +1,90 @@ +package issuetracker_test + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "net/http" + "os" + "testing" + "time" + + "github.com/vtmocanu/uzi/api/internal/forge" + "github.com/vtmocanu/uzi/api/internal/issuetracker/issuetrackertest" +) + +// forgeFakeProjectID is the project id of the Node fake's first project. +const forgeFakeProjectID = 1 + +// TestIssueTrackerForgeFakeContracts runs the shared issue tracker contract +// against the real GitLab, GitHub and Forgejo drivers over the Node HTTPS fake +// (e2e/forge-fake/forge-fake.mjs). The launcher +// e2e/forge-fake/issue-tracker-contracts.test.mjs starts the fake with +// FORGE_FAKE_ISSUE_CONTRACT=1 and this process with SSL_CERT_FILE set before +// TLS roots load. +func TestIssueTrackerForgeFakeContracts(t *testing.T) { + switch os.Getenv("UZI_FORGE_FAKE_REQUIRED") { + case "": + t.Skip("standalone contract requires task test:issue-tracker-forge-fake-contract") + case "1": + default: + t.Fatal("invalid required-mode marker") + } + endpoint := os.Getenv("UZI_FORGE_FAKE_ENDPOINT") + if endpoint == "" { + t.Fatal("UZI_FORGE_FAKE_ENDPOINT is required when UZI_FORGE_FAKE_REQUIRED=1") + } + for _, dialect := range []forge.Type{forge.TypeGitLab, forge.TypeGitHub, forge.TypeForgejo} { + t.Run(string(dialect), func(t *testing.T) { + issuetrackertest.RunContract(t, func(t *testing.T) issuetrackertest.Subject { + // The fake does not check auth; the value only has to be non-empty. + driver, err := forge.New(dialect, endpoint, "contract-"+"fixture", 5*time.Second) + if err != nil { + t.Fatalf("forge.New(%s): %v", dialect, err) + } + return issuetrackertest.Subject{ + IssueTracker: driver, + ProjectID: forgeFakeProjectID, + SeedHumanComment: func(t *testing.T, iid int64, username, body string) { + postFake(t, endpoint, fmt.Sprintf("/_e2e/issues/%d/comments", iid), + map[string]any{"username": username, "body": body}) + }, + SeedLabelEvent: func(t *testing.T, iid int64, action, label, username string) { + postFake(t, endpoint, fmt.Sprintf("/_e2e/issues/%d/label-events", iid), + map[string]any{"action": action, "label": label, "username": username}) + }, + } + }) + }) + } +} + +// postFake POSTs a JSON body to one of the fake's /_e2e mutators and fails the +// test unless it answers 201. +func postFake(t *testing.T, endpoint, path string, body any) { + t.Helper() + raw, err := json.Marshal(body) + if err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithTimeout(t.Context(), 5*time.Second) + defer cancel() + req, err := http.NewRequestWithContext(ctx, http.MethodPost, endpoint+path, bytes.NewReader(raw)) + if err != nil { + t.Fatal(err) + } + req.Header.Set("Content-Type", "application/json") + resp, err := http.DefaultClient.Do(req) + if err != nil { + t.Fatalf("POST %s: %v", path, err) + } + defer func() { + if err := resp.Body.Close(); err != nil { + t.Errorf("close response body: %v", err) + } + }() + if resp.StatusCode != http.StatusCreated { + t.Fatalf("POST %s = %d, want 201", path, resp.StatusCode) + } +} diff --git a/e2e/forge-fake/forge-fake.mjs b/e2e/forge-fake/forge-fake.mjs index b986d8700a..06842f82f6 100644 --- a/e2e/forge-fake/forge-fake.mjs +++ b/e2e/forge-fake/forge-fake.mjs @@ -93,6 +93,21 @@ const GIT_HTTP_BACKEND = `${GIT_EXEC_PATH}/git-http-backend`; const EXPECT_USER = process.env.FORGE_FAKE_EXPECT_USER || ""; const EXPECT_PAT = process.env.FORGE_FAKE_EXPECT_PAT || ""; +// FORGE_FAKE_ISSUE_CONTRACT=1 turns on the issue-tracker contract fidelity the +// default e2e fake deliberately lacks: a persisted label catalog on GitLab and +// GitHub, issue state/description writes on all three dialects, GitHub's per-label +// add/remove routes, and the /_e2e/issues/{iid}/comments mutator. Unset, every +// answer is byte-identical to before (e2e/phases/60-schedules-sweep.sh relies on +// GitLab GET /labels answering [] and POST not persisting). +const ISSUE_CONTRACT = process.env.FORGE_FAKE_ISSUE_CONTRACT === "1"; +// name -> {id, name, color}; only read or written when ISSUE_CONTRACT is on. +const labelCatalog = /** @type {Record} */ ({}); +let nextCatalogLabelId = 300; +function catalogLabel(name, color) { + if (!labelCatalog[name]) labelCatalog[name] = { id: nextCatalogLabelId++, name, color }; + return labelCatalog[name]; +} + const state = { issues: /** @type {Record} */ ({}), mrs: /** @type {any[]} */ ([]), @@ -864,6 +879,25 @@ const server = https.createServer( return send(res, 201, issue); } + // Seed a human comment on an issue, visible to all three dialects' comment + // routes (they share state.notes). Only under FORGE_FAKE_ISSUE_CONTRACT=1. + const commentEv = ISSUE_CONTRACT && method === "POST" && path.match(/^\/_e2e\/issues\/(\d+)\/comments$/); + if (commentEv) { + const body = await readBody(req); + const iid = Number(commentEv[1]); + const issue = state.issues[iid]; + if (!issue) return send(res, 404, { message: "404 Not found (no such issue)" }); + const note = { + id: state.nextNoteId++, project_id: issue.project_id, issue_iid: iid, + body: String(body.body || ""), created_at: new Date().toISOString(), system: false, + author: { id: Number(body.id) || 2, username: String(body.username || "reviewer") }, + }; + state.notes.push(note); + persist(); + log("_e2e comment on issue", iid, "by", note.author.username); + return send(res, 201, note); + } + // Append a resource label event to an issue — the harness stand-in for a // human adding/removing a label. The rising id models GitLab's globally- // monotonic event ids, so a remove+re-add mints a larger id (the autopilot @@ -896,8 +930,8 @@ const server = https.createServer( // // MEASURED before adding it: makeIssue stamps state:"opened" and NOTHING else in // this file assigns issue.state — the GitLab PUT /issues/{iid} handler applies - // only add_labels/remove_labels, by design, and the Forgejo lane's PATCH does the - // same. So this is a new capability, not a second way to do an existing one. + // only add_labels/remove_labels by default, and the Forgejo lane has no issue PATCH + // (both gain state writes only under FORGE_FAKE_ISSUE_CONTRACT=1). So this is a new capability, not a second way to do an existing one. // // LANE-NEUTRAL BY CONSTRUCTION: it mutates the shared state.issues store, and the // Forgejo lane serves the same objects through toForgejoIssue (which maps @@ -1195,6 +1229,18 @@ const server = https.createServer( const issue = state.issues[Number(fjIssueGet[1])]; return issue ? send(res, 200, toForgejoIssue(issue)) : send(res, 404, { message: "issue does not exist" }); } + if (ISSUE_CONTRACT && method === "PATCH" && fjIssueGet) { + // EditIssue: body and state ("open"/"closed"), applied only under the contract flag. + const issue = state.issues[Number(fjIssueGet[1])]; + if (!issue) return send(res, 404, { message: "issue does not exist" }); + const body = await readBody(req); + if (typeof body.body === "string") issue.description = body.body; + if (body.state === "open") issue.state = "opened"; + if (body.state === "closed") issue.state = "closed"; + issue.updated_at = new Date().toISOString(); + persist(); + return send(res, 201, toForgejoIssue(issue)); + } if (method === "POST" && rest === "/issues") { // CreateIssue: body {title, body, labels:[ids]}. Map ids -> names. const body = await readBody(req); @@ -1540,6 +1586,8 @@ const server = https.createServer( if (!issue) return send(res, 404, { message: "Not Found" }); const body = await readBody(req); if (typeof body.body === "string") issue.description = body.body; + if (ISSUE_CONTRACT && body.state === "open") issue.state = "opened"; + if (ISSUE_CONTRACT && body.state === "closed") issue.state = "closed"; issue.updated_at = new Date().toISOString(); persist(); log("gh issue", issue.iid, "description updated"); @@ -1559,6 +1607,30 @@ const server = https.createServer( log("gh issue", issue.iid, "labels ->", JSON.stringify(issue.labels)); return send(res, 200, names.map((n) => ({ id: 1000 + issue.iid, name: n, color: "ededed" }))); } + if (ISSUE_CONTRACT && ghIssueLabels && method === "POST") { + // AddLabelsToIssue: a bare JSON array of names; add, deduplicated. + const issue = state.issues[Number(ghIssueLabels[1])]; + if (!issue) return send(res, 404, { message: "Not Found" }); + const body = await readBody(req); + const names = Array.isArray(body) ? body.map(String) : normLabels(body.labels); + const next = [...(issue.labels || [])]; + for (const n of names) if (!next.includes(n)) next.push(n); + issue.labels = next; + issue.updated_at = new Date().toISOString(); + persist(); + return send(res, 200, next.map((n) => ({ id: 1000 + issue.iid, name: n, color: "ededed" }))); + } + const ghIssueLabel = ISSUE_CONTRACT && method === "DELETE" && rest.match(/^\/issues\/(\d+)\/labels\/(.+)$/); + if (ghIssueLabel) { + // RemoveLabelForIssue: remove one label by name, answer the remaining set. + const issue = state.issues[Number(ghIssueLabel[1])]; + if (!issue) return send(res, 404, { message: "Not Found" }); + const name = decodeURIComponent(ghIssueLabel[2]); + issue.labels = (issue.labels || []).filter((l) => l !== name); + issue.updated_at = new Date().toISOString(); + persist(); + return send(res, 200, issue.labels.map((n) => ({ id: 1000 + issue.iid, name: n, color: "ededed" }))); + } // Issue comments (CreateIssueNote). Recorded in the shared notes list so the // harness reads them back via /_e2e/state exactly like the other lanes. const ghComments = rest.match(/^\/issues\/(\d+)\/comments$/); @@ -1597,10 +1669,15 @@ const server = https.createServer( if (method === "GET") { // Defensive: the harness critical path does not depend on a pre-seeded // catalog (CreateIssue auto-creates a referenced label on GitHub). + if (ISSUE_CONTRACT) return send(res, 200, Object.values(labelCatalog)); return send(res, 200, []); } if (method === "POST") { const body = await readBody(req); + if (ISSUE_CONTRACT) { + const color = (body.color || "ededed").replace(/^#/, ""); + return send(res, 201, catalogLabel(String(body.name || "label"), color)); + } return send(res, 201, { id: 1, name: body.name || "label", color: (body.color || "ededed").replace(/^#/, "") }); } } @@ -1854,6 +1931,11 @@ const server = https.createServer( const next = (issue.labels || []).filter((l) => !remove.has(l)); for (const l of normLabels(body.add_labels)) if (!next.includes(l)) next.push(l); issue.labels = next; + if (ISSUE_CONTRACT) { + if (body.state_event === "close") issue.state = "closed"; + if (body.state_event === "reopen") issue.state = "opened"; + if (typeof body.description === "string") issue.description = body.description; + } issue.updated_at = new Date().toISOString(); persist(); log("issue", issue.iid, "labels ->", JSON.stringify(issue.labels)); @@ -1948,9 +2030,10 @@ const server = https.createServer( // Labels. On the harness critical path: enabling a repo (the startup seed, // the web enable, /server setup) creates the board column labels first, and // a refusal here keeps the repo off (PRD #3760). - if (method === "GET" && rest === "/labels") return send(res, 200, []); + if (method === "GET" && rest === "/labels") return send(res, 200, ISSUE_CONTRACT ? Object.values(labelCatalog) : []); if (method === "POST" && rest === "/labels") { const body = await readBody(req); + if (ISSUE_CONTRACT) return send(res, 201, catalogLabel(String(body.name || "label"), body.color || "#cccccc")); return send(res, 201, { id: 1, name: body.name || "label", color: body.color || "#cccccc" }); } diff --git a/e2e/forge-fake/issue-tracker-contracts.test.mjs b/e2e/forge-fake/issue-tracker-contracts.test.mjs new file mode 100644 index 0000000000..f5bf51a8f1 --- /dev/null +++ b/e2e/forge-fake/issue-tracker-contracts.test.mjs @@ -0,0 +1,142 @@ +import assert from "node:assert/strict"; +import { execFileSync, spawn } from "node:child_process"; +import { once } from "node:events"; +import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import path from "node:path"; +import test, { run as runTests } from "node:test"; +import { tap } from "node:test/reporters"; +import { fileURLToPath } from "node:url"; + +// Runs the shared issue tracker contract (issuetrackertest.RunContract) against +// the real GitLab, GitHub and Forgejo drivers over the Node HTTPS fake started +// with FORGE_FAKE_ISSUE_CONTRACT=1. A separate launcher from +// operation-contracts.test.mjs because that one pins a single Go argv. +const root = fileURLToPath(new URL("../../", import.meta.url)); +const separator = process.argv.indexOf("--"); +const command = process.argv.slice(separator + 1); +assert.ok(separator >= 2, "precondition: pass the Go command after --"); +assert.equal(command[0], "go", "precondition: expected go executable"); +assert.equal(command[1], "test"); +const flags = command.slice(2, -1); +assert.equal(command.at(-1), "./internal/issuetracker"); +for (const pattern of [/^-buildvcs=false$/, /^-count=1$/, /^-v$/, /^-timeout=[1-9]\d*s$/, /^-run=\^TestIssueTrackerForgeFakeContracts\$$/]) { + assert.equal(flags.filter((flag) => pattern.test(flag)).length, 1, `precondition: Go command missing/duplicated ${pattern}`); +} +assert.equal(flags.length, 5); +console.log("# Go command:", command.join(" ")); + +function run(executable, args) { + try { + return execFileSync(executable, args, { cwd: root, encoding: "utf8", timeout: 10000, stdio: "pipe" }).trim(); + } catch (error) { + throw new Error(`precondition/fixture command failed: ${executable} ${args.join(" ")}\n${error.stderr ?? ""}`, { cause: error }); + } +} + +// Only owned children are signalled. TERM has a two-second grace, then KILL and reap. +async function stop(child) { + if (!child || child.exitCode !== null || child.signalCode !== null) return; + const closed = once(child, "close"); + child.kill("SIGTERM"); + const timer = setTimeout(() => child.kill("SIGKILL"), 2000); + try { await closed; } finally { clearTimeout(timer); } +} + +// The only subtests that may skip: YouTrack-only behaviours the forge dialects +// do not have (an issue that moves project, a deletable comment). +const ALLOWED_SKIPS = new Set(["MovedIssueIsRefused"]); +const CASES = [ + "EnsureLabelsIsIdempotent", "CreateIssueRoundTrips", "ListIssuesFilters", "UpdateLabelsAndDescription", + "SetIssueStateClosesAndReopens", "CommentsOldestFirst", "LabelEventsOldestFirst", "MissingIssueIsNotFound", "MovedIssueIsRefused", +]; +const DIALECTS = ["gitlab", "github", "forgejo"]; + +async function contracts(t) { + console.log("#", run("go", ["version"])); + console.log("#", run("openssl", ["version"])); + mkdirSync(path.join(root, ".uzi/scratch"), { recursive: true }); + const scratch = mkdtempSync(path.join(root, ".uzi/scratch/issue-tracker-contracts-")); + let fake; + let go; + let diagnostics = ""; + t.after(async () => { + await stop(go); + await stop(fake); + rmSync(scratch, { recursive: true, force: true }); + }); + const cert = path.join(scratch, "cert.pem"); + const key = path.join(scratch, "key.pem"); + run("openssl", ["req", "-x509", "-newkey", "rsa:2048", "-nodes", "-days", "1", "-subj", "/CN=localhost", "-addext", "subjectAltName=IP:127.0.0.1", "-keyout", key, "-out", cert]); + const gitRoot = path.join(scratch, "gitroot"); + mkdirSync(gitRoot); + const statePath = path.join(scratch, "state.json"); + writeFileSync(statePath, "{}"); + fake = spawn(process.execPath, [path.join(root, "e2e/forge-fake/forge-fake.mjs")], { + cwd: root, env: { PATH: process.env.PATH, FORGE_FAKE_PORT: "0", FORGE_FAKE_CERT: cert, FORGE_FAKE_KEY: key, + FORGE_FAKE_STATE: statePath, FORGE_FAKE_BASE_URL: "https://configured.invalid", + FORGE_FAKE_PROJECT: "group/repo", FORGE_FAKE_GIT_ROOT: gitRoot, FORGE_FAKE_ISSUE_CONTRACT: "1" }, + stdio: ["ignore", "pipe", "pipe"], + }); + fake.stdout.on("data", (data) => { diagnostics = (diagnostics + data).slice(-65536); }); + fake.stderr.on("data", (data) => { diagnostics = (diagnostics + data).slice(-65536); }); + const port = await new Promise((resolve, reject) => { + const timer = setTimeout(() => finish(new Error(`fake startup timed out\n${diagnostics}`)), 10000); + const onData = () => { const match = diagnostics.match(/listening on :(\d+)/); if (match) finish(null, Number(match[1])); }; + const onExit = (code, signal) => finish(new Error(`fake startup exit ${code}/${signal}\n${diagnostics}`)); + const onError = (error) => finish(error); + function finish(error, value) { + clearTimeout(timer); + fake.stdout.off("data", onData); + fake.off("exit", onExit); + fake.off("error", onError); + if (error) reject(error); else resolve(value); + } + fake.stdout.on("data", onData); + fake.once("exit", onExit); + fake.once("error", onError); + onData(); + }); + const endpoint = `https://127.0.0.1:${port}`; + + await t.test("Go issue tracker contract over trusted HTTPS", async () => { + let output = ""; + const started = performance.now(); + go = spawn(command[0], command.slice(1), { cwd: path.join(root, "api"), + env: { ...process.env, SSL_CERT_FILE: cert, HTTPS_PROXY: "", HTTP_PROXY: "", ALL_PROXY: "", NO_PROXY: "127.0.0.1", + UZI_FORGE_FAKE_REQUIRED: "1", UZI_FORGE_FAKE_ENDPOINT: endpoint }, + stdio: ["ignore", "pipe", "pipe"] }); + for (const stream of [go.stdout, go.stderr]) stream.on("data", (data) => { + output += data; + if (output.length > 2 * 1024 * 1024) go.kill("SIGKILL"); + }); + const timer = setTimeout(() => { console.error("Go child exceeded 240s including compilation"); go.kill("SIGKILL"); }, 240000); + let code; + try { [code] = await once(go, "close"); } finally { clearTimeout(timer); } + console.log(output.trim().split("\n").map((line) => "# " + line).join("\n")); + console.log(`# Go child wall ${((performance.now() - started) / 1000).toFixed(2)}s (includes compilation)`); + assert.equal(code, 0, `Go contract failed\n${output}\nfake diagnostics:\n${diagnostics}`); + assert.match(output, /^=== RUN\s+TestIssueTrackerForgeFakeContracts$/m); + assert.match(output, /^--- PASS: TestIssueTrackerForgeFakeContracts /m); + for (const dialect of DIALECTS) { + for (const name of CASES) { + const full = `TestIssueTrackerForgeFakeContracts/${dialect}/${name}`; + const escaped = full.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); + assert.match(output, new RegExp(`^\\s*--- (PASS|SKIP): ${escaped} `, "m"), `subtest did not run: ${full}`); + if (!ALLOWED_SKIPS.has(name)) { + assert.match(output, new RegExp(`^\\s*--- PASS: ${escaped} `, "m"), `subtest must pass, not skip: ${full}`); + } + } + } + for (const line of output.split("\n").filter((l) => /--- SKIP:/.test(l))) { + const match = line.match(/--- SKIP: TestIssueTrackerForgeFakeContracts\/(gitlab|github|forgejo)\/(\w+) /); + assert.ok(match && ALLOWED_SKIPS.has(match[2]), `unexpected skip: ${line.trim()}`); + } + }); +} + +// Select TAP explicitly: Node versions differ in their default reporter. +const results = runTests({ files: [], isolation: "none", setup() { + test("issue tracker contract against the forge fake over HTTPS", { timeout: 330000 }, contracts); +} }); +results.on("test:fail", () => { process.exitCode = 1; }); +results.compose(tap).pipe(process.stdout); From fdfcb56125e6dd8f968a55e15e2955deb80722fe Mon Sep 17 00:00:00 2001 From: uzi-agent Date: Wed, 11 Nov 2026 11:10:30 +0000 Subject: [PATCH 15/17] docs: PRD #4173 M3 delivered; decision log and changelog fragment Tick M3 with its verification, record the M3 choices in the Decision Log, add the changelog fragment, and correct two forge-fake comments. Co-Authored-By: Claude Opus 5.5 --- .../a8164ae4-6df4-477e-8255-2d74987f2aa4.md | 4 ++++ e2e/forge-fake/forge-fake.mjs | 2 +- e2e/forge-fake/issue-tracker-contracts.test.mjs | 4 ++-- prds/4173-external-issue-trackers.md | 16 ++++++++++++++-- 4 files changed, 21 insertions(+), 5 deletions(-) create mode 100644 changelog.d/a8164ae4-6df4-477e-8255-2d74987f2aa4.md diff --git a/changelog.d/a8164ae4-6df4-477e-8255-2d74987f2aa4.md b/changelog.d/a8164ae4-6df4-477e-8255-2d74987f2aa4.md new file mode 100644 index 0000000000..f088fd3370 --- /dev/null +++ b/changelog.d/a8164ae4-6df4-477e-8255-2d74987f2aa4.md @@ -0,0 +1,4 @@ +### Added + +- **Issue tracker connections and a YouTrack driver, not yet reachable from any repo (#4173).** + A YouTrack driver for every issue-side operation, one contract suite run against it and the GitLab, GitHub and Forgejo drivers, and issue tracker connections stored with the token sealed and an https base URL on the separate `ISSUE_TRACKER_ALLOWED_BASE_URLS` allowlist (empty by default, so none can be created yet); behaviour is unchanged for every forge. diff --git a/e2e/forge-fake/forge-fake.mjs b/e2e/forge-fake/forge-fake.mjs index 06842f82f6..e77bfe1492 100644 --- a/e2e/forge-fake/forge-fake.mjs +++ b/e2e/forge-fake/forge-fake.mjs @@ -94,7 +94,7 @@ const EXPECT_USER = process.env.FORGE_FAKE_EXPECT_USER || ""; const EXPECT_PAT = process.env.FORGE_FAKE_EXPECT_PAT || ""; // FORGE_FAKE_ISSUE_CONTRACT=1 turns on the issue-tracker contract fidelity the -// default e2e fake deliberately lacks: a persisted label catalog on GitLab and +// default e2e fake deliberately lacks: an in-memory label catalog on GitLab and // GitHub, issue state/description writes on all three dialects, GitHub's per-label // add/remove routes, and the /_e2e/issues/{iid}/comments mutator. Unset, every // answer is byte-identical to before (e2e/phases/60-schedules-sweep.sh relies on diff --git a/e2e/forge-fake/issue-tracker-contracts.test.mjs b/e2e/forge-fake/issue-tracker-contracts.test.mjs index f5bf51a8f1..fe70dc15e4 100644 --- a/e2e/forge-fake/issue-tracker-contracts.test.mjs +++ b/e2e/forge-fake/issue-tracker-contracts.test.mjs @@ -42,8 +42,8 @@ async function stop(child) { try { await closed; } finally { clearTimeout(timer); } } -// The only subtests that may skip: YouTrack-only behaviours the forge dialects -// do not have (an issue that moves project, a deletable comment). +// The only subtest that may skip: the forge subjects supply no MoveIssue seam, +// so the moved-issue case runs on YouTrack only. const ALLOWED_SKIPS = new Set(["MovedIssueIsRefused"]); const CASES = [ "EnsureLabelsIsIdempotent", "CreateIssueRoundTrips", "ListIssuesFilters", "UpdateLabelsAndDescription", diff --git a/prds/4173-external-issue-trackers.md b/prds/4173-external-issue-trackers.md index f0ad3c86ed..02578b62fb 100644 --- a/prds/4173-external-issue-trackers.md +++ b/prds/4173-external-issue-trackers.md @@ -2,7 +2,7 @@ **Issue:** #4173 **Priority:** Medium -**Status:** In progress, r3 2026-11-11. M1 and M2 delivered. r3 follows the maintainer's scope review: one real issue tracker instead of abstract identity work, and the comment, command and withdrawal behaviours leave this PRD. +**Status:** In progress, r3 2026-11-11. M1, M2 and M3 delivered. r3 follows the maintainer's scope review: one real issue tracker instead of abstract identity work, and the comment, command and withdrawal behaviours leave this PRD. **Delivery:** one PR per milestone, each merged before the next starts. No implementation or validation step touches `.github/workflows/**`. **Builds on:** @@ -101,7 +101,7 @@ Progress (one PR per milestone, each merged before the next starts): - [x] **M1:** the `IssueTracker` and `CodeHost` interfaces - [x] **M2:** every caller on its half. Verified in the run: the type-checked inventory shows 0 issue-side calls through `forge.Forge` and 45 through `IssueTracker` (plus `GetIssueBounded` and `EnsureBoardColumns`), and handing `privcheck` an `IssueTracker`-only value fails to compile. `task gate:api` slots green except `deadcode:api` (not run). -- [ ] **M3:** issue tracker connections and the YouTrack driver +- [x] **M3:** issue tracker connections and the YouTrack driver. Verified in the run: the contract suite passes on the YouTrack fake (`test:api`) and on the GitLab, GitHub and Forgejo drivers over the Node forge fake (`task test:issue-tracker-forge-fake-contract`, a `gate:repo` member; only the moved-issue case skips there); with the allowlist empty `Connect` refuses before a driver is built and `config.Load` succeeds; a redirect, an oversized body and an error carrying the token are each refused or scrubbed in a test. `task gate:api` slots green except `deadcode:api` (not run). - [ ] **M4:** issue identity, the repo link and issue intake from YouTrack - [ ] **M5:** the board and issue writes on YouTrack - [ ] **M6:** web and CLI surfaces, docs, ADR and live acceptance @@ -169,3 +169,15 @@ Settings → Issue trackers, the CLI commands, a setup page under `docs/`, an AD - **2026-11-10: the PRD link patch takes both halves.** D3 placed it on `IssueTracker`, but it also reads the merge request (`GetMergeRequest`), so `SyncPRDLinkPatches` takes `(ch forge.CodeHost, tr forge.IssueTracker)`. Comment intake (`issueinput.Fetch`), the autopilot, CI-autofix and MR-rework detectors, the MR state watcher, self-improve and the worker get-issue route are mixed the same way. - **2026-11-10: "`privcheck` compiles against `CodeHost` only" means every privcheck parameter, local and forge call is `CodeHost`.** Its `ForgeBuilder` factory interface still returns `forge.Forge`, converted to `CodeHost` straight after the build; handing privcheck an `IssueTracker`-only value is a compile error. - **2026-11-10: scope after the maintainer's review.** YouTrack becomes part of this PRD and drives the identity design. The living comment, the outbox, withdrawal, comments to the agent and `/uzi` commands move out. Asana follows in its own PRD. +- **2026-11-11: M3 layout.** Drivers in `api/internal/issuetracker` (`Driver` = `forge.IssueTracker` plus `VerifyToken`, `ListProjects`, `ListStates`; one kind constant and one registry entry per issue tracker); the loud fake `issuetrackertest.BaseFake`, the contract suite `issuetrackertest.RunContract` and the in-process HTTPS YouTrack fake `youtracktest`; connections in `api/internal/issuetrackersvc`, not wired into `cmd/server` until a handler exists (M6). +- **2026-11-11: "the three forge fakes" are the Node fake `e2e/forge-fake/forge-fake.mjs`** (owner answer). The suite runs the real forge drivers against it in the `gate:repo` task `test:issue-tracker-forge-fake-contract`; the fake's extra issue routes answer only under `FORGE_FAKE_ISSUE_CONTRACT=1`. The moved-issue case skips there, because the forge subjects supply no way to move an issue; every other case must pass. +- **2026-11-11: a YouTrack driver is bound to one YouTrack project when it is built.** The `projectID int64` argument of `forge.IssueTracker` is ignored, since M4's `IssueTrackerForRepo` returns a driver for the repo's linked project while callers keep passing the forge project id. An unbound driver refuses issue-side calls with `ErrProjectNotBound`. +- **2026-11-11: the issue number is YouTrack's `numberInProject` (D10's candidate), and every per-issue call first checks the issue is still in the bound project.** YouTrack still resolves an old `SHORT-n` after a move, so a moved issue reads as not found and is never written to or read. YouTrack has no conditional write, so a move between that check and the write is not caught. M4 still owns the identity storage. +- **2026-11-11: labels are YouTrack tags,** global per account. `ListLabels` lists the tags visible to the bot, once per name; `EnsureLabels` creates missing ones; `Label.Color` is ignored. `UpdateIssueLabels` is one call per tag (adds, then removes), so not atomic; the commands API is not used, so no name enters a command query. `CreateIssue` adds tags after creating the issue and does not roll back if that fails. +- **2026-11-11: closing and reopening use explicit State values from the driver options;** unset, `SetIssueState` returns `ErrStateMappingUnset` and never guesses. M5 owns the column and state mapping and the "never won't fix" rule. +- **2026-11-11: YouTrack account ids are strings, so the numeric author ids are 0 ("unresolved");** D11 withholds such comments anyway. `LabelEvent.ID` is the oldest-first ordinal, `IssueNote.ID` the numeric part of the comment id. +- **2026-11-11: issue tracker base URLs are scheme and host only,** https, ASCII host, no path, userinfo, query or fragment (refused, not stripped), so a YouTrack under a context path is not supported yet. The allowlist is `ISSUE_TRACKER_ALLOWED_BASE_URLS`, empty by default; it is documented with the M6 setup page. `Connect` checks the strict form before the allowlist, so the allowlist is always compared against the canonical value that is stored. +- **2026-11-11: the token is sealed with AAD `uzi/issue-tracker-token/v1|||`,** so a ciphertext copied to another row does not open; the canonical base URL is frozen at v1 for that reason. The token must be 16 to 4096 bytes with no control or invisible character. There is no least-privilege check like the forge's: YouTrack permanent tokens expose no scopes. A credential reset does not delete issue tracker tokens, as for forge connections; recorded as a residual risk. `DriverFor` re-checks the allowlist before opening a stored token. +- **2026-11-11: a hostile or compromised allowlisted YouTrack cannot make the driver allocate far beyond what it sent.** Responses are capped (8 MiB a list page, 2 MiB an object, 4 KiB an error body, 64 MiB a listing); a page holds at most the requested 100 items; nested tags are capped at 1000 per item and count toward the listing's item ceiling; a byte scan bounds the JSON values before decoding; list pages decode one element at a time and stop at the first error. The worst case found is about 34 MiB for an 8 MiB page, against about 13 MiB for the largest legitimate one. +- **2026-11-11: left for M6's live pass** (taken from YouTrack's documentation, checked only against the fake): the `TagsCategory` activity category, the `updated:` query date format, and that tag names can repeat across owners. +- **2026-11-11: migration `00424_issue_tracker_connections`** is the next free number on `prd/ws-2`; it is renumbered at upstream landing. Rolling it back deletes every stored issue tracker token. From f536964467db868cf2e32433c73b7841eb00b0b0 Mon Sep 17 00:00:00 2001 From: uzi-agent Date: Wed, 11 Nov 2026 11:31:17 +0000 Subject: [PATCH 16/17] issuetracker: clear the lint:api findings errcheck on deferred closes and test helpers, staticcheck struct conversions and the embedded selector, and reasoned gosec suppressions where a test deliberately redirects or posts to a local fake; plus three Decision Log and one Taskfile wording corrections. Co-Authored-By: Claude Opus 5.5 --- Taskfile.yml | 2 +- api/internal/issuetracker/bounded_test.go | 2 +- .../issuetracker/connection_service_test.go | 2 +- .../issuetracker/forgefake_contract_test.go | 4 +-- api/internal/issuetracker/httpclient_test.go | 4 +-- api/internal/issuetracker/youtrack_auth.go | 2 +- .../issuetracker/youtrack_guards_test.go | 2 +- .../issuetracker/youtrack_projects.go | 2 +- api/internal/issuetracker/youtrack_request.go | 2 +- .../issuetracker/youtrack_security_test.go | 2 +- .../issuetracker/youtracktest/server.go | 2 +- api/internal/issuetrackersvc/service_test.go | 34 +++++++++---------- prds/4173-external-issue-trackers.md | 6 ++-- 13 files changed, 33 insertions(+), 33 deletions(-) diff --git a/Taskfile.yml b/Taskfile.yml index 41ebc200a1..f4537bd457 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -1222,7 +1222,7 @@ tasks: cmds: # Same shape as test:forge-fake-contract: the launcher starts the fake with # FORGE_FAKE_ISSUE_CONTRACT=1 and a self-signed cert, then runs this exact Go - # argv from api. It allows only the YouTrack-only subtests to skip. + # argv from api. It allows only the moved-issue subtest to skip (the forge subjects supply no MoveIssue seam). - env GIT_CONFIG_GLOBAL=/dev/null GIT_CONFIG_SYSTEM=/dev/null node e2e/forge-fake/issue-tracker-contracts.test.mjs -- go test -buildvcs=false -count=1 -v -timeout=60s -run='^TestIssueTrackerForgeFakeContracts$' ./internal/issuetracker test:create-gitlab-bot: diff --git a/api/internal/issuetracker/bounded_test.go b/api/internal/issuetracker/bounded_test.go index 82d818a1b8..d190a44e1b 100644 --- a/api/internal/issuetracker/bounded_test.go +++ b/api/internal/issuetracker/bounded_test.go @@ -55,7 +55,7 @@ func TestReadBoundedCapsDecompressedBytes(t *testing.T) { if err != nil { t.Fatal(err) } - defer resp.Body.Close() + defer func() { _ = resp.Body.Close() }() if resp.ContentLength != -1 { t.Fatalf("declared length %d", resp.ContentLength) } diff --git a/api/internal/issuetracker/connection_service_test.go b/api/internal/issuetracker/connection_service_test.go index c04b0aa045..4b455a4a56 100644 --- a/api/internal/issuetracker/connection_service_test.go +++ b/api/internal/issuetracker/connection_service_test.go @@ -170,7 +170,7 @@ func TestConnectRefusesRedirect(t *testing.T) { t.Cleanup(other.Close) srv := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { primary.Add(1) - http.Redirect(w, r, other.URL+r.URL.Path, http.StatusFound) + http.Redirect(w, r, other.URL+r.URL.Path, http.StatusFound) //nolint:gosec // G710: test redirect to a second httptest server, the redirect-refusal behaviour under test })) t.Cleanup(srv.Close) h := newSvcHarness(t, srv.Client().Transport, allowOnly(srv.URL)) diff --git a/api/internal/issuetracker/forgefake_contract_test.go b/api/internal/issuetracker/forgefake_contract_test.go index 7066ade4e9..e4fb9b8b60 100644 --- a/api/internal/issuetracker/forgefake_contract_test.go +++ b/api/internal/issuetracker/forgefake_contract_test.go @@ -70,12 +70,12 @@ func postFake(t *testing.T, endpoint, path string, body any) { } ctx, cancel := context.WithTimeout(t.Context(), 5*time.Second) defer cancel() - req, err := http.NewRequestWithContext(ctx, http.MethodPost, endpoint+path, bytes.NewReader(raw)) + req, err := http.NewRequestWithContext(ctx, http.MethodPost, endpoint+path, bytes.NewReader(raw)) //nolint:gosec // G704: endpoint is the local httptest fake's URL, never user input if err != nil { t.Fatal(err) } req.Header.Set("Content-Type", "application/json") - resp, err := http.DefaultClient.Do(req) + resp, err := http.DefaultClient.Do(req) //nolint:gosec // G704: request targets the local httptest fake only if err != nil { t.Fatalf("POST %s: %v", path, err) } diff --git a/api/internal/issuetracker/httpclient_test.go b/api/internal/issuetracker/httpclient_test.go index a6383993ea..a4452612b6 100644 --- a/api/internal/issuetracker/httpclient_test.go +++ b/api/internal/issuetracker/httpclient_test.go @@ -23,7 +23,7 @@ func TestClientDoesNotFollowRedirect(t *testing.T) { if err != nil { t.Fatal(err) } - defer resp.Body.Close() + defer func() { _ = resp.Body.Close() }() if !isRedirect(resp) { t.Fatalf("status %d not detected as redirect", resp.StatusCode) } @@ -53,7 +53,7 @@ func TestClientNotModifiedWithoutLocation(t *testing.T) { if err != nil { t.Fatal(err) } - defer resp.Body.Close() + defer func() { _ = resp.Body.Close() }() if !isRedirect(resp) { t.Fatal("304 not detected") } diff --git a/api/internal/issuetracker/youtrack_auth.go b/api/internal/issuetracker/youtrack_auth.go index a7cc3ce45c..0fceedd73b 100644 --- a/api/internal/issuetracker/youtrack_auth.go +++ b/api/internal/issuetracker/youtrack_auth.go @@ -21,5 +21,5 @@ func (y *youTrack) VerifyToken(ctx context.Context) (Account, error) { if err != nil { return Account{}, err } - return Account{ID: u.ID, Login: u.Login}, nil + return Account(u), nil } diff --git a/api/internal/issuetracker/youtrack_guards_test.go b/api/internal/issuetracker/youtrack_guards_test.go index 1aaaec27bb..e37332ac14 100644 --- a/api/internal/issuetracker/youtrack_guards_test.go +++ b/api/internal/issuetracker/youtrack_guards_test.go @@ -228,7 +228,7 @@ func TestResponseFailuresAreClassifiedAndRedacted(t *testing.T) { t.Run("read error on success", func(t *testing.T) { y, _ := loggedDriver(t, truncated(http.StatusOK)) _, err := y.GetIssue(context.Background(), 1, 1) - classified(t, err) + _ = classified(t, err) }) t.Run("read error on a server error", func(t *testing.T) { y, _ := loggedDriver(t, truncated(http.StatusInternalServerError)) diff --git a/api/internal/issuetracker/youtrack_projects.go b/api/internal/issuetracker/youtrack_projects.go index e8fface8a5..733af7d81e 100644 --- a/api/internal/issuetracker/youtrack_projects.go +++ b/api/internal/issuetracker/youtrack_projects.go @@ -16,7 +16,7 @@ func (y *youTrack) ListProjects(ctx context.Context) ([]Project, error) { if !entityIDRE.MatchString(p.ID) || !shortNameRE.MatchString(p.ShortName) { return nil, y.fail("youtrack: project with a malformed id in the response") } - out = append(out, Project{ID: p.ID, ShortName: p.ShortName, Name: p.Name}) + out = append(out, Project(p)) } return out, nil } diff --git a/api/internal/issuetracker/youtrack_request.go b/api/internal/issuetracker/youtrack_request.go index 871eeb00e7..dacacdfa13 100644 --- a/api/internal/issuetracker/youtrack_request.go +++ b/api/internal/issuetracker/youtrack_request.go @@ -71,7 +71,7 @@ func (y *youTrack) do(ctx context.Context, c ytCall, out any) (status, n int, er if err != nil { return 0, 0, y.red.error(err) } - defer resp.Body.Close() + defer func() { _ = resp.Body.Close() }() status = resp.StatusCode if isRedirect(resp) { return status, 0, y.red.sentinel(ErrRedirectRefused, status, fmt.Sprintf("%s %s answered HTTP %d", c.method, path, status)) diff --git a/api/internal/issuetracker/youtrack_security_test.go b/api/internal/issuetracker/youtrack_security_test.go index 01c33973e4..0c7ba2afe5 100644 --- a/api/internal/issuetracker/youtrack_security_test.go +++ b/api/internal/issuetracker/youtrack_security_test.go @@ -74,7 +74,7 @@ func TestRedirectsAreRefused(t *testing.T) { other := httptest.NewTLSServer(http.HandlerFunc(func(http.ResponseWriter, *http.Request) { hits.Add(1) })) t.Cleanup(other.Close) y, _ := tlsDriver(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - http.Redirect(w, r, other.URL+r.URL.Path, code) + http.Redirect(w, r, other.URL+r.URL.Path, code) //nolint:gosec // G710: test redirect to a second httptest server, the redirect-refusal behaviour under test })) ctx := context.Background() calls := map[string]func() error{ diff --git a/api/internal/issuetracker/youtracktest/server.go b/api/internal/issuetracker/youtracktest/server.go index 728817c9ad..82c0cbecd1 100644 --- a/api/internal/issuetracker/youtracktest/server.go +++ b/api/internal/issuetracker/youtracktest/server.go @@ -50,7 +50,7 @@ func NewServer(t testing.TB) *Server { s.mux = s.routes() s.Server = httptest.NewTLSServer(http.HandlerFunc(s.serve)) t.Cleanup(func() { - s.Server.Close() + s.Close() s.mu.Lock() defer s.mu.Unlock() if len(s.unmodelled) > 0 { diff --git a/api/internal/issuetrackersvc/service_test.go b/api/internal/issuetrackersvc/service_test.go index 68e4e65c21..b4a67e492e 100644 --- a/api/internal/issuetrackersvc/service_test.go +++ b/api/internal/issuetrackersvc/service_test.go @@ -156,7 +156,7 @@ func TestConnectRefusesWhenAllowlistEmpty(t *testing.T) { h := newHarness(t) h.allow = map[string]bool{} _, err := h.svc.Connect(context.Background(), uuid.New(), "youtrack", testHost, testToken("a", "b")) - wantKind(t, err, KindBadRequest) + _ = wantKind(t, err, KindBadRequest) if len(h.built) != 0 { t.Fatalf("driver factory called %d times for a refused host", len(h.built)) } @@ -189,7 +189,7 @@ func TestConnectRefusesBadInput(t *testing.T) { t.Run(tc.name, func(t *testing.T) { h := newHarness(t) _, err := h.svc.Connect(context.Background(), uuid.New(), tc.kind, tc.url, tc.token) - wantKind(t, err, KindBadRequest) + _ = wantKind(t, err, KindBadRequest) if len(h.built) != 0 { t.Error("driver built for refused input") } @@ -307,7 +307,7 @@ func TestConnectRefusesBadAccount(t *testing.T) { h := newHarness(t) h.template.account = acct _, err := h.svc.Connect(context.Background(), uuid.New(), "youtrack", testHost, testToken("t")) - wantKind(t, err, KindBadGateway) + _ = wantKind(t, err, KindBadGateway) if len(h.q.rows) != 0 { t.Error("row stored for a bad account") } @@ -393,21 +393,21 @@ func TestReadsAreOwnerScoped(t *testing.T) { t.Fatalf("owner get = %v, %v", got, err) } _, err = h.svc.Get(ctx, other, c.ID) - wantKind(t, err, KindNotFound) + _ = wantKind(t, err, KindNotFound) if list, err := h.svc.List(ctx, other); err != nil || len(list) != 0 { t.Errorf("foreign list = %v, %v", list, err) } if list, err := h.svc.List(ctx, owner); err != nil || len(list) != 1 { t.Errorf("owner list = %v, %v", list, err) } - wantKind(t, h.svc.Delete(ctx, other, c.ID), KindNotFound) + _ = wantKind(t, h.svc.Delete(ctx, other, c.ID), KindNotFound) if _, err := h.svc.Get(ctx, owner, c.ID); err != nil { t.Fatalf("foreign delete removed the row: %v", err) } if err := h.svc.Delete(ctx, owner, c.ID); err != nil { t.Fatal(err) } - wantKind(t, h.svc.Delete(ctx, owner, c.ID), KindNotFound) + _ = wantKind(t, h.svc.Delete(ctx, owner, c.ID), KindNotFound) } func TestDriverForOpensTokenAndPassesOptions(t *testing.T) { @@ -429,7 +429,7 @@ func TestDriverForOpensTokenAndPassesOptions(t *testing.T) { t.Errorf("built = %+v, want %+v", h.built, want) } _, err = h.svc.DriverFor(ctx, uuid.New(), c.ID, ref, "", "", "") - wantKind(t, err, KindNotFound) + _ = wantKind(t, err, KindNotFound) } func TestDriverForFailsClosedWhenAllowlistEmptied(t *testing.T) { @@ -443,13 +443,13 @@ func TestDriverForFailsClosedWhenAllowlistEmptied(t *testing.T) { h.built = nil h.allow = map[string]bool{} _, err = h.svc.DriverFor(ctx, user, c.ID, issuetracker.ProjectRef{}, "", "", "") - wantKind(t, err, KindBadRequest) + _ = wantKind(t, err, KindBadRequest) if len(h.built) != 0 { t.Error("driver built after the allowlist was emptied") } // Discovery goes through the same gate. _, err = h.svc.ListProjects(ctx, user, c.ID) - wantKind(t, err, KindBadRequest) + _ = wantKind(t, err, KindBadRequest) } func TestDriverForRefusesTamperedRow(t *testing.T) { @@ -473,7 +473,7 @@ func TestDriverForRefusesTamperedRow(t *testing.T) { h.allow["https://evil.example.com/x"] = true h.q.rows[0].BaseUrl = "https://evil.example.com/x" _, err = h.svc.DriverFor(ctx, user, c.ID, issuetracker.ProjectRef{}, "", "", "") - wantKind(t, err, KindBadRequest) + _ = wantKind(t, err, KindBadRequest) } func TestDriverForRefusesNonCanonicalStoredURL(t *testing.T) { @@ -489,7 +489,7 @@ func TestDriverForRefusesNonCanonicalStoredURL(t *testing.T) { h.allow["https://ISSUES.example.com"] = true h.q.rows[0].BaseUrl = "https://ISSUES.example.com" _, err = h.svc.DriverFor(ctx, user, c.ID, issuetracker.ProjectRef{}, "", "", "") - wantKind(t, err, KindBadRequest) + _ = wantKind(t, err, KindBadRequest) if len(h.built) != 0 { t.Error("driver built for a non-canonical stored URL") } @@ -506,7 +506,7 @@ func TestDriverForRefusesMalformedProjectRef(t *testing.T) { h.built = nil for _, ref := range []issuetracker.ProjectRef{{ID: "0-1"}, {ShortName: "UZI"}, {ID: "x", ShortName: "UZI"}, {ID: "0-1", ShortName: "1bad"}} { _, err = h.svc.DriverFor(ctx, user, c.ID, ref, "", "", "") - wantKind(t, err, KindBadRequest) + _ = wantKind(t, err, KindBadRequest) } if len(h.built) != 0 { t.Error("driver built for a malformed project reference") @@ -530,7 +530,7 @@ func TestListStatesUsesEmptyProjectRefAndValidatesID(t *testing.T) { h.built = nil for _, bad := range []string{"", "abc", "0-1/../x", "1-", "0-1\n"} { _, err = h.svc.ListStates(ctx, user, c.ID, bad) - wantKind(t, err, KindBadRequest) + _ = wantKind(t, err, KindBadRequest) } if len(h.built) != 0 { t.Error("driver built for a bad project id") @@ -552,12 +552,12 @@ func TestListProjectsAndStates(t *testing.T) { } h.template.projectErr = errors.New("boom") _, err = h.svc.ListProjects(ctx, user, c.ID) - wantKind(t, err, KindBadGateway) + _ = wantKind(t, err, KindBadGateway) // ListStates is not stubbed on the double: its error surfaces as a bad gateway, not a panic. _, err = h.svc.ListStates(ctx, user, c.ID, "0-1") - wantKind(t, err, KindBadGateway) + _ = wantKind(t, err, KindBadGateway) _, err = h.svc.ListStates(ctx, uuid.New(), c.ID, "0-1") - wantKind(t, err, KindNotFound) + _ = wantKind(t, err, KindNotFound) } func TestNewDefaultsToRegistry(t *testing.T) { @@ -585,7 +585,7 @@ func TestConnectRefusesAnInvisibleHostCharacterBeforeTheAllowlist(t *testing.T) asked := 0 h.svc.allowed = func(string) bool { asked++; return true } _, err := h.svc.Connect(context.Background(), uuid.New(), "youtrack", u, testToken("v")) - wantKind(t, err, KindBadRequest) + _ = wantKind(t, err, KindBadRequest) if asked != 0 || len(h.built) != 0 { t.Errorf("allowlist asked %d times, driver built %d times", asked, len(h.built)) } diff --git a/prds/4173-external-issue-trackers.md b/prds/4173-external-issue-trackers.md index 02578b62fb..3bcddbba0a 100644 --- a/prds/4173-external-issue-trackers.md +++ b/prds/4173-external-issue-trackers.md @@ -173,11 +173,11 @@ Settings → Issue trackers, the CLI commands, a setup page under `docs/`, an AD - **2026-11-11: "the three forge fakes" are the Node fake `e2e/forge-fake/forge-fake.mjs`** (owner answer). The suite runs the real forge drivers against it in the `gate:repo` task `test:issue-tracker-forge-fake-contract`; the fake's extra issue routes answer only under `FORGE_FAKE_ISSUE_CONTRACT=1`. The moved-issue case skips there, because the forge subjects supply no way to move an issue; every other case must pass. - **2026-11-11: a YouTrack driver is bound to one YouTrack project when it is built.** The `projectID int64` argument of `forge.IssueTracker` is ignored, since M4's `IssueTrackerForRepo` returns a driver for the repo's linked project while callers keep passing the forge project id. An unbound driver refuses issue-side calls with `ErrProjectNotBound`. - **2026-11-11: the issue number is YouTrack's `numberInProject` (D10's candidate), and every per-issue call first checks the issue is still in the bound project.** YouTrack still resolves an old `SHORT-n` after a move, so a moved issue reads as not found and is never written to or read. YouTrack has no conditional write, so a move between that check and the write is not caught. M4 still owns the identity storage. -- **2026-11-11: labels are YouTrack tags,** global per account. `ListLabels` lists the tags visible to the bot, once per name; `EnsureLabels` creates missing ones; `Label.Color` is ignored. `UpdateIssueLabels` is one call per tag (adds, then removes), so not atomic; the commands API is not used, so no name enters a command query. `CreateIssue` adds tags after creating the issue and does not roll back if that fails. +- **2026-11-11: labels are YouTrack tags,** global per account. `ListLabels` lists the tags visible to the token, once per name; `EnsureLabels` creates missing ones; `Label.Color` is ignored. `UpdateIssueLabels` is one call per tag (adds, then removes), so not atomic; the commands API is not used, so no name enters a command query. `CreateIssue` adds tags after creating the issue and does not roll back if that fails. - **2026-11-11: closing and reopening use explicit State values from the driver options;** unset, `SetIssueState` returns `ErrStateMappingUnset` and never guesses. M5 owns the column and state mapping and the "never won't fix" rule. - **2026-11-11: YouTrack account ids are strings, so the numeric author ids are 0 ("unresolved");** D11 withholds such comments anyway. `LabelEvent.ID` is the oldest-first ordinal, `IssueNote.ID` the numeric part of the comment id. -- **2026-11-11: issue tracker base URLs are scheme and host only,** https, ASCII host, no path, userinfo, query or fragment (refused, not stripped), so a YouTrack under a context path is not supported yet. The allowlist is `ISSUE_TRACKER_ALLOWED_BASE_URLS`, empty by default; it is documented with the M6 setup page. `Connect` checks the strict form before the allowlist, so the allowlist is always compared against the canonical value that is stored. +- **2026-11-11: issue tracker base URLs are scheme and host only,** https, ASCII host, no path other than `/`, no userinfo, query or fragment (refused, not stripped), so a YouTrack under a context path is not supported yet. The allowlist is `ISSUE_TRACKER_ALLOWED_BASE_URLS`, empty by default; it is documented with the M6 setup page. `Connect` checks the strict form before the allowlist, so the allowlist is always compared against the canonical value that is stored. - **2026-11-11: the token is sealed with AAD `uzi/issue-tracker-token/v1|||`,** so a ciphertext copied to another row does not open; the canonical base URL is frozen at v1 for that reason. The token must be 16 to 4096 bytes with no control or invisible character. There is no least-privilege check like the forge's: YouTrack permanent tokens expose no scopes. A credential reset does not delete issue tracker tokens, as for forge connections; recorded as a residual risk. `DriverFor` re-checks the allowlist before opening a stored token. -- **2026-11-11: a hostile or compromised allowlisted YouTrack cannot make the driver allocate far beyond what it sent.** Responses are capped (8 MiB a list page, 2 MiB an object, 4 KiB an error body, 64 MiB a listing); a page holds at most the requested 100 items; nested tags are capped at 1000 per item and count toward the listing's item ceiling; a byte scan bounds the JSON values before decoding; list pages decode one element at a time and stop at the first error. The worst case found is about 34 MiB for an 8 MiB page, against about 13 MiB for the largest legitimate one. +- **2026-11-11: a hostile or compromised allowlisted YouTrack cannot make the driver allocate far beyond what it sent.** Responses are capped (8 MiB a list page, 2 MiB an object, 4 KiB an error body, 64 MiB a listing); a page holds at most the requested 100 items; nested tags are capped at 1000 per item and count toward the listing's item ceiling; a byte scan bounds the JSON values before decoding; list pages decode one element at a time and stop at the first error. The worst case found is about 34 MiB for an 8 MiB page, against about 13 MiB for the largest legitimate one (measured in the security review). - **2026-11-11: left for M6's live pass** (taken from YouTrack's documentation, checked only against the fake): the `TagsCategory` activity category, the `updated:` query date format, and that tag names can repeat across owners. - **2026-11-11: migration `00424_issue_tracker_connections`** is the next free number on `prd/ws-2`; it is renumbered at upstream landing. Rolling it back deletes every stored issue tracker token. From f627b3b9d4994e1f487532101a7afb3d22622edf Mon Sep 17 00:00:00 2001 From: uzi-agent Date: Wed, 11 Nov 2026 11:40:15 +0000 Subject: [PATCH 17/17] docs: PRD #4173 M3 records how ListIssues queries YouTrack Co-Authored-By: Claude Opus 5.5 --- prds/4173-external-issue-trackers.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/prds/4173-external-issue-trackers.md b/prds/4173-external-issue-trackers.md index 3bcddbba0a..e4a604b961 100644 --- a/prds/4173-external-issue-trackers.md +++ b/prds/4173-external-issue-trackers.md @@ -174,10 +174,11 @@ Settings → Issue trackers, the CLI commands, a setup page under `docs/`, an AD - **2026-11-11: a YouTrack driver is bound to one YouTrack project when it is built.** The `projectID int64` argument of `forge.IssueTracker` is ignored, since M4's `IssueTrackerForRepo` returns a driver for the repo's linked project while callers keep passing the forge project id. An unbound driver refuses issue-side calls with `ErrProjectNotBound`. - **2026-11-11: the issue number is YouTrack's `numberInProject` (D10's candidate), and every per-issue call first checks the issue is still in the bound project.** YouTrack still resolves an old `SHORT-n` after a move, so a moved issue reads as not found and is never written to or read. YouTrack has no conditional write, so a move between that check and the write is not caught. M4 still owns the identity storage. - **2026-11-11: labels are YouTrack tags,** global per account. `ListLabels` lists the tags visible to the token, once per name; `EnsureLabels` creates missing ones; `Label.Color` is ignored. `UpdateIssueLabels` is one call per tag (adds, then removes), so not atomic; the commands API is not used, so no name enters a command query. `CreateIssue` adds tags after creating the issue and does not roll back if that fails. +- **2026-11-11: how `ListIssues` queries YouTrack.** The query is `project: {SHORT}`, one `tag: {name}` per label (AND), `#Unresolved` or `#Resolved` for the state, and for `UpdatedAfter` an `updated:` range from the UTC date one day early, sorted by `updated asc`. The driver then re-filters client-side on exact tag names, the exact `updated` time and the bound project. A label name holding `{`, `}` or a control or invisible character is refused with `ErrInvalidLabelName`, so no name can break out of the query. - **2026-11-11: closing and reopening use explicit State values from the driver options;** unset, `SetIssueState` returns `ErrStateMappingUnset` and never guesses. M5 owns the column and state mapping and the "never won't fix" rule. - **2026-11-11: YouTrack account ids are strings, so the numeric author ids are 0 ("unresolved");** D11 withholds such comments anyway. `LabelEvent.ID` is the oldest-first ordinal, `IssueNote.ID` the numeric part of the comment id. - **2026-11-11: issue tracker base URLs are scheme and host only,** https, ASCII host, no path other than `/`, no userinfo, query or fragment (refused, not stripped), so a YouTrack under a context path is not supported yet. The allowlist is `ISSUE_TRACKER_ALLOWED_BASE_URLS`, empty by default; it is documented with the M6 setup page. `Connect` checks the strict form before the allowlist, so the allowlist is always compared against the canonical value that is stored. - **2026-11-11: the token is sealed with AAD `uzi/issue-tracker-token/v1|||`,** so a ciphertext copied to another row does not open; the canonical base URL is frozen at v1 for that reason. The token must be 16 to 4096 bytes with no control or invisible character. There is no least-privilege check like the forge's: YouTrack permanent tokens expose no scopes. A credential reset does not delete issue tracker tokens, as for forge connections; recorded as a residual risk. `DriverFor` re-checks the allowlist before opening a stored token. - **2026-11-11: a hostile or compromised allowlisted YouTrack cannot make the driver allocate far beyond what it sent.** Responses are capped (8 MiB a list page, 2 MiB an object, 4 KiB an error body, 64 MiB a listing); a page holds at most the requested 100 items; nested tags are capped at 1000 per item and count toward the listing's item ceiling; a byte scan bounds the JSON values before decoding; list pages decode one element at a time and stop at the first error. The worst case found is about 34 MiB for an 8 MiB page, against about 13 MiB for the largest legitimate one (measured in the security review). -- **2026-11-11: left for M6's live pass** (taken from YouTrack's documentation, checked only against the fake): the `TagsCategory` activity category, the `updated:` query date format, and that tag names can repeat across owners. +- **2026-11-11: left for M6's live pass** (taken from YouTrack's documentation, checked only against the fake): the `TagsCategory` activity category, the `updated:` query date format, that tag names can repeat across owners, and whether YouTrack accepts tags on issue create (the driver adds them after create, so it does not depend on it). - **2026-11-11: migration `00424_issue_tracker_connections`** is the next free number on `prd/ws-2`; it is renumbered at upstream landing. Rolling it back deletes every stored issue tracker token.