diff --git a/README.md b/README.md index ea59744..145bc48 100644 --- a/README.md +++ b/README.md @@ -99,6 +99,7 @@ make demo | Send event | `POST /v1/events` (header `X-Signature`, optional `X-Key-Id`) | | Rotate HMAC | `POST /v1/hmac/rotate` (header `X-API-Key`; overlap for that brand only) | | Read player | `GET /v1/players/{id}` (header `X-API-Key`) | +| List dead letters | `GET /v1/dlq` (header `X-API-Key`) | | Tool call | `POST /v1/tools/{name}` (header `X-API-Key`; `ingest` also `X-Signature`) | | Read player (gRPC) | `GetPlayer` (metadata `x-api-key`) | | Health | `GET /healthz` | diff --git a/docs/architecture.md b/docs/architecture.md index 2c0f09c..ddd0459 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -103,7 +103,9 @@ Code: `internal/store/outbox.go`, `internal/outbox/publisher.go`. Local demos can turn on `FAIL_INJECT=true`. Then an event id that starts with `poison-` fails after mark, retries three times, and lands on `player.events.dlq`. Default ingest is unchanged. Leave this off on the public API. -Code: `internal/kafka/consumer.go`, `internal/kafka/dlq.go`. +`GET /v1/dlq` (API key) lists the last N dead letters for that brand. Redrive is a new signed `POST /v1/events` of the same `event_id` — not an admin credit. Arena’s Architecture page shows the list; a human has to click Redrive. The ledger unique key still blocks a second bonus. + +Code: `internal/kafka/consumer.go`, `internal/kafka/dlq.go`, `internal/kafka/inspect.go`. ## Why bonus credits use a unique event id @@ -143,6 +145,7 @@ Details: [concepts.md](./concepts.md). Code: `internal/rules`. | `POST /v1/hmac/rotate` | Header `X-API-Key`. Overlap window for that brand only. Response has key ids, not secrets. | | `POST /v1/tools/{name}` | Header `X-API-Key`; `ingest` also `X-Signature` of the event body | | `GET /v1/players/{id}` | Header `X-API-Key` | +| `GET /v1/dlq` | Header `X-API-Key`; last N dead letters for that brand | | gRPC `GetPlayer` | metadata `x-api-key` | Unknown brand and bad signature both return **401** (same answer on purpose). @@ -160,11 +163,12 @@ Useful coverage for this path: - worker rollback if processing fails after the mark (`internal/worker`) - bad HMAC / unknown tenant → 401, cross-tenant player read → 404 (`internal/api/httpapi`) +- last N DLQ inspect + signed redrive credit-once (`internal/kafka`, `internal/api/httpapi`) - ledger credit-once under concurrent delivery (`internal/ledger`) ## Not in this project - Game UI / website for players - Real money payments -- Full fraud team tools for DLQ messages +- Auto-redrive of DLQ messages (inspect is read; redrive is a human click through HMAC ingest) - Machine-learning “personal offers” diff --git a/internal/api/httpapi/dlq.go b/internal/api/httpapi/dlq.go new file mode 100644 index 0000000..da3e6f2 --- /dev/null +++ b/internal/api/httpapi/dlq.go @@ -0,0 +1,53 @@ +package httpapi + +import ( + "context" + "encoding/json" + "net/http" + "strconv" + "time" + + "github.com/o-mid/engagepulse/internal/kafka" +) + +func (s *Server) handleListDLQ(w http.ResponseWriter, r *http.Request) { + tenant, ok := TenantFromContext(r.Context()) + if !ok { + http.Error(w, "unauthorized", http.StatusUnauthorized) + return + } + n := 20 + if q := r.URL.Query().Get("n"); q != "" { + parsed, err := strconv.Atoi(q) + if err == nil && parsed > 0 { + n = parsed + } + } + if n > 100 { + n = 100 + } + + items := []kafka.DeadLetter{} + if s.dlq != nil { + ctx, cancel := context.WithTimeout(r.Context(), 8*time.Second) + defer cancel() + all, err := s.dlq.Recent(ctx, 100) + if err != nil { + s.logger.Error("list dlq", "err", err) + writeError(w, http.StatusBadGateway, "dlq unavailable") + return + } + for _, dl := range all { + if dl.Event.TenantID != tenant.ID { + continue + } + items = append(items, dl) + if len(items) >= n { + break + } + } + } + + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(map[string]any{"items": items}) +} diff --git a/internal/api/httpapi/dlq_test.go b/internal/api/httpapi/dlq_test.go new file mode 100644 index 0000000..a69e09e --- /dev/null +++ b/internal/api/httpapi/dlq_test.go @@ -0,0 +1,221 @@ +package httpapi_test + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "log/slog" + "net/http" + "net/http/httptest" + "os" + "testing" + "time" + + "github.com/o-mid/engagepulse/internal/api/httpapi" + "github.com/o-mid/engagepulse/internal/domain" + "github.com/o-mid/engagepulse/internal/ingest" + "github.com/o-mid/engagepulse/internal/kafka" + "github.com/o-mid/engagepulse/internal/store" + "github.com/o-mid/engagepulse/internal/worker" +) + +type memDLQList struct { + items []kafka.DeadLetter +} + +func (m memDLQList) Recent(_ context.Context, n int) ([]kafka.DeadLetter, error) { + if n > len(m.items) { + n = len(m.items) + } + return m.items[:n], nil +} + +func TestListDLQRequiresAPIKey(t *testing.T) { + st, srv := newSecurityTestServer(t) + defer st.Close() + defer srv.Close() + + resp, err := http.Get(srv.URL + "/v1/dlq") + if err != nil { + t.Fatalf("get: %v", err) + } + defer func() { _ = resp.Body.Close() }() + if resp.StatusCode != http.StatusUnauthorized { + t.Fatalf("status=%d want 401", resp.StatusCode) + } +} + +func TestListDLQHidesOtherTenant(t *testing.T) { + st, api, srv := newDLQTestServer(t) + defer st.Close() + defer srv.Close() + + suffix := fmt.Sprintf("%d", time.Now().UnixNano()) + api.SetDLQ(memDLQList{items: []kafka.DeadLetter{ + { + FailedAt: time.Now().UTC(), + Error: "injected", + Attempts: 3, + Event: domain.Event{ + EventID: "acme-dlq-" + suffix, + TenantID: "acme-casino", + PlayerID: "p-acme", + Type: domain.EventDeposit, + Amount: 50, + }, + }, + { + FailedAt: time.Now().UTC(), + Error: "injected", + Attempts: 3, + Event: domain.Event{ + EventID: "nova-dlq-" + suffix, + TenantID: "nova-sports", + PlayerID: "p-nova", + Type: domain.EventDeposit, + Amount: 50, + }, + }, + }}) + + req, err := http.NewRequest(http.MethodGet, srv.URL+"/v1/dlq?n=20", nil) + if err != nil { + t.Fatalf("request: %v", err) + } + req.Header.Set("X-API-Key", "ak_acme_dev_001") + resp, err := http.DefaultClient.Do(req) + if err != nil { + t.Fatalf("do: %v", err) + } + defer func() { _ = resp.Body.Close() }() + if resp.StatusCode != http.StatusOK { + body, _ := io.ReadAll(resp.Body) + t.Fatalf("status=%d body=%s", resp.StatusCode, body) + } + var out struct { + Items []kafka.DeadLetter `json:"items"` + } + if err = json.NewDecoder(resp.Body).Decode(&out); err != nil { + t.Fatalf("decode: %v", err) + } + if len(out.Items) != 1 || out.Items[0].Event.EventID != "acme-dlq-"+suffix { + t.Fatalf("items=%+v", out.Items) + } +} + +func TestRedriveSignedIngestIsCreditOnce(t *testing.T) { + st, api, srv := newDLQTestServer(t) + defer st.Close() + defer srv.Close() + + ctx := context.Background() + suffix := fmt.Sprintf("%d", time.Now().UnixNano()) + playerID := "redrive-" + suffix + eventID := "redrive-evt-" + suffix + evt := domain.Event{ + EventID: eventID, + TenantID: "acme-casino", + PlayerID: playerID, + Type: domain.EventDeposit, + Amount: 50, + OccurredAt: time.Now().UTC(), + } + api.SetDLQ(memDLQList{items: []kafka.DeadLetter{{ + FailedAt: time.Now().UTC(), + Error: "injected", + Attempts: 3, + Event: evt, + }}}) + + req, err := http.NewRequest(http.MethodGet, srv.URL+"/v1/dlq", nil) + if err != nil { + t.Fatalf("request: %v", err) + } + req.Header.Set("X-API-Key", "ak_acme_dev_001") + resp, err := http.DefaultClient.Do(req) + if err != nil { + t.Fatalf("list: %v", err) + } + _ = resp.Body.Close() + if resp.StatusCode != http.StatusOK { + t.Fatalf("list status=%d", resp.StatusCode) + } + + tenant, err := st.GetTenant(ctx, "acme-casino") + if err != nil { + t.Fatalf("tenant: %v", err) + } + w := worker.New(st, slog.New(slog.NewTextHandler(io.Discard, nil))) + + status := postSignedEvent(t, srv.URL, tenant.HMACSecret, evt) + if status != http.StatusAccepted { + t.Fatalf("redrive status=%d", status) + } + if err = w.Handle(ctx, evt); err != nil { + t.Fatalf("handle: %v", err) + } + snap, err := st.GetPlayerSnapshot(ctx, "acme-casino", playerID) + if err != nil { + t.Fatalf("snapshot: %v", err) + } + if snap.Balance <= 0 { + t.Fatalf("first redrive balance=%d", snap.Balance) + } + first := snap.Balance + + status = postSignedEvent(t, srv.URL, tenant.HMACSecret, evt) + if status != http.StatusAccepted { + t.Fatalf("second ingest status=%d", status) + } + if err = w.Handle(ctx, evt); err != nil { + t.Fatalf("handle again: %v", err) + } + snap, err = st.GetPlayerSnapshot(ctx, "acme-casino", playerID) + if err != nil { + t.Fatalf("snapshot: %v", err) + } + if snap.Balance != first { + t.Fatalf("second redrive balance=%d want %d", snap.Balance, first) + } +} + +func newDLQTestServer(t *testing.T) (*store.Store, *httpapi.Server, *httptest.Server) { + t.Helper() + dsn := os.Getenv("DATABASE_URL") + if dsn == "" { + t.Skip("DATABASE_URL not set") + } + ctx := context.Background() + st, err := store.Open(ctx, dsn) + if err != nil { + t.Fatalf("open: %v", err) + } + if err = st.Migrate(ctx); err != nil { + st.Close() + t.Fatalf("migrate: %v", err) + } + api := httpapi.New(st, st, slog.New(slog.NewTextHandler(io.Discard, nil))) + return st, api, httptest.NewServer(api.Handler()) +} + +func postSignedEvent(t *testing.T, base, secret string, evt domain.Event) int { + t.Helper() + body, err := json.Marshal(evt) + if err != nil { + t.Fatalf("marshal: %v", err) + } + req, err := http.NewRequest(http.MethodPost, base+"/v1/events", bytes.NewReader(body)) + if err != nil { + t.Fatalf("request: %v", err) + } + req.Header.Set("Content-Type", "application/json") + req.Header.Set("X-Signature", ingest.Sign(secret, body)) + resp, err := http.DefaultClient.Do(req) + if err != nil { + t.Fatalf("do: %v", err) + } + defer func() { _ = resp.Body.Close() }() + return resp.StatusCode +} diff --git a/internal/api/httpapi/server.go b/internal/api/httpapi/server.go index 4b79446..c4ac702 100644 --- a/internal/api/httpapi/server.go +++ b/internal/api/httpapi/server.go @@ -11,6 +11,7 @@ import ( "github.com/o-mid/engagepulse/internal/domain" "github.com/o-mid/engagepulse/internal/ingest" + "github.com/o-mid/engagepulse/internal/kafka" "github.com/o-mid/engagepulse/internal/metrics" "github.com/o-mid/engagepulse/internal/store" ) @@ -19,24 +20,34 @@ type EventAccepter interface { EnqueueEvent(ctx context.Context, evt domain.Event) error } +type DLQLister interface { + Recent(ctx context.Context, n int) ([]kafka.DeadLetter, error) +} + type Server struct { - store *store.Store accept EventAccepter + dlq DLQLister + store *store.Store logger *slog.Logger mux *http.ServeMux } func New(st *store.Store, accept EventAccepter, logger *slog.Logger) *Server { - s := &Server{store: st, accept: accept, logger: logger, mux: http.NewServeMux()} + s := &Server{accept: accept, store: st, logger: logger, mux: http.NewServeMux()} s.mux.HandleFunc("GET /healthz", s.handleHealth) s.mux.Handle("GET /metrics", metrics.Handler()) s.mux.HandleFunc("POST /v1/events", s.handleIngest) s.mux.HandleFunc("POST /v1/hmac/rotate", s.requireAPIKey(s.handleRotateHMAC)) s.mux.HandleFunc("POST /v1/tools/{name}", s.requireAPIKey(s.handleTool)) s.mux.HandleFunc("GET /v1/players/{id}", s.requireAPIKey(s.handleGetPlayer)) + s.mux.HandleFunc("GET /v1/dlq", s.requireAPIKey(s.handleListDLQ)) return s } +func (s *Server) SetDLQ(l DLQLister) { + s.dlq = l +} + func (s *Server) Handler() http.Handler { return s.mux } diff --git a/internal/app/app.go b/internal/app/app.go index a248f43..aadb46a 100644 --- a/internal/app/app.go +++ b/internal/app/app.go @@ -73,6 +73,7 @@ func (a *App) Run(ctx context.Context) error { defer func() { _ = consumer.Close() }() api := httpapi.New(a.store, a.store, a.logger) + api.SetDLQ(kafka.NewInspector(a.cfg.KafkaBrokers, a.cfg.KafkaDLQTopic)) srv := &http.Server{ Addr: a.cfg.HTTPAddr, Handler: api.Handler(), diff --git a/internal/app/pipeline_test.go b/internal/app/pipeline_test.go index ff6fac2..a50e056 100644 --- a/internal/app/pipeline_test.go +++ b/internal/app/pipeline_test.go @@ -347,6 +347,200 @@ func TestPoisonSignedEventRetriesThenKafkaDLQ(t *testing.T) { } } +// Proves: list last N dead letters, then a new signed ingest of the same event_id credits once. +func TestDLQInspectThenRedriveCreditOnce(t *testing.T) { + dsn := os.Getenv("DATABASE_URL") + brokers := os.Getenv("KAFKA_BROKERS") + if dsn == "" || brokers == "" { + t.Skip("DATABASE_URL and KAFKA_BROKERS required") + } + + ctx := context.Background() + suffix := fmt.Sprintf("%d", time.Now().UnixNano()) + topic := "player.events.redrive-" + suffix + dlqTopic := "player.events.dlq.redrive-" + suffix + if err := kafka.EnsureTopic([]string{brokers}, topic); err != nil { + t.Fatalf("topic: %v", err) + } + if err := kafka.EnsureTopic([]string{brokers}, dlqTopic); err != nil { + t.Fatalf("dlq topic: %v", err) + } + + st, err := store.Open(ctx, dsn) + if err != nil { + t.Fatalf("open store: %v", err) + } + defer st.Close() + if err = st.Migrate(ctx); err != nil { + t.Fatalf("migrate: %v", err) + } + + logger := slog.New(slog.NewTextHandler(io.Discard, nil)) + api := httpapi.New(st, st, logger) + api.SetDLQ(kafka.NewInspector([]string{brokers}, dlqTopic)) + srv := httptest.NewServer(api.Handler()) + defer srv.Close() + + tenantID := "acme-casino" + playerID := "redrive-player-" + suffix + eventID := "redrive-evt-" + suffix + tenant, err := st.GetTenant(ctx, tenantID) + if err != nil { + t.Fatalf("tenant: %v", err) + } + + evt := domain.Event{ + EventID: eventID, + TenantID: tenantID, + PlayerID: playerID, + Type: domain.EventDeposit, + Amount: 50, + OccurredAt: time.Now().UTC(), + } + dlqPub := kafka.NewProducer([]string{brokers}, dlqTopic) + if err = dlqPub.PublishDLQ(ctx, evt, "injected", kafka.MaxAttempts); err != nil { + t.Fatalf("publish dlq: %v", err) + } + if err = dlqPub.Close(); err != nil { + t.Fatalf("close dlq: %v", err) + } + + listCtx, listCancel := context.WithTimeout(ctx, 15*time.Second) + defer listCancel() + var listed kafka.DeadLetter + for { + req, reqErr := http.NewRequest(http.MethodGet, srv.URL+"/v1/dlq?n=20", nil) + if reqErr != nil { + t.Fatalf("list request: %v", reqErr) + } + req.Header.Set("X-API-Key", "ak_acme_dev_001") + resp, doErr := http.DefaultClient.Do(req) + if doErr != nil { + t.Fatalf("list: %v", doErr) + } + var out struct { + Items []kafka.DeadLetter `json:"items"` + } + if err = json.NewDecoder(resp.Body).Decode(&out); err != nil { + _ = resp.Body.Close() + t.Fatalf("decode list: %v", err) + } + _ = resp.Body.Close() + if resp.StatusCode != http.StatusOK { + t.Fatalf("list status=%d", resp.StatusCode) + } + for _, item := range out.Items { + if item.Event.EventID == eventID { + listed = item + break + } + } + if listed.Event.EventID == eventID { + break + } + select { + case <-listCtx.Done(): + t.Fatalf("letter %s not listed", eventID) + case <-time.After(200 * time.Millisecond): + } + } + if listed.Attempts != kafka.MaxAttempts { + t.Fatalf("listed attempts=%d", listed.Attempts) + } + + body, err := json.Marshal(evt) + if err != nil { + t.Fatalf("marshal: %v", err) + } + req, err := http.NewRequest(http.MethodPost, srv.URL+"/v1/events", bytes.NewReader(body)) + if err != nil { + t.Fatalf("request: %v", err) + } + req.Header.Set("Content-Type", "application/json") + req.Header.Set("X-Signature", ingest.Sign(tenant.HMACSecret, body)) + resp, err := http.DefaultClient.Do(req) + if err != nil { + t.Fatalf("redrive: %v", err) + } + _ = resp.Body.Close() + if resp.StatusCode != http.StatusAccepted { + t.Fatalf("redrive status=%d", resp.StatusCode) + } + + pub := kafka.NewProducer([]string{brokers}, topic) + defer func() { _ = pub.Close() }() + consumerDLQ := kafka.NewProducer([]string{brokers}, dlqTopic) + defer func() { _ = consumerDLQ.Close() }() + w := worker.New(st, logger) + consumer := kafka.NewConsumer( + []string{brokers}, + topic, + "engagepulse-redrive-"+suffix, + logger, + w.Handle, + consumerDLQ, + ) + defer func() { _ = consumer.Close() }() + runCtx, cancel := context.WithCancel(ctx) + defer cancel() + go func() { _ = consumer.Run(runCtx) }() + + op := outbox.NewPublisher(st, pub, logger) + deadline := time.Now().Add(20 * time.Second) + for { + if err = op.FlushOnce(ctx); err != nil { + t.Fatalf("flush: %v", err) + } + row, getErr := st.GetOutbox(ctx, tenantID, eventID) + if getErr != nil { + t.Fatalf("outbox: %v", getErr) + } + if row.Status == store.OutboxPublished { + break + } + if time.Now().After(deadline) { + t.Fatalf("outbox not published; status=%s", row.Status) + } + time.Sleep(100 * time.Millisecond) + } + + var snap domain.PlayerSnapshot + for { + snap, err = st.GetPlayerSnapshot(ctx, tenantID, playerID) + if err == nil && snap.Balance > 0 { + break + } + if time.Now().After(deadline) { + t.Fatalf("redrive not credited: %v balance=%d", err, snap.Balance) + } + time.Sleep(100 * time.Millisecond) + } + first := snap.Balance + + req, err = http.NewRequest(http.MethodPost, srv.URL+"/v1/events", bytes.NewReader(body)) + if err != nil { + t.Fatalf("request: %v", err) + } + req.Header.Set("Content-Type", "application/json") + req.Header.Set("X-Signature", ingest.Sign(tenant.HMACSecret, body)) + resp, err = http.DefaultClient.Do(req) + if err != nil { + t.Fatalf("second ingest: %v", err) + } + _ = resp.Body.Close() + if resp.StatusCode != http.StatusAccepted { + t.Fatalf("second ingest status=%d", resp.StatusCode) + } + time.Sleep(300 * time.Millisecond) + snap, err = st.GetPlayerSnapshot(ctx, tenantID, playerID) + if err != nil { + t.Fatalf("snapshot: %v", err) + } + if snap.Balance != first { + t.Fatalf("second redrive balance=%d want %d", snap.Balance, first) + } +} + func envOr(key, fallback string) string { if v := os.Getenv(key); v != "" { return v diff --git a/internal/kafka/inspect.go b/internal/kafka/inspect.go new file mode 100644 index 0000000..edd463b --- /dev/null +++ b/internal/kafka/inspect.go @@ -0,0 +1,144 @@ +package kafka + +import ( + "context" + "fmt" + "sort" + "strings" + "time" + + kafkago "github.com/segmentio/kafka-go" +) + +const maxInspect = 100 + +// Inspector reads recent dead letters from a DLQ topic. +type Inspector struct { + topic string + brokers []string +} + +func NewInspector(brokers []string, topic string) *Inspector { + copied := make([]string, len(brokers)) + copy(copied, brokers) + return &Inspector{topic: topic, brokers: copied} +} + +func (i *Inspector) Recent(ctx context.Context, n int) ([]DeadLetter, error) { + if i == nil { + return nil, nil + } + return ReadRecent(ctx, i.brokers, i.topic, n) +} + +// ReadRecent returns the last n dead letters, newest first. +func ReadRecent(ctx context.Context, brokers []string, topic string, n int) ([]DeadLetter, error) { + if len(brokers) == 0 { + return nil, fmt.Errorf("kafka brokers required") + } + topic = strings.TrimSpace(topic) + if topic == "" { + return nil, fmt.Errorf("dlq topic required") + } + if n <= 0 { + n = 20 + } + if n > maxInspect { + n = maxInspect + } + if err := EnsureTopic(brokers, topic); err != nil { + return nil, err + } + + dialCtx, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + conn, err := kafkago.DialContext(dialCtx, "tcp", brokers[0]) + if err != nil { + return nil, fmt.Errorf("dlq dial: %w", err) + } + defer func() { _ = conn.Close() }() + parts, err := conn.ReadPartitions(topic) + if err != nil { + return nil, fmt.Errorf("dlq partitions: %w", err) + } + + all := make([]DeadLetter, 0, n) + seen := map[int]struct{}{} + for _, p := range parts { + if p.Topic != topic { + continue + } + if _, ok := seen[p.ID]; ok { + continue + } + seen[p.ID] = struct{}{} + var batch []DeadLetter + batch, err = readPartitionRecent(ctx, brokers, topic, p.ID, n) + if err != nil { + return nil, err + } + all = append(all, batch...) + } + sort.SliceStable(all, func(i, j int) bool { + if all[i].FailedAt.Equal(all[j].FailedAt) { + return all[i].Event.EventID > all[j].Event.EventID + } + return all[i].FailedAt.After(all[j].FailedAt) + }) + if len(all) > n { + all = all[:n] + } + return all, nil +} + +func readPartitionRecent(ctx context.Context, brokers []string, topic string, partition, n int) ([]DeadLetter, error) { + dialCtx, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + conn, err := kafkago.DialLeader(dialCtx, "tcp", brokers[0], topic, partition) + if err != nil { + return nil, fmt.Errorf("dlq dial partition %d: %w", partition, err) + } + first, last, err := conn.ReadOffsets() + _ = conn.Close() + if err != nil { + return nil, fmt.Errorf("dlq offsets partition %d: %w", partition, err) + } + // ReadLastOffset is the high watermark (next offset to be written). + if last <= first { + return nil, nil + } + start := last - int64(n) + if start < first { + start = first + } + want := int(last - start) + + r := kafkago.NewReader(kafkago.ReaderConfig{ + Brokers: brokers, + Topic: topic, + Partition: partition, + MinBytes: 1, + MaxBytes: 10e6, + MaxWait: time.Second, + }) + defer func() { _ = r.Close() }() + if err = r.SetOffset(start); err != nil { + return nil, fmt.Errorf("dlq seek partition %d: %w", partition, err) + } + + out := make([]DeadLetter, 0, want) + for len(out) < want { + var msg kafkago.Message + msg, err = r.ReadMessage(ctx) + if err != nil { + return nil, fmt.Errorf("dlq read partition %d: %w", partition, err) + } + var dl DeadLetter + dl, err = ParseDeadLetter(msg.Value) + if err != nil { + continue + } + out = append(out, dl) + } + return out, nil +} diff --git a/internal/kafka/inspect_test.go b/internal/kafka/inspect_test.go new file mode 100644 index 0000000..1c52355 --- /dev/null +++ b/internal/kafka/inspect_test.go @@ -0,0 +1,79 @@ +package kafka + +import ( + "context" + "fmt" + "os" + "testing" + "time" + + "github.com/o-mid/engagepulse/internal/domain" +) + +func TestReadRecentReturnsLastNNewestFirst(t *testing.T) { + brokers := os.Getenv("KAFKA_BROKERS") + if brokers == "" { + t.Skip("KAFKA_BROKERS required") + } + ctx := context.Background() + suffix := fmt.Sprintf("%d", time.Now().UnixNano()) + topic := "player.events.dlq.inspect-" + suffix + if err := EnsureTopic([]string{brokers}, topic); err != nil { + t.Fatalf("topic: %v", err) + } + + pub := NewProducer([]string{brokers}, topic) + for i := 1; i <= 3; i++ { + evt := domain.Event{ + EventID: fmt.Sprintf("inspect-%s-%d", suffix, i), + TenantID: "acme-casino", + PlayerID: "inspect-player", + Type: domain.EventDeposit, + Amount: 50, + OccurredAt: time.Now().UTC(), + } + if err := pub.PublishDLQ(ctx, evt, fmt.Sprintf("fail-%d", i), 3); err != nil { + t.Fatalf("publish %d: %v", i, err) + } + } + if err := pub.Close(); err != nil { + t.Fatalf("close producer: %v", err) + } + + readCtx, cancel := context.WithTimeout(ctx, 15*time.Second) + defer cancel() + got, err := ReadRecent(readCtx, []string{brokers}, topic, 2) + if err != nil { + t.Fatalf("read: %v", err) + } + if len(got) != 2 { + t.Fatalf("len=%d want 2: %+v", len(got), got) + } + if got[0].Event.EventID != "inspect-"+suffix+"-3" { + t.Fatalf("newest=%s want inspect-%s-3", got[0].Event.EventID, suffix) + } + if got[1].Event.EventID != "inspect-"+suffix+"-2" { + t.Fatalf("second=%s want inspect-%s-2", got[1].Event.EventID, suffix) + } + if got[0].Attempts != 3 || got[0].Error != "fail-3" { + t.Fatalf("payload=%+v", got[0]) + } +} + +func TestReadRecentEmptyTopic(t *testing.T) { + brokers := os.Getenv("KAFKA_BROKERS") + if brokers == "" { + t.Skip("KAFKA_BROKERS required") + } + suffix := fmt.Sprintf("%d", time.Now().UnixNano()) + topic := "player.events.dlq.empty-" + suffix + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + got, err := ReadRecent(ctx, []string{brokers}, topic, 20) + if err != nil { + t.Fatalf("read: %v", err) + } + if len(got) != 0 { + t.Fatalf("len=%d want 0", len(got)) + } +} diff --git a/web/README.md b/web/README.md index aeabac3..d74b28e 100644 --- a/web/README.md +++ b/web/README.md @@ -7,7 +7,7 @@ Next.js App Router console for the Go service. HMAC is signed in the BFF. The br | Path | What a hiring manager sees | | --- | --- | | `/` | Arena. Dual-tenant stage (`acme-casino` VIP vs `nova-sports` velocity). Ignite runs HMAC, outbox, Kafka, worker tx, then GET player. | -| `/architecture` | Live counters and the event path. Select a node. The reading at the bottom states the credit-once guarantee. | +| `/architecture` | Live counters and the event path. Select a node. Dead letters: last N from Kafka; Redrive is a human-clicked signed ingest of the same event_id. | | `/contract` | `make replay` snapshot contract. Tools tab: `ingest` / `get_player` / `get_metrics`. Not tools: `credit` / `set_vip`. Shadow tab: mock vs velocity disagreements. | Tab URLs: `/contract?tab=tools`, `/contract?tab=shadow`. diff --git a/web/app/api/dlq/route.ts b/web/app/api/dlq/route.ts new file mode 100644 index 0000000..be50116 --- /dev/null +++ b/web/app/api/dlq/route.ts @@ -0,0 +1,43 @@ +import { NextResponse } from "next/server"; +import { listDeadLetters, redriveDeadLetter } from "@/lib/engagepulse"; +import type { TenantId } from "@/lib/tenants"; + +export const dynamic = "force-dynamic"; +export const runtime = "nodejs"; + +function isTenant(value: unknown): value is TenantId { + return value === "acme-casino" || value === "nova-sports"; +} + +export async function GET(req: Request) { + try { + const n = Number(new URL(req.url).searchParams.get("n") ?? "20"); + const limit = Number.isFinite(n) && n > 0 ? Math.min(n, 100) : 20; + const [acme, nova] = await Promise.all([ + listDeadLetters("acme-casino", limit), + listDeadLetters("nova-sports", limit), + ]); + const items = [...acme, ...nova].sort((a, b) => + a.failed_at < b.failed_at ? 1 : a.failed_at > b.failed_at ? -1 : 0, + ); + return NextResponse.json({ items: items.slice(0, limit) }); + } catch (err) { + const message = err instanceof Error ? err.message : "dlq list failed"; + return NextResponse.json({ error: message }, { status: 502 }); + } +} + +export async function POST(req: Request) { + try { + const body = (await req.json()) as { tenant_id?: unknown; event_id?: unknown }; + if (!isTenant(body.tenant_id) || typeof body.event_id !== "string" || !body.event_id) { + return NextResponse.json({ error: "tenant_id and event_id required" }, { status: 400 }); + } + const result = await redriveDeadLetter(body.tenant_id, body.event_id); + return NextResponse.json(result, { status: 202 }); + } catch (err) { + const message = err instanceof Error ? err.message : "redrive failed"; + const status = message.includes("not found") ? 404 : 502; + return NextResponse.json({ error: message }, { status }); + } +} diff --git a/web/components/architecture-live.tsx b/web/components/architecture-live.tsx index af45b54..2452724 100644 --- a/web/components/architecture-live.tsx +++ b/web/components/architecture-live.tsx @@ -3,6 +3,7 @@ import { useState } from "react"; import Link from "next/link"; import { ArchitectureDiagram } from "@/components/architecture-diagram"; +import { DlqInspect } from "@/components/dlq-inspect"; import { Network4Icon } from "@/components/icons/network-4"; import { StatusBanner } from "@/components/status-banner"; import { Button } from "@/components/ui/button"; @@ -123,6 +124,10 @@ export function ArchitectureLive() { +
+ +
+
diff --git a/web/components/dlq-inspect.tsx b/web/components/dlq-inspect.tsx new file mode 100644 index 0000000..39473fa --- /dev/null +++ b/web/components/dlq-inspect.tsx @@ -0,0 +1,151 @@ +"use client"; + +import { useCallback, useEffect, useState } from "react"; +import { Button } from "@/components/ui/button"; +import { Card, CardContent, CardHeader, CardTitle } from "@/components/ui/card"; +import { + Table, + TableBody, + TableCaption, + TableCell, + TableHead, + TableHeader, + TableRow, +} from "@/components/ui/table"; +import type { DeadLetter, RedriveResult } from "@/lib/types"; + +type ListResponse = { items?: DeadLetter[]; error?: string }; + +export function DlqInspect() { + const [items, setItems] = useState([]); + const [error, setError] = useState(null); + const [busy, setBusy] = useState(false); + const [redriving, setRedriving] = useState(null); + const [note, setNote] = useState(null); + + const load = useCallback(async () => { + setBusy(true); + setError(null); + try { + const res = await fetch("/api/dlq?n=20", { cache: "no-store" }); + const data = (await res.json()) as ListResponse; + if (!res.ok) throw new Error(data.error ?? "dlq list failed"); + setItems(data.items ?? []); + } catch (err) { + setError(err instanceof Error ? err.message : "dlq list failed"); + } finally { + setBusy(false); + } + }, []); + + useEffect(() => { + void load(); + }, [load]); + + async function redrive(tenantId: string, eventId: string) { + if (redriving) return; + setRedriving(eventId); + setError(null); + try { + const res = await fetch("/api/dlq", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ tenant_id: tenantId, event_id: eventId }), + }); + const data = (await res.json()) as RedriveResult & { error?: string }; + if (!res.ok) throw new Error(data.error ?? "redrive failed"); + setNote(data); + await load(); + } catch (err) { + setError(err instanceof Error ? err.message : "redrive failed"); + } finally { + setRedriving(null); + } + } + + return ( + + +
+ Dead letters +

+ Last N from player.events.dlq. + Redrive is a new signed ingest of the same event_id. No skip-HMAC + credit. +

+
+ +
+ + {error ?

{error}

: null} + {note ? ( +

+ {note.message} {note.event_id}: balance {note.balance_before ?? "—"}{" "} + → {note.balance_after ?? "—"}. +

+ ) : null} + + + Human click only. Same event_id stays credit-once. + + + + Event + Player + Error + Redrive + + + + {items.length === 0 ? ( + + + No dead letters in the last 20. + + + ) : ( + items.map((item) => ( + + + + {item.event.event_id} + + + {item.event.tenant_id} · {item.attempts} tries + + + + {item.event.player_id} + + + {item.error} + + + + + + )) + )} + +
+
+
+ ); +} diff --git a/web/lib/architecture.ts b/web/lib/architecture.ts index 2068244..ac4f674 100644 --- a/web/lib/architecture.ts +++ b/web/lib/architecture.ts @@ -66,7 +66,7 @@ export const ARCH_NODES: ArchNode[] = [ id: "dlq", label: "DLQ", plain: "Hard fail", - detail: "After 3 worker retries, the event is published to player.events.dlq and is not credited.", + detail: "After 3 worker retries, the event is published to player.events.dlq and is not credited. Inspect last N; redrive is a new signed ingest of the same event_id.", code: "player.events.dlq", branch: true, }, @@ -150,7 +150,7 @@ export function liveReading( rules: `Hits: welcome ${live.welcome}, vip ${live.vip}, velocity ${live.velocity}.`, ledger: `Successful credits ${live.credits.toLocaleString()} (unique event ids).`, read: "GET /v1/players/{id}. Tools wrap the same reads. They do not credit.", - dlq: `${live.dlq.toLocaleString()} events on the dead-letter topic.`, + dlq: `${live.dlq.toLocaleString()} events on the dead-letter topic. Inspect last N; redrive is a human-clicked signed ingest.`, }; return lines[id]; } diff --git a/web/lib/engagepulse.ts b/web/lib/engagepulse.ts index c03c0b7..07ecd91 100644 --- a/web/lib/engagepulse.ts +++ b/web/lib/engagepulse.ts @@ -1,6 +1,12 @@ import { apiBase, getTenants, type TenantId } from "./tenants"; import { signatureHeaders } from "./hmac"; -import type { IngestResult, MetricsMap, PlayerSnapshot } from "./types"; +import type { + DeadLetter, + IngestResult, + MetricsMap, + PlayerSnapshot, + RedriveResult, +} from "./types"; type EventPayload = { event_id: string; @@ -63,6 +69,65 @@ export async function ingestEvent( }; } +export async function listDeadLetters( + tenantId: TenantId, + n = 20, +): Promise { + const tenant = getTenants()[tenantId]; + const res = await fetch( + `${apiBase()}/v1/dlq?n=${encodeURIComponent(String(n))}`, + { + headers: { "X-API-Key": tenant.apiKey }, + cache: "no-store", + }, + ); + if (!res.ok) { + const text = await res.text(); + throw new Error(`dlq ${res.status}: ${text}`); + } + const data = (await res.json()) as { items?: DeadLetter[] }; + return data.items ?? []; +} + +export async function redriveDeadLetter( + tenantId: TenantId, + eventId: string, +): Promise { + const items = await listDeadLetters(tenantId, 100); + const letter = items.find((item) => item.event.event_id === eventId); + if (!letter) { + throw new Error("dead letter not found"); + } + const before = await fetchPlayer(tenantId, letter.event.player_id); + const result = await ingestEvent(tenantId, { + event_id: letter.event.event_id, + player_id: letter.event.player_id, + type: letter.event.type, + amount: letter.event.amount ?? 0, + occurred_at: letter.event.occurred_at || new Date().toISOString(), + }); + let after = before; + for (let i = 0; i < 6; i++) { + await new Promise((r) => setTimeout(r, 400)); + after = await fetchPlayer(tenantId, letter.event.player_id); + if ( + after != null && + (before == null || after.balance !== before.balance) + ) { + break; + } + } + return { + status: result.status, + event_id: result.event_id, + tenant_id: tenantId, + player_id: letter.event.player_id, + balance_before: before?.balance ?? null, + balance_after: after?.balance ?? null, + message: "redrive accepted, balance unchanged if already credited.", + }; +} + export async function postTool( name: string, body: Record = {}, diff --git a/web/lib/types.ts b/web/lib/types.ts index 060acf5..384f96f 100644 --- a/web/lib/types.ts +++ b/web/lib/types.ts @@ -22,6 +22,32 @@ export type IngestResult = { export type MetricsMap = Record; +export type DeadLetterEvent = { + event_id: string; + tenant_id: string; + player_id: string; + type: string; + amount?: number; + occurred_at: string; +}; + +export type DeadLetter = { + failed_at: string; + error: string; + attempts: number; + event: DeadLetterEvent; +}; + +export type RedriveResult = { + status: string; + event_id: string; + tenant_id: string; + player_id: string; + balance_before: number | null; + balance_after: number | null; + message: string; +}; + export type DemoResponse = { accepted: IngestResult[]; players: {