diff --git a/docs/architecture.md b/docs/architecture.md index ddd0459..3bf9635 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -80,7 +80,7 @@ Locally, **Redpanda** is a smaller tool that speaks the same Kafka style of API. 3. A background loop sends pending rows to Kafka. 4. Same `event_id` cannot be inserted twice for the same brand. -If the app crashes while sending, rows stuck as “publishing” are put back to “pending” and tried again. +If the app crashes while sending, rows stuck as “publishing” are put back to “pending” and tried again. A store test leaves a row in `publishing`, reclaims it, then flushes: Kafka sees the event once. The gauge `engagepulse_outbox_pending` shows how many rows are still waiting (or mid-send). The demo script waits until that gauge is `0` and processed events catch up to accepted ones before printing player JSON. diff --git a/internal/store/outbox_test.go b/internal/store/outbox_test.go index 76d5dbb..10c3a55 100644 --- a/internal/store/outbox_test.go +++ b/internal/store/outbox_test.go @@ -86,6 +86,95 @@ func (m *memKafka) Publish(_ context.Context, evt domain.Event) error { return nil } +func TestReclaimStuckPublishingThenPublishOnce(t *testing.T) { + st := openTestStore(t) + ctx := context.Background() + id := uniqueID(t) + evt := domain.Event{ + EventID: "outbox-reclaim-" + id, + TenantID: "acme-casino", + PlayerID: "outbox-player-reclaim-" + id, + Type: domain.EventDeposit, + Amount: 50, + OccurredAt: time.Now().UTC(), + } + if err := st.EnqueueEvent(ctx, evt); err != nil { + t.Fatalf("enqueue: %v", err) + } + + claimed, err := st.ClaimPendingOutbox(ctx, 100) + if err != nil { + t.Fatalf("claim: %v", err) + } + foundClaim := false + for _, row := range claimed { + if row.EventID == evt.EventID { + foundClaim = true + if row.Status != store.OutboxPublishing { + t.Fatalf("claimed status=%s want publishing", row.Status) + } + } + } + if !foundClaim { + t.Fatalf("claim missed %s", evt.EventID) + } + + stuck, err := st.GetOutbox(ctx, evt.TenantID, evt.EventID) + if err != nil { + t.Fatalf("get stuck: %v", err) + } + if stuck.Status != store.OutboxPublishing { + t.Fatalf("stuck status=%s want publishing", stuck.Status) + } + + if err = st.ReclaimPublishingOutbox(ctx); err != nil { + t.Fatalf("reclaim: %v", err) + } + reclaimed, err := st.GetOutbox(ctx, evt.TenantID, evt.EventID) + if err != nil { + t.Fatalf("get reclaimed: %v", err) + } + if reclaimed.Status != store.OutboxPending { + t.Fatalf("reclaimed status=%s want pending", reclaimed.Status) + } + if reclaimed.LastError != "reclaimed stale publish" { + t.Fatalf("last_error=%q want reclaimed stale publish", reclaimed.LastError) + } + + k := &memKafka{} + p := outbox.NewPublisher(st, k, nil) + if err = p.FlushOnce(ctx); err != nil { + t.Fatalf("flush: %v", err) + } + if n := countPublished(k, evt.EventID); n != 1 { + t.Fatalf("published=%d want 1 after reclaim", n) + } + row, err := st.GetOutbox(ctx, evt.TenantID, evt.EventID) + if err != nil { + t.Fatalf("get published: %v", err) + } + if row.Status != store.OutboxPublished { + t.Fatalf("status=%s want published", row.Status) + } + + if err = p.FlushOnce(ctx); err != nil { + t.Fatalf("second flush: %v", err) + } + if n := countPublished(k, evt.EventID); n != 1 { + t.Fatalf("published=%d want 1 after second flush", n) + } +} + +func countPublished(k *memKafka, eventID string) int { + n := 0 + for _, published := range k.events { + if published.EventID == eventID { + n++ + } + } + return n +} + func TestPublisherMarksPublished(t *testing.T) { st := openTestStore(t) ctx := context.Background() diff --git a/web/lib/architecture.ts b/web/lib/architecture.ts index ac4f674..6a334de 100644 --- a/web/lib/architecture.ts +++ b/web/lib/architecture.ts @@ -45,7 +45,7 @@ export const ARCH_NODES: ArchNode[] = [ id: "outbox", label: "Outbox", plain: "Save first", - detail: "Postgres outbox row before Kafka — crash-safe accept, then 202.", + detail: "Postgres outbox row before Kafka — crash-safe accept, then 202. Rows left publishing after a crash go back to pending and publish once.", code: "internal/outbox", }, { @@ -140,8 +140,8 @@ export function liveReading( : "API probe failed.", outbox: live.outboxPending > 0 - ? `${live.outboxPending} row(s) waiting to publish.` - : "Outbox drained. Publisher caught up.", + ? `${live.outboxPending} row(s) waiting to publish (pending or reclaimed mid-send).` + : "Outbox drained. Crash reclaim runs before each publish flush.", kafka: lag > 0 ? `${lag} accepted event(s) not yet processed.`