Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion docs/architecture.md
Original file line number Diff line number Diff line change
Expand Up @@ -80,7 +80,7 @@ Locally, **Redpanda** is a smaller tool that speaks the same Kafka style of API.
3. A background loop sends pending rows to Kafka.
4. Same `event_id` cannot be inserted twice for the same brand.

If the app crashes while sending, rows stuck as “publishing” are put back to “pending” and tried again.
If the app crashes while sending, rows stuck as “publishing” are put back to “pending” and tried again. A store test leaves a row in `publishing`, reclaims it, then flushes: Kafka sees the event once.

The gauge `engagepulse_outbox_pending` shows how many rows are still waiting (or mid-send).
The demo script waits until that gauge is `0` and processed events catch up to accepted ones before printing player JSON.
Expand Down
89 changes: 89 additions & 0 deletions internal/store/outbox_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -86,6 +86,95 @@ func (m *memKafka) Publish(_ context.Context, evt domain.Event) error {
return nil
}

func TestReclaimStuckPublishingThenPublishOnce(t *testing.T) {
st := openTestStore(t)
ctx := context.Background()
id := uniqueID(t)
evt := domain.Event{
EventID: "outbox-reclaim-" + id,
TenantID: "acme-casino",
PlayerID: "outbox-player-reclaim-" + id,
Type: domain.EventDeposit,
Amount: 50,
OccurredAt: time.Now().UTC(),
}
if err := st.EnqueueEvent(ctx, evt); err != nil {
t.Fatalf("enqueue: %v", err)
}

claimed, err := st.ClaimPendingOutbox(ctx, 100)
if err != nil {
t.Fatalf("claim: %v", err)
}
foundClaim := false
for _, row := range claimed {
if row.EventID == evt.EventID {
foundClaim = true
if row.Status != store.OutboxPublishing {
t.Fatalf("claimed status=%s want publishing", row.Status)
}
}
}
if !foundClaim {
t.Fatalf("claim missed %s", evt.EventID)
}

stuck, err := st.GetOutbox(ctx, evt.TenantID, evt.EventID)
if err != nil {
t.Fatalf("get stuck: %v", err)
}
if stuck.Status != store.OutboxPublishing {
t.Fatalf("stuck status=%s want publishing", stuck.Status)
}

if err = st.ReclaimPublishingOutbox(ctx); err != nil {
t.Fatalf("reclaim: %v", err)
}
reclaimed, err := st.GetOutbox(ctx, evt.TenantID, evt.EventID)
if err != nil {
t.Fatalf("get reclaimed: %v", err)
}
if reclaimed.Status != store.OutboxPending {
t.Fatalf("reclaimed status=%s want pending", reclaimed.Status)
}
if reclaimed.LastError != "reclaimed stale publish" {
t.Fatalf("last_error=%q want reclaimed stale publish", reclaimed.LastError)
}

k := &memKafka{}
p := outbox.NewPublisher(st, k, nil)
if err = p.FlushOnce(ctx); err != nil {
t.Fatalf("flush: %v", err)
}
if n := countPublished(k, evt.EventID); n != 1 {
t.Fatalf("published=%d want 1 after reclaim", n)
}
row, err := st.GetOutbox(ctx, evt.TenantID, evt.EventID)
if err != nil {
t.Fatalf("get published: %v", err)
}
if row.Status != store.OutboxPublished {
t.Fatalf("status=%s want published", row.Status)
}

if err = p.FlushOnce(ctx); err != nil {
t.Fatalf("second flush: %v", err)
}
if n := countPublished(k, evt.EventID); n != 1 {
t.Fatalf("published=%d want 1 after second flush", n)
}
}

func countPublished(k *memKafka, eventID string) int {
n := 0
for _, published := range k.events {
if published.EventID == eventID {
n++
}
}
return n
}

func TestPublisherMarksPublished(t *testing.T) {
st := openTestStore(t)
ctx := context.Background()
Expand Down
6 changes: 3 additions & 3 deletions web/lib/architecture.ts
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,7 @@ export const ARCH_NODES: ArchNode[] = [
id: "outbox",
label: "Outbox",
plain: "Save first",
detail: "Postgres outbox row before Kafka — crash-safe accept, then 202.",
detail: "Postgres outbox row before Kafka — crash-safe accept, then 202. Rows left publishing after a crash go back to pending and publish once.",
code: "internal/outbox",
},
{
Expand Down Expand Up @@ -140,8 +140,8 @@ export function liveReading(
: "API probe failed.",
outbox:
live.outboxPending > 0
? `${live.outboxPending} row(s) waiting to publish.`
: "Outbox drained. Publisher caught up.",
? `${live.outboxPending} row(s) waiting to publish (pending or reclaimed mid-send).`
: "Outbox drained. Crash reclaim runs before each publish flush.",
kafka:
lag > 0
? `${lag} accepted event(s) not yet processed.`
Expand Down
Loading