Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions .env.example
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,10 @@ LOG_LEVEL=info

# make soak is local only. It refuses Railway URLs.

# Optional. stdout OpenTelemetry spans for ingest → outbox → Kafka → worker.
# Default discards spans. HMAC secrets are not span attributes.
# OTEL_TRACES=stdout

# Seed tenants (local demo only). HMAC key id defaults to v1.
# acme-casino: api_key=ak_acme_dev_001 hmac=hmac_acme_dev_secret
# nova-sports: api_key=ak_nova_dev_001 hmac=hmac_nova_dev_secret
2 changes: 1 addition & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -96,7 +96,7 @@ make demo

| What | Path |
| --- | --- |
| Send event | `POST /v1/events` (header `X-Signature`, optional `X-Key-Id`; `schema_version` omitted or `1`) |
| Send event | `POST /v1/events` (header `X-Signature`, optional `X-Key-Id`; `schema_version` omitted or `1`; response `X-Trace-Id`) |
| Rotate HMAC | `POST /v1/hmac/rotate` (header `X-API-Key`; overlap for that brand only) |
| Read player | `GET /v1/players/{id}` (header `X-API-Key`) |
| List dead letters | `GET /v1/dlq` (header `X-API-Key`) |
Expand Down
2 changes: 1 addition & 1 deletion docs/architecture.md
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@ loadgen / partner
POST /v1/events
- check JSON (`schema_version` omitted or 1; unknown versions 400)
- check brand secret (HMAC signature, optional X-Key-Id)
- save row in outbox table
- save row in outbox table (response `X-Trace-Id` is the OpenTelemetry trace)
- reply 202 Accepted
|
v
Expand Down
10 changes: 9 additions & 1 deletion go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -5,23 +5,31 @@ go 1.22
require (
github.com/jackc/pgx/v5 v5.7.4
github.com/prometheus/client_golang v1.20.5
github.com/prometheus/client_model v0.6.1
github.com/segmentio/kafka-go v0.4.47
go.opentelemetry.io/otel v1.31.0
go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.31.0
go.opentelemetry.io/otel/sdk v1.31.0
go.opentelemetry.io/otel/trace v1.31.0
google.golang.org/grpc v1.69.2
google.golang.org/protobuf v1.36.1
)

require (
github.com/beorn7/perks v1.0.1 // indirect
github.com/cespare/xxhash/v2 v2.3.0 // indirect
github.com/go-logr/logr v1.4.2 // indirect
github.com/go-logr/stdr v1.2.2 // indirect
github.com/google/uuid v1.6.0 // indirect
github.com/jackc/pgpassfile v1.0.0 // indirect
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect
github.com/jackc/puddle/v2 v2.2.2 // indirect
github.com/klauspost/compress v1.17.9 // indirect
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
github.com/pierrec/lz4/v4 v4.1.15 // indirect
github.com/prometheus/client_model v0.6.1 // indirect
github.com/prometheus/common v0.55.0 // indirect
github.com/prometheus/procfs v0.15.1 // indirect
go.opentelemetry.io/otel/metric v1.31.0 // indirect
golang.org/x/crypto v0.31.0 // indirect
golang.org/x/net v0.30.0 // indirect
golang.org/x/sync v0.10.0 // indirect
Expand Down
3 changes: 3 additions & 0 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XL
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY=
github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=
Expand Down Expand Up @@ -61,6 +62,8 @@ github.com/xdg-go/stringprep v1.0.4/go.mod h1:mPGuuIYwz7CmR2bT9j4GbQqutWS1zV24gi
github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
go.opentelemetry.io/otel v1.31.0 h1:NsJcKPIW0D0H3NgzPDHmo0WW6SptzPdqg/L1zsIm2hY=
go.opentelemetry.io/otel v1.31.0/go.mod h1:O0C14Yl9FgkjqcCZAsE053C13OaddMYr/hz6clDkEJE=
go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.31.0 h1:UGZ1QwZWY67Z6BmckTU+9Rxn04m2bD3gD6Mk0OIOCPk=
go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.31.0/go.mod h1:fcwWuDuaObkkChiDlhEpSq9+X1C0omv+s5mBtToAQ64=
go.opentelemetry.io/otel/metric v1.31.0 h1:FSErL0ATQAmYHUIzSezZibnyVlft1ybhy4ozRPcF2fE=
go.opentelemetry.io/otel/metric v1.31.0/go.mod h1:C3dEloVbLuYoX41KpmAhOqNriGbA+qqH6PQ5E5mUfnY=
go.opentelemetry.io/otel/sdk v1.31.0 h1:xLY3abVHYZ5HSfOg3l2E5LUj2Cwva5Y7yGxnSW9H5Gk=
Expand Down
34 changes: 20 additions & 14 deletions internal/api/httpapi/server.go
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ import (
"github.com/o-mid/engagepulse/internal/kafka"
"github.com/o-mid/engagepulse/internal/metrics"
"github.com/o-mid/engagepulse/internal/store"
"github.com/o-mid/engagepulse/internal/tracing"
)

type EventAccepter interface {
Expand Down Expand Up @@ -138,17 +139,15 @@ func (s *Server) acceptSignedEvent(w http.ResponseWriter, r *http.Request, body
return
}

ctx, span := tracing.Start(r.Context(), "ingest", tracing.EventAttrs(evt.EventID, evt.TenantID)...)
defer span.End()
evt.TraceID = tracing.TraceID(ctx)

// 202 = saved in the outbox. Kafka send happens in the background.
err = s.accept.EnqueueEvent(r.Context(), evt)
err = s.accept.EnqueueEvent(ctx, evt)
if errors.Is(err, store.ErrDuplicateEvent) {
// Same event_id again — already saved; still return 202.
metrics.EventsIngested.Inc()
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(http.StatusAccepted)
_ = json.NewEncoder(w).Encode(map[string]string{
"status": "accepted",
"event_id": evt.EventID,
})
writeAccepted(w, evt)
return
}
if err != nil {
Expand All @@ -158,12 +157,7 @@ func (s *Server) acceptSignedEvent(w http.ResponseWriter, r *http.Request, body
}

metrics.EventsIngested.Inc()
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(http.StatusAccepted)
_ = json.NewEncoder(w).Encode(map[string]string{
"status": "accepted",
"event_id": evt.EventID,
})
writeAccepted(w, evt)
}

func (s *Server) handleGetPlayer(w http.ResponseWriter, r *http.Request) {
Expand All @@ -190,3 +184,15 @@ func (s *Server) handleGetPlayer(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
_ = json.NewEncoder(w).Encode(snap)
}

func writeAccepted(w http.ResponseWriter, evt domain.Event) {
if evt.TraceID != "" {
w.Header().Set("X-Trace-Id", evt.TraceID)
}
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(http.StatusAccepted)
_ = json.NewEncoder(w).Encode(map[string]string{
"status": "accepted",
"event_id": evt.EventID,
})
}
55 changes: 55 additions & 0 deletions internal/api/httpapi/trace_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,55 @@
package httpapi_test

import (
"bytes"
"encoding/json"
"fmt"
"io"
"net/http"
"testing"
"time"

"github.com/o-mid/engagepulse/internal/domain"
"github.com/o-mid/engagepulse/internal/ingest"
"go.opentelemetry.io/otel/trace"
)

func TestIngestReturnsTraceID(t *testing.T) {
st, srv := newSecurityTestServer(t)
defer st.Close()
defer srv.Close()

suffix := fmt.Sprintf("%d", time.Now().UnixNano())
evt := domain.Event{
EventID: "trace-" + suffix,
TenantID: "acme-casino",
PlayerID: "trace-player-" + suffix,
Type: domain.EventDeposit,
Amount: 50,
OccurredAt: time.Now().UTC(),
}
body, err := json.Marshal(evt)
if err != nil {
t.Fatalf("marshal: %v", err)
}
req, err := http.NewRequest(http.MethodPost, srv.URL+"/v1/events", bytes.NewReader(body))
if err != nil {
t.Fatalf("request: %v", err)
}
req.Header.Set("Content-Type", "application/json")
req.Header.Set("X-Signature", ingest.Sign("hmac_acme_dev_secret", body))

resp, err := http.DefaultClient.Do(req)
if err != nil {
t.Fatalf("do: %v", err)
}
defer func() { _ = resp.Body.Close() }()
if resp.StatusCode != http.StatusAccepted {
b, _ := io.ReadAll(resp.Body)
t.Fatalf("status=%d body=%s", resp.StatusCode, b)
}
id := resp.Header.Get("X-Trace-Id")
if _, err := trace.TraceIDFromHex(id); err != nil {
t.Fatalf("X-Trace-Id %q: %v", id, err)
}
}
7 changes: 7 additions & 0 deletions internal/app/app.go
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ import (
"github.com/o-mid/engagepulse/internal/metrics"
"github.com/o-mid/engagepulse/internal/outbox"
"github.com/o-mid/engagepulse/internal/store"
"github.com/o-mid/engagepulse/internal/tracing"
"github.com/o-mid/engagepulse/internal/worker"
)

Expand All @@ -44,6 +45,12 @@ func New(cfg config.Config) *App {
}

func (a *App) Run(ctx context.Context) error {
tp, err := tracing.Setup()
if err != nil {
return fmt.Errorf("tracing: %w", err)
}
defer func() { _ = tp.Shutdown(context.Background()) }()

st, err := store.Open(ctx, a.cfg.DatabaseURL)
if err != nil {
return err
Expand Down
1 change: 1 addition & 0 deletions internal/domain/event.go
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ type Event struct {
TenantID string `json:"tenant_id"`
PlayerID string `json:"player_id"`
Type string `json:"type"`
TraceID string `json:"trace_id,omitempty"`
Amount int64 `json:"amount,omitempty"`
SchemaVersion int `json:"schema_version,omitempty"`
}
23 changes: 17 additions & 6 deletions internal/kafka/consumer.go
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ import (

"github.com/o-mid/engagepulse/internal/domain"
"github.com/o-mid/engagepulse/internal/metrics"
"github.com/o-mid/engagepulse/internal/tracing"
kafkago "github.com/segmentio/kafka-go"
)

Expand Down Expand Up @@ -72,25 +73,35 @@ func (c *Consumer) Run(ctx context.Context) error {
}

var evt domain.Event
if err := json.Unmarshal(msg.Value, &evt); err != nil {
if err = json.Unmarshal(msg.Value, &evt); err != nil {
c.logger.Error("unmarshal event", "err", err)
// Unreadable payloads will never succeed — park a stub in the DLQ and move on.
if c.dlq != nil {
_ = c.dlq.PublishDLQ(ctx, domain.Event{EventID: "unmarshal"}, err.Error(), 1)
}
if err := c.reader.CommitMessages(ctx, msg); err != nil {
if err = c.reader.CommitMessages(ctx, msg); err != nil {
return fmt.Errorf("commit poison message: %w", err)
}
continue
}

if err := c.processEvent(ctx, evt); err != nil {
if evt.TraceID == "" {
for _, h := range msg.Headers {
if h.Key == "trace_id" {
evt.TraceID = string(h.Value)
break
}
}
}
msgCtx := tracing.ContextWithTraceID(ctx, evt.TraceID)
msgCtx, span := tracing.Start(msgCtx, "kafka.consume", tracing.EventAttrs(evt.EventID, evt.TenantID)...)
err = c.processEvent(msgCtx, evt)
span.End()
if err != nil {
c.logger.Error("process event", "event_id", evt.EventID, "err", err)
// DLQ publish failed: keep the offset uncommitted so Kafka can redeliver.
continue
}

if err := c.reader.CommitMessages(ctx, msg); err != nil {
if err = c.reader.CommitMessages(ctx, msg); err != nil {
return fmt.Errorf("commit message: %w", err)
}
}
Expand Down
3 changes: 3 additions & 0 deletions internal/kafka/producer.go
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,9 @@ func (p *Producer) Publish(ctx context.Context, evt domain.Event) error {
Value: payload,
Time: time.Now().UTC(),
}
if evt.TraceID != "" {
msg.Headers = []kafkago.Header{{Key: "trace_id", Value: []byte(evt.TraceID)}}
}
if err := p.writer.WriteMessages(ctx, msg); err != nil {
return fmt.Errorf("write kafka message: %w", err)
}
Expand Down
8 changes: 6 additions & 2 deletions internal/outbox/publisher.go
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ import (
"github.com/o-mid/engagepulse/internal/domain"
"github.com/o-mid/engagepulse/internal/metrics"
"github.com/o-mid/engagepulse/internal/store"
"github.com/o-mid/engagepulse/internal/tracing"
)

type EventPublisher interface {
Expand Down Expand Up @@ -82,12 +83,15 @@ func (p *Publisher) flush(ctx context.Context) error {
p.logger.Error("outbox unmarshal", "id", row.ID, "err", err)
continue
}
if err := p.kafka.Publish(ctx, evt); err != nil {
// Send failed — mark pending again and try on the next loop.
pubCtx := tracing.ContextWithTraceID(ctx, evt.TraceID)
pubCtx, span := tracing.Start(pubCtx, "outbox.publish", tracing.EventAttrs(evt.EventID, evt.TenantID)...)
if err := p.kafka.Publish(pubCtx, evt); err != nil {
span.End()
_ = p.store.MarkOutboxPublishFailed(ctx, row.ID, err)
p.logger.Error("outbox publish", "event_id", evt.EventID, "err", err)
continue
}
span.End()
if err := p.store.MarkOutboxPublished(ctx, row.ID); err != nil {
return err
}
Expand Down
Loading
Loading