diff --git a/.env.example b/.env.example index ee8af4e..d784594 100644 --- a/.env.example +++ b/.env.example @@ -23,6 +23,10 @@ LOG_LEVEL=info # make soak is local only. It refuses Railway URLs. +# Optional. stdout OpenTelemetry spans for ingest → outbox → Kafka → worker. +# Default discards spans. HMAC secrets are not span attributes. +# OTEL_TRACES=stdout + # Seed tenants (local demo only). HMAC key id defaults to v1. # acme-casino: api_key=ak_acme_dev_001 hmac=hmac_acme_dev_secret # nova-sports: api_key=ak_nova_dev_001 hmac=hmac_nova_dev_secret diff --git a/README.md b/README.md index 21cc62e..e04334f 100644 --- a/README.md +++ b/README.md @@ -96,7 +96,7 @@ make demo | What | Path | | --- | --- | -| Send event | `POST /v1/events` (header `X-Signature`, optional `X-Key-Id`; `schema_version` omitted or `1`) | +| Send event | `POST /v1/events` (header `X-Signature`, optional `X-Key-Id`; `schema_version` omitted or `1`; response `X-Trace-Id`) | | Rotate HMAC | `POST /v1/hmac/rotate` (header `X-API-Key`; overlap for that brand only) | | Read player | `GET /v1/players/{id}` (header `X-API-Key`) | | List dead letters | `GET /v1/dlq` (header `X-API-Key`) | diff --git a/docs/architecture.md b/docs/architecture.md index f4b6117..f5f5587 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -30,7 +30,7 @@ loadgen / partner POST /v1/events - check JSON (`schema_version` omitted or 1; unknown versions 400) - check brand secret (HMAC signature, optional X-Key-Id) - - save row in outbox table + - save row in outbox table (response `X-Trace-Id` is the OpenTelemetry trace) - reply 202 Accepted | v diff --git a/go.mod b/go.mod index 5fbb226..9d7563e 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,12 @@ go 1.22 require ( github.com/jackc/pgx/v5 v5.7.4 github.com/prometheus/client_golang v1.20.5 + github.com/prometheus/client_model v0.6.1 github.com/segmentio/kafka-go v0.4.47 + go.opentelemetry.io/otel v1.31.0 + go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.31.0 + go.opentelemetry.io/otel/sdk v1.31.0 + go.opentelemetry.io/otel/trace v1.31.0 google.golang.org/grpc v1.69.2 google.golang.org/protobuf v1.36.1 ) @@ -13,15 +18,18 @@ require ( require ( github.com/beorn7/perks v1.0.1 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/go-logr/logr v1.4.2 // indirect + github.com/go-logr/stdr v1.2.2 // indirect + github.com/google/uuid v1.6.0 // indirect github.com/jackc/pgpassfile v1.0.0 // indirect github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect github.com/jackc/puddle/v2 v2.2.2 // indirect github.com/klauspost/compress v1.17.9 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/pierrec/lz4/v4 v4.1.15 // indirect - github.com/prometheus/client_model v0.6.1 // indirect github.com/prometheus/common v0.55.0 // indirect github.com/prometheus/procfs v0.15.1 // indirect + go.opentelemetry.io/otel/metric v1.31.0 // indirect golang.org/x/crypto v0.31.0 // indirect golang.org/x/net v0.30.0 // indirect golang.org/x/sync v0.10.0 // indirect diff --git a/go.sum b/go.sum index e4eb985..3ac41eb 100644 --- a/go.sum +++ b/go.sum @@ -5,6 +5,7 @@ github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XL github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= @@ -61,6 +62,8 @@ github.com/xdg-go/stringprep v1.0.4/go.mod h1:mPGuuIYwz7CmR2bT9j4GbQqutWS1zV24gi github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= go.opentelemetry.io/otel v1.31.0 h1:NsJcKPIW0D0H3NgzPDHmo0WW6SptzPdqg/L1zsIm2hY= go.opentelemetry.io/otel v1.31.0/go.mod h1:O0C14Yl9FgkjqcCZAsE053C13OaddMYr/hz6clDkEJE= +go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.31.0 h1:UGZ1QwZWY67Z6BmckTU+9Rxn04m2bD3gD6Mk0OIOCPk= +go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.31.0/go.mod h1:fcwWuDuaObkkChiDlhEpSq9+X1C0omv+s5mBtToAQ64= go.opentelemetry.io/otel/metric v1.31.0 h1:FSErL0ATQAmYHUIzSezZibnyVlft1ybhy4ozRPcF2fE= go.opentelemetry.io/otel/metric v1.31.0/go.mod h1:C3dEloVbLuYoX41KpmAhOqNriGbA+qqH6PQ5E5mUfnY= go.opentelemetry.io/otel/sdk v1.31.0 h1:xLY3abVHYZ5HSfOg3l2E5LUj2Cwva5Y7yGxnSW9H5Gk= diff --git a/internal/api/httpapi/server.go b/internal/api/httpapi/server.go index 410c932..2f5e36d 100644 --- a/internal/api/httpapi/server.go +++ b/internal/api/httpapi/server.go @@ -14,6 +14,7 @@ import ( "github.com/o-mid/engagepulse/internal/kafka" "github.com/o-mid/engagepulse/internal/metrics" "github.com/o-mid/engagepulse/internal/store" + "github.com/o-mid/engagepulse/internal/tracing" ) type EventAccepter interface { @@ -138,17 +139,15 @@ func (s *Server) acceptSignedEvent(w http.ResponseWriter, r *http.Request, body return } + ctx, span := tracing.Start(r.Context(), "ingest", tracing.EventAttrs(evt.EventID, evt.TenantID)...) + defer span.End() + evt.TraceID = tracing.TraceID(ctx) + // 202 = saved in the outbox. Kafka send happens in the background. - err = s.accept.EnqueueEvent(r.Context(), evt) + err = s.accept.EnqueueEvent(ctx, evt) if errors.Is(err, store.ErrDuplicateEvent) { - // Same event_id again — already saved; still return 202. metrics.EventsIngested.Inc() - w.Header().Set("Content-Type", "application/json") - w.WriteHeader(http.StatusAccepted) - _ = json.NewEncoder(w).Encode(map[string]string{ - "status": "accepted", - "event_id": evt.EventID, - }) + writeAccepted(w, evt) return } if err != nil { @@ -158,12 +157,7 @@ func (s *Server) acceptSignedEvent(w http.ResponseWriter, r *http.Request, body } metrics.EventsIngested.Inc() - w.Header().Set("Content-Type", "application/json") - w.WriteHeader(http.StatusAccepted) - _ = json.NewEncoder(w).Encode(map[string]string{ - "status": "accepted", - "event_id": evt.EventID, - }) + writeAccepted(w, evt) } func (s *Server) handleGetPlayer(w http.ResponseWriter, r *http.Request) { @@ -190,3 +184,15 @@ func (s *Server) handleGetPlayer(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "application/json") _ = json.NewEncoder(w).Encode(snap) } + +func writeAccepted(w http.ResponseWriter, evt domain.Event) { + if evt.TraceID != "" { + w.Header().Set("X-Trace-Id", evt.TraceID) + } + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusAccepted) + _ = json.NewEncoder(w).Encode(map[string]string{ + "status": "accepted", + "event_id": evt.EventID, + }) +} diff --git a/internal/api/httpapi/trace_test.go b/internal/api/httpapi/trace_test.go new file mode 100644 index 0000000..b16e5e7 --- /dev/null +++ b/internal/api/httpapi/trace_test.go @@ -0,0 +1,55 @@ +package httpapi_test + +import ( + "bytes" + "encoding/json" + "fmt" + "io" + "net/http" + "testing" + "time" + + "github.com/o-mid/engagepulse/internal/domain" + "github.com/o-mid/engagepulse/internal/ingest" + "go.opentelemetry.io/otel/trace" +) + +func TestIngestReturnsTraceID(t *testing.T) { + st, srv := newSecurityTestServer(t) + defer st.Close() + defer srv.Close() + + suffix := fmt.Sprintf("%d", time.Now().UnixNano()) + evt := domain.Event{ + EventID: "trace-" + suffix, + TenantID: "acme-casino", + PlayerID: "trace-player-" + suffix, + Type: domain.EventDeposit, + Amount: 50, + OccurredAt: time.Now().UTC(), + } + body, err := json.Marshal(evt) + if err != nil { + t.Fatalf("marshal: %v", err) + } + req, err := http.NewRequest(http.MethodPost, srv.URL+"/v1/events", bytes.NewReader(body)) + if err != nil { + t.Fatalf("request: %v", err) + } + req.Header.Set("Content-Type", "application/json") + req.Header.Set("X-Signature", ingest.Sign("hmac_acme_dev_secret", body)) + + resp, err := http.DefaultClient.Do(req) + if err != nil { + t.Fatalf("do: %v", err) + } + defer func() { _ = resp.Body.Close() }() + if resp.StatusCode != http.StatusAccepted { + b, _ := io.ReadAll(resp.Body) + t.Fatalf("status=%d body=%s", resp.StatusCode, b) + } + id := resp.Header.Get("X-Trace-Id") + if _, err := trace.TraceIDFromHex(id); err != nil { + t.Fatalf("X-Trace-Id %q: %v", id, err) + } +} diff --git a/internal/app/app.go b/internal/app/app.go index c956deb..91895e1 100644 --- a/internal/app/app.go +++ b/internal/app/app.go @@ -20,6 +20,7 @@ import ( "github.com/o-mid/engagepulse/internal/metrics" "github.com/o-mid/engagepulse/internal/outbox" "github.com/o-mid/engagepulse/internal/store" + "github.com/o-mid/engagepulse/internal/tracing" "github.com/o-mid/engagepulse/internal/worker" ) @@ -44,6 +45,12 @@ func New(cfg config.Config) *App { } func (a *App) Run(ctx context.Context) error { + tp, err := tracing.Setup() + if err != nil { + return fmt.Errorf("tracing: %w", err) + } + defer func() { _ = tp.Shutdown(context.Background()) }() + st, err := store.Open(ctx, a.cfg.DatabaseURL) if err != nil { return err diff --git a/internal/domain/event.go b/internal/domain/event.go index df67164..15e7895 100644 --- a/internal/domain/event.go +++ b/internal/domain/event.go @@ -15,6 +15,7 @@ type Event struct { TenantID string `json:"tenant_id"` PlayerID string `json:"player_id"` Type string `json:"type"` + TraceID string `json:"trace_id,omitempty"` Amount int64 `json:"amount,omitempty"` SchemaVersion int `json:"schema_version,omitempty"` } diff --git a/internal/kafka/consumer.go b/internal/kafka/consumer.go index d60ad2e..b39730e 100644 --- a/internal/kafka/consumer.go +++ b/internal/kafka/consumer.go @@ -9,6 +9,7 @@ import ( "github.com/o-mid/engagepulse/internal/domain" "github.com/o-mid/engagepulse/internal/metrics" + "github.com/o-mid/engagepulse/internal/tracing" kafkago "github.com/segmentio/kafka-go" ) @@ -72,25 +73,35 @@ func (c *Consumer) Run(ctx context.Context) error { } var evt domain.Event - if err := json.Unmarshal(msg.Value, &evt); err != nil { + if err = json.Unmarshal(msg.Value, &evt); err != nil { c.logger.Error("unmarshal event", "err", err) // Unreadable payloads will never succeed — park a stub in the DLQ and move on. if c.dlq != nil { _ = c.dlq.PublishDLQ(ctx, domain.Event{EventID: "unmarshal"}, err.Error(), 1) } - if err := c.reader.CommitMessages(ctx, msg); err != nil { + if err = c.reader.CommitMessages(ctx, msg); err != nil { return fmt.Errorf("commit poison message: %w", err) } continue } - - if err := c.processEvent(ctx, evt); err != nil { + if evt.TraceID == "" { + for _, h := range msg.Headers { + if h.Key == "trace_id" { + evt.TraceID = string(h.Value) + break + } + } + } + msgCtx := tracing.ContextWithTraceID(ctx, evt.TraceID) + msgCtx, span := tracing.Start(msgCtx, "kafka.consume", tracing.EventAttrs(evt.EventID, evt.TenantID)...) + err = c.processEvent(msgCtx, evt) + span.End() + if err != nil { c.logger.Error("process event", "event_id", evt.EventID, "err", err) - // DLQ publish failed: keep the offset uncommitted so Kafka can redeliver. continue } - if err := c.reader.CommitMessages(ctx, msg); err != nil { + if err = c.reader.CommitMessages(ctx, msg); err != nil { return fmt.Errorf("commit message: %w", err) } } diff --git a/internal/kafka/producer.go b/internal/kafka/producer.go index 3a845c5..e7bcce8 100644 --- a/internal/kafka/producer.go +++ b/internal/kafka/producer.go @@ -40,6 +40,9 @@ func (p *Producer) Publish(ctx context.Context, evt domain.Event) error { Value: payload, Time: time.Now().UTC(), } + if evt.TraceID != "" { + msg.Headers = []kafkago.Header{{Key: "trace_id", Value: []byte(evt.TraceID)}} + } if err := p.writer.WriteMessages(ctx, msg); err != nil { return fmt.Errorf("write kafka message: %w", err) } diff --git a/internal/outbox/publisher.go b/internal/outbox/publisher.go index 51d997c..dae45cc 100644 --- a/internal/outbox/publisher.go +++ b/internal/outbox/publisher.go @@ -9,6 +9,7 @@ import ( "github.com/o-mid/engagepulse/internal/domain" "github.com/o-mid/engagepulse/internal/metrics" "github.com/o-mid/engagepulse/internal/store" + "github.com/o-mid/engagepulse/internal/tracing" ) type EventPublisher interface { @@ -82,12 +83,15 @@ func (p *Publisher) flush(ctx context.Context) error { p.logger.Error("outbox unmarshal", "id", row.ID, "err", err) continue } - if err := p.kafka.Publish(ctx, evt); err != nil { - // Send failed — mark pending again and try on the next loop. + pubCtx := tracing.ContextWithTraceID(ctx, evt.TraceID) + pubCtx, span := tracing.Start(pubCtx, "outbox.publish", tracing.EventAttrs(evt.EventID, evt.TenantID)...) + if err := p.kafka.Publish(pubCtx, evt); err != nil { + span.End() _ = p.store.MarkOutboxPublishFailed(ctx, row.ID, err) p.logger.Error("outbox publish", "event_id", evt.EventID, "err", err) continue } + span.End() if err := p.store.MarkOutboxPublished(ctx, row.ID); err != nil { return err } diff --git a/internal/tracing/tracing.go b/internal/tracing/tracing.go new file mode 100644 index 0000000..7cbd3c7 --- /dev/null +++ b/internal/tracing/tracing.go @@ -0,0 +1,90 @@ +package tracing + +import ( + "context" + "crypto/rand" + "encoding/hex" + "io" + "os" + + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/exporters/stdout/stdouttrace" + "go.opentelemetry.io/otel/sdk/resource" + sdktrace "go.opentelemetry.io/otel/sdk/trace" + semconv "go.opentelemetry.io/otel/semconv/v1.26.0" + "go.opentelemetry.io/otel/trace" +) + +const TracerName = "engagepulse" + +func Setup() (*sdktrace.TracerProvider, error) { + var w io.Writer = io.Discard + if os.Getenv("OTEL_TRACES") == "stdout" { + w = os.Stdout + } + exp, err := stdouttrace.New(stdouttrace.WithWriter(w)) + if err != nil { + return nil, err + } + tp := sdktrace.NewTracerProvider( + sdktrace.WithBatcher(exp), + sdktrace.WithSampler(sdktrace.AlwaysSample()), + sdktrace.WithResource(resource.NewWithAttributes( + semconv.SchemaURL, + semconv.ServiceName("engagepulse"), + )), + ) + otel.SetTracerProvider(tp) + return tp, nil +} + +func Start(ctx context.Context, name string, attrs ...attribute.KeyValue) (context.Context, trace.Span) { + return otel.Tracer(TracerName).Start(ctx, name, trace.WithAttributes(attrs...)) +} + +func EventAttrs(eventID, tenantID string) []attribute.KeyValue { + return []attribute.KeyValue{ + attribute.String("event_id", eventID), + attribute.String("tenant_id", tenantID), + } +} + +func TraceID(ctx context.Context) string { + sc := trace.SpanContextFromContext(ctx) + if sc.IsValid() { + return sc.TraceID().String() + } + return newTraceID() +} + +func ContextWithTraceID(ctx context.Context, id string) context.Context { + tid, err := trace.TraceIDFromHex(id) + if err != nil { + return ctx + } + sc := trace.NewSpanContext(trace.SpanContextConfig{ + TraceID: tid, + SpanID: newSpanID(), + TraceFlags: trace.FlagsSampled, + Remote: true, + }) + if !sc.IsValid() { + return ctx + } + return trace.ContextWithRemoteSpanContext(ctx, sc) +} + +func newTraceID() string { + var b [16]byte + if _, err := rand.Read(b[:]); err != nil { + return hex.EncodeToString(make([]byte, 16)) + } + return hex.EncodeToString(b[:]) +} + +func newSpanID() trace.SpanID { + var b [8]byte + _, _ = rand.Read(b[:]) + return trace.SpanID(b) +} diff --git a/internal/tracing/tracing_test.go b/internal/tracing/tracing_test.go new file mode 100644 index 0000000..e52aef6 --- /dev/null +++ b/internal/tracing/tracing_test.go @@ -0,0 +1,36 @@ +package tracing + +import ( + "context" + "testing" + + "go.opentelemetry.io/otel/trace" +) + +func TestSetupProducesValidTraceID(t *testing.T) { + tp, err := Setup() + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + _ = tp.Shutdown(context.Background()) + }) + ctx, span := Start(context.Background(), "test") + defer span.End() + id := TraceID(ctx) + if len(id) != 32 { + t.Fatalf("trace id len=%d want 32", len(id)) + } + if _, err := trace.TraceIDFromHex(id); err != nil { + t.Fatalf("hex: %v", err) + } +} + +func TestContextWithTraceID(t *testing.T) { + id := newTraceID() + ctx := ContextWithTraceID(context.Background(), id) + got := trace.SpanContextFromContext(ctx).TraceID().String() + if got != id { + t.Fatalf("got %s want %s", got, id) + } +} diff --git a/internal/worker/worker.go b/internal/worker/worker.go index 494dc19..5fabeb7 100644 --- a/internal/worker/worker.go +++ b/internal/worker/worker.go @@ -13,6 +13,7 @@ import ( "github.com/o-mid/engagepulse/internal/metrics" "github.com/o-mid/engagepulse/internal/rules" "github.com/o-mid/engagepulse/internal/store" + "github.com/o-mid/engagepulse/internal/tracing" ) const FailInjectPrefix = "poison-" @@ -54,6 +55,10 @@ func NewWithRules(st *store.Store, logger *slog.Logger, eng *rules.Engine) *Work } func (w *Worker) Handle(ctx context.Context, evt domain.Event) error { + ctx = tracing.ContextWithTraceID(ctx, evt.TraceID) + ctx, span := tracing.Start(ctx, "worker.handle", tracing.EventAttrs(evt.EventID, evt.TenantID)...) + defer span.End() + var ( ruleHits []string creditAmount int64