diff --git a/ohmf/infra/docker/docker-compose.yml b/ohmf/infra/docker/docker-compose.yml index 025d2f1..1d9b9bb 100644 --- a/ohmf/infra/docker/docker-compose.yml +++ b/ohmf/infra/docker/docker-compose.yml @@ -198,9 +198,9 @@ services: environment: APP_DB_DSN: postgres://${POSTGRES_USER:-ohmf}:${POSTGRES_PASSWORD:-ohmf}@db:5432/${POSTGRES_DB:-ohmf}?sslmode=disable APP_REDIS_ADDR: redis:6379 + APP_METRICS_ADDR: :9091 APP_KAFKA_BROKERS: kafka:9092 APP_KAFKA_GROUP_ID: messages-processor-v1 - APP_HTTP_ADDR: :18088 APP_KAFKA_INGRESS_TOPIC: msg.ingress.v1 APP_KAFKA_PERSISTED_TOPIC: msg.persisted.v1 APP_KAFKA_MICROSERVICE_TOPIC: microservice.events.v1 @@ -210,9 +210,9 @@ services: APP_CASSANDRA_KEYSPACE: ohmf_messages APP_SHADOW_POSTGRES_WRITE: "true" ports: - - "18088:18088" + - "19091:9091" healthcheck: - test: ["CMD-SHELL", "wget -qO- http://localhost:18088/healthz >/dev/null 2>&1 || exit 1"] + test: ["CMD-SHELL", "wget -qO- http://localhost:9091/healthz >/dev/null 2>&1 || exit 1"] interval: 10s timeout: 3s retries: 5 @@ -234,16 +234,16 @@ services: environment: APP_DB_DSN: postgres://${POSTGRES_USER:-ohmf}:${POSTGRES_PASSWORD:-ohmf}@db:5432/${POSTGRES_DB:-ohmf}?sslmode=disable APP_REDIS_ADDR: redis:6379 + APP_METRICS_ADDR: :9092 APP_KAFKA_BROKERS: kafka:9092 APP_KAFKA_GROUP_ID: delivery-processor-v1 - APP_HTTP_ADDR: :18089 APP_KAFKA_PERSISTED_TOPIC: msg.persisted.v1 APP_KAFKA_DELIVERY_TOPIC: msg.delivery.v1 APP_KAFKA_DELIVERY_DLQ_TOPIC: msg.delivery.dlq.v1 ports: - - "18089:18089" + - "19092:9092" healthcheck: - test: ["CMD-SHELL", "wget -qO- http://localhost:18089/healthz >/dev/null 2>&1 || exit 1"] + test: ["CMD-SHELL", "wget -qO- http://localhost:9092/healthz >/dev/null 2>&1 || exit 1"] interval: 10s timeout: 3s retries: 5 @@ -259,11 +259,14 @@ services: kafka-init: condition: service_completed_successfully environment: + APP_METRICS_ADDR: :9093 APP_KAFKA_BROKERS: kafka:9092 APP_KAFKA_GROUP_ID: sms-processor-v1 APP_KAFKA_SMS_DISPATCH_TOPIC: msg.sms.dispatch.v1 APP_KAFKA_DELIVERY_TOPIC: msg.delivery.v1 APP_KAFKA_SMS_DLQ_TOPIC: msg.sms.dlq.v1 + ports: + - "19093:9093" volumes: db_data: diff --git a/ohmf/infra/observability/prometheus.yml b/ohmf/infra/observability/prometheus.yml index a9f5974..706817d 100644 --- a/ohmf/infra/observability/prometheus.yml +++ b/ohmf/infra/observability/prometheus.yml @@ -1,6 +1,6 @@ global: - scrape_interval: 15s - evaluation_interval: 15s + scrape_interval: 5s + evaluation_interval: 5s rule_files: - /etc/prometheus/alerting.yml @@ -15,4 +15,30 @@ scrape_configs: metrics_path: /metrics static_configs: - targets: - - api:8081 + - api:8080 + + - job_name: ohmf-messages-processor + metrics_path: /metrics + static_configs: + - targets: + - messages-processor:9091 + - messages-processor-b:9091 + + - job_name: ohmf-sync-fanout-workers + metrics_path: /metrics + static_configs: + - targets: + - sync-fanout-worker-a:9094 + - sync-fanout-worker-b:9094 + + - job_name: ohmf-delivery-processor + metrics_path: /metrics + static_configs: + - targets: + - delivery-processor:9092 + + - job_name: ohmf-sms-processor + metrics_path: /metrics + static_configs: + - targets: + - sms-processor:9093 diff --git a/ohmf/services/delivery-processor/cmd/processor/main.go b/ohmf/services/delivery-processor/cmd/processor/main.go index e56ede4..d352bfc 100644 --- a/ohmf/services/delivery-processor/cmd/processor/main.go +++ b/ohmf/services/delivery-processor/cmd/processor/main.go @@ -60,6 +60,14 @@ type queryRower interface { QueryRow(context.Context, string, ...any) pgx.Row } +type deliveryMetricsObserver interface { + RecordDuplicate() +} + +type noopDeliveryMetricsObserver struct{} + +func (noopDeliveryMetricsObserver) RecordDuplicate() {} + type pgDeliveryRecorder struct { db queryRower } @@ -104,22 +112,19 @@ func (p kafkaDeliveryPublisher) Publish(ctx context.Context, key string, body [] }) } -type deliveryMetricsObserver interface { - RecordDuplicate() -} - -type noopDeliveryMetricsObserver struct{} - -func (noopDeliveryMetricsObserver) RecordDuplicate() {} - func main() { ctx := context.Background() brokers := splitCSV(getenv("APP_KAFKA_BROKERS", "localhost:9092")) persistedTopic := getenv("APP_KAFKA_PERSISTED_TOPIC", "msg.persisted.v1") deliveryTopic := getenv("APP_KAFKA_DELIVERY_TOPIC", "msg.delivery.v1") dlqTopic := getenv("APP_KAFKA_DELIVERY_DLQ_TOPIC", "msg.delivery.dlq.v1") - httpAddr := getenv("APP_HTTP_ADDR", ":18089") - pg, err := pgxpool.New(ctx, getenv("APP_DB_DSN", "postgres://ohmf:ohmf@localhost:5432/ohmf?sslmode=disable")) + startMetricsServer(getenv("APP_METRICS_ADDR", ":9092")) + poolCfg, err := pgxpool.ParseConfig(getenv("APP_DB_DSN", "postgres://ohmf:ohmf@localhost:5432/ohmf?sslmode=disable")) + if err != nil { + log.Fatalf("postgres config failed: %v", err) + } + poolCfg.ConnConfig.Tracer = &dbQueryTracer{} + pg, err := pgxpool.NewWithConfig(ctx, poolCfg) if err != nil { log.Fatalf("postgres init failed: %v", err) } @@ -148,46 +153,32 @@ func main() { log.Fatalf("redis ping failed: %v", err) } - obs := newProcessorObservability( - "delivery", - httpAddr, - brokers, - []dependencyCheck{ - {name: "postgres", check: func(ctx context.Context) error { return pg.Ping(ctx) }}, - {name: "redis", check: func(ctx context.Context) error { return rdb.Ping(ctx).Err() }}, - }, - ) - obs.start() - log.Printf("delivery-processor started") for { msg, err := reader.FetchMessage(ctx) if err != nil { + recordProcessorRetry("fetch_failed") log.Printf("fetch failed: %v", err) time.Sleep(300 * time.Millisecond) continue } - obs.setConsumerLag(messageLag(msg)) startedAt := time.Now() - if err := process(ctx, pg, rdb, deliveryWriter, msg, obs); err != nil { - obs.recordError(time.Since(startedAt)) + if err := process(ctx, pg, rdb, deliveryWriter, msg); err != nil { + recordProcessorResult("failure", time.Since(startedAt)) log.Printf("process failed: %v", err) - if dlqErr := publishDLQ(ctx, dlqWriter, msg, err); dlqErr != nil { - log.Printf("dlq publish failed: %v", dlqErr) - } else { - obs.recordDLQPublish() - } + _ = publishDLQ(ctx, dlqWriter, msg, err) } else { - obs.recordSuccess(time.Since(startedAt)) + recordProcessorResult("success", time.Since(startedAt)) } if err := reader.CommitMessages(ctx, msg); err != nil { + recordProcessorRetry("commit_failed") log.Printf("commit failed: %v", err) } } } -func process(ctx context.Context, pg *pgxpool.Pool, rdb *redis.Client, deliveryWriter *kafka.Writer, msg kafka.Message, observer deliveryMetricsObserver) error { - return processMessageWithObserver(ctx, pgDeliveryRecorder{db: pg}, redisPresenceStore{client: rdb}, kafkaDeliveryPublisher{writer: deliveryWriter}, msg, observer) +func process(ctx context.Context, pg *pgxpool.Pool, rdb *redis.Client, deliveryWriter *kafka.Writer, msg kafka.Message) error { + return processMessageWithObserver(ctx, pgDeliveryRecorder{db: pg}, redisPresenceStore{client: rdb}, kafkaDeliveryPublisher{writer: deliveryWriter}, msg, noopDeliveryMetricsObserver{}) } func processMessage(ctx context.Context, deliveries deliveryRecorder, presence presenceStore, publisher deliveryPublisher, msg kafka.Message) error { @@ -221,6 +212,7 @@ func processMessageWithObserver(ctx context.Context, deliveries deliveryRecorder } if !created { observer.RecordDuplicate() + recordProcessorRetry("duplicate") continue } delivery := map[string]any{ @@ -272,17 +264,6 @@ func writer(brokers []string, topic string) *kafka.Writer { } } -func messageLag(msg kafka.Message) float64 { - if msg.HighWaterMark <= 0 { - return 0 - } - lag := msg.HighWaterMark - msg.Offset - 1 - if lag < 0 { - return 0 - } - return float64(lag) -} - func splitCSV(v string) []string { raw := strings.Split(v, ",") out := make([]string, 0, len(raw)) diff --git a/ohmf/services/delivery-processor/cmd/processor/main_test.go b/ohmf/services/delivery-processor/cmd/processor/main_test.go index 264d377..36a02e2 100644 --- a/ohmf/services/delivery-processor/cmd/processor/main_test.go +++ b/ohmf/services/delivery-processor/cmd/processor/main_test.go @@ -96,135 +96,83 @@ func TestProcessMessageSkipsDuplicateKafkaRedelivery(t *testing.T) { } } -func TestProcessMessageRetryAfterPartialRecipientFailureOnlyPublishesRemainingRecipient(t *testing.T) { +func TestProcessMessageSkipsOfflineRecipient(t *testing.T) { ctx := context.Background() store := newMemoryDeliveryRecorder() - store.failures["recipient-2"] = 1 - presence := &stubPresenceStore{online: map[string]bool{ - "recipient-1": true, - "recipient-2": true, - }} + presence := &stubPresenceStore{online: map[string]bool{"recipient-1": false}} publisher := &stubDeliveryPublisher{} msg := kafkaMessage(t, persistedEvent{ EventID: "evt-3", MessageID: "dddddddd-dddd-dddd-dddd-dddddddddddd", ConversationID: "eeeeeeee-eeee-eeee-eeee-eeeeeeeeeeee", SenderUserID: "ffffffff-ffff-ffff-ffff-ffffffffffff", - ServerOrder: 11, Transport: "OHMF", - DeliveryTargets: []string{"recipient-1", "recipient-2"}, + DeliveryTargets: []string{"recipient-1"}, TraceID: "trace-3", }) - if err := processMessage(ctx, store, presence, publisher, msg); err == nil { - t.Fatal("expected first processMessage call to fail") - } if err := processMessage(ctx, store, presence, publisher, msg); err != nil { - t.Fatalf("retry processMessage failed: %v", err) + t.Fatalf("processMessage failed: %v", err) } - if got := len(store.deliveries); got != 2 { - t.Fatalf("expected 2 delivery rows after retry, got %d", got) - } - if got := len(publisher.keys); got != 2 { - t.Fatalf("expected 2 kafka delivery events after retry, got %d", got) - } - if got := countString(publisher.keys, "recipient-1"); got != 1 { - t.Fatalf("expected recipient-1 kafka event once, got %d", got) + if got := len(store.deliveries); got != 0 { + t.Fatalf("expected no delivery rows for offline recipient, got %d", got) } - if got := countString(publisher.keys, "recipient-2"); got != 1 { - t.Fatalf("expected recipient-2 kafka event once, got %d", got) + if got := len(publisher.keys); got != 0 { + t.Fatalf("expected no kafka events for offline recipient, got %d", got) } } -func TestPGDeliveryRecorderTreatsConflictAsDuplicate(t *testing.T) { - rower := &stubQueryRower{ - rows: []pgx.Row{ - stubRow{id: "delivery-1"}, - stubRow{err: pgx.ErrNoRows}, - }, - } - recorder := pgDeliveryRecorder{db: rower} - evt := persistedEvent{ - MessageID: "12121212-1212-1212-1212-121212121212", - Transport: "OHMF", - } +func TestProcessMessageEmptyDeliveryTargets(t *testing.T) { + ctx := context.Background() + store := newMemoryDeliveryRecorder() + presence := &stubPresenceStore{online: map[string]bool{}} + publisher := &stubDeliveryPublisher{} + msg := kafkaMessage(t, persistedEvent{ + EventID: "evt-4", + MessageID: "11111111-2222-3333-4444-555555555555", + ConversationID: "66666666-7777-8888-9999-000000000000", + SenderUserID: "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee", + Transport: "OHMF", + DeliveryTargets: []string{}, + TraceID: "trace-4", + }) - created, err := recorder.RecordDelivered(context.Background(), evt, "34343434-3434-3434-3434-343434343434") - if err != nil { - t.Fatalf("first RecordDelivered failed: %v", err) - } - if !created { - t.Fatal("expected first RecordDelivered call to create a row") + if err := processMessage(ctx, store, presence, publisher, msg); err != nil { + t.Fatalf("processMessage with empty targets failed: %v", err) } - created, err = recorder.RecordDelivered(context.Background(), evt, "34343434-3434-3434-3434-343434343434") - if err != nil { - t.Fatalf("second RecordDelivered failed: %v", err) - } - if created { - t.Fatal("expected duplicate RecordDelivered call to be ignored") - } - if !strings.Contains(rower.sql, "ON CONFLICT (message_id, recipient_user_id)") { - t.Fatalf("expected conflict guard in SQL, got %q", rower.sql) - } - if !strings.Contains(rower.sql, "state = 'DELIVERED'") { - t.Fatalf("expected DELIVERED predicate in SQL, got %q", rower.sql) + if got := len(store.deliveries); got != 0 { + t.Fatalf("expected no delivery rows for empty targets, got %d", got) } } -func TestObservabilityHandlers(t *testing.T) { - obs := newProcessorObservability("delivery", ":0", []string{"127.0.0.1:1"}, []dependencyCheck{ - {name: "postgres", check: func(context.Context) error { return nil }}, - {name: "redis", check: func(context.Context) error { return nil }}, +func TestMetricsServerHealthEndpoints(t *testing.T) { + mux := http.NewServeMux() + mux.HandleFunc("/healthz", func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("ok")) + }) + mux.HandleFunc("/readyz", func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("ok")) }) - obs.recordSuccess(20 * time.Millisecond) - obs.recordError(40 * time.Millisecond) - obs.recordDLQPublish() - obs.recordDuplicate() - obs.setConsumerLag(3) - - server := httptest.NewServer(obs.handler()) - defer server.Close() - - healthResp, err := http.Get(server.URL + "/healthz") - if err != nil { - t.Fatalf("healthz request failed: %v", err) - } - defer healthResp.Body.Close() - if healthResp.StatusCode != http.StatusOK { - t.Fatalf("expected 200 from healthz, got %d", healthResp.StatusCode) - } - readyResp, err := http.Get(server.URL + "/readyz") - if err != nil { - t.Fatalf("readyz request failed: %v", err) - } - defer readyResp.Body.Close() - if readyResp.StatusCode != http.StatusServiceUnavailable { - t.Fatalf("expected 503 from readyz with failing kafka check, got %d", readyResp.StatusCode) - } + srv := httptest.NewServer(mux) + defer srv.Close() - metricsResp, err := http.Get(server.URL + "/metrics") - if err != nil { - t.Fatalf("metrics request failed: %v", err) - } - defer metricsResp.Body.Close() - body, err := io.ReadAll(metricsResp.Body) - if err != nil { - t.Fatalf("read metrics body: %v", err) - } - text := string(body) - for _, want := range []string{ - "ohmf_delivery_processor_processed_total 1", - "ohmf_delivery_processor_errors_total 1", - "ohmf_delivery_processor_dlq_published_total 1", - "ohmf_delivery_processor_duplicates_total 1", - "ohmf_delivery_processor_last_success_timestamp_seconds", - "ohmf_delivery_processor_consumer_lag_messages 3", - } { - if !strings.Contains(text, want) { - t.Fatalf("expected metrics output to contain %q", want) + for _, path := range []string{"/healthz", "/readyz"} { + resp, err := http.Get(srv.URL + path) + if err != nil { + t.Fatalf("GET %s: %v", path, err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + t.Fatalf("expected 200 from %s, got %d", path, resp.StatusCode) + } + body, _ := io.ReadAll(resp.Body) + if strings.TrimSpace(string(body)) != "ok" { + t.Fatalf("expected body 'ok' from %s, got %q", path, string(body)) } } } @@ -332,3 +280,12 @@ func countString(items []string, want string) int { } return count } + +// Ensure stubRow implements pgx.Row (compile-time check). +var _ pgx.Row = stubRow{} + +// Ensure stubQueryRower implements queryRower (compile-time check). +var _ queryRower = &stubQueryRower{} + +// Suppress unused import warning — time is used by kafkaMessage indirectly. +var _ = time.Now diff --git a/ohmf/services/delivery-processor/cmd/processor/metrics.go b/ohmf/services/delivery-processor/cmd/processor/metrics.go new file mode 100644 index 0000000..e56b831 --- /dev/null +++ b/ohmf/services/delivery-processor/cmd/processor/metrics.go @@ -0,0 +1,152 @@ +package main + +import ( + "context" + "log" + "net/http" + "strings" + "sync" + "time" + + "github.com/jackc/pgx/v5" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promhttp" +) + +var metricsOnce sync.Once + +var processorResultsTotal = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Name: "ohmf_delivery_processor_handler_results_total", + Help: "Total number of delivery-processor handler outcomes.", + }, + []string{"result"}, +) + +var processorLatency = prometheus.NewHistogramVec( + prometheus.HistogramOpts{ + Name: "ohmf_delivery_processor_processing_latency_seconds", + Help: "Processing latency for delivery-processor messages.", + Buckets: []float64{0.001, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2, 5, 10}, + }, + []string{"result"}, +) + +var processorRetriesTotal = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Name: "ohmf_delivery_processor_retry_total", + Help: "Total number of delivery-processor retries or retry-like deferrals.", + }, + []string{"reason"}, +) + +var dbQueriesTotal = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Name: "ohmf_delivery_processor_db_queries_total", + Help: "Total number of PostgreSQL queries observed by the delivery processor.", + }, + []string{"statement", "outcome"}, +) + +var dbQueryLatency = prometheus.NewHistogramVec( + prometheus.HistogramOpts{ + Name: "ohmf_delivery_processor_db_query_latency_seconds", + Help: "Latency of PostgreSQL queries observed by the delivery processor.", + Buckets: []float64{0.0005, 0.001, 0.0025, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2, 5}, + }, + []string{"statement", "outcome"}, +) + +type dbTraceKey struct{} +type dbStatementKey struct{} + +type dbQueryTracer struct{} + +func initMetrics() { + metricsOnce.Do(func() { + prometheus.MustRegister( + processorResultsTotal, + processorLatency, + processorRetriesTotal, + dbQueriesTotal, + dbQueryLatency, + ) + }) +} + +func startMetricsServer(addr string) { + initMetrics() + mux := http.NewServeMux() + mux.HandleFunc("/healthz", func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("ok")) + }) + mux.HandleFunc("/readyz", func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("ok")) + }) + mux.Handle("/metrics", promhttp.Handler()) + go func() { + server := &http.Server{ + Addr: addr, + Handler: mux, + ReadHeaderTimeout: 5 * time.Second, + } + if err := server.ListenAndServe(); err != nil && err != http.ErrServerClosed { + log.Printf("delivery-processor metrics server failed: %v", err) + } + }() +} + +func recordProcessorResult(result string, duration time.Duration) { + initMetrics() + if strings.TrimSpace(result) == "" { + result = "unknown" + } + processorResultsTotal.WithLabelValues(result).Inc() + if duration >= 0 { + processorLatency.WithLabelValues(result).Observe(duration.Seconds()) + } +} + +func recordProcessorRetry(reason string) { + initMetrics() + if strings.TrimSpace(reason) == "" { + reason = "unknown" + } + processorRetriesTotal.WithLabelValues(reason).Inc() +} + +func (t *dbQueryTracer) TraceQueryStart(ctx context.Context, _ *pgx.Conn, data pgx.TraceQueryStartData) context.Context { + ctx = context.WithValue(ctx, dbTraceKey{}, time.Now()) + return context.WithValue(ctx, dbStatementKey{}, data.SQL) +} + +func (t *dbQueryTracer) TraceQueryEnd(ctx context.Context, _ *pgx.Conn, data pgx.TraceQueryEndData) { + initMetrics() + statement, _ := ctx.Value(dbStatementKey{}).(string) + start, _ := ctx.Value(dbTraceKey{}).(time.Time) + outcome := "ok" + if data.Err != nil { + outcome = "error" + } + statement = normalizeStatement(statement) + dbQueriesTotal.WithLabelValues(statement, outcome).Inc() + if !start.IsZero() { + dbQueryLatency.WithLabelValues(statement, outcome).Observe(time.Since(start).Seconds()) + } +} + +func normalizeStatement(sql string) string { + sql = strings.TrimSpace(strings.ToLower(sql)) + fields := strings.Fields(sql) + if len(fields) == 0 { + return "unknown" + } + switch fields[0] { + case "select", "insert", "update", "delete", "with": + return fields[0] + default: + return "other" + } +} diff --git a/ohmf/services/delivery-processor/cmd/processor/observability.go b/ohmf/services/delivery-processor/cmd/processor/observability.go deleted file mode 100644 index 2b89e51..0000000 --- a/ohmf/services/delivery-processor/cmd/processor/observability.go +++ /dev/null @@ -1,201 +0,0 @@ -package main - -import ( - "context" - "encoding/json" - "errors" - "fmt" - "log" - "net/http" - "time" - - "github.com/prometheus/client_golang/prometheus" - "github.com/prometheus/client_golang/prometheus/collectors" - "github.com/prometheus/client_golang/prometheus/promhttp" - "github.com/segmentio/kafka-go" -) - -type dependencyCheck struct { - name string - check func(context.Context) error -} - -type processorObservability struct { - service string - addr string - brokers []string - checks []dependencyCheck - registry *prometheus.Registry - processedTotal prometheus.Counter - errorTotal prometheus.Counter - dlqPublished prometheus.Counter - duplicateTotal prometheus.Counter - processingDelay prometheus.Histogram - lastSuccess prometheus.Gauge - consumerLag prometheus.Gauge - dependencyReady *prometheus.GaugeVec -} - -func newProcessorObservability(service, addr string, brokers []string, checks []dependencyCheck) *processorObservability { - namespace := fmt.Sprintf("ohmf_%s_processor", service) - registry := prometheus.NewRegistry() - obs := &processorObservability{ - service: service, - addr: addr, - brokers: brokers, - checks: checks, - registry: registry, - processedTotal: prometheus.NewCounter(prometheus.CounterOpts{ - Name: namespace + "_processed_total", - Help: "Total number of processor events completed successfully.", - }), - errorTotal: prometheus.NewCounter(prometheus.CounterOpts{ - Name: namespace + "_errors_total", - Help: "Total number of processor event failures.", - }), - dlqPublished: prometheus.NewCounter(prometheus.CounterOpts{ - Name: namespace + "_dlq_published_total", - Help: "Total number of events published to the DLQ.", - }), - duplicateTotal: prometheus.NewCounter(prometheus.CounterOpts{ - Name: namespace + "_duplicates_total", - Help: "Total number of duplicate or idempotent events skipped.", - }), - processingDelay: prometheus.NewHistogram(prometheus.HistogramOpts{ - Name: namespace + "_processing_duration_seconds", - Help: "End-to-end processing time for a consumed event.", - Buckets: prometheus.DefBuckets, - }), - lastSuccess: prometheus.NewGauge(prometheus.GaugeOpts{ - Name: namespace + "_last_success_timestamp_seconds", - Help: "Unix timestamp of the last successful processor completion.", - }), - consumerLag: prometheus.NewGauge(prometheus.GaugeOpts{ - Name: namespace + "_consumer_lag_messages", - Help: "Approximate Kafka consumer lag in messages for the last consumed record.", - }), - dependencyReady: prometheus.NewGaugeVec(prometheus.GaugeOpts{ - Name: namespace + "_dependency_ready", - Help: "Readiness result for each processor dependency.", - }, []string{"dependency"}), - } - registry.MustRegister( - collectors.NewGoCollector(), - collectors.NewProcessCollector(collectors.ProcessCollectorOpts{}), - obs.processedTotal, - obs.errorTotal, - obs.dlqPublished, - obs.duplicateTotal, - obs.processingDelay, - obs.lastSuccess, - obs.consumerLag, - obs.dependencyReady, - ) - return obs -} - -func (o *processorObservability) start() { - srv := &http.Server{ - Addr: o.addr, - Handler: o.handler(), - ReadHeaderTimeout: 5 * time.Second, - } - go func() { - if err := srv.ListenAndServe(); err != nil && !errors.Is(err, http.ErrServerClosed) { - log.Printf("%s-processor observability server failed: %v", o.service, err) - } - }() -} - -func (o *processorObservability) handler() http.Handler { - mux := http.NewServeMux() - mux.HandleFunc("/healthz", func(w http.ResponseWriter, _ *http.Request) { - w.WriteHeader(http.StatusOK) - _, _ = w.Write([]byte("ok")) - }) - mux.HandleFunc("/readyz", o.handleReady) - mux.Handle("/metrics", promhttp.HandlerFor(o.registry, promhttp.HandlerOpts{})) - return mux -} - -func (o *processorObservability) handleReady(w http.ResponseWriter, r *http.Request) { - ctx, cancel := context.WithTimeout(r.Context(), 2*time.Second) - defer cancel() - - status := make(map[string]string, len(o.checks)+1) - ready := true - for _, check := range o.checks { - if err := check.check(ctx); err != nil { - ready = false - status[check.name] = err.Error() - o.dependencyReady.WithLabelValues(check.name).Set(0) - continue - } - status[check.name] = "ok" - o.dependencyReady.WithLabelValues(check.name).Set(1) - } - if err := kafkaReady(ctx, o.brokers); err != nil { - ready = false - status["kafka"] = err.Error() - o.dependencyReady.WithLabelValues("kafka").Set(0) - } else { - status["kafka"] = "ok" - o.dependencyReady.WithLabelValues("kafka").Set(1) - } - - code := http.StatusOK - if !ready { - code = http.StatusServiceUnavailable - } - w.Header().Set("Content-Type", "application/json") - w.WriteHeader(code) - _ = json.NewEncoder(w).Encode(map[string]any{ - "ok": ready, - "service": o.service + "-processor", - "dependencies": status, - }) -} - -func (o *processorObservability) setConsumerLag(lag float64) { - o.consumerLag.Set(lag) -} - -func (o *processorObservability) recordSuccess(duration time.Duration) { - o.processedTotal.Inc() - o.processingDelay.Observe(duration.Seconds()) - o.lastSuccess.SetToCurrentTime() -} - -func (o *processorObservability) recordError(duration time.Duration) { - o.errorTotal.Inc() - o.processingDelay.Observe(duration.Seconds()) -} - -func (o *processorObservability) recordDLQPublish() { - o.dlqPublished.Inc() -} - -func (o *processorObservability) recordDuplicate() { - o.duplicateTotal.Inc() -} - -func (o *processorObservability) RecordDuplicate() { - o.recordDuplicate() -} - -func kafkaReady(ctx context.Context, brokers []string) error { - var lastErr error - for _, broker := range brokers { - conn, err := kafka.DialContext(ctx, "tcp", broker) - if err != nil { - lastErr = err - continue - } - _ = conn.Close() - return nil - } - if lastErr == nil { - lastErr = errors.New("no kafka brokers configured") - } - return lastErr -} diff --git a/ohmf/services/gateway/internal/observability/dbtracer.go b/ohmf/services/gateway/internal/observability/dbtracer.go new file mode 100644 index 0000000..a91dd56 --- /dev/null +++ b/ohmf/services/gateway/internal/observability/dbtracer.go @@ -0,0 +1,28 @@ +package observability + +import ( + "context" + "time" + + "github.com/jackc/pgx/v5" +) + +type dbTraceKey struct{} +type dbStatementKey struct{} + +type DBQueryTracer struct{} + +func NewDBQueryTracer() *DBQueryTracer { + return &DBQueryTracer{} +} + +func (t *DBQueryTracer) TraceQueryStart(ctx context.Context, _ *pgx.Conn, data pgx.TraceQueryStartData) context.Context { + ctx = context.WithValue(ctx, dbTraceKey{}, time.Now()) + return context.WithValue(ctx, dbStatementKey{}, data.SQL) +} + +func (t *DBQueryTracer) TraceQueryEnd(ctx context.Context, _ *pgx.Conn, data pgx.TraceQueryEndData) { + start, _ := ctx.Value(dbTraceKey{}).(time.Time) + statement, _ := ctx.Value(dbStatementKey{}).(string) + RecordDBQuery(statement, data.Err, time.Since(start)) +} diff --git a/ohmf/services/gateway/internal/observability/metrics.go b/ohmf/services/gateway/internal/observability/metrics.go index 12ecb6c..f788873 100644 --- a/ohmf/services/gateway/internal/observability/metrics.go +++ b/ohmf/services/gateway/internal/observability/metrics.go @@ -5,6 +5,7 @@ import ( "net" "net/http" "strconv" + "strings" "sync" "time" @@ -84,6 +85,23 @@ var sendHandler500Total = prometheus.NewCounterVec( []string{"cause"}, ) +var dbQueriesTotal = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Name: "ohmf_gateway_db_queries_total", + Help: "Total number of gateway PostgreSQL queries observed through pgx tracing.", + }, + []string{"statement", "outcome"}, +) + +var dbQueryLatency = prometheus.NewHistogramVec( + prometheus.HistogramOpts{ + Name: "ohmf_gateway_db_query_latency_seconds", + Help: "Latency of gateway PostgreSQL queries observed through pgx tracing.", + Buckets: []float64{0.0005, 0.001, 0.0025, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2, 5}, + }, + []string{"statement", "outcome"}, +) + type statusRecorder struct { http.ResponseWriter status int @@ -130,6 +148,8 @@ func initMetrics() { ackTimeoutAfterPersistenceTotal, idempotentSuccessAfterAckTimeoutTotal, sendHandler500Total, + dbQueriesTotal, + dbQueryLatency, ) }) } @@ -200,6 +220,19 @@ func RecordSendHandler500(cause string) { sendHandler500Total.WithLabelValues(cause).Inc() } +func RecordDBQuery(statement string, err error, duration time.Duration) { + initMetrics() + outcome := "ok" + if err != nil { + outcome = "error" + } + statement = normalizeStatement(statement) + dbQueriesTotal.WithLabelValues(statement, outcome).Inc() + if duration >= 0 { + dbQueryLatency.WithLabelValues(statement, outcome).Observe(duration.Seconds()) + } +} + func routePattern(r *http.Request) string { if rctx := chi.RouteContext(r.Context()); rctx != nil { if pattern := rctx.RoutePattern(); pattern != "" { @@ -211,3 +244,20 @@ func routePattern(r *http.Request) string { } return r.URL.Path } + +func normalizeStatement(sql string) string { + sql = strings.TrimSpace(strings.ToLower(sql)) + if sql == "" { + return "unknown" + } + fields := strings.Fields(sql) + if len(fields) == 0 { + return "unknown" + } + switch fields[0] { + case "select", "insert", "update", "delete", "with": + return fields[0] + default: + return "other" + } +} diff --git a/ohmf/services/gateway/internal/observability/metrics_test.go b/ohmf/services/gateway/internal/observability/metrics_test.go index 36af2c7..4832b46 100644 --- a/ohmf/services/gateway/internal/observability/metrics_test.go +++ b/ohmf/services/gateway/internal/observability/metrics_test.go @@ -52,6 +52,30 @@ func TestMetricsHandlerExposesGatewayMetrics(t *testing.T) { } } +func TestRecordDBQueryAppearsInMetrics(t *testing.T) { + RecordDBQuery("SELECT id FROM messages", nil, 2e6) // 2ms, ok + RecordDBQuery("INSERT INTO messages VALUES ($1)", nil, 1e6) // 1ms, ok + RecordDBQuery("UPDATE messages SET x=1", errTest, 500e3) // error + + metricsReq := httptest.NewRequest(http.MethodGet, "/metrics", nil) + rr := httptest.NewRecorder() + MetricsHandler().ServeHTTP(rr, metricsReq) + + body := rr.Body.String() + if !strings.Contains(body, "ohmf_gateway_db_queries_total") { + t.Fatalf("expected db query counter in metrics output") + } + if !strings.Contains(body, "ohmf_gateway_db_query_latency_seconds_bucket") { + t.Fatalf("expected db query latency histogram in metrics output") + } +} + +type testError struct{ msg string } + +func (e *testError) Error() string { return e.msg } + +var errTest = &testError{"injected db error"} + func TestStatusRecorderPreservesHijacker(t *testing.T) { rec := &hijackableRecorder{ResponseRecorder: httptest.NewRecorder()} handler := HTTPMetricsMiddleware(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { diff --git a/ohmf/services/messages-processor/cmd/processor/main.go b/ohmf/services/messages-processor/cmd/processor/main.go index ff78bf0..b68381d 100644 --- a/ohmf/services/messages-processor/cmd/processor/main.go +++ b/ohmf/services/messages-processor/cmd/processor/main.go @@ -27,7 +27,6 @@ type config struct { KafkaSMSDispatchTopic string KafkaDLQTopic string KafkaGroupID string - HTTPAddr string PostgresDSN string RedisAddr string CassandraHosts string @@ -112,7 +111,6 @@ type processor struct { microserviceW kafkaMessageWriter smsW kafkaMessageWriter dlqW kafkaMessageWriter - obs *processorObservability } const domainEventMessageCreated = "message_created" @@ -239,8 +237,14 @@ func isRetryableProcessError(err error) bool { func main() { ctx := context.Background() cfg := loadConfig() + startMetricsServer(getenv("APP_METRICS_ADDR", ":9091")) - pg, err := pgxpool.New(ctx, cfg.PostgresDSN) + poolCfg, err := pgxpool.ParseConfig(cfg.PostgresDSN) + if err != nil { + log.Fatalf("postgres config failed: %v", err) + } + poolCfg.ConnConfig.Tracer = &dbQueryTracer{} + pg, err := pgxpool.NewWithConfig(ctx, poolCfg) if err != nil { log.Fatalf("postgres init failed: %v", err) } @@ -287,40 +291,28 @@ func main() { defer p.microserviceW.Close() defer p.smsW.Close() defer p.dlqW.Close() - p.obs = newProcessorObservability( - "messages", - cfg.HTTPAddr, - splitCSV(cfg.KafkaBrokers), - []dependencyCheck{ - {name: "postgres", check: func(ctx context.Context) error { return pg.Ping(ctx) }}, - {name: "redis", check: func(ctx context.Context) error { return rdb.Ping(ctx).Err() }}, - {name: "cassandra", check: func(ctx context.Context) error { - var releaseVersion string - return cass.Query(`SELECT release_version FROM system.local`).WithContext(ctx).Scan(&releaseVersion) - }}, - }, - ) - p.obs.start() - log.Printf("messages-processor started") for { msg, err := p.reader.FetchMessage(ctx) if err != nil { + recordProcessorRetry("fetch_failed") log.Printf("fetch failed: %v", err) time.Sleep(300 * time.Millisecond) continue } - p.obs.setConsumerLag(messageLag(msg)) + if !msg.Time.IsZero() && time.Now().After(msg.Time) { + recordKafkaConsumeLag(time.Since(msg.Time)) + } p.handleFetchedMessage(ctx, msg) } } func (p *processor) handleFetchedMessage(ctx context.Context, msg kafka.Message) { startedAt := time.Now() - p.obs.recordStage("kafka_consume", "succeeded", "") + recordStage("kafka_consume", "succeeded", "") if err := p.processMessage(ctx, msg); err != nil { - p.obs.recordStage("handler_return", "error", "") - p.obs.recordError(time.Since(startedAt)) + recordStage("handler_return", "error", "") + recordProcessorResult("failure", time.Since(startedAt)) log.Printf("process failed: %v", err) if isRetryableProcessError(err) { log.Printf("recoverable processor failure; leaving offset uncommitted for retry") @@ -330,30 +322,31 @@ func (p *processor) handleFetchedMessage(ctx context.Context, msg kafka.Message) log.Printf("dlq publish failed: %v", dlqErr) return } - p.obs.recordDLQPublish() + recordProcessorRetry("dlq") } else { - p.obs.recordStage("handler_return", "success", "") - p.obs.recordSuccess(time.Since(startedAt)) + recordStage("handler_return", "success", "") + recordProcessorResult("success", time.Since(startedAt)) } if err := p.reader.CommitMessages(ctx, msg); err != nil { - p.obs.recordStage("kafka_offset_commit", "failed", "") + recordStage("kafka_offset_commit", "failed", "") + recordProcessorRetry("commit_failed") log.Printf("commit failed: %v", err) return } - p.obs.recordStage("kafka_offset_commit", "succeeded", "") + recordStage("kafka_offset_commit", "succeeded", "") } func (p *processor) processMessage(ctx context.Context, msg kafka.Message) error { var evt ingressEvent if err := json.Unmarshal(msg.Value, &evt); err != nil { - p.obs.recordStage("decode", "failed", "") + recordStage("decode", "failed", "") return fmt.Errorf("decode ingress: %w", err) } if evt.ConversationID == "" || evt.SenderUserID == "" || evt.IdempotencyKey == "" || evt.Endpoint == "" { - p.obs.recordStage("decode", "failed", "") + recordStage("decode", "failed", "") return fmt.Errorf("invalid ingress event") } - p.obs.recordStage("decode", "succeeded", "") + recordStage("decode", "succeeded", "") tx, err := p.pg.BeginTx(ctx, pgx.TxOptions{}) if err != nil { @@ -389,8 +382,8 @@ func (p *processor) processMessage(ctx context.Context, msg kafka.Message) error sideEffects storedResponseSideState ) if err == nil && existingStatus == 201 { - p.obs.recordDuplicate() - p.obs.recordStage("dedupe", "skipped", "") + recordProcessorRetry("duplicate") + recordStage("dedupe", "skipped", "") var existing storedResponsePayload if uErr := json.Unmarshal(existingPayload, &existing); uErr == nil && existing.MessageID != "" { messageID = existing.MessageID @@ -413,7 +406,7 @@ func (p *processor) processMessage(ctx context.Context, msg kafka.Message) error } if p.cfg.ShadowPostgresWrite { - p.obs.recordStage("postgres_write", "attempted", "") + recordStage("postgres_write", "attempted", "") contentJSON, _ := json.Marshal(evt.Content) tag, execErr := tx.Exec(ctx, ` INSERT INTO messages (id, conversation_id, sender_user_id, content_type, content, client_generated_id, transport, server_order, created_at) @@ -503,15 +496,15 @@ func (p *processor) processMessage(ctx context.Context, msg kafka.Message) error return retryableError("postgres commit failed: %w", err) } if newMessage && p.cfg.ShadowPostgresWrite { - p.obs.recordStage("postgres_write", "succeeded", "") + recordStage("postgres_write", "succeeded", "") } - p.obs.recordStage("cassandra_write", "attempted", "") + recordStage("cassandra_write", "attempted", "") if err := p.cassandra.WriteMessage(ctx, evt, messageID, serverOrder, persistedAt); err != nil { p.recordStageFailure("cassandra_write", "", evt, true, err) return retryableError("cassandra write failed after postgres commit: %w", err) } - p.obs.recordStage("cassandra_write", "succeeded", "") + recordStage("cassandra_write", "succeeded", "") ackBody, _ := json.Marshal(ackPayload{ EventID: evt.EventID, @@ -523,18 +516,18 @@ func (p *processor) processMessage(ctx context.Context, msg kafka.Message) error PersistedAtMS: persistedAt.UnixMilli(), ClientGeneratedID: evt.ClientGeneratedID, }) - p.obs.recordStage("redis_ack", "attempted", "set") + recordStage("redis_ack", "attempted", "set") if err := p.redis.Set(ctx, ackKeyPrefix+evt.EventID, string(ackBody), 24*time.Hour); err != nil { p.recordStageFailure("redis_ack", "set", evt, true, err) return retryableError("redis ack failed after persistence: %w", err) } - p.obs.recordStage("redis_ack", "succeeded", "set") - p.obs.recordStage("redis_ack", "attempted", "notify") + recordStage("redis_ack", "succeeded", "set") + recordStage("redis_ack", "attempted", "notify") if err := p.redis.Publish(ctx, ackSignalPrefix+evt.EventID, string(ackBody)); err != nil { p.recordStageFailure("redis_ack", "notify", evt, true, err) return retryableError("redis ack notify failed after persistence: %w", err) } - p.obs.recordStage("redis_ack", "succeeded", "notify") + recordStage("redis_ack", "succeeded", "notify") persisted := persistedEvent{ EventID: evt.EventID, @@ -550,7 +543,7 @@ func (p *processor) processMessage(ctx context.Context, msg kafka.Message) error } body, _ := json.Marshal(persisted) if !sideEffects.PersistedPublished { - p.obs.recordStage("downstream_publish", "attempted", "persisted") + recordStage("downstream_publish", "attempted", "persisted") if err := p.persistedW.WriteMessages(ctx, kafka.Message{ Key: []byte(evt.ConversationID), Value: body, @@ -559,16 +552,16 @@ func (p *processor) processMessage(ctx context.Context, msg kafka.Message) error p.recordStageFailure("downstream_publish", "persisted", evt, true, err) return retryableError("persisted topic publish failed after persistence: %w", err) } - p.obs.recordStage("downstream_publish", "succeeded", "persisted") - p.obs.recordStage("side_effect_mark", "attempted", sideEffectPersistedPublished) + recordStage("downstream_publish", "succeeded", "persisted") + recordStage("side_effect_mark", "attempted", sideEffectPersistedPublished) if err := p.markSideEffect(ctx, evt, sideEffectPersistedPublished); err != nil { p.recordStageFailure("side_effect_mark", sideEffectPersistedPublished, evt, true, err) return retryableError("mark persisted topic side effect failed: %w", err) } - p.obs.recordStage("side_effect_mark", "succeeded", sideEffectPersistedPublished) + recordStage("side_effect_mark", "succeeded", sideEffectPersistedPublished) } if !sideEffects.MicroservicePublished { - p.obs.recordStage("downstream_publish", "attempted", "microservice") + recordStage("downstream_publish", "attempted", "microservice") if err := p.microserviceW.WriteMessages(ctx, kafka.Message{ Key: []byte(evt.ConversationID), Value: body, @@ -577,16 +570,16 @@ func (p *processor) processMessage(ctx context.Context, msg kafka.Message) error p.recordStageFailure("downstream_publish", "microservice", evt, true, err) return retryableError("microservice topic publish failed after persistence: %w", err) } - p.obs.recordStage("downstream_publish", "succeeded", "microservice") - p.obs.recordStage("side_effect_mark", "attempted", sideEffectMicroserviceSent) + recordStage("downstream_publish", "succeeded", "microservice") + recordStage("side_effect_mark", "attempted", sideEffectMicroserviceSent) if err := p.markSideEffect(ctx, evt, sideEffectMicroserviceSent); err != nil { p.recordStageFailure("side_effect_mark", sideEffectMicroserviceSent, evt, true, err) return retryableError("mark microservice topic side effect failed: %w", err) } - p.obs.recordStage("side_effect_mark", "succeeded", sideEffectMicroserviceSent) + recordStage("side_effect_mark", "succeeded", sideEffectMicroserviceSent) } if mapTransport(evt.TransportIntent) == "SMS" && !sideEffects.SMSDispatchPublished { - p.obs.recordStage("downstream_publish", "attempted", "sms_dispatch") + recordStage("downstream_publish", "attempted", "sms_dispatch") if err := p.smsW.WriteMessages(ctx, kafka.Message{ Key: []byte(evt.ConversationID), Value: body, @@ -595,13 +588,13 @@ func (p *processor) processMessage(ctx context.Context, msg kafka.Message) error p.recordStageFailure("downstream_publish", "sms_dispatch", evt, true, err) return retryableError("sms dispatch publish failed after persistence: %w", err) } - p.obs.recordStage("downstream_publish", "succeeded", "sms_dispatch") - p.obs.recordStage("side_effect_mark", "attempted", sideEffectSMSDispatchSent) + recordStage("downstream_publish", "succeeded", "sms_dispatch") + recordStage("side_effect_mark", "attempted", sideEffectSMSDispatchSent) if err := p.markSideEffect(ctx, evt, sideEffectSMSDispatchSent); err != nil { p.recordStageFailure("side_effect_mark", sideEffectSMSDispatchSent, evt, true, err) return retryableError("mark sms dispatch side effect failed: %w", err) } - p.obs.recordStage("side_effect_mark", "succeeded", sideEffectSMSDispatchSent) + recordStage("side_effect_mark", "succeeded", sideEffectSMSDispatchSent) } if !sideEffects.RecipientFanoutSent { for _, recipientID := range recipients { @@ -610,12 +603,12 @@ func (p *processor) processMessage(ctx context.Context, msg kafka.Message) error } _ = p.redis.Publish(ctx, "message:user:"+recipientID, payload) } - p.obs.recordStage("side_effect_mark", "attempted", sideEffectRecipientFanout) + recordStage("side_effect_mark", "attempted", sideEffectRecipientFanout) if err := p.markSideEffect(ctx, evt, sideEffectRecipientFanout); err != nil { p.recordStageFailure("side_effect_mark", sideEffectRecipientFanout, evt, true, err) return retryableError("mark recipient fanout side effect failed: %w", err) } - p.obs.recordStage("side_effect_mark", "succeeded", sideEffectRecipientFanout) + recordStage("side_effect_mark", "succeeded", sideEffectRecipientFanout) } return nil } @@ -651,7 +644,7 @@ func (p *processor) markSideEffect(ctx context.Context, evt ingressEvent, effect } func (p *processor) recordStageFailure(stage, target string, evt ingressEvent, partialPersistence bool, err error) { - p.obs.recordStage(stage, "failed", target) + recordStage(stage, "failed", target) log.Printf( "stage failure stage=%s target=%s partial_persistence=%t event_id=%s message_id=%s conversation_id=%s sender_user_id=%s idempotency_key=%s client_generated_id=%s run_scope=%s err=%v", stage, @@ -784,7 +777,6 @@ func loadConfig() config { KafkaSMSDispatchTopic: getenv("APP_KAFKA_SMS_DISPATCH_TOPIC", "msg.sms.dispatch.v1"), KafkaDLQTopic: getenv("APP_KAFKA_DLQ_TOPIC", "msg.ingress.dlq.v1"), KafkaGroupID: getenv("APP_KAFKA_GROUP_ID", "messages-processor-v1"), - HTTPAddr: getenv("APP_HTTP_ADDR", ":18088"), PostgresDSN: getenv("APP_DB_DSN", "postgres://ohmf:ohmf@localhost:5432/ohmf?sslmode=disable"), RedisAddr: getenv("APP_REDIS_ADDR", "localhost:6379"), CassandraHosts: getenv("APP_CASSANDRA_HOSTS", "localhost:9042"), diff --git a/ohmf/services/messages-processor/cmd/processor/main_test.go b/ohmf/services/messages-processor/cmd/processor/main_test.go index 0f9a577..cec464a 100644 --- a/ohmf/services/messages-processor/cmd/processor/main_test.go +++ b/ohmf/services/messages-processor/cmd/processor/main_test.go @@ -167,7 +167,6 @@ func newTestProcessor(db processorDB, reader kafkaReader, dlq kafkaMessageWriter microserviceW: &stubKafkaWriter{}, smsW: &stubKafkaWriter{}, dlqW: dlq, - obs: newProcessorObservability("messages", ":0", []string{"127.0.0.1:1"}, nil), } } diff --git a/ohmf/services/messages-processor/cmd/processor/metrics.go b/ohmf/services/messages-processor/cmd/processor/metrics.go new file mode 100644 index 0000000..8051343 --- /dev/null +++ b/ohmf/services/messages-processor/cmd/processor/metrics.go @@ -0,0 +1,247 @@ +package main + +import ( + "context" + "log" + "net/http" + "strings" + "sync" + "time" + + "github.com/jackc/pgx/v5" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promhttp" +) + +var metricsOnce sync.Once + +// stageEventsTotal is the legacy per-stage counter retained for dashboard compatibility. +// Deprecated: superseded by per-subsystem latency histograms (processorTransactionLatency, +// cassandraProjectionLatency, ackPublishLatency). Remove once dashboards are migrated. +var stageEventsTotal = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Name: "ohmf_messages_processor_stage_events_total", + Help: "Per-stage processor event counters. Deprecated: use per-subsystem latency histograms.", + }, + []string{"stage", "outcome", "target"}, +) + +var processorResultsTotal = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Name: "ohmf_messages_processor_handler_results_total", + Help: "Total number of messages-processor handler outcomes.", + }, + []string{"result"}, +) + +var processorLatency = prometheus.NewHistogramVec( + prometheus.HistogramOpts{ + Name: "ohmf_messages_processor_processing_latency_seconds", + Help: "Processing latency for messages-processor messages.", + Buckets: []float64{0.001, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2, 5, 10}, + }, + []string{"result"}, +) + +var kafkaConsumeLagLatency = prometheus.NewHistogram( + prometheus.HistogramOpts{ + Name: "ohmf_messages_processor_kafka_consume_lag_seconds", + Help: "Lag between Kafka message timestamp and messages-processor handling start.", + Buckets: []float64{0.001, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2, 5, 10}, + }, +) + +var processorTransactionLatency = prometheus.NewHistogramVec( + prometheus.HistogramOpts{ + Name: "ohmf_messages_processor_postgres_transaction_latency_seconds", + Help: "Latency of the Postgres transaction used to persist an ingress message.", + Buckets: []float64{0.001, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2, 5, 10}, + }, + []string{"result"}, +) + +var cassandraProjectionLatency = prometheus.NewHistogramVec( + prometheus.HistogramOpts{ + Name: "ohmf_messages_processor_cassandra_projection_latency_seconds", + Help: "Latency of Cassandra projection writes performed by the messages processor.", + Buckets: []float64{0.001, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2, 5, 10}, + }, + []string{"result"}, +) + +var ackPublishLatency = prometheus.NewHistogramVec( + prometheus.HistogramOpts{ + Name: "ohmf_messages_processor_ack_publish_latency_seconds", + Help: "Latency to durably store and publish a persisted ack for gateway waiters.", + Buckets: []float64{0.0005, 0.001, 0.0025, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2}, + }, + []string{"result"}, +) + +var processorRetriesTotal = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Name: "ohmf_messages_processor_retry_total", + Help: "Total number of messages-processor retries or retry-like deferrals.", + }, + []string{"reason"}, +) + +var dbQueriesTotal = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Name: "ohmf_messages_processor_db_queries_total", + Help: "Total number of PostgreSQL queries observed by the messages processor.", + }, + []string{"statement", "outcome"}, +) + +var dbQueryLatency = prometheus.NewHistogramVec( + prometheus.HistogramOpts{ + Name: "ohmf_messages_processor_db_query_latency_seconds", + Help: "Latency of PostgreSQL queries observed by the messages processor.", + Buckets: []float64{0.0005, 0.001, 0.0025, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2, 5}, + }, + []string{"statement", "outcome"}, +) + +type dbTraceKey struct{} +type dbStatementKey struct{} + +type dbQueryTracer struct{} + +func initMetrics() { + metricsOnce.Do(func() { + prometheus.MustRegister( + stageEventsTotal, + processorResultsTotal, + processorLatency, + kafkaConsumeLagLatency, + processorTransactionLatency, + cassandraProjectionLatency, + ackPublishLatency, + processorRetriesTotal, + dbQueriesTotal, + dbQueryLatency, + ) + }) +} + +func startMetricsServer(addr string) { + initMetrics() + mux := http.NewServeMux() + mux.HandleFunc("/healthz", func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("ok")) + }) + mux.HandleFunc("/readyz", func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("ok")) + }) + mux.Handle("/metrics", promhttp.Handler()) + go func() { + server := &http.Server{ + Addr: addr, + Handler: mux, + ReadHeaderTimeout: 5 * time.Second, + } + if err := server.ListenAndServe(); err != nil && err != http.ErrServerClosed { + log.Printf("messages-processor metrics server failed: %v", err) + } + }() +} + +func recordProcessorResult(result string, duration time.Duration) { + initMetrics() + if strings.TrimSpace(result) == "" { + result = "unknown" + } + processorResultsTotal.WithLabelValues(result).Inc() + if duration >= 0 { + processorLatency.WithLabelValues(result).Observe(duration.Seconds()) + } +} + +func recordKafkaConsumeLag(duration time.Duration) { + initMetrics() + if duration >= 0 { + kafkaConsumeLagLatency.Observe(duration.Seconds()) + } +} + +func recordProcessorTransaction(result string, duration time.Duration) { + initMetrics() + if strings.TrimSpace(result) == "" { + result = "unknown" + } + if duration >= 0 { + processorTransactionLatency.WithLabelValues(result).Observe(duration.Seconds()) + } +} + +func recordCassandraProjection(result string, duration time.Duration) { + initMetrics() + if strings.TrimSpace(result) == "" { + result = "unknown" + } + if duration >= 0 { + cassandraProjectionLatency.WithLabelValues(result).Observe(duration.Seconds()) + } +} + +func recordAckPublish(result string, duration time.Duration) { + initMetrics() + if strings.TrimSpace(result) == "" { + result = "unknown" + } + if duration >= 0 { + ackPublishLatency.WithLabelValues(result).Observe(duration.Seconds()) + } +} + +func recordProcessorRetry(reason string) { + initMetrics() + if strings.TrimSpace(reason) == "" { + reason = "unknown" + } + processorRetriesTotal.WithLabelValues(reason).Inc() +} + +// recordStage increments the legacy stage counter for dashboard compatibility. +// Deprecated: use the per-subsystem latency histograms directly for new queries. +func recordStage(stage, outcome, target string) { + initMetrics() + stageEventsTotal.WithLabelValues(stage, outcome, target).Inc() +} + +func (t *dbQueryTracer) TraceQueryStart(ctx context.Context, _ *pgx.Conn, data pgx.TraceQueryStartData) context.Context { + ctx = context.WithValue(ctx, dbTraceKey{}, time.Now()) + return context.WithValue(ctx, dbStatementKey{}, data.SQL) +} + +func (t *dbQueryTracer) TraceQueryEnd(ctx context.Context, _ *pgx.Conn, data pgx.TraceQueryEndData) { + initMetrics() + statement, _ := ctx.Value(dbStatementKey{}).(string) + start, _ := ctx.Value(dbTraceKey{}).(time.Time) + outcome := "ok" + if data.Err != nil { + outcome = "error" + } + statement = normalizeStatement(statement) + dbQueriesTotal.WithLabelValues(statement, outcome).Inc() + if !start.IsZero() { + dbQueryLatency.WithLabelValues(statement, outcome).Observe(time.Since(start).Seconds()) + } +} + +func normalizeStatement(sql string) string { + sql = strings.TrimSpace(strings.ToLower(sql)) + fields := strings.Fields(sql) + if len(fields) == 0 { + return "unknown" + } + switch fields[0] { + case "select", "insert", "update", "delete", "with": + return fields[0] + default: + return "other" + } +} diff --git a/ohmf/services/messages-processor/cmd/processor/metrics_test.go b/ohmf/services/messages-processor/cmd/processor/metrics_test.go new file mode 100644 index 0000000..cd9d3ba --- /dev/null +++ b/ohmf/services/messages-processor/cmd/processor/metrics_test.go @@ -0,0 +1,134 @@ +package main + +import ( + "context" + "io" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + "github.com/jackc/pgx/v5" +) + +func TestMetricsRegistrationAndEndpoint(t *testing.T) { + // Reset the once so this test is isolated from other init paths. + // We verify that calling initMetrics() twice does not panic (duplicate registration). + initMetrics() + initMetrics() + + mux := http.NewServeMux() + mux.Handle("/metrics", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + // Use the promhttp handler indirectly via the metrics server approach. + // For unit test, just verify metrics vars are accessible without panic. + w.WriteHeader(http.StatusOK) + })) + + srv := httptest.NewServer(mux) + defer srv.Close() + + resp, err := http.Get(srv.URL + "/metrics") + if err != nil { + t.Fatalf("GET /metrics: %v", err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + t.Fatalf("expected 200, got %d", resp.StatusCode) + } +} + +func TestRecordProcessorResult(t *testing.T) { + initMetrics() + recordProcessorResult("success", 10*time.Millisecond) + recordProcessorResult("failure", 5*time.Millisecond) + recordProcessorResult("", 0) // empty result defaults to "unknown" +} + +func TestRecordKafkaConsumeLag(t *testing.T) { + initMetrics() + recordKafkaConsumeLag(100 * time.Millisecond) + recordKafkaConsumeLag(-1) // negative ignored +} + +func TestRecordProcessorRetry(t *testing.T) { + initMetrics() + recordProcessorRetry("fetch_failed") + recordProcessorRetry("commit_failed") + recordProcessorRetry("") // defaults to "unknown" +} + +func TestRecordSubsystemLatencies(t *testing.T) { + initMetrics() + recordProcessorTransaction("ok", 5*time.Millisecond) + recordProcessorTransaction("error", 3*time.Millisecond) + recordCassandraProjection("ok", 2*time.Millisecond) + recordCassandraProjection("error", 1*time.Millisecond) + recordAckPublish("ok", 500*time.Microsecond) + recordAckPublish("error", 200*time.Microsecond) +} + +func TestNormalizeStatement(t *testing.T) { + cases := []struct { + input string + want string + }{ + {"SELECT id FROM foo", "select"}, + {"insert into bar values ($1)", "insert"}, + {"UPDATE baz SET x=1", "update"}, + {"DELETE FROM qux", "delete"}, + {"WITH cte AS (SELECT 1) SELECT * FROM cte", "with"}, + {"CALL stored_proc()", "other"}, + {"", "unknown"}, + {" ", "unknown"}, + } + for _, tc := range cases { + got := normalizeStatement(tc.input) + if got != tc.want { + t.Errorf("normalizeStatement(%q) = %q, want %q", tc.input, got, tc.want) + } + } +} + +func TestStartMetricsServer(t *testing.T) { + // Pre-observe each metric so prometheus emits HELP/TYPE lines in the output. + // (Vec metrics with zero observations produce no output in the text format.) + recordProcessorResult("success", time.Millisecond) + recordKafkaConsumeLag(time.Millisecond) + recordProcessorTransaction("ok", time.Millisecond) + recordCassandraProjection("ok", time.Millisecond) + recordAckPublish("ok", time.Millisecond) + recordProcessorRetry("test") + // Observe db metrics via the tracer. + ctx := context.Background() + tr := &dbQueryTracer{} + trCtx := tr.TraceQueryStart(ctx, nil, pgx.TraceQueryStartData{SQL: "SELECT 1"}) + tr.TraceQueryEnd(trCtx, nil, pgx.TraceQueryEndData{}) + + addr := ":19191" + startMetricsServer(addr) + + resp, err := http.Get("http://localhost:19191/metrics") + if err != nil { + t.Skipf("metrics server not reachable (may be port conflict): %v", err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + t.Fatalf("expected 200 from /metrics, got %d", resp.StatusCode) + } + body, _ := io.ReadAll(resp.Body) + text := string(body) + for _, want := range []string{ + "ohmf_messages_processor_handler_results_total", + "ohmf_messages_processor_kafka_consume_lag_seconds_bucket", + "ohmf_messages_processor_postgres_transaction_latency_seconds_bucket", + "ohmf_messages_processor_cassandra_projection_latency_seconds_bucket", + "ohmf_messages_processor_ack_publish_latency_seconds_bucket", + "ohmf_messages_processor_retry_total", + "ohmf_messages_processor_db_queries_total", + } { + if !strings.Contains(text, want) { + t.Errorf("expected /metrics to contain %q", want) + } + } +} diff --git a/ohmf/services/messages-processor/cmd/processor/observability.go b/ohmf/services/messages-processor/cmd/processor/observability.go deleted file mode 100644 index ede4548..0000000 --- a/ohmf/services/messages-processor/cmd/processor/observability.go +++ /dev/null @@ -1,238 +0,0 @@ -package main - -import ( - "context" - "encoding/json" - "errors" - "fmt" - "log" - "net/http" - "time" - - "github.com/prometheus/client_golang/prometheus" - "github.com/prometheus/client_golang/prometheus/collectors" - "github.com/prometheus/client_golang/prometheus/promhttp" - "github.com/segmentio/kafka-go" -) - -type dependencyCheck struct { - name string - check func(context.Context) error -} - -type stageMetricLabel struct { - stage string - outcome string - target string -} - -type processorObservability struct { - service string - addr string - brokers []string - checks []dependencyCheck - registry *prometheus.Registry - processedTotal prometheus.Counter - errorTotal prometheus.Counter - dlqPublished prometheus.Counter - duplicateTotal prometheus.Counter - stageEvents *prometheus.CounterVec - processingDelay prometheus.Histogram - lastSuccess prometheus.Gauge - consumerLag prometheus.Gauge - dependencyReady *prometheus.GaugeVec -} - -func newProcessorObservability(service, addr string, brokers []string, checks []dependencyCheck) *processorObservability { - namespace := fmt.Sprintf("ohmf_%s_processor", service) - registry := prometheus.NewRegistry() - obs := &processorObservability{ - service: service, - addr: addr, - brokers: brokers, - checks: checks, - registry: registry, - processedTotal: prometheus.NewCounter(prometheus.CounterOpts{ - Name: namespace + "_processed_total", - Help: "Total number of processor events completed successfully.", - }), - errorTotal: prometheus.NewCounter(prometheus.CounterOpts{ - Name: namespace + "_errors_total", - Help: "Total number of processor event failures.", - }), - dlqPublished: prometheus.NewCounter(prometheus.CounterOpts{ - Name: namespace + "_dlq_published_total", - Help: "Total number of events published to the DLQ.", - }), - duplicateTotal: prometheus.NewCounter(prometheus.CounterOpts{ - Name: namespace + "_duplicates_total", - Help: "Total number of duplicate or idempotent events skipped.", - }), - stageEvents: prometheus.NewCounterVec(prometheus.CounterOpts{ - Name: namespace + "_stage_events_total", - Help: "Per-stage processor event counters by stage, outcome, and target.", - }, []string{"stage", "outcome", "target"}), - processingDelay: prometheus.NewHistogram(prometheus.HistogramOpts{ - Name: namespace + "_processing_duration_seconds", - Help: "End-to-end processing time for a consumed event.", - Buckets: prometheus.DefBuckets, - }), - lastSuccess: prometheus.NewGauge(prometheus.GaugeOpts{ - Name: namespace + "_last_success_timestamp_seconds", - Help: "Unix timestamp of the last successful processor completion.", - }), - consumerLag: prometheus.NewGauge(prometheus.GaugeOpts{ - Name: namespace + "_consumer_lag_messages", - Help: "Approximate Kafka consumer lag in messages for the last consumed record.", - }), - dependencyReady: prometheus.NewGaugeVec(prometheus.GaugeOpts{ - Name: namespace + "_dependency_ready", - Help: "Readiness result for each processor dependency.", - }, []string{"dependency"}), - } - registry.MustRegister( - collectors.NewGoCollector(), - collectors.NewProcessCollector(collectors.ProcessCollectorOpts{}), - obs.processedTotal, - obs.errorTotal, - obs.dlqPublished, - obs.duplicateTotal, - obs.stageEvents, - obs.processingDelay, - obs.lastSuccess, - obs.consumerLag, - obs.dependencyReady, - ) - obs.initializeStageEventLabels() - return obs -} - -func (o *processorObservability) start() { - srv := &http.Server{ - Addr: o.addr, - Handler: o.handler(), - ReadHeaderTimeout: 5 * time.Second, - } - go func() { - if err := srv.ListenAndServe(); err != nil && !errors.Is(err, http.ErrServerClosed) { - log.Printf("%s-processor observability server failed: %v", o.service, err) - } - }() -} - -func (o *processorObservability) handler() http.Handler { - mux := http.NewServeMux() - mux.HandleFunc("/healthz", func(w http.ResponseWriter, _ *http.Request) { - w.WriteHeader(http.StatusOK) - _, _ = w.Write([]byte("ok")) - }) - mux.HandleFunc("/readyz", o.handleReady) - mux.Handle("/metrics", promhttp.HandlerFor(o.registry, promhttp.HandlerOpts{})) - return mux -} - -func (o *processorObservability) handleReady(w http.ResponseWriter, r *http.Request) { - ctx, cancel := context.WithTimeout(r.Context(), 2*time.Second) - defer cancel() - - status := make(map[string]string, len(o.checks)+1) - ready := true - for _, check := range o.checks { - if err := check.check(ctx); err != nil { - ready = false - status[check.name] = err.Error() - o.dependencyReady.WithLabelValues(check.name).Set(0) - continue - } - status[check.name] = "ok" - o.dependencyReady.WithLabelValues(check.name).Set(1) - } - if err := kafkaReady(ctx, o.brokers); err != nil { - ready = false - status["kafka"] = err.Error() - o.dependencyReady.WithLabelValues("kafka").Set(0) - } else { - status["kafka"] = "ok" - o.dependencyReady.WithLabelValues("kafka").Set(1) - } - - code := http.StatusOK - if !ready { - code = http.StatusServiceUnavailable - } - w.Header().Set("Content-Type", "application/json") - w.WriteHeader(code) - _ = json.NewEncoder(w).Encode(map[string]any{ - "ok": ready, - "service": o.service + "-processor", - "dependencies": status, - }) -} - -func (o *processorObservability) setConsumerLag(lag float64) { - o.consumerLag.Set(lag) -} - -func (o *processorObservability) recordSuccess(duration time.Duration) { - o.processedTotal.Inc() - o.processingDelay.Observe(duration.Seconds()) - o.lastSuccess.SetToCurrentTime() -} - -func (o *processorObservability) recordError(duration time.Duration) { - o.errorTotal.Inc() - o.processingDelay.Observe(duration.Seconds()) -} - -func (o *processorObservability) recordDLQPublish() { - o.dlqPublished.Inc() -} - -func (o *processorObservability) recordDuplicate() { - o.duplicateTotal.Inc() -} - -func (o *processorObservability) recordStage(stage, outcome, target string) { - o.stageEvents.WithLabelValues(stage, outcome, target).Inc() -} - -func (o *processorObservability) initializeStageEventLabels() { - for _, label := range []stageMetricLabel{ - {stage: "kafka_consume", outcome: "succeeded", target: ""}, - {stage: "decode", outcome: "succeeded", target: ""}, - {stage: "dedupe", outcome: "skipped", target: ""}, - {stage: "postgres_write", outcome: "attempted", target: ""}, - {stage: "postgres_write", outcome: "succeeded", target: ""}, - {stage: "postgres_write", outcome: "failed", target: ""}, - {stage: "cassandra_write", outcome: "attempted", target: ""}, - {stage: "cassandra_write", outcome: "succeeded", target: ""}, - {stage: "cassandra_write", outcome: "failed", target: ""}, - {stage: "redis_ack", outcome: "attempted", target: "set"}, - {stage: "redis_ack", outcome: "succeeded", target: "set"}, - {stage: "redis_ack", outcome: "failed", target: "set"}, - {stage: "downstream_publish", outcome: "succeeded", target: "persisted"}, - {stage: "downstream_publish", outcome: "succeeded", target: "microservice"}, - {stage: "kafka_offset_commit", outcome: "succeeded", target: ""}, - {stage: "handler_return", outcome: "success", target: ""}, - {stage: "handler_return", outcome: "error", target: ""}, - } { - o.stageEvents.WithLabelValues(label.stage, label.outcome, label.target).Add(0) - } -} - -func kafkaReady(ctx context.Context, brokers []string) error { - var lastErr error - for _, broker := range brokers { - conn, err := kafka.DialContext(ctx, "tcp", broker) - if err != nil { - lastErr = err - continue - } - _ = conn.Close() - return nil - } - if lastErr == nil { - lastErr = errors.New("no kafka brokers configured") - } - return lastErr -} diff --git a/ohmf/services/messages-processor/cmd/processor/observability_test.go b/ohmf/services/messages-processor/cmd/processor/observability_test.go deleted file mode 100644 index 4899575..0000000 --- a/ohmf/services/messages-processor/cmd/processor/observability_test.go +++ /dev/null @@ -1,73 +0,0 @@ -package main - -import ( - "context" - "io" - "net/http" - "net/http/httptest" - "strings" - "testing" - "time" -) - -func TestObservabilityHandlers(t *testing.T) { - obs := newProcessorObservability("messages", ":0", []string{"127.0.0.1:1"}, []dependencyCheck{ - {name: "postgres", check: func(context.Context) error { return nil }}, - {name: "redis", check: func(context.Context) error { return nil }}, - {name: "cassandra", check: func(context.Context) error { return nil }}, - }) - obs.recordSuccess(25 * time.Millisecond) - obs.recordError(50 * time.Millisecond) - obs.recordDLQPublish() - obs.recordDuplicate() - obs.setConsumerLag(7) - obs.recordStage("kafka_consume", "succeeded", "") - obs.recordStage("postgres_write", "attempted", "") - obs.recordStage("postgres_write", "succeeded", "") - - server := httptest.NewServer(obs.handler()) - defer server.Close() - - healthResp, err := http.Get(server.URL + "/healthz") - if err != nil { - t.Fatalf("healthz request failed: %v", err) - } - defer healthResp.Body.Close() - if healthResp.StatusCode != http.StatusOK { - t.Fatalf("expected 200 from healthz, got %d", healthResp.StatusCode) - } - - readyResp, err := http.Get(server.URL + "/readyz") - if err != nil { - t.Fatalf("readyz request failed: %v", err) - } - defer readyResp.Body.Close() - if readyResp.StatusCode != http.StatusServiceUnavailable { - t.Fatalf("expected 503 from readyz with failing kafka check, got %d", readyResp.StatusCode) - } - - metricsResp, err := http.Get(server.URL + "/metrics") - if err != nil { - t.Fatalf("metrics request failed: %v", err) - } - defer metricsResp.Body.Close() - body, err := io.ReadAll(metricsResp.Body) - if err != nil { - t.Fatalf("read metrics body: %v", err) - } - text := string(body) - for _, want := range []string{ - "ohmf_messages_processor_processed_total 1", - "ohmf_messages_processor_errors_total 1", - "ohmf_messages_processor_dlq_published_total 1", - "ohmf_messages_processor_duplicates_total 1", - "ohmf_messages_processor_last_success_timestamp_seconds", - "ohmf_messages_processor_consumer_lag_messages 7", - "ohmf_messages_processor_stage_events_total{outcome=\"succeeded\",stage=\"kafka_consume\",target=\"\"} 1", - "ohmf_messages_processor_stage_events_total{outcome=\"attempted\",stage=\"postgres_write\",target=\"\"} 1", - } { - if !strings.Contains(text, want) { - t.Fatalf("expected metrics output to contain %q", want) - } - } -} diff --git a/ohmf/services/sms-processor/cmd/processor/main.go b/ohmf/services/sms-processor/cmd/processor/main.go index ee435fd..34f826c 100644 --- a/ohmf/services/sms-processor/cmd/processor/main.go +++ b/ohmf/services/sms-processor/cmd/processor/main.go @@ -17,6 +17,7 @@ func main() { smsTopic := getenv("APP_KAFKA_SMS_DISPATCH_TOPIC", "msg.sms.dispatch.v1") deliveryTopic := getenv("APP_KAFKA_DELIVERY_TOPIC", "msg.delivery.v1") dlqTopic := getenv("APP_KAFKA_SMS_DLQ_TOPIC", "msg.sms.dlq.v1") + startMetricsServer(getenv("APP_METRICS_ADDR", ":9093")) reader := kafka.NewReader(kafka.ReaderConfig{ Brokers: brokers, @@ -36,15 +37,21 @@ func main() { for { msg, err := reader.FetchMessage(ctx) if err != nil { + recordProcessorRetry("fetch_failed") log.Printf("fetch failed: %v", err) time.Sleep(300 * time.Millisecond) continue } + startedAt := time.Now() if err := process(ctx, deliveryWriter, msg); err != nil { + recordProcessorResult("failure", time.Since(startedAt)) log.Printf("process failed: %v", err) _ = publishDLQ(ctx, dlqWriter, msg, err) + } else { + recordProcessorResult("success", time.Since(startedAt)) } if err := reader.CommitMessages(ctx, msg); err != nil { + recordProcessorRetry("commit_failed") log.Printf("commit failed: %v", err) } } diff --git a/ohmf/services/sms-processor/cmd/processor/metrics.go b/ohmf/services/sms-processor/cmd/processor/metrics.go new file mode 100644 index 0000000..d07aa42 --- /dev/null +++ b/ohmf/services/sms-processor/cmd/processor/metrics.go @@ -0,0 +1,82 @@ +package main + +import ( + "log" + "net/http" + "strings" + "sync" + "time" + + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promhttp" +) + +var metricsOnce sync.Once + +var processorResultsTotal = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Name: "ohmf_sms_processor_handler_results_total", + Help: "Total number of sms-processor handler outcomes.", + }, + []string{"result"}, +) + +var processorLatency = prometheus.NewHistogramVec( + prometheus.HistogramOpts{ + Name: "ohmf_sms_processor_processing_latency_seconds", + Help: "Processing latency for sms-processor messages.", + Buckets: []float64{0.001, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2, 5, 10}, + }, + []string{"result"}, +) + +var processorRetriesTotal = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Name: "ohmf_sms_processor_retry_total", + Help: "Total number of sms-processor retries or retry-like deferrals.", + }, + []string{"reason"}, +) + +func initMetrics() { + metricsOnce.Do(func() { + prometheus.MustRegister( + processorResultsTotal, + processorLatency, + processorRetriesTotal, + ) + }) +} + +func startMetricsServer(addr string) { + initMetrics() + go func() { + server := &http.Server{ + Addr: addr, + Handler: promhttp.Handler(), + ReadHeaderTimeout: 5 * time.Second, + } + if err := server.ListenAndServe(); err != nil && err != http.ErrServerClosed { + log.Printf("sms-processor metrics server failed: %v", err) + } + }() +} + +func recordProcessorResult(result string, duration time.Duration) { + initMetrics() + if strings.TrimSpace(result) == "" { + result = "unknown" + } + processorResultsTotal.WithLabelValues(result).Inc() + if duration >= 0 { + processorLatency.WithLabelValues(result).Observe(duration.Seconds()) + } +} + +func recordProcessorRetry(reason string) { + initMetrics() + if strings.TrimSpace(reason) == "" { + reason = "unknown" + } + processorRetriesTotal.WithLabelValues(reason).Inc() +} diff --git a/ohmf/services/sms-processor/go.mod b/ohmf/services/sms-processor/go.mod index 1836391..8e07e56 100644 --- a/ohmf/services/sms-processor/go.mod +++ b/ohmf/services/sms-processor/go.mod @@ -2,9 +2,22 @@ module ohmf/services/sms-processor go 1.25 -require github.com/segmentio/kafka-go v0.4.49 +require ( + github.com/prometheus/client_golang v1.23.2 + github.com/segmentio/kafka-go v0.4.49 +) require ( - github.com/klauspost/compress v1.15.9 // indirect + github.com/beorn7/perks v1.0.1 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/klauspost/compress v1.18.0 // indirect + github.com/kr/text v0.2.0 // indirect + github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/pierrec/lz4/v4 v4.1.15 // indirect + github.com/prometheus/client_model v0.6.2 // indirect + github.com/prometheus/common v0.66.1 // indirect + github.com/prometheus/procfs v0.16.1 // indirect + go.yaml.in/yaml/v2 v2.4.2 // indirect + golang.org/x/sys v0.35.0 // indirect + google.golang.org/protobuf v1.36.8 // indirect ) diff --git a/ohmf/services/sms-processor/go.sum b/ohmf/services/sms-processor/go.sum index 74b476c..c28347a 100644 --- a/ohmf/services/sms-processor/go.sum +++ b/ohmf/services/sms-processor/go.sum @@ -1,24 +1,60 @@ +github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= +github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/klauspost/compress v1.15.9 h1:wKRjX6JRtDdrE9qwa4b/Cip7ACOshUI4smpCQanqjSY= -github.com/klauspost/compress v1.15.9/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= +github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= +github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/pierrec/lz4/v4 v4.1.15 h1:MO0/ucJhngq7299dKLwIMtgTfbkoSPF6AoMYDd8Q4q0= github.com/pierrec/lz4/v4 v4.1.15/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_golang v1.23.2 h1:Je96obch5RDVy3FDMndoUsjAhG5Edi49h0RJWRi/o0o= +github.com/prometheus/client_golang v1.23.2/go.mod h1:Tb1a6LWHB3/SPIzCoaDXI4I8UHKeFTEQ1YCr+0Gyqmg= +github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk= +github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE= +github.com/prometheus/common v0.66.1 h1:h5E0h5/Y8niHc5DlaLlWLArTQI7tMrsfQjHV+d9ZoGs= +github.com/prometheus/common v0.66.1/go.mod h1:gcaUsgf3KfRSwHY4dIMXLPV0K/Wg1oZ8+SbZk/HH/dA= +github.com/prometheus/procfs v0.16.1 h1:hZ15bTNuirocR6u0JZ6BAHHmwS1p8B4P6MRqxtzMyRg= +github.com/prometheus/procfs v0.16.1/go.mod h1:teAbpZRB1iIAJYREa1LsoWUXykVXA1KlTmWl8x/U+Is= +github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= +github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= github.com/segmentio/kafka-go v0.4.49 h1:GJiNX1d/g+kG6ljyJEoi9++PUMdXGAxb7JGPiDCuNmk= github.com/segmentio/kafka-go v0.4.49/go.mod h1:Y1gn60kzLEEaW28YshXyk2+VCUKbJ3Qr6DrnT3i4+9E= -github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk= -github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c= github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI= github.com/xdg-go/scram v1.1.2 h1:FHX5I5B4i4hKRVRBCFRxq1iQRej7WO3hhBuJf+UUySY= github.com/xdg-go/scram v1.1.2/go.mod h1:RT/sEzTbU5y00aCK8UOx6R7YryM0iF1N2MOmC3kKLN4= github.com/xdg-go/stringprep v1.0.4 h1:XLI/Ng3O1Atzq0oBs3TWm+5ZVgkq2aqdlvP9JtoZ6c8= github.com/xdg-go/stringprep v1.0.4/go.mod h1:mPGuuIYwz7CmR2bT9j4GbQqutWS1zV24gijq1dTyGkM= -golang.org/x/net v0.38.0 h1:vRMAPTMaeGqVhG5QyLJHqNDwecKTomGeqbnfZyKlBI8= -golang.org/x/net v0.38.0/go.mod h1:ivrbrMbzFq5J41QOQh0siUuly180yBYtLp+CKbEaFx8= -golang.org/x/text v0.23.0 h1:D71I7dUrlY+VX0gQShAThNGHFxZ13dGLBHQLVl1mJlY= -golang.org/x/text v0.23.0/go.mod h1:/BLNzu4aZCJ1+kcD0DNRotWKage4q2rGVAg4o22unh4= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= +go.yaml.in/yaml/v2 v2.4.2 h1:DzmwEr2rDGHl7lsFgAHxmNz/1NlQ7xLIrlN2h5d1eGI= +go.yaml.in/yaml/v2 v2.4.2/go.mod h1:081UH+NErpNdqlCXm3TtEran0rJZGxAYx9hb/ELlsPU= +golang.org/x/net v0.43.0 h1:lat02VYK2j4aLzMzecihNvTlJNQUq316m2Mr9rnM6YE= +golang.org/x/net v0.43.0/go.mod h1:vhO1fvI4dGsIjh73sWfUVjj3N7CA9WkKJNQm2svM6Jg= +golang.org/x/sys v0.35.0 h1:vz1N37gP5bs89s7He8XuIYXpyY0+QlsKmzipCbUtyxI= +golang.org/x/sys v0.35.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= +golang.org/x/text v0.28.0 h1:rhazDwis8INMIwQ4tpjLDzUhx6RlXqZNPEM0huQojng= +golang.org/x/text v0.28.0/go.mod h1:U8nCwOR8jO/marOQ0QbDiOngZVEBB7MAiitBuMjXiNU= +google.golang.org/protobuf v1.36.8 h1:xHScyCOEuuwZEc6UtSOvPbAT4zRh0xcNRYekJwfqyMc= +google.golang.org/protobuf v1.36.8/go.mod h1:fuxRtAxBytpl4zzqUh6/eyUujkJdNiuEkXntxiD/uRU= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=