Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 9 additions & 6 deletions ohmf/infra/docker/docker-compose.yml
Original file line number Diff line number Diff line change
Expand Up @@ -198,9 +198,9 @@ services:
environment:
APP_DB_DSN: postgres://${POSTGRES_USER:-ohmf}:${POSTGRES_PASSWORD:-ohmf}@db:5432/${POSTGRES_DB:-ohmf}?sslmode=disable
APP_REDIS_ADDR: redis:6379
APP_METRICS_ADDR: :9091
APP_KAFKA_BROKERS: kafka:9092
APP_KAFKA_GROUP_ID: messages-processor-v1
APP_HTTP_ADDR: :18088
APP_KAFKA_INGRESS_TOPIC: msg.ingress.v1
APP_KAFKA_PERSISTED_TOPIC: msg.persisted.v1
APP_KAFKA_MICROSERVICE_TOPIC: microservice.events.v1
Expand All @@ -210,9 +210,9 @@ services:
APP_CASSANDRA_KEYSPACE: ohmf_messages
APP_SHADOW_POSTGRES_WRITE: "true"
ports:
- "18088:18088"
- "19091:9091"
healthcheck:
test: ["CMD-SHELL", "wget -qO- http://localhost:18088/healthz >/dev/null 2>&1 || exit 1"]
test: ["CMD-SHELL", "wget -qO- http://localhost:9091/healthz >/dev/null 2>&1 || exit 1"]
interval: 10s
timeout: 3s
retries: 5
Expand All @@ -234,16 +234,16 @@ services:
environment:
APP_DB_DSN: postgres://${POSTGRES_USER:-ohmf}:${POSTGRES_PASSWORD:-ohmf}@db:5432/${POSTGRES_DB:-ohmf}?sslmode=disable
APP_REDIS_ADDR: redis:6379
APP_METRICS_ADDR: :9092
APP_KAFKA_BROKERS: kafka:9092
APP_KAFKA_GROUP_ID: delivery-processor-v1
APP_HTTP_ADDR: :18089
APP_KAFKA_PERSISTED_TOPIC: msg.persisted.v1
APP_KAFKA_DELIVERY_TOPIC: msg.delivery.v1
APP_KAFKA_DELIVERY_DLQ_TOPIC: msg.delivery.dlq.v1
ports:
- "18089:18089"
- "19092:9092"
healthcheck:
test: ["CMD-SHELL", "wget -qO- http://localhost:18089/healthz >/dev/null 2>&1 || exit 1"]
test: ["CMD-SHELL", "wget -qO- http://localhost:9092/healthz >/dev/null 2>&1 || exit 1"]
interval: 10s
timeout: 3s
retries: 5
Expand All @@ -259,11 +259,14 @@ services:
kafka-init:
condition: service_completed_successfully
environment:
APP_METRICS_ADDR: :9093
APP_KAFKA_BROKERS: kafka:9092
APP_KAFKA_GROUP_ID: sms-processor-v1
APP_KAFKA_SMS_DISPATCH_TOPIC: msg.sms.dispatch.v1
APP_KAFKA_DELIVERY_TOPIC: msg.delivery.v1
APP_KAFKA_SMS_DLQ_TOPIC: msg.sms.dlq.v1
ports:
- "19093:9093"

volumes:
db_data:
Expand Down
32 changes: 29 additions & 3 deletions ohmf/infra/observability/prometheus.yml
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
global:
scrape_interval: 15s
evaluation_interval: 15s
scrape_interval: 5s
evaluation_interval: 5s

rule_files:
- /etc/prometheus/alerting.yml
Expand All @@ -15,4 +15,30 @@ scrape_configs:
metrics_path: /metrics
static_configs:
- targets:
- api:8081
- api:8080

- job_name: ohmf-messages-processor
metrics_path: /metrics
static_configs:
- targets:
- messages-processor:9091
- messages-processor-b:9091

Comment on lines +23 to +26
- job_name: ohmf-sync-fanout-workers
metrics_path: /metrics
static_configs:
- targets:
- sync-fanout-worker-a:9094
- sync-fanout-worker-b:9094
Comment on lines +27 to +32

- job_name: ohmf-delivery-processor
metrics_path: /metrics
static_configs:
- targets:
- delivery-processor:9092

- job_name: ohmf-sms-processor
metrics_path: /metrics
static_configs:
- targets:
- sms-processor:9093
67 changes: 24 additions & 43 deletions ohmf/services/delivery-processor/cmd/processor/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,14 @@ type queryRower interface {
QueryRow(context.Context, string, ...any) pgx.Row
}

type deliveryMetricsObserver interface {
RecordDuplicate()
}

type noopDeliveryMetricsObserver struct{}

func (noopDeliveryMetricsObserver) RecordDuplicate() {}

type pgDeliveryRecorder struct {
db queryRower
}
Expand Down Expand Up @@ -104,22 +112,19 @@ func (p kafkaDeliveryPublisher) Publish(ctx context.Context, key string, body []
})
}

type deliveryMetricsObserver interface {
RecordDuplicate()
}

type noopDeliveryMetricsObserver struct{}

func (noopDeliveryMetricsObserver) RecordDuplicate() {}

func main() {
ctx := context.Background()
brokers := splitCSV(getenv("APP_KAFKA_BROKERS", "localhost:9092"))
persistedTopic := getenv("APP_KAFKA_PERSISTED_TOPIC", "msg.persisted.v1")
deliveryTopic := getenv("APP_KAFKA_DELIVERY_TOPIC", "msg.delivery.v1")
dlqTopic := getenv("APP_KAFKA_DELIVERY_DLQ_TOPIC", "msg.delivery.dlq.v1")
httpAddr := getenv("APP_HTTP_ADDR", ":18089")
pg, err := pgxpool.New(ctx, getenv("APP_DB_DSN", "postgres://ohmf:ohmf@localhost:5432/ohmf?sslmode=disable"))
startMetricsServer(getenv("APP_METRICS_ADDR", ":9092"))
poolCfg, err := pgxpool.ParseConfig(getenv("APP_DB_DSN", "postgres://ohmf:ohmf@localhost:5432/ohmf?sslmode=disable"))
if err != nil {
log.Fatalf("postgres config failed: %v", err)
}
poolCfg.ConnConfig.Tracer = &dbQueryTracer{}
pg, err := pgxpool.NewWithConfig(ctx, poolCfg)
if err != nil {
log.Fatalf("postgres init failed: %v", err)
}
Expand Down Expand Up @@ -148,46 +153,32 @@ func main() {
log.Fatalf("redis ping failed: %v", err)
}

obs := newProcessorObservability(
"delivery",
httpAddr,
brokers,
[]dependencyCheck{
{name: "postgres", check: func(ctx context.Context) error { return pg.Ping(ctx) }},
{name: "redis", check: func(ctx context.Context) error { return rdb.Ping(ctx).Err() }},
},
)
obs.start()

log.Printf("delivery-processor started")
for {
msg, err := reader.FetchMessage(ctx)
if err != nil {
recordProcessorRetry("fetch_failed")
log.Printf("fetch failed: %v", err)
time.Sleep(300 * time.Millisecond)
continue
}
obs.setConsumerLag(messageLag(msg))
startedAt := time.Now()
if err := process(ctx, pg, rdb, deliveryWriter, msg, obs); err != nil {
obs.recordError(time.Since(startedAt))
if err := process(ctx, pg, rdb, deliveryWriter, msg); err != nil {
recordProcessorResult("failure", time.Since(startedAt))
log.Printf("process failed: %v", err)
if dlqErr := publishDLQ(ctx, dlqWriter, msg, err); dlqErr != nil {
log.Printf("dlq publish failed: %v", dlqErr)
} else {
obs.recordDLQPublish()
}
_ = publishDLQ(ctx, dlqWriter, msg, err)
} else {
obs.recordSuccess(time.Since(startedAt))
recordProcessorResult("success", time.Since(startedAt))
}
if err := reader.CommitMessages(ctx, msg); err != nil {
recordProcessorRetry("commit_failed")
log.Printf("commit failed: %v", err)
}
}
}

func process(ctx context.Context, pg *pgxpool.Pool, rdb *redis.Client, deliveryWriter *kafka.Writer, msg kafka.Message, observer deliveryMetricsObserver) error {
return processMessageWithObserver(ctx, pgDeliveryRecorder{db: pg}, redisPresenceStore{client: rdb}, kafkaDeliveryPublisher{writer: deliveryWriter}, msg, observer)
func process(ctx context.Context, pg *pgxpool.Pool, rdb *redis.Client, deliveryWriter *kafka.Writer, msg kafka.Message) error {
return processMessageWithObserver(ctx, pgDeliveryRecorder{db: pg}, redisPresenceStore{client: rdb}, kafkaDeliveryPublisher{writer: deliveryWriter}, msg, noopDeliveryMetricsObserver{})
}

func processMessage(ctx context.Context, deliveries deliveryRecorder, presence presenceStore, publisher deliveryPublisher, msg kafka.Message) error {
Expand Down Expand Up @@ -221,6 +212,7 @@ func processMessageWithObserver(ctx context.Context, deliveries deliveryRecorder
}
if !created {
observer.RecordDuplicate()
recordProcessorRetry("duplicate")
continue
}
delivery := map[string]any{
Expand Down Expand Up @@ -272,17 +264,6 @@ func writer(brokers []string, topic string) *kafka.Writer {
}
}

func messageLag(msg kafka.Message) float64 {
if msg.HighWaterMark <= 0 {
return 0
}
lag := msg.HighWaterMark - msg.Offset - 1
if lag < 0 {
return 0
}
return float64(lag)
}

func splitCSV(v string) []string {
raw := strings.Split(v, ",")
out := make([]string, 0, len(raw))
Expand Down
Loading
Loading