Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion ohmf/services/gateway/cmd/api/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -151,6 +151,7 @@ func main() {
if cfg.UseKafkaSend {
kafkaProducer = bus.NewKafkaProducer(cfg.KafkaBrokers, cfg.KafkaClientID, cfg.KafkaIngressTopic)
asyncPipeline = messages.NewAsyncPipeline(kafkaProducer, rdb)
defer asyncPipeline.Stop()
defer kafkaProducer.Close()
}

Expand Down Expand Up @@ -237,7 +238,7 @@ func main() {
msgHandler := messages.NewHandler(msgSvc)
syncSvc := sync.NewService(pool, replicationStore)
syncHandler := &sync.Handler{Svc: syncSvc}
syncFanoutWorker := wk.NewSyncFanoutWorker(replicationStore)
syncFanoutWorker := wk.NewSyncFanoutWorker(replicationStore, cfg.DBDSN, cfg.SyncFanoutBatchSize, cfg.SyncFanoutFallbackPoll, cfg.SyncFanoutNotifyChannel)
go func() {
if err := syncFanoutWorker.Start(ctx); err != nil {
logger.Error().Err(err).Msg("sync fanout worker stopped")
Expand Down
46 changes: 41 additions & 5 deletions ohmf/services/gateway/cmd/worker/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import (
"context"
"os"
"os/signal"
"strings"
"syscall"
"time"

Expand Down Expand Up @@ -85,11 +86,22 @@ func main() {

// create runner and workers
runner := wk.NewRunner()
runner.Add(wk.NewMediaWorker(pool))
runner.Add(wk.NewNotificationWorker(notificationHandler))
runner.Add(wk.NewAbuseAggregatorWorker(pool))
runner.Add(wk.NewRelayRetryWorker(pool))
runner.Add(wk.NewSyncFanoutWorker(replicationStore))
enabledWorkers := workerSetFromEnv(os.Getenv("APP_ENABLED_WORKERS"))
if workerEnabled(enabledWorkers, "media") {
runner.Add(wk.NewMediaWorker(pool))
}
if workerEnabled(enabledWorkers, "notification") {
runner.Add(wk.NewNotificationWorker(notificationHandler))
}
if workerEnabled(enabledWorkers, "abuse") {
runner.Add(wk.NewAbuseAggregatorWorker(pool))
}
if workerEnabled(enabledWorkers, "relay_retry") {
runner.Add(wk.NewRelayRetryWorker(pool))
}
if workerEnabled(enabledWorkers, "sync_fanout") {
runner.Add(wk.NewSyncFanoutWorker(replicationStore, cfg.DBDSN, cfg.SyncFanoutBatchSize, cfg.SyncFanoutFallbackPoll, cfg.SyncFanoutNotifyChannel))
}

ctx, cancel := context.WithCancel(ctx)
defer cancel()
Expand All @@ -109,3 +121,27 @@ func main() {
defer cancelStop()
_ = runner.StopAll(stopCtx)
}

func workerSetFromEnv(value string) map[string]struct{} {
value = strings.TrimSpace(value)
if value == "" {
return nil
}
out := make(map[string]struct{})
for _, item := range strings.Split(value, ",") {
item = strings.TrimSpace(strings.ToLower(item))
if item == "" {
continue
}
out[item] = struct{}{}
}
return out
}

func workerEnabled(enabled map[string]struct{}, name string) bool {
if len(enabled) == 0 {
return true
}
_, ok := enabled[strings.TrimSpace(strings.ToLower(name))]
return ok
}
10 changes: 8 additions & 2 deletions ohmf/services/gateway/internal/config/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -30,8 +30,11 @@ type Config struct {
KafkaPresenceTopic string
KafkaMicroserviceTopic string
KafkaClientID string
AckTimeout time.Duration
CassandraHosts string
AckTimeout time.Duration
SyncFanoutBatchSize int
SyncFanoutFallbackPoll time.Duration
SyncFanoutNotifyChannel string
CassandraHosts string
CassandraKeyspace string
CassandraUsername string
CassandraPassword string
Expand Down Expand Up @@ -123,6 +126,9 @@ func Load() Config {
KafkaMicroserviceTopic: get("APP_KAFKA_MICROSERVICE_TOPIC", "microservice.events.v1"),
KafkaClientID: get("APP_KAFKA_CLIENT_ID", "ohmf-gateway"),
AckTimeout: time.Duration(getInt("APP_ACK_TIMEOUT_MS", 2000)) * time.Millisecond,
SyncFanoutBatchSize: getInt("APP_SYNC_FANOUT_BATCH_SIZE", 100),
SyncFanoutFallbackPoll: time.Duration(getInt("APP_SYNC_FANOUT_FALLBACK_POLL_MS", 1000)) * time.Millisecond,
SyncFanoutNotifyChannel: get("APP_SYNC_FANOUT_NOTIFY_CHANNEL", "ohmf_domain_events"),
CassandraHosts: get("APP_CASSANDRA_HOSTS", "localhost:9042"),
CassandraKeyspace: get("APP_CASSANDRA_KEYSPACE", "ohmf_messages"),
CassandraUsername: get("APP_CASSANDRA_USERNAME", ""),
Expand Down
224 changes: 145 additions & 79 deletions ohmf/services/gateway/internal/messages/async.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,8 +3,8 @@ package messages
import (
"context"
"encoding/json"
"errors"
"fmt"
"sync"
"time"

"github.com/google/uuid"
Expand All @@ -13,9 +13,8 @@ import (
)

const (
ackKeyPrefix = "msg:ack:"
ackSignalPrefix = "msg:ack:notify:"
legacyAckPollInterval = time.Second
ackKeyPrefix = "msg:ack:"
ackChannelName = "msg:ack:events"
)

type IngressEvent struct {
Expand Down Expand Up @@ -48,16 +47,39 @@ type PersistedAck struct {
type AsyncPipeline struct {
producer bus.IngressProducer
redis *redis.Client
waiters map[string][]chan PersistedAck
mu sync.Mutex
cancel context.CancelFunc
stopOnce sync.Once
wg sync.WaitGroup
}

func NewAsyncPipeline(producer bus.IngressProducer, redisClient *redis.Client) *AsyncPipeline {
if producer == nil || redisClient == nil {
return nil
}
return &AsyncPipeline{
ctx, cancel := context.WithCancel(context.Background())
pipeline := &AsyncPipeline{
producer: producer,
redis: redisClient,
waiters: make(map[string][]chan PersistedAck),
cancel: cancel,
}
pipeline.wg.Add(1)
go pipeline.runAckSubscriber(ctx)
return pipeline
}

func (p *AsyncPipeline) Stop() {
if p == nil {
return
}
p.stopOnce.Do(func() {
if p.cancel != nil {
p.cancel()
}
p.wg.Wait()
})
}

func (p *AsyncPipeline) PublishIngress(ctx context.Context, evt IngressEvent) error {
Expand All @@ -79,59 +101,34 @@ func (p *AsyncPipeline) WaitAck(ctx context.Context, eventID string, timeout tim
if p == nil || p.redis == nil {
return PersistedAck{}, false, nil
}
waitCtx, cancel := context.WithTimeout(ctx, timeout)
defer cancel()

key := AckRedisKey(eventID)
if ack, ok, err := p.readAckWithTimeoutHandling(ctx, waitCtx, key); ok || err != nil {
return ack, ok, err
}

pubsub := p.redis.Subscribe(waitCtx, ackSignalChannel(eventID))
defer pubsub.Close()
if _, err := pubsub.Receive(waitCtx); err != nil {
return ackWaitResult(ctx, waitCtx, err)
if timeout <= 0 {
timeout = 2 * time.Second
}
key := ackKeyPrefix + eventID
waiter := make(chan PersistedAck, 1)
p.registerWaiter(eventID, waiter)
defer p.unregisterWaiter(eventID, waiter)

// Re-check the durable ack key after the subscription is live so a key write
// that raced with subscription setup is still observed without polling.
if ack, ok, err := p.readAckWithTimeoutHandling(ctx, waitCtx, key); ok || err != nil {
return ack, ok, err
if ack, ok, err := p.loadPersistedAck(ctx, key); err != nil {
return PersistedAck{}, false, err
} else if ok {
return ack, true, nil
}

msgCh := pubsub.Channel()
legacyTicker := time.NewTicker(legacyAckPollInterval)
defer legacyTicker.Stop()

for {
select {
case <-waitCtx.Done():
// One final authoritative key read avoids timing out on an ack that
// landed just before the wait context expired.
if ack, ok, err := p.readAck(context.Background(), key); ok || err != nil {
return ack, ok, err
}
return ackWaitResult(ctx, waitCtx, waitCtx.Err())
case msg, ok := <-msgCh:
if !ok {
if ack, found, err := p.readAckWithTimeoutHandling(ctx, waitCtx, key); found || err != nil {
return ack, found, err
}
return ackWaitResult(ctx, waitCtx, errors.New("redis ack subscription closed"))
}
ack, err := decodeAckPayload(msg.Payload)
if err != nil {
return PersistedAck{}, false, err
}
timer := time.NewTimer(timeout)
defer timer.Stop()
select {
case <-ctx.Done():
return PersistedAck{}, false, ctx.Err()
case ack := <-waiter:
return ack, true, nil
case <-timer.C:
if ack, ok, err := p.loadPersistedAck(ctx, key); err != nil {
return PersistedAck{}, false, err
} else if ok {
return ack, true, nil
case <-legacyTicker.C:
// Sparse fallback polling is retained for rolling-deploy compatibility
// and missed/lost Pub/Sub notifications. The durable ack key remains
// authoritative; Pub/Sub is only a wake-up optimization.
if ack, ok, err := p.readAckWithTimeoutHandling(ctx, waitCtx, key); ok || err != nil {
return ack, ok, err
}
}
return PersistedAck{}, false, nil
}
}

Expand Down Expand Up @@ -176,47 +173,116 @@ func AckRedisKey(eventID string) string {
return fmt.Sprintf("%s%s", ackKeyPrefix, eventID)
}

func ackSignalChannel(eventID string) string {
return fmt.Sprintf("%s%s", ackSignalPrefix, eventID)
func AckRedisChannel() string {
return ackChannelName
}

func (p *AsyncPipeline) readAck(ctx context.Context, key string) (PersistedAck, bool, error) {
func (p *AsyncPipeline) loadPersistedAck(ctx context.Context, key string) (PersistedAck, bool, error) {
payload, err := p.redis.Get(ctx, key).Result()
if err == nil {
ack, err := decodeAckPayload(payload)
if err != nil {
return PersistedAck{}, false, err
}
return ack, true, nil
if err == redis.Nil {
return PersistedAck{}, false, nil
}
if err != nil {
return PersistedAck{}, false, err
}
if err != nil && err != redis.Nil {
var ack PersistedAck
if err := json.Unmarshal([]byte(payload), &ack); err != nil {
return PersistedAck{}, false, err
}
return PersistedAck{}, false, nil
return ack, true, nil
}

func (p *AsyncPipeline) registerWaiter(eventID string, waiter chan PersistedAck) {
p.mu.Lock()
defer p.mu.Unlock()
p.waiters[eventID] = append(p.waiters[eventID], waiter)
}

func (p *AsyncPipeline) readAckWithTimeoutHandling(parentCtx, waitCtx context.Context, key string) (PersistedAck, bool, error) {
ack, ok, err := p.readAck(waitCtx, key)
if err == nil {
return ack, ok, nil
func (p *AsyncPipeline) unregisterWaiter(eventID string, waiter chan PersistedAck) {
p.mu.Lock()
defer p.mu.Unlock()
waiters := p.waiters[eventID]
if len(waiters) == 0 {
return
}
filtered := waiters[:0]
for _, item := range waiters {
if item == waiter {
continue
}
filtered = append(filtered, item)
}
if len(filtered) == 0 {
delete(p.waiters, eventID)
return
}
return ackWaitResult(parentCtx, waitCtx, err)
p.waiters[eventID] = filtered
}

func decodeAckPayload(payload string) (PersistedAck, error) {
var ack PersistedAck
if err := json.Unmarshal([]byte(payload), &ack); err != nil {
return PersistedAck{}, err
func (p *AsyncPipeline) dispatchAck(ack PersistedAck) {
if p == nil || ack.EventID == "" {
return
}
p.mu.Lock()
waiters := append([]chan PersistedAck(nil), p.waiters[ack.EventID]...)
p.mu.Unlock()
for _, waiter := range waiters {
select {
case waiter <- ack:
default:
}
}
return ack, nil
}

func ackWaitResult(parentCtx, waitCtx context.Context, err error) (PersistedAck, bool, error) {
if parentErr := parentCtx.Err(); parentErr != nil {
return PersistedAck{}, false, parentErr
func (p *AsyncPipeline) runAckSubscriber(ctx context.Context) {
if p == nil {
return
}
if errors.Is(err, context.DeadlineExceeded) || errors.Is(waitCtx.Err(), context.DeadlineExceeded) {
return PersistedAck{}, false, nil
defer p.wg.Done()
if p.redis == nil {
return
}
for {
select {
case <-ctx.Done():
return
default:
}
pubsub := p.redis.Subscribe(ctx, ackChannelName)
if _, err := pubsub.Receive(ctx); err != nil {
_ = pubsub.Close()
if !sleepWithContext(ctx, 250*time.Millisecond) {
return
}
continue
}
func() {
defer pubsub.Close()
for {
msg, err := pubsub.ReceiveMessage(ctx)
if err != nil {
return
}
var ack PersistedAck
if err := json.Unmarshal([]byte(msg.Payload), &ack); err != nil {
continue
}
p.dispatchAck(ack)
}
}()
if !sleepWithContext(ctx, 250*time.Millisecond) {
return
}
}
}

func sleepWithContext(ctx context.Context, d time.Duration) bool {
timer := time.NewTimer(d)
defer timer.Stop()
select {
case <-ctx.Done():
return false
case <-timer.C:
return true
}
return PersistedAck{}, false, err
}
Loading
Loading