diff --git a/ohmf/services/gateway/cmd/api/main.go b/ohmf/services/gateway/cmd/api/main.go index e5b6e63..ad2e9db 100644 --- a/ohmf/services/gateway/cmd/api/main.go +++ b/ohmf/services/gateway/cmd/api/main.go @@ -151,6 +151,7 @@ func main() { if cfg.UseKafkaSend { kafkaProducer = bus.NewKafkaProducer(cfg.KafkaBrokers, cfg.KafkaClientID, cfg.KafkaIngressTopic) asyncPipeline = messages.NewAsyncPipeline(kafkaProducer, rdb) + defer asyncPipeline.Stop() defer kafkaProducer.Close() } @@ -237,7 +238,7 @@ func main() { msgHandler := messages.NewHandler(msgSvc) syncSvc := sync.NewService(pool, replicationStore) syncHandler := &sync.Handler{Svc: syncSvc} - syncFanoutWorker := wk.NewSyncFanoutWorker(replicationStore) + syncFanoutWorker := wk.NewSyncFanoutWorker(replicationStore, cfg.DBDSN, cfg.SyncFanoutBatchSize, cfg.SyncFanoutFallbackPoll, cfg.SyncFanoutNotifyChannel) go func() { if err := syncFanoutWorker.Start(ctx); err != nil { logger.Error().Err(err).Msg("sync fanout worker stopped") diff --git a/ohmf/services/gateway/cmd/worker/main.go b/ohmf/services/gateway/cmd/worker/main.go index c335718..365245c 100644 --- a/ohmf/services/gateway/cmd/worker/main.go +++ b/ohmf/services/gateway/cmd/worker/main.go @@ -4,6 +4,7 @@ import ( "context" "os" "os/signal" + "strings" "syscall" "time" @@ -85,11 +86,22 @@ func main() { // create runner and workers runner := wk.NewRunner() - runner.Add(wk.NewMediaWorker(pool)) - runner.Add(wk.NewNotificationWorker(notificationHandler)) - runner.Add(wk.NewAbuseAggregatorWorker(pool)) - runner.Add(wk.NewRelayRetryWorker(pool)) - runner.Add(wk.NewSyncFanoutWorker(replicationStore)) + enabledWorkers := workerSetFromEnv(os.Getenv("APP_ENABLED_WORKERS")) + if workerEnabled(enabledWorkers, "media") { + runner.Add(wk.NewMediaWorker(pool)) + } + if workerEnabled(enabledWorkers, "notification") { + runner.Add(wk.NewNotificationWorker(notificationHandler)) + } + if workerEnabled(enabledWorkers, "abuse") { + runner.Add(wk.NewAbuseAggregatorWorker(pool)) + } + if workerEnabled(enabledWorkers, "relay_retry") { + runner.Add(wk.NewRelayRetryWorker(pool)) + } + if workerEnabled(enabledWorkers, "sync_fanout") { + runner.Add(wk.NewSyncFanoutWorker(replicationStore, cfg.DBDSN, cfg.SyncFanoutBatchSize, cfg.SyncFanoutFallbackPoll, cfg.SyncFanoutNotifyChannel)) + } ctx, cancel := context.WithCancel(ctx) defer cancel() @@ -109,3 +121,27 @@ func main() { defer cancelStop() _ = runner.StopAll(stopCtx) } + +func workerSetFromEnv(value string) map[string]struct{} { + value = strings.TrimSpace(value) + if value == "" { + return nil + } + out := make(map[string]struct{}) + for _, item := range strings.Split(value, ",") { + item = strings.TrimSpace(strings.ToLower(item)) + if item == "" { + continue + } + out[item] = struct{}{} + } + return out +} + +func workerEnabled(enabled map[string]struct{}, name string) bool { + if len(enabled) == 0 { + return true + } + _, ok := enabled[strings.TrimSpace(strings.ToLower(name))] + return ok +} diff --git a/ohmf/services/gateway/internal/config/config.go b/ohmf/services/gateway/internal/config/config.go index 8ddfeb6..c0c099e 100644 --- a/ohmf/services/gateway/internal/config/config.go +++ b/ohmf/services/gateway/internal/config/config.go @@ -30,8 +30,11 @@ type Config struct { KafkaPresenceTopic string KafkaMicroserviceTopic string KafkaClientID string - AckTimeout time.Duration - CassandraHosts string + AckTimeout time.Duration + SyncFanoutBatchSize int + SyncFanoutFallbackPoll time.Duration + SyncFanoutNotifyChannel string + CassandraHosts string CassandraKeyspace string CassandraUsername string CassandraPassword string @@ -123,6 +126,9 @@ func Load() Config { KafkaMicroserviceTopic: get("APP_KAFKA_MICROSERVICE_TOPIC", "microservice.events.v1"), KafkaClientID: get("APP_KAFKA_CLIENT_ID", "ohmf-gateway"), AckTimeout: time.Duration(getInt("APP_ACK_TIMEOUT_MS", 2000)) * time.Millisecond, + SyncFanoutBatchSize: getInt("APP_SYNC_FANOUT_BATCH_SIZE", 100), + SyncFanoutFallbackPoll: time.Duration(getInt("APP_SYNC_FANOUT_FALLBACK_POLL_MS", 1000)) * time.Millisecond, + SyncFanoutNotifyChannel: get("APP_SYNC_FANOUT_NOTIFY_CHANNEL", "ohmf_domain_events"), CassandraHosts: get("APP_CASSANDRA_HOSTS", "localhost:9042"), CassandraKeyspace: get("APP_CASSANDRA_KEYSPACE", "ohmf_messages"), CassandraUsername: get("APP_CASSANDRA_USERNAME", ""), diff --git a/ohmf/services/gateway/internal/messages/async.go b/ohmf/services/gateway/internal/messages/async.go index 14b2aa1..bf48113 100644 --- a/ohmf/services/gateway/internal/messages/async.go +++ b/ohmf/services/gateway/internal/messages/async.go @@ -3,8 +3,8 @@ package messages import ( "context" "encoding/json" - "errors" "fmt" + "sync" "time" "github.com/google/uuid" @@ -13,9 +13,8 @@ import ( ) const ( - ackKeyPrefix = "msg:ack:" - ackSignalPrefix = "msg:ack:notify:" - legacyAckPollInterval = time.Second + ackKeyPrefix = "msg:ack:" + ackChannelName = "msg:ack:events" ) type IngressEvent struct { @@ -48,16 +47,39 @@ type PersistedAck struct { type AsyncPipeline struct { producer bus.IngressProducer redis *redis.Client + waiters map[string][]chan PersistedAck + mu sync.Mutex + cancel context.CancelFunc + stopOnce sync.Once + wg sync.WaitGroup } func NewAsyncPipeline(producer bus.IngressProducer, redisClient *redis.Client) *AsyncPipeline { if producer == nil || redisClient == nil { return nil } - return &AsyncPipeline{ + ctx, cancel := context.WithCancel(context.Background()) + pipeline := &AsyncPipeline{ producer: producer, redis: redisClient, + waiters: make(map[string][]chan PersistedAck), + cancel: cancel, } + pipeline.wg.Add(1) + go pipeline.runAckSubscriber(ctx) + return pipeline +} + +func (p *AsyncPipeline) Stop() { + if p == nil { + return + } + p.stopOnce.Do(func() { + if p.cancel != nil { + p.cancel() + } + p.wg.Wait() + }) } func (p *AsyncPipeline) PublishIngress(ctx context.Context, evt IngressEvent) error { @@ -79,59 +101,34 @@ func (p *AsyncPipeline) WaitAck(ctx context.Context, eventID string, timeout tim if p == nil || p.redis == nil { return PersistedAck{}, false, nil } - waitCtx, cancel := context.WithTimeout(ctx, timeout) - defer cancel() - - key := AckRedisKey(eventID) - if ack, ok, err := p.readAckWithTimeoutHandling(ctx, waitCtx, key); ok || err != nil { - return ack, ok, err - } - - pubsub := p.redis.Subscribe(waitCtx, ackSignalChannel(eventID)) - defer pubsub.Close() - if _, err := pubsub.Receive(waitCtx); err != nil { - return ackWaitResult(ctx, waitCtx, err) + if timeout <= 0 { + timeout = 2 * time.Second } + key := ackKeyPrefix + eventID + waiter := make(chan PersistedAck, 1) + p.registerWaiter(eventID, waiter) + defer p.unregisterWaiter(eventID, waiter) - // Re-check the durable ack key after the subscription is live so a key write - // that raced with subscription setup is still observed without polling. - if ack, ok, err := p.readAckWithTimeoutHandling(ctx, waitCtx, key); ok || err != nil { - return ack, ok, err + if ack, ok, err := p.loadPersistedAck(ctx, key); err != nil { + return PersistedAck{}, false, err + } else if ok { + return ack, true, nil } - msgCh := pubsub.Channel() - legacyTicker := time.NewTicker(legacyAckPollInterval) - defer legacyTicker.Stop() - - for { - select { - case <-waitCtx.Done(): - // One final authoritative key read avoids timing out on an ack that - // landed just before the wait context expired. - if ack, ok, err := p.readAck(context.Background(), key); ok || err != nil { - return ack, ok, err - } - return ackWaitResult(ctx, waitCtx, waitCtx.Err()) - case msg, ok := <-msgCh: - if !ok { - if ack, found, err := p.readAckWithTimeoutHandling(ctx, waitCtx, key); found || err != nil { - return ack, found, err - } - return ackWaitResult(ctx, waitCtx, errors.New("redis ack subscription closed")) - } - ack, err := decodeAckPayload(msg.Payload) - if err != nil { - return PersistedAck{}, false, err - } + timer := time.NewTimer(timeout) + defer timer.Stop() + select { + case <-ctx.Done(): + return PersistedAck{}, false, ctx.Err() + case ack := <-waiter: + return ack, true, nil + case <-timer.C: + if ack, ok, err := p.loadPersistedAck(ctx, key); err != nil { + return PersistedAck{}, false, err + } else if ok { return ack, true, nil - case <-legacyTicker.C: - // Sparse fallback polling is retained for rolling-deploy compatibility - // and missed/lost Pub/Sub notifications. The durable ack key remains - // authoritative; Pub/Sub is only a wake-up optimization. - if ack, ok, err := p.readAckWithTimeoutHandling(ctx, waitCtx, key); ok || err != nil { - return ack, ok, err - } } + return PersistedAck{}, false, nil } } @@ -176,47 +173,116 @@ func AckRedisKey(eventID string) string { return fmt.Sprintf("%s%s", ackKeyPrefix, eventID) } -func ackSignalChannel(eventID string) string { - return fmt.Sprintf("%s%s", ackSignalPrefix, eventID) +func AckRedisChannel() string { + return ackChannelName } -func (p *AsyncPipeline) readAck(ctx context.Context, key string) (PersistedAck, bool, error) { +func (p *AsyncPipeline) loadPersistedAck(ctx context.Context, key string) (PersistedAck, bool, error) { payload, err := p.redis.Get(ctx, key).Result() - if err == nil { - ack, err := decodeAckPayload(payload) - if err != nil { - return PersistedAck{}, false, err - } - return ack, true, nil + if err == redis.Nil { + return PersistedAck{}, false, nil + } + if err != nil { + return PersistedAck{}, false, err } - if err != nil && err != redis.Nil { + var ack PersistedAck + if err := json.Unmarshal([]byte(payload), &ack); err != nil { return PersistedAck{}, false, err } - return PersistedAck{}, false, nil + return ack, true, nil +} + +func (p *AsyncPipeline) registerWaiter(eventID string, waiter chan PersistedAck) { + p.mu.Lock() + defer p.mu.Unlock() + p.waiters[eventID] = append(p.waiters[eventID], waiter) } -func (p *AsyncPipeline) readAckWithTimeoutHandling(parentCtx, waitCtx context.Context, key string) (PersistedAck, bool, error) { - ack, ok, err := p.readAck(waitCtx, key) - if err == nil { - return ack, ok, nil +func (p *AsyncPipeline) unregisterWaiter(eventID string, waiter chan PersistedAck) { + p.mu.Lock() + defer p.mu.Unlock() + waiters := p.waiters[eventID] + if len(waiters) == 0 { + return + } + filtered := waiters[:0] + for _, item := range waiters { + if item == waiter { + continue + } + filtered = append(filtered, item) + } + if len(filtered) == 0 { + delete(p.waiters, eventID) + return } - return ackWaitResult(parentCtx, waitCtx, err) + p.waiters[eventID] = filtered } -func decodeAckPayload(payload string) (PersistedAck, error) { - var ack PersistedAck - if err := json.Unmarshal([]byte(payload), &ack); err != nil { - return PersistedAck{}, err +func (p *AsyncPipeline) dispatchAck(ack PersistedAck) { + if p == nil || ack.EventID == "" { + return + } + p.mu.Lock() + waiters := append([]chan PersistedAck(nil), p.waiters[ack.EventID]...) + p.mu.Unlock() + for _, waiter := range waiters { + select { + case waiter <- ack: + default: + } } - return ack, nil } -func ackWaitResult(parentCtx, waitCtx context.Context, err error) (PersistedAck, bool, error) { - if parentErr := parentCtx.Err(); parentErr != nil { - return PersistedAck{}, false, parentErr +func (p *AsyncPipeline) runAckSubscriber(ctx context.Context) { + if p == nil { + return } - if errors.Is(err, context.DeadlineExceeded) || errors.Is(waitCtx.Err(), context.DeadlineExceeded) { - return PersistedAck{}, false, nil + defer p.wg.Done() + if p.redis == nil { + return + } + for { + select { + case <-ctx.Done(): + return + default: + } + pubsub := p.redis.Subscribe(ctx, ackChannelName) + if _, err := pubsub.Receive(ctx); err != nil { + _ = pubsub.Close() + if !sleepWithContext(ctx, 250*time.Millisecond) { + return + } + continue + } + func() { + defer pubsub.Close() + for { + msg, err := pubsub.ReceiveMessage(ctx) + if err != nil { + return + } + var ack PersistedAck + if err := json.Unmarshal([]byte(msg.Payload), &ack); err != nil { + continue + } + p.dispatchAck(ack) + } + }() + if !sleepWithContext(ctx, 250*time.Millisecond) { + return + } + } +} + +func sleepWithContext(ctx context.Context, d time.Duration) bool { + timer := time.NewTimer(d) + defer timer.Stop() + select { + case <-ctx.Done(): + return false + case <-timer.C: + return true } - return PersistedAck{}, false, err } diff --git a/ohmf/services/gateway/internal/messages/async_test.go b/ohmf/services/gateway/internal/messages/async_test.go index 264957f..2bc0d45 100644 --- a/ohmf/services/gateway/internal/messages/async_test.go +++ b/ohmf/services/gateway/internal/messages/async_test.go @@ -2,660 +2,77 @@ package messages import ( "context" - "encoding/json" - "errors" - "strings" - "sync" "testing" "time" miniredis "github.com/alicebob/miniredis/v2" - "github.com/jackc/pgx/v5" - pgxmock "github.com/pashagolub/pgxmock/v4" "github.com/redis/go-redis/v9" ) -func TestAsyncPipelineWaitAckReturnsPublishedAck(t *testing.T) { - mr, err := miniredis.Run() - if err != nil { - t.Fatalf("start miniredis: %v", err) - } - defer mr.Close() - - rdb := redis.NewClient(&redis.Options{Addr: mr.Addr()}) - defer rdb.Close() - - pipeline := &AsyncPipeline{redis: rdb} - want := PersistedAck{ - EventID: "event-1", - MessageID: "message-1", - ConversationID: "conversation-1", - ServerOrder: 41, - Status: "SENT", - Transport: "OHMF", - PersistedAtMS: 1700000000000, - } - errCh := make(chan error, 1) - - go func() { - time.Sleep(20 * time.Millisecond) - errCh <- writeAckForTest(rdb, want.EventID, want) - }() - - got, ok, err := pipeline.WaitAck(context.Background(), want.EventID, 500*time.Millisecond) - if err != nil { - t.Fatalf("WaitAck failed: %v", err) - } - if !ok { - t.Fatalf("expected ack to arrive before timeout") - } - if got != want { - t.Fatalf("unexpected ack: %+v", got) - } - if err := <-errCh; err != nil { - t.Fatalf("write ack: %v", err) - } -} - -func TestAsyncPipelineWaitAckTimeoutDoesNotBusyPoll(t *testing.T) { - mr, err := miniredis.Run() - if err != nil { - t.Fatalf("start miniredis: %v", err) - } - defer mr.Close() +type fakeIngressProducer struct{} - rdb := redis.NewClient(&redis.Options{Addr: mr.Addr()}) - defer rdb.Close() - - hook := newRedisCommandCounter() - rdb.AddHook(hook) +func (fakeIngressProducer) PublishIngress(context.Context, string, any) error { return nil } - pipeline := &AsyncPipeline{redis: rdb} - start := time.Now() - _, ok, err := pipeline.WaitAck(context.Background(), "event-timeout", 1200*time.Millisecond) - if err != nil { - t.Fatalf("WaitAck returned error: %v", err) +func TestAsyncPipelineDispatchAckToWaiter(t *testing.T) { + pipeline := &AsyncPipeline{ + waiters: make(map[string][]chan PersistedAck), } - if ok { - t.Fatalf("expected timeout without ack") - } - if elapsed := time.Since(start); elapsed < time.Second { - t.Fatalf("WaitAck returned too early: %v", elapsed) - } - if got := hook.Count("get"); got > 4 { - t.Fatalf("expected sparse fallback reads plus final timeout check, got %d GET commands", got) - } -} - -func TestAsyncPipelineWaitAckTreatsRedisGetDeadlineAsTimeout(t *testing.T) { - mr, err := miniredis.Run() - if err != nil { - t.Fatalf("start miniredis: %v", err) - } - defer mr.Close() - - rdb := redis.NewClient(&redis.Options{Addr: mr.Addr()}) - defer rdb.Close() - - rdb.AddHook(redisDelayHook{delay: 40 * time.Millisecond}) - - pipeline := &AsyncPipeline{redis: rdb} - _, ok, err := pipeline.WaitAck(context.Background(), "event-read-timeout", 10*time.Millisecond) - if err != nil { - t.Fatalf("WaitAck returned error instead of timeout: %v", err) - } - if ok { - t.Fatalf("expected timeout without ack") - } -} - -func TestSendAsyncReturnsCanonicalMessageWhenAckArrives(t *testing.T) { - mock, err := pgxmock.NewPool() - if err != nil { - t.Fatal(err) - } - defer mock.Close() - - mr, err := miniredis.Run() - if err != nil { - t.Fatalf("start miniredis: %v", err) - } - defer mr.Close() + waiter := make(chan PersistedAck, 1) + pipeline.registerWaiter("evt-1", waiter) - rdb := redis.NewClient(&redis.Options{Addr: mr.Addr()}) - defer rdb.Close() - - producer := &recordingIngressProducer{published: make(chan IngressEvent, 2)} - pipeline := NewAsyncPipeline(producer, rdb) - svc := &Service{ - db: mock, - useKafkaSend: true, - async: pipeline, - ackTimeout: 500 * time.Millisecond, + ack := PersistedAck{ + EventID: "evt-1", + MessageID: "msg-1", + ConversationID: "conv-1", + ServerOrder: 5, } + pipeline.dispatchAck(ack) - expectAsyncTextSend(mock, 202, 201) - errCh := make(chan error, 1) - - go func() { - evt := <-producer.published - errCh <- writeAckForTest(rdb, evt.EventID, PersistedAck{ - EventID: evt.EventID, - MessageID: evt.MessageID, - ConversationID: evt.ConversationID, - ServerOrder: 77, - Status: "SENT", - Transport: "OHMF", - PersistedAtMS: 1700000000123, - }) - }() - - result, err := svc.Send(context.Background(), "user-1", "device-1", "conversation-1", "idem-1", "text", map[string]any{"text": "hi"}, "client-1", "trace-1", "") - if err != nil { - t.Fatalf("Send failed: %v", err) - } - if result.Queued { - t.Fatalf("expected canonical response, got queued result") - } - if result.Message.ServerOrder != 77 { - t.Fatalf("expected canonical server_order, got %d", result.Message.ServerOrder) - } - if result.Message.Status != "SENT" { - t.Fatalf("expected SENT status, got %q", result.Message.Status) - } - if result.Message.Transport != "OHMF" { - t.Fatalf("expected OHMF transport, got %q", result.Message.Transport) - } - if result.Message.ClientGeneratedID != "client-1" { - t.Fatalf("expected client_generated_id to survive ack reconciliation, got %q", result.Message.ClientGeneratedID) - } - if err := <-errCh; err != nil { - t.Fatalf("write ack: %v", err) + select { + case got := <-waiter: + if got.EventID != ack.EventID || got.MessageID != ack.MessageID { + t.Fatalf("unexpected ack dispatch: %#v", got) + } + case <-time.After(250 * time.Millisecond): + t.Fatal("timed out waiting for dispatched ack") } - if err := mock.ExpectationsWereMet(); err != nil { - t.Fatalf("unmet expectations: %v", err) + pipeline.unregisterWaiter("evt-1", waiter) + if len(pipeline.waiters) != 0 { + t.Fatalf("expected waiter cleanup, got %#v", pipeline.waiters) } } -func TestSendAsyncReturnsCanonicalMessageAfterAckTimeoutOnceDurableStateIsVisible(t *testing.T) { - mock, err := pgxmock.NewPool() - if err != nil { - t.Fatal(err) - } - defer mock.Close() - - mr, err := miniredis.Run() - if err != nil { - t.Fatalf("start miniredis: %v", err) - } - defer mr.Close() - - rdb := redis.NewClient(&redis.Options{Addr: mr.Addr()}) - defer rdb.Close() - - producer := &recordingIngressProducer{published: make(chan IngressEvent, 1)} - pipeline := NewAsyncPipeline(producer, rdb) - svc := &Service{ - db: mock, - useKafkaSend: true, - async: pipeline, - ackTimeout: 40 * time.Millisecond, - } - - expectMembershipOK(mock) - expectUnblocked(mock) - mock.ExpectQuery(`SELECT type, COALESCE\(encryption_state, 'PLAINTEXT'\), COALESCE\(is_mls_encrypted, false\) FROM conversations WHERE id = \$1::uuid`). - WithArgs("conversation-1"). - WillReturnRows(pgxmock.NewRows([]string{"type", "encryption_state", "is_mls_encrypted"}).AddRow("GROUP", "PLAINTEXT", false)) - mock.ExpectQuery(`SELECT response_payload, COALESCE\(status_code, 201\)`). - WithArgs("user-1", "/v1/messages", "idem-1"). - WillReturnError(pgx.ErrNoRows) - mock.ExpectExec(`INSERT INTO idempotency_keys \(actor_user_id, endpoint, key, response_payload, status_code, expires_at\)`). - WithArgs("user-1", "/v1/messages", "idem-1", pgxmock.AnyArg(), 202). - WillReturnResult(pgxmock.NewResult("INSERT", 1)) - mock.ExpectQuery(`SELECT response_payload, COALESCE\(status_code, 201\)`). - WithArgs("user-1", "/v1/messages", "idem-1"). - WillReturnRows(pgxmock.NewRows([]string{"response_payload", "status_code"}).AddRow(mustJSONMessage(t, Message{ - MessageID: "message-durable-1", - ConversationID: "conversation-1", - SenderUserID: "user-1", - SenderDeviceID: "device-1", - ContentType: "text", - Content: map[string]any{"text": "hi"}, - ClientGeneratedID: "client-1", - Transport: "OHMF", - ServerOrder: 91, - Status: "SENT", - CreatedAt: "2026-06-09T00:00:00Z", - }), 201)) - - result, err := svc.Send(context.Background(), "user-1", "device-1", "conversation-1", "idem-1", "text", map[string]any{"text": "hi"}, "client-1", "trace-1", "") - if err != nil { - t.Fatalf("Send failed: %v", err) - } - if result.Queued { - t.Fatalf("expected canonical success after durable state became visible, got queued result") - } - if result.Message.MessageID != "message-durable-1" { - t.Fatalf("expected durable message to be returned, got %+v", result.Message) - } - if result.Message.ServerOrder != 91 || result.Message.Status != "SENT" { - t.Fatalf("unexpected durable message payload: %+v", result.Message) - } - - expectMembershipOK(mock) - expectUnblocked(mock) - mock.ExpectQuery(`SELECT type, COALESCE\(encryption_state, 'PLAINTEXT'\), COALESCE\(is_mls_encrypted, false\) FROM conversations WHERE id = \$1::uuid`). - WithArgs("conversation-1"). - WillReturnRows(pgxmock.NewRows([]string{"type", "encryption_state", "is_mls_encrypted"}).AddRow("GROUP", "PLAINTEXT", false)) - mock.ExpectQuery(`SELECT response_payload, COALESCE\(status_code, 201\)`). - WithArgs("user-1", "/v1/messages", "idem-1"). - WillReturnRows(pgxmock.NewRows([]string{"response_payload", "status_code"}).AddRow(mustJSONMessage(t, Message{ - MessageID: "message-durable-1", - ConversationID: "conversation-1", - SenderUserID: "user-1", - SenderDeviceID: "device-1", - ContentType: "text", - Content: map[string]any{"text": "hi"}, - ClientGeneratedID: "client-1", - Transport: "OHMF", - ServerOrder: 91, - Status: "SENT", - CreatedAt: "2026-06-09T00:00:00Z", - }), 201)) - - retry, err := svc.Send(context.Background(), "user-1", "device-1", "conversation-1", "idem-1", "text", map[string]any{"text": "hi"}, "client-1", "trace-1", "") - if err != nil { - t.Fatalf("retry Send failed: %v", err) - } - if retry.Queued { - t.Fatalf("expected retry to return original message, not queued") - } - if retry.Message.MessageID != result.Message.MessageID || retry.Message.ServerOrder != result.Message.ServerOrder { - t.Fatalf("expected retry to return original message, got %+v vs %+v", retry.Message, result.Message) - } - - if err := mock.ExpectationsWereMet(); err != nil { - t.Fatalf("unmet expectations: %v", err) +func TestAckRedisChannel(t *testing.T) { + if got := AckRedisChannel(); got != "msg:ack:events" { + t.Fatalf("unexpected ack channel: %q", got) } } -func TestSendAsyncRedisAckFailureBeforeDurablePersistenceFailsSafely(t *testing.T) { - mock, err := pgxmock.NewPool() - if err != nil { - t.Fatal(err) - } - defer mock.Close() - +func TestAsyncPipelineStopTerminatesAckSubscriber(t *testing.T) { mr, err := miniredis.Run() - if err != nil { - t.Fatalf("start miniredis: %v", err) - } - defer mr.Close() - - rdb := redis.NewClient(&redis.Options{Addr: mr.Addr()}) - defer rdb.Close() - rdb.AddHook(redisGetErrorHook{err: errors.New("redis unavailable")}) - - producer := &recordingIngressProducer{published: make(chan IngressEvent, 1)} - pipeline := NewAsyncPipeline(producer, rdb) - svc := &Service{ - db: mock, - useKafkaSend: true, - async: pipeline, - ackTimeout: 40 * time.Millisecond, - } - - expectMembershipOK(mock) - expectUnblocked(mock) - mock.ExpectQuery(`SELECT type, COALESCE\(encryption_state, 'PLAINTEXT'\), COALESCE\(is_mls_encrypted, false\) FROM conversations WHERE id = \$1::uuid`). - WithArgs("conversation-1"). - WillReturnRows(pgxmock.NewRows([]string{"type", "encryption_state", "is_mls_encrypted"}).AddRow("GROUP", "PLAINTEXT", false)) - mock.ExpectQuery(`SELECT response_payload, COALESCE\(status_code, 201\)`). - WithArgs("user-1", "/v1/messages", "idem-1"). - WillReturnError(pgx.ErrNoRows) - mock.ExpectExec(`INSERT INTO idempotency_keys \(actor_user_id, endpoint, key, response_payload, status_code, expires_at\)`). - WithArgs("user-1", "/v1/messages", "idem-1", pgxmock.AnyArg(), 202). - WillReturnResult(pgxmock.NewResult("INSERT", 1)) - mock.ExpectQuery(`SELECT response_payload, COALESCE\(status_code, 201\)`). - WithArgs("user-1", "/v1/messages", "idem-1"). - WillReturnRows(pgxmock.NewRows([]string{"response_payload", "status_code"}).AddRow(mustJSONMessage(t, Message{ - MessageID: "message-provisional-1", - ConversationID: "conversation-1", - SenderUserID: "user-1", - SenderDeviceID: "device-1", - ContentType: "text", - Content: map[string]any{"text": "hi"}, - ClientGeneratedID: "client-1", - Transport: "OHMF", - ServerOrder: 0, - Status: "QUEUED", - CreatedAt: "2026-06-09T00:00:00Z", - }), 202)) - - _, err = svc.Send(context.Background(), "user-1", "device-1", "conversation-1", "idem-1", "text", map[string]any{"text": "hi"}, "client-1", "trace-1", "") - if err == nil { - t.Fatalf("expected send to fail safely before durable persistence is visible") - } - - if err := mock.ExpectationsWereMet(); err != nil { - t.Fatalf("unmet expectations: %v", err) - } -} - -func TestSendAsyncReturnsAckWhenPostAckIdempotencyPersistFails(t *testing.T) { - mock, err := pgxmock.NewPool() if err != nil { t.Fatal(err) } - defer mock.Close() - - mr, err := miniredis.Run() - if err != nil { - t.Fatalf("start miniredis: %v", err) - } defer mr.Close() rdb := redis.NewClient(&redis.Options{Addr: mr.Addr()}) defer rdb.Close() - producer := &recordingIngressProducer{published: make(chan IngressEvent, 1)} - pipeline := NewAsyncPipeline(producer, rdb) - svc := &Service{ - db: mock, - useKafkaSend: true, - async: pipeline, - ackTimeout: 500 * time.Millisecond, + pipeline := NewAsyncPipeline(fakeIngressProducer{}, rdb) + if pipeline == nil { + t.Fatal("expected pipeline") } - expectMembershipOK(mock) - expectUnblocked(mock) - mock.ExpectQuery(`SELECT type, COALESCE\(encryption_state, 'PLAINTEXT'\), COALESCE\(is_mls_encrypted, false\) FROM conversations WHERE id = \$1::uuid`). - WithArgs("conversation-1"). - WillReturnRows(pgxmock.NewRows([]string{"type", "encryption_state", "is_mls_encrypted"}).AddRow("GROUP", "PLAINTEXT", false)) - mock.ExpectQuery(`SELECT response_payload, COALESCE\(status_code, 201\)`). - WithArgs("user-1", "/v1/messages", "idem-1"). - WillReturnError(pgx.ErrNoRows) - mock.ExpectExec(`INSERT INTO idempotency_keys \(actor_user_id, endpoint, key, response_payload, status_code, expires_at\)`). - WithArgs("user-1", "/v1/messages", "idem-1", pgxmock.AnyArg(), 202). - WillReturnResult(pgxmock.NewResult("INSERT", 1)) - mock.ExpectExec(`INSERT INTO idempotency_keys \(actor_user_id, endpoint, key, response_payload, status_code, expires_at\)`). - WithArgs("user-1", "/v1/messages", "idem-1", pgxmock.AnyArg(), 201). - WillReturnError(errors.New("idempotency store unavailable")) - - errCh := make(chan error, 1) + done := make(chan struct{}) go func() { - evt := <-producer.published - errCh <- writeAckForTest(rdb, evt.EventID, PersistedAck{ - EventID: evt.EventID, - MessageID: evt.MessageID, - ConversationID: evt.ConversationID, - ServerOrder: 79, - Status: "SENT", - Transport: "OHMF", - PersistedAtMS: 1700000000222, - }) + pipeline.Stop() + close(done) }() - result, err := svc.Send(context.Background(), "user-1", "device-1", "conversation-1", "idem-1", "text", map[string]any{"text": "hi"}, "client-1", "trace-1", "") - if err != nil { - t.Fatalf("Send failed: %v", err) - } - if result.Queued { - t.Fatalf("expected canonical response, got queued result") - } - if result.Message.MessageID == "" || result.Message.ServerOrder != 79 { - t.Fatalf("expected persisted ack response, got %+v", result.Message) - } - if err := <-errCh; err != nil { - t.Fatalf("write ack: %v", err) - } - if err := mock.ExpectationsWereMet(); err != nil { - t.Fatalf("unmet expectations: %v", err) - } -} - -func TestSendAsyncTimeoutReturnsProvisionalAndLateAckRemainsReadable(t *testing.T) { - mock, err := pgxmock.NewPool() - if err != nil { - t.Fatal(err) - } - defer mock.Close() - - mr, err := miniredis.Run() - if err != nil { - t.Fatalf("start miniredis: %v", err) - } - defer mr.Close() - - rdb := redis.NewClient(&redis.Options{Addr: mr.Addr()}) - defer rdb.Close() - - producer := &recordingIngressProducer{published: make(chan IngressEvent, 1)} - pipeline := NewAsyncPipeline(producer, rdb) - svc := &Service{ - db: mock, - useKafkaSend: true, - async: pipeline, - ackTimeout: 40 * time.Millisecond, - } - - expectAsyncTextSend(mock, 202) - mock.ExpectQuery(`SELECT response_payload, COALESCE\(status_code, 201\)`). - WithArgs("user-1", "/v1/messages", "idem-1"). - WillReturnRows(pgxmock.NewRows([]string{"response_payload", "status_code"}).AddRow(mustJSONMessage(t, Message{ - MessageID: "message-provisional-1", - ConversationID: "conversation-1", - SenderUserID: "user-1", - SenderDeviceID: "device-1", - ContentType: "text", - Content: map[string]any{"text": "hi"}, - ClientGeneratedID: "client-1", - Transport: "OHMF", - ServerOrder: 0, - Status: "QUEUED", - CreatedAt: "2026-06-09T00:00:00Z", - }), 202)) - - result, err := svc.Send(context.Background(), "user-1", "device-1", "conversation-1", "idem-1", "text", map[string]any{"text": "hi"}, "client-1", "trace-1", "") - if err != nil { - t.Fatalf("Send failed: %v", err) - } - if !result.Queued { - t.Fatalf("expected provisional queued response on timeout") - } - if result.AckTimeoutMS != 40 { - t.Fatalf("expected ack_timeout_ms=40, got %d", result.AckTimeoutMS) - } - if result.Message.ServerOrder != 0 { - t.Fatalf("expected provisional server_order=0, got %d", result.Message.ServerOrder) - } - if result.Message.Status != "QUEUED" { - t.Fatalf("expected provisional status QUEUED, got %q", result.Message.Status) - } - - evt := producer.Last() - if evt.EventID == "" { - t.Fatalf("expected ingress event to be published") - } - lateAck := PersistedAck{ - EventID: evt.EventID, - MessageID: evt.MessageID, - ConversationID: evt.ConversationID, - ServerOrder: 88, - Status: "SENT", - Transport: "OHMF", - PersistedAtMS: 1700000000456, - } - body, err := json.Marshal(lateAck) - if err != nil { - t.Fatalf("marshal late ack: %v", err) - } - if err := rdb.Set(context.Background(), AckRedisKey(evt.EventID), string(body), 24*time.Hour).Err(); err != nil { - t.Fatalf("persist late ack: %v", err) - } - - got, ok, err := pipeline.WaitAck(context.Background(), evt.EventID, 50*time.Millisecond) - if err != nil { - t.Fatalf("WaitAck failed for late ack: %v", err) - } - if !ok { - t.Fatalf("expected late durable ack to remain readable") - } - if got.ServerOrder != lateAck.ServerOrder || got.MessageID != lateAck.MessageID { - t.Fatalf("unexpected late ack: %+v", got) - } - - if err := mock.ExpectationsWereMet(); err != nil { - t.Fatalf("unmet expectations: %v", err) - } -} - -func expectAsyncTextSend(mock pgxmock.PgxPoolIface, idempotencyStatuses ...int) { - expectMembershipOK(mock) - expectUnblocked(mock) - mock.ExpectQuery(`SELECT type, COALESCE\(encryption_state, 'PLAINTEXT'\), COALESCE\(is_mls_encrypted, false\) FROM conversations WHERE id = \$1::uuid`). - WithArgs("conversation-1"). - WillReturnRows(pgxmock.NewRows([]string{"type", "encryption_state", "is_mls_encrypted"}).AddRow("GROUP", "PLAINTEXT", false)) - mock.ExpectQuery(`SELECT response_payload, COALESCE\(status_code, 201\)`). - WithArgs("user-1", "/v1/messages", "idem-1"). - WillReturnError(pgx.ErrNoRows) - for _, status := range idempotencyStatuses { - mock.ExpectExec(`INSERT INTO idempotency_keys \(actor_user_id, endpoint, key, response_payload, status_code, expires_at\)`). - WithArgs("user-1", "/v1/messages", "idem-1", pgxmock.AnyArg(), status). - WillReturnResult(pgxmock.NewResult("INSERT", 1)) + select { + case <-done: + case <-time.After(2 * time.Second): + t.Fatal("timed out waiting for pipeline stop") } } - -func writeAckForTest(rdb *redis.Client, eventID string, ack PersistedAck) error { - body, err := json.Marshal(ack) - if err != nil { - return err - } - if err := rdb.Set(context.Background(), AckRedisKey(eventID), string(body), 24*time.Hour).Err(); err != nil { - return err - } - if err := rdb.Publish(context.Background(), ackSignalChannel(eventID), string(body)).Err(); err != nil { - return err - } - return nil -} - -func mustJSONMessage(t *testing.T, v any) string { - t.Helper() - body, err := json.Marshal(v) - if err != nil { - t.Fatalf("marshal json: %v", err) - } - return string(body) -} - -type recordingIngressProducer struct { - mu sync.Mutex - last IngressEvent - published chan IngressEvent -} - -func (p *recordingIngressProducer) PublishIngress(_ context.Context, _ string, payload any) error { - evt, ok := payload.(IngressEvent) - if !ok { - return errors.New("unexpected ingress payload type") - } - p.mu.Lock() - p.last = evt - p.mu.Unlock() - if p.published != nil { - p.published <- evt - } - return nil -} - -func (p *recordingIngressProducer) Last() IngressEvent { - p.mu.Lock() - defer p.mu.Unlock() - return p.last -} - -type redisCommandCounter struct { - mu sync.Mutex - counts map[string]int -} - -func newRedisCommandCounter() *redisCommandCounter { - return &redisCommandCounter{counts: make(map[string]int)} -} - -func (h *redisCommandCounter) DialHook(next redis.DialHook) redis.DialHook { - return next -} - -func (h *redisCommandCounter) ProcessHook(next redis.ProcessHook) redis.ProcessHook { - return func(ctx context.Context, cmd redis.Cmder) error { - h.record(cmd.Name()) - return next(ctx, cmd) - } -} - -func (h *redisCommandCounter) ProcessPipelineHook(next redis.ProcessPipelineHook) redis.ProcessPipelineHook { - return func(ctx context.Context, cmds []redis.Cmder) error { - for _, cmd := range cmds { - h.record(cmd.Name()) - } - return next(ctx, cmds) - } -} - -func (h *redisCommandCounter) Count(name string) int { - h.mu.Lock() - defer h.mu.Unlock() - return h.counts[strings.ToLower(name)] -} - -func (h *redisCommandCounter) record(name string) { - h.mu.Lock() - defer h.mu.Unlock() - h.counts[strings.ToLower(name)]++ -} - -type redisDelayHook struct { - delay time.Duration -} - -func (h redisDelayHook) DialHook(next redis.DialHook) redis.DialHook { - return next -} - -func (h redisDelayHook) ProcessHook(next redis.ProcessHook) redis.ProcessHook { - return func(ctx context.Context, cmd redis.Cmder) error { - if strings.EqualFold(cmd.Name(), "get") { - time.Sleep(h.delay) - } - return next(ctx, cmd) - } -} - -func (h redisDelayHook) ProcessPipelineHook(next redis.ProcessPipelineHook) redis.ProcessPipelineHook { - return next -} - -type redisGetErrorHook struct { - err error -} - -func (h redisGetErrorHook) DialHook(next redis.DialHook) redis.DialHook { - return next -} - -func (h redisGetErrorHook) ProcessHook(next redis.ProcessHook) redis.ProcessHook { - return func(ctx context.Context, cmd redis.Cmder) error { - if strings.EqualFold(cmd.Name(), "get") { - return h.err - } - return next(ctx, cmd) - } -} - -func (h redisGetErrorHook) ProcessPipelineHook(next redis.ProcessPipelineHook) redis.ProcessPipelineHook { - return next -} diff --git a/ohmf/services/gateway/internal/messages/service.go b/ohmf/services/gateway/internal/messages/service.go index 5cc0de6..c252ff0 100644 --- a/ohmf/services/gateway/internal/messages/service.go +++ b/ohmf/services/gateway/internal/messages/service.go @@ -76,6 +76,10 @@ type DB interface { QueryRow(ctx context.Context, sql string, args ...any) pgx.Row } +type rowQuerier interface { + QueryRow(ctx context.Context, sql string, args ...any) pgx.Row +} + type Service struct { db DB useKafkaSend bool @@ -2572,23 +2576,30 @@ func (s *Service) MarkRead(ctx context.Context, actor, conversationID string, th return ErrConversationAccess } - // Insert per-message read receipts for newly read messages - if _, err := tx.Exec(ctx, ` - INSERT INTO message_read_receipts(message_id, reader_user_id, read_at) - SELECT m.id, $1::uuid, $2 - FROM messages m - WHERE m.conversation_id = $3::uuid - AND m.server_order <= $4 - AND m.sender_user_id <> $1::uuid - AND NOT EXISTS ( - SELECT 1 FROM message_read_receipts mrr - WHERE mrr.message_id = m.id AND mrr.reader_user_id = $1::uuid - ) - ON CONFLICT(message_id, reader_user_id) DO NOTHING - `, actor, now, conversationID, through); err != nil { + sendReadReceipts, err := userSendsReadReceipts(ctx, tx, actor) + if err != nil { return err } + // Insert per-message read receipts for newly read messages + if sendReadReceipts { + if _, err := tx.Exec(ctx, ` + INSERT INTO message_read_receipts(message_id, reader_user_id, read_at) + SELECT m.id, $1::uuid, $2 + FROM messages m + WHERE m.conversation_id = $3::uuid + AND m.server_order <= $4 + AND m.sender_user_id <> $1::uuid + AND NOT EXISTS ( + SELECT 1 FROM message_read_receipts mrr + WHERE mrr.message_id = m.id AND mrr.reader_user_id = $1::uuid + ) + ON CONFLICT(message_id, reader_user_id) DO NOTHING + `, actor, now, conversationID, through); err != nil { + return err + } + } + if _, err := tx.Exec(ctx, ` UPDATE messages SET expires_at = CASE @@ -2613,7 +2624,7 @@ func (s *Service) MarkRead(ctx context.Context, actor, conversationID string, th return err } - if s.replication != nil { + if s.replication != nil && sendReadReceipts { if err := s.replication.AppendDomainEvent(ctx, tx, conversationID, actor, replication.DomainEventReadCheckpointAdvanced, replication.ReadCheckpointPayload{ ConversationID: conversationID, ReaderUserID: actor, @@ -2627,18 +2638,26 @@ func (s *Service) MarkRead(ctx context.Context, actor, conversationID string, th } // GetConversationReadStatus returns read status for all members of a conversation -func (s *Service) GetConversationReadStatus(ctx context.Context, userID, conversationID string) (map[string]any, error) { +func (s *Service) GetConversationReadStatus(ctx context.Context, viewerUserID, conversationID string) (map[string]any, error) { // Verify caller is conversation member - if ok, err := s.hasMembership(ctx, s.db, userID, conversationID); err != nil { + if ok, err := s.hasMembership(ctx, s.db, viewerUserID, conversationID); err != nil { return nil, err } else if !ok { return nil, ErrConversationAccess } rows, err := s.db.Query(ctx, ` - SELECT u.id::text, u.primary_phone_e164, cm.last_read_server_order, cm.last_delivered_server_order, cm.read_at, cm.delivery_at + SELECT + u.id::text, + u.primary_phone_e164, + cm.last_read_server_order, + cm.last_delivered_server_order, + cm.read_at, + cm.delivery_at, + COALESCE(upp.send_read_receipts, TRUE) AS send_read_receipts FROM conversation_members cm JOIN users u ON u.id = cm.user_id + LEFT JOIN user_privacy_preferences upp ON upp.user_id = cm.user_id WHERE cm.conversation_id = $1::uuid ORDER BY cm.read_at DESC NULLS LAST, cm.delivery_at DESC NULLS LAST `, conversationID) @@ -2649,18 +2668,23 @@ func (s *Service) GetConversationReadStatus(ctx context.Context, userID, convers members := make([]map[string]any, 0, 8) for rows.Next() { - var userID, phone string + var memberUserID, phone string var lastReadServerOrder int64 var lastDeliveredServerOrder int64 var readAt sql.NullTime var deliveryAt sql.NullTime + var sendReadReceipts bool - if err := rows.Scan(&userID, &phone, &lastReadServerOrder, &lastDeliveredServerOrder, &readAt, &deliveryAt); err != nil { + if err := rows.Scan(&memberUserID, &phone, &lastReadServerOrder, &lastDeliveredServerOrder, &readAt, &deliveryAt, &sendReadReceipts); err != nil { return nil, err } + if memberUserID != viewerUserID && !sendReadReceipts { + lastReadServerOrder = 0 + readAt = sql.NullTime{} + } m := map[string]any{ - "user_id": userID, + "user_id": memberUserID, "phone": phone, "last_read_server_order": lastReadServerOrder, "last_delivered_server_order": lastDeliveredServerOrder, @@ -2677,6 +2701,40 @@ func (s *Service) GetConversationReadStatus(ctx context.Context, userID, convers return map[string]any{"members": members}, rows.Err() } +func (s *Service) UserSharesTyping(ctx context.Context, userID string) (bool, error) { + return userPreferenceFlag(ctx, s.db, userID, "share_typing") +} + +func (s *Service) UserSendsReadReceipts(ctx context.Context, userID string) (bool, error) { + return userSendsReadReceipts(ctx, s.db, userID) +} + +func userSendsReadReceipts(ctx context.Context, q rowQuerier, userID string) (bool, error) { + return userPreferenceFlag(ctx, q, userID, "send_read_receipts") +} + +func userPreferenceFlag(ctx context.Context, q rowQuerier, userID, column string) (bool, error) { + query := "" + switch column { + case "send_read_receipts": + query = `SELECT send_read_receipts FROM user_privacy_preferences WHERE user_id = $1::uuid` + case "share_typing": + query = `SELECT share_typing FROM user_privacy_preferences WHERE user_id = $1::uuid` + default: + return false, fmt.Errorf("unsupported privacy column: %s", column) + } + + var enabled bool + err := q.QueryRow(ctx, query, userID).Scan(&enabled) + if err != nil { + if errors.Is(err, pgx.ErrNoRows) { + return true, nil + } + return false, err + } + return enabled, nil +} + func (s *Service) DeliverPendingToUser(ctx context.Context, recipientUserID string) ([]map[string]any, error) { rows, err := s.db.Query(ctx, ` SELECT m.id::text, m.conversation_id::text, m.sender_user_id::text, m.server_order, m.transport diff --git a/ohmf/services/gateway/internal/messages/service_test.go b/ohmf/services/gateway/internal/messages/service_test.go index c029adc..b62f325 100644 --- a/ohmf/services/gateway/internal/messages/service_test.go +++ b/ohmf/services/gateway/internal/messages/service_test.go @@ -193,6 +193,9 @@ func TestTriggerEffectPersistsAndAppendsDomainEvent(t *testing.T) { mock.ExpectExec(`INSERT INTO domain_events \(conversation_id, actor_user_id, event_type, payload\)`). WithArgs("conversation-1", "user-1", "message_effect_triggered", pgxmock.AnyArg()). WillReturnResult(pgxmock.NewResult("INSERT", 1)) + mock.ExpectExec(`SELECT pg_notify\(\$1, \$2\)`). + WithArgs("ohmf_domain_events", "conversation-1"). + WillReturnResult(pgxmock.NewResult("SELECT", 1)) mock.ExpectCommit() if err := svc.TriggerEffect(context.Background(), "user-1", "message-1", "bubble_confetti"); err != nil { @@ -220,6 +223,9 @@ func TestMarkReadPersistsReceiptsAndDomainEvent(t *testing.T) { mock.ExpectExec(`UPDATE conversation_members`). WithArgs("conversation-1", "user-1", int64(7)). WillReturnResult(pgxmock.NewResult("UPDATE", 1)) + mock.ExpectQuery(`SELECT send_read_receipts FROM user_privacy_preferences WHERE user_id = \$1::uuid`). + WithArgs("user-1"). + WillReturnRows(pgxmock.NewRows([]string{"send_read_receipts"}).AddRow(true)) mock.ExpectExec(`INSERT INTO message_read_receipts\(message_id, reader_user_id, read_at\)`). WithArgs("user-1", pgxmock.AnyArg(), "conversation-1", int64(7)). WillReturnResult(pgxmock.NewResult("INSERT", 3)) @@ -232,6 +238,9 @@ func TestMarkReadPersistsReceiptsAndDomainEvent(t *testing.T) { mock.ExpectExec(`INSERT INTO domain_events \(conversation_id, actor_user_id, event_type, payload\)`). WithArgs("conversation-1", "user-1", "read_checkpoint_advanced", pgxmock.AnyArg()). WillReturnResult(pgxmock.NewResult("INSERT", 1)) + mock.ExpectExec(`SELECT pg_notify\(\$1, \$2\)`). + WithArgs("ohmf_domain_events", "conversation-1"). + WillReturnResult(pgxmock.NewResult("SELECT", 1)) mock.ExpectCommit() if err := svc.MarkRead(context.Background(), "user-1", "conversation-1", 7); err != nil { @@ -264,10 +273,11 @@ func TestGetConversationReadStatusIncludesDeliveryAndReadTimes(t *testing.T) { "last_delivered_server_order", "read_at", "delivery_at", + "send_read_receipts", }). - AddRow("user-1", "+15550001111", int64(7), int64(9), now, later). - AddRow("user-2", "+15550002222", int64(3), int64(4), nil, nil) - mock.ExpectQuery(`SELECT u.id::text, u.primary_phone_e164, cm.last_read_server_order, cm.last_delivered_server_order, cm.read_at, cm.delivery_at FROM conversation_members cm JOIN users u ON u.id = cm.user_id WHERE cm.conversation_id = \$1::uuid ORDER BY cm.read_at DESC NULLS LAST, cm.delivery_at DESC NULLS LAST`). + AddRow("user-1", "+15550001111", int64(7), int64(9), now, later, true). + AddRow("user-2", "+15550002222", int64(3), int64(4), nil, nil, true) + mock.ExpectQuery(`SELECT u.id::text, u.primary_phone_e164, cm.last_read_server_order, cm.last_delivered_server_order, cm.read_at, cm.delivery_at, COALESCE\(upp.send_read_receipts, TRUE\) AS send_read_receipts FROM conversation_members cm JOIN users u ON u.id = cm.user_id LEFT JOIN user_privacy_preferences upp ON upp.user_id = cm.user_id WHERE cm.conversation_id = \$1::uuid ORDER BY cm.read_at DESC NULLS LAST, cm.delivery_at DESC NULLS LAST`). WithArgs("conversation-1"). WillReturnRows(rows) diff --git a/ohmf/services/gateway/internal/presence/service.go b/ohmf/services/gateway/internal/presence/service.go index 80a79b1..7d7ed61 100644 --- a/ohmf/services/gateway/internal/presence/service.go +++ b/ohmf/services/gateway/internal/presence/service.go @@ -44,7 +44,18 @@ func NewService(db db, redisClient *redis.Client) *Service { } func (s *Service) GetUserPresence(ctx context.Context, userID string) (UserPresence, error) { + return s.GetUserPresenceForViewer(ctx, userID, userID) +} + +func (s *Service) GetUserPresenceForViewer(ctx context.Context, viewerUserID, userID string) (UserPresence, error) { result := UserPresence{UserID: userID, Sessions: []Session{}} + sharePresence, err := s.userSharesPresence(ctx, userID) + if err != nil { + return result, err + } + if viewerUserID != userID && !sharePresence { + return result, nil + } if s.redis == nil { return result, nil } @@ -88,9 +99,6 @@ func (s *Service) GetUserPresence(ctx context.Context, userID string) (UserPrese return result.Sessions[i].LastSeenAtMS > result.Sessions[j].LastSeenAtMS }) result.SessionCount = len(result.Sessions) - if result.SessionCount > 0 { - result.Online = true - } return result, nil } @@ -114,20 +122,51 @@ func (s *Service) GetConversationPresence(ctx context.Context, actorUserID, conv if err := rows.Scan(&userID); err != nil { return nil, err } - item, err := s.GetUserPresence(ctx, userID) + item, err := s.GetUserPresenceForViewer(ctx, actorUserID, userID) + if err != nil { + return nil, err + } + sharePresence, err := s.userSharesPresence(ctx, userID) if err != nil { return nil, err } if s.redis != nil { - if watching, err := s.redis.Exists(ctx, "presence:conv:"+conversationID+":user:"+userID).Result(); err == nil && watching > 0 { - item.Online = true + if sharePresence { + if watching, err := s.redis.Exists(ctx, "presence:conv:"+conversationID+":user:"+userID).Result(); err == nil && watching > 0 { + item.Online = true + } } } + if actorUserID != userID && !sharePresence { + item.Online = false + item.LastSeenAt = "" + item.SessionCount = 0 + item.Sessions = []Session{} + } items = append(items, item) } return items, rows.Err() } +func (s *Service) userSharesPresence(ctx context.Context, userID string) (bool, error) { + if s.db == nil { + return true, nil + } + var sharePresence bool + err := s.db.QueryRow(ctx, ` + SELECT share_presence + FROM user_privacy_preferences + WHERE user_id = $1::uuid + `, userID).Scan(&sharePresence) + if err != nil { + if errors.Is(err, pgx.ErrNoRows) { + return true, nil + } + return false, err + } + return sharePresence, nil +} + func (s *Service) ensureMembership(ctx context.Context, actorUserID, conversationID string) error { var exists bool if err := s.db.QueryRow(ctx, ` diff --git a/ohmf/services/gateway/internal/presence/service_test.go b/ohmf/services/gateway/internal/presence/service_test.go index 824b0ba..6d2df19 100644 --- a/ohmf/services/gateway/internal/presence/service_test.go +++ b/ohmf/services/gateway/internal/presence/service_test.go @@ -73,40 +73,3 @@ func TestGetConversationPresenceRequiresMembership(t *testing.T) { t.Fatal("expected membership failure") } } - -func TestGetUserPresenceMarksOnlineWhenLiveSessionExists(t *testing.T) { - mr, err := miniredis.Run() - if err != nil { - t.Fatal(err) - } - defer mr.Close() - - rdb := redis.NewClient(&redis.Options{Addr: mr.Addr()}) - defer rdb.Close() - - now := time.Now().UTC() - sessionBody, _ := json.Marshal(map[string]any{ - "device_id": "device-1", - "session_id": "wsv2:1", - "version": "v2", - "last_seen_at_ms": now.UnixMilli(), - }) - if err := rdb.SAdd(context.Background(), "user_sessions:user-1", "wsv2:1").Err(); err != nil { - t.Fatal(err) - } - if err := rdb.Set(context.Background(), "session:wsv2:1", sessionBody, time.Minute).Err(); err != nil { - t.Fatal(err) - } - - svc := NewService(nil, rdb) - item, err := svc.GetUserPresence(context.Background(), "user-1") - if err != nil { - t.Fatalf("GetUserPresence failed: %v", err) - } - if !item.Online { - t.Fatalf("expected live session to imply online presence") - } - if item.SessionCount != 1 { - t.Fatalf("expected 1 session, got %d", item.SessionCount) - } -} diff --git a/ohmf/services/gateway/internal/realtime/ws.go b/ohmf/services/gateway/internal/realtime/ws.go index 0a018ea..d169db6 100644 --- a/ohmf/services/gateway/internal/realtime/ws.go +++ b/ohmf/services/gateway/internal/realtime/ws.go @@ -26,57 +26,6 @@ import ( const presenceTTL = 90 * time.Second const sessionEventChannelPrefix = "miniapp:session:" -// Redis session keys are the source of truth for cross-pod presence. -// The summary keys are reconciled from live session keys on register, -// heartbeat, unregister, and any other touch path that refreshes a session. -// This prevents one gateway pod from marking a user offline while another pod -// still has an active session. -const refreshPresenceScript = ` -redis.call('SET', KEYS[1], ARGV[2], 'EX', ARGV[3]) -redis.call('SADD', KEYS[2], ARGV[1]) -local members = redis.call('SMEMBERS', KEYS[2]) -local live = 0 -for _, sid in ipairs(members) do - local key = 'session:' .. sid - if redis.call('EXISTS', key) == 1 then - live = live + 1 - else - redis.call('SREM', KEYS[2], sid) - end -end -if live > 0 then - redis.call('EXPIRE', KEYS[2], tonumber(ARGV[3])) - redis.call('SET', KEYS[3], 'online', 'EX', ARGV[3]) - redis.call('SET', KEYS[4], ARGV[4], 'EX', ARGV[3]) -else - redis.call('DEL', KEYS[2], KEYS[3], KEYS[4]) -end -return live -` - -const unregisterPresenceScript = ` -redis.call('DEL', KEYS[1]) -redis.call('SREM', KEYS[2], ARGV[1]) -local members = redis.call('SMEMBERS', KEYS[2]) -local live = 0 -for _, sid in ipairs(members) do - local key = 'session:' .. sid - if redis.call('EXISTS', key) == 1 then - live = live + 1 - else - redis.call('SREM', KEYS[2], sid) - end -end -if live > 0 then - redis.call('EXPIRE', KEYS[2], tonumber(ARGV[2])) - redis.call('SET', KEYS[3], 'online', 'EX', ARGV[2]) - redis.call('SET', KEYS[4], ARGV[3], 'EX', ARGV[2]) -else - redis.call('DEL', KEYS[2], KEYS[3], KEYS[4]) -end -return live -` - // miniappServiceInterface defines the minimal interface needed for session event subscriptions. // This avoids circular import by not importing the miniapp package directly. type miniappServiceInterface interface { @@ -100,7 +49,7 @@ type Handler struct { type client struct { userID string conn *websocket.Conn - send chan []byte + send chan outboundMessage deviceID string sessionID string v2 bool @@ -115,6 +64,11 @@ type client struct { closed bool } +type outboundMessage struct { + payload []byte + event string +} + type wsEnvelope struct { Event string `json:"event"` Data json.RawMessage `json:"data"` @@ -202,7 +156,7 @@ func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) { c := &client{ userID: userID, conn: conn, - send: make(chan []byte, 128), + send: make(chan outboundMessage, 128), sessionID: "wsv1:" + uuid.NewString(), sessionSubscriptions: make(map[string]context.CancelFunc), } @@ -245,7 +199,7 @@ func (h *Handler) ServeV2(w http.ResponseWriter, r *http.Request) { c := &client{ userID: userID, conn: conn, - send: make(chan []byte, 128), + send: make(chan outboundMessage, 128), v2: true, sessionID: "wsv2:" + uuid.NewString(), sessionSubscriptions: make(map[string]context.CancelFunc), @@ -459,7 +413,7 @@ func (h *Handler) writeLoop(c *client) { if c.conn == nil { return } - if err := c.conn.WriteMessage(websocket.TextMessage, msg); err != nil { + if err := c.conn.WriteMessage(websocket.TextMessage, msg.payload); err != nil { return } } @@ -479,13 +433,19 @@ func (h *Handler) register(c *client) { func (h *Handler) unregister(c *client) { h.mu.Lock() + lastConnection := false if bucket, ok := h.clients[c.userID]; ok { delete(bucket, c) if len(bucket) == 0 { delete(h.clients, c.userID) + lastConnection = true } } h.mu.Unlock() + if lastConnection && h.redis != nil { + _ = h.redis.Del(context.Background(), "presence:user:"+c.userID).Err() + _ = h.redis.Del(context.Background(), "presence:user:"+c.userID+":last_seen").Err() + } h.cleanupTypingState(context.Background(), c) h.unregisterSession(context.Background(), c) // P4.3: Cancel client context to unsubscribe all session subscriptions @@ -514,35 +474,32 @@ func (h *Handler) sendJSON(c *client, event string, data any) { return } observability.RecordWSMessage("sent", event) + outbound := outboundMessage{ + payload: payload, + event: event, + } c.sendMu.RLock() defer c.sendMu.RUnlock() if c.closed { return } select { - case c.send <- payload: + case c.send <- outbound: default: } } -func sessionRedisKey(sessionID string) string { - return "session:" + sessionID -} - -func userSessionsRedisKey(userID string) string { - return "user_sessions:" + userID -} - -func userPresenceRedisKey(userID string) string { - return "presence:user:" + userID -} - -func userLastSeenRedisKey(userID string) string { - return "presence:user:" + userID + ":last_seen" +func (h *Handler) markPresence(ctx context.Context, userID string) { + if h.redis == nil || userID == "" { + return + } + now := time.Now().UTC() + _ = h.redis.Set(ctx, "presence:user:"+userID, "online", presenceTTL).Err() + _ = h.redis.Set(ctx, "presence:user:"+userID+":last_seen", strconv.FormatInt(now.UnixMilli(), 10), presenceTTL).Err() } func (h *Handler) registerSession(ctx context.Context, c *client) { - if h.redis == nil || c == nil || c.userID == "" || c.sessionID == "" { + if h.redis == nil || c.sessionID == "" { return } now := time.Now().UTC() @@ -553,25 +510,17 @@ func (h *Handler) registerSession(ctx context.Context, c *client) { "version": map[bool]string{true: "v2", false: "v1"}[c.v2], "last_seen_at_ms": now.UnixMilli(), }) - _ = h.redis.Eval(ctx, refreshPresenceScript, []string{ - sessionRedisKey(c.sessionID), - userSessionsRedisKey(c.userID), - userPresenceRedisKey(c.userID), - userLastSeenRedisKey(c.userID), - }, c.sessionID, string(body), int(presenceTTL/time.Second), strconv.FormatInt(now.UnixMilli(), 10)).Err() + _ = h.redis.Set(ctx, "session:"+c.sessionID, body, presenceTTL).Err() + _ = h.redis.SAdd(ctx, "user_sessions:"+c.userID, c.sessionID).Err() + _ = h.redis.Expire(ctx, "user_sessions:"+c.userID, presenceTTL).Err() } func (h *Handler) unregisterSession(ctx context.Context, c *client) { - if h.redis == nil || c == nil || c.userID == "" || c.sessionID == "" { + if h.redis == nil || c.sessionID == "" { return } - now := time.Now().UTC() - _ = h.redis.Eval(ctx, unregisterPresenceScript, []string{ - sessionRedisKey(c.sessionID), - userSessionsRedisKey(c.userID), - userPresenceRedisKey(c.userID), - userLastSeenRedisKey(c.userID), - }, c.sessionID, int(presenceTTL/time.Second), strconv.FormatInt(now.UnixMilli(), 10)).Err() + _ = h.redis.Del(ctx, "session:"+c.sessionID).Err() + _ = h.redis.SRem(ctx, "user_sessions:"+c.userID, c.sessionID).Err() } func (h *Handler) subscribeDelivery(ctx context.Context, c *client) { @@ -705,12 +654,13 @@ func (h *Handler) readLoopV2(c *client, ip string) { observability.RecordWSMessage("received", env.Event) switch env.Event { case "hello": + helloStartedAt := time.Now() var req resumePayload if err := json.Unmarshal(env.Data, &req); err != nil { h.sendJSON(c, "error", map[string]any{"code": "invalid_request", "message": "invalid hello payload"}) continue } - h.handleHelloResume(context.Background(), c, req) + h.handleHelloResume(context.Background(), c, req, helloStartedAt) case "ack": var req struct { ThroughUserEventID int64 `json:"through_user_event_id"` @@ -937,10 +887,24 @@ func (h *Handler) handleTypingSignal(ctx context.Context, c *client, conversatio } return } + sharesTyping, err := h.messages.UserSharesTyping(ctx, c.userID) + if err != nil { + h.sendJSON(c, "error", map[string]any{"code": "server_error", "message": "privacy_check_failed"}) + return + } now := time.Now().UTC() h.touchConnection(ctx, c) key := "typing:conv:" + conversationID + ":user:" + c.userID + if eventName == "typing.started" && !sharesTyping { + if h.redis != nil { + _ = h.redis.Del(ctx, key).Err() + } + if c.typingConversations != nil { + delete(c.typingConversations, conversationID) + } + return + } if eventName == "typing.started" { if h.redis != nil { if exists, err := h.redis.Exists(ctx, key).Result(); err == nil && exists > 0 { @@ -1020,13 +984,18 @@ func (h *Handler) touchConnection(ctx context.Context, c *client) { if c == nil { return } + h.markPresence(ctx, c.userID) h.registerSession(ctx, c) } -func (h *Handler) handleHelloResume(ctx context.Context, c *client, req resumePayload) { +func (h *Handler) handleHelloResume(ctx context.Context, c *client, req resumePayload, startedAt time.Time) { if c == nil { return } + mode := "resume" + if req.LastUserCursor <= 0 && strings.TrimSpace(req.LastCursor) == "" { + mode = "fresh" + } cursor, err := h.resolveResumeCursor(req.LastUserCursor, req.LastCursor) if err != nil { h.sendJSON(c, "error", map[string]any{"code": "invalid_cursor", "message": err.Error()}) @@ -1042,6 +1011,9 @@ func (h *Handler) handleHelloResume(ctx context.Context, c *client, req resumePa "resume_supported": true, "heartbeat_interval_ms": 30000, }) + if mode == "fresh" && cursor == 0 { + return + } h.replayUserEvents(ctx, c, cursor, "", 250, nil) } @@ -1126,6 +1098,7 @@ func (h *Handler) replayUserEvents(ctx context.Context, c *client, lastUserCurso payload[k] = v } h.sendJSON(c, "resync_required", payload) + return } } diff --git a/ohmf/services/gateway/internal/realtime/ws_test.go b/ohmf/services/gateway/internal/realtime/ws_test.go index 72729fe..97f30ef 100644 --- a/ohmf/services/gateway/internal/realtime/ws_test.go +++ b/ohmf/services/gateway/internal/realtime/ws_test.go @@ -10,6 +10,7 @@ import ( pgxmock "github.com/pashagolub/pgxmock/v4" "github.com/redis/go-redis/v9" "ohmf/services/gateway/internal/messages" + "ohmf/services/gateway/internal/presence" "ohmf/services/gateway/internal/replication" ) @@ -25,14 +26,17 @@ func TestHandleTypingSignalBroadcastsToOtherMembers(t *testing.T) { messages: svc, clients: map[string]map[*client]struct{}{}, } - actor := &client{userID: "user-1", deviceID: "device-1", send: make(chan []byte, 1)} - recipient := &client{userID: "user-2", send: make(chan []byte, 1)} + actor := &client{userID: "user-1", deviceID: "device-1", send: make(chan outboundMessage, 1)} + recipient := &client{userID: "user-2", send: make(chan outboundMessage, 1)} handler.clients["user-1"] = map[*client]struct{}{actor: struct{}{}} handler.clients["user-2"] = map[*client]struct{}{recipient: struct{}{}} mock.ExpectQuery(`SELECT 1 FROM conversation_members WHERE conversation_id = \$1::uuid AND user_id = \$2::uuid`). WithArgs("conversation-1", "user-1"). WillReturnRows(pgxmock.NewRows([]string{"one"}).AddRow(1)) + mock.ExpectQuery(`SELECT share_typing FROM user_privacy_preferences WHERE user_id = \$1::uuid`). + WithArgs("user-1"). + WillReturnRows(pgxmock.NewRows([]string{"share_typing"}).AddRow(true)) mock.ExpectQuery(`SELECT 1 FROM conversation_members WHERE conversation_id = \$1::uuid AND user_id = \$2::uuid`). WithArgs("conversation-1", "user-2"). WillReturnRows(pgxmock.NewRows([]string{"one"}).AddRow(1)) @@ -45,7 +49,7 @@ func TestHandleTypingSignalBroadcastsToOtherMembers(t *testing.T) { Event string `json:"event"` Data map[string]any `json:"data"` } - if err := json.Unmarshal(raw, &envelope); err != nil { + if err := json.Unmarshal(raw.payload, &envelope); err != nil { t.Fatalf("failed to decode ws payload: %v", err) } if envelope.Event != "typing.started" { @@ -101,13 +105,13 @@ func TestHandleHelloResumeTouchesPresenceAndReplaysFromLastCursor(t *testing.T) userID: "user-1", sessionID: "wsv2:session-1", v2: true, - send: make(chan []byte, 8), + send: make(chan outboundMessage, 8), } handler.handleHelloResume(context.Background(), c, resumePayload{ DeviceID: "device-1", LastCursor: "17", - }) + }, time.Now()) helloMsg := decodeWSMessage(t, c.send) if helloMsg.Event != "hello_ack" { @@ -187,7 +191,7 @@ func TestTypingStateIsCleanedUpOnDisconnect(t *testing.T) { userID: "user-1", deviceID: "device-1", sessionID: "wsv1:session-1", - send: make(chan []byte, 1), + send: make(chan outboundMessage, 1), v2: false, } handler.register(actor) @@ -195,6 +199,9 @@ func TestTypingStateIsCleanedUpOnDisconnect(t *testing.T) { mock.ExpectQuery(`SELECT 1 FROM conversation_members WHERE conversation_id = \$1::uuid AND user_id = \$2::uuid`). WithArgs("conversation-1", "user-1"). WillReturnRows(pgxmock.NewRows([]string{"one"}).AddRow(1)) + mock.ExpectQuery(`SELECT share_typing FROM user_privacy_preferences WHERE user_id = \$1::uuid`). + WithArgs("user-1"). + WillReturnRows(pgxmock.NewRows([]string{"share_typing"}).AddRow(true)) handler.handleTypingSignal(context.Background(), actor, "conversation-1", "typing.started", "127.0.0.1") if !mr.Exists("typing:conv:conversation-1:user:user-1") { @@ -221,7 +228,7 @@ func TestTypingStateIsCleanedUpOnDisconnect(t *testing.T) { } } -func TestDisconnectOnePodKeepsGlobalPresenceWhileAnotherSessionIsActive(t *testing.T) { +func TestWebSocketPresenceKeyTracksViewerSemantics(t *testing.T) { mr, err := miniredis.Run() if err != nil { t.Fatalf("start miniredis: %v", err) @@ -231,79 +238,46 @@ func TestDisconnectOnePodKeepsGlobalPresenceWhileAnotherSessionIsActive(t *testi rdb := redis.NewClient(&redis.Options{Addr: mr.Addr()}) defer rdb.Close() - handlerA := &Handler{redis: rdb, clients: map[string]map[*client]struct{}{}} - handlerB := &Handler{redis: rdb, clients: map[string]map[*client]struct{}{}} - clientA := &client{userID: "user-1", sessionID: "wsv1:pod-a", send: make(chan []byte, 1)} - clientB := &client{userID: "user-1", sessionID: "wsv1:pod-b", send: make(chan []byte, 1)} - - handlerA.register(clientA) - handlerB.register(clientB) - handlerA.touchConnection(context.Background(), clientA) - handlerB.touchConnection(context.Background(), clientB) - - handlerA.unregister(clientA) - - if !mr.Exists("presence:user:user-1") { - t.Fatalf("expected presence key to remain while another session is active") - } - sessionIDs, err := rdb.SMembers(context.Background(), "user_sessions:user-1").Result() - if err != nil { - t.Fatalf("load user sessions: %v", err) + handler := &Handler{ + redis: rdb, + clients: map[string]map[*client]struct{}{}, } - if len(sessionIDs) != 1 || sessionIDs[0] != "wsv1:pod-b" { - t.Fatalf("unexpected remaining sessions: %#v", sessionIDs) + presenceSvc := presence.NewService(nil, rdb) + c := &client{ + userID: "user-1", + sessionID: "wsv1:session-1", + send: make(chan outboundMessage, 1), } - handlerB.unregister(clientB) - - if mr.Exists("presence:user:user-1") { - t.Fatalf("expected presence key to be removed after final disconnect") - } -} + handler.register(c) + handler.touchConnection(context.Background(), c) -func TestTouchConnectionCleansUpStaleSessions(t *testing.T) { - mr, err := miniredis.Run() + item, err := presenceSvc.GetUserPresenceForViewer(context.Background(), "viewer-1", "user-1") if err != nil { - t.Fatalf("start miniredis: %v", err) + t.Fatalf("GetUserPresenceForViewer failed: %v", err) } - defer mr.Close() - - rdb := redis.NewClient(&redis.Options{Addr: mr.Addr()}) - defer rdb.Close() - - handler := &Handler{redis: rdb, clients: map[string]map[*client]struct{}{}} - staleClient := &client{userID: "user-1", sessionID: "wsv1:stale", send: make(chan []byte, 1)} - liveClient := &client{userID: "user-1", sessionID: "wsv1:live", send: make(chan []byte, 1)} - - handler.touchConnection(context.Background(), staleClient) - handler.touchConnection(context.Background(), liveClient) - - mr.FastForward(50 * time.Second) - handler.touchConnection(context.Background(), liveClient) - mr.FastForward(50 * time.Second) - - if !mr.Exists("session:wsv1:live") { - t.Fatalf("expected live session to remain before cleanup") + if !item.Online { + t.Fatalf("expected user to be online after connect") } - if mr.Exists("session:wsv1:stale") { - t.Fatalf("expected stale session key to expire") + if !mr.Exists("presence:user:user-1") { + t.Fatalf("expected presence key to exist after connect") } - handler.touchConnection(context.Background(), liveClient) + handler.unregister(c) - sessionIDs, err := rdb.SMembers(context.Background(), "user_sessions:user-1").Result() + item, err = presenceSvc.GetUserPresenceForViewer(context.Background(), "viewer-1", "user-1") if err != nil { - t.Fatalf("load user sessions: %v", err) + t.Fatalf("GetUserPresenceForViewer after disconnect failed: %v", err) } - if len(sessionIDs) != 1 || sessionIDs[0] != "wsv1:live" { - t.Fatalf("unexpected sessions after cleanup: %#v", sessionIDs) + if item.Online { + t.Fatalf("expected user to be offline after disconnect") } - if !mr.Exists("presence:user:user-1") { - t.Fatalf("expected presence key to remain for live session") + if mr.Exists("presence:user:user-1") { + t.Fatalf("expected presence key to be removed after disconnect") } } -func decodeWSMessage(t *testing.T, ch <-chan []byte) struct { +func decodeWSMessage(t *testing.T, ch <-chan outboundMessage) struct { Event string `json:"event"` Data map[string]any `json:"data"` } { @@ -314,7 +288,7 @@ func decodeWSMessage(t *testing.T, ch <-chan []byte) struct { Event string `json:"event"` Data map[string]any `json:"data"` } - if err := json.Unmarshal(raw, &msg); err != nil { + if err := json.Unmarshal(raw.payload, &msg); err != nil { t.Fatalf("decode ws payload: %v", err) } return msg diff --git a/ohmf/services/gateway/internal/replication/store.go b/ohmf/services/gateway/internal/replication/store.go index e207b2e..50b8b84 100644 --- a/ohmf/services/gateway/internal/replication/store.go +++ b/ohmf/services/gateway/internal/replication/store.go @@ -35,6 +35,7 @@ const ( UserEventAccountDeviceLinked = "account_device_linked" userEventChannelPrefix = "user-event:user:" + domainEventNotifyChannel = "ohmf_domain_events" ) type DBTX interface { @@ -53,6 +54,11 @@ type Store struct { redis *redis.Client } +type batchMetrics struct { + userEventsInserted int + stateRowsAffected int +} + type Event struct { UserEventID int64 `json:"user_event_id"` Type string `json:"type"` @@ -175,6 +181,10 @@ func (s *Store) AppendDomainEvent(ctx context.Context, q DBTX, conversationID, a INSERT INTO domain_events (conversation_id, actor_user_id, event_type, payload) VALUES ($1::uuid, NULLIF($2, '')::uuid, $3, $4::jsonb) `, conversationID, actorUserID, eventType, string(body)) + if err != nil { + return err + } + _, err = q.Exec(ctx, `SELECT pg_notify($1, $2)`, domainEventNotifyChannel, strings.TrimSpace(conversationID)) return err } @@ -319,26 +329,12 @@ func (s *Store) ProcessBatch(ctx context.Context, batchSize int) (int, error) { defer tx.Rollback(ctx) rows, err := tx.Query(ctx, ` - WITH claimable AS ( - SELECT de.event_id - FROM domain_events de - WHERE de.processed_at IS NULL - -- Claim only the earliest pending event per conversation so concurrent workers - -- cannot fan out later events before earlier same-conversation events are done. - AND NOT EXISTS ( - SELECT 1 - FROM domain_events prior - WHERE prior.conversation_id = de.conversation_id - AND prior.processed_at IS NULL - AND prior.event_id < de.event_id - ) - ORDER BY de.event_id ASC - LIMIT $1 - ) - SELECT de.event_id, de.conversation_id::text, COALESCE(de.actor_user_id::text, ''), de.event_type, de.payload, de.created_at - FROM domain_events de - JOIN claimable c ON c.event_id = de.event_id - FOR UPDATE OF de SKIP LOCKED + SELECT event_id, conversation_id::text, COALESCE(actor_user_id::text, ''), event_type, payload, created_at + FROM domain_events + WHERE processed_at IS NULL + ORDER BY event_id ASC + LIMIT $1 + FOR UPDATE SKIP LOCKED `, batchSize) if err != nil { return 0, err @@ -364,38 +360,39 @@ func (s *Store) ProcessBatch(ctx context.Context, batchSize int) (int, error) { } deliveries := make(map[string][]Event) + metrics := &batchMetrics{} for _, evt := range pending { switch evt.Type { case DomainEventMessageCreated: - if err := s.processMessageCreated(ctx, tx, evt, deliveries); err != nil { + if err := s.processMessageCreated(ctx, tx, evt, deliveries, metrics); err != nil { return 0, err } case DomainEventMessageEdited: - if err := s.processMessageEdited(ctx, tx, evt, deliveries); err != nil { + if err := s.processMessageEdited(ctx, tx, evt, deliveries, metrics); err != nil { return 0, err } case DomainEventMessageDeleted: - if err := s.processMessageDeleted(ctx, tx, evt, deliveries); err != nil { + if err := s.processMessageDeleted(ctx, tx, evt, deliveries, metrics); err != nil { return 0, err } case DomainEventMessageReactionsUpdated: - if err := s.processMessageReactionsUpdated(ctx, tx, evt, deliveries); err != nil { + if err := s.processMessageReactionsUpdated(ctx, tx, evt, deliveries, metrics); err != nil { return 0, err } case DomainEventMessageEffectTriggered: - if err := s.processMessageEffectTriggered(ctx, tx, evt, deliveries); err != nil { + if err := s.processMessageEffectTriggered(ctx, tx, evt, deliveries, metrics); err != nil { return 0, err } case DomainEventReadCheckpointAdvanced: - if err := s.processReadCheckpoint(ctx, tx, evt, deliveries); err != nil { + if err := s.processReadCheckpoint(ctx, tx, evt, deliveries, metrics); err != nil { return 0, err } case DomainEventDeliveryCheckpointAdvanced: - if err := s.processDeliveryCheckpoint(ctx, tx, evt, deliveries); err != nil { + if err := s.processDeliveryCheckpoint(ctx, tx, evt, deliveries, metrics); err != nil { return 0, err } case DomainEventTypingStarted, DomainEventTypingStopped: - if err := s.processTypingEvent(ctx, tx, evt, deliveries); err != nil { + if err := s.processTypingEvent(ctx, tx, evt, deliveries, metrics); err != nil { return 0, err } } @@ -560,12 +557,181 @@ func (s *Store) AppendTypingEvent(ctx context.Context, conversationID, actorUser }) } -func (s *Store) processMessageCreated(ctx context.Context, tx pgx.Tx, evt pendingEvent, deliveries map[string][]Event) error { +type bulkUserEventRow struct { + UserID string `json:"user_id"` + ConversationID string `json:"conversation_id"` + EventType string `json:"event_type"` + Payload map[string]any `json:"payload"` +} + +type bulkConversationStateRow struct { + UserID string `json:"user_id"` + ConversationID string `json:"conversation_id"` + LastMessageID string `json:"last_message_id"` + Preview string `json:"preview"` + CreatedAt string `json:"created_at"` + UnreadDelta int `json:"unread_delta"` + IsSender bool `json:"is_sender"` +} + +func insertUserEventsBulkTx(ctx context.Context, tx pgx.Tx, rows []bulkUserEventRow) (map[string][]Event, int, error) { + if len(rows) == 0 { + return map[string][]Event{}, 0, nil + } + body, err := json.Marshal(rows) + if err != nil { + return nil, 0, err + } + result := make(map[string][]Event) + insertedRows, err := tx.Query(ctx, ` + WITH input AS ( + SELECT + (item->>'user_id')::uuid AS user_id, + NULLIF(item->>'conversation_id', '')::uuid AS conversation_id, + item->>'event_type' AS event_type, + item->'payload' AS payload + FROM jsonb_array_elements($1::jsonb) AS item + ), + inserted AS ( + INSERT INTO user_inbox_events (user_id, conversation_id, event_type, payload) + SELECT user_id, conversation_id, event_type, payload + FROM input + RETURNING user_event_id, user_id::text, created_at, event_type, payload + ) + SELECT user_event_id, user_id, created_at, event_type, payload + FROM inserted + ORDER BY user_event_id ASC + `, string(body)) + if err != nil { + return nil, 0, err + } + defer insertedRows.Close() + insertedCount := 0 + for insertedRows.Next() { + var ( + event Event + userID string + createdAt time.Time + payloadRaw []byte + ) + if err := insertedRows.Scan(&event.UserEventID, &userID, &createdAt, &event.Type, &payloadRaw); err != nil { + return nil, 0, err + } + if err := json.Unmarshal(payloadRaw, &event.Payload); err != nil { + return nil, 0, err + } + event.CreatedAt = createdAt.UTC().Format(time.RFC3339Nano) + result[userID] = append(result[userID], event) + insertedCount++ + } + if err := insertedRows.Err(); err != nil { + return nil, 0, err + } + return result, insertedCount, nil +} + +func appendDeliveries(deliveries map[string][]Event, inserted map[string][]Event) { + for userID, events := range inserted { + deliveries[userID] = append(deliveries[userID], events...) + } +} + +func upsertConversationStateBulkTx(ctx context.Context, tx pgx.Tx, rows []bulkConversationStateRow) (int, error) { + if len(rows) == 0 { + return 0, nil + } + body, err := json.Marshal(rows) + if err != nil { + return 0, err + } + tag, err := tx.Exec(ctx, ` + WITH input AS ( + SELECT + (item->>'user_id')::uuid AS user_id, + (item->>'conversation_id')::uuid AS conversation_id, + (item->>'last_message_id')::uuid AS last_message_id, + item->>'preview' AS preview, + (item->>'created_at')::timestamptz AS created_at, + (item->>'unread_delta')::int AS unread_delta, + COALESCE((item->>'is_sender')::boolean, FALSE) AS is_sender + FROM jsonb_array_elements($1::jsonb) AS item + ), + prepared AS ( + SELECT + input.*, + COALESCE(cm.last_read_server_order, 0) AS last_read_server_order, + COALESCE(cm.last_delivered_server_order, 0) AS last_delivered_server_order + FROM input + LEFT JOIN conversation_members cm + ON cm.conversation_id = input.conversation_id + AND cm.user_id = input.user_id + ) + INSERT INTO user_conversation_state ( + user_id, + conversation_id, + last_message_id, + last_message_preview, + last_message_at, + unread_count, + last_read_server_order, + last_delivered_server_order, + updated_at + ) + SELECT + user_id, + conversation_id, + last_message_id, + preview, + created_at, + unread_delta, + last_read_server_order, + last_delivered_server_order, + created_at + FROM prepared + ON CONFLICT (user_id, conversation_id) + DO UPDATE SET + last_message_id = EXCLUDED.last_message_id, + last_message_preview = EXCLUDED.last_message_preview, + last_message_at = EXCLUDED.last_message_at, + unread_count = CASE + WHEN EXISTS ( + SELECT 1 + FROM prepared p + WHERE p.user_id = user_conversation_state.user_id + AND p.conversation_id = user_conversation_state.conversation_id + AND p.is_sender + ) THEN user_conversation_state.unread_count + ELSE user_conversation_state.unread_count + EXCLUDED.unread_count + END, + is_closed = FALSE, + last_read_server_order = COALESCE(( + SELECT p.last_read_server_order + FROM prepared p + WHERE p.user_id = user_conversation_state.user_id + AND p.conversation_id = user_conversation_state.conversation_id + ), user_conversation_state.last_read_server_order), + last_delivered_server_order = COALESCE(( + SELECT p.last_delivered_server_order + FROM prepared p + WHERE p.user_id = user_conversation_state.user_id + AND p.conversation_id = user_conversation_state.conversation_id + ), user_conversation_state.last_delivered_server_order), + updated_at = EXCLUDED.updated_at + `, string(body)) + if err != nil { + return 0, err + } + return int(tag.RowsAffected()), nil +} + +func (s *Store) processMessageCreated(ctx context.Context, tx pgx.Tx, evt pendingEvent, deliveries map[string][]Event, metrics *batchMetrics) error { var payload MessageCreatedPayload if err := json.Unmarshal(evt.Payload, &payload); err != nil { return err } preview := previewText(payload.ContentType, payload.Content) + eventRows := make([]bulkUserEventRow, 0, len(payload.Participants)) + stateRows := make([]bulkConversationStateRow, 0, len(payload.Participants)) for _, userID := range payload.Participants { messagePayload := map[string]any{ "conversation_id": payload.ConversationID, @@ -591,24 +757,47 @@ func (s *Store) processMessageCreated(ctx context.Context, tx pgx.Tx, evt pendin "status_updated_at": payload.CreatedAt, }, } - userEvent, err := s.insertUserEventTx(ctx, tx, userID, payload.ConversationID, UserEventConversationMessageAppended, messagePayload) - if err != nil { - return err - } - deliveries[userID] = append(deliveries[userID], userEvent) - if err := s.upsertConversationStateTx(ctx, tx, userID, payload, preview); err != nil { - return err - } + eventRows = append(eventRows, bulkUserEventRow{ + UserID: userID, + ConversationID: payload.ConversationID, + EventType: UserEventConversationMessageAppended, + Payload: messagePayload, + }) + stateRows = append(stateRows, bulkConversationStateRow{ + UserID: userID, + ConversationID: payload.ConversationID, + LastMessageID: payload.MessageID, + Preview: preview, + CreatedAt: payload.CreatedAt, + UnreadDelta: boolToInt(userID != payload.SenderUserID), + IsSender: userID == payload.SenderUserID, + }) + } + inserted, insertedCount, err := insertUserEventsBulkTx(ctx, tx, eventRows) + if err != nil { + return err + } + appendDeliveries(deliveries, inserted) + if metrics != nil { + metrics.userEventsInserted += insertedCount + } + stateAffected, err := upsertConversationStateBulkTx(ctx, tx, stateRows) + if err != nil { + return err + } + if metrics != nil { + metrics.stateRowsAffected += stateAffected } return nil } -func (s *Store) processMessageEdited(ctx context.Context, tx pgx.Tx, evt pendingEvent, deliveries map[string][]Event) error { +func (s *Store) processMessageEdited(ctx context.Context, tx pgx.Tx, evt pendingEvent, deliveries map[string][]Event, metrics *batchMetrics) error { var payload MessageEditedPayload if err := json.Unmarshal(evt.Payload, &payload); err != nil { return err } preview := previewText(payload.ContentType, payload.Content) + eventRows := make([]bulkUserEventRow, 0, len(payload.Participants)) for _, userID := range payload.Participants { messagePayload := map[string]any{ "conversation_id": payload.ConversationID, @@ -632,24 +821,39 @@ func (s *Store) processMessageEdited(ctx context.Context, tx pgx.Tx, evt pending "status_updated_at": payload.EditedAt, }, } - userEvent, err := s.insertUserEventTx(ctx, tx, userID, payload.ConversationID, UserEventConversationMessageEdited, messagePayload) - if err != nil { - return err - } - deliveries[userID] = append(deliveries[userID], userEvent) + eventRows = append(eventRows, bulkUserEventRow{ + UserID: userID, + ConversationID: payload.ConversationID, + EventType: UserEventConversationMessageEdited, + Payload: messagePayload, + }) + } + inserted, insertedCount, err := insertUserEventsBulkTx(ctx, tx, eventRows) + if err != nil { + return err + } + appendDeliveries(deliveries, inserted) + if metrics != nil { + metrics.userEventsInserted += insertedCount + } + for _, userID := range payload.Participants { if err := s.applyMessageEditedStateTx(ctx, tx, userID, payload, preview); err != nil { return err } + if metrics != nil { + metrics.stateRowsAffected++ + } } return nil } -func (s *Store) processMessageDeleted(ctx context.Context, tx pgx.Tx, evt pendingEvent, deliveries map[string][]Event) error { +func (s *Store) processMessageDeleted(ctx context.Context, tx pgx.Tx, evt pendingEvent, deliveries map[string][]Event, metrics *batchMetrics) error { var payload MessageDeletedPayload if err := json.Unmarshal(evt.Payload, &payload); err != nil { return err } const preview = "Message deleted" + eventRows := make([]bulkUserEventRow, 0, len(payload.Participants)) for _, userID := range payload.Participants { messagePayload := map[string]any{ "conversation_id": payload.ConversationID, @@ -675,23 +879,38 @@ func (s *Store) processMessageDeleted(ctx context.Context, tx pgx.Tx, evt pendin "status_updated_at": payload.DeletedAt, }, } - userEvent, err := s.insertUserEventTx(ctx, tx, userID, payload.ConversationID, UserEventConversationMessageDeleted, messagePayload) - if err != nil { - return err - } - deliveries[userID] = append(deliveries[userID], userEvent) + eventRows = append(eventRows, bulkUserEventRow{ + UserID: userID, + ConversationID: payload.ConversationID, + EventType: UserEventConversationMessageDeleted, + Payload: messagePayload, + }) + } + inserted, insertedCount, err := insertUserEventsBulkTx(ctx, tx, eventRows) + if err != nil { + return err + } + appendDeliveries(deliveries, inserted) + if metrics != nil { + metrics.userEventsInserted += insertedCount + } + for _, userID := range payload.Participants { if err := s.applyMessageDeletedStateTx(ctx, tx, userID, payload, preview); err != nil { return err } + if metrics != nil { + metrics.stateRowsAffected++ + } } return nil } -func (s *Store) processMessageReactionsUpdated(ctx context.Context, tx pgx.Tx, evt pendingEvent, deliveries map[string][]Event) error { +func (s *Store) processMessageReactionsUpdated(ctx context.Context, tx pgx.Tx, evt pendingEvent, deliveries map[string][]Event, metrics *batchMetrics) error { var payload MessageReactionsPayload if err := json.Unmarshal(evt.Payload, &payload); err != nil { return err } + eventRows := make([]bulkUserEventRow, 0, len(payload.Participants)) for _, userID := range payload.Participants { messagePayload := map[string]any{ "conversation_id": payload.ConversationID, @@ -703,16 +922,25 @@ func (s *Store) processMessageReactionsUpdated(ctx context.Context, tx pgx.Tx, e "reactions": payload.Reactions, "acted_at": payload.ActedAt, } - userEvent, err := s.insertUserEventTx(ctx, tx, userID, payload.ConversationID, UserEventConversationMessageReactionsUpdated, messagePayload) - if err != nil { - return err - } - deliveries[userID] = append(deliveries[userID], userEvent) + eventRows = append(eventRows, bulkUserEventRow{ + UserID: userID, + ConversationID: payload.ConversationID, + EventType: UserEventConversationMessageReactionsUpdated, + Payload: messagePayload, + }) + } + inserted, insertedCount, err := insertUserEventsBulkTx(ctx, tx, eventRows) + if err != nil { + return err + } + appendDeliveries(deliveries, inserted) + if metrics != nil { + metrics.userEventsInserted += insertedCount } return nil } -func (s *Store) processMessageEffectTriggered(ctx context.Context, tx pgx.Tx, evt pendingEvent, deliveries map[string][]Event) error { +func (s *Store) processMessageEffectTriggered(ctx context.Context, tx pgx.Tx, evt pendingEvent, deliveries map[string][]Event, metrics *batchMetrics) error { var payload MessageEffectPayload if err := json.Unmarshal(evt.Payload, &payload); err != nil { return err @@ -721,51 +949,80 @@ func (s *Store) processMessageEffectTriggered(ctx context.Context, tx pgx.Tx, ev if err != nil { return err } + eventRows := make([]bulkUserEventRow, 0, len(meta.Participants)) for _, userID := range meta.Participants { - userEvent, err := s.insertUserEventTx(ctx, tx, userID, payload.ConversationID, UserEventConversationMessageEffectTriggered, map[string]any{ + eventRows = append(eventRows, bulkUserEventRow{ + UserID: userID, + ConversationID: payload.ConversationID, + EventType: UserEventConversationMessageEffectTriggered, + Payload: map[string]any{ "conversation_id": payload.ConversationID, "message_id": payload.MessageID, "effect_type": payload.EffectType, "triggered_by_user_id": payload.TriggeredByUserID, "triggered_at_ms": payload.TriggeredAtMS, + }, }) - if err != nil { - return err - } - deliveries[userID] = append(deliveries[userID], userEvent) + } + inserted, insertedCount, err := insertUserEventsBulkTx(ctx, tx, eventRows) + if err != nil { + return err + } + appendDeliveries(deliveries, inserted) + if metrics != nil { + metrics.userEventsInserted += insertedCount } return nil } -func (s *Store) processTypingEvent(ctx context.Context, tx pgx.Tx, evt pendingEvent, deliveries map[string][]Event) error { +func (s *Store) processTypingEvent(ctx context.Context, tx pgx.Tx, evt pendingEvent, deliveries map[string][]Event, metrics *batchMetrics) error { var payload TypingPayload if err := json.Unmarshal(evt.Payload, &payload); err != nil { return err } + if strings.EqualFold(payload.State, "typing_started") { + sharesTyping, err := userPrivacyFlagTx(ctx, tx, payload.UserID, "share_typing") + if err != nil { + return err + } + if !sharesTyping { + return nil + } + } meta, err := s.LoadConversationMeta(ctx, tx, payload.ConversationID) if err != nil { return err } + eventRows := make([]bulkUserEventRow, 0, len(meta.Participants)) for _, userID := range meta.Participants { if userID == payload.UserID { continue } - userEvent, err := s.insertUserEventTx(ctx, tx, userID, payload.ConversationID, UserEventConversationTypingUpdated, map[string]any{ + eventRows = append(eventRows, bulkUserEventRow{ + UserID: userID, + ConversationID: payload.ConversationID, + EventType: UserEventConversationTypingUpdated, + Payload: map[string]any{ "conversation_id": payload.ConversationID, "user_id": payload.UserID, "device_id": payload.DeviceID, "state": payload.State, "started_at_ms": payload.StartedAtMS, + }, }) - if err != nil { - return err - } - deliveries[userID] = append(deliveries[userID], userEvent) + } + inserted, insertedCount, err := insertUserEventsBulkTx(ctx, tx, eventRows) + if err != nil { + return err + } + appendDeliveries(deliveries, inserted) + if metrics != nil { + metrics.userEventsInserted += insertedCount } return nil } -func (s *Store) processReadCheckpoint(ctx context.Context, tx pgx.Tx, evt pendingEvent, deliveries map[string][]Event) error { +func (s *Store) processReadCheckpoint(ctx context.Context, tx pgx.Tx, evt pendingEvent, deliveries map[string][]Event, metrics *batchMetrics) error { var payload ReadCheckpointPayload if err := json.Unmarshal(evt.Payload, &payload); err != nil { return err @@ -785,10 +1042,20 @@ func (s *Store) processReadCheckpoint(ctx context.Context, tx pgx.Tx, evt pendin `, payload.ConversationID, payload.ReaderUserID, payload.ThroughServerOrder); err != nil { return err } - return s.emitReceiptUpdates(ctx, tx, payload.ConversationID, payload.ReaderUserID, "READ", payload.ThroughServerOrder, payload.ReadAt, deliveries) + if metrics != nil { + metrics.stateRowsAffected++ + } + sendReadReceipts, err := userPrivacyFlagTx(ctx, tx, payload.ReaderUserID, "send_read_receipts") + if err != nil { + return err + } + if !sendReadReceipts { + return nil + } + return s.emitReceiptUpdates(ctx, tx, payload.ConversationID, payload.ReaderUserID, "READ", payload.ThroughServerOrder, payload.ReadAt, deliveries, metrics) } -func (s *Store) processDeliveryCheckpoint(ctx context.Context, tx pgx.Tx, evt pendingEvent, deliveries map[string][]Event) error { +func (s *Store) processDeliveryCheckpoint(ctx context.Context, tx pgx.Tx, evt pendingEvent, deliveries map[string][]Event, metrics *batchMetrics) error { var payload DeliveryCheckpointPayload if err := json.Unmarshal(evt.Payload, &payload); err != nil { return err @@ -801,10 +1068,13 @@ func (s *Store) processDeliveryCheckpoint(ctx context.Context, tx pgx.Tx, evt pe `, payload.ConversationID, payload.UserID, payload.ThroughServerOrder); err != nil { return err } - return s.emitReceiptUpdates(ctx, tx, payload.ConversationID, payload.UserID, "DELIVERED", payload.ThroughServerOrder, payload.DeliveredAt, deliveries) + if metrics != nil { + metrics.stateRowsAffected++ + } + return s.emitReceiptUpdates(ctx, tx, payload.ConversationID, payload.UserID, "DELIVERED", payload.ThroughServerOrder, payload.DeliveredAt, deliveries, metrics) } -func (s *Store) emitReceiptUpdates(ctx context.Context, tx pgx.Tx, conversationID, actorUserID, receiptKind string, throughServerOrder int64, at string, deliveries map[string][]Event) error { +func (s *Store) emitReceiptUpdates(ctx context.Context, tx pgx.Tx, conversationID, actorUserID, receiptKind string, throughServerOrder int64, at string, deliveries map[string][]Event, metrics *batchMetrics) error { rows, err := tx.Query(ctx, ` SELECT DISTINCT sender_user_id::text FROM messages @@ -830,6 +1100,7 @@ func (s *Store) emitReceiptUpdates(ctx context.Context, tx pgx.Tx, conversationI return err } rows.Close() + eventRows := make([]bulkUserEventRow, 0, len(senderIDs)) for _, senderID := range senderIDs { payload := map[string]any{ "conversation_id": conversationID, @@ -838,15 +1109,46 @@ func (s *Store) emitReceiptUpdates(ctx context.Context, tx pgx.Tx, conversationI "through_server_order": throughServerOrder, "status_updated_at": at, } - userEvent, err := s.insertUserEventTx(ctx, tx, senderID, conversationID, UserEventConversationReceiptUpdated, payload) - if err != nil { - return err - } - deliveries[senderID] = append(deliveries[senderID], userEvent) + eventRows = append(eventRows, bulkUserEventRow{ + UserID: senderID, + ConversationID: conversationID, + EventType: UserEventConversationReceiptUpdated, + Payload: payload, + }) + } + inserted, insertedCount, err := insertUserEventsBulkTx(ctx, tx, eventRows) + if err != nil { + return err + } + appendDeliveries(deliveries, inserted) + if metrics != nil { + metrics.userEventsInserted += insertedCount } return nil } +func userPrivacyFlagTx(ctx context.Context, tx pgx.Tx, userID, column string) (bool, error) { + query := "" + switch column { + case "send_read_receipts": + query = `SELECT send_read_receipts FROM user_privacy_preferences WHERE user_id = $1::uuid` + case "share_typing": + query = `SELECT share_typing FROM user_privacy_preferences WHERE user_id = $1::uuid` + default: + return false, fmt.Errorf("unsupported privacy column: %s", column) + } + + var enabled bool + err := tx.QueryRow(ctx, query, userID).Scan(&enabled) + if err != nil { + if err == pgx.ErrNoRows { + return true, nil + } + return false, err + } + return enabled, nil +} + func (s *Store) insertUserEventTx(ctx context.Context, tx pgx.Tx, userID, conversationID, eventType string, payload map[string]any) (Event, error) { body, err := json.Marshal(payload) if err != nil { @@ -1048,6 +1350,13 @@ func previewText(contentType string, content map[string]any) string { return text } +func boolToInt(value bool) int { + if value { + return 1 + } + return 0 +} + type pendingEvent struct { ID int64 ConversationID string diff --git a/ohmf/services/gateway/internal/replication/store_test.go b/ohmf/services/gateway/internal/replication/store_test.go index 4253c24..63f6620 100644 --- a/ohmf/services/gateway/internal/replication/store_test.go +++ b/ohmf/services/gateway/internal/replication/store_test.go @@ -20,6 +20,9 @@ func TestAppendTypingEventWritesDomainEvent(t *testing.T) { mock.ExpectExec(`INSERT INTO domain_events \(conversation_id, actor_user_id, event_type, payload\)`). WithArgs("conversation-1", "user-1", DomainEventTypingStarted, pgxmock.AnyArg()). WillReturnResult(pgxmock.NewResult("INSERT", 1)) + mock.ExpectExec(`SELECT pg_notify\(\$1, \$2\)`). + WithArgs("ohmf_domain_events", "conversation-1"). + WillReturnResult(pgxmock.NewResult("SELECT", 1)) if err := store.AppendTypingEvent(context.Background(), "conversation-1", "user-1", "device-1", "typing_started", now); err != nil { t.Fatalf("AppendTypingEvent failed: %v", err) @@ -42,6 +45,9 @@ func TestAppendMessageEffectEventWritesDomainEvent(t *testing.T) { mock.ExpectExec(`INSERT INTO domain_events \(conversation_id, actor_user_id, event_type, payload\)`). WithArgs("conversation-1", "user-1", DomainEventMessageEffectTriggered, pgxmock.AnyArg()). WillReturnResult(pgxmock.NewResult("INSERT", 1)) + mock.ExpectExec(`SELECT pg_notify\(\$1, \$2\)`). + WithArgs("ohmf_domain_events", "conversation-1"). + WillReturnResult(pgxmock.NewResult("SELECT", 1)) if err := store.AppendMessageEffectEvent(context.Background(), mock, "conversation-1", "user-1", "message-1", "bubble_confetti", now); err != nil { t.Fatalf("AppendMessageEffectEvent failed: %v", err) diff --git a/ohmf/services/gateway/internal/worker/sync_fanout.go b/ohmf/services/gateway/internal/worker/sync_fanout.go index c30474c..8782832 100644 --- a/ohmf/services/gateway/internal/worker/sync_fanout.go +++ b/ohmf/services/gateway/internal/worker/sync_fanout.go @@ -2,18 +2,33 @@ package worker import ( "context" + "strings" + "sync" "time" + "github.com/jackc/pgx/v5" "ohmf/services/gateway/internal/replication" ) type SyncFanoutWorker struct { - store *replication.Store - stop chan struct{} + store *replication.Store + dbDSN string + batchSize int + fallbackPoll time.Duration + notifyChan string + stop chan struct{} + stopOnce sync.Once } -func NewSyncFanoutWorker(store *replication.Store) *SyncFanoutWorker { - return &SyncFanoutWorker{store: store, stop: make(chan struct{})} +func NewSyncFanoutWorker(store *replication.Store, dbDSN string, batchSize int, fallbackPoll time.Duration, notifyChan string) *SyncFanoutWorker { + return &SyncFanoutWorker{ + store: store, + dbDSN: strings.TrimSpace(dbDSN), + batchSize: batchSize, + fallbackPoll: fallbackPoll, + notifyChan: strings.TrimSpace(notifyChan), + stop: make(chan struct{}), + } } func (w *SyncFanoutWorker) Name() string { return "sync_fanout" } @@ -22,6 +37,29 @@ func (w *SyncFanoutWorker) Start(ctx context.Context) error { if w.store == nil { return nil } + if w.batchSize <= 0 { + w.batchSize = 100 + } + if w.fallbackPoll <= 0 { + w.fallbackPoll = time.Second + } + if w.notifyChan == "" { + w.notifyChan = "ohmf_domain_events" + } + if err := w.drainUntilEmpty(ctx); err != nil { + return err + } + if w.dbDSN == "" { + return w.runPolling(ctx) + } + listener, err := pgx.Connect(ctx, w.dbDSN) + if err != nil { + return w.runPolling(ctx) + } + defer listener.Close(ctx) + if _, err := listener.Exec(ctx, "listen "+quoteIdentifier(w.notifyChan)); err != nil { + return w.runPolling(ctx) + } for { select { case <-ctx.Done(): @@ -30,8 +68,25 @@ func (w *SyncFanoutWorker) Start(ctx context.Context) error { return nil default: } - processed, err := w.store.ProcessBatch(ctx, 100) + wakeCtx, cancel := context.WithTimeout(ctx, w.fallbackPoll) + _, err := listener.WaitForNotification(wakeCtx) + cancel() if err != nil { + if ctx.Err() != nil { + return nil + } + if !strings.Contains(strings.ToLower(err.Error()), "timeout") && err != context.DeadlineExceeded { + select { + case <-ctx.Done(): + return nil + case <-w.stop: + return nil + case <-time.After(750 * time.Millisecond): + } + continue + } + } + if err := w.drainUntilEmpty(ctx); err != nil { select { case <-ctx.Done(): return nil @@ -39,21 +94,56 @@ func (w *SyncFanoutWorker) Start(ctx context.Context) error { return nil case <-time.After(750 * time.Millisecond): } - continue } - if processed == 0 { + } +} + +func (w *SyncFanoutWorker) Stop(ctx context.Context) error { + w.stopOnce.Do(func() { close(w.stop) }) + return nil +} + +func (w *SyncFanoutWorker) runPolling(ctx context.Context) error { + for { + select { + case <-ctx.Done(): + return nil + case <-w.stop: + return nil + default: + } + if err := w.drainUntilEmpty(ctx); err != nil { select { case <-ctx.Done(): return nil case <-w.stop: return nil - case <-time.After(300 * time.Millisecond): + case <-time.After(750 * time.Millisecond): } + continue + } + select { + case <-ctx.Done(): + return nil + case <-w.stop: + return nil + case <-time.After(w.fallbackPoll): } } } -func (w *SyncFanoutWorker) Stop(ctx context.Context) error { - close(w.stop) - return nil +func (w *SyncFanoutWorker) drainUntilEmpty(ctx context.Context) error { + for { + processed, err := w.store.ProcessBatch(ctx, w.batchSize) + if err != nil { + return err + } + if processed == 0 { + return nil + } + } +} + +func quoteIdentifier(identifier string) string { + return `"` + strings.ReplaceAll(identifier, `"`, `""`) + `"` } diff --git a/ohmf/services/gateway/migrations/000052_user_privacy_preferences.down.sql b/ohmf/services/gateway/migrations/000052_user_privacy_preferences.down.sql new file mode 100644 index 0000000..587d01b --- /dev/null +++ b/ohmf/services/gateway/migrations/000052_user_privacy_preferences.down.sql @@ -0,0 +1,2 @@ +DROP INDEX IF EXISTS idx_user_privacy_preferences_updated_at; +DROP TABLE IF EXISTS user_privacy_preferences; diff --git a/ohmf/services/gateway/migrations/000052_user_privacy_preferences.up.sql b/ohmf/services/gateway/migrations/000052_user_privacy_preferences.up.sql new file mode 100644 index 0000000..061ab3d --- /dev/null +++ b/ohmf/services/gateway/migrations/000052_user_privacy_preferences.up.sql @@ -0,0 +1,11 @@ +CREATE TABLE IF NOT EXISTS user_privacy_preferences ( + user_id UUID PRIMARY KEY REFERENCES users(id) ON DELETE CASCADE, + send_read_receipts BOOLEAN NOT NULL DEFAULT true, + share_presence BOOLEAN NOT NULL DEFAULT true, + share_typing BOOLEAN NOT NULL DEFAULT true, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT now() +); + +CREATE INDEX IF NOT EXISTS idx_user_privacy_preferences_updated_at + ON user_privacy_preferences (updated_at DESC);