From 83df77ab7b496eee8233e33130215aaf981e2147 Mon Sep 17 00:00:00 2001 From: Konstantin Zaytcev Date: Thu, 15 Oct 2026 12:33:42 +0300 Subject: [PATCH] fix(secretary): cancel unsent work after owner manual replies --- adapters/lo/README.md | 29 ++- cmd/flock-lo/secretary.go | 76 +++++- cmd/flock-lo/secretary_manual.go | 78 ++++++ cmd/flock-lo/secretary_manual_test.go | 336 ++++++++++++++++++++++++++ cmd/flock-lo/secretary_review.go | 26 +- 5 files changed, 521 insertions(+), 24 deletions(-) create mode 100644 cmd/flock-lo/secretary_manual.go create mode 100644 cmd/flock-lo/secretary_manual_test.go diff --git a/adapters/lo/README.md b/adapters/lo/README.md index 503a8d1..3b6291a 100644 --- a/adapters/lo/README.md +++ b/adapters/lo/README.md @@ -214,7 +214,13 @@ The connection owner must be in `LO_ALLOWED_USERS` and independently grant both `receive_messages` and `send_messages`. The peer is not checked against the bot command allow-list. Owner messages and delegated bot echoes never trigger replies. Consent and source policy are checked again before transcription, agent execution -and delivery. Edits, deletion and connection changes cancel pending work. Native +and delivery. Edits, deletion, connection changes and authenticated owner manual replies cancel pending work. +A manual reply cancels unsent work only within its owner, connection, conversation, +peer and policy scope, up to its conversation `source_revision`. Message IDs are +identities, not an ordering fence. No-text manual messages also count. The bot +persists the highest observed revision for 48 hours, so delayed older sources and +replayed events cannot regenerate the cancelled reply after restart. Attempted +sends keep their exact retry/idempotency and known-or-unknown receipt semantics. Native server guards remain authoritative for races with manual takeover or revocation. Secretary runs use only authorized read-only knowledge tools (`kb_search`, @@ -228,13 +234,16 @@ this root in `secretary_workspace`. Host runs must set team paths. Changing the configured mode cancels retained work from the old mode. A delivery already attempted keeps an unknown-outcome tombstone instead of falsely claiming cancellation. Legacy version-1 native review proposals are never implicitly -approved during upgrade. Upgrading writes state version 2, which version-1 -runtimes cannot read. Keep a protected pre-upgrade state backup; after any v2 -activity, use a v2-compatible runtime or roll forward. Do not delete the state or +approved during upgrade. Upgrading writes state version 3, migrating versions 1 +and 2; older runtimes reject version 3 before processing. Keep a protected +pre-upgrade state backup; after any v3 activity, use a v3-compatible runtime or +roll forward. Do not delete the state or restore an older checkpoint that loses approvals, attempts or update watermarks. -Do not downgrade to an intermediate v2 runtime that ignores the context and -preview attempt markers: it cannot preserve publication limits. Use the released -runtime with these bounds or a newer compatible build. +Deploy this Flock runtime before enabling backend delivery of owner manual +`business_message` events. Keep the previous private state backup for recovery, +but do not blindly restore it after new approvals, attempts or manual watermarks: +that would lose their safety fences. Older runtimes do reject owner messages +before generation, but cannot immediately retire earlier pending review buttons. The private, atomically written `secretary-state.json` records admission before poll acknowledgement and stores each completed response before the delegated @@ -244,12 +253,14 @@ backoff starts at 15 seconds and doubles to five minutes; a larger server `retry_after` is respected up to the 48-hour lifetime. A crashed agent run is cancelled instead of replaying potentially completed tool actions. -State is bounded to 10,000 jobs/invalidation keys, 1,024 connection snapshots +State is bounded to 10,000 jobs/invalidation keys and manual watermarks, 1,024 connection snapshots and 32 MiB, including reserved space for eventual replies and metadata. Capacity pressure evicts the oldest terminal jobs, invalidations or inactive connections; if live jobs or reserved bytes fill the queue, only the incoming message is logged and skipped, with polling acknowledgement preserved. Live prepared -responses and their request IDs are never evicted for new work. Owners outside +responses and their request IDs are never evicted for new work. A full manual +watermark store pauses acknowledgement of a new takeover until expired entries +can be reclaimed; it never silently loses that replay fence. Owners outside `LO_ALLOWED_USERS` do not occupy connection snapshots. Terminal payloads are compacted, and the persisted native update watermark rejects acknowledged update redelivery after compaction/restart. Deduplication and invalidation records are diff --git a/cmd/flock-lo/secretary.go b/cmd/flock-lo/secretary.go index 3a2ff59..ba1b48c 100644 --- a/cmd/flock-lo/secretary.go +++ b/cmd/flock-lo/secretary.go @@ -32,7 +32,7 @@ import ( const ( secretaryRetryMultiplier = 2 - secretaryStateVersion = 2 + secretaryStateVersion = 3 secretaryMaxJobs = 10000 secretaryMaxConnections = 1024 secretaryMaxStateBytes = 32 << 20 @@ -90,12 +90,13 @@ type secretaryJob struct { } type secretaryState struct { - UpdateFloor int64 `json:"updateFloor"` - Version int `json:"version"` - BotID int64 `json:"botId"` - Jobs map[string]secretaryJob `json:"jobs"` - Connections map[string]lo.SecretaryConnection `json:"connections"` - Invalid map[string]int64 `json:"invalid"` + UpdateFloor int64 `json:"updateFloor"` + Version int `json:"version"` + BotID int64 `json:"botId"` + Jobs map[string]secretaryJob `json:"jobs"` + Connections map[string]lo.SecretaryConnection `json:"connections"` + Invalid map[string]int64 `json:"invalid"` + ManualReplies map[string]secretaryManualReply `json:"manualReplies"` } type secretaryRuntime struct { @@ -148,6 +149,7 @@ func newSecretaryManager(cfg config.Config, api secretaryAPI, botID int64, runti state: secretaryState{ Version: secretaryStateVersion, BotID: botID, Jobs: map[string]secretaryJob{}, Connections: map[string]lo.SecretaryConnection{}, Invalid: map[string]int64{}, + ManualReplies: map[string]secretaryManualReply{}, }, } if err := manager.load(); err != nil { @@ -212,6 +214,9 @@ func (m *secretaryManager) load() error { if info.Size() > secretaryMaxStateBytes { return errors.New("LO secretary state exceeds size limit") } + // Existing v3 files must carry their replay fence explicitly. Older schema + // versions initialize it only through the migration below. + m.state.ManualReplies = nil if err := json.NewDecoder(file).Decode(&m.state); err != nil { return fmt.Errorf("decode LO secretary state: %w", err) } @@ -225,12 +230,13 @@ func (m *secretaryManager) load() error { m.state.Jobs[key] = compactSecretaryJob(job) } } + } + if legacy || m.state.Version == 2 { m.state.Version = secretaryStateVersion + m.state.ManualReplies = map[string]secretaryManualReply{} } - if m.state.UpdateFloor < 0 || m.state.Version != secretaryStateVersion || m.state.BotID != m.botID || m.state.Jobs == nil || - m.state.Connections == nil || m.state.Invalid == nil || len(m.state.Invalid) > secretaryMaxJobs || - len(m.state.Jobs) > secretaryMaxJobs || len(m.state.Connections) > secretaryMaxConnections { - return errors.New("invalid LO secretary state identity or bounds") + if err := m.validateState(); err != nil { + return err } for key, job := range m.state.Jobs { if err := m.validateJob(key, job); err != nil { @@ -265,6 +271,21 @@ func (m *secretaryManager) load() error { return m.saveLocked() } +func (m *secretaryManager) validateState() error { + if m.state.UpdateFloor < 0 || m.state.Version != secretaryStateVersion || m.state.BotID != m.botID || m.state.Jobs == nil || + m.state.Connections == nil || m.state.Invalid == nil || m.state.ManualReplies == nil || + len(m.state.ManualReplies) > secretaryMaxJobs || len(m.state.Invalid) > secretaryMaxJobs || + len(m.state.Jobs) > secretaryMaxJobs || len(m.state.Connections) > secretaryMaxConnections { + return errors.New("invalid LO secretary state identity or bounds") + } + for key, marker := range m.state.ManualReplies { + if !marker.valid() || key != marker.key() { + return errors.New("invalid persisted LO manual reply watermark") + } + } + return nil +} + func (m *secretaryManager) validateJob(key string, job secretaryJob) error { if !job.Message.Valid() || job.CreatedAt <= 0 || job.Attempts < 0 || job.NextAttemptAt < 0 || job.ContextNoticeID < 0 || job.NoticeAttempts < 0 || job.NoticeAttempts > secretaryMaxNoticeAttempts || @@ -343,6 +364,12 @@ func (m *secretaryManager) err() error { func (m *secretaryManager) pruneLocked() bool { changed := false cutoff := time.Now().Add(-secretaryLifetime).Unix() + for key, marker := range m.state.ManualReplies { + if marker.ObservedAt < cutoff { + delete(m.state.ManualReplies, key) + changed = true + } + } for key, at := range m.state.Invalid { if at < cutoff { delete(m.state.Invalid, key) @@ -364,7 +391,24 @@ func (m *secretaryManager) pruneLocked() bool { } // Handle persists the native update watermark and changes before polling acknowledgement. -func (m *secretaryManager) Handle(_ context.Context, update lo.Update) (err error) { +func (m *secretaryManager) Handle(ctx context.Context, update lo.Update) (err error) { + m.mu.Lock() + failure := m.failure + duplicate := update.ID > 0 && update.ID <= m.state.UpdateFloor + m.mu.Unlock() + if failure != nil || duplicate { + return failure + } + // A direction candidate must be authenticated before it can invalidate jobs. + // Never perform the connection lookup while holding the state lock. + var manualConnection *lo.SecretaryConnection + if msg := update.BusinessMessage; isSecretaryManualCandidate(msg) { + conn, lookupErr := m.api.GetBusinessConnection(ctx, msg.ConnectionID) + if lookupErr != nil { + return lookupErr + } + manualConnection = &conn + } m.mu.Lock() defer m.mu.Unlock() if m.failure != nil { @@ -397,6 +441,9 @@ func (m *secretaryManager) Handle(_ context.Context, update lo.Update) (err erro m.invalidateSourceLocked(deleted.ConnectionID, deleted.Context, deleted.MessageIDs) return nil } + if manualConnection != nil { + return m.observeManualReplyLocked(*update.BusinessMessage, *manualConnection) + } m.admitLocked(update) return nil } @@ -453,6 +500,9 @@ func (m *secretaryManager) admitLocked(update lo.Update) { conn.PolicyVersion >= msg.Context.PolicyVersion && !conn.CanReply(msg.Context) { job.Status = secretaryCancelled } + if conn, exists := m.state.Connections[msg.ConnectionID]; exists && m.manuallyAnsweredLocked(conn.User.ID, job) { + job.Status = secretaryCancelled + } if !m.admissionFitsLocked(job) { slog.Warn("LO secretary state byte capacity reached; message skipped", "update_id", update.ID) return @@ -810,7 +860,7 @@ func (m *secretaryManager) connection(ctx context.Context, key string, job secre // A newer poll snapshot can revoke an in-flight lookup; an older one cannot reject a fresh generation. latest, exists := m.state.Connections[conn.ID] current := m.state.Jobs[key] - if m.failure != nil || secretaryTerminal(current.Status) || + if m.failure != nil || secretaryTerminal(current.Status) || m.manuallyAnsweredLocked(conn.User.ID, current) || (exists && latest.PolicyVersion >= conn.PolicyVersion && (!latest.CanReply(job.Message.Context) || latest.User.ID != conn.User.ID)) { return conn, errSecretaryConsent diff --git a/cmd/flock-lo/secretary_manual.go b/cmd/flock-lo/secretary_manual.go new file mode 100644 index 0000000..e254795 --- /dev/null +++ b/cmd/flock-lo/secretary_manual.go @@ -0,0 +1,78 @@ +package main + +import ( + "errors" + "slices" + "strconv" + "time" + + "github.com/duckbugio/flock/adapters/lo" +) + +// Revision is the server's conversation change sequence, assigned under its +// conversation lock. Message IDs can be allocated before that lock is acquired. +type secretaryManualReply struct { + OwnerID int64 `json:"ownerId"` + ConnectionID string `json:"connectionId"` + Context lo.SecretaryContext `json:"context"` + ObservedAt int64 `json:"observedAt"` +} + +func (marker secretaryManualReply) key() string { + return strconv.FormatInt(marker.OwnerID, 10) + ":" + marker.ConnectionID + ":" + + marker.Context.ConversationID.String() + ":" + marker.Context.ChatID.String() + ":" + + marker.Context.PolicyVersion.String() +} + +func (marker secretaryManualReply) valid() bool { + return marker.OwnerID > 0 && marker.OwnerID < 1000000000000000 && + lo.ValidSecretaryConnectionID(marker.ConnectionID) && marker.Context.Valid() && marker.ObservedAt > 0 +} + +func isSecretaryManualCandidate(msg *lo.SecretaryMessage) bool { + // Incoming private messages have the correspondent as both From and Chat. + return msg != nil && msg.Valid() && msg.BotID == 0 && msg.From.ID != int64(msg.Chat.ID) +} + +func (m *secretaryManager) observeManualReplyLocked(msg lo.SecretaryMessage, conn lo.SecretaryConnection) error { + if !conn.Valid() || conn.ID != msg.ConnectionID || !conn.Enabled || conn.PolicyVersion != msg.Context.PolicyVersion || + conn.User.ID != msg.From.ID || !m.cfg.IsLOAllowed(conn.User.ID) || !slices.Contains(conn.Rights, "receive_messages") { + return nil + } + if latest, exists := m.state.Connections[conn.ID]; exists && latest.PolicyVersion >= conn.PolicyVersion && + (latest.PolicyVersion != conn.PolicyVersion || latest.User.ID != conn.User.ID || !latest.Enabled || + !slices.Contains(latest.Rights, "receive_messages")) { + return nil + } + marker := secretaryManualReply{ + OwnerID: conn.User.ID, ConnectionID: conn.ID, Context: msg.Context, ObservedAt: time.Now().Unix(), + } + key := marker.key() + previous, exists := m.state.ManualReplies[key] + if !exists && len(m.state.ManualReplies) >= secretaryMaxJobs { + // Do not acknowledge a takeover that cannot be retained against replay. + return errors.New("LO secretary manual reply watermark capacity reached") + } + if !exists || marker.Context.SourceRevision > previous.Context.SourceRevision { + m.state.ManualReplies[key] = marker + } + for jobKey, job := range m.state.Jobs { + scope := secretaryManualReply{OwnerID: conn.User.ID, ConnectionID: job.Message.ConnectionID, Context: job.Message.Context} + if scope.key() == key && m.manuallyAnsweredLocked(conn.User.ID, job) { + m.invalidateLocked(jobKey, job) + } + } + return nil +} + +func (m *secretaryManager) manuallyAnsweredLocked(ownerID int64, job secretaryJob) bool { + // A send already attempted must keep its exact request, retry and receipt + // semantics. Manual takeover cannot prove whether that write took effect. + if job.DeliveryAttempted || secretaryTerminal(job.Status) || (job.OwnerID != 0 && job.OwnerID != ownerID) || + job.Message.From.ID != int64(job.Message.Chat.ID) { + return false + } + marker := secretaryManualReply{OwnerID: ownerID, ConnectionID: job.Message.ConnectionID, Context: job.Message.Context} + previous, exists := m.state.ManualReplies[marker.key()] + return exists && job.Message.Context.SourceRevision <= previous.Context.SourceRevision +} diff --git a/cmd/flock-lo/secretary_manual_test.go b/cmd/flock-lo/secretary_manual_test.go new file mode 100644 index 0000000..c2fd20e --- /dev/null +++ b/cmd/flock-lo/secretary_manual_test.go @@ -0,0 +1,336 @@ +package main + +import ( + "context" + "encoding/json" + "errors" + "os" + "reflect" + "strings" + "testing" + "time" + + "github.com/duckbugio/flock/adapters/lo" + "github.com/duckbugio/flock/internal/config" +) + +const secretaryOwnerScopeTest = "owner" + +func secretaryManualMessage() lo.SecretaryMessage { + msg := secretaryMessage() + msg.ID, msg.Context.SourceMessageID, msg.Context.SourceRevision = 13, 13, 2 + msg.From.ID, msg.EventID, msg.Text = 1, "manual", "" + return msg +} + +func handleSecretaryManual(t *testing.T, m *secretaryManager, msg lo.SecretaryMessage, updateID int64) { + t.Helper() + if err := m.Handle(t.Context(), lo.Update{ID: updateID, BusinessMessage: &msg}); err != nil { + t.Fatal(err) + } +} + +func TestSecretaryManualReplyCancelsReviewAndSurvivesRestart(t *testing.T) { + t.Parallel() + m, api, runner := secretaryFixture(t, config.SecretaryModeApproval) + key := admitSecretary(t, m, secretaryMessage()) + m.process(t.Context(), key) + query := secretaryReviewQuery(m, key, lo.SecretaryActionSend) + manual := secretaryManualMessage() // Empty text is also a valid media-only takeover. + handleSecretaryManual(t, m, manual, 100) + if m.state.Jobs[key].Status != secretaryCancelled || len(m.state.Jobs) != 1 || runner.calls != 1 || api.sends != 0 { + t.Fatal("manual reply did not cancel only the existing review") + } + m.syncReviewNotices(t.Context()) + if len(api.noticeStatuses) != 1 || m.state.Jobs[key].NoticeStatus != secretaryCancelled { + t.Fatal("cancelled review buttons were not closed") + } + reloaded, err := newSecretaryManager(m.cfg, api, m.botID, m.runtime) + if err != nil { + t.Fatal(err) + } + if got := reloaded.ReviewCallback(t.Context(), query); !strings.Contains(got, "no longer") { + t.Fatal("old approval remained actionable", got) + } + handleSecretaryManual(t, reloaded, manual, 101) + late := secretaryMessage() + late.ID, late.Context.SourceMessageID = 99, 99 // IDs do not establish conversation order. + handleSecretaryManual(t, reloaded, late, 102) + lateKey := secretaryJobKey(m.botID, late) + reloaded.process(t.Context(), lateKey) + if reloaded.state.Jobs[lateKey].Status != secretaryCancelled || runner.calls != 1 || api.sends != 0 { + t.Fatal("late older revision was generated after restart") + } + newer := secretaryMessage() + newer.ID, newer.Context.SourceMessageID, newer.Context.SourceRevision = 3, 3, 3 + handleSecretaryManual(t, reloaded, newer, 103) + newKey := secretaryJobKey(m.botID, newer) + reloaded.process(t.Context(), newKey) + if runner.calls != 2 || reloaded.state.Jobs[newKey].Status != secretaryAwaiting { + t.Fatal("newer revision with lower message ID was suppressed") + } + handleSecretaryManual(t, reloaded, manual, 104) + if reloaded.state.Jobs[newKey].Status != secretaryAwaiting { + t.Fatal("replayed manual event cancelled a newer incoming revision") + } +} + +func TestSecretaryManualReplyScopeAndAuthority(t *testing.T) { + t.Parallel() + for _, kind := range []string{ + "owner", "connection", "conversation", "peer", "policy", "sender", "bot actor", "bot attribution", + "revoked", "no receive", "receive only", "newer cached policy", "disallowed owner", + } { + t.Run(kind, func(t *testing.T) { + t.Parallel() + m, api, runner := secretaryFixture(t, config.SecretaryModeApproval) + key := admitSecretary(t, m, secretaryMessage()) + m.process(t.Context(), key) + manual := secretaryManualMessage() + switch kind { + case secretaryOwnerScopeTest, "disallowed owner": + api.connection.User.ID, manual.From.ID = 2, 2 + if kind == secretaryOwnerScopeTest { + m.cfg.LOAllowedUsers = append(m.cfg.LOAllowedUsers, 2) + } + case "connection": + api.connection.ID = "25eac687-9383-4da1-9644-826693923e44" + manual.ConnectionID = api.connection.ID + case "conversation": + manual.Context.ConversationID++ + case "peer": + manual.Chat.ID++ + manual.Context.ChatID = manual.Chat.ID + case "policy": + api.connection.PolicyVersion++ + manual.Context.PolicyVersion++ + case "sender": + manual.From.ID = 3 + case "bot actor": + manual.From.IsBot = true + case "bot attribution": + manual.BotID = secretaryTestBot + case "revoked": + api.connection.Enabled = false + case "no receive": + api.connection.Rights = []string{"send_messages"} + case "receive only": + api.connection.Rights = []string{"receive_messages"} + case "newer cached policy": + latest := api.connection + latest.PolicyVersion++ + m.state.Connections[latest.ID] = latest + } + handleSecretaryManual(t, m, manual, 100) + want := secretaryAwaiting + if kind == "receive only" { + want = secretaryCancelled + } + if m.state.Jobs[key].Status != want || len(m.state.Jobs) != 1 || runner.calls != 1 || api.sends != 0 { + t.Fatalf("manual event crossed scope or generated work: %s", m.state.Jobs[key].Status) + } + }) + } +} + +func TestSecretaryManualReplyDuringGenerationAndNotice(t *testing.T) { + t.Parallel() + for _, phase := range []string{"queued", "running", "notice", "approved"} { + t.Run(phase, func(t *testing.T) { + t.Parallel() + m, api, runner := secretaryFixture(t, config.SecretaryModeApproval) + key := admitSecretary(t, m, secretaryMessage()) + cancel := func() { handleSecretaryManual(t, m, secretaryManualMessage(), 100) } + switch phase { + case "queued": + cancel() + case "running": + runner.onRun = func(ctx context.Context) { + cancel() + if ctx.Err() == nil { + t.Fatal("manual takeover did not cancel active generation") + } + } + case "notice": + api.onNotice = cancel + case "approved": + m.process(t.Context(), key) + m.ReviewCallback(t.Context(), secretaryReviewQuery(m, key, lo.SecretaryActionSend)) + cancel() + } + m.process(t.Context(), key) + if m.state.Jobs[key].Status != secretaryCancelled || api.sends != 0 { + t.Fatal("manual takeover failed before delegated send") + } + if phase == "notice" { + m.syncReviewNotices(t.Context()) + if m.state.Jobs[key].NoticeID != 99 || m.state.Jobs[key].NoticeStatus != secretaryCancelled { + t.Fatal("late notice receipt lost or resurrected cancelled review") + } + } + }) + } +} + +func TestSecretaryManualReplyPreservesAttemptedDelivery(t *testing.T) { + t.Parallel() + for _, phase := range []string{"inflight success", "inflight timeout", "retry", "unknown", "done"} { + t.Run(phase, func(t *testing.T) { + t.Parallel() + m, api, runner := secretaryFixture(t, config.SecretaryModeApproval) + key := admitSecretary(t, m, secretaryMessage()) + m.process(t.Context(), key) + m.ReviewCallback(t.Context(), secretaryReviewQuery(m, key, lo.SecretaryActionSend)) + cancel := func() { + before := m.state.Jobs[key] + handleSecretaryManual(t, m, secretaryManualMessage(), 100) + if !reflect.DeepEqual(m.state.Jobs[key], before) { + t.Fatal("manual reply rewrote an attempted delivery or receipt") + } + } + if phase != "done" && phase != "inflight success" { + api.err = errors.New("lost response") + } + if strings.HasPrefix(phase, "inflight") { + api.onSend = cancel + } + m.process(t.Context(), key) + api.onSend = nil + if phase == "unknown" { + m.finish(key, secretaryUnknown) + } + if !strings.HasPrefix(phase, "inflight") { + cancel() + } + reloaded, err := newSecretaryManager(m.cfg, api, m.botID, m.runtime) + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(reloaded.state.Jobs[key], m.state.Jobs[key]) || runner.calls != 1 || api.sends != 1 { + t.Fatal("restart changed the exact attempted-delivery outcome") + } + if phase == "retry" { + api.err = nil + reloaded.process(t.Context(), key) + if api.sends != 2 || api.actions[0] != api.actions[1] || runner.calls != 1 || + reloaded.state.Jobs[key].Status != secretaryDone { + t.Fatal("manual marker changed idempotent retry semantics") + } + } + }) + } +} + +func TestSecretaryManualStateMigrationBoundsAndValidation(t *testing.T) { + t.Parallel() + for _, kind := range []string{"v2", "bad key", "bad owner", "capacity"} { + t.Run(kind, func(t *testing.T) { + t.Parallel() + m, api, _ := secretaryFixture(t, config.SecretaryModeApproval) + manual := secretaryManualMessage() + handleSecretaryManual(t, m, manual, 100) + switch kind { + case "v2": + m.state.Version, m.state.ManualReplies = 2, nil + case "bad key": + for _, marker := range m.state.ManualReplies { + m.state.ManualReplies["wrong"] = marker + } + case "bad owner": + for key, marker := range m.state.ManualReplies { + marker.OwnerID = 0 + m.state.ManualReplies[key] = marker + } + case "capacity": + for id := range secretaryMaxJobs { + marker := secretaryManualReply{ + OwnerID: 1, ConnectionID: manual.ConnectionID, Context: manual.Context, + ObservedAt: time.Now().Unix(), + } + marker.Context.ConversationID = lo.SecretaryNumber(id + 1) + m.state.ManualReplies[marker.key()] = marker + } + manual.Context.ConversationID = secretaryMaxJobs + 1 + if err := m.Handle(t.Context(), lo.Update{ID: 101, BusinessMessage: &manual}); err == nil || m.state.UpdateFloor != 100 { + t.Fatal("unretained manual takeover was acknowledged") + } + return + } + if err := m.saveLocked(); err != nil { + t.Fatal(err) + } + reloaded, err := newSecretaryManager(m.cfg, api, m.botID, m.runtime) + if kind != "v2" { + if err == nil { + t.Fatal("invalid manual watermark state loaded") + } + return + } + if err != nil || reloaded.state.Version != 3 || reloaded.state.ManualReplies == nil { + t.Fatal("v2 migration failed", err) + } + data, err := os.ReadFile(m.path) + if err != nil { + t.Fatal(err) + } + var disk secretaryState + if err := json.Unmarshal(data, &disk); err != nil || disk.Version != 3 { + t.Fatal("migration was not durable", err) + } + }) + } +} + +func TestSecretaryManualDuplicateDoesNotLookupRevokedConnection(t *testing.T) { + t.Parallel() + m, api, _ := secretaryFixture(t, config.SecretaryModeApproval) + manual := secretaryManualMessage() + handleSecretaryManual(t, m, manual, 100) + api.onLookup = func() { t.Fatal("acknowledged replay queried a possibly deleted connection") } + handleSecretaryManual(t, m, manual, 100) +} + +func TestSecretaryManualPersistenceFailureNeverAcknowledges(t *testing.T) { + t.Parallel() + m, api, _ := secretaryFixture(t, config.SecretaryModeApproval) + key := admitSecretary(t, m, secretaryMessage()) + m.process(t.Context(), key) + // A full state must fail closed rather than acknowledging an unretained fence. + m.state.Invalid[strings.Repeat("x", secretaryMaxStateBytes)] = time.Now().Unix() + manual := secretaryManualMessage() + if err := m.Handle(t.Context(), lo.Update{ID: 100, BusinessMessage: &manual}); err == nil || m.err() == nil { + t.Fatal("state byte overflow acknowledged manual takeover") + } + reloaded, err := newSecretaryManager(m.cfg, api, m.botID, m.runtime) + if err != nil || reloaded.state.UpdateFloor != 12 || len(reloaded.state.ManualReplies) != 0 { + t.Fatal("failed write changed durable polling watermark", err) + } + handleSecretaryManual(t, reloaded, manual, 100) + if reloaded.state.Jobs[key].Status != secretaryCancelled { + t.Fatal("unacknowledged event did not apply after restart") + } +} + +func TestSecretaryManualStateRejectsMissingV3Watermarks(t *testing.T) { + t.Parallel() + m, api, _ := secretaryFixture(t, config.SecretaryModeApproval) + data, err := os.ReadFile(m.path) + if err != nil { + t.Fatal(err) + } + var fields map[string]json.RawMessage + if err := json.Unmarshal(data, &fields); err != nil { + t.Fatal(err) + } + delete(fields, "manualReplies") + data, err = json.Marshal(fields) + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile(m.path, data, 0o600); err != nil { + t.Fatal(err) + } + if _, err := newSecretaryManager(m.cfg, api, m.botID, m.runtime); err == nil { + t.Fatal("v3 state without its mandatory replay fence was accepted") + } +} diff --git a/cmd/flock-lo/secretary_review.go b/cmd/flock-lo/secretary_review.go index 9a57d53..b7eac6b 100644 --- a/cmd/flock-lo/secretary_review.go +++ b/cmd/flock-lo/secretary_review.go @@ -133,8 +133,7 @@ func (m *secretaryManager) publishReviewNotice(ctx context.Context, key string, id, err := m.api.SendSecretaryReviewNotice(ctx, job.OwnerID, m.botID, int64(job.Message.Chat.ID), job.Action.Text, job.ReviewToken) if err == nil && id > 0 { - job.NoticeID, job.Status = id, secretaryAwaiting - m.store(key, job) + m.recordReviewNotice(key, job.ReviewToken, id) return } var apiErr *lo.APIError @@ -147,6 +146,29 @@ func (m *secretaryManager) publishReviewNotice(ctx context.Context, key string, slog.Warn("LO secretary review preview failed", "update_id", job.UpdateID, "attempt", job.NoticeAttempts) } +// A confirmed preview can return after cancellation. Retain its receipt without +// restoring the draft, so the pump can remove its now-invalid callback buttons. +func (m *secretaryManager) recordReviewNotice(key, token string, id int64) { + m.mu.Lock() + defer m.mu.Unlock() + job, exists := m.state.Jobs[key] + if !exists || m.failure != nil || job.ReviewToken != token || job.NoticeID != 0 { + return + } + job.NoticeID = id + if job.Status == secretaryPrepared { + job.Status = secretaryAwaiting + } + m.state.Jobs[key] = job + if err := m.saveLocked(); err != nil { + slog.Error("persist LO secretary review receipt", "error", err) + } + select { + case m.wake <- struct{}{}: + default: + } +} + func (m *secretaryManager) publishReviewContext(ctx context.Context, key string, job secretaryJob) (secretaryJob, bool) { if job.ContextAttempted { m.finish(key, secretaryCancelled)