Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
29 changes: 20 additions & 9 deletions adapters/lo/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -214,7 +214,13 @@ The connection owner must be in `LO_ALLOWED_USERS` and independently grant both
`receive_messages` and `send_messages`. The peer is not checked against the bot
command allow-list. Owner messages and delegated bot echoes never trigger replies.
Consent and source policy are checked again before transcription, agent execution
and delivery. Edits, deletion and connection changes cancel pending work. Native
and delivery. Edits, deletion, connection changes and authenticated owner manual replies cancel pending work.
A manual reply cancels unsent work only within its owner, connection, conversation,
peer and policy scope, up to its conversation `source_revision`. Message IDs are
identities, not an ordering fence. No-text manual messages also count. The bot
persists the highest observed revision for 48 hours, so delayed older sources and
replayed events cannot regenerate the cancelled reply after restart. Attempted
sends keep their exact retry/idempotency and known-or-unknown receipt semantics. Native
server guards remain authoritative for races with manual takeover or revocation.

Secretary runs use only authorized read-only knowledge tools (`kb_search`,
Expand All @@ -228,13 +234,16 @@ this root in `secretary_workspace`. Host runs must set
team paths. Changing the configured mode cancels retained work from the old mode.
A delivery already attempted keeps an unknown-outcome tombstone instead of falsely
claiming cancellation. Legacy version-1 native review proposals are never implicitly
approved during upgrade. Upgrading writes state version 2, which version-1
runtimes cannot read. Keep a protected pre-upgrade state backup; after any v2
activity, use a v2-compatible runtime or roll forward. Do not delete the state or
approved during upgrade. Upgrading writes state version 3, migrating versions 1
and 2; older runtimes reject version 3 before processing. Keep a protected
pre-upgrade state backup; after any v3 activity, use a v3-compatible runtime or
roll forward. Do not delete the state or
restore an older checkpoint that loses approvals, attempts or update watermarks.
Do not downgrade to an intermediate v2 runtime that ignores the context and
preview attempt markers: it cannot preserve publication limits. Use the released
runtime with these bounds or a newer compatible build.
Deploy this Flock runtime before enabling backend delivery of owner manual
`business_message` events. Keep the previous private state backup for recovery,
but do not blindly restore it after new approvals, attempts or manual watermarks:
that would lose their safety fences. Older runtimes do reject owner messages
before generation, but cannot immediately retire earlier pending review buttons.

The private, atomically written `secretary-state.json` records admission before
poll acknowledgement and stores each completed response before the delegated
Expand All @@ -244,12 +253,14 @@ backoff starts at 15 seconds and doubles to five minutes; a larger server
`retry_after` is respected up to the 48-hour lifetime. A crashed agent run is
cancelled instead of replaying potentially completed tool actions.

State is bounded to 10,000 jobs/invalidation keys, 1,024 connection snapshots
State is bounded to 10,000 jobs/invalidation keys and manual watermarks, 1,024 connection snapshots
and 32 MiB, including reserved space for eventual replies and metadata. Capacity
pressure evicts the oldest terminal jobs, invalidations or inactive connections;
if live jobs or reserved bytes fill the queue, only the incoming message is
logged and skipped, with polling acknowledgement preserved. Live prepared
responses and their request IDs are never evicted for new work. Owners outside
responses and their request IDs are never evicted for new work. A full manual
watermark store pauses acknowledgement of a new takeover until expired entries
can be reclaimed; it never silently loses that replay fence. Owners outside
`LO_ALLOWED_USERS` do not occupy connection snapshots. Terminal payloads are
compacted, and the persisted native update watermark rejects acknowledged update
redelivery after compaction/restart. Deduplication and invalidation records are
Expand Down
76 changes: 63 additions & 13 deletions cmd/flock-lo/secretary.go
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@ import (

const (
secretaryRetryMultiplier = 2
secretaryStateVersion = 2
secretaryStateVersion = 3
secretaryMaxJobs = 10000
secretaryMaxConnections = 1024
secretaryMaxStateBytes = 32 << 20
Expand Down Expand Up @@ -90,12 +90,13 @@ type secretaryJob struct {
}

type secretaryState struct {
UpdateFloor int64 `json:"updateFloor"`
Version int `json:"version"`
BotID int64 `json:"botId"`
Jobs map[string]secretaryJob `json:"jobs"`
Connections map[string]lo.SecretaryConnection `json:"connections"`
Invalid map[string]int64 `json:"invalid"`
UpdateFloor int64 `json:"updateFloor"`
Version int `json:"version"`
BotID int64 `json:"botId"`
Jobs map[string]secretaryJob `json:"jobs"`
Connections map[string]lo.SecretaryConnection `json:"connections"`
Invalid map[string]int64 `json:"invalid"`
ManualReplies map[string]secretaryManualReply `json:"manualReplies"`
}

type secretaryRuntime struct {
Expand Down Expand Up @@ -148,6 +149,7 @@ func newSecretaryManager(cfg config.Config, api secretaryAPI, botID int64, runti
state: secretaryState{
Version: secretaryStateVersion, BotID: botID, Jobs: map[string]secretaryJob{},
Connections: map[string]lo.SecretaryConnection{}, Invalid: map[string]int64{},
ManualReplies: map[string]secretaryManualReply{},
},
}
if err := manager.load(); err != nil {
Expand Down Expand Up @@ -212,6 +214,9 @@ func (m *secretaryManager) load() error {
if info.Size() > secretaryMaxStateBytes {
return errors.New("LO secretary state exceeds size limit")
}
// Existing v3 files must carry their replay fence explicitly. Older schema
// versions initialize it only through the migration below.
m.state.ManualReplies = nil
if err := json.NewDecoder(file).Decode(&m.state); err != nil {
return fmt.Errorf("decode LO secretary state: %w", err)
}
Expand All @@ -225,12 +230,13 @@ func (m *secretaryManager) load() error {
m.state.Jobs[key] = compactSecretaryJob(job)
}
}
}
if legacy || m.state.Version == 2 {
m.state.Version = secretaryStateVersion
m.state.ManualReplies = map[string]secretaryManualReply{}
}
if m.state.UpdateFloor < 0 || m.state.Version != secretaryStateVersion || m.state.BotID != m.botID || m.state.Jobs == nil ||
m.state.Connections == nil || m.state.Invalid == nil || len(m.state.Invalid) > secretaryMaxJobs ||
len(m.state.Jobs) > secretaryMaxJobs || len(m.state.Connections) > secretaryMaxConnections {
return errors.New("invalid LO secretary state identity or bounds")
if err := m.validateState(); err != nil {
return err
}
for key, job := range m.state.Jobs {
if err := m.validateJob(key, job); err != nil {
Expand Down Expand Up @@ -265,6 +271,21 @@ func (m *secretaryManager) load() error {
return m.saveLocked()
}

func (m *secretaryManager) validateState() error {
if m.state.UpdateFloor < 0 || m.state.Version != secretaryStateVersion || m.state.BotID != m.botID || m.state.Jobs == nil ||
m.state.Connections == nil || m.state.Invalid == nil || m.state.ManualReplies == nil ||
len(m.state.ManualReplies) > secretaryMaxJobs || len(m.state.Invalid) > secretaryMaxJobs ||
len(m.state.Jobs) > secretaryMaxJobs || len(m.state.Connections) > secretaryMaxConnections {
return errors.New("invalid LO secretary state identity or bounds")
}
for key, marker := range m.state.ManualReplies {
if !marker.valid() || key != marker.key() {
return errors.New("invalid persisted LO manual reply watermark")
}
}
return nil
}

func (m *secretaryManager) validateJob(key string, job secretaryJob) error {
if !job.Message.Valid() || job.CreatedAt <= 0 || job.Attempts < 0 || job.NextAttemptAt < 0 ||
job.ContextNoticeID < 0 || job.NoticeAttempts < 0 || job.NoticeAttempts > secretaryMaxNoticeAttempts ||
Expand Down Expand Up @@ -343,6 +364,12 @@ func (m *secretaryManager) err() error {
func (m *secretaryManager) pruneLocked() bool {
changed := false
cutoff := time.Now().Add(-secretaryLifetime).Unix()
for key, marker := range m.state.ManualReplies {
if marker.ObservedAt < cutoff {
delete(m.state.ManualReplies, key)
changed = true
}
}
for key, at := range m.state.Invalid {
if at < cutoff {
delete(m.state.Invalid, key)
Expand All @@ -364,7 +391,24 @@ func (m *secretaryManager) pruneLocked() bool {
}

// Handle persists the native update watermark and changes before polling acknowledgement.
func (m *secretaryManager) Handle(_ context.Context, update lo.Update) (err error) {
func (m *secretaryManager) Handle(ctx context.Context, update lo.Update) (err error) {
m.mu.Lock()
failure := m.failure
duplicate := update.ID > 0 && update.ID <= m.state.UpdateFloor
m.mu.Unlock()
if failure != nil || duplicate {
return failure
}
// A direction candidate must be authenticated before it can invalidate jobs.
// Never perform the connection lookup while holding the state lock.
var manualConnection *lo.SecretaryConnection
if msg := update.BusinessMessage; isSecretaryManualCandidate(msg) {
conn, lookupErr := m.api.GetBusinessConnection(ctx, msg.ConnectionID)
if lookupErr != nil {
return lookupErr
}
manualConnection = &conn
}
m.mu.Lock()
defer m.mu.Unlock()
if m.failure != nil {
Expand Down Expand Up @@ -397,6 +441,9 @@ func (m *secretaryManager) Handle(_ context.Context, update lo.Update) (err erro
m.invalidateSourceLocked(deleted.ConnectionID, deleted.Context, deleted.MessageIDs)
return nil
}
if manualConnection != nil {
return m.observeManualReplyLocked(*update.BusinessMessage, *manualConnection)
}
m.admitLocked(update)
return nil
}
Expand Down Expand Up @@ -453,6 +500,9 @@ func (m *secretaryManager) admitLocked(update lo.Update) {
conn.PolicyVersion >= msg.Context.PolicyVersion && !conn.CanReply(msg.Context) {
job.Status = secretaryCancelled
}
if conn, exists := m.state.Connections[msg.ConnectionID]; exists && m.manuallyAnsweredLocked(conn.User.ID, job) {
job.Status = secretaryCancelled
}
if !m.admissionFitsLocked(job) {
slog.Warn("LO secretary state byte capacity reached; message skipped", "update_id", update.ID)
return
Expand Down Expand Up @@ -810,7 +860,7 @@ func (m *secretaryManager) connection(ctx context.Context, key string, job secre
// A newer poll snapshot can revoke an in-flight lookup; an older one cannot reject a fresh generation.
latest, exists := m.state.Connections[conn.ID]
current := m.state.Jobs[key]
if m.failure != nil || secretaryTerminal(current.Status) ||
if m.failure != nil || secretaryTerminal(current.Status) || m.manuallyAnsweredLocked(conn.User.ID, current) ||
(exists && latest.PolicyVersion >= conn.PolicyVersion &&
(!latest.CanReply(job.Message.Context) || latest.User.ID != conn.User.ID)) {
return conn, errSecretaryConsent
Expand Down
78 changes: 78 additions & 0 deletions cmd/flock-lo/secretary_manual.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,78 @@
package main

import (
"errors"
"slices"
"strconv"
"time"

"github.com/duckbugio/flock/adapters/lo"
)

// Revision is the server's conversation change sequence, assigned under its
// conversation lock. Message IDs can be allocated before that lock is acquired.
type secretaryManualReply struct {
OwnerID int64 `json:"ownerId"`
ConnectionID string `json:"connectionId"`
Context lo.SecretaryContext `json:"context"`
ObservedAt int64 `json:"observedAt"`
}

func (marker secretaryManualReply) key() string {
return strconv.FormatInt(marker.OwnerID, 10) + ":" + marker.ConnectionID + ":" +
marker.Context.ConversationID.String() + ":" + marker.Context.ChatID.String() + ":" +
marker.Context.PolicyVersion.String()
}

func (marker secretaryManualReply) valid() bool {
return marker.OwnerID > 0 && marker.OwnerID < 1000000000000000 &&
lo.ValidSecretaryConnectionID(marker.ConnectionID) && marker.Context.Valid() && marker.ObservedAt > 0
}

func isSecretaryManualCandidate(msg *lo.SecretaryMessage) bool {
// Incoming private messages have the correspondent as both From and Chat.
return msg != nil && msg.Valid() && msg.BotID == 0 && msg.From.ID != int64(msg.Chat.ID)
}

func (m *secretaryManager) observeManualReplyLocked(msg lo.SecretaryMessage, conn lo.SecretaryConnection) error {
if !conn.Valid() || conn.ID != msg.ConnectionID || !conn.Enabled || conn.PolicyVersion != msg.Context.PolicyVersion ||
conn.User.ID != msg.From.ID || !m.cfg.IsLOAllowed(conn.User.ID) || !slices.Contains(conn.Rights, "receive_messages") {
return nil
}
if latest, exists := m.state.Connections[conn.ID]; exists && latest.PolicyVersion >= conn.PolicyVersion &&
(latest.PolicyVersion != conn.PolicyVersion || latest.User.ID != conn.User.ID || !latest.Enabled ||
!slices.Contains(latest.Rights, "receive_messages")) {
return nil
}
marker := secretaryManualReply{
OwnerID: conn.User.ID, ConnectionID: conn.ID, Context: msg.Context, ObservedAt: time.Now().Unix(),
}
key := marker.key()
previous, exists := m.state.ManualReplies[key]
if !exists && len(m.state.ManualReplies) >= secretaryMaxJobs {
// Do not acknowledge a takeover that cannot be retained against replay.
return errors.New("LO secretary manual reply watermark capacity reached")
}
if !exists || marker.Context.SourceRevision > previous.Context.SourceRevision {
m.state.ManualReplies[key] = marker
}
for jobKey, job := range m.state.Jobs {
scope := secretaryManualReply{OwnerID: conn.User.ID, ConnectionID: job.Message.ConnectionID, Context: job.Message.Context}
if scope.key() == key && m.manuallyAnsweredLocked(conn.User.ID, job) {
m.invalidateLocked(jobKey, job)
}
}
return nil
}

func (m *secretaryManager) manuallyAnsweredLocked(ownerID int64, job secretaryJob) bool {
// A send already attempted must keep its exact request, retry and receipt
// semantics. Manual takeover cannot prove whether that write took effect.
if job.DeliveryAttempted || secretaryTerminal(job.Status) || (job.OwnerID != 0 && job.OwnerID != ownerID) ||
job.Message.From.ID != int64(job.Message.Chat.ID) {
return false
}
marker := secretaryManualReply{OwnerID: ownerID, ConnectionID: job.Message.ConnectionID, Context: job.Message.Context}
previous, exists := m.state.ManualReplies[marker.key()]
return exists && job.Message.Context.SourceRevision <= previous.Context.SourceRevision
}
Loading
Loading