Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
134 changes: 134 additions & 0 deletions internal/app/console_lab_clock_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,134 @@
package app

import (
"context"
"crypto/sha256"
"encoding/hex"
"errors"
"path/filepath"
"testing"
"time"

"github.com/Horcag/agent-machine-control/internal/approval"
"github.com/Horcag/agent-machine-control/internal/domain"
)

func TestGuestLabAdmissionSurvivesBackwardClockStep(t *testing.T) {
h := newGuestHarness(t)
h.safety.destructive = true
h.actor.AuthenticatedCaller, h.actor.EffectiveActor = "agent:mcp-local", "agent:mcp-local"
coordinator := h.service.session
authority, operator := persistentGuestAuthority(t, h)
admittedAt := time.Now().UTC()
authority.recovery.nowFn = func() time.Time { return admittedAt }
grant, _, err := authority.IssueLabGrant(t.Context(), operator, ConsoleLabGrantIssueRequest{Mode: ConsoleLabModePersistentVM, Target: "default", Reason: "operate disposable VM", IdempotencyKey: "clock-lab", Beneficiary: "agent:mcp-local", AcknowledgeExternalEffects: true})
if err != nil {
t.Fatal(err)
}
coordinator.labAuthority = authority
p := h.params("prepare", "clock-step-prepare")
p.Deadline = admittedAt.Add(time.Minute)
digest := sha256.Sum256([]byte(grant.GrantID + "\x00" + string(h.actor.EffectiveActor) + "\x00" + p.IdempotencyKey))
id := ConsoleLabApprovalPrefix + hex.EncodeToString(digest[:16])
// A server clock can step backwards between issuance and policy/consumption.
// The separate recovery clock must not choose this admission's issuance time.
authority.recovery.nowFn = func() time.Time { return admittedAt.Add(50 * time.Millisecond) }
coordinator.nowFn = func() time.Time {
if issued, _ := coordinator.approvalStore.LoadIssuedContext(context.Background(), id); issued != nil {
return admittedAt.Add(-50 * time.Millisecond)
}
return admittedAt
}
result, err := h.service.Execute(t.Context(), p, nil, nil)
if err != nil || result.Receipt == nil || result.Receipt.Outcome.Status != domain.OutcomeSuccess || h.transport.executed != 1 {
t.Fatalf("backwards clock denied fresh local authority: result=%+v err=%v dispatches=%d", result, err, h.transport.executed)
}
issued, err := coordinator.approvalStore.LoadIssuedContext(t.Context(), id)
if err != nil || !issued.IssuedAt.Equal(admittedAt) {
t.Fatalf("issuance used a different clock: approval=%+v err=%v", issued, err)
}
loaded, err := approval.LoadFromFile(filepath.Join(filepath.Dir(h.service.store.directory), "approvals", id+".json"))
if err != nil || !loaded.Consumed || loaded.ConsumedAt.Before(loaded.IssuedAt) {
t.Fatalf("invalid durable consumption: approval=%+v err=%v", loaded, err)
}
if _, err := h.service.Execute(t.Context(), p, nil, nil); err != nil || h.transport.executed != 1 {
t.Fatalf("exact retry dispatched again: %v dispatches=%d", err, h.transport.executed)
}
}

func TestGuestLabClockBoundariesRemainStrict(t *testing.T) {
for _, scenario := range []string{"existing future", "forward expiry", "consumption expiry", "cancelled"} {
t.Run(scenario, func(t *testing.T) { testGuestLabClockBoundary(t, scenario) })
}
}

func testGuestLabClockBoundary(t *testing.T, scenario string) {
t.Helper()
h := newGuestHarness(t)
h.safety.destructive = true
h.actor.AuthenticatedCaller, h.actor.EffectiveActor = "agent:mcp-local", "agent:mcp-local"
authority, operator := persistentGuestAuthority(t, h)
admittedAt := time.Now().UTC()
authority.recovery.nowFn = func() time.Time { return admittedAt }
grant, _, err := authority.IssueLabGrant(t.Context(), operator, ConsoleLabGrantIssueRequest{Mode: ConsoleLabModePersistentVM, Target: "default", Reason: "operate disposable VM", IdempotencyKey: "clock-boundaries", Beneficiary: "agent:mcp-local", AcknowledgeExternalEffects: true})
if err != nil {
t.Fatal(err)
}
coordinator := h.service.session
coordinator.labAuthority = authority
p := h.params("prepare", "clock-boundaries-prepare")
p.Deadline = admittedAt.Add(time.Minute)
digest := sha256.Sum256([]byte(grant.GrantID + "\x00" + string(h.actor.EffectiveActor) + "\x00" + p.IdempotencyKey))
id := ConsoleLabApprovalPrefix + hex.EncodeToString(digest[:16])
ctx, cancel := context.WithCancel(t.Context())
defer cancel()
if scenario == "existing future" {
config, err := h.config.Current(ctx, grant.Target, h.actor)
if err != nil {
t.Fatal(err)
}
op, err := guestOperation(p, grant.Target, config)
if err != nil {
t.Fatal(err)
}
op.Classification = domain.ClassDestructivePrivileged
if _, err := authority.deriveLabApproval(ctx, grant, op, admittedAt.Add(50*time.Millisecond)); err != nil {
t.Fatal(err)
}
}
coordinator.nowFn = guestBoundaryClock(coordinator, id, scenario, admittedAt, p.Deadline, cancel)
_, err = h.service.Execute(ctx, p, nil, nil)
if err == nil || h.transport.executed != 0 {
t.Fatalf("clock boundary dispatched: err=%v dispatches=%d", err, h.transport.executed)
}
if scenario == "existing future" && !errors.Is(err, domain.ErrApprovalNotYetValid) {
var denied *PolicyDeniedError
if !errors.As(err, &denied) || denied.Reason != "approval_record_not_yet_valid" {
t.Fatalf("future authority changed: %v", err)
}
}
consumed, err := coordinator.approvalStore.IsConsumedContext(t.Context(), id)
if err != nil || consumed {
t.Fatalf("rejected authority was consumed: %v %v", consumed, err)
}
}

func guestBoundaryClock(coordinator *SessionService, id, scenario string, admittedAt, deadline time.Time, cancel context.CancelFunc) func() time.Time {
afterIssuance := 0
return func() time.Time {
if issued, _ := coordinator.approvalStore.LoadIssuedContext(context.Background(), id); issued != nil {
afterIssuance++
switch scenario {
case "forward expiry":
return deadline
case "consumption expiry":
if afterIssuance > 1 {
return deadline
}
case "cancelled":
cancel()
}
}
return admittedAt
}
}
18 changes: 15 additions & 3 deletions internal/app/console_lab_coordination.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ package app
import (
"context"
"errors"
"time"

"github.com/Horcag/agent-machine-control/internal/domain"
"github.com/Horcag/agent-machine-control/internal/target"
Expand Down Expand Up @@ -57,12 +58,12 @@ func (s *ConsoleService) admitPersistentLab(ctx context.Context, op domain.Opera
return &grant, release, nil
}

func (s *ConsoleService) derivePersistentLab(ctx context.Context, op domain.Operation) (*domain.Approval, *ConsoleLabGrant, func() error, error) {
func (s *ConsoleService) derivePersistentLab(ctx context.Context, op domain.Operation, admittedAt time.Time) (*domain.Approval, *ConsoleLabGrant, func() error, error) {
grant, release, err := s.admitPersistentLab(ctx, op)
if err != nil || grant == nil {
return nil, grant, release, err
}
approval, err := s.deriveLabApproval(ctx, *grant, op)
approval, err := s.deriveLabApproval(ctx, *grant, op, admittedAt)
if err != nil {
return nil, nil, func() error { return nil }, errors.Join(err, release())
}
Expand All @@ -86,16 +87,27 @@ func (s *RecoveryService) coordinatePersistentRecovery(ctx context.Context, op d
if req.Approval != nil || req.ApprovalID != "" || req.ApprovalError != nil {
return req, execute, release, nil
}
derived, grant, release, err := s.labAuthority.derivePersistentLab(ctx, op)
admittedAt := s.now()
derived, grant, release, err := s.labAuthority.derivePersistentLab(ctx, op, admittedAt)
if err != nil {
return req, execute, release, err
}
if derived == nil {
return req, execute, release, nil
}
req.Approval, req.ApprovalID = derived, string(derived.ID)
req.labAdmissionAt = admittedAt
fenced := func(effectCtx context.Context) error {
return s.labAuthority.withPersistentLabEffect(effectCtx, grant, execute)
}
return req, fenced, release, nil
}

// labAdmissionTime preserves causality only for locally derived authority.
// Fresh forward readings still enforce expiry; external approvals get no floor.
func labAdmissionTime(now, admittedAt time.Time, local bool) time.Time {
if local && now.Before(admittedAt) {
return admittedAt
}
return now
}
7 changes: 4 additions & 3 deletions internal/app/console_lab_mutation.go
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,8 @@ func (s *ConsoleService) executeLabMutation(ctx context.Context, actor domain.Ac
if err != nil {
return result, err
}
issued, err := s.deriveLabApproval(ctx, grant, op)
req.labAdmissionAt = s.recovery.now()
issued, err := s.deriveLabApproval(ctx, grant, op, req.labAdmissionAt)
if err != nil {
return result, err
}
Expand All @@ -63,7 +64,7 @@ func labOperationAllowed(op domain.Operation) bool {
op.Classification == domain.ClassDestructivePrivileged && len(op.RequiredScopes) == 1 && op.RequiredScopes[0] == domain.ScopeMachineWrite
}

func (s *ConsoleService) deriveLabApproval(ctx context.Context, grant ConsoleLabGrant, op domain.Operation) (*domain.Approval, error) {
func (s *ConsoleService) deriveLabApproval(ctx context.Context, grant ConsoleLabGrant, op domain.Operation, issuedAt time.Time) (*domain.Approval, error) {
fingerprint, err := op.Fingerprint()
if err != nil {
return nil, err
Expand All @@ -75,7 +76,7 @@ func (s *ConsoleService) deriveLabApproval(ctx context.Context, grant ConsoleLab
return nil, err
}
issued := domain.Approval{ID: id, Actor: op.Actor.EffectiveActor, Target: op.Target, AuthorizedClass: domain.ClassDestructivePrivileged,
Fingerprint: fingerprint, IdempotencyKey: op.IdempotencyKey, IssuedAt: s.recovery.now(), ExpiresAt: op.Deadline}
Fingerprint: fingerprint, IdempotencyKey: op.IdempotencyKey, IssuedAt: issuedAt, ExpiresAt: op.Deadline}
if existing != nil {
issued.IssuedAt = existing.IssuedAt
if !equalIssuedOperationApproval(*existing, issued) {
Expand Down
32 changes: 32 additions & 0 deletions internal/app/console_lab_time_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,9 +2,14 @@ package app_test

import (
"context"
"crypto/sha256"
"encoding/hex"
"path/filepath"
"testing"
"time"

"github.com/Horcag/agent-machine-control/internal/app"
"github.com/Horcag/agent-machine-control/internal/approval"
"github.com/Horcag/agent-machine-control/internal/domain"
)

Expand Down Expand Up @@ -44,3 +49,30 @@ func TestConsoleLabEquivalentDeadlineDispatchesOnce(t *testing.T) {
})
}
}

func TestNativeLabAdmissionSurvivesBackwardClockStep(t *testing.T) {
f := newConsoleFixture(t)
configureLab(f)
grant := issueLab(t, f)
op, req, providerID := labOperation(t, grant)
digest := sha256.Sum256([]byte(grant.GrantID + "\x00" + string(op.Actor.EffectiveActor) + "\x00" + op.IdempotencyKey))
id := app.ConsoleLabApprovalPrefix + hex.EncodeToString(digest[:16])
store := approval.NewStore(filepath.Join(f.root, "approvals"))
*f.now = f.now.Add(time.Second)
admittedAt := *f.now
app.WithRecoveryClock(func() time.Time {
if issued, _ := store.LoadIssuedContext(context.Background(), id); issued != nil {
return admittedAt.Add(-50 * time.Millisecond)
}
return admittedAt
})(f.recovery)
dispatched := 0
result, err := f.service.ExecuteLabMutation(t.Context(), req.Actor, grant.GrantID, op, req, providerID, func(context.Context) error { dispatched++; return nil })
if err != nil || result.Outcome.Status != domain.OutcomeSuccess || dispatched != 1 {
t.Fatalf("native lab clock step: result=%+v err=%v dispatches=%d", result, err, dispatched)
}
loaded, err := approval.LoadFromFile(filepath.Join(f.root, "approvals", id+".json"))
if err != nil || !loaded.Consumed || loaded.ConsumedAt.Before(loaded.IssuedAt) {
t.Fatalf("native consumption is invalid: approval=%+v err=%v", loaded, err)
}
}
1 change: 1 addition & 0 deletions internal/app/guest_types.go
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ var ErrGuestUncertain = errors.New("app: guest operation outcome is uncertain; d
// GuestConfiguration is refreshed at admission and at the effect boundary.
// Current must revalidate enrollment, credentials and key pins, current actor
// authority, and the server-provisioned workspace; Generation binds all of them.
// Caller preserves requested authority without expanding it to the live principal.
// WorkspaceIdentity must come from authorized provisioning, never path adoption.
type GuestConfiguration struct {
Generation string
Expand Down
2 changes: 2 additions & 0 deletions internal/app/recovery.go
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,8 @@ type MutationRequest struct {
// Set only when the normal identity-bound receipt lookup returns a terminal result.
cachedReceipt *bool
providerDispatched *bool
// Only trusted lab coordination sets this causal floor for local issuance.
labAdmissionAt time.Time
}

// RecoveryService orchestrates in-process direct recovery operations, policy, leases, and receipts.
Expand Down
71 changes: 71 additions & 0 deletions internal/app/recovery_lab_clock_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,71 @@
package app

import (
"context"
"crypto/sha256"
"encoding/hex"
"testing"
"time"

"github.com/Horcag/agent-machine-control/internal/domain"
)

type clockLabBackend struct {
Backend
onCapabilities func()
}

func (b clockLabBackend) Capabilities(context.Context, string) (domain.CapabilitySet, error) {
b.onCapabilities()
return domain.NewCapabilitySet(domain.CapabilityConsoleInput), nil
}

func TestPersistentRecoveryLabClockBoundaries(t *testing.T) {
for _, forward := range []bool{false, true} {
name := "backward step"
if forward {
name = "expiry before consumption"
}
t.Run(name, func(t *testing.T) { testPersistentRecoveryLabClock(t, forward) })
}
}

func testPersistentRecoveryLabClock(t *testing.T, forward bool) {
t.Helper()
authority, grant, op, _ := coordinationGrantFixture(t)
recovery := authority.recovery
WithRecoveryLabAuthority(authority)(recovery)
digest := sha256.Sum256([]byte(grant.GrantID + "\x00" + string(op.Actor.EffectiveActor) + "\x00" + op.IdempotencyKey))
id := ConsoleLabApprovalPrefix + hex.EncodeToString(digest[:16])
admittedAt := grant.IssuedAt.Add(time.Second)
op.Deadline = admittedAt.Add(time.Minute)
authorized := false
recovery.backend = clockLabBackend{onCapabilities: func() { authorized = true }}
recovery.nowFn = func() time.Time {
if forward && authorized {
return op.Deadline
}
if issued, _ := recovery.approvalStore.LoadIssuedContext(context.Background(), id); issued != nil {
return admittedAt.Add(-50 * time.Millisecond)
}
return admittedAt
}
req := MutationRequest{TargetID: string(op.Target), Actor: op.Actor, Reason: op.Reason, IdempotencyKey: op.IdempotencyKey, Deadline: op.Deadline}
locator, err := domain.ParseMachineLocator(string(op.Target))
if err != nil {
t.Fatal(err)
}
dispatched := 0
result, err := recovery.executeMutation(t.Context(), op, req, locator.VMID, func(context.Context) error { dispatched++; return nil })
if forward {
if err == nil || dispatched != 0 {
t.Fatalf("expired admission dispatched: result=%+v err=%v dispatches=%d", result, err, dispatched)
}
} else if err != nil || result.Outcome.Status != domain.OutcomeSuccess || dispatched != 1 {
t.Fatalf("backwards clock rejected causal admission: result=%+v err=%v dispatches=%d", result, err, dispatched)
}
consumed, err := recovery.approvalStore.IsConsumedContext(t.Context(), id)
if err != nil || consumed == forward {
t.Fatalf("consumption=%v err=%v", consumed, err)
}
}
10 changes: 8 additions & 2 deletions internal/app/recovery_mutation.go
Original file line number Diff line number Diff line change
Expand Up @@ -130,7 +130,7 @@ func (s *RecoveryService) executeMutation(
}
defer func() { resultErr = errors.Join(resultErr, releaseGrant()) }()

now := s.now()
now := labAdmissionTime(s.now(), req.labAdmissionAt, !req.labAdmissionAt.IsZero())

// 3. Rollback discovery & 4. Policy evaluation & 5. Approval check
decision, rollbackRef, err := s.prepareAndAuthorizeMutation(ctx, op, req, providerTargetID, now)
Expand All @@ -151,7 +151,8 @@ func (s *RecoveryService) executeMutation(
}

// 7. Audit Admission Intent & Approval Consumption
approvalConsumed, err := s.recordAdmissionAndConsume(ctx, op, decision, req, now)
consumedAt := labAdmissionTime(s.now(), now, !req.labAdmissionAt.IsZero())
approvalConsumed, err := s.recordAdmissionAndConsume(ctx, op, decision, req, consumedAt)
if err != nil {
return s.preProviderFailure(ctx, op, fp, decision, now, errors.Join(err, releaseLease()), rollbackRef, req.ApprovalID)
}
Expand Down Expand Up @@ -219,6 +220,11 @@ func (s *RecoveryService) recordAdmissionAndConsume(
req MutationRequest,
now time.Time,
) (bool, error) {
if decision.EffectiveClass.RequiresApproval() && req.Approval != nil {
if err := req.Approval.IsActive(now); err != nil {
return false, approvalActivityDenial(err)
}
}
if s.auditStore != nil {
if err := s.auditStore.RecordAdmissionIntentContext(ctx, op); err != nil {
return false, fmt.Errorf("%w: %w", ErrAuditUnavailable, err)
Expand Down
Loading
Loading