From 4cde8beed1cde5f64686eaed8d14e8f02ea4f1f4 Mon Sep 17 00:00:00 2001 From: Aaron Stannard Date: Sat, 1 Aug 2026 00:18:53 +0000 Subject: [PATCH 1/6] Reject incompatible session input before model calls --- ...SPEC-002-session-lifecycle-and-protocol.md | 25 ++- .../references/providers.md | 12 ++ .../.openspec.yaml | 2 + .../design.md | 72 ++++++++ .../proposal.md | 54 ++++++ .../specs/netclaw-model-capabilities/spec.md | 47 +++++ .../tasks.md | 35 ++++ .../Sessions/ModalityGateTests.cs | 60 +++---- .../Sessions/ModelInputCompatibilityTests.cs | 90 ++++++++++ ...ssionInputCompatibilityIntegrationTests.cs | 167 ++++++++++++++++++ .../Sessions/LlmSessionActor.cs | 85 ++++++--- .../Sessions/ModelInputCompatibility.cs | 82 +++++++++ .../Sessions/SessionProtocol.Outputs.cs | 5 +- 13 files changed, 665 insertions(+), 71 deletions(-) create mode 100644 openspec/changes/reject-incompatible-session-history/.openspec.yaml create mode 100644 openspec/changes/reject-incompatible-session-history/design.md create mode 100644 openspec/changes/reject-incompatible-session-history/proposal.md create mode 100644 openspec/changes/reject-incompatible-session-history/specs/netclaw-model-capabilities/spec.md create mode 100644 openspec/changes/reject-incompatible-session-history/tasks.md create mode 100644 src/Netclaw.Actors.Tests/Sessions/ModelInputCompatibilityTests.cs create mode 100644 src/Netclaw.Actors.Tests/Sessions/SessionInputCompatibilityIntegrationTests.cs create mode 100644 src/Netclaw.Actors/Sessions/ModelInputCompatibility.cs diff --git a/docs/spec/SPEC-002-session-lifecycle-and-protocol.md b/docs/spec/SPEC-002-session-lifecycle-and-protocol.md index 90977dfcf..5ee3e762d 100644 --- a/docs/spec/SPEC-002-session-lifecycle-and-protocol.md +++ b/docs/spec/SPEC-002-session-lifecycle-and-protocol.md @@ -34,12 +34,25 @@ This enables: ## Turn Lifecycle -1. `SendUserMessage` command accepted after policy checks. -2. Actor appends user message to `SessionState.History`. -3. Actor invokes configured `IChatClient` via `ChatMessageConverter`. -4. Actor persists `TurnRecorded` event and applies to state. -5. Actor emits typed `SessionOutput` events to subscribers. -6. Actor checks compaction threshold. +1. `SendUserMessage` passes policy and complete input compatibility checks. +2. Actor appends the user message to `SessionState.History`. +3. Actor checks active history again before each model call. +4. Actor invokes the configured `IChatClient` via `ChatMessageConverter`. +5. Actor persists the `TurnRecorded` event and applies it to state. +6. Actor emits typed `SessionOutput` events to subscribers. +7. Actor checks the compaction threshold. + +### Model Input Compatibility + +The actor checks all active media references against the main model input +modalities. The check includes recovered history, new input, buffered input, +and tool-result media. An unknown persisted modality fails closed. + +The actor rejects incompatible new input before it changes the session state. +It checks again before each model call to protect paths that add media during a +turn. The actor emits `ErrorCategory.InputCompatibility` with the unsupported +modalities and recovery guidance. It does not call the primary client, +fallback client, or provider when this local check fails. ### Tool Execution Pipeline diff --git a/feeds/skills/.system/files/netclaw-operations/references/providers.md b/feeds/skills/.system/files/netclaw-operations/references/providers.md index 462defd0e..5cfd3a7a5 100644 --- a/feeds/skills/.system/files/netclaw-operations/references/providers.md +++ b/feeds/skills/.system/files/netclaw-operations/references/providers.md @@ -113,6 +113,18 @@ still read. `model list` reports an unparseable config instead of crashing. `netclaw doctor --fix` applies only repairs it can derive safely; it does not invent missing named definitions or role assignments. +### Session input compatibility errors + +A saved session can contain image, audio, or video input from an earlier model. +Netclaw checks the complete active history before each model call. If the new +main model lacks a required modality, the turn stops before any provider or +fallback call. + +The error names the unsupported modalities and the active model. Select a model +that accepts those modalities, or start a new conversation. Do not diagnose +this result as a provider outage. Netclaw also rejects an unknown saved modality +value instead of omitting that media. + ### Adding GitHub Copilot GitHub Copilot uses the OAuth device flow only — no API key. The operator diff --git a/openspec/changes/reject-incompatible-session-history/.openspec.yaml b/openspec/changes/reject-incompatible-session-history/.openspec.yaml new file mode 100644 index 000000000..ffa710fcc --- /dev/null +++ b/openspec/changes/reject-incompatible-session-history/.openspec.yaml @@ -0,0 +1,2 @@ +schema: spec-driven +created: 2026-07-31 diff --git a/openspec/changes/reject-incompatible-session-history/design.md b/openspec/changes/reject-incompatible-session-history/design.md new file mode 100644 index 000000000..c941927cb --- /dev/null +++ b/openspec/changes/reject-incompatible-session-history/design.md @@ -0,0 +1,72 @@ +## Context + +The session actor persists media references with each chat message. +The message assembler later restores those references as model input. +The current ingress check only examines media on the new user command. +A model change can therefore make recovered history incompatible with the active model. + +The routing chat client treats request failures as provider failures. +The actor must reject incompatible input before that boundary. + +## Goals / Non-Goals + +**Goals:** + +- Check the complete active session input before every model call. +- Reject current, recovered, and tool-produced unsupported media. +- Fail closed for an unknown persisted media modality. +- Keep provider fallback and health signals out of this local error path. + +**Non-Goals:** + +- Convert media to another modality. +- Select another model. +- Change the persisted media format. +- Add audio or video support. + +## Decisions + +### The session actor owns the compatibility check + +The actor has the active model capabilities and the canonical session history. +It will check persisted media references before it calls `IChatClient`. + +The provider client was rejected as the owner. +That location cannot separate local input errors from provider failover without wider routing changes. + +### One pure check covers all media references + +A pure helper will map each `MediaModality` value to a `ModelModality` flag. +The result will list required, unsupported, and unknown modalities. + +The actor will use the helper before it accepts a new user turn. +The actor will use it again before each model call after a tool result. + +### The actor will reject instead of removing content + +The actor will not remove an unsupported media reference. +It will emit an input compatibility error with the active model and missing modalities. + +This choice preserves the session record and prevents silent context loss. + +### Local compatibility errors will not enter model routing + +The actor will complete a rejected new command without a provider call. +If a tool adds incompatible media, the actor will fail the current turn before the next model call. +Neither path will persist a provider failure or activate fallback. + +## Risks / Trade-offs + +- [A historical session remains unusable with a text-only model] -> The error names the required modalities and gives model-selection guidance. +- [A corrupt modality value exists in storage] -> The check rejects the call and reports an unknown modality. +- [A future call path bypasses the ingress check] -> The second check at the model-call boundary remains authoritative. + +## Migration Plan + +The change needs no data migration. +Deployment changes only the result for an incompatible session. +A rollback restores the old provider-error behavior. + +## Open Questions + +None. diff --git a/openspec/changes/reject-incompatible-session-history/proposal.md b/openspec/changes/reject-incompatible-session-history/proposal.md new file mode 100644 index 000000000..638a84014 --- /dev/null +++ b/openspec/changes/reject-incompatible-session-history/proposal.md @@ -0,0 +1,54 @@ +## Why + +A resumed session can contain image content that the active model cannot accept. +The actor now discovers this mismatch only after it calls a provider. + +Source PRDs: `PRD-001`, `PRD-005` +GitHub issue: `#1727` + +## What Changes + +- Check all active session media before each model call. +- Include recovered history, the new user message, and tool-produced media in the check. +- Reject unsupported or unknown media before the routing client receives a request. +- Show the unsupported modalities and clear operator recovery steps. +- Classify the result as an input compatibility error, not a provider failure. +- Do not activate a fallback model or a provider alert for this local error. + +### In Scope + +- Image, audio, and video compatibility checks for existing media records. +- A fail-closed result for unknown persisted modality values. +- Tests for recovery, new input, tool-loop input, and zero provider calls. + +### Out of Scope + +- A media proxy. +- Session model pins. +- An automatic switch to a compatible model. +- Audio or video feature support. + +## Capabilities + +### New Capabilities + +None. + +### Modified Capabilities + +- `netclaw-model-capabilities`: Require a complete session-input compatibility check before each model call. + +## Impact + +The change affects the session actor, media conversion, error output, and session tests. +It does not change provider APIs or persisted media records. + +### Security Impact + +The check fails closed for unknown media types. +It prevents incompatible content from crossing the provider boundary. + +### Operational Impact + +Operators receive a local compatibility error with model-selection guidance. +Provider health alerts and fallback logs remain reserved for provider failures. diff --git a/openspec/changes/reject-incompatible-session-history/specs/netclaw-model-capabilities/spec.md b/openspec/changes/reject-incompatible-session-history/specs/netclaw-model-capabilities/spec.md new file mode 100644 index 000000000..695e389c6 --- /dev/null +++ b/openspec/changes/reject-incompatible-session-history/specs/netclaw-model-capabilities/spec.md @@ -0,0 +1,47 @@ +## ADDED Requirements + +### Requirement: Complete session input compatibility check + +The session actor SHALL check all active persisted media and all new media against the active model input modalities before each model call. +The check SHALL include recovered history and media that a tool adds during the current turn. +The actor SHALL reject an unsupported or unknown modality before any primary, fallback, or provider client receives a request. +The actor SHALL preserve all original media references and SHALL identify the incompatible modalities in the session error. + +#### Scenario: Recovered image history meets a text-only model + +- **GIVEN** a recovered session contains an image media reference +- **AND** the active model accepts text only +- **WHEN** the user resumes the session +- **THEN** the actor SHALL emit an input compatibility error +- **AND** the error SHALL identify image input as unsupported +- **AND** no primary, fallback, or provider client SHALL receive a request + +#### Scenario: New unsupported media is rejected before turn admission + +- **GIVEN** a new user command contains an image media reference +- **AND** the active model accepts text only +- **WHEN** the actor receives the command +- **THEN** the actor SHALL reject the command before it adds the user message to session state +- **AND** no model client SHALL receive a request + +#### Scenario: Tool-produced media is checked before the next call + +- **GIVEN** the active model call starts with compatible text input +- **AND** a tool result adds media that the active model cannot accept +- **WHEN** the actor prepares the next model call +- **THEN** the actor SHALL fail the current turn with an input compatibility error +- **AND** no later model client SHALL receive the incompatible request + +#### Scenario: Unknown persisted modality fails closed + +- **GIVEN** a session contains a media reference with an unknown modality value +- **WHEN** the actor prepares a model call +- **THEN** the actor SHALL emit an input compatibility error +- **AND** no model client SHALL receive a request + +#### Scenario: Compatible media reaches the model + +- **GIVEN** all session media modalities are accepted by the active model +- **WHEN** the actor prepares a model call +- **THEN** the actor SHALL preserve the media references +- **AND** the model call SHALL proceed through normal routing diff --git a/openspec/changes/reject-incompatible-session-history/tasks.md b/openspec/changes/reject-incompatible-session-history/tasks.md new file mode 100644 index 000000000..1778df9b2 --- /dev/null +++ b/openspec/changes/reject-incompatible-session-history/tasks.md @@ -0,0 +1,35 @@ +## 1. Compatibility Contract + +- [x] 1.1 Add a pure media-to-model compatibility check with an unknown-modality result. +- [x] 1.2 Add a distinct input compatibility error category and user guidance. + +## 2. Session Boundary + +- [x] 2.1 Reject incompatible current and recovered media before turn admission. +- [x] 2.2 Check active history again before every model call after state changes. +- [x] 2.3 Preserve original media and keep local errors outside provider fallback and alerts. + +## 3. Automated Proof + +- [x] 3.1 Add unit tests for supported, unsupported, combined, and unknown modalities. +- [x] 3.2 Add actor tests for current media and recovered history with zero provider calls. +- [x] 3.3 Add a tool-message test and a second-boundary actor test. + +## 4. Documentation and Gates + +- [x] 4.1 Update operator guidance and the `netclaw-operations` system skill. +- [x] 4.2 Run targeted tests, the eval suite, repository quality gates, and OpenSpec validation. +- [x] 4.3 Update this checklist with final verification evidence. + +## Verification Evidence + +- Focused compatibility suite: 10 tests passed. +- `Netclaw.Actors.Tests`: 2,657 tests passed. +- `dotnet test Netclaw.slnx --no-restore`: all enabled tests passed. +- `dotnet slopwatch analyze`: 0 issues. +- `pwsh ./scripts/Add-FileHeaders.ps1 -Verify`: passed. +- `openspec validate reject-incompatible-session-history --strict`: passed. +- `git diff --check`: passed. +- Changed production and new test files pass the scoped format check. +- The full format check still reports pre-existing repository format debt. +- `./evals/run-evals.sh` could not start because no `NETCLAW_EVAL_*` target exists in this environment. diff --git a/src/Netclaw.Actors.Tests/Sessions/ModalityGateTests.cs b/src/Netclaw.Actors.Tests/Sessions/ModalityGateTests.cs index 5296ff7dc..5c3ac12c2 100644 --- a/src/Netclaw.Actors.Tests/Sessions/ModalityGateTests.cs +++ b/src/Netclaw.Actors.Tests/Sessions/ModalityGateTests.cs @@ -16,8 +16,7 @@ namespace Netclaw.Actors.Tests.Sessions; /// -/// Tests the modality gate in : -/// images sent to a text-only model are stripped; images sent to a vision model pass through. +/// Tests the modality gate in . /// public class ModalityGateTextOnlyTests : LlmSessionTestBase { @@ -46,14 +45,8 @@ protected override void ConfigureSessionServices(IServiceCollection services) } [Fact] - public async Task Image_with_text_on_text_only_model_surfaces_ingress_bug_and_still_calls_llm() + public async Task Image_with_text_on_text_only_model_is_rejected_before_model_call() { - // The strict-consumer contract treats an unsupported-modality media - // ref reaching the session actor as an ingress bug. The session still - // completes the turn (so the user gets a reply) but the offending refs - // are dropped and a [system] notice about the ingress bug is appended - // to the user message before it goes to the model. No legacy - // "[Images removed]" placeholder is emitted. var sessionId = new SessionId("test-channel/modality-text-only"); var sessionManager = ActorRegistry.Get(); var subscriber = CreateTestProbe("modality-sub"); @@ -80,29 +73,27 @@ await sessionManager.Ask(new SendUserMessage ] }, TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); - // The first output is the LLM response itself — there is no longer a - // separate "[Images removed]" TextOutput before the reply. - var textOutput = await subscriber.ExpectMsgAsync(cancellationToken: TestContext.Current.CancellationToken); - Assert.Contains("fake", textOutput.Text, StringComparison.OrdinalIgnoreCase); - Assert.DoesNotContain("Images removed", textOutput.Text); + var error = await subscriber.ExpectMsgAsync(cancellationToken: TestContext.Current.CancellationToken); + Assert.Equal(ErrorCategory.InputCompatibility, error.Category); + Assert.Contains("Image", error.Message, StringComparison.Ordinal); + Assert.Contains("text-only-model", error.Message, StringComparison.Ordinal); - await subscriber.ExpectMsgAsync(cancellationToken: TestContext.Current.CancellationToken); + var completed = await subscriber.ExpectMsgAsync(cancellationToken: TestContext.Current.CancellationToken); + Assert.Equal(TurnOutcome.Skipped, completed.Outcome); + Assert.Equal(0, _fakeChatClient.CallCount); - // LLM was called and saw the ingress-bug notice appended to the user text. - Assert.Equal(1, _fakeChatClient.CallCount); - Assert.NotEmpty(_fakeChatClient.ReceivedMessages); - var lastRequest = _fakeChatClient.ReceivedMessages[^1]; - var concatenated = string.Join("\n", lastRequest.Select(m => m.Text ?? string.Empty)); - Assert.Contains("ingress bug", concatenated, StringComparison.OrdinalIgnoreCase); + var joined = await sessionManager.Ask(new JoinSession(subscriber) + { + SessionId = sessionId, + Filter = OutputFilter.Full + }, TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + Assert.Equal(0, joined.TurnCount); + Assert.Empty(joined.RecentMessages ?? []); } [Fact] - public async Task Image_only_message_on_text_only_model_still_calls_llm_with_ingress_bug_notice() + public async Task Image_only_message_on_text_only_model_is_rejected_before_model_call() { - // Empty text body + only unsupported media. The strict-consumer - // contract appends the [system] ingress bug notice to the user - // content so the LLM has something to respond to. We'd rather the - // user get a reply explaining the situation than silence. var sessionId = new SessionId("test-channel/modality-image-only"); var sessionManager = ActorRegistry.Get(); var subscriber = CreateTestProbe("modality-image-only-sub"); @@ -129,18 +120,13 @@ await sessionManager.Ask(new SendUserMessage ] }, TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); - var reply = await subscriber.ExpectMsgAsync(cancellationToken: TestContext.Current.CancellationToken); - Assert.DoesNotContain("Images removed", reply.Text); + var error = await subscriber.ExpectMsgAsync(cancellationToken: TestContext.Current.CancellationToken); + Assert.Equal(ErrorCategory.InputCompatibility, error.Category); + Assert.Contains("start a new conversation", error.Message, StringComparison.OrdinalIgnoreCase); - await subscriber.ExpectMsgAsync(cancellationToken: TestContext.Current.CancellationToken); - - // LLM was called once, and the user-visible content we sent it - // included the ingress-bug notice (not a legacy placeholder). - Assert.Equal(1, _fakeChatClient.CallCount); - Assert.NotEmpty(_fakeChatClient.ReceivedMessages); - var lastRequest = _fakeChatClient.ReceivedMessages[^1]; - var concatenated = string.Join("\n", lastRequest.Select(m => m.Text ?? string.Empty)); - Assert.Contains("ingress bug", concatenated, StringComparison.OrdinalIgnoreCase); + var completed = await subscriber.ExpectMsgAsync(cancellationToken: TestContext.Current.CancellationToken); + Assert.Equal(TurnOutcome.Skipped, completed.Outcome); + Assert.Equal(0, _fakeChatClient.CallCount); } } diff --git a/src/Netclaw.Actors.Tests/Sessions/ModelInputCompatibilityTests.cs b/src/Netclaw.Actors.Tests/Sessions/ModelInputCompatibilityTests.cs new file mode 100644 index 000000000..a6584fba8 --- /dev/null +++ b/src/Netclaw.Actors.Tests/Sessions/ModelInputCompatibilityTests.cs @@ -0,0 +1,90 @@ +// ----------------------------------------------------------------------- +// +// Copyright (C) 2026 - 2026 Petabridge, LLC +// +// ----------------------------------------------------------------------- +using Netclaw.Actors.Protocol; +using Netclaw.Actors.Sessions; +using Netclaw.Configuration; +using Xunit; + +namespace Netclaw.Actors.Tests.Sessions; + +public sealed class ModelInputCompatibilityTests +{ + [Fact] + public void Compatible_modalities_pass() + { + var result = ModelInputCompatibility.Evaluate( + ModelModality.Text | ModelModality.Image, + [MessageWith(MediaModality.Image)]); + + Assert.True(result.IsCompatible); + Assert.Equal(ModelModality.Image, result.RequiredModalities); + Assert.Equal(ModelModality.None, result.UnsupportedModalities); + } + + [Fact] + public void Combined_unsupported_modalities_are_reported() + { + var result = ModelInputCompatibility.Evaluate( + ModelModality.Text | ModelModality.Image, + [MessageWith(MediaModality.Image, MediaModality.Audio, MediaModality.Video)]); + + Assert.False(result.IsCompatible); + Assert.Equal(ModelModality.Audio | ModelModality.Video, result.UnsupportedModalities); + } + + [Fact] + public void Pending_media_and_history_use_one_check() + { + var result = ModelInputCompatibility.Evaluate( + ModelModality.Text | ModelModality.Image, + [MessageWith(MediaModality.Image)], + [Media(MediaModality.Audio)]); + + Assert.False(result.IsCompatible); + Assert.Equal(ModelModality.Image | ModelModality.Audio, result.RequiredModalities); + Assert.Equal(ModelModality.Audio, result.UnsupportedModalities); + } + + [Fact] + public void Tool_message_media_is_checked() + { + var result = ModelInputCompatibility.Evaluate( + ModelModality.Text, + [new SerializableChatMessage + { + Role = ChatRole.Tool, + MediaReferences = [Media(MediaModality.Image)] + }]); + + Assert.False(result.IsCompatible); + Assert.Equal(ModelModality.Image, result.UnsupportedModalities); + } + + [Fact] + public void Unknown_modality_fails_closed() + { + var unknown = Media(MediaModality.Image) with { Modality = 99 }; + + var result = ModelInputCompatibility.Evaluate( + ModelModality.Text | ModelModality.Image | ModelModality.Audio | ModelModality.Video, + [new SerializableChatMessage { MediaReferences = [unknown] }]); + + Assert.False(result.IsCompatible); + Assert.Equal([99], result.UnknownModalityValues); + } + + private static SerializableChatMessage MessageWith(params MediaModality[] modalities) => new() + { + MediaReferences = [.. modalities.Select(Media)] + }; + + private static SerializableMediaReference Media(MediaModality modality) => new() + { + RelativePath = $"{modality}.bin", + MimeType = new Netclaw.Media.MimeType("application/octet-stream"), + Modality = (int)modality + }; +} diff --git a/src/Netclaw.Actors.Tests/Sessions/SessionInputCompatibilityIntegrationTests.cs b/src/Netclaw.Actors.Tests/Sessions/SessionInputCompatibilityIntegrationTests.cs new file mode 100644 index 000000000..05d9661f8 --- /dev/null +++ b/src/Netclaw.Actors.Tests/Sessions/SessionInputCompatibilityIntegrationTests.cs @@ -0,0 +1,167 @@ +// ----------------------------------------------------------------------- +// +// Copyright (C) 2026 - 2026 Petabridge, LLC +// +// ----------------------------------------------------------------------- +using Akka; +using Akka.Actor; +using Akka.Hosting; +using Akka.Persistence; +using Microsoft.Extensions.AI; +using Microsoft.Extensions.DependencyInjection; +using Netclaw.Actors.Hosting; +using Netclaw.Actors.Protocol; +using Netclaw.Actors.Sessions; +using Netclaw.Configuration; +using Xunit; +using static Netclaw.Actors.Sessions.SessionProtocol; + +namespace Netclaw.Actors.Tests.Sessions; + +public sealed class SessionInputCompatibilityIntegrationTests : LlmSessionTestBase +{ + private readonly FakeChatClient _chatClient = new(); + + public SessionInputCompatibilityIntegrationTests(ITestOutputHelper output) : base(output) { } + + protected override void ConfigureSessionServices(IServiceCollection services) + { + services.AddSingleton(new SingleClientProvider(_chatClient)); + services.AddSingleton(new ModelCapabilities + { + ModelId = "text-only-model", + ContextWindowTokens = 128_000, + InputModalities = ModelModality.Text, + }); + services.AddSingleton(new SessionConfig + { + Tuning = new SessionTuning { TitleGenerationInterval = 0 } + }); + services.AddSingleton(new StaticSystemPromptProvider( + "You are a test assistant with tools.")); + } + + [Fact] + public async Task Recovered_image_history_is_rejected_before_model_call() + { + var sessionId = new SessionId("test-channel/recovered-image-compatibility"); + var seeder = Sys.ActorOf(Props.Create(() => new SessionEventSeeder($"session-{sessionId.Value}"))); + await seeder.Ask(new TurnRecorded + { + SessionId = sessionId, + UserMessage = new SerializableChatMessage + { + Role = Netclaw.Actors.Protocol.ChatRole.User, + Content = "Describe this image.", + MediaReferences = [ImageReference("historical.png")] + }, + AssistantReply = new SerializableChatMessage + { + Role = Netclaw.Actors.Protocol.ChatRole.Assistant, + Content = "A prior response." + }, + RecordedAtMs = 1 + }, TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + Watch(seeder); + Sys.Stop(seeder); + await ExpectTerminatedAsync(seeder, cancellationToken: TestContext.Current.CancellationToken); + + var sessionManager = ActorRegistry.Get(); + var subscriber = CreateTestProbe("recovered-image-compatibility-sub"); + var joined = await sessionManager.Ask(new JoinSession(subscriber) + { + SessionId = sessionId, + Filter = OutputFilter.Full + }, TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + await subscriber.ExpectMsgAsync(cancellationToken: TestContext.Current.CancellationToken); + Assert.Equal(1, joined.TurnCount); + + await sessionManager.Ask(new SendUserMessage + { + SessionId = sessionId, + Content = "Continue." + }, TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + + var error = await subscriber.ExpectMsgAsync(cancellationToken: TestContext.Current.CancellationToken); + Assert.Equal(ErrorCategory.InputCompatibility, error.Category); + Assert.Contains("Image", error.Message, StringComparison.Ordinal); + var completed = await subscriber.ExpectMsgAsync(cancellationToken: TestContext.Current.CancellationToken); + Assert.Equal(TurnOutcome.Skipped, completed.Outcome); + Assert.Equal(0, _chatClient.CallCount); + } + + [Fact] + public async Task Buffered_image_is_rejected_before_follow_up_model_call() + { + var responseGate = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + _chatClient.NextResponseGate = responseGate; + + var sessionId = new SessionId("test-channel/buffered-image-compatibility"); + var sessionManager = ActorRegistry.Get(); + var subscriber = CreateTestProbe("buffered-image-compatibility-sub"); + + await sessionManager.Ask(new JoinSession(subscriber) + { + SessionId = sessionId, + Filter = OutputFilter.Full + }, TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + await subscriber.ExpectMsgAsync(cancellationToken: TestContext.Current.CancellationToken); + + await sessionManager.Ask(new SendUserMessage + { + SessionId = sessionId, + Content = "First message." + }, TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + await AwaitAssertAsync( + () => Assert.Equal(1, _chatClient.CallCount), + TimeSpan.FromSeconds(5), + cancellationToken: TestContext.Current.CancellationToken); + + await sessionManager.Ask(new SendUserMessage + { + SessionId = sessionId, + Content = "Describe this buffered image.", + MediaReferences = [ImageReference("buffered.png")] + }, TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + + responseGate.TrySetResult(); + + var error = (ErrorOutput)await subscriber.FishForMessageAsync( + message => message is ErrorOutput, + TimeSpan.FromSeconds(10), + cancellationToken: TestContext.Current.CancellationToken); + Assert.Equal(ErrorCategory.InputCompatibility, error.Category); + + var completed = await subscriber.FishForMessageAsync( + _ => true, + TimeSpan.FromSeconds(5), + cancellationToken: TestContext.Current.CancellationToken); + Assert.Equal(TurnOutcome.Failed, completed.Outcome); + Assert.Equal(1, _chatClient.CallCount); + } + + private static SerializableMediaReference ImageReference(string path) => new() + { + RelativePath = path, + MimeType = new Netclaw.Media.MimeType("image/png"), + Modality = (int)MediaModality.Image + }; + + private sealed class SessionEventSeeder : ReceivePersistentActor + { + public override string PersistenceId { get; } + + public SessionEventSeeder(string persistenceId) + { + PersistenceId = persistenceId; + RecoverAny(_ => { }); + + Command(turn => + { + var replyTo = Sender; + Persist(turn, _ => replyTo.Tell(Done.Instance)); + }); + } + } + +} diff --git a/src/Netclaw.Actors/Sessions/LlmSessionActor.cs b/src/Netclaw.Actors/Sessions/LlmSessionActor.cs index df8557770..ecbb68c7b 100644 --- a/src/Netclaw.Actors/Sessions/LlmSessionActor.cs +++ b/src/Netclaw.Actors/Sessions/LlmSessionActor.cs @@ -2227,6 +2227,9 @@ private void HandleIncomingUserMessage(SendUserMessage cmd) return; } + if (TryRejectIncompatibleInput(cmd.MediaReferences, cmd.Source)) + return; + _inFlightDedup.ReserveReminder(reminderId); _inFlightDedup.ReserveBackgroundJob(bgJobId); @@ -2318,33 +2321,6 @@ private void ContinueIncomingUserMessage(SendUserMessage cmd) _config.Tuning.DiscoveredToolMaxCount, _fullRegistry); - // Strict modality consumer contract: the session actor trusts ingress - // to have routed attachments through its own capability gate. If an - // unsupported modality still reaches here, the originating channel - // skipped the contract in netclaw-input-adapters and that's a bug - // the operator needs to see — surface it loudly and continue. - if (mediaRefs.Count > 0 && !_model.InputModalities.HasFlag(Configuration.ModelModality.Image)) - { - var offendingRefs = mediaRefs.Where(r => r.Modality == (int)MediaModality.Image).ToList(); - if (offendingRefs.Count > 0) - { - var offendingDesc = string.Join(",", - offendingRefs.Select(r => $"{r.RelativePath}:modality={r.Modality}")); - _log.Error( - "ingress_bug model={ModelId} modalities={Modalities} offending={Offending}", - _model.ModelId, _model.InputModalities, offendingDesc); - - mediaRefs = [.. mediaRefs.Where(r => r.Modality != (int)MediaModality.Image)]; - - const string ingressBugNotice = - "[system] An attachment was received but could not be delivered to the model due to an ingress bug. " + - "Please retry, or notify the operator if this persists."; - userContent = string.IsNullOrEmpty(userContent) - ? ingressBugNotice - : userContent + "\n\n" + ingressBugNotice; - } - } - if (TryHandleSlashCommand(executableUserContent, mediaRefs)) return; @@ -2667,6 +2643,17 @@ private static string ShortContentHash(string content) private void FireLlmCall(string? recallQuery = null, bool forceNoTools = false) { + var compatibility = ModelInputCompatibility.Evaluate(_model.InputModalities, _state.History); + if (!compatibility.IsCompatible) + { + var message = ModelInputCompatibility.BuildErrorMessage(_model, compatibility); + FailCurrentTurn( + message, + new InvalidOperationException(message), + ErrorCategory.InputCompatibility); + return; + } + _anyContentStreamed = false; CancelAndDisposeLlmCts(); _activeLlmCts = new CancellationTokenSource(); @@ -2745,6 +2732,50 @@ private void FireLlmCall(string? recallQuery = null, bool forceNoTools = false) ContinueFireLlmCall(forceNoTools); } + private bool TryRejectIncompatibleInput( + IReadOnlyList pendingMedia, + MessageSource? source) + { + var compatibility = ModelInputCompatibility.Evaluate( + _model.InputModalities, + _state.History, + pendingMedia); + if (compatibility.IsCompatible) + return false; + + var message = ModelInputCompatibility.BuildErrorMessage(_model, compatibility); + var cause = new InvalidOperationException(message); + var correlationId = Guid.NewGuid(); + + _log.Error( + cause, + "session_input_incompatible model={ModelId} supported={Supported} required={Required} unsupported={Unsupported} unknown={Unknown} correlationId={CorrelationId}", + _model.ModelId, + _model.InputModalities, + compatibility.RequiredModalities, + compatibility.UnsupportedModalities, + string.Join(",", compatibility.UnknownModalityValues), + correlationId); + + EmitOutput(new ErrorOutput + { + SessionId = _sessionId, + Message = message, + Category = ErrorCategory.InputCompatibility, + CorrelationId = correlationId, + Cause = cause + }); + EmitOutput(new TurnCompleted + { + SessionId = _sessionId, + TurnNumber = new TurnNumber(_state.TurnCount), + Outcome = TurnOutcome.Skipped, + SourceReminderId = source?.ReminderId + }); + TryReplyAck(); + return true; + } + private void ContinueFireLlmCall(bool forceNoTools) { _activeRecall = _recallManager.TurnRecallCache; diff --git a/src/Netclaw.Actors/Sessions/ModelInputCompatibility.cs b/src/Netclaw.Actors/Sessions/ModelInputCompatibility.cs new file mode 100644 index 000000000..2006d4c69 --- /dev/null +++ b/src/Netclaw.Actors/Sessions/ModelInputCompatibility.cs @@ -0,0 +1,82 @@ +// ----------------------------------------------------------------------- +// +// Copyright (C) 2026 - 2026 Petabridge, LLC +// +// ----------------------------------------------------------------------- +using Netclaw.Actors.Protocol; +using Netclaw.Configuration; + +namespace Netclaw.Actors.Sessions; + +internal sealed record ModelInputCompatibilityResult( + ModelModality RequiredModalities, + ModelModality UnsupportedModalities, + IReadOnlyList UnknownModalityValues) +{ + public bool IsCompatible => UnsupportedModalities == ModelModality.None + && UnknownModalityValues.Count == 0; +} + +internal static class ModelInputCompatibility +{ + public static ModelInputCompatibilityResult Evaluate( + ModelModality supportedModalities, + IEnumerable history, + IEnumerable? pendingMedia = null) + { + var required = ModelModality.None; + var unknown = new HashSet(); + + foreach (var message in history) + AddRequirements(message.MediaReferences, ref required, unknown); + + if (pendingMedia is not null) + AddRequirements(pendingMedia, ref required, unknown); + + return new ModelInputCompatibilityResult( + required, + required & ~supportedModalities, + unknown.Order().ToArray()); + } + + public static string BuildErrorMessage( + ModelCapabilities model, + ModelInputCompatibilityResult result) + { + var unsupported = result.UnsupportedModalities == ModelModality.None + ? "none" + : result.UnsupportedModalities.ToString(); + var unknown = result.UnknownModalityValues.Count == 0 + ? "none" + : string.Join(", ", result.UnknownModalityValues); + + return $"The session input is not compatible with model '{model.ModelId}'. " + + $"Unsupported modalities: {unsupported}. Unknown modality values: {unknown}. " + + "Select a model that supports this input, or start a new conversation."; + } + + private static void AddRequirements( + IEnumerable media, + ref ModelModality required, + HashSet unknown) + { + foreach (var reference in media) + { + switch ((MediaModality)reference.Modality) + { + case MediaModality.Image: + required |= ModelModality.Image; + break; + case MediaModality.Audio: + required |= ModelModality.Audio; + break; + case MediaModality.Video: + required |= ModelModality.Video; + break; + default: + unknown.Add(reference.Modality); + break; + } + } + } +} diff --git a/src/Netclaw.Actors/Sessions/SessionProtocol.Outputs.cs b/src/Netclaw.Actors/Sessions/SessionProtocol.Outputs.cs index deff64d8e..8fa0e4612 100644 --- a/src/Netclaw.Actors/Sessions/SessionProtocol.Outputs.cs +++ b/src/Netclaw.Actors/Sessions/SessionProtocol.Outputs.cs @@ -196,7 +196,10 @@ public enum ErrorCategory Timeout, /// Error source is unclassified (e.g. compaction failures). - Unknown + Unknown, + + /// The active model cannot accept the complete session input. + InputCompatibility } /// From fd30964b239fa6af7402afcc0ee18d2cd7e2b394 Mon Sep 17 00:00:00 2001 From: Aaron Stannard Date: Sat, 1 Aug 2026 01:40:59 +0000 Subject: [PATCH 2/6] Report required and supported modalities --- .../Sessions/ModelInputCompatibilityTests.cs | 19 +++++++++++++++++++ .../Sessions/ModelInputCompatibility.cs | 7 +++++++ 2 files changed, 26 insertions(+) diff --git a/src/Netclaw.Actors.Tests/Sessions/ModelInputCompatibilityTests.cs b/src/Netclaw.Actors.Tests/Sessions/ModelInputCompatibilityTests.cs index a6584fba8..b9a06e82e 100644 --- a/src/Netclaw.Actors.Tests/Sessions/ModelInputCompatibilityTests.cs +++ b/src/Netclaw.Actors.Tests/Sessions/ModelInputCompatibilityTests.cs @@ -76,6 +76,25 @@ public void Unknown_modality_fails_closed() Assert.Equal([99], result.UnknownModalityValues); } + [Fact] + public void Error_message_reports_required_and_supported_modalities() + { + var model = new ModelCapabilities + { + ModelId = "text-and-image-model", + InputModalities = ModelModality.Text | ModelModality.Image + }; + var result = ModelInputCompatibility.Evaluate( + model.InputModalities, + [MessageWith(MediaModality.Image, MediaModality.Audio)]); + + var message = ModelInputCompatibility.BuildErrorMessage(model, result); + + Assert.Contains("Required modalities: Image, Audio.", message, StringComparison.Ordinal); + Assert.Contains("Supported modalities: Text, Image.", message, StringComparison.Ordinal); + Assert.Contains("Unsupported modalities: Audio.", message, StringComparison.Ordinal); + } + private static SerializableChatMessage MessageWith(params MediaModality[] modalities) => new() { MediaReferences = [.. modalities.Select(Media)] diff --git a/src/Netclaw.Actors/Sessions/ModelInputCompatibility.cs b/src/Netclaw.Actors/Sessions/ModelInputCompatibility.cs index 2006d4c69..9d74794e9 100644 --- a/src/Netclaw.Actors/Sessions/ModelInputCompatibility.cs +++ b/src/Netclaw.Actors/Sessions/ModelInputCompatibility.cs @@ -43,6 +43,12 @@ public static string BuildErrorMessage( ModelCapabilities model, ModelInputCompatibilityResult result) { + var required = result.RequiredModalities == ModelModality.None + ? "none" + : result.RequiredModalities.ToString(); + var supported = model.InputModalities == ModelModality.None + ? "none" + : model.InputModalities.ToString(); var unsupported = result.UnsupportedModalities == ModelModality.None ? "none" : result.UnsupportedModalities.ToString(); @@ -51,6 +57,7 @@ public static string BuildErrorMessage( : string.Join(", ", result.UnknownModalityValues); return $"The session input is not compatible with model '{model.ModelId}'. " + + $"Required modalities: {required}. Supported modalities: {supported}. " + $"Unsupported modalities: {unsupported}. Unknown modality values: {unknown}. " + "Select a model that supports this input, or start a new conversation."; } From db71e8a570b5d303649d4b240eafdc8c1deb8b5a Mon Sep 17 00:00:00 2001 From: Aaron Stannard Date: Sat, 1 Aug 2026 02:55:10 +0000 Subject: [PATCH 3/6] Degrade incompatible session history instead of rejecting it MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When a session contains historical media the current model cannot accept (e.g. images from a multimodal model in a text-only session), degrade gracefully: - ChatMessageConverter strips incompatible DataContent at assembly time without touching persisted MediaReferences. - SessionMessageAssembler injects a volatile [system: media-filtered] notice so the user knows content was omitted. - FireLlmCall logs a warning instead of failing the turn. - TryRejectIncompatibleInput only hard-rejects new user-supplied media (defense-in-depth; the adapter already handles this). The session stays usable. The assembler boundary is the single correct place to filter — after history reconstruction, before the provider. Tests: 11/11 pass (compatibility unit tests + modality gate tests + integration tests for recovered history and buffered drain paths). --- ...ssionInputCompatibilityIntegrationTests.cs | 50 +++++++--------- .../Protocol/ChatMessageConverter.cs | 60 +++++++++++++++++-- .../Sessions/LlmSessionActor.cs | 34 ++++++++--- .../Sessions/SessionMessageAssembler.cs | 43 ++++++++++++- 4 files changed, 145 insertions(+), 42 deletions(-) diff --git a/src/Netclaw.Actors.Tests/Sessions/SessionInputCompatibilityIntegrationTests.cs b/src/Netclaw.Actors.Tests/Sessions/SessionInputCompatibilityIntegrationTests.cs index 05d9661f8..63b856c7d 100644 --- a/src/Netclaw.Actors.Tests/Sessions/SessionInputCompatibilityIntegrationTests.cs +++ b/src/Netclaw.Actors.Tests/Sessions/SessionInputCompatibilityIntegrationTests.cs @@ -42,7 +42,7 @@ protected override void ConfigureSessionServices(IServiceCollection services) } [Fact] - public async Task Recovered_image_history_is_rejected_before_model_call() + public async Task Recovered_image_history_is_degraded_not_rejected() { var sessionId = new SessionId("test-channel/recovered-image-compatibility"); var seeder = Sys.ActorOf(Props.Create(() => new SessionEventSeeder($"session-{sessionId.Value}"))); @@ -82,20 +82,20 @@ await sessionManager.Ask(new SendUserMessage Content = "Continue." }, TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); - var error = await subscriber.ExpectMsgAsync(cancellationToken: TestContext.Current.CancellationToken); - Assert.Equal(ErrorCategory.InputCompatibility, error.Category); - Assert.Contains("Image", error.Message, StringComparison.Ordinal); - var completed = await subscriber.ExpectMsgAsync(cancellationToken: TestContext.Current.CancellationToken); - Assert.Equal(TurnOutcome.Skipped, completed.Outcome); - Assert.Equal(0, _chatClient.CallCount); + // The session should proceed normally — the historical image is stripped + // at assembly time, not rejected. The model is called with text-only content. + var completed = await subscriber.FishForMessageAsync( + _ => true, + TimeSpan.FromSeconds(10), + cancellationToken: TestContext.Current.CancellationToken); + Assert.NotEqual(TurnOutcome.Skipped, completed.Outcome); + Assert.NotEqual(TurnOutcome.Failed, completed.Outcome); + Assert.True(_chatClient.CallCount >= 1); } [Fact] - public async Task Buffered_image_is_rejected_before_follow_up_model_call() + public async Task Buffered_image_is_degraded_not_rejected() { - var responseGate = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); - _chatClient.NextResponseGate = responseGate; - var sessionId = new SessionId("test-channel/buffered-image-compatibility"); var sessionManager = ActorRegistry.Get(); var subscriber = CreateTestProbe("buffered-image-compatibility-sub"); @@ -107,15 +107,13 @@ await sessionManager.Ask(new JoinSession(subscriber) }, TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); await subscriber.ExpectMsgAsync(cancellationToken: TestContext.Current.CancellationToken); + // Send two messages back-to-back. The second one carries an image + // that the text-only model will strip at assembly time. await sessionManager.Ask(new SendUserMessage { SessionId = sessionId, Content = "First message." }, TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); - await AwaitAssertAsync( - () => Assert.Equal(1, _chatClient.CallCount), - TimeSpan.FromSeconds(5), - cancellationToken: TestContext.Current.CancellationToken); await sessionManager.Ask(new SendUserMessage { @@ -124,20 +122,16 @@ await sessionManager.Ask(new SendUserMessage MediaReferences = [ImageReference("buffered.png")] }, TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); - responseGate.TrySetResult(); - - var error = (ErrorOutput)await subscriber.FishForMessageAsync( - message => message is ErrorOutput, - TimeSpan.FromSeconds(10), - cancellationToken: TestContext.Current.CancellationToken); - Assert.Equal(ErrorCategory.InputCompatibility, error.Category); + // Collect both TurnCompleted events. Neither turn should fail — + // the buffered image is stripped at assembly time. + for (var i = 0; i < 2; i++) + { + var completed = await subscriber.FishForMessageAsync( + _ => true, TimeSpan.FromSeconds(10), cancellationToken: TestContext.Current.CancellationToken); + Assert.NotEqual(TurnOutcome.Failed, completed.Outcome); + } - var completed = await subscriber.FishForMessageAsync( - _ => true, - TimeSpan.FromSeconds(5), - cancellationToken: TestContext.Current.CancellationToken); - Assert.Equal(TurnOutcome.Failed, completed.Outcome); - Assert.Equal(1, _chatClient.CallCount); + Assert.Equal(2, _chatClient.CallCount); } private static SerializableMediaReference ImageReference(string path) => new() diff --git a/src/Netclaw.Actors/Protocol/ChatMessageConverter.cs b/src/Netclaw.Actors/Protocol/ChatMessageConverter.cs index 9443bd96a..2ec8fd940 100644 --- a/src/Netclaw.Actors/Protocol/ChatMessageConverter.cs +++ b/src/Netclaw.Actors/Protocol/ChatMessageConverter.cs @@ -1,4 +1,4 @@ -// ----------------------------------------------------------------------- +// ----------------------------------------------------------------------- // // Copyright (C) 2026 - 2026 Petabridge, LLC // @@ -6,6 +6,7 @@ using System.Text.Json; using Microsoft.Extensions.AI; using Microsoft.Extensions.Logging; +using Netclaw.Configuration; using Netclaw.Media; using Netclaw.Tools; using AiChatMessage = Microsoft.Extensions.AI.ChatMessage; @@ -42,12 +43,18 @@ public static class ChatMessageConverter /// rationale instead of silently falling back to defaults. Must stay false for /// outbound provider history — the model must never receive meta keys. /// + /// + /// When set, media references whose modality is not in this mask are dropped + /// from the wire DataContent list. Persisted + /// are not changed. + /// public static AiChatMessage ToAiMessage( SerializableChatMessage msg, string? sessionDir = null, ILogger? logger = null, Func? toolNameResolver = null, - bool reinjectMeta = false) + bool reinjectMeta = false, + ModelModality supportedModalities = ModelModality.Text | ModelModality.Image | ModelModality.Audio | ModelModality.Video) { var role = msg.Role switch { @@ -113,6 +120,15 @@ public static AiChatMessage ToAiMessage( foreach (var media in msg.MediaReferences) { + if (!AcceptsModality(supportedModalities, (MediaModality)media.Modality)) + { + logger?.LogDebug( + "Skipping media reference modality={Modality} unsupported by model capabilities={Supported}", + (MediaModality)media.Modality, + supportedModalities); + continue; + } + var fullPath = SessionMediaStore.GetMediaPath(sessionDir, media.RelativePath); if (!File.Exists(fullPath)) { @@ -131,15 +147,51 @@ public static AiChatMessage ToAiMessage( return new AiChatMessage(role, msg.Content); } + /// + /// Convert a sequence of persisted messages to MEAI messages. When + /// excludes a modality present in + /// a message's media references, that DataContent is silently + /// dropped from the wire representation while the persisted + /// remain untouched. + /// public static List ToAiMessages( IEnumerable messages, string? sessionDir = null, ILogger? logger = null, - Func? toolNameResolver = null) + Func? toolNameResolver = null, + ModelModality supportedModalities = ModelModality.Text | ModelModality.Image | ModelModality.Audio | ModelModality.Video) { - return [.. messages.Select(m => ToAiMessage(m, sessionDir, logger, toolNameResolver))]; + return [.. messages.Select(m => ToAiMessage(m, sessionDir, logger, toolNameResolver, false, supportedModalities))]; } + /// + /// Count how many media references across would + /// be stripped given . + /// + public static int CountStrippedMedia( + IEnumerable messages, + ModelModality supportedModalities) + { + var count = 0; + foreach (var m in messages) + { + foreach (var media in m.MediaReferences) + { + if (!AcceptsModality(supportedModalities, (MediaModality)media.Modality)) + count++; + } + } + return count; + } + + private static bool AcceptsModality(ModelModality supported, MediaModality media) => media switch + { + MediaModality.Image => (supported & ModelModality.Image) != 0, + MediaModality.Audio => (supported & ModelModality.Audio) != 0, + MediaModality.Video => (supported & ModelModality.Video) != 0, + _ => false // unknown modality → not accepted + }; + /// /// Optional schema-aware interpreter (the executor's PrepareToolCall) used to /// extract meta + strip meta keys per tool call. When supplied, persisted history diff --git a/src/Netclaw.Actors/Sessions/LlmSessionActor.cs b/src/Netclaw.Actors/Sessions/LlmSessionActor.cs index ecbb68c7b..2420491d4 100644 --- a/src/Netclaw.Actors/Sessions/LlmSessionActor.cs +++ b/src/Netclaw.Actors/Sessions/LlmSessionActor.cs @@ -2646,12 +2646,16 @@ private void FireLlmCall(string? recallQuery = null, bool forceNoTools = false) var compatibility = ModelInputCompatibility.Evaluate(_model.InputModalities, _state.History); if (!compatibility.IsCompatible) { - var message = ModelInputCompatibility.BuildErrorMessage(_model, compatibility); - FailCurrentTurn( - message, - new InvalidOperationException(message), - ErrorCategory.InputCompatibility); - return; + TurnLog().Warning( + "turn_media_history_incompatible required={Required} unsupported={Unsupported} unknownCount={UnknownCount} " + + "model={ModelId} — incompatible media references will be stripped from wire messages by the assembler", + compatibility.RequiredModalities, + compatibility.UnsupportedModalities, + compatibility.UnknownModalityValues.Count, + _model.ModelId); + // Do not fail the turn — ChatMessageConverter.ToAiMessages strips + // incompatible DataContent at assembly time, and the assembler + // injects a volatile system notice. The session stays usable. } _anyContentStreamed = false; @@ -2743,6 +2747,21 @@ private bool TryRejectIncompatibleInput( if (compatibility.IsCompatible) return false; + // History-only incompatibility: debug log and let the assembler strip + // incompatible media at wire time. Only new user-supplied media on this + // specific command triggers a hard rejection. + if (pendingMedia.Count == 0) + { + TurnLog().Info( + "session_history_media_stripped model={ModelId} required={Required} unsupported={Unsupported} unknown={Unknown} " + + "— historical media references are incompatible with the current model and will be stripped by the assembler", + _model.ModelId, + compatibility.RequiredModalities, + compatibility.UnsupportedModalities, + string.Join(",", compatibility.UnknownModalityValues)); + return false; + } + var message = ModelInputCompatibility.BuildErrorMessage(_model, compatibility); var cause = new InvalidOperationException(message); var correlationId = Guid.NewGuid(); @@ -2807,7 +2826,8 @@ private void ContinueFireLlmCall(bool forceNoTools) SkillHint: skillHint, // Canonical names live in history (post-PR follow-up); the // LLM provider wants the sanitized alias back on the wire. - ToolNameToLlmFacing: _toolRegistry is null ? null : _toolRegistry.ToLlmFacingName)); + ToolNameToLlmFacing: _toolRegistry is null ? null : _toolRegistry.ToLlmFacingName, + SupportedInputModalities: _model.InputModalities)); _startupContextInjected = true; var self = Self; diff --git a/src/Netclaw.Actors/Sessions/SessionMessageAssembler.cs b/src/Netclaw.Actors/Sessions/SessionMessageAssembler.cs index 749411716..3567b2d01 100644 --- a/src/Netclaw.Actors/Sessions/SessionMessageAssembler.cs +++ b/src/Netclaw.Actors/Sessions/SessionMessageAssembler.cs @@ -1,4 +1,4 @@ -// ----------------------------------------------------------------------- +// ----------------------------------------------------------------------- // // Copyright (C) 2026 - 2026 Petabridge, LLC // @@ -35,7 +35,11 @@ public sealed record ContextAssemblyInput( // MCP tool calls, the LLM provider will return 400. Production // callers should pass `toolRegistry.ToLlmFacingName`; unit tests // that don't exercise MCP can leave it null. - Func? ToolNameToLlmFacing = null); + Func? ToolNameToLlmFacing = null, + // When set, media references whose modality is not supported by the + // active model are dropped from the wire message list. The assembler + // injects a volatile system notice when any media is stripped. + ModelModality SupportedInputModalities = ModelModality.Text | ModelModality.Image | ModelModality.Audio | ModelModality.Video); /// /// Pure-function assembly of the list sent to @@ -115,7 +119,19 @@ public static List Assemble(ContextAssemblyInput input) var messages = ChatMessageConverter.ToAiMessages( input.State.History, sessionDir, - toolNameResolver: input.ToolNameToLlmFacing); + toolNameResolver: input.ToolNameToLlmFacing, + supportedModalities: input.SupportedInputModalities); + + // Inject volatile notice when media was stripped from history + var strippedCount = ChatMessageConverter.CountStrippedMedia( + input.State.History, input.SupportedInputModalities); + if (strippedCount > 0) + { + var notice = BuildMediaStrippedNotice(strippedCount, input.SupportedInputModalities); + messages.Insert(0, new AiChatMessage( + Microsoft.Extensions.AI.ChatRole.User, + notice)); + } var staticBlock = BuildStaticContextBlock(input, sessionDir); if (!string.IsNullOrEmpty(staticBlock)) @@ -283,4 +299,25 @@ private static string FormatRecallForLlm(AutomaticRecallResult recall) } return sb.ToString().TrimEnd(); } + + /// + /// Build a volatile (non-persisted) system notice when media references + /// were stripped from the wire message list because the active model does + /// not support their modality. + /// + internal static string BuildMediaStrippedNotice(int strippedCount, ModelModality supported) + { + var required = string.Empty; + if ((supported & ModelModality.Image) == 0) + required = "image"; + if ((supported & ModelModality.Audio) == 0) + required = (required.Length > 0 ? required + ", " : "") + "audio"; + if ((supported & ModelModality.Video) == 0) + required = (required.Length > 0 ? required + ", " : "") + "video"; + + return $"[system: media-filtered] {strippedCount} media reference(s) from earlier in this " + + $"conversation were omitted because the current model does not support " + + $"{(required.Length > 0 ? required : "this")} input. " + + "Switch to a multimodal model in your Netclaw configuration to view them."; + } } From 60d54edb531ade29a13a21676bdabea4a492a2fd Mon Sep 17 00:00:00 2001 From: Aaron Stannard Date: Sat, 1 Aug 2026 03:09:39 +0000 Subject: [PATCH 4/6] Fix flaky buffered-image test with response gate Without the gate, the first FakeChatClient response completes instantly and the second turn's drain + model call races against the FishForMessageAsync timeout. Gating the first response ensures both TurnCompleted events arrive deterministically. --- .../SessionInputCompatibilityIntegrationTests.cs | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/src/Netclaw.Actors.Tests/Sessions/SessionInputCompatibilityIntegrationTests.cs b/src/Netclaw.Actors.Tests/Sessions/SessionInputCompatibilityIntegrationTests.cs index 63b856c7d..0ce6edb97 100644 --- a/src/Netclaw.Actors.Tests/Sessions/SessionInputCompatibilityIntegrationTests.cs +++ b/src/Netclaw.Actors.Tests/Sessions/SessionInputCompatibilityIntegrationTests.cs @@ -96,6 +96,12 @@ await sessionManager.Ask(new SendUserMessage [Fact] public async Task Buffered_image_is_degraded_not_rejected() { + // Gate the first model response so the buffered second message + // drains after we release the gate — both TurnCompleted events + // arrive deterministically rather than racing. + var responseGate = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + _chatClient.NextResponseGate = responseGate; + var sessionId = new SessionId("test-channel/buffered-image-compatibility"); var sessionManager = ActorRegistry.Get(); var subscriber = CreateTestProbe("buffered-image-compatibility-sub"); @@ -107,8 +113,6 @@ await sessionManager.Ask(new JoinSession(subscriber) }, TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); await subscriber.ExpectMsgAsync(cancellationToken: TestContext.Current.CancellationToken); - // Send two messages back-to-back. The second one carries an image - // that the text-only model will strip at assembly time. await sessionManager.Ask(new SendUserMessage { SessionId = sessionId, @@ -122,8 +126,10 @@ await sessionManager.Ask(new SendUserMessage MediaReferences = [ImageReference("buffered.png")] }, TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); - // Collect both TurnCompleted events. Neither turn should fail — - // the buffered image is stripped at assembly time. + // Release the gate — first turn completes, drain fires, second call + // proceeds with the image stripped at assembly time. + responseGate.TrySetResult(); + for (var i = 0; i < 2; i++) { var completed = await subscriber.FishForMessageAsync( From 8eabf0d52d0df7270d211f067e0f693ee8b25f46 Mon Sep 17 00:00:00 2001 From: Aaron Stannard Date: Sat, 1 Aug 2026 03:27:47 +0000 Subject: [PATCH 5/6] Restore ingress-bug modality guard MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This guard was removed in the original PR but should have been kept. It catches media that slipped past the adapter's capability gate (a contract violation in netclaw-input-adapters), strips the offending references, logs an operator-visible error, and appends a system notice so the user knows something went wrong. This is the new-message counterpart to the assembly-time history filter added in the previous commit — both degrade instead of rejecting. --- .../Sessions/LlmSessionActor.cs | 27 +++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/src/Netclaw.Actors/Sessions/LlmSessionActor.cs b/src/Netclaw.Actors/Sessions/LlmSessionActor.cs index 2420491d4..052b4d4e4 100644 --- a/src/Netclaw.Actors/Sessions/LlmSessionActor.cs +++ b/src/Netclaw.Actors/Sessions/LlmSessionActor.cs @@ -2321,6 +2321,33 @@ private void ContinueIncomingUserMessage(SendUserMessage cmd) _config.Tuning.DiscoveredToolMaxCount, _fullRegistry); + // Strict modality consumer contract: the session actor trusts ingress + // to have routed attachments through its own capability gate. If an + // unsupported modality still reaches here, the originating channel + // skipped the contract in netclaw-input-adapters and that is a bug + // the operator needs to see — surface it loudly and continue. + if (mediaRefs.Count > 0 && !_model.InputModalities.HasFlag(Configuration.ModelModality.Image)) + { + var offendingRefs = mediaRefs.Where(r => r.Modality == (int)MediaModality.Image).ToList(); + if (offendingRefs.Count > 0) + { + var offendingDesc = string.Join(",", + offendingRefs.Select(r => $"{r.RelativePath}:modality={r.Modality}")); + _log.Error( + "ingress_bug model={ModelId} modalities={Modalities} offending={Offending}", + _model.ModelId, _model.InputModalities, offendingDesc); + + mediaRefs = [.. mediaRefs.Where(r => r.Modality != (int)MediaModality.Image)]; + + const string ingressBugNotice = + "[system] An attachment was received but could not be delivered to the model due to an ingress bug. " + + "Please retry, or notify the operator if this persists."; + userContent = string.IsNullOrEmpty(userContent) + ? ingressBugNotice + : userContent + "\n\n" + ingressBugNotice; + } + } + if (TryHandleSlashCommand(executableUserContent, mediaRefs)) return; From abbc622955cb86b05e7a797bab6b7f20c5670823 Mon Sep 17 00:00:00 2001 From: Aaron Stannard Date: Sat, 1 Aug 2026 03:29:23 +0000 Subject: [PATCH 6/6] Revert "Restore ingress-bug modality guard" This reverts commit 8921afa0a604e40d12d67b8f8f73c30d435f7f63. --- .../Sessions/LlmSessionActor.cs | 27 ------------------- 1 file changed, 27 deletions(-) diff --git a/src/Netclaw.Actors/Sessions/LlmSessionActor.cs b/src/Netclaw.Actors/Sessions/LlmSessionActor.cs index 052b4d4e4..2420491d4 100644 --- a/src/Netclaw.Actors/Sessions/LlmSessionActor.cs +++ b/src/Netclaw.Actors/Sessions/LlmSessionActor.cs @@ -2321,33 +2321,6 @@ private void ContinueIncomingUserMessage(SendUserMessage cmd) _config.Tuning.DiscoveredToolMaxCount, _fullRegistry); - // Strict modality consumer contract: the session actor trusts ingress - // to have routed attachments through its own capability gate. If an - // unsupported modality still reaches here, the originating channel - // skipped the contract in netclaw-input-adapters and that is a bug - // the operator needs to see — surface it loudly and continue. - if (mediaRefs.Count > 0 && !_model.InputModalities.HasFlag(Configuration.ModelModality.Image)) - { - var offendingRefs = mediaRefs.Where(r => r.Modality == (int)MediaModality.Image).ToList(); - if (offendingRefs.Count > 0) - { - var offendingDesc = string.Join(",", - offendingRefs.Select(r => $"{r.RelativePath}:modality={r.Modality}")); - _log.Error( - "ingress_bug model={ModelId} modalities={Modalities} offending={Offending}", - _model.ModelId, _model.InputModalities, offendingDesc); - - mediaRefs = [.. mediaRefs.Where(r => r.Modality != (int)MediaModality.Image)]; - - const string ingressBugNotice = - "[system] An attachment was received but could not be delivered to the model due to an ingress bug. " + - "Please retry, or notify the operator if this persists."; - userContent = string.IsNullOrEmpty(userContent) - ? ingressBugNotice - : userContent + "\n\n" + ingressBugNotice; - } - } - if (TryHandleSlashCommand(executableUserContent, mediaRefs)) return;