From d4c27aaaaeb5582ecd3e72d3247b9536beaf3260 Mon Sep 17 00:00:00 2001 From: "@daniel-lxs" <57051444+daniel-lxs@users.noreply.github.com> Date: Sat, 12 Sep 2026 05:13:43 +0000 Subject: [PATCH 1/2] fix: retain Telegram first-turn attachments --- .../handlers/telegram/__tests__/index.test.ts | 216 ++++++++++++++++++ apps/api/src/handlers/telegram/attachments.ts | 71 ++++-- apps/api/src/handlers/telegram/index.ts | 6 + apps/api/src/handlers/telegram/types.ts | 1 + .../providers/communications/telegram.mdx | 9 +- .../lib/fast-agent-parent-event.test.ts | 2 + .../src/server/lib/fast-agent-parent-event.ts | 3 + .../lib/fast-agent-surface-reply.test.ts | 8 + .../server/lib/fast-agent-surface-reply.ts | 5 + packages/types/src/fast-agent.ts | 1 + 10 files changed, 304 insertions(+), 18 deletions(-) diff --git a/apps/api/src/handlers/telegram/__tests__/index.test.ts b/apps/api/src/handlers/telegram/__tests__/index.test.ts index bcd724c8d2..8a11895b9a 100644 --- a/apps/api/src/handlers/telegram/__tests__/index.test.ts +++ b/apps/api/src/handlers/telegram/__tests__/index.test.ts @@ -8,11 +8,13 @@ const { taskRunsFindFirstMock, consumeLinkCodeMock, createForumTopicMock, + describeVideoAttachmentMock, downloadFileMock, restoreLinkCodeMock, editMessageReplyMarkupMock, editMessageTextMock, enqueueTaskMock, + extractPromptTextAttachmentsMock, environmentsFindFirstMock, envMock, getAvailableEnvironmentsMock, @@ -43,6 +45,7 @@ const { getFastSessionMock, isFastProviderMessageMock, recordFastConversationMessageMock, + transcribeAudioAttachmentMock, } = vi.hoisted(() => ({ addReactionMock: vi.fn(), answerCallbackQueryMock: vi.fn(), @@ -50,11 +53,13 @@ const { taskRunsFindFirstMock: vi.fn(), consumeLinkCodeMock: vi.fn(), createForumTopicMock: vi.fn(), + describeVideoAttachmentMock: vi.fn(), downloadFileMock: vi.fn(), restoreLinkCodeMock: vi.fn(), editMessageReplyMarkupMock: vi.fn(), editMessageTextMock: vi.fn(), enqueueTaskMock: vi.fn(), + extractPromptTextAttachmentsMock: vi.fn(), environmentsFindFirstMock: vi.fn(), getAvailableEnvironmentsMock: vi.fn(), getBotInfoMock: vi.fn(), @@ -90,6 +95,7 @@ const { getFastSessionMock: vi.fn(), isFastProviderMessageMock: vi.fn(), recordFastConversationMessageMock: vi.fn(), + transcribeAudioAttachmentMock: vi.fn(), })); vi.mock('@roomote/env', () => ({ @@ -308,14 +314,31 @@ vi.mock('../../tasks/task-stop.js', () => ({ })); vi.mock('@roomote/cloud-agents/server', () => ({ + AUDIO_TRANSCRIPTION_MAX_SIZE_BYTES: 20 * 1024 * 1024, buildFastAgentReactionExternalInputQuestion: vi.fn( (input: unknown) => `${JSON.stringify(input)}`, ), + describeVideoAttachment: describeVideoAttachmentMock, enqueueTask: enqueueTaskMock, + extractPromptTextAttachments: extractPromptTextAttachmentsMock, + formatAudioAttachmentWarning: (filename: string, reason: string) => + `[Audio attachment ${filename} ${reason}.]`, + formatAudioTranscriptionResult: ( + filename: string, + result: { transcript?: string }, + ) => `Audio attachment transcript: ${filename}\n${result.transcript ?? ''}`, getAvailableEnvironments: getAvailableEnvironmentsMock, getTaskUrl: getTaskUrlMock, getOrCreateFastAgentSession: getFastSessionMock, + isVideoAgentSupportedMimeType: (mimeType: string) => + ['video/mp4', 'video/quicktime', 'video/webm', 'video/mpeg'].includes( + mimeType, + ), + resolveAudioTranscriptionMimeType: ({ mimeType }: { mimeType?: string }) => + mimeType?.startsWith('audio/') ? mimeType : null, + transcribeAudioAttachment: transcribeAudioAttachmentMock, + VIDEO_AGENT_MAX_VIDEO_SIZE_BYTES: 20 * 1024 * 1024, })); import { telegram } from '../index'; @@ -392,6 +415,15 @@ describe('Telegram webhook handler', () => { filePath: 'photos/example.jpg', contentType: 'image/jpeg', }); + describeVideoAttachmentMock.mockResolvedValue('The video shows an error.'); + extractPromptTextAttachmentsMock.mockResolvedValue({ + attachmentTexts: ['Attachment: notes.txt\nDeployment failed.'], + warnings: [], + }); + transcribeAudioAttachmentMock.mockResolvedValue({ + status: 'transcribed', + transcript: 'Run the focused tests.', + }); taskRunsFindFirstMock.mockReset(); telegramMappingsFindFirstMock.mockReset(); consumeLinkCodeMock.mockReset(); @@ -851,6 +883,190 @@ describe('Telegram webhook handler', () => { expect(enqueueTaskMock).not.toHaveBeenCalled(); }); + it('passes Telegram image documents to a new Fast session as images', async () => { + mockTelegramLinkedSender('mapped-user-1'); + + const response = await postTelegramUpdate( + createTelegramUpdate({ + message: { + text: undefined, + caption: 'Inspect the uncompressed screenshot', + document: { + file_id: 'screenshot-file', + file_unique_id: 'screenshot-1', + file_name: 'failure.png', + mime_type: 'image/png', + }, + }, + }), + ); + + await expect(response.json()).resolves.toMatchObject({ + fastAnswered: true, + fastDefaulted: true, + }); + expect(continueFastReplyMock).toHaveBeenCalledWith( + expect.objectContaining({ + question: 'Inspect the uncompressed screenshot', + images: ['data:image/jpeg;base64,AQID'], + }), + ); + }); + + it('passes extracted Telegram documents to a new Fast session', async () => { + mockTelegramLinkedSender('mapped-user-1'); + downloadFileMock.mockResolvedValueOnce({ + bytes: new TextEncoder().encode('Deployment failed.'), + filePath: 'documents/notes.txt', + contentType: 'text/plain', + }); + + const response = await postTelegramUpdate( + createTelegramUpdate({ + message: { + text: undefined, + caption: 'Diagnose this log', + document: { + file_id: 'document-file', + file_unique_id: 'document-1', + file_name: 'notes.txt', + mime_type: 'text/plain', + }, + }, + }), + ); + + await expect(response.json()).resolves.toMatchObject({ + fastAnswered: true, + fastDefaulted: true, + }); + expect(continueFastReplyMock).toHaveBeenCalledWith( + expect.objectContaining({ + question: + 'Diagnose this log\n\nAttachment: notes.txt\nDeployment failed.', + attachmentTexts: ['Attachment: notes.txt\nDeployment failed.'], + }), + ); + }); + + it.each([ + [ + 'audio', + { + audio: { + file_id: 'audio-file', + file_unique_id: 'audio-1', + duration: 3, + file_name: 'request.mp3', + mime_type: 'audio/mpeg', + }, + }, + ], + [ + 'voice note', + { + voice: { + file_id: 'voice-file', + file_unique_id: 'voice-1', + duration: 3, + mime_type: 'audio/ogg', + }, + }, + ], + ])( + 'passes transcribed Telegram %s to a new Fast session', + async (_, message) => { + mockTelegramLinkedSender('mapped-user-1'); + + const response = await postTelegramUpdate( + createTelegramUpdate({ message: { text: undefined, ...message } }), + ); + + await expect(response.json()).resolves.toMatchObject({ + fastAnswered: true, + fastDefaulted: true, + }); + expect(continueFastReplyMock).toHaveBeenCalledWith( + expect.objectContaining({ + attachmentTexts: [expect.stringContaining('Run the focused tests.')], + }), + ); + }, + ); + + it('passes bounded Telegram video descriptions to a new Fast session', async () => { + mockTelegramLinkedSender('mapped-user-1'); + downloadFileMock.mockResolvedValueOnce({ + bytes: new Uint8Array([1, 2, 3]), + filePath: 'videos/repro.mp4', + contentType: 'video/mp4', + }); + + const response = await postTelegramUpdate( + createTelegramUpdate({ + message: { + text: undefined, + caption: 'Review this recording', + document: { + file_id: 'video-file', + file_unique_id: 'video-1', + file_name: 'repro.mp4', + mime_type: 'video/mp4', + }, + }, + }), + ); + + await expect(response.json()).resolves.toMatchObject({ + fastAnswered: true, + fastDefaulted: true, + }); + expect(downloadFileMock).toHaveBeenCalledWith( + 'video-file', + 20 * 1024 * 1024, + ); + expect(continueFastReplyMock).toHaveBeenCalledWith( + expect.objectContaining({ + question: expect.stringContaining('The video shows an error.'), + attachmentTexts: [ + 'Video attachment description: repro.mp4\nThe video shows an error.', + ], + }), + ); + }); + + it('keeps unsupported Telegram documents out of new Fast attachment context', async () => { + mockTelegramLinkedSender('mapped-user-1'); + + const response = await postTelegramUpdate( + createTelegramUpdate({ + message: { + text: undefined, + caption: 'Use this file', + document: { + file_id: 'archive-file', + file_unique_id: 'archive-1', + file_name: 'bundle.zip', + mime_type: 'application/zip', + }, + }, + }), + ); + + await expect(response.json()).resolves.toMatchObject({ + fastAnswered: true, + fastDefaulted: true, + }); + expect(downloadFileMock).not.toHaveBeenCalled(); + expect(continueFastReplyMock).toHaveBeenCalledWith({ + sessionId: 'fast-session-default', + userId: 'mapped-user-1', + senderDisplayName: 'Ada Lovelace', + question: 'Use this file', + currentMessageId: '456', + }); + }); + it('uses a user-scoped Fast session for a Telegram group topic mention', async () => { mockTelegramLinkedSender('mapped-user-1'); getFastSessionMock.mockResolvedValueOnce({ diff --git a/apps/api/src/handlers/telegram/attachments.ts b/apps/api/src/handlers/telegram/attachments.ts index 1bc8b33992..0d27b37951 100644 --- a/apps/api/src/handlers/telegram/attachments.ts +++ b/apps/api/src/handlers/telegram/attachments.ts @@ -1,14 +1,18 @@ import { appendAttachmentTextsToPromptText, + isRoomoteImageAttachment, isRoomoteTextExtractableAttachment, } from '@roomote/cloud-agents'; import { AUDIO_TRANSCRIPTION_MAX_SIZE_BYTES, + describeVideoAttachment, extractPromptTextAttachments, formatAudioAttachmentWarning, formatAudioTranscriptionResult, + isVideoAgentSupportedMimeType, resolveAudioTranscriptionMimeType, transcribeAudioAttachment, + VIDEO_AGENT_MAX_VIDEO_SIZE_BYTES, } from '@roomote/cloud-agents/server'; import { TelegramCommunicationProvider } from '@roomote/communication/telegram-provider'; import type { TelegramMessage } from '@roomote/communication/telegram-update'; @@ -63,27 +67,65 @@ export async function attachTelegramMediaToQueuedMessage(input: { } const document = input.message.document; - if ( + const documentMimeType = document?.mime_type?.trim().toLowerCase(); + const documentIsImage = Boolean( + document && + isRoomoteImageAttachment({ + filename: document.file_name, + mimeType: document.mime_type, + }), + ); + const documentIsText = Boolean( document && isRoomoteTextExtractableAttachment({ filename: document.file_name, mimeType: document.mime_type, - }) - ) { + }), + ); + const documentIsVideo = Boolean( + documentMimeType && isVideoAgentSupportedMimeType(documentMimeType), + ); + if (document && (documentIsImage || documentIsText || documentIsVideo)) { const downloaded = await provider.downloadFile( document.file_id, - MAX_DOCUMENT_BYTES, + documentIsImage + ? MAX_IMAGE_BYTES + : documentIsVideo + ? VIDEO_AGENT_MAX_VIDEO_SIZE_BYTES + : MAX_DOCUMENT_BYTES, ); - const extracted = await extractPromptTextAttachments([ - { - filename: document.file_name ?? downloaded.filePath, - mimeType: document.mime_type ?? downloaded.contentType ?? undefined, - bytes: downloaded.bytes, - }, - ]); - attachmentTexts.push(...extracted.attachmentTexts); - for (const warning of extracted.warnings) { - console.warn(`[telegram] Attachment extraction warning: ${warning}`); + if (documentIsImage) { + const downloadedMimeType = downloaded.contentType?.split(';')[0]; + const mimeType = downloadedMimeType?.startsWith('image/') + ? downloadedMimeType + : (documentMimeType ?? 'image/png'); + images.push( + `data:${mimeType};base64,${Buffer.from(downloaded.bytes).toString('base64')}`, + ); + } else if (documentIsVideo && documentMimeType) { + const description = await describeVideoAttachment({ + videoBytes: Buffer.from(downloaded.bytes), + mimeType: documentMimeType, + userId: input.queuedMessage.userId, + userTextContext: input.queuedMessage.text, + }); + if (description) { + attachmentTexts.push( + `Video attachment description${document.file_name ? `: ${document.file_name}` : ''}\n${description}`, + ); + } + } else { + const extracted = await extractPromptTextAttachments([ + { + filename: document.file_name ?? downloaded.filePath, + mimeType: document.mime_type ?? downloaded.contentType ?? undefined, + bytes: downloaded.bytes, + }, + ]); + attachmentTexts.push(...extracted.attachmentTexts); + for (const warning of extracted.warnings) { + console.warn(`[telegram] Attachment extraction warning: ${warning}`); + } } } } catch (error) { @@ -149,5 +191,6 @@ export async function attachTelegramMediaToQueuedMessage(input: { attachmentTexts, }), ...(images.length ? { images } : {}), + ...(attachmentTexts.length ? { attachmentTexts } : {}), }; } diff --git a/apps/api/src/handlers/telegram/index.ts b/apps/api/src/handlers/telegram/index.ts index 70db20e332..9123e2f3e1 100644 --- a/apps/api/src/handlers/telegram/index.ts +++ b/apps/api/src/handlers/telegram/index.ts @@ -629,6 +629,9 @@ telegram.post('/', async (c) => { ? { agentContext: fastMessage.agentContext } : {}), ...(fastMessage.images ? { images: fastMessage.images } : {}), + ...(fastMessage.attachmentTexts + ? { attachmentTexts: fastMessage.attachmentTexts } + : {}), }); if (!continued) { apiLogger.warn( @@ -996,6 +999,9 @@ telegram.post('/', async (c) => { ? { agentContext: queuedMessage.agentContext } : {}), ...(queuedMessage.images ? { images: queuedMessage.images } : {}), + ...(queuedMessage.attachmentTexts + ? { attachmentTexts: queuedMessage.attachmentTexts } + : {}), }) .then((continued) => { if (!continued) { diff --git a/apps/api/src/handlers/telegram/types.ts b/apps/api/src/handlers/telegram/types.ts index c7578e5199..35979bdde1 100644 --- a/apps/api/src/handlers/telegram/types.ts +++ b/apps/api/src/handlers/telegram/types.ts @@ -3,6 +3,7 @@ import type { QueuedCommunicationMessage } from '@roomote/types'; export type QueuedTelegramCommunicationMessage = QueuedCommunicationMessage & { provider: 'telegram'; userId: string; + attachmentTexts?: string[]; }; export type TelegramConversationRef = { diff --git a/apps/docs/providers/communications/telegram.mdx b/apps/docs/providers/communications/telegram.mdx index bcc8246ca5..e183dc954b 100644 --- a/apps/docs/providers/communications/telegram.mdx +++ b/apps/docs/providers/communications/telegram.mdx @@ -141,10 +141,11 @@ Roomote keeps the footer on the latest reply current as delegated tasks start and finish (checking about every 30 seconds while work is running), and earlier replies drop their footer when a new reply posts. -Photos are passed to Fast or the task as image input. Supported text documents -are downloaded server-side and their extracted content is added to the request; -voice and audio messages are transcribed when supported. The bot token is never -included in the prompt or attachment URL. +Photos and supported image documents are passed as image input. Supported text +documents are downloaded server-side and their extracted content is added to the +request; voice and audio messages are transcribed, and supported video documents +are described. Fast can explicitly forward those current-message attachments to +a task. The bot token is never included in the prompt or attachment URL. ## Local URL changes diff --git a/packages/sdk/src/server/lib/fast-agent-parent-event.test.ts b/packages/sdk/src/server/lib/fast-agent-parent-event.test.ts index 4516e06c06..632b5a2fb3 100644 --- a/packages/sdk/src/server/lib/fast-agent-parent-event.test.ts +++ b/packages/sdk/src/server/lib/fast-agent-parent-event.test.ts @@ -518,6 +518,7 @@ describe('deliverFastAgentParentEvent', () => { userId: 'user-2', question: 'Use the corrected requirement.', images: ['data:image/png;base64,aGVsbG8='], + attachmentTexts: ['Attachment: plan.md\nUse the corrected value.'], senderDisplayName: 'Matt', senderExternalId: 'U123', }, @@ -529,6 +530,7 @@ describe('deliverFastAgentParentEvent', () => { expect.objectContaining({ question: 'Use the corrected requirement.', images: ['data:image/png;base64,aGVsbG8='], + attachmentTexts: ['Attachment: plan.md\nUse the corrected value.'], userId: 'user-2', currentMessageId: '100.003', currentDurableHumanFollowUpEventId: '100.003', diff --git a/packages/sdk/src/server/lib/fast-agent-parent-event.ts b/packages/sdk/src/server/lib/fast-agent-parent-event.ts index 4dc4f6ae6a..3db73993df 100644 --- a/packages/sdk/src/server/lib/fast-agent-parent-event.ts +++ b/packages/sdk/src/server/lib/fast-agent-parent-event.ts @@ -2711,6 +2711,9 @@ export async function deliverFastAgentParentEventWithLock( humanFollowUp?.question ?? `${JSON.stringify(params.event)}`, ...(humanFollowUp?.images ? { images: humanFollowUp.images } : {}), + ...(humanFollowUp?.attachmentTexts + ? { attachmentTexts: humanFollowUp.attachmentTexts } + : {}), userId: humanFollowUp?.userId ?? parentTurn.userId, conversation: parentTurn.conversation, currentMessageId: diff --git a/packages/sdk/src/server/lib/fast-agent-surface-reply.test.ts b/packages/sdk/src/server/lib/fast-agent-surface-reply.test.ts index 44c0f80cc2..2669b9e559 100644 --- a/packages/sdk/src/server/lib/fast-agent-surface-reply.test.ts +++ b/packages/sdk/src/server/lib/fast-agent-surface-reply.test.ts @@ -951,6 +951,7 @@ describe('continueFastAgentSurfaceReply admission hooks', () => { userId: user.id, senderDisplayName: 'Matt', question: 'Follow up', + attachmentTexts: ['Attachment: notes.txt\nUse the new requirement.'], currentMessageId: 'message-1', onAccepted, onRejected, @@ -959,6 +960,13 @@ describe('continueFastAgentSurfaceReply admission hooks', () => { expect(onAccepted).toHaveBeenCalledWith(abort); expect(onRejected).not.toHaveBeenCalled(); + expect(mocks.admitHumanFollowUp).toHaveBeenCalledWith( + expect.objectContaining({ + event: expect.objectContaining({ + attachmentTexts: ['Attachment: notes.txt\nUse the new requirement.'], + }), + }), + ); }); it('admits a reaction turn durably with its input and resumes a still-pending row', async () => { diff --git a/packages/sdk/src/server/lib/fast-agent-surface-reply.ts b/packages/sdk/src/server/lib/fast-agent-surface-reply.ts index 01e32b7bb7..446e7f326f 100644 --- a/packages/sdk/src/server/lib/fast-agent-surface-reply.ts +++ b/packages/sdk/src/server/lib/fast-agent-surface-reply.ts @@ -169,6 +169,7 @@ type FastAgentSurfaceReplyParams = { currentMessageId: string; replyToMessageId?: string; images?: string[]; + attachmentTexts?: string[]; /** * Tasks the Session may steer on this turn beyond the ones it delegated, * for example the task that already owns the pull request a comment is on. @@ -779,6 +780,9 @@ function buildSurfaceHumanFollowUpEvent( userId: params.userId, question: params.question, ...(params.images?.length ? { images: params.images } : {}), + ...(params.attachmentTexts?.length + ? { attachmentTexts: params.attachmentTexts } + : {}), ...(params.senderDisplayName ? { senderDisplayName: params.senderDisplayName } : {}), @@ -940,6 +944,7 @@ async function runFastAgentSurfaceReplyWithLock( return answerFastAgentQuestion({ question: params.question, images: params.images, + attachmentTexts: params.attachmentTexts, ...(params.agentContext ? { currentMessageAgentContext: params.agentContext } : {}), diff --git a/packages/types/src/fast-agent.ts b/packages/types/src/fast-agent.ts index 635a21e138..2b1394a092 100644 --- a/packages/types/src/fast-agent.ts +++ b/packages/types/src/fast-agent.ts @@ -269,6 +269,7 @@ export const fastAgentHumanFollowUpEventSchema = z.object({ userId: z.string().min(1), question: z.string().min(1), images: z.array(z.string()).optional(), + attachmentTexts: z.array(z.string()).optional(), senderDisplayName: z.string().min(1).optional(), senderExternalId: z.string().min(1).optional(), /** From 30a1c845640ca1ae492a5a368d475a1ed8caa638 Mon Sep 17 00:00:00 2001 From: "@daniel-lxs" <57051444+daniel-lxs@users.noreply.github.com> Date: Sat, 12 Sep 2026 05:27:50 +0000 Subject: [PATCH 2/2] fix: preserve Telegram image document MIME --- apps/api/src/handlers/telegram/__tests__/index.test.ts | 9 +++++++-- apps/api/src/handlers/telegram/attachments.ts | 4 +++- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/apps/api/src/handlers/telegram/__tests__/index.test.ts b/apps/api/src/handlers/telegram/__tests__/index.test.ts index 8a11895b9a..2d72c6fd42 100644 --- a/apps/api/src/handlers/telegram/__tests__/index.test.ts +++ b/apps/api/src/handlers/telegram/__tests__/index.test.ts @@ -885,6 +885,11 @@ describe('Telegram webhook handler', () => { it('passes Telegram image documents to a new Fast session as images', async () => { mockTelegramLinkedSender('mapped-user-1'); + downloadFileMock.mockResolvedValueOnce({ + bytes: new Uint8Array([1, 2, 3]), + filePath: 'documents/failure.png', + contentType: 'application/octet-stream', + }); const response = await postTelegramUpdate( createTelegramUpdate({ @@ -895,7 +900,7 @@ describe('Telegram webhook handler', () => { file_id: 'screenshot-file', file_unique_id: 'screenshot-1', file_name: 'failure.png', - mime_type: 'image/png', + mime_type: 'application/octet-stream', }, }, }), @@ -908,7 +913,7 @@ describe('Telegram webhook handler', () => { expect(continueFastReplyMock).toHaveBeenCalledWith( expect.objectContaining({ question: 'Inspect the uncompressed screenshot', - images: ['data:image/jpeg;base64,AQID'], + images: ['data:image/png;base64,AQID'], }), ); }); diff --git a/apps/api/src/handlers/telegram/attachments.ts b/apps/api/src/handlers/telegram/attachments.ts index 0d27b37951..1164c7e086 100644 --- a/apps/api/src/handlers/telegram/attachments.ts +++ b/apps/api/src/handlers/telegram/attachments.ts @@ -98,7 +98,9 @@ export async function attachTelegramMediaToQueuedMessage(input: { const downloadedMimeType = downloaded.contentType?.split(';')[0]; const mimeType = downloadedMimeType?.startsWith('image/') ? downloadedMimeType - : (documentMimeType ?? 'image/png'); + : documentMimeType?.startsWith('image/') + ? documentMimeType + : 'image/png'; images.push( `data:${mimeType};base64,${Buffer.from(downloaded.bytes).toString('base64')}`, );