diff --git a/apps/api/src/handlers/telegram/__tests__/index.test.ts b/apps/api/src/handlers/telegram/__tests__/index.test.ts
index fe4f9ea8a..eb679c538 100644
--- a/apps/api/src/handlers/telegram/__tests__/index.test.ts
+++ b/apps/api/src/handlers/telegram/__tests__/index.test.ts
@@ -8,11 +8,13 @@ const {
taskRunsFindFirstMock,
consumeLinkCodeMock,
createForumTopicMock,
+ describeVideoAttachmentMock,
downloadFileMock,
restoreLinkCodeMock,
editMessageReplyMarkupMock,
editMessageTextMock,
enqueueTaskMock,
+ extractPromptTextAttachmentsMock,
environmentsFindFirstMock,
envMock,
getAvailableEnvironmentsMock,
@@ -43,6 +45,7 @@ const {
getFastSessionMock,
isFastProviderMessageMock,
recordFastConversationMessageMock,
+ transcribeAudioAttachmentMock,
claimPendingPrReviewActionMock,
completePendingPrReviewActionDispatchMock,
dispatchPrReviewFollowUpMock,
@@ -55,11 +58,13 @@ const {
taskRunsFindFirstMock: vi.fn(),
consumeLinkCodeMock: vi.fn(),
createForumTopicMock: vi.fn(),
+ describeVideoAttachmentMock: vi.fn(),
downloadFileMock: vi.fn(),
restoreLinkCodeMock: vi.fn(),
editMessageReplyMarkupMock: vi.fn(),
editMessageTextMock: vi.fn(),
enqueueTaskMock: vi.fn(),
+ extractPromptTextAttachmentsMock: vi.fn(),
environmentsFindFirstMock: vi.fn(),
getAvailableEnvironmentsMock: vi.fn(),
getBotInfoMock: vi.fn(),
@@ -95,6 +100,7 @@ const {
getFastSessionMock: vi.fn(),
isFastProviderMessageMock: vi.fn(),
recordFastConversationMessageMock: vi.fn(),
+ transcribeAudioAttachmentMock: vi.fn(),
claimPendingPrReviewActionMock: vi.fn(),
completePendingPrReviewActionDispatchMock: vi.fn(),
dispatchPrReviewFollowUpMock: vi.fn(),
@@ -322,14 +328,31 @@ vi.mock('../../tasks/task-stop.js', () => ({
}));
vi.mock('@roomote/cloud-agents/server', () => ({
+ AUDIO_TRANSCRIPTION_MAX_SIZE_BYTES: 20 * 1024 * 1024,
buildFastAgentReactionExternalInputQuestion: vi.fn(
(input: unknown) =>
`${JSON.stringify(input)}`,
),
+ describeVideoAttachment: describeVideoAttachmentMock,
enqueueTask: enqueueTaskMock,
+ extractPromptTextAttachments: extractPromptTextAttachmentsMock,
+ formatAudioAttachmentWarning: (filename: string, reason: string) =>
+ `[Audio attachment ${filename} ${reason}.]`,
+ formatAudioTranscriptionResult: (
+ filename: string,
+ result: { transcript?: string },
+ ) => `Audio attachment transcript: ${filename}\n${result.transcript ?? ''}`,
getAvailableEnvironments: getAvailableEnvironmentsMock,
getTaskUrl: getTaskUrlMock,
getOrCreateFastAgentSession: getFastSessionMock,
+ isVideoAgentSupportedMimeType: (mimeType: string) =>
+ ['video/mp4', 'video/quicktime', 'video/webm', 'video/mpeg'].includes(
+ mimeType,
+ ),
+ resolveAudioTranscriptionMimeType: ({ mimeType }: { mimeType?: string }) =>
+ mimeType?.startsWith('audio/') ? mimeType : null,
+ transcribeAudioAttachment: transcribeAudioAttachmentMock,
+ VIDEO_AGENT_MAX_VIDEO_SIZE_BYTES: 20 * 1024 * 1024,
}));
import { telegram } from '../index';
@@ -406,6 +429,15 @@ describe('Telegram webhook handler', () => {
filePath: 'photos/example.jpg',
contentType: 'image/jpeg',
});
+ describeVideoAttachmentMock.mockResolvedValue('The video shows an error.');
+ extractPromptTextAttachmentsMock.mockResolvedValue({
+ attachmentTexts: ['Attachment: notes.txt\nDeployment failed.'],
+ warnings: [],
+ });
+ transcribeAudioAttachmentMock.mockResolvedValue({
+ status: 'transcribed',
+ transcript: 'Run the focused tests.',
+ });
taskRunsFindFirstMock.mockReset();
telegramMappingsFindFirstMock.mockReset();
consumeLinkCodeMock.mockReset();
@@ -936,6 +968,195 @@ describe('Telegram webhook handler', () => {
expect(enqueueTaskMock).not.toHaveBeenCalled();
});
+ it('passes Telegram image documents to a new Fast session as images', async () => {
+ mockTelegramLinkedSender('mapped-user-1');
+ downloadFileMock.mockResolvedValueOnce({
+ bytes: new Uint8Array([1, 2, 3]),
+ filePath: 'documents/failure.png',
+ contentType: 'application/octet-stream',
+ });
+
+ const response = await postTelegramUpdate(
+ createTelegramUpdate({
+ message: {
+ text: undefined,
+ caption: 'Inspect the uncompressed screenshot',
+ document: {
+ file_id: 'screenshot-file',
+ file_unique_id: 'screenshot-1',
+ file_name: 'failure.png',
+ mime_type: 'application/octet-stream',
+ },
+ },
+ }),
+ );
+
+ await expect(response.json()).resolves.toMatchObject({
+ fastAnswered: true,
+ fastDefaulted: true,
+ });
+ expect(continueFastReplyMock).toHaveBeenCalledWith(
+ expect.objectContaining({
+ question: 'Inspect the uncompressed screenshot',
+ images: ['data:image/png;base64,AQID'],
+ }),
+ );
+ });
+
+ it('passes extracted Telegram documents to a new Fast session', async () => {
+ mockTelegramLinkedSender('mapped-user-1');
+ downloadFileMock.mockResolvedValueOnce({
+ bytes: new TextEncoder().encode('Deployment failed.'),
+ filePath: 'documents/notes.txt',
+ contentType: 'text/plain',
+ });
+
+ const response = await postTelegramUpdate(
+ createTelegramUpdate({
+ message: {
+ text: undefined,
+ caption: 'Diagnose this log',
+ document: {
+ file_id: 'document-file',
+ file_unique_id: 'document-1',
+ file_name: 'notes.txt',
+ mime_type: 'text/plain',
+ },
+ },
+ }),
+ );
+
+ await expect(response.json()).resolves.toMatchObject({
+ fastAnswered: true,
+ fastDefaulted: true,
+ });
+ expect(continueFastReplyMock).toHaveBeenCalledWith(
+ expect.objectContaining({
+ question:
+ 'Diagnose this log\n\nAttachment: notes.txt\nDeployment failed.',
+ attachmentTexts: ['Attachment: notes.txt\nDeployment failed.'],
+ }),
+ );
+ });
+
+ it.each([
+ [
+ 'audio',
+ {
+ audio: {
+ file_id: 'audio-file',
+ file_unique_id: 'audio-1',
+ duration: 3,
+ file_name: 'request.mp3',
+ mime_type: 'audio/mpeg',
+ },
+ },
+ ],
+ [
+ 'voice note',
+ {
+ voice: {
+ file_id: 'voice-file',
+ file_unique_id: 'voice-1',
+ duration: 3,
+ mime_type: 'audio/ogg',
+ },
+ },
+ ],
+ ])(
+ 'passes transcribed Telegram %s to a new Fast session',
+ async (_, message) => {
+ mockTelegramLinkedSender('mapped-user-1');
+
+ const response = await postTelegramUpdate(
+ createTelegramUpdate({ message: { text: undefined, ...message } }),
+ );
+
+ await expect(response.json()).resolves.toMatchObject({
+ fastAnswered: true,
+ fastDefaulted: true,
+ });
+ expect(continueFastReplyMock).toHaveBeenCalledWith(
+ expect.objectContaining({
+ attachmentTexts: [expect.stringContaining('Run the focused tests.')],
+ }),
+ );
+ },
+ );
+
+ it('passes bounded Telegram video descriptions to a new Fast session', async () => {
+ mockTelegramLinkedSender('mapped-user-1');
+ downloadFileMock.mockResolvedValueOnce({
+ bytes: new Uint8Array([1, 2, 3]),
+ filePath: 'videos/repro.mp4',
+ contentType: 'video/mp4',
+ });
+
+ const response = await postTelegramUpdate(
+ createTelegramUpdate({
+ message: {
+ text: undefined,
+ caption: 'Review this recording',
+ document: {
+ file_id: 'video-file',
+ file_unique_id: 'video-1',
+ file_name: 'repro.mp4',
+ mime_type: 'video/mp4',
+ },
+ },
+ }),
+ );
+
+ await expect(response.json()).resolves.toMatchObject({
+ fastAnswered: true,
+ fastDefaulted: true,
+ });
+ expect(downloadFileMock).toHaveBeenCalledWith(
+ 'video-file',
+ 20 * 1024 * 1024,
+ );
+ expect(continueFastReplyMock).toHaveBeenCalledWith(
+ expect.objectContaining({
+ question: expect.stringContaining('The video shows an error.'),
+ attachmentTexts: [
+ 'Video attachment description: repro.mp4\nThe video shows an error.',
+ ],
+ }),
+ );
+ });
+
+ it('keeps unsupported Telegram documents out of new Fast attachment context', async () => {
+ mockTelegramLinkedSender('mapped-user-1');
+
+ const response = await postTelegramUpdate(
+ createTelegramUpdate({
+ message: {
+ text: undefined,
+ caption: 'Use this file',
+ document: {
+ file_id: 'archive-file',
+ file_unique_id: 'archive-1',
+ file_name: 'bundle.zip',
+ mime_type: 'application/zip',
+ },
+ },
+ }),
+ );
+
+ await expect(response.json()).resolves.toMatchObject({
+ fastAnswered: true,
+ fastDefaulted: true,
+ });
+ expect(downloadFileMock).not.toHaveBeenCalled();
+ expect(continueFastReplyMock).toHaveBeenCalledWith({
+ sessionId: 'fast-session-default',
+ userId: 'mapped-user-1',
+ senderDisplayName: 'Ada Lovelace',
+ question: 'Use this file',
+ currentMessageId: '456',
+ });
+ });
+
it('uses a user-scoped Fast session for a Telegram group topic mention', async () => {
mockTelegramLinkedSender('mapped-user-1');
getFastSessionMock.mockResolvedValueOnce({
diff --git a/apps/api/src/handlers/telegram/attachments.ts b/apps/api/src/handlers/telegram/attachments.ts
index 1bc8b3399..1164c7e08 100644
--- a/apps/api/src/handlers/telegram/attachments.ts
+++ b/apps/api/src/handlers/telegram/attachments.ts
@@ -1,14 +1,18 @@
import {
appendAttachmentTextsToPromptText,
+ isRoomoteImageAttachment,
isRoomoteTextExtractableAttachment,
} from '@roomote/cloud-agents';
import {
AUDIO_TRANSCRIPTION_MAX_SIZE_BYTES,
+ describeVideoAttachment,
extractPromptTextAttachments,
formatAudioAttachmentWarning,
formatAudioTranscriptionResult,
+ isVideoAgentSupportedMimeType,
resolveAudioTranscriptionMimeType,
transcribeAudioAttachment,
+ VIDEO_AGENT_MAX_VIDEO_SIZE_BYTES,
} from '@roomote/cloud-agents/server';
import { TelegramCommunicationProvider } from '@roomote/communication/telegram-provider';
import type { TelegramMessage } from '@roomote/communication/telegram-update';
@@ -63,27 +67,67 @@ export async function attachTelegramMediaToQueuedMessage(input: {
}
const document = input.message.document;
- if (
+ const documentMimeType = document?.mime_type?.trim().toLowerCase();
+ const documentIsImage = Boolean(
+ document &&
+ isRoomoteImageAttachment({
+ filename: document.file_name,
+ mimeType: document.mime_type,
+ }),
+ );
+ const documentIsText = Boolean(
document &&
isRoomoteTextExtractableAttachment({
filename: document.file_name,
mimeType: document.mime_type,
- })
- ) {
+ }),
+ );
+ const documentIsVideo = Boolean(
+ documentMimeType && isVideoAgentSupportedMimeType(documentMimeType),
+ );
+ if (document && (documentIsImage || documentIsText || documentIsVideo)) {
const downloaded = await provider.downloadFile(
document.file_id,
- MAX_DOCUMENT_BYTES,
+ documentIsImage
+ ? MAX_IMAGE_BYTES
+ : documentIsVideo
+ ? VIDEO_AGENT_MAX_VIDEO_SIZE_BYTES
+ : MAX_DOCUMENT_BYTES,
);
- const extracted = await extractPromptTextAttachments([
- {
- filename: document.file_name ?? downloaded.filePath,
- mimeType: document.mime_type ?? downloaded.contentType ?? undefined,
- bytes: downloaded.bytes,
- },
- ]);
- attachmentTexts.push(...extracted.attachmentTexts);
- for (const warning of extracted.warnings) {
- console.warn(`[telegram] Attachment extraction warning: ${warning}`);
+ if (documentIsImage) {
+ const downloadedMimeType = downloaded.contentType?.split(';')[0];
+ const mimeType = downloadedMimeType?.startsWith('image/')
+ ? downloadedMimeType
+ : documentMimeType?.startsWith('image/')
+ ? documentMimeType
+ : 'image/png';
+ images.push(
+ `data:${mimeType};base64,${Buffer.from(downloaded.bytes).toString('base64')}`,
+ );
+ } else if (documentIsVideo && documentMimeType) {
+ const description = await describeVideoAttachment({
+ videoBytes: Buffer.from(downloaded.bytes),
+ mimeType: documentMimeType,
+ userId: input.queuedMessage.userId,
+ userTextContext: input.queuedMessage.text,
+ });
+ if (description) {
+ attachmentTexts.push(
+ `Video attachment description${document.file_name ? `: ${document.file_name}` : ''}\n${description}`,
+ );
+ }
+ } else {
+ const extracted = await extractPromptTextAttachments([
+ {
+ filename: document.file_name ?? downloaded.filePath,
+ mimeType: document.mime_type ?? downloaded.contentType ?? undefined,
+ bytes: downloaded.bytes,
+ },
+ ]);
+ attachmentTexts.push(...extracted.attachmentTexts);
+ for (const warning of extracted.warnings) {
+ console.warn(`[telegram] Attachment extraction warning: ${warning}`);
+ }
}
}
} catch (error) {
@@ -149,5 +193,6 @@ export async function attachTelegramMediaToQueuedMessage(input: {
attachmentTexts,
}),
...(images.length ? { images } : {}),
+ ...(attachmentTexts.length ? { attachmentTexts } : {}),
};
}
diff --git a/apps/api/src/handlers/telegram/index.ts b/apps/api/src/handlers/telegram/index.ts
index 70db20e33..9123e2f3e 100644
--- a/apps/api/src/handlers/telegram/index.ts
+++ b/apps/api/src/handlers/telegram/index.ts
@@ -629,6 +629,9 @@ telegram.post('/', async (c) => {
? { agentContext: fastMessage.agentContext }
: {}),
...(fastMessage.images ? { images: fastMessage.images } : {}),
+ ...(fastMessage.attachmentTexts
+ ? { attachmentTexts: fastMessage.attachmentTexts }
+ : {}),
});
if (!continued) {
apiLogger.warn(
@@ -996,6 +999,9 @@ telegram.post('/', async (c) => {
? { agentContext: queuedMessage.agentContext }
: {}),
...(queuedMessage.images ? { images: queuedMessage.images } : {}),
+ ...(queuedMessage.attachmentTexts
+ ? { attachmentTexts: queuedMessage.attachmentTexts }
+ : {}),
})
.then((continued) => {
if (!continued) {
diff --git a/apps/api/src/handlers/telegram/types.ts b/apps/api/src/handlers/telegram/types.ts
index c7578e519..35979bdde 100644
--- a/apps/api/src/handlers/telegram/types.ts
+++ b/apps/api/src/handlers/telegram/types.ts
@@ -3,6 +3,7 @@ import type { QueuedCommunicationMessage } from '@roomote/types';
export type QueuedTelegramCommunicationMessage = QueuedCommunicationMessage & {
provider: 'telegram';
userId: string;
+ attachmentTexts?: string[];
};
export type TelegramConversationRef = {
diff --git a/apps/docs/providers/communications/telegram.mdx b/apps/docs/providers/communications/telegram.mdx
index bcc8246ca..e183dc954 100644
--- a/apps/docs/providers/communications/telegram.mdx
+++ b/apps/docs/providers/communications/telegram.mdx
@@ -141,10 +141,11 @@ Roomote keeps the footer on the latest reply current as delegated tasks start
and finish (checking about every 30 seconds while work is running), and
earlier replies drop their footer when a new reply posts.
-Photos are passed to Fast or the task as image input. Supported text documents
-are downloaded server-side and their extracted content is added to the request;
-voice and audio messages are transcribed when supported. The bot token is never
-included in the prompt or attachment URL.
+Photos and supported image documents are passed as image input. Supported text
+documents are downloaded server-side and their extracted content is added to the
+request; voice and audio messages are transcribed, and supported video documents
+are described. Fast can explicitly forward those current-message attachments to
+a task. The bot token is never included in the prompt or attachment URL.
## Local URL changes
diff --git a/packages/sdk/src/server/lib/fast-agent-parent-event.test.ts b/packages/sdk/src/server/lib/fast-agent-parent-event.test.ts
index bb44032bb..02ad1b29a 100644
--- a/packages/sdk/src/server/lib/fast-agent-parent-event.test.ts
+++ b/packages/sdk/src/server/lib/fast-agent-parent-event.test.ts
@@ -518,6 +518,7 @@ describe('deliverFastAgentParentEvent', () => {
userId: 'user-2',
question: 'Use the corrected requirement.',
images: ['data:image/png;base64,aGVsbG8='],
+ attachmentTexts: ['Attachment: plan.md\nUse the corrected value.'],
senderDisplayName: 'Matt',
senderExternalId: 'U123',
},
@@ -529,6 +530,7 @@ describe('deliverFastAgentParentEvent', () => {
expect.objectContaining({
question: 'Use the corrected requirement.',
images: ['data:image/png;base64,aGVsbG8='],
+ attachmentTexts: ['Attachment: plan.md\nUse the corrected value.'],
userId: 'user-2',
currentMessageId: '100.003',
currentDurableHumanFollowUpEventId: '100.003',
diff --git a/packages/sdk/src/server/lib/fast-agent-parent-event.ts b/packages/sdk/src/server/lib/fast-agent-parent-event.ts
index 906c6b31d..ad1fc74b3 100644
--- a/packages/sdk/src/server/lib/fast-agent-parent-event.ts
+++ b/packages/sdk/src/server/lib/fast-agent-parent-event.ts
@@ -2721,6 +2721,9 @@ export async function deliverFastAgentParentEventWithLock(
humanFollowUp?.question ??
`${JSON.stringify(params.event)}`,
...(humanFollowUp?.images ? { images: humanFollowUp.images } : {}),
+ ...(humanFollowUp?.attachmentTexts
+ ? { attachmentTexts: humanFollowUp.attachmentTexts }
+ : {}),
userId: humanFollowUp?.userId ?? parentTurn.userId,
conversation: parentTurn.conversation,
currentMessageId:
diff --git a/packages/sdk/src/server/lib/fast-agent-surface-reply.test.ts b/packages/sdk/src/server/lib/fast-agent-surface-reply.test.ts
index 44c0f80cc..2669b9e55 100644
--- a/packages/sdk/src/server/lib/fast-agent-surface-reply.test.ts
+++ b/packages/sdk/src/server/lib/fast-agent-surface-reply.test.ts
@@ -951,6 +951,7 @@ describe('continueFastAgentSurfaceReply admission hooks', () => {
userId: user.id,
senderDisplayName: 'Matt',
question: 'Follow up',
+ attachmentTexts: ['Attachment: notes.txt\nUse the new requirement.'],
currentMessageId: 'message-1',
onAccepted,
onRejected,
@@ -959,6 +960,13 @@ describe('continueFastAgentSurfaceReply admission hooks', () => {
expect(onAccepted).toHaveBeenCalledWith(abort);
expect(onRejected).not.toHaveBeenCalled();
+ expect(mocks.admitHumanFollowUp).toHaveBeenCalledWith(
+ expect.objectContaining({
+ event: expect.objectContaining({
+ attachmentTexts: ['Attachment: notes.txt\nUse the new requirement.'],
+ }),
+ }),
+ );
});
it('admits a reaction turn durably with its input and resumes a still-pending row', async () => {
diff --git a/packages/sdk/src/server/lib/fast-agent-surface-reply.ts b/packages/sdk/src/server/lib/fast-agent-surface-reply.ts
index 9f628cf4c..f98744e8c 100644
--- a/packages/sdk/src/server/lib/fast-agent-surface-reply.ts
+++ b/packages/sdk/src/server/lib/fast-agent-surface-reply.ts
@@ -172,6 +172,7 @@ type FastAgentSurfaceReplyParams = {
currentMessageId: string;
replyToMessageId?: string;
images?: string[];
+ attachmentTexts?: string[];
/**
* Tasks the Session may steer on this turn beyond the ones it delegated,
* for example the task that already owns the pull request a comment is on.
@@ -787,6 +788,9 @@ function buildSurfaceHumanFollowUpEvent(
userId: params.userId,
question: params.question,
...(params.images?.length ? { images: params.images } : {}),
+ ...(params.attachmentTexts?.length
+ ? { attachmentTexts: params.attachmentTexts }
+ : {}),
...(params.senderDisplayName
? { senderDisplayName: params.senderDisplayName }
: {}),
@@ -948,6 +952,7 @@ async function runFastAgentSurfaceReplyWithLock(
return answerFastAgentQuestion({
question: params.question,
images: params.images,
+ attachmentTexts: params.attachmentTexts,
...(params.agentContext
? { currentMessageAgentContext: params.agentContext }
: {}),
diff --git a/packages/types/src/fast-agent.ts b/packages/types/src/fast-agent.ts
index 635a21e13..2b1394a09 100644
--- a/packages/types/src/fast-agent.ts
+++ b/packages/types/src/fast-agent.ts
@@ -269,6 +269,7 @@ export const fastAgentHumanFollowUpEventSchema = z.object({
userId: z.string().min(1),
question: z.string().min(1),
images: z.array(z.string()).optional(),
+ attachmentTexts: z.array(z.string()).optional(),
senderDisplayName: z.string().min(1).optional(),
senderExternalId: z.string().min(1).optional(),
/**