From a62a55c49380ceca322b9ff249ff71933cf5cf0c Mon Sep 17 00:00:00 2001 From: ziad Date: Sun, 12 Jul 2026 22:24:00 +1000 Subject: [PATCH] Correct rotated thumbnails for videos uploaded as a Telegram document MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Telegram's document.thumb for a video sent as a document is a raw frame grab that ignores the container's moov/trak/tkhd display-rotation matrix, so a portrait-recorded video (landscape pixel data + rotation matrix) got a sideways preview image. Read the video track's tkhd matrix from the already-downloaded video buffer, compute the clockwise rotation (0/90/180/270), and apply it to the thumbnail with sharp before uploading. tkhd is a standard ISO-BMFF box, so this also corrects rotation on plain MP4 uploads, not just Apple's MOV. Falls back to the unmodified thumbnail when no rotation is detected. Mux-hosted playback video is untouched — Mux already respects the rotation matrix. Co-Authored-By: Claude Sonnet 5 --- .../readMp4VideoRotationDegrees.test.ts | 91 +++++++++++++++++++ .../media/__tests__/readTkhdMatrix.test.ts | 89 ++++++++++++++++++ .../media/__tests__/rotateImageBuffer.test.ts | 32 +++++++ src/lib/media/readMp4VideoRotationDegrees.ts | 46 ++++++++++ src/lib/media/readTkhdMatrix.ts | 42 +++++++++ src/lib/media/rotateImageBuffer.ts | 10 ++ .../__tests__/uploadVideoAttachment.test.ts | 39 ++++++++ .../chat/moment/uploadVideoAttachment.ts | 11 ++- 8 files changed, 359 insertions(+), 1 deletion(-) create mode 100644 src/lib/media/__tests__/readMp4VideoRotationDegrees.test.ts create mode 100644 src/lib/media/__tests__/readTkhdMatrix.test.ts create mode 100644 src/lib/media/__tests__/rotateImageBuffer.test.ts create mode 100644 src/lib/media/readMp4VideoRotationDegrees.ts create mode 100644 src/lib/media/readTkhdMatrix.ts create mode 100644 src/lib/media/rotateImageBuffer.ts diff --git a/src/lib/media/__tests__/readMp4VideoRotationDegrees.test.ts b/src/lib/media/__tests__/readMp4VideoRotationDegrees.test.ts new file mode 100644 index 00000000..b6cfa284 --- /dev/null +++ b/src/lib/media/__tests__/readMp4VideoRotationDegrees.test.ts @@ -0,0 +1,91 @@ +import { describe, it, expect } from 'vitest'; +import readMp4VideoRotationDegrees from '../readMp4VideoRotationDegrees'; + +const fixed1616 = (value: number): number => Math.round(value * 65536); + +const box = (type: string, payload: Buffer): Buffer => { + const header = Buffer.alloc(8); + header.writeUInt32BE(8 + payload.length, 0); + header.write(type, 4, 'ascii'); + return Buffer.concat([header, payload]); +}; + +const tkhdV0 = (matrix: { + a: number; + b: number; + c: number; + d: number; + width: number; + height: number; +}): Buffer => { + const payload = Buffer.alloc(84); + const matrixOffset = 4 + 20 + 8 + 8; + payload.writeInt32BE(fixed1616(matrix.a), matrixOffset); + payload.writeInt32BE(fixed1616(matrix.b), matrixOffset + 4); + payload.writeInt32BE(fixed1616(matrix.c), matrixOffset + 12); + payload.writeInt32BE(fixed1616(matrix.d), matrixOffset + 16); + payload.writeInt32BE(0x40000000, matrixOffset + 32); + payload.writeInt32BE(fixed1616(matrix.width), matrixOffset + 36); + payload.writeInt32BE(fixed1616(matrix.height), matrixOffset + 40); + return box('tkhd', payload); +}; + +const trak = (tkhdBuffer: Buffer): Buffer => box('trak', tkhdBuffer); + +const buildMoovBuffer = (traks: Buffer[]): Buffer => + box('moov', Buffer.concat(traks)); + +describe('readMp4VideoRotationDegrees', () => { + it('returns 0 for an unrotated (landscape, identity matrix) video track', () => { + const buffer = buildMoovBuffer([ + trak(tkhdV0({ a: 1, b: 0, c: 0, d: 1, width: 1920, height: 1080 })), + ]); + + expect(readMp4VideoRotationDegrees(buffer)).toBe(0); + }); + + it('returns 90 for the real iPhone portrait-recording matrix (a=0,b=1,c=-1,d=0)', () => { + const buffer = buildMoovBuffer([ + trak(tkhdV0({ a: 0, b: 1, c: -1, d: 0, width: 1920, height: 1080 })), + ]); + + expect(readMp4VideoRotationDegrees(buffer)).toBe(90); + }); + + it('returns 180 for an upside-down matrix', () => { + const buffer = buildMoovBuffer([ + trak(tkhdV0({ a: -1, b: 0, c: 0, d: -1, width: 1920, height: 1080 })), + ]); + + expect(readMp4VideoRotationDegrees(buffer)).toBe(180); + }); + + it('returns 270 for a -90-class matrix', () => { + const buffer = buildMoovBuffer([ + trak(tkhdV0({ a: 0, b: -1, c: 1, d: 0, width: 1920, height: 1080 })), + ]); + + expect(readMp4VideoRotationDegrees(buffer)).toBe(270); + }); + + it('skips audio/metadata tracks (zero width/height) and finds the video track', () => { + const buffer = buildMoovBuffer([ + trak(tkhdV0({ a: 1, b: 0, c: 0, d: 1, width: 0, height: 0 })), + trak(tkhdV0({ a: 0, b: 1, c: -1, d: 0, width: 1920, height: 1080 })), + ]); + + expect(readMp4VideoRotationDegrees(buffer)).toBe(90); + }); + + it('returns undefined when there is no moov box', () => { + expect(readMp4VideoRotationDegrees(Buffer.from('not-mp4'))).toBeUndefined(); + }); + + it('returns undefined when no track has a non-zero-dimension tkhd', () => { + const buffer = buildMoovBuffer([ + trak(tkhdV0({ a: 1, b: 0, c: 0, d: 1, width: 0, height: 0 })), + ]); + + expect(readMp4VideoRotationDegrees(buffer)).toBeUndefined(); + }); +}); diff --git a/src/lib/media/__tests__/readTkhdMatrix.test.ts b/src/lib/media/__tests__/readTkhdMatrix.test.ts new file mode 100644 index 00000000..a1548f06 --- /dev/null +++ b/src/lib/media/__tests__/readTkhdMatrix.test.ts @@ -0,0 +1,89 @@ +import { describe, it, expect } from 'vitest'; +import readTkhdMatrix from '../readTkhdMatrix'; +import readMp4BoxHeader from '../readMp4BoxHeader'; + +const fixed1616 = (value: number): number => Math.round(value * 65536); + +const tkhdV0 = (matrix: { + a: number; + b: number; + c: number; + d: number; + width: number; + height: number; +}): Buffer => { + const payload = Buffer.alloc(84); + payload.writeUInt8(0, 0); // version + // bytes 1-3 flags, 4..23 = creation/modification/trackId/reserved/duration (5 x 4 bytes) + // 24..31 = reserved[2], 32..39 = layer/altgroup/volume/reserved + const matrixOffset = 4 + 20 + 8 + 8; + payload.writeInt32BE(fixed1616(matrix.a), matrixOffset); + payload.writeInt32BE(fixed1616(matrix.b), matrixOffset + 4); + payload.writeInt32BE(0, matrixOffset + 8); // u + payload.writeInt32BE(fixed1616(matrix.c), matrixOffset + 12); + payload.writeInt32BE(fixed1616(matrix.d), matrixOffset + 16); + payload.writeInt32BE(0, matrixOffset + 20); // v + payload.writeInt32BE(0, matrixOffset + 24); // x + payload.writeInt32BE(0, matrixOffset + 28); // y + payload.writeInt32BE(0x40000000, matrixOffset + 32); // w (2.30 fixed, 1.0) + payload.writeInt32BE(fixed1616(matrix.width), matrixOffset + 36); + payload.writeInt32BE(fixed1616(matrix.height), matrixOffset + 40); + + const header = Buffer.alloc(8); + header.writeUInt32BE(8 + payload.length, 0); + header.write('tkhd', 4, 'ascii'); + return Buffer.concat([header, payload]); +}; + +describe('readTkhdMatrix', () => { + it('reads the identity matrix and dimensions of a landscape track', () => { + const buffer = tkhdV0({ + a: 1, + b: 0, + c: 0, + d: 1, + width: 1920, + height: 1080, + }); + const tkhd = readMp4BoxHeader(buffer, 0)!; + + expect(readTkhdMatrix(buffer, tkhd)).toEqual({ + a: 1, + b: 0, + c: 0, + d: 1, + width: 1920, + height: 1080, + }); + }); + + it('reads a 90-degree portrait-recording matrix (matches real iPhone MOV bytes)', () => { + const buffer = tkhdV0({ + a: 0, + b: 1, + c: -1, + d: 0, + width: 1920, + height: 1080, + }); + const tkhd = readMp4BoxHeader(buffer, 0)!; + + expect(readTkhdMatrix(buffer, tkhd)).toEqual({ + a: 0, + b: 1, + c: -1, + d: 0, + width: 1920, + height: 1080, + }); + }); + + it('returns undefined when the box is too short', () => { + const buffer = Buffer.alloc(8); + buffer.writeUInt32BE(8, 0); + buffer.write('tkhd', 4, 'ascii'); + const tkhd = readMp4BoxHeader(buffer, 0)!; + + expect(readTkhdMatrix(buffer, tkhd)).toBeUndefined(); + }); +}); diff --git a/src/lib/media/__tests__/rotateImageBuffer.test.ts b/src/lib/media/__tests__/rotateImageBuffer.test.ts new file mode 100644 index 00000000..ad8765b4 --- /dev/null +++ b/src/lib/media/__tests__/rotateImageBuffer.test.ts @@ -0,0 +1,32 @@ +import { describe, it, expect } from 'vitest'; +import sharp from 'sharp'; +import rotateImageBuffer from '../rotateImageBuffer'; + +const makeImageBuffer = (width: number, height: number): Promise => + sharp({ + create: { width, height, channels: 3, background: { r: 255, g: 0, b: 0 } }, + }) + .png() + .toBuffer(); + +describe('rotateImageBuffer', () => { + it('swaps width/height when rotated 90 degrees', async () => { + const input = await makeImageBuffer(20, 10); + + const rotated = await rotateImageBuffer(input, 90); + const metadata = await sharp(rotated).metadata(); + + expect(metadata.width).toBe(10); + expect(metadata.height).toBe(20); + }); + + it('leaves dimensions unchanged when rotated 180 degrees', async () => { + const input = await makeImageBuffer(20, 10); + + const rotated = await rotateImageBuffer(input, 180); + const metadata = await sharp(rotated).metadata(); + + expect(metadata.width).toBe(20); + expect(metadata.height).toBe(10); + }); +}); diff --git a/src/lib/media/readMp4VideoRotationDegrees.ts b/src/lib/media/readMp4VideoRotationDegrees.ts new file mode 100644 index 00000000..2f5f2d7d --- /dev/null +++ b/src/lib/media/readMp4VideoRotationDegrees.ts @@ -0,0 +1,46 @@ +import listMp4ChildBoxes from './listMp4ChildBoxes'; +import readTkhdMatrix from './readTkhdMatrix'; + +/** + * Reads the clockwise rotation (0/90/180/270) needed to display a video's + * raw encoded frames correctly, from the video track's `tkhd` display + * matrix (moov/trak/tkhd). iPhone videos recorded in portrait store the + * frame as landscape pixels plus this matrix instead of rotating the + * pixels themselves — Mux/native players apply it automatically, but a + * thumbnail taken from the raw frame bytes needs it applied manually. + * + * Returns 0 when no rotation is needed, and undefined when no video track + * (a `tkhd` with non-zero width/height) or matrix can be found — callers + * should treat both the same way (skip correction). + */ +const readMp4VideoRotationDegrees = (buffer: Buffer): number | undefined => { + const moov = listMp4ChildBoxes(buffer, 0, buffer.length).find( + (box) => box.type === 'moov' + ); + if (!moov) return undefined; + + const traks = listMp4ChildBoxes( + buffer, + moov.contentStart, + moov.contentEnd + ).filter((box) => box.type === 'trak'); + + for (const trak of traks) { + const tkhd = listMp4ChildBoxes( + buffer, + trak.contentStart, + trak.contentEnd + ).find((box) => box.type === 'tkhd'); + if (!tkhd) continue; + + const matrix = readTkhdMatrix(buffer, tkhd); + if (!matrix || matrix.width <= 0 || matrix.height <= 0) continue; + + const degrees = Math.atan2(matrix.b, matrix.a) * (180 / Math.PI); + return (((Math.round(degrees / 90) * 90) % 360) + 360) % 360; + } + + return undefined; +}; + +export default readMp4VideoRotationDegrees; diff --git a/src/lib/media/readTkhdMatrix.ts b/src/lib/media/readTkhdMatrix.ts new file mode 100644 index 00000000..97ef5dfe --- /dev/null +++ b/src/lib/media/readTkhdMatrix.ts @@ -0,0 +1,42 @@ +import type { Mp4Box } from './readMp4BoxHeader'; + +export type TkhdMatrix = { + a: number; + b: number; + c: number; + d: number; + width: number; + height: number; +}; + +const readInt32Fixed1616 = (buffer: Buffer, offset: number): number => + buffer.readInt32BE(offset) / 65536; + +/** + * Reads the display transformation matrix (a, b, c, d of the 3x3 row-major + * matrix; the translation/perspective terms are irrelevant for rotation) + * plus the track's declared width/height from a `tkhd` full box. Handles + * both the 32-bit (version 0) and 64-bit (version 1) time-field layouts. + */ +const readTkhdMatrix = ( + buffer: Buffer, + tkhd: Mp4Box +): TkhdMatrix | undefined => { + if (tkhd.contentEnd - tkhd.contentStart < 5) return undefined; + + const version = buffer.readUInt8(tkhd.contentStart); + const timesSize = version === 1 ? 8 + 8 + 4 + 4 + 8 : 4 + 4 + 4 + 4 + 4; + const matrixOffset = tkhd.contentStart + 4 + timesSize + 8 + 8; + if (matrixOffset + 44 > tkhd.contentEnd) return undefined; + + return { + a: readInt32Fixed1616(buffer, matrixOffset), + b: readInt32Fixed1616(buffer, matrixOffset + 4), + c: readInt32Fixed1616(buffer, matrixOffset + 12), + d: readInt32Fixed1616(buffer, matrixOffset + 16), + width: readInt32Fixed1616(buffer, matrixOffset + 36), + height: readInt32Fixed1616(buffer, matrixOffset + 40), + }; +}; + +export default readTkhdMatrix; diff --git a/src/lib/media/rotateImageBuffer.ts b/src/lib/media/rotateImageBuffer.ts new file mode 100644 index 00000000..f05fc6d4 --- /dev/null +++ b/src/lib/media/rotateImageBuffer.ts @@ -0,0 +1,10 @@ +import sharp from 'sharp'; + +/** + * Rotates an image buffer clockwise by `degrees` (must be a multiple of 90), + * preserving the source format. + */ +const rotateImageBuffer = (buffer: Buffer, degrees: number): Promise => + sharp(buffer).rotate(degrees).toBuffer(); + +export default rotateImageBuffer; diff --git a/src/lib/telegram/chat/moment/__tests__/uploadVideoAttachment.test.ts b/src/lib/telegram/chat/moment/__tests__/uploadVideoAttachment.test.ts index e687c949..841368ac 100644 --- a/src/lib/telegram/chat/moment/__tests__/uploadVideoAttachment.test.ts +++ b/src/lib/telegram/chat/moment/__tests__/uploadVideoAttachment.test.ts @@ -14,15 +14,22 @@ vi.mock('@/lib/telegram/chat/attachment/getTelegramFilePath', () => ({ vi.mock('@/lib/telegram/chat/attachment/fetchTelegramFile', () => ({ default: vi.fn(), })); +vi.mock('@/lib/media/readMp4VideoRotationDegrees', () => ({ + default: vi.fn(), +})); +vi.mock('@/lib/media/rotateImageBuffer', () => ({ default: vi.fn() })); import uploadFileToSupabase from '@/lib/supabase/storage/uploadFileToSupabase'; import uploadJsonToSupabase from '@/lib/supabase/storage/uploadJsonToSupabase'; import uploadVideoToMux from '@/lib/mux/uploadVideoToMux'; import getTelegramFilePath from '@/lib/telegram/chat/attachment/getTelegramFilePath'; import fetchTelegramFile from '@/lib/telegram/chat/attachment/fetchTelegramFile'; +import readMp4VideoRotationDegrees from '@/lib/media/readMp4VideoRotationDegrees'; +import rotateImageBuffer from '@/lib/media/rotateImageBuffer'; const BUFFER = Buffer.from('video data'); const THUMB_BUFFER = Buffer.from('thumb data'); +const ROTATED_THUMB_BUFFER = Buffer.from('rotated thumb data'); const THUMB_URL = 'https://supabase.co/storage/v1/object/public/bucket/thumb.jpg'; const META_URL = @@ -41,6 +48,8 @@ beforeEach(() => { }); vi.mocked(uploadFileToSupabase).mockResolvedValue(THUMB_URL); vi.mocked(uploadJsonToSupabase).mockResolvedValue(META_URL); + vi.mocked(readMp4VideoRotationDegrees).mockReturnValue(0); + vi.mocked(rotateImageBuffer).mockResolvedValue(ROTATED_THUMB_BUFFER); }); const makeAttachment = (overrides: Record = {}) => ({ @@ -102,6 +111,36 @@ describe('uploadVideoAttachment', () => { expect(uploadFileToSupabase).toHaveBeenCalledOnce(); }); + it('uploads the thumbnail unmodified when no rotation is needed', async () => { + vi.mocked(readMp4VideoRotationDegrees).mockReturnValue(0); + + await uploadVideoAttachment( + makeAttachment() as never, + 'file-id', + 'My Video', + 'thumb-id' + ); + + expect(rotateImageBuffer).not.toHaveBeenCalled(); + const uploadedFile = vi.mocked(uploadFileToSupabase).mock.calls[0][0]; + expect(await uploadedFile.text()).toBe('thumb data'); + }); + + it('rotates the thumbnail before uploading when the video track is rotated', async () => { + vi.mocked(readMp4VideoRotationDegrees).mockReturnValue(90); + + await uploadVideoAttachment( + makeAttachment() as never, + 'file-id', + 'My Video', + 'thumb-id' + ); + + expect(rotateImageBuffer).toHaveBeenCalledWith(THUMB_BUFFER, 90); + const uploadedFile = vi.mocked(uploadFileToSupabase).mock.calls[0][0]; + expect(await uploadedFile.text()).toBe('rotated thumb data'); + }); + it('includes image in metadata', async () => { await uploadVideoAttachment( makeAttachment() as never, diff --git a/src/lib/telegram/chat/moment/uploadVideoAttachment.ts b/src/lib/telegram/chat/moment/uploadVideoAttachment.ts index 300c9a8c..efcb1424 100644 --- a/src/lib/telegram/chat/moment/uploadVideoAttachment.ts +++ b/src/lib/telegram/chat/moment/uploadVideoAttachment.ts @@ -6,6 +6,8 @@ import getTelegramFilePath from '@/lib/telegram/chat/attachment/getTelegramFileP import getMimeTypeFromFilePath from '@/lib/telegram/chat/attachment/getMimeTypeFromFilePath'; import fetchTelegramFile from '@/lib/telegram/chat/attachment/fetchTelegramFile'; import toFile from '@/lib/telegram/chat/attachment/toFile'; +import readMp4VideoRotationDegrees from '@/lib/media/readMp4VideoRotationDegrees'; +import rotateImageBuffer from '@/lib/media/rotateImageBuffer'; const uploadVideoAttachment = async ( attachment: Attachment, @@ -20,6 +22,10 @@ const uploadVideoAttachment = async ( getTelegramFilePath(fileId), ]); const mimeType = attachment.mimeType ?? getMimeTypeFromFilePath(filePath); + // Telegram's own thumbnail for a video sent as a document is a raw frame + // grab that ignores the container's display-rotation matrix, so correct + // it ourselves before storing it as the token's preview image. + const rotationDegrees = readMp4VideoRotationDegrees(buffer); const [{ playbackUrl, downloadUrl }, thumbResult] = await Promise.all([ uploadVideoToMux(buffer, mimeType), @@ -28,8 +34,11 @@ const uploadVideoAttachment = async ( let imageUri = ''; if (thumbResult) { + const thumbBuffer = rotationDegrees + ? await rotateImageBuffer(thumbResult.buffer, rotationDegrees) + : thumbResult.buffer; const thumbFile = toFile( - thumbResult.buffer, + thumbBuffer, `${name}-thumb`, thumbResult.mimeType );