Correct rotated thumbnails for videos uploaded as a Telegram document

src/lib/media/__tests__/readMp4VideoRotationDegrees.test.ts

@@ -0,0 +1,91 @@
+import { describe, it, expect } from 'vitest';
+import readMp4VideoRotationDegrees from '../readMp4VideoRotationDegrees';
+
+const fixed1616 = (value: number): number => Math.round(value * 65536);
+
+const box = (type: string, payload: Buffer): Buffer => {
+  const header = Buffer.alloc(8);
+  header.writeUInt32BE(8 + payload.length, 0);
+  header.write(type, 4, 'ascii');
+  return Buffer.concat([header, payload]);
+};
+
+const tkhdV0 = (matrix: {
+  a: number;
+  b: number;
+  c: number;
+  d: number;
+  width: number;
+  height: number;
+}): Buffer => {
+  const payload = Buffer.alloc(84);
+  const matrixOffset = 4 + 20 + 8 + 8;
+  payload.writeInt32BE(fixed1616(matrix.a), matrixOffset);
+  payload.writeInt32BE(fixed1616(matrix.b), matrixOffset + 4);
+  payload.writeInt32BE(fixed1616(matrix.c), matrixOffset + 12);
+  payload.writeInt32BE(fixed1616(matrix.d), matrixOffset + 16);
+  payload.writeInt32BE(0x40000000, matrixOffset + 32);
+  payload.writeInt32BE(fixed1616(matrix.width), matrixOffset + 36);
+  payload.writeInt32BE(fixed1616(matrix.height), matrixOffset + 40);
+  return box('tkhd', payload);
+};
+
+const trak = (tkhdBuffer: Buffer): Buffer => box('trak', tkhdBuffer);
+
+const buildMoovBuffer = (traks: Buffer[]): Buffer =>
+  box('moov', Buffer.concat(traks));
+
+describe('readMp4VideoRotationDegrees', () => {
+  it('returns 0 for an unrotated (landscape, identity matrix) video track', () => {
+    const buffer = buildMoovBuffer([
+      trak(tkhdV0({ a: 1, b: 0, c: 0, d: 1, width: 1920, height: 1080 })),
+    ]);
+
+    expect(readMp4VideoRotationDegrees(buffer)).toBe(0);
+  });
+
+  it('returns 90 for the real iPhone portrait-recording matrix (a=0,b=1,c=-1,d=0)', () => {
+    const buffer = buildMoovBuffer([
+      trak(tkhdV0({ a: 0, b: 1, c: -1, d: 0, width: 1920, height: 1080 })),
+    ]);
+
+    expect(readMp4VideoRotationDegrees(buffer)).toBe(90);
+  });
+
+  it('returns 180 for an upside-down matrix', () => {
+    const buffer = buildMoovBuffer([
+      trak(tkhdV0({ a: -1, b: 0, c: 0, d: -1, width: 1920, height: 1080 })),
+    ]);
+
+    expect(readMp4VideoRotationDegrees(buffer)).toBe(180);
+  });
+
+  it('returns 270 for a -90-class matrix', () => {
+    const buffer = buildMoovBuffer([
+      trak(tkhdV0({ a: 0, b: -1, c: 1, d: 0, width: 1920, height: 1080 })),
+    ]);
+
+    expect(readMp4VideoRotationDegrees(buffer)).toBe(270);
+  });
+
+  it('skips audio/metadata tracks (zero width/height) and finds the video track', () => {
+    const buffer = buildMoovBuffer([
+      trak(tkhdV0({ a: 1, b: 0, c: 0, d: 1, width: 0, height: 0 })),
+      trak(tkhdV0({ a: 0, b: 1, c: -1, d: 0, width: 1920, height: 1080 })),
+    ]);
+
+    expect(readMp4VideoRotationDegrees(buffer)).toBe(90);
+  });
+
+  it('returns undefined when there is no moov box', () => {
+    expect(readMp4VideoRotationDegrees(Buffer.from('not-mp4'))).toBeUndefined();
+  });
+
+  it('returns undefined when no track has a non-zero-dimension tkhd', () => {
+    const buffer = buildMoovBuffer([
+      trak(tkhdV0({ a: 1, b: 0, c: 0, d: 1, width: 0, height: 0 })),
+    ]);
+
+    expect(readMp4VideoRotationDegrees(buffer)).toBeUndefined();
+  });
+});

src/lib/media/__tests__/readTkhdMatrix.test.ts

@@ -0,0 +1,89 @@
+import { describe, it, expect } from 'vitest';
+import readTkhdMatrix from '../readTkhdMatrix';
+import readMp4BoxHeader from '../readMp4BoxHeader';
+
+const fixed1616 = (value: number): number => Math.round(value * 65536);
+
+const tkhdV0 = (matrix: {
+  a: number;
+  b: number;
+  c: number;
+  d: number;
+  width: number;
+  height: number;
+}): Buffer => {
+  const payload = Buffer.alloc(84);
+  payload.writeUInt8(0, 0); // version
+  // bytes 1-3 flags, 4..23 = creation/modification/trackId/reserved/duration (5 x 4 bytes)
+  // 24..31 = reserved[2], 32..39 = layer/altgroup/volume/reserved
+  const matrixOffset = 4 + 20 + 8 + 8;
+  payload.writeInt32BE(fixed1616(matrix.a), matrixOffset);
+  payload.writeInt32BE(fixed1616(matrix.b), matrixOffset + 4);
+  payload.writeInt32BE(0, matrixOffset + 8); // u
+  payload.writeInt32BE(fixed1616(matrix.c), matrixOffset + 12);
+  payload.writeInt32BE(fixed1616(matrix.d), matrixOffset + 16);
+  payload.writeInt32BE(0, matrixOffset + 20); // v
+  payload.writeInt32BE(0, matrixOffset + 24); // x
+  payload.writeInt32BE(0, matrixOffset + 28); // y
+  payload.writeInt32BE(0x40000000, matrixOffset + 32); // w (2.30 fixed, 1.0)
+  payload.writeInt32BE(fixed1616(matrix.width), matrixOffset + 36);
+  payload.writeInt32BE(fixed1616(matrix.height), matrixOffset + 40);
+
+  const header = Buffer.alloc(8);
+  header.writeUInt32BE(8 + payload.length, 0);
+  header.write('tkhd', 4, 'ascii');
+  return Buffer.concat([header, payload]);
+};
+
+describe('readTkhdMatrix', () => {
+  it('reads the identity matrix and dimensions of a landscape track', () => {
+    const buffer = tkhdV0({
+      a: 1,
+      b: 0,
+      c: 0,
+      d: 1,
+      width: 1920,
+      height: 1080,
+    });
+    const tkhd = readMp4BoxHeader(buffer, 0)!;
+
+    expect(readTkhdMatrix(buffer, tkhd)).toEqual({
+      a: 1,
+      b: 0,
+      c: 0,
+      d: 1,
+      width: 1920,
+      height: 1080,
+    });
+  });
+
+  it('reads a 90-degree portrait-recording matrix (matches real iPhone MOV bytes)', () => {
+    const buffer = tkhdV0({
+      a: 0,
+      b: 1,
+      c: -1,
+      d: 0,
+      width: 1920,
+      height: 1080,
+    });
+    const tkhd = readMp4BoxHeader(buffer, 0)!;
+
+    expect(readTkhdMatrix(buffer, tkhd)).toEqual({
+      a: 0,
+      b: 1,
+      c: -1,
+      d: 0,
+      width: 1920,
+      height: 1080,
+    });
+  });
+
+  it('returns undefined when the box is too short', () => {
+    const buffer = Buffer.alloc(8);
+    buffer.writeUInt32BE(8, 0);
+    buffer.write('tkhd', 4, 'ascii');
+    const tkhd = readMp4BoxHeader(buffer, 0)!;
+
+    expect(readTkhdMatrix(buffer, tkhd)).toBeUndefined();
+  });
+});

src/lib/media/__tests__/rotateImageBuffer.test.ts

@@ -0,0 +1,32 @@
+import { describe, it, expect } from 'vitest';
+import sharp from 'sharp';
+import rotateImageBuffer from '../rotateImageBuffer';
+
+const makeImageBuffer = (width: number, height: number): Promise<Buffer> =>
+  sharp({
+    create: { width, height, channels: 3, background: { r: 255, g: 0, b: 0 } },
+  })
+    .png()
+    .toBuffer();
+
+describe('rotateImageBuffer', () => {
+  it('swaps width/height when rotated 90 degrees', async () => {
+    const input = await makeImageBuffer(20, 10);
+
+    const rotated = await rotateImageBuffer(input, 90);
+    const metadata = await sharp(rotated).metadata();
+
+    expect(metadata.width).toBe(10);
+    expect(metadata.height).toBe(20);
+  });
+
+  it('leaves dimensions unchanged when rotated 180 degrees', async () => {
+    const input = await makeImageBuffer(20, 10);
+
+    const rotated = await rotateImageBuffer(input, 180);
+    const metadata = await sharp(rotated).metadata();
+
+    expect(metadata.width).toBe(20);
+    expect(metadata.height).toBe(10);
+  });
+});

src/lib/media/readMp4VideoRotationDegrees.ts

@@ -0,0 +1,46 @@
+import listMp4ChildBoxes from './listMp4ChildBoxes';
+import readTkhdMatrix from './readTkhdMatrix';
+
+/**
+ * Reads the clockwise rotation (0/90/180/270) needed to display a video's
+ * raw encoded frames correctly, from the video track's `tkhd` display
+ * matrix (moov/trak/tkhd). iPhone videos recorded in portrait store the
+ * frame as landscape pixels plus this matrix instead of rotating the
+ * pixels themselves — Mux/native players apply it automatically, but a
+ * thumbnail taken from the raw frame bytes needs it applied manually.
+ *
+ * Returns 0 when no rotation is needed, and undefined when no video track
+ * (a `tkhd` with non-zero width/height) or matrix can be found — callers
+ * should treat both the same way (skip correction).
+ */
+const readMp4VideoRotationDegrees = (buffer: Buffer): number | undefined => {
+  const moov = listMp4ChildBoxes(buffer, 0, buffer.length).find(
+    (box) => box.type === 'moov'
+  );
+  if (!moov) return undefined;
+
+  const traks = listMp4ChildBoxes(
+    buffer,
+    moov.contentStart,
+    moov.contentEnd
+  ).filter((box) => box.type === 'trak');
+
+  for (const trak of traks) {
+    const tkhd = listMp4ChildBoxes(
+      buffer,
+      trak.contentStart,
+      trak.contentEnd
+    ).find((box) => box.type === 'tkhd');
+    if (!tkhd) continue;
+
+    const matrix = readTkhdMatrix(buffer, tkhd);
+    if (!matrix || matrix.width <= 0 || matrix.height <= 0) continue;
+
+    const degrees = Math.atan2(matrix.b, matrix.a) * (180 / Math.PI);
+    return (((Math.round(degrees / 90) * 90) % 360) + 360) % 360;
+  }
+
+  return undefined;
+};
+
+export default readMp4VideoRotationDegrees;

src/lib/media/readTkhdMatrix.ts

@@ -0,0 +1,42 @@
+import type { Mp4Box } from './readMp4BoxHeader';
+
+export type TkhdMatrix = {
+  a: number;
+  b: number;
+  c: number;
+  d: number;
+  width: number;
+  height: number;
+};
+
+const readInt32Fixed1616 = (buffer: Buffer, offset: number): number =>
+  buffer.readInt32BE(offset) / 65536;
+
+/**
+ * Reads the display transformation matrix (a, b, c, d of the 3x3 row-major
+ * matrix; the translation/perspective terms are irrelevant for rotation)
+ * plus the track's declared width/height from a `tkhd` full box. Handles
+ * both the 32-bit (version 0) and 64-bit (version 1) time-field layouts.
+ */
+const readTkhdMatrix = (
+  buffer: Buffer,
+  tkhd: Mp4Box
+): TkhdMatrix | undefined => {
+  if (tkhd.contentEnd - tkhd.contentStart < 5) return undefined;
+
+  const version = buffer.readUInt8(tkhd.contentStart);
+  const timesSize = version === 1 ? 8 + 8 + 4 + 4 + 8 : 4 + 4 + 4 + 4 + 4;
+  const matrixOffset = tkhd.contentStart + 4 + timesSize + 8 + 8;
+  if (matrixOffset + 44 > tkhd.contentEnd) return undefined;
+
+  return {
+    a: readInt32Fixed1616(buffer, matrixOffset),
+    b: readInt32Fixed1616(buffer, matrixOffset + 4),
+    c: readInt32Fixed1616(buffer, matrixOffset + 12),
+    d: readInt32Fixed1616(buffer, matrixOffset + 16),
+    width: readInt32Fixed1616(buffer, matrixOffset + 36),
+    height: readInt32Fixed1616(buffer, matrixOffset + 40),
+  };
+};
+
+export default readTkhdMatrix;

src/lib/media/rotateImageBuffer.ts

@@ -0,0 +1,10 @@
+import sharp from 'sharp';
+
+/**
+ * Rotates an image buffer clockwise by `degrees` (must be a multiple of 90),
+ * preserving the source format.
+ */
+const rotateImageBuffer = (buffer: Buffer, degrees: number): Promise<Buffer> =>
+  sharp(buffer).rotate(degrees).toBuffer();
+
+export default rotateImageBuffer;

src/lib/telegram/chat/moment/__tests__/uploadVideoAttachment.test.ts

@@ -14,15 +14,22 @@ vi.mock('@/lib/telegram/chat/attachment/getTelegramFilePath', () => ({
 vi.mock('@/lib/telegram/chat/attachment/fetchTelegramFile', () => ({
   default: vi.fn(),
 }));
+vi.mock('@/lib/media/readMp4VideoRotationDegrees', () => ({
+  default: vi.fn(),
+}));
+vi.mock('@/lib/media/rotateImageBuffer', () => ({ default: vi.fn() }));
 
 import uploadFileToSupabase from '@/lib/supabase/storage/uploadFileToSupabase';
 import uploadJsonToSupabase from '@/lib/supabase/storage/uploadJsonToSupabase';
 import uploadVideoToMux from '@/lib/mux/uploadVideoToMux';
 import getTelegramFilePath from '@/lib/telegram/chat/attachment/getTelegramFilePath';
 import fetchTelegramFile from '@/lib/telegram/chat/attachment/fetchTelegramFile';
+import readMp4VideoRotationDegrees from '@/lib/media/readMp4VideoRotationDegrees';
+import rotateImageBuffer from '@/lib/media/rotateImageBuffer';
 
 const BUFFER = Buffer.from('video data');
 const THUMB_BUFFER = Buffer.from('thumb data');
+const ROTATED_THUMB_BUFFER = Buffer.from('rotated thumb data');
 const THUMB_URL =
   'https://supabase.co/storage/v1/object/public/bucket/thumb.jpg';
 const META_URL =
@@ -41,6 +48,8 @@ beforeEach(() => {
   });
   vi.mocked(uploadFileToSupabase).mockResolvedValue(THUMB_URL);
   vi.mocked(uploadJsonToSupabase).mockResolvedValue(META_URL);
+  vi.mocked(readMp4VideoRotationDegrees).mockReturnValue(0);
+  vi.mocked(rotateImageBuffer).mockResolvedValue(ROTATED_THUMB_BUFFER);
 });
 
 const makeAttachment = (overrides: Record<string, unknown> = {}) => ({
@@ -102,6 +111,36 @@ describe('uploadVideoAttachment', () => {
       expect(uploadFileToSupabase).toHaveBeenCalledOnce();
     });
 
+    it('uploads the thumbnail unmodified when no rotation is needed', async () => {
+      vi.mocked(readMp4VideoRotationDegrees).mockReturnValue(0);
+
+      await uploadVideoAttachment(
+        makeAttachment() as never,
+        'file-id',
+        'My Video',
+        'thumb-id'
+      );
+
+      expect(rotateImageBuffer).not.toHaveBeenCalled();
+      const uploadedFile = vi.mocked(uploadFileToSupabase).mock.calls[0][0];
+      expect(await uploadedFile.text()).toBe('thumb data');
+    });
+
+    it('rotates the thumbnail before uploading when the video track is rotated', async () => {
+      vi.mocked(readMp4VideoRotationDegrees).mockReturnValue(90);
+
+      await uploadVideoAttachment(
+        makeAttachment() as never,
+        'file-id',
+        'My Video',
+        'thumb-id'
+      );
+
+      expect(rotateImageBuffer).toHaveBeenCalledWith(THUMB_BUFFER, 90);
+      const uploadedFile = vi.mocked(uploadFileToSupabase).mock.calls[0][0];
+      expect(await uploadedFile.text()).toBe('rotated thumb data');
+    });
+
     it('includes image in metadata', async () => {
       await uploadVideoAttachment(
         makeAttachment() as never,