diff --git a/src/main/emulator/avcc-frame-parser.test.ts b/src/main/emulator/avcc-frame-parser.test.ts index 63e202338143..908eaec3ecba 100644 --- a/src/main/emulator/avcc-frame-parser.test.ts +++ b/src/main/emulator/avcc-frame-parser.test.ts @@ -35,4 +35,16 @@ describe('AvccFrameParser', () => { [false, false, [7]] ]) }) + + it('forwards the connect-time JPEG seed before the description', () => { + const wire = [...packet(4, [0xff, 0xd8]), ...packet(1, [1, 0x64, 0, 0x33])] + const frames = parse([wire.slice(0, 6), wire.slice(6)]) + + expect( + frames.map(({ frame, bytes }) => [frame.seed, frame.config, frame.keyFrame, bytes]) + ).toEqual([ + [true, false, false, [0xff, 0xd8]], + [false, true, false, [1, 0x64, 0, 0x33]] + ]) + }) }) diff --git a/src/main/emulator/avcc-frame-parser.ts b/src/main/emulator/avcc-frame-parser.ts index ef445583d76b..22af5dca75bc 100644 --- a/src/main/emulator/avcc-frame-parser.ts +++ b/src/main/emulator/avcc-frame-parser.ts @@ -5,7 +5,7 @@ const KEY_FRAME_TAG = 2 const SEED_TAG = 4 const MAX_PACKET_BYTES = 32 * 1024 * 1024 -export type AvccFrame = { config: boolean; keyFrame: boolean; bytes: ArrayBuffer } +export type AvccFrame = { config: boolean; keyFrame: boolean; seed: boolean; bytes: ArrayBuffer } export class AvccFrameParser { private readonly header = new Uint8Array(5) @@ -51,14 +51,12 @@ export class AvccFrameParser { } const bytes = this.payload.buffer this.payload = null - // The connect-time JPEG seed only previews; the decoder starts from the description and IDR. - if (this.tag !== SEED_TAG) { - this.onFrame({ - config: this.tag === DESCRIPTION_TAG, - keyFrame: this.tag === KEY_FRAME_TAG, - bytes - }) - } + this.onFrame({ + config: this.tag === DESCRIPTION_TAG, + keyFrame: this.tag === KEY_FRAME_TAG, + seed: this.tag === SEED_TAG, + bytes + }) } } } diff --git a/src/main/ipc/emulator-renderer-lifetime.test.ts b/src/main/ipc/emulator-renderer-lifetime.test.ts index 23e8414e39a3..6b266ecaf674 100644 --- a/src/main/ipc/emulator-renderer-lifetime.test.ts +++ b/src/main/ipc/emulator-renderer-lifetime.test.ts @@ -258,12 +258,13 @@ it('streams a registered iOS device from its helper and releases it with the ren const [callbacks] = mocks.avccCallbacks const bytes = new ArrayBuffer(4) - callbacks.onFrame({ config: true, keyFrame: false, bytes }) + callbacks.onFrame({ config: true, keyFrame: false, seed: false, bytes }) expect(sender.send).toHaveBeenCalledWith('emulator:videoStreamFrame', { streamId: 'ios-stream', deviceId: IOS_UDID, config: true, keyFrame: false, + seed: false, bytes }) diff --git a/src/preload/api/emulator-api.ts b/src/preload/api/emulator-api.ts index 1638b4e4420c..55ffbe0e910c 100644 --- a/src/preload/api/emulator-api.ts +++ b/src/preload/api/emulator-api.ts @@ -41,6 +41,7 @@ export type EmulatorApi = { deviceId: string config: boolean keyFrame: boolean + seed?: boolean bytes: ArrayBuffer }) => void ) => () => void diff --git a/src/preload/api/emulator-bridge.ts b/src/preload/api/emulator-bridge.ts index 50fb7c3c1664..c239909fb8a1 100644 --- a/src/preload/api/emulator-bridge.ts +++ b/src/preload/api/emulator-bridge.ts @@ -67,6 +67,7 @@ export const emulatorApi = { deviceId: string config: boolean keyFrame: boolean + seed?: boolean bytes: ArrayBuffer }) => void ): (() => void) => { @@ -77,6 +78,7 @@ export const emulatorApi = { deviceId: string config: boolean keyFrame: boolean + seed?: boolean bytes: ArrayBuffer } ) => callback(data) diff --git a/src/renderer/src/components/emulator-pane/emulator-screen-stream-content.test.tsx b/src/renderer/src/components/emulator-pane/emulator-screen-stream-content.test.tsx index bbd6fc813a93..e0baa860f264 100644 --- a/src/renderer/src/components/emulator-pane/emulator-screen-stream-content.test.tsx +++ b/src/renderer/src/components/emulator-pane/emulator-screen-stream-content.test.tsx @@ -2,7 +2,7 @@ import { act } from 'react' import { createRoot, type Root } from 'react-dom/client' -import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { afterEach, beforeEach, describe, expect, it, vi, type Mock } from 'vitest' import { EmulatorScreenStreamContent } from './emulator-screen-stream-content' type FrameListener = (data: { streamId: string; bytes: ArrayBuffer }) => void @@ -12,6 +12,7 @@ type VideoFrameListener = (data: { deviceId: string config: boolean keyFrame: boolean + seed?: boolean bytes: ArrayBuffer }) => void @@ -62,6 +63,7 @@ let originalCreateObjectURL: typeof URL.createObjectURL | undefined let originalRevokeObjectURL: typeof URL.revokeObjectURL | undefined let objectUrlCounter: number let streamCounter: number +let drawImage: Mock beforeEach(() => { ;( @@ -77,6 +79,13 @@ beforeEach(() => { stopFrameStream = vi.fn(async () => {}) startVideoStream = vi.fn(async (args: { streamId: string }) => ({ streamId: args.streamId })) stopVideoStream = vi.fn(async () => {}) + drawImage = vi.fn() + vi.spyOn(HTMLCanvasElement.prototype, 'getContext').mockImplementation(() => { + const context: CanvasRenderingContext2D = Object.create(null) + context.drawImage = drawImage + context.clearRect = vi.fn() + return context + }) vi.stubGlobal('VideoDecoder', FakeVideoDecoder) vi.stubGlobal( 'EncodedVideoChunk', @@ -179,14 +188,14 @@ async function renderStream( }) } -async function renderIosStream(onStreamError = vi.fn()): Promise { +async function renderIosStream(onStreamError = vi.fn(), onStreamSize = vi.fn()): Promise { await act(async () => { root.render( { +async function sendVideo( + streamId: string, + packet: 'config' | 'key' | 'delta' | 'seed' +): Promise { await act(async () => { for (const listener of videoFrameListeners) { listener({ @@ -208,6 +220,7 @@ async function sendVideo(streamId: string, packet: 'config' | 'key' | 'delta'): deviceId: IOS_UDID, config: packet === 'config', keyFrame: packet === 'key', + seed: packet === 'seed', bytes: new Uint8Array([1, 0x64, 0x00, 0x33]).buffer }) } @@ -263,6 +276,46 @@ describe('EmulatorScreenStreamContent', () => { expect(container.querySelector('canvas')).not.toBeNull() }) + it('paints the JPEG seed and reports its size before H.264 arrives', async () => { + const seed = { width: 400, height: 800, close: vi.fn() } + vi.stubGlobal('createImageBitmap', vi.fn().mockResolvedValue(seed)) + const onStreamSize = vi.fn() + await renderIosStream(vi.fn(), onStreamSize) + + await sendVideo(videoStreamId(0), 'seed') + + expect(drawImage).toHaveBeenLastCalledWith(seed, 0, 0) + expect(container.querySelector('canvas')?.width).toBe(400) + expect(container.querySelector('canvas')?.height).toBe(800) + expect(onStreamSize).toHaveBeenLastCalledWith({ width: 400, height: 800 }) + expect(seed.close).toHaveBeenCalledOnce() + await sendVideo(videoStreamId(0), 'config') + await sendVideo(videoStreamId(0), 'key') + expect(drawImage).toHaveBeenLastCalledWith( + expect.objectContaining({ displayWidth: 1206, displayHeight: 2622 }), + 0, + 0 + ) + }) + + it('does not let a late JPEG seed paint over a decoded H.264 frame', async () => { + const seed = { width: 400, height: 800, close: vi.fn() } + const pending = Promise.withResolvers() + vi.stubGlobal( + 'createImageBitmap', + vi.fn(() => pending.promise) + ) + await renderIosStream() + await sendVideo(videoStreamId(0), 'seed') + await sendVideo(videoStreamId(0), 'config') + await sendVideo(videoStreamId(0), 'key') + + await act(async () => pending.resolve(seed)) + + expect(drawImage).not.toHaveBeenCalledWith(seed, 0, 0) + expect(seed.close).toHaveBeenCalledOnce() + }) + it('falls back to the iOS JPEG stream without reporting a disconnect when H.264 fails', async () => { const onStreamError = vi.fn() await renderIosStream(onStreamError) diff --git a/src/renderer/src/components/emulator-pane/use-emulator-video-stream.ts b/src/renderer/src/components/emulator-pane/use-emulator-video-stream.ts index abda16c6831f..e1fe1992c7bf 100644 --- a/src/renderer/src/components/emulator-pane/use-emulator-video-stream.ts +++ b/src/renderer/src/components/emulator-pane/use-emulator-video-stream.ts @@ -6,6 +6,7 @@ type VideoFrameMessage = { deviceId: string config: boolean keyFrame: boolean + seed?: boolean bytes: ArrayBuffer } type VideoMetaMessage = { @@ -76,6 +77,7 @@ export function useEmulatorVideoStream( let disposed = false let configured = false let waitingForKeyFrame = true + let hasVideoFrame = false let timestamp = 0 let configBytes: Uint8Array | null = null let reportedSize: StreamSize | null = null @@ -108,6 +110,7 @@ export function useEmulatorVideoStream( } reportSize(frame.displayWidth, frame.displayHeight) ctx.drawImage(frame, 0, 0) + hasVideoFrame = true } frame.close() }, @@ -139,6 +142,7 @@ export function useEmulatorVideoStream( const startStream = (): void => { const id = newVideoStreamId() streamId = id + hasVideoFrame = false void startVideoStream({ deviceId, streamId: id }) .then((started) => { if (disposed) { @@ -212,6 +216,22 @@ export function useEmulatorVideoStream( if (disposed || msg.streamId !== streamId || msg.deviceId !== deviceId) { return } + if (avcc && msg.seed) { + if (!hasVideoFrame && canvas && ctx) { + void createImageBitmap(new Blob([msg.bytes], { type: 'image/jpeg' })) + .then((bitmap) => { + if (!disposed && msg.streamId === streamId && !hasVideoFrame) { + canvas.width = bitmap.width + canvas.height = bitmap.height + ctx.drawImage(bitmap, 0, 0) + reportSize(bitmap.width, bitmap.height) + } + bitmap.close() + }) + .catch(() => {}) + } + return + } const data = new Uint8Array(msg.bytes) if (msg.config && avcc) { // serve-sim sends a new avcC description whenever its encoder restarts, e.g. on resize.