Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 12 additions & 0 deletions src/main/emulator/avcc-frame-parser.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -35,4 +35,16 @@ describe('AvccFrameParser', () => {
[false, false, [7]]
])
})

it('forwards the connect-time JPEG seed before the description', () => {
const wire = [...packet(4, [0xff, 0xd8]), ...packet(1, [1, 0x64, 0, 0x33])]
const frames = parse([wire.slice(0, 6), wire.slice(6)])

expect(
frames.map(({ frame, bytes }) => [frame.seed, frame.config, frame.keyFrame, bytes])
).toEqual([
[true, false, false, [0xff, 0xd8]],
[false, true, false, [1, 0x64, 0, 0x33]]
])
})
})
16 changes: 7 additions & 9 deletions src/main/emulator/avcc-frame-parser.ts
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ const KEY_FRAME_TAG = 2
const SEED_TAG = 4
const MAX_PACKET_BYTES = 32 * 1024 * 1024

export type AvccFrame = { config: boolean; keyFrame: boolean; bytes: ArrayBuffer }
export type AvccFrame = { config: boolean; keyFrame: boolean; seed: boolean; bytes: ArrayBuffer }

export class AvccFrameParser {
private readonly header = new Uint8Array(5)
Expand Down Expand Up @@ -51,14 +51,12 @@ export class AvccFrameParser {
}
const bytes = this.payload.buffer
this.payload = null
// The connect-time JPEG seed only previews; the decoder starts from the description and IDR.
if (this.tag !== SEED_TAG) {
this.onFrame({
config: this.tag === DESCRIPTION_TAG,
keyFrame: this.tag === KEY_FRAME_TAG,
bytes
})
}
this.onFrame({
config: this.tag === DESCRIPTION_TAG,
keyFrame: this.tag === KEY_FRAME_TAG,
seed: this.tag === SEED_TAG,
bytes
})
}
}
}
3 changes: 2 additions & 1 deletion src/main/ipc/emulator-renderer-lifetime.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -258,12 +258,13 @@ it('streams a registered iOS device from its helper and releases it with the ren

const [callbacks] = mocks.avccCallbacks
const bytes = new ArrayBuffer(4)
callbacks.onFrame({ config: true, keyFrame: false, bytes })
callbacks.onFrame({ config: true, keyFrame: false, seed: false, bytes })
expect(sender.send).toHaveBeenCalledWith('emulator:videoStreamFrame', {
streamId: 'ios-stream',
deviceId: IOS_UDID,
config: true,
keyFrame: false,
seed: false,
bytes
})

Expand Down
1 change: 1 addition & 0 deletions src/preload/api/emulator-api.ts
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,7 @@ export type EmulatorApi = {
deviceId: string
config: boolean
keyFrame: boolean
seed?: boolean
bytes: ArrayBuffer
}) => void
) => () => void
Expand Down
2 changes: 2 additions & 0 deletions src/preload/api/emulator-bridge.ts
Original file line number Diff line number Diff line change
Expand Up @@ -67,6 +67,7 @@ export const emulatorApi = {
deviceId: string
config: boolean
keyFrame: boolean
seed?: boolean
bytes: ArrayBuffer
}) => void
): (() => void) => {
Expand All @@ -77,6 +78,7 @@ export const emulatorApi = {
deviceId: string
config: boolean
keyFrame: boolean
seed?: boolean
bytes: ArrayBuffer
}
) => callback(data)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@

import { act } from 'react'
import { createRoot, type Root } from 'react-dom/client'
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
import { afterEach, beforeEach, describe, expect, it, vi, type Mock } from 'vitest'
import { EmulatorScreenStreamContent } from './emulator-screen-stream-content'

type FrameListener = (data: { streamId: string; bytes: ArrayBuffer }) => void
Expand All @@ -12,6 +12,7 @@ type VideoFrameListener = (data: {
deviceId: string
config: boolean
keyFrame: boolean
seed?: boolean
bytes: ArrayBuffer
}) => void

Expand Down Expand Up @@ -62,6 +63,7 @@ let originalCreateObjectURL: typeof URL.createObjectURL | undefined
let originalRevokeObjectURL: typeof URL.revokeObjectURL | undefined
let objectUrlCounter: number
let streamCounter: number
let drawImage: Mock

beforeEach(() => {
;(
Expand All @@ -77,6 +79,13 @@ beforeEach(() => {
stopFrameStream = vi.fn(async () => {})
startVideoStream = vi.fn(async (args: { streamId: string }) => ({ streamId: args.streamId }))
stopVideoStream = vi.fn(async () => {})
drawImage = vi.fn()
vi.spyOn(HTMLCanvasElement.prototype, 'getContext').mockImplementation(() => {
const context: CanvasRenderingContext2D = Object.create(null)
context.drawImage = drawImage
context.clearRect = vi.fn()
return context
})
vi.stubGlobal('VideoDecoder', FakeVideoDecoder)
vi.stubGlobal(
'EncodedVideoChunk',
Expand Down Expand Up @@ -179,14 +188,14 @@ async function renderStream(
})
}

async function renderIosStream(onStreamError = vi.fn()): Promise<void> {
async function renderIosStream(onStreamError = vi.fn(), onStreamSize = vi.fn()): Promise<void> {
await act(async () => {
root.render(
<EmulatorScreenStreamContent
deviceId={IOS_UDID}
loading={false}
onStreamError={onStreamError}
onStreamSize={vi.fn()}
onStreamSize={onStreamSize}
previewUrl={IOS_PREVIEW_URL}
showStream={true}
streamError={false}
Expand All @@ -200,14 +209,18 @@ function videoStreamId(call: number): string {
return startVideoStream.mock.calls[call][0].streamId
}

async function sendVideo(streamId: string, packet: 'config' | 'key' | 'delta'): Promise<void> {
async function sendVideo(
streamId: string,
packet: 'config' | 'key' | 'delta' | 'seed'
): Promise<void> {
await act(async () => {
for (const listener of videoFrameListeners) {
listener({
streamId,
deviceId: IOS_UDID,
config: packet === 'config',
keyFrame: packet === 'key',
seed: packet === 'seed',
bytes: new Uint8Array([1, 0x64, 0x00, 0x33]).buffer
})
}
Expand Down Expand Up @@ -263,6 +276,46 @@ describe('EmulatorScreenStreamContent', () => {
expect(container.querySelector('canvas')).not.toBeNull()
})

it('paints the JPEG seed and reports its size before H.264 arrives', async () => {
const seed = { width: 400, height: 800, close: vi.fn() }
vi.stubGlobal('createImageBitmap', vi.fn().mockResolvedValue(seed))
const onStreamSize = vi.fn()
await renderIosStream(vi.fn(), onStreamSize)

await sendVideo(videoStreamId(0), 'seed')

expect(drawImage).toHaveBeenLastCalledWith(seed, 0, 0)
expect(container.querySelector('canvas')?.width).toBe(400)
expect(container.querySelector('canvas')?.height).toBe(800)
expect(onStreamSize).toHaveBeenLastCalledWith({ width: 400, height: 800 })
expect(seed.close).toHaveBeenCalledOnce()
await sendVideo(videoStreamId(0), 'config')
await sendVideo(videoStreamId(0), 'key')
expect(drawImage).toHaveBeenLastCalledWith(
expect.objectContaining({ displayWidth: 1206, displayHeight: 2622 }),
0,
0
)
})

it('does not let a late JPEG seed paint over a decoded H.264 frame', async () => {
const seed = { width: 400, height: 800, close: vi.fn() }
const pending = Promise.withResolvers<typeof seed>()
vi.stubGlobal(
'createImageBitmap',
vi.fn(() => pending.promise)
)
await renderIosStream()
await sendVideo(videoStreamId(0), 'seed')
await sendVideo(videoStreamId(0), 'config')
await sendVideo(videoStreamId(0), 'key')

await act(async () => pending.resolve(seed))

expect(drawImage).not.toHaveBeenCalledWith(seed, 0, 0)
expect(seed.close).toHaveBeenCalledOnce()
})

it('falls back to the iOS JPEG stream without reporting a disconnect when H.264 fails', async () => {
const onStreamError = vi.fn()
await renderIosStream(onStreamError)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ type VideoFrameMessage = {
deviceId: string
config: boolean
keyFrame: boolean
seed?: boolean
bytes: ArrayBuffer
}
type VideoMetaMessage = {
Expand Down Expand Up @@ -76,6 +77,7 @@ export function useEmulatorVideoStream(
let disposed = false
let configured = false
let waitingForKeyFrame = true
let hasVideoFrame = false
let timestamp = 0
let configBytes: Uint8Array | null = null
let reportedSize: StreamSize | null = null
Expand Down Expand Up @@ -108,6 +110,7 @@ export function useEmulatorVideoStream(
}
reportSize(frame.displayWidth, frame.displayHeight)
ctx.drawImage(frame, 0, 0)
hasVideoFrame = true
}
frame.close()
},
Expand Down Expand Up @@ -139,6 +142,7 @@ export function useEmulatorVideoStream(
const startStream = (): void => {
const id = newVideoStreamId()
streamId = id
hasVideoFrame = false
void startVideoStream({ deviceId, streamId: id })
.then((started) => {
if (disposed) {
Expand Down Expand Up @@ -212,6 +216,22 @@ export function useEmulatorVideoStream(
if (disposed || msg.streamId !== streamId || msg.deviceId !== deviceId) {
return
}
if (avcc && msg.seed) {
if (!hasVideoFrame && canvas && ctx) {
void createImageBitmap(new Blob([msg.bytes], { type: 'image/jpeg' }))
.then((bitmap) => {
if (!disposed && msg.streamId === streamId && !hasVideoFrame) {
canvas.width = bitmap.width
canvas.height = bitmap.height
ctx.drawImage(bitmap, 0, 0)
reportSize(bitmap.width, bitmap.height)
}
bitmap.close()
})
.catch(() => {})
}
return
}
const data = new Uint8Array(msg.bytes)
if (msg.config && avcc) {
// serve-sim sends a new avcC description whenever its encoder restarts, e.g. on resize.
Expand Down