diff --git a/packages/react-user-media/src/hooks/index.ts b/packages/react-user-media/src/hooks/index.ts index 08a580d..dce09db 100644 --- a/packages/react-user-media/src/hooks/index.ts +++ b/packages/react-user-media/src/hooks/index.ts @@ -3,3 +3,6 @@ export * from "./use-media-devices"; export * from "./use-media-devices-ext"; export * from "./use-media-ext"; export * from "./use-media-recorder"; + +// Worker-based hooks +export * from "../workers"; diff --git a/packages/react-user-media/src/index.ts b/packages/react-user-media/src/index.ts index 9562774..f49331d 100644 --- a/packages/react-user-media/src/index.ts +++ b/packages/react-user-media/src/index.ts @@ -1,5 +1,6 @@ export * from "./components"; export * from "./hooks"; +export * from "./workers"; export function getSupportedConstraints() { return navigator.mediaDevices.getSupportedConstraints(); diff --git a/packages/react-user-media/src/workers/README.md b/packages/react-user-media/src/workers/README.md new file mode 100644 index 0000000..182a5a1 --- /dev/null +++ b/packages/react-user-media/src/workers/README.md @@ -0,0 +1,226 @@ +# Media Workers + +This module provides worker-based abstractions for media operations, allowing for testing without the full web worker stack while maintaining production compatibility. + +## Overview + +The media workers module consists of: + +- **Types**: Core interfaces and type definitions +- **Mock Worker**: Test-friendly implementation that doesn't require actual web workers +- **Web Worker**: Production implementation using actual web workers +- **Factory**: Management and creation of worker instances +- **Hooks**: React hooks for easy integration + +## Key Features + +- **Testable**: Mock implementation for unit testing without web workers +- **Production Ready**: Real web worker implementation for production use +- **Type Safe**: Full TypeScript support with comprehensive type definitions +- **React Integration**: Custom hooks for seamless React integration +- **Flexible**: Configurable worker behavior and lifecycle management + +## Basic Usage + +### Using the Media Worker Hook + +```tsx +import { useMediaWorker } from '@react-user-media/workers'; + +function MyComponent() { + const mediaWorker = useMediaWorker({ + useMockWorker: process.env.NODE_ENV === 'test', // Use mock in tests + autoInitialize: true, + }); + + const handleStartRecording = async () => { + try { + await mediaWorker.startRecording({ + mimeType: 'video/webm', + videoBitsPerSecond: 2500000, + }); + } catch (error) { + console.error('Failed to start recording:', error); + } + }; + + return ( +
+

Initialized: {mediaWorker.isInitialized ? 'Yes' : 'No'}

+

Recording: {mediaWorker.isRecording ? 'Yes' : 'No'}

+ +
+ ); +} +``` + +### Using the Media Worker Recorder + +```tsx +import { useMediaWorkerRecorder } from '@react-user-media/workers'; + +function RecordingComponent() { + const recorder = useMediaWorkerRecorder({ + useMockWorker: process.env.NODE_ENV === 'test', + defaultRecordingConfig: { + mimeType: 'video/webm', + timeslice: 1000, // 1 second chunks + }, + }); + + const handleStopRecording = async () => { + const segments = await recorder.stopRecording(); + console.log('Recorded segments:', segments); + }; + + return ( +
+

Duration: {recorder.getDuration() ? Math.round(recorder.getDuration()! / 1000) : 0}s

+

Finalized: {recorder.isFinalized ? 'Yes' : 'No'}

+ +
+ ); +} +``` + +## Configuration + +### Worker Factory Configuration + +```tsx +import { getWorkerFactory } from '@react-user-media/workers'; + +const factory = getWorkerFactory({ + useMockWorker: process.env.NODE_ENV === 'test', + workerScript: '/path/to/custom-worker.js', + isWorkerSupported: () => typeof Worker !== 'undefined', +}); +``` + +### Hook Configuration + +```tsx +const mediaWorker = useMediaWorker({ + workerId: 'my-worker', // Unique identifier + useMockWorker: false, // Use real web worker + workerScript: '/custom-worker.js', // Custom worker script + autoInitialize: true, // Auto-initialize on mount +}); +``` + +## Testing + +The mock worker implementation allows you to test worker functionality without actual web workers: + +```tsx +import { renderHook, act } from '@testing-library/react'; +import { useMediaWorker } from '@react-user-media/workers'; + +test('should start recording', async () => { + const { result } = renderHook(() => useMediaWorker({ useMockWorker: true })); + + await act(async () => { + await result.current.initialize(); + await result.current.startRecording(); + }); + + expect(result.current.isRecording).toBe(true); +}); +``` + +## Worker Messages + +The worker system uses a message-based communication pattern: + +```tsx +const unsubscribe = mediaWorker.subscribe((message) => { + switch (message.type) { + case 'SUCCESS': + console.log('Operation succeeded:', message.payload); + break; + case 'ERROR': + console.error('Operation failed:', message.error); + break; + case 'DATA_AVAILABLE': + console.log('New data available:', message.payload); + break; + } +}); +``` + +## Media Processing + +The worker can process media data for various operations: + +```tsx +const processedBlob = await mediaWorker.processMedia(originalBlob, { + operation: 'compress', + options: { quality: 0.8 }, +}); +``` + +Supported operations: +- `compress`: Compress media data +- `convert`: Convert between formats +- `extract_audio`: Extract audio track +- `extract_video`: Extract video track +- `resize`: Resize video dimensions + +## Device Management + +Get available media devices through the worker: + +```tsx +const devices = await mediaWorker.getDevices(); +console.log('Available devices:', devices); +``` + +## Lifecycle Management + +Workers are automatically managed, but you can control them manually: + +```tsx +// Terminate a specific worker +mediaWorker.terminate(); + +// Or use the factory to manage multiple workers +const factory = getWorkerFactory(); +factory.removeWorker('worker-id'); +factory.removeAllWorkers(); +``` + +## Error Handling + +The worker system provides comprehensive error handling: + +```tsx +const mediaWorker = useMediaWorker(); + +if (mediaWorker.isError) { + console.error('Worker error:', mediaWorker.error); +} + +// Subscribe to error messages +mediaWorker.subscribe((message) => { + if (message.type === 'ERROR') { + // Handle error + } +}); +``` + +## Performance Considerations + +- **Mock Workers**: Use for testing and development - no performance overhead +- **Web Workers**: Use for production - offloads processing to background thread +- **Memory Management**: Workers are automatically cleaned up when components unmount +- **Resource Limits**: Consider worker limits in your target environment + +## Browser Support + +- **Web Workers**: Modern browsers with Web Worker support +- **Mock Workers**: All environments (fallback for unsupported browsers) +- **Media APIs**: Requires secure context (HTTPS) for media device access \ No newline at end of file diff --git a/packages/react-user-media/src/workers/__tests__/mock-worker.spec.ts b/packages/react-user-media/src/workers/__tests__/mock-worker.spec.ts new file mode 100644 index 0000000..457d09a --- /dev/null +++ b/packages/react-user-media/src/workers/__tests__/mock-worker.spec.ts @@ -0,0 +1,186 @@ +import { describe, it, expect, beforeEach, vi } from 'vitest'; +import { MockMediaWorkerController } from '../mock-worker'; + +// Mock performance.now +const mockPerformanceNow = vi.fn(() => 1000); +Object.defineProperty(globalThis, 'performance', { + value: { + now: mockPerformanceNow, + }, + writable: true, +}); + +describe('MockMediaWorkerController', () => { + let controller: MockMediaWorkerController; + + beforeEach(() => { + controller = new MockMediaWorkerController(); + vi.clearAllMocks(); + }); + + it('should initialize with correct default state', () => { + const state = controller.getState(); + + expect(state.isInitialized).toBe(false); + expect(state.isProcessing).toBe(false); + expect(state.error).toBe(null); + expect(state.videoConfig).toBe(null); + expect(state.audioConfig).toBe(null); + expect(state.codecCapabilities).toEqual([]); + }); + + it('should initialize asynchronously', async () => { + const initPromise = controller.initialize(); + + // Should not be initialized immediately + expect(controller.getState().isInitialized).toBe(false); + + await initPromise; + + // Should be initialized after promise resolves + expect(controller.getState().isInitialized).toBe(true); + }); + + it('should process video frame', async () => { + await controller.initialize(); + + // Create a mock VideoFrame + const canvas = new OffscreenCanvas(1920, 1080); + const ctx = canvas.getContext('2d')!; + ctx.fillStyle = 'red'; + ctx.fillRect(0, 0, 1920, 1080); + const frame = new VideoFrame(canvas, { timestamp: performance.now() }); + + const result = await controller.processVideoFrame(frame, { + operation: 'resize', + options: { width: 640, height: 480, format: 'RGBA' }, + }); + + expect(result.data).toBeInstanceOf(ArrayBuffer); + expect(result.width).toBe(640); + expect(result.height).toBe(480); + expect(result.format).toBe('RGBA'); + expect(result.timestamp).toBeGreaterThan(0); + + frame.close(); + }); + + it('should process audio data', async () => { + await controller.initialize(); + + // Create mock AudioData + const audioData = new AudioData({ + format: 'f32', + sampleRate: 48000, + numberOfChannels: 2, + numberOfFrames: 1024, + timestamp: performance.now(), + data: new Float32Array(2048), + }); + + const result = await controller.processAudioData(audioData, { + operation: 'resample', + options: { sampleRate: 44100, channels: 1 }, + }); + + expect(result.data).toBeInstanceOf(ArrayBuffer); + expect(result.sampleRate).toBe(44100); + expect(result.channels).toBe(1); + expect(result.format).toBe('f32'); + expect(result.duration).toBeGreaterThan(0); + }); + + it('should encode video frames', async () => { + await controller.initialize(); + + // Create mock VideoFrames + const canvas = new OffscreenCanvas(1920, 1080); + const ctx = canvas.getContext('2d')!; + ctx.fillStyle = 'blue'; + ctx.fillRect(0, 0, 1920, 1080); + const frame = new VideoFrame(canvas, { timestamp: performance.now() }); + + const chunks = await controller.encodeVideo([frame], { + codec: 'vp8', + width: 1920, + height: 1080, + bitrate: 1000000, + frameRate: 30, + }); + + expect(chunks).toHaveLength(1); + expect(chunks[0]).toBeInstanceOf(EncodedVideoChunk); + expect(chunks[0].type).toBe('key'); + + frame.close(); + }); + + it('should get codec capabilities', async () => { + await controller.initialize(); + + const capabilities = await controller.getCodecCapabilities(); + + expect(capabilities).toHaveLength(6); + expect(capabilities[0]).toEqual({ + supported: true, + codec: 'avc1.42E01E', + hardwareAccelerated: true, + maxWidth: 4096, + maxHeight: 4096, + maxFrameRate: 60, + maxBitrate: 100000000, + }); + }); + + it('should handle subscriptions', async () => { + const callback = vi.fn(); + const unsubscribe = controller.subscribe(callback); + + expect(typeof unsubscribe).toBe('function'); + + // Trigger a message by initializing + await controller.initialize(); + + // Should have received messages + expect(callback).toHaveBeenCalled(); + + // Unsubscribe should work + unsubscribe(); + }); + + it('should terminate and reset state', () => { + controller.terminate(); + + const state = controller.getState(); + expect(state.isInitialized).toBe(false); + expect(state.isProcessing).toBe(false); + expect(state.error).toBe(null); + expect(state.videoConfig).toBe(null); + expect(state.audioConfig).toBe(null); + expect(state.codecCapabilities).toEqual([]); + }); + + it('should throw error when not initialized', async () => { + // Create a proper canvas with content + const canvas = new OffscreenCanvas(100, 100); + const ctx = canvas.getContext('2d')!; + ctx.fillStyle = 'red'; + ctx.fillRect(0, 0, 100, 100); + + const frame = new VideoFrame(canvas, { timestamp: performance.now() }); + const audioData = new AudioData({ + format: 'f32', + sampleRate: 48000, + numberOfChannels: 2, + numberOfFrames: 1024, + timestamp: performance.now(), + data: new Float32Array(2048), + }); + + await expect(controller.processVideoFrame(frame, { operation: 'resize' })).rejects.toThrow('Worker not initialized'); + await expect(controller.processAudioData(audioData, { operation: 'resample' })).rejects.toThrow('Worker not initialized'); + await expect(controller.getCodecCapabilities()).rejects.toThrow('Worker not initialized'); + + frame.close(); + }); +}); \ No newline at end of file diff --git a/packages/react-user-media/src/workers/__tests__/worker-types.spec.ts b/packages/react-user-media/src/workers/__tests__/worker-types.spec.ts new file mode 100644 index 0000000..b88aeec --- /dev/null +++ b/packages/react-user-media/src/workers/__tests__/worker-types.spec.ts @@ -0,0 +1,96 @@ +import { describe, it, expect } from 'vitest'; +import type { + MediaWorkerMessage, + MediaWorkerMessageType, + MediaWorkerRecordingConfig, + MediaWorkerProcessingOptions, + MediaWorkerDeviceInfo, + MediaWorkerState, + MediaWorkerController, +} from '../types'; + +describe('Worker Types', () => { + it('should define correct message types', () => { + const messageTypes: MediaWorkerMessageType[] = [ + 'INIT', + 'START_RECORDING', + 'STOP_RECORDING', + 'PROCESS_MEDIA', + 'GET_DEVICES', + 'ERROR', + 'SUCCESS', + 'DATA_AVAILABLE', + ]; + + expect(messageTypes).toHaveLength(8); + expect(messageTypes).toContain('INIT'); + expect(messageTypes).toContain('START_RECORDING'); + expect(messageTypes).toContain('STOP_RECORDING'); + }); + + it('should create valid message structure', () => { + const message: MediaWorkerMessage = { + id: 'test-1', + type: 'SUCCESS', + payload: { message: 'Test success' }, + }; + + expect(message.id).toBe('test-1'); + expect(message.type).toBe('SUCCESS'); + expect(message.payload).toEqual({ message: 'Test success' }); + }); + + it('should define recording configuration', () => { + const config: MediaWorkerRecordingConfig = { + mimeType: 'video/webm', + timeslice: 1000, + videoBitsPerSecond: 2500000, + audioBitsPerSecond: 128000, + }; + + expect(config.mimeType).toBe('video/webm'); + expect(config.timeslice).toBe(1000); + expect(config.videoBitsPerSecond).toBe(2500000); + }); + + it('should define processing options', () => { + const options: MediaWorkerProcessingOptions = { + operation: 'compress', + options: { quality: 0.8 }, + }; + + expect(options.operation).toBe('compress'); + expect(options.options).toEqual({ quality: 0.8 }); + }); + + it('should define device info structure', () => { + const device: MediaWorkerDeviceInfo = { + deviceId: 'test-camera', + kind: 'videoinput', + label: 'Test Camera', + groupId: 'test-group', + }; + + expect(device.deviceId).toBe('test-camera'); + expect(device.kind).toBe('videoinput'); + expect(device.label).toBe('Test Camera'); + }); + + it('should define worker state structure', () => { + const state: MediaWorkerState = { + isInitialized: true, + isRecording: false, + isProcessing: false, + error: null, + recordingStartTime: null, + recordingEndTime: null, + segments: [], + mimeType: null, + }; + + expect(state.isInitialized).toBe(true); + expect(state.isRecording).toBe(false); + expect(state.isProcessing).toBe(false); + expect(state.error).toBe(null); + }); +}); \ No newline at end of file diff --git a/packages/react-user-media/src/workers/examples/MediaWorkerExample.tsx b/packages/react-user-media/src/workers/examples/MediaWorkerExample.tsx new file mode 100644 index 0000000..51e2c5f --- /dev/null +++ b/packages/react-user-media/src/workers/examples/MediaWorkerExample.tsx @@ -0,0 +1,235 @@ +import React, { useState, useEffect } from 'react'; +import { useMediaWorker, useMediaWorkerRecorder } from '../use-media-worker'; + +/** + * Example component demonstrating media worker usage + */ +export function MediaWorkerExample() { + const [useMockWorker, setUseMockWorker] = useState(true); + const [isRecording, setIsRecording] = useState(false); + const [segments, setSegments] = useState([]); + const [devices, setDevices] = useState([]); + + // Basic media worker + const mediaWorker = useMediaWorker({ + useMockWorker, + autoInitialize: true, + }); + + // Specialized recorder + const recorder = useMediaWorkerRecorder({ + useMockWorker, + autoInitialize: true, + defaultRecordingConfig: { + mimeType: 'video/webm', + timeslice: 1000, // 1 second chunks + }, + }); + + // Load devices on mount + useEffect(() => { + if (mediaWorker.isInitialized) { + mediaWorker.getDevices().then(setDevices); + } + }, [mediaWorker.isInitialized]); + + // Subscribe to worker messages + useEffect(() => { + const unsubscribe = mediaWorker.subscribe((message) => { + console.log('Worker message:', message); + }); + + return unsubscribe; + }, [mediaWorker]); + + const handleStartRecording = async () => { + try { + await recorder.startRecording({ + mimeType: 'video/webm', + videoBitsPerSecond: 2500000, + audioBitsPerSecond: 128000, + }); + setIsRecording(true); + } catch (error) { + console.error('Failed to start recording:', error); + } + }; + + const handleStopRecording = async () => { + try { + const recordedSegments = await recorder.stopRecording(); + setSegments(recordedSegments); + setIsRecording(false); + } catch (error) { + console.error('Failed to stop recording:', error); + } + }; + + const handleProcessMedia = async () => { + if (segments.length === 0) return; + + try { + const processedBlob = await mediaWorker.processMedia(segments[0], { + operation: 'compress', + options: { quality: 0.8 }, + }); + + console.log('Processed media:', processedBlob); + // You could download or display the processed media here + } catch (error) { + console.error('Failed to process media:', error); + } + }; + + const handleClearSegments = () => { + setSegments([]); + recorder.clearSegments(); + }; + + return ( +
+

Media Worker Example

+ + {/* Worker Configuration */} +
+ +
+ + {/* Worker Status */} +
+

Worker Status

+

Initialized: {mediaWorker.isInitialized ? 'Yes' : 'No'}

+

Recording: {recorder.isRecording ? 'Yes' : 'No'}

+

Processing: {mediaWorker.isProcessing ? 'Yes' : 'No'}

+

Error: {mediaWorker.error || 'None'}

+ {recorder.getDuration() && ( +

Duration: {Math.round(recorder.getDuration()! / 1000)}s

+ )} +
+ + {/* Available Devices */} +
+

Available Devices

+ {mediaWorker.isLoadingDevices ? ( +

Loading devices...

+ ) : ( +
    + {devices.map((device, index) => ( +
  • + {device.label} ({device.kind}) +
    • + ))} +
+ )} +
+ + {/* Recording Controls */} +
+

Recording Controls

+ + +
+ + {/* Recorded Segments */} +
+

Recorded Segments

+

Count: {segments.length}

+ {segments.length > 0 && ( +
+ + +
+ )} +
+ + {/* Worker State Debug */} +
+

Worker State (Debug)

+ 
+          {JSON.stringify(mediaWorker.workerState, null, 2)}
+
+
+ + {/* Cleanup */} +
+ +
+
+ ); +} \ No newline at end of file diff --git a/packages/react-user-media/src/workers/examples/SimpleWorkerExample.tsx b/packages/react-user-media/src/workers/examples/SimpleWorkerExample.tsx new file mode 100644 index 0000000..56d079d --- /dev/null +++ b/packages/react-user-media/src/workers/examples/SimpleWorkerExample.tsx @@ -0,0 +1,142 @@ +import React, { useState } from 'react'; +import { useMediaWorker, useMediaWorkerRecorder } from '../use-media-worker'; + +/** + * Simple example demonstrating basic worker usage + */ +export function SimpleWorkerExample() { + const [useMock, setUseMock] = useState(true); + const [status, setStatus] = useState('Ready'); + + // Basic media worker + const mediaWorker = useMediaWorker({ + useMockWorker: useMock, + autoInitialize: true, + }); + + // Specialized recorder + const recorder = useMediaWorkerRecorder({ + useMockWorker: useMock, + autoInitialize: true, + }); + + const handleStartRecording = async () => { + try { + setStatus('Starting recording...'); + await recorder.startRecording({ + mimeType: 'video/webm', + timeslice: 1000, + }); + setStatus('Recording...'); + } catch (error) { + setStatus(`Error: ${error instanceof Error ? error.message : 'Unknown error'}`); + } + }; + + const handleStopRecording = async () => { + try { + setStatus('Stopping recording...'); + const segments = await recorder.stopRecording(); + setStatus(`Recording stopped. Got ${segments.length} segments.`); + } catch (error) { + setStatus(`Error: ${error instanceof Error ? error.message : 'Unknown error'}`); + } + }; + + const handleGetDevices = async () => { + try { + setStatus('Getting devices...'); + const devices = await mediaWorker.getDevices(); + setStatus(`Found ${devices.length} devices`); + } catch (error) { + setStatus(`Error: ${error instanceof Error ? error.message : 'Unknown error'}`); + } + }; + + return ( +
+

Simple Media Worker Example

+ +
+ +
+ +
+

Status: {status}

+

Initialized: {mediaWorker.isInitialized ? 'Yes' : 'No'}

+

Recording: {recorder.isRecording ? 'Yes' : 'No'}

+

Error: {mediaWorker.error || 'None'}

+
+ +
+ + + + + +
+ +
+

Worker State

+ 
+          {JSON.stringify(mediaWorker.workerState, null, 2)}
+
+
+
+ ); +} \ No newline at end of file diff --git a/packages/react-user-media/src/workers/examples/WebCodecsExample.tsx b/packages/react-user-media/src/workers/examples/WebCodecsExample.tsx new file mode 100644 index 0000000..2b43b83 --- /dev/null +++ b/packages/react-user-media/src/workers/examples/WebCodecsExample.tsx @@ -0,0 +1,167 @@ +import React, { useState, useRef } from 'react'; +import { useMediaWorker } from '../use-media-worker'; + +/** + * Example component demonstrating WebCodecs worker usage + */ +export function WebCodecsExample() { + const [useMockWorker, setUseMockWorker] = useState(true); + const [status, setStatus] = useState('Ready'); + const [processedFrame, setProcessedFrame] = useState(null); + const videoRef = useRef(null); + const canvasRef = useRef(null); + + const mediaWorker = useMediaWorker({ + useMockWorker, + autoInitialize: true, + }); + + const handleProcessVideoFrame = async () => { + if (!videoRef.current || !canvasRef.current) return; + + try { + setStatus('Processing video frame...'); + + // Create a VideoFrame from the video element + const videoFrame = new VideoFrame(videoRef.current, { + timestamp: performance.now(), + }); + + // Process the frame + const result = await mediaWorker.processVideoFrame(videoFrame, { + operation: 'resize', + options: { width: 640, height: 480, format: 'RGBA' }, + }); + + setProcessedFrame(result.data); + setStatus(`Processed frame: ${result.width}x${result.height} ${result.format}`); + + // Draw the processed frame to canvas + const canvas = canvasRef.current; + const ctx = canvas.getContext('2d')!; + const imageData = new ImageData( + new Uint8ClampedArray(result.data), + result.width, + result.height + ); + canvas.width = result.width; + canvas.height = result.height; + ctx.putImageData(imageData, 0, 0); + + videoFrame.close(); + } catch (error) { + setStatus(`Error: ${error instanceof Error ? error.message : 'Unknown error'}`); + } + }; + + const handleGetCodecCapabilities = async () => { + try { + setStatus('Getting codec capabilities...'); + const capabilities = await mediaWorker.getCodecCapabilities(); + setStatus(`Found ${capabilities.length} codec capabilities`); + console.log('Codec capabilities:', capabilities); + } catch (error) { + setStatus(`Error: ${error instanceof Error ? error.message : 'Unknown error'}`); + } + }; + + return ( +
+

WebCodecs Media Worker Example

+ +
+ +
+ +
+

Status: {status}

+

Initialized: {mediaWorker.isInitialized ? 'Yes' : 'No'}

+

Processing: {mediaWorker.isProcessing ? 'Yes' : 'No'}

+

Error: {mediaWorker.error || 'None'}

+

Codec Capabilities: {mediaWorker.codecCapabilities.length}

+
+ +
+ + + +
+ +
+ + + +
+ + {processedFrame && ( +
+

Processed Frame Data

+

Size: {processedFrame.byteLength} bytes

+
+ )} + +
+

Codec Capabilities

+
    + {mediaWorker.codecCapabilities.map((cap, index) => ( +
  • + {cap.codec} - + {cap.supported ? 'Supported' : 'Not Supported'} - + {cap.hardwareAccelerated ? 'Hardware Accelerated' : 'Software Only'} + {cap.maxWidth && ` (Max: ${cap.maxWidth}x${cap.maxHeight})`} +
    • + ))} +
+
+
+ ); +} \ No newline at end of file diff --git a/packages/react-user-media/src/workers/index.ts b/packages/react-user-media/src/workers/index.ts new file mode 100644 index 0000000..955c9b4 --- /dev/null +++ b/packages/react-user-media/src/workers/index.ts @@ -0,0 +1,39 @@ +/** + * Media Worker Module + * + * This module provides worker-based abstractions for media operations, + * allowing for testing without full web worker stack while maintaining + * production compatibility. + */ + +// Types +export * from './types'; + +// Worker implementations +export { MockMediaWorkerController } from './mock-worker'; +export { WebMediaWorkerController } from './web-worker'; + +// Factory and management +export { + MediaWorkerFactory, + getWorkerFactory, + createWorkerFactory, + createMediaWorker, + type WorkerFactoryConfig, +} from './worker-factory'; + +// Hooks +export { + useMediaWorker, + type UseMediaWorkerConfig, + type MediaWorkerHookState, +} from './use-media-worker'; + +export { + useMediaWorkerRecorder, + type MediaWorkerRecorderConfig, + type MediaWorkerRecorderState, +} from './use-media-worker-recorder'; + +// Utility functions +export { createWorkerFactory as createMediaWorkerFactory } from './worker-factory'; \ No newline at end of file diff --git a/packages/react-user-media/src/workers/mock-worker.ts b/packages/react-user-media/src/workers/mock-worker.ts new file mode 100644 index 0000000..28a215b --- /dev/null +++ b/packages/react-user-media/src/workers/mock-worker.ts @@ -0,0 +1,388 @@ +import type { + MediaWorkerController, + MediaWorkerMessage, + MediaWorkerMessageType, + VideoProcessingConfig, + AudioProcessingConfig, + VideoFrameProcessingOptions, + AudioDataProcessingOptions, + CodecCapabilities, + ProcessedVideoFrame, + ProcessedAudioData, + MediaWorkerState, +} from './types'; + +/** + * Mock implementation of MediaWorkerController for testing and development + * This allows testing WebCodecs functionality without actual web workers + */ +export class MockMediaWorkerController implements MediaWorkerController { + private state: MediaWorkerState = { + isInitialized: false, + isProcessing: false, + error: null, + videoConfig: null, + audioConfig: null, + codecCapabilities: [], + }; + + private subscribers: Set<(message: MediaWorkerMessage) => void> = new Set(); + private videoConfig: VideoProcessingConfig | null = null; + private audioConfig: AudioProcessingConfig | null = null; + private messageId = 0; + + constructor() { + // Simulate initialization delay + setTimeout(() => { + this.state.isInitialized = true; + this.state.codecCapabilities = this.getMockCodecCapabilities(); + this.notifySubscribers({ + id: this.generateId(), + type: 'SUCCESS', + payload: { message: 'Worker initialized' }, + }); + }, 10); + } + + async initialize(): Promise { + return new Promise((resolve) => { + setTimeout(() => { + this.state.isInitialized = true; + this.state.error = null; + this.state.codecCapabilities = this.getMockCodecCapabilities(); + resolve(); + }, 10); + }); + } + + async processVideoFrame( + frame: VideoFrame, + options: VideoFrameProcessingOptions + ): Promise { + if (!this.state.isInitialized) { + throw new Error('Worker not initialized'); + } + + this.state.isProcessing = true; + + // Simulate processing delay + await new Promise(resolve => setTimeout(resolve, 50)); + + this.state.isProcessing = false; + + // Mock processed frame data + const processedFrame: ProcessedVideoFrame = { + data: new ArrayBuffer(1920 * 1080 * 4), // Mock RGBA data + width: options.options?.width || frame.displayWidth, + height: options.options?.height || frame.displayHeight, + format: options.options?.format || 'RGBA', + timestamp: performance.now(), + }; + + this.notifySubscribers({ + id: this.generateId(), + type: 'FRAME_PROCESSED', + payload: { frame: processedFrame }, + }); + + return processedFrame; + } + + async processAudioData( + data: AudioData, + options: AudioDataProcessingOptions + ): Promise { + if (!this.state.isInitialized) { + throw new Error('Worker not initialized'); + } + + this.state.isProcessing = true; + + // Simulate processing delay + await new Promise(resolve => setTimeout(resolve, 30)); + + this.state.isProcessing = false; + + // Mock processed audio data + const processedAudio: ProcessedAudioData = { + data: new ArrayBuffer(data.numberOfFrames * data.numberOfChannels * 4), // Mock f32 data + sampleRate: options.options?.sampleRate || data.sampleRate, + channels: options.options?.channels || data.numberOfChannels, + format: options.options?.format || 'f32', + duration: data.numberOfFrames / data.sampleRate, + }; + + this.notifySubscribers({ + id: this.generateId(), + type: 'SUCCESS', + payload: { audio: processedAudio }, + }); + + return processedAudio; + } + + async encodeVideo( + frames: VideoFrame[], + config: VideoProcessingConfig + ): Promise { + if (!this.state.isInitialized) { + throw new Error('Worker not initialized'); + } + + this.state.isProcessing = true; + this.state.videoConfig = config; + + // Simulate encoding delay + await new Promise(resolve => setTimeout(resolve, 100)); + + this.state.isProcessing = false; + + // Mock encoded chunks + const chunks: EncodedVideoChunk[] = frames.map((frame, index) => { + const data = new Uint8Array(1024); // Mock encoded data + return new EncodedVideoChunk({ + type: index === 0 ? 'key' : 'delta', + timestamp: performance.now() + index * 33, // ~30fps + duration: 33333, // ~30fps + data, + }); + }); + + this.notifySubscribers({ + id: this.generateId(), + type: 'ENCODED_DATA', + payload: { chunks, type: 'video' }, + }); + + return chunks; + } + + async encodeAudio( + data: AudioData[], + config: AudioProcessingConfig + ): Promise { + if (!this.state.isInitialized) { + throw new Error('Worker not initialized'); + } + + this.state.isProcessing = true; + this.state.audioConfig = config; + + // Simulate encoding delay + await new Promise(resolve => setTimeout(resolve, 50)); + + this.state.isProcessing = false; + + // Mock encoded chunks + const chunks: EncodedAudioChunk[] = data.map((audioData, index) => { + const data = new Uint8Array(512); // Mock encoded data + return new EncodedAudioChunk({ + type: 'key', + timestamp: performance.now() + index * 1000, + duration: 1000000, // 1 second + data, + }); + }); + + this.notifySubscribers({ + id: this.generateId(), + type: 'ENCODED_DATA', + payload: { chunks, type: 'audio' }, + }); + + return chunks; + } + + async decodeVideo( + chunks: EncodedVideoChunk[], + config: VideoProcessingConfig + ): Promise { + if (!this.state.isInitialized) { + throw new Error('Worker not initialized'); + } + + this.state.isProcessing = true; + + // Simulate decoding delay + await new Promise(resolve => setTimeout(resolve, 80)); + + this.state.isProcessing = false; + + // Mock decoded frames + const frames: VideoFrame[] = chunks.map((chunk, index) => { + // Create a mock VideoFrame - in real implementation this would come from decoder + const canvas = new OffscreenCanvas(config.width || 1920, config.height || 1080); + const ctx = canvas.getContext('2d')!; + ctx.fillStyle = `hsl(${index * 30}, 50%, 50%)`; + ctx.fillRect(0, 0, canvas.width, canvas.height); + + return new VideoFrame(canvas, { timestamp: chunk.timestamp }); + }); + + this.notifySubscribers({ + id: this.generateId(), + type: 'SUCCESS', + payload: { frames, type: 'video' }, + }); + + return frames; + } + + async decodeAudio( + chunks: EncodedAudioChunk[], + config: AudioProcessingConfig + ): Promise { + if (!this.state.isInitialized) { + throw new Error('Worker not initialized'); + } + + this.state.isProcessing = true; + + // Simulate decoding delay + await new Promise(resolve => setTimeout(resolve, 60)); + + this.state.isProcessing = false; + + // Mock decoded audio data + const audioData: AudioData[] = chunks.map((chunk) => { + const sampleRate = config.sampleRate || 48000; + const channels = config.channels || 2; + const frames = 1024; // Mock frame count + + return new AudioData({ + format: config.format || 'f32', + sampleRate, + numberOfChannels: channels, + numberOfFrames: frames, + data: new Float32Array(frames * channels), + }); + }); + + this.notifySubscribers({ + id: this.generateId(), + type: 'SUCCESS', + payload: { audioData, type: 'audio' }, + }); + + return audioData; + } + + async configureCodec( + type: 'video' | 'audio', + config: VideoProcessingConfig | AudioProcessingConfig + ): Promise { + if (!this.state.isInitialized) { + throw new Error('Worker not initialized'); + } + + if (type === 'video') { + this.state.videoConfig = config as VideoProcessingConfig; + } else { + this.state.audioConfig = config as AudioProcessingConfig; + } + + this.notifySubscribers({ + id: this.generateId(), + type: 'SUCCESS', + payload: { message: `${type} codec configured` }, + }); + } + + async getCodecCapabilities(): Promise { + if (!this.state.isInitialized) { + throw new Error('Worker not initialized'); + } + + return this.state.codecCapabilities; + } + + getState(): MediaWorkerState { + return { ...this.state }; + } + + subscribe(callback: (message: MediaWorkerMessage) => void): () => void { + this.subscribers.add(callback); + return () => { + this.subscribers.delete(callback); + }; + } + + terminate(): void { + this.state = { + isInitialized: false, + isProcessing: false, + error: null, + videoConfig: null, + audioConfig: null, + codecCapabilities: [], + }; + this.subscribers.clear(); + } + + private generateId(): string { + return `mock-${++this.messageId}`; + } + + private notifySubscribers(message: MediaWorkerMessage): void { + this.subscribers.forEach(callback => { + try { + callback(message); + } catch (error) { + console.error('Error in worker subscriber:', error); + } + }); + } + + private getMockCodecCapabilities(): CodecCapabilities[] { + return [ + { + supported: true, + codec: 'avc1.42E01E', // H.264 Baseline + hardwareAccelerated: true, + maxWidth: 4096, + maxHeight: 4096, + maxFrameRate: 60, + maxBitrate: 100000000, // 100 Mbps + }, + { + supported: true, + codec: 'vp8', + hardwareAccelerated: false, + maxWidth: 4096, + maxHeight: 4096, + maxFrameRate: 60, + maxBitrate: 50000000, // 50 Mbps + }, + { + supported: true, + codec: 'vp9', + hardwareAccelerated: true, + maxWidth: 8192, + maxHeight: 8192, + maxFrameRate: 60, + maxBitrate: 200000000, // 200 Mbps + }, + { + supported: true, + codec: 'av01.0.08M.08', // AV1 Main + hardwareAccelerated: true, + maxWidth: 8192, + maxHeight: 8192, + maxFrameRate: 60, + maxBitrate: 300000000, // 300 Mbps + }, + { + supported: true, + codec: 'mp4a.40.2', // AAC-LC + hardwareAccelerated: true, + maxBitrate: 320000, // 320 kbps + }, + { + supported: true, + codec: 'opus', + hardwareAccelerated: false, + maxBitrate: 256000, // 256 kbps + }, + ]; + } +} \ No newline at end of file diff --git a/packages/react-user-media/src/workers/types.ts b/packages/react-user-media/src/workers/types.ts new file mode 100644 index 0000000..036a18d --- /dev/null +++ b/packages/react-user-media/src/workers/types.ts @@ -0,0 +1,222 @@ +/** + * Core types for WebCodecs-based media worker operations + */ + +/** + * Message types that can be sent to/from the media worker + */ +export type MediaWorkerMessageType = + | 'INIT' + | 'PROCESS_VIDEO_FRAME' + | 'PROCESS_AUDIO_DATA' + | 'ENCODE_VIDEO' + | 'ENCODE_AUDIO' + | 'DECODE_VIDEO' + | 'DECODE_AUDIO' + | 'CONFIGURE_CODEC' + | 'GET_CODEC_CAPABILITIES' + | 'ERROR' + | 'SUCCESS' + | 'FRAME_PROCESSED' + | 'ENCODED_DATA'; + +/** + * Base message structure for worker communication + */ +export interface MediaWorkerMessage { + id: string; + type: MediaWorkerMessageType; + payload?: T; + error?: string; +} + +/** + * Video processing configuration + */ +export interface VideoProcessingConfig { + width?: number; + height?: number; + frameRate?: number; + bitrate?: number; + codec?: string; + format?: 'I420' | 'I422' | 'I444' | 'NV12' | 'RGBA' | 'BGRA' | 'RGB24' | 'BGR24'; +} + +/** + * Audio processing configuration + */ +export interface AudioProcessingConfig { + sampleRate?: number; + channels?: number; + bitDepth?: number; + codec?: string; + format?: 'f32' | 's16' | 's24' | 's32'; +} + +/** + * Video frame processing options + */ +export interface VideoFrameProcessingOptions { + operation: 'resize' | 'crop' | 'rotate' | 'filter' | 'convert_format' | 'extract_region'; + options?: { + width?: number; + height?: number; + x?: number; + y?: number; + angle?: number; + filter?: string; + format?: string; + }; +} + +/** + * Audio data processing options + */ +export interface AudioDataProcessingOptions { + operation: 'resample' | 'mix' | 'filter' | 'normalize' | 'convert_format' | 'extract_channels'; + options?: { + sampleRate?: number; + channels?: number; + format?: string; + filter?: string; + gain?: number; + }; +} + +/** + * Codec capabilities information + */ +export interface CodecCapabilities { + supported: boolean; + codec: string; + hardwareAccelerated: boolean; + maxWidth?: number; + maxHeight?: number; + maxFrameRate?: number; + maxBitrate?: number; +} + +/** + * Processed video frame result + */ +export interface ProcessedVideoFrame { + data: ArrayBuffer; + width: number; + height: number; + format: string; + timestamp: number; +} + +/** + * Processed audio data result + */ +export interface ProcessedAudioData { + data: ArrayBuffer; + sampleRate: number; + channels: number; + format: string; + duration: number; +} + +/** + * Worker state information + */ +export interface MediaWorkerState { + isInitialized: boolean; + isProcessing: boolean; + error: string | null; + videoConfig: VideoProcessingConfig | null; + audioConfig: AudioProcessingConfig | null; + codecCapabilities: CodecCapabilities[]; +} + +/** + * Worker controller interface for WebCodecs operations + */ +export interface MediaWorkerController { + /** + * Initialize the worker + */ + initialize(): Promise; + + /** + * Process a video frame + */ + processVideoFrame( + frame: VideoFrame, + options: VideoFrameProcessingOptions + ): Promise; + + /** + * Process audio data + */ + processAudioData( + data: AudioData, + options: AudioDataProcessingOptions + ): Promise; + + /** + * Encode video frames + */ + encodeVideo( + frames: VideoFrame[], + config: VideoProcessingConfig + ): Promise; + + /** + * Encode audio data + */ + encodeAudio( + data: AudioData[], + config: AudioProcessingConfig + ): Promise; + + /** + * Decode video chunks + */ + decodeVideo( + chunks: EncodedVideoChunk[], + config: VideoProcessingConfig + ): Promise; + + /** + * Decode audio chunks + */ + decodeAudio( + chunks: EncodedAudioChunk[], + config: AudioProcessingConfig + ): Promise; + + /** + * Configure codec + */ + configureCodec( + type: 'video' | 'audio', + config: VideoProcessingConfig | AudioProcessingConfig + ): Promise; + + /** + * Get codec capabilities + */ + getCodecCapabilities(): Promise; + + /** + * Get current worker state + */ + getState(): MediaWorkerState; + + /** + * Subscribe to worker events + */ + subscribe(callback: (message: MediaWorkerMessage) => void): () => void; + + /** + * Clean up and terminate worker + */ + terminate(): void; +} + +/** + * Worker factory function type + */ +export type MediaWorkerFactory = () => MediaWorkerController; \ No newline at end of file diff --git a/packages/react-user-media/src/workers/use-media-worker-recorder.ts b/packages/react-user-media/src/workers/use-media-worker-recorder.ts new file mode 100644 index 0000000..d210066 --- /dev/null +++ b/packages/react-user-media/src/workers/use-media-worker-recorder.ts @@ -0,0 +1,194 @@ +import { useCallback, useEffect, useMemo, useState } from 'react'; +import type { ShallowShapeOf } from '../types'; +import type { MediaWorkerRecordingConfig } from './types'; +import { useMediaWorker, type UseMediaWorkerConfig } from './use-media-worker'; + +/** + * Configuration for the media worker recorder + */ +export interface MediaWorkerRecorderConfig extends UseMediaWorkerConfig { + /** + * Default recording configuration + */ + defaultRecordingConfig?: MediaWorkerRecordingConfig; + + /** + * Auto-start recording when initialized + * @default false + */ + autoStartRecording?: boolean; +} + +/** + * State of the media worker recorder + */ +export interface MediaWorkerRecorderState { + /** + * Whether the recorder is initialized + */ + isInitialized: boolean; + + /** + * Whether the recorder is currently recording + */ + isRecording: boolean; + + /** + * Whether the recorder is in an error state + */ + isError: boolean; + + /** + * Current error message, if any + */ + error: string | null; + + /** + * Time when recording started + */ + startTime: number | null; + + /** + * Time when recording ended + */ + endTime: number | null; + + /** + * Recorded segments + */ + segments: Blob[]; + + /** + * MIME type of the recording + */ + mimeType: string | null; + + /** + * Whether recording is finalized (stopped and ready) + */ + isFinalized: boolean; + + /** + * Start recording with the given configuration + */ + startRecording(config?: MediaWorkerRecordingConfig): Promise; + + /** + * Stop recording and return recorded segments + */ + stopRecording(): Promise; + + /** + * Get the current recording configuration + */ + getRecordingConfig(): MediaWorkerRecordingConfig | null; + + /** + * Clear recorded segments + */ + clearSegments(): void; + + /** + * Get recording duration in milliseconds + */ + getDuration(): number | null; + + /** + * Terminate the recorder + */ + terminate(): void; +} + +/** + * Hook for managing media recording through workers + * @param config Configuration for the recorder + * @returns MediaWorkerRecorderState + */ +export function useMediaWorkerRecorder(config: MediaWorkerRecorderConfig = {}): MediaWorkerRecorderState { + const { + defaultRecordingConfig, + autoStartRecording = false, + ...workerConfig + } = config; + + const worker = useMediaWorker(workerConfig); + const [recordingConfig, setRecordingConfig] = useState( + defaultRecordingConfig || null + ); + + // Auto-start recording if configured + useEffect(() => { + if (autoStartRecording && worker.isInitialized && !worker.isRecording) { + startRecording(); + } + }, [autoStartRecording, worker.isInitialized, worker.isRecording]); + + const startRecording = useCallback(async (config?: MediaWorkerRecordingConfig): Promise => { + const finalConfig = config || recordingConfig || defaultRecordingConfig; + if (finalConfig) { + setRecordingConfig(finalConfig); + } + await worker.startRecording(finalConfig); + }, [worker, recordingConfig, defaultRecordingConfig]); + + const stopRecording = useCallback(async (): Promise => { + return await worker.stopRecording(); + }, [worker]); + + const getRecordingConfig = useCallback((): MediaWorkerRecordingConfig | null => { + return recordingConfig; + }, [recordingConfig]); + + const clearSegments = useCallback((): void => { + // This would need to be implemented in the worker + // For now, we'll just clear the local state + setRecordingConfig(null); + }, []); + + const getDuration = useCallback((): number | null => { + const { startTime, endTime } = worker.workerState; + if (startTime && endTime) { + return endTime - startTime; + } + if (startTime && worker.isRecording) { + return performance.now() - startTime; + } + return null; + }, [worker.workerState, worker.isRecording]); + + // Computed state + const isInitialized = useMemo(() => worker.isInitialized, [worker.isInitialized]); + const isRecording = useMemo(() => worker.isRecording, [worker.isRecording]); + const isError = useMemo(() => worker.isError, [worker.isError]); + const error = useMemo(() => worker.error, [worker.error]); + const startTime = useMemo(() => worker.workerState.recordingStartTime, [worker.workerState.recordingStartTime]); + const endTime = useMemo(() => worker.workerState.recordingEndTime, [worker.workerState.recordingEndTime]); + const segments = useMemo(() => worker.workerState.segments, [worker.workerState.segments]); + const mimeType = useMemo(() => worker.workerState.mimeType, [worker.workerState.mimeType]); + + const isFinalized = useMemo(() => { + return segments.length > 0 && + worker.workerState.recordingEndTime !== null && + !worker.isRecording; + }, [segments.length, worker.workerState.recordingEndTime, worker.isRecording]); + + const state: MediaWorkerRecorderState = { + isInitialized, + isRecording, + isError, + error, + startTime, + endTime, + segments, + mimeType, + isFinalized, + startRecording, + stopRecording, + getRecordingConfig, + clearSegments, + getDuration, + terminate: worker.terminate, + }; + + return state; +} \ No newline at end of file diff --git a/packages/react-user-media/src/workers/use-media-worker.ts b/packages/react-user-media/src/workers/use-media-worker.ts new file mode 100644 index 0000000..dfdace6 --- /dev/null +++ b/packages/react-user-media/src/workers/use-media-worker.ts @@ -0,0 +1,416 @@ +import { useCallback, useEffect, useMemo, useRef, useState } from 'react'; +import type { ShallowShapeOf } from '../types'; +import type { + MediaWorkerController, + MediaWorkerState, + VideoProcessingConfig, + AudioProcessingConfig, + VideoFrameProcessingOptions, + AudioDataProcessingOptions, + CodecCapabilities, + ProcessedVideoFrame, + ProcessedAudioData, + MediaWorkerMessage, +} from './types'; +import { getWorkerFactory } from './worker-factory'; + +/** + * Configuration for the useMediaWorker hook + */ +export interface UseMediaWorkerConfig { + /** + * Worker instance ID + * @default 'default' + */ + workerId?: string; + + /** + * Whether to use mock worker for testing + * @default false + */ + useMockWorker?: boolean; + + /** + * Custom worker script for production + */ + workerScript?: string; + + /** + * Auto-initialize worker on mount + * @default true + */ + autoInitialize?: boolean; +} + +/** + * State of the media worker hook + */ +export interface MediaWorkerHookState { + /** + * Whether the worker is initialized + */ + isInitialized: boolean; + + /** + * Whether the worker is currently processing media + */ + isProcessing: boolean; + + /** + * Whether the worker is in an error state + */ + isError: boolean; + + /** + * Current error message, if any + */ + error: string | null; + + /** + * Current worker state + */ + workerState: MediaWorkerState; + + /** + * Available codec capabilities + */ + codecCapabilities: CodecCapabilities[]; + + /** + * Whether codec capabilities are being loaded + */ + isLoadingCapabilities: boolean; + + /** + * Initialize the worker + */ + initialize(): Promise; + + /** + * Process a video frame + */ + processVideoFrame( + frame: VideoFrame, + options: VideoFrameProcessingOptions + ): Promise; + + /** + * Process audio data + */ + processAudioData( + data: AudioData, + options: AudioDataProcessingOptions + ): Promise; + + /** + * Encode video frames + */ + encodeVideo( + frames: VideoFrame[], + config: VideoProcessingConfig + ): Promise; + + /** + * Encode audio data + */ + encodeAudio( + data: AudioData[], + config: AudioProcessingConfig + ): Promise; + + /** + * Decode video chunks + */ + decodeVideo( + chunks: EncodedVideoChunk[], + config: VideoProcessingConfig + ): Promise; + + /** + * Decode audio chunks + */ + decodeAudio( + chunks: EncodedAudioChunk[], + config: AudioProcessingConfig + ): Promise; + + /** + * Configure codec + */ + configureCodec( + type: 'video' | 'audio', + config: VideoProcessingConfig | AudioProcessingConfig + ): Promise; + + /** + * Get codec capabilities + */ + getCodecCapabilities(): Promise; + + /** + * Subscribe to worker messages + */ + subscribe(callback: (message: MediaWorkerMessage) => void): () => void; + + /** + * Terminate the worker + */ + terminate(): void; +} + +/** + * Hook for managing media operations through workers + * @param config Configuration for the worker + * @returns MediaWorkerHookState + */ +export function useMediaWorker(config: UseMediaWorkerConfig = {}): MediaWorkerHookState { + const { + workerId = 'default', + useMockWorker = false, + workerScript, + autoInitialize = true, + } = config; + + const [workerState, setWorkerState] = useState({ + isInitialized: false, + isRecording: false, + isProcessing: false, + error: null, + recordingStartTime: null, + recordingEndTime: null, + segments: [], + mimeType: null, + }); + + const [codecCapabilities, setCodecCapabilities] = useState([]); + const [isLoadingCapabilities, setIsLoadingCapabilities] = useState(false); + const [error, setError] = useState(null); + + const workerRef = useRef(null); + const factoryRef = useRef(getWorkerFactory({ useMockWorker, workerScript })); + + // Update factory config when it changes + useEffect(() => { + factoryRef.current.updateConfig({ useMockWorker, workerScript }); + }, [useMockWorker, workerScript]); + + // Get or create worker instance + const getWorker = useCallback((): MediaWorkerController => { + if (!workerRef.current) { + workerRef.current = factoryRef.current.createWorker(workerId); + } + return workerRef.current; + }, [workerId]); + + // Auto-initialize worker + useEffect(() => { + if (autoInitialize && !workerState.isInitialized) { + initialize(); + } + }, [autoInitialize, workerState.isInitialized]); + + // Subscribe to worker state changes + useEffect(() => { + const worker = getWorker(); + const unsubscribe = worker.subscribe((message) => { + // Update local state based on worker messages + if (message.type === 'SUCCESS' || message.type === 'ERROR') { + const newState = worker.getState(); + setWorkerState(newState); + + if (message.type === 'ERROR') { + setError(message.error || 'Unknown error'); + } else { + setError(null); + } + } + }); + + return unsubscribe; + }, [getWorker]); + + const initialize = useCallback(async (): Promise => { + try { + setError(null); + const worker = getWorker(); + await worker.initialize(); + } catch (err) { + const errorMessage = err instanceof Error ? err.message : 'Failed to initialize worker'; + setError(errorMessage); + throw err; + } + }, [getWorker]); + + const processVideoFrame = useCallback(async ( + frame: VideoFrame, + options: VideoFrameProcessingOptions + ): Promise => { + try { + setError(null); + const worker = getWorker(); + return await worker.processVideoFrame(frame, options); + } catch (err) { + const errorMessage = err instanceof Error ? err.message : 'Failed to process video frame'; + setError(errorMessage); + throw err; + } + }, [getWorker]); + + const processAudioData = useCallback(async ( + data: AudioData, + options: AudioDataProcessingOptions + ): Promise => { + try { + setError(null); + const worker = getWorker(); + return await worker.processAudioData(data, options); + } catch (err) { + const errorMessage = err instanceof Error ? err.message : 'Failed to process audio data'; + setError(errorMessage); + throw err; + } + }, [getWorker]); + + const encodeVideo = useCallback(async ( + frames: VideoFrame[], + config: VideoProcessingConfig + ): Promise => { + try { + setError(null); + const worker = getWorker(); + return await worker.encodeVideo(frames, config); + } catch (err) { + const errorMessage = err instanceof Error ? err.message : 'Failed to encode video'; + setError(errorMessage); + throw err; + } + }, [getWorker]); + + const encodeAudio = useCallback(async ( + data: AudioData[], + config: AudioProcessingConfig + ): Promise => { + try { + setError(null); + const worker = getWorker(); + return await worker.encodeAudio(data, config); + } catch (err) { + const errorMessage = err instanceof Error ? err.message : 'Failed to encode audio'; + setError(errorMessage); + throw err; + } + }, [getWorker]); + + const decodeVideo = useCallback(async ( + chunks: EncodedVideoChunk[], + config: VideoProcessingConfig + ): Promise => { + try { + setError(null); + const worker = getWorker(); + return await worker.decodeVideo(chunks, config); + } catch (err) { + const errorMessage = err instanceof Error ? err.message : 'Failed to decode video'; + setError(errorMessage); + throw err; + } + }, [getWorker]); + + const decodeAudio = useCallback(async ( + chunks: EncodedAudioChunk[], + config: AudioProcessingConfig + ): Promise => { + try { + setError(null); + const worker = getWorker(); + return await worker.decodeAudio(chunks, config); + } catch (err) { + const errorMessage = err instanceof Error ? err.message : 'Failed to decode audio'; + setError(errorMessage); + throw err; + } + }, [getWorker]); + + const configureCodec = useCallback(async ( + type: 'video' | 'audio', + config: VideoProcessingConfig | AudioProcessingConfig + ): Promise => { + try { + setError(null); + const worker = getWorker(); + return await worker.configureCodec(type, config); + } catch (err) { + const errorMessage = err instanceof Error ? err.message : 'Failed to configure codec'; + setError(errorMessage); + throw err; + } + }, [getWorker]); + + const getCodecCapabilities = useCallback(async (): Promise => { + try { + setIsLoadingCapabilities(true); + setError(null); + const worker = getWorker(); + const capabilities = await worker.getCodecCapabilities(); + setCodecCapabilities(capabilities); + return capabilities; + } catch (err) { + const errorMessage = err instanceof Error ? err.message : 'Failed to get codec capabilities'; + setError(errorMessage); + throw err; + } finally { + setIsLoadingCapabilities(false); + } + }, [getWorker]); + + const subscribe = useCallback((callback: (message: MediaWorkerMessage) => void): (() => void) => { + const worker = getWorker(); + return worker.subscribe(callback); + }, [getWorker]); + + const terminate = useCallback((): void => { + if (workerRef.current) { + workerRef.current.terminate(); + workerRef.current = null; + } + factoryRef.current.removeWorker(workerId); + setWorkerState({ + isInitialized: false, + isProcessing: false, + error: null, + videoConfig: null, + audioConfig: null, + codecCapabilities: [], + }); + setCodecCapabilities([]); + setError(null); + }, [workerId]); + + // Computed state + const isInitialized = useMemo(() => workerState.isInitialized, [workerState.isInitialized]); + const isProcessing = useMemo(() => workerState.isProcessing, [workerState.isProcessing]); + const isError = useMemo(() => error !== null || workerState.error !== null, [error, workerState.error]); + + const state: MediaWorkerHookState = { + isInitialized, + isProcessing, + isError, + error: error || workerState.error, + workerState, + codecCapabilities, + isLoadingCapabilities, + initialize, + processVideoFrame, + processAudioData, + encodeVideo, + encodeAudio, + decodeVideo, + decodeAudio, + configureCodec, + getCodecCapabilities, + subscribe, + terminate, + }; + + return state; +} \ No newline at end of file diff --git a/packages/react-user-media/src/workers/web-worker.ts b/packages/react-user-media/src/workers/web-worker.ts new file mode 100644 index 0000000..7e1fc25 --- /dev/null +++ b/packages/react-user-media/src/workers/web-worker.ts @@ -0,0 +1,610 @@ +import type { + MediaWorkerController, + MediaWorkerMessage, + VideoProcessingConfig, + AudioProcessingConfig, + VideoFrameProcessingOptions, + AudioDataProcessingOptions, + CodecCapabilities, + ProcessedVideoFrame, + ProcessedAudioData, + MediaWorkerState, +} from './types'; + +/** + * Production implementation of MediaWorkerController using actual Web Workers + */ +export class WebMediaWorkerController implements MediaWorkerController { + private worker: Worker | null = null; + private state: MediaWorkerState = { + isInitialized: false, + isProcessing: false, + error: null, + videoConfig: null, + audioConfig: null, + codecCapabilities: [], + }; + + private subscribers: Set<(message: MediaWorkerMessage) => void> = new Set(); + private pendingRequests: Map void; + reject: (error: Error) => void; + }> = new Map(); + private messageId = 0; + + constructor(workerScript?: string) { + this.initializeWorker(workerScript); + } + + private initializeWorker(workerScript?: string): void { + try { + // Create worker from inline script or external file + if (workerScript) { + const blob = new Blob([workerScript], { type: 'application/javascript' }); + this.worker = new Worker(URL.createObjectURL(blob)); + } else { + // Use default worker implementation + this.worker = this.createDefaultWorker(); + } + + this.worker.onmessage = this.handleWorkerMessage.bind(this); + this.worker.onerror = this.handleWorkerError.bind(this); + } catch (error) { + this.state.error = `Failed to create worker: ${error}`; + console.error('Worker creation failed:', error); + } + } + + private createDefaultWorker(): Worker { + const workerScript = ` + // WebCodecs-based media worker implementation + let videoEncoder = null; + let audioEncoder = null; + let videoDecoder = null; + let audioDecoder = null; + let videoConfig = null; + let audioConfig = null; + let codecCapabilities = []; + + // Initialize codec capabilities + async function initCodecCapabilities() { + try { + // Check video codec support + const videoCodecs = [ + 'avc1.42E01E', // H.264 Baseline + 'vp8', + 'vp9', + 'av01.0.08M.08', // AV1 Main + ]; + + for (const codec of videoCodecs) { + try { + const support = await VideoEncoder.isConfigSupported({ + codec, + width: 1920, + height: 1080, + bitrate: 1000000, + framerate: 30, + }); + + codecCapabilities.push({ + supported: support.supported, + codec, + hardwareAccelerated: support.config?.hardwareAcceleration === 'prefer-hardware', + maxWidth: 4096, + maxHeight: 4096, + maxFrameRate: 60, + maxBitrate: 100000000, + }); + } catch (e) { + // Codec not supported + } + } + + // Check audio codec support + const audioCodecs = [ + 'mp4a.40.2', // AAC-LC + 'opus', + ]; + + for (const codec of audioCodecs) { + try { + const support = await AudioEncoder.isConfigSupported({ + codec, + sampleRate: 48000, + numberOfChannels: 2, + bitrate: 128000, + }); + + codecCapabilities.push({ + supported: support.supported, + codec, + hardwareAccelerated: support.config?.hardwareAcceleration === 'prefer-hardware', + maxBitrate: 320000, + }); + } catch (e) { + // Codec not supported + } + } + } catch (error) { + console.error('Error initializing codec capabilities:', error); + } + } + + self.onmessage = async function(e) { + const { id, type, payload } = e.data; + + try { + switch (type) { + case 'INIT': + await initCodecCapabilities(); + self.postMessage({ + id, + type: 'SUCCESS', + payload: { + message: 'Worker initialized', + capabilities: codecCapabilities + } + }); + break; + + case 'PROCESS_VIDEO_FRAME': + // Process video frame using WebCodecs + const frame = payload.frame; + const options = payload.options; + + // Create a canvas to process the frame + const canvas = new OffscreenCanvas(frame.displayWidth, frame.displayHeight); + const ctx = canvas.getContext('2d'); + + // Draw frame to canvas + ctx.drawImage(frame, 0, 0); + + // Apply processing based on options + if (options.operation === 'resize' && options.options) { + const { width, height } = options.options; + const resizedCanvas = new OffscreenCanvas(width, height); + const resizedCtx = resizedCanvas.getContext('2d'); + resizedCtx.drawImage(canvas, 0, 0, width, height); + + const imageData = resizedCtx.getImageData(0, 0, width, height); + const processedFrame = { + data: imageData.data.buffer, + width, + height, + format: 'RGBA', + timestamp: performance.now(), + }; + + self.postMessage({ + id, + type: 'FRAME_PROCESSED', + payload: { frame: processedFrame } + }); + } else { + // Default processing + const imageData = ctx.getImageData(0, 0, canvas.width, canvas.height); + const processedFrame = { + data: imageData.data.buffer, + width: canvas.width, + height: canvas.height, + format: 'RGBA', + timestamp: performance.now(), + }; + + self.postMessage({ + id, + type: 'FRAME_PROCESSED', + payload: { frame: processedFrame } + }); + } + break; + + case 'PROCESS_AUDIO_DATA': + // Process audio data using WebCodecs + const audioData = payload.data; + const audioOptions = payload.options; + + // Simple audio processing simulation + const processedAudio = { + data: audioData.data.buffer, + sampleRate: audioOptions.options?.sampleRate || audioData.sampleRate, + channels: audioOptions.options?.channels || audioData.numberOfChannels, + format: audioOptions.options?.format || 'f32', + duration: audioData.numberOfFrames / audioData.sampleRate, + }; + + self.postMessage({ + id, + type: 'SUCCESS', + payload: { audio: processedAudio } + }); + break; + + case 'ENCODE_VIDEO': + // Encode video frames + const frames = payload.frames; + const vConfig = payload.config; + + if (!videoEncoder) { + videoEncoder = new VideoEncoder({ + output: (chunk) => { + self.postMessage({ + id, + type: 'ENCODED_DATA', + payload: { chunks: [chunk], type: 'video' } + }); + }, + error: (error) => { + self.postMessage({ + id, + type: 'ERROR', + error: error.message + }); + } + }); + } + + videoEncoder.configure({ + codec: vConfig.codec || 'vp8', + width: vConfig.width || 1920, + height: vConfig.height || 1080, + bitrate: vConfig.bitrate || 1000000, + framerate: vConfig.frameRate || 30, + }); + + for (const frame of frames) { + videoEncoder.encode(frame); + } + + self.postMessage({ + id, + type: 'SUCCESS', + payload: { message: 'Video encoding started' } + }); + break; + + case 'ENCODE_AUDIO': + // Encode audio data + const audioFrames = payload.data; + const aConfig = payload.config; + + if (!audioEncoder) { + audioEncoder = new AudioEncoder({ + output: (chunk) => { + self.postMessage({ + id, + type: 'ENCODED_DATA', + payload: { chunks: [chunk], type: 'audio' } + }); + }, + error: (error) => { + self.postMessage({ + id, + type: 'ERROR', + error: error.message + }); + } + }); + } + + audioEncoder.configure({ + codec: aConfig.codec || 'mp4a.40.2', + sampleRate: aConfig.sampleRate || 48000, + numberOfChannels: aConfig.channels || 2, + bitrate: aConfig.bitrate || 128000, + }); + + for (const audioData of audioFrames) { + audioEncoder.encode(audioData); + } + + self.postMessage({ + id, + type: 'SUCCESS', + payload: { message: 'Audio encoding started' } + }); + break; + + case 'CONFIGURE_CODEC': + const codecType = payload.type; + const config = payload.config; + + if (codecType === 'video') { + videoConfig = config; + } else { + audioConfig = config; + } + + self.postMessage({ + id, + type: 'SUCCESS', + payload: { message: \`\${codecType} codec configured\` } + }); + break; + + case 'GET_CODEC_CAPABILITIES': + self.postMessage({ + id, + type: 'SUCCESS', + payload: { capabilities: codecCapabilities } + }); + break; + + default: + throw new Error(\`Unknown message type: \${type}\`); + } + } catch (error) { + self.postMessage({ + id, + type: 'ERROR', + error: error.message + }); + } + }; + `; + + const blob = new Blob([workerScript], { type: 'application/javascript' }); + return new Worker(URL.createObjectURL(blob)); + } + + private handleWorkerMessage(event: MessageEvent): void { + const message: MediaWorkerMessage = event.data; + + // Handle pending requests + const pendingRequest = this.pendingRequests.get(message.id); + if (pendingRequest) { + this.pendingRequests.delete(message.id); + + if (message.type === 'ERROR') { + pendingRequest.reject(new Error(message.error || 'Unknown worker error')); + } else { + pendingRequest.resolve(message.payload); + } + } + + // Update state based on message + this.updateStateFromMessage(message); + + // Notify subscribers + this.notifySubscribers(message); + } + + private handleWorkerError(error: ErrorEvent): void { + this.state.error = `Worker error: ${error.message}`; + console.error('Worker error:', error); + + this.notifySubscribers({ + id: this.generateId(), + type: 'ERROR', + error: error.message, + }); + } + + private updateStateFromMessage(message: MediaWorkerMessage): void { + switch (message.type) { + case 'SUCCESS': + if (message.payload?.message === 'Worker initialized') { + this.state.isInitialized = true; + this.state.error = null; + this.state.codecCapabilities = message.payload.capabilities || []; + } else if (message.payload?.message?.includes('codec configured')) { + // Codec configuration handled in individual methods + } + break; + case 'ERROR': + this.state.error = message.error || 'Unknown error'; + break; + case 'FRAME_PROCESSED': + // Frame processing completed + break; + case 'ENCODED_DATA': + // Encoding completed + break; + } + } + + async initialize(): Promise { + return this.sendMessage('INIT'); + } + + async processVideoFrame( + frame: VideoFrame, + options: VideoFrameProcessingOptions + ): Promise { + if (!this.state.isInitialized) { + throw new Error('Worker not initialized'); + } + + this.state.isProcessing = true; + + try { + const result = await this.sendMessage('PROCESS_VIDEO_FRAME', { frame, options }); + return result.frame; + } finally { + this.state.isProcessing = false; + } + } + + async processAudioData( + data: AudioData, + options: AudioDataProcessingOptions + ): Promise { + if (!this.state.isInitialized) { + throw new Error('Worker not initialized'); + } + + this.state.isProcessing = true; + + try { + const result = await this.sendMessage('PROCESS_AUDIO_DATA', { data, options }); + return result.audio; + } finally { + this.state.isProcessing = false; + } + } + + async encodeVideo( + frames: VideoFrame[], + config: VideoProcessingConfig + ): Promise { + if (!this.state.isInitialized) { + throw new Error('Worker not initialized'); + } + + this.state.isProcessing = true; + this.state.videoConfig = config; + + try { + const result = await this.sendMessage('ENCODE_VIDEO', { frames, config }); + return result.chunks || []; + } finally { + this.state.isProcessing = false; + } + } + + async encodeAudio( + data: AudioData[], + config: AudioProcessingConfig + ): Promise { + if (!this.state.isInitialized) { + throw new Error('Worker not initialized'); + } + + this.state.isProcessing = true; + this.state.audioConfig = config; + + try { + const result = await this.sendMessage('ENCODE_AUDIO', { data, config }); + return result.chunks || []; + } finally { + this.state.isProcessing = false; + } + } + + async decodeVideo( + chunks: EncodedVideoChunk[], + config: VideoProcessingConfig + ): Promise { + if (!this.state.isInitialized) { + throw new Error('Worker not initialized'); + } + + this.state.isProcessing = true; + + try { + const result = await this.sendMessage('DECODE_VIDEO', { chunks, config }); + return result.frames || []; + } finally { + this.state.isProcessing = false; + } + } + + async decodeAudio( + chunks: EncodedAudioChunk[], + config: AudioProcessingConfig + ): Promise { + if (!this.state.isInitialized) { + throw new Error('Worker not initialized'); + } + + this.state.isProcessing = true; + + try { + const result = await this.sendMessage('DECODE_AUDIO', { chunks, config }); + return result.audioData || []; + } finally { + this.state.isProcessing = false; + } + } + + async configureCodec( + type: 'video' | 'audio', + config: VideoProcessingConfig | AudioProcessingConfig + ): Promise { + if (!this.state.isInitialized) { + throw new Error('Worker not initialized'); + } + + return this.sendMessage('CONFIGURE_CODEC', { type, config }); + } + + async getCodecCapabilities(): Promise { + if (!this.state.isInitialized) { + throw new Error('Worker not initialized'); + } + + const result = await this.sendMessage('GET_CODEC_CAPABILITIES'); + return result.capabilities || []; + } + + getState(): MediaWorkerState { + return { ...this.state }; + } + + subscribe(callback: (message: MediaWorkerMessage) => void): () => void { + this.subscribers.add(callback); + return () => { + this.subscribers.delete(callback); + }; + } + + terminate(): void { + if (this.worker) { + this.worker.terminate(); + this.worker = null; + } + + this.state = { + isInitialized: false, + isProcessing: false, + error: null, + videoConfig: null, + audioConfig: null, + codecCapabilities: [], + }; + + this.subscribers.clear(); + this.pendingRequests.clear(); + } + + private sendMessage(type: string, payload?: any): Promise { + return new Promise((resolve, reject) => { + if (!this.worker) { + reject(new Error('Worker not available')); + return; + } + + const id = this.generateId(); + this.pendingRequests.set(id, { resolve, reject }); + + this.worker.postMessage({ + id, + type, + payload, + }); + + // Timeout after 10 seconds + setTimeout(() => { + if (this.pendingRequests.has(id)) { + this.pendingRequests.delete(id); + reject(new Error('Worker request timeout')); + } + }, 10000); + }); + } + + private generateId(): string { + return `web-${++this.messageId}-${Date.now()}`; + } + + private notifySubscribers(message: MediaWorkerMessage): void { + this.subscribers.forEach(callback => { + try { + callback(message); + } catch (error) { + console.error('Error in worker subscriber:', error); + } + }); + } +} \ No newline at end of file diff --git a/packages/react-user-media/src/workers/worker-factory.ts b/packages/react-user-media/src/workers/worker-factory.ts new file mode 100644 index 0000000..a43289d --- /dev/null +++ b/packages/react-user-media/src/workers/worker-factory.ts @@ -0,0 +1,166 @@ +import type { MediaWorkerController, MediaWorkerFactory } from './types'; +import { MockMediaWorkerController } from './mock-worker'; +import { WebMediaWorkerController } from './web-worker'; + +/** + * Configuration for worker factory + */ +export interface WorkerFactoryConfig { + /** + * Whether to use mock worker for testing/development + * @default false + */ + useMockWorker?: boolean; + + /** + * Custom worker script for production workers + */ + workerScript?: string; + + /** + * Environment detection function + * @default () => typeof Worker !== 'undefined' + */ + isWorkerSupported?: () => boolean; +} + +/** + * Default configuration + */ +const defaultConfig: Required = { + useMockWorker: false, + workerScript: undefined, + isWorkerSupported: () => typeof Worker !== 'undefined', +}; + +/** + * Worker factory for creating appropriate worker implementations + */ +export class MediaWorkerFactory { + private config: Required; + private workerInstances: Map = new Map(); + + constructor(config: WorkerFactoryConfig = {}) { + this.config = { ...defaultConfig, ...config }; + } + + /** + * Create a new worker instance + * @param id Unique identifier for the worker instance + * @returns MediaWorkerController instance + */ + createWorker(id: string = 'default'): MediaWorkerController { + // Return existing instance if available + if (this.workerInstances.has(id)) { + return this.workerInstances.get(id)!; + } + + let worker: MediaWorkerController; + + if (this.config.useMockWorker) { + worker = new MockMediaWorkerController(); + } else if (this.config.isWorkerSupported()) { + worker = new WebMediaWorkerController(this.config.workerScript); + } else { + console.warn('Web Workers not supported, falling back to mock worker'); + worker = new MockMediaWorkerController(); + } + + this.workerInstances.set(id, worker); + return worker; + } + + /** + * Get an existing worker instance + * @param id Worker instance identifier + * @returns MediaWorkerController instance or undefined + */ + getWorker(id: string = 'default'): MediaWorkerController | undefined { + return this.workerInstances.get(id); + } + + /** + * Remove and terminate a worker instance + * @param id Worker instance identifier + */ + removeWorker(id: string = 'default'): void { + const worker = this.workerInstances.get(id); + if (worker) { + worker.terminate(); + this.workerInstances.delete(id); + } + } + + /** + * Remove and terminate all worker instances + */ + removeAllWorkers(): void { + this.workerInstances.forEach(worker => worker.terminate()); + this.workerInstances.clear(); + } + + /** + * Update factory configuration + * @param config New configuration + */ + updateConfig(config: Partial): void { + this.config = { ...this.config, ...config }; + } + + /** + * Get current configuration + */ + getConfig(): Required { + return { ...this.config }; + } + + /** + * Check if workers are supported in current environment + */ + isWorkerSupported(): boolean { + return this.config.isWorkerSupported(); + } + + /** + * Get list of active worker IDs + */ + getActiveWorkerIds(): string[] { + return Array.from(this.workerInstances.keys()); + } +} + +/** + * Global worker factory instance + */ +let globalWorkerFactory: MediaWorkerFactory | null = null; + +/** + * Get or create the global worker factory + * @param config Optional configuration for the factory + * @returns MediaWorkerFactory instance + */ +export function getWorkerFactory(config?: WorkerFactoryConfig): MediaWorkerFactory { + if (!globalWorkerFactory) { + globalWorkerFactory = new MediaWorkerFactory(config); + } + return globalWorkerFactory; +} + +/** + * Create a worker factory function + * @param config Factory configuration + * @returns MediaWorkerFactory function + */ +export function createWorkerFactory(config?: WorkerFactoryConfig): MediaWorkerFactory { + return new MediaWorkerFactory(config); +} + +/** + * Default factory function for creating workers + * @param config Optional configuration + * @returns MediaWorkerController instance + */ +export const createMediaWorker: MediaWorkerFactory = (config?: WorkerFactoryConfig) => { + const factory = getWorkerFactory(config); + return factory.createWorker(); +}; \ No newline at end of file