From eaea70b67f28536d78ddd09aa11d7a33f2605aa3 Mon Sep 17 00:00:00 2001 From: Kevin Hanna Date: Tue, 4 Nov 2025 17:59:53 +0000 Subject: [PATCH 1/2] media: add Safari12 handler as Safari17 handler this is taken directly from the mediasoup-client repo (with imports adjusted) the next commit will add the changes that make it work with maxBandwidth for clarity --- .../media/src/utils/getMediasoupDevice.ts | 10 +- .../webrtc/VegaRtcManager/Safari17Handler.ts | 1272 +++++++++++++++++ 2 files changed, 1281 insertions(+), 1 deletion(-) create mode 100644 packages/media/src/webrtc/VegaRtcManager/Safari17Handler.ts diff --git a/packages/media/src/utils/getMediasoupDevice.ts b/packages/media/src/utils/getMediasoupDevice.ts index d177bf870..f35242622 100644 --- a/packages/media/src/utils/getMediasoupDevice.ts +++ b/packages/media/src/utils/getMediasoupDevice.ts @@ -1,7 +1,11 @@ import { detectDeviceAsync, Device } from "mediasoup-client"; import { BuiltinHandlerName } from "mediasoup-client/lib/types"; +import { Safari17 } from "../webrtc/VegaRtcManager/Safari17Handler"; -type SupportedDevice = BuiltinHandlerName | "NodeJS" | undefined; +type SupportedDevice = BuiltinHandlerName | "NodeJS" | "Safari17" | undefined; +const SAFARI_17_REGEXP = new RegExp( + /^(?=.*\bSafari\/)(?!.*\b(?:Chrome|Chromium|CriOS|Edg|OPR)\b).*?\bVersion\/(?:1[8-9]|[2-9]\d|17\.(?:0\.(?:[1-9]\d*)|[1-9]\d*))\b/, +); export const getMediasoupDeviceAsync = async (features: Record): Promise => { if (features.isNodeSdk) { return new Device({ handlerName: "Safari12" }); @@ -16,5 +20,9 @@ export const getMediasoupDeviceAsync = async (features: Record + implements HandlerInterface +{ + // Closed flag. + private _closed = false; + // Handler direction. + private _direction: 'send' | 'recv'; + // Remote SDP handler. + private _remoteSdp: RemoteSdp; + // Callback to request sending extended RTP capabilities on demand. + private _getSendExtendedRtpCapabilities: ( + nativeRtpCapabilities: RtpCapabilities + ) => ExtendedRtpCapabilities; + // Initial server side DTLS role. If not 'auto', it will force the opposite + // value in client side. + private _forcedLocalDtlsRole?: DtlsRole; + // RTCPeerConnection instance. + private _pc: RTCPeerConnection; + // Map of RTCTransceivers indexed by MID. + private readonly _mapMidTransceiver: Map = + new Map(); + // Local stream for sending. + private readonly _sendStream = new MediaStream(); + // Whether a DataChannel m=application section has been created. + private _hasDataChannelMediaSection = false; + // Sending DataChannel id value counter. Incremented for each new DataChannel. + private _nextSendSctpStreamId = 0; + // Got transport local and remote parameters. + private _transportReady = false; + + /** + * Creates a factory function. + */ + static createFactory(): HandlerFactory { + return { + name: NAME, + factory: (options: HandlerOptions): Safari17 => new Safari17(options), + getNativeRtpCapabilities: async (): Promise => { + logger.debug('getNativeRtpCapabilities()'); + + let pc: RTCPeerConnection | undefined = new RTCPeerConnection({ + iceServers: [], + iceTransportPolicy: 'all', + bundlePolicy: 'max-bundle', + rtcpMuxPolicy: 'require', + }); + + try { + pc.addTransceiver('audio'); + pc.addTransceiver('video'); + + const offer = await pc.createOffer(); + + try { + pc.close(); + } catch (error) {} + + pc = undefined; + + const sdpObject = sdpTransform.parse(offer.sdp!); + const nativeRtpCapabilities = + Safari17.getLocalRtpCapabilities(sdpObject); + + return nativeRtpCapabilities; + } catch (error) { + try { + pc?.close(); + } catch (error2) {} + + pc = undefined; + + throw error; + } + }, + getNativeSctpCapabilities: async (): Promise => { + logger.debug('getNativeSctpCapabilities()'); + + return { + numStreams: SCTP_NUM_STREAMS, + }; + }, + }; + } + + private static getLocalRtpCapabilities( + localSdpObject: SdpTransform.SessionDescription + ): RtpCapabilities { + const nativeRtpCapabilities = sdpCommonUtils.extractRtpCapabilities({ + sdpObject: localSdpObject, + }); + + // Need to validate and normalize native RTP capabilities. + ortc.validateAndNormalizeRtpCapabilities(nativeRtpCapabilities); + + // libwebrtc supports NACK for OPUS but doesn't announce it. + ortcUtils.addNackSupportForOpus(nativeRtpCapabilities); + + return nativeRtpCapabilities; + } + + constructor({ + direction, + iceParameters, + iceCandidates, + dtlsParameters, + sctpParameters, + iceServers, + iceTransportPolicy, + additionalSettings, + getSendExtendedRtpCapabilities, + }: HandlerOptions) { + super(); + + logger.debug('constructor()'); + + this._direction = direction; + + this._remoteSdp = new RemoteSdp({ + iceParameters, + iceCandidates, + dtlsParameters, + sctpParameters, + }); + + this._getSendExtendedRtpCapabilities = getSendExtendedRtpCapabilities; + + if (dtlsParameters.role && dtlsParameters.role !== 'auto') { + this._forcedLocalDtlsRole = + dtlsParameters.role === 'server' ? 'client' : 'server'; + } + + this._pc = new RTCPeerConnection({ + iceServers: iceServers ?? [], + iceTransportPolicy: iceTransportPolicy ?? 'all', + bundlePolicy: 'max-bundle', + rtcpMuxPolicy: 'require', + ...additionalSettings, + }); + + this._pc.addEventListener('icegatheringstatechange', () => { + this.emit('@icegatheringstatechange', this._pc.iceGatheringState); + }); + + this._pc.addEventListener( + 'icecandidateerror', + (event: RTCPeerConnectionIceErrorEvent) => { + this.emit('@icecandidateerror', event); + } + ); + + this._pc.addEventListener( + 'icegatheringstatechange', + this.onIceGatheringStateChange + ); + + this._pc.addEventListener('icecandidateerror', this.onIceCandidateError); + + if (this._pc.connectionState) { + this._pc.addEventListener( + 'connectionstatechange', + this.onConnectionStateChange + ); + } else { + logger.warn( + 'run() | pc.connectionState not supported, using pc.iceConnectionState' + ); + + this._pc.addEventListener( + 'iceconnectionstatechange', + this.onIceConnectionStateChange + ); + } + } + + get name(): string { + return NAME; + } + + override close(): void { + logger.debug('close()'); + + if (this._closed) { + return; + } + + this._closed = true; + + // Close RTCPeerConnection. + try { + this._pc.close(); + } catch (error) {} + + this._pc.removeEventListener( + 'icegatheringstatechange', + this.onIceGatheringStateChange + ); + + this._pc.removeEventListener('icecandidateerror', this.onIceCandidateError); + + this._pc.removeEventListener( + 'connectionstatechange', + this.onConnectionStateChange + ); + + this._pc.removeEventListener( + 'iceconnectionstatechange', + this.onIceConnectionStateChange + ); + + this.emit('@close'); + + // Invoke close() in EnhancedEventEmitter classes. + super.close(); + } + + async updateIceServers(iceServers: RTCIceServer[]): Promise { + this.assertNotClosed(); + + logger.debug('updateIceServers()'); + + const configuration = this._pc.getConfiguration(); + + configuration.iceServers = iceServers; + + this._pc.setConfiguration(configuration); + } + + async restartIce(iceParameters: IceParameters): Promise { + this.assertNotClosed(); + + logger.debug('restartIce()'); + + // Provide the remote SDP handler with new remote ICE parameters. + this._remoteSdp.updateIceParameters(iceParameters); + + if (!this._transportReady) { + return; + } + + if (this._direction === 'send') { + const offer = await this._pc.createOffer({ iceRestart: true }); + + logger.debug( + 'restartIce() | calling pc.setLocalDescription() [offer:%o]', + offer + ); + + await this._pc.setLocalDescription(offer); + + const answer = { + type: 'answer' as RTCSdpType, + sdp: this._remoteSdp.getSdp(), + }; + + logger.debug( + 'restartIce() | calling pc.setRemoteDescription() [answer:%o]', + answer + ); + + await this._pc.setRemoteDescription(answer); + } else { + const offer = { + type: 'offer' as RTCSdpType, + sdp: this._remoteSdp.getSdp(), + }; + + logger.debug( + 'restartIce() | calling pc.setRemoteDescription() [offer:%o]', + offer + ); + + await this._pc.setRemoteDescription(offer); + + const answer = await this._pc.createAnswer(); + + logger.debug( + 'restartIce() | calling pc.setLocalDescription() [answer:%o]', + answer + ); + + await this._pc.setLocalDescription(answer); + } + } + + async getTransportStats(): Promise { + this.assertNotClosed(); + + return this._pc.getStats(); + } + + async send({ + track, + encodings, + codecOptions, + codec, + onRtpSender, + }: HandlerSendOptions): Promise { + this.assertNotClosed(); + this.assertSendDirection(); + + logger.debug('send() [kind:%s, track.id:%s]', track.kind, track.id); + + const mediaSectionIdx = this._remoteSdp.getNextMediaSectionIdx(); + const transceiver = this._pc.addTransceiver(track, { + direction: 'sendonly', + streams: [this._sendStream], + }); + + if (onRtpSender) { + onRtpSender(transceiver.sender); + } + + let offer = await this._pc.createOffer(); + let localSdpObject = sdpTransform.parse(offer.sdp!); + + // @ts-expect-error property doesn't exist + if (localSdpObject.extmapAllowMixed) { + this._remoteSdp.setSessionExtmapAllowMixed(); + } + + const nativeRtpCapabilities = + Safari17.getLocalRtpCapabilities(localSdpObject); + const sendExtendedRtpCapabilities = this._getSendExtendedRtpCapabilities( + nativeRtpCapabilities + ); + + // Generic sending RTP parameters. + const sendingRtpParameters = ortc.getSendingRtpParameters( + track.kind as MediaKind, + sendExtendedRtpCapabilities + ); + + // This may throw. + sendingRtpParameters.codecs = ortc.reduceCodecs( + sendingRtpParameters.codecs, + codec + ); + + // Generic sending RTP parameters suitable for the SDP remote answer. + const sendingRemoteRtpParameters = ortc.getSendingRemoteRtpParameters( + track.kind as MediaKind, + sendExtendedRtpCapabilities + ); + + // This may throw. + sendingRemoteRtpParameters.codecs = ortc.reduceCodecs( + sendingRemoteRtpParameters.codecs, + codec + ); + + let offerMediaObject; + + if (!this._transportReady) { + await this.setupTransport({ + localDtlsRole: this._forcedLocalDtlsRole ?? 'client', + localSdpObject, + }); + } + + const layers = parseScalabilityMode( + (encodings ?? [{}])[0]!.scalabilityMode + ); + + if (encodings && encodings.length > 1) { + logger.debug('send() | enabling legacy simulcast'); + + localSdpObject = sdpTransform.parse(offer.sdp!); + offerMediaObject = localSdpObject.media[mediaSectionIdx.idx]!; + + sdpUnifiedPlanUtils.addLegacySimulcast({ + offerMediaObject, + numStreams: encodings.length, + }); + + offer = { + type: 'offer' as RTCSdpType, + sdp: sdpTransform.write(localSdpObject), + }; + } + + logger.debug('send() | calling pc.setLocalDescription() [offer:%o]', offer); + + await this._pc.setLocalDescription(offer); + + // We can now get the transceiver.mid. + const localId = transceiver.mid!; + + // Set MID. + sendingRtpParameters.mid = localId; + + localSdpObject = sdpTransform.parse(this._pc.localDescription!.sdp); + offerMediaObject = localSdpObject.media[mediaSectionIdx.idx]!; + + // Set RTCP CNAME. + sendingRtpParameters.rtcp!.cname = sdpCommonUtils.getCname({ + offerMediaObject, + }); + + // Set RTP encodings. + sendingRtpParameters.encodings = sdpUnifiedPlanUtils.getRtpEncodings({ + offerMediaObject, + }); + + // Complete encodings with given values. + if (encodings) { + for (let idx = 0; idx < sendingRtpParameters.encodings.length; ++idx) { + if (encodings[idx]) { + Object.assign(sendingRtpParameters.encodings[idx]!, encodings[idx]); + } + } + } + + // If VP8 or H264 and there is effective simulcast, add scalabilityMode to + // each encoding. + if ( + sendingRtpParameters.encodings.length > 1 && + (sendingRtpParameters.codecs[0]!.mimeType.toLowerCase() === 'video/vp8' || + sendingRtpParameters.codecs[0]!.mimeType.toLowerCase() === 'video/h264') + ) { + for (const encoding of sendingRtpParameters.encodings) { + if (encoding.scalabilityMode) { + encoding.scalabilityMode = `L1T${layers.temporalLayers}`; + } else { + encoding.scalabilityMode = 'L1T3'; + } + } + } + + this._remoteSdp.send({ + offerMediaObject, + reuseMid: mediaSectionIdx.reuseMid, + offerRtpParameters: sendingRtpParameters, + answerRtpParameters: sendingRemoteRtpParameters, + codecOptions, + }); + + const answer = { + type: 'answer' as RTCSdpType, + sdp: this._remoteSdp.getSdp(), + }; + + logger.debug( + 'send() | calling pc.setRemoteDescription() [answer:%o]', + answer + ); + + await this._pc.setRemoteDescription(answer); + + // Store in the map. + this._mapMidTransceiver.set(localId, transceiver); + + return { + localId, + rtpParameters: sendingRtpParameters, + rtpSender: transceiver.sender, + }; + } + + async stopSending(localId: string): Promise { + this.assertSendDirection(); + + if (this._closed) { + return; + } + + logger.debug('stopSending() [localId:%s]', localId); + + const transceiver = this._mapMidTransceiver.get(localId); + + if (!transceiver) { + throw new Error('associated RTCRtpTransceiver not found'); + } + + void transceiver.sender.replaceTrack(null); + + this._pc.removeTrack(transceiver.sender); + + const mediaSectionClosed = this._remoteSdp.closeMediaSection( + transceiver.mid! + ); + + if (mediaSectionClosed) { + try { + transceiver.stop(); + } catch (error) {} + } + + const offer = await this._pc.createOffer(); + + logger.debug( + 'stopSending() | calling pc.setLocalDescription() [offer:%o]', + offer + ); + + await this._pc.setLocalDescription(offer); + + const answer = { + type: 'answer' as RTCSdpType, + sdp: this._remoteSdp.getSdp(), + }; + + logger.debug( + 'stopSending() | calling pc.setRemoteDescription() [answer:%o]', + answer + ); + + await this._pc.setRemoteDescription(answer); + + this._mapMidTransceiver.delete(localId); + } + + async pauseSending(localId: string): Promise { + this.assertNotClosed(); + this.assertSendDirection(); + + logger.debug('pauseSending() [localId:%s]', localId); + + const transceiver = this._mapMidTransceiver.get(localId); + + if (!transceiver) { + throw new Error('associated RTCRtpTransceiver not found'); + } + + transceiver.direction = 'inactive'; + this._remoteSdp.pauseMediaSection(localId); + + const offer = await this._pc.createOffer(); + + logger.debug( + 'pauseSending() | calling pc.setLocalDescription() [offer:%o]', + offer + ); + + await this._pc.setLocalDescription(offer); + + const answer = { + type: 'answer' as RTCSdpType, + sdp: this._remoteSdp.getSdp(), + }; + + logger.debug( + 'pauseSending() | calling pc.setRemoteDescription() [answer:%o]', + answer + ); + + await this._pc.setRemoteDescription(answer); + } + + async resumeSending(localId: string): Promise { + this.assertNotClosed(); + this.assertSendDirection(); + + logger.debug('resumeSending() [localId:%s]', localId); + + const transceiver = this._mapMidTransceiver.get(localId); + + if (!transceiver) { + throw new Error('associated RTCRtpTransceiver not found'); + } + + transceiver.direction = 'sendonly'; + this._remoteSdp.resumeSendingMediaSection(localId); + + const offer = await this._pc.createOffer(); + + logger.debug( + 'resumeSending() | calling pc.setLocalDescription() [offer:%o]', + offer + ); + + await this._pc.setLocalDescription(offer); + + const answer = { + type: 'answer' as RTCSdpType, + sdp: this._remoteSdp.getSdp(), + }; + + logger.debug( + 'resumeSending() | calling pc.setRemoteDescription() [answer:%o]', + answer + ); + + await this._pc.setRemoteDescription(answer); + } + + async replaceTrack( + localId: string, + track: MediaStreamTrack | null + ): Promise { + this.assertNotClosed(); + this.assertSendDirection(); + + if (track) { + logger.debug( + 'replaceTrack() [localId:%s, track.id:%s]', + localId, + track.id + ); + } else { + logger.debug('replaceTrack() [localId:%s, no track]', localId); + } + + const transceiver = this._mapMidTransceiver.get(localId); + + if (!transceiver) { + throw new Error('associated RTCRtpTransceiver not found'); + } + + await transceiver.sender.replaceTrack(track); + } + + async setMaxSpatialLayer( + localId: string, + spatialLayer: number + ): Promise { + this.assertNotClosed(); + this.assertSendDirection(); + + logger.debug( + 'setMaxSpatialLayer() [localId:%s, spatialLayer:%s]', + localId, + spatialLayer + ); + + const transceiver = this._mapMidTransceiver.get(localId); + + if (!transceiver) { + throw new Error('associated RTCRtpTransceiver not found'); + } + + const parameters = transceiver.sender.getParameters(); + + parameters.encodings.forEach( + (encoding: RTCRtpEncodingParameters, idx: number) => { + if (idx <= spatialLayer) { + encoding.active = true; + } else { + encoding.active = false; + } + } + ); + + await transceiver.sender.setParameters(parameters); + + this._remoteSdp.muxMediaSectionSimulcast(localId, parameters.encodings); + + const offer = await this._pc.createOffer(); + + logger.debug( + 'setMaxSpatialLayer() | calling pc.setLocalDescription() [offer:%o]', + offer + ); + + await this._pc.setLocalDescription(offer); + + const answer = { + type: 'answer' as RTCSdpType, + sdp: this._remoteSdp.getSdp(), + }; + + logger.debug( + 'setMaxSpatialLayer() | calling pc.setRemoteDescription() [answer:%o]', + answer + ); + + await this._pc.setRemoteDescription(answer); + } + + async setRtpEncodingParameters( + localId: string, + params: Partial + ): Promise { + this.assertNotClosed(); + this.assertSendDirection(); + + logger.debug( + 'setRtpEncodingParameters() [localId:%s, params:%o]', + localId, + params + ); + + const transceiver = this._mapMidTransceiver.get(localId); + + if (!transceiver) { + throw new Error('associated RTCRtpTransceiver not found'); + } + + const parameters = transceiver.sender.getParameters(); + + parameters.encodings.forEach( + (encoding: RTCRtpEncodingParameters, idx: number) => { + parameters.encodings[idx] = { ...encoding, ...params }; + } + ); + + await transceiver.sender.setParameters(parameters); + + this._remoteSdp.muxMediaSectionSimulcast(localId, parameters.encodings); + + const offer = await this._pc.createOffer(); + + logger.debug( + 'setRtpEncodingParameters() | calling pc.setLocalDescription() [offer:%o]', + offer + ); + + await this._pc.setLocalDescription(offer); + + const answer = { + type: 'answer' as RTCSdpType, + sdp: this._remoteSdp.getSdp(), + }; + + logger.debug( + 'setRtpEncodingParameters() | calling pc.setRemoteDescription() [answer:%o]', + answer + ); + + await this._pc.setRemoteDescription(answer); + } + + async getSenderStats(localId: string): Promise { + this.assertNotClosed(); + this.assertSendDirection(); + + const transceiver = this._mapMidTransceiver.get(localId); + + if (!transceiver) { + throw new Error('associated RTCRtpTransceiver not found'); + } + + return transceiver.sender.getStats(); + } + + async sendDataChannel({ + ordered, + maxPacketLifeTime, + maxRetransmits, + label, + protocol, + }: HandlerSendDataChannelOptions): Promise { + this.assertNotClosed(); + this.assertSendDirection(); + + const options = { + negotiated: true, + id: this._nextSendSctpStreamId, + ordered, + maxPacketLifeTime, + maxRetransmits, + protocol, + }; + + logger.debug('sendDataChannel() [options:%o]', options); + + const dataChannel = this._pc.createDataChannel(label!, options); + + // Increase next id. + this._nextSendSctpStreamId = + ++this._nextSendSctpStreamId % SCTP_NUM_STREAMS.MIS; + + // If this is the first DataChannel we need to create the SDP answer with + // m=application section. + if (!this._hasDataChannelMediaSection) { + const offer = await this._pc.createOffer(); + const localSdpObject = sdpTransform.parse(offer.sdp!); + const offerMediaObject = localSdpObject.media.find( + m => m.type === 'application' + )!; + + if (!this._transportReady) { + await this.setupTransport({ + localDtlsRole: this._forcedLocalDtlsRole ?? 'client', + localSdpObject, + }); + } + + logger.debug( + 'sendDataChannel() | calling pc.setLocalDescription() [offer:%o]', + offer + ); + + await this._pc.setLocalDescription(offer); + + this._remoteSdp.sendSctpAssociation({ offerMediaObject }); + + const answer = { + type: 'answer' as RTCSdpType, + sdp: this._remoteSdp.getSdp(), + }; + + logger.debug( + 'sendDataChannel() | calling pc.setRemoteDescription() [answer:%o]', + answer + ); + + await this._pc.setRemoteDescription(answer); + + this._hasDataChannelMediaSection = true; + } + + const sctpStreamParameters: SctpStreamParameters = { + streamId: options.id, + ordered: options.ordered, + maxPacketLifeTime: options.maxPacketLifeTime, + maxRetransmits: options.maxRetransmits, + }; + + return { dataChannel, sctpStreamParameters }; + } + + async receive( + optionsList: HandlerReceiveOptions[] + ): Promise { + this.assertNotClosed(); + this.assertRecvDirection(); + + const results: HandlerReceiveResult[] = []; + const mapLocalId: Map = new Map(); + + for (const options of optionsList) { + const { trackId, kind, rtpParameters, streamId } = options; + + logger.debug('receive() [trackId:%s, kind:%s]', trackId, kind); + + const localId = rtpParameters.mid ?? String(this._mapMidTransceiver.size); + + mapLocalId.set(trackId, localId); + + this._remoteSdp.receive({ + mid: localId, + kind, + offerRtpParameters: rtpParameters, + streamId: streamId ?? rtpParameters.rtcp!.cname!, + trackId, + }); + } + + const offer = { + type: 'offer' as RTCSdpType, + sdp: this._remoteSdp.getSdp(), + }; + + logger.debug( + 'receive() | calling pc.setRemoteDescription() [offer:%o]', + offer + ); + + await this._pc.setRemoteDescription(offer); + + for (const options of optionsList) { + const { trackId, onRtpReceiver } = options; + + if (onRtpReceiver) { + const localId = mapLocalId.get(trackId); + const transceiver = this._pc + .getTransceivers() + .find((t: RTCRtpTransceiver) => t.mid === localId); + + if (!transceiver) { + throw new Error('transceiver not found'); + } + + onRtpReceiver(transceiver.receiver); + } + } + + let answer = await this._pc.createAnswer(); + const localSdpObject = sdpTransform.parse(answer.sdp!); + + for (const options of optionsList) { + const { trackId, rtpParameters } = options; + const localId = mapLocalId.get(trackId); + const answerMediaObject = localSdpObject.media.find( + m => String(m.mid) === localId + )!; + + // May need to modify codec parameters in the answer based on codec + // parameters in the offer. + sdpCommonUtils.applyCodecParameters({ + offerRtpParameters: rtpParameters, + answerMediaObject, + }); + } + + answer = { + type: 'answer' as RTCSdpType, + sdp: sdpTransform.write(localSdpObject), + }; + + if (!this._transportReady) { + await this.setupTransport({ + localDtlsRole: this._forcedLocalDtlsRole ?? 'client', + localSdpObject, + }); + } + + logger.debug( + 'receive() | calling pc.setLocalDescription() [answer:%o]', + answer + ); + + await this._pc.setLocalDescription(answer); + + for (const options of optionsList) { + const { trackId } = options; + const localId = mapLocalId.get(trackId)!; + const transceiver = this._pc + .getTransceivers() + .find((t: RTCRtpTransceiver) => t.mid === localId); + + if (!transceiver) { + throw new Error('new RTCRtpTransceiver not found'); + } + + // Store in the map. + this._mapMidTransceiver.set(localId, transceiver); + + results.push({ + localId, + track: transceiver.receiver.track, + rtpReceiver: transceiver.receiver, + }); + } + + return results; + } + + async stopReceiving(localIds: string[]): Promise { + this.assertRecvDirection(); + + if (this._closed) { + return; + } + + for (const localId of localIds) { + logger.debug('stopReceiving() [localId:%s]', localId); + + const transceiver = this._mapMidTransceiver.get(localId); + + if (!transceiver) { + throw new Error('associated RTCRtpTransceiver not found'); + } + + this._remoteSdp.closeMediaSection(transceiver.mid!); + } + + const offer = { + type: 'offer' as RTCSdpType, + sdp: this._remoteSdp.getSdp(), + }; + + logger.debug( + 'stopReceiving() | calling pc.setRemoteDescription() [offer:%o]', + offer + ); + + await this._pc.setRemoteDescription(offer); + + const answer = await this._pc.createAnswer(); + + logger.debug( + 'stopReceiving() | calling pc.setLocalDescription() [answer:%o]', + answer + ); + + await this._pc.setLocalDescription(answer); + + for (const localId of localIds) { + this._mapMidTransceiver.delete(localId); + } + } + + async pauseReceiving(localIds: string[]): Promise { + this.assertNotClosed(); + this.assertRecvDirection(); + + for (const localId of localIds) { + logger.debug('pauseReceiving() [localId:%s]', localId); + + const transceiver = this._mapMidTransceiver.get(localId); + + if (!transceiver) { + throw new Error('associated RTCRtpTransceiver not found'); + } + + transceiver.direction = 'inactive'; + this._remoteSdp.pauseMediaSection(localId); + } + + const offer = { + type: 'offer' as RTCSdpType, + sdp: this._remoteSdp.getSdp(), + }; + + logger.debug( + 'pauseReceiving() | calling pc.setRemoteDescription() [offer:%o]', + offer + ); + + await this._pc.setRemoteDescription(offer); + + const answer = await this._pc.createAnswer(); + + logger.debug( + 'pauseReceiving() | calling pc.setLocalDescription() [answer:%o]', + answer + ); + + await this._pc.setLocalDescription(answer); + } + + async resumeReceiving(localIds: string[]): Promise { + this.assertNotClosed(); + this.assertRecvDirection(); + + for (const localId of localIds) { + logger.debug('resumeReceiving() [localId:%s]', localId); + + const transceiver = this._mapMidTransceiver.get(localId); + + if (!transceiver) { + throw new Error('associated RTCRtpTransceiver not found'); + } + + transceiver.direction = 'recvonly'; + this._remoteSdp.resumeReceivingMediaSection(localId); + } + + const offer = { + type: 'offer' as RTCSdpType, + sdp: this._remoteSdp.getSdp(), + }; + + logger.debug( + 'resumeReceiving() | calling pc.setRemoteDescription() [offer:%o]', + offer + ); + + await this._pc.setRemoteDescription(offer); + + const answer = await this._pc.createAnswer(); + + logger.debug( + 'resumeReceiving() | calling pc.setLocalDescription() [answer:%o]', + answer + ); + + await this._pc.setLocalDescription(answer); + } + + async getReceiverStats(localId: string): Promise { + this.assertNotClosed(); + this.assertRecvDirection(); + + const transceiver = this._mapMidTransceiver.get(localId); + + if (!transceiver) { + throw new Error('associated RTCRtpTransceiver not found'); + } + + return transceiver.receiver.getStats(); + } + + async receiveDataChannel({ + sctpStreamParameters, + label, + protocol, + }: HandlerReceiveDataChannelOptions): Promise { + this.assertNotClosed(); + this.assertRecvDirection(); + + const { + streamId, + ordered, + maxPacketLifeTime, + maxRetransmits, + }: SctpStreamParameters = sctpStreamParameters; + + const options = { + negotiated: true, + id: streamId, + ordered, + maxPacketLifeTime, + maxRetransmits, + protocol, + }; + + logger.debug('receiveDataChannel() [options:%o]', options); + + const dataChannel = this._pc.createDataChannel(label!, options); + + // If this is the first DataChannel we need to create the SDP offer with + // m=application section. + if (!this._hasDataChannelMediaSection) { + this._remoteSdp.receiveSctpAssociation(); + + const offer = { + type: 'offer' as RTCSdpType, + sdp: this._remoteSdp.getSdp(), + }; + + logger.debug( + 'receiveDataChannel() | calling pc.setRemoteDescription() [offer:%o]', + offer + ); + + await this._pc.setRemoteDescription(offer); + + const answer = await this._pc.createAnswer(); + + if (!this._transportReady) { + const localSdpObject = sdpTransform.parse(answer.sdp!); + + await this.setupTransport({ + localDtlsRole: this._forcedLocalDtlsRole ?? 'client', + localSdpObject, + }); + } + + logger.debug( + 'receiveDataChannel() | calling pc.setRemoteDescription() [answer:%o]', + answer + ); + + await this._pc.setLocalDescription(answer); + + this._hasDataChannelMediaSection = true; + } + + return { dataChannel }; + } + + private async setupTransport({ + localDtlsRole, + localSdpObject, + }: { + localDtlsRole: DtlsRole; + localSdpObject?: SdpTransform.SessionDescription; + }): Promise { + if (!localSdpObject) { + localSdpObject = sdpTransform.parse(this._pc.localDescription!.sdp); + } + + // Get our local DTLS parameters. + const dtlsParameters = sdpCommonUtils.extractDtlsParameters({ + sdpObject: localSdpObject, + }); + + // Set our DTLS role. + dtlsParameters.role = localDtlsRole; + + // Update the remote DTLS role in the SDP. + this._remoteSdp.updateDtlsRole( + localDtlsRole === 'client' ? 'server' : 'client' + ); + + // Need to tell the remote transport about our parameters. + await new Promise((resolve, reject) => { + this.safeEmit('@connect', { dtlsParameters }, resolve, reject); + }); + + this._transportReady = true; + } + + private onIceGatheringStateChange = (): void => { + this.emit('@icegatheringstatechange', this._pc.iceGatheringState); + }; + + private onIceCandidateError = ( + event: RTCPeerConnectionIceErrorEvent + ): void => { + this.emit('@icecandidateerror', event); + }; + + private onConnectionStateChange = (): void => { + this.emit('@connectionstatechange', this._pc.connectionState); + }; + + private onIceConnectionStateChange = (): void => { + switch (this._pc.iceConnectionState) { + case 'checking': { + this.emit('@connectionstatechange', 'connecting'); + + break; + } + + case 'connected': + case 'completed': { + this.emit('@connectionstatechange', 'connected'); + + break; + } + + case 'failed': { + this.emit('@connectionstatechange', 'failed'); + + break; + } + + case 'disconnected': { + this.emit('@connectionstatechange', 'disconnected'); + + break; + } + + case 'closed': { + this.emit('@connectionstatechange', 'closed'); + + break; + } + } + }; + + private assertNotClosed(): void { + if (this._closed) { + throw new InvalidStateError('method called in a closed handler'); + } + } + + private assertSendDirection(): void { + if (this._direction !== 'send') { + throw new Error( + 'method can just be called for handlers with "send" direction' + ); + } + } + + private assertRecvDirection(): void { + if (this._direction !== 'recv') { + throw new Error( + 'method can just be called for handlers with "recv" direction' + ); + } + } +} From bbd2e1e6674dcc7e5c546d9cff3570c3a7f76db1 Mon Sep 17 00:00:00 2001 From: Kevin Hanna Date: Tue, 4 Nov 2025 18:26:00 +0000 Subject: [PATCH 2/2] media: Fix simulcast bandwidth limiting in Safari17Handler encodings aren't respected if provided in `addTransceiver` so we wait until the connection is negotiated --- .changeset/moody-experts-admire.md | 5 ++++ packages/media/jest.config.mjs | 3 +++ .../__tests__/getMediasoupDevice.spec.ts | 23 ++++++++++++++++++- .../webrtc/VegaRtcManager/Safari17Handler.ts | 10 ++++++++ 4 files changed, 40 insertions(+), 1 deletion(-) create mode 100644 .changeset/moody-experts-admire.md diff --git a/.changeset/moody-experts-admire.md b/.changeset/moody-experts-admire.md new file mode 100644 index 000000000..e77b3d5e2 --- /dev/null +++ b/.changeset/moody-experts-admire.md @@ -0,0 +1,5 @@ +--- +"@whereby.com/media": minor +--- + +Fix Safari simulcast maxBitrate bug diff --git a/packages/media/jest.config.mjs b/packages/media/jest.config.mjs index 616cd7ecf..63234e8a3 100644 --- a/packages/media/jest.config.mjs +++ b/packages/media/jest.config.mjs @@ -9,6 +9,9 @@ export default buildConfig(__dirname, { transform: { "^.+\\.(j|t)sx?$": "ts-jest", }, + moduleNameMapper: { + "^mediasoup-client/lib/(.*)$": "/../../node_modules/mediasoup-client/lib/$1", + }, roots: [""], coverageDirectory: "test-report/unit-tests", }); diff --git a/packages/media/src/utils/__tests__/getMediasoupDevice.spec.ts b/packages/media/src/utils/__tests__/getMediasoupDevice.spec.ts index 8b881bdb8..e9cb19ec7 100644 --- a/packages/media/src/utils/__tests__/getMediasoupDevice.spec.ts +++ b/packages/media/src/utils/__tests__/getMediasoupDevice.spec.ts @@ -1,15 +1,23 @@ import { getMediasoupDeviceAsync } from "../getMediasoupDevice"; +import { Safari17 } from "../../webrtc/VegaRtcManager/Safari17Handler"; jest.mock("mediasoup-client", () => ({ Device: jest.fn(), detectDeviceAsync: jest.fn(), })); + +jest.mock("../../webrtc/VegaRtcManager/Safari17Handler") + const mediasoupClient = jest.requireMock("mediasoup-client"); const safari17UserAgent = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.4 Safari/605.1.15"; const safari18UserAgent = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/18.1.1 Safari/605.1.15"; +const safari26UserAgent = + "Mozilla/5.0 (Macintosh; Intel Mac OS X 15_7_2) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/26.0 Safari/605.1.15"; +const safari26MobileUserAgent = + "Mozilla/5.0 (iPhone; CPU iPhone OS 18_7 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/26.0.1 Mobile/15E148 Safari/604.1"; describe("getMediasoupClient", () => { const features = {}; @@ -37,7 +45,9 @@ describe("getMediasoupClient", () => { describe.each([ { version: 17, userAgent: safari17UserAgent }, { version: 18, userAgent: safari18UserAgent }, - ])("when the user agent version is $version", ({ userAgent }) => { + { version: 26, userAgent: safari26UserAgent }, + { version: 26, userAgent: safari26MobileUserAgent }, + ])("when the safari user agent version is $version", ({ userAgent }) => { beforeEach(() => { (global as any).userAgent.mockReturnValue(userAgent); }); @@ -47,6 +57,17 @@ describe("getMediasoupClient", () => { expect(mediasoupClient.Device).toHaveBeenCalledWith({ handlerName: "Safari12" }); }); + + describe("and the safari17 handler is enabled", () => { + it("returns the Safari17 Handler", async () => { + const factory = jest.fn(); + (Safari17.createFactory as jest.Mock).mockImplementationOnce(() => factory); + + await getMediasoupDeviceAsync({ ...features, safari17HandlerOn: true }); + + expect(mediasoupClient.Device).toHaveBeenCalledWith({ handlerFactory: factory }); + }); + }); }); }); diff --git a/packages/media/src/webrtc/VegaRtcManager/Safari17Handler.ts b/packages/media/src/webrtc/VegaRtcManager/Safari17Handler.ts index b9462472c..b93c78b26 100644 --- a/packages/media/src/webrtc/VegaRtcManager/Safari17Handler.ts +++ b/packages/media/src/webrtc/VegaRtcManager/Safari17Handler.ts @@ -485,6 +485,16 @@ export class Safari17 // Store in the map. this._mapMidTransceiver.set(localId, transceiver); + if (track.kind === "video") { + const parameters = transceiver.sender.getParameters(); + for (const [index, encoding] of sendingRtpParameters.encodings.entries()) { + parameters.encodings[index]!.maxBitrate = encoding.maxBitrate; + parameters.encodings[index]!.scaleResolutionDownBy = encoding.scaleResolutionDownBy; + } + // here we set the encodings with maxBitrate using the values provided + transceiver.sender.setParameters(parameters); + } + return { localId, rtpParameters: sendingRtpParameters,