fix(sfu): upgrade mediasoup to 3.31 and port the sdp bridge

.changeset/mediasoup-sdp-bridge-3.26.md

@@ -0,0 +1,5 @@
+---
+'@epicgames-ps/mediasoup-sdp-bridge': minor
+---
+
+Support mediasoup 3.31 and later and mediasoup-client 3.25/3.26 (#1147). The peer dependencies are now `mediasoup` `^3.31.0` and `mediasoup-client` `>=3.25.0 <3.27.0`, and `sdp-transform` is upgraded to 3.0.0 to match mediasoup-client, which changed how `a=msid` is parsed. Types are imported from `mediasoup/types`, `validateRtpCapabilities` is replaced with `validateAndNormalizeRtpCapabilities`, `getExtendedRtpCapabilities` keeps the remote codec order it used before, the removed `planB` option is dropped, and `getRtpEncodings` is given the codecs so it only adds RTX when an RTX codec is negotiated. The `toffset` header extension, which mediasoup no longer supports, is removed from the browser capabilities.

.changeset/sfu-mediasoup-3.31.md

@@ -0,0 +1,5 @@
+---
+'@epicgames-ps/pixelstreaming-sfu': minor
+---
+
+Upgrade mediasoup from 3.15.5 to 3.31.0 (#1147). 3.15.5 has no prebuilt worker for Linux kernel 7, so installing the SFU on a kernel 7 host compiled the worker from source; 3.31.0 ships kernel 6 and kernel 7 prebuilts. Hosts on kernel 5 (Debian 11, Amazon Linux 2, Ubuntu 22.04 on its original kernel) no longer have a prebuilt and now compile the worker, and the kernel 7 prebuilt needs glibc 2.38 or newer. mediasoup is held below 3.32, whose new bandwidth estimation drops Firefox viewers to no video layer. The SFU now reserves data channel SCTP stream ids with mediasoup's `allocateSctpStreamId()`, as `getNextSctpStreamId()` no longer exists, and exits on SIGINT/SIGTERM itself, because mediasoup now handles those signals to close its worker and would otherwise leave the SFU running without one.

Extras/mediasoup-sdp-bridge/package.json

@@ -46,15 +46,15 @@
     "debug": "^4.1.1",
     "h264-profile-level-id": "^1.0.1",
     "lodash": "^4.18.1",
-    "sdp-transform": "^2.14.0",
+    "sdp-transform": "^3.0.0",
     "supports-color": "^7.1.0"
   },
   "devDependencies": {
     "@tsconfig/node16": "^1.0.0",
     "@types/debug": "^4.1.5",
     "@types/lodash": "^4.0.0",
     "@types/node": "^22.14.0",
-    "@types/sdp-transform": "^2.4.3",
+    "@types/sdp-transform": "^3.0.0",
     "@typescript-eslint/eslint-plugin": "^8.48.0",
     "@typescript-eslint/parser": "^8.48.0",
     "eslint": "^9.39.0",
@@ -63,15 +63,15 @@
     "eslint-plugin-jest": "^29.2.0",
     "jest": "^29.7.0",
     "jest-tobetype": "^1.2.3",
-    "mediasoup": "3.15.5",
-    "mediasoup-client": "~3.10.0",
+    "mediasoup": "3.31.0",
+    "mediasoup-client": "~3.26.0",
     "open-cli": "^6.0.1",
     "prettier": "~2.5.0",
     "tsc-watch": "^4.2.8",
     "typescript": "^5.7.3"
   },
   "peerDependencies": {
-    "mediasoup": "^3.0.0",
-    "mediasoup-client": ">=3.10.0 <3.11.0"
+    "mediasoup": "^3.31.0",
+    "mediasoup-client": ">=3.25.0 <3.27.0"
   }
 }

Extras/mediasoup-sdp-bridge/src/BrowserRtpCapabilities.ts

@@ -14,7 +14,7 @@
 // * Chrome 116.
 // * Safari 14.1.1 (macOS).
 
-import { RtpCapabilities, RtpHeaderExtensionUri } from "mediasoup/node/lib/types";
+import { RtpCapabilities, RtpHeaderExtensionUri } from "mediasoup/types";
 
 export const firefox: RtpCapabilities = {
   codecs: [
@@ -244,11 +244,6 @@ export const firefox: RtpCapabilities = {
       uri: "http://www.webrtc.org/experiments/rtp-hdrext/abs-send-time",
       preferredId: 4,
     },
-    {
-      kind: "video",
-      uri: "urn:ietf:params:rtp-hdrext:toffset",
-      preferredId: 5,
-    },
     {
       kind: "video",
       uri: "http://www.ietf.org/id/draft-holmer-rmcat-transport-wide-cc-extensions-01",
@@ -768,11 +763,6 @@ export const chrome: RtpCapabilities = {
       uri: "urn:ietf:params:rtp-hdrext:sdes:mid",
       preferredId: 4,
     },
-    {
-      kind: "video",
-      uri: "urn:ietf:params:rtp-hdrext:toffset",
-      preferredId: 14,
-    },
     {
       kind: "video",
       uri: "http://www.webrtc.org/experiments/rtp-hdrext/abs-send-time",
@@ -1170,11 +1160,6 @@ export const safari: RtpCapabilities = {
       uri: "urn:ietf:params:rtp-hdrext:sdes:repaired-rtp-stream-id",
       preferredId: 6,
     },
-    {
-      kind: "video",
-      uri: "urn:ietf:params:rtp-hdrext:toffset",
-      preferredId: 14,
-    },
     {
       kind: "video",
       uri: "http://www.webrtc.org/experiments/rtp-hdrext/abs-send-time",

Extras/mediasoup-sdp-bridge/src/SdpUtils.ts

@@ -9,8 +9,8 @@ import {
   MediaKind,
   RtpCapabilities,
   RtpParameters,
-} from "mediasoup/node/lib/types";
-import { MediaAttributes } from "sdp-transform";
+} from "mediasoup/types";
+import { MediaDescription, SessionDescription } from "sdp-transform";
 import _ from "lodash";
 
 // SDP to RTP Capabilities and Parameters
@@ -20,7 +20,7 @@ import _ from "lodash";
 // of each kind.
 // MsSdpUtils.extractRtpCapabilities() only works for 1 audio and 1 video.
 export function sdpToConsumerRtpCapabilities(
-  sdpObject: object,
+  sdpObject: SessionDescription,
   localCaps: RtpCapabilities
 ): RtpCapabilities {
   // Clone input to avoid side effect modifications.
@@ -37,10 +37,12 @@ export function sdpToConsumerRtpCapabilities(
   // }
 
   // This may throw.
-  MsOrtc.validateRtpCapabilities(_localCaps);
-  MsOrtc.validateRtpCapabilities(caps);
+  MsOrtc.validateAndNormalizeRtpCapabilities(_localCaps);
+  MsOrtc.validateAndNormalizeRtpCapabilities(caps);
 
-  const extendedCaps = MsOrtc.getExtendedRtpCapabilities(caps, _localCaps);
+  // Keep the codec order of the remote caps, as before mediasoup-client made
+  // it configurable.
+  const extendedCaps = MsOrtc.getExtendedRtpCapabilities(caps, _localCaps, false);
   const consumerCaps = MsOrtc.getRecvRtpCapabilities(extendedCaps);
 
   // DEBUG: Uncomment for details.
@@ -57,7 +59,7 @@ export function sdpToConsumerRtpCapabilities(
 // of each kind.
 // MsSdpUtils.extractRtpCapabilities() only works for 1 audio and 1 video.
 export function sdpToProducerRtpParameters(
-  sdpObject: any,
+  sdpObject: SessionDescription,
   localCaps: RtpCapabilities,
   kind: MediaKind,
   scalabilityMode: string
@@ -80,10 +82,12 @@ export function sdpToProducerRtpParameters(
   caps.headerExtensions = caps.headerExtensions?.filter((e) => e.kind === kind);
 
   // This may throw.
-  MsOrtc.validateRtpCapabilities(_localCaps);
-  MsOrtc.validateRtpCapabilities(caps);
+  MsOrtc.validateAndNormalizeRtpCapabilities(_localCaps);
+  MsOrtc.validateAndNormalizeRtpCapabilities(caps);
 
-  const extendedCaps = MsOrtc.getExtendedRtpCapabilities(_localCaps, caps);
+  // Keep the codec order of the remote caps, as before mediasoup-client made
+  // it configurable.
+  const extendedCaps = MsOrtc.getExtendedRtpCapabilities(_localCaps, caps, false);
   const producerParams = MsOrtc.getSendingRtpParameters(kind, extendedCaps);
 
   // DEBUG: Uncomment for details.
@@ -100,9 +104,13 @@ export function sdpToProducerRtpParameters(
   for (const codec of producerParams.codecs) {
     if (rtxCodecRegex.test(codec.mimeType)) {
       const extendedCodec = extendedCaps.codecs.find(
-        (c: any) => c.localPayloadType === codec.parameters.apt
+        (c: any) => c.localPayloadType === codec.parameters?.apt
       );
-      if (extendedCodec) {
+      if (
+        extendedCodec &&
+        extendedCodec.remoteRtxPayloadType !== undefined &&
+        codec.parameters
+      ) {
         codec.payloadType = extendedCodec.remoteRtxPayloadType;
         codec.parameters.apt = extendedCodec.remotePayloadType;
       }
@@ -128,9 +136,14 @@ export function sdpToProducerRtpParameters(
     }
   }
 
-  const sdpMediaObj: MediaAttributes =
-    (sdpObject.media || []).find((m: { type: MediaKind }) => m.type === kind) ||
-    {};
+  const sdpMediaObj: MediaDescription | undefined = sdpObject.media.find(
+    (m) => m.type === kind
+  );
+  if (!sdpMediaObj) {
+    throw new Error(
+      `[SdpUtils.sdpToProducerRtpParameters] No '${kind}' media in the SDP`
+    );
+  }
 
   // Fill `RtpParameters.mid`.
   if ("mid" in sdpMediaObj) {
@@ -144,6 +157,7 @@ export function sdpToProducerRtpParameters(
     if ("ssrcs" in sdpMediaObj) {
       producerParams.encodings = MsSdpUnifiedPlanUtils.getRtpEncodings({
         offerMediaObject: sdpMediaObj,
+        codecs: producerParams.codecs,
       });
     } else {
       producerParams.encodings = [];

Extras/mediasoup-sdp-bridge/src/index.ts

@@ -10,7 +10,7 @@ import {
   RtpParameters,
   Transport,
   WebRtcTransport,
-} from "mediasoup/node/lib/types";
+} from "mediasoup/types";
 
 import { randomUUID } from "node:crypto";
 import * as SdpTransform from "sdp-transform";
@@ -33,9 +33,9 @@ export class SdpEndpoint {
   private remoteSdp: string | undefined;
 
   private producers: Producer[] = [];
-  private producerOfferMedias: object[] = [];
+  private producerOfferMedias: SdpTransform.MediaDescription[] = [];
   private producerOfferParams: RtpParameters[] = [];
-  private producerOfferSctpMedia: object | undefined;
+  private producerOfferSctpMedia: SdpTransform.MediaDescription | undefined;
 
   private consumers: Consumer[] = [];
   private _receiveData: boolean = false;
@@ -194,7 +194,6 @@ export class SdpEndpoint {
       iceCandidates: this.webRtcTransport.iceCandidates as ClientIceCandidate[],
       dtlsParameters: this.webRtcTransport.dtlsParameters,
       sctpParameters: this.webRtcTransport.sctpParameters,
-      planB: false,
     });
 
     console.log("[SdpEndpoint.createAnswer] Make 'recvonly' SDP Answer");
@@ -249,7 +248,6 @@ export class SdpEndpoint {
       iceCandidates: this.webRtcTransport.iceCandidates as ClientIceCandidate[],
       dtlsParameters: this.webRtcTransport.dtlsParameters,
       sctpParameters: this.webRtcTransport.sctpParameters,
-      planB: false,
     });
 
     // Make an MSID to be used for both "audio" and "video" kinds.

SFU/package.json

@@ -18,7 +18,7 @@
         "lint": ""
     },
     "dependencies": {
-        "mediasoup": "^3.15.5",
+        "mediasoup": "~3.31.0",
         "@epicgames-ps/mediasoup-sdp-bridge": "^1.0.0",
         "minimist": "^1.2.8",
         "run-script-os": "^1.1.6",

SFU/sfu_server.js

@@ -227,8 +227,20 @@ async function onStreamerOffer(msg) {
     streamer = { transport: transport, producers: producers, dataEnabled: dataEnabled, multiplexChannels: multiplex };
 }
 
+// Reserves an SCTP stream id on the transport for a DataProducer, so that DataConsumers created on
+// the same transport later are never given the same id. mediasoup has no public API for this;
+// allocateSctpStreamId() is private (older versions called it getNextSctpStreamId()). mediasoup only frees
+// the ids of DataConsumers, so ids reserved here stay in use until the transport closes.
+function allocateSctpStreamId(transport) {
+    try {
+        return transport.allocateSctpStreamId();
+    } catch (error) {
+        throw new Error(`Could not reserve an SCTP stream id on transport ${transport.id}: ${error.message}`);
+    }
+}
+
 function getNextStreamerSCTPId() {
-    return streamer.transport.getNextSctpStreamId();
+    return allocateSctpStreamId(streamer.transport);
 }
 
 // Drops all players and closes anything we hold for the current streamer. Sends nothing, so it is
@@ -358,7 +370,7 @@ async function setupPeerDataChannels(peerId) {
 
 async function setupMultiplexPeerDataChannels(peer) {
     //this will be always 0 as we are using only one producer
-    const nextPeerSCTPStreamId = peer.transport.getNextSctpStreamId();
+    const nextPeerSCTPStreamId = allocateSctpStreamId(peer.transport);
     peer.peerDataProducer = await peer.transport.produceData({ label: 'send-datachannel', sctpStreamParameters: { streamId: nextPeerSCTPStreamId, ordered: true } });
 
     const dataProducerId = await dataRouter.handlePlayer(peer.peerDataProducer, peer.id);
@@ -505,6 +517,15 @@ async function startMediasoup() {
         process.exit(1);
     });
 
+    // mediasoup listens for SIGINT/SIGTERM to close the worker, which stops Node exiting on them and
+    // does not emit 'died', so exit here or the SFU stays connected to signalling without a worker.
+    for (const signal of ['SIGINT', 'SIGTERM']) {
+        process.once(signal, () => {
+            console.log(`${signal} received, exiting`);
+            process.exit(0);
+        });
+    }
+
     const mediaCodecs = config.mediasoup.router.mediaCodecs;
     const mediasoupRouter = await worker.createRouter({ mediaCodecs });
 
@@ -516,7 +537,7 @@ async function onICEStateChange(identifier, iceState) {
 
     if (identifier === 'Streamer' && iceState === 'completed') {
         if (streamer.multiplexChannels) {
-            const nextStreamerSCTPStreamId = streamer.transport.getNextSctpStreamId();
+            const nextStreamerSCTPStreamId = allocateSctpStreamId(streamer.transport);
             //this will always be 0 since we are using one producer only
             console.log(`Attempting streamer SCTP id=${nextStreamerSCTPStreamId}`);