VideoConduit.cpp (79652B)
1 /* This Source Code Form is subject to the terms of the Mozilla Public 2 * License, v. 2.0. If a copy of the MPL was not distributed with this file, 3 * You can obtain one at http://mozilla.org/MPL/2.0/. */ 4 5 #include "VideoConduit.h" 6 7 #include <algorithm> 8 #include <cmath> 9 #include <string> 10 #include <utility> 11 #include <vector> 12 13 #include "MediaConduitControl.h" 14 #include "RtpRtcpConfig.h" 15 #include "Tracing.h" 16 #include "VideoStreamFactory.h" 17 #include "WebrtcCallWrapper.h" 18 #include "WebrtcTaskQueueWrapper.h" 19 #include "common/YuvStamper.h" 20 #include "common/browser_logging/CSFLog.h" 21 #include "libwebrtcglue/FrameTransformer.h" 22 #include "libwebrtcglue/FrameTransformerProxy.h" 23 #include "mozilla/ErrorResult.h" 24 #include "mozilla/Maybe.h" 25 #include "mozilla/RefPtr.h" 26 #include "mozilla/StateMirroring.h" 27 #include "nsIGfxInfo.h" 28 #include "nsServiceManagerUtils.h" 29 #include "nsThreadUtils.h" 30 #include "transport/SrtpFlow.h" // For SRTP_MAX_EXPANSION 31 32 // libwebrtc includes 33 #include <stdint.h> 34 35 #include <iomanip> 36 #include <ios> 37 #include <limits> 38 #include <sstream> 39 #include <utility> 40 41 #include "CodecConfig.h" 42 #include "MainThreadUtils.h" 43 #include "MediaConduitErrors.h" 44 #include "MediaConduitInterface.h" 45 #include "MediaEventSource.h" 46 #include "PerformanceRecorder.h" 47 #include "VideoUtils.h" 48 #include "WebrtcVideoCodecFactory.h" 49 #include "api/call/transport.h" 50 #include "api/media_types.h" 51 #include "api/rtp_headers.h" 52 #include "api/rtp_parameters.h" 53 #include "api/scoped_refptr.h" 54 #include "api/transport/bitrate_settings.h" 55 #include "api/transport/rtp/rtp_source.h" 56 #include "api/video/video_codec_constants.h" 57 #include "api/video/video_codec_type.h" 58 #include "api/video/video_frame_buffer.h" 59 #include "api/video/video_sink_interface.h" 60 #include "api/video/video_source_interface.h" 61 #include "api/video_codecs/h264_profile_level_id.h" 62 #include "api/video_codecs/sdp_video_format.h" 63 #include "api/video_codecs/video_codec.h" 64 #include "api/video_codecs/video_encoder.h" 65 #include "call/call.h" 66 #include "call/rtp_config.h" 67 #include "call/video_receive_stream.h" 68 #include "call/video_send_stream.h" 69 #include "common_video/include/video_frame_buffer_pool.h" 70 #include "domstubs.h" 71 #include "jsapi/RTCStatsReport.h" 72 #include "media/base/media_constants.h" 73 #include "modules/rtp_rtcp/include/rtp_rtcp_defines.h" 74 #include "modules/rtp_rtcp/source/rtp_packet_received.h" 75 #include "mozilla/Assertions.h" 76 #include "mozilla/DataMutex.h" 77 #include "mozilla/MozPromise.h" 78 #include "mozilla/Mutex.h" 79 #include "mozilla/ProfilerState.h" 80 #include "mozilla/ReentrantMonitor.h" 81 #include "mozilla/ReverseIterator.h" 82 #include "mozilla/StateWatching.h" 83 #include "mozilla/UniquePtr.h" 84 #include "mozilla/dom/BindingDeclarations.h" 85 #include "mozilla/dom/RTCStatsReportBinding.h" 86 #include "mozilla/fallible.h" 87 #include "mozilla/glean/DomMediaWebrtcMetrics.h" 88 #include "mozilla/mozalloc_oom.h" 89 #include "nsCOMPtr.h" 90 #include "nsDebug.h" 91 #include "nsError.h" 92 #include "nsIDirectTaskDispatcher.h" 93 #include "nsISerialEventTarget.h" 94 #include "nsStringFwd.h" 95 #include "rtc_base/copy_on_write_buffer.h" 96 #include "rtc_base/network/sent_packet.h" 97 #include "rtc_base/ref_counted_object.h" 98 #include "transport/mediapacket.h" 99 #include "video/config/video_encoder_config.h" 100 101 #ifdef MOZ_WIDGET_ANDROID 102 # include "VideoEngine.h" 103 #endif 104 105 // for ntohs 106 #ifdef WIN32 107 # include "winsock2.h" 108 #else 109 # include <netinet/in.h> 110 #endif 111 112 #define INVALID_RTP_PAYLOAD 255 // valid payload types are 0 to 127 113 114 namespace mozilla { 115 116 namespace { 117 118 const char* vcLogTag = "WebrtcVideoSessionConduit"; 119 #ifdef LOGTAG 120 # undef LOGTAG 121 #endif 122 #define LOGTAG vcLogTag 123 124 using namespace webrtc; 125 using LocalDirection = MediaSessionConduitLocalDirection; 126 127 const int kNullPayloadType = -1; 128 const char kRtcpFbCcmParamTmmbr[] = "tmmbr"; 129 130 template <class t> 131 void ConstrainPreservingAspectRatioExact(uint32_t max_fs, t* width, t* height) { 132 // We could try to pick a better starting divisor, but it won't make any real 133 // performance difference. 134 for (size_t d = 1; d < std::min(*width, *height); ++d) { 135 if ((*width % d) || (*height % d)) { 136 continue; // Not divisible 137 } 138 139 if (((*width) * (*height)) / (d * d) <= max_fs) { 140 *width /= d; 141 *height /= d; 142 return; 143 } 144 } 145 146 *width = 0; 147 *height = 0; 148 } 149 150 /** 151 * Perform validation on the codecConfig to be applied 152 */ 153 MediaConduitErrorCode ValidateCodecConfig(const VideoCodecConfig& codecInfo) { 154 if (codecInfo.mName.empty()) { 155 CSFLogError(LOGTAG, "%s Empty Payload Name ", __FUNCTION__); 156 return kMediaConduitMalformedArgument; 157 } 158 159 return kMediaConduitNoError; 160 } 161 162 webrtc::VideoCodecType SupportedCodecType(webrtc::VideoCodecType aType) { 163 switch (aType) { 164 case webrtc::VideoCodecType::kVideoCodecVP8: 165 case webrtc::VideoCodecType::kVideoCodecVP9: 166 case webrtc::VideoCodecType::kVideoCodecH264: 167 case webrtc::VideoCodecType::kVideoCodecAV1: 168 return aType; 169 default: 170 return webrtc::VideoCodecType::kVideoCodecGeneric; 171 } 172 // NOTREACHED 173 } 174 175 // Call thread only. 176 webrtc::scoped_refptr<webrtc::VideoEncoderConfig::EncoderSpecificSettings> 177 ConfigureVideoEncoderSettings(const VideoCodecConfig& aConfig, 178 const WebrtcVideoConduit* aConduit, 179 webrtc::CodecParameterMap& aParameters) { 180 bool is_screencast = 181 aConduit->CodecMode() == webrtc::VideoCodecMode::kScreensharing; 182 // No automatic resizing when using simulcast or screencast. 183 bool automatic_resize = !is_screencast && aConfig.mEncodings.size() <= 1; 184 bool denoising; 185 bool codec_default_denoising = false; 186 if (is_screencast) { 187 denoising = false; 188 } else { 189 // Use codec default if video_noise_reduction is unset. 190 denoising = aConduit->Denoising(); 191 codec_default_denoising = !denoising; 192 } 193 194 using Av1Config = JsepVideoCodecDescription::Av1Config; 195 aConfig.mAv1Config.apply([&](const Av1Config& config) { 196 MOZ_ASSERT(aConfig.mName == kAv1CodecName); 197 config.mProfile.apply([&](uint8_t value) { 198 aParameters[kAv1FmtpProfile] = std::to_string(value); 199 }); 200 config.mLevelIdx.apply([&](uint8_t value) { 201 aParameters[kAv1FmtpLevelIdx] = std::to_string(value); 202 }); 203 config.mTier.apply([&](uint8_t value) { 204 aParameters[kAv1FmtpTier] = std::to_string(value); 205 }); 206 }); 207 208 if (aConfig.mName == kH264CodecName) { 209 aParameters[kH264FmtpPacketizationMode] = 210 std::to_string(aConfig.mPacketizationMode); 211 { 212 std::stringstream ss; 213 ss << std::hex << std::setfill('0'); 214 ss << std::setw(2) << static_cast<uint32_t>(aConfig.mProfile); 215 ss << std::setw(2) << static_cast<uint32_t>(aConfig.mConstraints); 216 ss << std::setw(2) << static_cast<uint32_t>(aConfig.mLevel); 217 std::string profileLevelId = ss.str(); 218 auto parsedProfileLevelId = 219 webrtc::ParseH264ProfileLevelId(profileLevelId.c_str()); 220 MOZ_DIAGNOSTIC_ASSERT(parsedProfileLevelId); 221 if (parsedProfileLevelId) { 222 aParameters[kH264FmtpProfileLevelId] = profileLevelId; 223 } 224 } 225 aParameters[kH264FmtpSpropParameterSets] = aConfig.mSpropParameterSets; 226 } 227 if (aConfig.mName == kVp8CodecName) { 228 webrtc::VideoCodecVP8 vp8_settings = 229 webrtc::VideoEncoder::GetDefaultVp8Settings(); 230 vp8_settings.automaticResizeOn = automatic_resize; 231 // VP8 denoising is enabled by default. 232 vp8_settings.denoisingOn = codec_default_denoising ? true : denoising; 233 return webrtc::scoped_refptr< 234 webrtc::VideoEncoderConfig::EncoderSpecificSettings>( 235 new webrtc::RefCountedObject< 236 webrtc::VideoEncoderConfig::Vp8EncoderSpecificSettings>( 237 vp8_settings)); 238 } 239 if (aConfig.mName == kVp9CodecName) { 240 webrtc::VideoCodecVP9 vp9_settings = 241 webrtc::VideoEncoder::GetDefaultVp9Settings(); 242 if (!is_screencast) { 243 // Always configure only 1 spatial layer for screencapture as libwebrtc 244 // has some special requirements when SVC is active. For non-screencapture 245 // the spatial layers are experimentally configurable via a pref. 246 vp9_settings.numberOfSpatialLayers = aConduit->SpatialLayers(); 247 } 248 // VP9 denoising is disabled by default. 249 vp9_settings.denoisingOn = codec_default_denoising ? false : denoising; 250 return webrtc::scoped_refptr< 251 webrtc::VideoEncoderConfig::EncoderSpecificSettings>( 252 new webrtc::RefCountedObject< 253 webrtc::VideoEncoderConfig::Vp9EncoderSpecificSettings>( 254 vp9_settings)); 255 } 256 return nullptr; 257 } 258 259 uint32_t GenerateRandomSSRC() { 260 uint32_t ssrc; 261 do { 262 SECStatus rv = PK11_GenerateRandom(reinterpret_cast<unsigned char*>(&ssrc), 263 sizeof(ssrc)); 264 MOZ_RELEASE_ASSERT(rv == SECSuccess); 265 } while (ssrc == 0); // webrtc.org code has fits if you select an SSRC of 0 266 267 return ssrc; 268 } 269 270 // TODO: Make this a defaulted operator when we have c++20 (bug 1731036). 271 bool operator!=( 272 const webrtc::VideoReceiveStreamInterface::Config::Rtp& aThis, 273 const webrtc::VideoReceiveStreamInterface::Config::Rtp& aOther) { 274 return aThis.remote_ssrc != aOther.remote_ssrc || 275 aThis.local_ssrc != aOther.local_ssrc || 276 aThis.rtcp_mode != aOther.rtcp_mode || 277 aThis.rtcp_xr.receiver_reference_time_report != 278 aOther.rtcp_xr.receiver_reference_time_report || 279 aThis.remb != aOther.remb || aThis.tmmbr != aOther.tmmbr || 280 aThis.keyframe_method != aOther.keyframe_method || 281 aThis.lntf.enabled != aOther.lntf.enabled || 282 aThis.nack.rtp_history_ms != aOther.nack.rtp_history_ms || 283 aThis.ulpfec_payload_type != aOther.ulpfec_payload_type || 284 aThis.red_payload_type != aOther.red_payload_type || 285 aThis.rtx_ssrc != aOther.rtx_ssrc || 286 aThis.protected_by_flexfec != aOther.protected_by_flexfec || 287 aThis.rtx_associated_payload_types != 288 aOther.rtx_associated_payload_types || 289 aThis.raw_payload_types != aOther.raw_payload_types; 290 } 291 292 #ifdef DEBUG 293 // TODO: Make this a defaulted operator when we have c++20 (bug 1731036). 294 bool operator==( 295 const webrtc::VideoReceiveStreamInterface::Config::Rtp& aThis, 296 const webrtc::VideoReceiveStreamInterface::Config::Rtp& aOther) { 297 return !(aThis != aOther); 298 } 299 #endif 300 301 // TODO: Make this a defaulted operator when we have c++20 (bug 1731036). 302 bool operator!=(const webrtc::RtpConfig& aThis, 303 const webrtc::RtpConfig& aOther) { 304 return aThis.ssrcs != aOther.ssrcs || aThis.rids != aOther.rids || 305 aThis.mid != aOther.mid || aThis.rtcp_mode != aOther.rtcp_mode || 306 aThis.max_packet_size != aOther.max_packet_size || 307 aThis.extmap_allow_mixed != aOther.extmap_allow_mixed || 308 aThis.extensions != aOther.extensions || 309 aThis.payload_name != aOther.payload_name || 310 aThis.payload_type != aOther.payload_type || 311 aThis.raw_payload != aOther.raw_payload || 312 aThis.lntf.enabled != aOther.lntf.enabled || 313 aThis.nack.rtp_history_ms != aOther.nack.rtp_history_ms || 314 !(aThis.ulpfec == aOther.ulpfec) || 315 aThis.flexfec.payload_type != aOther.flexfec.payload_type || 316 aThis.flexfec.ssrc != aOther.flexfec.ssrc || 317 aThis.flexfec.protected_media_ssrcs != 318 aOther.flexfec.protected_media_ssrcs || 319 aThis.rtx.ssrcs != aOther.rtx.ssrcs || 320 aThis.rtx.payload_type != aOther.rtx.payload_type || 321 aThis.c_name != aOther.c_name; 322 } 323 324 #ifdef DEBUG 325 // TODO: Make this a defaulted operator when we have c++20 (bug 1731036). 326 bool operator==(const webrtc::RtpConfig& aThis, 327 const webrtc::RtpConfig& aOther) { 328 return !(aThis != aOther); 329 } 330 #endif 331 332 } // namespace 333 334 void RecvSinkProxy::OnFrame(const webrtc::VideoFrame& aFrame) { 335 MOZ_ALWAYS_SUCCEEDS(mOwner->mFrameRecvThread->Dispatch(NS_NewRunnableFunction( 336 __FUNCTION__, 337 [owner = RefPtr(mOwner), aFrame] { owner->OnRecvFrame(aFrame); }))); 338 } 339 340 void SendSinkProxy::OnFrame(const webrtc::VideoFrame& aFrame) { 341 MOZ_ALWAYS_SUCCEEDS(mOwner->mCallThread->Dispatch(NS_NewRunnableFunction( 342 __FUNCTION__, 343 [owner = RefPtr(mOwner), aFrame] { owner->OnSendFrame(aFrame); }))); 344 } 345 346 /** 347 * Factory Method for VideoConduit 348 */ 349 RefPtr<VideoSessionConduit> VideoSessionConduit::Create( 350 RefPtr<WebrtcCallWrapper> aCall, nsCOMPtr<nsISerialEventTarget> aStsThread, 351 Options aOptions, std::string aPCHandle, 352 const TrackingId& aRecvTrackingId) { 353 MOZ_ASSERT(NS_IsMainThread()); 354 MOZ_ASSERT(aCall, "missing required parameter: aCall"); 355 CSFLogVerbose(LOGTAG, "%s", __FUNCTION__); 356 357 if (!aCall) { 358 return nullptr; 359 } 360 361 auto obj = MakeRefPtr<WebrtcVideoConduit>( 362 std::move(aCall), std::move(aStsThread), std::move(aOptions), 363 std::move(aPCHandle), aRecvTrackingId); 364 if (obj->Init() != kMediaConduitNoError) { 365 CSFLogError(LOGTAG, "%s VideoConduit Init Failed ", __FUNCTION__); 366 return nullptr; 367 } 368 CSFLogVerbose(LOGTAG, "%s Successfully created VideoConduit %p", __FUNCTION__, 369 obj.get()); 370 return obj; 371 } 372 373 #define INIT_MIRROR(name, val) \ 374 name(aCallThread, val, "WebrtcVideoConduit::Control::" #name " (Mirror)") 375 WebrtcVideoConduit::Control::Control(const RefPtr<AbstractThread>& aCallThread) 376 : INIT_MIRROR(mReceiving, false), 377 INIT_MIRROR(mTransmitting, false), 378 INIT_MIRROR(mLocalSsrcs, Ssrcs()), 379 INIT_MIRROR(mLocalRtxSsrcs, Ssrcs()), 380 INIT_MIRROR(mLocalCname, std::string()), 381 INIT_MIRROR(mMid, std::string()), 382 INIT_MIRROR(mRemoteSsrc, 0), 383 INIT_MIRROR(mRemoteRtxSsrc, 0), 384 INIT_MIRROR(mSyncGroup, std::string()), 385 INIT_MIRROR(mLocalRecvRtpExtensions, RtpExtList()), 386 INIT_MIRROR(mLocalSendRtpExtensions, RtpExtList()), 387 INIT_MIRROR(mSendCodec, Nothing()), 388 INIT_MIRROR(mSendRtpRtcpConfig, Nothing()), 389 INIT_MIRROR(mRecvCodecs, std::vector<VideoCodecConfig>()), 390 INIT_MIRROR(mRecvRtpRtcpConfig, Nothing()), 391 INIT_MIRROR(mCodecMode, webrtc::VideoCodecMode::kRealtimeVideo), 392 INIT_MIRROR(mFrameTransformerProxySend, nullptr), 393 INIT_MIRROR(mFrameTransformerProxyRecv, nullptr), 394 INIT_MIRROR(mVideoDegradationPreference, 395 webrtc::DegradationPreference::DISABLED) {} 396 #undef INIT_MIRROR 397 398 #define INIT_CANONICAL(name, thread, val) \ 399 name(thread, val, "WebrtcVideoConduit::" #name " (Canonical)") 400 #define INIT_MIRROR(name, val) \ 401 name(AbstractThread::MainThread(), val, \ 402 "WebrtcVideoConduit::" #name " (Mirror)") 403 WebrtcVideoConduit::WebrtcVideoConduit( 404 RefPtr<WebrtcCallWrapper> aCall, nsCOMPtr<nsISerialEventTarget> aStsThread, 405 Options aOptions, std::string aPCHandle, const TrackingId& aRecvTrackingId) 406 : mRendererMonitor("WebrtcVideoConduit::mRendererMonitor"), 407 mCall(std::move(aCall)), 408 mCallThread(mCall->mCallThread), 409 mStsThread(std::move(aStsThread)), 410 mFrameRecvThread(CreateWebrtcTaskQueueWrapper( 411 GetMediaThreadPool(MediaThreadType::WEBRTC_WORKER), 412 "WebrtcVideoConduit::mFrameRecvThread"_ns, 413 /* aSupportsTailDispatch= */ true)), 414 mControl(mCall->mCallThread), 415 INIT_CANONICAL(mReceivingSize, mFrameRecvThread, {}), 416 mWatchManager(this, mCall->mCallThread), 417 mMutex("WebrtcVideoConduit::mMutex"), 418 mDecoderFactory(MakeUnique<WebrtcVideoDecoderFactory>( 419 mCallThread.get(), aPCHandle, aRecvTrackingId)), 420 mEncoderFactory(MakeUnique<WebrtcVideoEncoderFactory>( 421 mCallThread.get(), std::move(aPCHandle))), 422 mRecvSinkProxy(this), 423 mSendSinkProxy(this), 424 mEngineTransmitting(false), 425 mEngineReceiving(false), 426 mVideoLatencyTestEnable(aOptions.mVideoLatencyTestEnable), 427 mMinBitrate(aOptions.mMinBitrate), 428 mStartBitrate(aOptions.mStartBitrate), 429 mPrefMaxBitrate(aOptions.mPrefMaxBitrate), 430 mMinBitrateEstimate(aOptions.mMinBitrateEstimate), 431 mDenoising(aOptions.mDenoising), 432 mLockScaling(aOptions.mLockScaling), 433 mSpatialLayers(aOptions.mSpatialLayers), 434 mTemporalLayers(aOptions.mTemporalLayers), 435 mSendTransport(this), 436 mRecvTransport(this), 437 mSendStreamConfig(&mSendTransport), 438 mVideoStreamFactory("WebrtcVideoConduit::mVideoStreamFactory"), 439 mRecvStreamConfig(&mRecvTransport), 440 INIT_CANONICAL(mCanonicalRtpSources, mCall->mCallThread, {}), 441 INIT_MIRROR(mRtpSources, {}) { 442 mRecvStreamConfig.rtp.rtcp_event_observer = this; 443 } 444 #undef INIT_MIRROR 445 #undef INIT_CANONICAL 446 447 WebrtcVideoConduit::~WebrtcVideoConduit() { 448 CSFLogDebug(LOGTAG, "%s ", __FUNCTION__); 449 MOZ_ASSERT(!mSendStream && !mRecvStream, 450 "Call DeleteStreams prior to ~WebrtcVideoConduit."); 451 } 452 453 #define CONNECT(aCanonical, aMirror) \ 454 do { \ 455 /* Ensure the watchmanager is wired up before the mirror receives its \ 456 * initial mirrored value. */ \ 457 mCall->mCallThread->DispatchStateChange( \ 458 NS_NewRunnableFunction(__func__, [this, self = RefPtr(this)] { \ 459 mWatchManager.Watch(aMirror, \ 460 &WebrtcVideoConduit::OnControlConfigChange); \ 461 })); \ 462 (aCanonical).ConnectMirror(&(aMirror)); \ 463 } while (0) 464 465 void WebrtcVideoConduit::InitControl(VideoConduitControlInterface* aControl) { 466 MOZ_ASSERT(NS_IsMainThread()); 467 468 CONNECT(aControl->CanonicalReceiving(), mControl.mReceiving); 469 CONNECT(aControl->CanonicalTransmitting(), mControl.mTransmitting); 470 CONNECT(aControl->CanonicalLocalSsrcs(), mControl.mLocalSsrcs); 471 CONNECT(aControl->CanonicalLocalVideoRtxSsrcs(), mControl.mLocalRtxSsrcs); 472 CONNECT(aControl->CanonicalLocalCname(), mControl.mLocalCname); 473 CONNECT(aControl->CanonicalMid(), mControl.mMid); 474 CONNECT(aControl->CanonicalRemoteSsrc(), mControl.mRemoteSsrc); 475 CONNECT(aControl->CanonicalRemoteVideoRtxSsrc(), mControl.mRemoteRtxSsrc); 476 CONNECT(aControl->CanonicalSyncGroup(), mControl.mSyncGroup); 477 CONNECT(aControl->CanonicalLocalRecvRtpExtensions(), 478 mControl.mLocalRecvRtpExtensions); 479 CONNECT(aControl->CanonicalLocalSendRtpExtensions(), 480 mControl.mLocalSendRtpExtensions); 481 CONNECT(aControl->CanonicalVideoSendCodec(), mControl.mSendCodec); 482 CONNECT(aControl->CanonicalVideoSendRtpRtcpConfig(), 483 mControl.mSendRtpRtcpConfig); 484 CONNECT(aControl->CanonicalVideoRecvCodecs(), mControl.mRecvCodecs); 485 CONNECT(aControl->CanonicalVideoRecvRtpRtcpConfig(), 486 mControl.mRecvRtpRtcpConfig); 487 CONNECT(aControl->CanonicalVideoCodecMode(), mControl.mCodecMode); 488 CONNECT(aControl->CanonicalFrameTransformerProxySend(), 489 mControl.mFrameTransformerProxySend); 490 CONNECT(aControl->CanonicalFrameTransformerProxyRecv(), 491 mControl.mFrameTransformerProxyRecv); 492 CONNECT(aControl->CanonicalVideoDegradationPreference(), 493 mControl.mVideoDegradationPreference); 494 } 495 496 #undef CONNECT 497 498 void WebrtcVideoConduit::OnControlConfigChange() { 499 MOZ_ASSERT(mCallThread->IsOnCurrentThread()); 500 501 bool encoderReconfigureNeeded = false; 502 bool remoteSsrcUpdateNeeded = false; 503 bool sendStreamRecreationNeeded = false; 504 bool sendSourceUpdateNeeded = false; 505 506 if (mControl.mRemoteSsrc.Ref() != mControl.mConfiguredRemoteSsrc) { 507 mControl.mConfiguredRemoteSsrc = mControl.mRemoteSsrc; 508 remoteSsrcUpdateNeeded = true; 509 } 510 511 if (mControl.mRemoteRtxSsrc.Ref() != mControl.mConfiguredRemoteRtxSsrc) { 512 mControl.mConfiguredRemoteRtxSsrc = mControl.mRemoteRtxSsrc; 513 remoteSsrcUpdateNeeded = true; 514 } 515 516 if (mControl.mSyncGroup.Ref() != mRecvStreamConfig.sync_group) { 517 mRecvStreamConfig.sync_group = mControl.mSyncGroup; 518 } 519 520 if (const auto [codecConfigList, rtpRtcpConfig] = std::make_pair( 521 mControl.mRecvCodecs.Ref(), mControl.mRecvRtpRtcpConfig.Ref()); 522 !codecConfigList.empty() && rtpRtcpConfig.isSome() && 523 (codecConfigList != mControl.mConfiguredRecvCodecs || 524 rtpRtcpConfig != mControl.mConfiguredRecvRtpRtcpConfig)) { 525 mControl.mConfiguredRecvCodecs = codecConfigList; 526 mControl.mConfiguredRecvRtpRtcpConfig = rtpRtcpConfig; 527 webrtc::VideoReceiveStreamInterface::Config::Rtp newRtp( 528 mRecvStreamConfig.rtp); 529 MOZ_ASSERT(newRtp == mRecvStreamConfig.rtp); 530 newRtp.rtx_associated_payload_types.clear(); 531 newRtp.rtcp_mode = rtpRtcpConfig->GetRtcpMode(); 532 newRtp.nack.rtp_history_ms = 0; 533 newRtp.remb = false; 534 newRtp.tmmbr = false; 535 newRtp.keyframe_method = webrtc::KeyFrameReqMethod::kNone; 536 newRtp.ulpfec_payload_type = kNullPayloadType; 537 newRtp.red_payload_type = kNullPayloadType; 538 bool use_fec = false; 539 std::vector<webrtc::VideoReceiveStreamInterface::Decoder> recv_codecs; 540 541 // Try Applying the codecs in the list 542 // we treat as success if at least one codec was applied and reception was 543 // started successfully. 544 for (const auto& codec_config : codecConfigList) { 545 if (auto condError = ValidateCodecConfig(codec_config); 546 condError != kMediaConduitNoError) { 547 CSFLogError(LOGTAG, "Invalid recv codec config for %s decoder: %i", 548 codec_config.mName.c_str(), condError); 549 continue; 550 } 551 552 if (codec_config.mName == kUlpfecCodecName) { 553 newRtp.ulpfec_payload_type = codec_config.mType; 554 continue; 555 } 556 557 // Set RTX associated PT here so we can set it for RED without additional 558 // checks for things like preventing creating an unncessary decoder for 559 // RED. This assumes that any codecs created with RTX enabled 560 // (including those not found in SupportedCodecType) intend to use it. 561 if (codec_config.RtxPayloadTypeIsSet()) { 562 newRtp.rtx_associated_payload_types[codec_config.mRTXPayloadType] = 563 codec_config.mType; 564 } 565 566 if (codec_config.mName == kRedCodecName) { 567 newRtp.red_payload_type = codec_config.mType; 568 continue; 569 } 570 571 if (SupportedCodecType( 572 webrtc::PayloadStringToCodecType(codec_config.mName)) == 573 webrtc::VideoCodecType::kVideoCodecGeneric) { 574 CSFLogError(LOGTAG, "%s Unknown decoder type: %s", __FUNCTION__, 575 codec_config.mName.c_str()); 576 continue; 577 } 578 579 // Check for the keyframe request type: PLI is preferred over FIR, and FIR 580 // is preferred over none. 581 if (codec_config.RtcpFbNackIsSet(kRtcpFbNackParamPli)) { 582 newRtp.keyframe_method = webrtc::KeyFrameReqMethod::kPliRtcp; 583 } else if (newRtp.keyframe_method != 584 webrtc::KeyFrameReqMethod::kPliRtcp && 585 codec_config.RtcpFbCcmIsSet(kRtcpFbCcmParamFir)) { 586 newRtp.keyframe_method = webrtc::KeyFrameReqMethod::kFirRtcp; 587 } 588 589 // What if codec A has Nack and REMB, and codec B has TMMBR, and codec C 590 // has none? In practice, that's not a useful configuration, and 591 // VideoReceiveStream::Config can't represent that, so simply union the 592 // (boolean) settings 593 if (codec_config.RtcpFbNackIsSet(kParamValueEmpty)) { 594 newRtp.nack.rtp_history_ms = 1000; 595 } 596 newRtp.tmmbr |= codec_config.RtcpFbCcmIsSet(kRtcpFbCcmParamTmmbr); 597 newRtp.remb |= codec_config.RtcpFbRembIsSet(); 598 use_fec |= codec_config.RtcpFbFECIsSet(); 599 600 auto& decoder = recv_codecs.emplace_back(); 601 decoder.video_format = webrtc::SdpVideoFormat(codec_config.mName); 602 decoder.payload_type = codec_config.mType; 603 } 604 605 if (!use_fec) { 606 // Reset to defaults 607 newRtp.ulpfec_payload_type = kNullPayloadType; 608 newRtp.red_payload_type = kNullPayloadType; 609 } 610 611 // TODO: This would be simpler, but for some reason gives 612 // "error: invalid operands to binary expression 613 // ('webrtc::VideoReceiveStreamInterface::Decoder' and 614 // 'webrtc::VideoReceiveStreamInterface::Decoder')" 615 // if (recv_codecs != mRecvStreamConfig.decoders) { 616 if (!std::equal(recv_codecs.begin(), recv_codecs.end(), 617 mRecvStreamConfig.decoders.begin(), 618 mRecvStreamConfig.decoders.end(), 619 [](const auto& aLeft, const auto& aRight) { 620 return aLeft == aRight; 621 })) { 622 if (recv_codecs.empty()) { 623 CSFLogError(LOGTAG, "%s Found no valid receive codecs", __FUNCTION__); 624 } 625 mRecvStreamConfig.decoders = std::move(recv_codecs); 626 } 627 628 if (mRecvStreamConfig.rtp != newRtp) { 629 mRecvStreamConfig.rtp = newRtp; 630 remoteSsrcUpdateNeeded = true; 631 } 632 } 633 634 { 635 // mSendStreamConfig and other members need the lock 636 MutexAutoLock lock(mMutex); 637 if (mControl.mLocalSsrcs.Ref() != mSendStreamConfig.rtp.ssrcs) { 638 mSendStreamConfig.rtp.ssrcs = mControl.mLocalSsrcs; 639 sendStreamRecreationNeeded = true; 640 641 const uint32_t localSsrc = mSendStreamConfig.rtp.ssrcs.empty() 642 ? 0 643 : mSendStreamConfig.rtp.ssrcs.front(); 644 if (localSsrc != mRecvStreamConfig.rtp.local_ssrc) { 645 mRecvStreamConfig.rtp.local_ssrc = localSsrc; 646 } 647 } 648 649 { 650 Ssrcs localRtxSsrcs = mControl.mLocalRtxSsrcs.Ref(); 651 if (!mControl.mSendCodec.Ref() 652 .map([](const auto& aCodec) { 653 return aCodec.RtxPayloadTypeIsSet(); 654 }) 655 .valueOr(false)) { 656 localRtxSsrcs.clear(); 657 } 658 if (localRtxSsrcs != mSendStreamConfig.rtp.rtx.ssrcs) { 659 mSendStreamConfig.rtp.rtx.ssrcs = localRtxSsrcs; 660 sendStreamRecreationNeeded = true; 661 } 662 } 663 664 if (mControl.mLocalCname.Ref() != mSendStreamConfig.rtp.c_name) { 665 mSendStreamConfig.rtp.c_name = mControl.mLocalCname; 666 sendStreamRecreationNeeded = true; 667 } 668 669 if (mControl.mMid.Ref() != mSendStreamConfig.rtp.mid) { 670 mSendStreamConfig.rtp.mid = mControl.mMid; 671 sendStreamRecreationNeeded = true; 672 } 673 674 if (mControl.mLocalSendRtpExtensions.Ref() != 675 mSendStreamConfig.rtp.extensions) { 676 mSendStreamConfig.rtp.extensions = mControl.mLocalSendRtpExtensions; 677 sendStreamRecreationNeeded = true; 678 } 679 680 if (const auto [codecConfig, rtpRtcpConfig] = std::make_pair( 681 mControl.mSendCodec.Ref(), mControl.mSendRtpRtcpConfig.Ref()); 682 codecConfig.isSome() && rtpRtcpConfig.isSome() && 683 (codecConfig != mControl.mConfiguredSendCodec || 684 rtpRtcpConfig != mControl.mConfiguredSendRtpRtcpConfig)) { 685 CSFLogDebug(LOGTAG, "Configuring codec %s", codecConfig->mName.c_str()); 686 687 if (mControl.mConfiguredSendCodec.isSome() && 688 (mControl.mConfiguredSendCodec->mName != codecConfig->mName)) { 689 // This tends to happen if the codec is changed mid call. 690 // We need to delete the stream now, if we continue to setup the new 691 // codec before deleting the send stream libwebrtc will throw erros. 692 MemoSendStreamStats(); 693 DeleteSendStream(); 694 } 695 mControl.mConfiguredSendCodec = codecConfig; 696 mControl.mConfiguredSendRtpRtcpConfig = rtpRtcpConfig; 697 698 if (ValidateCodecConfig(*codecConfig) == kMediaConduitNoError) { 699 encoderReconfigureNeeded = true; 700 sendSourceUpdateNeeded = true; 701 702 mCurSendCodecConfig = codecConfig; 703 704 size_t streamCount = std::min(codecConfig->mEncodings.size(), 705 (size_t)webrtc::kMaxSimulcastStreams); 706 MOZ_RELEASE_ASSERT(streamCount >= 1, 707 "streamCount should be at least one"); 708 709 CSFLogDebug(LOGTAG, 710 "Updating send codec for VideoConduit:%p stream count:%zu", 711 this, streamCount); 712 713 // So we can comply with b=TIAS/b=AS/maxbr=X when input resolution 714 // changes 715 MOZ_ASSERT(codecConfig->mTias < INT_MAX); 716 mNegotiatedMaxBitrate = static_cast<int>(codecConfig->mTias); 717 718 if (!mLastSize && mMinBitrateEstimate != 0) { 719 // Only do this at the start; use "have we sent a frame" as a 720 // reasonable stand-in. min <= start <= max (but all three parameters 721 // are optional) 722 webrtc::BitrateSettings settings; 723 settings.min_bitrate_bps = mMinBitrateEstimate; 724 settings.start_bitrate_bps = mMinBitrateEstimate; 725 mCall->Call()->SetClientBitratePreferences(settings); 726 } 727 728 // XXX parse the encoded SPS/PPS data and set 729 // spsData/spsLen/ppsData/ppsLen 730 mEncoderConfig.video_format = 731 webrtc::SdpVideoFormat(codecConfig->mName); 732 mEncoderConfig.encoder_specific_settings = 733 ConfigureVideoEncoderSettings( 734 *codecConfig, this, mEncoderConfig.video_format.parameters); 735 736 mEncoderConfig.codec_type = SupportedCodecType( 737 webrtc::PayloadStringToCodecType(codecConfig->mName)); 738 MOZ_RELEASE_ASSERT(mEncoderConfig.codec_type != 739 webrtc::VideoCodecType::kVideoCodecGeneric); 740 741 mEncoderConfig.content_type = 742 mControl.mCodecMode.Ref() == webrtc::VideoCodecMode::kRealtimeVideo 743 ? webrtc::VideoEncoderConfig::ContentType::kRealtimeVideo 744 : webrtc::VideoEncoderConfig::ContentType::kScreen; 745 746 mEncoderConfig.frame_drop_enabled = true; 747 748 mEncoderConfig.min_transmit_bitrate_bps = mMinBitrate; 749 750 // Set the max bitrate, defaulting to 10Mbps, checking: 751 // - pref 752 // - b=TIAS 753 // - codec constraints 754 // - encoding parameter if there's a single stream 755 int maxBps = KBPS(10000); 756 maxBps = MinIgnoreZero(maxBps, mPrefMaxBitrate); 757 maxBps = MinIgnoreZero(maxBps, mNegotiatedMaxBitrate); 758 maxBps = MinIgnoreZero( 759 maxBps, static_cast<int>(codecConfig->mEncodingConstraints.maxBr)); 760 if (codecConfig->mEncodings.size() == 1) { 761 maxBps = MinIgnoreZero( 762 maxBps, 763 static_cast<int>(codecConfig->mEncodings[0].constraints.maxBr)); 764 } 765 mEncoderConfig.max_bitrate_bps = maxBps; 766 767 // TODO this is for webrtc-priority, but needs plumbing bits 768 mEncoderConfig.bitrate_priority = 1.0; 769 770 // Populate simulcast_layers with their config. 771 mEncoderConfig.simulcast_layers.clear(); 772 const auto& codecConstraints = codecConfig->mEncodingConstraints; 773 for (size_t idx = 0; idx < streamCount; ++idx) { 774 webrtc::VideoStream video_stream; 775 const auto& encoding = codecConfig->mEncodings[idx]; 776 const auto& encodingConstraints = encoding.constraints; 777 778 video_stream.active = encoding.active; 779 780 // Dimensions set here are used by libwebrtc to set the maximum to 781 // request from the source, to support scaleDownTo without another 782 // encoder reconfiguration on the first frame. 783 // We don't support scaleDownTo but use max-width and max-height if 784 // signaled. 785 video_stream.width = codecConfig->mEncodingConstraints.maxWidth; 786 video_stream.height = codecConfig->mEncodingConstraints.maxHeight; 787 788 // Max framerate is also used to cap the source, to avoid processing 789 // frames that will have to be dropped. Our signals here are both 790 // RTCRtpEncodingParameters.maxFramerate (per encoding) and max-fr 791 // for supported codecs. 792 video_stream.max_framerate = static_cast<int>(([&]() { 793 if (codecConstraints.maxFps && encodingConstraints.maxFps) { 794 return std::min(*codecConstraints.maxFps, 795 *encodingConstraints.maxFps); 796 } 797 return codecConstraints.maxFps 798 .orElse([&] { return encodingConstraints.maxFps; }) 799 .valueOr(-1); 800 })()); 801 802 // Set each layer's max-bitrate explicitly or libwebrtc may ignore all 803 // stream-specific max-bitrate settings later on, as provided by the 804 // VideoStreamFactory. Default to our max of 10Mbps, overriden by 805 // SDP/JS. 806 int maxBps = KBPS(10000); 807 maxBps = MinIgnoreZero(maxBps, mPrefMaxBitrate); 808 maxBps = MinIgnoreZero(maxBps, mNegotiatedMaxBitrate); 809 maxBps = MinIgnoreZero(maxBps, 810 static_cast<int>(encodingConstraints.maxBr)); 811 video_stream.max_bitrate_bps = maxBps; 812 813 // At this time, other values are not used until after 814 // CreateEncoderStreams(). We fill these in directly from the codec 815 // config in VideoStreamFactory. 816 817 mEncoderConfig.simulcast_layers.push_back(video_stream); 818 } 819 820 // Expected max number of encodings 821 mEncoderConfig.number_of_streams = 822 mEncoderConfig.simulcast_layers.size(); 823 824 // libwebrtc disables this by default. 825 mSendStreamConfig.suspend_below_min_bitrate = false; 826 827 webrtc::RtpConfig newRtp = mSendStreamConfig.rtp; 828 MOZ_ASSERT(newRtp == mSendStreamConfig.rtp); 829 newRtp.payload_name = codecConfig->mName; 830 newRtp.payload_type = codecConfig->mType; 831 newRtp.rtcp_mode = rtpRtcpConfig->GetRtcpMode(); 832 newRtp.extmap_allow_mixed = rtpRtcpConfig->GetExtmapAllowMixed(); 833 newRtp.max_packet_size = kVideoMtu; 834 newRtp.rtx.payload_type = codecConfig->RtxPayloadTypeIsSet() 835 ? codecConfig->mRTXPayloadType 836 : kNullPayloadType; 837 838 { 839 // See Bug 1297058, enabling FEC when basic NACK is to be enabled in 840 // H.264 is problematic 841 const bool useFECDefaults = 842 !codecConfig->RtcpFbFECIsSet() || 843 (codecConfig->mName == kH264CodecName && 844 codecConfig->RtcpFbNackIsSet(kParamValueEmpty)); 845 newRtp.ulpfec.ulpfec_payload_type = 846 useFECDefaults ? kNullPayloadType 847 : codecConfig->mULPFECPayloadType; 848 newRtp.ulpfec.red_payload_type = 849 useFECDefaults ? kNullPayloadType : codecConfig->mREDPayloadType; 850 newRtp.ulpfec.red_rtx_payload_type = 851 useFECDefaults ? kNullPayloadType 852 : codecConfig->mREDRTXPayloadType; 853 } 854 855 newRtp.nack.rtp_history_ms = 856 codecConfig->RtcpFbNackIsSet(kParamValueEmpty) ? 1000 : 0; 857 858 newRtp.rids.clear(); 859 if (!codecConfig->mEncodings.empty() && 860 !codecConfig->mEncodings[0].rid.empty()) { 861 for (const auto& encoding : codecConfig->mEncodings) { 862 newRtp.rids.push_back(encoding.rid); 863 } 864 } 865 866 if (mSendStreamConfig.rtp != newRtp) { 867 mSendStreamConfig.rtp = newRtp; 868 sendStreamRecreationNeeded = true; 869 } 870 } 871 } 872 873 if (mControl.mConfiguredDegradationPreference != 874 mControl.mVideoDegradationPreference) { 875 mControl.mConfiguredDegradationPreference = 876 mControl.mVideoDegradationPreference.Ref(); 877 if (mSendStream) { 878 mSendStream->SetSource(mTrackSource, DegradationPreference()); 879 } 880 } 881 882 { 883 const auto& mode = mControl.mCodecMode.Ref(); 884 MOZ_ASSERT(mode == webrtc::VideoCodecMode::kRealtimeVideo || 885 mode == webrtc::VideoCodecMode::kScreensharing); 886 887 auto contentType = 888 mode == webrtc::VideoCodecMode::kRealtimeVideo 889 ? webrtc::VideoEncoderConfig::ContentType::kRealtimeVideo 890 : webrtc::VideoEncoderConfig::ContentType::kScreen; 891 892 if (contentType != mEncoderConfig.content_type) { 893 encoderReconfigureNeeded = true; 894 sendSourceUpdateNeeded = true; 895 mEncoderConfig.content_type = contentType; 896 } 897 } 898 899 if (mControl.mConfiguredFrameTransformerProxySend.get() != 900 mControl.mFrameTransformerProxySend.Ref().get()) { 901 mControl.mConfiguredFrameTransformerProxySend = 902 mControl.mFrameTransformerProxySend.Ref(); 903 if (!mSendStreamConfig.frame_transformer) { 904 mSendStreamConfig.frame_transformer = 905 new webrtc::RefCountedObject<FrameTransformer>(true); 906 sendStreamRecreationNeeded = true; 907 } 908 static_cast<FrameTransformer*>(mSendStreamConfig.frame_transformer.get()) 909 ->SetProxy(mControl.mConfiguredFrameTransformerProxySend); 910 } 911 912 if (mControl.mConfiguredFrameTransformerProxyRecv.get() != 913 mControl.mFrameTransformerProxyRecv.Ref().get()) { 914 mControl.mConfiguredFrameTransformerProxyRecv = 915 mControl.mFrameTransformerProxyRecv.Ref(); 916 if (!mRecvStreamConfig.frame_transformer) { 917 mRecvStreamConfig.frame_transformer = 918 new webrtc::RefCountedObject<FrameTransformer>(true); 919 } 920 static_cast<FrameTransformer*>(mRecvStreamConfig.frame_transformer.get()) 921 ->SetProxy(mControl.mConfiguredFrameTransformerProxyRecv); 922 // No flag to set, we always recreate recv streams 923 } 924 925 if (remoteSsrcUpdateNeeded) { 926 SetRemoteSSRCConfig(mControl.mConfiguredRemoteSsrc, 927 mControl.mConfiguredRemoteRtxSsrc); 928 } 929 930 // Handle un-signalled SSRCs by creating random ones and then when they 931 // actually get set, we'll destroy and recreate. 932 if (mControl.mReceiving || mControl.mTransmitting) { 933 const auto remoteSsrc = mRecvStreamConfig.rtp.remote_ssrc; 934 const auto localSsrc = mRecvStreamConfig.rtp.local_ssrc; 935 const auto localSsrcs = mSendStreamConfig.rtp.ssrcs; 936 EnsureLocalSSRC(); 937 if (mControl.mReceiving) { 938 EnsureRemoteSSRC(); 939 } 940 if (localSsrc != mRecvStreamConfig.rtp.local_ssrc || 941 remoteSsrc != mRecvStreamConfig.rtp.remote_ssrc) { 942 } 943 if (localSsrcs != mSendStreamConfig.rtp.ssrcs) { 944 sendStreamRecreationNeeded = true; 945 } 946 } 947 948 // Recreate receiving streams 949 if (mControl.mReceiving) { 950 DeleteRecvStream(); 951 CreateRecvStream(); 952 } 953 if (sendStreamRecreationNeeded) { 954 encoderReconfigureNeeded = false; 955 MemoSendStreamStats(); 956 DeleteSendStream(); 957 } 958 if (mControl.mTransmitting) { 959 CreateSendStream(); 960 } 961 } 962 963 // We make sure to not hold the lock while stopping/starting/reconfiguring 964 // streams, so as to not cause deadlocks. These methods can cause our platform 965 // codecs to dispatch sync runnables to main, and main may grab the lock. 966 967 if (mSendStream) { 968 if (encoderReconfigureNeeded) { 969 MOZ_DIAGNOSTIC_ASSERT( 970 mSendStreamConfig.rtp.ssrcs.size() == 971 mEncoderConfig.number_of_streams, 972 "Each video substream must have a corresponding ssrc."); 973 mEncoderConfig.video_stream_factory = CreateVideoStreamFactory(); 974 for (const auto& stream : mEncoderConfig.simulcast_layers) { 975 CSFLogDebug( 976 LOGTAG, 977 "%s Reconfigure with simulcast stream maxFps=%d, " 978 "bitrate=[%dkbps, %dkbps, %dkbps]", 979 __FUNCTION__, stream.max_framerate, stream.min_bitrate_bps / 1000, 980 stream.target_bitrate_bps / 1000, stream.max_bitrate_bps / 1000); 981 } 982 mSendStream->ReconfigureVideoEncoder(mEncoderConfig.Copy()); 983 } 984 if (sendSourceUpdateNeeded && mTrackSource) { 985 mSendStream->SetSource(mTrackSource, DegradationPreference()); 986 } 987 } 988 989 if (!mControl.mReceiving) { 990 StopReceiving(); 991 } 992 if (!mControl.mTransmitting) { 993 StopTransmitting(); 994 } 995 996 if (mControl.mReceiving) { 997 StartReceiving(); 998 } 999 if (mControl.mTransmitting) { 1000 StartTransmitting(); 1001 } 1002 } 1003 1004 std::vector<unsigned int> WebrtcVideoConduit::GetLocalSSRCs() const { 1005 MOZ_ASSERT(mCallThread->IsOnCurrentThread()); 1006 return mSendStreamConfig.rtp.ssrcs; 1007 } 1008 1009 Maybe<Ssrc> WebrtcVideoConduit::GetAssociatedLocalRtxSSRC(Ssrc aSsrc) const { 1010 MOZ_ASSERT(mCallThread->IsOnCurrentThread()); 1011 for (size_t i = 0; i < mSendStreamConfig.rtp.ssrcs.size() && 1012 i < mSendStreamConfig.rtp.rtx.ssrcs.size(); 1013 ++i) { 1014 if (mSendStreamConfig.rtp.ssrcs[i] == aSsrc) { 1015 return Some(mSendStreamConfig.rtp.rtx.ssrcs[i]); 1016 } 1017 } 1018 return Nothing(); 1019 } 1020 1021 Maybe<gfx::IntSize> WebrtcVideoConduit::GetLastResolution() const { 1022 MOZ_ASSERT(mCallThread->IsOnCurrentThread()); 1023 return mLastSize; 1024 } 1025 1026 void WebrtcVideoConduit::DeleteSendStream() { 1027 MOZ_ASSERT(mCallThread->IsOnCurrentThread()); 1028 mMutex.AssertCurrentThreadOwns(); 1029 1030 if (!mSendStream) { 1031 return; 1032 } 1033 mCall->Call()->DestroyVideoSendStream(mSendStream); 1034 mEngineTransmitting = false; 1035 mSendStream = nullptr; 1036 1037 // Reset base_seqs in case ssrcs get re-used. 1038 mRtpSendBaseSeqs.clear(); 1039 } 1040 1041 void WebrtcVideoConduit::MemoSendStreamStats() { 1042 MOZ_ASSERT(mCallThread->IsOnCurrentThread()); 1043 // If we are going to be recreating the send stream, we hold onto stats until 1044 // libwebrtc stats collection catches up. 1045 if (mControl.mTransmitting && mSendStream) { 1046 const auto stats = mSendStream->GetStats(); 1047 // If we have no streams we don't need to hold onto anything 1048 if (stats.substreams.size()) { 1049 mTransitionalSendStreamStats = Some(stats); 1050 } 1051 } 1052 } 1053 1054 void WebrtcVideoConduit::CreateSendStream() { 1055 MOZ_ASSERT(mCallThread->IsOnCurrentThread()); 1056 mMutex.AssertCurrentThreadOwns(); 1057 1058 if (mSendStream) { 1059 return; 1060 } 1061 1062 glean::webrtc_video::send_codec_used 1063 .Get(nsDependentCString(mSendStreamConfig.rtp.payload_name.c_str())) 1064 .Add(1); 1065 1066 mEncoderConfig.video_stream_factory = CreateVideoStreamFactory(); 1067 1068 mSendStreamConfig.encoder_settings.encoder_factory = mEncoderFactory.get(); 1069 mSendStreamConfig.encoder_settings.bitrate_allocator_factory = 1070 mCall->mVideoBitrateAllocatorFactory.get(); 1071 1072 MOZ_DIAGNOSTIC_ASSERT( 1073 mSendStreamConfig.rtp.ssrcs.size() == mEncoderConfig.number_of_streams, 1074 "Each video substream must have a corresponding ssrc."); 1075 1076 mSendStream = mCall->Call()->CreateVideoSendStream(mSendStreamConfig.Copy(), 1077 mEncoderConfig.Copy()); 1078 1079 if (mTrackSource) { 1080 mSendStream->SetSource(mTrackSource, DegradationPreference()); 1081 } 1082 } 1083 1084 void WebrtcVideoConduit::DeleteRecvStream() { 1085 MOZ_ASSERT(mCallThread->IsOnCurrentThread()); 1086 mMutex.AssertCurrentThreadOwns(); 1087 1088 if (!mRecvStream) { 1089 return; 1090 } 1091 1092 mCall->Call()->DestroyVideoReceiveStream(mRecvStream); 1093 mEngineReceiving = false; 1094 mRecvStream = nullptr; 1095 } 1096 1097 void WebrtcVideoConduit::CreateRecvStream() { 1098 MOZ_ASSERT(mCallThread->IsOnCurrentThread()); 1099 mMutex.AssertCurrentThreadOwns(); 1100 1101 if (mRecvStream) { 1102 return; 1103 } 1104 1105 mRecvStreamConfig.renderer = &mRecvSinkProxy; 1106 1107 for (auto& decoder : mRecvStreamConfig.decoders) { 1108 glean::webrtc_video::recv_codec_used 1109 .Get(nsDependentCString(decoder.video_format.name.c_str())) 1110 .Add(1); 1111 } 1112 1113 mRecvStreamConfig.decoder_factory = mDecoderFactory.get(); 1114 1115 mRecvStream = 1116 mCall->Call()->CreateVideoReceiveStream(mRecvStreamConfig.Copy()); 1117 // Ensure that we set the jitter buffer target on this stream. 1118 mRecvStream->SetBaseMinimumPlayoutDelayMs(mJitterBufferTargetMs); 1119 1120 CSFLogDebug(LOGTAG, "Created VideoReceiveStream %p for SSRC %u (0x%x)", 1121 mRecvStream, mRecvStreamConfig.rtp.remote_ssrc, 1122 mRecvStreamConfig.rtp.remote_ssrc); 1123 } 1124 1125 void WebrtcVideoConduit::NotifyUnsetCurrentRemoteSSRC() { 1126 MOZ_ASSERT(mCallThread->IsOnCurrentThread()); 1127 CSFLogDebug(LOGTAG, "%s (%p): Unsetting SSRC %u in other conduits", 1128 __FUNCTION__, this, mRecvStreamConfig.rtp.remote_ssrc); 1129 mCall->UnregisterConduit(this); 1130 mCall->UnsetRemoteSSRC(mRecvStreamConfig.rtp.remote_ssrc); 1131 mCall->RegisterConduit(this); 1132 } 1133 1134 void WebrtcVideoConduit::SetRemoteSSRCConfig(uint32_t aSsrc, 1135 uint32_t aRtxSsrc) { 1136 MOZ_ASSERT(mCallThread->IsOnCurrentThread()); 1137 1138 CSFLogDebug(LOGTAG, "%s: SSRC %u (0x%x)", __FUNCTION__, aSsrc, aSsrc); 1139 1140 if (mRecvStreamConfig.rtp.remote_ssrc != aSsrc) { 1141 nsCOMPtr<nsIDirectTaskDispatcher> dtd = do_QueryInterface(mCallThread); 1142 MOZ_ALWAYS_SUCCEEDS(dtd->DispatchDirectTask(NewRunnableMethod( 1143 "WebrtcVideoConduit::NotifyUnsetCurrentRemoteSSRC", this, 1144 &WebrtcVideoConduit::NotifyUnsetCurrentRemoteSSRC))); 1145 } 1146 1147 mRecvSSRC = mRecvStreamConfig.rtp.remote_ssrc = aSsrc; 1148 // If we have no associated PT then ensure we dont have an rtx_ssrc set. 1149 mRecvStreamConfig.rtp.rtx_ssrc = 1150 mRecvStreamConfig.rtp.rtx_associated_payload_types.empty() ? 0 : aRtxSsrc; 1151 } 1152 1153 void WebrtcVideoConduit::SetRemoteSSRCAndRestartAsNeeded(uint32_t aSsrc, 1154 uint32_t aRtxSsrc) { 1155 MOZ_ASSERT(mCallThread->IsOnCurrentThread()); 1156 1157 if (mRecvStreamConfig.rtp.remote_ssrc == aSsrc && 1158 mRecvStreamConfig.rtp.rtx_ssrc == aRtxSsrc) { 1159 return; 1160 } 1161 1162 SetRemoteSSRCConfig(aSsrc, aRtxSsrc); 1163 1164 const bool wasReceiving = mEngineReceiving; 1165 const bool hadRecvStream = mRecvStream; 1166 1167 StopReceiving(); 1168 1169 if (hadRecvStream) { 1170 MutexAutoLock lock(mMutex); 1171 DeleteRecvStream(); 1172 CreateRecvStream(); 1173 } 1174 1175 if (wasReceiving) { 1176 StartReceiving(); 1177 } 1178 } 1179 1180 void WebrtcVideoConduit::EnsureRemoteSSRC() { 1181 MOZ_ASSERT(mCallThread->IsOnCurrentThread()); 1182 mMutex.AssertCurrentThreadOwns(); 1183 1184 const auto& ssrcs = mSendStreamConfig.rtp.ssrcs; 1185 if (mRecvStreamConfig.rtp.remote_ssrc != 0 && 1186 std::find(ssrcs.begin(), ssrcs.end(), 1187 mRecvStreamConfig.rtp.remote_ssrc) == ssrcs.end()) { 1188 return; 1189 } 1190 1191 uint32_t ssrc; 1192 do { 1193 ssrc = GenerateRandomSSRC(); 1194 } while ( 1195 NS_WARN_IF(std::find(ssrcs.begin(), ssrcs.end(), ssrc) != ssrcs.end())); 1196 CSFLogDebug(LOGTAG, "VideoConduit %p: Generated remote SSRC %u", this, ssrc); 1197 SetRemoteSSRCConfig(ssrc, 0); 1198 } 1199 1200 void WebrtcVideoConduit::EnsureLocalSSRC() { 1201 MOZ_ASSERT(mCallThread->IsOnCurrentThread()); 1202 mMutex.AssertCurrentThreadOwns(); 1203 1204 auto& ssrcs = mSendStreamConfig.rtp.ssrcs; 1205 if (ssrcs.empty()) { 1206 ssrcs.push_back(GenerateRandomSSRC()); 1207 } 1208 1209 // Reverse-iterating here so that the first dupe in `ssrcs` always wins. 1210 for (auto& ssrc : Reversed(ssrcs)) { 1211 if (ssrc != 0 && ssrc != mRecvStreamConfig.rtp.remote_ssrc && 1212 std::count(ssrcs.begin(), ssrcs.end(), ssrc) == 1) { 1213 continue; 1214 } 1215 do { 1216 ssrc = GenerateRandomSSRC(); 1217 } while (NS_WARN_IF(ssrc == mRecvStreamConfig.rtp.remote_ssrc) || 1218 NS_WARN_IF(std::count(ssrcs.begin(), ssrcs.end(), ssrc) > 1)); 1219 CSFLogDebug(LOGTAG, "%s (%p): Generated local SSRC %u", __FUNCTION__, this, 1220 ssrc); 1221 } 1222 mRecvStreamConfig.rtp.local_ssrc = ssrcs[0]; 1223 } 1224 1225 void WebrtcVideoConduit::UnsetRemoteSSRC(uint32_t aSsrc) { 1226 MOZ_ASSERT(mCallThread->IsOnCurrentThread()); 1227 mMutex.AssertNotCurrentThreadOwns(); 1228 1229 if (mRecvStreamConfig.rtp.remote_ssrc != aSsrc && 1230 mRecvStreamConfig.rtp.rtx_ssrc != aSsrc) { 1231 return; 1232 } 1233 1234 const auto& ssrcs = mSendStreamConfig.rtp.ssrcs; 1235 uint32_t our_ssrc = 0; 1236 do { 1237 our_ssrc = GenerateRandomSSRC(); 1238 } while (NS_WARN_IF(our_ssrc == aSsrc) || 1239 NS_WARN_IF(std::find(ssrcs.begin(), ssrcs.end(), our_ssrc) != 1240 ssrcs.end())); 1241 1242 CSFLogDebug(LOGTAG, "%s (%p): Generated remote SSRC %u", __FUNCTION__, this, 1243 our_ssrc); 1244 1245 // There is a (tiny) chance that this new random ssrc will collide with some 1246 // other conduit's remote ssrc, in which case that conduit will choose a new 1247 // one. 1248 SetRemoteSSRCAndRestartAsNeeded(our_ssrc, 0); 1249 } 1250 1251 /*static*/ 1252 unsigned WebrtcVideoConduit::ToLibwebrtcMaxFramerate( 1253 const Maybe<double>& aMaxFramerate) { 1254 Maybe<unsigned> negotiatedMaxFps; 1255 if (aMaxFramerate.isSome()) { 1256 // libwebrtc does not handle non-integer max framerate. 1257 unsigned integerMaxFps = static_cast<unsigned>(std::round(*aMaxFramerate)); 1258 // libwebrtc crashes with a max framerate of 0, even though the 1259 // spec says this is valid. For now, we treat this as no limit. 1260 if (integerMaxFps) { 1261 negotiatedMaxFps = Some(integerMaxFps); 1262 } 1263 } 1264 // We do not use DEFAULT_VIDEO_MAX_FRAMERATE here; that is used at the very 1265 // end in VideoStreamFactory, once codec-wide and per-encoding limits are 1266 // known. 1267 return negotiatedMaxFps.refOr(std::numeric_limits<unsigned int>::max()); 1268 } 1269 1270 Maybe<Ssrc> WebrtcVideoConduit::GetRemoteSSRC() const { 1271 MOZ_ASSERT(mCallThread->IsOnCurrentThread()); 1272 // libwebrtc uses 0 to mean a lack of SSRC. That is not to spec. 1273 return mRecvStreamConfig.rtp.remote_ssrc == 0 1274 ? Nothing() 1275 : Some(mRecvStreamConfig.rtp.remote_ssrc); 1276 } 1277 1278 Maybe<webrtc::VideoReceiveStreamInterface::Stats> 1279 WebrtcVideoConduit::GetReceiverStats() const { 1280 MOZ_ASSERT(mCallThread->IsOnCurrentThread()); 1281 if (!mRecvStream) { 1282 return Nothing(); 1283 } 1284 return Some(mRecvStream->GetStats()); 1285 } 1286 1287 Maybe<webrtc::VideoSendStream::Stats> WebrtcVideoConduit::GetSenderStats() 1288 const { 1289 MOZ_ASSERT(mCallThread->IsOnCurrentThread()); 1290 if (!mSendStream) { 1291 return Nothing(); 1292 } 1293 auto stats = mSendStream->GetStats(); 1294 if (stats.substreams.empty()) { 1295 if (!mTransitionalSendStreamStats) { 1296 CSFLogError(LOGTAG, "%s: No SSRC in send stream stats", __FUNCTION__); 1297 } 1298 return mTransitionalSendStreamStats; 1299 } 1300 // Successfully got stats, so clear the transitional stats. 1301 mTransitionalSendStreamStats = Nothing(); 1302 return Some(mSendStream->GetStats()); 1303 } 1304 1305 Maybe<webrtc::Call::Stats> WebrtcVideoConduit::GetCallStats() const { 1306 MOZ_ASSERT(mCallThread->IsOnCurrentThread()); 1307 if (!mCall->Call()) { 1308 return Nothing(); 1309 } 1310 return Some(mCall->Call()->GetStats()); 1311 } 1312 1313 MediaConduitErrorCode WebrtcVideoConduit::Init() { 1314 MOZ_ASSERT(NS_IsMainThread()); 1315 1316 CSFLogDebug(LOGTAG, "%s this=%p", __FUNCTION__, this); 1317 1318 #ifdef MOZ_WIDGET_ANDROID 1319 if (mozilla::camera::VideoEngine::SetAndroidObjects() != 0) { 1320 CSFLogError(LOGTAG, "%s: could not set Android objects", __FUNCTION__); 1321 return kMediaConduitSessionNotInited; 1322 } 1323 #endif // MOZ_WIDGET_ANDROID 1324 1325 mSendPluginCreated = mEncoderFactory->CreatedGmpPluginEvent().Connect( 1326 GetMainThreadSerialEventTarget(), 1327 [self = detail::RawPtr(this)](uint64_t aPluginID) { 1328 self.get()->mSendCodecPluginIDs.AppendElement(aPluginID); 1329 }); 1330 mSendPluginReleased = mEncoderFactory->ReleasedGmpPluginEvent().Connect( 1331 GetMainThreadSerialEventTarget(), 1332 [self = detail::RawPtr(this)](uint64_t aPluginID) { 1333 self.get()->mSendCodecPluginIDs.RemoveElement(aPluginID); 1334 }); 1335 mRecvPluginCreated = mDecoderFactory->CreatedGmpPluginEvent().Connect( 1336 GetMainThreadSerialEventTarget(), 1337 [self = detail::RawPtr(this)](uint64_t aPluginID) { 1338 self.get()->mRecvCodecPluginIDs.AppendElement(aPluginID); 1339 }); 1340 mRecvPluginReleased = mDecoderFactory->ReleasedGmpPluginEvent().Connect( 1341 GetMainThreadSerialEventTarget(), 1342 [self = detail::RawPtr(this)](uint64_t aPluginID) { 1343 self.get()->mRecvCodecPluginIDs.RemoveElement(aPluginID); 1344 }); 1345 mRtpSources.Connect(&mCanonicalRtpSources); 1346 1347 MOZ_ALWAYS_SUCCEEDS(mCallThread->Dispatch(NS_NewRunnableFunction( 1348 __func__, [this, self = RefPtr<WebrtcVideoConduit>(this)] { 1349 mCall->RegisterConduit(this); 1350 }))); 1351 1352 CSFLogDebug(LOGTAG, "%s Initialization Done", __FUNCTION__); 1353 return kMediaConduitNoError; 1354 } 1355 1356 RefPtr<GenericPromise> WebrtcVideoConduit::Shutdown() { 1357 MOZ_ASSERT(NS_IsMainThread()); 1358 1359 mSendPluginCreated.DisconnectIfExists(); 1360 mSendPluginReleased.DisconnectIfExists(); 1361 mRecvPluginCreated.DisconnectIfExists(); 1362 mRecvPluginReleased.DisconnectIfExists(); 1363 mReceiverRtpEventListener.DisconnectIfExists(); 1364 mReceiverRtcpEventListener.DisconnectIfExists(); 1365 mSenderRtcpEventListener.DisconnectIfExists(); 1366 mRtpSources.DisconnectIfConnected(); 1367 1368 return InvokeAsync( 1369 mCallThread, __func__, [this, self = RefPtr<WebrtcVideoConduit>(this)] { 1370 if (mSendBitrate.NumDataValues() > 0) { 1371 glean::webrtc::video_encoder_bitrate_avg_per_call_kbps 1372 .AccumulateSingleSample( 1373 static_cast<unsigned>(mSendBitrate.Mean() / 1000)); 1374 glean::webrtc::video_encoder_bitrate_std_dev_per_call_kbps 1375 .AccumulateSingleSample(static_cast<unsigned>( 1376 mSendBitrate.StandardDeviation() / 1000)); 1377 mSendBitrate.Clear(); 1378 } 1379 if (mSendFramerate.NumDataValues() > 0) { 1380 glean::webrtc::video_encoder_framerate_avg_per_call 1381 .AccumulateSingleSample( 1382 static_cast<unsigned>(mSendFramerate.Mean())); 1383 glean::webrtc::video_encoder_framerate_10x_std_dev_per_call 1384 .AccumulateSingleSample(static_cast<unsigned>( 1385 mSendFramerate.StandardDeviation() * 10)); 1386 mSendFramerate.Clear(); 1387 } 1388 1389 if (mRecvBitrate.NumDataValues() > 0) { 1390 glean::webrtc::video_decoder_bitrate_avg_per_call_kbps 1391 .AccumulateSingleSample( 1392 static_cast<unsigned>(mRecvBitrate.Mean() / 1000)); 1393 glean::webrtc::video_decoder_bitrate_std_dev_per_call_kbps 1394 .AccumulateSingleSample(static_cast<unsigned>( 1395 mRecvBitrate.StandardDeviation() / 1000)); 1396 mRecvBitrate.Clear(); 1397 } 1398 if (mRecvFramerate.NumDataValues() > 0) { 1399 glean::webrtc::video_decoder_framerate_avg_per_call 1400 .AccumulateSingleSample( 1401 static_cast<unsigned>(mRecvFramerate.Mean())); 1402 glean::webrtc::video_decoder_framerate_10x_std_dev_per_call 1403 .AccumulateSingleSample(static_cast<unsigned>( 1404 mRecvFramerate.StandardDeviation() * 10)); 1405 mRecvFramerate.Clear(); 1406 } 1407 1408 mControl.mReceiving.DisconnectIfConnected(); 1409 mControl.mTransmitting.DisconnectIfConnected(); 1410 mControl.mLocalSsrcs.DisconnectIfConnected(); 1411 mControl.mLocalRtxSsrcs.DisconnectIfConnected(); 1412 mControl.mLocalCname.DisconnectIfConnected(); 1413 mControl.mMid.DisconnectIfConnected(); 1414 mControl.mRemoteSsrc.DisconnectIfConnected(); 1415 mControl.mRemoteRtxSsrc.DisconnectIfConnected(); 1416 mControl.mSyncGroup.DisconnectIfConnected(); 1417 mControl.mLocalRecvRtpExtensions.DisconnectIfConnected(); 1418 mControl.mLocalSendRtpExtensions.DisconnectIfConnected(); 1419 mControl.mSendCodec.DisconnectIfConnected(); 1420 mControl.mSendRtpRtcpConfig.DisconnectIfConnected(); 1421 mControl.mRecvCodecs.DisconnectIfConnected(); 1422 mControl.mRecvRtpRtcpConfig.DisconnectIfConnected(); 1423 mControl.mCodecMode.DisconnectIfConnected(); 1424 mControl.mFrameTransformerProxySend.DisconnectIfConnected(); 1425 mControl.mFrameTransformerProxyRecv.DisconnectIfConnected(); 1426 mControl.mVideoDegradationPreference.DisconnectIfConnected(); 1427 mWatchManager.Shutdown(); 1428 1429 if (mTrackSource) { 1430 mTrackSource->RemoveSink(&mSendSinkProxy); 1431 } 1432 1433 mCall->UnregisterConduit(this); 1434 mDecoderFactory->DisconnectAll(); 1435 mEncoderFactory->DisconnectAll(); 1436 { 1437 MutexAutoLock lock(mMutex); 1438 DeleteSendStream(); 1439 DeleteRecvStream(); 1440 } 1441 // Clear the stats send stream stats cache 1442 mTransitionalSendStreamStats = Nothing(); 1443 1444 SetIsShutdown(); 1445 1446 return GenericPromise::CreateAndResolve(true, __func__); 1447 }); 1448 } 1449 1450 bool WebrtcVideoConduit::IsShutdown() const { 1451 MOZ_ASSERT(mCallThread->IsOnCurrentThread()); 1452 return mIsShutdown; 1453 } 1454 1455 void WebrtcVideoConduit::SetIsShutdown() { 1456 MOZ_ASSERT(mCallThread->IsOnCurrentThread()); 1457 mIsShutdown = true; 1458 } 1459 1460 webrtc::VideoCodecMode WebrtcVideoConduit::CodecMode() const { 1461 MOZ_ASSERT(mCallThread->IsOnCurrentThread()); 1462 return mControl.mCodecMode; 1463 } 1464 1465 webrtc::DegradationPreference WebrtcVideoConduit::DegradationPreference() 1466 const { 1467 MOZ_ASSERT(mCallThread->IsOnCurrentThread()); 1468 if (mControl.mConfiguredDegradationPreference != 1469 webrtc::DegradationPreference::DISABLED) { 1470 return mControl.mConfiguredDegradationPreference; 1471 } 1472 1473 if (mLockScaling || CodecMode() == webrtc::VideoCodecMode::kScreensharing) { 1474 return webrtc::DegradationPreference::MAINTAIN_RESOLUTION; 1475 } 1476 // Fall back to MAINTAIN_FRAMERATE by default. This is what libwebrtc/Chrome 1477 // uses, because BALANCED hasn't been tuned yet. See 1478 // https://source.chromium.org/chromium/chromium/src/+/main:third_party/webrtc/media/engine/webrtc_video_engine.cc;l=1939;drc=7c2b25f6a19cfeeea67f0f43ed33617840bab33d 1479 return webrtc::DegradationPreference::MAINTAIN_FRAMERATE; 1480 } 1481 1482 MediaConduitErrorCode WebrtcVideoConduit::AttachRenderer( 1483 RefPtr<mozilla::VideoRenderer> aVideoRenderer) { 1484 MOZ_ASSERT(NS_IsMainThread()); 1485 1486 CSFLogDebug(LOGTAG, "%s", __FUNCTION__); 1487 1488 // null renderer 1489 if (!aVideoRenderer) { 1490 CSFLogError(LOGTAG, "%s NULL Renderer", __FUNCTION__); 1491 MOZ_ASSERT(false); 1492 return kMediaConduitInvalidRenderer; 1493 } 1494 1495 // This function is called only from main, so we only need to protect against 1496 // modifying mRenderer while any webrtc.org code is trying to use it. 1497 { 1498 ReentrantMonitorAutoEnter enter(mRendererMonitor); 1499 mRenderer = aVideoRenderer; 1500 } 1501 1502 return kMediaConduitNoError; 1503 } 1504 1505 void WebrtcVideoConduit::DetachRenderer() { 1506 MOZ_ASSERT(NS_IsMainThread()); 1507 1508 ReentrantMonitorAutoEnter enter(mRendererMonitor); 1509 if (mRenderer) { 1510 mRenderer = nullptr; 1511 } 1512 } 1513 1514 webrtc::RefCountedObject<mozilla::VideoStreamFactory>* 1515 WebrtcVideoConduit::CreateVideoStreamFactory() { 1516 auto videoStreamFactory = mVideoStreamFactory.Lock(); 1517 *videoStreamFactory = new webrtc::RefCountedObject<VideoStreamFactory>( 1518 *mCurSendCodecConfig, mMinBitrate, mStartBitrate, mPrefMaxBitrate, 1519 mNegotiatedMaxBitrate); 1520 return videoStreamFactory->get(); 1521 } 1522 1523 void WebrtcVideoConduit::OnSendFrame(const webrtc::VideoFrame& aFrame) { 1524 MOZ_ASSERT(mCallThread->IsOnCurrentThread()); 1525 // XXX Google uses a "timestamp_aligner" to translate timestamps from the 1526 // camera via TranslateTimestamp(); we should look at doing the same. This 1527 // avoids sampling error when capturing frames, but google had to deal with 1528 // some broken cameras, include Logitech c920's IIRC. 1529 1530 const gfx::IntSize size{aFrame.width(), aFrame.height()}; 1531 1532 CSFLogVerbose(LOGTAG, "WebrtcVideoConduit %p %s (send SSRC %u (0x%x))", this, 1533 __FUNCTION__, mSendStreamConfig.rtp.ssrcs.front(), 1534 mSendStreamConfig.rtp.ssrcs.front()); 1535 1536 if (Some(size) != mLastSize) { 1537 MOZ_ASSERT(size != gfx::IntSize(0, 0)); 1538 // Note coverity will flag this since it thinks they can be 0 1539 MOZ_ASSERT(mCurSendCodecConfig); 1540 mLastSize = Some(size); 1541 } 1542 1543 MOZ_ASSERT(!aFrame.color_space(), "Unexpected use of color space"); 1544 MOZ_ASSERT(!aFrame.has_update_rect(), "Unexpected use of update rect"); 1545 1546 #ifdef MOZ_REAL_TIME_TRACING 1547 if (profiler_is_active()) { 1548 MutexAutoLock lock(mMutex); 1549 nsAutoCStringN<256> ssrcsCommaSeparated; 1550 bool first = true; 1551 for (auto ssrc : mSendStreamConfig.rtp.ssrcs) { 1552 if (!first) { 1553 ssrcsCommaSeparated.AppendASCII(", "); 1554 } else { 1555 first = false; 1556 } 1557 ssrcsCommaSeparated.AppendInt(ssrc); 1558 } 1559 1560 const webrtc::Timestamp currentTimestamp = 1561 webrtc::Timestamp::Micros(aFrame.timestamp_us()); 1562 // The first frame has a delta of zero. 1563 webrtc::TimeDelta timestampDelta = 1564 mLastTimestampSend.isSome() 1565 ? currentTimestamp - mLastTimestampSend.value() 1566 : webrtc::TimeDelta::Zero(); 1567 mLastTimestampSend = Some(currentTimestamp); 1568 TRACE_COMMENT("VideoConduit::OnSendFrame (async)", 1569 "t-delta=%.1fms, ssrcs=%s", timestampDelta.ms<double>(), 1570 ssrcsCommaSeparated.get()); 1571 } 1572 #endif 1573 } 1574 1575 void WebrtcVideoConduit::SetTrackSource( 1576 webrtc::VideoTrackSourceInterface* aSource) { 1577 MOZ_ASSERT(NS_IsMainThread()); 1578 MOZ_ALWAYS_SUCCEEDS(mCallThread->Dispatch(NS_NewRunnableFunction( 1579 __func__, [this, self = RefPtr(this), src = RefPtr(aSource)] { 1580 if (mTrackSource) { 1581 mTrackSource->RemoveSink(&mSendSinkProxy); 1582 } 1583 if (src) { 1584 src->AddOrUpdateSink(&mSendSinkProxy, {}); 1585 } 1586 mTrackSource = src; 1587 if (mSendStream) { 1588 mSendStream->SetSource(mTrackSource, DegradationPreference()); 1589 } 1590 }))); 1591 } 1592 1593 // Transport Layer Callbacks 1594 1595 void WebrtcVideoConduit::DeliverPacket(webrtc::CopyOnWriteBuffer packet, 1596 PacketType type) { 1597 // Currently unused. 1598 MOZ_ASSERT(false); 1599 } 1600 1601 void WebrtcVideoConduit::OnRtpReceived(webrtc::RtpPacketReceived&& aPacket, 1602 webrtc::RTPHeader&& aHeader) { 1603 MOZ_ASSERT(mCallThread->IsOnCurrentThread()); 1604 1605 // We should only be handling packets on this conduit if we are set to receive 1606 // them. 1607 if (!mControl.mReceiving) { 1608 CSFLogVerbose(LOGTAG, 1609 "VideoConduit %p: Discarding packet SEQ# %u SSRC %u as not " 1610 "configured to receive.", 1611 this, aPacket.SequenceNumber(), aHeader.ssrc); 1612 return; 1613 } 1614 1615 mRemoteSendSSRC = aHeader.ssrc; 1616 1617 if (mAllowSsrcChange || mRecvStreamConfig.rtp.remote_ssrc == 0) { 1618 bool switchRequired = mRecvStreamConfig.rtp.remote_ssrc != aHeader.ssrc; 1619 if (switchRequired) { 1620 // Handle the unknown ssrc (and ssrc-not-signaled case). 1621 1622 // We need to check that the newly received ssrc is not already 1623 // associated with ulpfec or rtx. This is how webrtc.org handles 1624 // things, see https://codereview.webrtc.org/1226093002. 1625 const webrtc::VideoReceiveStreamInterface::Config::Rtp& rtp = 1626 mRecvStreamConfig.rtp; 1627 switchRequired = 1628 rtp.rtx_associated_payload_types.find(aHeader.payloadType) == 1629 rtp.rtx_associated_payload_types.end() && 1630 rtp.ulpfec_payload_type != aHeader.payloadType; 1631 } 1632 1633 if (switchRequired) { 1634 CSFLogInfo(LOGTAG, "VideoConduit %p: Switching remote SSRC from %u to %u", 1635 this, mRecvStreamConfig.rtp.remote_ssrc, aHeader.ssrc); 1636 SetRemoteSSRCAndRestartAsNeeded(aHeader.ssrc, 0); 1637 } 1638 } 1639 1640 CSFLogVerbose(LOGTAG, "%s: seq# %u, Len %zu, SSRC %u (0x%x) ", __FUNCTION__, 1641 aPacket.SequenceNumber(), aPacket.size(), aPacket.Ssrc(), 1642 aPacket.Ssrc()); 1643 1644 // Libwebrtc commit cde4b67d9d now expect calls to 1645 // SourceTracker::GetSources() to happen on the call thread. We'll 1646 // grab the value now while on the call thread, and dispatch to main 1647 // to store the cached value if we have new source information. 1648 // See Bug 1845621. 1649 if (mRecvStream) { 1650 mCanonicalRtpSources = mRecvStream->GetSources(); 1651 } 1652 1653 mRtpPacketEvent.Notify(); 1654 if (mCall->Call()) { 1655 mCall->Call()->Receiver()->DeliverRtpPacket( 1656 webrtc::MediaType::VIDEO, std::move(aPacket), 1657 [self = RefPtr<WebrtcVideoConduit>(this)]( 1658 const webrtc::RtpPacketReceived& packet) { 1659 CSFLogVerbose( 1660 LOGTAG, 1661 "VideoConduit %p: failed demuxing packet, ssrc: %u seq: %u", 1662 self.get(), packet.Ssrc(), packet.SequenceNumber()); 1663 return false; 1664 }); 1665 } 1666 } 1667 1668 void WebrtcVideoConduit::OnRtcpReceived(webrtc::CopyOnWriteBuffer&& aPacket) { 1669 MOZ_ASSERT(mCallThread->IsOnCurrentThread()); 1670 1671 if (mCall->Call()) { 1672 mCall->Call()->Receiver()->DeliverRtcpPacket( 1673 std::forward<webrtc::CopyOnWriteBuffer>(aPacket)); 1674 } 1675 } 1676 1677 Maybe<uint16_t> WebrtcVideoConduit::RtpSendBaseSeqFor(uint32_t aSsrc) const { 1678 MOZ_ASSERT(mCallThread->IsOnCurrentThread()); 1679 auto it = mRtpSendBaseSeqs.find(aSsrc); 1680 if (it == mRtpSendBaseSeqs.end()) { 1681 return Nothing(); 1682 } 1683 return Some(it->second); 1684 } 1685 1686 const dom::RTCStatsTimestampMaker& WebrtcVideoConduit::GetTimestampMaker() 1687 const { 1688 return mCall->GetTimestampMaker(); 1689 } 1690 1691 void WebrtcVideoConduit::StopTransmitting() { 1692 MOZ_ASSERT(mCallThread->IsOnCurrentThread()); 1693 mMutex.AssertNotCurrentThreadOwns(); 1694 1695 if (!mEngineTransmitting) { 1696 return; 1697 } 1698 1699 if (mSendStream) { 1700 CSFLogDebug(LOGTAG, "%s Stopping send stream", __FUNCTION__); 1701 mSendStream->Stop(); 1702 } 1703 1704 mEngineTransmitting = false; 1705 } 1706 1707 void WebrtcVideoConduit::StartTransmitting() { 1708 MOZ_ASSERT(mCallThread->IsOnCurrentThread()); 1709 MOZ_ASSERT(mSendStream); 1710 mMutex.AssertNotCurrentThreadOwns(); 1711 1712 if (mEngineTransmitting) { 1713 return; 1714 } 1715 1716 CSFLogDebug(LOGTAG, "%s Starting send stream", __FUNCTION__); 1717 1718 mSendStream->Start(); 1719 // XXX File a bug to consider hooking this up to the state of mtransport 1720 mCall->Call()->SignalChannelNetworkState(webrtc::MediaType::VIDEO, 1721 webrtc::kNetworkUp); 1722 mEngineTransmitting = true; 1723 } 1724 1725 void WebrtcVideoConduit::StopReceiving() { 1726 MOZ_ASSERT(mCallThread->IsOnCurrentThread()); 1727 mMutex.AssertNotCurrentThreadOwns(); 1728 1729 // Are we receiving already? If so, stop receiving and playout 1730 // since we can't apply new recv codec when the engine is playing. 1731 if (!mEngineReceiving) { 1732 return; 1733 } 1734 1735 if (mRecvStream) { 1736 CSFLogDebug(LOGTAG, "%s Stopping receive stream", __FUNCTION__); 1737 mRecvStream->Stop(); 1738 } 1739 1740 mEngineReceiving = false; 1741 } 1742 1743 void WebrtcVideoConduit::StartReceiving() { 1744 MOZ_ASSERT(mCallThread->IsOnCurrentThread()); 1745 MOZ_ASSERT(mRecvStream); 1746 mMutex.AssertNotCurrentThreadOwns(); 1747 1748 if (mEngineReceiving) { 1749 return; 1750 } 1751 1752 CSFLogDebug(LOGTAG, "%s Starting receive stream (SSRC %u (0x%x))", 1753 __FUNCTION__, mRecvStreamConfig.rtp.remote_ssrc, 1754 mRecvStreamConfig.rtp.remote_ssrc); 1755 // Start Receiving on the video engine 1756 mRecvStream->Start(); 1757 1758 // XXX File a bug to consider hooking this up to the state of mtransport 1759 mCall->Call()->SignalChannelNetworkState(webrtc::MediaType::VIDEO, 1760 webrtc::kNetworkUp); 1761 mEngineReceiving = true; 1762 } 1763 1764 bool WebrtcVideoConduit::SendRtp(const uint8_t* aData, size_t aLength, 1765 const webrtc::PacketOptions& aOptions) { 1766 MOZ_ASSERT(aLength >= 12); 1767 const uint16_t seqno = ntohs(*((uint16_t*)&aData[2])); 1768 const uint32_t ssrc = ntohl(*((uint32_t*)&aData[8])); 1769 1770 CSFLogVerbose( 1771 LOGTAG, 1772 "VideoConduit %p: Sending RTP Packet seq# %u, len %zu, SSRC %u (0x%x)", 1773 this, seqno, aLength, ssrc, ssrc); 1774 1775 if (!mTransportActive) { 1776 CSFLogError(LOGTAG, "VideoConduit %p: RTP Packet Send Failed", this); 1777 return false; 1778 } 1779 1780 MediaPacket packet; 1781 packet.Copy(aData, aLength, aLength + SRTP_MAX_EXPANSION); 1782 packet.SetType(MediaPacket::RTP); 1783 mSenderRtpSendEvent.Notify(std::move(packet)); 1784 1785 // Parse the sequence number of the first rtp packet as base_seq. 1786 const auto inserted = mRtpSendBaseSeqs_n.insert({ssrc, seqno}).second; 1787 1788 if (inserted || aOptions.packet_id >= 0) { 1789 int64_t now_ms = PR_Now() / 1000; 1790 MOZ_ALWAYS_SUCCEEDS(mCallThread->Dispatch(NS_NewRunnableFunction( 1791 __func__, [this, self = RefPtr<WebrtcVideoConduit>(this), 1792 packet_id = aOptions.packet_id, now_ms, ssrc, seqno] { 1793 mRtpSendBaseSeqs.insert({ssrc, seqno}); 1794 if (packet_id >= 0) { 1795 if (mCall->Call()) { 1796 // TODO: This notification should ideally happen after the 1797 // transport layer has sent the packet on the wire. 1798 mCall->Call()->OnSentPacket({packet_id, now_ms}); 1799 } 1800 } 1801 }))); 1802 } 1803 return true; 1804 } 1805 1806 bool WebrtcVideoConduit::SendSenderRtcp(const uint8_t* aData, size_t aLength) { 1807 CSFLogVerbose( 1808 LOGTAG, 1809 "VideoConduit %p: Sending RTCP SR Packet, len %zu, SSRC %u (0x%x)", this, 1810 aLength, (uint32_t)ntohl(*((uint32_t*)&aData[4])), 1811 (uint32_t)ntohl(*((uint32_t*)&aData[4]))); 1812 1813 if (!mTransportActive) { 1814 CSFLogError(LOGTAG, "VideoConduit %p: RTCP SR Packet Send Failed", this); 1815 return false; 1816 } 1817 1818 MediaPacket packet; 1819 packet.Copy(aData, aLength, aLength + SRTP_MAX_EXPANSION); 1820 packet.SetType(MediaPacket::RTCP); 1821 mSenderRtcpSendEvent.Notify(std::move(packet)); 1822 return true; 1823 } 1824 1825 bool WebrtcVideoConduit::SendReceiverRtcp(const uint8_t* aData, 1826 size_t aLength) { 1827 CSFLogVerbose( 1828 LOGTAG, 1829 "VideoConduit %p: Sending RTCP RR Packet, len %zu, SSRC %u (0x%x)", this, 1830 aLength, (uint32_t)ntohl(*((uint32_t*)&aData[4])), 1831 (uint32_t)ntohl(*((uint32_t*)&aData[4]))); 1832 1833 if (!mTransportActive) { 1834 CSFLogError(LOGTAG, "VideoConduit %p: RTCP RR Packet Send Failed", this); 1835 return false; 1836 } 1837 1838 MediaPacket packet; 1839 packet.Copy(aData, aLength, aLength + SRTP_MAX_EXPANSION); 1840 packet.SetType(MediaPacket::RTCP); 1841 mReceiverRtcpSendEvent.Notify(std::move(packet)); 1842 return true; 1843 } 1844 1845 void WebrtcVideoConduit::OnRecvFrame(const webrtc::VideoFrame& aFrame) { 1846 MOZ_ASSERT(mFrameRecvThread->IsOnCurrentThread()); 1847 const uint32_t localRecvSsrc = mRecvSSRC; 1848 const uint32_t remoteSendSsrc = mRemoteSendSSRC; 1849 1850 CSFLogVerbose( 1851 LOGTAG, 1852 "VideoConduit %p: Rendering frame, Remote SSRC %u (0x%x), size %ux%u", 1853 this, static_cast<uint32_t>(remoteSendSsrc), 1854 static_cast<uint32_t>(remoteSendSsrc), aFrame.width(), aFrame.height()); 1855 ReentrantMonitorAutoEnter enter(mRendererMonitor); 1856 1857 if (!mRenderer) { 1858 CSFLogError(LOGTAG, "VideoConduit %p: Cannot render frame, no renderer", 1859 this); 1860 return; 1861 } 1862 1863 bool needsNewHistoryElement = mReceivedFrameHistory.mEntries.IsEmpty(); 1864 { 1865 const Maybe frameSize = Some(gfx::IntSize{aFrame.width(), aFrame.height()}); 1866 if (frameSize != mReceivingSize.Ref()) { 1867 mReceivingSize = frameSize; 1868 needsNewHistoryElement = true; 1869 } 1870 } 1871 1872 if (!needsNewHistoryElement) { 1873 auto& currentEntry = mReceivedFrameHistory.mEntries.LastElement(); 1874 needsNewHistoryElement = 1875 currentEntry.mRotationAngle != 1876 static_cast<unsigned long>(aFrame.rotation()) || 1877 currentEntry.mLocalSsrc != localRecvSsrc || 1878 currentEntry.mRemoteSsrc != remoteSendSsrc; 1879 } 1880 1881 // Record frame history 1882 const auto historyNow = mCall->GetTimestampMaker().GetNow().ToDom(); 1883 if (needsNewHistoryElement) { 1884 dom::RTCVideoFrameHistoryEntryInternal frameHistoryElement; 1885 frameHistoryElement.mConsecutiveFrames = 0; 1886 frameHistoryElement.mWidth = aFrame.width(); 1887 frameHistoryElement.mHeight = aFrame.height(); 1888 frameHistoryElement.mRotationAngle = 1889 static_cast<unsigned long>(aFrame.rotation()); 1890 frameHistoryElement.mFirstFrameTimestamp = historyNow; 1891 frameHistoryElement.mLocalSsrc = localRecvSsrc; 1892 frameHistoryElement.mRemoteSsrc = remoteSendSsrc; 1893 if (!mReceivedFrameHistory.mEntries.AppendElement(frameHistoryElement, 1894 fallible)) { 1895 mozalloc_handle_oom(0); 1896 } 1897 } 1898 auto& currentEntry = mReceivedFrameHistory.mEntries.LastElement(); 1899 1900 currentEntry.mConsecutiveFrames++; 1901 currentEntry.mLastFrameTimestamp = historyNow; 1902 // Attempt to retrieve an timestamp encoded in the image pixels if enabled. 1903 if (mVideoLatencyTestEnable && mReceivingSize.Ref()) { 1904 uint64_t now = PR_Now(); 1905 uint64_t timestamp = 0; 1906 uint8_t* data = 1907 const_cast<uint8_t*>(aFrame.video_frame_buffer()->GetI420()->DataY()); 1908 bool ok = YuvStamper::Decode( 1909 mReceivingSize.Ref()->width, mReceivingSize.Ref()->height, 1910 mReceivingSize.Ref()->width, data, 1911 reinterpret_cast<unsigned char*>(×tamp), sizeof(timestamp), 0, 0); 1912 if (ok) { 1913 VideoLatencyUpdate(now - timestamp); 1914 } 1915 } 1916 #ifdef MOZ_REAL_TIME_TRACING 1917 if (profiler_is_active()) { 1918 MutexAutoLock lock(mMutex); 1919 // The first frame has a delta of zero. 1920 uint32_t rtpTimestamp = aFrame.rtp_timestamp(); 1921 uint32_t timestampDelta = 1922 mLastRTPTimestampReceive.isSome() 1923 ? rtpTimestamp - mLastRTPTimestampReceive.value() 1924 : 0; 1925 mLastRTPTimestampReceive = Some(rtpTimestamp); 1926 TRACE_COMMENT("VideoConduit::OnRecvFrame", "t-delta=%.1fms, ssrc=%u", 1927 timestampDelta * 1000.f / webrtc::kVideoPayloadTypeFrequency, 1928 localRecvSsrc); 1929 } 1930 #endif 1931 1932 mRenderer->RenderVideoFrame(aFrame); 1933 } 1934 1935 bool WebrtcVideoConduit::AddFrameHistory( 1936 dom::Sequence<dom::RTCVideoFrameHistoryInternal>* outHistories) const { 1937 ReentrantMonitorAutoEnter enter(mRendererMonitor); 1938 if (!outHistories->AppendElement(mReceivedFrameHistory, fallible)) { 1939 mozalloc_handle_oom(0); 1940 return false; 1941 } 1942 return true; 1943 } 1944 1945 void WebrtcVideoConduit::SetJitterBufferTarget(DOMHighResTimeStamp aTargetMs) { 1946 MOZ_RELEASE_ASSERT(aTargetMs <= std::numeric_limits<uint16_t>::max()); 1947 MOZ_RELEASE_ASSERT(aTargetMs >= 0); 1948 1949 MOZ_ALWAYS_SUCCEEDS(mCallThread->Dispatch(NS_NewRunnableFunction( 1950 __func__, 1951 [this, self = RefPtr<WebrtcVideoConduit>(this), targetMs = aTargetMs] { 1952 mJitterBufferTargetMs = static_cast<uint16_t>(targetMs); 1953 if (mRecvStream) { 1954 mRecvStream->SetBaseMinimumPlayoutDelayMs(targetMs); 1955 } 1956 }))); 1957 } 1958 1959 void WebrtcVideoConduit::DumpCodecDB() const { 1960 MOZ_ASSERT(mCallThread->IsOnCurrentThread()); 1961 1962 for (const auto& entry : mControl.mConfiguredRecvCodecs) { 1963 CSFLogDebug(LOGTAG, "Payload Name: %s", entry.mName.c_str()); 1964 CSFLogDebug(LOGTAG, "Payload Type: %d", entry.mType); 1965 CSFLogDebug(LOGTAG, "Payload Max Frame Size: %d", 1966 entry.mEncodingConstraints.maxFs); 1967 if (entry.mEncodingConstraints.maxFps.isSome()) { 1968 CSFLogDebug(LOGTAG, "Payload Max Frame Rate: %f", 1969 *entry.mEncodingConstraints.maxFps); 1970 } 1971 } 1972 } 1973 1974 void WebrtcVideoConduit::VideoLatencyUpdate(uint64_t aNewSample) { 1975 mRendererMonitor.AssertCurrentThreadIn(); 1976 1977 mVideoLatencyAvg = 1978 (sRoundingPadding * aNewSample + sAlphaNum * mVideoLatencyAvg) / 1979 sAlphaDen; 1980 } 1981 1982 uint64_t WebrtcVideoConduit::MozVideoLatencyAvg() { 1983 mRendererMonitor.AssertCurrentThreadIn(); 1984 1985 return mVideoLatencyAvg / sRoundingPadding; 1986 } 1987 1988 void WebrtcVideoConduit::CollectTelemetryData() { 1989 MOZ_ASSERT(mCallThread->IsOnCurrentThread()); 1990 1991 if (mEngineTransmitting) { 1992 webrtc::VideoSendStream::Stats stats = mSendStream->GetStats(); 1993 mSendBitrate.Push(stats.media_bitrate_bps); 1994 mSendFramerate.Push(stats.encode_frame_rate); 1995 } 1996 if (mEngineReceiving) { 1997 webrtc::VideoReceiveStreamInterface::Stats stats = mRecvStream->GetStats(); 1998 mRecvBitrate.Push(stats.total_bitrate_bps); 1999 mRecvFramerate.Push(stats.decode_frame_rate); 2000 } 2001 } 2002 2003 void WebrtcVideoConduit::OnRtcpBye() { mRtcpByeEvent.Notify(); } 2004 2005 void WebrtcVideoConduit::OnRtcpTimeout() { mRtcpTimeoutEvent.Notify(); } 2006 2007 void WebrtcVideoConduit::SetTransportActive(bool aActive) { 2008 MOZ_ASSERT(mStsThread->IsOnCurrentThread()); 2009 if (mTransportActive == aActive) { 2010 return; 2011 } 2012 2013 // If false, This stops us from sending 2014 mTransportActive = aActive; 2015 } 2016 2017 const std::vector<webrtc::RtpSource>& 2018 WebrtcVideoConduit::GetUpstreamRtpSources() const { 2019 MOZ_ASSERT(NS_IsMainThread()); 2020 return mRtpSources; 2021 } 2022 2023 void WebrtcVideoConduit::RequestKeyFrame(FrameTransformerProxy* aProxy) { 2024 mCallThread->Dispatch(NS_NewRunnableFunction( 2025 __func__, [this, self = RefPtr<WebrtcVideoConduit>(this), 2026 proxy = RefPtr<FrameTransformerProxy>(aProxy)] { 2027 if (mRecvStream && mEngineReceiving) { 2028 // This is a misnomer. This requests a keyframe from the other side. 2029 mRecvStream->GenerateKeyFrame(); 2030 } 2031 proxy->KeyFrameRequestDone(); 2032 })); 2033 } 2034 2035 void WebrtcVideoConduit::GenerateKeyFrame(const Maybe<std::string>& aRid, 2036 FrameTransformerProxy* aProxy) { 2037 // libwebrtc does not implement error handling in the way that 2038 // webrtc-encoded-transform specifies. So, we'll need to do that here. 2039 // Also, spec wants us to synchronously check whether there's an encoder, but 2040 // that's not something that can be checked synchronously. 2041 2042 mCallThread->Dispatch(NS_NewRunnableFunction( 2043 __func__, [this, self = RefPtr(this), proxy = RefPtr(aProxy), aRid] { 2044 // If encoder is undefined, reject promise with InvalidStateError, 2045 // abort these steps. 2046 2047 // If encoder is not processing video frames, reject promise with 2048 // InvalidStateError, abort these steps. 2049 if (!mSendStream || !mCurSendCodecConfig) { 2050 CopyableErrorResult result; 2051 result.ThrowInvalidStateError("No encoders"); 2052 proxy->GenerateKeyFrameError(aRid, result); 2053 return; 2054 } 2055 2056 // Gather a list of video encoders, named videoEncoders from encoder, 2057 // ordered according negotiated RIDs if any. 2058 // NOTE: This is represented by mCurSendCodecConfig->mEncodings 2059 2060 // If rid is defined, remove from videoEncoders any video encoder that 2061 // does not match rid. 2062 2063 // If rid is undefined, remove from videoEncoders all video encoders 2064 // except the first one. 2065 bool found = false; 2066 std::vector<std::string> rids; 2067 if (!aRid.isSome()) { 2068 // If rid is undefined, set rid to the RID value corresponding to 2069 // videoEncoder. 2070 if (!mCurSendCodecConfig->mEncodings.empty()) { 2071 if (!mCurSendCodecConfig->mEncodings[0].rid.empty()) { 2072 rids.push_back(mCurSendCodecConfig->mEncodings[0].rid); 2073 } 2074 found = true; 2075 } 2076 } else { 2077 for (const auto& encoding : mCurSendCodecConfig->mEncodings) { 2078 if (encoding.rid == *aRid) { 2079 found = true; 2080 rids.push_back(encoding.rid); 2081 break; 2082 } 2083 } 2084 } 2085 2086 // If videoEncoders is empty, reject promise with NotFoundError and 2087 // abort these steps. videoEncoders is expected to be empty if the 2088 // corresponding RTCRtpSender is not active, or the corresponding 2089 // RTCRtpSender track is ended. 2090 if (!found) { 2091 CopyableErrorResult result; 2092 result.ThrowNotFoundError("Rid not in use"); 2093 proxy->GenerateKeyFrameError(aRid, result); 2094 } 2095 2096 // NOTE: We don't do this stuff, because libwebrtc's interface is 2097 // rid-based. 2098 // Let videoEncoder be the first encoder in videoEncoders. 2099 // If rid is undefined, set rid to the RID value corresponding to 2100 // videoEncoder. 2101 2102 mSendStream->GenerateKeyFrame(rids); 2103 })); 2104 } 2105 2106 bool WebrtcVideoConduit::HasCodecPluginID(uint64_t aPluginID) const { 2107 MOZ_ASSERT(NS_IsMainThread()); 2108 2109 return mSendCodecPluginIDs.Contains(aPluginID) || 2110 mRecvCodecPluginIDs.Contains(aPluginID); 2111 } 2112 2113 bool WebrtcVideoConduit::HasH264Hardware() { 2114 nsCOMPtr<nsIGfxInfo> gfxInfo = do_GetService("@mozilla.org/gfx/info;1"); 2115 if (!gfxInfo) { 2116 return false; 2117 } 2118 int32_t status; 2119 nsCString discardFailureId; 2120 return NS_SUCCEEDED(gfxInfo->GetFeatureStatus( 2121 nsIGfxInfo::FEATURE_WEBRTC_HW_ACCELERATION_H264, discardFailureId, 2122 &status)) && 2123 status == nsIGfxInfo::FEATURE_STATUS_OK; 2124 } 2125 2126 bool WebrtcVideoConduit::HasAv1() { 2127 #if defined(MOZ_AV1) 2128 return true; 2129 #else 2130 return false; 2131 #endif 2132 } 2133 2134 Maybe<int> WebrtcVideoConduit::ActiveSendPayloadType() const { 2135 MOZ_ASSERT(mCallThread->IsOnCurrentThread()); 2136 2137 if (!mSendStream) { 2138 return Nothing(); 2139 } 2140 2141 if (mSendStreamConfig.rtp.payload_type == -1) { 2142 return Nothing(); 2143 } 2144 2145 return Some(mSendStreamConfig.rtp.payload_type); 2146 } 2147 2148 Maybe<int> WebrtcVideoConduit::ActiveRecvPayloadType() const { 2149 MOZ_ASSERT(mCallThread->IsOnCurrentThread()); 2150 2151 auto stats = GetReceiverStats(); 2152 if (!stats) { 2153 return Nothing(); 2154 } 2155 2156 if (stats->current_payload_type == -1) { 2157 return Nothing(); 2158 } 2159 2160 return Some(stats->current_payload_type); 2161 } 2162 2163 } // namespace mozilla