AudioConduit.cpp (36063B)
1 /* This Source Code Form is subject to the terms of the Mozilla Public 2 * License, v. 2.0. If a copy of the MPL was not distributed with this file, 3 * You can obtain one at http://mozilla.org/MPL/2.0/. */ 4 5 #include "AudioConduit.h" 6 7 #include <vector> 8 9 #include "CodecConfig.h" 10 #include "MediaConduitControl.h" 11 #include "WebrtcCallWrapper.h" 12 #include "common/browser_logging/CSFLog.h" 13 #include "libwebrtcglue/FrameTransformer.h" 14 #include "mozilla/MozPromise.h" 15 #include "mozilla/RWLock.h" 16 #include "mozilla/RefPtr.h" 17 #include "mozilla/StateMirroring.h" 18 #include "transport/SrtpFlow.h" // For SRTP_MAX_EXPANSION 19 20 // libwebrtc includes 21 #include <stdint.h> 22 23 #include <limits> 24 #include <memory> 25 #include <string> 26 #include <utility> 27 28 #include "MainThreadUtils.h" 29 #include "MediaConduitErrors.h" 30 #include "MediaConduitInterface.h" 31 #include "api/audio/audio_frame.h" 32 #include "api/audio/audio_mixer.h" 33 #include "api/audio_codecs/audio_format.h" 34 #include "api/audio_codecs/builtin_audio_encoder_factory.h" 35 #include "api/call/transport.h" 36 #include "api/media_types.h" 37 #include "api/rtp_headers.h" 38 #include "api/rtp_parameters.h" 39 #include "api/transport/rtp/rtp_source.h" 40 #include "audio/audio_receive_stream.h" 41 #include "call/audio_receive_stream.h" 42 #include "call/audio_send_stream.h" 43 #include "call/call_basic_stats.h" 44 #include "domstubs.h" 45 #include "jsapi/RTCStatsReport.h" 46 #include "media/base/media_constants.h" 47 #include "modules/rtp_rtcp/source/rtp_packet_received.h" 48 #include "mozilla/Assertions.h" 49 #include "mozilla/Maybe.h" 50 #include "mozilla/StateWatching.h" 51 #include "nsCOMPtr.h" 52 #include "nsError.h" 53 #include "nsISerialEventTarget.h" 54 #include "nsThreadUtils.h" 55 #include "rtc_base/copy_on_write_buffer.h" 56 #include "rtc_base/network/sent_packet.h" 57 #include "rtc_base/ref_counted_object.h" 58 #include "transport/mediapacket.h" 59 60 // for ntohs 61 #ifdef HAVE_NETINET_IN_H 62 # include <netinet/in.h> 63 #elif defined XP_WIN 64 # include <winsock2.h> 65 #endif 66 67 #ifdef MOZ_WIDGET_ANDROID 68 # include "AndroidBridge.h" 69 #endif 70 71 namespace mozilla { 72 73 namespace { 74 75 static const char* acLogTag = "WebrtcAudioSessionConduit"; 76 #ifdef LOGTAG 77 # undef LOGTAG 78 #endif 79 #define LOGTAG acLogTag 80 81 using namespace webrtc; 82 using LocalDirection = MediaSessionConduitLocalDirection; 83 84 const char kCodecParamCbr[] = "cbr"; 85 86 } // namespace 87 88 /** 89 * Factory Method for AudioConduit 90 */ 91 RefPtr<AudioSessionConduit> AudioSessionConduit::Create( 92 RefPtr<WebrtcCallWrapper> aCall, 93 nsCOMPtr<nsISerialEventTarget> aStsThread) { 94 CSFLogDebug(LOGTAG, "%s ", __FUNCTION__); 95 MOZ_ASSERT(NS_IsMainThread()); 96 97 auto conduit = 98 MakeRefPtr<WebrtcAudioConduit>(std::move(aCall), std::move(aStsThread)); 99 if (conduit->Init() != kMediaConduitNoError) { 100 CSFLogError(LOGTAG, "%s AudioConduit Init Failed ", __FUNCTION__); 101 return nullptr; 102 } 103 104 CSFLogDebug(LOGTAG, "%s Successfully created AudioConduit %p", __FUNCTION__, 105 conduit.get()); 106 return conduit; 107 } 108 109 #define INIT_MIRROR(name, val) \ 110 name(aCallThread, val, "WebrtcAudioConduit::Control::" #name " (Mirror)") 111 WebrtcAudioConduit::Control::Control(const RefPtr<AbstractThread>& aCallThread) 112 : INIT_MIRROR(mReceiving, false), 113 INIT_MIRROR(mTransmitting, false), 114 INIT_MIRROR(mLocalSsrcs, Ssrcs()), 115 INIT_MIRROR(mLocalCname, std::string()), 116 INIT_MIRROR(mMid, std::string()), 117 INIT_MIRROR(mRemoteSsrc, 0), 118 INIT_MIRROR(mSyncGroup, std::string()), 119 INIT_MIRROR(mLocalRecvRtpExtensions, RtpExtList()), 120 INIT_MIRROR(mLocalSendRtpExtensions, RtpExtList()), 121 INIT_MIRROR(mSendCodec, Nothing()), 122 INIT_MIRROR(mRecvCodecs, std::vector<AudioCodecConfig>()), 123 INIT_MIRROR(mFrameTransformerProxySend, nullptr), 124 INIT_MIRROR(mFrameTransformerProxyRecv, nullptr) {} 125 #undef INIT_MIRROR 126 127 RefPtr<GenericPromise> WebrtcAudioConduit::Shutdown() { 128 MOZ_ASSERT(NS_IsMainThread()); 129 130 mControl.mOnDtmfEventListener.DisconnectIfExists(); 131 mReceiverRtpEventListener.DisconnectIfExists(); 132 mReceiverRtcpEventListener.DisconnectIfExists(); 133 mSenderRtcpEventListener.DisconnectIfExists(); 134 mRtpSources.DisconnectIfConnected(); 135 136 return InvokeAsync( 137 mCallThread, "WebrtcAudioConduit::Shutdown (main thread)", 138 [this, self = RefPtr<WebrtcAudioConduit>(this)] { 139 mControl.mReceiving.DisconnectIfConnected(); 140 mControl.mTransmitting.DisconnectIfConnected(); 141 mControl.mLocalSsrcs.DisconnectIfConnected(); 142 mControl.mLocalCname.DisconnectIfConnected(); 143 mControl.mMid.DisconnectIfConnected(); 144 mControl.mRemoteSsrc.DisconnectIfConnected(); 145 mControl.mSyncGroup.DisconnectIfConnected(); 146 mControl.mLocalRecvRtpExtensions.DisconnectIfConnected(); 147 mControl.mLocalSendRtpExtensions.DisconnectIfConnected(); 148 mControl.mSendCodec.DisconnectIfConnected(); 149 mControl.mRecvCodecs.DisconnectIfConnected(); 150 mControl.mFrameTransformerProxySend.DisconnectIfConnected(); 151 mControl.mFrameTransformerProxyRecv.DisconnectIfConnected(); 152 mWatchManager.Shutdown(); 153 154 { 155 AutoWriteLock lock(mLock); 156 DeleteSendStream(); 157 DeleteRecvStream(); 158 } 159 // Clear the stats send stream stats cache 160 mTransitionalSendStreamStats = Nothing(); 161 162 SetIsShutdown(); 163 164 return GenericPromise::CreateAndResolve( 165 true, "WebrtcAudioConduit::Shutdown (call thread)"); 166 }); 167 } 168 169 bool WebrtcAudioConduit::IsShutdown() const { 170 MOZ_ASSERT(mCallThread->IsOnCurrentThread()); 171 return mIsShutdown; 172 } 173 174 void WebrtcAudioConduit::SetIsShutdown() { 175 MOZ_ASSERT(mCallThread->IsOnCurrentThread()); 176 mIsShutdown = true; 177 } 178 179 #define INIT_CANONICAL(name, val) \ 180 name(mCallThread, val, "WebrtcAudioConduit::" #name " (Canonical)") 181 #define INIT_MIRROR(name, val) \ 182 name(AbstractThread::MainThread(), val, \ 183 "WebrtcAudioConduit::" #name " (Mirror)") 184 185 WebrtcAudioConduit::WebrtcAudioConduit( 186 RefPtr<WebrtcCallWrapper> aCall, nsCOMPtr<nsISerialEventTarget> aStsThread) 187 : mCall(std::move(aCall)), 188 mSendTransport(this), 189 mRecvTransport(this), 190 mRecvStream(nullptr), 191 mSendStreamConfig(&mSendTransport), 192 mSendStream(nullptr), 193 mSendStreamRunning(false), 194 mRecvStreamRunning(false), 195 mDtmfEnabled(false), 196 mLock("WebrtcAudioConduit::mLock"), 197 mCallThread(mCall->mCallThread), 198 mStsThread(std::move(aStsThread)), 199 mControl(mCall->mCallThread), 200 mWatchManager(this, mCall->mCallThread), 201 INIT_CANONICAL(mCanonicalRtpSources, {}), 202 INIT_MIRROR(mRtpSources, {}) { 203 mRecvStreamConfig.rtcp_send_transport = &mRecvTransport; 204 mRecvStreamConfig.rtp.rtcp_event_observer = this; 205 } 206 #undef INIT_MIRROR 207 #undef INIT_CANONICAL 208 209 /** 210 * Destruction defines for our super-classes 211 */ 212 WebrtcAudioConduit::~WebrtcAudioConduit() { 213 CSFLogDebug(LOGTAG, "%s ", __FUNCTION__); 214 MOZ_ASSERT(!mSendStream && !mRecvStream, 215 "Call DeleteStreams prior to ~WebrtcAudioConduit."); 216 } 217 218 #define CONNECT(aCanonical, aMirror) \ 219 do { \ 220 /* Ensure the watchmanager is wired up before the mirror receives its \ 221 * initial mirrored value. */ \ 222 mCall->mCallThread->DispatchStateChange( \ 223 NS_NewRunnableFunction(__func__, [this, self = RefPtr(this)] { \ 224 mWatchManager.Watch(aMirror, \ 225 &WebrtcAudioConduit::OnControlConfigChange); \ 226 })); \ 227 (aCanonical).ConnectMirror(&(aMirror)); \ 228 } while (0) 229 230 void WebrtcAudioConduit::InitControl(AudioConduitControlInterface* aControl) { 231 MOZ_ASSERT(NS_IsMainThread()); 232 233 CONNECT(aControl->CanonicalReceiving(), mControl.mReceiving); 234 CONNECT(aControl->CanonicalTransmitting(), mControl.mTransmitting); 235 CONNECT(aControl->CanonicalLocalSsrcs(), mControl.mLocalSsrcs); 236 CONNECT(aControl->CanonicalLocalCname(), mControl.mLocalCname); 237 CONNECT(aControl->CanonicalMid(), mControl.mMid); 238 CONNECT(aControl->CanonicalRemoteSsrc(), mControl.mRemoteSsrc); 239 CONNECT(aControl->CanonicalSyncGroup(), mControl.mSyncGroup); 240 CONNECT(aControl->CanonicalLocalRecvRtpExtensions(), 241 mControl.mLocalRecvRtpExtensions); 242 CONNECT(aControl->CanonicalLocalSendRtpExtensions(), 243 mControl.mLocalSendRtpExtensions); 244 CONNECT(aControl->CanonicalAudioSendCodec(), mControl.mSendCodec); 245 CONNECT(aControl->CanonicalAudioRecvCodecs(), mControl.mRecvCodecs); 246 CONNECT(aControl->CanonicalFrameTransformerProxySend(), 247 mControl.mFrameTransformerProxySend); 248 CONNECT(aControl->CanonicalFrameTransformerProxyRecv(), 249 mControl.mFrameTransformerProxyRecv); 250 mControl.mOnDtmfEventListener = aControl->OnDtmfEvent().Connect( 251 mCall->mCallThread, this, &WebrtcAudioConduit::OnDtmfEvent); 252 } 253 254 #undef CONNECT 255 256 MediaConduitErrorCode WebrtcAudioConduit::Init() { 257 MOZ_ASSERT(NS_IsMainThread()); 258 mRtpSources.Connect(&mCanonicalRtpSources); 259 return kMediaConduitNoError; 260 } 261 262 void WebrtcAudioConduit::OnDtmfEvent(const DtmfEvent& aEvent) { 263 MOZ_ASSERT(mCallThread->IsOnCurrentThread()); 264 MOZ_ASSERT(mSendStream); 265 MOZ_ASSERT(mDtmfEnabled); 266 mSendStream->SendTelephoneEvent(aEvent.mPayloadType, aEvent.mPayloadFrequency, 267 aEvent.mEventCode, aEvent.mLengthMs); 268 } 269 270 void WebrtcAudioConduit::OnControlConfigChange() { 271 MOZ_ASSERT(mCallThread->IsOnCurrentThread()); 272 273 bool recvStreamReconfigureNeeded = false; 274 bool sendStreamReconfigureNeeded = false; 275 bool recvStreamRecreationNeeded = false; 276 bool sendStreamRecreationNeeded = false; 277 278 if (!mControl.mLocalSsrcs.Ref().empty()) { 279 if (mControl.mLocalSsrcs.Ref()[0] != mSendStreamConfig.rtp.ssrc) { 280 sendStreamRecreationNeeded = true; 281 282 // For now... 283 recvStreamRecreationNeeded = true; 284 } 285 mRecvStreamConfig.rtp.local_ssrc = mControl.mLocalSsrcs.Ref()[0]; 286 mSendStreamConfig.rtp.ssrc = mControl.mLocalSsrcs.Ref()[0]; 287 288 // In the future we can do this instead of recreating the recv stream: 289 // if (mRecvStream) { 290 // mCall->Call()->OnLocalSsrcUpdated(mRecvStream, 291 // mControl.mLocalSsrcs.Ref()[0]); 292 // } 293 } 294 295 if (mControl.mLocalCname.Ref() != mSendStreamConfig.rtp.c_name) { 296 mSendStreamConfig.rtp.c_name = mControl.mLocalCname.Ref(); 297 sendStreamReconfigureNeeded = true; 298 } 299 300 if (mControl.mMid.Ref() != mSendStreamConfig.rtp.mid) { 301 mSendStreamConfig.rtp.mid = mControl.mMid.Ref(); 302 sendStreamReconfigureNeeded = true; 303 } 304 305 if (mControl.mRemoteSsrc.Ref() != mControl.mConfiguredRemoteSsrc) { 306 mRecvStreamConfig.rtp.remote_ssrc = mControl.mConfiguredRemoteSsrc = 307 mControl.mRemoteSsrc.Ref(); 308 recvStreamRecreationNeeded = true; 309 } 310 311 if (mControl.mSyncGroup.Ref() != mRecvStreamConfig.sync_group) { 312 mRecvStreamConfig.sync_group = mControl.mSyncGroup.Ref(); 313 // For now... 314 recvStreamRecreationNeeded = true; 315 // In the future we can do this instead of recreating the recv stream: 316 // if (mRecvStream) { 317 // mCall->Call()->OnUpdateSyncGroup(mRecvStream, 318 // mRecvStreamConfig.sync_group); 319 // } 320 } 321 322 if (auto filteredExtensions = FilterExtensions( 323 LocalDirection::kSend, mControl.mLocalSendRtpExtensions); 324 filteredExtensions != mSendStreamConfig.rtp.extensions) { 325 // At the very least, we need a reconfigure. Recreation needed if the 326 // extmap for any extension has changed, but not for adding/removing 327 // extensions. 328 sendStreamReconfigureNeeded = true; 329 330 for (const auto& newExt : filteredExtensions) { 331 if (sendStreamRecreationNeeded) { 332 break; 333 } 334 for (const auto& oldExt : mSendStreamConfig.rtp.extensions) { 335 if (newExt.uri == oldExt.uri) { 336 if (newExt.id != oldExt.id) { 337 sendStreamRecreationNeeded = true; 338 } 339 // We're done handling newExt, one way or another 340 break; 341 } 342 } 343 } 344 345 mSendStreamConfig.rtp.extensions = std::move(filteredExtensions); 346 } 347 348 mControl.mSendCodec.Ref().apply([&](const auto& aConfig) { 349 if (mControl.mConfiguredSendCodec != mControl.mSendCodec.Ref()) { 350 mControl.mConfiguredSendCodec = mControl.mSendCodec; 351 if (ValidateCodecConfig(aConfig, true) == kMediaConduitNoError) { 352 mSendStreamConfig.encoder_factory = 353 webrtc::CreateBuiltinAudioEncoderFactory(); 354 355 webrtc::AudioSendStream::Config::SendCodecSpec spec( 356 aConfig.mType, CodecConfigToLibwebrtcFormat(aConfig)); 357 if (const auto& maxBps = mControl.mConfiguredSendCodec 358 ->mEncodingConstraints.maxBitrateBps) { 359 const auto& info = 360 mSendStreamConfig.encoder_factory->QueryAudioEncoder(spec.format); 361 spec.target_bitrate_bps = 362 std::clamp(AssertedCast<int>(*maxBps), info->min_bitrate_bps, 363 info->max_bitrate_bps); 364 } 365 mSendStreamConfig.send_codec_spec = spec; 366 367 mDtmfEnabled = aConfig.mDtmfEnabled; 368 sendStreamReconfigureNeeded = true; 369 } 370 } 371 }); 372 373 if (mControl.mConfiguredRecvCodecs != mControl.mRecvCodecs.Ref()) { 374 mControl.mConfiguredRecvCodecs = mControl.mRecvCodecs; 375 mRecvStreamConfig.decoder_factory = mCall->mAudioDecoderFactory; 376 mRecvStreamConfig.decoder_map.clear(); 377 378 for (const auto& codec : mControl.mRecvCodecs.Ref()) { 379 if (ValidateCodecConfig(codec, false) != kMediaConduitNoError) { 380 continue; 381 } 382 mRecvStreamConfig.decoder_map.emplace( 383 codec.mType, CodecConfigToLibwebrtcFormat(codec)); 384 } 385 386 recvStreamReconfigureNeeded = true; 387 } 388 389 if (mControl.mConfiguredFrameTransformerProxySend.get() != 390 mControl.mFrameTransformerProxySend.Ref().get()) { 391 mControl.mConfiguredFrameTransformerProxySend = 392 mControl.mFrameTransformerProxySend.Ref(); 393 if (!mSendStreamConfig.frame_transformer) { 394 mSendStreamConfig.frame_transformer = 395 new webrtc::RefCountedObject<FrameTransformer>(false); 396 sendStreamRecreationNeeded = true; 397 } 398 static_cast<FrameTransformer*>(mSendStreamConfig.frame_transformer.get()) 399 ->SetProxy(mControl.mConfiguredFrameTransformerProxySend); 400 } 401 402 if (mControl.mConfiguredFrameTransformerProxyRecv.get() != 403 mControl.mFrameTransformerProxyRecv.Ref().get()) { 404 mControl.mConfiguredFrameTransformerProxyRecv = 405 mControl.mFrameTransformerProxyRecv.Ref(); 406 if (!mRecvStreamConfig.frame_transformer) { 407 mRecvStreamConfig.frame_transformer = 408 new webrtc::RefCountedObject<FrameTransformer>(false); 409 recvStreamRecreationNeeded = true; 410 } 411 static_cast<FrameTransformer*>(mRecvStreamConfig.frame_transformer.get()) 412 ->SetProxy(mControl.mConfiguredFrameTransformerProxyRecv); 413 } 414 415 if (!recvStreamReconfigureNeeded && !sendStreamReconfigureNeeded && 416 !recvStreamRecreationNeeded && !sendStreamRecreationNeeded && 417 mControl.mReceiving == mRecvStreamRunning && 418 mControl.mTransmitting == mSendStreamRunning) { 419 // No changes applied -- no need to lock. 420 return; 421 } 422 423 if (recvStreamRecreationNeeded) { 424 recvStreamReconfigureNeeded = false; 425 } 426 if (sendStreamRecreationNeeded) { 427 sendStreamReconfigureNeeded = false; 428 } 429 430 { 431 AutoWriteLock lock(mLock); 432 // Recreate/Stop/Start streams as needed. 433 if (recvStreamRecreationNeeded) { 434 DeleteRecvStream(); 435 } 436 if (mControl.mReceiving) { 437 CreateRecvStream(); 438 } 439 if (sendStreamRecreationNeeded) { 440 if (mControl.mTransmitting) { 441 MemoSendStreamStats(); 442 } 443 DeleteSendStream(); 444 } 445 if (mControl.mTransmitting) { 446 CreateSendStream(); 447 } 448 } 449 450 // We make sure to not hold the lock while stopping/starting/reconfiguring 451 // streams, so as to not cause deadlocks. These methods can cause our platform 452 // codecs to dispatch sync runnables to main, and main may grab the lock. 453 454 if (mRecvStream && recvStreamReconfigureNeeded) { 455 MOZ_ASSERT(!recvStreamRecreationNeeded); 456 mRecvStream->SetDecoderMap(mRecvStreamConfig.decoder_map); 457 } 458 459 if (mSendStream && sendStreamReconfigureNeeded) { 460 MOZ_ASSERT(!sendStreamRecreationNeeded); 461 // TODO: Pass a callback here, so we can react to RTCErrors thrown by 462 // libwebrtc. 463 mSendStream->Reconfigure(mSendStreamConfig, nullptr); 464 } 465 466 if (!mControl.mReceiving) { 467 StopReceiving(); 468 } 469 if (!mControl.mTransmitting) { 470 StopTransmitting(); 471 } 472 473 if (mControl.mReceiving) { 474 StartReceiving(); 475 } 476 if (mControl.mTransmitting) { 477 StartTransmitting(); 478 } 479 } 480 481 std::vector<uint32_t> WebrtcAudioConduit::GetLocalSSRCs() const { 482 MOZ_ASSERT(mCallThread->IsOnCurrentThread()); 483 return std::vector<uint32_t>(1, mRecvStreamConfig.rtp.local_ssrc); 484 } 485 486 bool WebrtcAudioConduit::OverrideRemoteSSRC(uint32_t aSsrc) { 487 MOZ_ASSERT(mCallThread->IsOnCurrentThread()); 488 489 if (mRecvStreamConfig.rtp.remote_ssrc == aSsrc) { 490 return true; 491 } 492 mRecvStreamConfig.rtp.remote_ssrc = aSsrc; 493 494 const bool wasReceiving = mRecvStreamRunning; 495 const bool hadRecvStream = mRecvStream; 496 497 StopReceiving(); 498 499 if (hadRecvStream) { 500 AutoWriteLock lock(mLock); 501 DeleteRecvStream(); 502 CreateRecvStream(); 503 } 504 505 if (wasReceiving) { 506 StartReceiving(); 507 } 508 return true; 509 } 510 511 Maybe<Ssrc> WebrtcAudioConduit::GetRemoteSSRC() const { 512 MOZ_ASSERT(mCallThread->IsOnCurrentThread()); 513 // libwebrtc uses 0 to mean a lack of SSRC. That is not to spec. 514 return mRecvStreamConfig.rtp.remote_ssrc == 0 515 ? Nothing() 516 : Some(mRecvStreamConfig.rtp.remote_ssrc); 517 } 518 519 Maybe<webrtc::AudioReceiveStreamInterface::Stats> 520 WebrtcAudioConduit::GetReceiverStats() const { 521 MOZ_ASSERT(mCallThread->IsOnCurrentThread()); 522 if (!mRecvStream) { 523 return Nothing(); 524 } 525 return Some(mRecvStream->GetStats()); 526 } 527 528 Maybe<webrtc::AudioSendStream::Stats> WebrtcAudioConduit::GetSenderStats() 529 const { 530 MOZ_ASSERT(mCallThread->IsOnCurrentThread()); 531 if (!mSendStream) { 532 // Might be nothing 533 return mTransitionalSendStreamStats; 534 } 535 // Successfully got stats, so clear the transitional stats. 536 mTransitionalSendStreamStats = Nothing(); 537 return Some(mSendStream->GetStats()); 538 } 539 540 Maybe<webrtc::CallBasicStats> WebrtcAudioConduit::GetCallStats() const { 541 MOZ_ASSERT(mCallThread->IsOnCurrentThread()); 542 if (!mCall->Call()) { 543 return Nothing(); 544 } 545 return Some(mCall->Call()->GetStats()); 546 } 547 548 void WebrtcAudioConduit::OnRtcpBye() { mRtcpByeEvent.Notify(); } 549 550 void WebrtcAudioConduit::OnRtcpTimeout() { mRtcpTimeoutEvent.Notify(); } 551 552 void WebrtcAudioConduit::SetTransportActive(bool aActive) { 553 MOZ_ASSERT(mStsThread->IsOnCurrentThread()); 554 if (mTransportActive == aActive) { 555 return; 556 } 557 558 // If false, This stops us from sending 559 mTransportActive = aActive; 560 } 561 562 // AudioSessionConduit Implementation 563 MediaConduitErrorCode WebrtcAudioConduit::SendAudioFrame( 564 std::unique_ptr<webrtc::AudioFrame> frame) { 565 CSFLogDebug(LOGTAG, "%s ", __FUNCTION__); 566 // Following checks need to be performed 567 // 1. Non null audio buffer pointer, and 568 // 2. Valid sample rate, and 569 // 3. Appropriate Sample Length for 10 ms audio-frame. This represents the 570 // block size used upstream for processing. 571 // Ex: for 16000 sample rate , valid block-length is 160. 572 // Similarly for 32000 sample rate, valid block length is 320. 573 574 if (!frame->data() || 575 (IsSamplingFreqSupported(frame->sample_rate_hz()) == false) || 576 ((frame->samples_per_channel() % (frame->sample_rate_hz() / 100) != 0))) { 577 CSFLogError(LOGTAG, "%s Invalid Parameters ", __FUNCTION__); 578 MOZ_ASSERT(PR_FALSE); 579 return kMediaConduitMalformedArgument; 580 } 581 582 // This is the AudioProxyThread, blocking it for a bit is fine. 583 AutoReadLock lock(mLock); 584 if (!mSendStreamRunning) { 585 CSFLogError(LOGTAG, "%s Engine not transmitting ", __FUNCTION__); 586 return kMediaConduitSessionNotInited; 587 } 588 589 mSendStream->SendAudioData(std::move(frame)); 590 return kMediaConduitNoError; 591 } 592 593 MediaConduitErrorCode WebrtcAudioConduit::GetAudioFrame( 594 int32_t samplingFreqHz, webrtc::AudioFrame* frame) { 595 CSFLogDebug(LOGTAG, "%s ", __FUNCTION__); 596 597 // validate params 598 if (!frame) { 599 CSFLogError(LOGTAG, "%s Null Audio Buffer Pointer", __FUNCTION__); 600 MOZ_ASSERT(PR_FALSE); 601 return kMediaConduitMalformedArgument; 602 } 603 604 // Validate sample length 605 if (GetNum10msSamplesForFrequency(samplingFreqHz) == 0) { 606 CSFLogError(LOGTAG, "%s Invalid Sampling Frequency ", __FUNCTION__); 607 MOZ_ASSERT(PR_FALSE); 608 return kMediaConduitMalformedArgument; 609 } 610 611 // If the lock is taken, skip this chunk to avoid blocking the audio thread. 612 AutoTryReadLock tryLock(mLock); 613 if (!tryLock) { 614 CSFLogError(LOGTAG, "%s Conduit going through negotiation ", __FUNCTION__); 615 return kMediaConduitPlayoutError; 616 } 617 618 // Conduit should have reception enabled before we ask for decoded 619 // samples 620 if (!mRecvStreamRunning) { 621 CSFLogError(LOGTAG, "%s Engine not Receiving ", __FUNCTION__); 622 return kMediaConduitSessionNotInited; 623 } 624 625 // Unfortunate to have to cast to an internal class, but that looks like the 626 // only way short of interfacing with a layer above (which mixes all streams, 627 // which we don't want) or a layer below (which we try to avoid because it is 628 // less stable). 629 auto info = static_cast<webrtc::AudioReceiveStreamImpl*>(mRecvStream) 630 ->GetAudioFrameWithInfo(samplingFreqHz, frame); 631 632 if (info == webrtc::AudioMixer::Source::AudioFrameInfo::kError) { 633 CSFLogError(LOGTAG, "%s Getting audio frame failed", __FUNCTION__); 634 return kMediaConduitPlayoutError; 635 } 636 637 CSFLogDebug(LOGTAG, "%s Got %zu channels of %zu samples", __FUNCTION__, 638 frame->num_channels(), frame->samples_per_channel()); 639 return kMediaConduitNoError; 640 } 641 642 // Transport Layer Callbacks 643 void WebrtcAudioConduit::OnRtpReceived(webrtc::RtpPacketReceived&& aPacket, 644 webrtc::RTPHeader&& aHeader) { 645 MOZ_ASSERT(mCallThread->IsOnCurrentThread()); 646 647 // We should only be handling packets on this conduit if we are set to receive 648 // them. 649 if (!mControl.mReceiving) { 650 CSFLogVerbose(LOGTAG, 651 "AudioConduit %p: Discarding packet SEQ# %u SSRC %u as not " 652 "configured to receive.", 653 this, aPacket.SequenceNumber(), aHeader.ssrc); 654 return; 655 } 656 657 if (mAllowSsrcChange && mRecvStreamConfig.rtp.remote_ssrc != aHeader.ssrc) { 658 CSFLogDebug(LOGTAG, "%s: switching from SSRC %u to %u", __FUNCTION__, 659 mRecvStreamConfig.rtp.remote_ssrc, aHeader.ssrc); 660 OverrideRemoteSSRC(aHeader.ssrc); 661 } 662 663 CSFLogVerbose(LOGTAG, "%s: seq# %u, Len %zu, SSRC %u (0x%x) ", __FUNCTION__, 664 aPacket.SequenceNumber(), aPacket.size(), aPacket.Ssrc(), 665 aPacket.Ssrc()); 666 667 // Libwebrtc commit cde4b67d9d now expect calls to 668 // SourceTracker::GetSources() to happen on the call thread. We'll 669 // grab the value now while on the call thread, and dispatch to main 670 // to store the cached value if we have new source information. 671 // See Bug 1845621. 672 if (mRecvStream) { 673 mCanonicalRtpSources = mRecvStream->GetSources(); 674 } 675 676 mRtpPacketEvent.Notify(); 677 if (mCall->Call()) { 678 mCall->Call()->Receiver()->DeliverRtpPacket( 679 webrtc::MediaType::AUDIO, std::move(aPacket), 680 [self = RefPtr<WebrtcAudioConduit>(this)]( 681 const webrtc::RtpPacketReceived& packet) { 682 CSFLogVerbose( 683 LOGTAG, 684 "AudioConduit %p: failed demuxing packet, ssrc: %u seq: %u", 685 self.get(), packet.Ssrc(), packet.SequenceNumber()); 686 return false; 687 }); 688 } 689 } 690 691 void WebrtcAudioConduit::OnRtcpReceived(webrtc::CopyOnWriteBuffer&& aPacket) { 692 MOZ_ASSERT(mCallThread->IsOnCurrentThread()); 693 694 if (mCall->Call()) { 695 mCall->Call()->Receiver()->DeliverRtcpPacket( 696 std::forward<webrtc::CopyOnWriteBuffer>(aPacket)); 697 } 698 } 699 700 Maybe<uint16_t> WebrtcAudioConduit::RtpSendBaseSeqFor(uint32_t aSsrc) const { 701 MOZ_ASSERT(mCallThread->IsOnCurrentThread()); 702 auto it = mRtpSendBaseSeqs.find(aSsrc); 703 if (it == mRtpSendBaseSeqs.end()) { 704 return Nothing(); 705 } 706 return Some(it->second); 707 } 708 709 const dom::RTCStatsTimestampMaker& WebrtcAudioConduit::GetTimestampMaker() 710 const { 711 return mCall->GetTimestampMaker(); 712 } 713 714 void WebrtcAudioConduit::StopTransmitting() { 715 MOZ_ASSERT(mCallThread->IsOnCurrentThread()); 716 MOZ_ASSERT(!mLock.LockedForWritingByCurrentThread()); 717 718 if (!mSendStreamRunning) { 719 return; 720 } 721 722 if (mSendStream) { 723 CSFLogDebug(LOGTAG, "%s Stopping send stream", __FUNCTION__); 724 mSendStream->Stop(); 725 } 726 727 mSendStreamRunning = false; 728 } 729 730 void WebrtcAudioConduit::StartTransmitting() { 731 MOZ_ASSERT(mCallThread->IsOnCurrentThread()); 732 MOZ_ASSERT(mSendStream); 733 MOZ_ASSERT(!mLock.LockedForWritingByCurrentThread()); 734 735 if (mSendStreamRunning) { 736 return; 737 } 738 739 CSFLogDebug(LOGTAG, "%s Starting send stream", __FUNCTION__); 740 741 mCall->Call()->SignalChannelNetworkState(webrtc::MediaType::AUDIO, 742 webrtc::kNetworkUp); 743 mSendStream->Start(); 744 mSendStreamRunning = true; 745 } 746 747 void WebrtcAudioConduit::StopReceiving() { 748 MOZ_ASSERT(mCallThread->IsOnCurrentThread()); 749 MOZ_ASSERT(!mLock.LockedForWritingByCurrentThread()); 750 751 if (!mRecvStreamRunning) { 752 return; 753 } 754 755 if (mRecvStream) { 756 CSFLogDebug(LOGTAG, "%s Stopping recv stream", __FUNCTION__); 757 mRecvStream->Stop(); 758 } 759 760 mRecvStreamRunning = false; 761 } 762 763 void WebrtcAudioConduit::StartReceiving() { 764 MOZ_ASSERT(mCallThread->IsOnCurrentThread()); 765 MOZ_ASSERT(mRecvStream); 766 MOZ_ASSERT(!mLock.LockedForWritingByCurrentThread()); 767 768 if (mRecvStreamRunning) { 769 return; 770 } 771 772 CSFLogDebug(LOGTAG, "%s Starting receive stream (SSRC %u (0x%x))", 773 __FUNCTION__, mRecvStreamConfig.rtp.remote_ssrc, 774 mRecvStreamConfig.rtp.remote_ssrc); 775 776 mCall->Call()->SignalChannelNetworkState(webrtc::MediaType::AUDIO, 777 webrtc::kNetworkUp); 778 mRecvStream->Start(); 779 mRecvStreamRunning = true; 780 } 781 782 bool WebrtcAudioConduit::SendRtp(const uint8_t* aData, size_t aLength, 783 const webrtc::PacketOptions& aOptions) { 784 MOZ_ASSERT(aLength >= 12); 785 const uint16_t seqno = ntohs(*((uint16_t*)&aData[2])); 786 const uint32_t ssrc = ntohl(*((uint32_t*)&aData[8])); 787 788 CSFLogVerbose( 789 LOGTAG, 790 "AudioConduit %p: Sending RTP Packet seq# %u, len %zu, SSRC %u (0x%x)", 791 this, seqno, aLength, ssrc, ssrc); 792 793 if (!mTransportActive) { 794 CSFLogError(LOGTAG, "AudioConduit %p: RTP Packet Send Failed ", this); 795 return false; 796 } 797 798 MediaPacket packet; 799 packet.Copy(aData, aLength, aLength + SRTP_MAX_EXPANSION); 800 packet.SetType(MediaPacket::RTP); 801 mSenderRtpSendEvent.Notify(std::move(packet)); 802 803 // Parse the sequence number of the first rtp packet as base_seq. 804 const auto inserted = mRtpSendBaseSeqs_n.insert({ssrc, seqno}).second; 805 806 if (inserted || aOptions.packet_id >= 0) { 807 int64_t now_ms = PR_Now() / 1000; 808 MOZ_ALWAYS_SUCCEEDS(mCallThread->Dispatch(NS_NewRunnableFunction( 809 __func__, [this, self = RefPtr<WebrtcAudioConduit>(this), 810 packet_id = aOptions.packet_id, now_ms, ssrc, seqno] { 811 mRtpSendBaseSeqs.insert({ssrc, seqno}); 812 if (packet_id >= 0) { 813 if (mCall->Call()) { 814 // TODO: This notification should ideally happen after the 815 // transport layer has sent the packet on the wire. 816 mCall->Call()->OnSentPacket({packet_id, now_ms}); 817 } 818 } 819 }))); 820 } 821 return true; 822 } 823 824 bool WebrtcAudioConduit::SendSenderRtcp(const uint8_t* aData, size_t aLength) { 825 CSFLogVerbose( 826 LOGTAG, 827 "AudioConduit %p: Sending RTCP SR Packet, len %zu, SSRC %u (0x%x)", this, 828 aLength, (uint32_t)ntohl(*((uint32_t*)&aData[4])), 829 (uint32_t)ntohl(*((uint32_t*)&aData[4]))); 830 831 if (!mTransportActive) { 832 CSFLogError(LOGTAG, "%s RTCP SR Packet Send Failed ", __FUNCTION__); 833 return false; 834 } 835 836 MediaPacket packet; 837 packet.Copy(aData, aLength, aLength + SRTP_MAX_EXPANSION); 838 packet.SetType(MediaPacket::RTCP); 839 mSenderRtcpSendEvent.Notify(std::move(packet)); 840 return true; 841 } 842 843 bool WebrtcAudioConduit::SendReceiverRtcp(const uint8_t* aData, 844 size_t aLength) { 845 CSFLogVerbose( 846 LOGTAG, 847 "AudioConduit %p: Sending RTCP RR Packet, len %zu, SSRC %u (0x%x)", this, 848 aLength, (uint32_t)ntohl(*((uint32_t*)&aData[4])), 849 (uint32_t)ntohl(*((uint32_t*)&aData[4]))); 850 851 if (!mTransportActive) { 852 CSFLogError(LOGTAG, "AudioConduit %p: RTCP RR Packet Send Failed", this); 853 return false; 854 } 855 856 MediaPacket packet; 857 packet.Copy(aData, aLength, aLength + SRTP_MAX_EXPANSION); 858 packet.SetType(MediaPacket::RTCP); 859 mReceiverRtcpSendEvent.Notify(std::move(packet)); 860 return true; 861 } 862 863 /** 864 * Supported Sampling Frequencies. 865 */ 866 bool WebrtcAudioConduit::IsSamplingFreqSupported(int freq) const { 867 return GetNum10msSamplesForFrequency(freq) != 0; 868 } 869 870 const std::vector<webrtc::RtpSource>& 871 WebrtcAudioConduit::GetUpstreamRtpSources() const { 872 MOZ_ASSERT(NS_IsMainThread()); 873 return mRtpSources; 874 } 875 876 /* Return block-length of 10 ms audio frame in number of samples */ 877 unsigned int WebrtcAudioConduit::GetNum10msSamplesForFrequency( 878 int samplingFreqHz) const { 879 switch (samplingFreqHz) { 880 case 16000: 881 return 160; // 160 samples 882 case 32000: 883 return 320; // 320 samples 884 case 44100: 885 return 441; // 441 samples 886 case 48000: 887 return 480; // 480 samples 888 default: 889 return 0; // invalid or unsupported 890 } 891 } 892 893 /** 894 * Perform validation on the codecConfig to be applied. 895 * Verifies if the codec is already applied. 896 */ 897 MediaConduitErrorCode WebrtcAudioConduit::ValidateCodecConfig( 898 const AudioCodecConfig& codecInfo, bool send) { 899 if (codecInfo.mName.empty()) { 900 CSFLogError(LOGTAG, "%s Empty Payload Name ", __FUNCTION__); 901 return kMediaConduitMalformedArgument; 902 } 903 904 // Only mono or stereo channels supported 905 if ((codecInfo.mChannels != 1) && (codecInfo.mChannels != 2)) { 906 CSFLogError(LOGTAG, "%s Channel Unsupported ", __FUNCTION__); 907 return kMediaConduitMalformedArgument; 908 } 909 910 return kMediaConduitNoError; 911 } 912 913 RtpExtList WebrtcAudioConduit::FilterExtensions(LocalDirection aDirection, 914 const RtpExtList& aExtensions) { 915 const bool isSend = aDirection == LocalDirection::kSend; 916 RtpExtList filteredExtensions; 917 918 for (const auto& extension : aExtensions) { 919 // ssrc-audio-level RTP header extension 920 if (extension.uri == webrtc::RtpExtension::kAudioLevelUri) { 921 filteredExtensions.push_back( 922 webrtc::RtpExtension(extension.uri, extension.id)); 923 } 924 925 // csrc-audio-level RTP header extension 926 if (extension.uri == webrtc::RtpExtension::kCsrcAudioLevelsUri) { 927 if (isSend) { 928 continue; 929 } 930 filteredExtensions.push_back( 931 webrtc::RtpExtension(extension.uri, extension.id)); 932 } 933 934 // MID RTP header extension 935 if (extension.uri == webrtc::RtpExtension::kMidUri) { 936 if (!isSend) { 937 // TODO: recv mid support, see also bug 1727211 938 continue; 939 } 940 filteredExtensions.push_back( 941 webrtc::RtpExtension(extension.uri, extension.id)); 942 } 943 } 944 945 return filteredExtensions; 946 } 947 948 webrtc::SdpAudioFormat WebrtcAudioConduit::CodecConfigToLibwebrtcFormat( 949 const AudioCodecConfig& aConfig) { 950 webrtc::CodecParameterMap parameters; 951 if (aConfig.mName == kOpusCodecName) { 952 if (aConfig.mChannels == 2) { 953 parameters[kCodecParamStereo] = kParamValueTrue; 954 } 955 if (aConfig.mFECEnabled) { 956 parameters[kCodecParamUseInbandFec] = kParamValueTrue; 957 } 958 if (aConfig.mDTXEnabled) { 959 parameters[kCodecParamUseDtx] = kParamValueTrue; 960 } 961 if (aConfig.mMaxPlaybackRate) { 962 parameters[kCodecParamMaxPlaybackRate] = 963 std::to_string(aConfig.mMaxPlaybackRate); 964 } 965 if (aConfig.mMaxAverageBitrate) { 966 parameters[kCodecParamMaxAverageBitrate] = 967 std::to_string(aConfig.mMaxAverageBitrate); 968 } 969 if (aConfig.mFrameSizeMs) { 970 parameters[kCodecParamPTime] = std::to_string(aConfig.mFrameSizeMs); 971 } 972 if (aConfig.mMinFrameSizeMs) { 973 parameters[kCodecParamMinPTime] = std::to_string(aConfig.mMinFrameSizeMs); 974 } 975 if (aConfig.mMaxFrameSizeMs) { 976 parameters[kCodecParamMaxPTime] = std::to_string(aConfig.mMaxFrameSizeMs); 977 } 978 if (aConfig.mCbrEnabled) { 979 parameters[kCodecParamCbr] = kParamValueTrue; 980 } 981 } 982 983 return webrtc::SdpAudioFormat(aConfig.mName, aConfig.mFreq, aConfig.mChannels, 984 parameters); 985 } 986 987 void WebrtcAudioConduit::DeleteSendStream() { 988 MOZ_ASSERT(mCallThread->IsOnCurrentThread()); 989 MOZ_ASSERT(mLock.LockedForWritingByCurrentThread()); 990 991 if (!mSendStream) { 992 return; 993 } 994 995 mCall->Call()->DestroyAudioSendStream(mSendStream); 996 mSendStreamRunning = false; 997 mSendStream = nullptr; 998 999 // Reset base_seqs in case ssrcs get re-used. 1000 mRtpSendBaseSeqs.clear(); 1001 } 1002 1003 void WebrtcAudioConduit::MemoSendStreamStats() { 1004 MOZ_ASSERT(mCallThread->IsOnCurrentThread()); 1005 // If we are going to be recreating the send stream, we hold onto stats until 1006 // libwebrtc stats collection catches up. 1007 if (mControl.mTransmitting && mSendStream) { 1008 const auto stats = mSendStream->GetStats(); 1009 mTransitionalSendStreamStats = Some(stats); 1010 } 1011 } 1012 1013 void WebrtcAudioConduit::CreateSendStream() { 1014 MOZ_ASSERT(mCallThread->IsOnCurrentThread()); 1015 MOZ_ASSERT(mLock.LockedForWritingByCurrentThread()); 1016 1017 if (mSendStream) { 1018 return; 1019 } 1020 1021 mSendStream = mCall->Call()->CreateAudioSendStream(mSendStreamConfig); 1022 } 1023 1024 void WebrtcAudioConduit::DeleteRecvStream() { 1025 MOZ_ASSERT(mCallThread->IsOnCurrentThread()); 1026 MOZ_ASSERT(mLock.LockedForWritingByCurrentThread()); 1027 1028 if (!mRecvStream) { 1029 return; 1030 } 1031 1032 mCall->Call()->DestroyAudioReceiveStream(mRecvStream); 1033 mRecvStreamRunning = false; 1034 mRecvStream = nullptr; 1035 } 1036 1037 void WebrtcAudioConduit::CreateRecvStream() { 1038 MOZ_ASSERT(mCallThread->IsOnCurrentThread()); 1039 MOZ_ASSERT(mLock.LockedForWritingByCurrentThread()); 1040 1041 if (mRecvStream) { 1042 return; 1043 } 1044 1045 mRecvStream = mCall->Call()->CreateAudioReceiveStream(mRecvStreamConfig); 1046 // Ensure that we set the jitter buffer target on this stream. 1047 mRecvStream->SetBaseMinimumPlayoutDelayMs(mJitterBufferTargetMs); 1048 } 1049 1050 void WebrtcAudioConduit::SetJitterBufferTarget(DOMHighResTimeStamp aTargetMs) { 1051 MOZ_RELEASE_ASSERT(aTargetMs <= std::numeric_limits<uint16_t>::max()); 1052 MOZ_RELEASE_ASSERT(aTargetMs >= 0); 1053 1054 MOZ_ALWAYS_SUCCEEDS(mCallThread->Dispatch(NS_NewRunnableFunction( 1055 __func__, 1056 [this, self = RefPtr<WebrtcAudioConduit>(this), targetMs = aTargetMs] { 1057 mJitterBufferTargetMs = static_cast<uint16_t>(targetMs); 1058 if (mRecvStream) { 1059 mRecvStream->SetBaseMinimumPlayoutDelayMs(targetMs); 1060 } 1061 }))); 1062 } 1063 1064 void WebrtcAudioConduit::DeliverPacket(webrtc::CopyOnWriteBuffer packet, 1065 PacketType type) { 1066 // Currently unused. 1067 MOZ_ASSERT(false); 1068 } 1069 1070 Maybe<int> WebrtcAudioConduit::ActiveSendPayloadType() const { 1071 MOZ_ASSERT(mCallThread->IsOnCurrentThread()); 1072 1073 auto stats = GetSenderStats(); 1074 if (!stats) { 1075 return Nothing(); 1076 } 1077 1078 if (!stats->codec_payload_type) { 1079 return Nothing(); 1080 } 1081 1082 return Some(*stats->codec_payload_type); 1083 } 1084 1085 Maybe<int> WebrtcAudioConduit::ActiveRecvPayloadType() const { 1086 MOZ_ASSERT(mCallThread->IsOnCurrentThread()); 1087 1088 auto stats = GetReceiverStats(); 1089 if (!stats) { 1090 return Nothing(); 1091 } 1092 1093 if (!stats->codec_payload_type) { 1094 return Nothing(); 1095 } 1096 1097 return Some(*stats->codec_payload_type); 1098 } 1099 1100 } // namespace mozilla