frame_combiner.cc (7414B)
1 /* 2 * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. 3 * 4 * Use of this source code is governed by a BSD-style license 5 * that can be found in the LICENSE file in the root of the source 6 * tree. An additional intellectual property rights grant can be found 7 * in the file PATENTS. All contributing project authors may 8 * be found in the AUTHORS file in the root of the source tree. 9 */ 10 11 #include "modules/audio_mixer/frame_combiner.h" 12 13 #include <algorithm> 14 #include <array> 15 #include <cstddef> 16 #include <cstdint> 17 #include <memory> 18 #include <utility> 19 #include <vector> 20 21 #include "api/array_view.h" 22 #include "api/audio/audio_frame.h" 23 #include "api/audio/audio_view.h" 24 #include "api/rtp_packet_info.h" 25 #include "api/rtp_packet_infos.h" 26 #include "common_audio/include/audio_util.h" 27 #include "modules/audio_mixer/audio_frame_manipulator.h" 28 #include "modules/audio_processing/agc2/limiter.h" 29 #include "modules/audio_processing/logging/apm_data_dumper.h" 30 #include "rtc_base/checks.h" 31 32 namespace webrtc { 33 namespace { 34 35 void SetAudioFrameFields(ArrayView<const AudioFrame* const> mix_list, 36 size_t number_of_channels, 37 int sample_rate, 38 size_t /* number_of_streams */, 39 AudioFrame* audio_frame_for_mixing) { 40 const size_t samples_per_channel = 41 SampleRateToDefaultChannelSize(sample_rate); 42 43 // TODO(minyue): Issue bugs.webrtc.org/3390. 44 // Audio frame timestamp. The 'timestamp_' field is set to dummy 45 // value '0', because it is only supported in the one channel case and 46 // is then updated in the helper functions. 47 audio_frame_for_mixing->UpdateFrame( 48 0, nullptr, samples_per_channel, sample_rate, AudioFrame::kUndefined, 49 AudioFrame::kVadUnknown, number_of_channels); 50 51 if (mix_list.empty()) { 52 audio_frame_for_mixing->elapsed_time_ms_ = -1; 53 } else { 54 audio_frame_for_mixing->timestamp_ = mix_list[0]->timestamp_; 55 audio_frame_for_mixing->elapsed_time_ms_ = mix_list[0]->elapsed_time_ms_; 56 audio_frame_for_mixing->ntp_time_ms_ = mix_list[0]->ntp_time_ms_; 57 std::vector<RtpPacketInfo> packet_infos; 58 for (const auto& frame : mix_list) { 59 audio_frame_for_mixing->timestamp_ = 60 std::min(audio_frame_for_mixing->timestamp_, frame->timestamp_); 61 audio_frame_for_mixing->ntp_time_ms_ = 62 std::min(audio_frame_for_mixing->ntp_time_ms_, frame->ntp_time_ms_); 63 audio_frame_for_mixing->elapsed_time_ms_ = std::max( 64 audio_frame_for_mixing->elapsed_time_ms_, frame->elapsed_time_ms_); 65 packet_infos.insert(packet_infos.end(), frame->packet_infos_.begin(), 66 frame->packet_infos_.end()); 67 } 68 audio_frame_for_mixing->packet_infos_ = 69 RtpPacketInfos(std::move(packet_infos)); 70 } 71 } 72 73 void MixFewFramesWithNoLimiter(ArrayView<const AudioFrame* const> mix_list, 74 AudioFrame* audio_frame_for_mixing) { 75 if (mix_list.empty()) { 76 audio_frame_for_mixing->Mute(); 77 return; 78 } 79 RTC_DCHECK_LE(mix_list.size(), 1); 80 InterleavedView<int16_t> dst = audio_frame_for_mixing->mutable_data( 81 mix_list[0]->samples_per_channel_, mix_list[0]->num_channels_); 82 CopySamples(dst, mix_list[0]->data_view()); 83 } 84 85 void MixToFloatFrame(ArrayView<const AudioFrame* const> mix_list, 86 DeinterleavedView<float>& mixing_buffer) { 87 const size_t number_of_channels = NumChannels(mixing_buffer); 88 // Clear the mixing buffer. 89 mixing_buffer.Clear(); 90 91 // Convert to FloatS16 and mix. 92 for (size_t i = 0; i < mix_list.size(); ++i) { 93 InterleavedView<const int16_t> frame_data = mix_list[i]->data_view(); 94 RTC_CHECK(!frame_data.empty()); 95 for (size_t j = 0; j < number_of_channels; ++j) { 96 MonoView<float> channel = mixing_buffer[j]; 97 for (size_t k = 0; k < SamplesPerChannel(channel); ++k) { 98 channel[k] += frame_data[number_of_channels * k + j]; 99 } 100 } 101 } 102 } 103 104 void RunLimiter(DeinterleavedView<float> deinterleaved, Limiter* limiter) { 105 limiter->SetSamplesPerChannel(deinterleaved.samples_per_channel()); 106 limiter->Process(deinterleaved); 107 } 108 109 // Both interleaves and rounds. 110 void InterleaveToAudioFrame(DeinterleavedView<float> deinterleaved, 111 AudioFrame* audio_frame_for_mixing) { 112 InterleavedView<int16_t> mixing_data = audio_frame_for_mixing->mutable_data( 113 deinterleaved.samples_per_channel(), deinterleaved.num_channels()); 114 // Put data in the result frame. 115 for (size_t i = 0; i < mixing_data.num_channels(); ++i) { 116 auto channel = deinterleaved[i]; 117 for (size_t j = 0; j < mixing_data.samples_per_channel(); ++j) { 118 mixing_data[mixing_data.num_channels() * j + i] = 119 FloatS16ToS16(channel[j]); 120 } 121 } 122 } 123 } // namespace 124 125 FrameCombiner::FrameCombiner(bool use_limiter) 126 : data_dumper_(new ApmDataDumper(0)), 127 limiter_(data_dumper_.get(), kMaximumChannelSize, "AudioMixer"), 128 use_limiter_(use_limiter) { 129 static_assert(kMaximumChannelSize * kMaximumNumberOfChannels <= 130 AudioFrame::kMaxDataSizeSamples, 131 ""); 132 } 133 134 FrameCombiner::~FrameCombiner() = default; 135 136 void FrameCombiner::Combine(ArrayView<AudioFrame* const> mix_list, 137 size_t number_of_channels, 138 int sample_rate, 139 size_t number_of_streams, 140 AudioFrame* audio_frame_for_mixing) { 141 RTC_DCHECK(audio_frame_for_mixing); 142 RTC_DCHECK_GT(sample_rate, 0); 143 144 // Note: `mix_list` is allowed to be empty. 145 // See FrameCombiner.CombiningZeroFramesShouldProduceSilence. 146 147 // Make sure to cap `number_of_channels` to the kMaximumNumberOfChannels 148 // limits since processing from hereon out will be bound by them. 149 number_of_channels = std::min(number_of_channels, kMaximumNumberOfChannels); 150 151 SetAudioFrameFields(mix_list, number_of_channels, sample_rate, 152 number_of_streams, audio_frame_for_mixing); 153 154 size_t samples_per_channel = SampleRateToDefaultChannelSize(sample_rate); 155 156 #if RTC_DCHECK_IS_ON 157 for (const auto* frame : mix_list) { 158 RTC_DCHECK_EQ(samples_per_channel, frame->samples_per_channel_); 159 RTC_DCHECK_EQ(sample_rate, frame->sample_rate_hz_); 160 } 161 #endif 162 163 // The 'num_channels_' field of frames in 'mix_list' could be 164 // different from 'number_of_channels'. 165 for (auto* frame : mix_list) { 166 RemixFrame(number_of_channels, frame); 167 } 168 169 if (number_of_streams <= 1) { 170 MixFewFramesWithNoLimiter(mix_list, audio_frame_for_mixing); 171 return; 172 } 173 174 // Make sure that the size of the view based on the desired 175 // `samples_per_channel` and `number_of_channels` doesn't exceed the size of 176 // the `mixing_buffer_` buffer. 177 RTC_DCHECK_LE(samples_per_channel, kMaximumChannelSize); 178 // Since the above check is a DCHECK only, clamp down on `samples_per_channel` 179 // to make sure we don't exceed the buffer size in non-dcheck builds. 180 // See also FrameCombinerDeathTest.DebugBuildCrashesWithHighRate. 181 samples_per_channel = std::min(samples_per_channel, kMaximumChannelSize); 182 DeinterleavedView<float> deinterleaved( 183 mixing_buffer_.data(), samples_per_channel, number_of_channels); 184 MixToFloatFrame(mix_list, deinterleaved); 185 186 if (use_limiter_) { 187 RunLimiter(deinterleaved, &limiter_); 188 } 189 190 InterleaveToAudioFrame(deinterleaved, audio_frame_for_mixing); 191 } 192 193 } // namespace webrtc