channel_mixing_matrix.cc (12744B)
1 /* 2 * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved. 3 * 4 * Use of this source code is governed by a BSD-style license 5 * that can be found in the LICENSE file in the root of the source 6 * tree. An additional intellectual property rights grant can be found 7 * in the file PATENTS. All contributing project authors may 8 * be found in the AUTHORS file in the root of the source tree. 9 */ 10 11 #include "audio/utility/channel_mixing_matrix.h" 12 13 #include <algorithm> 14 #include <cstddef> 15 #include <vector> 16 17 #include "api/audio/channel_layout.h" 18 #include "audio/utility/channel_mixer.h" 19 #include "rtc_base/checks.h" 20 21 namespace webrtc { 22 23 namespace { 24 25 ChannelLayout CheckInputLayout(ChannelLayout input_layout, 26 ChannelLayout output_layout) { 27 // Special case for 5.0, 5.1 with back channels when upmixed to 7.0, 7.1, 28 // which should map the back LR to side LR. 29 if (input_layout == CHANNEL_LAYOUT_5_0_BACK && 30 output_layout == CHANNEL_LAYOUT_7_0) { 31 return CHANNEL_LAYOUT_5_0; 32 } else if (input_layout == CHANNEL_LAYOUT_5_1_BACK && 33 output_layout == CHANNEL_LAYOUT_7_1) { 34 return CHANNEL_LAYOUT_5_1; 35 } 36 37 return input_layout; 38 } 39 40 } // namespace 41 42 static void ValidateLayout(ChannelLayout layout) { 43 RTC_CHECK_NE(layout, CHANNEL_LAYOUT_NONE); 44 RTC_CHECK_LE(layout, CHANNEL_LAYOUT_MAX); 45 RTC_CHECK_NE(layout, CHANNEL_LAYOUT_UNSUPPORTED); 46 RTC_CHECK_NE(layout, CHANNEL_LAYOUT_DISCRETE); 47 RTC_CHECK_NE(layout, CHANNEL_LAYOUT_STEREO_AND_KEYBOARD_MIC); 48 49 // Verify there's at least one channel. Should always be true here by virtue 50 // of not being one of the invalid layouts, but lets double check to be sure. 51 int channel_count = ChannelLayoutToChannelCount(layout); 52 RTC_DCHECK_GT(channel_count, 0); 53 54 // If we have more than one channel, verify a symmetric layout for sanity. 55 // The unit test will verify all possible layouts, so this can be a DCHECK. 56 // Symmetry allows simplifying the matrix building code by allowing us to 57 // assume that if one channel of a pair exists, the other will too. 58 if (channel_count > 1) { 59 // Assert that LEFT exists if and only if RIGHT exists, and so on. 60 RTC_DCHECK_EQ(ChannelOrder(layout, LEFT) >= 0, 61 ChannelOrder(layout, RIGHT) >= 0); 62 RTC_DCHECK_EQ(ChannelOrder(layout, SIDE_LEFT) >= 0, 63 ChannelOrder(layout, SIDE_RIGHT) >= 0); 64 RTC_DCHECK_EQ(ChannelOrder(layout, BACK_LEFT) >= 0, 65 ChannelOrder(layout, BACK_RIGHT) >= 0); 66 RTC_DCHECK_EQ(ChannelOrder(layout, LEFT_OF_CENTER) >= 0, 67 ChannelOrder(layout, RIGHT_OF_CENTER) >= 0); 68 } else { 69 RTC_DCHECK_EQ(layout, CHANNEL_LAYOUT_MONO); 70 } 71 } 72 73 ChannelMixingMatrix::ChannelMixingMatrix(ChannelLayout input_layout, 74 int input_channels, 75 ChannelLayout output_layout, 76 int output_channels) 77 : input_layout_(CheckInputLayout(input_layout, output_layout)), 78 input_channels_(input_channels), 79 output_layout_(output_layout), 80 output_channels_(output_channels) { 81 // Stereo down mix should never be the output layout. 82 RTC_CHECK_NE(output_layout, CHANNEL_LAYOUT_STEREO_DOWNMIX); 83 84 // Verify that the layouts are supported 85 if (input_layout != CHANNEL_LAYOUT_DISCRETE) 86 ValidateLayout(input_layout); 87 if (output_layout != CHANNEL_LAYOUT_DISCRETE) 88 ValidateLayout(output_layout); 89 } 90 91 ChannelMixingMatrix::~ChannelMixingMatrix() = default; 92 93 bool ChannelMixingMatrix::CreateTransformationMatrix( 94 std::vector<std::vector<float>>* matrix) { 95 matrix_ = matrix; 96 97 // Size out the initial matrix. 98 matrix_->reserve(output_channels_); 99 for (int output_ch = 0; output_ch < output_channels_; ++output_ch) 100 matrix_->push_back(std::vector<float>(input_channels_, 0)); 101 102 // First check for discrete case. 103 if (input_layout_ == CHANNEL_LAYOUT_DISCRETE || 104 output_layout_ == CHANNEL_LAYOUT_DISCRETE) { 105 // If the number of input channels is more than output channels, then 106 // copy as many as we can then drop the remaining input channels. 107 // If the number of input channels is less than output channels, then 108 // copy them all, then zero out the remaining output channels. 109 int passthrough_channels = std::min(input_channels_, output_channels_); 110 for (int i = 0; i < passthrough_channels; ++i) 111 (*matrix_)[i][i] = 1; 112 113 return true; 114 } 115 116 // If specified, use adjusted channel mapping for the VoIP scenario. 117 if (input_layout_ == CHANNEL_LAYOUT_MONO && 118 ChannelLayoutToChannelCount(output_layout_) >= 2) { 119 // Only place the mono input in the front left and right channels. 120 (*matrix_)[0][0] = 1.f; 121 (*matrix_)[1][0] = 1.f; 122 123 for (size_t output_ch = 2; output_ch < matrix_->size(); ++output_ch) { 124 (*matrix_)[output_ch][0] = 0.f; 125 } 126 return true; 127 } 128 129 // Route matching channels and figure out which ones aren't accounted for. 130 for (Channels ch = LEFT; ch < CHANNELS_MAX + 1; 131 ch = static_cast<Channels>(ch + 1)) { 132 int input_ch_index = ChannelOrder(input_layout_, ch); 133 if (input_ch_index < 0) 134 continue; 135 136 int output_ch_index = ChannelOrder(output_layout_, ch); 137 if (output_ch_index < 0) { 138 unaccounted_inputs_.push_back(ch); 139 continue; 140 } 141 142 RTC_DCHECK_LT(static_cast<size_t>(output_ch_index), matrix_->size()); 143 RTC_DCHECK_LT(static_cast<size_t>(input_ch_index), 144 (*matrix_)[output_ch_index].size()); 145 (*matrix_)[output_ch_index][input_ch_index] = 1; 146 } 147 148 // If all input channels are accounted for, there's nothing left to do. 149 if (unaccounted_inputs_.empty()) { 150 // Since all output channels map directly to inputs we can optimize. 151 return true; 152 } 153 154 // Mix front LR into center. 155 if (IsUnaccounted(LEFT)) { 156 // When down mixing to mono from stereo, we need to be careful of full scale 157 // stereo mixes. Scaling by 1 / sqrt(2) here will likely lead to clipping 158 // so we use 1 / 2 instead. 159 float scale = 160 (output_layout_ == CHANNEL_LAYOUT_MONO && input_channels_ == 2) 161 ? 0.5 162 : ChannelMixer::kHalfPower; 163 Mix(LEFT, CENTER, scale); 164 Mix(RIGHT, CENTER, scale); 165 } 166 167 // Mix center into front LR. 168 if (IsUnaccounted(CENTER)) { 169 // When up mixing from mono, just do a copy to front LR. 170 float scale = 171 (input_layout_ == CHANNEL_LAYOUT_MONO) ? 1 : ChannelMixer::kHalfPower; 172 MixWithoutAccounting(CENTER, LEFT, scale); 173 Mix(CENTER, RIGHT, scale); 174 } 175 176 // Mix back LR into: side LR || back center || front LR || front center. 177 if (IsUnaccounted(BACK_LEFT)) { 178 if (HasOutputChannel(SIDE_LEFT)) { 179 // If the input has side LR, mix back LR into side LR, but instead if the 180 // input doesn't have side LR (but output does) copy back LR to side LR. 181 float scale = HasInputChannel(SIDE_LEFT) ? ChannelMixer::kHalfPower : 1; 182 Mix(BACK_LEFT, SIDE_LEFT, scale); 183 Mix(BACK_RIGHT, SIDE_RIGHT, scale); 184 } else if (HasOutputChannel(BACK_CENTER)) { 185 // Mix back LR into back center. 186 Mix(BACK_LEFT, BACK_CENTER, ChannelMixer::kHalfPower); 187 Mix(BACK_RIGHT, BACK_CENTER, ChannelMixer::kHalfPower); 188 } else if (output_layout_ > CHANNEL_LAYOUT_MONO) { 189 // Mix back LR into front LR. 190 Mix(BACK_LEFT, LEFT, ChannelMixer::kHalfPower); 191 Mix(BACK_RIGHT, RIGHT, ChannelMixer::kHalfPower); 192 } else { 193 // Mix back LR into front center. 194 Mix(BACK_LEFT, CENTER, ChannelMixer::kHalfPower); 195 Mix(BACK_RIGHT, CENTER, ChannelMixer::kHalfPower); 196 } 197 } 198 199 // Mix side LR into: back LR || back center || front LR || front center. 200 if (IsUnaccounted(SIDE_LEFT)) { 201 if (HasOutputChannel(BACK_LEFT)) { 202 // If the input has back LR, mix side LR into back LR, but instead if the 203 // input doesn't have back LR (but output does) copy side LR to back LR. 204 float scale = HasInputChannel(BACK_LEFT) ? ChannelMixer::kHalfPower : 1; 205 Mix(SIDE_LEFT, BACK_LEFT, scale); 206 Mix(SIDE_RIGHT, BACK_RIGHT, scale); 207 } else if (HasOutputChannel(BACK_CENTER)) { 208 // Mix side LR into back center. 209 Mix(SIDE_LEFT, BACK_CENTER, ChannelMixer::kHalfPower); 210 Mix(SIDE_RIGHT, BACK_CENTER, ChannelMixer::kHalfPower); 211 } else if (output_layout_ > CHANNEL_LAYOUT_MONO) { 212 // Mix side LR into front LR. 213 Mix(SIDE_LEFT, LEFT, ChannelMixer::kHalfPower); 214 Mix(SIDE_RIGHT, RIGHT, ChannelMixer::kHalfPower); 215 } else { 216 // Mix side LR into front center. 217 Mix(SIDE_LEFT, CENTER, ChannelMixer::kHalfPower); 218 Mix(SIDE_RIGHT, CENTER, ChannelMixer::kHalfPower); 219 } 220 } 221 222 // Mix back center into: back LR || side LR || front LR || front center. 223 if (IsUnaccounted(BACK_CENTER)) { 224 if (HasOutputChannel(BACK_LEFT)) { 225 // Mix back center into back LR. 226 MixWithoutAccounting(BACK_CENTER, BACK_LEFT, ChannelMixer::kHalfPower); 227 Mix(BACK_CENTER, BACK_RIGHT, ChannelMixer::kHalfPower); 228 } else if (HasOutputChannel(SIDE_LEFT)) { 229 // Mix back center into side LR. 230 MixWithoutAccounting(BACK_CENTER, SIDE_LEFT, ChannelMixer::kHalfPower); 231 Mix(BACK_CENTER, SIDE_RIGHT, ChannelMixer::kHalfPower); 232 } else if (output_layout_ > CHANNEL_LAYOUT_MONO) { 233 // Mix back center into front LR. 234 // TODO(dalecurtis): Not sure about these values? 235 MixWithoutAccounting(BACK_CENTER, LEFT, ChannelMixer::kHalfPower); 236 Mix(BACK_CENTER, RIGHT, ChannelMixer::kHalfPower); 237 } else { 238 // Mix back center into front center. 239 // TODO(dalecurtis): Not sure about these values? 240 Mix(BACK_CENTER, CENTER, ChannelMixer::kHalfPower); 241 } 242 } 243 244 // Mix LR of center into: front LR || front center. 245 if (IsUnaccounted(LEFT_OF_CENTER)) { 246 if (HasOutputChannel(LEFT)) { 247 // Mix LR of center into front LR. 248 Mix(LEFT_OF_CENTER, LEFT, ChannelMixer::kHalfPower); 249 Mix(RIGHT_OF_CENTER, RIGHT, ChannelMixer::kHalfPower); 250 } else { 251 // Mix LR of center into front center. 252 Mix(LEFT_OF_CENTER, CENTER, ChannelMixer::kHalfPower); 253 Mix(RIGHT_OF_CENTER, CENTER, ChannelMixer::kHalfPower); 254 } 255 } 256 257 // Mix LFE into: front center || front LR. 258 if (IsUnaccounted(LFE)) { 259 if (!HasOutputChannel(CENTER)) { 260 // Mix LFE into front LR. 261 MixWithoutAccounting(LFE, LEFT, ChannelMixer::kHalfPower); 262 Mix(LFE, RIGHT, ChannelMixer::kHalfPower); 263 } else { 264 // Mix LFE into front center. 265 Mix(LFE, CENTER, ChannelMixer::kHalfPower); 266 } 267 } 268 269 // All channels should now be accounted for. 270 RTC_DCHECK(unaccounted_inputs_.empty()); 271 272 // See if the output `matrix_` is simply a remapping matrix. If each input 273 // channel maps to a single output channel we can simply remap. Doing this 274 // programmatically is less fragile than logic checks on channel mappings. 275 for (int output_ch = 0; output_ch < output_channels_; ++output_ch) { 276 int input_mappings = 0; 277 for (int input_ch = 0; input_ch < input_channels_; ++input_ch) { 278 // We can only remap if each row contains a single scale of 1. I.e., each 279 // output channel is mapped from a single unscaled input channel. 280 if ((*matrix_)[output_ch][input_ch] != 1 || ++input_mappings > 1) 281 return false; 282 } 283 } 284 285 // If we've gotten here, `matrix_` is simply a remapping. 286 return true; 287 } 288 289 void ChannelMixingMatrix::AccountFor(Channels ch) { 290 unaccounted_inputs_.erase( 291 std::find(unaccounted_inputs_.begin(), unaccounted_inputs_.end(), ch)); 292 } 293 294 bool ChannelMixingMatrix::IsUnaccounted(Channels ch) const { 295 return std::find(unaccounted_inputs_.begin(), unaccounted_inputs_.end(), 296 ch) != unaccounted_inputs_.end(); 297 } 298 299 bool ChannelMixingMatrix::HasInputChannel(Channels ch) const { 300 return ChannelOrder(input_layout_, ch) >= 0; 301 } 302 303 bool ChannelMixingMatrix::HasOutputChannel(Channels ch) const { 304 return ChannelOrder(output_layout_, ch) >= 0; 305 } 306 307 void ChannelMixingMatrix::Mix(Channels input_ch, 308 Channels output_ch, 309 float scale) { 310 MixWithoutAccounting(input_ch, output_ch, scale); 311 AccountFor(input_ch); 312 } 313 314 void ChannelMixingMatrix::MixWithoutAccounting(Channels input_ch, 315 Channels output_ch, 316 float scale) { 317 int input_ch_index = ChannelOrder(input_layout_, input_ch); 318 int output_ch_index = ChannelOrder(output_layout_, output_ch); 319 320 RTC_DCHECK(IsUnaccounted(input_ch)); 321 RTC_DCHECK_GE(input_ch_index, 0); 322 RTC_DCHECK_GE(output_ch_index, 0); 323 324 RTC_DCHECK_EQ((*matrix_)[output_ch_index][input_ch_index], 0); 325 (*matrix_)[output_ch_index][input_ch_index] = scale; 326 } 327 328 } // namespace webrtc