MediaTrackConstraints.cpp (18482B)
1 /* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- 2 * This Source Code Form is subject to the terms of the Mozilla Public 3 * License, v. 2.0. If a copy of the MPL was not distributed with this file, 4 * You can obtain one at http://mozilla.org/MPL/2.0/. */ 5 6 #include "MediaTrackConstraints.h" 7 8 #include <algorithm> 9 #include <iterator> 10 #include <limits> 11 12 #include "mozilla/MediaManager.h" 13 #include "mozilla/dom/MediaStreamTrackBinding.h" 14 15 #ifdef MOZ_WEBRTC 16 namespace mozilla { 17 extern LazyLogModule gMediaManagerLog; 18 } 19 #else 20 static mozilla::LazyLogModule gMediaManagerLog("MediaManager"); 21 #endif 22 #define LOG(...) MOZ_LOG(gMediaManagerLog, LogLevel::Debug, (__VA_ARGS__)) 23 24 namespace mozilla { 25 26 using dom::CallerType; 27 using dom::VideoResizeModeEnum; 28 29 template <class ValueType> 30 template <class ConstrainRange> 31 void NormalizedConstraintSet::Range<ValueType>::SetFrom( 32 const ConstrainRange& aOther) { 33 if (aOther.mIdeal.WasPassed()) { 34 mIdeal.emplace(aOther.mIdeal.Value()); 35 } 36 if (aOther.mExact.WasPassed()) { 37 mMin = aOther.mExact.Value(); 38 mMax = aOther.mExact.Value(); 39 } else { 40 if (aOther.mMin.WasPassed()) { 41 mMin = aOther.mMin.Value(); 42 } 43 if (aOther.mMax.WasPassed()) { 44 mMax = aOther.mMax.Value(); 45 } 46 } 47 } 48 49 // The Range code works surprisingly well for bool, except when averaging 50 // ideals. 51 template <> 52 bool NormalizedConstraintSet::Range<bool>::Merge(const Range& aOther) { 53 if (!Intersects(aOther)) { 54 return false; 55 } 56 Intersect(aOther); 57 58 // To avoid "unsafe use of type 'bool'", we keep counter in mMergeDenominator 59 uint32_t counter = mMergeDenominator >> 16; 60 uint32_t denominator = mMergeDenominator & 0xffff; 61 62 if (aOther.mIdeal.isSome()) { 63 if (mIdeal.isNothing()) { 64 mIdeal.emplace(aOther.Get(false)); 65 counter = aOther.Get(false); 66 denominator = 1; 67 } else { 68 if (!denominator) { 69 counter = Get(false); 70 denominator = 1; 71 } 72 counter += aOther.Get(false); 73 denominator++; 74 } 75 } 76 mMergeDenominator = ((counter & 0xffff) << 16) + (denominator & 0xffff); 77 return true; 78 } 79 80 template <> 81 void NormalizedConstraintSet::Range<bool>::FinalizeMerge() { 82 if (mMergeDenominator) { 83 uint32_t counter = mMergeDenominator >> 16; 84 uint32_t denominator = mMergeDenominator & 0xffff; 85 86 *mIdeal = !!(counter / denominator); 87 mMergeDenominator = 0; 88 } 89 } 90 91 NormalizedConstraintSet::LongRange::LongRange( 92 const nsCString& aName, 93 const dom::Optional<dom::OwningLongOrConstrainLongRange>& aOther, 94 bool advanced) 95 : Range<int32_t>(aName, 96 1 + INT32_MIN, // +1 avoids Windows compiler bug 97 INT32_MAX) { 98 if (!aOther.WasPassed()) { 99 return; 100 } 101 const auto& other = aOther.Value(); 102 if (other.IsLong()) { 103 if (advanced) { 104 mMin = mMax = other.GetAsLong(); 105 } else { 106 mIdeal.emplace(other.GetAsLong()); 107 } 108 } else { 109 SetFrom(other.GetAsConstrainLongRange()); 110 } 111 } 112 113 NormalizedConstraintSet::LongLongRange::LongLongRange( 114 const nsCString& aName, const dom::Optional<int64_t>& aOther) 115 : Range<int64_t>(aName, 116 1 + INT64_MIN, // +1 avoids Windows compiler bug 117 INT64_MAX) { 118 if (aOther.WasPassed()) { 119 mIdeal.emplace(aOther.Value()); 120 } 121 } 122 123 NormalizedConstraintSet::DoubleRange::DoubleRange( 124 const nsCString& aName, 125 const dom::Optional<dom::OwningDoubleOrConstrainDoubleRange>& aOther, 126 bool advanced) 127 : Range<double>(aName, -std::numeric_limits<double>::infinity(), 128 std::numeric_limits<double>::infinity()) { 129 if (!aOther.WasPassed()) { 130 return; 131 } 132 const auto& other = aOther.Value(); 133 if (other.IsDouble()) { 134 if (advanced) { 135 mMin = mMax = other.GetAsDouble(); 136 } else { 137 mIdeal.emplace(other.GetAsDouble()); 138 } 139 } else { 140 SetFrom(other.GetAsConstrainDoubleRange()); 141 } 142 } 143 144 NormalizedConstraintSet::BooleanRange::BooleanRange( 145 const nsCString& aName, 146 const dom::Optional<dom::OwningBooleanOrConstrainBooleanParameters>& aOther, 147 bool advanced) 148 : Range<bool>(aName, false, true) { 149 if (!aOther.WasPassed()) { 150 return; 151 } 152 const auto& other = aOther.Value(); 153 if (other.IsBoolean()) { 154 if (advanced) { 155 mMin = mMax = other.GetAsBoolean(); 156 } else { 157 mIdeal.emplace(other.GetAsBoolean()); 158 } 159 } else { 160 const auto& r = other.GetAsConstrainBooleanParameters(); 161 if (r.mIdeal.WasPassed()) { 162 mIdeal.emplace(r.mIdeal.Value()); 163 } 164 if (r.mExact.WasPassed()) { 165 mMin = r.mExact.Value(); 166 mMax = r.mExact.Value(); 167 } 168 } 169 } 170 171 NormalizedConstraintSet::StringRange::StringRange( 172 const nsCString& aName, 173 const dom::Optional< 174 dom::OwningStringOrStringSequenceOrConstrainDOMStringParameters>& 175 aOther, 176 bool advanced) 177 : BaseRange(aName) { 178 if (!aOther.WasPassed()) { 179 return; 180 } 181 const auto& other = aOther.Value(); 182 if (other.IsString()) { 183 if (advanced) { 184 mExact.insert(other.GetAsString()); 185 } else { 186 mIdeal.insert(other.GetAsString()); 187 } 188 } else if (other.IsStringSequence()) { 189 if (advanced) { 190 mExact.clear(); 191 for (const auto& str : other.GetAsStringSequence()) { 192 mExact.insert(str); 193 } 194 } else { 195 mIdeal.clear(); 196 for (const auto& str : other.GetAsStringSequence()) { 197 mIdeal.insert(str); 198 } 199 } 200 } else { 201 SetFrom(other.GetAsConstrainDOMStringParameters()); 202 } 203 } 204 205 void NormalizedConstraintSet::StringRange::SetFrom( 206 const dom::ConstrainDOMStringParameters& aOther) { 207 if (aOther.mIdeal.WasPassed()) { 208 mIdeal.clear(); 209 if (aOther.mIdeal.Value().IsString()) { 210 mIdeal.insert(aOther.mIdeal.Value().GetAsString()); 211 } else { 212 for (const auto& str : aOther.mIdeal.Value().GetAsStringSequence()) { 213 mIdeal.insert(str); 214 } 215 } 216 } 217 if (aOther.mExact.WasPassed()) { 218 mExact.clear(); 219 if (aOther.mExact.Value().IsString()) { 220 mExact.insert(aOther.mExact.Value().GetAsString()); 221 } else { 222 for (const auto& str : aOther.mExact.Value().GetAsStringSequence()) { 223 mExact.insert(str); 224 } 225 } 226 } 227 } 228 229 auto NormalizedConstraintSet::StringRange::Clamp(const ValueType& n) const 230 -> ValueType { 231 if (mExact.empty()) { 232 return n; 233 } 234 ValueType result; 235 for (const auto& entry : n) { 236 if (mExact.find(entry) != mExact.end()) { 237 result.insert(entry); 238 } 239 } 240 return result; 241 } 242 243 bool NormalizedConstraintSet::StringRange::Intersects( 244 const StringRange& aOther) const { 245 if (mExact.empty() || aOther.mExact.empty()) { 246 return true; 247 } 248 249 ValueType intersection; 250 set_intersection(mExact.begin(), mExact.end(), aOther.mExact.begin(), 251 aOther.mExact.end(), 252 std::inserter(intersection, intersection.begin())); 253 return !intersection.empty(); 254 } 255 256 void NormalizedConstraintSet::StringRange::Intersect( 257 const StringRange& aOther) { 258 if (aOther.mExact.empty()) { 259 return; 260 } 261 262 ValueType intersection; 263 set_intersection(mExact.begin(), mExact.end(), aOther.mExact.begin(), 264 aOther.mExact.end(), 265 std::inserter(intersection, intersection.begin())); 266 mExact = intersection; 267 } 268 269 bool NormalizedConstraintSet::StringRange::Merge(const StringRange& aOther) { 270 if (!Intersects(aOther)) { 271 return false; 272 } 273 Intersect(aOther); 274 275 ValueType unioned; 276 set_union(mIdeal.begin(), mIdeal.end(), aOther.mIdeal.begin(), 277 aOther.mIdeal.end(), std::inserter(unioned, unioned.begin())); 278 mIdeal = unioned; 279 return true; 280 } 281 282 NormalizedConstraints::NormalizedConstraints( 283 const dom::MediaTrackConstraints& aOther) 284 : NormalizedConstraintSet(aOther, false) { 285 if (aOther.mAdvanced.WasPassed()) { 286 for (const auto& entry : aOther.mAdvanced.Value()) { 287 mAdvanced.push_back(NormalizedConstraintSet(entry, true)); 288 } 289 } 290 } 291 292 FlattenedConstraints::FlattenedConstraints(const NormalizedConstraints& aOther) 293 : NormalizedConstraintSet(aOther) { 294 for (const auto& set : aOther.mAdvanced) { 295 // Must only apply compatible i.e. inherently non-overconstraining sets 296 // This rule is pretty much why this code is centralized here. 297 if (mWidth.Intersects(set.mWidth) && mHeight.Intersects(set.mHeight) && 298 mFrameRate.Intersects(set.mFrameRate)) { 299 mWidth.Intersect(set.mWidth); 300 mHeight.Intersect(set.mHeight); 301 mFrameRate.Intersect(set.mFrameRate); 302 } 303 if (mEchoCancellation.Intersects(set.mEchoCancellation)) { 304 mEchoCancellation.Intersect(set.mEchoCancellation); 305 } 306 if (mNoiseSuppression.Intersects(set.mNoiseSuppression)) { 307 mNoiseSuppression.Intersect(set.mNoiseSuppression); 308 } 309 if (mAutoGainControl.Intersects(set.mAutoGainControl)) { 310 mAutoGainControl.Intersect(set.mAutoGainControl); 311 } 312 if (mChannelCount.Intersects(set.mChannelCount)) { 313 mChannelCount.Intersect(set.mChannelCount); 314 } 315 } 316 } 317 318 // MediaEngine helper 319 // 320 // The full algorithm for all devices. 321 // 322 // Fitness distance returned as integer math * 1000. Infinity = UINT32_MAX 323 324 // First, all devices have a minimum distance based on their deviceId. 325 // If you have no other constraints, use this one. Reused by all device types. 326 327 /* static */ 328 bool MediaConstraintsHelper::SomeSettingsFit( 329 const NormalizedConstraints& aConstraints, const MediaEnginePrefs& aPrefs, 330 const nsTArray<RefPtr<LocalMediaDevice>>& aDevices) { 331 nsTArray<const NormalizedConstraintSet*> sets; 332 sets.AppendElement(&aConstraints); 333 334 MOZ_ASSERT(!aDevices.IsEmpty()); 335 for (const auto& device : aDevices) { 336 auto distance = 337 device->GetBestFitnessDistance(sets, aPrefs, CallerType::NonSystem); 338 if (distance != UINT32_MAX) { 339 return true; 340 } 341 } 342 return false; 343 } 344 345 // Fitness distance returned as integer math * 1000. Infinity = UINT32_MAX 346 347 /* static */ 348 uint32_t MediaConstraintsHelper::FitnessDistance( 349 const Maybe<nsString>& aN, 350 const NormalizedConstraintSet::StringRange& aParams) { 351 if (!aParams.mExact.empty() && 352 (aN.isNothing() || aParams.mExact.find(*aN) == aParams.mExact.end())) { 353 return UINT32_MAX; 354 } 355 if (!aParams.mIdeal.empty() && 356 (aN.isNothing() || aParams.mIdeal.find(*aN) == aParams.mIdeal.end())) { 357 return 1000; 358 } 359 return 0; 360 } 361 362 /* static */ const char* MediaConstraintsHelper::SelectSettings( 363 const NormalizedConstraints& aConstraints, const MediaEnginePrefs& aPrefs, 364 nsTArray<RefPtr<LocalMediaDevice>>& aDevices, CallerType aCallerType) { 365 const auto& c = aConstraints; 366 LogConstraints(c); 367 368 if (!aDevices.IsEmpty() && 369 aDevices[0]->Kind() == dom::MediaDeviceKind::Videoinput && 370 aPrefs.mResizeModeEnabled) { 371 // Check invalid exact resizeMode constraint (not a device property) 372 nsString none = 373 NS_ConvertASCIItoUTF16(dom::GetEnumString(VideoResizeModeEnum::None)); 374 nsString crop = NS_ConvertASCIItoUTF16( 375 dom::GetEnumString(VideoResizeModeEnum::Crop_and_scale)); 376 if (FitnessDistance(Some(none), c.mResizeMode) == UINT32_MAX && 377 FitnessDistance(Some(crop), c.mResizeMode) == UINT32_MAX) { 378 return "resizeMode"; 379 } 380 } 381 382 // First apply top-level constraints. 383 384 // Stack constraintSets that pass, starting with the required one, because the 385 // whole stack must be re-satisfied each time a capability-set is ruled out 386 // (this avoids storing state or pushing algorithm into the lower-level code). 387 nsTArray<RefPtr<LocalMediaDevice>> unsatisfactory; 388 nsTArray<const NormalizedConstraintSet*> aggregateConstraints; 389 aggregateConstraints.AppendElement(&c); 390 391 std::multimap<uint32_t, RefPtr<LocalMediaDevice>> ordered; 392 393 for (uint32_t i = 0; i < aDevices.Length();) { 394 uint32_t distance = aDevices[i]->GetBestFitnessDistance( 395 aggregateConstraints, aPrefs, aCallerType); 396 if (distance == UINT32_MAX) { 397 unsatisfactory.AppendElement(std::move(aDevices[i])); 398 aDevices.RemoveElementAt(i); 399 } else { 400 ordered.insert(std::make_pair(distance, aDevices[i])); 401 ++i; 402 } 403 } 404 if (aDevices.IsEmpty()) { 405 return FindBadConstraint(c, aPrefs, unsatisfactory); 406 } 407 408 // Order devices by shortest distance 409 for (auto& ordinal : ordered) { 410 aDevices.RemoveElement(ordinal.second); 411 aDevices.AppendElement(ordinal.second); 412 } 413 414 // Then apply advanced constraints. 415 416 for (const auto& advanced : c.mAdvanced) { 417 aggregateConstraints.AppendElement(&advanced); 418 nsTArray<RefPtr<LocalMediaDevice>> rejects; 419 for (uint32_t j = 0; j < aDevices.Length();) { 420 uint32_t distance = aDevices[j]->GetBestFitnessDistance( 421 aggregateConstraints, aPrefs, aCallerType); 422 if (distance == UINT32_MAX) { 423 rejects.AppendElement(std::move(aDevices[j])); 424 aDevices.RemoveElementAt(j); 425 } else { 426 ++j; 427 } 428 } 429 if (aDevices.IsEmpty()) { 430 aDevices.AppendElements(std::move(rejects)); 431 aggregateConstraints.RemoveLastElement(); 432 } 433 } 434 return nullptr; 435 } 436 437 /* static */ const char* MediaConstraintsHelper::FindBadConstraint( 438 const NormalizedConstraints& aConstraints, const MediaEnginePrefs& aPrefs, 439 const nsTArray<RefPtr<LocalMediaDevice>>& aDevices) { 440 // The spec says to report a constraint that satisfies NONE 441 // of the sources. Unfortunately, this is a bit laborious to find out, and 442 // requires updating as new constraints are added! 443 const auto& c = aConstraints; 444 445 if (aDevices.IsEmpty() || 446 !SomeSettingsFit(NormalizedConstraints(), aPrefs, aDevices)) { 447 return ""; 448 } 449 { 450 NormalizedConstraints fresh; 451 fresh.mDeviceId = c.mDeviceId; 452 if (!SomeSettingsFit(fresh, aPrefs, aDevices)) { 453 return "deviceId"; 454 } 455 } 456 { 457 NormalizedConstraints fresh; 458 fresh.mGroupId = c.mGroupId; 459 if (!SomeSettingsFit(fresh, aPrefs, aDevices)) { 460 return "groupId"; 461 } 462 } 463 { 464 NormalizedConstraints fresh; 465 fresh.mWidth = c.mWidth; 466 if (!SomeSettingsFit(fresh, aPrefs, aDevices)) { 467 return "width"; 468 } 469 } 470 { 471 NormalizedConstraints fresh; 472 fresh.mHeight = c.mHeight; 473 if (!SomeSettingsFit(fresh, aPrefs, aDevices)) { 474 return "height"; 475 } 476 } 477 { 478 NormalizedConstraints fresh; 479 fresh.mFrameRate = c.mFrameRate; 480 if (!SomeSettingsFit(fresh, aPrefs, aDevices)) { 481 return "frameRate"; 482 } 483 } 484 { 485 NormalizedConstraints fresh; 486 fresh.mFacingMode = c.mFacingMode; 487 if (!SomeSettingsFit(fresh, aPrefs, aDevices)) { 488 return "facingMode"; 489 } 490 } 491 return ""; 492 } 493 494 /* static */ 495 const char* MediaConstraintsHelper::FindBadConstraint( 496 const NormalizedConstraints& aConstraints, const MediaEnginePrefs& aPrefs, 497 const MediaDevice* aMediaDevice) { 498 NormalizedConstraints c(aConstraints); 499 NormalizedConstraints empty; 500 c.mDeviceId = empty.mDeviceId; 501 c.mGroupId = empty.mGroupId; 502 AutoTArray<RefPtr<LocalMediaDevice>, 1> devices; 503 devices.EmplaceBack( 504 new LocalMediaDevice(aMediaDevice, u""_ns, u""_ns, u""_ns)); 505 return FindBadConstraint(c, aPrefs, devices); 506 } 507 508 static void LogConstraintStringRange( 509 const NormalizedConstraintSet::StringRange& aRange) { 510 if (aRange.mExact.size() <= 1 && aRange.mIdeal.size() <= 1) { 511 LOG(" %s: { exact: [%s], ideal: [%s] }", aRange.mName.get(), 512 (aRange.mExact.empty() 513 ? "" 514 : NS_ConvertUTF16toUTF8(*aRange.mExact.begin()).get()), 515 (aRange.mIdeal.empty() 516 ? "" 517 : NS_ConvertUTF16toUTF8(*aRange.mIdeal.begin()).get())); 518 } else { 519 LOG(" %s: { exact: [", aRange.mName.get()); 520 for (const auto& entry : aRange.mExact) { 521 LOG(" %s,", NS_ConvertUTF16toUTF8(entry).get()); 522 } 523 LOG(" ], ideal: ["); 524 for (const auto& entry : aRange.mIdeal) { 525 LOG(" %s,", NS_ConvertUTF16toUTF8(entry).get()); 526 } 527 LOG(" ]}"); 528 } 529 } 530 531 template <typename T> 532 static void LogConstraintRange( 533 const NormalizedConstraintSet::Range<T>& aRange) { 534 if (aRange.mIdeal.isSome()) { 535 LOG(" %s: { min: %d, max: %d, ideal: %d }", aRange.mName.get(), 536 aRange.mMin, aRange.mMax, aRange.mIdeal.valueOr(0)); 537 } else { 538 LOG(" %s: { min: %d, max: %d }", aRange.mName.get(), aRange.mMin, 539 aRange.mMax); 540 } 541 } 542 543 template <> 544 void LogConstraintRange(const NormalizedConstraintSet::Range<double>& aRange) { 545 if (aRange.mIdeal.isSome()) { 546 LOG(" %s: { min: %f, max: %f, ideal: %f }", aRange.mName.get(), 547 aRange.mMin, aRange.mMax, aRange.mIdeal.valueOr(0)); 548 } else { 549 LOG(" %s: { min: %f, max: %f }", aRange.mName.get(), aRange.mMin, 550 aRange.mMax); 551 } 552 } 553 554 /* static */ 555 void MediaConstraintsHelper::LogConstraints( 556 const NormalizedConstraintSet& aConstraints) { 557 const auto& c = aConstraints; 558 LOG("Constraints: {"); 559 LOG("%s", [&]() { 560 LogConstraintRange(c.mWidth); 561 LogConstraintRange(c.mHeight); 562 LogConstraintRange(c.mFrameRate); 563 LogConstraintStringRange(c.mMediaSource); 564 LogConstraintStringRange(c.mFacingMode); 565 LogConstraintStringRange(c.mResizeMode); 566 LogConstraintStringRange(c.mDeviceId); 567 LogConstraintStringRange(c.mGroupId); 568 LogConstraintRange(c.mEchoCancellation); 569 LogConstraintRange(c.mAutoGainControl); 570 LogConstraintRange(c.mNoiseSuppression); 571 LogConstraintRange(c.mChannelCount); 572 return "}"; 573 }()); 574 } 575 576 /* static */ 577 Maybe<VideoResizeModeEnum> MediaConstraintsHelper::GetResizeMode( 578 const NormalizedConstraintSet& aConstraints, 579 const MediaEnginePrefs& aPrefs) { 580 if (!aPrefs.mResizeModeEnabled) { 581 return Nothing(); 582 } 583 auto defaultResizeMode = aPrefs.mResizeMode; 584 nsString defaultResizeModeString = 585 NS_ConvertASCIItoUTF16(dom::GetEnumString(defaultResizeMode)); 586 uint32_t distanceToDefault = MediaConstraintsHelper::FitnessDistance( 587 Some(defaultResizeModeString), aConstraints.mResizeMode); 588 if (distanceToDefault == 0) { 589 return Some(defaultResizeMode); 590 } 591 VideoResizeModeEnum otherResizeMode = 592 (defaultResizeMode == VideoResizeModeEnum::None) 593 ? VideoResizeModeEnum::Crop_and_scale 594 : VideoResizeModeEnum::None; 595 nsString otherResizeModeString = 596 NS_ConvertASCIItoUTF16(dom::GetEnumString(otherResizeMode)); 597 uint32_t distanceToOther = MediaConstraintsHelper::FitnessDistance( 598 Some(otherResizeModeString), aConstraints.mResizeMode); 599 return Some((distanceToDefault <= distanceToOther) ? defaultResizeMode 600 : otherResizeMode); 601 } 602 603 } // namespace mozilla 604 605 #undef LOG