Add media stream release logging
[WebKit-https.git] / Source / WebCore / platform / mediastream / RealtimeMediaSource.cpp
1 /*
2  * Copyright (C) 2012 Google Inc. All rights reserved.
3  * Copyright (C) 2013-2019 Apple Inc. All rights reserved.
4  * Copyright (C) 2013 Nokia Corporation and/or its subsidiary(-ies).
5  * Copyright (C) 2015 Ericsson AB. All rights reserved.
6  *
7  * Redistribution and use in source and binary forms, with or without
8  * modification, are permitted provided that the following conditions
9  * are met:
10  *
11  * 1. Redistributions of source code must retain the above copyright
12  *    notice, this list of conditions and the following disclaimer.
13  * 2. Redistributions in binary form must reproduce the above copyright
14  *    notice, this list of conditions and the following disclaimer
15  *    in the documentation and/or other materials provided with the
16  *    distribution.
17  * 3. Neither the name of Google Inc. nor the names of its contributors
18  *    may be used to endorse or promote products derived from this
19  *    software without specific prior written permission.
20  *
21  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
22  * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
23  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
24  * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
25  * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
26  * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
27  * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
28  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
29  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
30  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
31  * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
32  */
33
34 #include "config.h"
35
36 #if ENABLE(MEDIA_STREAM)
37 #include "RealtimeMediaSource.h"
38
39 #include "Logging.h"
40 #include "MediaConstraints.h"
41 #include "NotImplemented.h"
42 #include "RealtimeMediaSourceCapabilities.h"
43 #include "RealtimeMediaSourceCenter.h"
44 #include <wtf/CompletionHandler.h>
45 #include <wtf/MainThread.h>
46 #include <wtf/UUID.h>
47 #include <wtf/text/StringHash.h>
48
49 namespace WebCore {
50
51 RealtimeMediaSource::RealtimeMediaSource(Type type, String&& name, String&& deviceID, String&& hashSalt)
52     : m_idHashSalt(WTFMove(hashSalt))
53     , m_persistentID(WTFMove(deviceID))
54     , m_type(type)
55     , m_name(WTFMove(name))
56 {
57     if (m_persistentID.isEmpty())
58         m_persistentID = createCanonicalUUIDString();
59
60     m_hashedID = RealtimeMediaSourceCenter::singleton().hashStringWithSalt(m_persistentID, m_idHashSalt);
61 }
62
63 void RealtimeMediaSource::addObserver(RealtimeMediaSource::Observer& observer)
64 {
65     auto locker = holdLock(m_observersLock);
66     m_observers.add(&observer);
67 }
68
69 void RealtimeMediaSource::removeObserver(RealtimeMediaSource::Observer& observer)
70 {
71     auto locker = holdLock(m_observersLock);
72
73     m_observers.remove(&observer);
74     if (m_observers.isEmpty())
75         stop();
76 }
77
78 void RealtimeMediaSource::setInterrupted(bool interrupted, bool pageMuted)
79 {
80     if (interrupted == m_interrupted)
81         return;
82
83     ALWAYS_LOG_IF(m_logger, LOGIDENTIFIER, interrupted, ", page muted : ", pageMuted);
84
85     m_interrupted = interrupted;
86     if (!interrupted && pageMuted)
87         return;
88
89     setMuted(interrupted);
90 }
91
92 void RealtimeMediaSource::setMuted(bool muted)
93 {
94     ALWAYS_LOG_IF(m_logger, LOGIDENTIFIER);
95
96     if (muted)
97         stop();
98     else {
99         if (interrupted())
100             return;
101
102         start();
103     }
104
105     notifyMutedChange(muted);
106 }
107
108 void RealtimeMediaSource::notifyMutedChange(bool muted)
109 {
110     if (m_muted == muted)
111         return;
112
113     ALWAYS_LOG_IF(m_logger, LOGIDENTIFIER);
114     m_muted = muted;
115
116     notifyMutedObservers();
117 }
118
119 void RealtimeMediaSource::setInterruptedForTesting(bool interrupted)
120 {
121     notifyMutedChange(interrupted);
122 }
123
124 void RealtimeMediaSource::forEachObserver(const WTF::Function<void(Observer&)>& apply) const
125 {
126     Vector<Observer*> observersCopy;
127     {
128         auto locker = holdLock(m_observersLock);
129         observersCopy = copyToVector(m_observers);
130     }
131     for (auto* observer : observersCopy) {
132         auto locker = holdLock(m_observersLock);
133         // Make sure the observer has not been destroyed.
134         if (!m_observers.contains(observer))
135             continue;
136         apply(*observer);
137     }
138 }
139
140 void RealtimeMediaSource::notifyMutedObservers() const
141 {
142     forEachObserver([](auto& observer) {
143         observer.sourceMutedChanged();
144     });
145 }
146
147 void RealtimeMediaSource::notifySettingsDidChangeObservers(OptionSet<RealtimeMediaSourceSettings::Flag> flags)
148 {
149     ASSERT(isMainThread());
150
151     settingsDidChange(flags);
152
153     if (m_pendingSettingsDidChangeNotification)
154         return;
155     m_pendingSettingsDidChangeNotification = true;
156
157     ALWAYS_LOG_IF(m_logger, LOGIDENTIFIER, flags);
158
159     scheduleDeferredTask([this] {
160         m_pendingSettingsDidChangeNotification = false;
161         forEachObserver([](auto& observer) {
162             observer.sourceSettingsChanged();
163         });
164     });
165 }
166
167 void RealtimeMediaSource::videoSampleAvailable(MediaSample& mediaSample)
168 {
169 #if !RELEASE_LOG_DISABLED
170     ++m_frameCount;
171
172     auto timestamp = MonotonicTime::now();
173     auto delta = timestamp - m_lastFrameLogTime;
174     if (!m_lastFrameLogTime || delta >= 1_s) {
175         if (m_lastFrameLogTime) {
176             INFO_LOG_IF(loggerPtr(), LOGIDENTIFIER, m_frameCount, " frames sent in ", delta.value(), " seconds");
177             m_frameCount = 0;
178         }
179         m_lastFrameLogTime = timestamp;
180     }
181 #endif
182
183     forEachObserver([&](auto& observer) {
184         observer.videoSampleAvailable(mediaSample);
185     });
186 }
187
188 void RealtimeMediaSource::audioSamplesAvailable(const MediaTime& time, const PlatformAudioData& audioData, const AudioStreamDescription& description, size_t numberOfFrames)
189 {
190     forEachObserver([&](auto& observer) {
191         observer.audioSamplesAvailable(time, audioData, description, numberOfFrames);
192     });
193 }
194
195 void RealtimeMediaSource::start()
196 {
197     if (m_isProducingData || m_isEnded)
198         return;
199
200     ALWAYS_LOG_IF(m_logger, LOGIDENTIFIER);
201
202     m_isProducingData = true;
203     startProducingData();
204
205     if (!m_isProducingData)
206         return;
207
208     forEachObserver([](auto& observer) {
209         observer.sourceStarted();
210     });
211 }
212
213 void RealtimeMediaSource::stop()
214 {
215     if (!m_isProducingData)
216         return;
217
218     ALWAYS_LOG_IF(m_logger, LOGIDENTIFIER);
219
220     m_isProducingData = false;
221     stopProducingData();
222 }
223
224 void RealtimeMediaSource::requestToEnd(Observer& callingObserver)
225 {
226     if (!m_isProducingData)
227         return;
228
229     bool hasObserverPreventingStopping = false;
230     forEachObserver([&](auto& observer) {
231         if (observer.preventSourceFromStopping())
232             hasObserverPreventingStopping = true;
233     });
234     if (hasObserverPreventingStopping)
235         return;
236
237     auto protectedThis = makeRef(*this);
238
239     stop();
240     m_isEnded = true;
241     hasEnded();
242
243     forEachObserver([callingObserver](auto& observer) {
244         if (&observer != &callingObserver)
245             observer.sourceStopped();
246     });
247 }
248
249 void RealtimeMediaSource::captureFailed()
250 {
251     ERROR_LOG_IF(m_logger, LOGIDENTIFIER);
252
253     m_isProducingData = false;
254     m_captureDidFailed = true;
255
256     forEachObserver([](auto& observer) {
257         observer.sourceStopped();
258     });
259 }
260
261 bool RealtimeMediaSource::supportsSizeAndFrameRate(Optional<int>, Optional<int>, Optional<double>)
262 {
263     // The size and frame rate are within the capability limits, so they are supported.
264     return true;
265 }
266
267 bool RealtimeMediaSource::supportsSizeAndFrameRate(Optional<IntConstraint> widthConstraint, Optional<IntConstraint> heightConstraint, Optional<DoubleConstraint> frameRateConstraint, String& badConstraint, double& distance)
268 {
269     if (!widthConstraint && !heightConstraint && !frameRateConstraint)
270         return true;
271
272     auto& capabilities = this->capabilities();
273
274     distance = std::numeric_limits<double>::infinity();
275
276     Optional<int> width;
277     if (widthConstraint && capabilities.supportsWidth()) {
278         double constraintDistance = fitnessDistance(*widthConstraint);
279         if (std::isinf(constraintDistance)) {
280             badConstraint = widthConstraint->name();
281             return false;
282         }
283
284         distance = std::min(distance, constraintDistance);
285         if (widthConstraint->isMandatory()) {
286             auto range = capabilities.width();
287             width = widthConstraint->valueForCapabilityRange(size().width(), range.rangeMin().asInt, range.rangeMax().asInt);
288         }
289     }
290
291     Optional<int> height;
292     if (heightConstraint && capabilities.supportsHeight()) {
293         double constraintDistance = fitnessDistance(*heightConstraint);
294         if (std::isinf(constraintDistance)) {
295             badConstraint = heightConstraint->name();
296             return false;
297         }
298
299         distance = std::min(distance, constraintDistance);
300         if (heightConstraint->isMandatory()) {
301             auto range = capabilities.height();
302             height = heightConstraint->valueForCapabilityRange(size().height(), range.rangeMin().asInt, range.rangeMax().asInt);
303         }
304     }
305
306     Optional<double> frameRate;
307     if (frameRateConstraint && capabilities.supportsFrameRate()) {
308         double constraintDistance = fitnessDistance(*frameRateConstraint);
309         if (std::isinf(constraintDistance)) {
310             badConstraint = frameRateConstraint->name();
311             return false;
312         }
313
314         distance = std::min(distance, constraintDistance);
315         if (frameRateConstraint->isMandatory()) {
316             auto range = capabilities.frameRate();
317             frameRate = frameRateConstraint->valueForCapabilityRange(this->frameRate(), range.rangeMin().asDouble, range.rangeMax().asDouble);
318         }
319     }
320
321     // Each of the non-null values is supported individually, see if they all can be applied at the same time.
322     if (!supportsSizeAndFrameRate(WTFMove(width), WTFMove(height), WTFMove(frameRate))) {
323         if (widthConstraint)
324             badConstraint = widthConstraint->name();
325         else if (heightConstraint)
326             badConstraint = heightConstraint->name();
327         else
328             badConstraint = frameRateConstraint->name();
329         return false;
330     }
331
332     return true;
333 }
334
335 double RealtimeMediaSource::fitnessDistance(const MediaConstraint& constraint)
336 {
337     auto& capabilities = this->capabilities();
338
339     switch (constraint.constraintType()) {
340     case MediaConstraintType::Width: {
341         ASSERT(constraint.isInt());
342         if (!capabilities.supportsWidth())
343             return 0;
344
345         auto range = capabilities.width();
346         return downcast<IntConstraint>(constraint).fitnessDistance(range.rangeMin().asInt, range.rangeMax().asInt);
347         break;
348     }
349
350     case MediaConstraintType::Height: {
351         ASSERT(constraint.isInt());
352         if (!capabilities.supportsHeight())
353             return 0;
354
355         auto range = capabilities.height();
356         return downcast<IntConstraint>(constraint).fitnessDistance(range.rangeMin().asInt, range.rangeMax().asInt);
357         break;
358     }
359
360     case MediaConstraintType::FrameRate: {
361         ASSERT(constraint.isDouble());
362         if (!capabilities.supportsFrameRate())
363             return 0;
364
365         auto range = capabilities.frameRate();
366         return downcast<DoubleConstraint>(constraint).fitnessDistance(range.rangeMin().asDouble, range.rangeMax().asDouble);
367         break;
368     }
369
370     case MediaConstraintType::AspectRatio: {
371         ASSERT(constraint.isDouble());
372         if (!capabilities.supportsAspectRatio())
373             return 0;
374
375         auto range = capabilities.aspectRatio();
376         return downcast<DoubleConstraint>(constraint).fitnessDistance(range.rangeMin().asDouble, range.rangeMax().asDouble);
377         break;
378     }
379
380     case MediaConstraintType::Volume: {
381         ASSERT(constraint.isDouble());
382         if (!capabilities.supportsVolume())
383             return 0;
384
385         auto range = capabilities.volume();
386         return downcast<DoubleConstraint>(constraint).fitnessDistance(range.rangeMin().asDouble, range.rangeMax().asDouble);
387         break;
388     }
389
390     case MediaConstraintType::SampleRate: {
391         ASSERT(constraint.isInt());
392         if (!capabilities.supportsSampleRate())
393             return 0;
394
395         if (auto discreteRates = discreteSampleRates())
396             return downcast<IntConstraint>(constraint).fitnessDistance(*discreteRates);
397
398         auto range = capabilities.sampleRate();
399         return downcast<IntConstraint>(constraint).fitnessDistance(range.rangeMin().asInt, range.rangeMax().asInt);
400         break;
401     }
402
403     case MediaConstraintType::SampleSize: {
404         ASSERT(constraint.isInt());
405         if (!capabilities.supportsSampleSize())
406             return 0;
407
408         if (auto discreteSizes = discreteSampleSizes())
409             return downcast<IntConstraint>(constraint).fitnessDistance(*discreteSizes);
410
411         auto range = capabilities.sampleSize();
412         return downcast<IntConstraint>(constraint).fitnessDistance(range.rangeMin().asInt, range.rangeMax().asInt);
413         break;
414     }
415
416     case MediaConstraintType::FacingMode: {
417         ASSERT(constraint.isString());
418         if (!capabilities.supportsFacingMode())
419             return 0;
420
421         auto& modes = capabilities.facingMode();
422         Vector<String> supportedModes;
423         supportedModes.reserveInitialCapacity(modes.size());
424         for (auto& mode : modes)
425             supportedModes.uncheckedAppend(RealtimeMediaSourceSettings::facingMode(mode));
426         return downcast<StringConstraint>(constraint).fitnessDistance(supportedModes);
427         break;
428     }
429
430     case MediaConstraintType::EchoCancellation: {
431         ASSERT(constraint.isBoolean());
432         if (!capabilities.supportsEchoCancellation())
433             return 0;
434
435         bool echoCancellationReadWrite = capabilities.echoCancellation() == RealtimeMediaSourceCapabilities::EchoCancellation::ReadWrite;
436         return downcast<BooleanConstraint>(constraint).fitnessDistance(echoCancellationReadWrite);
437         break;
438     }
439
440     case MediaConstraintType::DeviceId:
441         ASSERT(!m_hashedID.isEmpty());
442         return downcast<StringConstraint>(constraint).fitnessDistance(m_hashedID);
443         break;
444
445     case MediaConstraintType::GroupId: {
446         ASSERT(constraint.isString());
447         if (!capabilities.supportsDeviceId())
448             return 0;
449
450         return downcast<StringConstraint>(constraint).fitnessDistance(settings().groupId());
451         break;
452     }
453
454     case MediaConstraintType::DisplaySurface:
455     case MediaConstraintType::LogicalSurface:
456         break;
457
458     case MediaConstraintType::Unknown:
459         // Unknown (or unsupported) constraints should be ignored.
460         break;
461     }
462
463     return 0;
464 }
465
466 template <typename ValueType>
467 static void applyNumericConstraint(const NumericConstraint<ValueType>& constraint, ValueType current, Optional<Vector<ValueType>> discreteCapabilityValues, ValueType capabilityMin, ValueType capabilityMax, RealtimeMediaSource& source, void (RealtimeMediaSource::*applier)(ValueType))
468 {
469     if (discreteCapabilityValues) {
470         int value = constraint.valueForDiscreteCapabilityValues(current, *discreteCapabilityValues);
471         if (value != current)
472             (source.*applier)(value);
473         return;
474     }
475
476     ValueType value = constraint.valueForCapabilityRange(current, capabilityMin, capabilityMax);
477     if (value != current)
478         (source.*applier)(value);
479 }
480
481 void RealtimeMediaSource::setSizeAndFrameRate(Optional<int> width, Optional<int> height, Optional<double> frameRate)
482 {
483     IntSize size;
484     if (width)
485         size.setWidth(width.value());
486     if (height)
487         size.setHeight(height.value());
488     setSize(size);
489     if (frameRate)
490         setFrameRate(frameRate.value());
491 }
492
493 void RealtimeMediaSource::applyConstraint(const MediaConstraint& constraint)
494 {
495     ALWAYS_LOG_IF(m_logger, LOGIDENTIFIER, constraint.name());
496
497     auto& capabilities = this->capabilities();
498     switch (constraint.constraintType()) {
499     case MediaConstraintType::Width:
500         ASSERT_NOT_REACHED();
501         break;
502
503     case MediaConstraintType::Height:
504         ASSERT_NOT_REACHED();
505         break;
506
507     case MediaConstraintType::FrameRate:
508         ASSERT_NOT_REACHED();
509         break;
510
511     case MediaConstraintType::AspectRatio: {
512         ASSERT(constraint.isDouble());
513         if (!capabilities.supportsAspectRatio())
514             return;
515
516         auto range = capabilities.aspectRatio();
517         applyNumericConstraint(downcast<DoubleConstraint>(constraint), aspectRatio(), { }, range.rangeMin().asDouble, range.rangeMax().asDouble, *this, &RealtimeMediaSource::setAspectRatio);
518         break;
519     }
520
521     case MediaConstraintType::Volume: {
522         ASSERT(constraint.isDouble());
523         if (!capabilities.supportsVolume())
524             return;
525
526         auto range = capabilities.volume();
527         applyNumericConstraint(downcast<DoubleConstraint>(constraint), volume(), { }, range.rangeMin().asDouble, range.rangeMax().asDouble, *this, &RealtimeMediaSource::setVolume);
528         break;
529     }
530
531     case MediaConstraintType::SampleRate: {
532         ASSERT(constraint.isInt());
533         if (!capabilities.supportsSampleRate())
534             return;
535
536         auto range = capabilities.sampleRate();
537         applyNumericConstraint(downcast<IntConstraint>(constraint), sampleRate(), discreteSampleRates(), range.rangeMin().asInt, range.rangeMax().asInt, *this, &RealtimeMediaSource::setSampleRate);
538         break;
539     }
540
541     case MediaConstraintType::SampleSize: {
542         ASSERT(constraint.isInt());
543         if (!capabilities.supportsSampleSize())
544             return;
545
546         auto range = capabilities.sampleSize();
547         applyNumericConstraint(downcast<IntConstraint>(constraint), sampleSize(), { }, range.rangeMin().asInt, range.rangeMax().asInt, *this, &RealtimeMediaSource::setSampleSize);
548         break;
549     }
550
551     case MediaConstraintType::EchoCancellation: {
552         ASSERT(constraint.isBoolean());
553         if (!capabilities.supportsEchoCancellation())
554             return;
555
556         bool setting;
557         const BooleanConstraint& boolConstraint = downcast<BooleanConstraint>(constraint);
558         if (boolConstraint.getExact(setting) || boolConstraint.getIdeal(setting))
559             setEchoCancellation(setting);
560         break;
561     }
562
563     case MediaConstraintType::FacingMode: {
564         ASSERT(constraint.isString());
565         if (!capabilities.supportsFacingMode())
566             return;
567
568         auto& supportedModes = capabilities.facingMode();
569         auto filter = [supportedModes](const String& modeString) {
570             auto mode = RealtimeMediaSourceSettings::videoFacingModeEnum(modeString);
571             for (auto& supportedMode : supportedModes) {
572                 if (mode == supportedMode)
573                     return true;
574             }
575             return false;
576         };
577
578         auto modeString = downcast<StringConstraint>(constraint).find(WTFMove(filter));
579         if (!modeString.isEmpty())
580             setFacingMode(RealtimeMediaSourceSettings::videoFacingModeEnum(modeString));
581         break;
582     }
583
584     case MediaConstraintType::DeviceId:
585     case MediaConstraintType::GroupId:
586         ASSERT(constraint.isString());
587         // There is nothing to do here, neither can be changed.
588         break;
589
590     case MediaConstraintType::DisplaySurface:
591     case MediaConstraintType::LogicalSurface:
592         ASSERT(constraint.isBoolean());
593         break;
594
595     case MediaConstraintType::Unknown:
596         break;
597     }
598 }
599
600 bool RealtimeMediaSource::selectSettings(const MediaConstraints& constraints, FlattenedConstraint& candidates, String& failedConstraint)
601 {
602     double minimumDistance = std::numeric_limits<double>::infinity();
603
604     // https://w3c.github.io/mediacapture-main/#dfn-selectsettings
605     //
606     // 1. Each constraint specifies one or more values (or a range of values) for its property.
607     //    A property may appear more than once in the list of 'advanced' ConstraintSets. If an
608     //    empty object or list has been given as the value for a constraint, it must be interpreted
609     //    as if the constraint were not specified (in other words, an empty constraint == no constraint).
610     //
611     //    Note that unknown properties are discarded by WebIDL, which means that unknown/unsupported required
612     //    constraints will silently disappear. To avoid this being a surprise, application authors are
613     //    expected to first use the getSupportedConstraints() method as shown in the Examples below.
614
615     // 2. Let object be the ConstrainablePattern object on which this algorithm is applied. Let copy be an
616     //    unconstrained copy of object (i.e., copy should behave as if it were object with all ConstraintSets
617     //    removed.)
618
619     // 3. For every possible settings dictionary of copy compute its fitness distance, treating bare values of
620     //    properties as ideal values. Let candidates be the set of settings dictionaries for which the fitness
621     //    distance is finite.
622
623     failedConstraint = emptyString();
624
625     // Check width, height and frame rate jointly, because while they may be supported individually the combination may not be supported.
626     double distance = std::numeric_limits<double>::infinity();
627     if (!supportsSizeAndFrameRate(constraints.mandatoryConstraints.width(), constraints.mandatoryConstraints.height(), constraints.mandatoryConstraints.frameRate(), failedConstraint, minimumDistance))
628         return false;
629
630     constraints.mandatoryConstraints.filter([&](const MediaConstraint& constraint) {
631         if (!supportsConstraint(constraint))
632             return false;
633
634         if (constraint.constraintType() == MediaConstraintType::Width || constraint.constraintType() == MediaConstraintType::Height || constraint.constraintType() == MediaConstraintType::FrameRate) {
635             candidates.set(constraint);
636             return false;
637         }
638
639         double constraintDistance = fitnessDistance(constraint);
640         if (std::isinf(constraintDistance)) {
641             failedConstraint = constraint.name();
642             return true;
643         }
644
645         distance = std::min(distance, constraintDistance);
646         candidates.set(constraint);
647         return false;
648     });
649
650     if (!failedConstraint.isEmpty())
651         return false;
652
653     minimumDistance = distance;
654
655     // 4. If candidates is empty, return undefined as the result of the SelectSettings() algorithm.
656     if (candidates.isEmpty())
657         return true;
658
659     // 5. Iterate over the 'advanced' ConstraintSets in newConstraints in the order in which they were specified.
660     //    For each ConstraintSet:
661
662     // 5.1 compute the fitness distance between it and each settings dictionary in candidates, treating bare
663     //     values of properties as exact.
664     Vector<std::pair<double, MediaTrackConstraintSetMap>> supportedConstraints;
665
666     for (const auto& advancedConstraint : constraints.advancedConstraints) {
667         double constraintDistance = 0;
668         bool supported = false;
669
670         if (advancedConstraint.width() || advancedConstraint.height() || advancedConstraint.frameRate()) {
671             String dummy;
672             if (!supportsSizeAndFrameRate(advancedConstraint.width(), advancedConstraint.height(), advancedConstraint.frameRate(), dummy, constraintDistance))
673                 continue;
674
675             supported = true;
676         }
677
678         advancedConstraint.forEach([&](const MediaConstraint& constraint) {
679
680             if (constraint.constraintType() == MediaConstraintType::Width || constraint.constraintType() == MediaConstraintType::Height || constraint.constraintType() == MediaConstraintType::FrameRate)
681                 return;
682
683             distance = fitnessDistance(constraint);
684             constraintDistance += distance;
685             if (!std::isinf(distance))
686                 supported = true;
687         });
688
689         minimumDistance = std::min(minimumDistance, constraintDistance);
690
691         // 5.2 If the fitness distance is finite for one or more settings dictionaries in candidates, keep those
692         //     settings dictionaries in candidates, discarding others.
693         //     If the fitness distance is infinite for all settings dictionaries in candidates, ignore this ConstraintSet.
694         if (supported)
695             supportedConstraints.append({constraintDistance, advancedConstraint});
696     }
697
698     // 6. Select one settings dictionary from candidates, and return it as the result of the SelectSettings() algorithm.
699     //    The UA should use the one with the smallest fitness distance, as calculated in step 3.
700     if (!supportedConstraints.isEmpty()) {
701         supportedConstraints.removeAllMatching([&](const std::pair<double, MediaTrackConstraintSetMap>& pair) -> bool {
702             return std::isinf(pair.first) || pair.first > minimumDistance;
703         });
704
705         if (!supportedConstraints.isEmpty()) {
706             auto& advancedConstraint = supportedConstraints[0].second;
707             advancedConstraint.forEach([&](const MediaConstraint& constraint) {
708                 candidates.merge(constraint);
709             });
710
711             minimumDistance = std::min(minimumDistance, supportedConstraints[0].first);
712         }
713     }
714
715     return true;
716 }
717
718 bool RealtimeMediaSource::supportsConstraint(const MediaConstraint& constraint)
719 {
720     auto& capabilities = this->capabilities();
721
722     switch (constraint.constraintType()) {
723     case MediaConstraintType::Width:
724         ASSERT(constraint.isInt());
725         return capabilities.supportsWidth();
726         break;
727
728     case MediaConstraintType::Height:
729         ASSERT(constraint.isInt());
730         return capabilities.supportsHeight();
731         break;
732
733     case MediaConstraintType::FrameRate:
734         ASSERT(constraint.isDouble());
735         return capabilities.supportsFrameRate();
736         break;
737
738     case MediaConstraintType::AspectRatio:
739         ASSERT(constraint.isDouble());
740         return capabilities.supportsAspectRatio();
741         break;
742
743     case MediaConstraintType::Volume:
744         ASSERT(constraint.isDouble());
745         return capabilities.supportsVolume();
746         break;
747
748     case MediaConstraintType::SampleRate:
749         ASSERT(constraint.isInt());
750         return capabilities.supportsSampleRate();
751         break;
752
753     case MediaConstraintType::SampleSize:
754         ASSERT(constraint.isInt());
755         return capabilities.supportsSampleSize();
756         break;
757
758     case MediaConstraintType::FacingMode:
759         ASSERT(constraint.isString());
760         return capabilities.supportsFacingMode();
761         break;
762
763     case MediaConstraintType::EchoCancellation:
764         ASSERT(constraint.isBoolean());
765         return capabilities.supportsEchoCancellation();
766         break;
767
768     case MediaConstraintType::DeviceId:
769         ASSERT(constraint.isString());
770         return capabilities.supportsDeviceId();
771         break;
772
773     case MediaConstraintType::GroupId:
774         ASSERT(constraint.isString());
775         return capabilities.supportsDeviceId();
776         break;
777
778     case MediaConstraintType::DisplaySurface:
779     case MediaConstraintType::LogicalSurface:
780         // https://www.w3.org/TR/screen-capture/#new-constraints-for-captured-display-surfaces
781         // 5.2.1 New Constraints for Captured Display Surfaces
782         // Since the source of media cannot be changed after a MediaStreamTrack has been returned,
783         // these constraints cannot be changed by an application.
784         return false;
785         break;
786
787     case MediaConstraintType::Unknown:
788         // Unknown (or unsupported) constraints should be ignored.
789         break;
790     }
791     
792     return false;
793 }
794
795 bool RealtimeMediaSource::supportsConstraints(const MediaConstraints& constraints, String& invalidConstraint)
796 {
797     ASSERT(constraints.isValid);
798
799     ALWAYS_LOG_IF(m_logger, LOGIDENTIFIER);
800
801     FlattenedConstraint candidates;
802     if (!selectSettings(constraints, candidates, invalidConstraint))
803         return false;
804
805     m_fitnessScore = 0;
806     for (auto& variant : candidates) {
807         double distance = fitnessDistance(variant);
808         switch (variant.constraintType()) {
809         case MediaConstraintType::DeviceId:
810         case MediaConstraintType::FacingMode:
811             m_fitnessScore += distance ? 1 : 32;
812             break;
813
814         case MediaConstraintType::Width:
815         case MediaConstraintType::Height:
816         case MediaConstraintType::FrameRate:
817         case MediaConstraintType::AspectRatio:
818         case MediaConstraintType::Volume:
819         case MediaConstraintType::SampleRate:
820         case MediaConstraintType::SampleSize:
821         case MediaConstraintType::EchoCancellation:
822         case MediaConstraintType::GroupId:
823         case MediaConstraintType::DisplaySurface:
824         case MediaConstraintType::LogicalSurface:
825         case MediaConstraintType::Unknown:
826             m_fitnessScore += distance ? 1 : 2;
827             break;
828         }
829     }
830
831     ALWAYS_LOG_IF(m_logger, LOGIDENTIFIER, "fitness distance : ", m_fitnessScore);
832
833     return true;
834 }
835
836 void RealtimeMediaSource::applyConstraints(const FlattenedConstraint& constraints)
837 {
838     if (constraints.isEmpty())
839         return;
840
841     beginConfiguration();
842
843     auto& capabilities = this->capabilities();
844
845     Optional<int> width;
846     if (const MediaConstraint* constraint = constraints.find(MediaConstraintType::Width)) {
847         ASSERT(constraint->isInt());
848         if (capabilities.supportsWidth()) {
849             auto range = capabilities.width();
850             width = downcast<IntConstraint>(*constraint).valueForCapabilityRange(size().width(), range.rangeMin().asInt, range.rangeMax().asInt);
851         }
852     }
853
854     Optional<int> height;
855     if (const MediaConstraint* constraint = constraints.find(MediaConstraintType::Height)) {
856         ASSERT(constraint->isInt());
857         if (capabilities.supportsHeight()) {
858             auto range = capabilities.height();
859             height = downcast<IntConstraint>(*constraint).valueForCapabilityRange(size().height(), range.rangeMin().asInt, range.rangeMax().asInt);
860         }
861     }
862
863     Optional<double> frameRate;
864     if (const MediaConstraint* constraint = constraints.find(MediaConstraintType::FrameRate)) {
865         ASSERT(constraint->isDouble());
866         if (capabilities.supportsFrameRate()) {
867             auto range = capabilities.frameRate();
868             frameRate = downcast<DoubleConstraint>(*constraint).valueForCapabilityRange(this->frameRate(), range.rangeMin().asDouble, range.rangeMax().asDouble);
869         }
870     }
871
872     if (width || height || frameRate)
873         setSizeAndFrameRate(WTFMove(width), WTFMove(height), WTFMove(frameRate));
874
875     for (auto& variant : constraints) {
876         if (variant.constraintType() == MediaConstraintType::Width || variant.constraintType() == MediaConstraintType::Height || variant.constraintType() == MediaConstraintType::FrameRate)
877             continue;
878
879         applyConstraint(variant);
880     }
881
882     commitConfiguration();
883 }
884
885 Optional<RealtimeMediaSource::ApplyConstraintsError> RealtimeMediaSource::applyConstraints(const MediaConstraints& constraints)
886 {
887     ASSERT(constraints.isValid);
888
889     ALWAYS_LOG_IF(m_logger, LOGIDENTIFIER);
890
891     FlattenedConstraint candidates;
892     String failedConstraint;
893     if (!selectSettings(constraints, candidates, failedConstraint))
894         return ApplyConstraintsError { failedConstraint, "Constraint not supported"_s };
895
896     applyConstraints(candidates);
897     return { };
898 }
899
900 void RealtimeMediaSource::applyConstraints(const MediaConstraints& constraints, ApplyConstraintsHandler&& completionHandler)
901 {
902     completionHandler(applyConstraints(constraints));
903 }
904
905 void RealtimeMediaSource::setSize(const IntSize& size)
906 {
907     if (size == m_size)
908         return;
909
910     ALWAYS_LOG_IF(m_logger, LOGIDENTIFIER, size);
911     
912     m_size = size;
913     notifySettingsDidChangeObservers({ RealtimeMediaSourceSettings::Flag::Width, RealtimeMediaSourceSettings::Flag::Height });
914 }
915
916 const IntSize RealtimeMediaSource::size() const
917 {
918     auto size = m_size;
919
920     if (size.isEmpty() && !m_intrinsicSize.isEmpty()) {
921         if (size.isZero())
922             size = m_intrinsicSize;
923         else if (size.width())
924             size.setHeight(size.width() * (m_intrinsicSize.height() / static_cast<double>(m_intrinsicSize.width())));
925         else if (size.height())
926             size.setWidth(size.height() * (m_intrinsicSize.width() / static_cast<double>(m_intrinsicSize.height())));
927     }
928
929     return size;
930 }
931
932 void RealtimeMediaSource::setIntrinsicSize(const IntSize& size)
933 {
934     if (m_intrinsicSize == size)
935         return;
936
937     ALWAYS_LOG_IF(m_logger, LOGIDENTIFIER, size);
938     
939     auto currentSize = this->size();
940     m_intrinsicSize = size;
941
942     if (currentSize != this->size())
943         notifySettingsDidChangeObservers({ RealtimeMediaSourceSettings::Flag::Width, RealtimeMediaSourceSettings::Flag::Height });
944 }
945
946 const IntSize RealtimeMediaSource::intrinsicSize() const
947 {
948     return m_intrinsicSize;
949 }
950
951 void RealtimeMediaSource::setFrameRate(double rate)
952 {
953     if (m_frameRate == rate)
954         return;
955
956     ALWAYS_LOG_IF(m_logger, LOGIDENTIFIER, rate);
957     
958     m_frameRate = rate;
959     notifySettingsDidChangeObservers(RealtimeMediaSourceSettings::Flag::FrameRate);
960 }
961
962 void RealtimeMediaSource::setAspectRatio(double ratio)
963 {
964     if (m_aspectRatio == ratio)
965         return;
966
967     ALWAYS_LOG_IF(m_logger, LOGIDENTIFIER, ratio);
968     
969     m_aspectRatio = ratio;
970     m_size.setHeight(m_size.width() / ratio);
971     notifySettingsDidChangeObservers({ RealtimeMediaSourceSettings::Flag::AspectRatio, RealtimeMediaSourceSettings::Flag::Height });
972 }
973
974 void RealtimeMediaSource::setFacingMode(RealtimeMediaSourceSettings::VideoFacingMode mode)
975 {
976     if (m_facingMode == mode)
977         return;
978
979     ALWAYS_LOG_IF(m_logger, LOGIDENTIFIER, mode);
980
981     m_facingMode = mode;
982     notifySettingsDidChangeObservers(RealtimeMediaSourceSettings::Flag::FacingMode);
983 }
984
985 void RealtimeMediaSource::setVolume(double volume)
986 {
987     if (m_volume == volume)
988         return;
989
990     ALWAYS_LOG_IF(m_logger, LOGIDENTIFIER, volume);
991
992     m_volume = volume;
993     notifySettingsDidChangeObservers(RealtimeMediaSourceSettings::Flag::Volume);
994 }
995
996 void RealtimeMediaSource::setSampleRate(int rate)
997 {
998     if (m_sampleRate == rate)
999         return;
1000
1001     ALWAYS_LOG_IF(m_logger, LOGIDENTIFIER, rate);
1002
1003     m_sampleRate = rate;
1004     notifySettingsDidChangeObservers(RealtimeMediaSourceSettings::Flag::SampleRate);
1005 }
1006
1007 Optional<Vector<int>> RealtimeMediaSource::discreteSampleRates() const
1008 {
1009     return WTF::nullopt;
1010 }
1011
1012 void RealtimeMediaSource::setSampleSize(int size)
1013 {
1014     if (m_sampleSize == size)
1015         return;
1016
1017     ALWAYS_LOG_IF(m_logger, LOGIDENTIFIER, size);
1018
1019     m_sampleSize = size;
1020     notifySettingsDidChangeObservers(RealtimeMediaSourceSettings::Flag::SampleSize);
1021 }
1022
1023 Optional<Vector<int>> RealtimeMediaSource::discreteSampleSizes() const
1024 {
1025     return WTF::nullopt;
1026 }
1027
1028 void RealtimeMediaSource::setEchoCancellation(bool echoCancellation)
1029 {
1030     if (m_echoCancellation == echoCancellation)
1031         return;
1032
1033     ALWAYS_LOG_IF(m_logger, LOGIDENTIFIER, echoCancellation);
1034     m_echoCancellation = echoCancellation;
1035     notifySettingsDidChangeObservers(RealtimeMediaSourceSettings::Flag::EchoCancellation);
1036 }
1037
1038 void RealtimeMediaSource::scheduleDeferredTask(WTF::Function<void()>&& function)
1039 {
1040     ASSERT(function);
1041     callOnMainThread([weakThis = makeWeakPtr(*this), function = WTFMove(function)] {
1042         if (!weakThis)
1043             return;
1044
1045         function();
1046     });
1047 }
1048
1049 const String& RealtimeMediaSource::hashedId() const
1050 {
1051     ASSERT(!m_hashedID.isEmpty());
1052     return m_hashedID;
1053 }
1054
1055 String RealtimeMediaSource::deviceIDHashSalt() const
1056 {
1057     return m_idHashSalt;
1058 }
1059
1060 RealtimeMediaSource::Observer::~Observer()
1061 {
1062 }
1063
1064 #if !RELEASE_LOG_DISABLED
1065 void RealtimeMediaSource::setLogger(const Logger& newLogger, const void* newLogIdentifier)
1066 {
1067     m_logger = &newLogger;
1068     m_logIdentifier = newLogIdentifier;
1069     ALWAYS_LOG(LOGIDENTIFIER, m_type, ", ", m_name, ", ", m_hashedID);
1070 }
1071
1072 WTFLogChannel& RealtimeMediaSource::logChannel() const
1073 {
1074     return LogWebRTC;
1075 }
1076 #endif
1077
1078 String convertEnumerationToString(RealtimeMediaSource::Type enumerationValue)
1079 {
1080     static const NeverDestroyed<String> values[] = {
1081         MAKE_STATIC_STRING_IMPL("None"),
1082         MAKE_STATIC_STRING_IMPL("Audio"),
1083         MAKE_STATIC_STRING_IMPL("Video"),
1084     };
1085     static_assert(static_cast<size_t>(RealtimeMediaSource::Type::None) == 0, "RealtimeMediaSource::Type::None is not 0 as expected");
1086     static_assert(static_cast<size_t>(RealtimeMediaSource::Type::Audio) == 1, "RealtimeMediaSource::Type::Audio is not 1 as expected");
1087     static_assert(static_cast<size_t>(RealtimeMediaSource::Type::Video) == 2, "RealtimeMediaSource::Type::Video is not 2 as expected");
1088     ASSERT(static_cast<size_t>(enumerationValue) < WTF_ARRAY_LENGTH(values));
1089     return values[static_cast<size_t>(enumerationValue)];
1090 }
1091
1092 } // namespace WebCore
1093
1094 #endif // ENABLE(MEDIA_STREAM)