Add media stream release logging
[WebKit-https.git] / Source / WebCore / Modules / mediastream / MediaStreamTrack.cpp
1 /*
2  * Copyright (C) 2011 Google Inc. All rights reserved.
3  * Copyright (C) 2011, 2015 Ericsson AB. All rights reserved.
4  * Copyright (C) 2013-2019 Apple Inc. All rights reserved.
5  * Copyright (C) 2013 Nokia Corporation and/or its subsidiary(-ies).
6  *
7  * Redistribution and use in source and binary forms, with or without
8  * modification, are permitted provided that the following conditions
9  * are met:
10  * 1.  Redistributions of source code must retain the above copyright
11  *     notice, this list of conditions and the following disclaimer.
12  * 2.  Redistributions in binary form must reproduce the above copyright
13  *     notice, this list of conditions and the following disclaimer in the
14  *     documentation and/or other materials provided with the distribution.
15  *
16  * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' AND ANY
17  * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
18  * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
19  * DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS BE LIABLE FOR ANY
20  * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
21  * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
22  * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
23  * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
24  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
25  * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
26  */
27
28 #include "config.h"
29 #include "MediaStreamTrack.h"
30
31 #if ENABLE(MEDIA_STREAM)
32
33 #include "Document.h"
34 #include "Event.h"
35 #include "EventNames.h"
36 #include "JSOverconstrainedError.h"
37 #include "MediaConstraints.h"
38 #include "MediaStream.h"
39 #include "MediaStreamPrivate.h"
40 #include "NotImplemented.h"
41 #include "OverconstrainedError.h"
42 #include "Page.h"
43 #include "RealtimeMediaSourceCenter.h"
44 #include "ScriptExecutionContext.h"
45 #include <wtf/NeverDestroyed.h>
46
47 namespace WebCore {
48
49 Ref<MediaStreamTrack> MediaStreamTrack::create(ScriptExecutionContext& context, Ref<MediaStreamTrackPrivate>&& privateTrack)
50 {
51     return adoptRef(*new MediaStreamTrack(context, WTFMove(privateTrack)));
52 }
53
54 MediaStreamTrack::MediaStreamTrack(ScriptExecutionContext& context, Ref<MediaStreamTrackPrivate>&& privateTrack)
55     : ActiveDOMObject(&context)
56     , m_private(WTFMove(privateTrack))
57 #if !RELEASE_LOG_DISABLED
58     , m_logger(document()->logger())
59     , m_logIdentifier(uniqueLogIdentifier())
60 #endif
61     , m_taskQueue(context)
62 {
63     ALWAYS_LOG(LOGIDENTIFIER);
64     suspendIfNeeded();
65
66 #if !RELEASE_LOG_DISABLED
67     m_private->setLogger(logger(), logIdentifier());
68 #endif
69     m_private->addObserver(*this);
70
71     if (auto document = this->document())
72         document->addAudioProducer(*this);
73 }
74
75 MediaStreamTrack::~MediaStreamTrack()
76 {
77     m_private->removeObserver(*this);
78
79     if (auto document = this->document())
80         document->removeAudioProducer(*this);
81 }
82
83 const AtomicString& MediaStreamTrack::kind() const
84 {
85     static NeverDestroyed<AtomicString> audioKind("audio", AtomicString::ConstructFromLiteral);
86     static NeverDestroyed<AtomicString> videoKind("video", AtomicString::ConstructFromLiteral);
87
88     if (m_private->type() == RealtimeMediaSource::Type::Audio)
89         return audioKind;
90     return videoKind;
91 }
92
93 const String& MediaStreamTrack::id() const
94 {
95     return m_private->id();
96 }
97
98 const String& MediaStreamTrack::label() const
99 {
100     return m_private->label();
101 }
102
103 const AtomicString& MediaStreamTrack::contentHint() const
104 {
105     static NeverDestroyed<const AtomicString> speechHint("speech", AtomicString::ConstructFromLiteral);
106     static NeverDestroyed<const AtomicString> musicHint("music", AtomicString::ConstructFromLiteral);
107     static NeverDestroyed<const AtomicString> detailHint("detail", AtomicString::ConstructFromLiteral);
108     static NeverDestroyed<const AtomicString> textHint("text", AtomicString::ConstructFromLiteral);
109     static NeverDestroyed<const AtomicString> motionHint("motion", AtomicString::ConstructFromLiteral);
110
111     switch (m_private->contentHint()) {
112     case MediaStreamTrackPrivate::HintValue::Empty:
113         return emptyAtom();
114     case MediaStreamTrackPrivate::HintValue::Speech:
115         return speechHint;
116     case MediaStreamTrackPrivate::HintValue::Music:
117         return musicHint;
118     case MediaStreamTrackPrivate::HintValue::Motion:
119         return motionHint;
120     case MediaStreamTrackPrivate::HintValue::Detail:
121         return detailHint;
122     case MediaStreamTrackPrivate::HintValue::Text:
123         return textHint;
124     default:
125         return emptyAtom();
126     }
127 }
128
129 void MediaStreamTrack::setContentHint(const String& hintValue)
130 {
131     MediaStreamTrackPrivate::HintValue value;
132     if (m_private->type() == RealtimeMediaSource::Type::Audio) {
133         if (hintValue == "")
134             value = MediaStreamTrackPrivate::HintValue::Empty;
135         else if (hintValue == "speech")
136             value = MediaStreamTrackPrivate::HintValue::Speech;
137         else if (hintValue == "music")
138             value = MediaStreamTrackPrivate::HintValue::Music;
139         else
140             return;
141     } else {
142         if (hintValue == "")
143             value = MediaStreamTrackPrivate::HintValue::Empty;
144         else if (hintValue == "detail")
145             value = MediaStreamTrackPrivate::HintValue::Detail;
146         else if (hintValue == "motion")
147             value = MediaStreamTrackPrivate::HintValue::Motion;
148         else if (hintValue == "text")
149             value = MediaStreamTrackPrivate::HintValue::Text;
150         else
151             return;
152     }
153     m_private->setContentHint(value);
154 }
155
156 bool MediaStreamTrack::enabled() const
157 {
158     return m_private->enabled();
159 }
160
161 void MediaStreamTrack::setEnabled(bool enabled)
162 {
163     m_private->setEnabled(enabled);
164 }
165
166 bool MediaStreamTrack::muted() const
167 {
168     return m_private->muted();
169 }
170
171 auto MediaStreamTrack::readyState() const -> State
172 {
173     return ended() ? State::Ended : State::Live;
174 }
175
176 bool MediaStreamTrack::ended() const
177 {
178     return m_ended || m_private->ended();
179 }
180
181 RefPtr<MediaStreamTrack> MediaStreamTrack::clone()
182 {
183     if (!scriptExecutionContext())
184         return nullptr;
185
186     return MediaStreamTrack::create(*scriptExecutionContext(), m_private->clone());
187 }
188
189 void MediaStreamTrack::stopTrack(StopMode mode)
190 {
191     // NOTE: this method is called when the "stop" method is called from JS, using the "ImplementedAs" IDL attribute.
192     // This is done because ActiveDOMObject requires a "stop" method.
193
194     if (ended())
195         return;
196
197     // An 'ended' event is not posted if m_ended is true when trackEnded is called, so set it now if we are
198     // not supposed to post the event.
199     if (mode == StopMode::Silently)
200         m_ended = true;
201
202     m_private->endTrack();
203     m_ended = true;
204
205     configureTrackRendering();
206 }
207
208 MediaStreamTrack::TrackSettings MediaStreamTrack::getSettings() const
209 {
210     auto& settings = m_private->settings();
211     TrackSettings result;
212     if (settings.supportsWidth())
213         result.width = settings.width();
214     if (settings.supportsHeight())
215         result.height = settings.height();
216     if (settings.supportsAspectRatio() && settings.aspectRatio()) // FIXME: Why the check for zero here?
217         result.aspectRatio = settings.aspectRatio();
218     if (settings.supportsFrameRate())
219         result.frameRate = settings.frameRate();
220     if (settings.supportsFacingMode())
221         result.facingMode = RealtimeMediaSourceSettings::facingMode(settings.facingMode());
222     if (settings.supportsVolume())
223         result.volume = settings.volume();
224     if (settings.supportsSampleRate())
225         result.sampleRate = settings.sampleRate();
226     if (settings.supportsSampleSize())
227         result.sampleSize = settings.sampleSize();
228     if (settings.supportsEchoCancellation())
229         result.echoCancellation = settings.echoCancellation();
230     if (settings.supportsDeviceId())
231         result.deviceId = settings.deviceId();
232     if (settings.supportsGroupId())
233         result.groupId = settings.groupId();
234
235     // FIXME: shouldn't this include displaySurface and logicalSurface?
236
237     return result;
238 }
239
240 static DoubleRange capabilityDoubleRange(const CapabilityValueOrRange& value)
241 {
242     DoubleRange range;
243     switch (value.type()) {
244     case CapabilityValueOrRange::Double:
245         range.min = value.value().asDouble;
246         range.max = range.min;
247         break;
248     case CapabilityValueOrRange::DoubleRange:
249         range.min = value.rangeMin().asDouble;
250         range.max = value.rangeMax().asDouble;
251         break;
252     case CapabilityValueOrRange::Undefined:
253     case CapabilityValueOrRange::ULong:
254     case CapabilityValueOrRange::ULongRange:
255         ASSERT_NOT_REACHED();
256     }
257     return range;
258 }
259
260 static LongRange capabilityIntRange(const CapabilityValueOrRange& value)
261 {
262     LongRange range;
263     switch (value.type()) {
264     case CapabilityValueOrRange::ULong:
265         range.min = value.value().asInt;
266         range.max = range.min;
267         break;
268     case CapabilityValueOrRange::ULongRange:
269         range.min = value.rangeMin().asInt;
270         range.max = value.rangeMax().asInt;
271         break;
272     case CapabilityValueOrRange::Undefined:
273     case CapabilityValueOrRange::Double:
274     case CapabilityValueOrRange::DoubleRange:
275         ASSERT_NOT_REACHED();
276     }
277     return range;
278 }
279
280 static Vector<String> capabilityStringVector(const Vector<RealtimeMediaSourceSettings::VideoFacingMode>& modes)
281 {
282     Vector<String> result;
283     result.reserveCapacity(modes.size());
284     for (auto& mode : modes)
285         result.uncheckedAppend(RealtimeMediaSourceSettings::facingMode(mode));
286     return result;
287 }
288
289 static Vector<bool> capabilityBooleanVector(RealtimeMediaSourceCapabilities::EchoCancellation cancellation)
290 {
291     Vector<bool> result;
292     result.reserveCapacity(2);
293     result.uncheckedAppend(true);
294     result.uncheckedAppend(cancellation == RealtimeMediaSourceCapabilities::EchoCancellation::ReadWrite);
295     return result;
296 }
297
298 MediaStreamTrack::TrackCapabilities MediaStreamTrack::getCapabilities() const
299 {
300     auto capabilities = m_private->capabilities();
301     TrackCapabilities result;
302     if (capabilities.supportsWidth())
303         result.width = capabilityIntRange(capabilities.width());
304     if (capabilities.supportsHeight())
305         result.height = capabilityIntRange(capabilities.height());
306     if (capabilities.supportsAspectRatio())
307         result.aspectRatio = capabilityDoubleRange(capabilities.aspectRatio());
308     if (capabilities.supportsFrameRate())
309         result.frameRate = capabilityDoubleRange(capabilities.frameRate());
310     if (capabilities.supportsFacingMode())
311         result.facingMode = capabilityStringVector(capabilities.facingMode());
312     if (capabilities.supportsVolume())
313         result.volume = capabilityDoubleRange(capabilities.volume());
314     if (capabilities.supportsSampleRate())
315         result.sampleRate = capabilityIntRange(capabilities.sampleRate());
316     if (capabilities.supportsSampleSize())
317         result.sampleSize = capabilityIntRange(capabilities.sampleSize());
318     if (capabilities.supportsEchoCancellation())
319         result.echoCancellation = capabilityBooleanVector(capabilities.echoCancellation());
320     if (capabilities.supportsDeviceId())
321         result.deviceId = capabilities.deviceId();
322     if (capabilities.supportsGroupId())
323         result.groupId = capabilities.groupId();
324     return result;
325 }
326
327 static MediaConstraints createMediaConstraints(const Optional<MediaTrackConstraints>& constraints)
328 {
329     if (!constraints) {
330         MediaConstraints validConstraints;
331         validConstraints.isValid = true;
332         return validConstraints;
333     }
334     return createMediaConstraints(constraints.value());
335 }
336
337 void MediaStreamTrack::applyConstraints(const Optional<MediaTrackConstraints>& constraints, DOMPromiseDeferred<void>&& promise)
338 {
339     m_promise = WTFMove(promise);
340
341     auto completionHandler = [this, weakThis = makeWeakPtr(*this), constraints](auto&& error) mutable {
342         if (!weakThis || !m_promise)
343             return;
344         if (error) {
345             m_promise->rejectType<IDLInterface<OverconstrainedError>>(OverconstrainedError::create(WTFMove(error->badConstraint), WTFMove(error->message)));
346             return;
347         }
348         m_promise->resolve();
349         m_constraints = constraints.valueOr(MediaTrackConstraints { });
350     };
351     m_private->applyConstraints(createMediaConstraints(constraints), WTFMove(completionHandler));
352 }
353
354 void MediaStreamTrack::addObserver(Observer& observer)
355 {
356     m_observers.append(&observer);
357 }
358
359 void MediaStreamTrack::removeObserver(Observer& observer)
360 {
361     m_observers.removeFirst(&observer);
362 }
363
364 void MediaStreamTrack::pageMutedStateDidChange()
365 {
366     if (m_ended || !isCaptureTrack())
367         return;
368
369     Document* document = this->document();
370     if (!document || !document->page())
371         return;
372
373     m_private->setMuted(document->page()->isMediaCaptureMuted());
374 }
375
376 MediaProducer::MediaStateFlags MediaStreamTrack::mediaState() const
377 {
378     if (m_ended || !isCaptureTrack())
379         return IsNotPlaying;
380
381     Document* document = this->document();
382     if (!document || !document->page())
383         return IsNotPlaying;
384
385     bool pageCaptureMuted = document->page()->isMediaCaptureMuted();
386
387     if (source().type() == RealtimeMediaSource::Type::Audio) {
388         if (source().interrupted() && !pageCaptureMuted)
389             return HasInterruptedAudioCaptureDevice;
390         if (muted())
391             return HasMutedAudioCaptureDevice;
392         if (m_private->isProducingData())
393             return HasActiveAudioCaptureDevice;
394     } else {
395         auto deviceType = source().deviceType();
396         ASSERT(deviceType == CaptureDevice::DeviceType::Camera || deviceType == CaptureDevice::DeviceType::Screen || deviceType == CaptureDevice::DeviceType::Window);
397         if (source().interrupted() && !pageCaptureMuted)
398             return deviceType == CaptureDevice::DeviceType::Camera ? HasInterruptedVideoCaptureDevice : HasInterruptedDisplayCaptureDevice;
399         if (muted())
400             return deviceType == CaptureDevice::DeviceType::Camera ? HasMutedVideoCaptureDevice : HasMutedDisplayCaptureDevice;
401         if (m_private->isProducingData())
402             return deviceType == CaptureDevice::DeviceType::Camera ? HasActiveVideoCaptureDevice : HasActiveDisplayCaptureDevice;
403     }
404
405     return IsNotPlaying;
406 }
407
408 void MediaStreamTrack::trackStarted(MediaStreamTrackPrivate&)
409 {
410     configureTrackRendering();
411 }
412
413 void MediaStreamTrack::trackEnded(MediaStreamTrackPrivate&)
414 {
415     // http://w3c.github.io/mediacapture-main/#life-cycle
416     // When a MediaStreamTrack track ends for any reason other than the stop() method being invoked, the User Agent must queue a task that runs the following steps:
417     // 1. If the track's readyState attribute has the value ended already, then abort these steps.
418     if (m_ended)
419         return;
420
421     // 2. Set track's readyState attribute to ended.
422     m_ended = true;
423
424     if (scriptExecutionContext()->activeDOMObjectsAreSuspended() || scriptExecutionContext()->activeDOMObjectsAreStopped())
425         return;
426
427     // 3. Notify track's source that track is ended so that the source may be stopped, unless other MediaStreamTrack objects depend on it.
428     // 4. Fire a simple event named ended at the object.
429     dispatchEvent(Event::create(eventNames().endedEvent, Event::CanBubble::No, Event::IsCancelable::No));
430
431     for (auto& observer : m_observers)
432         observer->trackDidEnd();
433
434     configureTrackRendering();
435 }
436     
437 void MediaStreamTrack::trackMutedChanged(MediaStreamTrackPrivate&)
438 {
439     if (scriptExecutionContext()->activeDOMObjectsAreSuspended() || scriptExecutionContext()->activeDOMObjectsAreStopped() || m_ended)
440         return;
441
442     AtomicString eventType = muted() ? eventNames().muteEvent : eventNames().unmuteEvent;
443     dispatchEvent(Event::create(eventType, Event::CanBubble::No, Event::IsCancelable::No));
444
445     configureTrackRendering();
446 }
447
448 void MediaStreamTrack::trackSettingsChanged(MediaStreamTrackPrivate&)
449 {
450     configureTrackRendering();
451 }
452
453 void MediaStreamTrack::trackEnabledChanged(MediaStreamTrackPrivate&)
454 {
455     configureTrackRendering();
456 }
457
458 void MediaStreamTrack::configureTrackRendering()
459 {
460     m_taskQueue.enqueueTask([this] {
461         if (auto document = this->document())
462             document->updateIsPlayingMedia();
463     });
464
465     // 4.3.1
466     // ... media from the source only flows when a MediaStreamTrack object is both unmuted and enabled
467 }
468
469 void MediaStreamTrack::stop()
470 {
471     stopTrack();
472     m_taskQueue.close();
473 }
474
475 const char* MediaStreamTrack::activeDOMObjectName() const
476 {
477     return "MediaStreamTrack";
478 }
479
480 bool MediaStreamTrack::canSuspendForDocumentSuspension() const
481 {
482     return !hasPendingActivity();
483 }
484
485 bool MediaStreamTrack::hasPendingActivity() const
486 {
487     return !m_ended;
488 }
489
490 AudioSourceProvider* MediaStreamTrack::audioSourceProvider()
491 {
492     return m_private->audioSourceProvider();
493 }
494
495 Document* MediaStreamTrack::document() const
496 {
497     return downcast<Document>(scriptExecutionContext());
498 }
499
500 #if !RELEASE_LOG_DISABLED
501 WTFLogChannel& MediaStreamTrack::logChannel() const
502 {
503     return LogWebRTC;
504 }
505 #endif
506
507 } // namespace WebCore
508
509 #endif // ENABLE(MEDIA_STREAM)