Improve use of NeverDestroyed
[WebKit-https.git] / Source / WebCore / platform / graphics / avfoundation / MediaPlayerPrivateAVFoundation.cpp
1 /*
2  * Copyright (C) 2011-2014 Apple Inc. All rights reserved.
3  *
4  * Redistribution and use in source and binary forms, with or without
5  * modification, are permitted provided that the following conditions
6  * are met:
7  * 1. Redistributions of source code must retain the above copyright
8  *    notice, this list of conditions and the following disclaimer.
9  * 2. Redistributions in binary form must reproduce the above copyright
10  *    notice, this list of conditions and the following disclaimer in the
11  *    documentation and/or other materials provided with the distribution.
12  *
13  * THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY
14  * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
15  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
16  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL APPLE INC. OR
17  * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
18  * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
19  * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
20  * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
21  * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
22  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
23  * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 
24  */
25
26 #include "config.h"
27
28 #if ENABLE(VIDEO) && USE(AVFOUNDATION)
29
30 #include "MediaPlayerPrivateAVFoundation.h"
31
32 #include "DocumentLoader.h"
33 #include "FloatConversion.h"
34 #include "GraphicsContext.h"
35 #include "InbandTextTrackPrivateAVF.h"
36 #include "InbandTextTrackPrivateClient.h"
37 #include "Logging.h"
38 #include "PlatformLayer.h"
39 #include "PlatformTimeRanges.h"
40 #include "Settings.h"
41 #include "URL.h"
42 #include <CoreMedia/CoreMedia.h>
43 #include <heap/HeapInlines.h>
44 #include <runtime/DataView.h>
45 #include <runtime/TypedArrayInlines.h>
46 #include <runtime/Uint16Array.h>
47 #include <wtf/MainThread.h>
48 #include <wtf/NeverDestroyed.h>
49 #include <wtf/SoftLinking.h>
50 #include <wtf/StringPrintStream.h>
51 #include <wtf/text/CString.h>
52
53 namespace WebCore {
54
55 MediaPlayerPrivateAVFoundation::MediaPlayerPrivateAVFoundation(MediaPlayer* player)
56     : m_player(player)
57     , m_weakPtrFactory(this)
58     , m_queuedNotifications()
59     , m_queueMutex()
60     , m_networkState(MediaPlayer::Empty)
61     , m_readyState(MediaPlayer::HaveNothing)
62     , m_preload(MediaPlayer::Auto)
63     , m_cachedDuration(MediaTime::invalidTime())
64     , m_reportedDuration(MediaTime::invalidTime())
65     , m_maxTimeLoadedAtLastDidLoadingProgress(MediaTime::invalidTime())
66     , m_delayCallbacks(0)
67     , m_delayCharacteristicsChangedNotification(0)
68     , m_mainThreadCallPending(false)
69     , m_assetIsPlayable(false)
70     , m_visible(false)
71     , m_loadingMetadata(false)
72     , m_isAllowedToRender(false)
73     , m_cachedHasAudio(false)
74     , m_cachedHasVideo(false)
75     , m_cachedHasCaptions(false)
76     , m_ignoreLoadStateChanges(false)
77     , m_haveReportedFirstVideoFrame(false)
78     , m_playWhenFramesAvailable(false)
79     , m_inbandTrackConfigurationPending(false)
80     , m_characteristicsChanged(false)
81     , m_shouldMaintainAspectRatio(true)
82     , m_seeking(false)
83 {
84     LOG(Media, "MediaPlayerPrivateAVFoundation::MediaPlayerPrivateAVFoundation(%p)", this);
85 }
86
87 MediaPlayerPrivateAVFoundation::~MediaPlayerPrivateAVFoundation()
88 {
89     LOG(Media, "MediaPlayerPrivateAVFoundation::~MediaPlayerPrivateAVFoundation(%p)", this);
90     setIgnoreLoadStateChanges(true);
91 }
92
93 MediaPlayerPrivateAVFoundation::MediaRenderingMode MediaPlayerPrivateAVFoundation::currentRenderingMode() const
94 {
95     if (platformLayer())
96         return MediaRenderingToLayer;
97
98     if (hasContextRenderer())
99         return MediaRenderingToContext;
100
101     return MediaRenderingNone;
102 }
103
104 MediaPlayerPrivateAVFoundation::MediaRenderingMode MediaPlayerPrivateAVFoundation::preferredRenderingMode() const
105 {
106     if (!m_player->visible() || assetStatus() == MediaPlayerAVAssetStatusUnknown)
107         return MediaRenderingNone;
108
109     if (supportsAcceleratedRendering() && m_player->client().mediaPlayerRenderingCanBeAccelerated(m_player))
110         return MediaRenderingToLayer;
111
112     return MediaRenderingToContext;
113 }
114
115 void MediaPlayerPrivateAVFoundation::setUpVideoRendering()
116 {
117     if (!isReadyForVideoSetup())
118         return;
119
120     MediaRenderingMode currentMode = currentRenderingMode();
121     MediaRenderingMode preferredMode = preferredRenderingMode();
122
123     if (preferredMode == MediaRenderingNone)
124         preferredMode = MediaRenderingToContext;
125
126     if (currentMode == preferredMode && currentMode != MediaRenderingNone)
127         return;
128
129     LOG(Media, "MediaPlayerPrivateAVFoundation::setUpVideoRendering(%p) - current mode = %d, preferred mode = %d", 
130         this, static_cast<int>(currentMode), static_cast<int>(preferredMode));
131
132     if (currentMode != MediaRenderingNone)  
133         tearDownVideoRendering();
134
135     switch (preferredMode) {
136     case MediaRenderingNone:
137     case MediaRenderingToContext:
138         createContextVideoRenderer();
139         break;
140
141     case MediaRenderingToLayer:
142         createVideoLayer();
143         break;
144     }
145
146     // If using a movie layer, inform the client so the compositing tree is updated.
147     if (currentMode == MediaRenderingToLayer || preferredMode == MediaRenderingToLayer) {
148         LOG(Media, "MediaPlayerPrivateAVFoundation::setUpVideoRendering(%p) - calling mediaPlayerRenderingModeChanged()", this);
149         m_player->client().mediaPlayerRenderingModeChanged(m_player);
150     }
151 }
152
153 void MediaPlayerPrivateAVFoundation::tearDownVideoRendering()
154 {
155     LOG(Media, "MediaPlayerPrivateAVFoundation::tearDownVideoRendering(%p)", this);
156
157     destroyContextVideoRenderer();
158
159     if (platformLayer())
160         destroyVideoLayer();
161 }
162
163 bool MediaPlayerPrivateAVFoundation::hasSetUpVideoRendering() const
164 {
165     return hasLayerRenderer() || hasContextRenderer();
166 }
167
168 void MediaPlayerPrivateAVFoundation::load(const String& url)
169 {
170     LOG(Media, "MediaPlayerPrivateAVFoundation::load(%p)", this);
171
172     setNetworkState(m_preload == MediaPlayer::None ? MediaPlayer::Idle : MediaPlayer::Loading);
173     setReadyState(MediaPlayer::HaveNothing);
174
175     m_assetURL = url;
176
177     // Don't do any more work if the url is empty.
178     if (!url.length())
179         return;
180
181     setPreload(m_preload);
182 }
183
184 #if ENABLE(MEDIA_SOURCE)
185 void MediaPlayerPrivateAVFoundation::load(const String&, MediaSourcePrivateClient*)
186 {
187     setNetworkState(MediaPlayer::FormatError);
188 }
189 #endif
190
191
192 void MediaPlayerPrivateAVFoundation::playabilityKnown()
193 {
194     LOG(Media, "MediaPlayerPrivateAVFoundation::playabilityKnown(%p)", this);
195
196     if (m_assetIsPlayable)
197         return;
198
199     // Nothing more to do if we already have all of the item's metadata.
200     if (assetStatus() > MediaPlayerAVAssetStatusLoading) {
201         LOG(Media, "MediaPlayerPrivateAVFoundation::playabilityKnown(%p) - all metadata loaded", this);
202         return;
203     }
204
205     // At this point we are supposed to load metadata. It is OK to ask the asset to load the same 
206     // information multiple times, because if it has already been loaded the completion handler 
207     // will just be called synchronously.
208     m_loadingMetadata = true;
209     beginLoadingMetadata();
210 }
211
212 void MediaPlayerPrivateAVFoundation::prepareToPlay()
213 {
214     LOG(Media, "MediaPlayerPrivateAVFoundation::prepareToPlay(%p)", this);
215
216     setPreload(MediaPlayer::Auto);
217 }
218
219 void MediaPlayerPrivateAVFoundation::play()
220 {
221     LOG(Media, "MediaPlayerPrivateAVFoundation::play(%p)", this);
222
223     // If the file has video, don't request playback until the first frame of video is ready to display
224     // or the audio may start playing before we can render video.
225     if (!m_cachedHasVideo || hasAvailableVideoFrame())
226         platformPlay();
227     else {
228         LOG(Media, "MediaPlayerPrivateAVFoundation::play(%p) - waiting for first video frame", this);
229         m_playWhenFramesAvailable = true;
230     }
231 }
232
233 void MediaPlayerPrivateAVFoundation::pause()
234 {
235     LOG(Media, "MediaPlayerPrivateAVFoundation::pause(%p)", this);
236     m_playWhenFramesAvailable = false;
237     platformPause();
238 }
239
240 MediaTime MediaPlayerPrivateAVFoundation::durationMediaTime() const
241 {
242     if (m_cachedDuration.isValid())
243         return m_cachedDuration;
244
245     MediaTime duration = platformDuration();
246     if (!duration || duration.isInvalid())
247         return MediaTime::zeroTime();
248
249     m_cachedDuration = duration;
250     LOG(Media, "MediaPlayerPrivateAVFoundation::duration(%p) - caching %s", this, toString(m_cachedDuration).utf8().data());
251     return m_cachedDuration;
252 }
253
254 void MediaPlayerPrivateAVFoundation::seek(const MediaTime& time)
255 {
256     seekWithTolerance(time, MediaTime::zeroTime(), MediaTime::zeroTime());
257 }
258
259 void MediaPlayerPrivateAVFoundation::seekWithTolerance(const MediaTime& mediaTime, const MediaTime& negativeTolerance, const MediaTime& positiveTolerance)
260 {
261     MediaTime time = mediaTime;
262
263     if (m_seeking) {
264         LOG(Media, "MediaPlayerPrivateAVFoundation::seekWithTolerance(%p) - save pending seek", this);
265         m_pendingSeek = [this, time, negativeTolerance, positiveTolerance]() {
266             seekWithTolerance(time, negativeTolerance, positiveTolerance);
267         };
268         return;
269     }
270     m_seeking = true;
271
272     if (!metaDataAvailable())
273         return;
274
275     if (time > durationMediaTime())
276         time = durationMediaTime();
277
278     if (currentTextTrack())
279         currentTextTrack()->beginSeeking();
280
281     LOG(Media, "MediaPlayerPrivateAVFoundation::seek(%p) - seeking to %s", this, toString(time).utf8().data());
282
283     seekToTime(time, negativeTolerance, positiveTolerance);
284 }
285
286 bool MediaPlayerPrivateAVFoundation::paused() const
287 {
288     if (!metaDataAvailable())
289         return true;
290
291     return rate() == 0;
292 }
293
294 bool MediaPlayerPrivateAVFoundation::seeking() const
295 {
296     if (!metaDataAvailable())
297         return false;
298
299     return m_seeking;
300 }
301
302 FloatSize MediaPlayerPrivateAVFoundation::naturalSize() const
303 {
304     if (!metaDataAvailable())
305         return IntSize();
306
307     // In spite of the name of this method, return the natural size transformed by the 
308     // initial movie scale because the spec says intrinsic size is:
309     //
310     //    ... the dimensions of the resource in CSS pixels after taking into account the resource's 
311     //    dimensions, aspect ratio, clean aperture, resolution, and so forth, as defined for the 
312     //    format used by the resource
313
314     return m_cachedNaturalSize;
315 }
316
317 void MediaPlayerPrivateAVFoundation::setNaturalSize(FloatSize size)
318 {
319     LOG(Media, "MediaPlayerPrivateAVFoundation:setNaturalSize(%p) - size = %f x %f", this, size.width(), size.height());
320
321     FloatSize oldSize = m_cachedNaturalSize;
322     m_cachedNaturalSize = size;
323     if (oldSize != m_cachedNaturalSize)
324         m_player->sizeChanged();
325 }
326
327 void MediaPlayerPrivateAVFoundation::setHasVideo(bool b)
328 {
329     if (m_cachedHasVideo != b) {
330         m_cachedHasVideo = b;
331         characteristicsChanged();
332     }
333 }
334
335 void MediaPlayerPrivateAVFoundation::setHasAudio(bool b)
336 {
337     if (m_cachedHasAudio != b) {
338         m_cachedHasAudio = b;
339         characteristicsChanged();
340     }
341 }
342
343 void MediaPlayerPrivateAVFoundation::setHasClosedCaptions(bool b)
344 {
345     if (m_cachedHasCaptions != b) {
346         m_cachedHasCaptions = b;
347         characteristicsChanged();
348     }
349 }
350
351 void MediaPlayerPrivateAVFoundation::setNetworkState(MediaPlayer::NetworkState state)
352 {
353     if (state == m_networkState)
354         return;
355
356     m_networkState = state;
357     m_player->networkStateChanged();
358 }
359
360 void MediaPlayerPrivateAVFoundation::setReadyState(MediaPlayer::ReadyState state)
361 {
362     if (state == m_readyState)
363         return;
364
365     m_readyState = state;
366     m_player->readyStateChanged();
367 }
368
369 void MediaPlayerPrivateAVFoundation::characteristicsChanged()
370 {
371     if (m_delayCharacteristicsChangedNotification) {
372         m_characteristicsChanged = true;
373         return;
374     }
375
376     m_characteristicsChanged = false;
377     m_player->characteristicChanged();
378 }
379
380 void MediaPlayerPrivateAVFoundation::setDelayCharacteristicsChangedNotification(bool delay)
381 {
382     if (delay) {
383         m_delayCharacteristicsChangedNotification++;
384         return;
385     }
386     
387     ASSERT(m_delayCharacteristicsChangedNotification);
388     m_delayCharacteristicsChangedNotification--;
389     if (!m_delayCharacteristicsChangedNotification && m_characteristicsChanged)
390         characteristicsChanged();
391 }
392
393 std::unique_ptr<PlatformTimeRanges> MediaPlayerPrivateAVFoundation::buffered() const
394 {
395     if (!m_cachedLoadedTimeRanges)
396         m_cachedLoadedTimeRanges = platformBufferedTimeRanges();
397
398     return std::make_unique<PlatformTimeRanges>(*m_cachedLoadedTimeRanges);
399 }
400
401 MediaTime MediaPlayerPrivateAVFoundation::maxMediaTimeSeekable() const
402 {
403     if (!metaDataAvailable())
404         return MediaTime::zeroTime();
405
406     if (!m_cachedMaxTimeSeekable)
407         m_cachedMaxTimeSeekable = platformMaxTimeSeekable();
408
409     LOG(Media, "MediaPlayerPrivateAVFoundation::maxTimeSeekable(%p) - returning %s", this, toString(m_cachedMaxTimeSeekable).utf8().data());
410     return m_cachedMaxTimeSeekable;   
411 }
412
413 MediaTime MediaPlayerPrivateAVFoundation::minMediaTimeSeekable() const
414 {
415     if (!metaDataAvailable())
416         return MediaTime::zeroTime();
417
418     if (!m_cachedMinTimeSeekable)
419         m_cachedMinTimeSeekable = platformMinTimeSeekable();
420
421     LOG(Media, "MediaPlayerPrivateAVFoundation::minTimeSeekable(%p) - returning %s", this, toString(m_cachedMinTimeSeekable).utf8().data());
422     return m_cachedMinTimeSeekable;
423 }
424
425 double MediaPlayerPrivateAVFoundation::requestedRate() const
426 {
427     return m_player->requestedRate();
428 }
429
430 MediaTime MediaPlayerPrivateAVFoundation::maxTimeLoaded() const
431 {
432     if (!metaDataAvailable())
433         return MediaTime::zeroTime();
434
435     if (!m_cachedMaxTimeLoaded)
436         m_cachedMaxTimeLoaded = platformMaxTimeLoaded();
437
438     return m_cachedMaxTimeLoaded;   
439 }
440
441 bool MediaPlayerPrivateAVFoundation::didLoadingProgress() const
442 {
443     if (!durationMediaTime())
444         return false;
445     MediaTime currentMaxTimeLoaded = maxTimeLoaded();
446     bool didLoadingProgress = currentMaxTimeLoaded != m_maxTimeLoadedAtLastDidLoadingProgress;
447     m_maxTimeLoadedAtLastDidLoadingProgress = currentMaxTimeLoaded;
448
449     return didLoadingProgress;
450 }
451
452 bool MediaPlayerPrivateAVFoundation::isReadyForVideoSetup() const
453 {
454     // AVFoundation will not return true for firstVideoFrameAvailable until
455     // an AVPlayerLayer has been added to the AVPlayerItem, so allow video setup
456     // here if a video track to trigger allocation of a AVPlayerLayer.
457     return (m_isAllowedToRender || m_cachedHasVideo) && m_readyState >= MediaPlayer::HaveMetadata && m_player->visible();
458 }
459
460 void MediaPlayerPrivateAVFoundation::prepareForRendering()
461 {
462     if (m_isAllowedToRender)
463         return;
464     m_isAllowedToRender = true;
465
466     setUpVideoRendering();
467
468     if (currentRenderingMode() == MediaRenderingToLayer || preferredRenderingMode() == MediaRenderingToLayer)
469         m_player->client().mediaPlayerRenderingModeChanged(m_player);
470 }
471
472 bool MediaPlayerPrivateAVFoundation::supportsFullscreen() const
473 {
474 #if ENABLE(FULLSCREEN_API)
475     return true;
476 #else
477     // FIXME: WebVideoFullscreenController assumes a QTKit/QuickTime media engine
478 #if PLATFORM(IOS)
479     if (Settings::avKitEnabled())
480         return true;
481 #endif
482     return false;
483 #endif
484 }
485
486 void MediaPlayerPrivateAVFoundation::updateStates()
487 {
488     if (m_ignoreLoadStateChanges)
489         return;
490
491     MediaPlayer::NetworkState newNetworkState = m_networkState;
492     MediaPlayer::ReadyState newReadyState = m_readyState;
493
494     if (m_loadingMetadata)
495         newNetworkState = MediaPlayer::Loading;
496     else {
497         // -loadValuesAsynchronouslyForKeys:completionHandler: has invoked its handler; test status of keys and determine state.
498         AssetStatus assetStatus = this->assetStatus();
499         ItemStatus itemStatus = playerItemStatus();
500         
501         m_assetIsPlayable = (assetStatus == MediaPlayerAVAssetStatusPlayable);
502         if (m_readyState < MediaPlayer::HaveMetadata && assetStatus > MediaPlayerAVAssetStatusLoading) {
503             if (m_assetIsPlayable) {
504                 if (assetStatus >= MediaPlayerAVAssetStatusLoaded)
505                     newReadyState = MediaPlayer::HaveMetadata;
506                 if (itemStatus <= MediaPlayerAVPlayerItemStatusUnknown) {
507                     if (assetStatus == MediaPlayerAVAssetStatusFailed || m_preload > MediaPlayer::MetaData || isLiveStream()) {
508                         // The asset is playable but doesn't support inspection prior to playback (eg. streaming files),
509                         // or we are supposed to prepare for playback immediately, so create the player item now.
510                         newNetworkState = MediaPlayer::Loading;
511                         prepareToPlay();
512                     } else
513                         newNetworkState = MediaPlayer::Idle;
514                 }
515             } else {
516                 // FIX ME: fetch the error associated with the @"playable" key to distinguish between format 
517                 // and network errors.
518                 newNetworkState = MediaPlayer::FormatError;
519             }
520         }
521         
522         if (assetStatus >= MediaPlayerAVAssetStatusLoaded && itemStatus > MediaPlayerAVPlayerItemStatusUnknown) {
523             switch (itemStatus) {
524             case MediaPlayerAVPlayerItemStatusDoesNotExist:
525             case MediaPlayerAVPlayerItemStatusUnknown:
526             case MediaPlayerAVPlayerItemStatusFailed:
527                 break;
528
529             case MediaPlayerAVPlayerItemStatusPlaybackLikelyToKeepUp:
530             case MediaPlayerAVPlayerItemStatusPlaybackBufferFull:
531                 // If the status becomes PlaybackBufferFull, loading stops and the status will not
532                 // progress to LikelyToKeepUp. Set the readyState to  HAVE_ENOUGH_DATA, on the
533                 // presumption that if the playback buffer is full, playback will probably not stall.
534                 newReadyState = MediaPlayer::HaveEnoughData;
535                 break;
536
537             case MediaPlayerAVPlayerItemStatusReadyToPlay:
538                 if (m_readyState != MediaPlayer::HaveEnoughData && maxTimeLoaded() > currentMediaTime())
539                     newReadyState = MediaPlayer::HaveFutureData;
540                 break;
541
542             case MediaPlayerAVPlayerItemStatusPlaybackBufferEmpty:
543                 newReadyState = MediaPlayer::HaveCurrentData;
544                 break;
545             }
546
547             if (itemStatus == MediaPlayerAVPlayerItemStatusPlaybackBufferFull)
548                 newNetworkState = MediaPlayer::Idle;
549             else if (itemStatus == MediaPlayerAVPlayerItemStatusFailed)
550                 newNetworkState = MediaPlayer::DecodeError;
551             else if (itemStatus != MediaPlayerAVPlayerItemStatusPlaybackBufferFull && itemStatus >= MediaPlayerAVPlayerItemStatusReadyToPlay)
552                 newNetworkState = (maxTimeLoaded() == durationMediaTime()) ? MediaPlayer::Loaded : MediaPlayer::Loading;
553         }
554     }
555
556     if (isReadyForVideoSetup() && currentRenderingMode() != preferredRenderingMode())
557         setUpVideoRendering();
558
559     if (!m_haveReportedFirstVideoFrame && m_cachedHasVideo && hasAvailableVideoFrame()) {
560         if (m_readyState < MediaPlayer::HaveCurrentData)
561             newReadyState = MediaPlayer::HaveCurrentData;
562         m_haveReportedFirstVideoFrame = true;
563         m_player->firstVideoFrameAvailable();
564     } else if (!hasAvailableVideoFrame())
565         m_haveReportedFirstVideoFrame = false;
566
567 #if !LOG_DISABLED
568     if (m_networkState != newNetworkState || m_readyState != newReadyState) {
569         LOG(Media, "MediaPlayerPrivateAVFoundation::updateStates(%p) - entered with networkState = %i, readyState = %i,  exiting with networkState = %i, readyState = %i",
570             this, static_cast<int>(m_networkState), static_cast<int>(m_readyState), static_cast<int>(newNetworkState), static_cast<int>(newReadyState));
571     }
572 #endif
573
574     setNetworkState(newNetworkState);
575     setReadyState(newReadyState);
576
577     if (m_playWhenFramesAvailable && hasAvailableVideoFrame()) {
578         m_playWhenFramesAvailable = false;
579         platformPlay();
580     }
581 }
582
583 void MediaPlayerPrivateAVFoundation::setSize(const IntSize&) 
584
585 }
586
587 void MediaPlayerPrivateAVFoundation::setVisible(bool visible)
588 {
589     if (m_visible == visible)
590         return;
591
592     m_visible = visible;
593     if (visible)
594         setUpVideoRendering();
595     
596     platformSetVisible(visible);
597 }
598
599 void MediaPlayerPrivateAVFoundation::acceleratedRenderingStateChanged()
600 {
601     // Set up or change the rendering path if necessary.
602     setUpVideoRendering();
603 }
604
605 void MediaPlayerPrivateAVFoundation::setShouldMaintainAspectRatio(bool maintainAspectRatio)
606 {
607     if (maintainAspectRatio == m_shouldMaintainAspectRatio)
608         return;
609
610     m_shouldMaintainAspectRatio = maintainAspectRatio;
611     updateVideoLayerGravity();
612 }
613
614 void MediaPlayerPrivateAVFoundation::metadataLoaded()
615 {
616     m_loadingMetadata = false;
617     tracksChanged();
618 }
619
620 void MediaPlayerPrivateAVFoundation::rateChanged()
621 {
622 #if ENABLE(WIRELESS_PLAYBACK_TARGET) && PLATFORM(IOS)
623     LOG(Media, "MediaPlayerPrivateAVFoundation::rateChanged(%p) - rate = %f, requested rate = %f, item status = %i", this, rate(), requestedRate(), playerItemStatus());
624     if (isCurrentPlaybackTargetWireless() && playerItemStatus() >= MediaPlayerAVPlayerItemStatusPlaybackBufferFull) {
625         double rate = this->rate();
626         if (rate != requestedRate()) {
627             m_player->handlePlaybackCommand(rate ? PlatformMediaSession::PlayCommand : PlatformMediaSession::PauseCommand);
628             return;
629         }
630     }
631 #endif
632
633     m_player->rateChanged();
634 }
635
636 void MediaPlayerPrivateAVFoundation::loadedTimeRangesChanged()
637 {
638     m_cachedLoadedTimeRanges = nullptr;
639     m_cachedMaxTimeLoaded = MediaTime::zeroTime();
640     invalidateCachedDuration();
641 }
642
643 void MediaPlayerPrivateAVFoundation::seekableTimeRangesChanged()
644 {
645     m_cachedMaxTimeSeekable = MediaTime::zeroTime();
646     m_cachedMinTimeSeekable = MediaTime::zeroTime();
647 }
648
649 void MediaPlayerPrivateAVFoundation::timeChanged(const MediaTime& time)
650 {
651     LOG(Media, "MediaPlayerPrivateAVFoundation::timeChanged(%p) - time = %s", this, toString(time).utf8().data());
652     UNUSED_PARAM(time);
653 }
654
655 void MediaPlayerPrivateAVFoundation::seekCompleted(bool finished)
656 {
657     LOG(Media, "MediaPlayerPrivateAVFoundation::seekCompleted(%p) - finished = %d", this, finished);
658     UNUSED_PARAM(finished);
659
660     m_seeking = false;
661
662     WTF::Function<void()> pendingSeek;
663     std::swap(pendingSeek, m_pendingSeek);
664
665     if (pendingSeek) {
666         LOG(Media, "MediaPlayerPrivateAVFoundation::seekCompleted(%p) - issuing pending seek", this);
667         pendingSeek();
668         return;
669     }
670
671     if (currentTextTrack())
672         currentTextTrack()->endSeeking();
673
674     updateStates();
675     m_player->timeChanged();
676 }
677
678 void MediaPlayerPrivateAVFoundation::didEnd()
679 {
680     // Hang onto the current time and use it as duration from now on since we are definitely at
681     // the end of the movie. Do this because the initial duration is sometimes an estimate.
682     MediaTime now = currentMediaTime();
683     if (now > MediaTime::zeroTime() && !m_seeking)
684         m_cachedDuration = now;
685
686     updateStates();
687     m_player->timeChanged();
688 }
689
690 void MediaPlayerPrivateAVFoundation::invalidateCachedDuration()
691 {
692     m_cachedDuration = MediaTime::invalidTime();
693
694     // For some media files, reported duration is estimated and updated as media is loaded
695     // so report duration changed when the estimate is upated.
696     MediaTime duration = this->durationMediaTime();
697     if (duration != m_reportedDuration) {
698         if (m_reportedDuration.isValid())
699             m_player->durationChanged();
700         m_reportedDuration = duration;
701     }
702     
703 }
704
705 void MediaPlayerPrivateAVFoundation::repaint()
706 {
707     m_player->repaint();
708 }
709
710 MediaPlayer::MovieLoadType MediaPlayerPrivateAVFoundation::movieLoadType() const
711 {
712     if (!metaDataAvailable() || assetStatus() == MediaPlayerAVAssetStatusUnknown)
713         return MediaPlayer::Unknown;
714
715     if (isLiveStream())
716         return MediaPlayer::LiveStream;
717
718     return MediaPlayer::Download;
719 }
720
721 void MediaPlayerPrivateAVFoundation::setPreload(MediaPlayer::Preload preload)
722 {
723     m_preload = preload;
724     if (!m_assetURL.length())
725         return;
726
727     setDelayCallbacks(true);
728
729     if (m_preload >= MediaPlayer::MetaData && assetStatus() == MediaPlayerAVAssetStatusDoesNotExist) {
730         createAVAssetForURL(m_assetURL);
731         checkPlayability();
732     }
733
734     // Don't force creation of the player and player item unless we already know that the asset is playable. If we aren't
735     // there yet, or if we already know it is not playable, creating them now won't help.
736     if (m_preload == MediaPlayer::Auto && m_assetIsPlayable) {
737         createAVPlayerItem();
738         createAVPlayer();
739     }
740
741     setDelayCallbacks(false);
742 }
743
744 void MediaPlayerPrivateAVFoundation::setDelayCallbacks(bool delay) const
745 {
746     LockHolder lock(m_queueMutex);
747     if (delay)
748         ++m_delayCallbacks;
749     else {
750         ASSERT(m_delayCallbacks);
751         --m_delayCallbacks;
752     }
753 }
754
755 void MediaPlayerPrivateAVFoundation::mainThreadCallback()
756 {
757     LOG(Media, "MediaPlayerPrivateAVFoundation::mainThreadCallback(%p)", this);
758
759     clearMainThreadPendingFlag();
760     dispatchNotification();
761 }
762
763 void MediaPlayerPrivateAVFoundation::clearMainThreadPendingFlag()
764 {
765     LockHolder lock(m_queueMutex);
766     m_mainThreadCallPending = false;
767 }
768
769 void MediaPlayerPrivateAVFoundation::scheduleMainThreadNotification(Notification::Type type, const MediaTime& time)
770 {
771     scheduleMainThreadNotification(Notification(type, time));
772 }
773
774 void MediaPlayerPrivateAVFoundation::scheduleMainThreadNotification(Notification::Type type, bool finished)
775 {
776     scheduleMainThreadNotification(Notification(type, finished));
777 }
778
779 #if !LOG_DISABLED
780 static const char* notificationName(MediaPlayerPrivateAVFoundation::Notification& notification)
781 {
782 #define DEFINE_TYPE_STRING_CASE(type) case MediaPlayerPrivateAVFoundation::Notification::type: return #type;
783     switch (notification.type()) {
784     FOR_EACH_MEDIAPLAYERPRIVATEAVFOUNDATION_NOTIFICATION_TYPE(DEFINE_TYPE_STRING_CASE)
785     case MediaPlayerPrivateAVFoundation::Notification::FunctionType: return "FunctionType";
786     default: ASSERT_NOT_REACHED(); return "";
787     }
788 #undef DEFINE_TYPE_STRING_CASE
789 }
790 #endif // !LOG_DISABLED
791     
792
793 void MediaPlayerPrivateAVFoundation::scheduleMainThreadNotification(Notification&& notification)
794 {
795     auto notificationType = notification.type();
796     if (notificationType != Notification::FunctionType)
797         LOG(Media, "MediaPlayerPrivateAVFoundation::scheduleMainThreadNotification(%p) - notification %s", this, notificationName(notification));
798
799     m_queueMutex.lock();
800
801     // It is important to always process the properties in the order that we are notified,
802     // so always go through the queue because notifications happen on different threads.
803     m_queuedNotifications.append(WTFMove(notification));
804
805 #if OS(WINDOWS)
806     bool delayDispatch = true;
807 #else
808     bool delayDispatch = m_delayCallbacks || !isMainThread();
809 #endif
810     if (delayDispatch && !m_mainThreadCallPending) {
811         m_mainThreadCallPending = true;
812
813         callOnMainThread([weakThis = createWeakPtr()] {
814             if (!weakThis)
815                 return;
816
817             weakThis->mainThreadCallback();
818         });
819     }
820
821     m_queueMutex.unlock();
822
823     if (delayDispatch) {
824         if (notificationType != Notification::FunctionType)
825             LOG(Media, "MediaPlayerPrivateAVFoundation::scheduleMainThreadNotification(%p) - early return", this);
826         return;
827     }
828
829     dispatchNotification();
830 }
831
832 void MediaPlayerPrivateAVFoundation::dispatchNotification()
833 {
834     ASSERT(isMainThread());
835
836     Notification notification;
837     {
838         LockHolder lock(m_queueMutex);
839         
840         if (m_queuedNotifications.isEmpty())
841             return;
842         
843         if (!m_delayCallbacks) {
844             // Only dispatch one notification callback per invocation because they can cause recursion.
845             notification = m_queuedNotifications.takeFirst();
846         }
847         
848         if (!m_queuedNotifications.isEmpty() && !m_mainThreadCallPending) {
849             callOnMainThread([weakThis = createWeakPtr()] {
850                 if (!weakThis)
851                     return;
852
853                 weakThis->mainThreadCallback();
854             });
855         }
856
857         if (!notification.isValid())
858             return;
859     }
860
861     if (notification.type() != Notification::FunctionType)
862         LOG(Media, "MediaPlayerPrivateAVFoundation::dispatchNotification(%p) - dispatching %s", this, notificationName(notification));
863
864     switch (notification.type()) {
865     case Notification::ItemDidPlayToEndTime:
866         didEnd();
867         break;
868     case Notification::ItemTracksChanged:
869         tracksChanged();
870         updateStates();
871         break;
872     case Notification::ItemStatusChanged:
873         updateStates();
874         break;
875     case Notification::ItemSeekableTimeRangesChanged:
876         seekableTimeRangesChanged();
877         updateStates();
878         break;
879     case Notification::ItemLoadedTimeRangesChanged:
880         loadedTimeRangesChanged();
881         updateStates();
882         break;
883     case Notification::ItemPresentationSizeChanged:
884         sizeChanged();
885         updateStates();
886         break;
887     case Notification::ItemIsPlaybackLikelyToKeepUpChanged:
888         updateStates();
889         break;
890     case Notification::ItemIsPlaybackBufferEmptyChanged:
891         updateStates();
892         break;
893     case Notification::ItemIsPlaybackBufferFullChanged:
894         updateStates();
895         break;
896     case Notification::PlayerRateChanged:
897         updateStates();
898         rateChanged();
899         break;
900     case Notification::PlayerTimeChanged:
901         timeChanged(notification.time());
902         break;
903     case Notification::SeekCompleted:
904         seekCompleted(notification.finished());
905         break;
906     case Notification::AssetMetadataLoaded:
907         metadataLoaded();
908         updateStates();
909         break;
910     case Notification::AssetPlayabilityKnown:
911         updateStates();
912         playabilityKnown();
913         break;
914     case Notification::DurationChanged:
915         invalidateCachedDuration();
916         break;
917     case Notification::ContentsNeedsDisplay:
918         contentsNeedsDisplay();
919         break;
920     case Notification::InbandTracksNeedConfiguration:
921         m_inbandTrackConfigurationPending = false;
922         configureInbandTracks();
923         break;
924     case Notification::FunctionType:
925         notification.function()();
926         break;
927     case Notification::TargetIsWirelessChanged:
928 #if ENABLE(WIRELESS_PLAYBACK_TARGET)
929         playbackTargetIsWirelessChanged();
930 #endif
931         break;
932
933     case Notification::None:
934         ASSERT_NOT_REACHED();
935         break;
936     }
937 }
938
939 void MediaPlayerPrivateAVFoundation::configureInbandTracks()
940 {
941     RefPtr<InbandTextTrackPrivateAVF> trackToEnable;
942     
943 #if ENABLE(AVF_CAPTIONS)
944     synchronizeTextTrackState();
945 #endif
946
947     // AVFoundation can only emit cues for one track at a time, so enable the first track that is showing, or the first that
948     // is hidden if none are showing. Otherwise disable all tracks.
949     for (unsigned i = 0; i < m_textTracks.size(); ++i) {
950         RefPtr<InbandTextTrackPrivateAVF> track = m_textTracks[i];
951         if (track->mode() == InbandTextTrackPrivate::Showing) {
952             trackToEnable = track;
953             break;
954         }
955         if (track->mode() == InbandTextTrackPrivate::Hidden)
956             trackToEnable = track;
957     }
958
959     setCurrentTextTrack(trackToEnable.get());
960 }
961
962 void MediaPlayerPrivateAVFoundation::trackModeChanged()
963 {
964     if (m_inbandTrackConfigurationPending)
965         return;
966     m_inbandTrackConfigurationPending = true;
967     scheduleMainThreadNotification(Notification::InbandTracksNeedConfiguration);
968 }
969
970 void MediaPlayerPrivateAVFoundation::clearTextTracks()
971 {
972     for (auto& track : m_textTracks) {
973         player()->removeTextTrack(*track);
974         track->disconnect();
975     }
976     m_textTracks.clear();
977 }
978
979 void MediaPlayerPrivateAVFoundation::processNewAndRemovedTextTracks(const Vector<RefPtr<InbandTextTrackPrivateAVF>>& removedTextTracks)
980 {
981     if (removedTextTracks.size()) {
982         for (unsigned i = 0; i < m_textTracks.size(); ) {
983             if (!removedTextTracks.contains(m_textTracks[i])) {
984                 ++i;
985                 continue;
986             }
987             player()->removeTextTrack(*m_textTracks[i]);
988             m_textTracks.remove(i);
989         }
990     }
991
992     unsigned inBandCount = 0;
993     for (unsigned i = 0; i < m_textTracks.size(); ++i) {
994         RefPtr<InbandTextTrackPrivateAVF> track = m_textTracks[i];
995
996 #if ENABLE(AVF_CAPTIONS)
997         if (track->textTrackCategory() == InbandTextTrackPrivateAVF::OutOfBand)
998             continue;
999 #endif
1000
1001         track->setTextTrackIndex(inBandCount);
1002         ++inBandCount;
1003         if (track->hasBeenReported())
1004             continue;
1005         
1006         track->setHasBeenReported(true);
1007         player()->addTextTrack(*track);
1008     }
1009     LOG(Media, "MediaPlayerPrivateAVFoundation::processNewAndRemovedTextTracks(%p) - found %lu text tracks", this, m_textTracks.size());
1010 }
1011
1012 #if ENABLE(WIRELESS_PLAYBACK_TARGET)
1013
1014 void MediaPlayerPrivateAVFoundation::playbackTargetIsWirelessChanged()
1015 {
1016     if (m_player)
1017         m_player->currentPlaybackTargetIsWirelessChanged();
1018 }
1019
1020 #endif
1021
1022 #if ENABLE(LEGACY_ENCRYPTED_MEDIA)
1023
1024 bool MediaPlayerPrivateAVFoundation::extractKeyURIKeyIDAndCertificateFromInitData(Uint8Array* initData, String& keyURI, String& keyID, RefPtr<Uint8Array>& certificate)
1025 {
1026     // initData should have the following layout:
1027     // [4 bytes: keyURI length][N bytes: keyURI][4 bytes: contentID length], [N bytes: contentID], [4 bytes: certificate length][N bytes: certificate]
1028     if (initData->byteLength() < 4)
1029         return false;
1030
1031     RefPtr<ArrayBuffer> initDataBuffer = initData->unsharedBuffer();
1032
1033     // Use a DataView to read uint32 values from the buffer, as Uint32Array requires the reads be aligned on 4-byte boundaries. 
1034     RefPtr<JSC::DataView> initDataView = JSC::DataView::create(initDataBuffer.copyRef(), 0, initDataBuffer->byteLength());
1035     uint32_t offset = 0;
1036     bool status = true;
1037
1038     uint32_t keyURILength = initDataView->get<uint32_t>(offset, true, &status);
1039     offset += 4;
1040     if (!status || offset + keyURILength > initData->length())
1041         return false;
1042
1043     RefPtr<Uint16Array> keyURIArray = Uint16Array::create(initDataBuffer.copyRef(), offset, keyURILength);
1044     if (!keyURIArray)
1045         return false;
1046
1047     keyURI = String(reinterpret_cast<UChar*>(keyURIArray->data()), keyURILength / sizeof(unsigned short));
1048     offset += keyURILength;
1049
1050     uint32_t keyIDLength = initDataView->get<uint32_t>(offset, true, &status);
1051     offset += 4;
1052     if (!status || offset + keyIDLength > initData->length())
1053         return false;
1054
1055     RefPtr<Uint8Array> keyIDArray = Uint8Array::create(initDataBuffer.copyRef(), offset, keyIDLength);
1056     if (!keyIDArray)
1057         return false;
1058
1059     keyID = String(reinterpret_cast<UChar*>(keyIDArray->data()), keyIDLength / sizeof(unsigned short));
1060     offset += keyIDLength;
1061
1062     uint32_t certificateLength = initDataView->get<uint32_t>(offset, true, &status);
1063     offset += 4;
1064     if (!status || offset + certificateLength > initData->length())
1065         return false;
1066
1067     certificate = Uint8Array::create(WTFMove(initDataBuffer), offset, certificateLength);
1068     if (!certificate)
1069         return false;
1070
1071     return true;
1072 }
1073
1074 #endif
1075
1076 URL MediaPlayerPrivateAVFoundation::resolvedURL() const
1077 {
1078     if (!m_assetURL.length())
1079         return URL();
1080
1081     return URL(ParsedURLString, m_assetURL);
1082 }
1083
1084 bool MediaPlayerPrivateAVFoundation::canSaveMediaData() const
1085 {
1086     URL url = resolvedURL();
1087
1088     if (url.isLocalFile())
1089         return true;
1090
1091     if (!url.protocolIsInHTTPFamily())
1092         return false;
1093
1094     if (isLiveStream())
1095         return false;
1096
1097     return true;
1098 }
1099
1100 bool MediaPlayerPrivateAVFoundation::isUnsupportedMIMEType(const String& type)
1101 {
1102     String lowerCaseType = type.convertToASCIILowercase();
1103
1104     // AVFoundation will return non-video MIME types which it claims to support, but which we
1105     // do not support in the <video> element. Reject all non video/, audio/, and application/ types.
1106     if (!lowerCaseType.startsWith("video/") && !lowerCaseType.startsWith("audio/") && !lowerCaseType.startsWith("application/"))
1107         return true;
1108
1109     // Reject types we know AVFoundation does not support that sites commonly ask about.
1110     if (lowerCaseType == "video/webm" || lowerCaseType == "audio/webm" || lowerCaseType == "video/x-webm")
1111         return true;
1112
1113     if (lowerCaseType == "video/x-flv")
1114         return true;
1115
1116     if (lowerCaseType == "audio/ogg" || lowerCaseType == "video/ogg" || lowerCaseType == "application/ogg")
1117         return true;
1118
1119     if (lowerCaseType == "video/h264")
1120         return true;
1121
1122     return false;
1123 }
1124
1125 const HashSet<String, ASCIICaseInsensitiveHash>& MediaPlayerPrivateAVFoundation::staticMIMETypeList()
1126 {
1127     static const auto cache = makeNeverDestroyed(HashSet<String, ASCIICaseInsensitiveHash> {
1128         "application/vnd.apple.mpegurl",
1129         "application/x-mpegurl",
1130         "audio/3gpp",
1131         "audio/aac",
1132         "audio/aacp",
1133         "audio/aiff",
1134         "audio/basic",
1135         "audio/mp3",
1136         "audio/mp4",
1137         "audio/mpeg",
1138         "audio/mpeg3",
1139         "audio/mpegurl",
1140         "audio/mpg",
1141         "audio/vnd.wave",
1142         "audio/wav",
1143         "audio/wave",
1144         "audio/x-aac",
1145         "audio/x-aiff",
1146         "audio/x-m4a",
1147         "audio/x-mpegurl",
1148         "audio/x-wav",
1149         "video/3gpp",
1150         "video/3gpp2",
1151         "video/mp4",
1152         "video/mpeg",
1153         "video/mpeg2",
1154         "video/mpg",
1155         "video/quicktime",
1156         "video/x-m4v",
1157         "video/x-mpeg",
1158         "video/x-mpg",
1159     });
1160     return cache;
1161 }
1162
1163 } // namespace WebCore
1164
1165 #endif