7ab7ecbe614a5c4f13ce663dbdf29d2d765a2ffe
[WebKit-https.git] / Source / WebCore / platform / graphics / avfoundation / MediaPlayerPrivateAVFoundation.cpp
1 /*
2  * Copyright (C) 2011-2014 Apple Inc. All rights reserved.
3  *
4  * Redistribution and use in source and binary forms, with or without
5  * modification, are permitted provided that the following conditions
6  * are met:
7  * 1. Redistributions of source code must retain the above copyright
8  *    notice, this list of conditions and the following disclaimer.
9  * 2. Redistributions in binary form must reproduce the above copyright
10  *    notice, this list of conditions and the following disclaimer in the
11  *    documentation and/or other materials provided with the distribution.
12  *
13  * THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY
14  * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
15  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
16  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL APPLE INC. OR
17  * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
18  * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
19  * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
20  * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
21  * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
22  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
23  * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 
24  */
25
26 #include "config.h"
27
28 #if ENABLE(VIDEO) && USE(AVFOUNDATION)
29
30 #include "MediaPlayerPrivateAVFoundation.h"
31
32 #include "DocumentLoader.h"
33 #include "FloatConversion.h"
34 #include "GraphicsContext.h"
35 #include "InbandTextTrackPrivateAVF.h"
36 #include "InbandTextTrackPrivateClient.h"
37 #include "URL.h"
38 #include "Logging.h"
39 #include "PlatformLayer.h"
40 #include "PlatformTimeRanges.h"
41 #include "Settings.h"
42 #include "SoftLinking.h"
43 #include <CoreMedia/CoreMedia.h>
44 #include <runtime/DataView.h>
45 #include <runtime/Uint16Array.h>
46 #include <wtf/MainThread.h>
47 #include <wtf/NeverDestroyed.h>
48 #include <wtf/text/CString.h>
49 #include <wtf/StringPrintStream.h>
50
51 namespace WebCore {
52
53 MediaPlayerPrivateAVFoundation::MediaPlayerPrivateAVFoundation(MediaPlayer* player)
54     : m_player(player)
55     , m_weakPtrFactory(this)
56     , m_queuedNotifications()
57     , m_queueMutex()
58     , m_networkState(MediaPlayer::Empty)
59     , m_readyState(MediaPlayer::HaveNothing)
60     , m_preload(MediaPlayer::Auto)
61     , m_cachedDuration(MediaTime::invalidTime())
62     , m_reportedDuration(MediaTime::invalidTime())
63     , m_maxTimeLoadedAtLastDidLoadingProgress(MediaTime::invalidTime())
64     , m_delayCallbacks(0)
65     , m_delayCharacteristicsChangedNotification(0)
66     , m_mainThreadCallPending(false)
67     , m_assetIsPlayable(false)
68     , m_visible(false)
69     , m_loadingMetadata(false)
70     , m_isAllowedToRender(false)
71     , m_cachedHasAudio(false)
72     , m_cachedHasVideo(false)
73     , m_cachedHasCaptions(false)
74     , m_ignoreLoadStateChanges(false)
75     , m_haveReportedFirstVideoFrame(false)
76     , m_playWhenFramesAvailable(false)
77     , m_inbandTrackConfigurationPending(false)
78     , m_characteristicsChanged(false)
79     , m_shouldMaintainAspectRatio(true)
80     , m_seeking(false)
81 {
82     LOG(Media, "MediaPlayerPrivateAVFoundation::MediaPlayerPrivateAVFoundation(%p)", this);
83 }
84
85 MediaPlayerPrivateAVFoundation::~MediaPlayerPrivateAVFoundation()
86 {
87     LOG(Media, "MediaPlayerPrivateAVFoundation::~MediaPlayerPrivateAVFoundation(%p)", this);
88     setIgnoreLoadStateChanges(true);
89 }
90
91 MediaPlayerPrivateAVFoundation::MediaRenderingMode MediaPlayerPrivateAVFoundation::currentRenderingMode() const
92 {
93     if (platformLayer())
94         return MediaRenderingToLayer;
95
96     if (hasContextRenderer())
97         return MediaRenderingToContext;
98
99     return MediaRenderingNone;
100 }
101
102 MediaPlayerPrivateAVFoundation::MediaRenderingMode MediaPlayerPrivateAVFoundation::preferredRenderingMode() const
103 {
104     if (!m_player->visible() || assetStatus() == MediaPlayerAVAssetStatusUnknown)
105         return MediaRenderingNone;
106
107     if (supportsAcceleratedRendering() && m_player->client().mediaPlayerRenderingCanBeAccelerated(m_player))
108         return MediaRenderingToLayer;
109
110     return MediaRenderingToContext;
111 }
112
113 void MediaPlayerPrivateAVFoundation::setUpVideoRendering()
114 {
115     if (!isReadyForVideoSetup())
116         return;
117
118     MediaRenderingMode currentMode = currentRenderingMode();
119     MediaRenderingMode preferredMode = preferredRenderingMode();
120
121     if (preferredMode == MediaRenderingNone)
122         preferredMode = MediaRenderingToContext;
123
124     if (currentMode == preferredMode && currentMode != MediaRenderingNone)
125         return;
126
127     LOG(Media, "MediaPlayerPrivateAVFoundation::setUpVideoRendering(%p) - current mode = %d, preferred mode = %d", 
128         this, static_cast<int>(currentMode), static_cast<int>(preferredMode));
129
130     if (currentMode != MediaRenderingNone)  
131         tearDownVideoRendering();
132
133     switch (preferredMode) {
134     case MediaRenderingNone:
135     case MediaRenderingToContext:
136         createContextVideoRenderer();
137         break;
138
139     case MediaRenderingToLayer:
140         createVideoLayer();
141         break;
142     }
143
144     // If using a movie layer, inform the client so the compositing tree is updated.
145     if (currentMode == MediaRenderingToLayer || preferredMode == MediaRenderingToLayer) {
146         LOG(Media, "MediaPlayerPrivateAVFoundation::setUpVideoRendering(%p) - calling mediaPlayerRenderingModeChanged()", this);
147         m_player->client().mediaPlayerRenderingModeChanged(m_player);
148     }
149 }
150
151 void MediaPlayerPrivateAVFoundation::tearDownVideoRendering()
152 {
153     LOG(Media, "MediaPlayerPrivateAVFoundation::tearDownVideoRendering(%p)", this);
154
155     destroyContextVideoRenderer();
156
157     if (platformLayer())
158         destroyVideoLayer();
159 }
160
161 bool MediaPlayerPrivateAVFoundation::hasSetUpVideoRendering() const
162 {
163     return hasLayerRenderer() || hasContextRenderer();
164 }
165
166 void MediaPlayerPrivateAVFoundation::load(const String& url)
167 {
168     LOG(Media, "MediaPlayerPrivateAVFoundation::load(%p)", this);
169
170     setNetworkState(m_preload == MediaPlayer::None ? MediaPlayer::Idle : MediaPlayer::Loading);
171     setReadyState(MediaPlayer::HaveNothing);
172
173     m_assetURL = url;
174
175     // Don't do any more work if the url is empty.
176     if (!url.length())
177         return;
178
179     setPreload(m_preload);
180 }
181
182 #if ENABLE(MEDIA_SOURCE)
183 void MediaPlayerPrivateAVFoundation::load(const String&, MediaSourcePrivateClient*)
184 {
185     setNetworkState(MediaPlayer::FormatError);
186 }
187 #endif
188
189
190 void MediaPlayerPrivateAVFoundation::playabilityKnown()
191 {
192     LOG(Media, "MediaPlayerPrivateAVFoundation::playabilityKnown(%p)", this);
193
194     if (m_assetIsPlayable)
195         return;
196
197     // Nothing more to do if we already have all of the item's metadata.
198     if (assetStatus() > MediaPlayerAVAssetStatusLoading) {
199         LOG(Media, "MediaPlayerPrivateAVFoundation::playabilityKnown(%p) - all metadata loaded", this);
200         return;
201     }
202
203     // At this point we are supposed to load metadata. It is OK to ask the asset to load the same 
204     // information multiple times, because if it has already been loaded the completion handler 
205     // will just be called synchronously.
206     m_loadingMetadata = true;
207     beginLoadingMetadata();
208 }
209
210 void MediaPlayerPrivateAVFoundation::prepareToPlay()
211 {
212     LOG(Media, "MediaPlayerPrivateAVFoundation::prepareToPlay(%p)", this);
213
214     setPreload(MediaPlayer::Auto);
215 }
216
217 void MediaPlayerPrivateAVFoundation::play()
218 {
219     LOG(Media, "MediaPlayerPrivateAVFoundation::play(%p)", this);
220
221     // If the file has video, don't request playback until the first frame of video is ready to display
222     // or the audio may start playing before we can render video.
223     if (!m_cachedHasVideo || hasAvailableVideoFrame())
224         platformPlay();
225     else {
226         LOG(Media, "MediaPlayerPrivateAVFoundation::play(%p) - waiting for first video frame", this);
227         m_playWhenFramesAvailable = true;
228     }
229 }
230
231 void MediaPlayerPrivateAVFoundation::pause()
232 {
233     LOG(Media, "MediaPlayerPrivateAVFoundation::pause(%p)", this);
234     m_playWhenFramesAvailable = false;
235     platformPause();
236 }
237
238 MediaTime MediaPlayerPrivateAVFoundation::durationMediaTime() const
239 {
240     if (m_cachedDuration.isValid())
241         return m_cachedDuration;
242
243     MediaTime duration = platformDuration();
244     if (!duration || duration.isInvalid())
245         return MediaTime::zeroTime();
246
247     m_cachedDuration = duration;
248     LOG(Media, "MediaPlayerPrivateAVFoundation::duration(%p) - caching %s", this, toString(m_cachedDuration).utf8().data());
249     return m_cachedDuration;
250 }
251
252 void MediaPlayerPrivateAVFoundation::seek(const MediaTime& time)
253 {
254     seekWithTolerance(time, MediaTime::zeroTime(), MediaTime::zeroTime());
255 }
256
257 void MediaPlayerPrivateAVFoundation::seekWithTolerance(const MediaTime& mediaTime, const MediaTime& negativeTolerance, const MediaTime& positiveTolerance)
258 {
259     MediaTime time = mediaTime;
260
261     if (m_seeking) {
262         LOG(Media, "MediaPlayerPrivateAVFoundation::seekWithTolerance(%p) - save pending seek", this);
263         m_pendingSeek = [this, time, negativeTolerance, positiveTolerance]() {
264             seekWithTolerance(time, negativeTolerance, positiveTolerance);
265         };
266         return;
267     }
268     m_seeking = true;
269
270     if (!metaDataAvailable())
271         return;
272
273     if (time > durationMediaTime())
274         time = durationMediaTime();
275
276     if (currentTextTrack())
277         currentTextTrack()->beginSeeking();
278
279     LOG(Media, "MediaPlayerPrivateAVFoundation::seek(%p) - seeking to %s", this, toString(time).utf8().data());
280
281     seekToTime(time, negativeTolerance, positiveTolerance);
282 }
283
284 bool MediaPlayerPrivateAVFoundation::paused() const
285 {
286     if (!metaDataAvailable())
287         return true;
288
289     return rate() == 0;
290 }
291
292 bool MediaPlayerPrivateAVFoundation::seeking() const
293 {
294     if (!metaDataAvailable())
295         return false;
296
297     return m_seeking;
298 }
299
300 FloatSize MediaPlayerPrivateAVFoundation::naturalSize() const
301 {
302     if (!metaDataAvailable())
303         return IntSize();
304
305     // In spite of the name of this method, return the natural size transformed by the 
306     // initial movie scale because the spec says intrinsic size is:
307     //
308     //    ... the dimensions of the resource in CSS pixels after taking into account the resource's 
309     //    dimensions, aspect ratio, clean aperture, resolution, and so forth, as defined for the 
310     //    format used by the resource
311
312     return m_cachedNaturalSize;
313 }
314
315 void MediaPlayerPrivateAVFoundation::setNaturalSize(FloatSize size)
316 {
317     LOG(Media, "MediaPlayerPrivateAVFoundation:setNaturalSize(%p) - size = %f x %f", this, size.width(), size.height());
318
319     FloatSize oldSize = m_cachedNaturalSize;
320     m_cachedNaturalSize = size;
321     if (oldSize != m_cachedNaturalSize)
322         m_player->sizeChanged();
323 }
324
325 void MediaPlayerPrivateAVFoundation::setHasVideo(bool b)
326 {
327     if (m_cachedHasVideo != b) {
328         m_cachedHasVideo = b;
329         characteristicsChanged();
330     }
331 }
332
333 void MediaPlayerPrivateAVFoundation::setHasAudio(bool b)
334 {
335     if (m_cachedHasAudio != b) {
336         m_cachedHasAudio = b;
337         characteristicsChanged();
338     }
339 }
340
341 void MediaPlayerPrivateAVFoundation::setHasClosedCaptions(bool b)
342 {
343     if (m_cachedHasCaptions != b) {
344         m_cachedHasCaptions = b;
345         characteristicsChanged();
346     }
347 }
348
349 void MediaPlayerPrivateAVFoundation::setNetworkState(MediaPlayer::NetworkState state)
350 {
351     if (state == m_networkState)
352         return;
353
354     m_networkState = state;
355     m_player->networkStateChanged();
356 }
357
358 void MediaPlayerPrivateAVFoundation::setReadyState(MediaPlayer::ReadyState state)
359 {
360     if (state == m_readyState)
361         return;
362
363     m_readyState = state;
364     m_player->readyStateChanged();
365 }
366
367 void MediaPlayerPrivateAVFoundation::characteristicsChanged()
368 {
369     if (m_delayCharacteristicsChangedNotification) {
370         m_characteristicsChanged = true;
371         return;
372     }
373
374     m_characteristicsChanged = false;
375     m_player->characteristicChanged();
376 }
377
378 void MediaPlayerPrivateAVFoundation::setDelayCharacteristicsChangedNotification(bool delay)
379 {
380     if (delay) {
381         m_delayCharacteristicsChangedNotification++;
382         return;
383     }
384     
385     ASSERT(m_delayCharacteristicsChangedNotification);
386     m_delayCharacteristicsChangedNotification--;
387     if (!m_delayCharacteristicsChangedNotification && m_characteristicsChanged)
388         characteristicsChanged();
389 }
390
391 std::unique_ptr<PlatformTimeRanges> MediaPlayerPrivateAVFoundation::buffered() const
392 {
393     if (!m_cachedLoadedTimeRanges)
394         m_cachedLoadedTimeRanges = platformBufferedTimeRanges();
395
396     return std::make_unique<PlatformTimeRanges>(*m_cachedLoadedTimeRanges);
397 }
398
399 MediaTime MediaPlayerPrivateAVFoundation::maxMediaTimeSeekable() const
400 {
401     if (!metaDataAvailable())
402         return MediaTime::zeroTime();
403
404     if (!m_cachedMaxTimeSeekable)
405         m_cachedMaxTimeSeekable = platformMaxTimeSeekable();
406
407     LOG(Media, "MediaPlayerPrivateAVFoundation::maxTimeSeekable(%p) - returning %s", this, toString(m_cachedMaxTimeSeekable).utf8().data());
408     return m_cachedMaxTimeSeekable;   
409 }
410
411 MediaTime MediaPlayerPrivateAVFoundation::minMediaTimeSeekable() const
412 {
413     if (!metaDataAvailable())
414         return MediaTime::zeroTime();
415
416     if (!m_cachedMinTimeSeekable)
417         m_cachedMinTimeSeekable = platformMinTimeSeekable();
418
419     LOG(Media, "MediaPlayerPrivateAVFoundation::minTimeSeekable(%p) - returning %s", this, toString(m_cachedMinTimeSeekable).utf8().data());
420     return m_cachedMinTimeSeekable;
421 }
422
423 double MediaPlayerPrivateAVFoundation::requestedRate() const
424 {
425     return m_player->requestedRate();
426 }
427
428 MediaTime MediaPlayerPrivateAVFoundation::maxTimeLoaded() const
429 {
430     if (!metaDataAvailable())
431         return MediaTime::zeroTime();
432
433     if (!m_cachedMaxTimeLoaded)
434         m_cachedMaxTimeLoaded = platformMaxTimeLoaded();
435
436     return m_cachedMaxTimeLoaded;   
437 }
438
439 bool MediaPlayerPrivateAVFoundation::didLoadingProgress() const
440 {
441     if (!durationMediaTime())
442         return false;
443     MediaTime currentMaxTimeLoaded = maxTimeLoaded();
444     bool didLoadingProgress = currentMaxTimeLoaded != m_maxTimeLoadedAtLastDidLoadingProgress;
445     m_maxTimeLoadedAtLastDidLoadingProgress = currentMaxTimeLoaded;
446
447     return didLoadingProgress;
448 }
449
450 bool MediaPlayerPrivateAVFoundation::isReadyForVideoSetup() const
451 {
452     // AVFoundation will not return true for firstVideoFrameAvailable until
453     // an AVPlayerLayer has been added to the AVPlayerItem, so allow video setup
454     // here if a video track to trigger allocation of a AVPlayerLayer.
455     return (m_isAllowedToRender || m_cachedHasVideo) && m_readyState >= MediaPlayer::HaveMetadata && m_player->visible();
456 }
457
458 void MediaPlayerPrivateAVFoundation::prepareForRendering()
459 {
460     if (m_isAllowedToRender)
461         return;
462     m_isAllowedToRender = true;
463
464     setUpVideoRendering();
465
466     if (currentRenderingMode() == MediaRenderingToLayer || preferredRenderingMode() == MediaRenderingToLayer)
467         m_player->client().mediaPlayerRenderingModeChanged(m_player);
468 }
469
470 bool MediaPlayerPrivateAVFoundation::supportsFullscreen() const
471 {
472 #if ENABLE(FULLSCREEN_API)
473     return true;
474 #else
475     // FIXME: WebVideoFullscreenController assumes a QTKit/QuickTime media engine
476 #if PLATFORM(IOS)
477     if (Settings::avKitEnabled())
478         return true;
479 #endif
480     return false;
481 #endif
482 }
483
484 void MediaPlayerPrivateAVFoundation::updateStates()
485 {
486     if (m_ignoreLoadStateChanges)
487         return;
488
489     MediaPlayer::NetworkState newNetworkState = m_networkState;
490     MediaPlayer::ReadyState newReadyState = m_readyState;
491
492     if (m_loadingMetadata)
493         newNetworkState = MediaPlayer::Loading;
494     else {
495         // -loadValuesAsynchronouslyForKeys:completionHandler: has invoked its handler; test status of keys and determine state.
496         AssetStatus assetStatus = this->assetStatus();
497         ItemStatus itemStatus = playerItemStatus();
498         
499         m_assetIsPlayable = (assetStatus == MediaPlayerAVAssetStatusPlayable);
500         if (m_readyState < MediaPlayer::HaveMetadata && assetStatus > MediaPlayerAVAssetStatusLoading) {
501             if (m_assetIsPlayable) {
502                 if (assetStatus >= MediaPlayerAVAssetStatusLoaded)
503                     newReadyState = MediaPlayer::HaveMetadata;
504                 if (itemStatus <= MediaPlayerAVPlayerItemStatusUnknown) {
505                     if (assetStatus == MediaPlayerAVAssetStatusFailed || m_preload > MediaPlayer::MetaData || isLiveStream()) {
506                         // The asset is playable but doesn't support inspection prior to playback (eg. streaming files),
507                         // or we are supposed to prepare for playback immediately, so create the player item now.
508                         newNetworkState = MediaPlayer::Loading;
509                         prepareToPlay();
510                     } else
511                         newNetworkState = MediaPlayer::Idle;
512                 }
513             } else {
514                 // FIX ME: fetch the error associated with the @"playable" key to distinguish between format 
515                 // and network errors.
516                 newNetworkState = MediaPlayer::FormatError;
517             }
518         }
519         
520         if (assetStatus >= MediaPlayerAVAssetStatusLoaded && itemStatus > MediaPlayerAVPlayerItemStatusUnknown) {
521             switch (itemStatus) {
522             case MediaPlayerAVPlayerItemStatusDoesNotExist:
523             case MediaPlayerAVPlayerItemStatusUnknown:
524             case MediaPlayerAVPlayerItemStatusFailed:
525                 break;
526
527             case MediaPlayerAVPlayerItemStatusPlaybackLikelyToKeepUp:
528             case MediaPlayerAVPlayerItemStatusPlaybackBufferFull:
529                 // If the status becomes PlaybackBufferFull, loading stops and the status will not
530                 // progress to LikelyToKeepUp. Set the readyState to  HAVE_ENOUGH_DATA, on the
531                 // presumption that if the playback buffer is full, playback will probably not stall.
532                 newReadyState = MediaPlayer::HaveEnoughData;
533                 break;
534
535             case MediaPlayerAVPlayerItemStatusReadyToPlay:
536                 if (m_readyState != MediaPlayer::HaveEnoughData && maxTimeLoaded() > currentMediaTime())
537                     newReadyState = MediaPlayer::HaveFutureData;
538                 break;
539
540             case MediaPlayerAVPlayerItemStatusPlaybackBufferEmpty:
541                 newReadyState = MediaPlayer::HaveCurrentData;
542                 break;
543             }
544
545             if (itemStatus == MediaPlayerAVPlayerItemStatusPlaybackBufferFull)
546                 newNetworkState = MediaPlayer::Idle;
547             else if (itemStatus == MediaPlayerAVPlayerItemStatusFailed)
548                 newNetworkState = MediaPlayer::DecodeError;
549             else if (itemStatus != MediaPlayerAVPlayerItemStatusPlaybackBufferFull && itemStatus >= MediaPlayerAVPlayerItemStatusReadyToPlay)
550                 newNetworkState = (maxTimeLoaded() == durationMediaTime()) ? MediaPlayer::Loaded : MediaPlayer::Loading;
551         }
552     }
553
554     if (isReadyForVideoSetup() && currentRenderingMode() != preferredRenderingMode())
555         setUpVideoRendering();
556
557     if (!m_haveReportedFirstVideoFrame && m_cachedHasVideo && hasAvailableVideoFrame()) {
558         if (m_readyState < MediaPlayer::HaveCurrentData)
559             newReadyState = MediaPlayer::HaveCurrentData;
560         m_haveReportedFirstVideoFrame = true;
561         m_player->firstVideoFrameAvailable();
562     }
563
564 #if !LOG_DISABLED
565     if (m_networkState != newNetworkState || m_readyState != newReadyState) {
566         LOG(Media, "MediaPlayerPrivateAVFoundation::updateStates(%p) - entered with networkState = %i, readyState = %i,  exiting with networkState = %i, readyState = %i",
567             this, static_cast<int>(m_networkState), static_cast<int>(m_readyState), static_cast<int>(newNetworkState), static_cast<int>(newReadyState));
568     }
569 #endif
570
571     setNetworkState(newNetworkState);
572     setReadyState(newReadyState);
573
574     if (m_playWhenFramesAvailable && hasAvailableVideoFrame()) {
575         m_playWhenFramesAvailable = false;
576         platformPlay();
577     }
578 }
579
580 void MediaPlayerPrivateAVFoundation::setSize(const IntSize&) 
581
582 }
583
584 void MediaPlayerPrivateAVFoundation::setVisible(bool visible)
585 {
586     if (m_visible == visible)
587         return;
588
589     m_visible = visible;
590     if (visible)
591         setUpVideoRendering();
592     
593     platformSetVisible(visible);
594 }
595
596 void MediaPlayerPrivateAVFoundation::acceleratedRenderingStateChanged()
597 {
598     // Set up or change the rendering path if necessary.
599     setUpVideoRendering();
600 }
601
602 void MediaPlayerPrivateAVFoundation::setShouldMaintainAspectRatio(bool maintainAspectRatio)
603 {
604     if (maintainAspectRatio == m_shouldMaintainAspectRatio)
605         return;
606
607     m_shouldMaintainAspectRatio = maintainAspectRatio;
608     updateVideoLayerGravity();
609 }
610
611 void MediaPlayerPrivateAVFoundation::metadataLoaded()
612 {
613     m_loadingMetadata = false;
614     tracksChanged();
615 }
616
617 void MediaPlayerPrivateAVFoundation::rateChanged()
618 {
619 #if ENABLE(WIRELESS_PLAYBACK_TARGET) && PLATFORM(IOS)
620     LOG(Media, "MediaPlayerPrivateAVFoundation::rateChanged(%p) - rate = %f, requested rate = %f, item status = %i", this, rate(), requestedRate(), playerItemStatus());
621     if (isCurrentPlaybackTargetWireless() && playerItemStatus() >= MediaPlayerAVPlayerItemStatusPlaybackBufferFull) {
622         double rate = this->rate();
623         if (rate != requestedRate()) {
624             m_player->handlePlaybackCommand(rate ? PlatformMediaSession::PlayCommand : PlatformMediaSession::PauseCommand);
625             return;
626         }
627     }
628 #endif
629
630     m_player->rateChanged();
631 }
632
633 void MediaPlayerPrivateAVFoundation::loadedTimeRangesChanged()
634 {
635     m_cachedLoadedTimeRanges = nullptr;
636     m_cachedMaxTimeLoaded = MediaTime::zeroTime();
637     invalidateCachedDuration();
638 }
639
640 void MediaPlayerPrivateAVFoundation::seekableTimeRangesChanged()
641 {
642     m_cachedMaxTimeSeekable = MediaTime::zeroTime();
643     m_cachedMinTimeSeekable = MediaTime::zeroTime();
644 }
645
646 void MediaPlayerPrivateAVFoundation::timeChanged(const MediaTime& time)
647 {
648     LOG(Media, "MediaPlayerPrivateAVFoundation::timeChanged(%p) - time = %s", this, toString(time).utf8().data());
649     UNUSED_PARAM(time);
650 }
651
652 void MediaPlayerPrivateAVFoundation::seekCompleted(bool finished)
653 {
654     LOG(Media, "MediaPlayerPrivateAVFoundation::seekCompleted(%p) - finished = %d", this, finished);
655     UNUSED_PARAM(finished);
656
657     m_seeking = false;
658
659     std::function<void()> pendingSeek;
660     std::swap(pendingSeek, m_pendingSeek);
661
662     if (pendingSeek) {
663         LOG(Media, "MediaPlayerPrivateAVFoundation::seekCompleted(%p) - issuing pending seek", this);
664         pendingSeek();
665         return;
666     }
667
668     if (currentTextTrack())
669         currentTextTrack()->endSeeking();
670
671     updateStates();
672     m_player->timeChanged();
673 }
674
675 void MediaPlayerPrivateAVFoundation::didEnd()
676 {
677     // Hang onto the current time and use it as duration from now on since we are definitely at
678     // the end of the movie. Do this because the initial duration is sometimes an estimate.
679     MediaTime now = currentMediaTime();
680     if (now > MediaTime::zeroTime() && !m_seeking)
681         m_cachedDuration = now;
682
683     updateStates();
684     m_player->timeChanged();
685 }
686
687 void MediaPlayerPrivateAVFoundation::invalidateCachedDuration()
688 {
689     m_cachedDuration = MediaTime::invalidTime();
690
691     // For some media files, reported duration is estimated and updated as media is loaded
692     // so report duration changed when the estimate is upated.
693     MediaTime duration = this->durationMediaTime();
694     if (duration != m_reportedDuration) {
695         if (m_reportedDuration.isValid())
696             m_player->durationChanged();
697         m_reportedDuration = duration;
698     }
699     
700 }
701
702 void MediaPlayerPrivateAVFoundation::repaint()
703 {
704     m_player->repaint();
705 }
706
707 MediaPlayer::MovieLoadType MediaPlayerPrivateAVFoundation::movieLoadType() const
708 {
709     if (!metaDataAvailable() || assetStatus() == MediaPlayerAVAssetStatusUnknown)
710         return MediaPlayer::Unknown;
711
712     if (isLiveStream())
713         return MediaPlayer::LiveStream;
714
715     return MediaPlayer::Download;
716 }
717
718 void MediaPlayerPrivateAVFoundation::setPreload(MediaPlayer::Preload preload)
719 {
720     m_preload = preload;
721     if (!m_assetURL.length())
722         return;
723
724     setDelayCallbacks(true);
725
726     if (m_preload >= MediaPlayer::MetaData && assetStatus() == MediaPlayerAVAssetStatusDoesNotExist) {
727         createAVAssetForURL(m_assetURL);
728         checkPlayability();
729     }
730
731     // Don't force creation of the player and player item unless we already know that the asset is playable. If we aren't
732     // there yet, or if we already know it is not playable, creating them now won't help.
733     if (m_preload == MediaPlayer::Auto && m_assetIsPlayable) {
734         createAVPlayerItem();
735         createAVPlayer();
736     }
737
738     setDelayCallbacks(false);
739 }
740
741 void MediaPlayerPrivateAVFoundation::setDelayCallbacks(bool delay) const
742 {
743     LockHolder lock(m_queueMutex);
744     if (delay)
745         ++m_delayCallbacks;
746     else {
747         ASSERT(m_delayCallbacks);
748         --m_delayCallbacks;
749     }
750 }
751
752 void MediaPlayerPrivateAVFoundation::mainThreadCallback()
753 {
754     LOG(Media, "MediaPlayerPrivateAVFoundation::mainThreadCallback(%p)", this);
755
756     clearMainThreadPendingFlag();
757     dispatchNotification();
758 }
759
760 void MediaPlayerPrivateAVFoundation::clearMainThreadPendingFlag()
761 {
762     LockHolder lock(m_queueMutex);
763     m_mainThreadCallPending = false;
764 }
765
766 void MediaPlayerPrivateAVFoundation::scheduleMainThreadNotification(Notification::Type type, const MediaTime& time)
767 {
768     scheduleMainThreadNotification(Notification(type, time));
769 }
770
771 void MediaPlayerPrivateAVFoundation::scheduleMainThreadNotification(Notification::Type type, bool finished)
772 {
773     scheduleMainThreadNotification(Notification(type, finished));
774 }
775
776 #if !LOG_DISABLED
777 static const char* notificationName(MediaPlayerPrivateAVFoundation::Notification& notification)
778 {
779 #define DEFINE_TYPE_STRING_CASE(type) case MediaPlayerPrivateAVFoundation::Notification::type: return #type;
780     switch (notification.type()) {
781     FOR_EACH_MEDIAPLAYERPRIVATEAVFOUNDATION_NOTIFICATION_TYPE(DEFINE_TYPE_STRING_CASE)
782     case MediaPlayerPrivateAVFoundation::Notification::FunctionType: return "FunctionType";
783     default: ASSERT_NOT_REACHED(); return "";
784     }
785 #undef DEFINE_TYPE_STRING_CASE
786 }
787 #endif // !LOG_DISABLED
788     
789
790 void MediaPlayerPrivateAVFoundation::scheduleMainThreadNotification(Notification notification)
791 {
792     if (notification.type() != Notification::FunctionType)
793         LOG(Media, "MediaPlayerPrivateAVFoundation::scheduleMainThreadNotification(%p) - notification %s", this, notificationName(notification));
794
795     m_queueMutex.lock();
796
797     // It is important to always process the properties in the order that we are notified,
798     // so always go through the queue because notifications happen on different threads.
799     m_queuedNotifications.append(notification);
800
801 #if OS(WINDOWS)
802     bool delayDispatch = true;
803 #else
804     bool delayDispatch = m_delayCallbacks || !isMainThread();
805 #endif
806     if (delayDispatch && !m_mainThreadCallPending) {
807         m_mainThreadCallPending = true;
808
809         callOnMainThread([weakThis = createWeakPtr()] {
810             if (!weakThis)
811                 return;
812
813             weakThis->mainThreadCallback();
814         });
815     }
816
817     m_queueMutex.unlock();
818
819     if (delayDispatch) {
820         if (notification.type() != Notification::FunctionType)
821             LOG(Media, "MediaPlayerPrivateAVFoundation::scheduleMainThreadNotification(%p) - early return", this);
822         return;
823     }
824
825     dispatchNotification();
826 }
827
828 void MediaPlayerPrivateAVFoundation::dispatchNotification()
829 {
830     ASSERT(isMainThread());
831
832     Notification notification = Notification();
833     {
834         LockHolder lock(m_queueMutex);
835         
836         if (m_queuedNotifications.isEmpty())
837             return;
838         
839         if (!m_delayCallbacks) {
840             // Only dispatch one notification callback per invocation because they can cause recursion.
841             notification = m_queuedNotifications.first();
842             m_queuedNotifications.remove(0);
843         }
844         
845         if (!m_queuedNotifications.isEmpty() && !m_mainThreadCallPending) {
846             callOnMainThread([weakThis = createWeakPtr()] {
847                 if (!weakThis)
848                     return;
849
850                 weakThis->mainThreadCallback();
851             });
852         }
853
854         if (!notification.isValid())
855             return;
856     }
857
858     if (notification.type() != Notification::FunctionType)
859         LOG(Media, "MediaPlayerPrivateAVFoundation::dispatchNotification(%p) - dispatching %s", this, notificationName(notification));
860
861     switch (notification.type()) {
862     case Notification::ItemDidPlayToEndTime:
863         didEnd();
864         break;
865     case Notification::ItemTracksChanged:
866         tracksChanged();
867         updateStates();
868         break;
869     case Notification::ItemStatusChanged:
870         updateStates();
871         break;
872     case Notification::ItemSeekableTimeRangesChanged:
873         seekableTimeRangesChanged();
874         updateStates();
875         break;
876     case Notification::ItemLoadedTimeRangesChanged:
877         loadedTimeRangesChanged();
878         updateStates();
879         break;
880     case Notification::ItemPresentationSizeChanged:
881         sizeChanged();
882         updateStates();
883         break;
884     case Notification::ItemIsPlaybackLikelyToKeepUpChanged:
885         updateStates();
886         break;
887     case Notification::ItemIsPlaybackBufferEmptyChanged:
888         updateStates();
889         break;
890     case Notification::ItemIsPlaybackBufferFullChanged:
891         updateStates();
892         break;
893     case Notification::PlayerRateChanged:
894         updateStates();
895         rateChanged();
896         break;
897     case Notification::PlayerTimeChanged:
898         timeChanged(notification.time());
899         break;
900     case Notification::SeekCompleted:
901         seekCompleted(notification.finished());
902         break;
903     case Notification::AssetMetadataLoaded:
904         metadataLoaded();
905         updateStates();
906         break;
907     case Notification::AssetPlayabilityKnown:
908         updateStates();
909         playabilityKnown();
910         break;
911     case Notification::DurationChanged:
912         invalidateCachedDuration();
913         break;
914     case Notification::ContentsNeedsDisplay:
915         contentsNeedsDisplay();
916         break;
917     case Notification::InbandTracksNeedConfiguration:
918         m_inbandTrackConfigurationPending = false;
919         configureInbandTracks();
920         break;
921     case Notification::FunctionType:
922         notification.function()();
923         break;
924     case Notification::TargetIsWirelessChanged:
925 #if ENABLE(WIRELESS_PLAYBACK_TARGET)
926         playbackTargetIsWirelessChanged();
927 #endif
928         break;
929
930     case Notification::None:
931         ASSERT_NOT_REACHED();
932         break;
933     }
934 }
935
936 void MediaPlayerPrivateAVFoundation::configureInbandTracks()
937 {
938     RefPtr<InbandTextTrackPrivateAVF> trackToEnable;
939     
940 #if ENABLE(AVF_CAPTIONS)
941     synchronizeTextTrackState();
942 #endif
943
944     // AVFoundation can only emit cues for one track at a time, so enable the first track that is showing, or the first that
945     // is hidden if none are showing. Otherwise disable all tracks.
946     for (unsigned i = 0; i < m_textTracks.size(); ++i) {
947         RefPtr<InbandTextTrackPrivateAVF> track = m_textTracks[i];
948         if (track->mode() == InbandTextTrackPrivate::Showing) {
949             trackToEnable = track;
950             break;
951         }
952         if (track->mode() == InbandTextTrackPrivate::Hidden)
953             trackToEnable = track;
954     }
955
956     setCurrentTextTrack(trackToEnable.get());
957 }
958
959 void MediaPlayerPrivateAVFoundation::trackModeChanged()
960 {
961     if (m_inbandTrackConfigurationPending)
962         return;
963     m_inbandTrackConfigurationPending = true;
964     scheduleMainThreadNotification(Notification::InbandTracksNeedConfiguration);
965 }
966
967 void MediaPlayerPrivateAVFoundation::clearTextTracks()
968 {
969     for (auto& track : m_textTracks) {
970         player()->removeTextTrack(*track);
971         track->disconnect();
972     }
973     m_textTracks.clear();
974 }
975
976 void MediaPlayerPrivateAVFoundation::processNewAndRemovedTextTracks(const Vector<RefPtr<InbandTextTrackPrivateAVF>>& removedTextTracks)
977 {
978     if (removedTextTracks.size()) {
979         for (unsigned i = 0; i < m_textTracks.size(); ) {
980             if (!removedTextTracks.contains(m_textTracks[i])) {
981                 ++i;
982                 continue;
983             }
984             player()->removeTextTrack(*m_textTracks[i]);
985             m_textTracks.remove(i);
986         }
987     }
988
989     unsigned inBandCount = 0;
990     for (unsigned i = 0; i < m_textTracks.size(); ++i) {
991         RefPtr<InbandTextTrackPrivateAVF> track = m_textTracks[i];
992
993 #if ENABLE(AVF_CAPTIONS)
994         if (track->textTrackCategory() == InbandTextTrackPrivateAVF::OutOfBand)
995             continue;
996 #endif
997
998         track->setTextTrackIndex(inBandCount);
999         ++inBandCount;
1000         if (track->hasBeenReported())
1001             continue;
1002         
1003         track->setHasBeenReported(true);
1004         player()->addTextTrack(*track);
1005     }
1006     LOG(Media, "MediaPlayerPrivateAVFoundation::processNewAndRemovedTextTracks(%p) - found %lu text tracks", this, m_textTracks.size());
1007 }
1008
1009 #if ENABLE(WIRELESS_PLAYBACK_TARGET)
1010
1011 void MediaPlayerPrivateAVFoundation::playbackTargetIsWirelessChanged()
1012 {
1013     if (m_player)
1014         m_player->currentPlaybackTargetIsWirelessChanged();
1015 }
1016
1017 #endif
1018
1019 #if ENABLE(LEGACY_ENCRYPTED_MEDIA)
1020
1021 bool MediaPlayerPrivateAVFoundation::extractKeyURIKeyIDAndCertificateFromInitData(Uint8Array* initData, String& keyURI, String& keyID, RefPtr<Uint8Array>& certificate)
1022 {
1023     // initData should have the following layout:
1024     // [4 bytes: keyURI length][N bytes: keyURI][4 bytes: contentID length], [N bytes: contentID], [4 bytes: certificate length][N bytes: certificate]
1025     if (initData->byteLength() < 4)
1026         return false;
1027
1028     RefPtr<ArrayBuffer> initDataBuffer = initData->unsharedBuffer();
1029
1030     // Use a DataView to read uint32 values from the buffer, as Uint32Array requires the reads be aligned on 4-byte boundaries. 
1031     RefPtr<JSC::DataView> initDataView = JSC::DataView::create(initDataBuffer, 0, initDataBuffer->byteLength());
1032     uint32_t offset = 0;
1033     bool status = true;
1034
1035     uint32_t keyURILength = initDataView->get<uint32_t>(offset, true, &status);
1036     offset += 4;
1037     if (!status || offset + keyURILength > initData->length())
1038         return false;
1039
1040     RefPtr<Uint16Array> keyURIArray = Uint16Array::create(initDataBuffer, offset, keyURILength);
1041     if (!keyURIArray)
1042         return false;
1043
1044     keyURI = String(reinterpret_cast<UChar*>(keyURIArray->data()), keyURILength / sizeof(unsigned short));
1045     offset += keyURILength;
1046
1047     uint32_t keyIDLength = initDataView->get<uint32_t>(offset, true, &status);
1048     offset += 4;
1049     if (!status || offset + keyIDLength > initData->length())
1050         return false;
1051
1052     RefPtr<Uint8Array> keyIDArray = Uint8Array::create(initDataBuffer, offset, keyIDLength);
1053     if (!keyIDArray)
1054         return false;
1055
1056     keyID = String(reinterpret_cast<UChar*>(keyIDArray->data()), keyIDLength / sizeof(unsigned short));
1057     offset += keyIDLength;
1058
1059     uint32_t certificateLength = initDataView->get<uint32_t>(offset, true, &status);
1060     offset += 4;
1061     if (!status || offset + certificateLength > initData->length())
1062         return false;
1063
1064     certificate = Uint8Array::create(initDataBuffer, offset, certificateLength);
1065     if (!certificate)
1066         return false;
1067
1068     return true;
1069 }
1070
1071 #endif
1072
1073 URL MediaPlayerPrivateAVFoundation::resolvedURL() const
1074 {
1075     if (!m_assetURL.length())
1076         return URL();
1077
1078     return URL(ParsedURLString, m_assetURL);
1079 }
1080
1081 bool MediaPlayerPrivateAVFoundation::canSaveMediaData() const
1082 {
1083     URL url = resolvedURL();
1084
1085     if (url.isLocalFile())
1086         return true;
1087
1088     if (!url.protocolIsInHTTPFamily())
1089         return false;
1090
1091     if (isLiveStream())
1092         return false;
1093
1094     return true;
1095 }
1096
1097 bool MediaPlayerPrivateAVFoundation::isUnsupportedMIMEType(const String& type)
1098 {
1099     String lowerCaseType = type.convertToASCIILowercase();
1100
1101     // AVFoundation will return non-video MIME types which it claims to support, but which we
1102     // do not support in the <video> element. Reject all non video/, audio/, and application/ types.
1103     if (!lowerCaseType.startsWith("video/") && !lowerCaseType.startsWith("audio/") && !lowerCaseType.startsWith("application/"))
1104         return true;
1105
1106     // Reject types we know AVFoundation does not support that sites commonly ask about.
1107     if (lowerCaseType == "video/webm" || lowerCaseType == "audio/webm" || lowerCaseType == "video/x-webm")
1108         return true;
1109
1110     if (lowerCaseType == "video/x-flv")
1111         return true;
1112
1113     if (lowerCaseType == "audio/ogg" || lowerCaseType == "video/ogg" || lowerCaseType == "application/ogg")
1114         return true;
1115
1116     if (lowerCaseType == "video/h264")
1117         return true;
1118
1119     return false;
1120 }
1121
1122 const HashSet<String, ASCIICaseInsensitiveHash>& MediaPlayerPrivateAVFoundation::staticMIMETypeList()
1123 {
1124     static NeverDestroyed<HashSet<String, ASCIICaseInsensitiveHash>> cache = []() {
1125         HashSet<String, ASCIICaseInsensitiveHash> types;
1126
1127         static const char* const typeNames[] = {
1128             "application/vnd.apple.mpegurl",
1129             "application/x-mpegurl",
1130             "audio/3gpp",
1131             "audio/aac",
1132             "audio/aacp",
1133             "audio/aiff",
1134             "audio/basic",
1135             "audio/mp3",
1136             "audio/mp4",
1137             "audio/mpeg",
1138             "audio/mpeg3",
1139             "audio/mpegurl",
1140             "audio/mpg",
1141             "audio/wav",
1142             "audio/wave",
1143             "audio/x-aac",
1144             "audio/x-aiff",
1145             "audio/x-m4a",
1146             "audio/x-mpegurl",
1147             "audio/x-wav",
1148             "video/3gpp",
1149             "video/3gpp2",
1150             "video/mp4",
1151             "video/mpeg",
1152             "video/mpeg2",
1153             "video/mpg",
1154             "video/quicktime",
1155             "video/x-m4v",
1156             "video/x-mpeg",
1157             "video/x-mpg",
1158         };
1159         for (auto& type : typeNames)
1160             types.add(type);
1161
1162         return types;
1163     }();
1164
1165     return cache;
1166 }
1167
1168 } // namespace WebCore
1169
1170 #endif