Unreviewed, roll out http://trac.webkit.org/changeset/187972.
[WebKit-https.git] / Source / WebCore / platform / graphics / avfoundation / MediaPlayerPrivateAVFoundation.cpp
1 /*
2  * Copyright (C) 2011-2014 Apple Inc. All rights reserved.
3  *
4  * Redistribution and use in source and binary forms, with or without
5  * modification, are permitted provided that the following conditions
6  * are met:
7  * 1. Redistributions of source code must retain the above copyright
8  *    notice, this list of conditions and the following disclaimer.
9  * 2. Redistributions in binary form must reproduce the above copyright
10  *    notice, this list of conditions and the following disclaimer in the
11  *    documentation and/or other materials provided with the distribution.
12  *
13  * THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY
14  * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
15  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
16  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL APPLE INC. OR
17  * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
18  * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
19  * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
20  * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
21  * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
22  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
23  * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 
24  */
25
26 #include "config.h"
27
28 #if ENABLE(VIDEO) && USE(AVFOUNDATION)
29
30 #include "MediaPlayerPrivateAVFoundation.h"
31
32 #include "DocumentLoader.h"
33 #include "FloatConversion.h"
34 #include "GraphicsContext.h"
35 #include "InbandTextTrackPrivateAVF.h"
36 #include "InbandTextTrackPrivateClient.h"
37 #include "URL.h"
38 #include "Logging.h"
39 #include "PlatformLayer.h"
40 #include "PlatformTimeRanges.h"
41 #include "Settings.h"
42 #include "SoftLinking.h"
43 #include <CoreMedia/CoreMedia.h>
44 #include <runtime/DataView.h>
45 #include <runtime/Uint16Array.h>
46 #include <wtf/MainThread.h>
47 #include <wtf/NeverDestroyed.h>
48 #include <wtf/text/CString.h>
49 #include <wtf/StringPrintStream.h>
50
51 namespace WebCore {
52
53 MediaPlayerPrivateAVFoundation::MediaPlayerPrivateAVFoundation(MediaPlayer* player)
54     : m_player(player)
55     , m_weakPtrFactory(this)
56     , m_queuedNotifications()
57     , m_queueMutex()
58     , m_networkState(MediaPlayer::Empty)
59     , m_readyState(MediaPlayer::HaveNothing)
60     , m_preload(MediaPlayer::Auto)
61     , m_cachedDuration(MediaTime::invalidTime())
62     , m_reportedDuration(MediaTime::invalidTime())
63     , m_maxTimeLoadedAtLastDidLoadingProgress(MediaTime::invalidTime())
64     , m_delayCallbacks(0)
65     , m_delayCharacteristicsChangedNotification(0)
66     , m_mainThreadCallPending(false)
67     , m_assetIsPlayable(false)
68     , m_visible(false)
69     , m_loadingMetadata(false)
70     , m_isAllowedToRender(false)
71     , m_cachedHasAudio(false)
72     , m_cachedHasVideo(false)
73     , m_cachedHasCaptions(false)
74     , m_ignoreLoadStateChanges(false)
75     , m_haveReportedFirstVideoFrame(false)
76     , m_playWhenFramesAvailable(false)
77     , m_inbandTrackConfigurationPending(false)
78     , m_characteristicsChanged(false)
79     , m_shouldMaintainAspectRatio(true)
80     , m_seeking(false)
81 {
82     LOG(Media, "MediaPlayerPrivateAVFoundation::MediaPlayerPrivateAVFoundation(%p)", this);
83 }
84
85 MediaPlayerPrivateAVFoundation::~MediaPlayerPrivateAVFoundation()
86 {
87     LOG(Media, "MediaPlayerPrivateAVFoundation::~MediaPlayerPrivateAVFoundation(%p)", this);
88     setIgnoreLoadStateChanges(true);
89     cancelCallOnMainThread(mainThreadCallback, this);
90 }
91
92 MediaPlayerPrivateAVFoundation::MediaRenderingMode MediaPlayerPrivateAVFoundation::currentRenderingMode() const
93 {
94     if (platformLayer())
95         return MediaRenderingToLayer;
96
97     if (hasContextRenderer())
98         return MediaRenderingToContext;
99
100     return MediaRenderingNone;
101 }
102
103 MediaPlayerPrivateAVFoundation::MediaRenderingMode MediaPlayerPrivateAVFoundation::preferredRenderingMode() const
104 {
105     if (!m_player->visible() || assetStatus() == MediaPlayerAVAssetStatusUnknown)
106         return MediaRenderingNone;
107
108     if (supportsAcceleratedRendering() && m_player->client().mediaPlayerRenderingCanBeAccelerated(m_player))
109         return MediaRenderingToLayer;
110
111     return MediaRenderingToContext;
112 }
113
114 void MediaPlayerPrivateAVFoundation::setUpVideoRendering()
115 {
116     if (!isReadyForVideoSetup())
117         return;
118
119     MediaRenderingMode currentMode = currentRenderingMode();
120     MediaRenderingMode preferredMode = preferredRenderingMode();
121
122     if (preferredMode == MediaRenderingNone)
123         preferredMode = MediaRenderingToContext;
124
125     if (currentMode == preferredMode && currentMode != MediaRenderingNone)
126         return;
127
128     LOG(Media, "MediaPlayerPrivateAVFoundation::setUpVideoRendering(%p) - current mode = %d, preferred mode = %d", 
129         this, static_cast<int>(currentMode), static_cast<int>(preferredMode));
130
131     if (currentMode != MediaRenderingNone)  
132         tearDownVideoRendering();
133
134     switch (preferredMode) {
135     case MediaRenderingNone:
136     case MediaRenderingToContext:
137         createContextVideoRenderer();
138         break;
139
140     case MediaRenderingToLayer:
141         createVideoLayer();
142         break;
143     }
144
145     // If using a movie layer, inform the client so the compositing tree is updated.
146     if (currentMode == MediaRenderingToLayer || preferredMode == MediaRenderingToLayer) {
147         LOG(Media, "MediaPlayerPrivateAVFoundation::setUpVideoRendering(%p) - calling mediaPlayerRenderingModeChanged()", this);
148         m_player->client().mediaPlayerRenderingModeChanged(m_player);
149     }
150 }
151
152 void MediaPlayerPrivateAVFoundation::tearDownVideoRendering()
153 {
154     LOG(Media, "MediaPlayerPrivateAVFoundation::tearDownVideoRendering(%p)", this);
155
156     destroyContextVideoRenderer();
157
158     if (platformLayer())
159         destroyVideoLayer();
160 }
161
162 bool MediaPlayerPrivateAVFoundation::hasSetUpVideoRendering() const
163 {
164     return hasLayerRenderer() || hasContextRenderer();
165 }
166
167 void MediaPlayerPrivateAVFoundation::load(const String& url)
168 {
169     LOG(Media, "MediaPlayerPrivateAVFoundation::load(%p)", this);
170
171     setNetworkState(m_preload == MediaPlayer::None ? MediaPlayer::Idle : MediaPlayer::Loading);
172     setReadyState(MediaPlayer::HaveNothing);
173
174     m_assetURL = url;
175
176     // Don't do any more work if the url is empty.
177     if (!url.length())
178         return;
179
180     setPreload(m_preload);
181 }
182
183 #if ENABLE(MEDIA_SOURCE)
184 void MediaPlayerPrivateAVFoundation::load(const String&, MediaSourcePrivateClient*)
185 {
186     setNetworkState(MediaPlayer::FormatError);
187 }
188 #endif
189
190
191 void MediaPlayerPrivateAVFoundation::playabilityKnown()
192 {
193     LOG(Media, "MediaPlayerPrivateAVFoundation::playabilityKnown(%p)", this);
194
195     if (m_assetIsPlayable)
196         return;
197
198     // Nothing more to do if we already have all of the item's metadata.
199     if (assetStatus() > MediaPlayerAVAssetStatusLoading) {
200         LOG(Media, "MediaPlayerPrivateAVFoundation::playabilityKnown(%p) - all metadata loaded", this);
201         return;
202     }
203
204     // At this point we are supposed to load metadata. It is OK to ask the asset to load the same 
205     // information multiple times, because if it has already been loaded the completion handler 
206     // will just be called synchronously.
207     m_loadingMetadata = true;
208     beginLoadingMetadata();
209 }
210
211 void MediaPlayerPrivateAVFoundation::prepareToPlay()
212 {
213     LOG(Media, "MediaPlayerPrivateAVFoundation::prepareToPlay(%p)", this);
214
215     setPreload(MediaPlayer::Auto);
216 }
217
218 void MediaPlayerPrivateAVFoundation::play()
219 {
220     LOG(Media, "MediaPlayerPrivateAVFoundation::play(%p)", this);
221
222     // If the file has video, don't request playback until the first frame of video is ready to display
223     // or the audio may start playing before we can render video.
224     if (!m_cachedHasVideo || hasAvailableVideoFrame())
225         platformPlay();
226     else {
227         LOG(Media, "MediaPlayerPrivateAVFoundation::play(%p) - waiting for first video frame", this);
228         m_playWhenFramesAvailable = true;
229     }
230 }
231
232 void MediaPlayerPrivateAVFoundation::pause()
233 {
234     LOG(Media, "MediaPlayerPrivateAVFoundation::pause(%p)", this);
235     m_playWhenFramesAvailable = false;
236     platformPause();
237 }
238
239 MediaTime MediaPlayerPrivateAVFoundation::durationMediaTime() const
240 {
241     if (m_cachedDuration.isValid())
242         return m_cachedDuration;
243
244     MediaTime duration = platformDuration();
245     if (!duration || duration.isInvalid())
246         return MediaTime::zeroTime();
247
248     m_cachedDuration = duration;
249     LOG(Media, "MediaPlayerPrivateAVFoundation::duration(%p) - caching %s", this, toString(m_cachedDuration).utf8().data());
250     return m_cachedDuration;
251 }
252
253 void MediaPlayerPrivateAVFoundation::seek(const MediaTime& time)
254 {
255     seekWithTolerance(time, MediaTime::zeroTime(), MediaTime::zeroTime());
256 }
257
258 void MediaPlayerPrivateAVFoundation::seekWithTolerance(const MediaTime& mediaTime, const MediaTime& negativeTolerance, const MediaTime& positiveTolerance)
259 {
260     MediaTime time = mediaTime;
261
262     if (m_seeking) {
263         LOG(Media, "MediaPlayerPrivateAVFoundation::seekWithTolerance(%p) - save pending seek", this);
264         m_pendingSeek = [this, time, negativeTolerance, positiveTolerance]() {
265             seekWithTolerance(time, negativeTolerance, positiveTolerance);
266         };
267         return;
268     }
269     m_seeking = true;
270
271     if (!metaDataAvailable())
272         return;
273
274     if (time > durationMediaTime())
275         time = durationMediaTime();
276
277     if (currentMediaTime() == time)
278         return;
279
280     if (currentTextTrack())
281         currentTextTrack()->beginSeeking();
282
283     LOG(Media, "MediaPlayerPrivateAVFoundation::seek(%p) - seeking to %s", this, toString(time).utf8().data());
284
285     seekToTime(time, negativeTolerance, positiveTolerance);
286 }
287
288 bool MediaPlayerPrivateAVFoundation::paused() const
289 {
290     if (!metaDataAvailable())
291         return true;
292
293     return rate() == 0;
294 }
295
296 bool MediaPlayerPrivateAVFoundation::seeking() const
297 {
298     if (!metaDataAvailable())
299         return false;
300
301     return m_seeking;
302 }
303
304 FloatSize MediaPlayerPrivateAVFoundation::naturalSize() const
305 {
306     if (!metaDataAvailable())
307         return IntSize();
308
309     // In spite of the name of this method, return the natural size transformed by the 
310     // initial movie scale because the spec says intrinsic size is:
311     //
312     //    ... the dimensions of the resource in CSS pixels after taking into account the resource's 
313     //    dimensions, aspect ratio, clean aperture, resolution, and so forth, as defined for the 
314     //    format used by the resource
315
316     return m_cachedNaturalSize;
317 }
318
319 void MediaPlayerPrivateAVFoundation::setNaturalSize(FloatSize size)
320 {
321     LOG(Media, "MediaPlayerPrivateAVFoundation:setNaturalSize(%p) - size = %f x %f", this, size.width(), size.height());
322
323     FloatSize oldSize = m_cachedNaturalSize;
324     m_cachedNaturalSize = size;
325     if (oldSize != m_cachedNaturalSize)
326         m_player->sizeChanged();
327 }
328
329 void MediaPlayerPrivateAVFoundation::setHasVideo(bool b)
330 {
331     if (m_cachedHasVideo != b) {
332         m_cachedHasVideo = b;
333         characteristicsChanged();
334     }
335 }
336
337 void MediaPlayerPrivateAVFoundation::setHasAudio(bool b)
338 {
339     if (m_cachedHasAudio != b) {
340         m_cachedHasAudio = b;
341         characteristicsChanged();
342     }
343 }
344
345 void MediaPlayerPrivateAVFoundation::setHasClosedCaptions(bool b)
346 {
347     if (m_cachedHasCaptions != b) {
348         m_cachedHasCaptions = b;
349         characteristicsChanged();
350     }
351 }
352
353 void MediaPlayerPrivateAVFoundation::setNetworkState(MediaPlayer::NetworkState state)
354 {
355     if (state == m_networkState)
356         return;
357
358     m_networkState = state;
359     m_player->networkStateChanged();
360 }
361
362 void MediaPlayerPrivateAVFoundation::setReadyState(MediaPlayer::ReadyState state)
363 {
364     if (state == m_readyState)
365         return;
366
367     m_readyState = state;
368     m_player->readyStateChanged();
369 }
370
371 void MediaPlayerPrivateAVFoundation::characteristicsChanged()
372 {
373     if (m_delayCharacteristicsChangedNotification) {
374         m_characteristicsChanged = true;
375         return;
376     }
377
378     m_characteristicsChanged = false;
379     m_player->characteristicChanged();
380 }
381
382 void MediaPlayerPrivateAVFoundation::setDelayCharacteristicsChangedNotification(bool delay)
383 {
384     if (delay) {
385         m_delayCharacteristicsChangedNotification++;
386         return;
387     }
388     
389     ASSERT(m_delayCharacteristicsChangedNotification);
390     m_delayCharacteristicsChangedNotification--;
391     if (!m_delayCharacteristicsChangedNotification && m_characteristicsChanged)
392         characteristicsChanged();
393 }
394
395 std::unique_ptr<PlatformTimeRanges> MediaPlayerPrivateAVFoundation::buffered() const
396 {
397     if (!m_cachedLoadedTimeRanges)
398         m_cachedLoadedTimeRanges = platformBufferedTimeRanges();
399
400     return std::make_unique<PlatformTimeRanges>(*m_cachedLoadedTimeRanges);
401 }
402
403 MediaTime MediaPlayerPrivateAVFoundation::maxMediaTimeSeekable() const
404 {
405     if (!metaDataAvailable())
406         return MediaTime::zeroTime();
407
408     if (!m_cachedMaxTimeSeekable)
409         m_cachedMaxTimeSeekable = platformMaxTimeSeekable();
410
411     LOG(Media, "MediaPlayerPrivateAVFoundation::maxTimeSeekable(%p) - returning %s", this, toString(m_cachedMaxTimeSeekable).utf8().data());
412     return m_cachedMaxTimeSeekable;   
413 }
414
415 MediaTime MediaPlayerPrivateAVFoundation::minMediaTimeSeekable() const
416 {
417     if (!metaDataAvailable())
418         return MediaTime::zeroTime();
419
420     if (!m_cachedMinTimeSeekable)
421         m_cachedMinTimeSeekable = platformMinTimeSeekable();
422
423     LOG(Media, "MediaPlayerPrivateAVFoundation::minTimeSeekable(%p) - returning %s", this, toString(m_cachedMinTimeSeekable).utf8().data());
424     return m_cachedMinTimeSeekable;
425 }
426
427 double MediaPlayerPrivateAVFoundation::requestedRate() const
428 {
429     return m_player->requestedRate();
430 }
431
432 MediaTime MediaPlayerPrivateAVFoundation::maxTimeLoaded() const
433 {
434     if (!metaDataAvailable())
435         return MediaTime::zeroTime();
436
437     if (!m_cachedMaxTimeLoaded)
438         m_cachedMaxTimeLoaded = platformMaxTimeLoaded();
439
440     return m_cachedMaxTimeLoaded;   
441 }
442
443 bool MediaPlayerPrivateAVFoundation::didLoadingProgress() const
444 {
445     if (!durationMediaTime())
446         return false;
447     MediaTime currentMaxTimeLoaded = maxTimeLoaded();
448     bool didLoadingProgress = currentMaxTimeLoaded != m_maxTimeLoadedAtLastDidLoadingProgress;
449     m_maxTimeLoadedAtLastDidLoadingProgress = currentMaxTimeLoaded;
450
451     return didLoadingProgress;
452 }
453
454 bool MediaPlayerPrivateAVFoundation::isReadyForVideoSetup() const
455 {
456     // AVFoundation will not return true for firstVideoFrameAvailable until
457     // an AVPlayerLayer has been added to the AVPlayerItem, so allow video setup
458     // here if a video track to trigger allocation of a AVPlayerLayer.
459     return (m_isAllowedToRender || m_cachedHasVideo) && m_readyState >= MediaPlayer::HaveMetadata && m_player->visible();
460 }
461
462 void MediaPlayerPrivateAVFoundation::prepareForRendering()
463 {
464     if (m_isAllowedToRender)
465         return;
466     m_isAllowedToRender = true;
467
468     setUpVideoRendering();
469
470     if (currentRenderingMode() == MediaRenderingToLayer || preferredRenderingMode() == MediaRenderingToLayer)
471         m_player->client().mediaPlayerRenderingModeChanged(m_player);
472 }
473
474 bool MediaPlayerPrivateAVFoundation::supportsFullscreen() const
475 {
476 #if ENABLE(FULLSCREEN_API)
477     return true;
478 #else
479     // FIXME: WebVideoFullscreenController assumes a QTKit/QuickTime media engine
480 #if PLATFORM(IOS)
481     if (Settings::avKitEnabled())
482         return true;
483 #endif
484     return false;
485 #endif
486 }
487
488 void MediaPlayerPrivateAVFoundation::updateStates()
489 {
490     if (m_ignoreLoadStateChanges)
491         return;
492
493     MediaPlayer::NetworkState newNetworkState = m_networkState;
494     MediaPlayer::ReadyState newReadyState = m_readyState;
495
496     if (m_loadingMetadata)
497         newNetworkState = MediaPlayer::Loading;
498     else {
499         // -loadValuesAsynchronouslyForKeys:completionHandler: has invoked its handler; test status of keys and determine state.
500         AssetStatus assetStatus = this->assetStatus();
501         ItemStatus itemStatus = playerItemStatus();
502         
503         m_assetIsPlayable = (assetStatus == MediaPlayerAVAssetStatusPlayable);
504         if (m_readyState < MediaPlayer::HaveMetadata && assetStatus > MediaPlayerAVAssetStatusLoading) {
505             if (m_assetIsPlayable) {
506                 if (assetStatus >= MediaPlayerAVAssetStatusLoaded)
507                     newReadyState = MediaPlayer::HaveMetadata;
508                 if (itemStatus <= MediaPlayerAVPlayerItemStatusUnknown) {
509                     if (assetStatus == MediaPlayerAVAssetStatusFailed || m_preload > MediaPlayer::MetaData || isLiveStream()) {
510                         // The asset is playable but doesn't support inspection prior to playback (eg. streaming files),
511                         // or we are supposed to prepare for playback immediately, so create the player item now.
512                         newNetworkState = MediaPlayer::Loading;
513                         prepareToPlay();
514                     } else
515                         newNetworkState = MediaPlayer::Idle;
516                 }
517             } else {
518                 // FIX ME: fetch the error associated with the @"playable" key to distinguish between format 
519                 // and network errors.
520                 newNetworkState = MediaPlayer::FormatError;
521             }
522         }
523         
524         if (assetStatus >= MediaPlayerAVAssetStatusLoaded && itemStatus > MediaPlayerAVPlayerItemStatusUnknown) {
525             switch (itemStatus) {
526             case MediaPlayerAVPlayerItemStatusDoesNotExist:
527             case MediaPlayerAVPlayerItemStatusUnknown:
528             case MediaPlayerAVPlayerItemStatusFailed:
529                 break;
530
531             case MediaPlayerAVPlayerItemStatusPlaybackLikelyToKeepUp:
532             case MediaPlayerAVPlayerItemStatusPlaybackBufferFull:
533                 // If the status becomes PlaybackBufferFull, loading stops and the status will not
534                 // progress to LikelyToKeepUp. Set the readyState to  HAVE_ENOUGH_DATA, on the
535                 // presumption that if the playback buffer is full, playback will probably not stall.
536                 newReadyState = MediaPlayer::HaveEnoughData;
537                 break;
538
539             case MediaPlayerAVPlayerItemStatusReadyToPlay:
540                 if (m_readyState != MediaPlayer::HaveEnoughData && maxTimeLoaded() > currentMediaTime())
541                     newReadyState = MediaPlayer::HaveFutureData;
542                 break;
543
544             case MediaPlayerAVPlayerItemStatusPlaybackBufferEmpty:
545                 newReadyState = MediaPlayer::HaveCurrentData;
546                 break;
547             }
548
549             if (itemStatus == MediaPlayerAVPlayerItemStatusPlaybackBufferFull)
550                 newNetworkState = MediaPlayer::Idle;
551             else if (itemStatus == MediaPlayerAVPlayerItemStatusFailed)
552                 newNetworkState = MediaPlayer::DecodeError;
553             else if (itemStatus != MediaPlayerAVPlayerItemStatusPlaybackBufferFull && itemStatus >= MediaPlayerAVPlayerItemStatusReadyToPlay)
554                 newNetworkState = (maxTimeLoaded() == durationMediaTime()) ? MediaPlayer::Loaded : MediaPlayer::Loading;
555         }
556     }
557
558     if (isReadyForVideoSetup() && currentRenderingMode() != preferredRenderingMode())
559         setUpVideoRendering();
560
561     if (!m_haveReportedFirstVideoFrame && m_cachedHasVideo && hasAvailableVideoFrame()) {
562         if (m_readyState < MediaPlayer::HaveCurrentData)
563             newReadyState = MediaPlayer::HaveCurrentData;
564         m_haveReportedFirstVideoFrame = true;
565         m_player->firstVideoFrameAvailable();
566     }
567
568 #if !LOG_DISABLED
569     if (m_networkState != newNetworkState || m_readyState != newReadyState) {
570         LOG(Media, "MediaPlayerPrivateAVFoundation::updateStates(%p) - entered with networkState = %i, readyState = %i,  exiting with networkState = %i, readyState = %i",
571             this, static_cast<int>(m_networkState), static_cast<int>(m_readyState), static_cast<int>(newNetworkState), static_cast<int>(newReadyState));
572     }
573 #endif
574
575     setNetworkState(newNetworkState);
576     setReadyState(newReadyState);
577
578     if (m_playWhenFramesAvailable && hasAvailableVideoFrame()) {
579         m_playWhenFramesAvailable = false;
580         platformPlay();
581     }
582 }
583
584 void MediaPlayerPrivateAVFoundation::setSize(const IntSize&) 
585
586 }
587
588 void MediaPlayerPrivateAVFoundation::setVisible(bool visible)
589 {
590     if (m_visible == visible)
591         return;
592
593     m_visible = visible;
594     if (visible)
595         setUpVideoRendering();
596     
597     platformSetVisible(visible);
598 }
599
600 void MediaPlayerPrivateAVFoundation::acceleratedRenderingStateChanged()
601 {
602     // Set up or change the rendering path if necessary.
603     setUpVideoRendering();
604 }
605
606 void MediaPlayerPrivateAVFoundation::setShouldMaintainAspectRatio(bool maintainAspectRatio)
607 {
608     if (maintainAspectRatio == m_shouldMaintainAspectRatio)
609         return;
610
611     m_shouldMaintainAspectRatio = maintainAspectRatio;
612     updateVideoLayerGravity();
613 }
614
615 void MediaPlayerPrivateAVFoundation::metadataLoaded()
616 {
617     m_loadingMetadata = false;
618     tracksChanged();
619 }
620
621 void MediaPlayerPrivateAVFoundation::rateChanged()
622 {
623 #if ENABLE(WIRELESS_PLAYBACK_TARGET) && PLATFORM(IOS)
624     if (isCurrentPlaybackTargetWireless())
625         m_player->handlePlaybackCommand(rate() ? PlatformMediaSession::PlayCommand : PlatformMediaSession::PauseCommand);
626 #endif
627
628     m_player->rateChanged();
629 }
630
631 void MediaPlayerPrivateAVFoundation::loadedTimeRangesChanged()
632 {
633     m_cachedLoadedTimeRanges = nullptr;
634     m_cachedMaxTimeLoaded = MediaTime::zeroTime();
635     invalidateCachedDuration();
636 }
637
638 void MediaPlayerPrivateAVFoundation::seekableTimeRangesChanged()
639 {
640     m_cachedMaxTimeSeekable = MediaTime::zeroTime();
641     m_cachedMinTimeSeekable = MediaTime::zeroTime();
642 }
643
644 void MediaPlayerPrivateAVFoundation::timeChanged(const MediaTime& time)
645 {
646     LOG(Media, "MediaPlayerPrivateAVFoundation::timeChanged(%p) - time = %s", this, toString(time).utf8().data());
647     UNUSED_PARAM(time);
648 }
649
650 void MediaPlayerPrivateAVFoundation::seekCompleted(bool finished)
651 {
652     LOG(Media, "MediaPlayerPrivateAVFoundation::seekCompleted(%p) - finished = %d", this, finished);
653     UNUSED_PARAM(finished);
654
655     m_seeking = false;
656
657     std::function<void()> pendingSeek;
658     std::swap(pendingSeek, m_pendingSeek);
659
660     if (pendingSeek) {
661         LOG(Media, "MediaPlayerPrivateAVFoundation::seekCompleted(%p) - issuing pending seek", this);
662         pendingSeek();
663         return;
664     }
665
666     if (currentTextTrack())
667         currentTextTrack()->endSeeking();
668
669     updateStates();
670     m_player->timeChanged();
671 }
672
673 void MediaPlayerPrivateAVFoundation::didEnd()
674 {
675     // Hang onto the current time and use it as duration from now on since we are definitely at
676     // the end of the movie. Do this because the initial duration is sometimes an estimate.
677     MediaTime now = currentMediaTime();
678     if (now > MediaTime::zeroTime())
679         m_cachedDuration = now;
680
681     updateStates();
682     m_player->timeChanged();
683 }
684
685 void MediaPlayerPrivateAVFoundation::invalidateCachedDuration()
686 {
687     LOG(Media, "MediaPlayerPrivateAVFoundation::invalidateCachedDuration(%p)", this);
688     
689     m_cachedDuration = MediaTime::invalidTime();
690
691     // For some media files, reported duration is estimated and updated as media is loaded
692     // so report duration changed when the estimate is upated.
693     MediaTime duration = this->durationMediaTime();
694     if (duration != m_reportedDuration) {
695         if (m_reportedDuration.isValid())
696             m_player->durationChanged();
697         m_reportedDuration = duration;
698     }
699     
700 }
701
702 void MediaPlayerPrivateAVFoundation::repaint()
703 {
704     m_player->repaint();
705 }
706
707 MediaPlayer::MovieLoadType MediaPlayerPrivateAVFoundation::movieLoadType() const
708 {
709     if (!metaDataAvailable() || assetStatus() == MediaPlayerAVAssetStatusUnknown)
710         return MediaPlayer::Unknown;
711
712     if (isLiveStream())
713         return MediaPlayer::LiveStream;
714
715     return MediaPlayer::Download;
716 }
717
718 void MediaPlayerPrivateAVFoundation::setPreload(MediaPlayer::Preload preload)
719 {
720     m_preload = preload;
721     if (!m_assetURL.length())
722         return;
723
724     setDelayCallbacks(true);
725
726     if (m_preload >= MediaPlayer::MetaData && assetStatus() == MediaPlayerAVAssetStatusDoesNotExist) {
727         createAVAssetForURL(m_assetURL);
728         checkPlayability();
729     }
730
731     // Don't force creation of the player and player item unless we already know that the asset is playable. If we aren't
732     // there yet, or if we already know it is not playable, creating them now won't help.
733     if (m_preload == MediaPlayer::Auto && m_assetIsPlayable) {
734         createAVPlayerItem();
735         createAVPlayer();
736     }
737
738     setDelayCallbacks(false);
739 }
740
741 void MediaPlayerPrivateAVFoundation::setDelayCallbacks(bool delay) const
742 {
743     MutexLocker lock(m_queueMutex);
744     if (delay)
745         ++m_delayCallbacks;
746     else {
747         ASSERT(m_delayCallbacks);
748         --m_delayCallbacks;
749     }
750 }
751
752 void MediaPlayerPrivateAVFoundation::mainThreadCallback(void* context)
753 {
754     LOG(Media, "MediaPlayerPrivateAVFoundation::mainThreadCallback(%p)", context);
755     MediaPlayerPrivateAVFoundation* player = static_cast<MediaPlayerPrivateAVFoundation*>(context);
756     player->clearMainThreadPendingFlag();
757     player->dispatchNotification();
758 }
759
760 void MediaPlayerPrivateAVFoundation::clearMainThreadPendingFlag()
761 {
762     MutexLocker lock(m_queueMutex);
763     m_mainThreadCallPending = false;
764 }
765
766 void MediaPlayerPrivateAVFoundation::scheduleMainThreadNotification(Notification::Type type, const MediaTime& time)
767 {
768     scheduleMainThreadNotification(Notification(type, time));
769 }
770
771 void MediaPlayerPrivateAVFoundation::scheduleMainThreadNotification(Notification::Type type, bool finished)
772 {
773     scheduleMainThreadNotification(Notification(type, finished));
774 }
775
776 #if !LOG_DISABLED
777 static const char* notificationName(MediaPlayerPrivateAVFoundation::Notification& notification)
778 {
779 #define DEFINE_TYPE_STRING_CASE(type) case MediaPlayerPrivateAVFoundation::Notification::type: return #type;
780     switch (notification.type()) {
781     FOR_EACH_MEDIAPLAYERPRIVATEAVFOUNDATION_NOTIFICATION_TYPE(DEFINE_TYPE_STRING_CASE)
782     case MediaPlayerPrivateAVFoundation::Notification::FunctionType: return "FunctionType";
783     default: ASSERT_NOT_REACHED(); return "";
784     }
785 #undef DEFINE_TYPE_STRING_CASE
786 }
787 #endif // !LOG_DISABLED
788     
789
790 void MediaPlayerPrivateAVFoundation::scheduleMainThreadNotification(Notification notification)
791 {
792     if (notification.type() != Notification::FunctionType)
793         LOG(Media, "MediaPlayerPrivateAVFoundation::scheduleMainThreadNotification(%p) - notification %s", this, notificationName(notification));
794
795     m_queueMutex.lock();
796
797     // It is important to always process the properties in the order that we are notified,
798     // so always go through the queue because notifications happen on different threads.
799     m_queuedNotifications.append(notification);
800
801 #if OS(WINDOWS)
802     bool delayDispatch = true;
803 #else
804     bool delayDispatch = m_delayCallbacks || !isMainThread();
805 #endif
806     if (delayDispatch && !m_mainThreadCallPending) {
807         m_mainThreadCallPending = true;
808         callOnMainThread(mainThreadCallback, this);
809     }
810
811     m_queueMutex.unlock();
812
813     if (delayDispatch) {
814         if (notification.type() != Notification::FunctionType)
815             LOG(Media, "MediaPlayerPrivateAVFoundation::scheduleMainThreadNotification(%p) - early return", this);
816         return;
817     }
818
819     dispatchNotification();
820 }
821
822 void MediaPlayerPrivateAVFoundation::dispatchNotification()
823 {
824     ASSERT(isMainThread());
825
826     Notification notification = Notification();
827     {
828         MutexLocker lock(m_queueMutex);
829         
830         if (m_queuedNotifications.isEmpty())
831             return;
832         
833         if (!m_delayCallbacks) {
834             // Only dispatch one notification callback per invocation because they can cause recursion.
835             notification = m_queuedNotifications.first();
836             m_queuedNotifications.remove(0);
837         }
838         
839         if (!m_queuedNotifications.isEmpty() && !m_mainThreadCallPending)
840             callOnMainThread(mainThreadCallback, this);
841
842         if (!notification.isValid())
843             return;
844     }
845
846     if (notification.type() != Notification::FunctionType)
847         LOG(Media, "MediaPlayerPrivateAVFoundation::dispatchNotification(%p) - dispatching %s", this, notificationName(notification));
848
849     switch (notification.type()) {
850     case Notification::ItemDidPlayToEndTime:
851         didEnd();
852         break;
853     case Notification::ItemTracksChanged:
854         tracksChanged();
855         updateStates();
856         break;
857     case Notification::ItemStatusChanged:
858         updateStates();
859         break;
860     case Notification::ItemSeekableTimeRangesChanged:
861         seekableTimeRangesChanged();
862         updateStates();
863         break;
864     case Notification::ItemLoadedTimeRangesChanged:
865         loadedTimeRangesChanged();
866         updateStates();
867         break;
868     case Notification::ItemPresentationSizeChanged:
869         sizeChanged();
870         updateStates();
871         break;
872     case Notification::ItemIsPlaybackLikelyToKeepUpChanged:
873         updateStates();
874         break;
875     case Notification::ItemIsPlaybackBufferEmptyChanged:
876         updateStates();
877         break;
878     case Notification::ItemIsPlaybackBufferFullChanged:
879         updateStates();
880         break;
881     case Notification::PlayerRateChanged:
882         updateStates();
883         rateChanged();
884         break;
885     case Notification::PlayerTimeChanged:
886         timeChanged(notification.time());
887         break;
888     case Notification::SeekCompleted:
889         seekCompleted(notification.finished());
890         break;
891     case Notification::AssetMetadataLoaded:
892         metadataLoaded();
893         updateStates();
894         break;
895     case Notification::AssetPlayabilityKnown:
896         updateStates();
897         playabilityKnown();
898         break;
899     case Notification::DurationChanged:
900         invalidateCachedDuration();
901         break;
902     case Notification::ContentsNeedsDisplay:
903         contentsNeedsDisplay();
904         break;
905     case Notification::InbandTracksNeedConfiguration:
906         m_inbandTrackConfigurationPending = false;
907         configureInbandTracks();
908         break;
909     case Notification::FunctionType:
910         notification.function()();
911         break;
912     case Notification::TargetIsWirelessChanged:
913 #if ENABLE(WIRELESS_PLAYBACK_TARGET)
914         playbackTargetIsWirelessChanged();
915 #endif
916         break;
917
918     case Notification::None:
919         ASSERT_NOT_REACHED();
920         break;
921     }
922 }
923
924 void MediaPlayerPrivateAVFoundation::configureInbandTracks()
925 {
926     RefPtr<InbandTextTrackPrivateAVF> trackToEnable;
927     
928 #if ENABLE(AVF_CAPTIONS)
929     synchronizeTextTrackState();
930 #endif
931
932     // AVFoundation can only emit cues for one track at a time, so enable the first track that is showing, or the first that
933     // is hidden if none are showing. Otherwise disable all tracks.
934     for (unsigned i = 0; i < m_textTracks.size(); ++i) {
935         RefPtr<InbandTextTrackPrivateAVF> track = m_textTracks[i];
936         if (track->mode() == InbandTextTrackPrivate::Showing) {
937             trackToEnable = track;
938             break;
939         }
940         if (track->mode() == InbandTextTrackPrivate::Hidden)
941             trackToEnable = track;
942     }
943
944     setCurrentTextTrack(trackToEnable.get());
945 }
946
947 void MediaPlayerPrivateAVFoundation::trackModeChanged()
948 {
949     if (m_inbandTrackConfigurationPending)
950         return;
951     m_inbandTrackConfigurationPending = true;
952     scheduleMainThreadNotification(Notification::InbandTracksNeedConfiguration);
953 }
954
955 void MediaPlayerPrivateAVFoundation::clearTextTracks()
956 {
957     for (unsigned i = 0; i < m_textTracks.size(); ++i) {
958         RefPtr<InbandTextTrackPrivateAVF> track = m_textTracks[i];
959         player()->removeTextTrack(track);
960         track->disconnect();
961     }
962     m_textTracks.clear();
963 }
964
965 void MediaPlayerPrivateAVFoundation::processNewAndRemovedTextTracks(const Vector<RefPtr<InbandTextTrackPrivateAVF>>& removedTextTracks)
966 {
967     if (removedTextTracks.size()) {
968         for (unsigned i = 0; i < m_textTracks.size(); ++i) {
969             if (!removedTextTracks.contains(m_textTracks[i]))
970                 continue;
971             
972             player()->removeTextTrack(removedTextTracks[i].get());
973             m_textTracks.remove(i);
974         }
975     }
976
977     unsigned inBandCount = 0;
978     for (unsigned i = 0; i < m_textTracks.size(); ++i) {
979         RefPtr<InbandTextTrackPrivateAVF> track = m_textTracks[i];
980
981 #if ENABLE(AVF_CAPTIONS)
982         if (track->textTrackCategory() == InbandTextTrackPrivateAVF::OutOfBand)
983             continue;
984 #endif
985
986         track->setTextTrackIndex(inBandCount);
987         ++inBandCount;
988         if (track->hasBeenReported())
989             continue;
990         
991         track->setHasBeenReported(true);
992         player()->addTextTrack(track.get());
993     }
994     LOG(Media, "MediaPlayerPrivateAVFoundation::processNewAndRemovedTextTracks(%p) - found %lu text tracks", this, m_textTracks.size());
995 }
996
997 #if ENABLE(WIRELESS_PLAYBACK_TARGET)
998 void MediaPlayerPrivateAVFoundation::playbackTargetIsWirelessChanged()
999 {
1000     if (m_player)
1001         m_player->currentPlaybackTargetIsWirelessChanged();
1002 }
1003 #endif
1004
1005 #if ENABLE(ENCRYPTED_MEDIA) || ENABLE(ENCRYPTED_MEDIA_V2)
1006 bool MediaPlayerPrivateAVFoundation::extractKeyURIKeyIDAndCertificateFromInitData(Uint8Array* initData, String& keyURI, String& keyID, RefPtr<Uint8Array>& certificate)
1007 {
1008     // initData should have the following layout:
1009     // [4 bytes: keyURI length][N bytes: keyURI][4 bytes: contentID length], [N bytes: contentID], [4 bytes: certificate length][N bytes: certificate]
1010     if (initData->byteLength() < 4)
1011         return false;
1012
1013     RefPtr<ArrayBuffer> initDataBuffer = initData->buffer();
1014
1015     // Use a DataView to read uint32 values from the buffer, as Uint32Array requires the reads be aligned on 4-byte boundaries. 
1016     RefPtr<JSC::DataView> initDataView = JSC::DataView::create(initDataBuffer, 0, initDataBuffer->byteLength());
1017     uint32_t offset = 0;
1018     bool status = true;
1019
1020     uint32_t keyURILength = initDataView->get<uint32_t>(offset, true, &status);
1021     offset += 4;
1022     if (!status || offset + keyURILength > initData->length())
1023         return false;
1024
1025     RefPtr<Uint16Array> keyURIArray = Uint16Array::create(initDataBuffer, offset, keyURILength);
1026     if (!keyURIArray)
1027         return false;
1028
1029     keyURI = String(reinterpret_cast<UChar*>(keyURIArray->data()), keyURILength / sizeof(unsigned short));
1030     offset += keyURILength;
1031
1032     uint32_t keyIDLength = initDataView->get<uint32_t>(offset, true, &status);
1033     offset += 4;
1034     if (!status || offset + keyIDLength > initData->length())
1035         return false;
1036
1037     RefPtr<Uint16Array> keyIDArray = Uint16Array::create(initDataBuffer, offset, keyIDLength);
1038     if (!keyIDArray)
1039         return false;
1040
1041     keyID = String(reinterpret_cast<UChar*>(keyIDArray->data()), keyIDLength / sizeof(unsigned short));
1042     offset += keyIDLength;
1043
1044     uint32_t certificateLength = initDataView->get<uint32_t>(offset, true, &status);
1045     offset += 4;
1046     if (!status || offset + certificateLength > initData->length())
1047         return false;
1048
1049     certificate = Uint8Array::create(initDataBuffer, offset, certificateLength);
1050     if (!certificate)
1051         return false;
1052
1053     return true;
1054 }
1055 #endif
1056
1057 URL MediaPlayerPrivateAVFoundation::resolvedURL() const
1058 {
1059     if (!m_assetURL.length())
1060         return URL();
1061
1062     return URL(ParsedURLString, m_assetURL);
1063 }
1064
1065 bool MediaPlayerPrivateAVFoundation::canSaveMediaData() const
1066 {
1067     URL url = resolvedURL();
1068
1069     if (url.isLocalFile())
1070         return true;
1071
1072     if (!url.protocolIsInHTTPFamily())
1073         return false;
1074
1075     if (isLiveStream())
1076         return false;
1077
1078     return true;
1079 }
1080
1081 bool MediaPlayerPrivateAVFoundation::isUnsupportedMIMEType(const String& type)
1082 {
1083     String lowerCaseType = type.convertToASCIILowercase();
1084
1085     // AVFoundation will return non-video MIME types which it claims to support, but which we
1086     // do not support in the <video> element. Reject all non video/, audio/, and application/ types.
1087     if (!lowerCaseType.startsWith("video/") && !lowerCaseType.startsWith("audio/") && !lowerCaseType.startsWith("application/"))
1088         return true;
1089
1090     // Reject types we know AVFoundation does not support that sites commonly ask about.
1091     if (lowerCaseType == "video/webm" || lowerCaseType == "audio/webm" || lowerCaseType == "video/x-webm")
1092         return true;
1093
1094     if (lowerCaseType == "video/x-flv")
1095         return true;
1096
1097     if (lowerCaseType == "audio/ogg" || lowerCaseType == "video/ogg" || lowerCaseType == "application/ogg")
1098         return true;
1099
1100     if (lowerCaseType == "video/h264")
1101         return true;
1102
1103     return false;
1104 }
1105
1106 const HashSet<String>& MediaPlayerPrivateAVFoundation::staticMIMETypeList()
1107 {
1108     static NeverDestroyed<HashSet<String>> cache = []() {
1109         HashSet<String> types;
1110
1111         static const char* typeNames[] = {
1112             "application/vnd.apple.mpegurl",
1113             "application/x-mpegurl",
1114             "audio/3gpp",
1115             "audio/aac",
1116             "audio/aacp",
1117             "audio/aiff",
1118             "audio/basic",
1119             "audio/mp3",
1120             "audio/mp4",
1121             "audio/mpeg",
1122             "audio/mpeg3",
1123             "audio/mpegurl",
1124             "audio/mpg",
1125             "audio/wav",
1126             "audio/wave",
1127             "audio/x-aac",
1128             "audio/x-aiff",
1129             "audio/x-m4a",
1130             "audio/x-mpegurl",
1131             "audio/x-wav",
1132             "video/3gpp",
1133             "video/3gpp2",
1134             "video/mp4",
1135             "video/mpeg",
1136             "video/mpeg2",
1137             "video/mpg",
1138             "video/quicktime",
1139             "video/x-m4v",
1140             "video/x-mpeg",
1141             "video/x-mpg",
1142         };
1143         for (size_t i = 0; i < WTF_ARRAY_LENGTH(typeNames); ++i)
1144             types.add(typeNames[i]);
1145
1146         return types;
1147     }();
1148
1149     return cache;
1150 }
1151
1152 } // namespace WebCore
1153
1154 #endif