28992314ec9c90edf8c5333ab106cadc46f6a455
[WebKit-https.git] / Source / WebCore / platform / graphics / avfoundation / MediaPlayerPrivateAVFoundation.cpp
1 /*
2  * Copyright (C) 2011-2014 Apple Inc. All rights reserved.
3  *
4  * Redistribution and use in source and binary forms, with or without
5  * modification, are permitted provided that the following conditions
6  * are met:
7  * 1. Redistributions of source code must retain the above copyright
8  *    notice, this list of conditions and the following disclaimer.
9  * 2. Redistributions in binary form must reproduce the above copyright
10  *    notice, this list of conditions and the following disclaimer in the
11  *    documentation and/or other materials provided with the distribution.
12  *
13  * THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY
14  * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
15  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
16  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL APPLE INC. OR
17  * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
18  * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
19  * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
20  * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
21  * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
22  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
23  * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 
24  */
25
26 #include "config.h"
27
28 #if ENABLE(VIDEO) && USE(AVFOUNDATION)
29
30 #include "MediaPlayerPrivateAVFoundation.h"
31
32 #include "DeprecatedGlobalSettings.h"
33 #include "DocumentLoader.h"
34 #include "FloatConversion.h"
35 #include "GraphicsContext.h"
36 #include "InbandTextTrackPrivateAVF.h"
37 #include "InbandTextTrackPrivateClient.h"
38 #include "Logging.h"
39 #include "PlatformLayer.h"
40 #include "PlatformTimeRanges.h"
41 #include "Settings.h"
42 #include <CoreMedia/CoreMedia.h>
43 #include <JavaScriptCore/DataView.h>
44 #include <JavaScriptCore/HeapInlines.h>
45 #include <JavaScriptCore/TypedArrayInlines.h>
46 #include <JavaScriptCore/Uint16Array.h>
47 #include <wtf/MainThread.h>
48 #include <wtf/NeverDestroyed.h>
49 #include <wtf/SoftLinking.h>
50 #include <wtf/StringPrintStream.h>
51 #include <wtf/URL.h>
52 #include <wtf/text/CString.h>
53
54 namespace WebCore {
55
56 MediaPlayerPrivateAVFoundation::MediaPlayerPrivateAVFoundation(MediaPlayer* player)
57     : m_player(player)
58     , m_queuedNotifications()
59     , m_queueMutex()
60     , m_networkState(MediaPlayer::Empty)
61     , m_readyState(MediaPlayer::HaveNothing)
62     , m_preload(MediaPlayer::Auto)
63 #if !RELEASE_LOG_DISABLED
64     , m_logger(player->mediaPlayerLogger())
65     , m_logIdentifier(player->mediaPlayerLogIdentifier())
66 #endif
67     , m_cachedDuration(MediaTime::invalidTime())
68     , m_reportedDuration(MediaTime::invalidTime())
69     , m_maxTimeLoadedAtLastDidLoadingProgress(MediaTime::invalidTime())
70     , m_delayCallbacks(0)
71     , m_delayCharacteristicsChangedNotification(0)
72     , m_mainThreadCallPending(false)
73     , m_assetIsPlayable(false)
74     , m_visible(false)
75     , m_loadingMetadata(false)
76     , m_isAllowedToRender(false)
77     , m_cachedHasAudio(false)
78     , m_cachedHasVideo(false)
79     , m_cachedHasCaptions(false)
80     , m_ignoreLoadStateChanges(false)
81     , m_haveReportedFirstVideoFrame(false)
82     , m_inbandTrackConfigurationPending(false)
83     , m_characteristicsChanged(false)
84     , m_shouldMaintainAspectRatio(true)
85     , m_seeking(false)
86 {
87     INFO_LOG(LOGIDENTIFIER);
88 }
89
90 MediaPlayerPrivateAVFoundation::~MediaPlayerPrivateAVFoundation()
91 {
92     INFO_LOG(LOGIDENTIFIER);
93     setIgnoreLoadStateChanges(true);
94 }
95
96 MediaPlayerPrivateAVFoundation::MediaRenderingMode MediaPlayerPrivateAVFoundation::currentRenderingMode() const
97 {
98     if (platformLayer())
99         return MediaRenderingToLayer;
100
101     if (hasContextRenderer())
102         return MediaRenderingToContext;
103
104     return MediaRenderingNone;
105 }
106
107 MediaPlayerPrivateAVFoundation::MediaRenderingMode MediaPlayerPrivateAVFoundation::preferredRenderingMode() const
108 {
109     if (!m_player->visible() || assetStatus() == MediaPlayerAVAssetStatusUnknown)
110         return MediaRenderingNone;
111
112     if (supportsAcceleratedRendering() && m_player->client().mediaPlayerRenderingCanBeAccelerated(m_player))
113         return MediaRenderingToLayer;
114
115     return MediaRenderingToContext;
116 }
117
118 void MediaPlayerPrivateAVFoundation::setUpVideoRendering()
119 {
120     if (!isReadyForVideoSetup())
121         return;
122
123     MediaRenderingMode currentMode = currentRenderingMode();
124     MediaRenderingMode preferredMode = preferredRenderingMode();
125
126     if (preferredMode == MediaRenderingNone)
127         preferredMode = MediaRenderingToContext;
128
129     if (currentMode == preferredMode && currentMode != MediaRenderingNone)
130         return;
131
132     if (currentMode != MediaRenderingNone)
133         tearDownVideoRendering();
134
135     switch (preferredMode) {
136     case MediaRenderingNone:
137     case MediaRenderingToContext:
138         createContextVideoRenderer();
139         break;
140
141     case MediaRenderingToLayer:
142         createVideoLayer();
143         break;
144     }
145
146     // If using a movie layer, inform the client so the compositing tree is updated.
147     if (currentMode == MediaRenderingToLayer || preferredMode == MediaRenderingToLayer)
148         m_player->client().mediaPlayerRenderingModeChanged(m_player);
149 }
150
151 void MediaPlayerPrivateAVFoundation::tearDownVideoRendering()
152 {
153     INFO_LOG(LOGIDENTIFIER);
154
155     destroyContextVideoRenderer();
156
157     if (platformLayer())
158         destroyVideoLayer();
159 }
160
161 bool MediaPlayerPrivateAVFoundation::hasSetUpVideoRendering() const
162 {
163     return hasLayerRenderer() || hasContextRenderer();
164 }
165
166 void MediaPlayerPrivateAVFoundation::load(const String& url)
167 {
168     ALWAYS_LOG(LOGIDENTIFIER);
169
170     setNetworkState(m_preload == MediaPlayer::None ? MediaPlayer::Idle : MediaPlayer::Loading);
171     setReadyState(MediaPlayer::HaveNothing);
172
173     m_assetURL = URL({ }, url);
174     m_requestedOrigin = SecurityOrigin::create(m_assetURL);
175
176     // Don't do any more work if the url is empty.
177     if (!url.length())
178         return;
179
180     setPreload(m_preload);
181 }
182
183 #if ENABLE(MEDIA_SOURCE)
184 void MediaPlayerPrivateAVFoundation::load(const String&, MediaSourcePrivateClient*)
185 {
186     setNetworkState(MediaPlayer::FormatError);
187 }
188 #endif
189
190
191 void MediaPlayerPrivateAVFoundation::playabilityKnown()
192 {
193     INFO_LOG(LOGIDENTIFIER, "metadata loaded = ", assetStatus() > MediaPlayerAVAssetStatusLoading);
194
195     if (m_assetIsPlayable)
196         return;
197
198     // Nothing more to do if we already have all of the item's metadata.
199     if (assetStatus() > MediaPlayerAVAssetStatusLoading)
200         return;
201
202     // At this point we are supposed to load metadata. It is OK to ask the asset to load the same 
203     // information multiple times, because if it has already been loaded the completion handler 
204     // will just be called synchronously.
205     m_loadingMetadata = true;
206     beginLoadingMetadata();
207 }
208
209 void MediaPlayerPrivateAVFoundation::prepareToPlay()
210 {
211     ALWAYS_LOG(LOGIDENTIFIER);
212     setPreload(MediaPlayer::Auto);
213 }
214
215 void MediaPlayerPrivateAVFoundation::play()
216 {
217     ALWAYS_LOG(LOGIDENTIFIER);
218     platformPlay();
219 }
220
221 void MediaPlayerPrivateAVFoundation::pause()
222 {
223     ALWAYS_LOG(LOGIDENTIFIER);
224     platformPause();
225 }
226
227 MediaTime MediaPlayerPrivateAVFoundation::durationMediaTime() const
228 {
229     if (m_cachedDuration.isValid())
230         return m_cachedDuration;
231
232     MediaTime duration = platformDuration();
233     if (!duration || duration.isInvalid())
234         return MediaTime::zeroTime();
235
236     m_cachedDuration = duration;
237
238     return m_cachedDuration;
239 }
240
241 void MediaPlayerPrivateAVFoundation::seek(const MediaTime& time)
242 {
243     seekWithTolerance(time, MediaTime::zeroTime(), MediaTime::zeroTime());
244 }
245
246 void MediaPlayerPrivateAVFoundation::seekWithTolerance(const MediaTime& mediaTime, const MediaTime& negativeTolerance, const MediaTime& positiveTolerance)
247 {
248     MediaTime time = mediaTime;
249
250     if (m_seeking) {
251         ALWAYS_LOG(LOGIDENTIFIER, "saving pending seek");
252         m_pendingSeek = [this, time, negativeTolerance, positiveTolerance]() {
253             seekWithTolerance(time, negativeTolerance, positiveTolerance);
254         };
255         return;
256     }
257     m_seeking = true;
258
259     if (!metaDataAvailable())
260         return;
261
262     if (time > durationMediaTime())
263         time = durationMediaTime();
264
265     if (currentTextTrack())
266         currentTextTrack()->beginSeeking();
267
268     ALWAYS_LOG(LOGIDENTIFIER, "seeking to  ", time);
269
270     seekToTime(time, negativeTolerance, positiveTolerance);
271 }
272
273 bool MediaPlayerPrivateAVFoundation::paused() const
274 {
275     if (!metaDataAvailable())
276         return true;
277
278     return platformPaused();
279 }
280
281 bool MediaPlayerPrivateAVFoundation::seeking() const
282 {
283     if (!metaDataAvailable())
284         return false;
285
286     return m_seeking;
287 }
288
289 FloatSize MediaPlayerPrivateAVFoundation::naturalSize() const
290 {
291     if (!metaDataAvailable())
292         return IntSize();
293
294     // In spite of the name of this method, return the natural size transformed by the 
295     // initial movie scale because the spec says intrinsic size is:
296     //
297     //    ... the dimensions of the resource in CSS pixels after taking into account the resource's 
298     //    dimensions, aspect ratio, clean aperture, resolution, and so forth, as defined for the 
299     //    format used by the resource
300
301     return m_cachedNaturalSize;
302 }
303
304 void MediaPlayerPrivateAVFoundation::setNaturalSize(FloatSize size)
305 {
306     FloatSize oldSize = m_cachedNaturalSize;
307     m_cachedNaturalSize = size;
308     if (oldSize != m_cachedNaturalSize) {
309         INFO_LOG(LOGIDENTIFIER, "was ", oldSize.width(), " x ", oldSize.height(), ", is ", size.width(), " x ", size.height());
310         m_player->sizeChanged();
311     }
312 }
313
314 void MediaPlayerPrivateAVFoundation::setHasVideo(bool b)
315 {
316     if (m_cachedHasVideo != b) {
317         m_cachedHasVideo = b;
318         characteristicsChanged();
319     }
320 }
321
322 void MediaPlayerPrivateAVFoundation::setHasAudio(bool b)
323 {
324     if (m_cachedHasAudio != b) {
325         m_cachedHasAudio = b;
326         characteristicsChanged();
327     }
328 }
329
330 void MediaPlayerPrivateAVFoundation::setHasClosedCaptions(bool b)
331 {
332     if (m_cachedHasCaptions != b) {
333         m_cachedHasCaptions = b;
334         characteristicsChanged();
335     }
336 }
337
338 void MediaPlayerPrivateAVFoundation::setNetworkState(MediaPlayer::NetworkState state)
339 {
340     if (state == m_networkState)
341         return;
342
343     m_networkState = state;
344     m_player->networkStateChanged();
345 }
346
347 void MediaPlayerPrivateAVFoundation::setReadyState(MediaPlayer::ReadyState state)
348 {
349     if (state == m_readyState)
350         return;
351
352     m_readyState = state;
353     m_player->readyStateChanged();
354 }
355
356 void MediaPlayerPrivateAVFoundation::characteristicsChanged()
357 {
358     if (m_delayCharacteristicsChangedNotification) {
359         m_characteristicsChanged = true;
360         return;
361     }
362
363     m_characteristicsChanged = false;
364     m_player->characteristicChanged();
365 }
366
367 void MediaPlayerPrivateAVFoundation::setDelayCharacteristicsChangedNotification(bool delay)
368 {
369     if (delay) {
370         m_delayCharacteristicsChangedNotification++;
371         return;
372     }
373     
374     ASSERT(m_delayCharacteristicsChangedNotification);
375     m_delayCharacteristicsChangedNotification--;
376     if (!m_delayCharacteristicsChangedNotification && m_characteristicsChanged)
377         characteristicsChanged();
378 }
379
380 std::unique_ptr<PlatformTimeRanges> MediaPlayerPrivateAVFoundation::buffered() const
381 {
382     if (!m_cachedLoadedTimeRanges)
383         m_cachedLoadedTimeRanges = platformBufferedTimeRanges();
384
385     return std::make_unique<PlatformTimeRanges>(*m_cachedLoadedTimeRanges);
386 }
387
388 MediaTime MediaPlayerPrivateAVFoundation::maxMediaTimeSeekable() const
389 {
390     if (!metaDataAvailable())
391         return MediaTime::zeroTime();
392
393     if (!m_cachedMaxTimeSeekable)
394         m_cachedMaxTimeSeekable = platformMaxTimeSeekable();
395
396     return m_cachedMaxTimeSeekable;
397 }
398
399 MediaTime MediaPlayerPrivateAVFoundation::minMediaTimeSeekable() const
400 {
401     if (!metaDataAvailable())
402         return MediaTime::zeroTime();
403
404     if (!m_cachedMinTimeSeekable)
405         m_cachedMinTimeSeekable = platformMinTimeSeekable();
406
407     return m_cachedMinTimeSeekable;
408 }
409
410 double MediaPlayerPrivateAVFoundation::requestedRate() const
411 {
412     return m_player->requestedRate();
413 }
414
415 MediaTime MediaPlayerPrivateAVFoundation::maxTimeLoaded() const
416 {
417     if (!metaDataAvailable())
418         return MediaTime::zeroTime();
419
420     if (!m_cachedMaxTimeLoaded)
421         m_cachedMaxTimeLoaded = platformMaxTimeLoaded();
422
423     return m_cachedMaxTimeLoaded;   
424 }
425
426 bool MediaPlayerPrivateAVFoundation::didLoadingProgress() const
427 {
428     if (!durationMediaTime())
429         return false;
430     MediaTime currentMaxTimeLoaded = maxTimeLoaded();
431     bool didLoadingProgress = currentMaxTimeLoaded != m_maxTimeLoadedAtLastDidLoadingProgress;
432     m_maxTimeLoadedAtLastDidLoadingProgress = currentMaxTimeLoaded;
433
434     return didLoadingProgress;
435 }
436
437 bool MediaPlayerPrivateAVFoundation::isReadyForVideoSetup() const
438 {
439     // AVFoundation will not return true for firstVideoFrameAvailable until
440     // an AVPlayerLayer has been added to the AVPlayerItem, so allow video setup
441     // here if a video track to trigger allocation of a AVPlayerLayer.
442     return (m_isAllowedToRender || m_cachedHasVideo) && m_readyState >= MediaPlayer::HaveMetadata && m_player->visible();
443 }
444
445 void MediaPlayerPrivateAVFoundation::prepareForRendering()
446 {
447     if (m_isAllowedToRender)
448         return;
449     m_isAllowedToRender = true;
450
451     setUpVideoRendering();
452
453     if (currentRenderingMode() == MediaRenderingToLayer || preferredRenderingMode() == MediaRenderingToLayer)
454         m_player->client().mediaPlayerRenderingModeChanged(m_player);
455 }
456
457 bool MediaPlayerPrivateAVFoundation::supportsFullscreen() const
458 {
459 #if ENABLE(FULLSCREEN_API)
460     return true;
461 #else
462     // FIXME: WebVideoFullscreenController assumes a QTKit/QuickTime media engine
463 #if PLATFORM(IOS_FAMILY)
464     if (DeprecatedGlobalSettings::avKitEnabled())
465         return true;
466 #endif
467     return false;
468 #endif
469 }
470
471 bool MediaPlayerPrivateAVFoundation::hasSingleSecurityOrigin() const
472 {
473     if (m_resolvedOrigin && m_requestedOrigin)
474         return m_resolvedOrigin->isSameSchemeHostPort(*m_requestedOrigin);
475     return false;
476 }
477
478 void MediaPlayerPrivateAVFoundation::setResolvedURL(URL&& resolvedURL)
479 {
480     m_resolvedURL = WTFMove(resolvedURL);
481     m_resolvedOrigin = SecurityOrigin::create(m_resolvedURL);
482 }
483
484 void MediaPlayerPrivateAVFoundation::updateStates()
485 {
486     if (m_ignoreLoadStateChanges)
487         return;
488
489     MediaPlayer::NetworkState newNetworkState = m_networkState;
490     MediaPlayer::ReadyState newReadyState = m_readyState;
491
492     if (m_loadingMetadata)
493         newNetworkState = MediaPlayer::Loading;
494     else {
495         // -loadValuesAsynchronouslyForKeys:completionHandler: has invoked its handler; test status of keys and determine state.
496         AssetStatus assetStatus = this->assetStatus();
497         ItemStatus itemStatus = playerItemStatus();
498         
499         m_assetIsPlayable = (assetStatus == MediaPlayerAVAssetStatusPlayable);
500         if (m_readyState < MediaPlayer::HaveMetadata && assetStatus > MediaPlayerAVAssetStatusLoading) {
501             if (m_assetIsPlayable) {
502                 if (assetStatus >= MediaPlayerAVAssetStatusLoaded)
503                     newReadyState = MediaPlayer::HaveMetadata;
504                 if (itemStatus <= MediaPlayerAVPlayerItemStatusUnknown) {
505                     if (assetStatus == MediaPlayerAVAssetStatusFailed || m_preload > MediaPlayer::MetaData || isLiveStream()) {
506                         // The asset is playable but doesn't support inspection prior to playback (eg. streaming files),
507                         // or we are supposed to prepare for playback immediately, so create the player item now.
508                         newNetworkState = MediaPlayer::Loading;
509                         prepareToPlay();
510                     } else
511                         newNetworkState = MediaPlayer::Idle;
512                 }
513             } else {
514                 // FIX ME: fetch the error associated with the @"playable" key to distinguish between format 
515                 // and network errors.
516                 newNetworkState = MediaPlayer::FormatError;
517             }
518         }
519         
520         if (assetStatus >= MediaPlayerAVAssetStatusLoaded && itemStatus > MediaPlayerAVPlayerItemStatusUnknown) {
521             switch (itemStatus) {
522             case MediaPlayerAVPlayerItemStatusDoesNotExist:
523             case MediaPlayerAVPlayerItemStatusUnknown:
524             case MediaPlayerAVPlayerItemStatusFailed:
525                 break;
526
527             case MediaPlayerAVPlayerItemStatusPlaybackLikelyToKeepUp:
528             case MediaPlayerAVPlayerItemStatusPlaybackBufferFull:
529                 // If the status becomes PlaybackBufferFull, loading stops and the status will not
530                 // progress to LikelyToKeepUp. Set the readyState to  HAVE_ENOUGH_DATA, on the
531                 // presumption that if the playback buffer is full, playback will probably not stall.
532                 newReadyState = MediaPlayer::HaveEnoughData;
533                 break;
534
535             case MediaPlayerAVPlayerItemStatusReadyToPlay:
536                 if (m_readyState != MediaPlayer::HaveEnoughData && maxTimeLoaded() > currentMediaTime())
537                     newReadyState = MediaPlayer::HaveFutureData;
538                 break;
539
540             case MediaPlayerAVPlayerItemStatusPlaybackBufferEmpty:
541                 newReadyState = MediaPlayer::HaveCurrentData;
542                 break;
543             }
544
545             if (itemStatus == MediaPlayerAVPlayerItemStatusPlaybackBufferFull)
546                 newNetworkState = MediaPlayer::Idle;
547             else if (itemStatus == MediaPlayerAVPlayerItemStatusFailed)
548                 newNetworkState = MediaPlayer::DecodeError;
549             else if (itemStatus != MediaPlayerAVPlayerItemStatusPlaybackBufferFull && itemStatus >= MediaPlayerAVPlayerItemStatusReadyToPlay)
550                 newNetworkState = (maxTimeLoaded() == durationMediaTime()) ? MediaPlayer::Loaded : MediaPlayer::Loading;
551         }
552     }
553
554     if (isReadyForVideoSetup() && currentRenderingMode() != preferredRenderingMode())
555         setUpVideoRendering();
556
557     if (!m_haveReportedFirstVideoFrame && m_cachedHasVideo && hasAvailableVideoFrame()) {
558         if (m_readyState < MediaPlayer::HaveCurrentData)
559             newReadyState = MediaPlayer::HaveCurrentData;
560         m_haveReportedFirstVideoFrame = true;
561         m_player->firstVideoFrameAvailable();
562     } else if (!hasAvailableVideoFrame())
563         m_haveReportedFirstVideoFrame = false;
564
565     if (m_networkState != newNetworkState)
566         ALWAYS_LOG(LOGIDENTIFIER, "entered with networkState ", m_networkState, ", exiting with ", newNetworkState);
567     if (m_readyState != newReadyState)
568         ALWAYS_LOG(LOGIDENTIFIER, "entered with readyState ", m_readyState, ", exiting with ", newReadyState);
569
570     setNetworkState(newNetworkState);
571     setReadyState(newReadyState);
572 }
573
574 void MediaPlayerPrivateAVFoundation::setSize(const IntSize&) 
575
576 }
577
578 void MediaPlayerPrivateAVFoundation::setVisible(bool visible)
579 {
580     if (m_visible == visible)
581         return;
582
583     m_visible = visible;
584     if (visible)
585         setUpVideoRendering();
586     
587     platformSetVisible(visible);
588 }
589
590 void MediaPlayerPrivateAVFoundation::acceleratedRenderingStateChanged()
591 {
592     // Set up or change the rendering path if necessary.
593     setUpVideoRendering();
594 }
595
596 void MediaPlayerPrivateAVFoundation::setShouldMaintainAspectRatio(bool maintainAspectRatio)
597 {
598     if (maintainAspectRatio == m_shouldMaintainAspectRatio)
599         return;
600
601     m_shouldMaintainAspectRatio = maintainAspectRatio;
602     updateVideoLayerGravity();
603 }
604
605 void MediaPlayerPrivateAVFoundation::metadataLoaded()
606 {
607     m_loadingMetadata = false;
608     resolvedURLChanged();
609     tracksChanged();
610 }
611
612 void MediaPlayerPrivateAVFoundation::rateChanged()
613 {
614     m_player->rateChanged();
615 }
616
617 void MediaPlayerPrivateAVFoundation::loadedTimeRangesChanged()
618 {
619     m_cachedLoadedTimeRanges = nullptr;
620     m_cachedMaxTimeLoaded = MediaTime::zeroTime();
621     invalidateCachedDuration();
622 }
623
624 void MediaPlayerPrivateAVFoundation::seekableTimeRangesChanged()
625 {
626     m_cachedMaxTimeSeekable = MediaTime::zeroTime();
627     m_cachedMinTimeSeekable = MediaTime::zeroTime();
628 }
629
630 void MediaPlayerPrivateAVFoundation::timeChanged(const MediaTime& time)
631 {
632     UNUSED_PARAM(time);
633     INFO_LOG(LOGIDENTIFIER, "- ", time);
634 }
635
636 void MediaPlayerPrivateAVFoundation::seekCompleted(bool finished)
637 {
638     UNUSED_PARAM(finished);
639     ALWAYS_LOG(LOGIDENTIFIER, "finished = ", finished);
640
641     m_seeking = false;
642
643     WTF::Function<void()> pendingSeek;
644     std::swap(pendingSeek, m_pendingSeek);
645
646     if (pendingSeek) {
647         ALWAYS_LOG(LOGIDENTIFIER, "issuing pending seek");
648         pendingSeek();
649         return;
650     }
651
652     if (currentTextTrack())
653         currentTextTrack()->endSeeking();
654
655     updateStates();
656     m_player->timeChanged();
657 }
658
659 void MediaPlayerPrivateAVFoundation::didEnd()
660 {
661     // Hang onto the current time and use it as duration from now on since we are definitely at
662     // the end of the movie. Do this because the initial duration is sometimes an estimate.
663     MediaTime now = currentMediaTime();
664     if (now > MediaTime::zeroTime() && !m_seeking)
665         m_cachedDuration = now;
666
667     updateStates();
668     m_player->timeChanged();
669 }
670
671 void MediaPlayerPrivateAVFoundation::invalidateCachedDuration()
672 {
673     m_cachedDuration = MediaTime::invalidTime();
674
675     // For some media files, reported duration is estimated and updated as media is loaded
676     // so report duration changed when the estimate is upated.
677     MediaTime duration = this->durationMediaTime();
678     if (duration != m_reportedDuration) {
679         INFO_LOG(LOGIDENTIFIER, "- ", m_cachedDuration);
680         if (m_reportedDuration.isValid())
681             m_player->durationChanged();
682         m_reportedDuration = duration;
683     }
684     
685 }
686
687 void MediaPlayerPrivateAVFoundation::repaint()
688 {
689     m_player->repaint();
690 }
691
692 MediaPlayer::MovieLoadType MediaPlayerPrivateAVFoundation::movieLoadType() const
693 {
694     if (!metaDataAvailable() || assetStatus() == MediaPlayerAVAssetStatusUnknown)
695         return MediaPlayer::Unknown;
696
697     if (isLiveStream())
698         return MediaPlayer::LiveStream;
699
700     return MediaPlayer::Download;
701 }
702
703 void MediaPlayerPrivateAVFoundation::setPreload(MediaPlayer::Preload preload)
704 {
705     ALWAYS_LOG(LOGIDENTIFIER, " - ", static_cast<int>(preload));
706     m_preload = preload;
707     if (m_assetURL.isEmpty())
708         return;
709
710     setDelayCallbacks(true);
711
712     if (m_preload >= MediaPlayer::MetaData && assetStatus() == MediaPlayerAVAssetStatusDoesNotExist) {
713         createAVAssetForURL(m_assetURL);
714         checkPlayability();
715     }
716
717     // Don't force creation of the player and player item unless we already know that the asset is playable. If we aren't
718     // there yet, or if we already know it is not playable, creating them now won't help.
719     if (m_preload == MediaPlayer::Auto && m_assetIsPlayable) {
720         createAVPlayerItem();
721         createAVPlayer();
722     }
723
724     setDelayCallbacks(false);
725 }
726
727 void MediaPlayerPrivateAVFoundation::setDelayCallbacks(bool delay) const
728 {
729     LockHolder lock(m_queueMutex);
730     if (delay)
731         ++m_delayCallbacks;
732     else {
733         ASSERT(m_delayCallbacks);
734         --m_delayCallbacks;
735     }
736 }
737
738 void MediaPlayerPrivateAVFoundation::mainThreadCallback()
739 {
740     clearMainThreadPendingFlag();
741     dispatchNotification();
742 }
743
744 void MediaPlayerPrivateAVFoundation::clearMainThreadPendingFlag()
745 {
746     LockHolder lock(m_queueMutex);
747     m_mainThreadCallPending = false;
748 }
749
750 void MediaPlayerPrivateAVFoundation::scheduleMainThreadNotification(Notification::Type type, const MediaTime& time)
751 {
752     scheduleMainThreadNotification(Notification(type, time));
753 }
754
755 void MediaPlayerPrivateAVFoundation::scheduleMainThreadNotification(Notification::Type type, bool finished)
756 {
757     scheduleMainThreadNotification(Notification(type, finished));
758 }
759
760 void MediaPlayerPrivateAVFoundation::scheduleMainThreadNotification(Notification&& notification)
761 {
762     m_queueMutex.lock();
763
764     // It is important to always process the properties in the order that we are notified,
765     // so always go through the queue because notifications happen on different threads.
766     m_queuedNotifications.append(WTFMove(notification));
767
768 #if OS(WINDOWS)
769     bool delayDispatch = true;
770 #else
771     bool delayDispatch = m_delayCallbacks || !isMainThread();
772 #endif
773     if (delayDispatch && !m_mainThreadCallPending) {
774         m_mainThreadCallPending = true;
775
776         callOnMainThread([weakThis = makeWeakPtr(*this)] {
777             if (!weakThis)
778                 return;
779
780             weakThis->mainThreadCallback();
781         });
782     }
783
784     m_queueMutex.unlock();
785
786     if (delayDispatch)
787         return;
788
789     dispatchNotification();
790 }
791
792 void MediaPlayerPrivateAVFoundation::dispatchNotification()
793 {
794     ASSERT(isMainThread());
795
796     Notification notification;
797     {
798         LockHolder lock(m_queueMutex);
799         
800         if (m_queuedNotifications.isEmpty())
801             return;
802         
803         if (!m_delayCallbacks) {
804             // Only dispatch one notification callback per invocation because they can cause recursion.
805             notification = m_queuedNotifications.takeFirst();
806         }
807         
808         if (!m_queuedNotifications.isEmpty() && !m_mainThreadCallPending) {
809             callOnMainThread([weakThis = makeWeakPtr(*this)] {
810                 if (!weakThis)
811                     return;
812
813                 weakThis->mainThreadCallback();
814             });
815         }
816
817         if (!notification.isValid())
818             return;
819     }
820
821     switch (notification.type()) {
822     case Notification::ItemDidPlayToEndTime:
823         didEnd();
824         break;
825     case Notification::ItemTracksChanged:
826         tracksChanged();
827         updateStates();
828         break;
829     case Notification::ItemStatusChanged:
830         updateStates();
831         break;
832     case Notification::ItemSeekableTimeRangesChanged:
833         seekableTimeRangesChanged();
834         updateStates();
835         break;
836     case Notification::ItemLoadedTimeRangesChanged:
837         loadedTimeRangesChanged();
838         updateStates();
839         break;
840     case Notification::ItemPresentationSizeChanged:
841         sizeChanged();
842         updateStates();
843         break;
844     case Notification::ItemIsPlaybackLikelyToKeepUpChanged:
845         updateStates();
846         break;
847     case Notification::ItemIsPlaybackBufferEmptyChanged:
848         updateStates();
849         break;
850     case Notification::ItemIsPlaybackBufferFullChanged:
851         updateStates();
852         break;
853     case Notification::PlayerRateChanged:
854         updateStates();
855         rateChanged();
856         break;
857     case Notification::PlayerTimeChanged:
858         timeChanged(notification.time());
859         break;
860     case Notification::SeekCompleted:
861         seekCompleted(notification.finished());
862         break;
863     case Notification::AssetMetadataLoaded:
864         metadataLoaded();
865         updateStates();
866         break;
867     case Notification::AssetPlayabilityKnown:
868         updateStates();
869         playabilityKnown();
870         break;
871     case Notification::DurationChanged:
872         invalidateCachedDuration();
873         break;
874     case Notification::ContentsNeedsDisplay:
875         contentsNeedsDisplay();
876         break;
877     case Notification::InbandTracksNeedConfiguration:
878         m_inbandTrackConfigurationPending = false;
879         configureInbandTracks();
880         break;
881     case Notification::FunctionType:
882         notification.function()();
883         break;
884     case Notification::TargetIsWirelessChanged:
885 #if ENABLE(WIRELESS_PLAYBACK_TARGET)
886         playbackTargetIsWirelessChanged();
887 #endif
888         break;
889
890     case Notification::None:
891         ASSERT_NOT_REACHED();
892         break;
893     }
894 }
895
896 void MediaPlayerPrivateAVFoundation::configureInbandTracks()
897 {
898     RefPtr<InbandTextTrackPrivateAVF> trackToEnable;
899     
900 #if ENABLE(AVF_CAPTIONS)
901     synchronizeTextTrackState();
902 #endif
903
904     // AVFoundation can only emit cues for one track at a time, so enable the first track that is showing, or the first that
905     // is hidden if none are showing. Otherwise disable all tracks.
906     for (unsigned i = 0; i < m_textTracks.size(); ++i) {
907         RefPtr<InbandTextTrackPrivateAVF> track = m_textTracks[i];
908         if (track->mode() == InbandTextTrackPrivate::Showing) {
909             trackToEnable = track;
910             break;
911         }
912         if (track->mode() == InbandTextTrackPrivate::Hidden)
913             trackToEnable = track;
914     }
915
916     setCurrentTextTrack(trackToEnable.get());
917 }
918
919 void MediaPlayerPrivateAVFoundation::trackModeChanged()
920 {
921     if (m_inbandTrackConfigurationPending)
922         return;
923     m_inbandTrackConfigurationPending = true;
924     scheduleMainThreadNotification(Notification::InbandTracksNeedConfiguration);
925 }
926
927 void MediaPlayerPrivateAVFoundation::clearTextTracks()
928 {
929     for (auto& track : m_textTracks) {
930         player()->removeTextTrack(*track);
931         track->disconnect();
932     }
933     m_textTracks.clear();
934 }
935
936 void MediaPlayerPrivateAVFoundation::processNewAndRemovedTextTracks(const Vector<RefPtr<InbandTextTrackPrivateAVF>>& removedTextTracks)
937 {
938     if (removedTextTracks.size()) {
939         for (unsigned i = 0; i < m_textTracks.size(); ) {
940             if (!removedTextTracks.contains(m_textTracks[i])) {
941                 ++i;
942                 continue;
943             }
944             player()->removeTextTrack(*m_textTracks[i]);
945             m_textTracks.remove(i);
946         }
947     }
948
949     unsigned trackCount = m_textTracks.size();
950     unsigned inBandCount = 0;
951     for (unsigned i = 0; i < trackCount; ++i) {
952         RefPtr<InbandTextTrackPrivateAVF> track = m_textTracks[i];
953
954 #if ENABLE(AVF_CAPTIONS)
955         if (track->textTrackCategory() == InbandTextTrackPrivateAVF::OutOfBand)
956             continue;
957 #endif
958
959         track->setTextTrackIndex(inBandCount);
960         ++inBandCount;
961         if (track->hasBeenReported())
962             continue;
963         
964         track->setHasBeenReported(true);
965         player()->addTextTrack(*track);
966     }
967
968     if (trackCount != m_textTracks.size())
969         INFO_LOG(LOGIDENTIFIER, "found ", m_textTracks.size(), " text tracks");
970 }
971
972 #if ENABLE(WIRELESS_PLAYBACK_TARGET)
973 void MediaPlayerPrivateAVFoundation::playbackTargetIsWirelessChanged()
974 {
975     if (m_player)
976         m_player->currentPlaybackTargetIsWirelessChanged();
977 }
978 #endif
979
980 #if ENABLE(LEGACY_ENCRYPTED_MEDIA)
981 bool MediaPlayerPrivateAVFoundation::extractKeyURIKeyIDAndCertificateFromInitData(Uint8Array* initData, String& keyURI, String& keyID, RefPtr<Uint8Array>& certificate)
982 {
983     // initData should have the following layout:
984     // [4 bytes: keyURI length][N bytes: keyURI][4 bytes: contentID length], [N bytes: contentID], [4 bytes: certificate length][N bytes: certificate]
985     if (initData->byteLength() < 4)
986         return false;
987
988     RefPtr<ArrayBuffer> initDataBuffer = initData->unsharedBuffer();
989
990     // Use a DataView to read uint32 values from the buffer, as Uint32Array requires the reads be aligned on 4-byte boundaries. 
991     auto initDataView = JSC::DataView::create(initDataBuffer.copyRef(), 0, initDataBuffer->byteLength());
992     uint32_t offset = 0;
993     bool status = true;
994
995     uint32_t keyURILength = initDataView->get<uint32_t>(offset, true, &status);
996     offset += 4;
997     if (!status || offset + keyURILength > initData->length())
998         return false;
999
1000     auto keyURIArray = Uint16Array::tryCreate(initDataBuffer.copyRef(), offset, keyURILength);
1001     if (!keyURIArray)
1002         return false;
1003
1004     keyURI = String(reinterpret_cast<UChar*>(keyURIArray->data()), keyURILength / sizeof(unsigned short));
1005     offset += keyURILength;
1006
1007     uint32_t keyIDLength = initDataView->get<uint32_t>(offset, true, &status);
1008     offset += 4;
1009     if (!status || offset + keyIDLength > initData->length())
1010         return false;
1011
1012     auto keyIDArray = Uint8Array::tryCreate(initDataBuffer.copyRef(), offset, keyIDLength);
1013     if (!keyIDArray)
1014         return false;
1015
1016     keyID = String(reinterpret_cast<UChar*>(keyIDArray->data()), keyIDLength / sizeof(unsigned short));
1017     offset += keyIDLength;
1018
1019     uint32_t certificateLength = initDataView->get<uint32_t>(offset, true, &status);
1020     offset += 4;
1021     if (!status || offset + certificateLength > initData->length())
1022         return false;
1023
1024     certificate = Uint8Array::tryCreate(WTFMove(initDataBuffer), offset, certificateLength);
1025     if (!certificate)
1026         return false;
1027
1028     return true;
1029 }
1030 #endif
1031
1032 bool MediaPlayerPrivateAVFoundation::canSaveMediaData() const
1033 {
1034     URL url = resolvedURL();
1035
1036     if (url.isLocalFile())
1037         return true;
1038
1039     if (!url.protocolIsInHTTPFamily())
1040         return false;
1041
1042     if (isLiveStream())
1043         return false;
1044
1045     return true;
1046 }
1047
1048 bool MediaPlayerPrivateAVFoundation::isUnsupportedMIMEType(const String& type)
1049 {
1050     String lowerCaseType = type.convertToASCIILowercase();
1051
1052     // AVFoundation will return non-video MIME types which it claims to support, but which we
1053     // do not support in the <video> element. Reject all non video/, audio/, and application/ types.
1054     if (!lowerCaseType.startsWith("video/") && !lowerCaseType.startsWith("audio/") && !lowerCaseType.startsWith("application/"))
1055         return true;
1056
1057     // Reject types we know AVFoundation does not support that sites commonly ask about.
1058     if (lowerCaseType == "video/webm" || lowerCaseType == "audio/webm" || lowerCaseType == "video/x-webm")
1059         return true;
1060
1061     if (lowerCaseType == "video/x-flv")
1062         return true;
1063
1064     if (lowerCaseType == "audio/ogg" || lowerCaseType == "video/ogg" || lowerCaseType == "application/ogg")
1065         return true;
1066
1067     if (lowerCaseType == "video/h264")
1068         return true;
1069
1070     return false;
1071 }
1072
1073 #if !RELEASE_LOG_DISABLED
1074 WTFLogChannel& MediaPlayerPrivateAVFoundation::logChannel() const
1075 {
1076     return LogMedia;
1077 }
1078 #endif
1079
1080 const HashSet<String, ASCIICaseInsensitiveHash>& MediaPlayerPrivateAVFoundation::staticMIMETypeList()
1081 {
1082     static const auto cache = makeNeverDestroyed(HashSet<String, ASCIICaseInsensitiveHash> {
1083         "application/vnd.apple.mpegurl",
1084         "application/x-mpegurl",
1085         "audio/3gpp",
1086         "audio/aac",
1087         "audio/aacp",
1088         "audio/aiff",
1089         "audio/basic",
1090         "audio/mp3",
1091         "audio/mp4",
1092         "audio/mpeg",
1093         "audio/mpeg3",
1094         "audio/mpegurl",
1095         "audio/mpg",
1096         "audio/vnd.wave",
1097         "audio/wav",
1098         "audio/wave",
1099         "audio/x-aac",
1100         "audio/x-aiff",
1101         "audio/x-m4a",
1102         "audio/x-mpegurl",
1103         "audio/x-wav",
1104         "video/3gpp",
1105         "video/3gpp2",
1106         "video/mp4",
1107         "video/mpeg",
1108         "video/mpeg2",
1109         "video/mpg",
1110         "video/quicktime",
1111         "video/x-m4v",
1112         "video/x-mpeg",
1113         "video/x-mpg",
1114     });
1115     return cache;
1116 }
1117
1118 } // namespace WebCore
1119
1120 #endif