Move URL from WebCore to WTF
[WebKit-https.git] / Source / WebCore / platform / graphics / avfoundation / MediaPlayerPrivateAVFoundation.cpp
1 /*
2  * Copyright (C) 2011-2014 Apple Inc. All rights reserved.
3  *
4  * Redistribution and use in source and binary forms, with or without
5  * modification, are permitted provided that the following conditions
6  * are met:
7  * 1. Redistributions of source code must retain the above copyright
8  *    notice, this list of conditions and the following disclaimer.
9  * 2. Redistributions in binary form must reproduce the above copyright
10  *    notice, this list of conditions and the following disclaimer in the
11  *    documentation and/or other materials provided with the distribution.
12  *
13  * THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY
14  * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
15  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
16  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL APPLE INC. OR
17  * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
18  * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
19  * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
20  * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
21  * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
22  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
23  * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 
24  */
25
26 #include "config.h"
27
28 #if ENABLE(VIDEO) && USE(AVFOUNDATION)
29
30 #include "MediaPlayerPrivateAVFoundation.h"
31
32 #include "DeprecatedGlobalSettings.h"
33 #include "DocumentLoader.h"
34 #include "FloatConversion.h"
35 #include "GraphicsContext.h"
36 #include "InbandTextTrackPrivateAVF.h"
37 #include "InbandTextTrackPrivateClient.h"
38 #include "Logging.h"
39 #include "PlatformLayer.h"
40 #include "PlatformTimeRanges.h"
41 #include "Settings.h"
42 #include <CoreMedia/CoreMedia.h>
43 #include <JavaScriptCore/DataView.h>
44 #include <JavaScriptCore/HeapInlines.h>
45 #include <JavaScriptCore/TypedArrayInlines.h>
46 #include <JavaScriptCore/Uint16Array.h>
47 #include <wtf/MainThread.h>
48 #include <wtf/NeverDestroyed.h>
49 #include <wtf/SoftLinking.h>
50 #include <wtf/StringPrintStream.h>
51 #include <wtf/URL.h>
52 #include <wtf/text/CString.h>
53
54 namespace WebCore {
55
56 MediaPlayerPrivateAVFoundation::MediaPlayerPrivateAVFoundation(MediaPlayer* player)
57     : m_player(player)
58     , m_queuedNotifications()
59     , m_queueMutex()
60     , m_networkState(MediaPlayer::Empty)
61     , m_readyState(MediaPlayer::HaveNothing)
62     , m_preload(MediaPlayer::Auto)
63 #if !RELEASE_LOG_DISABLED
64     , m_logger(player->mediaPlayerLogger())
65     , m_logIdentifier(player->mediaPlayerLogIdentifier())
66 #endif
67     , m_cachedDuration(MediaTime::invalidTime())
68     , m_reportedDuration(MediaTime::invalidTime())
69     , m_maxTimeLoadedAtLastDidLoadingProgress(MediaTime::invalidTime())
70     , m_delayCallbacks(0)
71     , m_delayCharacteristicsChangedNotification(0)
72     , m_mainThreadCallPending(false)
73     , m_assetIsPlayable(false)
74     , m_visible(false)
75     , m_loadingMetadata(false)
76     , m_isAllowedToRender(false)
77     , m_cachedHasAudio(false)
78     , m_cachedHasVideo(false)
79     , m_cachedHasCaptions(false)
80     , m_ignoreLoadStateChanges(false)
81     , m_haveReportedFirstVideoFrame(false)
82     , m_playWhenFramesAvailable(false)
83     , m_inbandTrackConfigurationPending(false)
84     , m_characteristicsChanged(false)
85     , m_shouldMaintainAspectRatio(true)
86     , m_seeking(false)
87 {
88     INFO_LOG(LOGIDENTIFIER);
89 }
90
91 MediaPlayerPrivateAVFoundation::~MediaPlayerPrivateAVFoundation()
92 {
93     INFO_LOG(LOGIDENTIFIER);
94     setIgnoreLoadStateChanges(true);
95 }
96
97 MediaPlayerPrivateAVFoundation::MediaRenderingMode MediaPlayerPrivateAVFoundation::currentRenderingMode() const
98 {
99     if (platformLayer())
100         return MediaRenderingToLayer;
101
102     if (hasContextRenderer())
103         return MediaRenderingToContext;
104
105     return MediaRenderingNone;
106 }
107
108 MediaPlayerPrivateAVFoundation::MediaRenderingMode MediaPlayerPrivateAVFoundation::preferredRenderingMode() const
109 {
110     if (!m_player->visible() || assetStatus() == MediaPlayerAVAssetStatusUnknown)
111         return MediaRenderingNone;
112
113     if (supportsAcceleratedRendering() && m_player->client().mediaPlayerRenderingCanBeAccelerated(m_player))
114         return MediaRenderingToLayer;
115
116     return MediaRenderingToContext;
117 }
118
119 void MediaPlayerPrivateAVFoundation::setUpVideoRendering()
120 {
121     if (!isReadyForVideoSetup())
122         return;
123
124     MediaRenderingMode currentMode = currentRenderingMode();
125     MediaRenderingMode preferredMode = preferredRenderingMode();
126
127     if (preferredMode == MediaRenderingNone)
128         preferredMode = MediaRenderingToContext;
129
130     if (currentMode == preferredMode && currentMode != MediaRenderingNone)
131         return;
132
133     if (currentMode != MediaRenderingNone)
134         tearDownVideoRendering();
135
136     switch (preferredMode) {
137     case MediaRenderingNone:
138     case MediaRenderingToContext:
139         createContextVideoRenderer();
140         break;
141
142     case MediaRenderingToLayer:
143         createVideoLayer();
144         break;
145     }
146
147     // If using a movie layer, inform the client so the compositing tree is updated.
148     if (currentMode == MediaRenderingToLayer || preferredMode == MediaRenderingToLayer)
149         m_player->client().mediaPlayerRenderingModeChanged(m_player);
150 }
151
152 void MediaPlayerPrivateAVFoundation::tearDownVideoRendering()
153 {
154     INFO_LOG(LOGIDENTIFIER);
155
156     destroyContextVideoRenderer();
157
158     if (platformLayer())
159         destroyVideoLayer();
160 }
161
162 bool MediaPlayerPrivateAVFoundation::hasSetUpVideoRendering() const
163 {
164     return hasLayerRenderer() || hasContextRenderer();
165 }
166
167 void MediaPlayerPrivateAVFoundation::load(const String& url)
168 {
169     ALWAYS_LOG(LOGIDENTIFIER);
170
171     setNetworkState(m_preload == MediaPlayer::None ? MediaPlayer::Idle : MediaPlayer::Loading);
172     setReadyState(MediaPlayer::HaveNothing);
173
174     m_assetURL = URL({ }, url);
175     m_requestedOrigin = SecurityOrigin::create(m_assetURL);
176
177     // Don't do any more work if the url is empty.
178     if (!url.length())
179         return;
180
181     setPreload(m_preload);
182 }
183
184 #if ENABLE(MEDIA_SOURCE)
185 void MediaPlayerPrivateAVFoundation::load(const String&, MediaSourcePrivateClient*)
186 {
187     setNetworkState(MediaPlayer::FormatError);
188 }
189 #endif
190
191
192 void MediaPlayerPrivateAVFoundation::playabilityKnown()
193 {
194     INFO_LOG(LOGIDENTIFIER, "metadata loaded = ", assetStatus() > MediaPlayerAVAssetStatusLoading);
195
196     if (m_assetIsPlayable)
197         return;
198
199     // Nothing more to do if we already have all of the item's metadata.
200     if (assetStatus() > MediaPlayerAVAssetStatusLoading)
201         return;
202
203     // At this point we are supposed to load metadata. It is OK to ask the asset to load the same 
204     // information multiple times, because if it has already been loaded the completion handler 
205     // will just be called synchronously.
206     m_loadingMetadata = true;
207     beginLoadingMetadata();
208 }
209
210 void MediaPlayerPrivateAVFoundation::prepareToPlay()
211 {
212     ALWAYS_LOG(LOGIDENTIFIER);
213     setPreload(MediaPlayer::Auto);
214 }
215
216 void MediaPlayerPrivateAVFoundation::play()
217 {
218     ALWAYS_LOG(LOGIDENTIFIER);
219
220     // If the file has video, don't request playback until the first frame of video is ready to display
221     // or the audio may start playing before we can render video.
222     if (!m_cachedHasVideo || hasAvailableVideoFrame())
223         platformPlay();
224     else {
225         INFO_LOG(LOGIDENTIFIER, "waiting for first video frame");
226         m_playWhenFramesAvailable = true;
227     }
228 }
229
230 void MediaPlayerPrivateAVFoundation::pause()
231 {
232     ALWAYS_LOG(LOGIDENTIFIER);
233     m_playWhenFramesAvailable = false;
234     platformPause();
235 }
236
237 MediaTime MediaPlayerPrivateAVFoundation::durationMediaTime() const
238 {
239     if (m_cachedDuration.isValid())
240         return m_cachedDuration;
241
242     MediaTime duration = platformDuration();
243     if (!duration || duration.isInvalid())
244         return MediaTime::zeroTime();
245
246     m_cachedDuration = duration;
247
248     return m_cachedDuration;
249 }
250
251 void MediaPlayerPrivateAVFoundation::seek(const MediaTime& time)
252 {
253     seekWithTolerance(time, MediaTime::zeroTime(), MediaTime::zeroTime());
254 }
255
256 void MediaPlayerPrivateAVFoundation::seekWithTolerance(const MediaTime& mediaTime, const MediaTime& negativeTolerance, const MediaTime& positiveTolerance)
257 {
258     MediaTime time = mediaTime;
259
260     if (m_seeking) {
261         ALWAYS_LOG(LOGIDENTIFIER, "saving pending seek");
262         m_pendingSeek = [this, time, negativeTolerance, positiveTolerance]() {
263             seekWithTolerance(time, negativeTolerance, positiveTolerance);
264         };
265         return;
266     }
267     m_seeking = true;
268
269     if (!metaDataAvailable())
270         return;
271
272     if (time > durationMediaTime())
273         time = durationMediaTime();
274
275     if (currentTextTrack())
276         currentTextTrack()->beginSeeking();
277
278     ALWAYS_LOG(LOGIDENTIFIER, "seeking to  ", time);
279
280     seekToTime(time, negativeTolerance, positiveTolerance);
281 }
282
283 bool MediaPlayerPrivateAVFoundation::paused() const
284 {
285     if (!metaDataAvailable())
286         return true;
287
288     return rate() == 0;
289 }
290
291 bool MediaPlayerPrivateAVFoundation::seeking() const
292 {
293     if (!metaDataAvailable())
294         return false;
295
296     return m_seeking;
297 }
298
299 FloatSize MediaPlayerPrivateAVFoundation::naturalSize() const
300 {
301     if (!metaDataAvailable())
302         return IntSize();
303
304     // In spite of the name of this method, return the natural size transformed by the 
305     // initial movie scale because the spec says intrinsic size is:
306     //
307     //    ... the dimensions of the resource in CSS pixels after taking into account the resource's 
308     //    dimensions, aspect ratio, clean aperture, resolution, and so forth, as defined for the 
309     //    format used by the resource
310
311     return m_cachedNaturalSize;
312 }
313
314 void MediaPlayerPrivateAVFoundation::setNaturalSize(FloatSize size)
315 {
316     FloatSize oldSize = m_cachedNaturalSize;
317     m_cachedNaturalSize = size;
318     if (oldSize != m_cachedNaturalSize) {
319         INFO_LOG(LOGIDENTIFIER, "was ", oldSize.width(), " x ", oldSize.height(), ", is ", size.width(), " x ", size.height());
320         m_player->sizeChanged();
321     }
322 }
323
324 void MediaPlayerPrivateAVFoundation::setHasVideo(bool b)
325 {
326     if (m_cachedHasVideo != b) {
327         m_cachedHasVideo = b;
328         characteristicsChanged();
329     }
330 }
331
332 void MediaPlayerPrivateAVFoundation::setHasAudio(bool b)
333 {
334     if (m_cachedHasAudio != b) {
335         m_cachedHasAudio = b;
336         characteristicsChanged();
337     }
338 }
339
340 void MediaPlayerPrivateAVFoundation::setHasClosedCaptions(bool b)
341 {
342     if (m_cachedHasCaptions != b) {
343         m_cachedHasCaptions = b;
344         characteristicsChanged();
345     }
346 }
347
348 void MediaPlayerPrivateAVFoundation::setNetworkState(MediaPlayer::NetworkState state)
349 {
350     if (state == m_networkState)
351         return;
352
353     m_networkState = state;
354     m_player->networkStateChanged();
355 }
356
357 void MediaPlayerPrivateAVFoundation::setReadyState(MediaPlayer::ReadyState state)
358 {
359     if (state == m_readyState)
360         return;
361
362     m_readyState = state;
363     m_player->readyStateChanged();
364 }
365
366 void MediaPlayerPrivateAVFoundation::characteristicsChanged()
367 {
368     if (m_delayCharacteristicsChangedNotification) {
369         m_characteristicsChanged = true;
370         return;
371     }
372
373     m_characteristicsChanged = false;
374     m_player->characteristicChanged();
375 }
376
377 void MediaPlayerPrivateAVFoundation::setDelayCharacteristicsChangedNotification(bool delay)
378 {
379     if (delay) {
380         m_delayCharacteristicsChangedNotification++;
381         return;
382     }
383     
384     ASSERT(m_delayCharacteristicsChangedNotification);
385     m_delayCharacteristicsChangedNotification--;
386     if (!m_delayCharacteristicsChangedNotification && m_characteristicsChanged)
387         characteristicsChanged();
388 }
389
390 std::unique_ptr<PlatformTimeRanges> MediaPlayerPrivateAVFoundation::buffered() const
391 {
392     if (!m_cachedLoadedTimeRanges)
393         m_cachedLoadedTimeRanges = platformBufferedTimeRanges();
394
395     return std::make_unique<PlatformTimeRanges>(*m_cachedLoadedTimeRanges);
396 }
397
398 MediaTime MediaPlayerPrivateAVFoundation::maxMediaTimeSeekable() const
399 {
400     if (!metaDataAvailable())
401         return MediaTime::zeroTime();
402
403     if (!m_cachedMaxTimeSeekable)
404         m_cachedMaxTimeSeekable = platformMaxTimeSeekable();
405
406     return m_cachedMaxTimeSeekable;
407 }
408
409 MediaTime MediaPlayerPrivateAVFoundation::minMediaTimeSeekable() const
410 {
411     if (!metaDataAvailable())
412         return MediaTime::zeroTime();
413
414     if (!m_cachedMinTimeSeekable)
415         m_cachedMinTimeSeekable = platformMinTimeSeekable();
416
417     return m_cachedMinTimeSeekable;
418 }
419
420 double MediaPlayerPrivateAVFoundation::requestedRate() const
421 {
422     return m_player->requestedRate();
423 }
424
425 MediaTime MediaPlayerPrivateAVFoundation::maxTimeLoaded() const
426 {
427     if (!metaDataAvailable())
428         return MediaTime::zeroTime();
429
430     if (!m_cachedMaxTimeLoaded)
431         m_cachedMaxTimeLoaded = platformMaxTimeLoaded();
432
433     return m_cachedMaxTimeLoaded;   
434 }
435
436 bool MediaPlayerPrivateAVFoundation::didLoadingProgress() const
437 {
438     if (!durationMediaTime())
439         return false;
440     MediaTime currentMaxTimeLoaded = maxTimeLoaded();
441     bool didLoadingProgress = currentMaxTimeLoaded != m_maxTimeLoadedAtLastDidLoadingProgress;
442     m_maxTimeLoadedAtLastDidLoadingProgress = currentMaxTimeLoaded;
443
444     return didLoadingProgress;
445 }
446
447 bool MediaPlayerPrivateAVFoundation::isReadyForVideoSetup() const
448 {
449     // AVFoundation will not return true for firstVideoFrameAvailable until
450     // an AVPlayerLayer has been added to the AVPlayerItem, so allow video setup
451     // here if a video track to trigger allocation of a AVPlayerLayer.
452     return (m_isAllowedToRender || m_cachedHasVideo) && m_readyState >= MediaPlayer::HaveMetadata && m_player->visible();
453 }
454
455 void MediaPlayerPrivateAVFoundation::prepareForRendering()
456 {
457     if (m_isAllowedToRender)
458         return;
459     m_isAllowedToRender = true;
460
461     setUpVideoRendering();
462
463     if (currentRenderingMode() == MediaRenderingToLayer || preferredRenderingMode() == MediaRenderingToLayer)
464         m_player->client().mediaPlayerRenderingModeChanged(m_player);
465 }
466
467 bool MediaPlayerPrivateAVFoundation::supportsFullscreen() const
468 {
469 #if ENABLE(FULLSCREEN_API)
470     return true;
471 #else
472     // FIXME: WebVideoFullscreenController assumes a QTKit/QuickTime media engine
473 #if PLATFORM(IOS_FAMILY)
474     if (DeprecatedGlobalSettings::avKitEnabled())
475         return true;
476 #endif
477     return false;
478 #endif
479 }
480
481 bool MediaPlayerPrivateAVFoundation::hasSingleSecurityOrigin() const
482 {
483     if (m_resolvedOrigin && m_requestedOrigin)
484         return m_resolvedOrigin->isSameSchemeHostPort(*m_requestedOrigin);
485     return false;
486 }
487
488 void MediaPlayerPrivateAVFoundation::setResolvedURL(URL&& resolvedURL)
489 {
490     m_resolvedURL = WTFMove(resolvedURL);
491     m_resolvedOrigin = SecurityOrigin::create(m_resolvedURL);
492 }
493
494 void MediaPlayerPrivateAVFoundation::updateStates()
495 {
496     if (m_ignoreLoadStateChanges)
497         return;
498
499     MediaPlayer::NetworkState newNetworkState = m_networkState;
500     MediaPlayer::ReadyState newReadyState = m_readyState;
501
502     if (m_loadingMetadata)
503         newNetworkState = MediaPlayer::Loading;
504     else {
505         // -loadValuesAsynchronouslyForKeys:completionHandler: has invoked its handler; test status of keys and determine state.
506         AssetStatus assetStatus = this->assetStatus();
507         ItemStatus itemStatus = playerItemStatus();
508         
509         m_assetIsPlayable = (assetStatus == MediaPlayerAVAssetStatusPlayable);
510         if (m_readyState < MediaPlayer::HaveMetadata && assetStatus > MediaPlayerAVAssetStatusLoading) {
511             if (m_assetIsPlayable) {
512                 if (assetStatus >= MediaPlayerAVAssetStatusLoaded)
513                     newReadyState = MediaPlayer::HaveMetadata;
514                 if (itemStatus <= MediaPlayerAVPlayerItemStatusUnknown) {
515                     if (assetStatus == MediaPlayerAVAssetStatusFailed || m_preload > MediaPlayer::MetaData || isLiveStream()) {
516                         // The asset is playable but doesn't support inspection prior to playback (eg. streaming files),
517                         // or we are supposed to prepare for playback immediately, so create the player item now.
518                         newNetworkState = MediaPlayer::Loading;
519                         prepareToPlay();
520                     } else
521                         newNetworkState = MediaPlayer::Idle;
522                 }
523             } else {
524                 // FIX ME: fetch the error associated with the @"playable" key to distinguish between format 
525                 // and network errors.
526                 newNetworkState = MediaPlayer::FormatError;
527             }
528         }
529         
530         if (assetStatus >= MediaPlayerAVAssetStatusLoaded && itemStatus > MediaPlayerAVPlayerItemStatusUnknown) {
531             switch (itemStatus) {
532             case MediaPlayerAVPlayerItemStatusDoesNotExist:
533             case MediaPlayerAVPlayerItemStatusUnknown:
534             case MediaPlayerAVPlayerItemStatusFailed:
535                 break;
536
537             case MediaPlayerAVPlayerItemStatusPlaybackLikelyToKeepUp:
538             case MediaPlayerAVPlayerItemStatusPlaybackBufferFull:
539                 // If the status becomes PlaybackBufferFull, loading stops and the status will not
540                 // progress to LikelyToKeepUp. Set the readyState to  HAVE_ENOUGH_DATA, on the
541                 // presumption that if the playback buffer is full, playback will probably not stall.
542                 newReadyState = MediaPlayer::HaveEnoughData;
543                 break;
544
545             case MediaPlayerAVPlayerItemStatusReadyToPlay:
546                 if (m_readyState != MediaPlayer::HaveEnoughData && maxTimeLoaded() > currentMediaTime())
547                     newReadyState = MediaPlayer::HaveFutureData;
548                 break;
549
550             case MediaPlayerAVPlayerItemStatusPlaybackBufferEmpty:
551                 newReadyState = MediaPlayer::HaveCurrentData;
552                 break;
553             }
554
555             if (itemStatus == MediaPlayerAVPlayerItemStatusPlaybackBufferFull)
556                 newNetworkState = MediaPlayer::Idle;
557             else if (itemStatus == MediaPlayerAVPlayerItemStatusFailed)
558                 newNetworkState = MediaPlayer::DecodeError;
559             else if (itemStatus != MediaPlayerAVPlayerItemStatusPlaybackBufferFull && itemStatus >= MediaPlayerAVPlayerItemStatusReadyToPlay)
560                 newNetworkState = (maxTimeLoaded() == durationMediaTime()) ? MediaPlayer::Loaded : MediaPlayer::Loading;
561         }
562     }
563
564     if (isReadyForVideoSetup() && currentRenderingMode() != preferredRenderingMode())
565         setUpVideoRendering();
566
567     if (!m_haveReportedFirstVideoFrame && m_cachedHasVideo && hasAvailableVideoFrame()) {
568         if (m_readyState < MediaPlayer::HaveCurrentData)
569             newReadyState = MediaPlayer::HaveCurrentData;
570         m_haveReportedFirstVideoFrame = true;
571         m_player->firstVideoFrameAvailable();
572     } else if (!hasAvailableVideoFrame())
573         m_haveReportedFirstVideoFrame = false;
574
575     if (m_networkState != newNetworkState)
576         ALWAYS_LOG(LOGIDENTIFIER, "entered with networkState ", m_networkState, ", exiting with ", newNetworkState);
577     if (m_readyState != newReadyState)
578         ALWAYS_LOG(LOGIDENTIFIER, "entered with readyState ", m_readyState, ", exiting with ", newReadyState);
579
580     setNetworkState(newNetworkState);
581     setReadyState(newReadyState);
582
583     if (m_playWhenFramesAvailable && hasAvailableVideoFrame()) {
584         m_playWhenFramesAvailable = false;
585         platformPlay();
586     }
587 }
588
589 void MediaPlayerPrivateAVFoundation::setSize(const IntSize&) 
590
591 }
592
593 void MediaPlayerPrivateAVFoundation::setVisible(bool visible)
594 {
595     if (m_visible == visible)
596         return;
597
598     m_visible = visible;
599     if (visible)
600         setUpVideoRendering();
601     
602     platformSetVisible(visible);
603 }
604
605 void MediaPlayerPrivateAVFoundation::acceleratedRenderingStateChanged()
606 {
607     // Set up or change the rendering path if necessary.
608     setUpVideoRendering();
609 }
610
611 void MediaPlayerPrivateAVFoundation::setShouldMaintainAspectRatio(bool maintainAspectRatio)
612 {
613     if (maintainAspectRatio == m_shouldMaintainAspectRatio)
614         return;
615
616     m_shouldMaintainAspectRatio = maintainAspectRatio;
617     updateVideoLayerGravity();
618 }
619
620 void MediaPlayerPrivateAVFoundation::metadataLoaded()
621 {
622     m_loadingMetadata = false;
623     resolvedURLChanged();
624     tracksChanged();
625 }
626
627 void MediaPlayerPrivateAVFoundation::rateChanged()
628 {
629
630 #if ENABLE(WIRELESS_PLAYBACK_TARGET) && PLATFORM(IOS_FAMILY)
631     if (isCurrentPlaybackTargetWireless() && playerItemStatus() >= MediaPlayerAVPlayerItemStatusPlaybackBufferFull) {
632         double rate = this->rate();
633         if (rate != requestedRate()) {
634             m_player->handlePlaybackCommand(rate ? PlatformMediaSession::PlayCommand : PlatformMediaSession::PauseCommand);
635             return;
636         }
637     }
638 #endif
639
640     m_player->rateChanged();
641 }
642
643 void MediaPlayerPrivateAVFoundation::loadedTimeRangesChanged()
644 {
645     m_cachedLoadedTimeRanges = nullptr;
646     m_cachedMaxTimeLoaded = MediaTime::zeroTime();
647     invalidateCachedDuration();
648 }
649
650 void MediaPlayerPrivateAVFoundation::seekableTimeRangesChanged()
651 {
652     m_cachedMaxTimeSeekable = MediaTime::zeroTime();
653     m_cachedMinTimeSeekable = MediaTime::zeroTime();
654 }
655
656 void MediaPlayerPrivateAVFoundation::timeChanged(const MediaTime& time)
657 {
658     UNUSED_PARAM(time);
659     INFO_LOG(LOGIDENTIFIER, "- ", time);
660 }
661
662 void MediaPlayerPrivateAVFoundation::seekCompleted(bool finished)
663 {
664     UNUSED_PARAM(finished);
665     ALWAYS_LOG(LOGIDENTIFIER, "finished = ", finished);
666
667     m_seeking = false;
668
669     WTF::Function<void()> pendingSeek;
670     std::swap(pendingSeek, m_pendingSeek);
671
672     if (pendingSeek) {
673         ALWAYS_LOG(LOGIDENTIFIER, "issuing pending seek");
674         pendingSeek();
675         return;
676     }
677
678     if (currentTextTrack())
679         currentTextTrack()->endSeeking();
680
681     updateStates();
682     m_player->timeChanged();
683 }
684
685 void MediaPlayerPrivateAVFoundation::didEnd()
686 {
687     // Hang onto the current time and use it as duration from now on since we are definitely at
688     // the end of the movie. Do this because the initial duration is sometimes an estimate.
689     MediaTime now = currentMediaTime();
690     if (now > MediaTime::zeroTime() && !m_seeking)
691         m_cachedDuration = now;
692
693     updateStates();
694     m_player->timeChanged();
695 }
696
697 void MediaPlayerPrivateAVFoundation::invalidateCachedDuration()
698 {
699     m_cachedDuration = MediaTime::invalidTime();
700
701     // For some media files, reported duration is estimated and updated as media is loaded
702     // so report duration changed when the estimate is upated.
703     MediaTime duration = this->durationMediaTime();
704     if (duration != m_reportedDuration) {
705         INFO_LOG(LOGIDENTIFIER, "- ", m_cachedDuration);
706         if (m_reportedDuration.isValid())
707             m_player->durationChanged();
708         m_reportedDuration = duration;
709     }
710     
711 }
712
713 void MediaPlayerPrivateAVFoundation::repaint()
714 {
715     m_player->repaint();
716 }
717
718 MediaPlayer::MovieLoadType MediaPlayerPrivateAVFoundation::movieLoadType() const
719 {
720     if (!metaDataAvailable() || assetStatus() == MediaPlayerAVAssetStatusUnknown)
721         return MediaPlayer::Unknown;
722
723     if (isLiveStream())
724         return MediaPlayer::LiveStream;
725
726     return MediaPlayer::Download;
727 }
728
729 void MediaPlayerPrivateAVFoundation::setPreload(MediaPlayer::Preload preload)
730 {
731     ALWAYS_LOG(LOGIDENTIFIER, " - ", static_cast<int>(preload));
732     m_preload = preload;
733     if (m_assetURL.isEmpty())
734         return;
735
736     setDelayCallbacks(true);
737
738     if (m_preload >= MediaPlayer::MetaData && assetStatus() == MediaPlayerAVAssetStatusDoesNotExist) {
739         createAVAssetForURL(m_assetURL);
740         checkPlayability();
741     }
742
743     // Don't force creation of the player and player item unless we already know that the asset is playable. If we aren't
744     // there yet, or if we already know it is not playable, creating them now won't help.
745     if (m_preload == MediaPlayer::Auto && m_assetIsPlayable) {
746         createAVPlayerItem();
747         createAVPlayer();
748     }
749
750     setDelayCallbacks(false);
751 }
752
753 void MediaPlayerPrivateAVFoundation::setDelayCallbacks(bool delay) const
754 {
755     LockHolder lock(m_queueMutex);
756     if (delay)
757         ++m_delayCallbacks;
758     else {
759         ASSERT(m_delayCallbacks);
760         --m_delayCallbacks;
761     }
762 }
763
764 void MediaPlayerPrivateAVFoundation::mainThreadCallback()
765 {
766     clearMainThreadPendingFlag();
767     dispatchNotification();
768 }
769
770 void MediaPlayerPrivateAVFoundation::clearMainThreadPendingFlag()
771 {
772     LockHolder lock(m_queueMutex);
773     m_mainThreadCallPending = false;
774 }
775
776 void MediaPlayerPrivateAVFoundation::scheduleMainThreadNotification(Notification::Type type, const MediaTime& time)
777 {
778     scheduleMainThreadNotification(Notification(type, time));
779 }
780
781 void MediaPlayerPrivateAVFoundation::scheduleMainThreadNotification(Notification::Type type, bool finished)
782 {
783     scheduleMainThreadNotification(Notification(type, finished));
784 }
785
786 void MediaPlayerPrivateAVFoundation::scheduleMainThreadNotification(Notification&& notification)
787 {
788     m_queueMutex.lock();
789
790     // It is important to always process the properties in the order that we are notified,
791     // so always go through the queue because notifications happen on different threads.
792     m_queuedNotifications.append(WTFMove(notification));
793
794 #if OS(WINDOWS)
795     bool delayDispatch = true;
796 #else
797     bool delayDispatch = m_delayCallbacks || !isMainThread();
798 #endif
799     if (delayDispatch && !m_mainThreadCallPending) {
800         m_mainThreadCallPending = true;
801
802         callOnMainThread([weakThis = makeWeakPtr(*this)] {
803             if (!weakThis)
804                 return;
805
806             weakThis->mainThreadCallback();
807         });
808     }
809
810     m_queueMutex.unlock();
811
812     if (delayDispatch)
813         return;
814
815     dispatchNotification();
816 }
817
818 void MediaPlayerPrivateAVFoundation::dispatchNotification()
819 {
820     ASSERT(isMainThread());
821
822     Notification notification;
823     {
824         LockHolder lock(m_queueMutex);
825         
826         if (m_queuedNotifications.isEmpty())
827             return;
828         
829         if (!m_delayCallbacks) {
830             // Only dispatch one notification callback per invocation because they can cause recursion.
831             notification = m_queuedNotifications.takeFirst();
832         }
833         
834         if (!m_queuedNotifications.isEmpty() && !m_mainThreadCallPending) {
835             callOnMainThread([weakThis = makeWeakPtr(*this)] {
836                 if (!weakThis)
837                     return;
838
839                 weakThis->mainThreadCallback();
840             });
841         }
842
843         if (!notification.isValid())
844             return;
845     }
846
847     switch (notification.type()) {
848     case Notification::ItemDidPlayToEndTime:
849         didEnd();
850         break;
851     case Notification::ItemTracksChanged:
852         tracksChanged();
853         updateStates();
854         break;
855     case Notification::ItemStatusChanged:
856         updateStates();
857         break;
858     case Notification::ItemSeekableTimeRangesChanged:
859         seekableTimeRangesChanged();
860         updateStates();
861         break;
862     case Notification::ItemLoadedTimeRangesChanged:
863         loadedTimeRangesChanged();
864         updateStates();
865         break;
866     case Notification::ItemPresentationSizeChanged:
867         sizeChanged();
868         updateStates();
869         break;
870     case Notification::ItemIsPlaybackLikelyToKeepUpChanged:
871         updateStates();
872         break;
873     case Notification::ItemIsPlaybackBufferEmptyChanged:
874         updateStates();
875         break;
876     case Notification::ItemIsPlaybackBufferFullChanged:
877         updateStates();
878         break;
879     case Notification::PlayerRateChanged:
880         updateStates();
881         rateChanged();
882         break;
883     case Notification::PlayerTimeChanged:
884         timeChanged(notification.time());
885         break;
886     case Notification::SeekCompleted:
887         seekCompleted(notification.finished());
888         break;
889     case Notification::AssetMetadataLoaded:
890         metadataLoaded();
891         updateStates();
892         break;
893     case Notification::AssetPlayabilityKnown:
894         updateStates();
895         playabilityKnown();
896         break;
897     case Notification::DurationChanged:
898         invalidateCachedDuration();
899         break;
900     case Notification::ContentsNeedsDisplay:
901         contentsNeedsDisplay();
902         break;
903     case Notification::InbandTracksNeedConfiguration:
904         m_inbandTrackConfigurationPending = false;
905         configureInbandTracks();
906         break;
907     case Notification::FunctionType:
908         notification.function()();
909         break;
910     case Notification::TargetIsWirelessChanged:
911 #if ENABLE(WIRELESS_PLAYBACK_TARGET)
912         playbackTargetIsWirelessChanged();
913 #endif
914         break;
915
916     case Notification::None:
917         ASSERT_NOT_REACHED();
918         break;
919     }
920 }
921
922 void MediaPlayerPrivateAVFoundation::configureInbandTracks()
923 {
924     RefPtr<InbandTextTrackPrivateAVF> trackToEnable;
925     
926 #if ENABLE(AVF_CAPTIONS)
927     synchronizeTextTrackState();
928 #endif
929
930     // AVFoundation can only emit cues for one track at a time, so enable the first track that is showing, or the first that
931     // is hidden if none are showing. Otherwise disable all tracks.
932     for (unsigned i = 0; i < m_textTracks.size(); ++i) {
933         RefPtr<InbandTextTrackPrivateAVF> track = m_textTracks[i];
934         if (track->mode() == InbandTextTrackPrivate::Showing) {
935             trackToEnable = track;
936             break;
937         }
938         if (track->mode() == InbandTextTrackPrivate::Hidden)
939             trackToEnable = track;
940     }
941
942     setCurrentTextTrack(trackToEnable.get());
943 }
944
945 void MediaPlayerPrivateAVFoundation::trackModeChanged()
946 {
947     if (m_inbandTrackConfigurationPending)
948         return;
949     m_inbandTrackConfigurationPending = true;
950     scheduleMainThreadNotification(Notification::InbandTracksNeedConfiguration);
951 }
952
953 void MediaPlayerPrivateAVFoundation::clearTextTracks()
954 {
955     for (auto& track : m_textTracks) {
956         player()->removeTextTrack(*track);
957         track->disconnect();
958     }
959     m_textTracks.clear();
960 }
961
962 void MediaPlayerPrivateAVFoundation::processNewAndRemovedTextTracks(const Vector<RefPtr<InbandTextTrackPrivateAVF>>& removedTextTracks)
963 {
964     if (removedTextTracks.size()) {
965         for (unsigned i = 0; i < m_textTracks.size(); ) {
966             if (!removedTextTracks.contains(m_textTracks[i])) {
967                 ++i;
968                 continue;
969             }
970             player()->removeTextTrack(*m_textTracks[i]);
971             m_textTracks.remove(i);
972         }
973     }
974
975     unsigned trackCount = m_textTracks.size();
976     unsigned inBandCount = 0;
977     for (unsigned i = 0; i < trackCount; ++i) {
978         RefPtr<InbandTextTrackPrivateAVF> track = m_textTracks[i];
979
980 #if ENABLE(AVF_CAPTIONS)
981         if (track->textTrackCategory() == InbandTextTrackPrivateAVF::OutOfBand)
982             continue;
983 #endif
984
985         track->setTextTrackIndex(inBandCount);
986         ++inBandCount;
987         if (track->hasBeenReported())
988             continue;
989         
990         track->setHasBeenReported(true);
991         player()->addTextTrack(*track);
992     }
993
994     if (trackCount != m_textTracks.size())
995         INFO_LOG(LOGIDENTIFIER, "found ", m_textTracks.size(), " text tracks");
996 }
997
998 #if ENABLE(WIRELESS_PLAYBACK_TARGET)
999 void MediaPlayerPrivateAVFoundation::playbackTargetIsWirelessChanged()
1000 {
1001     if (m_player)
1002         m_player->currentPlaybackTargetIsWirelessChanged();
1003 }
1004 #endif
1005
1006 #if ENABLE(LEGACY_ENCRYPTED_MEDIA)
1007 bool MediaPlayerPrivateAVFoundation::extractKeyURIKeyIDAndCertificateFromInitData(Uint8Array* initData, String& keyURI, String& keyID, RefPtr<Uint8Array>& certificate)
1008 {
1009     // initData should have the following layout:
1010     // [4 bytes: keyURI length][N bytes: keyURI][4 bytes: contentID length], [N bytes: contentID], [4 bytes: certificate length][N bytes: certificate]
1011     if (initData->byteLength() < 4)
1012         return false;
1013
1014     RefPtr<ArrayBuffer> initDataBuffer = initData->unsharedBuffer();
1015
1016     // Use a DataView to read uint32 values from the buffer, as Uint32Array requires the reads be aligned on 4-byte boundaries. 
1017     auto initDataView = JSC::DataView::create(initDataBuffer.copyRef(), 0, initDataBuffer->byteLength());
1018     uint32_t offset = 0;
1019     bool status = true;
1020
1021     uint32_t keyURILength = initDataView->get<uint32_t>(offset, true, &status);
1022     offset += 4;
1023     if (!status || offset + keyURILength > initData->length())
1024         return false;
1025
1026     auto keyURIArray = Uint16Array::tryCreate(initDataBuffer.copyRef(), offset, keyURILength);
1027     if (!keyURIArray)
1028         return false;
1029
1030     keyURI = String(reinterpret_cast<UChar*>(keyURIArray->data()), keyURILength / sizeof(unsigned short));
1031     offset += keyURILength;
1032
1033     uint32_t keyIDLength = initDataView->get<uint32_t>(offset, true, &status);
1034     offset += 4;
1035     if (!status || offset + keyIDLength > initData->length())
1036         return false;
1037
1038     auto keyIDArray = Uint8Array::tryCreate(initDataBuffer.copyRef(), offset, keyIDLength);
1039     if (!keyIDArray)
1040         return false;
1041
1042     keyID = String(reinterpret_cast<UChar*>(keyIDArray->data()), keyIDLength / sizeof(unsigned short));
1043     offset += keyIDLength;
1044
1045     uint32_t certificateLength = initDataView->get<uint32_t>(offset, true, &status);
1046     offset += 4;
1047     if (!status || offset + certificateLength > initData->length())
1048         return false;
1049
1050     certificate = Uint8Array::tryCreate(WTFMove(initDataBuffer), offset, certificateLength);
1051     if (!certificate)
1052         return false;
1053
1054     return true;
1055 }
1056 #endif
1057
1058 bool MediaPlayerPrivateAVFoundation::canSaveMediaData() const
1059 {
1060     URL url = resolvedURL();
1061
1062     if (url.isLocalFile())
1063         return true;
1064
1065     if (!url.protocolIsInHTTPFamily())
1066         return false;
1067
1068     if (isLiveStream())
1069         return false;
1070
1071     return true;
1072 }
1073
1074 bool MediaPlayerPrivateAVFoundation::isUnsupportedMIMEType(const String& type)
1075 {
1076     String lowerCaseType = type.convertToASCIILowercase();
1077
1078     // AVFoundation will return non-video MIME types which it claims to support, but which we
1079     // do not support in the <video> element. Reject all non video/, audio/, and application/ types.
1080     if (!lowerCaseType.startsWith("video/") && !lowerCaseType.startsWith("audio/") && !lowerCaseType.startsWith("application/"))
1081         return true;
1082
1083     // Reject types we know AVFoundation does not support that sites commonly ask about.
1084     if (lowerCaseType == "video/webm" || lowerCaseType == "audio/webm" || lowerCaseType == "video/x-webm")
1085         return true;
1086
1087     if (lowerCaseType == "video/x-flv")
1088         return true;
1089
1090     if (lowerCaseType == "audio/ogg" || lowerCaseType == "video/ogg" || lowerCaseType == "application/ogg")
1091         return true;
1092
1093     if (lowerCaseType == "video/h264")
1094         return true;
1095
1096     return false;
1097 }
1098
1099 #if !RELEASE_LOG_DISABLED
1100 WTFLogChannel& MediaPlayerPrivateAVFoundation::logChannel() const
1101 {
1102     return LogMedia;
1103 }
1104 #endif
1105
1106 const HashSet<String, ASCIICaseInsensitiveHash>& MediaPlayerPrivateAVFoundation::staticMIMETypeList()
1107 {
1108     static const auto cache = makeNeverDestroyed(HashSet<String, ASCIICaseInsensitiveHash> {
1109         "application/vnd.apple.mpegurl",
1110         "application/x-mpegurl",
1111         "audio/3gpp",
1112         "audio/aac",
1113         "audio/aacp",
1114         "audio/aiff",
1115         "audio/basic",
1116         "audio/mp3",
1117         "audio/mp4",
1118         "audio/mpeg",
1119         "audio/mpeg3",
1120         "audio/mpegurl",
1121         "audio/mpg",
1122         "audio/vnd.wave",
1123         "audio/wav",
1124         "audio/wave",
1125         "audio/x-aac",
1126         "audio/x-aiff",
1127         "audio/x-m4a",
1128         "audio/x-mpegurl",
1129         "audio/x-wav",
1130         "video/3gpp",
1131         "video/3gpp2",
1132         "video/mp4",
1133         "video/mpeg",
1134         "video/mpeg2",
1135         "video/mpg",
1136         "video/quicktime",
1137         "video/x-m4v",
1138         "video/x-mpeg",
1139         "video/x-mpg",
1140     });
1141     return cache;
1142 }
1143
1144 } // namespace WebCore
1145
1146 #endif