Unreviewed, rolling out r245857.
[WebKit-https.git] / Source / WebCore / platform / graphics / avfoundation / objc / MediaPlayerPrivateAVFoundationObjC.h
1 /*
2  * Copyright (C) 2011-2018 Apple Inc. All rights reserved.
3  *
4  * Redistribution and use in source and binary forms, with or without
5  * modification, are permitted provided that the following conditions
6  * are met:
7  * 1. Redistributions of source code must retain the above copyright
8  *    notice, this list of conditions and the following disclaimer.
9  * 2. Redistributions in binary form must reproduce the above copyright
10  *    notice, this list of conditions and the following disclaimer in the
11  *    documentation and/or other materials provided with the distribution.
12  *
13  * THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY
14  * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
15  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
16  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL APPLE INC. OR
17  * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
18  * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
19  * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
20  * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
21  * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
22  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
23  * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 
24  */
25
26 #pragma once
27
28 #if ENABLE(VIDEO) && USE(AVFOUNDATION)
29
30 #include "MediaPlaybackTarget.h"
31 #include "MediaPlayerPrivateAVFoundation.h"
32 #include <wtf/Function.h>
33 #include <wtf/HashMap.h>
34 #include <wtf/threads/BinarySemaphore.h>
35
36 OBJC_CLASS AVAssetImageGenerator;
37 OBJC_CLASS AVAssetResourceLoadingRequest;
38 OBJC_CLASS AVMediaSelectionGroup;
39 OBJC_CLASS AVOutputContext;
40 OBJC_CLASS AVPlayerItem;
41 OBJC_CLASS AVPlayerItemLegibleOutput;
42 OBJC_CLASS AVPlayerItemVideoOutput;
43 OBJC_CLASS AVPlayerLayer;
44 OBJC_CLASS AVURLAsset;
45 OBJC_CLASS NSArray;
46 OBJC_CLASS WebCoreAVFLoaderDelegate;
47 OBJC_CLASS WebCoreAVFMovieObserver;
48 OBJC_CLASS WebCoreAVFPullDelegate;
49
50 typedef struct objc_object* id;
51
52 typedef struct CGImage *CGImageRef;
53 typedef struct __CVBuffer *CVPixelBufferRef;
54
55 namespace WebCore {
56
57 class AudioSourceProviderAVFObjC;
58 class AudioTrackPrivateAVFObjC;
59 class CDMInstanceFairPlayStreamingAVFObjC;
60 class CDMSessionAVFoundationObjC;
61 class InbandMetadataTextTrackPrivateAVF;
62 class MediaSelectionGroupAVFObjC;
63 class PixelBufferConformerCV;
64 class SharedBuffer;
65 class VideoFullscreenLayerManagerObjC;
66 class VideoTextureCopierCV;
67 class VideoTrackPrivateAVFObjC;
68 class WebCoreAVFResourceLoader;
69
70 class MediaPlayerPrivateAVFoundationObjC : public MediaPlayerPrivateAVFoundation {
71 public:
72     explicit MediaPlayerPrivateAVFoundationObjC(MediaPlayer*);
73     virtual ~MediaPlayerPrivateAVFoundationObjC();
74
75     static void registerMediaEngine(MediaEngineRegistrar);
76
77     static HashSet<RefPtr<SecurityOrigin>> originsInMediaCache(const String&);
78     static void clearMediaCache(const String&, WallTime modifiedSince);
79     static void clearMediaCacheForOrigins(const String&, const HashSet<RefPtr<SecurityOrigin>>&);
80
81     void setAsset(RetainPtr<id>&&);
82     void tracksChanged() override;
83     void didEnd() override;
84
85 #if HAVE(AVFOUNDATION_MEDIA_SELECTION_GROUP)
86     RetainPtr<AVPlayerItem> playerItem() const { return m_avPlayerItem; }
87     void processCue(NSArray *, NSArray *, const MediaTime&);
88     void flushCues();
89 #endif
90     AVPlayer *avPlayer() const { return m_avPlayer.get(); }
91
92 #if HAVE(AVFOUNDATION_LOADER_DELEGATE)
93     bool shouldWaitForLoadingOfResource(AVAssetResourceLoadingRequest*);
94     void didCancelLoadingRequest(AVAssetResourceLoadingRequest*);
95     void didStopLoadingRequest(AVAssetResourceLoadingRequest *);
96 #endif
97
98 #if ENABLE(LEGACY_ENCRYPTED_MEDIA)
99     RetainPtr<AVAssetResourceLoadingRequest> takeRequestForKeyURI(const String&);
100     void keyAdded() override;
101 #endif
102
103     void playerItemStatusDidChange(int);
104     void playbackLikelyToKeepUpWillChange();
105     void playbackLikelyToKeepUpDidChange(bool);
106     void playbackBufferEmptyWillChange();
107     void playbackBufferEmptyDidChange(bool);
108     void playbackBufferFullWillChange();
109     void playbackBufferFullDidChange(bool);
110     void loadedTimeRangesDidChange(RetainPtr<NSArray>&&);
111     void seekableTimeRangesDidChange(RetainPtr<NSArray>&&);
112     void tracksDidChange(const RetainPtr<NSArray>&);
113     void hasEnabledAudioDidChange(bool);
114     void presentationSizeDidChange(FloatSize);
115     void durationDidChange(const MediaTime&);
116     void rateDidChange(double);
117     void timeControlStatusDidChange(int);
118     void metadataDidArrive(const RetainPtr<NSArray>&, const MediaTime&);
119     void firstFrameAvailableDidChange(bool);
120     void trackEnabledDidChange(bool);
121     void canPlayFastReverseDidChange(bool);
122     void canPlayFastForwardDidChange(bool);
123
124     void setBufferingPolicy(MediaPlayerEnums::BufferingPolicy) override;
125
126 #if HAVE(AVFOUNDATION_VIDEO_OUTPUT)
127     void outputMediaDataWillChange(AVPlayerItemVideoOutput*);
128 #endif
129
130 #if ENABLE(WIRELESS_PLAYBACK_TARGET)
131     void playbackTargetIsWirelessDidChange();
132 #endif
133
134 #if ENABLE(LEGACY_ENCRYPTED_MEDIA) || (ENABLE(ENCRYPTED_MEDIA) && HAVE(AVCONTENTKEYSESSION))
135     void outputObscuredDueToInsufficientExternalProtectionChanged(bool);
136     void beginSimulatedHDCPError() override { outputObscuredDueToInsufficientExternalProtectionChanged(true); }
137     void endSimulatedHDCPError() override { outputObscuredDueToInsufficientExternalProtectionChanged(false); }
138 #endif
139
140 #if ENABLE(AVF_CAPTIONS)
141     void notifyTrackModeChanged() override;
142     void synchronizeTextTrackState() override;
143 #endif
144
145 #if ENABLE(LEGACY_ENCRYPTED_MEDIA)
146     void removeSession(LegacyCDMSession&);
147 #endif
148
149 #if ENABLE(ENCRYPTED_MEDIA)
150     void cdmInstanceAttached(CDMInstance&) final;
151     void cdmInstanceDetached(CDMInstance&) final;
152     void attemptToDecryptWithInstance(CDMInstance&) final;
153     void setWaitingForKey(bool);
154     bool waitingForKey() const final { return m_waitingForKey; }
155 #endif
156
157 private:
158     // engine support
159     static void getSupportedTypes(HashSet<String, ASCIICaseInsensitiveHash>& types);
160     static MediaPlayer::SupportsType supportsType(const MediaEngineSupportParameters&);
161     static bool supportsKeySystem(const String& keySystem, const String& mimeType);
162
163     static bool isAvailable();
164
165     void cancelLoad() override;
166
167     void platformSetVisible(bool) override;
168     void platformPlay() override;
169     void platformPause() override;
170     bool platformPaused() const override;
171     MediaTime currentMediaTime() const override;
172     void setVolume(float) override;
173     bool supportsMuting() const override { return true; }
174     void setMuted(bool) override;
175     void setClosedCaptionsVisible(bool) override;
176     void paint(GraphicsContext&, const FloatRect&) override;
177     void paintCurrentFrameInContext(GraphicsContext&, const FloatRect&) override;
178     PlatformLayer* platformLayer() const override;
179     void setVideoFullscreenLayer(PlatformLayer*, Function<void()>&& completionHandler) override;
180     void updateVideoFullscreenInlineImage() final;
181     void setVideoFullscreenFrame(FloatRect) override;
182     void setVideoFullscreenGravity(MediaPlayer::VideoGravity) override;
183     void setVideoFullscreenMode(MediaPlayer::VideoFullscreenMode) override;
184     void videoFullscreenStandbyChanged() override;
185     void setPlayerRate(double);
186
187 #if PLATFORM(IOS_FAMILY)
188     NSArray *timedMetadata() const override;
189     String accessLog() const override;
190     String errorLog() const override;
191 #endif
192
193     bool supportsAcceleratedRendering() const override { return true; }
194     MediaTime mediaTimeForTimeValue(const MediaTime&) const override;
195     double maximumDurationToCacheMediaTime() const override;
196
197     void createAVPlayer() override;
198     void createAVPlayerItem() override;
199     virtual void createAVPlayerLayer();
200     void createAVAssetForURL(const URL&) override;
201     MediaPlayerPrivateAVFoundation::ItemStatus playerItemStatus() const override;
202     MediaPlayerPrivateAVFoundation::AssetStatus assetStatus() const override;
203     long assetErrorCode() const override;
204
205     double seekableTimeRangesLastModifiedTime() const override;
206     double liveUpdateInterval() const override;
207
208     void checkPlayability() override;
209     void setRateDouble(double) override;
210     double rate() const override;
211     void setPreservesPitch(bool) override;
212     void seekToTime(const MediaTime&, const MediaTime& negativeTolerance, const MediaTime& positiveTolerance) override;
213     unsigned long long totalBytes() const override;
214     std::unique_ptr<PlatformTimeRanges> platformBufferedTimeRanges() const override;
215     MediaTime platformMinTimeSeekable() const override;
216     MediaTime platformMaxTimeSeekable() const override;
217     MediaTime platformDuration() const override;
218     MediaTime platformMaxTimeLoaded() const override;
219     void beginLoadingMetadata() override;
220     void sizeChanged() override;
221     void resolvedURLChanged() override;
222
223     bool hasAvailableVideoFrame() const override;
224
225     void createContextVideoRenderer() override;
226     void destroyContextVideoRenderer() override;
227
228     void createVideoLayer() override;
229     void destroyVideoLayer() override;
230
231     bool hasContextRenderer() const override;
232     bool hasLayerRenderer() const override;
233
234     void updateVideoLayerGravity() override;
235
236     bool didPassCORSAccessCheck() const override;
237     Optional<bool> wouldTaintOrigin(const SecurityOrigin&) const final;
238
239
240     MediaTime getStartDate() const override;
241
242 #if ENABLE(VIDEO_TRACK)
243     bool requiresTextTrackRepresentation() const override;
244     void setTextTrackRepresentation(TextTrackRepresentation*) override;
245     void syncTextTrackBounds() override;
246 #endif
247
248     void setAVPlayerItem(AVPlayerItem *);
249
250 #if ENABLE(WEB_AUDIO) && USE(MEDIATOOLBOX)
251     AudioSourceProvider* audioSourceProvider() override;
252 #endif
253
254     void createImageGenerator();
255     void destroyImageGenerator();
256     RetainPtr<CGImageRef> createImageForTimeInRect(float, const FloatRect&);
257     void paintWithImageGenerator(GraphicsContext&, const FloatRect&);
258
259 #if HAVE(AVFOUNDATION_VIDEO_OUTPUT)
260     enum class UpdateType { UpdateSynchronously, UpdateAsynchronously };
261     void updateLastImage(UpdateType type = UpdateType::UpdateAsynchronously);
262
263     void createVideoOutput();
264     void destroyVideoOutput();
265     bool updateLastPixelBuffer();
266     bool videoOutputHasAvailableFrame();
267     void paintWithVideoOutput(GraphicsContext&, const FloatRect&);
268     NativeImagePtr nativeImageForCurrentTime() override;
269     void waitForVideoOutputMediaDataWillChange();
270
271     bool copyVideoTextureToPlatformTexture(GraphicsContext3D*, Platform3DObject, GC3Denum target, GC3Dint level, GC3Denum internalFormat, GC3Denum format, GC3Denum type, bool premultiplyAlpha, bool flipY) override;
272 #endif
273
274 #if ENABLE(LEGACY_ENCRYPTED_MEDIA)
275     std::unique_ptr<LegacyCDMSession> createSession(const String& keySystem, LegacyCDMSessionClient*) override;
276 #endif
277
278     String languageOfPrimaryAudioTrack() const override;
279
280 #if HAVE(AVFOUNDATION_MEDIA_SELECTION_GROUP)
281     void processMediaSelectionOptions();
282     bool hasLoadedMediaSelectionGroups();
283
284     AVMediaSelectionGroup* safeMediaSelectionGroupForLegibleMedia();
285     AVMediaSelectionGroup* safeMediaSelectionGroupForAudibleMedia();
286     AVMediaSelectionGroup* safeMediaSelectionGroupForVisualMedia();
287 #endif
288
289     NSArray* safeAVAssetTracksForAudibleMedia();
290
291 #if ENABLE(DATACUE_VALUE)
292     void processMetadataTrack();
293 #endif
294
295     void setCurrentTextTrack(InbandTextTrackPrivateAVF*) override;
296     InbandTextTrackPrivateAVF* currentTextTrack() const override { return m_currentTextTrack; }
297
298 #if !HAVE(AVFOUNDATION_LEGIBLE_OUTPUT_SUPPORT)
299     void processLegacyClosedCaptionsTracks();
300 #endif
301
302 #if ENABLE(VIDEO_TRACK)
303     void updateAudioTracks();
304     void updateVideoTracks();
305 #endif
306
307 #if ENABLE(WIRELESS_PLAYBACK_TARGET)
308     bool isCurrentPlaybackTargetWireless() const override;
309     String wirelessPlaybackTargetName() const override;
310     MediaPlayer::WirelessPlaybackTargetType wirelessPlaybackTargetType() const override;
311     bool wirelessVideoPlaybackDisabled() const override;
312     void setWirelessVideoPlaybackDisabled(bool) override;
313     bool canPlayToWirelessPlaybackTarget() const override { return true; }
314     void updateDisableExternalPlayback();
315 #endif
316
317 #if ENABLE(WIRELESS_PLAYBACK_TARGET) && !PLATFORM(IOS_FAMILY)
318     void setWirelessPlaybackTarget(Ref<MediaPlaybackTarget>&&) override;
319     void setShouldPlayToPlaybackTarget(bool) override;
320 #endif
321
322     double maxFastForwardRate() const override { return m_cachedCanPlayFastForward ? std::numeric_limits<double>::infinity() : 2.0; }
323     double minFastReverseRate() const override { return m_cachedCanPlayFastReverse ? -std::numeric_limits<double>::infinity() : 0.0; }
324
325     Vector<String> preferredAudioCharacteristics() const;
326
327     void setShouldDisableSleep(bool) override;
328
329     Optional<VideoPlaybackQualityMetrics> videoPlaybackQualityMetrics() final;
330
331 #if !RELEASE_LOG_DISABLED
332     const char* logClassName() const final { return "MediaPlayerPrivateAVFoundationObjC"; }
333 #endif
334
335     AVPlayer *objCAVFoundationAVPlayer() const final { return m_avPlayer.get(); }
336
337     bool performTaskAtMediaTime(WTF::Function<void()>&&, MediaTime) final;
338     void setShouldObserveTimeControlStatus(bool);
339
340     WeakPtrFactory<MediaPlayerPrivateAVFoundationObjC> m_weakPtrFactory;
341     RetainPtr<AVURLAsset> m_avAsset;
342     RetainPtr<AVPlayer> m_avPlayer;
343     RetainPtr<AVPlayerItem> m_avPlayerItem;
344     RetainPtr<AVPlayerLayer> m_videoLayer;
345     std::unique_ptr<VideoFullscreenLayerManagerObjC> m_videoFullscreenLayerManager;
346     MediaPlayer::VideoGravity m_videoFullscreenGravity;
347     RetainPtr<WebCoreAVFMovieObserver> m_objcObserver;
348     RetainPtr<id> m_timeObserver;
349     mutable String m_languageOfPrimaryAudioTrack;
350     bool m_videoFrameHasDrawn;
351     bool m_haveCheckedPlayability;
352
353 #if ENABLE(WEB_AUDIO) && USE(MEDIATOOLBOX)
354     RefPtr<AudioSourceProviderAVFObjC> m_provider;
355 #endif
356
357     RetainPtr<AVAssetImageGenerator> m_imageGenerator;
358 #if HAVE(AVFOUNDATION_VIDEO_OUTPUT)
359     RetainPtr<AVPlayerItemVideoOutput> m_videoOutput;
360     RetainPtr<WebCoreAVFPullDelegate> m_videoOutputDelegate;
361     RetainPtr<CVPixelBufferRef> m_lastPixelBuffer;
362     RetainPtr<CGImageRef> m_lastImage;
363     BinarySemaphore m_videoOutputSemaphore;
364     std::unique_ptr<VideoTextureCopierCV> m_videoTextureCopier;
365 #endif
366
367 #if HAVE(CORE_VIDEO)
368     std::unique_ptr<PixelBufferConformerCV> m_pixelBufferConformer;
369 #endif
370
371 #if HAVE(AVFOUNDATION_LOADER_DELEGATE)
372     friend class WebCoreAVFResourceLoader;
373     HashMap<RetainPtr<CFTypeRef>, RefPtr<WebCoreAVFResourceLoader>> m_resourceLoaderMap;
374     RetainPtr<WebCoreAVFLoaderDelegate> m_loaderDelegate;
375     HashMap<String, RetainPtr<AVAssetResourceLoadingRequest>> m_keyURIToRequestMap;
376     HashMap<String, RetainPtr<AVAssetResourceLoadingRequest>> m_sessionIDToRequestMap;
377 #endif
378
379 #if HAVE(AVFOUNDATION_MEDIA_SELECTION_GROUP) && HAVE(AVFOUNDATION_LEGIBLE_OUTPUT_SUPPORT)
380     RetainPtr<AVPlayerItemLegibleOutput> m_legibleOutput;
381 #endif
382
383 #if ENABLE(VIDEO_TRACK)
384     Vector<RefPtr<AudioTrackPrivateAVFObjC>> m_audioTracks;
385     Vector<RefPtr<VideoTrackPrivateAVFObjC>> m_videoTracks;
386 #if HAVE(AVFOUNDATION_MEDIA_SELECTION_GROUP)
387     RefPtr<MediaSelectionGroupAVFObjC> m_audibleGroup;
388     RefPtr<MediaSelectionGroupAVFObjC> m_visualGroup;
389 #endif
390 #endif
391
392     InbandTextTrackPrivateAVF* m_currentTextTrack;
393
394 #if ENABLE(DATACUE_VALUE)
395     RefPtr<InbandMetadataTextTrackPrivateAVF> m_metadataTrack;
396 #endif
397
398 #if PLATFORM(MAC) && ENABLE(WIRELESS_PLAYBACK_TARGET)
399     RetainPtr<AVOutputContext> m_outputContext;
400     RefPtr<MediaPlaybackTarget> m_playbackTarget { nullptr };
401 #endif
402
403 #if ENABLE(LEGACY_ENCRYPTED_MEDIA)
404     WeakPtr<CDMSessionAVFoundationObjC> m_session;
405 #endif
406 #if ENABLE(ENCRYPTED_MEDIA)
407     bool m_waitingForKey { false };
408 #if HAVE(AVCONTENTKEYSESSION)
409     RefPtr<CDMInstanceFairPlayStreamingAVFObjC> m_cdmInstance;
410 #endif
411 #endif
412
413     mutable RetainPtr<NSArray> m_cachedSeekableRanges;
414     mutable RetainPtr<NSArray> m_cachedLoadedRanges;
415     RetainPtr<NSArray> m_cachedTracks;
416     RetainPtr<NSArray> m_currentMetaData;
417     FloatSize m_cachedPresentationSize;
418     MediaTime m_cachedDuration;
419     RefPtr<SharedBuffer> m_keyID;
420     double m_cachedRate;
421     bool m_requestedPlaying { false };
422     double m_requestedRate { 1.0 };
423     int m_cachedTimeControlStatus { 0 };
424     mutable long long m_cachedTotalBytes;
425     unsigned m_pendingStatusChanges;
426     int m_cachedItemStatus;
427     MediaPlayer::BufferingPolicy m_bufferingPolicy { MediaPlayer::BufferingPolicy::Default };
428     bool m_cachedLikelyToKeepUp;
429     bool m_cachedBufferEmpty;
430     bool m_cachedBufferFull;
431     bool m_cachedHasEnabledAudio;
432     bool m_cachedIsReadyForDisplay;
433     bool m_haveBeenAskedToCreateLayer;
434     bool m_cachedCanPlayFastForward;
435     bool m_cachedCanPlayFastReverse;
436     bool m_muted { false };
437     bool m_shouldObserveTimeControlStatus { false };
438     mutable Optional<bool> m_tracksArePlayable;
439 #if ENABLE(WIRELESS_PLAYBACK_TARGET)
440     mutable bool m_allowsWirelessVideoPlayback;
441     bool m_shouldPlayToPlaybackTarget { false };
442 #endif
443 };
444
445 }
446
447 #endif