99be9fb8ea27de7505d816987dda9afa6199a466
[WebKit-https.git] / Source / WebCore / platform / graphics / avfoundation / objc / MediaPlayerPrivateAVFoundationObjC.h
1 /*
2  * Copyright (C) 2011-2018 Apple Inc. All rights reserved.
3  *
4  * Redistribution and use in source and binary forms, with or without
5  * modification, are permitted provided that the following conditions
6  * are met:
7  * 1. Redistributions of source code must retain the above copyright
8  *    notice, this list of conditions and the following disclaimer.
9  * 2. Redistributions in binary form must reproduce the above copyright
10  *    notice, this list of conditions and the following disclaimer in the
11  *    documentation and/or other materials provided with the distribution.
12  *
13  * THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY
14  * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
15  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
16  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL APPLE INC. OR
17  * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
18  * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
19  * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
20  * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
21  * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
22  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
23  * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 
24  */
25
26 #pragma once
27
28 #if ENABLE(VIDEO) && USE(AVFOUNDATION)
29
30 #include "MediaPlaybackTarget.h"
31 #include "MediaPlayerPrivateAVFoundation.h"
32 #include <wtf/Function.h>
33 #include <wtf/HashMap.h>
34
35 OBJC_CLASS AVAssetImageGenerator;
36 OBJC_CLASS AVAssetResourceLoadingRequest;
37 OBJC_CLASS AVMediaSelectionGroup;
38 OBJC_CLASS AVOutputContext;
39 OBJC_CLASS AVPlayerItem;
40 OBJC_CLASS AVPlayerItemLegibleOutput;
41 OBJC_CLASS AVPlayerItemVideoOutput;
42 OBJC_CLASS AVPlayerLayer;
43 OBJC_CLASS AVURLAsset;
44 OBJC_CLASS NSArray;
45 OBJC_CLASS WebCoreAVFLoaderDelegate;
46 OBJC_CLASS WebCoreAVFMovieObserver;
47 OBJC_CLASS WebCoreAVFPullDelegate;
48
49 typedef struct objc_object* id;
50
51 typedef struct CGImage *CGImageRef;
52 typedef struct __CVBuffer *CVPixelBufferRef;
53
54 namespace WebCore {
55
56 class AudioSourceProviderAVFObjC;
57 class AudioTrackPrivateAVFObjC;
58 class CDMInstanceFairPlayStreamingAVFObjC;
59 class CDMSessionAVFoundationObjC;
60 class InbandMetadataTextTrackPrivateAVF;
61 class MediaSelectionGroupAVFObjC;
62 class PixelBufferConformerCV;
63 class VideoFullscreenLayerManagerObjC;
64 class VideoTextureCopierCV;
65 class VideoTrackPrivateAVFObjC;
66 class WebCoreAVFResourceLoader;
67
68 class MediaPlayerPrivateAVFoundationObjC : public MediaPlayerPrivateAVFoundation {
69 public:
70     explicit MediaPlayerPrivateAVFoundationObjC(MediaPlayer*);
71     virtual ~MediaPlayerPrivateAVFoundationObjC();
72
73     static void registerMediaEngine(MediaEngineRegistrar);
74
75     static HashSet<RefPtr<SecurityOrigin>> originsInMediaCache(const String&);
76     static void clearMediaCache(const String&, WallTime modifiedSince);
77     static void clearMediaCacheForOrigins(const String&, const HashSet<RefPtr<SecurityOrigin>>&);
78
79     void setAsset(RetainPtr<id>);
80     void tracksChanged() override;
81
82 #if HAVE(AVFOUNDATION_MEDIA_SELECTION_GROUP)
83     RetainPtr<AVPlayerItem> playerItem() const { return m_avPlayerItem; }
84     void processCue(NSArray *, NSArray *, const MediaTime&);
85     void flushCues();
86 #endif
87     AVPlayer *avPlayer() const { return m_avPlayer.get(); }
88
89 #if HAVE(AVFOUNDATION_LOADER_DELEGATE)
90     bool shouldWaitForLoadingOfResource(AVAssetResourceLoadingRequest*);
91     void didCancelLoadingRequest(AVAssetResourceLoadingRequest*);
92     void didStopLoadingRequest(AVAssetResourceLoadingRequest *);
93 #endif
94
95 #if ENABLE(LEGACY_ENCRYPTED_MEDIA)
96     RetainPtr<AVAssetResourceLoadingRequest> takeRequestForKeyURI(const String&);
97     void keyAdded() override;
98 #endif
99
100     void playerItemStatusDidChange(int);
101     void playbackLikelyToKeepUpWillChange();
102     void playbackLikelyToKeepUpDidChange(bool);
103     void playbackBufferEmptyWillChange();
104     void playbackBufferEmptyDidChange(bool);
105     void playbackBufferFullWillChange();
106     void playbackBufferFullDidChange(bool);
107     void loadedTimeRangesDidChange(RetainPtr<NSArray>);
108     void seekableTimeRangesDidChange(RetainPtr<NSArray>);
109     void tracksDidChange(RetainPtr<NSArray>);
110     void hasEnabledAudioDidChange(bool);
111     void presentationSizeDidChange(FloatSize);
112     void durationDidChange(const MediaTime&);
113     void rateDidChange(double);
114     void metadataDidArrive(RetainPtr<NSArray>, const MediaTime&);
115     void firstFrameAvailableDidChange(bool);
116     void trackEnabledDidChange(bool);
117     void canPlayFastReverseDidChange(bool);
118     void canPlayFastForwardDidChange(bool);
119
120     void setShouldBufferData(bool) override;
121
122 #if HAVE(AVFOUNDATION_VIDEO_OUTPUT)
123     void outputMediaDataWillChange(AVPlayerItemVideoOutput*);
124 #endif
125
126 #if ENABLE(WIRELESS_PLAYBACK_TARGET)
127     void playbackTargetIsWirelessDidChange();
128 #endif
129
130 #if ENABLE(LEGACY_ENCRYPTED_MEDIA) || (ENABLE(ENCRYPTED_MEDIA) && HAVE(AVCONTENTKEYSESSION))
131     void outputObscuredDueToInsufficientExternalProtectionChanged(bool);
132     void beginSimulatedHDCPError() override { outputObscuredDueToInsufficientExternalProtectionChanged(true); }
133     void endSimulatedHDCPError() override { outputObscuredDueToInsufficientExternalProtectionChanged(false); }
134 #endif
135
136 #if ENABLE(AVF_CAPTIONS)
137     void notifyTrackModeChanged() override;
138     void synchronizeTextTrackState() override;
139 #endif
140
141 #if ENABLE(LEGACY_ENCRYPTED_MEDIA)
142     void removeSession(LegacyCDMSession&);
143 #endif
144
145 #if ENABLE(ENCRYPTED_MEDIA)
146     void cdmInstanceAttached(CDMInstance&) final;
147     void cdmInstanceDetached(CDMInstance&) final;
148     void attemptToDecryptWithInstance(CDMInstance&) final;
149 #endif
150
151 private:
152     // engine support
153     static void getSupportedTypes(HashSet<String, ASCIICaseInsensitiveHash>& types);
154     static MediaPlayer::SupportsType supportsType(const MediaEngineSupportParameters&);
155     static bool supportsKeySystem(const String& keySystem, const String& mimeType);
156
157     static bool isAvailable();
158
159     void cancelLoad() override;
160
161     void platformSetVisible(bool) override;
162     void platformPlay() override;
163     void platformPause() override;
164     MediaTime currentMediaTime() const override;
165     void setVolume(float) override;
166 #if PLATFORM(IOS)
167     bool supportsMuting() const override { return true; }
168 #endif
169     void setMuted(bool) override;
170     void setClosedCaptionsVisible(bool) override;
171     void paint(GraphicsContext&, const FloatRect&) override;
172     void paintCurrentFrameInContext(GraphicsContext&, const FloatRect&) override;
173     PlatformLayer* platformLayer() const override;
174     void setVideoFullscreenLayer(PlatformLayer*, Function<void()>&& completionHandler) override;
175     void updateVideoFullscreenInlineImage() final;
176     void setVideoFullscreenFrame(FloatRect) override;
177     void setVideoFullscreenGravity(MediaPlayer::VideoGravity) override;
178     void setVideoFullscreenMode(MediaPlayer::VideoFullscreenMode) override;
179
180 #if PLATFORM(IOS)
181     NSArray *timedMetadata() const override;
182     String accessLog() const override;
183     String errorLog() const override;
184 #endif
185
186     bool supportsAcceleratedRendering() const override { return true; }
187     MediaTime mediaTimeForTimeValue(const MediaTime&) const override;
188     double maximumDurationToCacheMediaTime() const override;
189
190     void createAVPlayer() override;
191     void createAVPlayerItem() override;
192     virtual void createAVPlayerLayer();
193     void createAVAssetForURL(const URL&) override;
194     MediaPlayerPrivateAVFoundation::ItemStatus playerItemStatus() const override;
195     MediaPlayerPrivateAVFoundation::AssetStatus assetStatus() const override;
196     long assetErrorCode() const override;
197
198     double seekableTimeRangesLastModifiedTime() const override;
199     double liveUpdateInterval() const override;
200
201     void checkPlayability() override;
202     void setRateDouble(double) override;
203     double rate() const override;
204     void setPreservesPitch(bool) override;
205     void seekToTime(const MediaTime&, const MediaTime& negativeTolerance, const MediaTime& positiveTolerance) override;
206     unsigned long long totalBytes() const override;
207     std::unique_ptr<PlatformTimeRanges> platformBufferedTimeRanges() const override;
208     MediaTime platformMinTimeSeekable() const override;
209     MediaTime platformMaxTimeSeekable() const override;
210     MediaTime platformDuration() const override;
211     MediaTime platformMaxTimeLoaded() const override;
212     void beginLoadingMetadata() override;
213     void sizeChanged() override;
214     void resolvedURLChanged() override;
215
216     bool hasAvailableVideoFrame() const override;
217
218     void createContextVideoRenderer() override;
219     void destroyContextVideoRenderer() override;
220
221     void createVideoLayer() override;
222     void destroyVideoLayer() override;
223
224     bool hasContextRenderer() const override;
225     bool hasLayerRenderer() const override;
226
227     void updateVideoLayerGravity() override;
228
229     bool didPassCORSAccessCheck() const override;
230     std::optional<bool> wouldTaintOrigin(const SecurityOrigin&) const final;
231
232
233     MediaTime getStartDate() const override;
234
235 #if ENABLE(VIDEO_TRACK)
236     bool requiresTextTrackRepresentation() const override;
237     void setTextTrackRepresentation(TextTrackRepresentation*) override;
238     void syncTextTrackBounds() override;
239 #endif
240
241     void setAVPlayerItem(AVPlayerItem *);
242
243 #if ENABLE(WEB_AUDIO) && USE(MEDIATOOLBOX)
244     AudioSourceProvider* audioSourceProvider() override;
245 #endif
246
247     void createImageGenerator();
248     void destroyImageGenerator();
249     RetainPtr<CGImageRef> createImageForTimeInRect(float, const FloatRect&);
250     void paintWithImageGenerator(GraphicsContext&, const FloatRect&);
251
252 #if HAVE(AVFOUNDATION_VIDEO_OUTPUT)
253     enum class UpdateType { UpdateSynchronously, UpdateAsynchronously };
254     void updateLastImage(UpdateType type = UpdateType::UpdateAsynchronously);
255
256     void createVideoOutput();
257     void destroyVideoOutput();
258     bool updateLastPixelBuffer();
259     bool videoOutputHasAvailableFrame();
260     void paintWithVideoOutput(GraphicsContext&, const FloatRect&);
261     NativeImagePtr nativeImageForCurrentTime() override;
262     void waitForVideoOutputMediaDataWillChange();
263
264     bool copyVideoTextureToPlatformTexture(GraphicsContext3D*, Platform3DObject, GC3Denum target, GC3Dint level, GC3Denum internalFormat, GC3Denum format, GC3Denum type, bool premultiplyAlpha, bool flipY) override;
265 #endif
266
267 #if ENABLE(LEGACY_ENCRYPTED_MEDIA)
268     std::unique_ptr<LegacyCDMSession> createSession(const String& keySystem, LegacyCDMSessionClient*) override;
269 #endif
270
271     String languageOfPrimaryAudioTrack() const override;
272
273 #if HAVE(AVFOUNDATION_MEDIA_SELECTION_GROUP)
274     void processMediaSelectionOptions();
275     bool hasLoadedMediaSelectionGroups();
276
277     AVMediaSelectionGroup* safeMediaSelectionGroupForLegibleMedia();
278     AVMediaSelectionGroup* safeMediaSelectionGroupForAudibleMedia();
279     AVMediaSelectionGroup* safeMediaSelectionGroupForVisualMedia();
280 #endif
281
282     NSArray* safeAVAssetTracksForAudibleMedia();
283
284 #if ENABLE(DATACUE_VALUE)
285     void processMetadataTrack();
286 #endif
287
288     void setCurrentTextTrack(InbandTextTrackPrivateAVF*) override;
289     InbandTextTrackPrivateAVF* currentTextTrack() const override { return m_currentTextTrack; }
290
291 #if !HAVE(AVFOUNDATION_LEGIBLE_OUTPUT_SUPPORT)
292     void processLegacyClosedCaptionsTracks();
293 #endif
294
295 #if ENABLE(VIDEO_TRACK)
296     void updateAudioTracks();
297     void updateVideoTracks();
298 #endif
299
300 #if ENABLE(WIRELESS_PLAYBACK_TARGET)
301     bool isCurrentPlaybackTargetWireless() const override;
302     String wirelessPlaybackTargetName() const override;
303     MediaPlayer::WirelessPlaybackTargetType wirelessPlaybackTargetType() const override;
304     bool wirelessVideoPlaybackDisabled() const override;
305     void setWirelessVideoPlaybackDisabled(bool) override;
306     bool canPlayToWirelessPlaybackTarget() const override { return true; }
307     void updateDisableExternalPlayback();
308 #endif
309
310 #if ENABLE(WIRELESS_PLAYBACK_TARGET) && !PLATFORM(IOS)
311     void setWirelessPlaybackTarget(Ref<MediaPlaybackTarget>&&) override;
312     void setShouldPlayToPlaybackTarget(bool) override;
313 #endif
314
315     double maxFastForwardRate() const override { return m_cachedCanPlayFastForward ? std::numeric_limits<double>::infinity() : 2.0; }
316     double minFastReverseRate() const override { return m_cachedCanPlayFastReverse ? -std::numeric_limits<double>::infinity() : 0.0; }
317
318     Vector<String> preferredAudioCharacteristics() const;
319
320     void setShouldDisableSleep(bool) override;
321
322 #if !RELEASE_LOG_DISABLED
323     const char* logClassName() const final { return "MediaPlayerPrivateAVFoundationObjC"; }
324 #endif
325
326     AVPlayer *objCAVFoundationAVPlayer() const final { return m_avPlayer.get(); }
327
328     RetainPtr<AVURLAsset> m_avAsset;
329     RetainPtr<AVPlayer> m_avPlayer;
330     RetainPtr<AVPlayerItem> m_avPlayerItem;
331     RetainPtr<AVPlayerLayer> m_videoLayer;
332     std::unique_ptr<VideoFullscreenLayerManagerObjC> m_videoFullscreenLayerManager;
333     MediaPlayer::VideoGravity m_videoFullscreenGravity;
334     RetainPtr<WebCoreAVFMovieObserver> m_objcObserver;
335     RetainPtr<id> m_timeObserver;
336     mutable String m_languageOfPrimaryAudioTrack;
337     bool m_videoFrameHasDrawn;
338     bool m_haveCheckedPlayability;
339
340 #if ENABLE(WEB_AUDIO) && USE(MEDIATOOLBOX)
341     RefPtr<AudioSourceProviderAVFObjC> m_provider;
342 #endif
343
344     RetainPtr<AVAssetImageGenerator> m_imageGenerator;
345 #if HAVE(AVFOUNDATION_VIDEO_OUTPUT)
346     RetainPtr<AVPlayerItemVideoOutput> m_videoOutput;
347     RetainPtr<WebCoreAVFPullDelegate> m_videoOutputDelegate;
348     RetainPtr<CVPixelBufferRef> m_lastPixelBuffer;
349     RetainPtr<CGImageRef> m_lastImage;
350     dispatch_semaphore_t m_videoOutputSemaphore;
351     std::unique_ptr<VideoTextureCopierCV> m_videoTextureCopier;
352 #endif
353
354 #if HAVE(CORE_VIDEO)
355     std::unique_ptr<PixelBufferConformerCV> m_pixelBufferConformer;
356 #endif
357
358 #if HAVE(AVFOUNDATION_LOADER_DELEGATE)
359     friend class WebCoreAVFResourceLoader;
360     HashMap<RetainPtr<CFTypeRef>, RefPtr<WebCoreAVFResourceLoader>> m_resourceLoaderMap;
361     RetainPtr<WebCoreAVFLoaderDelegate> m_loaderDelegate;
362     HashMap<String, RetainPtr<AVAssetResourceLoadingRequest>> m_keyURIToRequestMap;
363     HashMap<String, RetainPtr<AVAssetResourceLoadingRequest>> m_sessionIDToRequestMap;
364 #endif
365
366 #if HAVE(AVFOUNDATION_MEDIA_SELECTION_GROUP) && HAVE(AVFOUNDATION_LEGIBLE_OUTPUT_SUPPORT)
367     RetainPtr<AVPlayerItemLegibleOutput> m_legibleOutput;
368 #endif
369
370 #if ENABLE(VIDEO_TRACK)
371     Vector<RefPtr<AudioTrackPrivateAVFObjC>> m_audioTracks;
372     Vector<RefPtr<VideoTrackPrivateAVFObjC>> m_videoTracks;
373 #if HAVE(AVFOUNDATION_MEDIA_SELECTION_GROUP)
374     RefPtr<MediaSelectionGroupAVFObjC> m_audibleGroup;
375     RefPtr<MediaSelectionGroupAVFObjC> m_visualGroup;
376 #endif
377 #endif
378
379     InbandTextTrackPrivateAVF* m_currentTextTrack;
380
381 #if ENABLE(DATACUE_VALUE)
382     RefPtr<InbandMetadataTextTrackPrivateAVF> m_metadataTrack;
383 #endif
384
385 #if PLATFORM(MAC) && ENABLE(WIRELESS_PLAYBACK_TARGET)
386     RetainPtr<AVOutputContext> m_outputContext;
387     RefPtr<MediaPlaybackTarget> m_playbackTarget { nullptr };
388 #endif
389
390 #if ENABLE(LEGACY_ENCRYPTED_MEDIA)
391     WeakPtr<CDMSessionAVFoundationObjC> m_session;
392 #endif
393 #if ENABLE(ENCRYPTED_MEDIA) && HAVE(AVCONTENTKEYSESSION)
394     RefPtr<CDMInstanceFairPlayStreamingAVFObjC> m_cdmInstance;
395 #endif
396
397     mutable RetainPtr<NSArray> m_cachedSeekableRanges;
398     mutable RetainPtr<NSArray> m_cachedLoadedRanges;
399     RetainPtr<NSArray> m_cachedTracks;
400     RetainPtr<NSArray> m_currentMetaData;
401     FloatSize m_cachedPresentationSize;
402     MediaTime m_cachedDuration;
403     double m_cachedRate;
404     mutable long long m_cachedTotalBytes;
405     unsigned m_pendingStatusChanges;
406     int m_cachedItemStatus;
407     bool m_cachedLikelyToKeepUp;
408     bool m_cachedBufferEmpty;
409     bool m_cachedBufferFull;
410     bool m_cachedHasEnabledAudio;
411     bool m_shouldBufferData;
412     bool m_cachedIsReadyForDisplay;
413     bool m_haveBeenAskedToCreateLayer;
414     bool m_cachedCanPlayFastForward;
415     bool m_cachedCanPlayFastReverse;
416     bool m_muted { false };
417     mutable std::optional<bool> m_tracksArePlayable;
418 #if ENABLE(WIRELESS_PLAYBACK_TARGET)
419     mutable bool m_allowsWirelessVideoPlayback;
420     bool m_shouldPlayToPlaybackTarget { false };
421 #endif
422 };
423
424 }
425
426 #endif