2222c94ea70a62932d1543b92fa3599227a3ba52
[WebKit.git] / Source / WebCore / platform / graphics / avfoundation / objc / MediaPlayerPrivateAVFoundationObjC.h
1 /*
2  * Copyright (C) 2011-2014 Apple Inc. All rights reserved.
3  *
4  * Redistribution and use in source and binary forms, with or without
5  * modification, are permitted provided that the following conditions
6  * are met:
7  * 1. Redistributions of source code must retain the above copyright
8  *    notice, this list of conditions and the following disclaimer.
9  * 2. Redistributions in binary form must reproduce the above copyright
10  *    notice, this list of conditions and the following disclaimer in the
11  *    documentation and/or other materials provided with the distribution.
12  *
13  * THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY
14  * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
15  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
16  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL APPLE INC. OR
17  * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
18  * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
19  * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
20  * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
21  * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
22  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
23  * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 
24  */
25
26 #ifndef MediaPlayerPrivateAVFoundationObjC_h
27 #define MediaPlayerPrivateAVFoundationObjC_h
28
29 #if ENABLE(VIDEO) && USE(AVFOUNDATION)
30
31 #include "MediaPlaybackTarget.h"
32 #include "MediaPlayerPrivateAVFoundation.h"
33 #include <wtf/Function.h>
34 #include <wtf/HashMap.h>
35
36 OBJC_CLASS AVAssetImageGenerator;
37 OBJC_CLASS AVAssetResourceLoadingRequest;
38 OBJC_CLASS AVMediaSelectionGroup;
39 OBJC_CLASS AVOutputContext;
40 OBJC_CLASS AVPlayer;
41 OBJC_CLASS AVPlayerItem;
42 OBJC_CLASS AVPlayerItemLegibleOutput;
43 OBJC_CLASS AVPlayerItemTrack;
44 OBJC_CLASS AVPlayerItemVideoOutput;
45 OBJC_CLASS AVPlayerLayer;
46 OBJC_CLASS AVURLAsset;
47 OBJC_CLASS NSArray;
48 OBJC_CLASS NSURLAuthenticationChallenge;
49 OBJC_CLASS WebCoreAVFMovieObserver;
50 OBJC_CLASS WebCoreAVFPullDelegate;
51
52 typedef struct objc_object* id;
53
54 #if HAVE(AVFOUNDATION_LOADER_DELEGATE)
55 OBJC_CLASS WebCoreAVFLoaderDelegate;
56 OBJC_CLASS AVAssetResourceLoadingRequest;
57 #endif
58
59 typedef struct CGImage *CGImageRef;
60 typedef struct __CVBuffer *CVPixelBufferRef;
61
62 namespace WebCore {
63
64 class AudioSourceProviderAVFObjC;
65 class AudioTrackPrivateAVFObjC;
66 class CDMInstanceFairPlayStreamingAVFObjC;
67 class CDMSessionAVFoundationObjC;
68 class InbandMetadataTextTrackPrivateAVF;
69 class InbandTextTrackPrivateAVFObjC;
70 class MediaSelectionGroupAVFObjC;
71 class PixelBufferConformerCV;
72 class VideoTrackPrivateAVFObjC;
73 class WebCoreAVFResourceLoader;
74 class TextureCacheCV;
75 class VideoTextureCopierCV;
76
77 #if PLATFORM(IOS) || (PLATFORM(MAC) && ENABLE(VIDEO_PRESENTATION_MODE))
78 class VideoFullscreenLayerManager;
79 #endif
80
81 class MediaPlayerPrivateAVFoundationObjC : public MediaPlayerPrivateAVFoundation {
82 public:
83     explicit MediaPlayerPrivateAVFoundationObjC(MediaPlayer*);
84     virtual ~MediaPlayerPrivateAVFoundationObjC();
85
86     static void registerMediaEngine(MediaEngineRegistrar);
87
88     static HashSet<RefPtr<SecurityOrigin>> originsInMediaCache(const String&);
89     static void clearMediaCache(const String&, std::chrono::system_clock::time_point modifiedSince);
90     static void clearMediaCacheForOrigins(const String&, const HashSet<RefPtr<SecurityOrigin>>&);
91
92     void setAsset(RetainPtr<id>);
93     void tracksChanged() override;
94
95 #if HAVE(AVFOUNDATION_MEDIA_SELECTION_GROUP)
96     RetainPtr<AVPlayerItem> playerItem() const { return m_avPlayerItem; }
97     void processCue(NSArray *, NSArray *, const MediaTime&);
98     void flushCues();
99 #endif
100     AVPlayer *avPlayer() const { return m_avPlayer.get(); }
101     
102 #if HAVE(AVFOUNDATION_LOADER_DELEGATE)
103     bool shouldWaitForLoadingOfResource(AVAssetResourceLoadingRequest*);
104     bool shouldWaitForResponseToAuthenticationChallenge(NSURLAuthenticationChallenge*);
105     void didCancelLoadingRequest(AVAssetResourceLoadingRequest*);
106     void didStopLoadingRequest(AVAssetResourceLoadingRequest *);
107 #endif
108
109 #if ENABLE(LEGACY_ENCRYPTED_MEDIA)
110     RetainPtr<AVAssetResourceLoadingRequest> takeRequestForKeyURI(const String&);
111     void keyAdded() override;
112 #endif
113
114     void playerItemStatusDidChange(int);
115     void playbackLikelyToKeepUpWillChange();
116     void playbackLikelyToKeepUpDidChange(bool);
117     void playbackBufferEmptyWillChange();
118     void playbackBufferEmptyDidChange(bool);
119     void playbackBufferFullWillChange();
120     void playbackBufferFullDidChange(bool);
121     void loadedTimeRangesDidChange(RetainPtr<NSArray>);
122     void seekableTimeRangesDidChange(RetainPtr<NSArray>);
123     void tracksDidChange(RetainPtr<NSArray>);
124     void hasEnabledAudioDidChange(bool);
125     void presentationSizeDidChange(FloatSize);
126     void durationDidChange(const MediaTime&);
127     void rateDidChange(double);
128     void metadataDidArrive(RetainPtr<NSArray>, const MediaTime&);
129     void firstFrameAvailableDidChange(bool);
130     void trackEnabledDidChange(bool);
131     void canPlayFastReverseDidChange(bool);
132     void canPlayFastForwardDidChange(bool);
133
134     void setShouldBufferData(bool) override;
135
136 #if HAVE(AVFOUNDATION_VIDEO_OUTPUT)
137     void outputMediaDataWillChange(AVPlayerItemVideoOutput*);
138 #endif
139
140 #if ENABLE(WIRELESS_PLAYBACK_TARGET)
141     void playbackTargetIsWirelessDidChange();
142 #endif
143
144 #if ENABLE(LEGACY_ENCRYPTED_MEDIA)
145     void outputObscuredDueToInsufficientExternalProtectionChanged(bool);
146 #endif
147
148 #if ENABLE(AVF_CAPTIONS)
149     void notifyTrackModeChanged() override;
150     void synchronizeTextTrackState() override;
151 #endif
152
153 #if ENABLE(LEGACY_ENCRYPTED_MEDIA)
154     void removeSession(LegacyCDMSession&);
155 #endif
156
157 #if ENABLE(ENCRYPTED_MEDIA)
158     void cdmInstanceAttached(CDMInstance&) final;
159     void cdmInstanceDetached(CDMInstance&) final;
160     void attemptToDecryptWithInstance(CDMInstance&) final;
161 #endif
162
163     WeakPtr<MediaPlayerPrivateAVFoundationObjC> createWeakPtr() { return m_weakPtrFactory.createWeakPtr(*this); }
164
165 private:
166     // engine support
167     static void getSupportedTypes(HashSet<String, ASCIICaseInsensitiveHash>& types);
168     static MediaPlayer::SupportsType supportsType(const MediaEngineSupportParameters&);
169     static bool supportsKeySystem(const String& keySystem, const String& mimeType);
170
171     static bool isAvailable();
172
173     void cancelLoad() override;
174
175     PlatformMedia platformMedia() const override;
176
177     void platformSetVisible(bool) override;
178     void platformPlay() override;
179     void platformPause() override;
180     MediaTime currentMediaTime() const override;
181     void setVolume(float) override;
182 #if PLATFORM(IOS)
183     bool supportsMuting() const override { return true; }
184 #endif
185     void setMuted(bool) override;
186     void setClosedCaptionsVisible(bool) override;
187     void paint(GraphicsContext&, const FloatRect&) override;
188     void paintCurrentFrameInContext(GraphicsContext&, const FloatRect&) override;
189     PlatformLayer* platformLayer() const override;
190 #if PLATFORM(IOS) || (PLATFORM(MAC) && ENABLE(VIDEO_PRESENTATION_MODE))
191     void setVideoFullscreenLayer(PlatformLayer*, WTF::Function<void()>&& completionHandler) override;
192     void setVideoFullscreenFrame(FloatRect) override;
193     void setVideoFullscreenGravity(MediaPlayer::VideoGravity) override;
194     void setVideoFullscreenMode(MediaPlayer::VideoFullscreenMode) override;
195 #endif
196
197 #if PLATFORM(IOS)
198     NSArray *timedMetadata() const override;
199     String accessLog() const override;
200     String errorLog() const override;
201 #endif
202
203     bool supportsAcceleratedRendering() const override { return true; }
204     MediaTime mediaTimeForTimeValue(const MediaTime&) const override;
205     double maximumDurationToCacheMediaTime() const override;
206
207     void createAVPlayer() override;
208     void createAVPlayerItem() override;
209     virtual void createAVPlayerLayer();
210     void createAVAssetForURL(const URL&) override;
211     MediaPlayerPrivateAVFoundation::ItemStatus playerItemStatus() const override;
212     MediaPlayerPrivateAVFoundation::AssetStatus assetStatus() const override;
213     long assetErrorCode() const override;
214
215     double seekableTimeRangesLastModifiedTime() const override;
216     double liveUpdateInterval() const override;
217
218     void checkPlayability() override;
219     void setRateDouble(double) override;
220     double rate() const override;
221     void setPreservesPitch(bool) override;
222     void seekToTime(const MediaTime&, const MediaTime& negativeTolerance, const MediaTime& positiveTolerance) override;
223     unsigned long long totalBytes() const override;
224     std::unique_ptr<PlatformTimeRanges> platformBufferedTimeRanges() const override;
225     MediaTime platformMinTimeSeekable() const override;
226     MediaTime platformMaxTimeSeekable() const override;
227     MediaTime platformDuration() const override;
228     MediaTime platformMaxTimeLoaded() const override;
229     void beginLoadingMetadata() override;
230     void sizeChanged() override;
231     void resolvedURLChanged() override;
232
233     bool hasAvailableVideoFrame() const override;
234
235     void createContextVideoRenderer() override;
236     void destroyContextVideoRenderer() override;
237
238     void createVideoLayer() override;
239     void destroyVideoLayer() override;
240
241     bool hasContextRenderer() const override;
242     bool hasLayerRenderer() const override;
243
244     void updateVideoLayerGravity() override;
245
246     bool didPassCORSAccessCheck() const override;
247
248     MediaTime getStartDate() const override;
249
250 #if ENABLE(VIDEO_TRACK)
251     bool requiresTextTrackRepresentation() const override;
252     void setTextTrackRepresentation(TextTrackRepresentation*) override;
253     void syncTextTrackBounds() override;
254 #endif
255
256     void setAVPlayerItem(AVPlayerItem *);
257
258 #if ENABLE(WEB_AUDIO) && USE(MEDIATOOLBOX)
259     AudioSourceProvider* audioSourceProvider() override;
260 #endif
261
262     void createImageGenerator();
263     void destroyImageGenerator();
264     RetainPtr<CGImageRef> createImageForTimeInRect(float, const FloatRect&);
265     void paintWithImageGenerator(GraphicsContext&, const FloatRect&);
266
267 #if HAVE(AVFOUNDATION_VIDEO_OUTPUT)
268     void createVideoOutput();
269     void destroyVideoOutput();
270     bool updateLastPixelBuffer();
271     void updateLastImage();
272     bool videoOutputHasAvailableFrame();
273     void paintWithVideoOutput(GraphicsContext&, const FloatRect&);
274     NativeImagePtr nativeImageForCurrentTime() override;
275     void waitForVideoOutputMediaDataWillChange();
276
277     bool copyVideoTextureToPlatformTexture(GraphicsContext3D*, Platform3DObject, GC3Denum target, GC3Dint level, GC3Denum internalFormat, GC3Denum format, GC3Denum type, bool premultiplyAlpha, bool flipY) override;
278 #endif
279
280 #if ENABLE(LEGACY_ENCRYPTED_MEDIA)
281     std::unique_ptr<LegacyCDMSession> createSession(const String& keySystem, LegacyCDMSessionClient*) override;
282 #endif
283
284     String languageOfPrimaryAudioTrack() const override;
285
286 #if HAVE(AVFOUNDATION_MEDIA_SELECTION_GROUP)
287     void processMediaSelectionOptions();
288     bool hasLoadedMediaSelectionGroups();
289
290     AVMediaSelectionGroup* safeMediaSelectionGroupForLegibleMedia();
291     AVMediaSelectionGroup* safeMediaSelectionGroupForAudibleMedia();
292     AVMediaSelectionGroup* safeMediaSelectionGroupForVisualMedia();
293 #endif
294
295     NSArray* safeAVAssetTracksForAudibleMedia();
296
297 #if ENABLE(DATACUE_VALUE)
298     void processMetadataTrack();
299 #endif
300
301     void setCurrentTextTrack(InbandTextTrackPrivateAVF*) override;
302     InbandTextTrackPrivateAVF* currentTextTrack() const override { return m_currentTextTrack; }
303
304 #if !HAVE(AVFOUNDATION_LEGIBLE_OUTPUT_SUPPORT)
305     void processLegacyClosedCaptionsTracks();
306 #endif
307
308 #if ENABLE(VIDEO_TRACK)
309     void updateAudioTracks();
310     void updateVideoTracks();
311 #endif
312
313 #if ENABLE(WIRELESS_PLAYBACK_TARGET)
314     bool isCurrentPlaybackTargetWireless() const override;
315     String wirelessPlaybackTargetName() const override;
316     MediaPlayer::WirelessPlaybackTargetType wirelessPlaybackTargetType() const override;
317     bool wirelessVideoPlaybackDisabled() const override;
318     void setWirelessVideoPlaybackDisabled(bool) override;
319     bool canPlayToWirelessPlaybackTarget() const override { return true; }
320     void updateDisableExternalPlayback();
321 #endif
322
323 #if ENABLE(WIRELESS_PLAYBACK_TARGET) && !PLATFORM(IOS)
324     void setWirelessPlaybackTarget(Ref<MediaPlaybackTarget>&&) override;
325     void setShouldPlayToPlaybackTarget(bool) override;
326 #endif
327
328     double maxFastForwardRate() const override { return m_cachedCanPlayFastForward ? std::numeric_limits<double>::infinity() : 2.0; }
329     double minFastReverseRate() const override { return m_cachedCanPlayFastReverse ? -std::numeric_limits<double>::infinity() : 0.0; }
330
331     Vector<String> preferredAudioCharacteristics() const;
332
333     void setShouldDisableSleep(bool) override;
334
335 #if !RELEASE_LOG_DISABLED
336     const char* logClassName() const final { return "MediaPlayerPrivateAVFoundationObjC"; }
337 #endif
338
339     WeakPtrFactory<MediaPlayerPrivateAVFoundationObjC> m_weakPtrFactory;
340
341     RetainPtr<AVURLAsset> m_avAsset;
342     RetainPtr<AVPlayer> m_avPlayer;
343     RetainPtr<AVPlayerItem> m_avPlayerItem;
344     RetainPtr<AVPlayerLayer> m_videoLayer;
345 #if PLATFORM(IOS) || (PLATFORM(MAC) && ENABLE(VIDEO_PRESENTATION_MODE))
346     std::unique_ptr<VideoFullscreenLayerManager> m_videoFullscreenLayerManager;
347     MediaPlayer::VideoGravity m_videoFullscreenGravity;
348     RetainPtr<PlatformLayer> m_textTrackRepresentationLayer;
349 #endif
350     RetainPtr<WebCoreAVFMovieObserver> m_objcObserver;
351     RetainPtr<id> m_timeObserver;
352     mutable String m_languageOfPrimaryAudioTrack;
353     bool m_videoFrameHasDrawn;
354     bool m_haveCheckedPlayability;
355
356 #if ENABLE(WEB_AUDIO) && USE(MEDIATOOLBOX)
357     RefPtr<AudioSourceProviderAVFObjC> m_provider;
358 #endif
359
360     RetainPtr<AVAssetImageGenerator> m_imageGenerator;
361 #if HAVE(AVFOUNDATION_VIDEO_OUTPUT)
362     RetainPtr<AVPlayerItemVideoOutput> m_videoOutput;
363     RetainPtr<WebCoreAVFPullDelegate> m_videoOutputDelegate;
364     RetainPtr<CVPixelBufferRef> m_lastPixelBuffer;
365     RetainPtr<CGImageRef> m_lastImage;
366     dispatch_semaphore_t m_videoOutputSemaphore;
367     std::unique_ptr<VideoTextureCopierCV> m_videoTextureCopier;
368 #endif
369
370     std::unique_ptr<PixelBufferConformerCV> m_pixelBufferConformer;
371
372 #if HAVE(AVFOUNDATION_LOADER_DELEGATE)
373     friend class WebCoreAVFResourceLoader;
374     HashMap<RetainPtr<AVAssetResourceLoadingRequest>, RefPtr<WebCoreAVFResourceLoader>> m_resourceLoaderMap;
375     RetainPtr<WebCoreAVFLoaderDelegate> m_loaderDelegate;
376     HashMap<String, RetainPtr<AVAssetResourceLoadingRequest>> m_keyURIToRequestMap;
377     HashMap<String, RetainPtr<AVAssetResourceLoadingRequest>> m_sessionIDToRequestMap;
378 #endif
379
380 #if HAVE(AVFOUNDATION_MEDIA_SELECTION_GROUP) && HAVE(AVFOUNDATION_LEGIBLE_OUTPUT_SUPPORT)
381     RetainPtr<AVPlayerItemLegibleOutput> m_legibleOutput;
382 #endif
383
384 #if ENABLE(VIDEO_TRACK)
385     Vector<RefPtr<AudioTrackPrivateAVFObjC>> m_audioTracks;
386     Vector<RefPtr<VideoTrackPrivateAVFObjC>> m_videoTracks;
387 #if HAVE(AVFOUNDATION_MEDIA_SELECTION_GROUP)
388     RefPtr<MediaSelectionGroupAVFObjC> m_audibleGroup;
389     RefPtr<MediaSelectionGroupAVFObjC> m_visualGroup;
390 #endif
391 #endif
392
393     InbandTextTrackPrivateAVF* m_currentTextTrack;
394
395 #if ENABLE(DATACUE_VALUE)
396     RefPtr<InbandMetadataTextTrackPrivateAVF> m_metadataTrack;
397 #endif
398
399 #if PLATFORM(MAC) && ENABLE(WIRELESS_PLAYBACK_TARGET)
400     RetainPtr<AVOutputContext> m_outputContext;
401     RefPtr<MediaPlaybackTarget> m_playbackTarget { nullptr };
402 #endif
403
404 #if ENABLE(LEGACY_ENCRYPTED_MEDIA)
405     WeakPtr<CDMSessionAVFoundationObjC> m_session;
406 #endif
407 #if ENABLE(ENCRYPTED_MEDIA) && HAVE(AVCONTENTKEYSESSION)
408     RefPtr<CDMInstanceFairPlayStreamingAVFObjC> m_cdmInstance;
409 #endif
410
411     mutable RetainPtr<NSArray> m_cachedSeekableRanges;
412     mutable RetainPtr<NSArray> m_cachedLoadedRanges;
413     RetainPtr<NSArray> m_cachedTracks;
414     RetainPtr<NSArray> m_currentMetaData;
415     FloatSize m_cachedPresentationSize;
416     MediaTime m_cachedDuration;
417     double m_cachedRate;
418     mutable long long m_cachedTotalBytes;
419     unsigned m_pendingStatusChanges;
420     int m_cachedItemStatus;
421     bool m_cachedLikelyToKeepUp;
422     bool m_cachedBufferEmpty;
423     bool m_cachedBufferFull;
424     bool m_cachedHasEnabledAudio;
425     bool m_shouldBufferData;
426     bool m_cachedIsReadyForDisplay;
427     bool m_haveBeenAskedToCreateLayer;
428     bool m_cachedCanPlayFastForward;
429     bool m_cachedCanPlayFastReverse;
430     bool m_muted { false };
431     mutable std::optional<bool> m_tracksArePlayable;
432 #if ENABLE(WIRELESS_PLAYBACK_TARGET)
433     mutable bool m_allowsWirelessVideoPlayback;
434     bool m_shouldPlayToPlaybackTarget { false };
435 #endif
436 };
437
438 }
439
440 #endif
441 #endif