Add support for reporting "display composited video frames" through the VideoPlayback...
[WebKit-https.git] / Source / WebCore / platform / graphics / avfoundation / objc / MediaPlayerPrivateAVFoundationObjC.h
1 /*
2  * Copyright (C) 2011-2018 Apple Inc. All rights reserved.
3  *
4  * Redistribution and use in source and binary forms, with or without
5  * modification, are permitted provided that the following conditions
6  * are met:
7  * 1. Redistributions of source code must retain the above copyright
8  *    notice, this list of conditions and the following disclaimer.
9  * 2. Redistributions in binary form must reproduce the above copyright
10  *    notice, this list of conditions and the following disclaimer in the
11  *    documentation and/or other materials provided with the distribution.
12  *
13  * THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY
14  * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
15  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
16  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL APPLE INC. OR
17  * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
18  * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
19  * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
20  * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
21  * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
22  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
23  * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 
24  */
25
26 #pragma once
27
28 #if ENABLE(VIDEO) && USE(AVFOUNDATION)
29
30 #include "MediaPlaybackTarget.h"
31 #include "MediaPlayerPrivateAVFoundation.h"
32 #include <wtf/Function.h>
33 #include <wtf/HashMap.h>
34 #include <wtf/threads/BinarySemaphore.h>
35
36 OBJC_CLASS AVAssetImageGenerator;
37 OBJC_CLASS AVAssetResourceLoadingRequest;
38 OBJC_CLASS AVMediaSelectionGroup;
39 OBJC_CLASS AVOutputContext;
40 OBJC_CLASS AVPlayerItem;
41 OBJC_CLASS AVPlayerItemLegibleOutput;
42 OBJC_CLASS AVPlayerItemVideoOutput;
43 OBJC_CLASS AVPlayerLayer;
44 OBJC_CLASS AVURLAsset;
45 OBJC_CLASS NSArray;
46 OBJC_CLASS WebCoreAVFLoaderDelegate;
47 OBJC_CLASS WebCoreAVFMovieObserver;
48 OBJC_CLASS WebCoreAVFPullDelegate;
49
50 typedef struct objc_object* id;
51
52 typedef struct CGImage *CGImageRef;
53 typedef struct __CVBuffer *CVPixelBufferRef;
54
55 namespace WebCore {
56
57 class AudioSourceProviderAVFObjC;
58 class AudioTrackPrivateAVFObjC;
59 class CDMInstanceFairPlayStreamingAVFObjC;
60 class CDMSessionAVFoundationObjC;
61 class InbandMetadataTextTrackPrivateAVF;
62 class MediaSelectionGroupAVFObjC;
63 class PixelBufferConformerCV;
64 class VideoFullscreenLayerManagerObjC;
65 class VideoTextureCopierCV;
66 class VideoTrackPrivateAVFObjC;
67 class WebCoreAVFResourceLoader;
68
69 class MediaPlayerPrivateAVFoundationObjC : public MediaPlayerPrivateAVFoundation {
70 public:
71     explicit MediaPlayerPrivateAVFoundationObjC(MediaPlayer*);
72     virtual ~MediaPlayerPrivateAVFoundationObjC();
73
74     static void registerMediaEngine(MediaEngineRegistrar);
75
76     static HashSet<RefPtr<SecurityOrigin>> originsInMediaCache(const String&);
77     static void clearMediaCache(const String&, WallTime modifiedSince);
78     static void clearMediaCacheForOrigins(const String&, const HashSet<RefPtr<SecurityOrigin>>&);
79
80     void setAsset(RetainPtr<id>);
81     void tracksChanged() override;
82
83 #if HAVE(AVFOUNDATION_MEDIA_SELECTION_GROUP)
84     RetainPtr<AVPlayerItem> playerItem() const { return m_avPlayerItem; }
85     void processCue(NSArray *, NSArray *, const MediaTime&);
86     void flushCues();
87 #endif
88     AVPlayer *avPlayer() const { return m_avPlayer.get(); }
89
90 #if HAVE(AVFOUNDATION_LOADER_DELEGATE)
91     bool shouldWaitForLoadingOfResource(AVAssetResourceLoadingRequest*);
92     void didCancelLoadingRequest(AVAssetResourceLoadingRequest*);
93     void didStopLoadingRequest(AVAssetResourceLoadingRequest *);
94 #endif
95
96 #if ENABLE(LEGACY_ENCRYPTED_MEDIA)
97     RetainPtr<AVAssetResourceLoadingRequest> takeRequestForKeyURI(const String&);
98     void keyAdded() override;
99 #endif
100
101     void playerItemStatusDidChange(int);
102     void playbackLikelyToKeepUpWillChange();
103     void playbackLikelyToKeepUpDidChange(bool);
104     void playbackBufferEmptyWillChange();
105     void playbackBufferEmptyDidChange(bool);
106     void playbackBufferFullWillChange();
107     void playbackBufferFullDidChange(bool);
108     void loadedTimeRangesDidChange(RetainPtr<NSArray>);
109     void seekableTimeRangesDidChange(RetainPtr<NSArray>);
110     void tracksDidChange(RetainPtr<NSArray>);
111     void hasEnabledAudioDidChange(bool);
112     void presentationSizeDidChange(FloatSize);
113     void durationDidChange(const MediaTime&);
114     void rateDidChange(double);
115     void metadataDidArrive(RetainPtr<NSArray>, const MediaTime&);
116     void firstFrameAvailableDidChange(bool);
117     void trackEnabledDidChange(bool);
118     void canPlayFastReverseDidChange(bool);
119     void canPlayFastForwardDidChange(bool);
120
121     void setShouldBufferData(bool) override;
122
123 #if HAVE(AVFOUNDATION_VIDEO_OUTPUT)
124     void outputMediaDataWillChange(AVPlayerItemVideoOutput*);
125 #endif
126
127 #if ENABLE(WIRELESS_PLAYBACK_TARGET)
128     void playbackTargetIsWirelessDidChange();
129 #endif
130
131 #if ENABLE(LEGACY_ENCRYPTED_MEDIA) || (ENABLE(ENCRYPTED_MEDIA) && HAVE(AVCONTENTKEYSESSION))
132     void outputObscuredDueToInsufficientExternalProtectionChanged(bool);
133     void beginSimulatedHDCPError() override { outputObscuredDueToInsufficientExternalProtectionChanged(true); }
134     void endSimulatedHDCPError() override { outputObscuredDueToInsufficientExternalProtectionChanged(false); }
135 #endif
136
137 #if ENABLE(AVF_CAPTIONS)
138     void notifyTrackModeChanged() override;
139     void synchronizeTextTrackState() override;
140 #endif
141
142 #if ENABLE(LEGACY_ENCRYPTED_MEDIA)
143     void removeSession(LegacyCDMSession&);
144 #endif
145
146 #if ENABLE(ENCRYPTED_MEDIA)
147     void cdmInstanceAttached(CDMInstance&) final;
148     void cdmInstanceDetached(CDMInstance&) final;
149     void attemptToDecryptWithInstance(CDMInstance&) final;
150     void setWaitingForKey(bool);
151     bool waitingForKey() const final { return m_waitingForKey; }
152 #endif
153
154 private:
155     // engine support
156     static void getSupportedTypes(HashSet<String, ASCIICaseInsensitiveHash>& types);
157     static MediaPlayer::SupportsType supportsType(const MediaEngineSupportParameters&);
158     static bool supportsKeySystem(const String& keySystem, const String& mimeType);
159
160     static bool isAvailable();
161
162     void cancelLoad() override;
163
164     void platformSetVisible(bool) override;
165     void platformPlay() override;
166     void platformPause() override;
167     MediaTime currentMediaTime() const override;
168     void setVolume(float) override;
169 #if PLATFORM(IOS)
170     bool supportsMuting() const override { return true; }
171 #endif
172     void setMuted(bool) override;
173     void setClosedCaptionsVisible(bool) override;
174     void paint(GraphicsContext&, const FloatRect&) override;
175     void paintCurrentFrameInContext(GraphicsContext&, const FloatRect&) override;
176     PlatformLayer* platformLayer() const override;
177     void setVideoFullscreenLayer(PlatformLayer*, Function<void()>&& completionHandler) override;
178     void updateVideoFullscreenInlineImage() final;
179     void setVideoFullscreenFrame(FloatRect) override;
180     void setVideoFullscreenGravity(MediaPlayer::VideoGravity) override;
181     void setVideoFullscreenMode(MediaPlayer::VideoFullscreenMode) override;
182
183 #if PLATFORM(IOS)
184     NSArray *timedMetadata() const override;
185     String accessLog() const override;
186     String errorLog() const override;
187 #endif
188
189     bool supportsAcceleratedRendering() const override { return true; }
190     MediaTime mediaTimeForTimeValue(const MediaTime&) const override;
191     double maximumDurationToCacheMediaTime() const override;
192
193     void createAVPlayer() override;
194     void createAVPlayerItem() override;
195     virtual void createAVPlayerLayer();
196     void createAVAssetForURL(const URL&) override;
197     MediaPlayerPrivateAVFoundation::ItemStatus playerItemStatus() const override;
198     MediaPlayerPrivateAVFoundation::AssetStatus assetStatus() const override;
199     long assetErrorCode() const override;
200
201     double seekableTimeRangesLastModifiedTime() const override;
202     double liveUpdateInterval() const override;
203
204     void checkPlayability() override;
205     void setRateDouble(double) override;
206     double rate() const override;
207     void setPreservesPitch(bool) override;
208     void seekToTime(const MediaTime&, const MediaTime& negativeTolerance, const MediaTime& positiveTolerance) override;
209     unsigned long long totalBytes() const override;
210     std::unique_ptr<PlatformTimeRanges> platformBufferedTimeRanges() const override;
211     MediaTime platformMinTimeSeekable() const override;
212     MediaTime platformMaxTimeSeekable() const override;
213     MediaTime platformDuration() const override;
214     MediaTime platformMaxTimeLoaded() const override;
215     void beginLoadingMetadata() override;
216     void sizeChanged() override;
217     void resolvedURLChanged() override;
218
219     bool hasAvailableVideoFrame() const override;
220
221     void createContextVideoRenderer() override;
222     void destroyContextVideoRenderer() override;
223
224     void createVideoLayer() override;
225     void destroyVideoLayer() override;
226
227     bool hasContextRenderer() const override;
228     bool hasLayerRenderer() const override;
229
230     void updateVideoLayerGravity() override;
231
232     bool didPassCORSAccessCheck() const override;
233     std::optional<bool> wouldTaintOrigin(const SecurityOrigin&) const final;
234
235
236     MediaTime getStartDate() const override;
237
238 #if ENABLE(VIDEO_TRACK)
239     bool requiresTextTrackRepresentation() const override;
240     void setTextTrackRepresentation(TextTrackRepresentation*) override;
241     void syncTextTrackBounds() override;
242 #endif
243
244     void setAVPlayerItem(AVPlayerItem *);
245
246 #if ENABLE(WEB_AUDIO) && USE(MEDIATOOLBOX)
247     AudioSourceProvider* audioSourceProvider() override;
248 #endif
249
250     void createImageGenerator();
251     void destroyImageGenerator();
252     RetainPtr<CGImageRef> createImageForTimeInRect(float, const FloatRect&);
253     void paintWithImageGenerator(GraphicsContext&, const FloatRect&);
254
255 #if HAVE(AVFOUNDATION_VIDEO_OUTPUT)
256     enum class UpdateType { UpdateSynchronously, UpdateAsynchronously };
257     void updateLastImage(UpdateType type = UpdateType::UpdateAsynchronously);
258
259     void createVideoOutput();
260     void destroyVideoOutput();
261     bool updateLastPixelBuffer();
262     bool videoOutputHasAvailableFrame();
263     void paintWithVideoOutput(GraphicsContext&, const FloatRect&);
264     NativeImagePtr nativeImageForCurrentTime() override;
265     void waitForVideoOutputMediaDataWillChange();
266
267     bool copyVideoTextureToPlatformTexture(GraphicsContext3D*, Platform3DObject, GC3Denum target, GC3Dint level, GC3Denum internalFormat, GC3Denum format, GC3Denum type, bool premultiplyAlpha, bool flipY) override;
268 #endif
269
270 #if ENABLE(LEGACY_ENCRYPTED_MEDIA)
271     std::unique_ptr<LegacyCDMSession> createSession(const String& keySystem, LegacyCDMSessionClient*) override;
272 #endif
273
274     String languageOfPrimaryAudioTrack() const override;
275
276 #if HAVE(AVFOUNDATION_MEDIA_SELECTION_GROUP)
277     void processMediaSelectionOptions();
278     bool hasLoadedMediaSelectionGroups();
279
280     AVMediaSelectionGroup* safeMediaSelectionGroupForLegibleMedia();
281     AVMediaSelectionGroup* safeMediaSelectionGroupForAudibleMedia();
282     AVMediaSelectionGroup* safeMediaSelectionGroupForVisualMedia();
283 #endif
284
285     NSArray* safeAVAssetTracksForAudibleMedia();
286
287 #if ENABLE(DATACUE_VALUE)
288     void processMetadataTrack();
289 #endif
290
291     void setCurrentTextTrack(InbandTextTrackPrivateAVF*) override;
292     InbandTextTrackPrivateAVF* currentTextTrack() const override { return m_currentTextTrack; }
293
294 #if !HAVE(AVFOUNDATION_LEGIBLE_OUTPUT_SUPPORT)
295     void processLegacyClosedCaptionsTracks();
296 #endif
297
298 #if ENABLE(VIDEO_TRACK)
299     void updateAudioTracks();
300     void updateVideoTracks();
301 #endif
302
303 #if ENABLE(WIRELESS_PLAYBACK_TARGET)
304     bool isCurrentPlaybackTargetWireless() const override;
305     String wirelessPlaybackTargetName() const override;
306     MediaPlayer::WirelessPlaybackTargetType wirelessPlaybackTargetType() const override;
307     bool wirelessVideoPlaybackDisabled() const override;
308     void setWirelessVideoPlaybackDisabled(bool) override;
309     bool canPlayToWirelessPlaybackTarget() const override { return true; }
310     void updateDisableExternalPlayback();
311 #endif
312
313 #if ENABLE(WIRELESS_PLAYBACK_TARGET) && !PLATFORM(IOS)
314     void setWirelessPlaybackTarget(Ref<MediaPlaybackTarget>&&) override;
315     void setShouldPlayToPlaybackTarget(bool) override;
316 #endif
317
318     double maxFastForwardRate() const override { return m_cachedCanPlayFastForward ? std::numeric_limits<double>::infinity() : 2.0; }
319     double minFastReverseRate() const override { return m_cachedCanPlayFastReverse ? -std::numeric_limits<double>::infinity() : 0.0; }
320
321     Vector<String> preferredAudioCharacteristics() const;
322
323     void setShouldDisableSleep(bool) override;
324
325     std::optional<VideoPlaybackQualityMetrics> videoPlaybackQualityMetrics() final;
326
327 #if !RELEASE_LOG_DISABLED
328     const char* logClassName() const final { return "MediaPlayerPrivateAVFoundationObjC"; }
329 #endif
330
331     AVPlayer *objCAVFoundationAVPlayer() const final { return m_avPlayer.get(); }
332
333     RetainPtr<AVURLAsset> m_avAsset;
334     RetainPtr<AVPlayer> m_avPlayer;
335     RetainPtr<AVPlayerItem> m_avPlayerItem;
336     RetainPtr<AVPlayerLayer> m_videoLayer;
337     std::unique_ptr<VideoFullscreenLayerManagerObjC> m_videoFullscreenLayerManager;
338     MediaPlayer::VideoGravity m_videoFullscreenGravity;
339     RetainPtr<WebCoreAVFMovieObserver> m_objcObserver;
340     RetainPtr<id> m_timeObserver;
341     mutable String m_languageOfPrimaryAudioTrack;
342     bool m_videoFrameHasDrawn;
343     bool m_haveCheckedPlayability;
344
345 #if ENABLE(WEB_AUDIO) && USE(MEDIATOOLBOX)
346     RefPtr<AudioSourceProviderAVFObjC> m_provider;
347 #endif
348
349     RetainPtr<AVAssetImageGenerator> m_imageGenerator;
350 #if HAVE(AVFOUNDATION_VIDEO_OUTPUT)
351     RetainPtr<AVPlayerItemVideoOutput> m_videoOutput;
352     RetainPtr<WebCoreAVFPullDelegate> m_videoOutputDelegate;
353     RetainPtr<CVPixelBufferRef> m_lastPixelBuffer;
354     RetainPtr<CGImageRef> m_lastImage;
355     BinarySemaphore m_videoOutputSemaphore;
356     std::unique_ptr<VideoTextureCopierCV> m_videoTextureCopier;
357 #endif
358
359 #if HAVE(CORE_VIDEO)
360     std::unique_ptr<PixelBufferConformerCV> m_pixelBufferConformer;
361 #endif
362
363 #if HAVE(AVFOUNDATION_LOADER_DELEGATE)
364     friend class WebCoreAVFResourceLoader;
365     HashMap<RetainPtr<CFTypeRef>, RefPtr<WebCoreAVFResourceLoader>> m_resourceLoaderMap;
366     RetainPtr<WebCoreAVFLoaderDelegate> m_loaderDelegate;
367     HashMap<String, RetainPtr<AVAssetResourceLoadingRequest>> m_keyURIToRequestMap;
368     HashMap<String, RetainPtr<AVAssetResourceLoadingRequest>> m_sessionIDToRequestMap;
369 #endif
370
371 #if HAVE(AVFOUNDATION_MEDIA_SELECTION_GROUP) && HAVE(AVFOUNDATION_LEGIBLE_OUTPUT_SUPPORT)
372     RetainPtr<AVPlayerItemLegibleOutput> m_legibleOutput;
373 #endif
374
375 #if ENABLE(VIDEO_TRACK)
376     Vector<RefPtr<AudioTrackPrivateAVFObjC>> m_audioTracks;
377     Vector<RefPtr<VideoTrackPrivateAVFObjC>> m_videoTracks;
378 #if HAVE(AVFOUNDATION_MEDIA_SELECTION_GROUP)
379     RefPtr<MediaSelectionGroupAVFObjC> m_audibleGroup;
380     RefPtr<MediaSelectionGroupAVFObjC> m_visualGroup;
381 #endif
382 #endif
383
384     InbandTextTrackPrivateAVF* m_currentTextTrack;
385
386 #if ENABLE(DATACUE_VALUE)
387     RefPtr<InbandMetadataTextTrackPrivateAVF> m_metadataTrack;
388 #endif
389
390 #if PLATFORM(MAC) && ENABLE(WIRELESS_PLAYBACK_TARGET)
391     RetainPtr<AVOutputContext> m_outputContext;
392     RefPtr<MediaPlaybackTarget> m_playbackTarget { nullptr };
393 #endif
394
395 #if ENABLE(LEGACY_ENCRYPTED_MEDIA)
396     WeakPtr<CDMSessionAVFoundationObjC> m_session;
397 #endif
398 #if ENABLE(ENCRYPTED_MEDIA)
399     bool m_waitingForKey { false };
400 #if HAVE(AVCONTENTKEYSESSION)
401     RefPtr<CDMInstanceFairPlayStreamingAVFObjC> m_cdmInstance;
402 #endif
403 #endif
404
405     mutable RetainPtr<NSArray> m_cachedSeekableRanges;
406     mutable RetainPtr<NSArray> m_cachedLoadedRanges;
407     RetainPtr<NSArray> m_cachedTracks;
408     RetainPtr<NSArray> m_currentMetaData;
409     FloatSize m_cachedPresentationSize;
410     MediaTime m_cachedDuration;
411     double m_cachedRate;
412     mutable long long m_cachedTotalBytes;
413     unsigned m_pendingStatusChanges;
414     int m_cachedItemStatus;
415     bool m_cachedLikelyToKeepUp;
416     bool m_cachedBufferEmpty;
417     bool m_cachedBufferFull;
418     bool m_cachedHasEnabledAudio;
419     bool m_shouldBufferData;
420     bool m_cachedIsReadyForDisplay;
421     bool m_haveBeenAskedToCreateLayer;
422     bool m_cachedCanPlayFastForward;
423     bool m_cachedCanPlayFastReverse;
424     bool m_muted { false };
425     mutable std::optional<bool> m_tracksArePlayable;
426 #if ENABLE(WIRELESS_PLAYBACK_TARGET)
427     mutable bool m_allowsWirelessVideoPlayback;
428     bool m_shouldPlayToPlaybackTarget { false };
429 #endif
430 };
431
432 }
433
434 #endif