Unreviewed watchOS build fix; Fix declaration for ports which USE(ENCRYPTED_MEDIA...
[WebKit-https.git] / Source / WebCore / platform / graphics / avfoundation / objc / MediaPlayerPrivateAVFoundationObjC.h
1 /*
2  * Copyright (C) 2011-2018 Apple Inc. All rights reserved.
3  *
4  * Redistribution and use in source and binary forms, with or without
5  * modification, are permitted provided that the following conditions
6  * are met:
7  * 1. Redistributions of source code must retain the above copyright
8  *    notice, this list of conditions and the following disclaimer.
9  * 2. Redistributions in binary form must reproduce the above copyright
10  *    notice, this list of conditions and the following disclaimer in the
11  *    documentation and/or other materials provided with the distribution.
12  *
13  * THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY
14  * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
15  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
16  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL APPLE INC. OR
17  * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
18  * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
19  * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
20  * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
21  * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
22  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
23  * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 
24  */
25
26 #pragma once
27
28 #if ENABLE(VIDEO) && USE(AVFOUNDATION)
29
30 #include "MediaPlaybackTarget.h"
31 #include "MediaPlayerPrivateAVFoundation.h"
32 #include <wtf/Function.h>
33 #include <wtf/HashMap.h>
34 #include <wtf/threads/BinarySemaphore.h>
35
36 OBJC_CLASS AVAssetImageGenerator;
37 OBJC_CLASS AVAssetResourceLoadingRequest;
38 OBJC_CLASS AVMediaSelectionGroup;
39 OBJC_CLASS AVOutputContext;
40 OBJC_CLASS AVPlayerItem;
41 OBJC_CLASS AVPlayerItemLegibleOutput;
42 OBJC_CLASS AVPlayerItemVideoOutput;
43 OBJC_CLASS AVPlayerLayer;
44 OBJC_CLASS AVURLAsset;
45 OBJC_CLASS NSArray;
46 OBJC_CLASS WebCoreAVFLoaderDelegate;
47 OBJC_CLASS WebCoreAVFMovieObserver;
48 OBJC_CLASS WebCoreAVFPullDelegate;
49
50 typedef struct objc_object* id;
51
52 typedef struct CGImage *CGImageRef;
53 typedef struct __CVBuffer *CVPixelBufferRef;
54
55 namespace WebCore {
56
57 class AudioSourceProviderAVFObjC;
58 class AudioTrackPrivateAVFObjC;
59 class CDMInstanceFairPlayStreamingAVFObjC;
60 class CDMSessionAVFoundationObjC;
61 class InbandMetadataTextTrackPrivateAVF;
62 class MediaSelectionGroupAVFObjC;
63 class PixelBufferConformerCV;
64 class VideoFullscreenLayerManagerObjC;
65 class VideoTextureCopierCV;
66 class VideoTrackPrivateAVFObjC;
67 class WebCoreAVFResourceLoader;
68
69 class MediaPlayerPrivateAVFoundationObjC : public MediaPlayerPrivateAVFoundation {
70 public:
71     explicit MediaPlayerPrivateAVFoundationObjC(MediaPlayer*);
72     virtual ~MediaPlayerPrivateAVFoundationObjC();
73
74     static void registerMediaEngine(MediaEngineRegistrar);
75
76     static HashSet<RefPtr<SecurityOrigin>> originsInMediaCache(const String&);
77     static void clearMediaCache(const String&, WallTime modifiedSince);
78     static void clearMediaCacheForOrigins(const String&, const HashSet<RefPtr<SecurityOrigin>>&);
79
80     void setAsset(RetainPtr<id>);
81     void tracksChanged() override;
82
83 #if HAVE(AVFOUNDATION_MEDIA_SELECTION_GROUP)
84     RetainPtr<AVPlayerItem> playerItem() const { return m_avPlayerItem; }
85     void processCue(NSArray *, NSArray *, const MediaTime&);
86     void flushCues();
87 #endif
88     AVPlayer *avPlayer() const { return m_avPlayer.get(); }
89
90 #if HAVE(AVFOUNDATION_LOADER_DELEGATE)
91     bool shouldWaitForLoadingOfResource(AVAssetResourceLoadingRequest*);
92     void didCancelLoadingRequest(AVAssetResourceLoadingRequest*);
93     void didStopLoadingRequest(AVAssetResourceLoadingRequest *);
94 #endif
95
96 #if ENABLE(LEGACY_ENCRYPTED_MEDIA)
97     RetainPtr<AVAssetResourceLoadingRequest> takeRequestForKeyURI(const String&);
98     void keyAdded() override;
99 #endif
100
101     void playerItemStatusDidChange(int);
102     void playbackLikelyToKeepUpWillChange();
103     void playbackLikelyToKeepUpDidChange(bool);
104     void playbackBufferEmptyWillChange();
105     void playbackBufferEmptyDidChange(bool);
106     void playbackBufferFullWillChange();
107     void playbackBufferFullDidChange(bool);
108     void loadedTimeRangesDidChange(RetainPtr<NSArray>);
109     void seekableTimeRangesDidChange(RetainPtr<NSArray>);
110     void tracksDidChange(RetainPtr<NSArray>);
111     void hasEnabledAudioDidChange(bool);
112     void presentationSizeDidChange(FloatSize);
113     void durationDidChange(const MediaTime&);
114     void rateDidChange(double);
115     void metadataDidArrive(RetainPtr<NSArray>, const MediaTime&);
116     void firstFrameAvailableDidChange(bool);
117     void trackEnabledDidChange(bool);
118     void canPlayFastReverseDidChange(bool);
119     void canPlayFastForwardDidChange(bool);
120
121     void setShouldBufferData(bool) override;
122
123 #if HAVE(AVFOUNDATION_VIDEO_OUTPUT)
124     void outputMediaDataWillChange(AVPlayerItemVideoOutput*);
125 #endif
126
127 #if ENABLE(WIRELESS_PLAYBACK_TARGET)
128     void playbackTargetIsWirelessDidChange();
129 #endif
130
131 #if ENABLE(LEGACY_ENCRYPTED_MEDIA) || (ENABLE(ENCRYPTED_MEDIA) && HAVE(AVCONTENTKEYSESSION))
132     void outputObscuredDueToInsufficientExternalProtectionChanged(bool);
133     void beginSimulatedHDCPError() override { outputObscuredDueToInsufficientExternalProtectionChanged(true); }
134     void endSimulatedHDCPError() override { outputObscuredDueToInsufficientExternalProtectionChanged(false); }
135 #endif
136
137 #if ENABLE(AVF_CAPTIONS)
138     void notifyTrackModeChanged() override;
139     void synchronizeTextTrackState() override;
140 #endif
141
142 #if ENABLE(LEGACY_ENCRYPTED_MEDIA)
143     void removeSession(LegacyCDMSession&);
144 #endif
145
146 #if ENABLE(ENCRYPTED_MEDIA)
147     void cdmInstanceAttached(CDMInstance&) final;
148     void cdmInstanceDetached(CDMInstance&) final;
149     void attemptToDecryptWithInstance(CDMInstance&) final;
150     void setWaitingForKey(bool);
151     bool waitingForKey() const final { return m_waitingForKey; }
152 #endif
153
154 private:
155     // engine support
156     static void getSupportedTypes(HashSet<String, ASCIICaseInsensitiveHash>& types);
157     static MediaPlayer::SupportsType supportsType(const MediaEngineSupportParameters&);
158     static bool supportsKeySystem(const String& keySystem, const String& mimeType);
159
160     static bool isAvailable();
161
162     void cancelLoad() override;
163
164     void platformSetVisible(bool) override;
165     void platformPlay() override;
166     void platformPause() override;
167     MediaTime currentMediaTime() const override;
168     void setVolume(float) override;
169 #if PLATFORM(IOS)
170     bool supportsMuting() const override { return true; }
171 #endif
172     void setMuted(bool) override;
173     void setClosedCaptionsVisible(bool) override;
174     void paint(GraphicsContext&, const FloatRect&) override;
175     void paintCurrentFrameInContext(GraphicsContext&, const FloatRect&) override;
176     PlatformLayer* platformLayer() const override;
177     void setVideoFullscreenLayer(PlatformLayer*, Function<void()>&& completionHandler) override;
178     void updateVideoFullscreenInlineImage() final;
179     void setVideoFullscreenFrame(FloatRect) override;
180     void setVideoFullscreenGravity(MediaPlayer::VideoGravity) override;
181     void setVideoFullscreenMode(MediaPlayer::VideoFullscreenMode) override;
182
183 #if PLATFORM(IOS)
184     NSArray *timedMetadata() const override;
185     String accessLog() const override;
186     String errorLog() const override;
187 #endif
188
189     bool supportsAcceleratedRendering() const override { return true; }
190     MediaTime mediaTimeForTimeValue(const MediaTime&) const override;
191     double maximumDurationToCacheMediaTime() const override;
192
193     void createAVPlayer() override;
194     void createAVPlayerItem() override;
195     virtual void createAVPlayerLayer();
196     void createAVAssetForURL(const URL&) override;
197     MediaPlayerPrivateAVFoundation::ItemStatus playerItemStatus() const override;
198     MediaPlayerPrivateAVFoundation::AssetStatus assetStatus() const override;
199     long assetErrorCode() const override;
200
201     double seekableTimeRangesLastModifiedTime() const override;
202     double liveUpdateInterval() const override;
203
204     void checkPlayability() override;
205     void setRateDouble(double) override;
206     double rate() const override;
207     void setPreservesPitch(bool) override;
208     void seekToTime(const MediaTime&, const MediaTime& negativeTolerance, const MediaTime& positiveTolerance) override;
209     unsigned long long totalBytes() const override;
210     std::unique_ptr<PlatformTimeRanges> platformBufferedTimeRanges() const override;
211     MediaTime platformMinTimeSeekable() const override;
212     MediaTime platformMaxTimeSeekable() const override;
213     MediaTime platformDuration() const override;
214     MediaTime platformMaxTimeLoaded() const override;
215     void beginLoadingMetadata() override;
216     void sizeChanged() override;
217     void resolvedURLChanged() override;
218
219     bool hasAvailableVideoFrame() const override;
220
221     void createContextVideoRenderer() override;
222     void destroyContextVideoRenderer() override;
223
224     void createVideoLayer() override;
225     void destroyVideoLayer() override;
226
227     bool hasContextRenderer() const override;
228     bool hasLayerRenderer() const override;
229
230     void updateVideoLayerGravity() override;
231
232     bool didPassCORSAccessCheck() const override;
233     std::optional<bool> wouldTaintOrigin(const SecurityOrigin&) const final;
234
235
236     MediaTime getStartDate() const override;
237
238 #if ENABLE(VIDEO_TRACK)
239     bool requiresTextTrackRepresentation() const override;
240     void setTextTrackRepresentation(TextTrackRepresentation*) override;
241     void syncTextTrackBounds() override;
242 #endif
243
244     void setAVPlayerItem(AVPlayerItem *);
245
246 #if ENABLE(WEB_AUDIO) && USE(MEDIATOOLBOX)
247     AudioSourceProvider* audioSourceProvider() override;
248 #endif
249
250     void createImageGenerator();
251     void destroyImageGenerator();
252     RetainPtr<CGImageRef> createImageForTimeInRect(float, const FloatRect&);
253     void paintWithImageGenerator(GraphicsContext&, const FloatRect&);
254
255 #if HAVE(AVFOUNDATION_VIDEO_OUTPUT)
256     enum class UpdateType { UpdateSynchronously, UpdateAsynchronously };
257     void updateLastImage(UpdateType type = UpdateType::UpdateAsynchronously);
258
259     void createVideoOutput();
260     void destroyVideoOutput();
261     bool updateLastPixelBuffer();
262     bool videoOutputHasAvailableFrame();
263     void paintWithVideoOutput(GraphicsContext&, const FloatRect&);
264     NativeImagePtr nativeImageForCurrentTime() override;
265     void waitForVideoOutputMediaDataWillChange();
266
267     bool copyVideoTextureToPlatformTexture(GraphicsContext3D*, Platform3DObject, GC3Denum target, GC3Dint level, GC3Denum internalFormat, GC3Denum format, GC3Denum type, bool premultiplyAlpha, bool flipY) override;
268 #endif
269
270 #if ENABLE(LEGACY_ENCRYPTED_MEDIA)
271     std::unique_ptr<LegacyCDMSession> createSession(const String& keySystem, LegacyCDMSessionClient*) override;
272 #endif
273
274     String languageOfPrimaryAudioTrack() const override;
275
276 #if HAVE(AVFOUNDATION_MEDIA_SELECTION_GROUP)
277     void processMediaSelectionOptions();
278     bool hasLoadedMediaSelectionGroups();
279
280     AVMediaSelectionGroup* safeMediaSelectionGroupForLegibleMedia();
281     AVMediaSelectionGroup* safeMediaSelectionGroupForAudibleMedia();
282     AVMediaSelectionGroup* safeMediaSelectionGroupForVisualMedia();
283 #endif
284
285     NSArray* safeAVAssetTracksForAudibleMedia();
286
287 #if ENABLE(DATACUE_VALUE)
288     void processMetadataTrack();
289 #endif
290
291     void setCurrentTextTrack(InbandTextTrackPrivateAVF*) override;
292     InbandTextTrackPrivateAVF* currentTextTrack() const override { return m_currentTextTrack; }
293
294 #if !HAVE(AVFOUNDATION_LEGIBLE_OUTPUT_SUPPORT)
295     void processLegacyClosedCaptionsTracks();
296 #endif
297
298 #if ENABLE(VIDEO_TRACK)
299     void updateAudioTracks();
300     void updateVideoTracks();
301 #endif
302
303 #if ENABLE(WIRELESS_PLAYBACK_TARGET)
304     bool isCurrentPlaybackTargetWireless() const override;
305     String wirelessPlaybackTargetName() const override;
306     MediaPlayer::WirelessPlaybackTargetType wirelessPlaybackTargetType() const override;
307     bool wirelessVideoPlaybackDisabled() const override;
308     void setWirelessVideoPlaybackDisabled(bool) override;
309     bool canPlayToWirelessPlaybackTarget() const override { return true; }
310     void updateDisableExternalPlayback();
311 #endif
312
313 #if ENABLE(WIRELESS_PLAYBACK_TARGET) && !PLATFORM(IOS)
314     void setWirelessPlaybackTarget(Ref<MediaPlaybackTarget>&&) override;
315     void setShouldPlayToPlaybackTarget(bool) override;
316 #endif
317
318     double maxFastForwardRate() const override { return m_cachedCanPlayFastForward ? std::numeric_limits<double>::infinity() : 2.0; }
319     double minFastReverseRate() const override { return m_cachedCanPlayFastReverse ? -std::numeric_limits<double>::infinity() : 0.0; }
320
321     Vector<String> preferredAudioCharacteristics() const;
322
323     void setShouldDisableSleep(bool) override;
324
325 #if !RELEASE_LOG_DISABLED
326     const char* logClassName() const final { return "MediaPlayerPrivateAVFoundationObjC"; }
327 #endif
328
329     AVPlayer *objCAVFoundationAVPlayer() const final { return m_avPlayer.get(); }
330
331     RetainPtr<AVURLAsset> m_avAsset;
332     RetainPtr<AVPlayer> m_avPlayer;
333     RetainPtr<AVPlayerItem> m_avPlayerItem;
334     RetainPtr<AVPlayerLayer> m_videoLayer;
335     std::unique_ptr<VideoFullscreenLayerManagerObjC> m_videoFullscreenLayerManager;
336     MediaPlayer::VideoGravity m_videoFullscreenGravity;
337     RetainPtr<WebCoreAVFMovieObserver> m_objcObserver;
338     RetainPtr<id> m_timeObserver;
339     mutable String m_languageOfPrimaryAudioTrack;
340     bool m_videoFrameHasDrawn;
341     bool m_haveCheckedPlayability;
342
343 #if ENABLE(WEB_AUDIO) && USE(MEDIATOOLBOX)
344     RefPtr<AudioSourceProviderAVFObjC> m_provider;
345 #endif
346
347     RetainPtr<AVAssetImageGenerator> m_imageGenerator;
348 #if HAVE(AVFOUNDATION_VIDEO_OUTPUT)
349     RetainPtr<AVPlayerItemVideoOutput> m_videoOutput;
350     RetainPtr<WebCoreAVFPullDelegate> m_videoOutputDelegate;
351     RetainPtr<CVPixelBufferRef> m_lastPixelBuffer;
352     RetainPtr<CGImageRef> m_lastImage;
353     BinarySemaphore m_videoOutputSemaphore;
354     std::unique_ptr<VideoTextureCopierCV> m_videoTextureCopier;
355 #endif
356
357 #if HAVE(CORE_VIDEO)
358     std::unique_ptr<PixelBufferConformerCV> m_pixelBufferConformer;
359 #endif
360
361 #if HAVE(AVFOUNDATION_LOADER_DELEGATE)
362     friend class WebCoreAVFResourceLoader;
363     HashMap<RetainPtr<CFTypeRef>, RefPtr<WebCoreAVFResourceLoader>> m_resourceLoaderMap;
364     RetainPtr<WebCoreAVFLoaderDelegate> m_loaderDelegate;
365     HashMap<String, RetainPtr<AVAssetResourceLoadingRequest>> m_keyURIToRequestMap;
366     HashMap<String, RetainPtr<AVAssetResourceLoadingRequest>> m_sessionIDToRequestMap;
367 #endif
368
369 #if HAVE(AVFOUNDATION_MEDIA_SELECTION_GROUP) && HAVE(AVFOUNDATION_LEGIBLE_OUTPUT_SUPPORT)
370     RetainPtr<AVPlayerItemLegibleOutput> m_legibleOutput;
371 #endif
372
373 #if ENABLE(VIDEO_TRACK)
374     Vector<RefPtr<AudioTrackPrivateAVFObjC>> m_audioTracks;
375     Vector<RefPtr<VideoTrackPrivateAVFObjC>> m_videoTracks;
376 #if HAVE(AVFOUNDATION_MEDIA_SELECTION_GROUP)
377     RefPtr<MediaSelectionGroupAVFObjC> m_audibleGroup;
378     RefPtr<MediaSelectionGroupAVFObjC> m_visualGroup;
379 #endif
380 #endif
381
382     InbandTextTrackPrivateAVF* m_currentTextTrack;
383
384 #if ENABLE(DATACUE_VALUE)
385     RefPtr<InbandMetadataTextTrackPrivateAVF> m_metadataTrack;
386 #endif
387
388 #if PLATFORM(MAC) && ENABLE(WIRELESS_PLAYBACK_TARGET)
389     RetainPtr<AVOutputContext> m_outputContext;
390     RefPtr<MediaPlaybackTarget> m_playbackTarget { nullptr };
391 #endif
392
393 #if ENABLE(LEGACY_ENCRYPTED_MEDIA)
394     WeakPtr<CDMSessionAVFoundationObjC> m_session;
395 #endif
396 #if ENABLE(ENCRYPTED_MEDIA)
397     bool m_waitingForKey { false };
398 #if HAVE(AVCONTENTKEYSESSION)
399     RefPtr<CDMInstanceFairPlayStreamingAVFObjC> m_cdmInstance;
400 #endif
401 #endif
402
403     mutable RetainPtr<NSArray> m_cachedSeekableRanges;
404     mutable RetainPtr<NSArray> m_cachedLoadedRanges;
405     RetainPtr<NSArray> m_cachedTracks;
406     RetainPtr<NSArray> m_currentMetaData;
407     FloatSize m_cachedPresentationSize;
408     MediaTime m_cachedDuration;
409     double m_cachedRate;
410     mutable long long m_cachedTotalBytes;
411     unsigned m_pendingStatusChanges;
412     int m_cachedItemStatus;
413     bool m_cachedLikelyToKeepUp;
414     bool m_cachedBufferEmpty;
415     bool m_cachedBufferFull;
416     bool m_cachedHasEnabledAudio;
417     bool m_shouldBufferData;
418     bool m_cachedIsReadyForDisplay;
419     bool m_haveBeenAskedToCreateLayer;
420     bool m_cachedCanPlayFastForward;
421     bool m_cachedCanPlayFastReverse;
422     bool m_muted { false };
423     mutable std::optional<bool> m_tracksArePlayable;
424 #if ENABLE(WIRELESS_PLAYBACK_TARGET)
425     mutable bool m_allowsWirelessVideoPlayback;
426     bool m_shouldPlayToPlaybackTarget { false };
427 #endif
428 };
429
430 }
431
432 #endif