[WTF] Use Semaphore and BinarySemaphore instead of dispatch_semaphore_t
[WebKit-https.git] / Source / WebCore / platform / graphics / avfoundation / objc / MediaPlayerPrivateAVFoundationObjC.h
1 /*
2  * Copyright (C) 2011-2018 Apple Inc. All rights reserved.
3  *
4  * Redistribution and use in source and binary forms, with or without
5  * modification, are permitted provided that the following conditions
6  * are met:
7  * 1. Redistributions of source code must retain the above copyright
8  *    notice, this list of conditions and the following disclaimer.
9  * 2. Redistributions in binary form must reproduce the above copyright
10  *    notice, this list of conditions and the following disclaimer in the
11  *    documentation and/or other materials provided with the distribution.
12  *
13  * THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY
14  * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
15  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
16  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL APPLE INC. OR
17  * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
18  * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
19  * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
20  * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
21  * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
22  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
23  * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 
24  */
25
26 #pragma once
27
28 #if ENABLE(VIDEO) && USE(AVFOUNDATION)
29
30 #include "MediaPlaybackTarget.h"
31 #include "MediaPlayerPrivateAVFoundation.h"
32 #include <wtf/Function.h>
33 #include <wtf/HashMap.h>
34 #include <wtf/threads/BinarySemaphore.h>
35
36 OBJC_CLASS AVAssetImageGenerator;
37 OBJC_CLASS AVAssetResourceLoadingRequest;
38 OBJC_CLASS AVMediaSelectionGroup;
39 OBJC_CLASS AVOutputContext;
40 OBJC_CLASS AVPlayerItem;
41 OBJC_CLASS AVPlayerItemLegibleOutput;
42 OBJC_CLASS AVPlayerItemVideoOutput;
43 OBJC_CLASS AVPlayerLayer;
44 OBJC_CLASS AVURLAsset;
45 OBJC_CLASS NSArray;
46 OBJC_CLASS WebCoreAVFLoaderDelegate;
47 OBJC_CLASS WebCoreAVFMovieObserver;
48 OBJC_CLASS WebCoreAVFPullDelegate;
49
50 typedef struct objc_object* id;
51
52 typedef struct CGImage *CGImageRef;
53 typedef struct __CVBuffer *CVPixelBufferRef;
54
55 namespace WebCore {
56
57 class AudioSourceProviderAVFObjC;
58 class AudioTrackPrivateAVFObjC;
59 class CDMInstanceFairPlayStreamingAVFObjC;
60 class CDMSessionAVFoundationObjC;
61 class InbandMetadataTextTrackPrivateAVF;
62 class MediaSelectionGroupAVFObjC;
63 class PixelBufferConformerCV;
64 class VideoFullscreenLayerManagerObjC;
65 class VideoTextureCopierCV;
66 class VideoTrackPrivateAVFObjC;
67 class WebCoreAVFResourceLoader;
68
69 class MediaPlayerPrivateAVFoundationObjC : public MediaPlayerPrivateAVFoundation {
70 public:
71     explicit MediaPlayerPrivateAVFoundationObjC(MediaPlayer*);
72     virtual ~MediaPlayerPrivateAVFoundationObjC();
73
74     static void registerMediaEngine(MediaEngineRegistrar);
75
76     static HashSet<RefPtr<SecurityOrigin>> originsInMediaCache(const String&);
77     static void clearMediaCache(const String&, WallTime modifiedSince);
78     static void clearMediaCacheForOrigins(const String&, const HashSet<RefPtr<SecurityOrigin>>&);
79
80     void setAsset(RetainPtr<id>);
81     void tracksChanged() override;
82
83 #if HAVE(AVFOUNDATION_MEDIA_SELECTION_GROUP)
84     RetainPtr<AVPlayerItem> playerItem() const { return m_avPlayerItem; }
85     void processCue(NSArray *, NSArray *, const MediaTime&);
86     void flushCues();
87 #endif
88     AVPlayer *avPlayer() const { return m_avPlayer.get(); }
89
90 #if HAVE(AVFOUNDATION_LOADER_DELEGATE)
91     bool shouldWaitForLoadingOfResource(AVAssetResourceLoadingRequest*);
92     void didCancelLoadingRequest(AVAssetResourceLoadingRequest*);
93     void didStopLoadingRequest(AVAssetResourceLoadingRequest *);
94 #endif
95
96 #if ENABLE(LEGACY_ENCRYPTED_MEDIA)
97     RetainPtr<AVAssetResourceLoadingRequest> takeRequestForKeyURI(const String&);
98     void keyAdded() override;
99 #endif
100
101     void playerItemStatusDidChange(int);
102     void playbackLikelyToKeepUpWillChange();
103     void playbackLikelyToKeepUpDidChange(bool);
104     void playbackBufferEmptyWillChange();
105     void playbackBufferEmptyDidChange(bool);
106     void playbackBufferFullWillChange();
107     void playbackBufferFullDidChange(bool);
108     void loadedTimeRangesDidChange(RetainPtr<NSArray>);
109     void seekableTimeRangesDidChange(RetainPtr<NSArray>);
110     void tracksDidChange(RetainPtr<NSArray>);
111     void hasEnabledAudioDidChange(bool);
112     void presentationSizeDidChange(FloatSize);
113     void durationDidChange(const MediaTime&);
114     void rateDidChange(double);
115     void metadataDidArrive(RetainPtr<NSArray>, const MediaTime&);
116     void firstFrameAvailableDidChange(bool);
117     void trackEnabledDidChange(bool);
118     void canPlayFastReverseDidChange(bool);
119     void canPlayFastForwardDidChange(bool);
120
121     void setShouldBufferData(bool) override;
122
123 #if HAVE(AVFOUNDATION_VIDEO_OUTPUT)
124     void outputMediaDataWillChange(AVPlayerItemVideoOutput*);
125 #endif
126
127 #if ENABLE(WIRELESS_PLAYBACK_TARGET)
128     void playbackTargetIsWirelessDidChange();
129 #endif
130
131 #if ENABLE(LEGACY_ENCRYPTED_MEDIA) || (ENABLE(ENCRYPTED_MEDIA) && HAVE(AVCONTENTKEYSESSION))
132     void outputObscuredDueToInsufficientExternalProtectionChanged(bool);
133     void beginSimulatedHDCPError() override { outputObscuredDueToInsufficientExternalProtectionChanged(true); }
134     void endSimulatedHDCPError() override { outputObscuredDueToInsufficientExternalProtectionChanged(false); }
135 #endif
136
137 #if ENABLE(AVF_CAPTIONS)
138     void notifyTrackModeChanged() override;
139     void synchronizeTextTrackState() override;
140 #endif
141
142 #if ENABLE(LEGACY_ENCRYPTED_MEDIA)
143     void removeSession(LegacyCDMSession&);
144 #endif
145
146 #if ENABLE(ENCRYPTED_MEDIA)
147     void cdmInstanceAttached(CDMInstance&) final;
148     void cdmInstanceDetached(CDMInstance&) final;
149     void attemptToDecryptWithInstance(CDMInstance&) final;
150 #endif
151
152 private:
153     // engine support
154     static void getSupportedTypes(HashSet<String, ASCIICaseInsensitiveHash>& types);
155     static MediaPlayer::SupportsType supportsType(const MediaEngineSupportParameters&);
156     static bool supportsKeySystem(const String& keySystem, const String& mimeType);
157
158     static bool isAvailable();
159
160     void cancelLoad() override;
161
162     void platformSetVisible(bool) override;
163     void platformPlay() override;
164     void platformPause() override;
165     MediaTime currentMediaTime() const override;
166     void setVolume(float) override;
167 #if PLATFORM(IOS)
168     bool supportsMuting() const override { return true; }
169 #endif
170     void setMuted(bool) override;
171     void setClosedCaptionsVisible(bool) override;
172     void paint(GraphicsContext&, const FloatRect&) override;
173     void paintCurrentFrameInContext(GraphicsContext&, const FloatRect&) override;
174     PlatformLayer* platformLayer() const override;
175     void setVideoFullscreenLayer(PlatformLayer*, Function<void()>&& completionHandler) override;
176     void updateVideoFullscreenInlineImage() final;
177     void setVideoFullscreenFrame(FloatRect) override;
178     void setVideoFullscreenGravity(MediaPlayer::VideoGravity) override;
179     void setVideoFullscreenMode(MediaPlayer::VideoFullscreenMode) override;
180
181 #if PLATFORM(IOS)
182     NSArray *timedMetadata() const override;
183     String accessLog() const override;
184     String errorLog() const override;
185 #endif
186
187     bool supportsAcceleratedRendering() const override { return true; }
188     MediaTime mediaTimeForTimeValue(const MediaTime&) const override;
189     double maximumDurationToCacheMediaTime() const override;
190
191     void createAVPlayer() override;
192     void createAVPlayerItem() override;
193     virtual void createAVPlayerLayer();
194     void createAVAssetForURL(const URL&) override;
195     MediaPlayerPrivateAVFoundation::ItemStatus playerItemStatus() const override;
196     MediaPlayerPrivateAVFoundation::AssetStatus assetStatus() const override;
197     long assetErrorCode() const override;
198
199     double seekableTimeRangesLastModifiedTime() const override;
200     double liveUpdateInterval() const override;
201
202     void checkPlayability() override;
203     void setRateDouble(double) override;
204     double rate() const override;
205     void setPreservesPitch(bool) override;
206     void seekToTime(const MediaTime&, const MediaTime& negativeTolerance, const MediaTime& positiveTolerance) override;
207     unsigned long long totalBytes() const override;
208     std::unique_ptr<PlatformTimeRanges> platformBufferedTimeRanges() const override;
209     MediaTime platformMinTimeSeekable() const override;
210     MediaTime platformMaxTimeSeekable() const override;
211     MediaTime platformDuration() const override;
212     MediaTime platformMaxTimeLoaded() const override;
213     void beginLoadingMetadata() override;
214     void sizeChanged() override;
215     void resolvedURLChanged() override;
216
217     bool hasAvailableVideoFrame() const override;
218
219     void createContextVideoRenderer() override;
220     void destroyContextVideoRenderer() override;
221
222     void createVideoLayer() override;
223     void destroyVideoLayer() override;
224
225     bool hasContextRenderer() const override;
226     bool hasLayerRenderer() const override;
227
228     void updateVideoLayerGravity() override;
229
230     bool didPassCORSAccessCheck() const override;
231     std::optional<bool> wouldTaintOrigin(const SecurityOrigin&) const final;
232
233
234     MediaTime getStartDate() const override;
235
236 #if ENABLE(VIDEO_TRACK)
237     bool requiresTextTrackRepresentation() const override;
238     void setTextTrackRepresentation(TextTrackRepresentation*) override;
239     void syncTextTrackBounds() override;
240 #endif
241
242     void setAVPlayerItem(AVPlayerItem *);
243
244 #if ENABLE(WEB_AUDIO) && USE(MEDIATOOLBOX)
245     AudioSourceProvider* audioSourceProvider() override;
246 #endif
247
248     void createImageGenerator();
249     void destroyImageGenerator();
250     RetainPtr<CGImageRef> createImageForTimeInRect(float, const FloatRect&);
251     void paintWithImageGenerator(GraphicsContext&, const FloatRect&);
252
253 #if HAVE(AVFOUNDATION_VIDEO_OUTPUT)
254     enum class UpdateType { UpdateSynchronously, UpdateAsynchronously };
255     void updateLastImage(UpdateType type = UpdateType::UpdateAsynchronously);
256
257     void createVideoOutput();
258     void destroyVideoOutput();
259     bool updateLastPixelBuffer();
260     bool videoOutputHasAvailableFrame();
261     void paintWithVideoOutput(GraphicsContext&, const FloatRect&);
262     NativeImagePtr nativeImageForCurrentTime() override;
263     void waitForVideoOutputMediaDataWillChange();
264
265     bool copyVideoTextureToPlatformTexture(GraphicsContext3D*, Platform3DObject, GC3Denum target, GC3Dint level, GC3Denum internalFormat, GC3Denum format, GC3Denum type, bool premultiplyAlpha, bool flipY) override;
266 #endif
267
268 #if ENABLE(LEGACY_ENCRYPTED_MEDIA)
269     std::unique_ptr<LegacyCDMSession> createSession(const String& keySystem, LegacyCDMSessionClient*) override;
270 #endif
271
272     String languageOfPrimaryAudioTrack() const override;
273
274 #if HAVE(AVFOUNDATION_MEDIA_SELECTION_GROUP)
275     void processMediaSelectionOptions();
276     bool hasLoadedMediaSelectionGroups();
277
278     AVMediaSelectionGroup* safeMediaSelectionGroupForLegibleMedia();
279     AVMediaSelectionGroup* safeMediaSelectionGroupForAudibleMedia();
280     AVMediaSelectionGroup* safeMediaSelectionGroupForVisualMedia();
281 #endif
282
283     NSArray* safeAVAssetTracksForAudibleMedia();
284
285 #if ENABLE(DATACUE_VALUE)
286     void processMetadataTrack();
287 #endif
288
289     void setCurrentTextTrack(InbandTextTrackPrivateAVF*) override;
290     InbandTextTrackPrivateAVF* currentTextTrack() const override { return m_currentTextTrack; }
291
292 #if !HAVE(AVFOUNDATION_LEGIBLE_OUTPUT_SUPPORT)
293     void processLegacyClosedCaptionsTracks();
294 #endif
295
296 #if ENABLE(VIDEO_TRACK)
297     void updateAudioTracks();
298     void updateVideoTracks();
299 #endif
300
301 #if ENABLE(WIRELESS_PLAYBACK_TARGET)
302     bool isCurrentPlaybackTargetWireless() const override;
303     String wirelessPlaybackTargetName() const override;
304     MediaPlayer::WirelessPlaybackTargetType wirelessPlaybackTargetType() const override;
305     bool wirelessVideoPlaybackDisabled() const override;
306     void setWirelessVideoPlaybackDisabled(bool) override;
307     bool canPlayToWirelessPlaybackTarget() const override { return true; }
308     void updateDisableExternalPlayback();
309 #endif
310
311 #if ENABLE(WIRELESS_PLAYBACK_TARGET) && !PLATFORM(IOS)
312     void setWirelessPlaybackTarget(Ref<MediaPlaybackTarget>&&) override;
313     void setShouldPlayToPlaybackTarget(bool) override;
314 #endif
315
316     double maxFastForwardRate() const override { return m_cachedCanPlayFastForward ? std::numeric_limits<double>::infinity() : 2.0; }
317     double minFastReverseRate() const override { return m_cachedCanPlayFastReverse ? -std::numeric_limits<double>::infinity() : 0.0; }
318
319     Vector<String> preferredAudioCharacteristics() const;
320
321     void setShouldDisableSleep(bool) override;
322
323 #if !RELEASE_LOG_DISABLED
324     const char* logClassName() const final { return "MediaPlayerPrivateAVFoundationObjC"; }
325 #endif
326
327     AVPlayer *objCAVFoundationAVPlayer() const final { return m_avPlayer.get(); }
328
329     RetainPtr<AVURLAsset> m_avAsset;
330     RetainPtr<AVPlayer> m_avPlayer;
331     RetainPtr<AVPlayerItem> m_avPlayerItem;
332     RetainPtr<AVPlayerLayer> m_videoLayer;
333     std::unique_ptr<VideoFullscreenLayerManagerObjC> m_videoFullscreenLayerManager;
334     MediaPlayer::VideoGravity m_videoFullscreenGravity;
335     RetainPtr<WebCoreAVFMovieObserver> m_objcObserver;
336     RetainPtr<id> m_timeObserver;
337     mutable String m_languageOfPrimaryAudioTrack;
338     bool m_videoFrameHasDrawn;
339     bool m_haveCheckedPlayability;
340
341 #if ENABLE(WEB_AUDIO) && USE(MEDIATOOLBOX)
342     RefPtr<AudioSourceProviderAVFObjC> m_provider;
343 #endif
344
345     RetainPtr<AVAssetImageGenerator> m_imageGenerator;
346 #if HAVE(AVFOUNDATION_VIDEO_OUTPUT)
347     RetainPtr<AVPlayerItemVideoOutput> m_videoOutput;
348     RetainPtr<WebCoreAVFPullDelegate> m_videoOutputDelegate;
349     RetainPtr<CVPixelBufferRef> m_lastPixelBuffer;
350     RetainPtr<CGImageRef> m_lastImage;
351     BinarySemaphore m_videoOutputSemaphore;
352     std::unique_ptr<VideoTextureCopierCV> m_videoTextureCopier;
353 #endif
354
355 #if HAVE(CORE_VIDEO)
356     std::unique_ptr<PixelBufferConformerCV> m_pixelBufferConformer;
357 #endif
358
359 #if HAVE(AVFOUNDATION_LOADER_DELEGATE)
360     friend class WebCoreAVFResourceLoader;
361     HashMap<RetainPtr<CFTypeRef>, RefPtr<WebCoreAVFResourceLoader>> m_resourceLoaderMap;
362     RetainPtr<WebCoreAVFLoaderDelegate> m_loaderDelegate;
363     HashMap<String, RetainPtr<AVAssetResourceLoadingRequest>> m_keyURIToRequestMap;
364     HashMap<String, RetainPtr<AVAssetResourceLoadingRequest>> m_sessionIDToRequestMap;
365 #endif
366
367 #if HAVE(AVFOUNDATION_MEDIA_SELECTION_GROUP) && HAVE(AVFOUNDATION_LEGIBLE_OUTPUT_SUPPORT)
368     RetainPtr<AVPlayerItemLegibleOutput> m_legibleOutput;
369 #endif
370
371 #if ENABLE(VIDEO_TRACK)
372     Vector<RefPtr<AudioTrackPrivateAVFObjC>> m_audioTracks;
373     Vector<RefPtr<VideoTrackPrivateAVFObjC>> m_videoTracks;
374 #if HAVE(AVFOUNDATION_MEDIA_SELECTION_GROUP)
375     RefPtr<MediaSelectionGroupAVFObjC> m_audibleGroup;
376     RefPtr<MediaSelectionGroupAVFObjC> m_visualGroup;
377 #endif
378 #endif
379
380     InbandTextTrackPrivateAVF* m_currentTextTrack;
381
382 #if ENABLE(DATACUE_VALUE)
383     RefPtr<InbandMetadataTextTrackPrivateAVF> m_metadataTrack;
384 #endif
385
386 #if PLATFORM(MAC) && ENABLE(WIRELESS_PLAYBACK_TARGET)
387     RetainPtr<AVOutputContext> m_outputContext;
388     RefPtr<MediaPlaybackTarget> m_playbackTarget { nullptr };
389 #endif
390
391 #if ENABLE(LEGACY_ENCRYPTED_MEDIA)
392     WeakPtr<CDMSessionAVFoundationObjC> m_session;
393 #endif
394 #if ENABLE(ENCRYPTED_MEDIA) && HAVE(AVCONTENTKEYSESSION)
395     RefPtr<CDMInstanceFairPlayStreamingAVFObjC> m_cdmInstance;
396 #endif
397
398     mutable RetainPtr<NSArray> m_cachedSeekableRanges;
399     mutable RetainPtr<NSArray> m_cachedLoadedRanges;
400     RetainPtr<NSArray> m_cachedTracks;
401     RetainPtr<NSArray> m_currentMetaData;
402     FloatSize m_cachedPresentationSize;
403     MediaTime m_cachedDuration;
404     double m_cachedRate;
405     mutable long long m_cachedTotalBytes;
406     unsigned m_pendingStatusChanges;
407     int m_cachedItemStatus;
408     bool m_cachedLikelyToKeepUp;
409     bool m_cachedBufferEmpty;
410     bool m_cachedBufferFull;
411     bool m_cachedHasEnabledAudio;
412     bool m_shouldBufferData;
413     bool m_cachedIsReadyForDisplay;
414     bool m_haveBeenAskedToCreateLayer;
415     bool m_cachedCanPlayFastForward;
416     bool m_cachedCanPlayFastReverse;
417     bool m_muted { false };
418     mutable std::optional<bool> m_tracksArePlayable;
419 #if ENABLE(WIRELESS_PLAYBACK_TARGET)
420     mutable bool m_allowsWirelessVideoPlayback;
421     bool m_shouldPlayToPlaybackTarget { false };
422 #endif
423 };
424
425 }
426
427 #endif