[Mac] process raw VTT in-band captions
[WebKit-https.git] / Source / WebCore / platform / graphics / avfoundation / objc / MediaPlayerPrivateAVFoundationObjC.h
1 /*
2  * Copyright (C) 2011-2014 Apple Inc. All rights reserved.
3  *
4  * Redistribution and use in source and binary forms, with or without
5  * modification, are permitted provided that the following conditions
6  * are met:
7  * 1. Redistributions of source code must retain the above copyright
8  *    notice, this list of conditions and the following disclaimer.
9  * 2. Redistributions in binary form must reproduce the above copyright
10  *    notice, this list of conditions and the following disclaimer in the
11  *    documentation and/or other materials provided with the distribution.
12  *
13  * THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY
14  * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
15  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
16  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL APPLE INC. OR
17  * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
18  * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
19  * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
20  * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
21  * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
22  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
23  * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 
24  */
25
26 #ifndef MediaPlayerPrivateAVFoundationObjC_h
27 #define MediaPlayerPrivateAVFoundationObjC_h
28
29 #if ENABLE(VIDEO) && USE(AVFOUNDATION)
30
31 #include "MediaPlayerPrivateAVFoundation.h"
32 #include <wtf/HashMap.h>
33
34 OBJC_CLASS AVAssetImageGenerator;
35 OBJC_CLASS AVAssetResourceLoadingRequest;
36 OBJC_CLASS AVMediaSelectionGroup;
37 OBJC_CLASS AVPlayer;
38 OBJC_CLASS AVPlayerItem;
39 OBJC_CLASS AVPlayerItemLegibleOutput;
40 OBJC_CLASS AVPlayerItemTrack;
41 OBJC_CLASS AVPlayerItemVideoOutput;
42 OBJC_CLASS AVPlayerLayer;
43 OBJC_CLASS AVURLAsset;
44 OBJC_CLASS NSArray;
45 OBJC_CLASS NSURLAuthenticationChallenge;
46 OBJC_CLASS WebCoreAVFMovieObserver;
47 OBJC_CLASS WebCoreAVFPullDelegate;
48
49 typedef struct objc_object* id;
50
51 #if HAVE(AVFOUNDATION_LOADER_DELEGATE)
52 OBJC_CLASS WebCoreAVFLoaderDelegate;
53 OBJC_CLASS AVAssetResourceLoadingRequest;
54 #endif
55
56 typedef struct CGImage *CGImageRef;
57 typedef struct __CVBuffer *CVPixelBufferRef;
58 typedef struct OpaqueVTPixelTransferSession* VTPixelTransferSessionRef;
59
60 namespace WebCore {
61
62 class WebCoreAVFResourceLoader;
63 class InbandMetadataTextTrackPrivateAVF;
64 class InbandTextTrackPrivateAVFObjC;
65 class AudioTrackPrivateAVFObjC;
66 class VideoTrackPrivateAVFObjC;
67
68 class MediaPlayerPrivateAVFoundationObjC : public MediaPlayerPrivateAVFoundation {
69 public:
70     virtual ~MediaPlayerPrivateAVFoundationObjC();
71
72     static void registerMediaEngine(MediaEngineRegistrar);
73
74     void setAsset(id);
75     virtual void tracksChanged() override;
76
77 #if HAVE(AVFOUNDATION_MEDIA_SELECTION_GROUP)
78     RetainPtr<AVPlayerItem> playerItem() const { return m_avPlayerItem; }
79     void processCue(NSArray *, NSArray *, double);
80     void flushCues();
81 #endif
82     
83 #if HAVE(AVFOUNDATION_LOADER_DELEGATE)
84     bool shouldWaitForLoadingOfResource(AVAssetResourceLoadingRequest*);
85     bool shouldWaitForResponseToAuthenticationChallenge(NSURLAuthenticationChallenge*);
86     void didCancelLoadingRequest(AVAssetResourceLoadingRequest*);
87     void didStopLoadingRequest(AVAssetResourceLoadingRequest *);
88 #endif
89
90 #if ENABLE(ENCRYPTED_MEDIA) || ENABLE(ENCRYPTED_MEDIA_V2)
91     static bool extractKeyURIKeyIDAndCertificateFromInitData(Uint8Array* initData, String& keyURI, String& keyID, RefPtr<Uint8Array>& certificate);
92 #endif
93
94 #if ENABLE(ENCRYPTED_MEDIA_V2)
95     RetainPtr<AVAssetResourceLoadingRequest> takeRequestForKeyURI(const String&);
96 #endif
97
98     void playerItemStatusDidChange(int);
99     void playbackLikelyToKeepUpWillChange();
100     void playbackLikelyToKeepUpDidChange(bool);
101     void playbackBufferEmptyWillChange();
102     void playbackBufferEmptyDidChange(bool);
103     void playbackBufferFullWillChange();
104     void playbackBufferFullDidChange(bool);
105     void loadedTimeRangesDidChange(RetainPtr<NSArray>);
106     void seekableTimeRangesDidChange(RetainPtr<NSArray>);
107     void tracksDidChange(RetainPtr<NSArray>);
108     void hasEnabledAudioDidChange(bool);
109     void presentationSizeDidChange(FloatSize);
110     void durationDidChange(double);
111     void rateDidChange(double);
112     void metadataDidArrive(RetainPtr<NSArray>, double);
113     void firstFrameAvailableDidChange(bool);
114     void trackEnabledDidChange(bool);
115
116     virtual void setShouldBufferData(bool);
117
118 #if HAVE(AVFOUNDATION_VIDEO_OUTPUT)
119     void outputMediaDataWillChange(AVPlayerItemVideoOutput*);
120 #endif
121
122 #if ENABLE(IOS_AIRPLAY)
123     void playbackTargetIsWirelessDidChange();
124 #endif
125     
126 #if ENABLE(AVF_CAPTIONS)
127     virtual void notifyTrackModeChanged() override;
128     virtual void synchronizeTextTrackState() override;
129 #endif
130     
131     WeakPtr<MediaPlayerPrivateAVFoundationObjC> createWeakPtr() { return m_weakPtrFactory.createWeakPtr(); }
132
133 private:
134     MediaPlayerPrivateAVFoundationObjC(MediaPlayer*);
135
136     // engine support
137     static PassOwnPtr<MediaPlayerPrivateInterface> create(MediaPlayer*);
138     static void getSupportedTypes(HashSet<String>& types);
139     static MediaPlayer::SupportsType supportsType(const MediaEngineSupportParameters&);
140     static bool supportsKeySystem(const String& keySystem, const String& mimeType);
141
142     static bool isAvailable();
143
144     virtual void cancelLoad();
145
146     virtual PlatformMedia platformMedia() const;
147
148     virtual void platformSetVisible(bool);
149     virtual void platformPlay();
150     virtual void platformPause();
151     virtual float currentTime() const;
152     virtual void setVolume(float);
153     virtual void setClosedCaptionsVisible(bool);
154     virtual void paint(GraphicsContext*, const IntRect&);
155     virtual void paintCurrentFrameInContext(GraphicsContext*, const IntRect&);
156     virtual PlatformLayer* platformLayer() const;
157 #if PLATFORM(IOS)
158     virtual void setVideoFullscreenLayer(PlatformLayer*);
159     virtual void setVideoFullscreenFrame(FloatRect);
160     virtual void setVideoFullscreenGravity(MediaPlayer::VideoGravity);
161
162     virtual NSArray *timedMetadata() const override;
163     virtual String accessLog() const;
164     virtual String errorLog() const;
165 #endif
166
167     virtual bool supportsAcceleratedRendering() const { return true; }
168     virtual float mediaTimeForTimeValue(float) const;
169     virtual double maximumDurationToCacheMediaTime() const { return 5; }
170
171     virtual void createAVPlayer();
172     virtual void createAVPlayerItem();
173     virtual void createAVPlayerLayer();
174     virtual void createAVAssetForURL(const String& url);
175     virtual MediaPlayerPrivateAVFoundation::ItemStatus playerItemStatus() const;
176     virtual MediaPlayerPrivateAVFoundation::AssetStatus assetStatus() const;
177
178     virtual void checkPlayability();
179     virtual void updateRate();
180     virtual float rate() const;
181     virtual void seekToTime(double time, double negativeTolerance, double positiveTolerance);
182     virtual unsigned long long totalBytes() const;
183     virtual std::unique_ptr<PlatformTimeRanges> platformBufferedTimeRanges() const;
184     virtual double platformMinTimeSeekable() const;
185     virtual double platformMaxTimeSeekable() const;
186     virtual float platformDuration() const;
187     virtual float platformMaxTimeLoaded() const;
188     virtual void beginLoadingMetadata();
189     virtual void sizeChanged();
190
191     virtual bool hasAvailableVideoFrame() const;
192
193     virtual void createContextVideoRenderer();
194     virtual void destroyContextVideoRenderer();
195
196     virtual void createVideoLayer();
197     virtual void destroyVideoLayer();
198
199     virtual bool hasContextRenderer() const;
200     virtual bool hasLayerRenderer() const;
201
202     virtual void updateVideoLayerGravity() override;
203
204     virtual bool hasSingleSecurityOrigin() const;
205
206 #if ENABLE(VIDEO_TRACK)
207     virtual bool requiresTextTrackRepresentation() const override;
208     virtual void setTextTrackRepresentation(TextTrackRepresentation*) override;
209     virtual void syncTextTrackBounds() override;
210 #endif
211
212     void createImageGenerator();
213     void destroyImageGenerator();
214     RetainPtr<CGImageRef> createImageForTimeInRect(float, const IntRect&);
215     void paintWithImageGenerator(GraphicsContext*, const IntRect&);
216
217 #if HAVE(AVFOUNDATION_VIDEO_OUTPUT)
218     void createVideoOutput();
219     void destroyVideoOutput();
220     RetainPtr<CVPixelBufferRef> createPixelBuffer();
221     void updateLastImage();
222     bool videoOutputHasAvailableFrame();
223     void paintWithVideoOutput(GraphicsContext*, const IntRect&);
224     virtual PassNativeImagePtr nativeImageForCurrentTime() override;
225     void waitForVideoOutputMediaDataWillChange();
226 #endif
227
228 #if ENABLE(ENCRYPTED_MEDIA)
229     virtual MediaPlayer::MediaKeyException addKey(const String&, const unsigned char*, unsigned, const unsigned char*, unsigned, const String&);
230     virtual MediaPlayer::MediaKeyException generateKeyRequest(const String&, const unsigned char*, unsigned);
231     virtual MediaPlayer::MediaKeyException cancelKeyRequest(const String&, const String&);
232 #endif
233
234 #if ENABLE(ENCRYPTED_MEDIA_V2)
235     std::unique_ptr<CDMSession> createSession(const String& keySystem);
236 #endif
237
238     virtual String languageOfPrimaryAudioTrack() const override;
239
240 #if HAVE(AVFOUNDATION_MEDIA_SELECTION_GROUP)
241     void processMediaSelectionOptions();
242     AVMediaSelectionGroup* safeMediaSelectionGroupForLegibleMedia();
243 #endif
244
245 #if ENABLE(DATACUE_VALUE)
246     void processMetadataTrack();
247 #endif
248
249     virtual void setCurrentTextTrack(InbandTextTrackPrivateAVF*) override;
250     virtual InbandTextTrackPrivateAVF* currentTextTrack() const override { return m_currentTextTrack; }
251
252 #if !HAVE(AVFOUNDATION_LEGIBLE_OUTPUT_SUPPORT)
253     void processLegacyClosedCaptionsTracks();
254 #endif
255
256 #if ENABLE(VIDEO_TRACK)
257     void updateAudioTracks();
258     void updateVideoTracks();
259 #endif
260
261 #if ENABLE(IOS_AIRPLAY)
262     virtual bool isCurrentPlaybackTargetWireless() const override;
263     virtual String wirelessPlaybackTargetName() const override;
264     virtual MediaPlayer::WirelessPlaybackTargetType wirelessPlaybackTargetType() const override;
265     virtual bool wirelessVideoPlaybackDisabled() const override;
266     virtual void setWirelessVideoPlaybackDisabled(bool) override;
267     void updateDisableExternalPlayback();
268 #endif
269
270     WeakPtrFactory<MediaPlayerPrivateAVFoundationObjC> m_weakPtrFactory;
271
272     RetainPtr<AVURLAsset> m_avAsset;
273     RetainPtr<AVPlayer> m_avPlayer;
274     RetainPtr<AVPlayerItem> m_avPlayerItem;
275     RetainPtr<AVPlayerLayer> m_videoLayer;
276 #if PLATFORM(IOS)
277     RetainPtr<PlatformLayer> m_videoFullscreenLayer;
278     FloatRect m_videoFullscreenFrame;
279     MediaPlayer::VideoGravity m_videoFullscreenGravity;
280     RetainPtr<PlatformLayer> m_textTrackRepresentationLayer;
281 #endif
282     RetainPtr<WebCoreAVFMovieObserver> m_objcObserver;
283     RetainPtr<id> m_timeObserver;
284     mutable String m_languageOfPrimaryAudioTrack;
285     bool m_videoFrameHasDrawn;
286     bool m_haveCheckedPlayability;
287
288     RetainPtr<AVAssetImageGenerator> m_imageGenerator;
289 #if HAVE(AVFOUNDATION_VIDEO_OUTPUT)
290     RetainPtr<AVPlayerItemVideoOutput> m_videoOutput;
291     RetainPtr<WebCoreAVFPullDelegate> m_videoOutputDelegate;
292     RetainPtr<CGImageRef> m_lastImage;
293     dispatch_semaphore_t m_videoOutputSemaphore;
294 #endif
295
296 #if USE(VIDEOTOOLBOX)
297     RetainPtr<VTPixelTransferSessionRef> m_pixelTransferSession;
298 #endif
299
300 #if HAVE(AVFOUNDATION_LOADER_DELEGATE)
301     friend class WebCoreAVFResourceLoader;
302     HashMap<RetainPtr<AVAssetResourceLoadingRequest>, RefPtr<WebCoreAVFResourceLoader>> m_resourceLoaderMap;
303     RetainPtr<WebCoreAVFLoaderDelegate> m_loaderDelegate;
304     HashMap<String, RetainPtr<AVAssetResourceLoadingRequest>> m_keyURIToRequestMap;
305     HashMap<String, RetainPtr<AVAssetResourceLoadingRequest>> m_sessionIDToRequestMap;
306 #endif
307
308 #if HAVE(AVFOUNDATION_MEDIA_SELECTION_GROUP) && HAVE(AVFOUNDATION_LEGIBLE_OUTPUT_SUPPORT)
309     RetainPtr<AVPlayerItemLegibleOutput> m_legibleOutput;
310 #endif
311
312 #if ENABLE(VIDEO_TRACK)
313     Vector<RefPtr<AudioTrackPrivateAVFObjC>> m_audioTracks;
314     Vector<RefPtr<VideoTrackPrivateAVFObjC>> m_videoTracks;
315 #endif
316
317     InbandTextTrackPrivateAVF* m_currentTextTrack;
318
319 #if ENABLE(DATACUE_VALUE)
320     RefPtr<InbandMetadataTextTrackPrivateAVF> m_metadataTrack;
321 #endif
322
323     mutable RetainPtr<NSArray> m_cachedSeekableRanges;
324     mutable RetainPtr<NSArray> m_cachedLoadedRanges;
325     RetainPtr<NSArray> m_cachedTracks;
326     RetainPtr<NSArray> m_currentMetaData;
327     FloatSize m_cachedPresentationSize;
328     double m_cachedDuration;
329     double m_cachedRate;
330     mutable long long m_cachedTotalBytes;
331     unsigned m_pendingStatusChanges;
332     int m_cachedItemStatus;
333     bool m_cachedLikelyToKeepUp;
334     bool m_cachedBufferEmpty;
335     bool m_cachedBufferFull;
336     bool m_cachedHasEnabledAudio;
337     bool m_shouldBufferData;
338     bool m_cachedIsReadyForDisplay;
339     bool m_haveBeenAskedToCreateLayer;
340 #if ENABLE(IOS_AIRPLAY)
341     mutable bool m_allowsWirelessVideoPlayback;
342 #endif
343 };
344
345 }
346
347 #endif
348 #endif