2 * Copyright (C) 2007, 2008, 2009, 2010, 2011, 2012 Apple Inc. All rights reserved.
4 * Redistribution and use in source and binary forms, with or without
5 * modification, are permitted provided that the following conditions
7 * 1. Redistributions of source code must retain the above copyright
8 * notice, this list of conditions and the following disclaimer.
9 * 2. Redistributions in binary form must reproduce the above copyright
10 * notice, this list of conditions and the following disclaimer in the
11 * documentation and/or other materials provided with the distribution.
13 * THIS SOFTWARE IS PROVIDED BY APPLE COMPUTER, INC. ``AS IS'' AND ANY
14 * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
15 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
16 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE COMPUTER, INC. OR
17 * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
18 * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
19 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
20 * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
21 * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
22 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
23 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
26 #ifndef HTMLMediaElement_h
27 #define HTMLMediaElement_h
31 #include "HTMLElement.h"
32 #include "ActiveDOMObject.h"
33 #include "GenericEventQueue.h"
34 #include "MediaCanStartListener.h"
35 #include "MediaControllerInterface.h"
36 #include "MediaPlayer.h"
38 #if ENABLE(PLUGIN_PROXY_FOR_VIDEO)
39 #include "MediaPlayerProxy.h"
42 #if ENABLE(VIDEO_TRACK)
43 #include "PODIntervalTree.h"
44 #include "TextTrack.h"
45 #include "TextTrackCue.h"
51 class AudioSourceProvider;
52 class MediaElementAudioSourceNode;
55 class HTMLSourceElement;
56 class HTMLTrackElement;
57 class MediaController;
63 #if ENABLE(PLUGIN_PROXY_FOR_VIDEO)
67 class DisplaySleepDisabler;
70 #if ENABLE(VIDEO_TRACK)
71 typedef PODIntervalTree<double, TextTrackCue*> CueIntervalTree;
72 typedef Vector<CueIntervalTree::IntervalType> CueList;
75 // FIXME: The inheritance from MediaPlayerClient here should be private inheritance.
76 // But it can't be until the Chromium WebMediaPlayerClientImpl class is fixed so it
77 // no longer depends on typecasting a MediaPlayerClient to an HTMLMediaElement.
79 class HTMLMediaElement : public HTMLElement, public MediaPlayerClient, private MediaCanStartListener, public ActiveDOMObject, public MediaControllerInterface
80 #if ENABLE(VIDEO_TRACK)
81 , private TextTrackClient
85 MediaPlayer* player() const { return m_player.get(); }
87 virtual bool isVideo() const = 0;
88 virtual bool hasVideo() const { return false; }
89 virtual bool hasAudio() const;
91 void rewind(float timeDelta);
92 void returnToRealtime();
94 // Eventually overloaded in HTMLVideoElement
95 virtual bool supportsFullscreen() const { return false; };
97 virtual bool supportsSave() const;
98 virtual bool supportsScanning() const;
100 PlatformMedia platformMedia() const;
101 #if USE(ACCELERATED_COMPOSITING)
102 PlatformLayer* platformLayer() const;
106 MediaResource = 1 << 0,
107 TextTrackResource = 1 << 1
109 void scheduleLoad(LoadType);
111 MediaPlayer::MovieLoadType movieLoadType() const;
113 bool inActiveDocument() const { return m_inActiveDocument; }
117 PassRefPtr<MediaError> error() const;
120 void setSrc(const String&);
121 const KURL& currentSrc() const { return m_currentSrc; }
123 enum NetworkState { NETWORK_EMPTY, NETWORK_IDLE, NETWORK_LOADING, NETWORK_NO_SOURCE };
124 NetworkState networkState() const;
126 String preload() const;
127 void setPreload(const String&);
129 PassRefPtr<TimeRanges> buffered() const;
130 void load(ExceptionCode&);
131 String canPlayType(const String& mimeType) const;
134 ReadyState readyState() const;
135 bool seeking() const;
138 float currentTime() const;
139 void setCurrentTime(float, ExceptionCode&);
140 double initialTime() const;
141 float startTime() const;
142 float duration() const;
144 float defaultPlaybackRate() const;
145 void setDefaultPlaybackRate(float);
146 float playbackRate() const;
147 void setPlaybackRate(float);
148 void updatePlaybackRate();
149 bool webkitPreservesPitch() const;
150 void setWebkitPreservesPitch(bool);
151 PassRefPtr<TimeRanges> played();
152 PassRefPtr<TimeRanges> seekable() const;
154 bool autoplay() const;
155 void setAutoplay(bool b);
157 void setLoop(bool b);
162 bool webkitHasClosedCaptions() const;
163 bool webkitClosedCaptionsVisible() const;
164 void setWebkitClosedCaptionsVisible(bool);
166 #if ENABLE(MEDIA_STATISTICS)
168 unsigned webkitAudioDecodedByteCount() const;
169 unsigned webkitVideoDecodedByteCount() const;
172 #if ENABLE(MEDIA_SOURCE)
174 const KURL& webkitMediaSourceURL() const { return m_mediaSourceURL; }
175 void webkitSourceAppend(PassRefPtr<Uint8Array> data, ExceptionCode&);
176 enum EndOfStreamStatus { EOS_NO_ERROR, EOS_NETWORK_ERR, EOS_DECODE_ERR };
177 void webkitSourceEndOfStream(unsigned short, ExceptionCode&);
178 enum SourceState { SOURCE_CLOSED, SOURCE_OPEN, SOURCE_ENDED };
179 SourceState webkitSourceState() const;
180 void setSourceState(SourceState);
184 bool controls() const;
185 void setControls(bool);
186 float volume() const;
187 void setVolume(float, ExceptionCode&);
191 void togglePlayState();
192 void beginScrubbing();
195 bool canPlay() const;
197 float percentLoaded() const;
199 #if ENABLE(VIDEO_TRACK)
200 PassRefPtr<TextTrack> addTextTrack(const String& kind, const String& label, const String& language, ExceptionCode&);
201 PassRefPtr<TextTrack> addTextTrack(const String& kind, const String& label, ExceptionCode& ec) { return addTextTrack(kind, label, emptyString(), ec); }
202 PassRefPtr<TextTrack> addTextTrack(const String& kind, ExceptionCode& ec) { return addTextTrack(kind, emptyString(), emptyString(), ec); }
204 TextTrackList* textTracks();
205 CueList currentlyActiveCues() const { return m_currentlyActiveCues; }
207 virtual void didAddTrack(HTMLTrackElement*);
208 virtual void willRemoveTrack(HTMLTrackElement*);
211 enum GroupKind { CaptionsAndSubtitles, Description, Chapter, Metadata, Other };
213 TrackGroup(GroupKind kind)
221 Vector<HTMLTrackElement*> tracks;
222 HTMLTrackElement* visibleTrack;
223 HTMLTrackElement* defaultTrack;
228 void configureTextTrackGroupForLanguage(const TrackGroup&) const;
229 void configureNewTextTracks();
230 void configureTextTrackGroup(const TrackGroup&) const;
232 bool userIsInterestedInThisTrackKind(String) const;
233 bool textTracksAreReady() const;
234 void configureTextTrackDisplay();
237 virtual void textTrackReadyStateChanged(TextTrack*);
238 virtual void textTrackKindChanged(TextTrack*);
239 virtual void textTrackModeChanged(TextTrack*);
240 virtual void textTrackAddCues(TextTrack*, const TextTrackCueList*);
241 virtual void textTrackRemoveCues(TextTrack*, const TextTrackCueList*);
242 virtual void textTrackAddCue(TextTrack*, PassRefPtr<TextTrackCue>);
243 virtual void textTrackRemoveCue(TextTrack*, PassRefPtr<TextTrackCue>);
246 #if ENABLE(PLUGIN_PROXY_FOR_VIDEO)
247 void allocateMediaPlayerIfNecessary();
248 void setNeedWidgetUpdate(bool needWidgetUpdate) { m_needWidgetUpdate = needWidgetUpdate; }
249 void deliverNotification(MediaPlayerProxyNotificationType notification);
250 void setMediaPlayerProxy(WebMediaPlayerProxy* proxy);
251 void getPluginProxyParams(KURL& url, Vector<String>& names, Vector<String>& values);
252 void createMediaPlayerProxy();
253 void updateWidget(PluginCreationOption);
256 bool hasSingleSecurityOrigin() const { return !m_player || m_player->hasSingleSecurityOrigin(); }
258 bool isFullscreen() const;
259 void enterFullscreen();
260 void exitFullscreen();
262 bool hasClosedCaptions() const;
263 bool closedCaptionsVisible() const;
264 void setClosedCaptionsVisible(bool);
266 MediaControls* mediaControls();
268 void sourceWillBeRemoved(HTMLSourceElement*);
269 void sourceWasAdded(HTMLSourceElement*);
271 void privateBrowsingStateDidChange();
273 // Media cache management.
274 static void getSitesInMediaCache(Vector<String>&);
275 static void clearMediaCache();
276 static void clearMediaCacheForSite(const String&);
278 bool isPlaying() const { return m_playing; }
280 virtual bool hasPendingActivity() const;
282 #if ENABLE(WEB_AUDIO)
283 MediaElementAudioSourceNode* audioSourceNode() { return m_audioSourceNode; }
284 void setAudioSourceNode(MediaElementAudioSourceNode*);
286 AudioSourceProvider* audioSourceProvider();
289 enum InvalidURLAction { DoNothing, Complain };
290 bool isSafeToLoadURL(const KURL&, InvalidURLAction);
292 const String& mediaGroup() const;
293 void setMediaGroup(const String&);
295 MediaController* controller() const;
296 void setController(PassRefPtr<MediaController>);
298 virtual bool dispatchEvent(PassRefPtr<Event>);
301 HTMLMediaElement(const QualifiedName&, Document*, bool);
302 virtual ~HTMLMediaElement();
304 virtual void parseAttribute(Attribute*) OVERRIDE;
305 virtual void finishParsingChildren();
306 virtual bool isURLAttribute(Attribute*) const;
307 virtual void attach();
309 virtual void didMoveToNewDocument(Document* oldDocument) OVERRIDE;
311 enum DisplayMode { Unknown, None, Poster, PosterWaitingForVideo, Video };
312 DisplayMode displayMode() const { return m_displayMode; }
313 virtual void setDisplayMode(DisplayMode mode) { m_displayMode = mode; }
315 virtual bool isMediaElement() const { return true; }
317 // Restrictions to change default behaviors.
318 enum BehaviorRestrictionFlags {
320 RequireUserGestureForLoadRestriction = 1 << 0,
321 RequireUserGestureForRateChangeRestriction = 1 << 1,
322 RequireUserGestureForFullscreenRestriction = 1 << 2,
323 RequirePageConsentToLoadMediaRestriction = 1 << 3,
325 typedef unsigned BehaviorRestrictions;
327 bool userGestureRequiredForLoad() const { return m_restrictions & RequireUserGestureForLoadRestriction; }
328 bool userGestureRequiredForRateChange() const { return m_restrictions & RequireUserGestureForRateChangeRestriction; }
329 bool userGestureRequiredForFullscreen() const { return m_restrictions & RequireUserGestureForFullscreenRestriction; }
330 bool pageConsentRequiredForLoad() const { return m_restrictions & RequirePageConsentToLoadMediaRestriction; }
332 void addBehaviorRestriction(BehaviorRestrictions restriction) { m_restrictions |= restriction; }
333 void removeBehaviorRestriction(BehaviorRestrictions restriction) { m_restrictions &= ~restriction; }
336 void createMediaPlayer();
338 virtual bool supportsFocus() const;
339 virtual bool isMouseFocusable() const;
340 virtual bool rendererIsNeeded(const NodeRenderingContext&);
341 virtual RenderObject* createRenderer(RenderArena*, RenderStyle*);
342 virtual bool childShouldCreateRenderer(const NodeRenderingContext&) const OVERRIDE;
343 virtual void insertedIntoDocument();
344 virtual void removedFromDocument();
345 virtual void didRecalcStyle(StyleChange);
347 virtual void defaultEventHandler(Event*);
349 virtual void didBecomeFullscreenElement();
350 virtual void willStopBeingFullscreenElement();
352 // ActiveDOMObject functions.
353 virtual bool canSuspend() const;
354 virtual void suspend(ReasonForSuspension);
355 virtual void resume();
358 virtual void mediaVolumeDidChange();
360 virtual void updateDisplayState() { }
362 void setReadyState(MediaPlayer::ReadyState);
363 void setNetworkState(MediaPlayer::NetworkState);
365 virtual Document* mediaPlayerOwningDocument();
366 virtual void mediaPlayerNetworkStateChanged(MediaPlayer*);
367 virtual void mediaPlayerReadyStateChanged(MediaPlayer*);
368 virtual void mediaPlayerTimeChanged(MediaPlayer*);
369 virtual void mediaPlayerVolumeChanged(MediaPlayer*);
370 virtual void mediaPlayerMuteChanged(MediaPlayer*);
371 virtual void mediaPlayerDurationChanged(MediaPlayer*);
372 virtual void mediaPlayerRateChanged(MediaPlayer*);
373 virtual void mediaPlayerPlaybackStateChanged(MediaPlayer*);
374 virtual void mediaPlayerSawUnsupportedTracks(MediaPlayer*);
375 virtual void mediaPlayerResourceNotSupported(MediaPlayer*);
376 virtual void mediaPlayerRepaint(MediaPlayer*);
377 virtual void mediaPlayerSizeChanged(MediaPlayer*);
378 #if USE(ACCELERATED_COMPOSITING)
379 virtual bool mediaPlayerRenderingCanBeAccelerated(MediaPlayer*);
380 virtual void mediaPlayerRenderingModeChanged(MediaPlayer*);
382 virtual void mediaPlayerEngineUpdated(MediaPlayer*);
384 virtual void mediaPlayerFirstVideoFrameAvailable(MediaPlayer*);
385 virtual void mediaPlayerCharacteristicChanged(MediaPlayer*);
387 #if ENABLE(MEDIA_SOURCE)
388 virtual void mediaPlayerSourceOpened();
389 virtual String mediaPlayerSourceURL() const;
392 virtual String mediaPlayerReferrer() const OVERRIDE;
393 virtual String mediaPlayerUserAgent() const OVERRIDE;
395 void loadTimerFired(Timer<HTMLMediaElement>*);
396 void progressEventTimerFired(Timer<HTMLMediaElement>*);
397 void playbackProgressTimerFired(Timer<HTMLMediaElement>*);
398 void startPlaybackProgressTimer();
399 void startProgressEventTimer();
400 void stopPeriodicTimers();
402 void seek(float time, ExceptionCode&);
404 void checkIfSeekNeeded();
405 void addPlayedRange(float start, float end);
407 void scheduleTimeupdateEvent(bool periodicEvent);
408 void scheduleEvent(const AtomicString& eventName);
411 void selectMediaResource();
412 void loadResource(const KURL&, ContentType&);
413 void scheduleNextSourceChild();
414 void loadNextSourceChild();
415 void userCancelledLoad();
416 bool havePotentialSourceChild();
417 void noneSupported();
418 void mediaEngineError(PassRefPtr<MediaError> err);
419 void cancelPendingEventsAndCallbacks();
420 void waitForSourceChange();
421 void prepareToPlay();
423 KURL selectNextSourceChild(ContentType*, InvalidURLAction);
424 void mediaLoadingFailed(MediaPlayer::NetworkState);
426 #if ENABLE(VIDEO_TRACK)
427 void updateActiveTextTrackCues(float);
428 bool userIsInterestedInThisLanguage(const String&) const;
429 HTMLTrackElement* showingTrackWithSameKind(HTMLTrackElement*) const;
431 bool ignoreTrackDisplayUpdateRequests() const { return m_ignoreTrackDisplayUpdate > 0; }
432 void beginIgnoringTrackDisplayUpdateRequests() { ++m_ignoreTrackDisplayUpdate; }
433 void endIgnoringTrackDisplayUpdateRequests() { ASSERT(m_ignoreTrackDisplayUpdate); --m_ignoreTrackDisplayUpdate; }
436 // These "internal" functions do not check user gesture restrictions.
439 void pauseInternal();
441 void prepareForLoad();
442 void allowVideoRendering();
444 bool processingMediaPlayerCallback() const { return m_processingMediaPlayerCallback > 0; }
445 void beginProcessingMediaPlayerCallback() { ++m_processingMediaPlayerCallback; }
446 void endProcessingMediaPlayerCallback() { ASSERT(m_processingMediaPlayerCallback); --m_processingMediaPlayerCallback; }
449 void updatePlayState();
450 bool potentiallyPlaying() const;
451 bool endedPlayback() const;
452 bool stoppedDueToErrors() const;
453 bool pausedForUserInteraction() const;
454 bool couldPlayIfEnoughData() const;
456 float minTimeSeekable() const;
457 float maxTimeSeekable() const;
459 // Pauses playback without changing any states or generating events
460 void setPausedInternal(bool);
462 void setPlaybackRateInternal(float);
464 virtual void mediaCanStart();
466 void setShouldDelayLoadEvent(bool);
467 void invalidateCachedTime();
468 void refreshCachedTime() const;
470 bool hasMediaControls();
471 bool createMediaControls();
472 void configureMediaControls();
474 void prepareMediaFragmentURI();
475 void applyMediaFragmentURI();
477 virtual void* preDispatchEventHandler(Event*);
479 void changeNetworkStateFromLoadingToIdle();
481 void removeBehaviorsRestrictionsAfterFirstUserGesture();
483 #if ENABLE(MICRODATA)
484 virtual String itemValueText() const;
485 virtual void setItemValueText(const String&, ExceptionCode&);
488 void updateMediaController();
489 bool isBlocked() const;
490 bool isBlockedOnMediaController() const;
491 bool hasCurrentSrc() const { return !m_currentSrc.isEmpty(); }
492 bool isLiveStream() const { return movieLoadType() == MediaPlayer::LiveStream; }
493 bool isAutoplaying() const { return m_autoplaying; }
496 void updateDisableSleep();
497 bool shouldDisableSleep() const;
500 Timer<HTMLMediaElement> m_loadTimer;
501 Timer<HTMLMediaElement> m_progressEventTimer;
502 Timer<HTMLMediaElement> m_playbackProgressTimer;
503 RefPtr<TimeRanges> m_playedTimeRanges;
504 GenericEventQueue m_asyncEventQueue;
506 float m_playbackRate;
507 float m_defaultPlaybackRate;
508 bool m_webkitPreservesPitch;
509 NetworkState m_networkState;
510 ReadyState m_readyState;
511 ReadyState m_readyStateMaximum;
514 RefPtr<MediaError> m_error;
517 float m_lastSeekTime;
519 unsigned m_previousProgress;
520 double m_previousProgressTime;
522 // The last time a timeupdate event was sent (wall clock).
523 double m_lastTimeUpdateEventWallTime;
525 // The last time a timeupdate event was sent in movie time.
526 float m_lastTimeUpdateEventMovieTime;
529 enum LoadState { WaitingForSource, LoadingFromSrcAttr, LoadingFromSourceElement };
530 LoadState m_loadState;
531 HTMLSourceElement* m_currentSourceNode;
532 Node* m_nextChildNodeToConsider;
533 Node* sourceChildEndOfListValue() { return static_cast<Node*>(this); }
535 OwnPtr<MediaPlayer> m_player;
536 #if ENABLE(PLUGIN_PROXY_FOR_VIDEO)
537 RefPtr<Widget> m_proxyWidget;
540 BehaviorRestrictions m_restrictions;
542 MediaPlayer::Preload m_preload;
544 DisplayMode m_displayMode;
546 // Counter incremented while processing a callback from the media player, so we can avoid
547 // calling the media engine recursively.
548 int m_processingMediaPlayerCallback;
550 #if ENABLE(MEDIA_SOURCE)
551 KURL m_mediaSourceURL;
552 SourceState m_sourceState;
555 mutable float m_cachedTime;
556 mutable double m_cachedTimeWallClockUpdateTime;
557 mutable double m_minimumWallClockTimeToCacheMediaTime;
559 double m_fragmentStartTime;
560 double m_fragmentEndTime;
562 typedef unsigned PendingLoadFlags;
563 PendingLoadFlags m_pendingLoadFlags;
566 bool m_isWaitingUntilMediaCanStart : 1;
567 bool m_shouldDelayLoadEvent : 1;
568 bool m_haveFiredLoadedData : 1;
569 bool m_inActiveDocument : 1;
570 bool m_autoplaying : 1;
575 // data has not been loaded since sending a "stalled" event
576 bool m_sentStalledEvent : 1;
578 // time has not changed since sending an "ended" event
579 bool m_sentEndEvent : 1;
581 bool m_pausedInternal : 1;
583 // Not all media engines provide enough information about a file to be able to
584 // support progress events so setting m_sendProgressEvents disables them
585 bool m_sendProgressEvents : 1;
587 bool m_isFullscreen : 1;
588 bool m_closedCaptionsVisible : 1;
590 #if ENABLE(PLUGIN_PROXY_FOR_VIDEO)
591 bool m_needWidgetUpdate : 1;
594 bool m_dispatchingCanPlayEvent : 1;
595 bool m_loadInitiatedByUserGesture : 1;
596 bool m_completelyLoaded : 1;
597 bool m_havePreparedToPlay : 1;
598 bool m_parsingInProgress : 1;
600 #if ENABLE(VIDEO_TRACK)
601 bool m_tracksAreReady : 1;
602 bool m_haveVisibleTextTrack : 1;
603 float m_lastTextTrackUpdateTime;
605 RefPtr<TextTrackList> m_textTracks;
606 Vector<RefPtr<TextTrack> > m_textTracksWhenResourceSelectionBegan;
608 CueIntervalTree m_cueTree;
610 CueList m_currentlyActiveCues;
611 int m_ignoreTrackDisplayUpdate;
614 #if ENABLE(WEB_AUDIO)
615 // This is a weak reference, since m_audioSourceNode holds a reference to us.
616 // The value is set just after the MediaElementAudioSourceNode is created.
617 // The value is cleared in MediaElementAudioSourceNode::~MediaElementAudioSourceNode().
618 MediaElementAudioSourceNode* m_audioSourceNode;
622 friend class MediaController;
623 RefPtr<MediaController> m_mediaController;
626 OwnPtr<DisplaySleepDisabler> m_sleepDisabler;
630 #if ENABLE(VIDEO_TRACK)
632 // Template specializations required by PodIntervalTree in debug mode.
634 struct ValueToString<double> {
635 static String string(const double value)
637 return String::number(value);
642 struct ValueToString<TextTrackCue*> {
643 static String string(TextTrackCue* const& cue)
645 return String::format("%p id=%s interval=%f-->%f cue=%s)", cue, cue->id().utf8().data(), cue->startTime(), cue->endTime(), cue->text().utf8().data());