76aee725f964dce651a87b29b23637c9b22ace0f
[WebKit-https.git] / Source / WebCore / platform / graphics / gstreamer / mse / MediaPlayerPrivateGStreamerMSE.cpp
1 /*
2  * Copyright (C) 2007, 2009 Apple Inc.  All rights reserved.
3  * Copyright (C) 2007 Collabora Ltd.  All rights reserved.
4  * Copyright (C) 2007 Alp Toker <alp@atoker.com>
5  * Copyright (C) 2009 Gustavo Noronha Silva <gns@gnome.org>
6  * Copyright (C) 2009, 2010, 2011, 2012, 2013, 2016 Igalia S.L
7  * Copyright (C) 2015 Sebastian Dröge <sebastian@centricular.com>
8  * Copyright (C) 2015, 2016 Metrological Group B.V.
9  *
10  * This library is free software; you can redistribute it and/or
11  * modify it under the terms of the GNU Library General Public
12  * License as published by the Free Software Foundation; either
13  * version 2 of the License, or (at your option) any later version.
14  *
15  * This library is distributed in the hope that it will be useful,
16  * but WITHOUT ANY WARRANTY; without even the implied warranty of
17  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
18  * Library General Public License for more details.
19  *
20  * You should have received a copy of the GNU Library General Public License
21  * aint with this library; see the file COPYING.LIB.  If not, write to
22  * the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
23  * Boston, MA 02110-1301, USA.
24  */
25
26 #include "config.h"
27 #include "MediaPlayerPrivateGStreamerMSE.h"
28
29 #if ENABLE(VIDEO) && USE(GSTREAMER) && ENABLE(MEDIA_SOURCE)
30
31 #include "AppendPipeline.h"
32 #include "AudioTrackPrivateGStreamer.h"
33 #include "GStreamerUtilities.h"
34 #include "InbandTextTrackPrivateGStreamer.h"
35 #include "MIMETypeRegistry.h"
36 #include "MediaDescription.h"
37 #include "MediaPlayer.h"
38 #include "NotImplemented.h"
39 #include "SourceBufferPrivateGStreamer.h"
40 #include "TimeRanges.h"
41 #include "URL.h"
42 #include "VideoTrackPrivateGStreamer.h"
43
44 #include <gst/app/gstappsink.h>
45 #include <gst/app/gstappsrc.h>
46 #include <gst/gst.h>
47 #include <gst/pbutils/pbutils.h>
48 #include <gst/video/video.h>
49 #include <wtf/Condition.h>
50 #include <wtf/NeverDestroyed.h>
51
52 static const char* dumpReadyState(WebCore::MediaPlayer::ReadyState readyState)
53 {
54     switch (readyState) {
55     case WebCore::MediaPlayer::HaveNothing: return "HaveNothing";
56     case WebCore::MediaPlayer::HaveMetadata: return "HaveMetadata";
57     case WebCore::MediaPlayer::HaveCurrentData: return "HaveCurrentData";
58     case WebCore::MediaPlayer::HaveFutureData: return "HaveFutureData";
59     case WebCore::MediaPlayer::HaveEnoughData: return "HaveEnoughData";
60     default: return "(unknown)";
61     }
62 }
63
64 // Max interval in seconds to stay in the READY state on manual state change requests.
65 static const unsigned gReadyStateTimerInterval = 60;
66
67 GST_DEBUG_CATEGORY(webkit_mse_debug);
68 #define GST_CAT_DEFAULT webkit_mse_debug
69
70 namespace WebCore {
71
72 void MediaPlayerPrivateGStreamerMSE::registerMediaEngine(MediaEngineRegistrar registrar)
73 {
74     if (isAvailable()) {
75         registrar([](MediaPlayer* player) { return std::make_unique<MediaPlayerPrivateGStreamerMSE>(player); },
76             getSupportedTypes, supportsType, nullptr, nullptr, nullptr, supportsKeySystem);
77     }
78 }
79
80 bool initializeGStreamerAndRegisterWebKitMSEElement()
81 {
82     if (UNLIKELY(!initializeGStreamer()))
83         return false;
84
85     registerWebKitGStreamerElements();
86
87     GST_DEBUG_CATEGORY_INIT(webkit_mse_debug, "webkitmse", 0, "WebKit MSE media player");
88
89     GRefPtr<GstElementFactory> WebKitMediaSrcFactory = adoptGRef(gst_element_factory_find("webkitmediasrc"));
90     if (UNLIKELY(!WebKitMediaSrcFactory))
91         gst_element_register(nullptr, "webkitmediasrc", GST_RANK_PRIMARY + 100, WEBKIT_TYPE_MEDIA_SRC);
92     return true;
93 }
94
95 bool MediaPlayerPrivateGStreamerMSE::isAvailable()
96 {
97     if (UNLIKELY(!initializeGStreamerAndRegisterWebKitMSEElement()))
98         return false;
99
100     GRefPtr<GstElementFactory> factory = adoptGRef(gst_element_factory_find("playbin"));
101     return factory;
102 }
103
104 MediaPlayerPrivateGStreamerMSE::MediaPlayerPrivateGStreamerMSE(MediaPlayer* player)
105     : MediaPlayerPrivateGStreamer(player)
106 {
107     GST_TRACE("creating the player (%p)", this);
108 }
109
110 MediaPlayerPrivateGStreamerMSE::~MediaPlayerPrivateGStreamerMSE()
111 {
112     GST_TRACE("destroying the player (%p)", this);
113
114     for (auto iterator : m_appendPipelinesMap)
115         iterator.value->clearPlayerPrivate();
116
117     if (m_source) {
118         webKitMediaSrcSetMediaPlayerPrivate(WEBKIT_MEDIA_SRC(m_source.get()), nullptr);
119         g_signal_handlers_disconnect_by_data(m_source.get(), this);
120     }
121
122     if (m_playbackPipeline)
123         m_playbackPipeline->setWebKitMediaSrc(nullptr);
124 }
125
126 void MediaPlayerPrivateGStreamerMSE::load(const String& urlString)
127 {
128     if (!urlString.startsWith("mediasource")) {
129         // Properly fail so the global MediaPlayer tries to fallback to the next MediaPlayerPrivate.
130         m_networkState = MediaPlayer::FormatError;
131         m_player->networkStateChanged();
132         return;
133     }
134
135     if (UNLIKELY(!initializeGStreamerAndRegisterWebKitMSEElement()))
136         return;
137
138     if (!m_playbackPipeline)
139         m_playbackPipeline = PlaybackPipeline::create();
140
141     MediaPlayerPrivateGStreamer::load(urlString);
142 }
143
144 void MediaPlayerPrivateGStreamerMSE::load(const String& url, MediaSourcePrivateClient* mediaSource)
145 {
146     m_mediaSource = mediaSource;
147     load(String::format("mediasource%s", url.utf8().data()));
148 }
149
150 void MediaPlayerPrivateGStreamerMSE::pause()
151 {
152     m_paused = true;
153     MediaPlayerPrivateGStreamer::pause();
154 }
155
156 MediaTime MediaPlayerPrivateGStreamerMSE::durationMediaTime() const
157 {
158     if (UNLIKELY(!m_pipeline || m_errorOccured))
159         return MediaTime();
160
161     return m_mediaTimeDuration;
162 }
163
164 void MediaPlayerPrivateGStreamerMSE::seek(float time)
165 {
166     if (UNLIKELY(!m_pipeline || m_errorOccured))
167         return;
168
169     GST_INFO("[Seek] seek attempt to %f secs", time);
170
171     // Avoid useless seeking.
172     float current = currentMediaTime().toFloat();
173     if (time == current) {
174         if (!m_seeking)
175             timeChanged();
176         return;
177     }
178
179     if (isLiveStream())
180         return;
181
182     if (m_seeking && m_seekIsPending) {
183         m_seekTime = time;
184         return;
185     }
186
187     GST_DEBUG("Seeking from %f to %f seconds", current, time);
188
189     float prevSeekTime = m_seekTime;
190     m_seekTime = time;
191
192     if (!doSeek()) {
193         m_seekTime = prevSeekTime;
194         GST_WARNING("Seeking to %f failed", time);
195         return;
196     }
197
198     m_isEndReached = false;
199     GST_DEBUG("m_seeking=%s, m_seekTime=%f", m_seeking ? "true" : "false", m_seekTime);
200 }
201
202 void MediaPlayerPrivateGStreamerMSE::configurePlaySink()
203 {
204     MediaPlayerPrivateGStreamer::configurePlaySink();
205
206     GRefPtr<GstElement> playsink = adoptGRef(gst_bin_get_by_name(GST_BIN(m_pipeline.get()), "playsink"));
207     if (playsink) {
208         // The default value (0) means "send events to all the sinks", instead
209         // of "only to the first that returns true". This is needed for MSE seek.
210         g_object_set(G_OBJECT(playsink.get()), "send-event-mode", 0, nullptr);
211     }
212 }
213
214 bool MediaPlayerPrivateGStreamerMSE::changePipelineState(GstState newState)
215 {
216     if (seeking()) {
217         GST_DEBUG("Rejected state change to %s while seeking",
218             gst_element_state_get_name(newState));
219         return true;
220     }
221
222     return MediaPlayerPrivateGStreamer::changePipelineState(newState);
223 }
224
225 void MediaPlayerPrivateGStreamerMSE::notifySeekNeedsDataForTime(const MediaTime& seekTime)
226 {
227     // Reenqueue samples needed to resume playback in the new position.
228     m_mediaSource->seekToTime(seekTime);
229
230     GST_DEBUG("MSE seek to %f finished", seekTime.toDouble());
231
232     if (!m_gstSeekCompleted) {
233         m_gstSeekCompleted = true;
234         maybeFinishSeek();
235     }
236 }
237
238 bool MediaPlayerPrivateGStreamerMSE::doSeek(gint64, float, GstSeekFlags)
239 {
240     // Use doSeek() instead. If anybody is calling this version of doSeek(), something is wrong.
241     ASSERT_NOT_REACHED();
242     return false;
243 }
244
245 bool MediaPlayerPrivateGStreamerMSE::doSeek()
246 {
247     GstClockTime position = toGstClockTime(m_seekTime);
248     MediaTime seekTime = MediaTime::createWithDouble(m_seekTime);
249     double rate = m_player->rate();
250     GstSeekFlags seekType = static_cast<GstSeekFlags>(GST_SEEK_FLAG_FLUSH | GST_SEEK_FLAG_ACCURATE);
251
252     // Always move to seeking state to report correct 'currentTime' while pending for actual seek to complete.
253     m_seeking = true;
254
255     // Check if playback pipeline is ready for seek.
256     GstState state, newState;
257     GstStateChangeReturn getStateResult = gst_element_get_state(m_pipeline.get(), &state, &newState, 0);
258     if (getStateResult == GST_STATE_CHANGE_FAILURE || getStateResult == GST_STATE_CHANGE_NO_PREROLL) {
259         GST_DEBUG("[Seek] cannot seek, current state change is %s", gst_element_state_change_return_get_name(getStateResult));
260         webKitMediaSrcSetReadyForSamples(WEBKIT_MEDIA_SRC(m_source.get()), true);
261         m_seeking = false;
262         return false;
263     }
264     if ((getStateResult == GST_STATE_CHANGE_ASYNC
265         && !(state == GST_STATE_PLAYING && newState == GST_STATE_PAUSED))
266         || state < GST_STATE_PAUSED
267         || m_isEndReached
268         || !m_gstSeekCompleted) {
269         CString reason = "Unknown reason";
270         if (getStateResult == GST_STATE_CHANGE_ASYNC) {
271             reason = String::format("In async change %s --> %s",
272                 gst_element_state_get_name(state),
273                 gst_element_state_get_name(newState)).utf8();
274         } else if (state < GST_STATE_PAUSED)
275             reason = "State less than PAUSED";
276         else if (m_isEndReached)
277             reason = "End reached";
278         else if (!m_gstSeekCompleted)
279             reason = "Previous seek is not finished yet";
280
281         GST_DEBUG("[Seek] Delaying the seek: %s", reason.data());
282
283         m_seekIsPending = true;
284
285         if (m_isEndReached) {
286             GST_DEBUG("[Seek] reset pipeline");
287             m_resetPipeline = true;
288             m_seeking = false;
289             if (!changePipelineState(GST_STATE_PAUSED))
290                 loadingFailed(MediaPlayer::Empty);
291             else
292                 m_seeking = true;
293         }
294
295         return m_seeking;
296     }
297
298     // Stop accepting new samples until actual seek is finished.
299     webKitMediaSrcSetReadyForSamples(WEBKIT_MEDIA_SRC(m_source.get()), false);
300
301     // Correct seek time if it helps to fix a small gap.
302     if (!isTimeBuffered(seekTime)) {
303         // Look if a near future time (<0.1 sec.) is buffered and change the seek target time.
304         if (m_mediaSource) {
305             const MediaTime miniGap = MediaTime::createWithDouble(0.1);
306             MediaTime nearest = m_mediaSource->buffered()->nearest(seekTime);
307             if (nearest.isValid() && nearest > seekTime && (nearest - seekTime) <= miniGap && isTimeBuffered(nearest + miniGap)) {
308                 GST_DEBUG("[Seek] Changed the seek target time from %f to %f, a near point in the future", seekTime.toFloat(), nearest.toFloat());
309                 seekTime = nearest;
310             }
311         }
312     }
313
314     // Check if MSE has samples for requested time and defer actual seek if needed.
315     if (!isTimeBuffered(seekTime)) {
316         GST_DEBUG("[Seek] Delaying the seek: MSE is not ready");
317         GstStateChangeReturn setStateResult = gst_element_set_state(m_pipeline.get(), GST_STATE_PAUSED);
318         if (setStateResult == GST_STATE_CHANGE_FAILURE) {
319             GST_DEBUG("[Seek] Cannot seek, failed to pause playback pipeline.");
320             webKitMediaSrcSetReadyForSamples(WEBKIT_MEDIA_SRC(m_source.get()), true);
321             m_seeking = false;
322             return false;
323         }
324         m_readyState = MediaPlayer::HaveMetadata;
325         notifySeekNeedsDataForTime(seekTime);
326         ASSERT(!m_mseSeekCompleted);
327         return true;
328     }
329
330     // Complete previous MSE seek if needed.
331     if (!m_mseSeekCompleted) {
332         m_mediaSource->monitorSourceBuffers();
333         ASSERT(m_mseSeekCompleted);
334         // Note: seekCompleted will recursively call us.
335         return m_seeking;
336     }
337
338     GST_DEBUG("We can seek now");
339
340     gint64 startTime = position, endTime = GST_CLOCK_TIME_NONE;
341     if (rate < 0) {
342         startTime = 0;
343         endTime = position;
344     }
345
346     if (!rate)
347         rate = 1;
348
349     GST_DEBUG("Actual seek to %" GST_TIME_FORMAT ", end time:  %" GST_TIME_FORMAT ", rate: %f", GST_TIME_ARGS(startTime), GST_TIME_ARGS(endTime), rate);
350
351     // This will call notifySeekNeedsData() after some time to tell that the pipeline is ready for sample enqueuing.
352     webKitMediaSrcPrepareSeek(WEBKIT_MEDIA_SRC(m_source.get()), seekTime);
353
354     m_gstSeekCompleted = false;
355     if (!gst_element_seek(m_pipeline.get(), rate, GST_FORMAT_TIME, seekType, GST_SEEK_TYPE_SET, startTime, GST_SEEK_TYPE_SET, endTime)) {
356         webKitMediaSrcSetReadyForSamples(WEBKIT_MEDIA_SRC(m_source.get()), true);
357         m_seeking = false;
358         m_gstSeekCompleted = true;
359         GST_DEBUG("doSeek(): gst_element_seek() failed, returning false");
360         return false;
361     }
362
363     // The samples will be enqueued in notifySeekNeedsData().
364     GST_DEBUG("doSeek(): gst_element_seek() succeeded, returning true");
365     return true;
366 }
367
368 void MediaPlayerPrivateGStreamerMSE::maybeFinishSeek()
369 {
370     if (!m_seeking || !m_mseSeekCompleted || !m_gstSeekCompleted)
371         return;
372
373     GstState state, newState;
374     GstStateChangeReturn getStateResult = gst_element_get_state(m_pipeline.get(), &state, &newState, 0);
375
376     if (getStateResult == GST_STATE_CHANGE_ASYNC
377         && !(state == GST_STATE_PLAYING && newState == GST_STATE_PAUSED)) {
378         GST_DEBUG("[Seek] Delaying seek finish");
379         return;
380     }
381
382     if (m_seekIsPending) {
383         GST_DEBUG("[Seek] Committing pending seek to %f", m_seekTime);
384         m_seekIsPending = false;
385         if (!doSeek()) {
386             GST_WARNING("[Seek] Seeking to %f failed", m_seekTime);
387             m_cachedPosition = -1;
388         }
389         return;
390     }
391
392     GST_DEBUG("[Seek] Seeked to %f", m_seekTime);
393
394     webKitMediaSrcSetReadyForSamples(WEBKIT_MEDIA_SRC(m_source.get()), true);
395     m_seeking = false;
396     m_cachedPosition = -1;
397     // The pipeline can still have a pending state. In this case a position query will fail.
398     // Right now we can use m_seekTime as a fallback.
399     m_canFallBackToLastFinishedSeekPosition = true;
400     timeChanged();
401 }
402
403 void MediaPlayerPrivateGStreamerMSE::updatePlaybackRate()
404 {
405     notImplemented();
406 }
407
408 bool MediaPlayerPrivateGStreamerMSE::seeking() const
409 {
410     return m_seeking;
411 }
412
413 // FIXME: MediaPlayerPrivateGStreamer manages the ReadyState on its own. We shouldn't change it manually.
414 void MediaPlayerPrivateGStreamerMSE::setReadyState(MediaPlayer::ReadyState readyState)
415 {
416     if (readyState == m_readyState)
417         return;
418
419     if (seeking()) {
420         GST_DEBUG("Skip ready state change(%s -> %s) due to seek\n", dumpReadyState(m_readyState), dumpReadyState(readyState));
421         return;
422     }
423
424     GST_DEBUG("Ready State Changed manually from %u to %u", m_readyState, readyState);
425     MediaPlayer::ReadyState oldReadyState = m_readyState;
426     m_readyState = readyState;
427     GST_DEBUG("m_readyState: %s -> %s", dumpReadyState(oldReadyState), dumpReadyState(m_readyState));
428
429     if (oldReadyState < MediaPlayer::HaveCurrentData && m_readyState >= MediaPlayer::HaveCurrentData) {
430         GST_DEBUG("[Seek] Reporting load state changed to trigger seek continuation");
431         loadStateChanged();
432     }
433     m_player->readyStateChanged();
434
435     GstState pipelineState;
436     GstStateChangeReturn getStateResult = gst_element_get_state(m_pipeline.get(), &pipelineState, nullptr, 250 * GST_NSECOND);
437     bool isPlaying = (getStateResult == GST_STATE_CHANGE_SUCCESS && pipelineState == GST_STATE_PLAYING);
438
439     if (m_readyState == MediaPlayer::HaveMetadata && oldReadyState > MediaPlayer::HaveMetadata && isPlaying) {
440         GST_TRACE("Changing pipeline to PAUSED...");
441         bool ok = changePipelineState(GST_STATE_PAUSED);
442         GST_TRACE("Changed pipeline to PAUSED: %s", ok ? "Success" : "Error");
443     }
444 }
445
446 void MediaPlayerPrivateGStreamerMSE::waitForSeekCompleted()
447 {
448     if (!m_seeking)
449         return;
450
451     GST_DEBUG("Waiting for MSE seek completed");
452     m_mseSeekCompleted = false;
453 }
454
455 void MediaPlayerPrivateGStreamerMSE::seekCompleted()
456 {
457     if (m_mseSeekCompleted)
458         return;
459
460     GST_DEBUG("MSE seek completed");
461     m_mseSeekCompleted = true;
462
463     doSeek();
464
465     if (!seeking() && m_readyState >= MediaPlayer::HaveFutureData)
466         changePipelineState(GST_STATE_PLAYING);
467
468     if (!seeking())
469         m_player->timeChanged();
470 }
471
472 void MediaPlayerPrivateGStreamerMSE::setRate(float)
473 {
474     notImplemented();
475 }
476
477 std::unique_ptr<PlatformTimeRanges> MediaPlayerPrivateGStreamerMSE::buffered() const
478 {
479     return m_mediaSource->buffered();
480 }
481
482 void MediaPlayerPrivateGStreamerMSE::sourceChanged()
483 {
484     m_source = nullptr;
485     g_object_get(m_pipeline.get(), "source", &m_source.outPtr(), nullptr);
486
487     ASSERT(WEBKIT_IS_MEDIA_SRC(m_source.get()));
488
489     m_playbackPipeline->setWebKitMediaSrc(WEBKIT_MEDIA_SRC(m_source.get()));
490
491     MediaSourceGStreamer::open(*m_mediaSource.get(), *this);
492     g_signal_connect_swapped(m_source.get(), "video-changed", G_CALLBACK(videoChangedCallback), this);
493     g_signal_connect_swapped(m_source.get(), "audio-changed", G_CALLBACK(audioChangedCallback), this);
494     g_signal_connect_swapped(m_source.get(), "text-changed", G_CALLBACK(textChangedCallback), this);
495     webKitMediaSrcSetMediaPlayerPrivate(WEBKIT_MEDIA_SRC(m_source.get()), this);
496 }
497
498 void MediaPlayerPrivateGStreamerMSE::updateStates()
499 {
500     if (UNLIKELY(!m_pipeline || m_errorOccured))
501         return;
502
503     MediaPlayer::NetworkState oldNetworkState = m_networkState;
504     MediaPlayer::ReadyState oldReadyState = m_readyState;
505     GstState state, pending;
506
507     GstStateChangeReturn getStateResult = gst_element_get_state(m_pipeline.get(), &state, &pending, 250 * GST_NSECOND);
508
509     bool shouldUpdatePlaybackState = false;
510     switch (getStateResult) {
511     case GST_STATE_CHANGE_SUCCESS: {
512         GST_DEBUG("State: %s, pending: %s", gst_element_state_get_name(state), gst_element_state_get_name(pending));
513
514         // Do nothing if on EOS and state changed to READY to avoid recreating the player
515         // on HTMLMediaElement and properly generate the video 'ended' event.
516         if (m_isEndReached && state == GST_STATE_READY)
517             break;
518
519         m_resetPipeline = (state <= GST_STATE_READY);
520         if (m_resetPipeline)
521             m_mediaTimeDuration = MediaTime::zeroTime();
522
523         // Update ready and network states.
524         switch (state) {
525         case GST_STATE_NULL:
526             m_readyState = MediaPlayer::HaveNothing;
527             GST_DEBUG("m_readyState=%s", dumpReadyState(m_readyState));
528             m_networkState = MediaPlayer::Empty;
529             break;
530         case GST_STATE_READY:
531             m_readyState = MediaPlayer::HaveMetadata;
532             GST_DEBUG("m_readyState=%s", dumpReadyState(m_readyState));
533             m_networkState = MediaPlayer::Empty;
534             break;
535         case GST_STATE_PAUSED:
536         case GST_STATE_PLAYING:
537             if (seeking()) {
538                 m_readyState = MediaPlayer::HaveMetadata;
539                 // FIXME: Should we manage NetworkState too?
540                 GST_DEBUG("m_readyState=%s", dumpReadyState(m_readyState));
541             } else if (m_buffering) {
542                 if (m_bufferingPercentage == 100) {
543                     GST_DEBUG("[Buffering] Complete.");
544                     m_buffering = false;
545                     m_readyState = MediaPlayer::HaveEnoughData;
546                     GST_DEBUG("m_readyState=%s", dumpReadyState(m_readyState));
547                     m_networkState = m_downloadFinished ? MediaPlayer::Idle : MediaPlayer::Loading;
548                 } else {
549                     m_readyState = MediaPlayer::HaveCurrentData;
550                     GST_DEBUG("m_readyState=%s", dumpReadyState(m_readyState));
551                     m_networkState = MediaPlayer::Loading;
552                 }
553             } else if (m_downloadFinished) {
554                 m_readyState = MediaPlayer::HaveEnoughData;
555                 GST_DEBUG("m_readyState=%s", dumpReadyState(m_readyState));
556                 m_networkState = MediaPlayer::Loaded;
557             } else {
558                 m_readyState = MediaPlayer::HaveFutureData;
559                 GST_DEBUG("m_readyState=%s", dumpReadyState(m_readyState));
560                 m_networkState = MediaPlayer::Loading;
561             }
562
563             if (m_eosMarked && state == GST_STATE_PLAYING)
564                 m_eosPending = true;
565
566             break;
567         default:
568             ASSERT_NOT_REACHED();
569             break;
570         }
571
572         // Sync states where needed.
573         if (state == GST_STATE_PAUSED) {
574             if (!m_volumeAndMuteInitialized) {
575                 notifyPlayerOfVolumeChange();
576                 notifyPlayerOfMute();
577                 m_volumeAndMuteInitialized = true;
578             }
579
580             if (!seeking() && !m_buffering && !m_paused && m_playbackRate) {
581                 GST_DEBUG("[Buffering] Restarting playback.");
582                 changePipelineState(GST_STATE_PLAYING);
583             }
584         } else if (state == GST_STATE_PLAYING) {
585             m_paused = false;
586
587             if ((m_buffering && !isLiveStream()) || !m_playbackRate) {
588                 GST_DEBUG("[Buffering] Pausing stream for buffering.");
589                 changePipelineState(GST_STATE_PAUSED);
590             }
591         } else
592             m_paused = true;
593
594         if (m_requestedState == GST_STATE_PAUSED && state == GST_STATE_PAUSED) {
595             shouldUpdatePlaybackState = true;
596             GST_DEBUG("Requested state change to %s was completed", gst_element_state_get_name(state));
597         }
598
599         break;
600     }
601     case GST_STATE_CHANGE_ASYNC:
602         GST_DEBUG("Async: State: %s, pending: %s", gst_element_state_get_name(state), gst_element_state_get_name(pending));
603         // Change in progress.
604         break;
605     case GST_STATE_CHANGE_FAILURE:
606         GST_WARNING("Failure: State: %s, pending: %s", gst_element_state_get_name(state), gst_element_state_get_name(pending));
607         // Change failed.
608         return;
609     case GST_STATE_CHANGE_NO_PREROLL:
610         GST_DEBUG("No preroll: State: %s, pending: %s", gst_element_state_get_name(state), gst_element_state_get_name(pending));
611
612         // Live pipelines go in PAUSED without prerolling.
613         m_isStreaming = true;
614
615         if (state == GST_STATE_READY) {
616             m_readyState = MediaPlayer::HaveNothing;
617             GST_DEBUG("m_readyState=%s", dumpReadyState(m_readyState));
618         } else if (state == GST_STATE_PAUSED) {
619             m_readyState = MediaPlayer::HaveEnoughData;
620             GST_DEBUG("m_readyState=%s", dumpReadyState(m_readyState));
621             m_paused = true;
622         } else if (state == GST_STATE_PLAYING)
623             m_paused = false;
624
625         if (!m_paused && m_playbackRate)
626             changePipelineState(GST_STATE_PLAYING);
627
628         m_networkState = MediaPlayer::Loading;
629         break;
630     default:
631         GST_DEBUG("Else : %d", getStateResult);
632         break;
633     }
634
635     m_requestedState = GST_STATE_VOID_PENDING;
636
637     if (shouldUpdatePlaybackState)
638         m_player->playbackStateChanged();
639
640     if (m_networkState != oldNetworkState) {
641         GST_DEBUG("Network State Changed from %u to %u", oldNetworkState, m_networkState);
642         m_player->networkStateChanged();
643     }
644     if (m_readyState != oldReadyState) {
645         GST_DEBUG("Ready State Changed from %u to %u", oldReadyState, m_readyState);
646         m_player->readyStateChanged();
647     }
648
649     if (getStateResult == GST_STATE_CHANGE_SUCCESS && state >= GST_STATE_PAUSED) {
650         updatePlaybackRate();
651         maybeFinishSeek();
652     }
653 }
654 void MediaPlayerPrivateGStreamerMSE::asyncStateChangeDone()
655 {
656     if (UNLIKELY(!m_pipeline || m_errorOccured))
657         return;
658
659     if (m_seeking)
660         maybeFinishSeek();
661     else
662         updateStates();
663 }
664
665 bool MediaPlayerPrivateGStreamerMSE::isTimeBuffered(const MediaTime &time) const
666 {
667     bool result = m_mediaSource && m_mediaSource->buffered()->contain(time);
668     GST_DEBUG("Time %f buffered? %s", time.toDouble(), result ? "Yes" : "No");
669     return result;
670 }
671
672 void MediaPlayerPrivateGStreamerMSE::setMediaSourceClient(Ref<MediaSourceClientGStreamerMSE> client)
673 {
674     m_mediaSourceClient = client.ptr();
675 }
676
677 RefPtr<MediaSourceClientGStreamerMSE> MediaPlayerPrivateGStreamerMSE::mediaSourceClient()
678 {
679     return m_mediaSourceClient;
680 }
681
682 void MediaPlayerPrivateGStreamerMSE::durationChanged()
683 {
684     if (!m_mediaSourceClient) {
685         GST_DEBUG("m_mediaSourceClient is null, doing nothing");
686         return;
687     }
688
689     MediaTime previousDuration = m_mediaTimeDuration;
690     m_mediaTimeDuration = m_mediaSourceClient->duration();
691
692     GST_TRACE("previous=%f, new=%f", previousDuration.toFloat(), m_mediaTimeDuration.toFloat());
693
694     // Avoid emiting durationchanged in the case where the previous duration was 0 because that case is already handled
695     // by the HTMLMediaElement.
696     if (m_mediaTimeDuration != previousDuration && m_mediaTimeDuration.isValid() && previousDuration.isValid()) {
697         m_player->durationChanged();
698         m_playbackPipeline->notifyDurationChanged();
699         m_mediaSource->durationChanged(m_mediaTimeDuration);
700     }
701 }
702
703 static HashSet<String, ASCIICaseInsensitiveHash>& mimeTypeCache()
704 {
705     static NeverDestroyed<HashSet<String, ASCIICaseInsensitiveHash>> cache = []()
706     {
707         initializeGStreamerAndRegisterWebKitMSEElement();
708         HashSet<String, ASCIICaseInsensitiveHash> set;
709         const char* mimeTypes[] = {
710             "video/mp4",
711             "audio/mp4"
712         };
713         for (auto& type : mimeTypes)
714             set.add(type);
715         return set;
716     }();
717     return cache;
718 }
719
720 void MediaPlayerPrivateGStreamerMSE::getSupportedTypes(HashSet<String, ASCIICaseInsensitiveHash>& types)
721 {
722     types = mimeTypeCache();
723 }
724
725 void MediaPlayerPrivateGStreamerMSE::trackDetected(RefPtr<AppendPipeline> appendPipeline, RefPtr<WebCore::TrackPrivateBase> oldTrack, RefPtr<WebCore::TrackPrivateBase> newTrack)
726 {
727     ASSERT(appendPipeline->track() == newTrack);
728
729     GstCaps* caps = appendPipeline->appsinkCaps();
730     ASSERT(caps);
731     GST_DEBUG("track ID: %s, caps: %" GST_PTR_FORMAT, newTrack->id().string().latin1().data(), caps);
732
733     GstStructure* structure = gst_caps_get_structure(caps, 0);
734     const gchar* mediaType = gst_structure_get_name(structure);
735     GstVideoInfo info;
736
737     if (g_str_has_prefix(mediaType, "video/") && gst_video_info_from_caps(&info, caps)) {
738         float width, height;
739
740         width = info.width;
741         height = info.height * ((float) info.par_d / (float) info.par_n);
742         m_videoSize.setWidth(width);
743         m_videoSize.setHeight(height);
744     }
745
746     if (!oldTrack)
747         m_playbackPipeline->attachTrack(appendPipeline->sourceBufferPrivate(), newTrack, structure, caps);
748     else
749         m_playbackPipeline->reattachTrack(appendPipeline->sourceBufferPrivate(), newTrack);
750 }
751
752 MediaPlayer::SupportsType MediaPlayerPrivateGStreamerMSE::supportsType(const MediaEngineSupportParameters& parameters)
753 {
754     MediaPlayer::SupportsType result = MediaPlayer::IsNotSupported;
755     if (!parameters.isMediaSource)
756         return result;
757
758     // Disable VPX/Opus on MSE for now, mp4/avc1 seems way more reliable currently.
759     if (parameters.type.endsWith("webm"))
760         return result;
761
762     // YouTube TV provides empty types for some videos and we want to be selected as best media engine for them.
763     if (parameters.type.isEmpty()) {
764         result = MediaPlayer::MayBeSupported;
765         return result;
766     }
767
768     // Spec says we should not return "probably" if the codecs string is empty.
769     if (mimeTypeCache().contains(parameters.type))
770         result = parameters.codecs.isEmpty() ? MediaPlayer::MayBeSupported : MediaPlayer::IsSupported;
771
772     return extendedSupportsType(parameters, result);
773 }
774
775 #if ENABLE(LEGACY_ENCRYPTED_MEDIA)
776 void MediaPlayerPrivateGStreamerMSE::dispatchDecryptionKey(GstBuffer* buffer)
777 {
778     for (auto iterator : m_appendPipelinesMap) {
779         if (iterator.value->appendState() == AppendPipeline::AppendState::KeyNegotiation) {
780             GST_TRACE("append pipeline %p in key negotiation, setting key", iterator.value.get());
781             gst_element_send_event(iterator.value->pipeline(), gst_event_new_custom(GST_EVENT_CUSTOM_DOWNSTREAM_OOB,
782                 gst_structure_new("drm-cipher", "key", GST_TYPE_BUFFER, buffer, nullptr)));
783             iterator.value->setAppendState(AppendPipeline::AppendState::Ongoing);
784         } else
785             GST_TRACE("append pipeline %p not in key negotiation", iterator.value.get());
786     }
787 }
788 #endif
789
790 void MediaPlayerPrivateGStreamerMSE::markEndOfStream(MediaSourcePrivate::EndOfStreamStatus status)
791 {
792     if (status != MediaSourcePrivate::EosNoError)
793         return;
794
795     GST_DEBUG("Marking end of stream");
796     m_eosMarked = true;
797     updateStates();
798 }
799
800 MediaTime MediaPlayerPrivateGStreamerMSE::currentMediaTime() const
801 {
802     MediaTime position = MediaPlayerPrivateGStreamer::currentMediaTime();
803
804     if (m_eosPending && (paused() || (position >= durationMediaTime()))) {
805         if (m_networkState != MediaPlayer::Loaded) {
806             m_networkState = MediaPlayer::Loaded;
807             m_player->networkStateChanged();
808         }
809
810         m_eosPending = false;
811         m_isEndReached = true;
812         m_cachedPosition = m_mediaTimeDuration.toFloat();
813         m_durationAtEOS = m_mediaTimeDuration.toFloat();
814         m_player->timeChanged();
815     }
816     return position;
817 }
818
819 float MediaPlayerPrivateGStreamerMSE::maxTimeSeekable() const
820 {
821     if (UNLIKELY(m_errorOccured))
822         return 0;
823
824     GST_DEBUG("maxTimeSeekable");
825     float result = durationMediaTime().toFloat();
826     // Infinite duration means live stream.
827     if (std::isinf(result)) {
828         MediaTime maxBufferedTime = buffered()->maximumBufferedTime();
829         // Return the highest end time reported by the buffered attribute.
830         result = maxBufferedTime.isValid() ? maxBufferedTime.toFloat() : 0;
831     }
832
833     return result;
834 }
835
836 } // namespace WebCore.
837
838 #endif // USE(GSTREAMER)