REGRESSION(r207879-207891): [GStreamer] Introduced many layout test failures and...
[WebKit-https.git] / Source / WebCore / platform / graphics / gstreamer / mse / MediaPlayerPrivateGStreamerMSE.cpp
1 /*
2  * Copyright (C) 2007, 2009 Apple Inc.  All rights reserved.
3  * Copyright (C) 2007 Collabora Ltd.  All rights reserved.
4  * Copyright (C) 2007 Alp Toker <alp@atoker.com>
5  * Copyright (C) 2009 Gustavo Noronha Silva <gns@gnome.org>
6  * Copyright (C) 2009, 2010, 2011, 2012, 2013, 2016 Igalia S.L
7  * Copyright (C) 2015 Sebastian Dröge <sebastian@centricular.com>
8  * Copyright (C) 2015, 2016 Metrological Group B.V.
9  *
10  * This library is free software; you can redistribute it and/or
11  * modify it under the terms of the GNU Library General Public
12  * License as published by the Free Software Foundation; either
13  * version 2 of the License, or (at your option) any later version.
14  *
15  * This library is distributed in the hope that it will be useful,
16  * but WITHOUT ANY WARRANTY; without even the implied warranty of
17  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
18  * Library General Public License for more details.
19  *
20  * You should have received a copy of the GNU Library General Public License
21  * aint with this library; see the file COPYING.LIB.  If not, write to
22  * the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
23  * Boston, MA 02110-1301, USA.
24  */
25
26 #include "config.h"
27 #include "MediaPlayerPrivateGStreamerMSE.h"
28
29 #if ENABLE(VIDEO) && USE(GSTREAMER) && ENABLE(MEDIA_SOURCE)
30
31 #include "AppendPipeline.h"
32 #include "AudioTrackPrivateGStreamer.h"
33 #include "GStreamerUtilities.h"
34 #include "InbandTextTrackPrivateGStreamer.h"
35 #include "MIMETypeRegistry.h"
36 #include "MediaDescription.h"
37 #include "MediaPlayer.h"
38 #include "NotImplemented.h"
39 #include "SourceBufferPrivateGStreamer.h"
40 #include "TimeRanges.h"
41 #include "URL.h"
42 #include "VideoTrackPrivateGStreamer.h"
43
44 #include <fnmatch.h>
45 #include <gst/app/gstappsink.h>
46 #include <gst/app/gstappsrc.h>
47 #include <gst/gst.h>
48 #include <gst/pbutils/pbutils.h>
49 #include <gst/video/video.h>
50 #include <wtf/Condition.h>
51 #include <wtf/NeverDestroyed.h>
52
53 static const char* dumpReadyState(WebCore::MediaPlayer::ReadyState readyState)
54 {
55     switch (readyState) {
56     case WebCore::MediaPlayer::HaveNothing: return "HaveNothing";
57     case WebCore::MediaPlayer::HaveMetadata: return "HaveMetadata";
58     case WebCore::MediaPlayer::HaveCurrentData: return "HaveCurrentData";
59     case WebCore::MediaPlayer::HaveFutureData: return "HaveFutureData";
60     case WebCore::MediaPlayer::HaveEnoughData: return "HaveEnoughData";
61     default: return "(unknown)";
62     }
63 }
64
65 // Max interval in seconds to stay in the READY state on manual state change requests.
66 static const unsigned gReadyStateTimerInterval = 60;
67
68 GST_DEBUG_CATEGORY(webkit_mse_debug);
69 #define GST_CAT_DEFAULT webkit_mse_debug
70
71 namespace WebCore {
72
73 void MediaPlayerPrivateGStreamerMSE::registerMediaEngine(MediaEngineRegistrar registrar)
74 {
75     if (isAvailable()) {
76         registrar([](MediaPlayer* player) { return std::make_unique<MediaPlayerPrivateGStreamerMSE>(player); },
77             getSupportedTypes, supportsType, nullptr, nullptr, nullptr, supportsKeySystem);
78     }
79 }
80
81 bool initializeGStreamerAndRegisterWebKitMSEElement()
82 {
83     if (UNLIKELY(!initializeGStreamer()))
84         return false;
85
86     registerWebKitGStreamerElements();
87
88     GST_DEBUG_CATEGORY_INIT(webkit_mse_debug, "webkitmse", 0, "WebKit MSE media player");
89
90     GRefPtr<GstElementFactory> WebKitMediaSrcFactory = adoptGRef(gst_element_factory_find("webkitmediasrc"));
91     if (UNLIKELY(!WebKitMediaSrcFactory))
92         gst_element_register(nullptr, "webkitmediasrc", GST_RANK_PRIMARY + 100, WEBKIT_TYPE_MEDIA_SRC);
93     return true;
94 }
95
96 bool MediaPlayerPrivateGStreamerMSE::isAvailable()
97 {
98     if (UNLIKELY(!initializeGStreamerAndRegisterWebKitMSEElement()))
99         return false;
100
101     GRefPtr<GstElementFactory> factory = adoptGRef(gst_element_factory_find("playbin"));
102     return factory;
103 }
104
105 MediaPlayerPrivateGStreamerMSE::MediaPlayerPrivateGStreamerMSE(MediaPlayer* player)
106     : MediaPlayerPrivateGStreamer(player)
107 {
108     GST_TRACE("creating the player (%p)", this);
109 }
110
111 MediaPlayerPrivateGStreamerMSE::~MediaPlayerPrivateGStreamerMSE()
112 {
113     GST_TRACE("destroying the player (%p)", this);
114
115     for (auto iterator : m_appendPipelinesMap)
116         iterator.value->clearPlayerPrivate();
117
118     if (m_source) {
119         webKitMediaSrcSetMediaPlayerPrivate(WEBKIT_MEDIA_SRC(m_source.get()), nullptr);
120         g_signal_handlers_disconnect_by_data(m_source.get(), this);
121     }
122
123     if (m_playbackPipeline)
124         m_playbackPipeline->setWebKitMediaSrc(nullptr);
125 }
126
127 void MediaPlayerPrivateGStreamerMSE::load(const String& urlString)
128 {
129     if (!urlString.startsWith("mediasource")) {
130         // Properly fail so the global MediaPlayer tries to fallback to the next MediaPlayerPrivate.
131         m_networkState = MediaPlayer::FormatError;
132         m_player->networkStateChanged();
133         return;
134     }
135
136     if (UNLIKELY(!initializeGStreamerAndRegisterWebKitMSEElement()))
137         return;
138
139     if (!m_playbackPipeline)
140         m_playbackPipeline = PlaybackPipeline::create();
141
142     MediaPlayerPrivateGStreamer::load(urlString);
143 }
144
145 void MediaPlayerPrivateGStreamerMSE::load(const String& url, MediaSourcePrivateClient* mediaSource)
146 {
147     m_mediaSource = mediaSource;
148     load(String::format("mediasource%s", url.utf8().data()));
149 }
150
151 void MediaPlayerPrivateGStreamerMSE::pause()
152 {
153     m_paused = true;
154     MediaPlayerPrivateGStreamer::pause();
155 }
156
157 MediaTime MediaPlayerPrivateGStreamerMSE::durationMediaTime() const
158 {
159     if (UNLIKELY(!m_pipeline || m_errorOccured))
160         return MediaTime();
161
162     return m_mediaTimeDuration;
163 }
164
165 void MediaPlayerPrivateGStreamerMSE::seek(float time)
166 {
167     if (UNLIKELY(!m_pipeline || m_errorOccured))
168         return;
169
170     GST_INFO("[Seek] seek attempt to %f secs", time);
171
172     // Avoid useless seeking.
173     float current = currentMediaTime().toFloat();
174     if (time == current) {
175         if (!m_seeking)
176             timeChanged();
177         return;
178     }
179
180     if (isLiveStream())
181         return;
182
183     if (m_seeking && m_seekIsPending) {
184         m_seekTime = time;
185         return;
186     }
187
188     GST_DEBUG("Seeking from %f to %f seconds", current, time);
189
190     float prevSeekTime = m_seekTime;
191     m_seekTime = time;
192
193     if (!doSeek()) {
194         m_seekTime = prevSeekTime;
195         GST_WARNING("Seeking to %f failed", time);
196         return;
197     }
198
199     m_isEndReached = false;
200     GST_DEBUG("m_seeking=%s, m_seekTime=%f", m_seeking ? "true" : "false", m_seekTime);
201 }
202
203 void MediaPlayerPrivateGStreamerMSE::configurePlaySink()
204 {
205     MediaPlayerPrivateGStreamer::configurePlaySink();
206
207     GRefPtr<GstElement> playsink = adoptGRef(gst_bin_get_by_name(GST_BIN(m_pipeline.get()), "playsink"));
208     if (playsink) {
209         // The default value (0) means "send events to all the sinks", instead
210         // of "only to the first that returns true". This is needed for MSE seek.
211         g_object_set(G_OBJECT(playsink.get()), "send-event-mode", 0, nullptr);
212     }
213 }
214
215 bool MediaPlayerPrivateGStreamerMSE::changePipelineState(GstState newState)
216 {
217     if (seeking()) {
218         GST_DEBUG("Rejected state change to %s while seeking",
219             gst_element_state_get_name(newState));
220         return true;
221     }
222
223     return MediaPlayerPrivateGStreamer::changePipelineState(newState);
224 }
225
226 void MediaPlayerPrivateGStreamerMSE::notifySeekNeedsDataForTime(const MediaTime& seekTime)
227 {
228     // Reenqueue samples needed to resume playback in the new position.
229     m_mediaSource->seekToTime(seekTime);
230
231     GST_DEBUG("MSE seek to %f finished", seekTime.toDouble());
232
233     if (!m_gstSeekCompleted) {
234         m_gstSeekCompleted = true;
235         maybeFinishSeek();
236     }
237 }
238
239 bool MediaPlayerPrivateGStreamerMSE::doSeek(gint64, float, GstSeekFlags)
240 {
241     // Use doSeek() instead. If anybody is calling this version of doSeek(), something is wrong.
242     ASSERT_NOT_REACHED();
243     return false;
244 }
245
246 bool MediaPlayerPrivateGStreamerMSE::doSeek()
247 {
248     GstClockTime position = toGstClockTime(m_seekTime);
249     MediaTime seekTime = MediaTime::createWithDouble(m_seekTime);
250     double rate = m_player->rate();
251     GstSeekFlags seekType = static_cast<GstSeekFlags>(GST_SEEK_FLAG_FLUSH | GST_SEEK_FLAG_ACCURATE);
252
253     // Always move to seeking state to report correct 'currentTime' while pending for actual seek to complete.
254     m_seeking = true;
255
256     // Check if playback pipeline is ready for seek.
257     GstState state, newState;
258     GstStateChangeReturn getStateResult = gst_element_get_state(m_pipeline.get(), &state, &newState, 0);
259     if (getStateResult == GST_STATE_CHANGE_FAILURE || getStateResult == GST_STATE_CHANGE_NO_PREROLL) {
260         GST_DEBUG("[Seek] cannot seek, current state change is %s", gst_element_state_change_return_get_name(getStateResult));
261         webKitMediaSrcSetReadyForSamples(WEBKIT_MEDIA_SRC(m_source.get()), true);
262         m_seeking = false;
263         return false;
264     }
265     if ((getStateResult == GST_STATE_CHANGE_ASYNC
266         && !(state == GST_STATE_PLAYING && newState == GST_STATE_PAUSED))
267         || state < GST_STATE_PAUSED
268         || m_isEndReached
269         || !m_gstSeekCompleted) {
270         CString reason = "Unknown reason";
271         if (getStateResult == GST_STATE_CHANGE_ASYNC) {
272             reason = String::format("In async change %s --> %s",
273                 gst_element_state_get_name(state),
274                 gst_element_state_get_name(newState)).utf8();
275         } else if (state < GST_STATE_PAUSED)
276             reason = "State less than PAUSED";
277         else if (m_isEndReached)
278             reason = "End reached";
279         else if (!m_gstSeekCompleted)
280             reason = "Previous seek is not finished yet";
281
282         GST_DEBUG("[Seek] Delaying the seek: %s", reason.data());
283
284         m_seekIsPending = true;
285
286         if (m_isEndReached) {
287             GST_DEBUG("[Seek] reset pipeline");
288             m_resetPipeline = true;
289             m_seeking = false;
290             if (!changePipelineState(GST_STATE_PAUSED))
291                 loadingFailed(MediaPlayer::Empty);
292             else
293                 m_seeking = true;
294         }
295
296         return m_seeking;
297     }
298
299     // Stop accepting new samples until actual seek is finished.
300     webKitMediaSrcSetReadyForSamples(WEBKIT_MEDIA_SRC(m_source.get()), false);
301
302     // Correct seek time if it helps to fix a small gap.
303     if (!isTimeBuffered(seekTime)) {
304         // Look if a near future time (<0.1 sec.) is buffered and change the seek target time.
305         if (m_mediaSource) {
306             const MediaTime miniGap = MediaTime::createWithDouble(0.1);
307             MediaTime nearest = m_mediaSource->buffered()->nearest(seekTime);
308             if (nearest.isValid() && nearest > seekTime && (nearest - seekTime) <= miniGap && isTimeBuffered(nearest + miniGap)) {
309                 GST_DEBUG("[Seek] Changed the seek target time from %f to %f, a near point in the future", seekTime.toFloat(), nearest.toFloat());
310                 seekTime = nearest;
311             }
312         }
313     }
314
315     // Check if MSE has samples for requested time and defer actual seek if needed.
316     if (!isTimeBuffered(seekTime)) {
317         GST_DEBUG("[Seek] Delaying the seek: MSE is not ready");
318         GstStateChangeReturn setStateResult = gst_element_set_state(m_pipeline.get(), GST_STATE_PAUSED);
319         if (setStateResult == GST_STATE_CHANGE_FAILURE) {
320             GST_DEBUG("[Seek] Cannot seek, failed to pause playback pipeline.");
321             webKitMediaSrcSetReadyForSamples(WEBKIT_MEDIA_SRC(m_source.get()), true);
322             m_seeking = false;
323             return false;
324         }
325         m_readyState = MediaPlayer::HaveMetadata;
326         notifySeekNeedsDataForTime(seekTime);
327         ASSERT(!m_mseSeekCompleted);
328         return true;
329     }
330
331     // Complete previous MSE seek if needed.
332     if (!m_mseSeekCompleted) {
333         m_mediaSource->monitorSourceBuffers();
334         ASSERT(m_mseSeekCompleted);
335         // Note: seekCompleted will recursively call us.
336         return m_seeking;
337     }
338
339     GST_DEBUG("We can seek now");
340
341     gint64 startTime = position, endTime = GST_CLOCK_TIME_NONE;
342     if (rate < 0) {
343         startTime = 0;
344         endTime = position;
345     }
346
347     if (!rate)
348         rate = 1;
349
350     GST_DEBUG("Actual seek to %" GST_TIME_FORMAT ", end time:  %" GST_TIME_FORMAT ", rate: %f", GST_TIME_ARGS(startTime), GST_TIME_ARGS(endTime), rate);
351
352     // This will call notifySeekNeedsData() after some time to tell that the pipeline is ready for sample enqueuing.
353     webKitMediaSrcPrepareSeek(WEBKIT_MEDIA_SRC(m_source.get()), seekTime);
354
355     m_gstSeekCompleted = false;
356     if (!gst_element_seek(m_pipeline.get(), rate, GST_FORMAT_TIME, seekType, GST_SEEK_TYPE_SET, startTime, GST_SEEK_TYPE_SET, endTime)) {
357         webKitMediaSrcSetReadyForSamples(WEBKIT_MEDIA_SRC(m_source.get()), true);
358         m_seeking = false;
359         m_gstSeekCompleted = true;
360         GST_DEBUG("doSeek(): gst_element_seek() failed, returning false");
361         return false;
362     }
363
364     // The samples will be enqueued in notifySeekNeedsData().
365     GST_DEBUG("doSeek(): gst_element_seek() succeeded, returning true");
366     return true;
367 }
368
369 void MediaPlayerPrivateGStreamerMSE::maybeFinishSeek()
370 {
371     if (!m_seeking || !m_mseSeekCompleted || !m_gstSeekCompleted)
372         return;
373
374     GstState state, newState;
375     GstStateChangeReturn getStateResult = gst_element_get_state(m_pipeline.get(), &state, &newState, 0);
376
377     if (getStateResult == GST_STATE_CHANGE_ASYNC
378         && !(state == GST_STATE_PLAYING && newState == GST_STATE_PAUSED)) {
379         GST_DEBUG("[Seek] Delaying seek finish");
380         return;
381     }
382
383     if (m_seekIsPending) {
384         GST_DEBUG("[Seek] Committing pending seek to %f", m_seekTime);
385         m_seekIsPending = false;
386         if (!doSeek()) {
387             GST_WARNING("[Seek] Seeking to %f failed", m_seekTime);
388             m_cachedPosition = -1;
389         }
390         return;
391     }
392
393     GST_DEBUG("[Seek] Seeked to %f", m_seekTime);
394
395     webKitMediaSrcSetReadyForSamples(WEBKIT_MEDIA_SRC(m_source.get()), true);
396     m_seeking = false;
397     m_cachedPosition = -1;
398     // The pipeline can still have a pending state. In this case a position query will fail.
399     // Right now we can use m_seekTime as a fallback.
400     m_canFallBackToLastFinishedSeekPosition = true;
401     timeChanged();
402 }
403
404 void MediaPlayerPrivateGStreamerMSE::updatePlaybackRate()
405 {
406     notImplemented();
407 }
408
409 bool MediaPlayerPrivateGStreamerMSE::seeking() const
410 {
411     return m_seeking;
412 }
413
414 // FIXME: MediaPlayerPrivateGStreamer manages the ReadyState on its own. We shouldn't change it manually.
415 void MediaPlayerPrivateGStreamerMSE::setReadyState(MediaPlayer::ReadyState readyState)
416 {
417     if (readyState == m_readyState)
418         return;
419
420     if (seeking()) {
421         GST_DEBUG("Skip ready state change(%s -> %s) due to seek\n", dumpReadyState(m_readyState), dumpReadyState(readyState));
422         return;
423     }
424
425     GST_DEBUG("Ready State Changed manually from %u to %u", m_readyState, readyState);
426     MediaPlayer::ReadyState oldReadyState = m_readyState;
427     m_readyState = readyState;
428     GST_DEBUG("m_readyState: %s -> %s", dumpReadyState(oldReadyState), dumpReadyState(m_readyState));
429
430     if (oldReadyState < MediaPlayer::HaveCurrentData && m_readyState >= MediaPlayer::HaveCurrentData) {
431         GST_DEBUG("[Seek] Reporting load state changed to trigger seek continuation");
432         loadStateChanged();
433     }
434     m_player->readyStateChanged();
435
436     GstState pipelineState;
437     GstStateChangeReturn getStateResult = gst_element_get_state(m_pipeline.get(), &pipelineState, nullptr, 250 * GST_NSECOND);
438     bool isPlaying = (getStateResult == GST_STATE_CHANGE_SUCCESS && pipelineState == GST_STATE_PLAYING);
439
440     if (m_readyState == MediaPlayer::HaveMetadata && oldReadyState > MediaPlayer::HaveMetadata && isPlaying) {
441         GST_TRACE("Changing pipeline to PAUSED...");
442         bool ok = changePipelineState(GST_STATE_PAUSED);
443         GST_TRACE("Changed pipeline to PAUSED: %s", ok ? "Success" : "Error");
444     }
445 }
446
447 void MediaPlayerPrivateGStreamerMSE::waitForSeekCompleted()
448 {
449     if (!m_seeking)
450         return;
451
452     GST_DEBUG("Waiting for MSE seek completed");
453     m_mseSeekCompleted = false;
454 }
455
456 void MediaPlayerPrivateGStreamerMSE::seekCompleted()
457 {
458     if (m_mseSeekCompleted)
459         return;
460
461     GST_DEBUG("MSE seek completed");
462     m_mseSeekCompleted = true;
463
464     doSeek();
465
466     if (!seeking() && m_readyState >= MediaPlayer::HaveFutureData)
467         changePipelineState(GST_STATE_PLAYING);
468
469     if (!seeking())
470         m_player->timeChanged();
471 }
472
473 void MediaPlayerPrivateGStreamerMSE::setRate(float)
474 {
475     notImplemented();
476 }
477
478 std::unique_ptr<PlatformTimeRanges> MediaPlayerPrivateGStreamerMSE::buffered() const
479 {
480     return m_mediaSource->buffered();
481 }
482
483 void MediaPlayerPrivateGStreamerMSE::sourceChanged()
484 {
485     m_source = nullptr;
486     g_object_get(m_pipeline.get(), "source", &m_source.outPtr(), nullptr);
487
488     ASSERT(WEBKIT_IS_MEDIA_SRC(m_source.get()));
489
490     m_playbackPipeline->setWebKitMediaSrc(WEBKIT_MEDIA_SRC(m_source.get()));
491
492     MediaSourceGStreamer::open(*m_mediaSource.get(), *this);
493     g_signal_connect_swapped(m_source.get(), "video-changed", G_CALLBACK(videoChangedCallback), this);
494     g_signal_connect_swapped(m_source.get(), "audio-changed", G_CALLBACK(audioChangedCallback), this);
495     g_signal_connect_swapped(m_source.get(), "text-changed", G_CALLBACK(textChangedCallback), this);
496     webKitMediaSrcSetMediaPlayerPrivate(WEBKIT_MEDIA_SRC(m_source.get()), this);
497 }
498
499 void MediaPlayerPrivateGStreamerMSE::updateStates()
500 {
501     if (UNLIKELY(!m_pipeline || m_errorOccured))
502         return;
503
504     MediaPlayer::NetworkState oldNetworkState = m_networkState;
505     MediaPlayer::ReadyState oldReadyState = m_readyState;
506     GstState state, pending;
507
508     GstStateChangeReturn getStateResult = gst_element_get_state(m_pipeline.get(), &state, &pending, 250 * GST_NSECOND);
509
510     bool shouldUpdatePlaybackState = false;
511     switch (getStateResult) {
512     case GST_STATE_CHANGE_SUCCESS: {
513         GST_DEBUG("State: %s, pending: %s", gst_element_state_get_name(state), gst_element_state_get_name(pending));
514
515         // Do nothing if on EOS and state changed to READY to avoid recreating the player
516         // on HTMLMediaElement and properly generate the video 'ended' event.
517         if (m_isEndReached && state == GST_STATE_READY)
518             break;
519
520         m_resetPipeline = (state <= GST_STATE_READY);
521         if (m_resetPipeline)
522             m_mediaTimeDuration = MediaTime::zeroTime();
523
524         // Update ready and network states.
525         switch (state) {
526         case GST_STATE_NULL:
527             m_readyState = MediaPlayer::HaveNothing;
528             GST_DEBUG("m_readyState=%s", dumpReadyState(m_readyState));
529             m_networkState = MediaPlayer::Empty;
530             break;
531         case GST_STATE_READY:
532             m_readyState = MediaPlayer::HaveMetadata;
533             GST_DEBUG("m_readyState=%s", dumpReadyState(m_readyState));
534             m_networkState = MediaPlayer::Empty;
535             break;
536         case GST_STATE_PAUSED:
537         case GST_STATE_PLAYING:
538             if (seeking()) {
539                 m_readyState = MediaPlayer::HaveMetadata;
540                 // FIXME: Should we manage NetworkState too?
541                 GST_DEBUG("m_readyState=%s", dumpReadyState(m_readyState));
542             } else if (m_buffering) {
543                 if (m_bufferingPercentage == 100) {
544                     GST_DEBUG("[Buffering] Complete.");
545                     m_buffering = false;
546                     m_readyState = MediaPlayer::HaveEnoughData;
547                     GST_DEBUG("m_readyState=%s", dumpReadyState(m_readyState));
548                     m_networkState = m_downloadFinished ? MediaPlayer::Idle : MediaPlayer::Loading;
549                 } else {
550                     m_readyState = MediaPlayer::HaveCurrentData;
551                     GST_DEBUG("m_readyState=%s", dumpReadyState(m_readyState));
552                     m_networkState = MediaPlayer::Loading;
553                 }
554             } else if (m_downloadFinished) {
555                 m_readyState = MediaPlayer::HaveEnoughData;
556                 GST_DEBUG("m_readyState=%s", dumpReadyState(m_readyState));
557                 m_networkState = MediaPlayer::Loaded;
558             } else {
559                 m_readyState = MediaPlayer::HaveFutureData;
560                 GST_DEBUG("m_readyState=%s", dumpReadyState(m_readyState));
561                 m_networkState = MediaPlayer::Loading;
562             }
563
564             if (m_eosMarked && state == GST_STATE_PLAYING)
565                 m_eosPending = true;
566
567             break;
568         default:
569             ASSERT_NOT_REACHED();
570             break;
571         }
572
573         // Sync states where needed.
574         if (state == GST_STATE_PAUSED) {
575             if (!m_volumeAndMuteInitialized) {
576                 notifyPlayerOfVolumeChange();
577                 notifyPlayerOfMute();
578                 m_volumeAndMuteInitialized = true;
579             }
580
581             if (!seeking() && !m_buffering && !m_paused && m_playbackRate) {
582                 GST_DEBUG("[Buffering] Restarting playback.");
583                 changePipelineState(GST_STATE_PLAYING);
584             }
585         } else if (state == GST_STATE_PLAYING) {
586             m_paused = false;
587
588             if ((m_buffering && !isLiveStream()) || !m_playbackRate) {
589                 GST_DEBUG("[Buffering] Pausing stream for buffering.");
590                 changePipelineState(GST_STATE_PAUSED);
591             }
592         } else
593             m_paused = true;
594
595         if (m_requestedState == GST_STATE_PAUSED && state == GST_STATE_PAUSED) {
596             shouldUpdatePlaybackState = true;
597             GST_DEBUG("Requested state change to %s was completed", gst_element_state_get_name(state));
598         }
599
600         break;
601     }
602     case GST_STATE_CHANGE_ASYNC:
603         GST_DEBUG("Async: State: %s, pending: %s", gst_element_state_get_name(state), gst_element_state_get_name(pending));
604         // Change in progress.
605         break;
606     case GST_STATE_CHANGE_FAILURE:
607         GST_WARNING("Failure: State: %s, pending: %s", gst_element_state_get_name(state), gst_element_state_get_name(pending));
608         // Change failed.
609         return;
610     case GST_STATE_CHANGE_NO_PREROLL:
611         GST_DEBUG("No preroll: State: %s, pending: %s", gst_element_state_get_name(state), gst_element_state_get_name(pending));
612
613         // Live pipelines go in PAUSED without prerolling.
614         m_isStreaming = true;
615
616         if (state == GST_STATE_READY) {
617             m_readyState = MediaPlayer::HaveNothing;
618             GST_DEBUG("m_readyState=%s", dumpReadyState(m_readyState));
619         } else if (state == GST_STATE_PAUSED) {
620             m_readyState = MediaPlayer::HaveEnoughData;
621             GST_DEBUG("m_readyState=%s", dumpReadyState(m_readyState));
622             m_paused = true;
623         } else if (state == GST_STATE_PLAYING)
624             m_paused = false;
625
626         if (!m_paused && m_playbackRate)
627             changePipelineState(GST_STATE_PLAYING);
628
629         m_networkState = MediaPlayer::Loading;
630         break;
631     default:
632         GST_DEBUG("Else : %d", getStateResult);
633         break;
634     }
635
636     m_requestedState = GST_STATE_VOID_PENDING;
637
638     if (shouldUpdatePlaybackState)
639         m_player->playbackStateChanged();
640
641     if (m_networkState != oldNetworkState) {
642         GST_DEBUG("Network State Changed from %u to %u", oldNetworkState, m_networkState);
643         m_player->networkStateChanged();
644     }
645     if (m_readyState != oldReadyState) {
646         GST_DEBUG("Ready State Changed from %u to %u", oldReadyState, m_readyState);
647         m_player->readyStateChanged();
648     }
649
650     if (getStateResult == GST_STATE_CHANGE_SUCCESS && state >= GST_STATE_PAUSED) {
651         updatePlaybackRate();
652         maybeFinishSeek();
653     }
654 }
655 void MediaPlayerPrivateGStreamerMSE::asyncStateChangeDone()
656 {
657     if (UNLIKELY(!m_pipeline || m_errorOccured))
658         return;
659
660     if (m_seeking)
661         maybeFinishSeek();
662     else
663         updateStates();
664 }
665
666 bool MediaPlayerPrivateGStreamerMSE::isTimeBuffered(const MediaTime &time) const
667 {
668     bool result = m_mediaSource && m_mediaSource->buffered()->contain(time);
669     GST_DEBUG("Time %f buffered? %s", time.toDouble(), result ? "Yes" : "No");
670     return result;
671 }
672
673 void MediaPlayerPrivateGStreamerMSE::setMediaSourceClient(Ref<MediaSourceClientGStreamerMSE> client)
674 {
675     m_mediaSourceClient = client.ptr();
676 }
677
678 RefPtr<MediaSourceClientGStreamerMSE> MediaPlayerPrivateGStreamerMSE::mediaSourceClient()
679 {
680     return m_mediaSourceClient;
681 }
682
683 void MediaPlayerPrivateGStreamerMSE::durationChanged()
684 {
685     if (!m_mediaSourceClient) {
686         GST_DEBUG("m_mediaSourceClient is null, doing nothing");
687         return;
688     }
689
690     MediaTime previousDuration = m_mediaTimeDuration;
691     m_mediaTimeDuration = m_mediaSourceClient->duration();
692
693     GST_TRACE("previous=%f, new=%f", previousDuration.toFloat(), m_mediaTimeDuration.toFloat());
694
695     // Avoid emiting durationchanged in the case where the previous duration was 0 because that case is already handled
696     // by the HTMLMediaElement.
697     if (m_mediaTimeDuration != previousDuration && m_mediaTimeDuration.isValid() && previousDuration.isValid()) {
698         m_player->durationChanged();
699         m_playbackPipeline->notifyDurationChanged();
700         m_mediaSource->durationChanged(m_mediaTimeDuration);
701     }
702 }
703
704 static HashSet<String, ASCIICaseInsensitiveHash>& mimeTypeCache()
705 {
706     static NeverDestroyed<HashSet<String, ASCIICaseInsensitiveHash>> cache = []()
707     {
708         initializeGStreamerAndRegisterWebKitMSEElement();
709         HashSet<String, ASCIICaseInsensitiveHash> set;
710         const char* mimeTypes[] = {
711             "video/mp4",
712             "audio/mp4"
713         };
714         for (auto& type : mimeTypes)
715             set.add(type);
716         return set;
717     }();
718     return cache;
719 }
720
721 void MediaPlayerPrivateGStreamerMSE::getSupportedTypes(HashSet<String, ASCIICaseInsensitiveHash>& types)
722 {
723     types = mimeTypeCache();
724 }
725
726 void MediaPlayerPrivateGStreamerMSE::trackDetected(RefPtr<AppendPipeline> appendPipeline, RefPtr<WebCore::TrackPrivateBase> oldTrack, RefPtr<WebCore::TrackPrivateBase> newTrack)
727 {
728     ASSERT(appendPipeline->track() == newTrack);
729
730     GstCaps* caps = appendPipeline->appsinkCaps();
731     ASSERT(caps);
732     GST_DEBUG("track ID: %s, caps: %" GST_PTR_FORMAT, newTrack->id().string().latin1().data(), caps);
733
734     GstStructure* structure = gst_caps_get_structure(caps, 0);
735     const gchar* mediaType = gst_structure_get_name(structure);
736     GstVideoInfo info;
737
738     if (g_str_has_prefix(mediaType, "video/") && gst_video_info_from_caps(&info, caps)) {
739         float width, height;
740
741         width = info.width;
742         height = info.height * ((float) info.par_d / (float) info.par_n);
743         m_videoSize.setWidth(width);
744         m_videoSize.setHeight(height);
745     }
746
747     if (!oldTrack)
748         m_playbackPipeline->attachTrack(appendPipeline->sourceBufferPrivate(), newTrack, structure, caps);
749     else
750         m_playbackPipeline->reattachTrack(appendPipeline->sourceBufferPrivate(), newTrack);
751 }
752
753 bool MediaPlayerPrivateGStreamerMSE::supportsCodecs(const String& codecs)
754 {
755     static Vector<const char*> supportedCodecs = { "avc*", "mp4a*", "mpeg", "x-h264" };
756     Vector<String> codecEntries;
757     codecs.split(',', false, codecEntries);
758
759     for (String codec : codecEntries) {
760         bool isCodecSupported = false;
761
762         // If the codec is named like a mimetype (eg: video/avc) remove the "video/" part.
763         size_t slashIndex = codec.find('/');
764         if (slashIndex != WTF::notFound)
765             codec = codec.substring(slashIndex+1);
766
767         const char* codecData = codec.utf8().data();
768         for (const auto& pattern : supportedCodecs) {
769             if (isCodecSupported = !fnmatch(pattern, codecData, 0))
770                 break;
771         }
772         if (!isCodecSupported)
773             return false;
774     }
775
776     return true;
777 }
778
779 MediaPlayer::SupportsType MediaPlayerPrivateGStreamerMSE::supportsType(const MediaEngineSupportParameters& parameters)
780 {
781     MediaPlayer::SupportsType result = MediaPlayer::IsNotSupported;
782     if (!parameters.isMediaSource)
783         return result;
784
785     // Disable VPX/Opus on MSE for now, mp4/avc1 seems way more reliable currently.
786     if (parameters.type.endsWith("webm"))
787         return result;
788
789     // YouTube TV provides empty types for some videos and we want to be selected as best media engine for them.
790     if (parameters.type.isEmpty()) {
791         result = MediaPlayer::MayBeSupported;
792         return result;
793     }
794
795     // Spec says we should not return "probably" if the codecs string is empty.
796     if (mimeTypeCache().contains(parameters.type)) {
797         if (parameters.codecs.isEmpty())
798             result = MediaPlayer::MayBeSupported;
799         else
800             result = supportsCodecs(parameters.codecs) ? MediaPlayer::IsSupported : MediaPlayer::IsNotSupported;
801     }
802
803     return extendedSupportsType(parameters, result);
804 }
805
806 #if ENABLE(LEGACY_ENCRYPTED_MEDIA)
807 void MediaPlayerPrivateGStreamerMSE::dispatchDecryptionKey(GstBuffer* buffer)
808 {
809     for (auto iterator : m_appendPipelinesMap) {
810         if (iterator.value->appendState() == AppendPipeline::AppendState::KeyNegotiation) {
811             GST_TRACE("append pipeline %p in key negotiation, setting key", iterator.value.get());
812             gst_element_send_event(iterator.value->pipeline(), gst_event_new_custom(GST_EVENT_CUSTOM_DOWNSTREAM_OOB,
813                 gst_structure_new("drm-cipher", "key", GST_TYPE_BUFFER, buffer, nullptr)));
814             iterator.value->setAppendState(AppendPipeline::AppendState::Ongoing);
815         } else
816             GST_TRACE("append pipeline %p not in key negotiation", iterator.value.get());
817     }
818 }
819 #endif
820
821 void MediaPlayerPrivateGStreamerMSE::markEndOfStream(MediaSourcePrivate::EndOfStreamStatus status)
822 {
823     if (status != MediaSourcePrivate::EosNoError)
824         return;
825
826     GST_DEBUG("Marking end of stream");
827     m_eosMarked = true;
828     updateStates();
829 }
830
831 MediaTime MediaPlayerPrivateGStreamerMSE::currentMediaTime() const
832 {
833     MediaTime position = MediaPlayerPrivateGStreamer::currentMediaTime();
834
835     if (m_eosPending && (paused() || (position >= durationMediaTime()))) {
836         if (m_networkState != MediaPlayer::Loaded) {
837             m_networkState = MediaPlayer::Loaded;
838             m_player->networkStateChanged();
839         }
840
841         m_eosPending = false;
842         m_isEndReached = true;
843         m_cachedPosition = m_mediaTimeDuration.toFloat();
844         m_durationAtEOS = m_mediaTimeDuration.toFloat();
845         m_player->timeChanged();
846     }
847     return position;
848 }
849
850 float MediaPlayerPrivateGStreamerMSE::maxTimeSeekable() const
851 {
852     if (UNLIKELY(m_errorOccured))
853         return 0;
854
855     GST_DEBUG("maxTimeSeekable");
856     float result = durationMediaTime().toFloat();
857     // Infinite duration means live stream.
858     if (std::isinf(result)) {
859         MediaTime maxBufferedTime = buffered()->maximumBufferedTime();
860         // Return the highest end time reported by the buffered attribute.
861         result = maxBufferedTime.isValid() ? maxBufferedTime.toFloat() : 0;
862     }
863
864     return result;
865 }
866
867 } // namespace WebCore.
868
869 #endif // USE(GSTREAMER)