Move URL from WebCore to WTF
[WebKit-https.git] / Source / WebCore / platform / graphics / gstreamer / mse / MediaPlayerPrivateGStreamerMSE.cpp
1 /*
2  * Copyright (C) 2007, 2009 Apple Inc.  All rights reserved.
3  * Copyright (C) 2007 Collabora Ltd.  All rights reserved.
4  * Copyright (C) 2007 Alp Toker <alp@atoker.com>
5  * Copyright (C) 2009 Gustavo Noronha Silva <gns@gnome.org>
6  * Copyright (C) 2009, 2010, 2011, 2012, 2013, 2016, 2017 Igalia S.L
7  * Copyright (C) 2015 Sebastian Dröge <sebastian@centricular.com>
8  * Copyright (C) 2015, 2016, 2017 Metrological Group B.V.
9  *
10  * This library is free software; you can redistribute it and/or
11  * modify it under the terms of the GNU Library General Public
12  * License as published by the Free Software Foundation; either
13  * version 2 of the License, or (at your option) any later version.
14  *
15  * This library is distributed in the hope that it will be useful,
16  * but WITHOUT ANY WARRANTY; without even the implied warranty of
17  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
18  * Library General Public License for more details.
19  *
20  * You should have received a copy of the GNU Library General Public License
21  * aint with this library; see the file COPYING.LIB.  If not, write to
22  * the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
23  * Boston, MA 02110-1301, USA.
24  */
25
26 #include "config.h"
27 #include "MediaPlayerPrivateGStreamerMSE.h"
28
29 #if ENABLE(VIDEO) && USE(GSTREAMER) && ENABLE(MEDIA_SOURCE)
30
31 #include "AppendPipeline.h"
32 #include "AudioTrackPrivateGStreamer.h"
33 #include "GStreamerCommon.h"
34 #include "InbandTextTrackPrivateGStreamer.h"
35 #include "MIMETypeRegistry.h"
36 #include "MediaDescription.h"
37 #include "MediaPlayer.h"
38 #include "NotImplemented.h"
39 #include "PlaybackPipeline.h"
40 #include "SourceBufferPrivateGStreamer.h"
41 #include "TimeRanges.h"
42 #include "VideoTrackPrivateGStreamer.h"
43
44 #include <fnmatch.h>
45 #include <gst/app/gstappsink.h>
46 #include <gst/app/gstappsrc.h>
47 #include <gst/gst.h>
48 #include <gst/pbutils/pbutils.h>
49 #include <gst/video/video.h>
50 #include <wtf/Condition.h>
51 #include <wtf/HashSet.h>
52 #include <wtf/NeverDestroyed.h>
53 #include <wtf/StringPrintStream.h>
54 #include <wtf/URL.h>
55 #include <wtf/text/AtomicString.h>
56 #include <wtf/text/AtomicStringHash.h>
57
58 #if ENABLE(ENCRYPTED_MEDIA)
59 #include "CDMClearKey.h"
60 #include "SharedBuffer.h"
61 #endif
62
63 static const char* dumpReadyState(WebCore::MediaPlayer::ReadyState readyState)
64 {
65     switch (readyState) {
66     case WebCore::MediaPlayer::HaveNothing: return "HaveNothing";
67     case WebCore::MediaPlayer::HaveMetadata: return "HaveMetadata";
68     case WebCore::MediaPlayer::HaveCurrentData: return "HaveCurrentData";
69     case WebCore::MediaPlayer::HaveFutureData: return "HaveFutureData";
70     case WebCore::MediaPlayer::HaveEnoughData: return "HaveEnoughData";
71     default: return "(unknown)";
72     }
73 }
74
75 GST_DEBUG_CATEGORY(webkit_mse_debug);
76 #define GST_CAT_DEFAULT webkit_mse_debug
77
78 namespace WebCore {
79
80 void MediaPlayerPrivateGStreamerMSE::registerMediaEngine(MediaEngineRegistrar registrar)
81 {
82     initializeGStreamerAndRegisterWebKitElements();
83     GST_DEBUG_CATEGORY_INIT(webkit_mse_debug, "webkitmse", 0, "WebKit MSE media player");
84     if (isAvailable()) {
85         registrar([](MediaPlayer* player) { return std::make_unique<MediaPlayerPrivateGStreamerMSE>(player); },
86             getSupportedTypes, supportsType, nullptr, nullptr, nullptr, supportsKeySystem);
87     }
88 }
89
90 MediaPlayerPrivateGStreamerMSE::MediaPlayerPrivateGStreamerMSE(MediaPlayer* player)
91     : MediaPlayerPrivateGStreamer(player)
92 {
93     GST_TRACE("creating the player (%p)", this);
94 }
95
96 MediaPlayerPrivateGStreamerMSE::~MediaPlayerPrivateGStreamerMSE()
97 {
98     GST_TRACE("destroying the player (%p)", this);
99
100     // Clear the AppendPipeline map. This should cause the destruction of all the AppendPipeline's since there should
101     // be no alive references at this point.
102 #ifndef NDEBUG
103     for (auto iterator : m_appendPipelinesMap)
104         ASSERT(iterator.value->hasOneRef());
105 #endif
106     m_appendPipelinesMap.clear();
107
108     if (m_source) {
109         webKitMediaSrcSetMediaPlayerPrivate(WEBKIT_MEDIA_SRC(m_source.get()), nullptr);
110         g_signal_handlers_disconnect_by_data(m_source.get(), this);
111     }
112
113     if (m_playbackPipeline)
114         m_playbackPipeline->setWebKitMediaSrc(nullptr);
115 }
116
117 void MediaPlayerPrivateGStreamerMSE::load(const String& urlString)
118 {
119     if (!urlString.startsWith("mediasource")) {
120         // Properly fail so the global MediaPlayer tries to fallback to the next MediaPlayerPrivate.
121         m_networkState = MediaPlayer::FormatError;
122         m_player->networkStateChanged();
123         return;
124     }
125
126     if (!m_playbackPipeline)
127         m_playbackPipeline = PlaybackPipeline::create();
128
129     MediaPlayerPrivateGStreamer::load(urlString);
130 }
131
132 void MediaPlayerPrivateGStreamerMSE::load(const String& url, MediaSourcePrivateClient* mediaSource)
133 {
134     m_mediaSource = mediaSource;
135     load(String::format("mediasource%s", url.utf8().data()));
136 }
137
138 void MediaPlayerPrivateGStreamerMSE::pause()
139 {
140     m_paused = true;
141     MediaPlayerPrivateGStreamer::pause();
142 }
143
144 MediaTime MediaPlayerPrivateGStreamerMSE::durationMediaTime() const
145 {
146     if (UNLIKELY(!m_pipeline || m_errorOccured))
147         return MediaTime();
148
149     return m_mediaTimeDuration;
150 }
151
152 void MediaPlayerPrivateGStreamerMSE::seek(const MediaTime& time)
153 {
154     if (UNLIKELY(!m_pipeline || m_errorOccured))
155         return;
156
157     GST_INFO("[Seek] seek attempt to %s secs", toString(time).utf8().data());
158
159     // Avoid useless seeking.
160     MediaTime current = currentMediaTime();
161     if (time == current) {
162         if (!m_seeking)
163             timeChanged();
164         return;
165     }
166
167     if (isLiveStream())
168         return;
169
170     if (m_seeking && m_seekIsPending) {
171         m_seekTime = time;
172         return;
173     }
174
175     GST_DEBUG("Seeking from %s to %s seconds", toString(current).utf8().data(), toString(time).utf8().data());
176
177     MediaTime previousSeekTime = m_seekTime;
178     m_seekTime = time;
179
180     if (!doSeek()) {
181         m_seekTime = previousSeekTime;
182         GST_WARNING("Seeking to %s failed", toString(time).utf8().data());
183         return;
184     }
185
186     m_isEndReached = false;
187     GST_DEBUG("m_seeking=%s, m_seekTime=%s", boolForPrinting(m_seeking), toString(m_seekTime).utf8().data());
188 }
189
190 void MediaPlayerPrivateGStreamerMSE::configurePlaySink()
191 {
192     MediaPlayerPrivateGStreamer::configurePlaySink();
193
194     GRefPtr<GstElement> playsink = adoptGRef(gst_bin_get_by_name(GST_BIN(m_pipeline.get()), "playsink"));
195     if (playsink) {
196         // The default value (0) means "send events to all the sinks", instead
197         // of "only to the first that returns true". This is needed for MSE seek.
198         g_object_set(G_OBJECT(playsink.get()), "send-event-mode", 0, nullptr);
199     }
200 }
201
202 bool MediaPlayerPrivateGStreamerMSE::changePipelineState(GstState newState)
203 {
204     if (seeking()) {
205         GST_DEBUG("Rejected state change to %s while seeking",
206             gst_element_state_get_name(newState));
207         return true;
208     }
209
210     return MediaPlayerPrivateGStreamer::changePipelineState(newState);
211 }
212
213 void MediaPlayerPrivateGStreamerMSE::notifySeekNeedsDataForTime(const MediaTime& seekTime)
214 {
215     // Reenqueue samples needed to resume playback in the new position.
216     m_mediaSource->seekToTime(seekTime);
217
218     GST_DEBUG("MSE seek to %s finished", toString(seekTime).utf8().data());
219
220     if (!m_gstSeekCompleted) {
221         m_gstSeekCompleted = true;
222         maybeFinishSeek();
223     }
224 }
225
226 bool MediaPlayerPrivateGStreamerMSE::doSeek(const MediaTime&, float, GstSeekFlags)
227 {
228     // Use doSeek() instead. If anybody is calling this version of doSeek(), something is wrong.
229     ASSERT_NOT_REACHED();
230     return false;
231 }
232
233 bool MediaPlayerPrivateGStreamerMSE::doSeek()
234 {
235     MediaTime seekTime = m_seekTime;
236     double rate = m_player->rate();
237     GstSeekFlags seekType = static_cast<GstSeekFlags>(GST_SEEK_FLAG_FLUSH | GST_SEEK_FLAG_ACCURATE);
238
239     // Always move to seeking state to report correct 'currentTime' while pending for actual seek to complete.
240     m_seeking = true;
241
242     // Check if playback pipeline is ready for seek.
243     GstState state, newState;
244     GstStateChangeReturn getStateResult = gst_element_get_state(m_pipeline.get(), &state, &newState, 0);
245     if (getStateResult == GST_STATE_CHANGE_FAILURE || getStateResult == GST_STATE_CHANGE_NO_PREROLL) {
246         GST_DEBUG("[Seek] cannot seek, current state change is %s", gst_element_state_change_return_get_name(getStateResult));
247         webKitMediaSrcSetReadyForSamples(WEBKIT_MEDIA_SRC(m_source.get()), true);
248         m_seeking = false;
249         return false;
250     }
251     if ((getStateResult == GST_STATE_CHANGE_ASYNC
252         && !(state == GST_STATE_PLAYING && newState == GST_STATE_PAUSED))
253         || state < GST_STATE_PAUSED
254         || m_isEndReached
255         || !m_gstSeekCompleted) {
256         CString reason = "Unknown reason";
257         if (getStateResult == GST_STATE_CHANGE_ASYNC) {
258             reason = String::format("In async change %s --> %s",
259                 gst_element_state_get_name(state),
260                 gst_element_state_get_name(newState)).utf8();
261         } else if (state < GST_STATE_PAUSED)
262             reason = "State less than PAUSED";
263         else if (m_isEndReached)
264             reason = "End reached";
265         else if (!m_gstSeekCompleted)
266             reason = "Previous seek is not finished yet";
267
268         GST_DEBUG("[Seek] Delaying the seek: %s", reason.data());
269
270         m_seekIsPending = true;
271
272         if (m_isEndReached) {
273             GST_DEBUG("[Seek] reset pipeline");
274             m_resetPipeline = true;
275             m_seeking = false;
276             if (!changePipelineState(GST_STATE_PAUSED))
277                 loadingFailed(MediaPlayer::Empty);
278             else
279                 m_seeking = true;
280         }
281
282         return m_seeking;
283     }
284
285     // Stop accepting new samples until actual seek is finished.
286     webKitMediaSrcSetReadyForSamples(WEBKIT_MEDIA_SRC(m_source.get()), false);
287
288     // Correct seek time if it helps to fix a small gap.
289     if (!isTimeBuffered(seekTime)) {
290         // Look if a near future time (<0.1 sec.) is buffered and change the seek target time.
291         if (m_mediaSource) {
292             const MediaTime miniGap = MediaTime(1, 10);
293             MediaTime nearest = m_mediaSource->buffered()->nearest(seekTime);
294             if (nearest.isValid() && nearest > seekTime && (nearest - seekTime) <= miniGap && isTimeBuffered(nearest + miniGap)) {
295                 GST_DEBUG("[Seek] Changed the seek target time from %s to %s, a near point in the future", toString(seekTime).utf8().data(), toString(nearest).utf8().data());
296                 seekTime = nearest;
297             }
298         }
299     }
300
301     // Check if MSE has samples for requested time and defer actual seek if needed.
302     if (!isTimeBuffered(seekTime)) {
303         GST_DEBUG("[Seek] Delaying the seek: MSE is not ready");
304         GstStateChangeReturn setStateResult = gst_element_set_state(m_pipeline.get(), GST_STATE_PAUSED);
305         if (setStateResult == GST_STATE_CHANGE_FAILURE) {
306             GST_DEBUG("[Seek] Cannot seek, failed to pause playback pipeline.");
307             webKitMediaSrcSetReadyForSamples(WEBKIT_MEDIA_SRC(m_source.get()), true);
308             m_seeking = false;
309             return false;
310         }
311         m_readyState = MediaPlayer::HaveMetadata;
312         notifySeekNeedsDataForTime(seekTime);
313         ASSERT(!m_mseSeekCompleted);
314         return true;
315     }
316
317     // Complete previous MSE seek if needed.
318     if (!m_mseSeekCompleted) {
319         m_mediaSource->monitorSourceBuffers();
320         ASSERT(m_mseSeekCompleted);
321         // Note: seekCompleted will recursively call us.
322         return m_seeking;
323     }
324
325     GST_DEBUG("We can seek now");
326
327     MediaTime startTime = seekTime, endTime = MediaTime::invalidTime();
328
329     if (rate < 0) {
330         startTime = MediaTime::zeroTime();
331         endTime = seekTime;
332     }
333
334     if (!rate)
335         rate = 1;
336
337     GST_DEBUG("Actual seek to %s, end time:  %s, rate: %f", toString(startTime).utf8().data(), toString(endTime).utf8().data(), rate);
338
339     // This will call notifySeekNeedsData() after some time to tell that the pipeline is ready for sample enqueuing.
340     webKitMediaSrcPrepareSeek(WEBKIT_MEDIA_SRC(m_source.get()), seekTime);
341
342     m_gstSeekCompleted = false;
343     if (!gst_element_seek(m_pipeline.get(), rate, GST_FORMAT_TIME, seekType, GST_SEEK_TYPE_SET, toGstClockTime(startTime), GST_SEEK_TYPE_SET, toGstClockTime(endTime))) {
344         webKitMediaSrcSetReadyForSamples(WEBKIT_MEDIA_SRC(m_source.get()), true);
345         m_seeking = false;
346         m_gstSeekCompleted = true;
347         GST_DEBUG("doSeek(): gst_element_seek() failed, returning false");
348         return false;
349     }
350
351     // The samples will be enqueued in notifySeekNeedsData().
352     GST_DEBUG("doSeek(): gst_element_seek() succeeded, returning true");
353     return true;
354 }
355
356 void MediaPlayerPrivateGStreamerMSE::maybeFinishSeek()
357 {
358     if (!m_seeking || !m_mseSeekCompleted || !m_gstSeekCompleted)
359         return;
360
361     GstState state, newState;
362     GstStateChangeReturn getStateResult = gst_element_get_state(m_pipeline.get(), &state, &newState, 0);
363
364     if (getStateResult == GST_STATE_CHANGE_ASYNC
365         && !(state == GST_STATE_PLAYING && newState == GST_STATE_PAUSED)) {
366         GST_DEBUG("[Seek] Delaying seek finish");
367         return;
368     }
369
370     if (m_seekIsPending) {
371         GST_DEBUG("[Seek] Committing pending seek to %s", toString(m_seekTime).utf8().data());
372         m_seekIsPending = false;
373         if (!doSeek()) {
374             GST_WARNING("[Seek] Seeking to %s failed", toString(m_seekTime).utf8().data());
375             m_cachedPosition = MediaTime::invalidTime();
376         }
377         return;
378     }
379
380     GST_DEBUG("[Seek] Seeked to %s", toString(m_seekTime).utf8().data());
381
382     webKitMediaSrcSetReadyForSamples(WEBKIT_MEDIA_SRC(m_source.get()), true);
383     m_seeking = false;
384     m_cachedPosition = MediaTime::invalidTime();
385     // The pipeline can still have a pending state. In this case a position query will fail.
386     // Right now we can use m_seekTime as a fallback.
387     m_canFallBackToLastFinishedSeekPosition = true;
388     timeChanged();
389 }
390
391 void MediaPlayerPrivateGStreamerMSE::updatePlaybackRate()
392 {
393     notImplemented();
394 }
395
396 bool MediaPlayerPrivateGStreamerMSE::seeking() const
397 {
398     return m_seeking;
399 }
400
401 // FIXME: MediaPlayerPrivateGStreamer manages the ReadyState on its own. We shouldn't change it manually.
402 void MediaPlayerPrivateGStreamerMSE::setReadyState(MediaPlayer::ReadyState readyState)
403 {
404     if (readyState == m_readyState)
405         return;
406
407     if (seeking()) {
408         GST_DEBUG("Skip ready state change(%s -> %s) due to seek\n", dumpReadyState(m_readyState), dumpReadyState(readyState));
409         return;
410     }
411
412     GST_DEBUG("Ready State Changed manually from %u to %u", m_readyState, readyState);
413     MediaPlayer::ReadyState oldReadyState = m_readyState;
414     m_readyState = readyState;
415     GST_DEBUG("m_readyState: %s -> %s", dumpReadyState(oldReadyState), dumpReadyState(m_readyState));
416
417     if (oldReadyState < MediaPlayer::HaveCurrentData && m_readyState >= MediaPlayer::HaveCurrentData) {
418         GST_DEBUG("[Seek] Reporting load state changed to trigger seek continuation");
419         loadStateChanged();
420     }
421     m_player->readyStateChanged();
422
423     GstState pipelineState;
424     GstStateChangeReturn getStateResult = gst_element_get_state(m_pipeline.get(), &pipelineState, nullptr, 250 * GST_NSECOND);
425     bool isPlaying = (getStateResult == GST_STATE_CHANGE_SUCCESS && pipelineState == GST_STATE_PLAYING);
426
427     if (m_readyState == MediaPlayer::HaveMetadata && oldReadyState > MediaPlayer::HaveMetadata && isPlaying) {
428         GST_TRACE("Changing pipeline to PAUSED...");
429         bool ok = changePipelineState(GST_STATE_PAUSED);
430         GST_TRACE("Changed pipeline to PAUSED: %s", ok ? "Success" : "Error");
431     }
432 }
433
434 void MediaPlayerPrivateGStreamerMSE::waitForSeekCompleted()
435 {
436     if (!m_seeking)
437         return;
438
439     GST_DEBUG("Waiting for MSE seek completed");
440     m_mseSeekCompleted = false;
441 }
442
443 void MediaPlayerPrivateGStreamerMSE::seekCompleted()
444 {
445     if (m_mseSeekCompleted)
446         return;
447
448     GST_DEBUG("MSE seek completed");
449     m_mseSeekCompleted = true;
450
451     doSeek();
452
453     if (!seeking() && m_readyState >= MediaPlayer::HaveFutureData)
454         changePipelineState(GST_STATE_PLAYING);
455
456     if (!seeking())
457         m_player->timeChanged();
458 }
459
460 void MediaPlayerPrivateGStreamerMSE::setRate(float)
461 {
462     notImplemented();
463 }
464
465 std::unique_ptr<PlatformTimeRanges> MediaPlayerPrivateGStreamerMSE::buffered() const
466 {
467     return m_mediaSource ? m_mediaSource->buffered() : std::make_unique<PlatformTimeRanges>();
468 }
469
470 void MediaPlayerPrivateGStreamerMSE::sourceSetup(GstElement* sourceElement)
471 {
472     m_source = sourceElement;
473
474     ASSERT(WEBKIT_IS_MEDIA_SRC(m_source.get()));
475
476     m_playbackPipeline->setWebKitMediaSrc(WEBKIT_MEDIA_SRC(m_source.get()));
477
478     MediaSourceGStreamer::open(*m_mediaSource.get(), *this);
479     g_signal_connect_swapped(m_source.get(), "video-changed", G_CALLBACK(videoChangedCallback), this);
480     g_signal_connect_swapped(m_source.get(), "audio-changed", G_CALLBACK(audioChangedCallback), this);
481     g_signal_connect_swapped(m_source.get(), "text-changed", G_CALLBACK(textChangedCallback), this);
482     webKitMediaSrcSetMediaPlayerPrivate(WEBKIT_MEDIA_SRC(m_source.get()), this);
483 }
484
485 void MediaPlayerPrivateGStreamerMSE::updateStates()
486 {
487     if (UNLIKELY(!m_pipeline || m_errorOccured))
488         return;
489
490     MediaPlayer::NetworkState oldNetworkState = m_networkState;
491     MediaPlayer::ReadyState oldReadyState = m_readyState;
492     GstState state, pending;
493
494     GstStateChangeReturn getStateResult = gst_element_get_state(m_pipeline.get(), &state, &pending, 250 * GST_NSECOND);
495
496     bool shouldUpdatePlaybackState = false;
497     switch (getStateResult) {
498     case GST_STATE_CHANGE_SUCCESS: {
499         GST_DEBUG("State: %s, pending: %s", gst_element_state_get_name(state), gst_element_state_get_name(pending));
500
501         // Do nothing if on EOS and state changed to READY to avoid recreating the player
502         // on HTMLMediaElement and properly generate the video 'ended' event.
503         if (m_isEndReached && state == GST_STATE_READY)
504             break;
505
506         m_resetPipeline = (state <= GST_STATE_READY);
507         if (m_resetPipeline)
508             m_mediaTimeDuration = MediaTime::zeroTime();
509
510         // Update ready and network states.
511         switch (state) {
512         case GST_STATE_NULL:
513             m_readyState = MediaPlayer::HaveNothing;
514             GST_DEBUG("m_readyState=%s", dumpReadyState(m_readyState));
515             m_networkState = MediaPlayer::Empty;
516             break;
517         case GST_STATE_READY:
518             m_readyState = MediaPlayer::HaveMetadata;
519             GST_DEBUG("m_readyState=%s", dumpReadyState(m_readyState));
520             m_networkState = MediaPlayer::Empty;
521             break;
522         case GST_STATE_PAUSED:
523         case GST_STATE_PLAYING:
524             if (seeking()) {
525                 m_readyState = MediaPlayer::HaveMetadata;
526                 // FIXME: Should we manage NetworkState too?
527                 GST_DEBUG("m_readyState=%s", dumpReadyState(m_readyState));
528             } else if (m_buffering) {
529                 if (m_bufferingPercentage == 100) {
530                     GST_DEBUG("[Buffering] Complete.");
531                     m_buffering = false;
532                     m_readyState = MediaPlayer::HaveEnoughData;
533                     GST_DEBUG("m_readyState=%s", dumpReadyState(m_readyState));
534                     m_networkState = m_downloadFinished ? MediaPlayer::Idle : MediaPlayer::Loading;
535                 } else {
536                     m_readyState = MediaPlayer::HaveCurrentData;
537                     GST_DEBUG("m_readyState=%s", dumpReadyState(m_readyState));
538                     m_networkState = MediaPlayer::Loading;
539                 }
540             } else if (m_downloadFinished) {
541                 m_readyState = MediaPlayer::HaveEnoughData;
542                 GST_DEBUG("m_readyState=%s", dumpReadyState(m_readyState));
543                 m_networkState = MediaPlayer::Loaded;
544             } else {
545                 m_readyState = MediaPlayer::HaveFutureData;
546                 GST_DEBUG("m_readyState=%s", dumpReadyState(m_readyState));
547                 m_networkState = MediaPlayer::Loading;
548             }
549
550             if (m_eosMarked && state == GST_STATE_PLAYING)
551                 m_eosPending = true;
552
553             break;
554         default:
555             ASSERT_NOT_REACHED();
556             break;
557         }
558
559         // Sync states where needed.
560         if (state == GST_STATE_PAUSED) {
561             if (!m_volumeAndMuteInitialized) {
562                 notifyPlayerOfVolumeChange();
563                 notifyPlayerOfMute();
564                 m_volumeAndMuteInitialized = true;
565             }
566
567             if (!seeking() && !m_buffering && !m_paused && m_playbackRate) {
568                 GST_DEBUG("[Buffering] Restarting playback.");
569                 changePipelineState(GST_STATE_PLAYING);
570             }
571         } else if (state == GST_STATE_PLAYING) {
572             m_paused = false;
573
574             if ((m_buffering && !isLiveStream()) || !m_playbackRate) {
575                 GST_DEBUG("[Buffering] Pausing stream for buffering.");
576                 changePipelineState(GST_STATE_PAUSED);
577             }
578         } else
579             m_paused = true;
580
581         if (m_requestedState == GST_STATE_PAUSED && state == GST_STATE_PAUSED) {
582             shouldUpdatePlaybackState = true;
583             GST_DEBUG("Requested state change to %s was completed", gst_element_state_get_name(state));
584         }
585
586         break;
587     }
588     case GST_STATE_CHANGE_ASYNC:
589         GST_DEBUG("Async: State: %s, pending: %s", gst_element_state_get_name(state), gst_element_state_get_name(pending));
590         // Change in progress.
591         break;
592     case GST_STATE_CHANGE_FAILURE:
593         GST_WARNING("Failure: State: %s, pending: %s", gst_element_state_get_name(state), gst_element_state_get_name(pending));
594         // Change failed.
595         return;
596     case GST_STATE_CHANGE_NO_PREROLL:
597         GST_DEBUG("No preroll: State: %s, pending: %s", gst_element_state_get_name(state), gst_element_state_get_name(pending));
598
599         // Live pipelines go in PAUSED without prerolling.
600         m_isStreaming = true;
601
602         if (state == GST_STATE_READY) {
603             m_readyState = MediaPlayer::HaveNothing;
604             GST_DEBUG("m_readyState=%s", dumpReadyState(m_readyState));
605         } else if (state == GST_STATE_PAUSED) {
606             m_readyState = MediaPlayer::HaveEnoughData;
607             GST_DEBUG("m_readyState=%s", dumpReadyState(m_readyState));
608             m_paused = true;
609         } else if (state == GST_STATE_PLAYING)
610             m_paused = false;
611
612         if (!m_paused && m_playbackRate)
613             changePipelineState(GST_STATE_PLAYING);
614
615         m_networkState = MediaPlayer::Loading;
616         break;
617     default:
618         GST_DEBUG("Else : %d", getStateResult);
619         break;
620     }
621
622     m_requestedState = GST_STATE_VOID_PENDING;
623
624     if (shouldUpdatePlaybackState)
625         m_player->playbackStateChanged();
626
627     if (m_networkState != oldNetworkState) {
628         GST_DEBUG("Network State Changed from %u to %u", oldNetworkState, m_networkState);
629         m_player->networkStateChanged();
630     }
631     if (m_readyState != oldReadyState) {
632         GST_DEBUG("Ready State Changed from %u to %u", oldReadyState, m_readyState);
633         m_player->readyStateChanged();
634     }
635
636     if (getStateResult == GST_STATE_CHANGE_SUCCESS && state >= GST_STATE_PAUSED) {
637         updatePlaybackRate();
638         maybeFinishSeek();
639     }
640 }
641 void MediaPlayerPrivateGStreamerMSE::asyncStateChangeDone()
642 {
643     if (UNLIKELY(!m_pipeline || m_errorOccured))
644         return;
645
646     if (m_seeking)
647         maybeFinishSeek();
648     else
649         updateStates();
650 }
651
652 bool MediaPlayerPrivateGStreamerMSE::isTimeBuffered(const MediaTime &time) const
653 {
654     bool result = m_mediaSource && m_mediaSource->buffered()->contain(time);
655     GST_DEBUG("Time %s buffered? %s", toString(time).utf8().data(), boolForPrinting(result));
656     return result;
657 }
658
659 void MediaPlayerPrivateGStreamerMSE::setMediaSourceClient(Ref<MediaSourceClientGStreamerMSE> client)
660 {
661     m_mediaSourceClient = client.ptr();
662 }
663
664 RefPtr<MediaSourceClientGStreamerMSE> MediaPlayerPrivateGStreamerMSE::mediaSourceClient()
665 {
666     return m_mediaSourceClient;
667 }
668
669 void MediaPlayerPrivateGStreamerMSE::durationChanged()
670 {
671     if (!m_mediaSourceClient) {
672         GST_DEBUG("m_mediaSourceClient is null, doing nothing");
673         return;
674     }
675
676     MediaTime previousDuration = m_mediaTimeDuration;
677     m_mediaTimeDuration = m_mediaSourceClient->duration();
678
679     GST_TRACE("previous=%s, new=%s", toString(previousDuration).utf8().data(), toString(m_mediaTimeDuration).utf8().data());
680
681     // Avoid emiting durationchanged in the case where the previous duration was 0 because that case is already handled
682     // by the HTMLMediaElement.
683     if (m_mediaTimeDuration != previousDuration && m_mediaTimeDuration.isValid() && previousDuration.isValid()) {
684         m_player->durationChanged();
685         m_playbackPipeline->notifyDurationChanged();
686         m_mediaSource->durationChanged(m_mediaTimeDuration);
687     }
688 }
689
690 static HashSet<String, ASCIICaseInsensitiveHash>& mimeTypeCache()
691 {
692     static NeverDestroyed<HashSet<String, ASCIICaseInsensitiveHash>> cache = []()
693     {
694         initializeGStreamerAndRegisterWebKitElements();
695         HashSet<String, ASCIICaseInsensitiveHash> set;
696         const char* mimeTypes[] = {
697             "video/mp4",
698             "audio/mp4",
699             "video/webm",
700             "audio/webm"
701         };
702         for (auto& type : mimeTypes)
703             set.add(type);
704         return set;
705     }();
706     return cache;
707 }
708
709 void MediaPlayerPrivateGStreamerMSE::getSupportedTypes(HashSet<String, ASCIICaseInsensitiveHash>& types)
710 {
711     types = mimeTypeCache();
712 }
713
714 void MediaPlayerPrivateGStreamerMSE::trackDetected(RefPtr<AppendPipeline> appendPipeline, RefPtr<WebCore::TrackPrivateBase> newTrack, bool firstTrackDetected)
715 {
716     ASSERT(appendPipeline->track() == newTrack);
717
718     GstCaps* caps = appendPipeline->appsinkCaps();
719     ASSERT(caps);
720     GST_DEBUG("track ID: %s, caps: %" GST_PTR_FORMAT, newTrack->id().string().latin1().data(), caps);
721
722     if (doCapsHaveType(caps, GST_VIDEO_CAPS_TYPE_PREFIX)) {
723         std::optional<FloatSize> size = getVideoResolutionFromCaps(caps);
724         if (size.has_value())
725             m_videoSize = size.value();
726     }
727
728     if (firstTrackDetected)
729         m_playbackPipeline->attachTrack(appendPipeline->sourceBufferPrivate(), newTrack, caps);
730     else
731         m_playbackPipeline->reattachTrack(appendPipeline->sourceBufferPrivate(), newTrack, caps);
732 }
733
734 const static HashSet<AtomicString>& codecSet()
735 {
736     static NeverDestroyed<HashSet<AtomicString>> codecTypes = []()
737     {
738         initializeGStreamerAndRegisterWebKitElements();
739         HashSet<AtomicString> set;
740
741         GList* audioDecoderFactories = gst_element_factory_list_get_elements(GST_ELEMENT_FACTORY_TYPE_DECODER | GST_ELEMENT_FACTORY_TYPE_MEDIA_AUDIO, GST_RANK_MARGINAL);
742         GList* videoDecoderFactories = gst_element_factory_list_get_elements(GST_ELEMENT_FACTORY_TYPE_DECODER | GST_ELEMENT_FACTORY_TYPE_MEDIA_VIDEO, GST_RANK_MARGINAL);
743
744         enum ElementType {
745             AudioDecoder = 0,
746             VideoDecoder
747         };
748         struct GstCapsWebKitMapping {
749             ElementType elementType;
750             const char* capsString;
751             Vector<AtomicString> webkitCodecs;
752         };
753
754         GstCapsWebKitMapping mapping[] = {
755             { VideoDecoder, "video/x-h264,  profile=(string){ constrained-baseline, baseline }", { "x-h264", "avc*" } },
756             { VideoDecoder, "video/mpeg, mpegversion=(int){1,2}, systemstream=(boolean)false", { "mpeg" } },
757             { VideoDecoder, "video/x-vp8", { "vp8", "x-vp8" } },
758             { VideoDecoder, "video/x-vp9", { "vp9", "x-vp9" } },
759             { AudioDecoder, "audio/x-vorbis", { "vorbis", "x-vorbis" } },
760             { AudioDecoder, "audio/x-opus", { "opus", "x-opus" } }
761         };
762
763         for (auto& current : mapping) {
764             GList* factories = nullptr;
765             switch (current.elementType) {
766             case AudioDecoder:
767                 factories = audioDecoderFactories;
768                 break;
769             case VideoDecoder:
770                 factories = videoDecoderFactories;
771                 break;
772             default:
773                 g_assert_not_reached();
774                 break;
775             }
776
777             g_assert_nonnull(factories);
778
779             if (gstRegistryHasElementForMediaType(factories, current.capsString) && factories != nullptr) {
780                 if (!current.webkitCodecs.isEmpty()) {
781                     for (const auto& mimeType : current.webkitCodecs)
782                         set.add(mimeType);
783                 } else
784                     set.add(AtomicString(current.capsString));
785             }
786         }
787
788         bool audioMpegSupported = false;
789         if (gstRegistryHasElementForMediaType(audioDecoderFactories, "audio/mpeg, mpegversion=(int)1, layer=(int)[1, 3]")) {
790             audioMpegSupported = true;
791             set.add(AtomicString("audio/mp3"));
792         }
793
794         if (gstRegistryHasElementForMediaType(audioDecoderFactories, "audio/mpeg, mpegversion=(int){2, 4}")) {
795             audioMpegSupported = true;
796             set.add(AtomicString("mp4a*"));
797         }
798
799         if (audioMpegSupported) {
800             set.add(AtomicString("audio/mpeg"));
801             set.add(AtomicString("audio/x-mpeg"));
802         }
803
804
805         gst_plugin_feature_list_free(audioDecoderFactories);
806         gst_plugin_feature_list_free(videoDecoderFactories);
807
808         return set;
809     }();
810     return codecTypes;
811 }
812
813 bool MediaPlayerPrivateGStreamerMSE::supportsCodec(String codec)
814 {
815     // If the codec is named like a mimetype (eg: video/avc) remove the "video/" part.
816     size_t slashIndex = codec.find('/');
817     if (slashIndex != WTF::notFound)
818         codec = codec.substring(slashIndex+1);
819
820     for (const auto& pattern : codecSet()) {
821         bool codecMatchesPattern = !fnmatch(pattern.string().utf8().data(), codec.utf8().data(), 0);
822         if (codecMatchesPattern)
823             return true;
824     }
825
826     return false;
827 }
828
829 bool MediaPlayerPrivateGStreamerMSE::supportsAllCodecs(const Vector<String>& codecs)
830 {
831     for (String codec : codecs) {
832         if (!supportsCodec(codec))
833             return false;
834     }
835
836     return true;
837 }
838
839 MediaPlayer::SupportsType MediaPlayerPrivateGStreamerMSE::supportsType(const MediaEngineSupportParameters& parameters)
840 {
841     MediaPlayer::SupportsType result = MediaPlayer::IsNotSupported;
842     if (!parameters.isMediaSource)
843         return result;
844
845     auto containerType = parameters.type.containerType();
846
847     // YouTube TV provides empty types for some videos and we want to be selected as best media engine for them.
848     if (containerType.isEmpty()) {
849         result = MediaPlayer::MayBeSupported;
850         return result;
851     }
852
853     // Spec says we should not return "probably" if the codecs string is empty.
854     if (mimeTypeCache().contains(containerType)) {
855         Vector<String> codecs = parameters.type.codecs();
856         if (codecs.isEmpty())
857             result = MediaPlayer::MayBeSupported;
858         else
859             result = supportsAllCodecs(codecs) ? MediaPlayer::IsSupported : MediaPlayer::IsNotSupported;
860     }
861
862     return extendedSupportsType(parameters, result);
863 }
864
865 void MediaPlayerPrivateGStreamerMSE::markEndOfStream(MediaSourcePrivate::EndOfStreamStatus status)
866 {
867     if (status != MediaSourcePrivate::EosNoError)
868         return;
869
870     GST_DEBUG("Marking end of stream");
871     m_eosMarked = true;
872     updateStates();
873 }
874
875 MediaTime MediaPlayerPrivateGStreamerMSE::currentMediaTime() const
876 {
877     MediaTime position = MediaPlayerPrivateGStreamer::currentMediaTime();
878
879     if (m_eosPending && (paused() || (position >= durationMediaTime()))) {
880         if (m_networkState != MediaPlayer::Loaded) {
881             m_networkState = MediaPlayer::Loaded;
882             m_player->networkStateChanged();
883         }
884
885         m_eosPending = false;
886         m_isEndReached = true;
887         m_cachedPosition = m_mediaTimeDuration;
888         m_durationAtEOS = m_mediaTimeDuration;
889         m_player->timeChanged();
890     }
891     return position;
892 }
893
894 MediaTime MediaPlayerPrivateGStreamerMSE::maxMediaTimeSeekable() const
895 {
896     if (UNLIKELY(m_errorOccured))
897         return MediaTime::zeroTime();
898
899     GST_DEBUG("maxMediaTimeSeekable");
900     MediaTime result = durationMediaTime();
901     // Infinite duration means live stream.
902     if (result.isPositiveInfinite()) {
903         MediaTime maxBufferedTime = buffered()->maximumBufferedTime();
904         // Return the highest end time reported by the buffered attribute.
905         result = maxBufferedTime.isValid() ? maxBufferedTime : MediaTime::zeroTime();
906     }
907
908     return result;
909 }
910
911 #if ENABLE(ENCRYPTED_MEDIA)
912 void MediaPlayerPrivateGStreamerMSE::attemptToDecryptWithLocalInstance()
913 {
914     if (is<CDMInstanceClearKey>(*m_cdmInstance)) {
915         auto& clearkeyCDMInstance = downcast<CDMInstanceClearKey>(*m_cdmInstance);
916         if (clearkeyCDMInstance.keys().isEmpty())
917             return;
918
919         GValue keyIDList = G_VALUE_INIT, keyValueList = G_VALUE_INIT;
920         g_value_init(&keyIDList, GST_TYPE_LIST);
921         g_value_init(&keyValueList, GST_TYPE_LIST);
922
923         auto appendBuffer =
924             [](GValue* valueList, const SharedBuffer& buffer)
925             {
926                 GValue* bufferValue = g_new0(GValue, 1);
927                 g_value_init(bufferValue, GST_TYPE_BUFFER);
928                 gst_value_take_buffer(bufferValue,
929                     gst_buffer_new_wrapped(g_memdup(buffer.data(), buffer.size()), buffer.size()));
930                 gst_value_list_append_and_take_value(valueList, bufferValue);
931             };
932
933         for (auto& key : clearkeyCDMInstance.keys()) {
934             appendBuffer(&keyIDList, *key.keyIDData);
935             appendBuffer(&keyValueList, *key.keyValueData);
936         }
937
938         GUniquePtr<GstStructure> structure(gst_structure_new_empty("drm-cipher-clearkey"));
939         gst_structure_set_value(structure.get(), "key-ids", &keyIDList);
940         gst_structure_set_value(structure.get(), "key-values", &keyValueList);
941
942         gst_element_send_event(m_playbackPipeline->pipeline(), gst_event_new_custom(GST_EVENT_CUSTOM_DOWNSTREAM_OOB, structure.release()));
943     }
944 }
945 #endif
946
947 } // namespace WebCore.
948
949 #endif // USE(GSTREAMER)