[GStreamer] Refactor media player to use MediaTime consistently
[WebKit-https.git] / Source / WebCore / platform / graphics / gstreamer / mse / MediaPlayerPrivateGStreamerMSE.cpp
1 /*
2  * Copyright (C) 2007, 2009 Apple Inc.  All rights reserved.
3  * Copyright (C) 2007 Collabora Ltd.  All rights reserved.
4  * Copyright (C) 2007 Alp Toker <alp@atoker.com>
5  * Copyright (C) 2009 Gustavo Noronha Silva <gns@gnome.org>
6  * Copyright (C) 2009, 2010, 2011, 2012, 2013, 2016, 2017 Igalia S.L
7  * Copyright (C) 2015 Sebastian Dröge <sebastian@centricular.com>
8  * Copyright (C) 2015, 2016, 2017 Metrological Group B.V.
9  *
10  * This library is free software; you can redistribute it and/or
11  * modify it under the terms of the GNU Library General Public
12  * License as published by the Free Software Foundation; either
13  * version 2 of the License, or (at your option) any later version.
14  *
15  * This library is distributed in the hope that it will be useful,
16  * but WITHOUT ANY WARRANTY; without even the implied warranty of
17  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
18  * Library General Public License for more details.
19  *
20  * You should have received a copy of the GNU Library General Public License
21  * aint with this library; see the file COPYING.LIB.  If not, write to
22  * the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
23  * Boston, MA 02110-1301, USA.
24  */
25
26 #include "config.h"
27 #include "MediaPlayerPrivateGStreamerMSE.h"
28
29 #if ENABLE(VIDEO) && USE(GSTREAMER) && ENABLE(MEDIA_SOURCE)
30
31 #include "AppendPipeline.h"
32 #include "AudioTrackPrivateGStreamer.h"
33 #include "GStreamerUtilities.h"
34 #include "InbandTextTrackPrivateGStreamer.h"
35 #include "MIMETypeRegistry.h"
36 #include "MediaDescription.h"
37 #include "MediaPlayer.h"
38 #include "NotImplemented.h"
39 #include "SourceBufferPrivateGStreamer.h"
40 #include "TimeRanges.h"
41 #include "URL.h"
42 #include "VideoTrackPrivateGStreamer.h"
43
44 #include <fnmatch.h>
45 #include <gst/app/gstappsink.h>
46 #include <gst/app/gstappsrc.h>
47 #include <gst/gst.h>
48 #include <gst/pbutils/pbutils.h>
49 #include <gst/video/video.h>
50 #include <wtf/Condition.h>
51 #include <wtf/HashSet.h>
52 #include <wtf/NeverDestroyed.h>
53 #include <wtf/StringPrintStream.h>
54 #include <wtf/text/AtomicString.h>
55 #include <wtf/text/AtomicStringHash.h>
56
57 #if ENABLE(ENCRYPTED_MEDIA)
58 #include "CDMClearKey.h"
59 #include "SharedBuffer.h"
60 #endif
61
62 static const char* dumpReadyState(WebCore::MediaPlayer::ReadyState readyState)
63 {
64     switch (readyState) {
65     case WebCore::MediaPlayer::HaveNothing: return "HaveNothing";
66     case WebCore::MediaPlayer::HaveMetadata: return "HaveMetadata";
67     case WebCore::MediaPlayer::HaveCurrentData: return "HaveCurrentData";
68     case WebCore::MediaPlayer::HaveFutureData: return "HaveFutureData";
69     case WebCore::MediaPlayer::HaveEnoughData: return "HaveEnoughData";
70     default: return "(unknown)";
71     }
72 }
73
74 GST_DEBUG_CATEGORY(webkit_mse_debug);
75 #define GST_CAT_DEFAULT webkit_mse_debug
76
77 namespace WebCore {
78
79 void MediaPlayerPrivateGStreamerMSE::registerMediaEngine(MediaEngineRegistrar registrar)
80 {
81     if (isAvailable()) {
82         registrar([](MediaPlayer* player) { return std::make_unique<MediaPlayerPrivateGStreamerMSE>(player); },
83             getSupportedTypes, supportsType, nullptr, nullptr, nullptr, supportsKeySystem);
84     }
85 }
86
87 bool initializeGStreamerAndRegisterWebKitMSEElement()
88 {
89     if (UNLIKELY(!initializeGStreamer()))
90         return false;
91
92     registerWebKitGStreamerElements();
93
94     GST_DEBUG_CATEGORY_INIT(webkit_mse_debug, "webkitmse", 0, "WebKit MSE media player");
95
96     GRefPtr<GstElementFactory> WebKitMediaSrcFactory = adoptGRef(gst_element_factory_find("webkitmediasrc"));
97     if (UNLIKELY(!WebKitMediaSrcFactory))
98         gst_element_register(nullptr, "webkitmediasrc", GST_RANK_PRIMARY + 100, WEBKIT_TYPE_MEDIA_SRC);
99     return true;
100 }
101
102 bool MediaPlayerPrivateGStreamerMSE::isAvailable()
103 {
104     if (UNLIKELY(!initializeGStreamerAndRegisterWebKitMSEElement()))
105         return false;
106
107     GRefPtr<GstElementFactory> factory = adoptGRef(gst_element_factory_find("playbin"));
108     return factory;
109 }
110
111 MediaPlayerPrivateGStreamerMSE::MediaPlayerPrivateGStreamerMSE(MediaPlayer* player)
112     : MediaPlayerPrivateGStreamer(player)
113 {
114     GST_TRACE("creating the player (%p)", this);
115 }
116
117 MediaPlayerPrivateGStreamerMSE::~MediaPlayerPrivateGStreamerMSE()
118 {
119     GST_TRACE("destroying the player (%p)", this);
120
121     for (auto iterator : m_appendPipelinesMap)
122         iterator.value->clearPlayerPrivate();
123
124     if (m_source) {
125         webKitMediaSrcSetMediaPlayerPrivate(WEBKIT_MEDIA_SRC(m_source.get()), nullptr);
126         g_signal_handlers_disconnect_by_data(m_source.get(), this);
127     }
128
129     if (m_playbackPipeline)
130         m_playbackPipeline->setWebKitMediaSrc(nullptr);
131 }
132
133 void MediaPlayerPrivateGStreamerMSE::load(const String& urlString)
134 {
135     if (!urlString.startsWith("mediasource")) {
136         // Properly fail so the global MediaPlayer tries to fallback to the next MediaPlayerPrivate.
137         m_networkState = MediaPlayer::FormatError;
138         m_player->networkStateChanged();
139         return;
140     }
141
142     if (UNLIKELY(!initializeGStreamerAndRegisterWebKitMSEElement()))
143         return;
144
145     if (!m_playbackPipeline)
146         m_playbackPipeline = PlaybackPipeline::create();
147
148     MediaPlayerPrivateGStreamer::load(urlString);
149 }
150
151 void MediaPlayerPrivateGStreamerMSE::load(const String& url, MediaSourcePrivateClient* mediaSource)
152 {
153     m_mediaSource = mediaSource;
154     load(String::format("mediasource%s", url.utf8().data()));
155 }
156
157 void MediaPlayerPrivateGStreamerMSE::pause()
158 {
159     m_paused = true;
160     MediaPlayerPrivateGStreamer::pause();
161 }
162
163 MediaTime MediaPlayerPrivateGStreamerMSE::durationMediaTime() const
164 {
165     if (UNLIKELY(!m_pipeline || m_errorOccured))
166         return MediaTime();
167
168     return m_mediaTimeDuration;
169 }
170
171 void MediaPlayerPrivateGStreamerMSE::seek(const MediaTime& time)
172 {
173     if (UNLIKELY(!m_pipeline || m_errorOccured))
174         return;
175
176     GST_INFO("[Seek] seek attempt to %s secs", toString(time).utf8().data());
177
178     // Avoid useless seeking.
179     MediaTime current = currentMediaTime();
180     if (time == current) {
181         if (!m_seeking)
182             timeChanged();
183         return;
184     }
185
186     if (isLiveStream())
187         return;
188
189     if (m_seeking && m_seekIsPending) {
190         m_seekTime = time;
191         return;
192     }
193
194     GST_DEBUG("Seeking from %s to %s seconds", toString(current).utf8().data(), toString(time).utf8().data());
195
196     MediaTime previousSeekTime = m_seekTime;
197     m_seekTime = time;
198
199     if (!doSeek()) {
200         m_seekTime = previousSeekTime;
201         GST_WARNING("Seeking to %s failed", toString(time).utf8().data());
202         return;
203     }
204
205     m_isEndReached = false;
206     GST_DEBUG("m_seeking=%s, m_seekTime=%s", boolForPrinting(m_seeking), toString(m_seekTime).utf8().data());
207 }
208
209 void MediaPlayerPrivateGStreamerMSE::configurePlaySink()
210 {
211     MediaPlayerPrivateGStreamer::configurePlaySink();
212
213     GRefPtr<GstElement> playsink = adoptGRef(gst_bin_get_by_name(GST_BIN(m_pipeline.get()), "playsink"));
214     if (playsink) {
215         // The default value (0) means "send events to all the sinks", instead
216         // of "only to the first that returns true". This is needed for MSE seek.
217         g_object_set(G_OBJECT(playsink.get()), "send-event-mode", 0, nullptr);
218     }
219 }
220
221 bool MediaPlayerPrivateGStreamerMSE::changePipelineState(GstState newState)
222 {
223     if (seeking()) {
224         GST_DEBUG("Rejected state change to %s while seeking",
225             gst_element_state_get_name(newState));
226         return true;
227     }
228
229     return MediaPlayerPrivateGStreamer::changePipelineState(newState);
230 }
231
232 void MediaPlayerPrivateGStreamerMSE::notifySeekNeedsDataForTime(const MediaTime& seekTime)
233 {
234     // Reenqueue samples needed to resume playback in the new position.
235     m_mediaSource->seekToTime(seekTime);
236
237     GST_DEBUG("MSE seek to %s finished", toString(seekTime).utf8().data());
238
239     if (!m_gstSeekCompleted) {
240         m_gstSeekCompleted = true;
241         maybeFinishSeek();
242     }
243 }
244
245 bool MediaPlayerPrivateGStreamerMSE::doSeek(const MediaTime&, float, GstSeekFlags)
246 {
247     // Use doSeek() instead. If anybody is calling this version of doSeek(), something is wrong.
248     ASSERT_NOT_REACHED();
249     return false;
250 }
251
252 bool MediaPlayerPrivateGStreamerMSE::doSeek()
253 {
254     MediaTime seekTime = m_seekTime;
255     double rate = m_player->rate();
256     GstSeekFlags seekType = static_cast<GstSeekFlags>(GST_SEEK_FLAG_FLUSH | GST_SEEK_FLAG_ACCURATE);
257
258     // Always move to seeking state to report correct 'currentTime' while pending for actual seek to complete.
259     m_seeking = true;
260
261     // Check if playback pipeline is ready for seek.
262     GstState state, newState;
263     GstStateChangeReturn getStateResult = gst_element_get_state(m_pipeline.get(), &state, &newState, 0);
264     if (getStateResult == GST_STATE_CHANGE_FAILURE || getStateResult == GST_STATE_CHANGE_NO_PREROLL) {
265         GST_DEBUG("[Seek] cannot seek, current state change is %s", gst_element_state_change_return_get_name(getStateResult));
266         webKitMediaSrcSetReadyForSamples(WEBKIT_MEDIA_SRC(m_source.get()), true);
267         m_seeking = false;
268         return false;
269     }
270     if ((getStateResult == GST_STATE_CHANGE_ASYNC
271         && !(state == GST_STATE_PLAYING && newState == GST_STATE_PAUSED))
272         || state < GST_STATE_PAUSED
273         || m_isEndReached
274         || !m_gstSeekCompleted) {
275         CString reason = "Unknown reason";
276         if (getStateResult == GST_STATE_CHANGE_ASYNC) {
277             reason = String::format("In async change %s --> %s",
278                 gst_element_state_get_name(state),
279                 gst_element_state_get_name(newState)).utf8();
280         } else if (state < GST_STATE_PAUSED)
281             reason = "State less than PAUSED";
282         else if (m_isEndReached)
283             reason = "End reached";
284         else if (!m_gstSeekCompleted)
285             reason = "Previous seek is not finished yet";
286
287         GST_DEBUG("[Seek] Delaying the seek: %s", reason.data());
288
289         m_seekIsPending = true;
290
291         if (m_isEndReached) {
292             GST_DEBUG("[Seek] reset pipeline");
293             m_resetPipeline = true;
294             m_seeking = false;
295             if (!changePipelineState(GST_STATE_PAUSED))
296                 loadingFailed(MediaPlayer::Empty);
297             else
298                 m_seeking = true;
299         }
300
301         return m_seeking;
302     }
303
304     // Stop accepting new samples until actual seek is finished.
305     webKitMediaSrcSetReadyForSamples(WEBKIT_MEDIA_SRC(m_source.get()), false);
306
307     // Correct seek time if it helps to fix a small gap.
308     if (!isTimeBuffered(seekTime)) {
309         // Look if a near future time (<0.1 sec.) is buffered and change the seek target time.
310         if (m_mediaSource) {
311             const MediaTime miniGap = MediaTime(1, 10);
312             MediaTime nearest = m_mediaSource->buffered()->nearest(seekTime);
313             if (nearest.isValid() && nearest > seekTime && (nearest - seekTime) <= miniGap && isTimeBuffered(nearest + miniGap)) {
314                 GST_DEBUG("[Seek] Changed the seek target time from %s to %s, a near point in the future", toString(seekTime).utf8().data(), toString(nearest).utf8().data());
315                 seekTime = nearest;
316             }
317         }
318     }
319
320     // Check if MSE has samples for requested time and defer actual seek if needed.
321     if (!isTimeBuffered(seekTime)) {
322         GST_DEBUG("[Seek] Delaying the seek: MSE is not ready");
323         GstStateChangeReturn setStateResult = gst_element_set_state(m_pipeline.get(), GST_STATE_PAUSED);
324         if (setStateResult == GST_STATE_CHANGE_FAILURE) {
325             GST_DEBUG("[Seek] Cannot seek, failed to pause playback pipeline.");
326             webKitMediaSrcSetReadyForSamples(WEBKIT_MEDIA_SRC(m_source.get()), true);
327             m_seeking = false;
328             return false;
329         }
330         m_readyState = MediaPlayer::HaveMetadata;
331         notifySeekNeedsDataForTime(seekTime);
332         ASSERT(!m_mseSeekCompleted);
333         return true;
334     }
335
336     // Complete previous MSE seek if needed.
337     if (!m_mseSeekCompleted) {
338         m_mediaSource->monitorSourceBuffers();
339         ASSERT(m_mseSeekCompleted);
340         // Note: seekCompleted will recursively call us.
341         return m_seeking;
342     }
343
344     GST_DEBUG("We can seek now");
345
346     MediaTime startTime = seekTime, endTime = MediaTime::invalidTime();
347
348     if (rate < 0) {
349         startTime = MediaTime::zeroTime();
350         endTime = seekTime;
351     }
352
353     if (!rate)
354         rate = 1;
355
356     GST_DEBUG("Actual seek to %s, end time:  %s, rate: %f", toString(startTime).utf8().data(), toString(endTime).utf8().data(), rate);
357
358     // This will call notifySeekNeedsData() after some time to tell that the pipeline is ready for sample enqueuing.
359     webKitMediaSrcPrepareSeek(WEBKIT_MEDIA_SRC(m_source.get()), seekTime);
360
361     m_gstSeekCompleted = false;
362     if (!gst_element_seek(m_pipeline.get(), rate, GST_FORMAT_TIME, seekType, GST_SEEK_TYPE_SET, toGstClockTime(startTime), GST_SEEK_TYPE_SET, toGstClockTime(endTime))) {
363         webKitMediaSrcSetReadyForSamples(WEBKIT_MEDIA_SRC(m_source.get()), true);
364         m_seeking = false;
365         m_gstSeekCompleted = true;
366         GST_DEBUG("doSeek(): gst_element_seek() failed, returning false");
367         return false;
368     }
369
370     // The samples will be enqueued in notifySeekNeedsData().
371     GST_DEBUG("doSeek(): gst_element_seek() succeeded, returning true");
372     return true;
373 }
374
375 void MediaPlayerPrivateGStreamerMSE::maybeFinishSeek()
376 {
377     if (!m_seeking || !m_mseSeekCompleted || !m_gstSeekCompleted)
378         return;
379
380     GstState state, newState;
381     GstStateChangeReturn getStateResult = gst_element_get_state(m_pipeline.get(), &state, &newState, 0);
382
383     if (getStateResult == GST_STATE_CHANGE_ASYNC
384         && !(state == GST_STATE_PLAYING && newState == GST_STATE_PAUSED)) {
385         GST_DEBUG("[Seek] Delaying seek finish");
386         return;
387     }
388
389     if (m_seekIsPending) {
390         GST_DEBUG("[Seek] Committing pending seek to %s", toString(m_seekTime).utf8().data());
391         m_seekIsPending = false;
392         if (!doSeek()) {
393             GST_WARNING("[Seek] Seeking to %s failed", toString(m_seekTime).utf8().data());
394             m_cachedPosition = MediaTime::invalidTime();
395         }
396         return;
397     }
398
399     GST_DEBUG("[Seek] Seeked to %s", toString(m_seekTime).utf8().data());
400
401     webKitMediaSrcSetReadyForSamples(WEBKIT_MEDIA_SRC(m_source.get()), true);
402     m_seeking = false;
403     m_cachedPosition = MediaTime::invalidTime();
404     // The pipeline can still have a pending state. In this case a position query will fail.
405     // Right now we can use m_seekTime as a fallback.
406     m_canFallBackToLastFinishedSeekPosition = true;
407     timeChanged();
408 }
409
410 void MediaPlayerPrivateGStreamerMSE::updatePlaybackRate()
411 {
412     notImplemented();
413 }
414
415 bool MediaPlayerPrivateGStreamerMSE::seeking() const
416 {
417     return m_seeking;
418 }
419
420 // FIXME: MediaPlayerPrivateGStreamer manages the ReadyState on its own. We shouldn't change it manually.
421 void MediaPlayerPrivateGStreamerMSE::setReadyState(MediaPlayer::ReadyState readyState)
422 {
423     if (readyState == m_readyState)
424         return;
425
426     if (seeking()) {
427         GST_DEBUG("Skip ready state change(%s -> %s) due to seek\n", dumpReadyState(m_readyState), dumpReadyState(readyState));
428         return;
429     }
430
431     GST_DEBUG("Ready State Changed manually from %u to %u", m_readyState, readyState);
432     MediaPlayer::ReadyState oldReadyState = m_readyState;
433     m_readyState = readyState;
434     GST_DEBUG("m_readyState: %s -> %s", dumpReadyState(oldReadyState), dumpReadyState(m_readyState));
435
436     if (oldReadyState < MediaPlayer::HaveCurrentData && m_readyState >= MediaPlayer::HaveCurrentData) {
437         GST_DEBUG("[Seek] Reporting load state changed to trigger seek continuation");
438         loadStateChanged();
439     }
440     m_player->readyStateChanged();
441
442     GstState pipelineState;
443     GstStateChangeReturn getStateResult = gst_element_get_state(m_pipeline.get(), &pipelineState, nullptr, 250 * GST_NSECOND);
444     bool isPlaying = (getStateResult == GST_STATE_CHANGE_SUCCESS && pipelineState == GST_STATE_PLAYING);
445
446     if (m_readyState == MediaPlayer::HaveMetadata && oldReadyState > MediaPlayer::HaveMetadata && isPlaying) {
447         GST_TRACE("Changing pipeline to PAUSED...");
448         bool ok = changePipelineState(GST_STATE_PAUSED);
449         GST_TRACE("Changed pipeline to PAUSED: %s", ok ? "Success" : "Error");
450     }
451 }
452
453 void MediaPlayerPrivateGStreamerMSE::waitForSeekCompleted()
454 {
455     if (!m_seeking)
456         return;
457
458     GST_DEBUG("Waiting for MSE seek completed");
459     m_mseSeekCompleted = false;
460 }
461
462 void MediaPlayerPrivateGStreamerMSE::seekCompleted()
463 {
464     if (m_mseSeekCompleted)
465         return;
466
467     GST_DEBUG("MSE seek completed");
468     m_mseSeekCompleted = true;
469
470     doSeek();
471
472     if (!seeking() && m_readyState >= MediaPlayer::HaveFutureData)
473         changePipelineState(GST_STATE_PLAYING);
474
475     if (!seeking())
476         m_player->timeChanged();
477 }
478
479 void MediaPlayerPrivateGStreamerMSE::setRate(float)
480 {
481     notImplemented();
482 }
483
484 std::unique_ptr<PlatformTimeRanges> MediaPlayerPrivateGStreamerMSE::buffered() const
485 {
486     return m_mediaSource ? m_mediaSource->buffered() : std::make_unique<PlatformTimeRanges>();
487 }
488
489 void MediaPlayerPrivateGStreamerMSE::sourceChanged()
490 {
491     m_source = nullptr;
492     g_object_get(m_pipeline.get(), "source", &m_source.outPtr(), nullptr);
493
494     ASSERT(WEBKIT_IS_MEDIA_SRC(m_source.get()));
495
496     m_playbackPipeline->setWebKitMediaSrc(WEBKIT_MEDIA_SRC(m_source.get()));
497
498     MediaSourceGStreamer::open(*m_mediaSource.get(), *this);
499     g_signal_connect_swapped(m_source.get(), "video-changed", G_CALLBACK(videoChangedCallback), this);
500     g_signal_connect_swapped(m_source.get(), "audio-changed", G_CALLBACK(audioChangedCallback), this);
501     g_signal_connect_swapped(m_source.get(), "text-changed", G_CALLBACK(textChangedCallback), this);
502     webKitMediaSrcSetMediaPlayerPrivate(WEBKIT_MEDIA_SRC(m_source.get()), this);
503 }
504
505 void MediaPlayerPrivateGStreamerMSE::updateStates()
506 {
507     if (UNLIKELY(!m_pipeline || m_errorOccured))
508         return;
509
510     MediaPlayer::NetworkState oldNetworkState = m_networkState;
511     MediaPlayer::ReadyState oldReadyState = m_readyState;
512     GstState state, pending;
513
514     GstStateChangeReturn getStateResult = gst_element_get_state(m_pipeline.get(), &state, &pending, 250 * GST_NSECOND);
515
516     bool shouldUpdatePlaybackState = false;
517     switch (getStateResult) {
518     case GST_STATE_CHANGE_SUCCESS: {
519         GST_DEBUG("State: %s, pending: %s", gst_element_state_get_name(state), gst_element_state_get_name(pending));
520
521         // Do nothing if on EOS and state changed to READY to avoid recreating the player
522         // on HTMLMediaElement and properly generate the video 'ended' event.
523         if (m_isEndReached && state == GST_STATE_READY)
524             break;
525
526         m_resetPipeline = (state <= GST_STATE_READY);
527         if (m_resetPipeline)
528             m_mediaTimeDuration = MediaTime::zeroTime();
529
530         // Update ready and network states.
531         switch (state) {
532         case GST_STATE_NULL:
533             m_readyState = MediaPlayer::HaveNothing;
534             GST_DEBUG("m_readyState=%s", dumpReadyState(m_readyState));
535             m_networkState = MediaPlayer::Empty;
536             break;
537         case GST_STATE_READY:
538             m_readyState = MediaPlayer::HaveMetadata;
539             GST_DEBUG("m_readyState=%s", dumpReadyState(m_readyState));
540             m_networkState = MediaPlayer::Empty;
541             break;
542         case GST_STATE_PAUSED:
543         case GST_STATE_PLAYING:
544             if (seeking()) {
545                 m_readyState = MediaPlayer::HaveMetadata;
546                 // FIXME: Should we manage NetworkState too?
547                 GST_DEBUG("m_readyState=%s", dumpReadyState(m_readyState));
548             } else if (m_buffering) {
549                 if (m_bufferingPercentage == 100) {
550                     GST_DEBUG("[Buffering] Complete.");
551                     m_buffering = false;
552                     m_readyState = MediaPlayer::HaveEnoughData;
553                     GST_DEBUG("m_readyState=%s", dumpReadyState(m_readyState));
554                     m_networkState = m_downloadFinished ? MediaPlayer::Idle : MediaPlayer::Loading;
555                 } else {
556                     m_readyState = MediaPlayer::HaveCurrentData;
557                     GST_DEBUG("m_readyState=%s", dumpReadyState(m_readyState));
558                     m_networkState = MediaPlayer::Loading;
559                 }
560             } else if (m_downloadFinished) {
561                 m_readyState = MediaPlayer::HaveEnoughData;
562                 GST_DEBUG("m_readyState=%s", dumpReadyState(m_readyState));
563                 m_networkState = MediaPlayer::Loaded;
564             } else {
565                 m_readyState = MediaPlayer::HaveFutureData;
566                 GST_DEBUG("m_readyState=%s", dumpReadyState(m_readyState));
567                 m_networkState = MediaPlayer::Loading;
568             }
569
570             if (m_eosMarked && state == GST_STATE_PLAYING)
571                 m_eosPending = true;
572
573             break;
574         default:
575             ASSERT_NOT_REACHED();
576             break;
577         }
578
579         // Sync states where needed.
580         if (state == GST_STATE_PAUSED) {
581             if (!m_volumeAndMuteInitialized) {
582                 notifyPlayerOfVolumeChange();
583                 notifyPlayerOfMute();
584                 m_volumeAndMuteInitialized = true;
585             }
586
587             if (!seeking() && !m_buffering && !m_paused && m_playbackRate) {
588                 GST_DEBUG("[Buffering] Restarting playback.");
589                 changePipelineState(GST_STATE_PLAYING);
590             }
591         } else if (state == GST_STATE_PLAYING) {
592             m_paused = false;
593
594             if ((m_buffering && !isLiveStream()) || !m_playbackRate) {
595                 GST_DEBUG("[Buffering] Pausing stream for buffering.");
596                 changePipelineState(GST_STATE_PAUSED);
597             }
598         } else
599             m_paused = true;
600
601         if (m_requestedState == GST_STATE_PAUSED && state == GST_STATE_PAUSED) {
602             shouldUpdatePlaybackState = true;
603             GST_DEBUG("Requested state change to %s was completed", gst_element_state_get_name(state));
604         }
605
606         break;
607     }
608     case GST_STATE_CHANGE_ASYNC:
609         GST_DEBUG("Async: State: %s, pending: %s", gst_element_state_get_name(state), gst_element_state_get_name(pending));
610         // Change in progress.
611         break;
612     case GST_STATE_CHANGE_FAILURE:
613         GST_WARNING("Failure: State: %s, pending: %s", gst_element_state_get_name(state), gst_element_state_get_name(pending));
614         // Change failed.
615         return;
616     case GST_STATE_CHANGE_NO_PREROLL:
617         GST_DEBUG("No preroll: State: %s, pending: %s", gst_element_state_get_name(state), gst_element_state_get_name(pending));
618
619         // Live pipelines go in PAUSED without prerolling.
620         m_isStreaming = true;
621
622         if (state == GST_STATE_READY) {
623             m_readyState = MediaPlayer::HaveNothing;
624             GST_DEBUG("m_readyState=%s", dumpReadyState(m_readyState));
625         } else if (state == GST_STATE_PAUSED) {
626             m_readyState = MediaPlayer::HaveEnoughData;
627             GST_DEBUG("m_readyState=%s", dumpReadyState(m_readyState));
628             m_paused = true;
629         } else if (state == GST_STATE_PLAYING)
630             m_paused = false;
631
632         if (!m_paused && m_playbackRate)
633             changePipelineState(GST_STATE_PLAYING);
634
635         m_networkState = MediaPlayer::Loading;
636         break;
637     default:
638         GST_DEBUG("Else : %d", getStateResult);
639         break;
640     }
641
642     m_requestedState = GST_STATE_VOID_PENDING;
643
644     if (shouldUpdatePlaybackState)
645         m_player->playbackStateChanged();
646
647     if (m_networkState != oldNetworkState) {
648         GST_DEBUG("Network State Changed from %u to %u", oldNetworkState, m_networkState);
649         m_player->networkStateChanged();
650     }
651     if (m_readyState != oldReadyState) {
652         GST_DEBUG("Ready State Changed from %u to %u", oldReadyState, m_readyState);
653         m_player->readyStateChanged();
654     }
655
656     if (getStateResult == GST_STATE_CHANGE_SUCCESS && state >= GST_STATE_PAUSED) {
657         updatePlaybackRate();
658         maybeFinishSeek();
659     }
660 }
661 void MediaPlayerPrivateGStreamerMSE::asyncStateChangeDone()
662 {
663     if (UNLIKELY(!m_pipeline || m_errorOccured))
664         return;
665
666     if (m_seeking)
667         maybeFinishSeek();
668     else
669         updateStates();
670 }
671
672 bool MediaPlayerPrivateGStreamerMSE::isTimeBuffered(const MediaTime &time) const
673 {
674     bool result = m_mediaSource && m_mediaSource->buffered()->contain(time);
675     GST_DEBUG("Time %s buffered? %s", toString(time).utf8().data(), boolForPrinting(result));
676     return result;
677 }
678
679 void MediaPlayerPrivateGStreamerMSE::setMediaSourceClient(Ref<MediaSourceClientGStreamerMSE> client)
680 {
681     m_mediaSourceClient = client.ptr();
682 }
683
684 RefPtr<MediaSourceClientGStreamerMSE> MediaPlayerPrivateGStreamerMSE::mediaSourceClient()
685 {
686     return m_mediaSourceClient;
687 }
688
689 void MediaPlayerPrivateGStreamerMSE::durationChanged()
690 {
691     if (!m_mediaSourceClient) {
692         GST_DEBUG("m_mediaSourceClient is null, doing nothing");
693         return;
694     }
695
696     MediaTime previousDuration = m_mediaTimeDuration;
697     m_mediaTimeDuration = m_mediaSourceClient->duration();
698
699     GST_TRACE("previous=%s, new=%s", toString(previousDuration).utf8().data(), toString(m_mediaTimeDuration).utf8().data());
700
701     // Avoid emiting durationchanged in the case where the previous duration was 0 because that case is already handled
702     // by the HTMLMediaElement.
703     if (m_mediaTimeDuration != previousDuration && m_mediaTimeDuration.isValid() && previousDuration.isValid()) {
704         m_player->durationChanged();
705         m_playbackPipeline->notifyDurationChanged();
706         m_mediaSource->durationChanged(m_mediaTimeDuration);
707     }
708 }
709
710 static HashSet<String, ASCIICaseInsensitiveHash>& mimeTypeCache()
711 {
712     static NeverDestroyed<HashSet<String, ASCIICaseInsensitiveHash>> cache = []()
713     {
714         initializeGStreamerAndRegisterWebKitMSEElement();
715         HashSet<String, ASCIICaseInsensitiveHash> set;
716         const char* mimeTypes[] = {
717             "video/mp4",
718             "audio/mp4",
719             "video/webm",
720             "audio/webm"
721         };
722         for (auto& type : mimeTypes)
723             set.add(type);
724         return set;
725     }();
726     return cache;
727 }
728
729 void MediaPlayerPrivateGStreamerMSE::getSupportedTypes(HashSet<String, ASCIICaseInsensitiveHash>& types)
730 {
731     types = mimeTypeCache();
732 }
733
734 void MediaPlayerPrivateGStreamerMSE::trackDetected(RefPtr<AppendPipeline> appendPipeline, RefPtr<WebCore::TrackPrivateBase> newTrack, bool firstTrackDetected)
735 {
736     ASSERT(appendPipeline->track() == newTrack);
737
738     GstCaps* caps = appendPipeline->appsinkCaps();
739     ASSERT(caps);
740     GST_DEBUG("track ID: %s, caps: %" GST_PTR_FORMAT, newTrack->id().string().latin1().data(), caps);
741
742     GstStructure* structure = gst_caps_get_structure(caps, 0);
743     const gchar* mediaType = gst_structure_get_name(structure);
744     GstVideoInfo info;
745
746     if (g_str_has_prefix(mediaType, "video/") && gst_video_info_from_caps(&info, caps)) {
747         float width, height;
748
749         width = info.width;
750         height = info.height * ((float) info.par_d / (float) info.par_n);
751         m_videoSize.setWidth(width);
752         m_videoSize.setHeight(height);
753     }
754
755     if (firstTrackDetected)
756         m_playbackPipeline->attachTrack(appendPipeline->sourceBufferPrivate(), newTrack, structure, caps);
757     else
758         m_playbackPipeline->reattachTrack(appendPipeline->sourceBufferPrivate(), newTrack, mediaType);
759 }
760
761 const static HashSet<AtomicString>& codecSet()
762 {
763     static NeverDestroyed<HashSet<AtomicString>> codecTypes = []()
764     {
765         MediaPlayerPrivateGStreamerBase::initializeGStreamerAndRegisterWebKitElements();
766         HashSet<AtomicString> set;
767
768         GList* audioDecoderFactories = gst_element_factory_list_get_elements(GST_ELEMENT_FACTORY_TYPE_DECODER | GST_ELEMENT_FACTORY_TYPE_MEDIA_AUDIO, GST_RANK_MARGINAL);
769         GList* videoDecoderFactories = gst_element_factory_list_get_elements(GST_ELEMENT_FACTORY_TYPE_DECODER | GST_ELEMENT_FACTORY_TYPE_MEDIA_VIDEO, GST_RANK_MARGINAL);
770
771         enum ElementType {
772             AudioDecoder = 0,
773             VideoDecoder
774         };
775         struct GstCapsWebKitMapping {
776             ElementType elementType;
777             const char* capsString;
778             Vector<AtomicString> webkitCodecs;
779         };
780
781         std::array<GstCapsWebKitMapping, 7> mapping = { {
782             { VideoDecoder, "video/x-h264,  profile=(string){ constrained-baseline, baseline }", { "x-h264" } },
783             { VideoDecoder, "video/x-h264, stream-format=avc", { "avc*"} },
784             { VideoDecoder, "video/mpeg, mpegversion=(int){1,2}, systemstream=(boolean)false", { "mpeg" } },
785             { VideoDecoder, "video/x-vp8", { "vp8", "x-vp8" } },
786             { VideoDecoder, "video/x-vp9", { "vp9", "x-vp9" } },
787             { AudioDecoder, "audio/x-vorbis", { "vorbis", "x-vorbis" } },
788             { AudioDecoder, "audio/x-opus", { "opus", "x-opus" } }
789         } };
790
791         for (auto& current : mapping) {
792             GList* factories = nullptr;
793             switch (current.elementType) {
794             case AudioDecoder:
795                 factories = audioDecoderFactories;
796                 break;
797             case VideoDecoder:
798                 factories = videoDecoderFactories;
799                 break;
800             default:
801                 g_assert_not_reached();
802                 break;
803             }
804
805             g_assert_nonnull(factories);
806
807             if (gstRegistryHasElementForMediaType(factories, current.capsString) && factories != nullptr) {
808                 if (!current.webkitCodecs.isEmpty()) {
809                     for (const auto& mimeType : current.webkitCodecs)
810                         set.add(mimeType);
811                 } else
812                     set.add(AtomicString(current.capsString));
813             }
814         }
815
816         bool audioMpegSupported = false;
817         if (gstRegistryHasElementForMediaType(audioDecoderFactories, "audio/mpeg, mpegversion=(int)1, layer=(int)[1, 3]")) {
818             audioMpegSupported = true;
819             set.add(AtomicString("audio/mp3"));
820         }
821
822         if (gstRegistryHasElementForMediaType(audioDecoderFactories, "audio/mpeg, mpegversion=(int){2, 4}")) {
823             audioMpegSupported = true;
824             set.add(AtomicString("mp4a*"));
825         }
826
827         if (audioMpegSupported) {
828             set.add(AtomicString("audio/mpeg"));
829             set.add(AtomicString("audio/x-mpeg"));
830         }
831
832
833         gst_plugin_feature_list_free(audioDecoderFactories);
834         gst_plugin_feature_list_free(videoDecoderFactories);
835
836         return set;
837     }();
838     return codecTypes;
839 }
840
841 bool MediaPlayerPrivateGStreamerMSE::supportsCodecs(const String& codecs)
842 {
843     Vector<String> codecEntries;
844     codecs.split(',', false, codecEntries);
845
846     for (String codec : codecEntries) {
847         bool isCodecSupported = false;
848
849         // If the codec is named like a mimetype (eg: video/avc) remove the "video/" part.
850         size_t slashIndex = codec.find('/');
851         if (slashIndex != WTF::notFound)
852             codec = codec.substring(slashIndex+1);
853
854         const char* codecData = codec.utf8().data();
855         for (const auto& pattern : codecSet()) {
856             isCodecSupported = !fnmatch(pattern.string().utf8().data(), codecData, 0);
857             if (isCodecSupported)
858                 break;
859         }
860         if (!isCodecSupported)
861             return false;
862     }
863
864     return true;
865 }
866
867 MediaPlayer::SupportsType MediaPlayerPrivateGStreamerMSE::supportsType(const MediaEngineSupportParameters& parameters)
868 {
869     MediaPlayer::SupportsType result = MediaPlayer::IsNotSupported;
870     if (!parameters.isMediaSource)
871         return result;
872
873     auto containerType = parameters.type.containerType();
874
875     // YouTube TV provides empty types for some videos and we want to be selected as best media engine for them.
876     if (containerType.isEmpty()) {
877         result = MediaPlayer::MayBeSupported;
878         return result;
879     }
880
881     // Spec says we should not return "probably" if the codecs string is empty.
882     if (mimeTypeCache().contains(containerType)) {
883         String codecs = parameters.type.parameter(ContentType::codecsParameter());
884         if (codecs.isEmpty())
885             result = MediaPlayer::MayBeSupported;
886         else
887             result = supportsCodecs(codecs) ? MediaPlayer::IsSupported : MediaPlayer::IsNotSupported;
888     }
889
890     return extendedSupportsType(parameters, result);
891 }
892
893 void MediaPlayerPrivateGStreamerMSE::markEndOfStream(MediaSourcePrivate::EndOfStreamStatus status)
894 {
895     if (status != MediaSourcePrivate::EosNoError)
896         return;
897
898     GST_DEBUG("Marking end of stream");
899     m_eosMarked = true;
900     updateStates();
901 }
902
903 MediaTime MediaPlayerPrivateGStreamerMSE::currentMediaTime() const
904 {
905     MediaTime position = MediaPlayerPrivateGStreamer::currentMediaTime();
906
907     if (m_eosPending && (paused() || (position >= durationMediaTime()))) {
908         if (m_networkState != MediaPlayer::Loaded) {
909             m_networkState = MediaPlayer::Loaded;
910             m_player->networkStateChanged();
911         }
912
913         m_eosPending = false;
914         m_isEndReached = true;
915         m_cachedPosition = m_mediaTimeDuration;
916         m_durationAtEOS = m_mediaTimeDuration;
917         m_player->timeChanged();
918     }
919     return position;
920 }
921
922 MediaTime MediaPlayerPrivateGStreamerMSE::maxMediaTimeSeekable() const
923 {
924     if (UNLIKELY(m_errorOccured))
925         return MediaTime::zeroTime();
926
927     GST_DEBUG("maxMediaTimeSeekable");
928     MediaTime result = durationMediaTime();
929     // Infinite duration means live stream.
930     if (result.isPositiveInfinite()) {
931         MediaTime maxBufferedTime = buffered()->maximumBufferedTime();
932         // Return the highest end time reported by the buffered attribute.
933         result = maxBufferedTime.isValid() ? maxBufferedTime : MediaTime::zeroTime();
934     }
935
936     return result;
937 }
938
939 #if ENABLE(ENCRYPTED_MEDIA)
940 void MediaPlayerPrivateGStreamerMSE::attemptToDecryptWithInstance(const CDMInstance& instance)
941 {
942     if (is<CDMInstanceClearKey>(instance)) {
943         auto& ckInstance = downcast<CDMInstanceClearKey>(instance);
944         if (ckInstance.keys().isEmpty())
945             return;
946
947         GValue keyIDList = G_VALUE_INIT, keyValueList = G_VALUE_INIT;
948         g_value_init(&keyIDList, GST_TYPE_LIST);
949         g_value_init(&keyValueList, GST_TYPE_LIST);
950
951         auto appendBuffer =
952             [](GValue* valueList, const SharedBuffer& buffer)
953             {
954                 GValue* bufferValue = g_new0(GValue, 1);
955                 g_value_init(bufferValue, GST_TYPE_BUFFER);
956                 gst_value_take_buffer(bufferValue,
957                     gst_buffer_new_wrapped(g_memdup(buffer.data(), buffer.size()), buffer.size()));
958                 gst_value_list_append_and_take_value(valueList, bufferValue);
959             };
960
961         for (auto& key : ckInstance.keys()) {
962             appendBuffer(&keyIDList, *key.keyIDData);
963             appendBuffer(&keyValueList, *key.keyValueData);
964         }
965
966         GUniquePtr<GstStructure> structure(gst_structure_new_empty("drm-cipher-clearkey"));
967         gst_structure_set_value(structure.get(), "key-ids", &keyIDList);
968         gst_structure_set_value(structure.get(), "key-values", &keyValueList);
969
970         for (auto it : m_appendPipelinesMap)
971             it.value->dispatchDecryptionStructure(GUniquePtr<GstStructure>(gst_structure_copy(structure.get())));
972     }
973 }
974 #endif
975
976 } // namespace WebCore.
977
978 #endif // USE(GSTREAMER)