Changing settings of a MediaStreamTrack clone should not alter the settings of the...
authoryouenn@apple.com <youenn@apple.com@268f45cc-cd09-0410-ab3c-d52691b4dbfc>
Thu, 20 Jun 2019 18:55:32 +0000 (18:55 +0000)
committeryouenn@apple.com <youenn@apple.com@268f45cc-cd09-0410-ab3c-d52691b4dbfc>
Thu, 20 Jun 2019 18:55:32 +0000 (18:55 +0000)
https://bugs.webkit.org/show_bug.cgi?id=198840

Reviewed by Eric Carlson.

Source/WebCore:

Rename RealtimeVideoSource in RealtimeVideoCaptureSource.
Introduce RealtimeVideoSource as a class wrapping RealtimeVideoCaptureSource.
Its goal is to be able to have independent settings from its underlying RealtimeVideoCaptureSource.
It can also adapt size based on its settings if different than its RealtimeVideoCaptureSource.
Apply this wrapping for AV video sources as well as mock video sources.
Test: fast/mediastream/mediastreamtrack-video-clone.html

* Sources.txt:
* WebCore.xcodeproj/project.pbxproj:
* platform/mediastream/MediaStreamTrackPrivate.cpp:
(WebCore::MediaStreamTrackPrivate::clone):
* platform/mediastream/RealtimeMediaSource.h:
* platform/mediastream/RealtimeVideoCaptureSource.cpp: Added.
(WebCore::RealtimeVideoCaptureSource::RealtimeVideoCaptureSource):
(WebCore::RealtimeVideoCaptureSource::~RealtimeVideoCaptureSource):
(WebCore::RealtimeVideoCaptureSource::prepareToProduceData):
(WebCore::RealtimeVideoCaptureSource::presets):
(WebCore::RealtimeVideoCaptureSource::setSupportedPresets):
(WebCore::RealtimeVideoCaptureSource::standardVideoSizes):
(WebCore::updateMinMax):
(WebCore::RealtimeVideoCaptureSource::updateCapabilities):
(WebCore::RealtimeVideoCaptureSource::supportsSizeAndFrameRate):
(WebCore::RealtimeVideoCaptureSource::frameRateRangeIncludesRate):
(WebCore::RealtimeVideoCaptureSource::presetSupportsFrameRate):
(WebCore::RealtimeVideoCaptureSource::supportsCaptureSize):
(WebCore::RealtimeVideoCaptureSource::shouldUsePreset):
(WebCore::RealtimeVideoCaptureSource::bestSupportedSizeAndFrameRate):
(WebCore::RealtimeVideoCaptureSource::setSizeAndFrameRate):
(WebCore::RealtimeVideoCaptureSource::adaptVideoSample):
(WebCore::RealtimeVideoCaptureSource::dispatchMediaSampleToObservers):
(WebCore::RealtimeVideoCaptureSource::clientUpdatedSizeAndFrameRate):
(WebCore::SizeAndFrameRate::toJSONObject const):
(WebCore::SizeAndFrameRate::toJSONString const):
* platform/mediastream/RealtimeVideoCaptureSource.h: Added.
(WebCore::RealtimeVideoCaptureSource::sampleRotation const):
(WebCore::RealtimeVideoCaptureSource::prefersPreset):
(WebCore::RealtimeVideoCaptureSource::setFrameRateWithPreset):
(WebCore::RealtimeVideoCaptureSource::canResizeVideoFrames const):
(WebCore::RealtimeVideoCaptureSource::setDefaultSize):
(WebCore::RealtimeVideoCaptureSource::observedFrameRate const):
(WTF::LogArgument<WebCore::SizeAndFrameRate>::toString):
* platform/mediastream/RealtimeVideoSource.cpp:
(WebCore::RealtimeVideoSource::RealtimeVideoSource):
(WebCore::m_source):
(WebCore::RealtimeVideoSource::~RealtimeVideoSource):
(WebCore::RealtimeVideoSource::startProducingData):
(WebCore::RealtimeVideoSource::stopProducingData):
(WebCore::RealtimeVideoSource::supportsSizeAndFrameRate):
(WebCore::RealtimeVideoSource::setSizeAndFrameRate):
(WebCore::RealtimeVideoSource::sourceMutedChanged):
(WebCore::RealtimeVideoSource::sourceSettingsChanged):
(WebCore::RealtimeVideoSource::preventSourceFromStopping):
(WebCore::RealtimeVideoSource::sourceStopped):
(WebCore::RealtimeVideoSource::videoSampleAvailable):
(WebCore::RealtimeVideoSource::clone):
* platform/mediastream/RealtimeVideoSource.h:
* platform/mediastream/gstreamer/GStreamerVideoCaptureSource.cpp:
(WebCore::GStreamerVideoCaptureSource::GStreamerVideoCaptureSource):
* platform/mediastream/gstreamer/GStreamerVideoCaptureSource.h:
* platform/mediastream/mac/AVVideoCaptureSource.h:
* platform/mediastream/mac/AVVideoCaptureSource.mm:
(WebCore::AVVideoCaptureSource::create):
(WebCore::AVVideoCaptureSource::AVVideoCaptureSource):
* platform/mediastream/mac/MockRealtimeVideoSourceMac.h:
* platform/mediastream/mac/MockRealtimeVideoSourceMac.mm:
(WebCore::MockRealtimeVideoSource::create):
* platform/mock/MockRealtimeVideoSource.cpp:
(WebCore::MockRealtimeVideoSource::create):
(WebCore::MockRealtimeVideoSource::MockRealtimeVideoSource):
(WebCore::MockRealtimeVideoSource::supportsSizeAndFrameRate):
(WebCore::MockRealtimeVideoSource::setSizeAndFrameRate):
* platform/mock/MockRealtimeVideoSource.h:

LayoutTests:

* fast/mediastream/mediastreamtrack-video-clone-expected.txt: Added.
* fast/mediastream/mediastreamtrack-video-clone.html: Added.

git-svn-id: https://svn.webkit.org/repository/webkit/trunk@246644 268f45cc-cd09-0410-ab3c-d52691b4dbfc

21 files changed:
LayoutTests/ChangeLog
LayoutTests/fast/mediastream/mediastreamtrack-video-clone-expected.txt [new file with mode: 0644]
LayoutTests/fast/mediastream/mediastreamtrack-video-clone.html [new file with mode: 0644]
Source/WebCore/ChangeLog
Source/WebCore/Headers.cmake
Source/WebCore/Sources.txt
Source/WebCore/WebCore.xcodeproj/project.pbxproj
Source/WebCore/platform/mediastream/MediaStreamTrackPrivate.cpp
Source/WebCore/platform/mediastream/RealtimeMediaSource.h
Source/WebCore/platform/mediastream/RealtimeVideoCaptureSource.cpp [new file with mode: 0644]
Source/WebCore/platform/mediastream/RealtimeVideoCaptureSource.h [new file with mode: 0644]
Source/WebCore/platform/mediastream/RealtimeVideoSource.cpp
Source/WebCore/platform/mediastream/RealtimeVideoSource.h
Source/WebCore/platform/mediastream/gstreamer/GStreamerVideoCaptureSource.cpp
Source/WebCore/platform/mediastream/gstreamer/GStreamerVideoCaptureSource.h
Source/WebCore/platform/mediastream/mac/AVVideoCaptureSource.h
Source/WebCore/platform/mediastream/mac/AVVideoCaptureSource.mm
Source/WebCore/platform/mediastream/mac/MockRealtimeVideoSourceMac.h
Source/WebCore/platform/mediastream/mac/MockRealtimeVideoSourceMac.mm
Source/WebCore/platform/mock/MockRealtimeVideoSource.cpp
Source/WebCore/platform/mock/MockRealtimeVideoSource.h

index 229184e..32b34df 100644 (file)
@@ -1,3 +1,13 @@
+2019-06-20  Youenn Fablet  <youenn@apple.com>
+
+        Changing settings of a MediaStreamTrack clone should not alter the settings of the original track
+        https://bugs.webkit.org/show_bug.cgi?id=198840
+
+        Reviewed by Eric Carlson.
+
+        * fast/mediastream/mediastreamtrack-video-clone-expected.txt: Added.
+        * fast/mediastream/mediastreamtrack-video-clone.html: Added.
+
 2019-06-20  Russell Epstein  <russell_e@apple.com>
 
         Layout Test imported/blink/fast/css/user-select-none.html is flaky.
diff --git a/LayoutTests/fast/mediastream/mediastreamtrack-video-clone-expected.txt b/LayoutTests/fast/mediastream/mediastreamtrack-video-clone-expected.txt
new file mode 100644 (file)
index 0000000..c36c281
--- /dev/null
@@ -0,0 +1,12 @@
+   
+
+PASS Setup for width test 
+PASS Setup for height test 
+PASS Setup for width+height test 
+PASS Check cloned track settings after applying width constraints 
+PASS Check cloned track settings after applying width constraint to original track 
+PASS Check cloned track settings after applying height constraints 
+PASS Check cloned track settings after applying height constraints to original track 
+PASS Check cloned track settings after applying width+height constraints 
+PASS Check cloned track settings after applying width+height constraints to original track 
+
diff --git a/LayoutTests/fast/mediastream/mediastreamtrack-video-clone.html b/LayoutTests/fast/mediastream/mediastreamtrack-video-clone.html
new file mode 100644 (file)
index 0000000..efdbf86
--- /dev/null
@@ -0,0 +1,110 @@
+<!DOCTYPE html>
+<html>
+<head>
+    <meta charset="utf-8">
+    <title>Clone a video track.</title>
+    <script src="../../resources/testharness.js"></script>
+    <script src="../../resources/testharnessreport.js"></script>
+</head>
+<body>
+    <video id='video1' autoplay playsinline></video>
+    <video id='video2' autoplay playsinline></video>
+    <video id='video3' autoplay playsinline></video>
+    <script>
+    promise_test(async (t) => {
+        const stream = await navigator.mediaDevices.getUserMedia({ video: { width: 160 } });
+        const streamClone = stream.clone();
+
+        video1.srcObject = stream;
+        video2.srcObject = streamClone;
+
+        const videoTrack = stream.getVideoTracks()[0];
+        const videoTrackClone = streamClone.getVideoTracks()[0];
+
+        assert_equals(videoTrack.enabled, videoTrackClone.enabled);
+        videoTrack.enabled = false;
+        assert_true(videoTrackClone.enabled);
+        videoTrack.enabled = true;
+
+        await videoTrackClone.applyConstraints({width: 640});
+        test(() => {
+            assert_equals(videoTrackClone.getSettings().width, 640);
+            assert_equals(videoTrack.getSettings().width, 160);
+        }, "Check cloned track settings after applying width constraints");
+
+        const videoTrackClone2 = videoTrackClone.clone();
+        await videoTrackClone.applyConstraints({width: 1280});
+        video3.srcObject = new MediaStream([videoTrackClone2]);
+        test(() => {
+            assert_equals(videoTrackClone.getSettings().width, 1280);
+            assert_equals(videoTrackClone2.getSettings().width, 640);
+        }, "Check cloned track settings after applying width constraint to original track");
+    }, "Setup for width test");
+
+    promise_test(async (t) => {
+        const stream = await navigator.mediaDevices.getUserMedia({ video: { height: 100 } });
+        const streamClone = stream.clone();
+
+        video1.srcObject = stream;
+        video2.srcObject = streamClone;
+
+        const videoTrack = stream.getVideoTracks()[0];
+        const videoTrackClone = streamClone.getVideoTracks()[0];
+
+        assert_equals(videoTrack.enabled, videoTrackClone.enabled);
+        videoTrack.enabled = false;
+        assert_true(videoTrackClone.enabled);
+        videoTrack.enabled = true;
+
+        await videoTrackClone.applyConstraints({height: 200});
+        test(() => {
+            assert_equals(videoTrackClone.getSettings().height, 200);
+            assert_equals(videoTrack.getSettings().height, 100);
+        }, "Check cloned track settings after applying height constraints");
+
+        const videoTrackClone2 = videoTrackClone.clone();
+        await videoTrackClone.applyConstraints({height: 400});
+        video3.srcObject = new MediaStream([videoTrackClone2]);
+        test(() => {
+            assert_equals(videoTrackClone.getSettings().height, 400);
+            assert_equals(videoTrackClone2.getSettings().height, 200);
+        }, "Check cloned track settings after applying height constraints to original track");
+    }, "Setup for height test");
+
+    promise_test(async (t) => {
+        const stream = await navigator.mediaDevices.getUserMedia({ video: { width: 100, height: 100 } });
+        const streamClone = stream.clone();
+
+        video1.srcObject = stream;
+        video2.srcObject = streamClone;
+
+        const videoTrack = stream.getVideoTracks()[0];
+        const videoTrackClone = streamClone.getVideoTracks()[0];
+
+        assert_equals(videoTrack.enabled, videoTrackClone.enabled);
+        videoTrack.enabled = false;
+        assert_true(videoTrackClone.enabled);
+        videoTrack.enabled = true;
+
+        await videoTrackClone.applyConstraints({width: 100, height: 200});
+        test(() => {
+            assert_equals(videoTrackClone.getSettings().width, 100);
+            assert_equals(videoTrackClone.getSettings().height, 200);
+            assert_equals(videoTrack.getSettings().height, 100);
+            assert_equals(videoTrack.getSettings().width, 100);
+        }, "Check cloned track settings after applying width+height constraints");
+
+        const videoTrackClone2 = videoTrackClone.clone();
+        await videoTrackClone.applyConstraints({width: 400, height: 200});
+        video3.srcObject = new MediaStream([videoTrackClone2]);
+        test(() => {
+            assert_equals(videoTrackClone.getSettings().width, 400);
+            assert_equals(videoTrackClone.getSettings().height, 200);
+            assert_equals(videoTrackClone2.getSettings().width, 100);
+            assert_equals(videoTrackClone2.getSettings().height, 200);
+        }, "Check cloned track settings after applying width+height constraints to original track");
+    }, "Setup for width+height test");
+
+    </script>
+</body>
+</html>
index d95ac7a..ca62184 100644 (file)
@@ -1,3 +1,83 @@
+2019-06-20  Youenn Fablet  <youenn@apple.com>
+
+        Changing settings of a MediaStreamTrack clone should not alter the settings of the original track
+        https://bugs.webkit.org/show_bug.cgi?id=198840
+
+        Reviewed by Eric Carlson.
+
+        Rename RealtimeVideoSource in RealtimeVideoCaptureSource.
+        Introduce RealtimeVideoSource as a class wrapping RealtimeVideoCaptureSource.
+        Its goal is to be able to have independent settings from its underlying RealtimeVideoCaptureSource.
+        It can also adapt size based on its settings if different than its RealtimeVideoCaptureSource.
+        Apply this wrapping for AV video sources as well as mock video sources.
+        Test: fast/mediastream/mediastreamtrack-video-clone.html
+
+        * Sources.txt:
+        * WebCore.xcodeproj/project.pbxproj:
+        * platform/mediastream/MediaStreamTrackPrivate.cpp:
+        (WebCore::MediaStreamTrackPrivate::clone):
+        * platform/mediastream/RealtimeMediaSource.h:
+        * platform/mediastream/RealtimeVideoCaptureSource.cpp: Added.
+        (WebCore::RealtimeVideoCaptureSource::RealtimeVideoCaptureSource):
+        (WebCore::RealtimeVideoCaptureSource::~RealtimeVideoCaptureSource):
+        (WebCore::RealtimeVideoCaptureSource::prepareToProduceData):
+        (WebCore::RealtimeVideoCaptureSource::presets):
+        (WebCore::RealtimeVideoCaptureSource::setSupportedPresets):
+        (WebCore::RealtimeVideoCaptureSource::standardVideoSizes):
+        (WebCore::updateMinMax):
+        (WebCore::RealtimeVideoCaptureSource::updateCapabilities):
+        (WebCore::RealtimeVideoCaptureSource::supportsSizeAndFrameRate):
+        (WebCore::RealtimeVideoCaptureSource::frameRateRangeIncludesRate):
+        (WebCore::RealtimeVideoCaptureSource::presetSupportsFrameRate):
+        (WebCore::RealtimeVideoCaptureSource::supportsCaptureSize):
+        (WebCore::RealtimeVideoCaptureSource::shouldUsePreset):
+        (WebCore::RealtimeVideoCaptureSource::bestSupportedSizeAndFrameRate):
+        (WebCore::RealtimeVideoCaptureSource::setSizeAndFrameRate):
+        (WebCore::RealtimeVideoCaptureSource::adaptVideoSample):
+        (WebCore::RealtimeVideoCaptureSource::dispatchMediaSampleToObservers):
+        (WebCore::RealtimeVideoCaptureSource::clientUpdatedSizeAndFrameRate):
+        (WebCore::SizeAndFrameRate::toJSONObject const):
+        (WebCore::SizeAndFrameRate::toJSONString const):
+        * platform/mediastream/RealtimeVideoCaptureSource.h: Added.
+        (WebCore::RealtimeVideoCaptureSource::sampleRotation const):
+        (WebCore::RealtimeVideoCaptureSource::prefersPreset):
+        (WebCore::RealtimeVideoCaptureSource::setFrameRateWithPreset):
+        (WebCore::RealtimeVideoCaptureSource::canResizeVideoFrames const):
+        (WebCore::RealtimeVideoCaptureSource::setDefaultSize):
+        (WebCore::RealtimeVideoCaptureSource::observedFrameRate const):
+        (WTF::LogArgument<WebCore::SizeAndFrameRate>::toString):
+        * platform/mediastream/RealtimeVideoSource.cpp:
+        (WebCore::RealtimeVideoSource::RealtimeVideoSource):
+        (WebCore::m_source):
+        (WebCore::RealtimeVideoSource::~RealtimeVideoSource):
+        (WebCore::RealtimeVideoSource::startProducingData):
+        (WebCore::RealtimeVideoSource::stopProducingData):
+        (WebCore::RealtimeVideoSource::supportsSizeAndFrameRate):
+        (WebCore::RealtimeVideoSource::setSizeAndFrameRate):
+        (WebCore::RealtimeVideoSource::sourceMutedChanged):
+        (WebCore::RealtimeVideoSource::sourceSettingsChanged):
+        (WebCore::RealtimeVideoSource::preventSourceFromStopping):
+        (WebCore::RealtimeVideoSource::sourceStopped):
+        (WebCore::RealtimeVideoSource::videoSampleAvailable):
+        (WebCore::RealtimeVideoSource::clone):
+        * platform/mediastream/RealtimeVideoSource.h:
+        * platform/mediastream/gstreamer/GStreamerVideoCaptureSource.cpp:
+        (WebCore::GStreamerVideoCaptureSource::GStreamerVideoCaptureSource):
+        * platform/mediastream/gstreamer/GStreamerVideoCaptureSource.h:
+        * platform/mediastream/mac/AVVideoCaptureSource.h:
+        * platform/mediastream/mac/AVVideoCaptureSource.mm:
+        (WebCore::AVVideoCaptureSource::create):
+        (WebCore::AVVideoCaptureSource::AVVideoCaptureSource):
+        * platform/mediastream/mac/MockRealtimeVideoSourceMac.h:
+        * platform/mediastream/mac/MockRealtimeVideoSourceMac.mm:
+        (WebCore::MockRealtimeVideoSource::create):
+        * platform/mock/MockRealtimeVideoSource.cpp:
+        (WebCore::MockRealtimeVideoSource::create):
+        (WebCore::MockRealtimeVideoSource::MockRealtimeVideoSource):
+        (WebCore::MockRealtimeVideoSource::supportsSizeAndFrameRate):
+        (WebCore::MockRealtimeVideoSource::setSizeAndFrameRate):
+        * platform/mock/MockRealtimeVideoSource.h:
+
 2019-06-20  Saam Barati  <sbarati@apple.com>
 
         Unreviewed. More speculative build fixing for watchOS after r246631.
index 34e8341..2dbefa0 100644 (file)
@@ -1145,6 +1145,7 @@ set(WebCore_PRIVATE_FRAMEWORK_HEADERS
     platform/mediastream/RealtimeMediaSourceFactory.h
     platform/mediastream/RealtimeMediaSourceSettings.h
     platform/mediastream/RealtimeMediaSourceSupportedConstraints.h
+    platform/mediastream/RealtimeVideoCaptureSource.h
     platform/mediastream/RealtimeVideoSource.h
     platform/mediastream/VideoPreset.h
     platform/mediastream/WebAudioSourceProvider.h
index de833de..07b3bdc 100644 (file)
@@ -1872,6 +1872,7 @@ platform/mediastream/RealtimeMediaSourceFactory.cpp
 platform/mediastream/RealtimeMediaSourceSettings.cpp
 platform/mediastream/RealtimeOutgoingAudioSource.cpp
 platform/mediastream/RealtimeOutgoingVideoSource.cpp
+platform/mediastream/RealtimeVideoCaptureSource.cpp
 platform/mediastream/RealtimeVideoSource.cpp
 
 platform/mediastream/libwebrtc/LibWebRTCProvider.cpp
index bc08b6f..e74d485 100644 (file)
                07277E4F17D018CC0015534D /* JSMediaStreamAudioDestinationNode.h in Headers */ = {isa = PBXBuildFile; fileRef = 07277E4317D018CC0015534D /* JSMediaStreamAudioDestinationNode.h */; };
                07277E5317D018CC0015534D /* JSMediaStreamTrack.h in Headers */ = {isa = PBXBuildFile; fileRef = 07277E4717D018CC0015534D /* JSMediaStreamTrack.h */; };
                07277E5517D018CC0015534D /* JSMediaStreamTrackEvent.h in Headers */ = {isa = PBXBuildFile; fileRef = 07277E4917D018CC0015534D /* JSMediaStreamTrackEvent.h */; };
-               072880D12010F1F60071B255 /* RealtimeVideoSource.h in Headers */ = {isa = PBXBuildFile; fileRef = 072880D02010EED70071B255 /* RealtimeVideoSource.h */; settings = {ATTRIBUTES = (Private, ); }; };
+               072880D12010F1F60071B255 /* RealtimeVideoCaptureSource.h in Headers */ = {isa = PBXBuildFile; fileRef = 072880D02010EED70071B255 /* RealtimeVideoCaptureSource.h */; settings = {ATTRIBUTES = (Private, ); }; };
                072A70401D6E8F6200DF0AFC /* OverconstrainedErrorEvent.h in Headers */ = {isa = PBXBuildFile; fileRef = 072A703E1D6E8F6200DF0AFC /* OverconstrainedErrorEvent.h */; };
                072AE1E5183C0741000A5988 /* PluginReplacement.h in Headers */ = {isa = PBXBuildFile; fileRef = 072AE1DF183C0741000A5988 /* PluginReplacement.h */; settings = {ATTRIBUTES = (Private, ); }; };
                072AE1E8183C0741000A5988 /* QuickTimePluginReplacement.h in Headers */ = {isa = PBXBuildFile; fileRef = 072AE1E2183C0741000A5988 /* QuickTimePluginReplacement.h */; };
                41B28B151F8501D300FB52AC /* MediaEndpointConfiguration.h in Headers */ = {isa = PBXBuildFile; fileRef = 41B28B121F8501A300FB52AC /* MediaEndpointConfiguration.h */; };
                41B28B3D1F860EF300FB52AC /* LibWebRTCProviderCocoa.h in Headers */ = {isa = PBXBuildFile; fileRef = 41B28B361F860BD000FB52AC /* LibWebRTCProviderCocoa.h */; settings = {ATTRIBUTES = (Private, ); }; };
                41B2A6261EF1BF6D002B9D7A /* WebAudioSourceProvider.h in Headers */ = {isa = PBXBuildFile; fileRef = 41B2A6251EF1BF60002B9D7A /* WebAudioSourceProvider.h */; settings = {ATTRIBUTES = (Private, ); }; };
+               41BF204922BA7BE80004F812 /* RealtimeVideoSource.h in Headers */ = {isa = PBXBuildFile; fileRef = 41BF204022B947160004F812 /* RealtimeVideoSource.h */; settings = {ATTRIBUTES = (Private, ); }; };
                41C760B10EDE03D300C1655F /* ScriptState.h in Headers */ = {isa = PBXBuildFile; fileRef = 41C760B00EDE03D300C1655F /* ScriptState.h */; settings = {ATTRIBUTES = (Private, ); }; };
                41D015CA0F4B5C71004A662F /* ContentType.h in Headers */ = {isa = PBXBuildFile; fileRef = 41D015C80F4B5C71004A662F /* ContentType.h */; settings = {ATTRIBUTES = (Private, ); }; };
                41D129CE1F3D0EF600D15E47 /* WorkerGlobalScopeCaches.h in Headers */ = {isa = PBXBuildFile; fileRef = 41FB278D1F34C28200795487 /* WorkerGlobalScopeCaches.h */; };
                07277E4817D018CC0015534D /* JSMediaStreamTrackEvent.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = JSMediaStreamTrackEvent.cpp; sourceTree = "<group>"; };
                07277E4917D018CC0015534D /* JSMediaStreamTrackEvent.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = JSMediaStreamTrackEvent.h; sourceTree = "<group>"; };
                072847E216EBC5B00043CFA4 /* PlatformTextTrack.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = PlatformTextTrack.h; sourceTree = "<group>"; };
-               072880CE2010EED60071B255 /* RealtimeVideoSource.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = RealtimeVideoSource.cpp; sourceTree = "<group>"; };
-               072880D02010EED70071B255 /* RealtimeVideoSource.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = RealtimeVideoSource.h; sourceTree = "<group>"; };
+               072880CE2010EED60071B255 /* RealtimeVideoCaptureSource.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = RealtimeVideoCaptureSource.cpp; sourceTree = "<group>"; };
+               072880D02010EED70071B255 /* RealtimeVideoCaptureSource.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = RealtimeVideoCaptureSource.h; sourceTree = "<group>"; };
                072A703E1D6E8F6200DF0AFC /* OverconstrainedErrorEvent.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = OverconstrainedErrorEvent.h; sourceTree = "<group>"; };
                072A703F1D6E8F6200DF0AFC /* OverconstrainedErrorEvent.idl */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = text; path = OverconstrainedErrorEvent.idl; sourceTree = "<group>"; };
                072AE1DF183C0741000A5988 /* PluginReplacement.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = PluginReplacement.h; sourceTree = "<group>"; };
                41B2A6251EF1BF60002B9D7A /* WebAudioSourceProvider.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = WebAudioSourceProvider.h; sourceTree = "<group>"; };
                41B459DA1F4CADB90000F6FD /* ReadableStream.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = ReadableStream.h; sourceTree = "<group>"; };
                41B459ED1F55EBC70000F6FD /* ReadableStream.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = ReadableStream.cpp; sourceTree = "<group>"; };
+               41BF204022B947160004F812 /* RealtimeVideoSource.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = RealtimeVideoSource.h; sourceTree = "<group>"; };
+               41BF204222B947170004F812 /* RealtimeVideoSource.cpp */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.cpp; path = RealtimeVideoSource.cpp; sourceTree = "<group>"; };
                41C760B00EDE03D300C1655F /* ScriptState.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = ScriptState.h; sourceTree = "<group>"; };
                41C7E1051E6A54360027B4DE /* CanvasCaptureMediaStreamTrack.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = CanvasCaptureMediaStreamTrack.cpp; sourceTree = "<group>"; };
                41C7E1061E6A54360027B4DE /* CanvasCaptureMediaStreamTrack.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = CanvasCaptureMediaStreamTrack.h; sourceTree = "<group>"; };
                                41103AA81E39790A00769F03 /* RealtimeOutgoingAudioSource.h */,
                                5CDD833B1E4324BB00621E92 /* RealtimeOutgoingVideoSource.cpp */,
                                5CDD833C1E4324BB00621E92 /* RealtimeOutgoingVideoSource.h */,
-                               072880CE2010EED60071B255 /* RealtimeVideoSource.cpp */,
-                               072880D02010EED70071B255 /* RealtimeVideoSource.h */,
+                               072880CE2010EED60071B255 /* RealtimeVideoCaptureSource.cpp */,
+                               072880D02010EED70071B255 /* RealtimeVideoCaptureSource.h */,
+                               41BF204222B947170004F812 /* RealtimeVideoSource.cpp */,
+                               41BF204022B947160004F812 /* RealtimeVideoSource.h */,
                                3135910C1E7DDCB600F30630 /* RTCBundlePolicy.h */,
                                07221BA217CF0AD400848E51 /* RTCDataChannelHandler.h */,
                                07221BA317CF0AD400848E51 /* RTCDataChannelHandlerClient.h */,
                                07C1C0E51BFB60ED00BD2256 /* RealtimeMediaSourceSupportedConstraints.h in Headers */,
                                41103AAC1E39791000769F03 /* RealtimeOutgoingAudioSource.h in Headers */,
                                41103AAC1E39791000769F14 /* RealtimeOutgoingAudioSourceCocoa.h in Headers */,
-                               072880D12010F1F60071B255 /* RealtimeVideoSource.h in Headers */,
+                               072880D12010F1F60071B255 /* RealtimeVideoCaptureSource.h in Headers */,
+                               41BF204922BA7BE80004F812 /* RealtimeVideoSource.h in Headers */,
                                91B952241F58A58F00931DC2 /* RecordingSwizzleTypes.h in Headers */,
                                BC4368E80C226E32005EFB5F /* Rect.h in Headers */,
                                FD45A958175D414C00C21EC8 /* RectangleShape.h in Headers */,
index 3c330b9..4da133b 100644 (file)
@@ -155,13 +155,16 @@ void MediaStreamTrackPrivate::endTrack()
 
 Ref<MediaStreamTrackPrivate> MediaStreamTrackPrivate::clone()
 {
-    auto clonedMediaStreamTrackPrivate = create(m_logger.copyRef(), m_source.copyRef());
+    auto clonedMediaStreamTrackPrivate = create(m_logger.copyRef(), m_source->clone());
 
     clonedMediaStreamTrackPrivate->m_isEnabled = this->m_isEnabled;
     clonedMediaStreamTrackPrivate->m_isEnded = this->m_isEnded;
     clonedMediaStreamTrackPrivate->m_contentHint = this->m_contentHint;
     clonedMediaStreamTrackPrivate->updateReadyState();
 
+    if (isProducingData())
+        clonedMediaStreamTrackPrivate->startProducingData();
+
     return clonedMediaStreamTrackPrivate;
 }
 
index ab3c997..226491f 100644 (file)
@@ -96,6 +96,8 @@ public:
 
     virtual ~RealtimeMediaSource() = default;
 
+    virtual Ref<RealtimeMediaSource> clone() { return *this; }
+
     const String& hashedId() const;
     String deviceIDHashSalt() const;
 
@@ -224,6 +226,8 @@ protected:
     void videoSampleAvailable(MediaSample&);
     void audioSamplesAvailable(const MediaTime&, const PlatformAudioData&, const AudioStreamDescription&, size_t);
 
+    void forEachObserver(const WTF::Function<void(Observer&)>&) const;
+
 private:
     virtual void startProducingData() { }
     virtual void stopProducingData() { }
@@ -231,8 +235,6 @@ private:
 
     virtual void hasEnded() { }
 
-    void forEachObserver(const WTF::Function<void(Observer&)>&) const;
-
 #if !RELEASE_LOG_DISABLED
     RefPtr<const Logger> m_logger;
     const void* m_logIdentifier;
diff --git a/Source/WebCore/platform/mediastream/RealtimeVideoCaptureSource.cpp b/Source/WebCore/platform/mediastream/RealtimeVideoCaptureSource.cpp
new file mode 100644 (file)
index 0000000..c7f8cdd
--- /dev/null
@@ -0,0 +1,468 @@
+/*
+ * Copyright (C) 2018-2019 Apple Inc. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL APPLE INC. OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+ * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
+ * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "config.h"
+#include "RealtimeVideoCaptureSource.h"
+
+#if ENABLE(MEDIA_STREAM)
+#include "CaptureDevice.h"
+#include "Logging.h"
+#include "RealtimeMediaSourceCenter.h"
+#include "RealtimeMediaSourceSettings.h"
+#include "RemoteVideoSample.h"
+#include <wtf/JSONValues.h>
+
+#if PLATFORM(COCOA)
+#include "ImageTransferSessionVT.h"
+#endif
+
+namespace WebCore {
+
+RealtimeVideoCaptureSource::RealtimeVideoCaptureSource(String&& name, String&& id, String&& hashSalt)
+    : RealtimeMediaSource(Type::Video, WTFMove(name), WTFMove(id), WTFMove(hashSalt))
+{
+}
+
+RealtimeVideoCaptureSource::~RealtimeVideoCaptureSource()
+{
+#if PLATFORM(IOS_FAMILY)
+    RealtimeMediaSourceCenter::singleton().videoCaptureFactory().unsetActiveSource(*this);
+#endif
+}
+
+void RealtimeVideoCaptureSource::prepareToProduceData()
+{
+    ASSERT(frameRate());
+
+#if PLATFORM(IOS_FAMILY)
+    RealtimeMediaSourceCenter::singleton().videoCaptureFactory().setActiveSource(*this);
+#endif
+
+    if (size().isEmpty() && !m_defaultSize.isEmpty())
+        setSize(m_defaultSize);
+}
+
+const Vector<Ref<VideoPreset>>& RealtimeVideoCaptureSource::presets()
+{
+    if (m_presets.isEmpty())
+        generatePresets();
+
+    ASSERT(!m_presets.isEmpty());
+    return m_presets;
+}
+
+void RealtimeVideoCaptureSource::setSupportedPresets(Vector<VideoPresetData>&& presetData)
+{
+    Vector<Ref<VideoPreset>> presets;
+
+    for (auto& data : presetData)
+        presets.append(VideoPreset::create(WTFMove(data)));
+
+    setSupportedPresets(WTFMove(presets));
+}
+
+void RealtimeVideoCaptureSource::setSupportedPresets(const Vector<Ref<VideoPreset>>& presets)
+{
+    m_presets = WTF::map(presets, [](auto& preset) {
+        return preset.copyRef();
+    });
+
+    for (auto& preset : m_presets) {
+        std::sort(preset->frameRateRanges.begin(), preset->frameRateRanges.end(),
+            [&] (const auto& a, const auto& b) -> bool {
+                return a.minimum < b.minimum;
+        });
+    }
+}
+
+const Vector<IntSize>& RealtimeVideoCaptureSource::standardVideoSizes()
+{
+    static const auto sizes = makeNeverDestroyed([] {
+        static IntSize videoSizes[] = {
+            { 112, 112 },
+            { 160, 160 },
+            { 160, 120 }, // 4:3, QQVGA
+            { 176, 144 }, // 4:3, QCIF
+            { 192, 192 },
+            { 192, 112 }, // 16:9
+            { 192, 144 }, // 3:4
+            { 240, 240 },
+            { 240, 160 }, // 3:2, HQVGA
+            { 320, 320 },
+            { 320, 180 }, // 16:9
+            { 320, 240 }, // 4:3, QVGA
+            { 352, 288 }, // CIF
+            { 480, 272 }, // 16:9
+            { 480, 360 }, // 4:3
+            { 480, 480 },
+            { 640, 640 },
+            { 640, 360 }, // 16:9, 360p nHD
+            { 640, 480 }, // 4:3
+            { 720, 720 },
+            { 800, 600 }, // 4:3, SVGA
+            { 960, 540 }, // 16:9, qHD
+            { 1024, 600 }, // 16:9, WSVGA
+            { 1024, 768 }, // 4:3, XGA
+            { 1280, 960 }, // 4:3
+            { 1280, 1024 }, // 5:4, SXGA
+            { 1280, 720 }, // 16:9, WXGA
+            { 1366, 768 }, // 16:9, HD
+            { 1600, 1200}, // 4:3, UXGA
+            { 1920, 1080 }, // 16:9, 1080p FHD
+            { 2560, 1440 }, // 16:9, QHD
+            { 2592, 1936 },
+            { 3264, 2448 }, // 3:4
+            { 3840, 2160 }, // 16:9, 4K UHD
+        };
+        Vector<IntSize> sizes;
+        for (auto& size : videoSizes)
+            sizes.append(size);
+
+        return sizes;
+    }());
+
+    return sizes.get();
+}
+template <typename ValueType>
+static void updateMinMax(ValueType& min, ValueType& max, ValueType value)
+{
+    min = std::min<ValueType>(min, value);
+    max = std::max<ValueType>(max, value);
+}
+
+void RealtimeVideoCaptureSource::updateCapabilities(RealtimeMediaSourceCapabilities& capabilities)
+{
+    ASSERT(!presets().isEmpty());
+
+    int minimumWidth = std::numeric_limits<int>::max();
+    int maximumWidth = 0;
+    int minimumHeight = std::numeric_limits<int>::max();
+    int maximumHeight = 0;
+    double minimumAspectRatio = std::numeric_limits<double>::max();
+    double maximumAspectRatio = 0;
+    double minimumFrameRate = std::numeric_limits<double>::max();
+    double maximumFrameRate = 0;
+    for (const auto& preset : presets()) {
+        const auto& size = preset->size;
+        updateMinMax(minimumWidth, maximumWidth, size.width());
+        updateMinMax(minimumHeight, maximumHeight, size.height());
+        updateMinMax(minimumAspectRatio, maximumAspectRatio, static_cast<double>(size.width()) / size.height());
+
+        for (const auto& rate : preset->frameRateRanges) {
+            updateMinMax(minimumFrameRate, maximumFrameRate, rate.minimum);
+            updateMinMax(minimumFrameRate, maximumFrameRate, rate.maximum);
+        }
+    }
+
+    if (canResizeVideoFrames()) {
+        minimumWidth = 1;
+        minimumHeight = 1;
+    }
+
+    capabilities.setWidth({ minimumWidth, maximumWidth });
+    capabilities.setHeight({ minimumHeight, maximumHeight });
+    capabilities.setAspectRatio({ minimumAspectRatio, maximumAspectRatio });
+    capabilities.setFrameRate({ minimumFrameRate, maximumFrameRate });
+}
+
+bool RealtimeVideoCaptureSource::supportsSizeAndFrameRate(Optional<int> width, Optional<int> height, Optional<double> frameRate)
+{
+    if (!width && !height && !frameRate)
+        return true;
+
+    return !!bestSupportedSizeAndFrameRate(width, height, frameRate);
+}
+
+bool RealtimeVideoCaptureSource::frameRateRangeIncludesRate(const FrameRateRange& range, double frameRate)
+{
+    const double epsilon = 0.001;
+    return frameRate + epsilon >= range.minimum && frameRate - epsilon <= range.maximum;
+}
+
+bool RealtimeVideoCaptureSource::presetSupportsFrameRate(RefPtr<VideoPreset> preset, double frameRate)
+{
+    for (const auto& range : preset->frameRateRanges) {
+        if (frameRateRangeIncludesRate(range, frameRate))
+            return true;
+    }
+
+    return false;
+}
+
+bool RealtimeVideoCaptureSource::supportsCaptureSize(Optional<int> width, Optional<int> height, const Function<bool(const IntSize&)>&& function)
+{
+    if (width && height)
+        return function({ width.value(), height.value() });
+
+    if (width) {
+        for (auto& size : standardVideoSizes()) {
+            if (width.value() == size.width() && function({ size.width(), size.height() }))
+                return true;
+        }
+
+        return false;
+    }
+
+    for (auto& size : standardVideoSizes()) {
+        if (height.value() == size.height() && function({ size.width(), size.height() }))
+            return true;
+    }
+
+    return false;
+}
+
+bool RealtimeVideoCaptureSource::shouldUsePreset(VideoPreset& current, VideoPreset& candidate)
+{
+    return candidate.size.width() <= current.size.width() && candidate.size.height() <= current.size.height() && prefersPreset(candidate);
+}
+
+Optional<RealtimeVideoCaptureSource::CaptureSizeAndFrameRate> RealtimeVideoCaptureSource::bestSupportedSizeAndFrameRate(Optional<int> requestedWidth, Optional<int> requestedHeight, Optional<double> requestedFrameRate)
+{
+    if (!requestedWidth && !requestedHeight && !requestedFrameRate)
+        return { };
+
+    if (!requestedWidth && !requestedHeight && !size().isEmpty()) {
+        requestedWidth = size().width();
+        requestedHeight = size().height();
+    }
+    if (!requestedFrameRate)
+        requestedFrameRate = frameRate();
+
+    CaptureSizeAndFrameRate result;
+    RefPtr<VideoPreset> exactSizePreset;
+    RefPtr<VideoPreset> aspectRatioPreset;
+    IntSize aspectRatioMatchSize;
+    RefPtr<VideoPreset> resizePreset;
+    IntSize resizeSize;
+
+    for (const auto& preset : presets()) {
+        const auto& presetSize = preset->size;
+
+        if (!presetSupportsFrameRate(&preset.get(), requestedFrameRate.value()))
+            continue;
+
+        if (!requestedWidth && !requestedHeight) {
+            result.requestedFrameRate = requestedFrameRate.value();
+            return result;
+        }
+
+        // Don't look at presets smaller than the requested resolution because we never want to resize larger.
+        if ((requestedWidth && presetSize.width() < requestedWidth.value()) || (requestedHeight && presetSize.height() < requestedHeight.value()))
+            continue;
+
+        auto lookForExactSizeMatch = [&] (const IntSize& size) -> bool {
+            return preset->size == size;
+        };
+        if (supportsCaptureSize(requestedWidth, requestedHeight, WTFMove(lookForExactSizeMatch))) {
+            if (!exactSizePreset || prefersPreset(preset))
+                exactSizePreset = &preset.get();
+            continue;
+        }
+
+        IntSize encodingSize;
+        auto lookForAspectRatioMatch = [this, &preset, &encodingSize] (const IntSize& size) -> bool {
+            auto aspectRatio = [] (const IntSize size) -> double {
+                return size.width() / static_cast<double>(size.height());
+            };
+            if (std::abs(aspectRatio(preset->size) - aspectRatio(size)) > 10e-7 || !canResizeVideoFrames())
+                return false;
+
+            encodingSize = size;
+            return true;
+        };
+        if (supportsCaptureSize(requestedWidth, requestedHeight, WTFMove(lookForAspectRatioMatch))) {
+            if (!aspectRatioPreset || shouldUsePreset(*aspectRatioPreset, preset)) {
+                aspectRatioPreset = &preset.get();
+                aspectRatioMatchSize = encodingSize;
+            }
+        }
+
+        if (exactSizePreset || aspectRatioPreset)
+            continue;
+
+        if ((requestedWidth && requestedWidth.value() > preset->size.width()) || (requestedHeight && requestedHeight.value() > preset->size.height()))
+            continue;
+
+        if (requestedWidth && requestedHeight) {
+            if (!resizePreset || shouldUsePreset(*resizePreset, preset)) {
+                resizePreset = &preset.get();
+                resizeSize = { requestedWidth.value(), requestedHeight.value() };
+            }
+        } else {
+            for (auto& standardSize : standardVideoSizes()) {
+                if (standardSize.width() > preset->size.width() || standardSize.height() > preset->size.height())
+                    break;
+                if ((requestedWidth && requestedWidth.value() != standardSize.width()) || (requestedHeight && requestedHeight.value() != standardSize.height()))
+                    continue;
+
+                if (!resizePreset || shouldUsePreset(*resizePreset, preset)) {
+                    resizePreset = &preset.get();
+                    resizeSize = standardSize;
+                }
+            }
+
+            if (!resizePreset || shouldUsePreset(*resizePreset, preset)) {
+                resizePreset = &preset.get();
+                if (requestedWidth)
+                    resizeSize = { requestedWidth.value(), requestedWidth.value() * preset->size.height() / preset->size.width()};
+                else
+                    resizeSize = { requestedHeight.value() * preset->size.width() / preset->size.height(), requestedHeight.value() };
+            }
+        }
+    }
+
+    if (!exactSizePreset && !aspectRatioPreset && !resizePreset)
+        return { };
+
+    result.requestedFrameRate = requestedFrameRate.value();
+    if (exactSizePreset) {
+        result.encodingPreset = exactSizePreset;
+        result.requestedSize = exactSizePreset->size;
+        return result;
+    }
+
+    if (aspectRatioPreset) {
+        result.encodingPreset = aspectRatioPreset;
+        result.requestedSize = aspectRatioMatchSize;
+        return result;
+    }
+
+    result.encodingPreset = resizePreset;
+    result.requestedSize = resizeSize;
+    return result;
+}
+
+void RealtimeVideoCaptureSource::setSizeAndFrameRate(Optional<int> width, Optional<int> height, Optional<double> frameRate)
+{
+    ALWAYS_LOG_IF(loggerPtr(), LOGIDENTIFIER, SizeAndFrameRate { width, height, frameRate });
+
+    auto size = this->size();
+    if (!width && !height && !size.isEmpty()) {
+        width = size.width();
+        height = size.height();
+    }
+
+    Optional<RealtimeVideoCaptureSource::CaptureSizeAndFrameRate> match = bestSupportedSizeAndFrameRate(width, height, frameRate);
+    ASSERT(match);
+    if (!match)
+        return;
+
+    setFrameRateWithPreset(match->requestedFrameRate, match->encodingPreset);
+
+    if (!match->requestedSize.isEmpty())
+        setSize(match->requestedSize);
+    setFrameRate(match->requestedFrameRate);
+}
+
+RefPtr<MediaSample> RealtimeVideoCaptureSource::adaptVideoSample(MediaSample& sample)
+{
+    MediaTime sampleTime = sample.outputPresentationTime();
+    if (!sampleTime || !sampleTime.isValid())
+        sampleTime = sample.presentationTime();
+
+    auto frameTime = sampleTime.toDouble();
+    m_observedFrameTimeStamps.append(frameTime);
+    m_observedFrameTimeStamps.removeAllMatching([&](auto time) {
+        return time <= frameTime - 2;
+    });
+
+    auto interval = m_observedFrameTimeStamps.last() - m_observedFrameTimeStamps.first();
+    if (interval > 1)
+        m_observedFrameRate = (m_observedFrameTimeStamps.size() / interval);
+
+    auto mediaSample = makeRefPtr(&sample);
+
+#if PLATFORM(COCOA)
+    if (!isRemote()) {
+        auto size = this->size();
+        if (!size.isEmpty() && size != expandedIntSize(sample.presentationSize())) {
+
+            if (!m_imageTransferSession || m_imageTransferSession->pixelFormat() != sample.videoPixelFormat())
+                m_imageTransferSession = ImageTransferSessionVT::create(sample.videoPixelFormat());
+
+            if (m_imageTransferSession) {
+                mediaSample = m_imageTransferSession->convertMediaSample(sample, size);
+                if (!mediaSample) {
+                    ASSERT_NOT_REACHED();
+                    return nullptr;
+                }
+            }
+        }
+    }
+#endif
+
+    return mediaSample.releaseNonNull();
+}
+
+void RealtimeVideoCaptureSource::dispatchMediaSampleToObservers(MediaSample& sample)
+{
+    if (auto mediaSample = adaptVideoSample(sample))
+        videoSampleAvailable(*mediaSample);
+}
+
+void RealtimeVideoCaptureSource::clientUpdatedSizeAndFrameRate(Optional<int> width, Optional<int> height, Optional<double> frameRate)
+{
+    // FIXME: We only change settings if capture resolution is below requested one. We should get the best preset for all clients.
+    auto& settings = this->settings();
+    if (width && *width < static_cast<int>(settings.width()))
+        width = { };
+    if (height && *height < static_cast<int>(settings.height()))
+        height = { };
+
+    // FIXME: handle frameRate potential increase.
+    if (!width && !height)
+        return;
+
+    auto match = bestSupportedSizeAndFrameRate(width, height, frameRate);
+    ERROR_LOG_IF(loggerPtr() && !match, LOGIDENTIFIER, "unable to find a preset that would match the size and frame rate");
+    if (!match)
+        return;
+
+    setFrameRateWithPreset(match->requestedFrameRate, match->encodingPreset);
+    setSize(match->encodingPreset->size);
+    setFrameRate(match->requestedFrameRate);
+}
+
+#if !RELEASE_LOG_DISABLED
+Ref<JSON::Object> SizeAndFrameRate::toJSONObject() const
+{
+    auto object = JSON::Object::create();
+
+    object->setDouble("width"_s, width ? width.value() : 0);
+    object->setDouble("height"_s, height ? height.value() : 0);
+    object->setDouble("frameRate"_s, frameRate ? frameRate.value() : 0);
+
+    return object;
+}
+
+String SizeAndFrameRate::toJSONString() const
+{
+    return toJSONObject()->toJSONString();
+}
+#endif
+
+} // namespace WebCore
+
+#endif // ENABLE(MEDIA_STREAM)
diff --git a/Source/WebCore/platform/mediastream/RealtimeVideoCaptureSource.h b/Source/WebCore/platform/mediastream/RealtimeVideoCaptureSource.h
new file mode 100644 (file)
index 0000000..a61c186
--- /dev/null
@@ -0,0 +1,124 @@
+/*
+ * Copyright (C) 2018-2019 Apple Inc. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL APPLE INC. OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+ * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
+ * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#pragma once
+
+#if ENABLE(MEDIA_STREAM)
+
+#include "ImageBuffer.h"
+#include "MediaSample.h"
+#include "RealtimeMediaSource.h"
+#include "VideoPreset.h"
+#include <wtf/Lock.h>
+#include <wtf/RunLoop.h>
+
+namespace WebCore {
+
+class ImageTransferSessionVT;
+
+class RealtimeVideoCaptureSource : public RealtimeMediaSource {
+public:
+    virtual ~RealtimeVideoCaptureSource();
+
+    void clientUpdatedSizeAndFrameRate(Optional<int> width, Optional<int> height, Optional<double> frameRate);
+
+    bool supportsSizeAndFrameRate(Optional<int> width, Optional<int> height, Optional<double>) override;
+    virtual void generatePresets() = 0;
+    virtual MediaSample::VideoRotation sampleRotation() const { return MediaSample::VideoRotation::None; }
+
+protected:
+    RealtimeVideoCaptureSource(String&& name, String&& id, String&& hashSalt);
+
+    void prepareToProduceData();
+    void setSizeAndFrameRate(Optional<int> width, Optional<int> height, Optional<double>) override;
+
+    virtual bool prefersPreset(VideoPreset&) { return true; }
+    virtual void setFrameRateWithPreset(double, RefPtr<VideoPreset>) { };
+    virtual bool canResizeVideoFrames() const { return false; }
+    bool shouldUsePreset(VideoPreset& current, VideoPreset& candidate);
+
+    void setSupportedPresets(const Vector<Ref<VideoPreset>>&);
+    void setSupportedPresets(Vector<VideoPresetData>&&);
+    const Vector<Ref<VideoPreset>>& presets();
+
+    bool frameRateRangeIncludesRate(const FrameRateRange&, double);
+
+    void updateCapabilities(RealtimeMediaSourceCapabilities&);
+
+    void setDefaultSize(const IntSize& size) { m_defaultSize = size; }
+
+    double observedFrameRate() const { return m_observedFrameRate; }
+
+    void dispatchMediaSampleToObservers(MediaSample&);
+    const Vector<IntSize>& standardVideoSizes();
+    RefPtr<MediaSample> adaptVideoSample(MediaSample&);
+
+private:
+    struct CaptureSizeAndFrameRate {
+        RefPtr<VideoPreset> encodingPreset;
+        IntSize requestedSize;
+        double requestedFrameRate { 0 };
+    };
+    bool supportsCaptureSize(Optional<int>, Optional<int>, const Function<bool(const IntSize&)>&&);
+    Optional<CaptureSizeAndFrameRate> bestSupportedSizeAndFrameRate(Optional<int> width, Optional<int> height, Optional<double>);
+    bool presetSupportsFrameRate(RefPtr<VideoPreset>, double);
+
+#if !RELEASE_LOG_DISABLED
+    const char* logClassName() const override { return "RealtimeVideoCaptureSource"; }
+#endif
+
+    Vector<Ref<VideoPreset>> m_presets;
+    Deque<double> m_observedFrameTimeStamps;
+    double m_observedFrameRate { 0 };
+    IntSize m_defaultSize;
+#if PLATFORM(COCOA)
+    std::unique_ptr<ImageTransferSessionVT> m_imageTransferSession;
+#endif
+};
+
+struct SizeAndFrameRate {
+    Optional<int> width;
+    Optional<int> height;
+    Optional<double> frameRate;
+
+    String toJSONString() const;
+    Ref<JSON::Object> toJSONObject() const;
+};
+
+} // namespace WebCore
+
+namespace WTF {
+template<typename Type> struct LogArgument;
+template <>
+struct LogArgument<WebCore::SizeAndFrameRate> {
+    static String toString(const WebCore::SizeAndFrameRate& size)
+    {
+        return size.toJSONString();
+    }
+};
+}; // namespace WTF
+
+#endif // ENABLE(MEDIA_STREAM)
+
index 065006e..1437d2d 100644 (file)
@@ -1,5 +1,5 @@
 /*
- * Copyright (C) 2018-2019 Apple Inc. All rights reserved.
+ * Copyright (C) 2019 Apple Inc. All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
 #include "RealtimeVideoSource.h"
 
 #if ENABLE(MEDIA_STREAM)
-#include "CaptureDevice.h"
-#include "Logging.h"
-#include "RealtimeMediaSourceCenter.h"
-#include "RealtimeMediaSourceSettings.h"
-#include "RemoteVideoSample.h"
-#include <wtf/JSONValues.h>
-
-#if PLATFORM(COCOA)
-#include "ImageTransferSessionVT.h"
-#endif
 
 namespace WebCore {
 
-RealtimeVideoSource::RealtimeVideoSource(String&& name, String&& id, String&& hashSalt)
-    : RealtimeMediaSource(Type::Video, WTFMove(name), WTFMove(id), WTFMove(hashSalt))
+RealtimeVideoSource::RealtimeVideoSource(Ref<RealtimeVideoCaptureSource>&& source)
+    : RealtimeVideoCaptureSource(String { source->name() }, String { source->persistentID() }, String { source->deviceIDHashSalt() })
+    , m_source(WTFMove(source))
 {
+    m_source->addObserver(*this);
+    m_currentSettings = m_source->settings();
 }
 
 RealtimeVideoSource::~RealtimeVideoSource()
 {
-#if PLATFORM(IOS_FAMILY)
-    RealtimeMediaSourceCenter::singleton().videoCaptureFactory().unsetActiveSource(*this);
-#endif
+    m_source->removeObserver(*this);
 }
 
-void RealtimeVideoSource::prepareToProduceData()
+void RealtimeVideoSource::startProducingData()
 {
-    ASSERT(frameRate());
-
-#if PLATFORM(IOS_FAMILY)
-    RealtimeMediaSourceCenter::singleton().videoCaptureFactory().setActiveSource(*this);
-#endif
-
-    if (size().isEmpty() && !m_defaultSize.isEmpty())
-        setSize(m_defaultSize);
+    m_source->start();
 }
 
-const Vector<Ref<VideoPreset>>& RealtimeVideoSource::presets()
+void RealtimeVideoSource::stopProducingData()
 {
-    if (m_presets.isEmpty())
-        generatePresets();
-
-    ASSERT(!m_presets.isEmpty());
-    return m_presets;
+    m_source->stop();
 }
 
-void RealtimeVideoSource::setSupportedPresets(Vector<VideoPresetData>&& presetData)
+bool RealtimeVideoSource::supportsSizeAndFrameRate(Optional<int> width, Optional<int> height, Optional<double> frameRate)
 {
-    Vector<Ref<VideoPreset>> presets;
-
-    for (auto& data : presetData)
-        presets.append(VideoPreset::create(WTFMove(data)));
-
-    setSupportedPresets(WTFMove(presets));
+    return m_source->supportsSizeAndFrameRate(width, height, frameRate);
 }
 
-void RealtimeVideoSource::setSupportedPresets(const Vector<Ref<VideoPreset>>& presets)
+void RealtimeVideoSource::setSizeAndFrameRate(Optional<int> width, Optional<int> height, Optional<double> frameRate)
 {
-    m_presets = WTF::map(presets, [](auto& preset) {
-        return preset.copyRef();
-    });
-
-    for (auto& preset : m_presets) {
-        std::sort(preset->frameRateRanges.begin(), preset->frameRateRanges.end(),
-            [&] (const auto& a, const auto& b) -> bool {
-                return a.minimum < b.minimum;
-        });
+    if (!width && !height) {
+        width = size().width();
+        height = size().height();
     }
-}
 
-const Vector<IntSize>& RealtimeVideoSource::standardVideoSizes()
-{
-    static const auto sizes = makeNeverDestroyed([] {
-        static IntSize videoSizes[] = {
-            { 112, 112 },
-            { 160, 160 },
-            { 160, 120 }, // 4:3, QQVGA
-            { 176, 144 }, // 4:3, QCIF
-            { 192, 192 },
-            { 192, 112 }, // 16:9
-            { 192, 144 }, // 3:4
-            { 240, 240 },
-            { 240, 160 }, // 3:2, HQVGA
-            { 320, 320 },
-            { 320, 180 }, // 16:9
-            { 320, 240 }, // 4:3, QVGA
-            { 352, 288 }, // CIF
-            { 480, 272 }, // 16:9
-            { 480, 360 }, // 4:3
-            { 480, 480 },
-            { 640, 640 },
-            { 640, 360 }, // 16:9, 360p nHD
-            { 640, 480 }, // 4:3
-            { 720, 720 },
-            { 800, 600 }, // 4:3, SVGA
-            { 960, 540 }, // 16:9, qHD
-            { 1024, 600 }, // 16:9, WSVGA
-            { 1024, 768 }, // 4:3, XGA
-            { 1280, 960 }, // 4:3
-            { 1280, 1024 }, // 5:4, SXGA
-            { 1280, 720 }, // 16:9, WXGA
-            { 1366, 768 }, // 16:9, HD
-            { 1600, 1200}, // 4:3, UXGA
-            { 1920, 1080 }, // 16:9, 1080p FHD
-            { 2560, 1440 }, // 16:9, QHD
-            { 2592, 1936 },
-            { 3264, 2448 }, // 3:4
-            { 3840, 2160 }, // 16:9, 4K UHD
-        };
-        Vector<IntSize> sizes;
-        for (auto& size : videoSizes)
-            sizes.append(size);
-
-        return sizes;
-    }());
-
-    return sizes.get();
-}
-template <typename ValueType>
-static void updateMinMax(ValueType& min, ValueType& max, ValueType value)
-{
-    min = std::min<ValueType>(min, value);
-    max = std::max<ValueType>(max, value);
-}
+    m_source->clientUpdatedSizeAndFrameRate(width, height, frameRate);
+    auto sourceSize = m_source->size();
+    ASSERT(sourceSize.height());
+    ASSERT(sourceSize.width());
 
-void RealtimeVideoSource::updateCapabilities(RealtimeMediaSourceCapabilities& capabilities)
-{
-    ASSERT(!presets().isEmpty());
+    if (!width)
+        width = sourceSize.width() * height.value() / sourceSize.height();
+    m_currentSettings.setWidth(*width);
 
-    int minimumWidth = std::numeric_limits<int>::max();
-    int maximumWidth = 0;
-    int minimumHeight = std::numeric_limits<int>::max();
-    int maximumHeight = 0;
-    double minimumAspectRatio = std::numeric_limits<double>::max();
-    double maximumAspectRatio = 0;
-    double minimumFrameRate = std::numeric_limits<double>::max();
-    double maximumFrameRate = 0;
-    for (const auto& preset : presets()) {
-        const auto& size = preset->size;
-        updateMinMax(minimumWidth, maximumWidth, size.width());
-        updateMinMax(minimumHeight, maximumHeight, size.height());
-        updateMinMax(minimumAspectRatio, maximumAspectRatio, static_cast<double>(size.width()) / size.height());
+    if (!height)
+        height = sourceSize.height() * width.value() / sourceSize.width();
+    m_currentSettings.setHeight(*height);
 
-        for (const auto& rate : preset->frameRateRanges) {
-            updateMinMax(minimumFrameRate, maximumFrameRate, rate.minimum);
-            updateMinMax(minimumFrameRate, maximumFrameRate, rate.maximum);
-        }
-    }
-
-    if (canResizeVideoFrames()) {
-        minimumWidth = 1;
-        minimumHeight = 1;
-        for (auto& size : standardVideoSizes()) {
-            if (size.width() < minimumWidth || size.height() < minimumHeight)
-                minimumAspectRatio = std::min(minimumAspectRatio, static_cast<double>(size.width()) / size.height());
-        }
-    }
+    if (frameRate)
+        m_currentSettings.setFrameRate(static_cast<float>(*frameRate));
 
-    capabilities.setWidth({ minimumWidth, maximumWidth });
-    capabilities.setHeight({ minimumHeight, maximumHeight });
-    capabilities.setAspectRatio({ minimumAspectRatio, maximumAspectRatio });
-    capabilities.setFrameRate({ minimumFrameRate, maximumFrameRate });
+    RealtimeMediaSource::setSizeAndFrameRate(width, height, frameRate);
 }
 
-bool RealtimeVideoSource::supportsSizeAndFrameRate(Optional<int> width, Optional<int> height, Optional<double> frameRate)
-{
-    if (!width && !height && !frameRate)
-        return true;
-
-    return !!bestSupportedSizeAndFrameRate(width, height, frameRate);
-}
-
-bool RealtimeVideoSource::frameRateRangeIncludesRate(const FrameRateRange& range, double frameRate)
+void RealtimeVideoSource::sourceMutedChanged()
 {
-    const double epsilon = 0.001;
-    return frameRate + epsilon >= range.minimum && frameRate - epsilon <= range.maximum;
+    notifyMutedChange(m_source->muted());
 }
 
-bool RealtimeVideoSource::presetSupportsFrameRate(RefPtr<VideoPreset> preset, double frameRate)
+void RealtimeVideoSource::sourceSettingsChanged()
 {
-    for (const auto& range : preset->frameRateRanges) {
-        if (frameRateRangeIncludesRate(range, frameRate))
-            return true;
+    auto rotation = m_source->sampleRotation();
+    if (rotation == MediaSample::VideoRotation::Left || rotation == MediaSample::VideoRotation::Right) {
+        auto size = this->size();
+        size = size.transposedSize();
+        m_currentSettings.setWidth(size.width());
+        m_currentSettings.setHeight(size.height());
     }
 
-    return false;
+    forEachObserver([&](auto& observer) {
+        observer.sourceSettingsChanged();
+    });
 }
 
-bool RealtimeVideoSource::supportsCaptureSize(Optional<int> width, Optional<int> height, const Function<bool(const IntSize&)>&& function)
+bool RealtimeVideoSource::preventSourceFromStopping()
 {
-    if (width && height)
-        return function({ width.value(), height.value() });
-
-    if (width) {
-        for (auto& size : standardVideoSizes()) {
-            if (width.value() == size.width() && function({ size.width(), size.height() }))
-                return true;
-        }
-
+    if (!isProducingData())
         return false;
-    }
 
-    for (auto& size : standardVideoSizes()) {
-        if (height.value() == size.height() && function({ size.width(), size.height() }))
-            return true;
-    }
-
-    return false;
-}
-
-bool RealtimeVideoSource::shouldUsePreset(VideoPreset& current, VideoPreset& candidate)
-{
-    return candidate.size.width() <= current.size.width() && candidate.size.height() <= current.size.height() && prefersPreset(candidate);
+    bool hasObserverPreventingStopping = false;
+    forEachObserver([&](auto& observer) {
+        if (observer.preventSourceFromStopping())
+            hasObserverPreventingStopping = true;
+    });
+    return hasObserverPreventingStopping;
 }
 
-Optional<RealtimeVideoSource::CaptureSizeAndFrameRate> RealtimeVideoSource::bestSupportedSizeAndFrameRate(Optional<int> requestedWidth, Optional<int> requestedHeight, Optional<double> requestedFrameRate)
+void RealtimeVideoSource::sourceStopped()
 {
-    if (!requestedWidth && !requestedHeight && !requestedFrameRate)
-        return { };
-
-    if (!requestedWidth && !requestedHeight && !size().isEmpty()) {
-        requestedWidth = size().width();
-        requestedHeight = size().height();
-    }
-    if (!requestedFrameRate)
-        requestedFrameRate = frameRate();
-
-    CaptureSizeAndFrameRate result;
-    RefPtr<VideoPreset> exactSizePreset;
-    RefPtr<VideoPreset> aspectRatioPreset;
-    IntSize aspectRatioMatchSize;
-    RefPtr<VideoPreset> resizePreset;
-    IntSize resizeSize;
-
-    for (const auto& preset : presets()) {
-        const auto& presetSize = preset->size;
-
-        if (!presetSupportsFrameRate(&preset.get(), requestedFrameRate.value()))
-            continue;
-
-        if (!requestedWidth && !requestedHeight) {
-            result.requestedFrameRate = requestedFrameRate.value();
-            return result;
-        }
-
-        // Don't look at presets smaller than the requested resolution because we never want to resize larger.
-        if ((requestedWidth && presetSize.width() < requestedWidth.value()) || (requestedHeight && presetSize.height() < requestedHeight.value()))
-            continue;
-
-        auto lookForExactSizeMatch = [&] (const IntSize& size) -> bool {
-            return preset->size == size;
-        };
-        if (supportsCaptureSize(requestedWidth, requestedHeight, WTFMove(lookForExactSizeMatch))) {
-            if (!exactSizePreset || prefersPreset(preset))
-                exactSizePreset = &preset.get();
-            continue;
-        }
-
-        IntSize encodingSize;
-        auto lookForAspectRatioMatch = [this, &preset, &encodingSize] (const IntSize& size) -> bool {
-            auto aspectRatio = [] (const IntSize size) -> double {
-                return size.width() / static_cast<double>(size.height());
-            };
-            if (std::abs(aspectRatio(preset->size) - aspectRatio(size)) > 10e-7 || !canResizeVideoFrames())
-                return false;
-
-            encodingSize = size;
-            return true;
-        };
-        if (supportsCaptureSize(requestedWidth, requestedHeight, WTFMove(lookForAspectRatioMatch))) {
-            if (!aspectRatioPreset || shouldUsePreset(*aspectRatioPreset, preset)) {
-                aspectRatioPreset = &preset.get();
-                aspectRatioMatchSize = encodingSize;
-            }
-        }
-
-        if (exactSizePreset || aspectRatioPreset)
-            continue;
-
-        if ((requestedWidth && requestedWidth.value() > preset->size.width()) || (requestedHeight && requestedHeight.value() > preset->size.height()))
-            continue;
-
-        if (requestedWidth && requestedHeight) {
-            if (!resizePreset || shouldUsePreset(*resizePreset, preset)) {
-                resizePreset = &preset.get();
-                resizeSize = { requestedWidth.value(), requestedHeight.value() };
-            }
-        } else {
-            for (auto& standardSize : standardVideoSizes()) {
-                if (standardSize.width() > preset->size.width() || standardSize.height() > preset->size.height())
-                    break;
-                if ((requestedWidth && requestedWidth.value() != standardSize.width()) || (requestedHeight && requestedHeight.value() != standardSize.height()))
-                    continue;
-
-                if (!resizePreset || shouldUsePreset(*resizePreset, preset)) {
-                    resizePreset = &preset.get();
-                    resizeSize = standardSize;
-                }
-            }
-
-            if (!resizePreset || shouldUsePreset(*resizePreset, preset)) {
-                resizePreset = &preset.get();
-                if (requestedWidth)
-                    resizeSize = { requestedWidth.value(), requestedWidth.value() * preset->size.height() / preset->size.width()};
-                else
-                    resizeSize = { requestedHeight.value() * preset->size.width() / preset->size.height(), requestedHeight.value() };
-            }
-        }
-    }
-
-    if (!exactSizePreset && !aspectRatioPreset && !resizePreset)
-        return { };
-
-    result.requestedFrameRate = requestedFrameRate.value();
-    if (exactSizePreset) {
-        result.encodingPreset = exactSizePreset;
-        result.requestedSize = exactSizePreset->size;
-        return result;
-    }
-
-    if (aspectRatioPreset) {
-        result.encodingPreset = aspectRatioPreset;
-        result.requestedSize = aspectRatioMatchSize;
-        return result;
+    if (m_source->captureDidFail()) {
+        captureFailed();
+        return;
     }
-
-    result.encodingPreset = resizePreset;
-    result.requestedSize = resizeSize;
-    return result;
+    stop();
+    forEachObserver([](auto& observer) {
+        observer.sourceStopped();
+    });
 }
 
-void RealtimeVideoSource::setSizeAndFrameRate(Optional<int> width, Optional<int> height, Optional<double> frameRate)
+void RealtimeVideoSource::videoSampleAvailable(MediaSample& sample)
 {
-    ALWAYS_LOG_IF(loggerPtr(), LOGIDENTIFIER, SizeAndFrameRate { width, height, frameRate });
-
-    auto size = this->size();
-    if (!width && !height && !size.isEmpty()) {
-        width = size.width();
-        height = size.height();
-    }
-
-    Optional<RealtimeVideoSource::CaptureSizeAndFrameRate> match = bestSupportedSizeAndFrameRate(width, height, frameRate);
-    ASSERT(match);
-    if (!match)
+    if (!isProducingData())
         return;
 
-    setFrameRateWithPreset(match->requestedFrameRate, match->encodingPreset);
-
-    if (!match->requestedSize.isEmpty())
-        setSize(match->requestedSize);
-    setFrameRate(match->requestedFrameRate);
-}
-
-void RealtimeVideoSource::dispatchMediaSampleToObservers(MediaSample& sample)
-{
-    MediaTime sampleTime = sample.outputPresentationTime();
-    if (!sampleTime || !sampleTime.isValid())
-        sampleTime = sample.presentationTime();
-
-    auto frameTime = sampleTime.toDouble();
-    m_observedFrameTimeStamps.append(frameTime);
-    m_observedFrameTimeStamps.removeAllMatching([&](auto time) {
-        return time <= frameTime - 2;
-    });
-
-    auto interval = m_observedFrameTimeStamps.last() - m_observedFrameTimeStamps.first();
-    if (interval > 1)
-        m_observedFrameRate = (m_observedFrameTimeStamps.size() / interval);
-
-    auto mediaSample = makeRefPtr(&sample);
-#if PLATFORM(COCOA)
-    if (!isRemote()) {
-        auto size = this->size();
-        if (!size.isEmpty() && size != expandedIntSize(sample.presentationSize())) {
-
-            if (!m_imageTransferSession || m_imageTransferSession->pixelFormat() != sample.videoPixelFormat())
-                m_imageTransferSession = ImageTransferSessionVT::create(sample.videoPixelFormat());
-
-            if (m_imageTransferSession) {
-                mediaSample = m_imageTransferSession->convertMediaSample(sample, size);
-                if (!mediaSample) {
-                    ASSERT_NOT_REACHED();
-                    return;
-                }
-            }
-        }
-    }
-#endif
-
-    videoSampleAvailable(mediaSample.releaseNonNull());
+    if (auto mediaSample = adaptVideoSample(sample))
+        RealtimeMediaSource::videoSampleAvailable(*mediaSample);
 }
 
-#if !RELEASE_LOG_DISABLED
-Ref<JSON::Object> SizeAndFrameRate::toJSONObject() const
+Ref<RealtimeMediaSource> RealtimeVideoSource::clone()
 {
-    auto object = JSON::Object::create();
-
-    object->setDouble("width"_s, width ? width.value() : 0);
-    object->setDouble("height"_s, height ? height.value() : 0);
-    object->setDouble("frameRate"_s, frameRate ? frameRate.value() : 0);
+    auto source = create(m_source.copyRef());
+    source->m_currentSettings = m_currentSettings;
 
-    return object;
-}
-
-String SizeAndFrameRate::toJSONString() const
-{
-    return toJSONObject()->toJSONString();
+    return source;
 }
-#endif
 
 } // namespace WebCore
 
index 1a17938..6f79468 100644 (file)
@@ -1,5 +1,5 @@
 /*
- * Copyright (C) 2018-2019 Apple Inc. All rights reserved.
+ * Copyright (C) 2019 Apple Inc. All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
 
 #if ENABLE(MEDIA_STREAM)
 
-#include "ImageBuffer.h"
-#include "MediaSample.h"
-#include "RealtimeMediaSource.h"
-#include "VideoPreset.h"
-#include <wtf/Lock.h>
-#include <wtf/RunLoop.h>
+#include "RealtimeVideoCaptureSource.h"
 
 namespace WebCore {
 
-class ImageTransferSessionVT;
-
-class RealtimeVideoSource : public RealtimeMediaSource {
+// FIXME: Make RealtimeVideoSource derive from RealtimeMediaSource directly.
+class RealtimeVideoSource final : public RealtimeVideoCaptureSource, public RealtimeMediaSource::Observer {
 public:
-    virtual ~RealtimeVideoSource();
-
-protected:
-    RealtimeVideoSource(String&& name, String&& id, String&& hashSalt);
-
-    void prepareToProduceData();
-    bool supportsSizeAndFrameRate(Optional<int> width, Optional<int> height, Optional<double>) override;
-    void setSizeAndFrameRate(Optional<int> width, Optional<int> height, Optional<double>) override;
-
-    virtual void generatePresets() = 0;
-    virtual bool prefersPreset(VideoPreset&) { return true; }
-    virtual void setFrameRateWithPreset(double, RefPtr<VideoPreset>) { };
-    virtual bool canResizeVideoFrames() const { return false; }
-    bool shouldUsePreset(VideoPreset& current, VideoPreset& candidate);
-
-    void setSupportedPresets(const Vector<Ref<VideoPreset>>&);
-    void setSupportedPresets(Vector<VideoPresetData>&&);
-    const Vector<Ref<VideoPreset>>& presets();
-
-    bool frameRateRangeIncludesRate(const FrameRateRange&, double);
-
-    void updateCapabilities(RealtimeMediaSourceCapabilities&);
-
-    void setDefaultSize(const IntSize& size) { m_defaultSize = size; }
-
-    double observedFrameRate() const { return m_observedFrameRate; }
-
-    void dispatchMediaSampleToObservers(MediaSample&);
-    const Vector<IntSize>& standardVideoSizes();
+    static Ref<RealtimeVideoSource> create(Ref<RealtimeVideoCaptureSource>&& source) { return adoptRef(*new RealtimeVideoSource(WTFMove(source))); }
 
 private:
-    struct CaptureSizeAndFrameRate {
-        RefPtr<VideoPreset> encodingPreset;
-        IntSize requestedSize;
-        double requestedFrameRate { 0 };
-    };
-    bool supportsCaptureSize(Optional<int>, Optional<int>, const Function<bool(const IntSize&)>&&);
-    Optional<CaptureSizeAndFrameRate> bestSupportedSizeAndFrameRate(Optional<int> width, Optional<int> height, Optional<double>);
-    bool presetSupportsFrameRate(RefPtr<VideoPreset>, double);
-
-#if !RELEASE_LOG_DISABLED
-    const char* logClassName() const override { return "RealtimeVideoSource"; }
-#endif
-
-    Vector<Ref<VideoPreset>> m_presets;
-    Deque<double> m_observedFrameTimeStamps;
-    double m_observedFrameRate { 0 };
-    IntSize m_defaultSize;
-#if PLATFORM(COCOA)
-    std::unique_ptr<ImageTransferSessionVT> m_imageTransferSession;
-#endif
-};
-
-struct SizeAndFrameRate {
-    Optional<int> width;
-    Optional<int> height;
-    Optional<double> frameRate;
-
-    String toJSONString() const;
-    Ref<JSON::Object> toJSONObject() const;
+    explicit RealtimeVideoSource(Ref<RealtimeVideoCaptureSource>&&);
+    ~RealtimeVideoSource();
+
+    // RealtimeVideoCaptureSource
+    void startProducingData() final;
+    void stopProducingData() final;
+    bool supportsSizeAndFrameRate(Optional<int> width, Optional<int> height, Optional<double> frameRate) final;
+    void setSizeAndFrameRate(Optional<int> width, Optional<int> height, Optional<double> frameRate) final;
+    Ref<RealtimeMediaSource> clone() final;
+
+    const RealtimeMediaSourceCapabilities& capabilities() final { return m_source->capabilities(); }
+    const RealtimeMediaSourceSettings& settings() final { return m_currentSettings; }
+    void generatePresets() final { m_source->generatePresets(); }
+    bool isCaptureSource() const final { return m_source->isCaptureSource(); }
+    CaptureDevice::DeviceType deviceType() const final { return m_source->deviceType(); }
+    void monitorOrientation(OrientationNotifier& notifier) final { m_source->monitorOrientation(notifier); }
+    bool interrupted() const final { return m_source->interrupted(); }
+
+    // Observer
+    void sourceMutedChanged() final;
+    void sourceSettingsChanged() final;
+    void sourceStopped() final;
+    bool preventSourceFromStopping() final;
+    void videoSampleAvailable(MediaSample&) final;
+
+    Ref<RealtimeVideoCaptureSource> m_source;
+    RealtimeMediaSourceSettings m_currentSettings;
 };
 
 } // namespace WebCore
 
-namespace WTF {
-template<typename Type> struct LogArgument;
-template <>
-struct LogArgument<WebCore::SizeAndFrameRate> {
-    static String toString(const WebCore::SizeAndFrameRate& size)
-    {
-        return size.toJSONString();
-    }
-};
-}; // namespace WTF
-
 #endif // ENABLE(MEDIA_STREAM)
 
index d7ca8d9..42eb4ac 100644 (file)
@@ -126,14 +126,14 @@ DisplayCaptureFactory& GStreamerVideoCaptureSource::displayFactory()
 }
 
 GStreamerVideoCaptureSource::GStreamerVideoCaptureSource(String&& deviceID, String&& name, String&& hashSalt, const gchar *source_factory)
-    : RealtimeVideoSource(WTFMove(deviceID), WTFMove(name), WTFMove(hashSalt))
+    : RealtimeVideoCaptureSource(WTFMove(deviceID), WTFMove(name), WTFMove(hashSalt))
     , m_capturer(std::make_unique<GStreamerVideoCapturer>(source_factory))
 {
     initializeGStreamerDebug();
 }
 
 GStreamerVideoCaptureSource::GStreamerVideoCaptureSource(GStreamerCaptureDevice device, String&& hashSalt)
-    : RealtimeVideoSource(String { device.persistentId() }, String { device.label() }, WTFMove(hashSalt))
+    : RealtimeVideoCaptureSource(String { device.persistentId() }, String { device.label() }, WTFMove(hashSalt))
     , m_capturer(std::make_unique<GStreamerVideoCapturer>(device))
 {
     initializeGStreamerDebug();
index 4681d11..380611b 100644 (file)
 #if ENABLE(MEDIA_STREAM) && USE(LIBWEBRTC) && USE(GSTREAMER)
 #include "CaptureDevice.h"
 #include "GStreamerVideoCapturer.h"
-#include "RealtimeVideoSource.h"
+#include "RealtimeVideoCaptureSource.h"
 
 namespace WebCore {
 
-class GStreamerVideoCaptureSource : public RealtimeVideoSource {
+class GStreamerVideoCaptureSource : public RealtimeVideoCaptureSource {
 public:
     static CaptureSourceOrError create(String&& deviceID, String&& hashSalt, const MediaConstraints*);
     WEBCORE_EXPORT static VideoCaptureFactory& factory();
index 5d6cb1e..2a85ad4 100644 (file)
@@ -29,7 +29,7 @@
 
 #include "IntSizeHash.h"
 #include "OrientationNotifier.h"
-#include "RealtimeVideoSource.h"
+#include "RealtimeVideoCaptureSource.h"
 #include <wtf/Lock.h>
 #include <wtf/text/StringHash.h>
 
@@ -51,7 +51,7 @@ namespace WebCore {
 class AVVideoPreset;
 class ImageTransferSessionVT;
 
-class AVVideoCaptureSource : public RealtimeVideoSource, private OrientationNotifier::Observer {
+class AVVideoCaptureSource : public RealtimeVideoCaptureSource, private OrientationNotifier::Observer {
 public:
     static CaptureSourceOrError create(String&& id, String&& hashSalt, const MediaConstraints*);
 
@@ -91,6 +91,7 @@ private:
     CaptureDevice::DeviceType deviceType() const final { return CaptureDevice::DeviceType::Camera; }
     bool interrupted() const final;
 
+    MediaSample::VideoRotation sampleRotation() const final { return m_sampleRotation; }
     void setFrameRateWithPreset(double, RefPtr<VideoPreset>) final;
     bool prefersPreset(VideoPreset&) final;
     void generatePresets() final;
index 8b40a29..5c612ed 100644 (file)
@@ -37,6 +37,7 @@
 #import "PlatformLayer.h"
 #import "RealtimeMediaSourceCenter.h"
 #import "RealtimeMediaSourceSettings.h"
+#import "RealtimeVideoSource.h"
 #import "RealtimeVideoUtilities.h"
 #import <AVFoundation/AVCaptureDevice.h>
 #import <AVFoundation/AVCaptureInput.h>
@@ -121,11 +122,11 @@ CaptureSourceOrError AVVideoCaptureSource::create(String&& id, String&& hashSalt
             return WTFMove(result.value().badConstraint);
     }
 
-    return CaptureSourceOrError(WTFMove(source));
+    return CaptureSourceOrError(RealtimeVideoSource::create(WTFMove(source)));
 }
 
 AVVideoCaptureSource::AVVideoCaptureSource(AVCaptureDevice* device, String&& id, String&& hashSalt)
-    : RealtimeVideoSource(device.localizedName, WTFMove(id), WTFMove(hashSalt))
+    : RealtimeVideoCaptureSource(device.localizedName, WTFMove(id), WTFMove(hashSalt))
     , m_objcObserver(adoptNS([[WebCoreAVVideoCaptureSourceObserver alloc] initWithCallback:this]))
     , m_device(device)
 {
@@ -637,7 +638,6 @@ void AVVideoCaptureSource::deviceDisconnected(RetainPtr<NSNotification> notifica
         captureFailed();
 }
 
-
 } // namespace WebCore
 
 @implementation WebCoreAVVideoCaptureSourceObserver
index 8843afa..5398b43 100644 (file)
@@ -59,6 +59,7 @@ private:
 
     void orientationChanged(int orientation) final;
     void monitorOrientation(OrientationNotifier&) final;
+    MediaSample::VideoRotation sampleRotation() const final { return m_deviceOrientation; }
 
     MediaSample::VideoRotation m_deviceOrientation { MediaSample::VideoRotation::None };
     std::unique_ptr<ImageTransferSessionVT> m_imageTransferSession;
index 5c78987..83c63b1 100644 (file)
@@ -41,6 +41,7 @@
 #import "NotImplemented.h"
 #import "PlatformLayer.h"
 #import "RealtimeMediaSourceSettings.h"
+#import "RealtimeVideoSource.h"
 #import "RealtimeVideoUtilities.h"
 #import <QuartzCore/CALayer.h>
 #import <QuartzCore/CATransaction.h>
@@ -66,7 +67,7 @@ CaptureSourceOrError MockRealtimeVideoSource::create(String&& deviceID, String&&
     if (constraints && source->applyConstraints(*constraints))
         return { };
 
-    return CaptureSourceOrError(WTFMove(source));
+    return CaptureSourceOrError(RealtimeVideoSource::create(WTFMove(source)));
 }
 
 MockRealtimeVideoSourceMac::MockRealtimeVideoSourceMac(String&& deviceID, String&& name, String&& hashSalt)
index 36d8c75..49a48be 100644 (file)
@@ -42,6 +42,7 @@
 #include "NotImplemented.h"
 #include "PlatformLayer.h"
 #include "RealtimeMediaSourceSettings.h"
+#include "RealtimeVideoSource.h"
 #include <math.h>
 #include <wtf/UUID.h>
 #include <wtf/text/StringConcatenateNumbers.h>
@@ -62,12 +63,12 @@ CaptureSourceOrError MockRealtimeVideoSource::create(String&& deviceID, String&&
     if (constraints && source->applyConstraints(*constraints))
         return { };
 
-    return CaptureSourceOrError(WTFMove(source));
+    return CaptureSourceOrError(RealtimeVideoSource::create(WTFMove(source)));
 }
 #endif
 
 MockRealtimeVideoSource::MockRealtimeVideoSource(String&& deviceID, String&& name, String&& hashSalt)
-    : RealtimeVideoSource(WTFMove(name), WTFMove(deviceID), WTFMove(hashSalt))
+    : RealtimeVideoCaptureSource(WTFMove(name), WTFMove(deviceID), WTFMove(hashSalt))
     , m_emitFrameTimer(RunLoop::current(), this, &MockRealtimeVideoSource::generateFrame)
 {
     auto device = MockRealtimeMediaSourceCenter::mockDeviceWithPersistentID(persistentID());
@@ -97,7 +98,7 @@ bool MockRealtimeVideoSource::supportsSizeAndFrameRate(Optional<int> width, Opti
     // FIXME: consider splitting mock display into another class so we don't don't have to do this silly dance
     // because of the RealtimeVideoSource inheritance.
     if (mockCamera())
-        return RealtimeVideoSource::supportsSizeAndFrameRate(width, height, rate);
+        return RealtimeVideoCaptureSource::supportsSizeAndFrameRate(width, height, rate);
 
     return RealtimeMediaSource::supportsSizeAndFrameRate(width, height, rate);
 }
@@ -107,7 +108,7 @@ void MockRealtimeVideoSource::setSizeAndFrameRate(Optional<int> width, Optional<
     // FIXME: consider splitting mock display into another class so we don't don't have to do this silly dance
     // because of the RealtimeVideoSource inheritance.
     if (mockCamera()) {
-        RealtimeVideoSource::setSizeAndFrameRate(width, height, rate);
+        RealtimeVideoCaptureSource::setSizeAndFrameRate(width, height, rate);
         return;
     }
 
index 866c1b0..cbe1a00 100644 (file)
@@ -37,7 +37,7 @@
 #include "ImageBuffer.h"
 #include "MockMediaDevice.h"
 #include "RealtimeMediaSourceFactory.h"
-#include "RealtimeVideoSource.h"
+#include "RealtimeVideoCaptureSource.h"
 #include <wtf/RunLoop.h>
 
 namespace WebCore {
@@ -45,7 +45,7 @@ namespace WebCore {
 class FloatRect;
 class GraphicsContext;
 
-class MockRealtimeVideoSource : public RealtimeVideoSource {
+class MockRealtimeVideoSource : public RealtimeVideoCaptureSource {
 public:
 
     static CaptureSourceOrError create(String&& deviceID, String&& name, String&& hashSalt, const MediaConstraints*);