[iOS] Local capture MediaStreamTrack does not render in portrait mode
authoryouenn@apple.com <youenn@apple.com@268f45cc-cd09-0410-ab3c-d52691b4dbfc>
Fri, 5 Jul 2019 18:54:14 +0000 (18:54 +0000)
committeryouenn@apple.com <youenn@apple.com@268f45cc-cd09-0410-ab3c-d52691b4dbfc>
Fri, 5 Jul 2019 18:54:14 +0000 (18:54 +0000)
https://bugs.webkit.org/show_bug.cgi?id=199519
<rdar://problem/52689720>

Reviewed by Eric Carlson.

Source/WebCore:

RealtimeVideoSource was badly computing its size in case of rotation.
Whenever its underlying source is notifying of settings change,
compute the size and transpose it only in left/right case.

Update mock video source to cover that case.
Covered by updated test.

* platform/mediastream/RealtimeVideoSource.cpp:
(WebCore::RealtimeVideoSource::sourceSettingsChanged):
* platform/mediastream/mac/MockRealtimeVideoSourceMac.h:
* platform/mediastream/mac/MockRealtimeVideoSourceMac.mm:
(WebCore::MockRealtimeVideoSourceMac::updateSampleBuffer):
* platform/mock/MockRealtimeVideoSource.cpp:
(WebCore::MockRealtimeVideoSource::settings):
(WebCore::MockRealtimeVideoSource::orientationChanged):
(WebCore::MockRealtimeVideoSource::monitorOrientation):
* platform/mock/MockRealtimeVideoSource.h:

LayoutTests:

* webrtc/video-rotation-expected.txt:
* webrtc/video-rotation.html:

git-svn-id: https://svn.webkit.org/repository/webkit/trunk@247165 268f45cc-cd09-0410-ab3c-d52691b4dbfc

LayoutTests/ChangeLog
LayoutTests/webrtc/video-rotation-expected.txt
LayoutTests/webrtc/video-rotation.html
Source/WebCore/ChangeLog
Source/WebCore/platform/mediastream/RealtimeVideoSource.cpp
Source/WebCore/platform/mediastream/mac/MockRealtimeVideoSourceMac.h
Source/WebCore/platform/mediastream/mac/MockRealtimeVideoSourceMac.mm
Source/WebCore/platform/mock/MockRealtimeVideoSource.cpp
Source/WebCore/platform/mock/MockRealtimeVideoSource.h

index 2ddac56..dade16f 100644 (file)
@@ -1,3 +1,14 @@
+2019-07-05  Youenn Fablet  <youenn@apple.com>
+
+        [iOS] Local capture MediaStreamTrack does not render in portrait mode
+        https://bugs.webkit.org/show_bug.cgi?id=199519
+        <rdar://problem/52689720>
+
+        Reviewed by Eric Carlson.
+
+        * webrtc/video-rotation-expected.txt:
+        * webrtc/video-rotation.html:
+
 2019-07-05  Ryan Haddad  <ryanhaddad@apple.com>
 
         Unreviewed, rolling out r247115.
index 70dd4fc..4ee3ffa 100644 (file)
@@ -1,7 +1,8 @@
-  
+    
 
 PASS Setting video exchange 
 PASS Track is enabled, video should not be black 
-PASS Track is enabled and rotated, video should not be black and should change size 
+PASS Track is enabled and rotated, local video should not be black and should change size 
+PASS Track is enabled and rotated, remote video should not be black and should change size 
 PASS Track is enabled and rotated again, video should not be black and should change size 
 
index a87698d..584056c 100644 (file)
@@ -7,15 +7,17 @@
         <script src="../resources/testharnessreport.js"></script>
     </head>
     <body>
-        <video id="video" autoplay playsInline width="320" height="240"></video>
+        <video id="localVideo" autoplay playsInline width="320" height="240"></video>
+        <video id="remoteVideo" autoplay playsInline width="320" height="240"></video>
+        <canvas id="canvas0" width="320" height="240"></canvas>
         <canvas id="canvas1" width="320" height="240"></canvas>
         <canvas id="canvas2" width="320" height="240"></canvas>
         <canvas id="canvas3" width="320" height="240"></canvas>
         <script src ="routines.js"></script>
         <script>
-function isVideoBlack(id)
+function isVideoBlack(video, canvasId)
 {
-    var canvas = document.getElementById(id);
+    var canvas = document.getElementById(canvasId);
     canvas.width = video.videoWidth;
     canvas.height = video.videoHeight;
     canvas.getContext('2d').drawImage(video, 0, 0, canvas.width, canvas.height);
@@ -30,21 +32,21 @@ function isVideoBlack(id)
     return true;
 }
 
-function pollVideoBlackCheck(expected, id, resolve)
+function pollVideoBlackCheck(expected, video, canvasId, resolve)
 {
-    if (isVideoBlack(id) === expected) {
+    if (isVideoBlack(video, canvasId) === expected) {
         resolve();
         return;
     }
 
-    setTimeout(() => pollVideoBlackCheck(expected, id, resolve), 100);
+    setTimeout(() => pollVideoBlackCheck(expected, video, canvasId, resolve), 100);
 }
 
-function checkVideoBlack(expected, id)
+function checkVideoBlack(expected, video, canvasId)
 {
     return new Promise((resolve, reject) => {
-        pollVideoBlackCheck(expected, id, resolve);
-        setTimeout(() => reject("checkVideoBlack timed out for '" + id + "', expected " + expected), 5000);
+        pollVideoBlackCheck(expected, video, canvasId, resolve);
+        setTimeout(() => reject("checkVideoBlack timed out for '" + canvasId + "', expected " + expected), 5000);
     });
 }
 
@@ -54,6 +56,7 @@ promise_test((test) => {
         testRunner.setUserMediaPermission(true);
 
     return navigator.mediaDevices.getUserMedia({video: {width: 320, height: 240, facingMode: "environment"}}).then((localStream) => {
+        localVideo.srcObject = localStream;
         return new Promise((resolve, reject) => {
             track = localStream.getVideoTracks()[0];
 
@@ -69,30 +72,42 @@ promise_test((test) => {
             setTimeout(() => reject("Test timed out"), 5000);
         });
     }).then((remoteStream) => {
-        video.srcObject = remoteStream;
-        return video.play();
+        remoteVideo.srcObject = remoteStream;
+        return remoteVideo.play();
     });
 }, "Setting video exchange");
 
-promise_test((test) => {
-   return checkVideoBlack(false, "canvas1");
+promise_test(async (test) => {
+   await checkVideoBlack(false, remoteVideo, "canvas0");
+   await waitForVideoSize(remoteVideo, 320, 240);
 }, "Track is enabled, video should not be black");
 
+promise_test(async (test) => {
+    await checkVideoBlack(false, localVideo, "canvas0");
+    await waitForVideoSize(localVideo, 320, 240);
+
+    if (window.internals)
+        window.internals.setCameraMediaStreamTrackOrientation(track, 90);
+
+    await checkVideoBlack(false, localVideo, "canvas1");
+    await waitForVideoSize(localVideo, 240, 320);
+}, "Track is enabled and rotated, local video should not be black and should change size");
+
 promise_test((test) => {
     if (window.internals)
         window.internals.setCameraMediaStreamTrackOrientation(track, 90);
 
-    return checkVideoBlack(false, "canvas1").then(() => {
-        return waitForVideoSize(video, 240, 320);
+    return checkVideoBlack(false, remoteVideo, "canvas2").then(() => {
+        return waitForVideoSize(remoteVideo, 240, 320);
     });
-}, "Track is enabled and rotated, video should not be black and should change size");
+}, "Track is enabled and rotated, remote video should not be black and should change size");
 
 promise_test((test) => {
     if (window.internals)
         window.internals.setCameraMediaStreamTrackOrientation(track, 180);
 
-    return checkVideoBlack(false, "canvas1").then(() => {
-        return waitForVideoSize(video, 320, 240);
+    return checkVideoBlack(false, remoteVideo, "canvas3").then(() => {
+        return waitForVideoSize(remoteVideo, 320, 240);
     });
 }, "Track is enabled and rotated again, video should not be black and should change size");
         </script>
index 6c9d172..aeefe81 100644 (file)
@@ -1,3 +1,29 @@
+2019-07-05  Youenn Fablet  <youenn@apple.com>
+
+        [iOS] Local capture MediaStreamTrack does not render in portrait mode
+        https://bugs.webkit.org/show_bug.cgi?id=199519
+        <rdar://problem/52689720>
+
+        Reviewed by Eric Carlson.
+
+        RealtimeVideoSource was badly computing its size in case of rotation.
+        Whenever its underlying source is notifying of settings change,
+        compute the size and transpose it only in left/right case.
+
+        Update mock video source to cover that case.
+        Covered by updated test.
+
+        * platform/mediastream/RealtimeVideoSource.cpp:
+        (WebCore::RealtimeVideoSource::sourceSettingsChanged):
+        * platform/mediastream/mac/MockRealtimeVideoSourceMac.h:
+        * platform/mediastream/mac/MockRealtimeVideoSourceMac.mm:
+        (WebCore::MockRealtimeVideoSourceMac::updateSampleBuffer):
+        * platform/mock/MockRealtimeVideoSource.cpp:
+        (WebCore::MockRealtimeVideoSource::settings):
+        (WebCore::MockRealtimeVideoSource::orientationChanged):
+        (WebCore::MockRealtimeVideoSource::monitorOrientation):
+        * platform/mock/MockRealtimeVideoSource.h:
+
 2019-07-05  Ryan Haddad  <ryanhaddad@apple.com>
 
         Unreviewed, rolling out r247115.
index 1437d2d..ec93705 100644 (file)
@@ -92,12 +92,13 @@ void RealtimeVideoSource::sourceMutedChanged()
 void RealtimeVideoSource::sourceSettingsChanged()
 {
     auto rotation = m_source->sampleRotation();
-    if (rotation == MediaSample::VideoRotation::Left || rotation == MediaSample::VideoRotation::Right) {
-        auto size = this->size();
+    auto size = this->size();
+    if (size.isEmpty())
+        size = m_source->size();
+    if (rotation == MediaSample::VideoRotation::Left || rotation == MediaSample::VideoRotation::Right)
         size = size.transposedSize();
-        m_currentSettings.setWidth(size.width());
-        m_currentSettings.setHeight(size.height());
-    }
+    m_currentSettings.setWidth(size.width());
+    m_currentSettings.setHeight(size.height());
 
     forEachObserver([&](auto& observer) {
         observer.sourceSettingsChanged();
index 5398b43..95a8cf0 100644 (file)
@@ -33,7 +33,6 @@
 #if ENABLE(MEDIA_STREAM)
 
 #include "MockRealtimeVideoSource.h"
-#include "OrientationNotifier.h"
 #include "PixelBufferConformerCV.h"
 
 typedef struct __CVBuffer *CVBufferRef;
@@ -45,7 +44,7 @@ namespace WebCore {
 
 class ImageTransferSessionVT;
 
-class MockRealtimeVideoSourceMac final : public MockRealtimeVideoSource, private OrientationNotifier::Observer {
+class MockRealtimeVideoSourceMac final : public MockRealtimeVideoSource {
 public:
     virtual ~MockRealtimeVideoSourceMac() = default;
 
@@ -57,11 +56,6 @@ private:
     void updateSampleBuffer() final;
     bool canResizeVideoFrames() const final { return true; }
 
-    void orientationChanged(int orientation) final;
-    void monitorOrientation(OrientationNotifier&) final;
-    MediaSample::VideoRotation sampleRotation() const final { return m_deviceOrientation; }
-
-    MediaSample::VideoRotation m_deviceOrientation { MediaSample::VideoRotation::None };
     std::unique_ptr<ImageTransferSessionVT> m_imageTransferSession;
     IntSize m_presetSize;
 };
index 83c63b1..9ddff40 100644 (file)
@@ -85,41 +85,13 @@ void MockRealtimeVideoSourceMac::updateSampleBuffer()
         m_imageTransferSession = ImageTransferSessionVT::create(preferedPixelBufferFormat());
 
     auto sampleTime = MediaTime::createWithDouble((elapsedTime() + 100_ms).seconds());
-    auto sampleBuffer = m_imageTransferSession->createMediaSample(imageBuffer->copyImage()->nativeImage().get(), sampleTime, size(), m_deviceOrientation);
+    auto sampleBuffer = m_imageTransferSession->createMediaSample(imageBuffer->copyImage()->nativeImage().get(), sampleTime, size(), sampleRotation());
     if (!sampleBuffer)
         return;
 
-    // We use m_deviceOrientation to emulate sensor orientation
     dispatchMediaSampleToObservers(*sampleBuffer);
 }
 
-void MockRealtimeVideoSourceMac::orientationChanged(int orientation)
-{
-    // FIXME: Do something with m_deviceOrientation. See bug 169822.
-    switch (orientation) {
-    case 0:
-        m_deviceOrientation = MediaSample::VideoRotation::None;
-        break;
-    case 90:
-        m_deviceOrientation = MediaSample::VideoRotation::Right;
-        break;
-    case -90:
-        m_deviceOrientation = MediaSample::VideoRotation::Left;
-        break;
-    case 180:
-        m_deviceOrientation = MediaSample::VideoRotation::UpsideDown;
-        break;
-    default:
-        return;
-    }
-}
-
-void MockRealtimeVideoSourceMac::monitorOrientation(OrientationNotifier& notifier)
-{
-    notifier.addObserver(*this);
-    orientationChanged(notifier.orientation());
-}
-
 } // namespace WebCore
 
 #endif // ENABLE(MEDIA_STREAM)
index 49a48be..04ee3b4 100644 (file)
@@ -157,7 +157,11 @@ const RealtimeMediaSourceSettings& MockRealtimeVideoSource::settings()
         settings.setLogicalSurface(false);
     }
     settings.setFrameRate(frameRate());
-    auto& size = this->size();
+    auto size = this->size();
+    if (mockCamera()) {
+        if (m_deviceOrientation == MediaSample::VideoRotation::Left || m_deviceOrientation == MediaSample::VideoRotation::Right)
+            size = size.transposedSize();
+    }
     settings.setWidth(size.width());
     settings.setHeight(size.height());
     if (aspectRatio())
@@ -472,6 +476,41 @@ bool MockRealtimeVideoSource::mockDisplayType(CaptureDevice::DeviceType type) co
     return WTF::get<MockDisplayProperties>(m_device.properties).type == type;
 }
 
+void MockRealtimeVideoSource::orientationChanged(int orientation)
+{
+    auto deviceOrientation = m_deviceOrientation;
+    switch (orientation) {
+    case 0:
+        m_deviceOrientation = MediaSample::VideoRotation::None;
+        break;
+    case 90:
+        m_deviceOrientation = MediaSample::VideoRotation::Right;
+        break;
+    case -90:
+        m_deviceOrientation = MediaSample::VideoRotation::Left;
+        break;
+    case 180:
+        m_deviceOrientation = MediaSample::VideoRotation::UpsideDown;
+        break;
+    default:
+        return;
+    }
+
+    if (deviceOrientation == m_deviceOrientation)
+        return;
+
+    notifySettingsDidChangeObservers({ RealtimeMediaSourceSettings::Flag::Width, RealtimeMediaSourceSettings::Flag::Height });
+}
+
+void MockRealtimeVideoSource::monitorOrientation(OrientationNotifier& notifier)
+{
+    if (!mockCamera())
+        return;
+
+    notifier.addObserver(*this);
+    orientationChanged(notifier.orientation());
+}
+
 } // namespace WebCore
 
 #endif // ENABLE(MEDIA_STREAM)
index cbe1a00..e599076 100644 (file)
@@ -36,6 +36,7 @@
 #include "FontCascade.h"
 #include "ImageBuffer.h"
 #include "MockMediaDevice.h"
+#include "OrientationNotifier.h"
 #include "RealtimeMediaSourceFactory.h"
 #include "RealtimeVideoCaptureSource.h"
 #include <wtf/RunLoop.h>
@@ -45,7 +46,7 @@ namespace WebCore {
 class FloatRect;
 class GraphicsContext;
 
-class MockRealtimeVideoSource : public RealtimeVideoCaptureSource {
+class MockRealtimeVideoSource : public RealtimeVideoCaptureSource, private OrientationNotifier::Observer {
 public:
 
     static CaptureSourceOrError create(String&& deviceID, String&& name, String&& hashSalt, const MediaConstraints*);
@@ -60,6 +61,7 @@ protected:
 
     Seconds elapsedTime();
     void settingsDidChange(OptionSet<RealtimeMediaSourceSettings::Flag>) override;
+    MediaSample::VideoRotation sampleRotation() const final { return m_deviceOrientation; }
 
 private:
     const RealtimeMediaSourceCapabilities& capabilities() final;
@@ -76,6 +78,10 @@ private:
 
     void generatePresets() final;
 
+    // OrientationNotifier::Observer
+    void orientationChanged(int orientation) final;
+    void monitorOrientation(OrientationNotifier&) final;
+
     void drawAnimation(GraphicsContext&);
     void drawText(GraphicsContext&);
     void drawBoxes(GraphicsContext&);
@@ -83,7 +89,7 @@ private:
     void generateFrame();
     void startCaptureTimer();
 
-    void delaySamples(Seconds) override;
+    void delaySamples(Seconds) final;
 
     bool mockCamera() const { return WTF::holds_alternative<MockCameraProperties>(m_device.properties); }
     bool mockDisplay() const { return WTF::holds_alternative<MockDisplayProperties>(m_device.properties); }
@@ -112,6 +118,7 @@ private:
     Color m_fillColor { Color::black };
     MockMediaDevice m_device;
     RefPtr<VideoPreset> m_preset;
+    MediaSample::VideoRotation m_deviceOrientation { MediaSample::VideoRotation::None };
 };
 
 } // namespace WebCore