vp8e_mr_alloc_mem() leaks LOWER_RES_FRAME_INFO if second memory allocation fails
authorddkilzer@apple.com <ddkilzer@apple.com@268f45cc-cd09-0410-ab3c-d52691b4dbfc>
Tue, 5 Feb 2019 03:24:54 +0000 (03:24 +0000)
committerddkilzer@apple.com <ddkilzer@apple.com@268f45cc-cd09-0410-ab3c-d52691b4dbfc>
Tue, 5 Feb 2019 03:24:54 +0000 (03:24 +0000)
<https://webkit.org/b/194265>

Reviewed by Youenn Fablet.

* Source/third_party/libvpx/source/libvpx/vp8/vp8_cx_iface.c:
(vp8e_mr_alloc_mem):
- Initialize `res` to VPX_CODEC_OK instead of 0.
- Return early if first calloc() fails instead of trying the
  second calloc().  The function would crash dereferencing
  nullptr in `shared_mem_loc->mb_info` otherwise.
- Call free(shared_mem_loc) if the second call to calloc()
  fails.  This fixes the leak.
* WebKit/0003-libwebrtc-fix-vp8e_mr_alloc_mem-leak.diff: Add.

git-svn-id: https://svn.webkit.org/repository/webkit/trunk@240961 268f45cc-cd09-0410-ab3c-d52691b4dbfc

Source/ThirdParty/libwebrtc/ChangeLog
Source/ThirdParty/libwebrtc/Source/third_party/libvpx/source/libvpx/vp8/vp8_cx_iface.c
Source/ThirdParty/libwebrtc/WebKit/0003-libwebrtc-fix-vp8e_mr_alloc_mem-leak.diff [new file with mode: 0644]

index 3f4f1c8..3d3129f 100644 (file)
@@ -1,3 +1,20 @@
+2019-02-04  David Kilzer  <ddkilzer@apple.com>
+
+        vp8e_mr_alloc_mem() leaks LOWER_RES_FRAME_INFO if second memory allocation fails
+        <https://webkit.org/b/194265>
+
+        Reviewed by Youenn Fablet.
+
+        * Source/third_party/libvpx/source/libvpx/vp8/vp8_cx_iface.c:
+        (vp8e_mr_alloc_mem):
+        - Initialize `res` to VPX_CODEC_OK instead of 0.
+        - Return early if first calloc() fails instead of trying the
+          second calloc().  The function would crash dereferencing
+          nullptr in `shared_mem_loc->mb_info` otherwise.
+        - Call free(shared_mem_loc) if the second call to calloc()
+          fails.  This fixes the leak.
+        * WebKit/0003-libwebrtc-fix-vp8e_mr_alloc_mem-leak.diff: Add.
+
 2019-01-30  Commit Queue  <commit-queue@webkit.org>
 
         Unreviewed, rolling out r240665.
index d3e2005..b67baab 100644 (file)
@@ -577,7 +577,7 @@ static vpx_codec_err_t set_screen_content_mode(vpx_codec_alg_priv_t *ctx,
 
 static vpx_codec_err_t vp8e_mr_alloc_mem(const vpx_codec_enc_cfg_t *cfg,
                                          void **mem_loc) {
-  vpx_codec_err_t res = 0;
+  vpx_codec_err_t res = VPX_CODEC_OK;
 
 #if CONFIG_MULTI_RES_ENCODING
   LOWER_RES_FRAME_INFO *shared_mem_loc;
@@ -586,12 +586,13 @@ static vpx_codec_err_t vp8e_mr_alloc_mem(const vpx_codec_enc_cfg_t *cfg,
 
   shared_mem_loc = calloc(1, sizeof(LOWER_RES_FRAME_INFO));
   if (!shared_mem_loc) {
-    res = VPX_CODEC_MEM_ERROR;
+    return VPX_CODEC_MEM_ERROR;
   }
 
   shared_mem_loc->mb_info =
       calloc(mb_rows * mb_cols, sizeof(LOWER_RES_MB_INFO));
   if (!(shared_mem_loc->mb_info)) {
+    free(shared_mem_loc);
     res = VPX_CODEC_MEM_ERROR;
   } else {
     *mem_loc = (void *)shared_mem_loc;
diff --git a/Source/ThirdParty/libwebrtc/WebKit/0003-libwebrtc-fix-vp8e_mr_alloc_mem-leak.diff b/Source/ThirdParty/libwebrtc/WebKit/0003-libwebrtc-fix-vp8e_mr_alloc_mem-leak.diff
new file mode 100644 (file)
index 0000000..6c591e0
--- /dev/null
@@ -0,0 +1,28 @@
+diff --git a/Source/ThirdParty/libwebrtc/Source/third_party/libvpx/source/libvpx/vp8/vp8_cx_iface.c b/Source/ThirdParty/libwebrtc/Source/third_party/libvpx/source/libvpx/vp8/vp8_cx_iface.c
+index d3e20059410..b67baab24d1 100644
+--- a/Source/ThirdParty/libwebrtc/Source/third_party/libvpx/source/libvpx/vp8/vp8_cx_iface.c
++++ b/Source/ThirdParty/libwebrtc/Source/third_party/libvpx/source/libvpx/vp8/vp8_cx_iface.c
+@@ -577,7 +577,7 @@ static vpx_codec_err_t set_screen_content_mode(vpx_codec_alg_priv_t *ctx,
+ static vpx_codec_err_t vp8e_mr_alloc_mem(const vpx_codec_enc_cfg_t *cfg,
+                                          void **mem_loc) {
+-  vpx_codec_err_t res = 0;
++  vpx_codec_err_t res = VPX_CODEC_OK;
+ #if CONFIG_MULTI_RES_ENCODING
+   LOWER_RES_FRAME_INFO *shared_mem_loc;
+@@ -586,12 +586,13 @@ static vpx_codec_err_t vp8e_mr_alloc_mem(const vpx_codec_enc_cfg_t *cfg,
+   shared_mem_loc = calloc(1, sizeof(LOWER_RES_FRAME_INFO));
+   if (!shared_mem_loc) {
+-    res = VPX_CODEC_MEM_ERROR;
++    return VPX_CODEC_MEM_ERROR;
+   }
+   shared_mem_loc->mb_info =
+       calloc(mb_rows * mb_cols, sizeof(LOWER_RES_MB_INFO));
+   if (!(shared_mem_loc->mb_info)) {
++    free(shared_mem_loc);
+     res = VPX_CODEC_MEM_ERROR;
+   } else {
+     *mem_loc = (void *)shared_mem_loc;