Origin: https://github.com/webmproject/libvpx/commit/c5640e3300690705c336966e2a8bb346a388c829

Backport of:

From c5640e3300690705c336966e2a8bb346a388c829 Mon Sep 17 00:00:00 2001
From: Wan-Teh Chang <wtc@google.com>
Date: Wed, 10 Apr 2024 17:01:10 -0700
Subject: [PATCH] Fix integer overflows in calc of stride_in_bytes

A port of the libaom CL
https://aomedia-review.googlesource.com/c/aom/+/188761.

Fix unsigned integer overflows in the calculation of stride_in_bytes in
img_alloc_helper() when d_w is huge.

Change the type of stride_in_bytes from unsigned int to int because it
will be assigned to img->stride[VPX_PLANE_Y], which is of the int type.

Test:
. ../libvpx/tools/set_analyzer_env.sh integer
../libvpx/configure --enable-debug --disable-optimizations
make -j
./test_libvpx --gtest_filter=VpxImageTest.VpxImgAllocHugeWidth

Bug: chromium:332382766
Change-Id: I3b39d78f61c7255e10cbf72ba2f4975425a05a82
(cherry picked from commit 2e32276277c0b1739707c5e861c96cf78794f1a0)
---
 test/vpx_image_test.cc | 36 ++++++++++++++++++++++++++++++++++++
 vpx/src/vpx_image.c    | 31 +++++++++++++++++++------------
 2 files changed, 55 insertions(+), 12 deletions(-)

--- a/test/vpx_image_test.cc
+++ b/test/vpx_image_test.cc
@@ -70,3 +70,34 @@ TEST(VpxImageTest, VpxImgAllocYv12) {
   EXPECT_EQ(img.planes[VPX_PLANE_V], img.planes[VPX_PLANE_U] + 1);
   vpx_img_free(&img);
 }
+
+TEST(VpxImageTest, VpxImgAllocHugeWidth) {
+  // The stride (0x80000000 * 2) would overflow unsigned int.
+  vpx_image_t *image =
+      vpx_img_alloc(nullptr, VPX_IMG_FMT_I42016, 0x80000000, 1, 1);
+  ASSERT_EQ(image, nullptr);
+
+  // The stride (0x80000000) would overflow int.
+  image = vpx_img_alloc(nullptr, VPX_IMG_FMT_I420, 0x80000000, 1, 1);
+  ASSERT_EQ(image, nullptr);
+
+  image = vpx_img_alloc(nullptr, VPX_IMG_FMT_I420, 0x7ffffffe, 1, 1);
+  if (image) {
+    vpx_img_free(image);
+  }
+
+  image = vpx_img_alloc(nullptr, VPX_IMG_FMT_I420, 285245883, 64, 1);
+  if (image) {
+    vpx_img_free(image);
+  }
+
+  image = vpx_img_alloc(nullptr, VPX_IMG_FMT_YV12, 285245883, 64, 1);
+  if (image) {
+    vpx_img_free(image);
+  }
+
+  image = vpx_img_alloc(nullptr, VPX_IMG_FMT_I42016, 285245883, 2, 1);
+  if (image) {
+    vpx_img_free(image);
+  }
+}
--- a/vpx/src/vpx_image.c
+++ b/vpx/src/vpx_image.c
@@ -8,6 +8,7 @@
  *  be found in the AUTHORS file in the root of the source tree.
  */
 
+#include <limits.h>
 #include <stdlib.h>
 #include <string.h>
 
@@ -20,8 +21,9 @@ static vpx_image_t *img_alloc_helper(vpx
                                      unsigned int buf_align,
                                      unsigned int stride_align,
                                      unsigned char *img_data) {
-  unsigned int h, w, s, xcs, ycs, bps;
-  unsigned int stride_in_bytes;
+  unsigned int h, w, xcs, ycs, bps;
+  uint64_t s;
+  int stride_in_bytes;
   int align;
 
   /* Treat align==0 like align==1 */
@@ -73,9 +75,11 @@ static vpx_image_t *img_alloc_helper(vpx
    * and height shouldn't be adjusted. */
   w = d_w;
   h = d_h;
-  s = (fmt & VPX_IMG_FMT_PLANAR) ? w : bps * w / 8;
-  s = (s + stride_align - 1) & ~(stride_align - 1);
-  stride_in_bytes = (fmt & VPX_IMG_FMT_HIGHBITDEPTH) ? s * 2 : s;
+  s = (fmt & VPX_IMG_FMT_PLANAR) ? w : (uint64_t)bps * w / 8;
+  s = (s + stride_align - 1) & ~((uint64_t)stride_align - 1);
+  s = (fmt & VPX_IMG_FMT_HIGHBITDEPTH) ? s * 2 : s;
+  if (s > INT_MAX) goto fail;
+  stride_in_bytes = (int)s;
 
   /* Allocate the new image */
   if (!img) {
@@ -98,9 +102,11 @@ static vpx_image_t *img_alloc_helper(vpx
     align = (1 << ycs) - 1;
     h = (d_h + align) & ~align;
 
-    s = (fmt & VPX_IMG_FMT_PLANAR) ? w : bps * w / 8;
-    s = (s + stride_align - 1) & ~(stride_align - 1);
-    stride_in_bytes = (fmt & VPX_IMG_FMT_HIGHBITDEPTH) ? s * 2 : s;
+    s = (fmt & VPX_IMG_FMT_PLANAR) ? w : (uint64_t)bps * w / 8;
+    s = (s + stride_align - 1) & ~((uint64_t)stride_align - 1);
+    s = (fmt & VPX_IMG_FMT_HIGHBITDEPTH) ? s * 2 : s;
+    if (s > INT_MAX) goto fail;
+    stride_in_bytes = (int)s;
     alloc_size = (fmt & VPX_IMG_FMT_PLANAR) ? (uint64_t)h * s * bps / 8
                                             : (uint64_t)h * s;
 
@@ -166,18 +172,19 @@ int vpx_img_set_rect(vpx_image_t *img, u
       if (img->fmt & VPX_IMG_FMT_HAS_ALPHA) {
         img->planes[VPX_PLANE_ALPHA] =
             data + x * bytes_per_sample + y * img->stride[VPX_PLANE_ALPHA];
-        data += img->h * img->stride[VPX_PLANE_ALPHA];
+        data += (size_t)img->h * img->stride[VPX_PLANE_ALPHA];
       }
 
       img->planes[VPX_PLANE_Y] =
           data + x * bytes_per_sample + y * img->stride[VPX_PLANE_Y];
-      data += img->h * img->stride[VPX_PLANE_Y];
+      data += (size_t)img->h * img->stride[VPX_PLANE_Y];
 
       if (!(img->fmt & VPX_IMG_FMT_UV_FLIP)) {
         img->planes[VPX_PLANE_U] =
             data + (x >> img->x_chroma_shift) * bytes_per_sample +
             (y >> img->y_chroma_shift) * img->stride[VPX_PLANE_U];
-        data += (img->h >> img->y_chroma_shift) * img->stride[VPX_PLANE_U];
+        data +=
+            (size_t)(img->h >> img->y_chroma_shift) * img->stride[VPX_PLANE_U];
         img->planes[VPX_PLANE_V] =
             data + (x >> img->x_chroma_shift) * bytes_per_sample +
             (y >> img->y_chroma_shift) * img->stride[VPX_PLANE_V];
@@ -185,7 +192,8 @@ int vpx_img_set_rect(vpx_image_t *img, u
         img->planes[VPX_PLANE_V] =
             data + (x >> img->x_chroma_shift) * bytes_per_sample +
             (y >> img->y_chroma_shift) * img->stride[VPX_PLANE_V];
-        data += (img->h >> img->y_chroma_shift) * img->stride[VPX_PLANE_V];
+        data +=
+            (size_t)(img->h >> img->y_chroma_shift) * img->stride[VPX_PLANE_V];
         img->planes[VPX_PLANE_U] =
             data + (x >> img->x_chroma_shift) * bytes_per_sample +
             (y >> img->y_chroma_shift) * img->stride[VPX_PLANE_U];

