vpx_image: reject zero dimensions in alloc/wrap/set_rect

test/vpx_image_test.cc

@@ -144,6 +144,27 @@ TEST(VpxImageTest, VpxImgSetRectRejectsFlipped) {
   }
 }
 
+TEST(VpxImageTest, VpxImgZeroDimensionsRejected) {
+  // Zero dimensions must be rejected: vpx_img_flip() computes
+  // (d_h - 1) * stride, which underflows for d_h == 0 and produces
+  // a wild plane pointer.
+  vpx_image_t img;
+  unsigned char buf[64];
+
+  EXPECT_EQ(vpx_img_wrap(&img, VPX_IMG_FMT_I420, 16, 0, 1, buf), nullptr);
+  EXPECT_EQ(vpx_img_wrap(&img, VPX_IMG_FMT_I420, 0, 16, 1, buf), nullptr);
+  EXPECT_EQ(vpx_img_alloc(&img, VPX_IMG_FMT_I420, 16, 0, 1), nullptr);
+  EXPECT_EQ(vpx_img_alloc(&img, VPX_IMG_FMT_I420, 0, 16, 1), nullptr);
+
+  // A zero-area viewport must also be rejected on a valid image.
+  ASSERT_NE(vpx_img_wrap(&img, VPX_IMG_FMT_I420, 16, 16, 1, buf), nullptr);
+  EXPECT_EQ(vpx_img_set_rect(&img, 0, 0, 16, 0), -1);
+  EXPECT_EQ(vpx_img_set_rect(&img, 0, 0, 0, 16), -1);
+  EXPECT_EQ(img.d_w, 16u);
+  EXPECT_EQ(img.d_h, 16u);
+  vpx_img_free(&img);
+}
+
 TEST(VpxImageTest, VpxImgAllocNone) {
   const int kWidth = 128;
   const int kHeight = 128;

vpx/src/vpx_image.c

@@ -50,9 +50,12 @@ static vpx_image_t *img_alloc_helper(vpx_image_t *img, vpx_img_fmt_t fmt,
 
   /* Impose maximum values on input parameters so that this function can
    * perform arithmetic operations without worrying about overflows.
+   * Zero dimensions are rejected: vpx_img_flip() computes
+   * (d_h - 1) * stride, which underflows for d_h == 0 and produces
+   * a wild plane pointer.
    */
-  if (d_w > 0x08000000 || d_h > 0x08000000 || buf_align > 65536 ||
-      stride_align > 65536) {
+  if (d_w == 0 || d_h == 0 || d_w > 0x08000000 || d_h > 0x08000000 ||
+      buf_align > 65536 || stride_align > 65536) {
     goto fail;
   }
 
@@ -209,6 +212,12 @@ int vpx_img_set_rect(vpx_image_t *img, unsigned int x, unsigned int y,
                      unsigned int w, unsigned int h) {
   if (img->stride[VPX_PLANE_Y] < 0) return -1;
 
+  /* A zero-area viewport is invalid: it would leave d_w or d_h at 0, and
+   * vpx_img_flip() computes (d_h - 1) * stride, which underflows for
+   * d_h == 0 and produces a wild plane pointer.
+   */
+  if (w == 0 || h == 0) return -1;
+
   if (x <= UINT_MAX - w && x + w <= img->w && y <= UINT_MAX - h &&
       y + h <= img->h) {
     img->d_w = w;