diff options
author | Wan-Teh Chang <wtc@google.com> | 2024-04-03 20:08:16 +0000 |
---|---|---|
committer | Jerome Jiang <jianj@google.com> | 2024-04-22 14:47:56 +0000 |
commit | 19d9966572a410804349e1a8ee2017fed49a6dab (patch) | |
tree | c9b2462fb6395a61bb06e47a68d3c870dec56070 | |
parent | 0f41d224de36ca2a68aa45e11654cf0de21f95e2 (diff) | |
download | libaom-19d9966572a410804349e1a8ee2017fed49a6dab.tar.gz |
Fix integer overflows in calc of stride_in_bytes
Fix unsigned integer overflows in the calculation of stride_in_bytes in
img_alloc_helper() when d_w is huge.
Change the type of stride_in_bytes from unsigned int to int because it
will be assigned to img->stride[AOM_PLANE_Y], which is of the int type.
Test:
cmake ../aom -G Ninja -DCMAKE_C_COMPILER=clang \
-DCMAKE_CXX_COMPILER=clang++ -DCMAKE_BUILD_TYPE=Debug \
-DSANITIZE=unsigned-integer-overflow
ninja
./test_libaom --gtest_filter=AomImageTest.AomImgAllocHugeWidth
Bug: chromium:332382766
Change-Id: Iaccb83bcd13ddc3ea5e6f01da91bb01215ddb461
(cherry picked from commit 7aa2edc2b09f98c32820923d813fd73eb23b5861)
-rw-r--r-- | aom/src/aom_image.c | 15 | ||||
-rw-r--r-- | test/aom_image_test.cc | 36 |
2 files changed, 44 insertions, 7 deletions
diff --git a/aom/src/aom_image.c b/aom/src/aom_image.c index 3b1c33d05..b68dc4c8f 100644 --- a/aom/src/aom_image.c +++ b/aom/src/aom_image.c @@ -36,8 +36,7 @@ static aom_image_t *img_alloc_helper( /* NOTE: In this function, bit_depth is either 8 or 16 (if * AOM_IMG_FMT_HIGHBITDEPTH is set), never 10 or 12. */ - unsigned int h, w, s, xcs, ycs, bps, bit_depth; - unsigned int stride_in_bytes; + unsigned int h, w, xcs, ycs, bps, bit_depth; if (img != NULL) memset(img, 0, sizeof(aom_image_t)); @@ -108,9 +107,11 @@ static aom_image_t *img_alloc_helper( w = align_image_dimension(d_w, xcs, size_align); h = align_image_dimension(d_h, ycs, size_align); - s = (fmt & AOM_IMG_FMT_PLANAR) ? w : bps * w / bit_depth; + uint64_t s = (fmt & AOM_IMG_FMT_PLANAR) ? w : (uint64_t)bps * w / bit_depth; s = (s + 2 * border + stride_align - 1) & ~(stride_align - 1); - stride_in_bytes = s * bit_depth / 8; + s = s * bit_depth / 8; + if (s > INT_MAX) goto fail; + const int stride_in_bytes = (int)s; /* Allocate the new image */ if (!img) { @@ -232,7 +233,7 @@ int aom_img_set_rect(aom_image_t *img, unsigned int x, unsigned int y, img->planes[AOM_PLANE_Y] = data + x * bytes_per_sample + y * img->stride[AOM_PLANE_Y]; - data += (img->h + 2 * border) * img->stride[AOM_PLANE_Y]; + data += ((size_t)img->h + 2 * border) * img->stride[AOM_PLANE_Y]; unsigned int uv_border_h = border >> img->y_chroma_shift; unsigned int uv_x = x >> img->x_chroma_shift; @@ -244,14 +245,14 @@ int aom_img_set_rect(aom_image_t *img, unsigned int x, unsigned int y, } else if (!(img->fmt & AOM_IMG_FMT_UV_FLIP)) { img->planes[AOM_PLANE_U] = data + uv_x * bytes_per_sample + uv_y * img->stride[AOM_PLANE_U]; - data += ((img->h >> img->y_chroma_shift) + 2 * uv_border_h) * + data += ((size_t)(img->h >> img->y_chroma_shift) + 2 * uv_border_h) * img->stride[AOM_PLANE_U]; img->planes[AOM_PLANE_V] = data + uv_x * bytes_per_sample + uv_y * img->stride[AOM_PLANE_V]; } else { img->planes[AOM_PLANE_V] = data + uv_x * bytes_per_sample + uv_y * img->stride[AOM_PLANE_V]; - data += ((img->h >> img->y_chroma_shift) + 2 * uv_border_h) * + data += ((size_t)(img->h >> img->y_chroma_shift) + 2 * uv_border_h) * img->stride[AOM_PLANE_V]; img->planes[AOM_PLANE_U] = data + uv_x * bytes_per_sample + uv_y * img->stride[AOM_PLANE_U]; diff --git a/test/aom_image_test.cc b/test/aom_image_test.cc index 03f4373f3..62f3c1274 100644 --- a/test/aom_image_test.cc +++ b/test/aom_image_test.cc @@ -70,3 +70,39 @@ TEST(AomImageTest, AomImgAllocNv12) { EXPECT_EQ(img.planes[AOM_PLANE_V], nullptr); aom_img_free(&img); } + +TEST(AomImageTest, AomImgAllocHugeWidth) { + // The stride (0x80000000 * 2) would overflow unsigned int. + aom_image_t *image = + aom_img_alloc(nullptr, AOM_IMG_FMT_I42016, 0x80000000, 1, 1); + ASSERT_EQ(image, nullptr); + + // The stride (0x80000000) would overflow int. + image = aom_img_alloc(nullptr, AOM_IMG_FMT_I420, 0x80000000, 1, 1); + ASSERT_EQ(image, nullptr); + + image = aom_img_alloc(nullptr, AOM_IMG_FMT_I420, 0x7ffffffe, 1, 1); + if (image) { + aom_img_free(image); + } + + image = aom_img_alloc(nullptr, AOM_IMG_FMT_I420, 285245883, 64, 1); + if (image) { + aom_img_free(image); + } + + image = aom_img_alloc(nullptr, AOM_IMG_FMT_NV12, 285245883, 64, 1); + if (image) { + aom_img_free(image); + } + + image = aom_img_alloc(nullptr, AOM_IMG_FMT_YV12, 285245883, 64, 1); + if (image) { + aom_img_free(image); + } + + image = aom_img_alloc(nullptr, AOM_IMG_FMT_I42016, 285245883, 2, 1); + if (image) { + aom_img_free(image); + } +} |