Fix potential integer overflow in rowBytes multiplications

Cast the first operand to (size_t) before multiplying two uint32_t
values involving rowBytes, alphaRowBytes, or yuvRowBytes to prevent
unsigned integer wrap-around on large images.

Author: rootvector2

src/codec_aom.c

@@ -1209,7 +1209,7 @@ static avifResult aomCodecEncodeImage(avifCodec * codec,
         if (aomImageAllocated) {
             const uint32_t bytesPerRow = ((image->depth > 8) ? 2 : 1) * image->width;
             for (uint32_t j = 0; j < image->height; ++j) {
-                const uint8_t * srcAlphaRow = &image->alphaPlane[j * image->alphaRowBytes];
+                const uint8_t * srcAlphaRow = &image->alphaPlane[(size_t)j * image->alphaRowBytes];
                 uint8_t * dstAlphaRow = &aomImage.planes[0][j * aomImage.stride[0]];
                 memcpy(dstAlphaRow, srcAlphaRow, bytesPerRow);
             }
@@ -1233,7 +1233,7 @@ static avifResult aomCodecEncodeImage(avifCodec * codec,
                 uint32_t bytesPerRow = bytesPerPixel * planeWidth;
 
                 for (uint32_t j = 0; j < planeHeight; ++j) {
-                    const uint8_t * srcRow = &image->yuvPlanes[yuvPlane][j * image->yuvRowBytes[yuvPlane]];
+                    const uint8_t * srcRow = &image->yuvPlanes[yuvPlane][(size_t)j * image->yuvRowBytes[yuvPlane]];
                     uint8_t * dstRow = &aomImage.planes[yuvPlane][j * aomImage.stride[yuvPlane]];
                     memcpy(dstRow, srcRow, bytesPerRow);
                 }

src/codec_avm.c

@@ -903,7 +903,7 @@ static avifResult avmCodecEncodeImage(avifCodec * codec,
         if (avmImageAllocated) {
             const uint32_t bytesPerRow = ((image->depth > 8) ? 2 : 1) * image->width;
             for (uint32_t j = 0; j < image->height; ++j) {
-                const uint8_t * srcAlphaRow = &image->alphaPlane[j * image->alphaRowBytes];
+                const uint8_t * srcAlphaRow = &image->alphaPlane[(size_t)j * image->alphaRowBytes];
                 uint8_t * dstAlphaRow = &avmImage.planes[0][j * avmImage.stride[0]];
                 memcpy(dstAlphaRow, srcAlphaRow, bytesPerRow);
             }
@@ -927,7 +927,7 @@ static avifResult avmCodecEncodeImage(avifCodec * codec,
                 uint32_t bytesPerRow = bytesPerPixel * planeWidth;
 
                 for (uint32_t j = 0; j < planeHeight; ++j) {
-                    const uint8_t * srcRow = &image->yuvPlanes[yuvPlane][j * image->yuvRowBytes[yuvPlane]];
+                    const uint8_t * srcRow = &image->yuvPlanes[yuvPlane][(size_t)j * image->yuvRowBytes[yuvPlane]];
                     uint8_t * dstRow = &avmImage.planes[yuvPlane][j * avmImage.stride[yuvPlane]];
                     memcpy(dstRow, srcRow, bytesPerRow);
                 }

src/codec_svt.c

@@ -274,13 +274,13 @@ static avifResult svtCodecEncodeImage(avifCodec * codec,
     if (alpha) {
         input_picture_buffer->y_stride = image->alphaRowBytes / bytesPerPixel;
         input_picture_buffer->luma = image->alphaPlane;
-        input_buffer->n_filled_len = image->alphaRowBytes * image->height;
+        input_buffer->n_filled_len = (size_t)image->alphaRowBytes * image->height;
 
 #if SVT_AV1_CHECK_VERSION(1, 8, 0)
         // Simulate 4:2:0 UV planes. SVT-AV1 does not support 4:0:0 samples.
         const uint32_t uvWidth = (image->width + y_shift) >> y_shift;
         const uint32_t uvRowBytes = uvWidth * bytesPerPixel;
-        const uint32_t uvSize = uvRowBytes * uvHeight;
+        const size_t uvSize = (size_t)uvRowBytes * uvHeight;
         uvPlanes = avifAlloc(uvSize);
         if (uvPlanes == NULL) {
             goto cleanup;
@@ -300,11 +300,11 @@ static avifResult svtCodecEncodeImage(avifCodec * codec,
     } else {
         input_picture_buffer->y_stride = image->yuvRowBytes[0] / bytesPerPixel;
         input_picture_buffer->luma = image->yuvPlanes[0];
-        input_buffer->n_filled_len = image->yuvRowBytes[0] * image->height;
+        input_buffer->n_filled_len = (size_t)image->yuvRowBytes[0] * image->height;
         input_picture_buffer->cb = image->yuvPlanes[1];
-        input_buffer->n_filled_len += image->yuvRowBytes[1] * uvHeight;
+        input_buffer->n_filled_len += (size_t)image->yuvRowBytes[1] * uvHeight;
         input_picture_buffer->cr = image->yuvPlanes[2];
-        input_buffer->n_filled_len += image->yuvRowBytes[2] * uvHeight;
+        input_buffer->n_filled_len += (size_t)image->yuvRowBytes[2] * uvHeight;
         input_picture_buffer->cb_stride = image->yuvRowBytes[1] / bytesPerPixel;
         input_picture_buffer->cr_stride = image->yuvRowBytes[2] / bytesPerPixel;
     }

src/read.c

@@ -6599,8 +6599,8 @@ static avifResult avifImageLimitedToFullAlpha(avifImage * image)
 
     if (image->depth > 8) {
         for (uint32_t j = 0; j < image->height; ++j) {
-            const uint8_t * srcRow = &alphaPlane[j * alphaRowBytes];
-            uint8_t * dstRow = &image->alphaPlane[j * image->alphaRowBytes];
+            const uint8_t * srcRow = &alphaPlane[(size_t)j * alphaRowBytes];
+            uint8_t * dstRow = &image->alphaPlane[(size_t)j * image->alphaRowBytes];
             for (uint32_t i = 0; i < image->width; ++i) {
                 int srcAlpha = *((const uint16_t *)&srcRow[i * 2]);
                 int dstAlpha = avifLimitedToFullY(image->depth, srcAlpha);
@@ -6609,8 +6609,8 @@ static avifResult avifImageLimitedToFullAlpha(avifImage * image)
         }
     } else {
         for (uint32_t j = 0; j < image->height; ++j) {
-            const uint8_t * srcRow = &alphaPlane[j * alphaRowBytes];
-            uint8_t * dstRow = &image->alphaPlane[j * image->alphaRowBytes];
+            const uint8_t * srcRow = &alphaPlane[(size_t)j * alphaRowBytes];
+            uint8_t * dstRow = &image->alphaPlane[(size_t)j * image->alphaRowBytes];
             for (uint32_t i = 0; i < image->width; ++i) {
                 int srcAlpha = srcRow[i];
                 int dstAlpha = avifLimitedToFullY(image->depth, srcAlpha);